embulk-input-presto 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 7162ef02ea3157af633bd47a7af58854966b48db
4
+ data.tar.gz: 5118f58ca84cb53855ee6d8787b67a486bb6e888
5
+ SHA512:
6
+ metadata.gz: a44ecbf5dd73f5826b8ba18236ba596ef442a2ad6b917677c169756cd353e9c587bc2a890a38338cfa44e764f65c0fd661cfbcf7a7ff7234c27c02dfb87f8285
7
+ data.tar.gz: 70183a801075057703fbeefabb650c0f2ae6ace0582f99d2b37eaf5ddea9c409aa2549d4f4b0f0b87d18c9ca7e7ed73c536a072e18bb1647e9a0ca741515cd6f
data/.gitignore ADDED
@@ -0,0 +1,6 @@
1
+ *~
2
+ /pkg/
3
+ /tmp/
4
+ /vendor/
5
+ /.bundle/
6
+ /Gemfile.lock
data/.ruby-version ADDED
@@ -0,0 +1 @@
1
+ jruby-9.0.0.0
data/Gemfile ADDED
@@ -0,0 +1,2 @@
1
+ source 'https://rubygems.org/'
2
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,21 @@
1
+
2
+ MIT License
3
+
4
+ Permission is hereby granted, free of charge, to any person obtaining
5
+ a copy of this software and associated documentation files (the
6
+ "Software"), to deal in the Software without restriction, including
7
+ without limitation the rights to use, copy, modify, merge, publish,
8
+ distribute, sublicense, and/or sell copies of the Software, and to
9
+ permit persons to whom the Software is furnished to do so, subject to
10
+ the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be
13
+ included in all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
19
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
20
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,64 @@
1
+ # Facebook Presto input plugin for Embulk
2
+
3
+ Facebook Presto input plugin for Embulk.
4
+ [see](https://prestodb.io/).
5
+
6
+ ## Overview
7
+
8
+ * **Plugin type**: input
9
+ * **Resume supported**: yes
10
+ * **Cleanup supported**: yes
11
+ * **Guess supported**: no
12
+
13
+ ## Configuration
14
+
15
+ - **host**: host (string, default: `"localhost"`)
16
+ - **port**: port (integer, default: `8080`)
17
+ - **schema**: schema (string, default: `"default"`)
18
+ - **catalog**: catalog (string, default: `"native"`)
19
+ - **query**: query (string, required)
20
+ - **user**: user (string, default: `"embulk"`)
21
+ - **columns**: columns (array, required)
22
+
23
+ ## Example
24
+
25
+ ```yaml
26
+ in:
27
+ type: presto
28
+ host: presto-cordinator
29
+ catalog: store
30
+ schema: public
31
+ query: |
32
+ SELECT
33
+ trim(upper(url_decode(keyword))) AS keyword,
34
+ count(*) as count
35
+ FROM search
36
+ CROSS JOIN UNNEST(split(keywords, ',')) AS t (keyword)
37
+ WHERE log_date >= (CURRENT_DATE - INTERVAL '90' DAY)
38
+ AND length(keywords) != 256
39
+ group by keyword
40
+ having count(*) >= 10
41
+ order by count(*) desc
42
+ columns:
43
+ - {name: keyword, type: string}
44
+ - {name: count, type: long}
45
+ out:
46
+ type: stdout
47
+ ```
48
+
49
+ ## Limited
50
+ * Only the data type that Embulk supports is possible.
51
+ * TIMESTAMP
52
+ * LONG
53
+ * DOUBLE
54
+ * BOOLEAN
55
+ * STRING
56
+
57
+ * Presto is not support Prepared statement.
58
+ * Can't fetch schema by sql
59
+
60
+ ## Build
61
+
62
+ ```
63
+ $ rake
64
+ ```
data/Rakefile ADDED
@@ -0,0 +1,3 @@
1
+ require "bundler/gem_tasks"
2
+
3
+ task default: :build
@@ -0,0 +1,20 @@
1
+
2
+ Gem::Specification.new do |spec|
3
+ spec.name = "embulk-input-presto"
4
+ spec.version = "0.1.0"
5
+ spec.authors = ["toyama0919"]
6
+ spec.summary = "Facebook Presto input plugin for Embulk"
7
+ spec.description = "Loads records from Presto."
8
+ spec.email = ["toyama0919@gmail.com"]
9
+ spec.licenses = ["MIT"]
10
+ spec.homepage = "https://github.com/toyama0919/embulk-input-presto"
11
+
12
+ spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
13
+ spec.test_files = spec.files.grep(%r{^(test|spec)/})
14
+ spec.require_paths = ["lib"]
15
+
16
+ spec.add_dependency "presto-client"
17
+ spec.add_development_dependency 'embulk', ['~> 0.7.10']
18
+ spec.add_development_dependency 'bundler', ['~> 1.0']
19
+ spec.add_development_dependency 'rake', ['>= 10.0']
20
+ end
@@ -0,0 +1,55 @@
1
+ module Embulk
2
+ class InputPresto < InputPlugin
3
+ require 'presto-client'
4
+
5
+ Plugin.register_input("presto", self)
6
+
7
+ def self.transaction(config, &control)
8
+ task = {
9
+ "host" => config.param("host", :string, default: "localhost"),
10
+ "port" => config.param("port", :integer, default: 8080),
11
+ "schema" => config.param("schema", :string, default: "default"),
12
+ "catalog" => config.param("catalog", :string, default: "native"),
13
+ "query" => config.param("query", :string),
14
+ "user" => config.param("user", :string, default: "embulk"),
15
+ "columns" => config.param("columns", :array, default: [])
16
+ }
17
+
18
+ columns = task['columns'].each_with_index.map do |c, i|
19
+ Column.new(i, c["name"], c["type"].to_sym)
20
+ end
21
+
22
+ resume(task, columns, 1, &control)
23
+ end
24
+
25
+ def self.resume(task, columns, count, &control)
26
+ task_reports = yield(task, columns, count)
27
+
28
+ next_config_diff = {}
29
+ return next_config_diff
30
+ end
31
+
32
+ def init
33
+ @client = Presto::Client.new(
34
+ server: "#{task['host']}:#{task['port']}",
35
+ catalog: task['catalog'],
36
+ user: task['user'],
37
+ schema: task['schema']
38
+ )
39
+ @query = task["query"]
40
+ end
41
+
42
+ def run
43
+ @client.query(@query) do |q|
44
+ q.each_row {|row|
45
+ page_builder.add(row)
46
+ }
47
+ end
48
+
49
+ page_builder.finish
50
+
51
+ task_report = {}
52
+ return task_report
53
+ end
54
+ end
55
+ end
metadata ADDED
@@ -0,0 +1,108 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: embulk-input-presto
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - toyama0919
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2015-12-07 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - ">="
17
+ - !ruby/object:Gem::Version
18
+ version: '0'
19
+ name: presto-client
20
+ prerelease: false
21
+ type: :runtime
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ requirement: !ruby/object:Gem::Requirement
29
+ requirements:
30
+ - - "~>"
31
+ - !ruby/object:Gem::Version
32
+ version: 0.7.10
33
+ name: embulk
34
+ prerelease: false
35
+ type: :development
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: 0.7.10
41
+ - !ruby/object:Gem::Dependency
42
+ requirement: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - "~>"
45
+ - !ruby/object:Gem::Version
46
+ version: '1.0'
47
+ name: bundler
48
+ prerelease: false
49
+ type: :development
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '1.0'
55
+ - !ruby/object:Gem::Dependency
56
+ requirement: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - ">="
59
+ - !ruby/object:Gem::Version
60
+ version: '10.0'
61
+ name: rake
62
+ prerelease: false
63
+ type: :development
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '10.0'
69
+ description: Loads records from Presto.
70
+ email:
71
+ - toyama0919@gmail.com
72
+ executables: []
73
+ extensions: []
74
+ extra_rdoc_files: []
75
+ files:
76
+ - ".gitignore"
77
+ - ".ruby-version"
78
+ - Gemfile
79
+ - LICENSE.txt
80
+ - README.md
81
+ - Rakefile
82
+ - embulk-input-presto.gemspec
83
+ - lib/embulk/input/presto.rb
84
+ homepage: https://github.com/toyama0919/embulk-input-presto
85
+ licenses:
86
+ - MIT
87
+ metadata: {}
88
+ post_install_message:
89
+ rdoc_options: []
90
+ require_paths:
91
+ - lib
92
+ required_ruby_version: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ required_rubygems_version: !ruby/object:Gem::Requirement
98
+ requirements:
99
+ - - ">="
100
+ - !ruby/object:Gem::Version
101
+ version: '0'
102
+ requirements: []
103
+ rubyforge_project:
104
+ rubygems_version: 2.4.5
105
+ signing_key:
106
+ specification_version: 4
107
+ summary: Facebook Presto input plugin for Embulk
108
+ test_files: []