embulk-input-presto 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7162ef02ea3157af633bd47a7af58854966b48db
+  data.tar.gz: 5118f58ca84cb53855ee6d8787b67a486bb6e888
+SHA512:
+  metadata.gz: a44ecbf5dd73f5826b8ba18236ba596ef442a2ad6b917677c169756cd353e9c587bc2a890a38338cfa44e764f65c0fd661cfbcf7a7ff7234c27c02dfb87f8285
+  data.tar.gz: 70183a801075057703fbeefabb650c0f2ae6ace0582f99d2b37eaf5ddea9c409aa2549d4f4b0f0b87d18c9ca7e7ed73c536a072e18bb1647e9a0ca741515cd6f

data/.gitignore

@@ -0,0 +1,6 @@
+*~
+/pkg/
+/tmp/
+/vendor/
+/.bundle/
+/Gemfile.lock

data/.ruby-version

@@ -0,0 +1 @@
+jruby-9.0.0.0

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org/'
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,64 @@
+# Facebook Presto input plugin for Embulk
+
+Facebook Presto input plugin for Embulk.
+[see](https://prestodb.io/).
+
+## Overview
+
+* **Plugin type**: input
+* **Resume supported**: yes
+* **Cleanup supported**: yes
+* **Guess supported**: no
+
+## Configuration
+
+- **host**: host (string, default: `"localhost"`)
+- **port**: port (integer, default: `8080`)
+- **schema**: schema (string, default: `"default"`)
+- **catalog**: catalog (string, default: `"native"`)
+- **query**: query (string, required)
+- **user**: user (string, default: `"embulk"`)
+- **columns**: columns (array, required)
+
+## Example
+
+```yaml
+in:
+  type: presto
+  host: presto-cordinator
+  catalog: store
+  schema: public
+  query: |
+    SELECT
+      trim(upper(url_decode(keyword))) AS keyword,
+      count(*) as count
+    FROM search
+    CROSS JOIN UNNEST(split(keywords, ',')) AS t (keyword)
+    WHERE log_date >= (CURRENT_DATE - INTERVAL '90' DAY)
+     AND length(keywords) != 256
+    group by keyword
+    having count(*) >= 10
+    order by count(*) desc
+  columns:
+    - {name: keyword, type: string}
+    - {name: count, type: long}
+out:
+  type: stdout
+```
+
+## Limited
+* Only the data type that Embulk supports is possible.
+  * TIMESTAMP
+  * LONG
+  * DOUBLE
+  * BOOLEAN
+  * STRING
+
+* Presto is not support Prepared statement.
+  * Can't fetch schema by sql
+
+## Build
+
+```
+$ rake
+```

data/Rakefile

@@ -0,0 +1,3 @@
+require "bundler/gem_tasks"
+
+task default: :build

data/embulk-input-presto.gemspec

@@ -0,0 +1,20 @@
+
+Gem::Specification.new do |spec|
+  spec.name          = "embulk-input-presto"
+  spec.version       = "0.1.0"
+  spec.authors       = ["toyama0919"]
+  spec.summary       = "Facebook Presto input plugin for Embulk"
+  spec.description   = "Loads records from Presto."
+  spec.email         = ["toyama0919@gmail.com"]
+  spec.licenses      = ["MIT"]
+  spec.homepage      = "https://github.com/toyama0919/embulk-input-presto"
+
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r{^(test|spec)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "presto-client"
+  spec.add_development_dependency 'embulk', ['~> 0.7.10']
+  spec.add_development_dependency 'bundler', ['~> 1.0']
+  spec.add_development_dependency 'rake', ['>= 10.0']
+end

data/lib/embulk/input/presto.rb

@@ -0,0 +1,55 @@
+module Embulk
+  class InputPresto < InputPlugin
+    require 'presto-client'
+
+    Plugin.register_input("presto", self)
+
+    def self.transaction(config, &control)
+      task = {
+        "host" => config.param("host", :string, default: "localhost"),
+        "port" => config.param("port", :integer, default: 8080),
+        "schema" => config.param("schema", :string, default: "default"),
+        "catalog" => config.param("catalog", :string, default: "native"),
+        "query" => config.param("query", :string),
+        "user" => config.param("user", :string, default: "embulk"),
+        "columns" => config.param("columns", :array, default: [])
+      }
+
+      columns = task['columns'].each_with_index.map do |c, i|
+        Column.new(i, c["name"], c["type"].to_sym)
+      end
+
+      resume(task, columns, 1, &control)
+    end
+
+    def self.resume(task, columns, count, &control)
+      task_reports = yield(task, columns, count)
+
+      next_config_diff = {}
+      return next_config_diff
+    end
+
+    def init
+      @client = Presto::Client.new(
+        server: "#{task['host']}:#{task['port']}",
+        catalog: task['catalog'],
+        user: task['user'],
+        schema: task['schema']
+      )
+      @query = task["query"]
+    end
+
+    def run
+      @client.query(@query) do |q|
+        q.each_row {|row|
+          page_builder.add(row)
+        }
+      end
+
+      page_builder.finish
+
+      task_report = {}
+      return task_report
+    end
+  end
+end

metadata

@@ -0,0 +1,108 @@
+--- !ruby/object:Gem::Specification
+name: embulk-input-presto
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- toyama0919
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-12-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: presto-client
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.10
+  name: embulk
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.10
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Loads records from Presto.
+email:
+- toyama0919@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".ruby-version"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- embulk-input-presto.gemspec
+- lib/embulk/input/presto.rb
+homepage: https://github.com/toyama0919/embulk-input-presto
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.5
+signing_key:
+specification_version: 4
+summary: Facebook Presto input plugin for Embulk
+test_files: []