embulk-input-presto 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cc144684e67c11aba628fa38f6ba455c453b87d2
-  data.tar.gz: 893c46973bfa5fc1c6a278b10ad7e81ea78be2dd
+  metadata.gz: 1a57f6dc5a7064378aba3325b8765ffc3e3f1cdd
+  data.tar.gz: f272abf3aad3768b0603366e404685cf33d4e61d
 SHA512:
-  metadata.gz: 9b79cc502d7a9f092116ce22a7abd72f6268b923429858c0ce2c822098f62be07a3b1a6d2f38cf16727a629507edf093d01f285c601144bde0f7f036cbae2dbe
-  data.tar.gz: 86cdb8c582355601eaebca39a37ca308c07af437e171b551620108c6223259215c1f2baf2e29fb28746c57ec61aec2ebae59ff104cd8c52e2880c006438bb222
+  metadata.gz: 13ce34fd728876372f389ac13cf422d23b8facba8c3069b60f946f4762cc37e003e2493ec5c1129ca10fdab20a8ed0ed860028a9109e02a5afd6bb62e9e30ccc
+  data.tar.gz: 1898c4f776b9570ecf9813372cfd80ff2c62c339f0872e633d85a722e8a844d911b1bb0716489cf243cde6b25aa6fc2452a8c628d3877cc6b13399a8924b279a

data/.ruby-version

@@ -1 +1 @@
-jruby-9.0.0.0
+jruby-9.0.5.0

data/Rakefile

@@ -1,3 +1,11 @@
 require "bundler/gem_tasks"
+require 'rake/testtask'
 
-task default: :build
+desc 'Run test_unit based test'
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.test_files = Dir["test/**/test_*.rb"].sort
+  t.verbose = true
+  t.warning = false
+end
+task :default => :test

data/embulk-input-presto.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-presto"
-  spec.version       = "0.1.1"
+  spec.version       = "0.1.2"
   spec.authors       = ["toyama0919"]
   spec.summary       = "Facebook Presto input plugin for Embulk"
   spec.description   = "Facebook Presto input plugin for Embulk."
@@ -14,7 +14,8 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "presto-client"
-  spec.add_development_dependency 'embulk', ['~> 0.7.10']
+  spec.add_development_dependency 'embulk', ['>= 0.8.9']
   spec.add_development_dependency 'bundler', ['~> 1.0']
   spec.add_development_dependency 'rake', ['>= 10.0']
+  spec.add_development_dependency 'test-unit'
 end

data/lib/embulk/input/presto.rb

@@ -1,59 +1,93 @@
 module Embulk
-  class InputPresto < InputPlugin
-    require 'presto-client'
-
-    Plugin.register_input("presto", self)
-
-    def self.transaction(config, &control)
-      task = {
-        "host" => config.param("host", :string, default: "localhost"),
-        "port" => config.param("port", :integer, default: 8080),
-        "schema" => config.param("schema", :string, default: "default"),
-        "catalog" => config.param("catalog", :string, default: "native"),
-        "query" => config.param("query", :string),
-        "user" => config.param("user", :string, default: "embulk"),
-        "columns" => config.param("columns", :array, default: [])
-      }
-
-      columns = task['columns'].each_with_index.map do |c, i|
-        Column.new(i, c["name"], c["type"].to_sym)
-      end
+  module Input
+    class Presto < InputPlugin
+      require 'presto-client'
 
-      resume(task, columns, 1, &control)
-    end
+      Plugin.register_input("presto", self)
 
-    def self.resume(task, columns, count, &control)
-      task_reports = yield(task, columns, count)
+      def self.transaction(config, &control)
+        task = {
+          "host" => config.param("host", :string, default: "localhost"),
+          "port" => config.param("port", :integer, default: 8080),
+          "schema" => config.param("schema", :string, default: "default"),
+          "catalog" => config.param("catalog", :string, default: "native"),
+          "query" => config.param("query", :string),
+          "user" => config.param("user", :string, default: "embulk"),
+          "columns" => config.param("columns", :array)
+        }
 
-      next_config_diff = {}
-      return next_config_diff
-    end
+        columns = task['columns'].each_with_index.map do |c, i|
+          Column.new(i, c["name"], c["type"].to_sym)
+        end
 
-    def init
-      @client = Presto::Client.new(
-        server: "#{task['host']}:#{task['port']}",
-        catalog: task['catalog'],
-        user: task['user'],
-        schema: task['schema']
-      )
-      @query = task["query"]
+        resume(task, columns, 1, &control)
+      end
 
-      Embulk.logger.info "SQL: #{@query}"
-    end
+      def self.resume(task, columns, count, &control)
+        task_reports = yield(task, columns, count)
 
-    def run
-      size = 0
-      @client.query(@query) do |q|
-        q.each_row {|row|
-          page_builder.add(row)
-        }
-        size = q.rows.size
+        next_config_diff = {}
+        return next_config_diff
       end
 
-      page_builder.finish
+      def init
+        @client = ::Presto::Client.new(
+          server: "#{task['host']}:#{task['port']}",
+          catalog: task['catalog'],
+          user: task['user'],
+          schema: task['schema']
+        )
+        @query = task["query"]
 
-      task_report = { size: size }
-      return task_report
+        Embulk.logger.info "SQL: #{@query}"
+      end
+
+      def run
+        size = 0
+        @client.query(@query) do |q|
+          q.each_row {|row|
+            converted_values = row.map.with_index { |value,i| convert_value(value, schema[i]) }
+            page_builder.add(converted_values)
+          }
+          size = q.rows.size
+        end
+
+        page_builder.finish
+
+        task_report = { size: size }
+        return task_report
+      end
+
+      def convert_value(value, field)
+        return nil if value.nil?
+        case field["type"]
+        when :string
+          value
+        when :long
+          value.to_i
+        when :double
+          value.to_f
+        when :boolean
+          if value.is_a?(TrueClass) || value.is_a?(FalseClass)
+            value
+          else
+            downcased_val = value.downcase
+            case downcased_val
+            when 'true' then true
+            when 'false' then false
+            when '1' then true
+            when '0' then false
+            else nil
+            end
+          end
+        when :timestamp
+          Time.parse(value)
+        when :json
+          value
+        else
+          raise "Unsupported type #{field['type']}"
+        end
+      end
     end
   end
 end

data/test/helper.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require 'test/unit'
+
+# require 'embulk/java/bootstrap'
+require 'embulk'
+Embulk.setup
+Embulk.logger = Embulk::Logger.new('/dev/null')
+
+APP_ROOT = File.expand_path('../', __dir__)
+TEST_ROOT = File.expand_path(File.dirname(__FILE__))

data/test/test_transaction.rb

@@ -0,0 +1,65 @@
+require_relative './helper'
+require 'embulk/input/presto'
+require 'yaml'
+
+Presto = Embulk::Input::Presto
+
+module Embulk
+  class Input::Presto
+    class TestTransaction < Test::Unit::TestCase
+      def control
+        Proc.new {|task| task_reports = [] }
+      end
+
+      sub_test_case "transaction" do
+        def test_normal
+          yaml = YAML.load(%(
+            host: presto01
+            catalog: store
+            schema: public
+            query: |
+              SELECT
+                trim(upper(url_decode(keyword))) AS keyword,
+                count(*) as count
+              FROM search
+              CROSS JOIN UNNEST(split(keywords, ',')) AS t (keyword)
+              WHERE log_date >= (CURRENT_DATE - INTERVAL '90' DAY)
+               AND length(keywords) != 256
+              group by keyword
+              having count(*) >= 10
+              order by count(*) desc
+            columns:
+              - {name: keyword, type: string}
+              - {name: count, type: long}
+            )
+          )
+          config = DataSource.new(yaml)
+          Presto.transaction(config, &control)
+        end
+
+        def test_minimum
+          yaml = YAML.load(%(
+            query: |
+              SELECT
+                trim(upper(url_decode(keyword))) AS keyword,
+                count(*) as count
+              FROM search
+              CROSS JOIN UNNEST(split(keywords, ',')) AS t (keyword)
+              WHERE log_date >= (CURRENT_DATE - INTERVAL '90' DAY)
+               AND length(keywords) != 256
+              group by keyword
+              having count(*) >= 10
+              order by count(*) desc
+            columns:
+              - {name: keyword, type: string}
+              - {name: count, type: long}
+
+            )
+          )
+          config = DataSource.new(yaml)
+          Presto.transaction(config, &control)
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-presto
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - toyama0919
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-12-08 00:00:00.000000000 Z
+date: 2016-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -27,17 +27,17 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.10
+        version: 0.8.9
   name: embulk
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.10
+        version: 0.8.9
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: test-unit
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Facebook Presto input plugin for Embulk.
 email:
 - toyama0919@gmail.com
@@ -81,6 +95,8 @@ files:
 - Rakefile
 - embulk-input-presto.gemspec
 - lib/embulk/input/presto.rb
+- test/helper.rb
+- test/test_transaction.rb
 homepage: https://github.com/toyama0919/embulk-input-presto
 licenses:
 - MIT
@@ -101,8 +117,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.5
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Facebook Presto input plugin for Embulk
-test_files: []
+test_files:
+- test/helper.rb
+- test/test_transaction.rb