embulk-input-presto 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1a57f6dc5a7064378aba3325b8765ffc3e3f1cdd
-  data.tar.gz: f272abf3aad3768b0603366e404685cf33d4e61d
+  metadata.gz: 042574a4df2a41bc869a625661001f50d0b371e6
+  data.tar.gz: 5e9536ea331fa538d6a09d390fa7cf07ea057320
 SHA512:
-  metadata.gz: 13ce34fd728876372f389ac13cf422d23b8facba8c3069b60f946f4762cc37e003e2493ec5c1129ca10fdab20a8ed0ed860028a9109e02a5afd6bb62e9e30ccc
-  data.tar.gz: 1898c4f776b9570ecf9813372cfd80ff2c62c339f0872e633d85a722e8a844d911b1bb0716489cf243cde6b25aa6fc2452a8c628d3877cc6b13399a8924b279a
+  metadata.gz: 17baab4b601e67d21c8b9fe91c23924594f1ccffa5ed22cd65a937ad3909edbf7cf3d070d0c7b1eb2bad965d757d3f2ea1ea6870748972990b9da4014578007f
+  data.tar.gz: 51b93b3c578d298f35399b1127a236cc64ea393aee7bef3088c62032c6b7e64640a61205ca5f4b27c5f4862a153d0212566080aec05255aca423f1f1e2bdcf84

data/.travis.yml

@@ -0,0 +1,12 @@
+language: ruby
+cache: bundler
+rvm:
+  - jruby-9.0.5.0
+  - jruby-head
+jdk:
+  - openjdk7
+before_install:
+  - gem install bundler
+matrix:
+  allow_failures:
+    - rvm: jruby-head

data/README.md

@@ -1,4 +1,4 @@
-# Facebook Presto input plugin for Embulk
+# Facebook Presto input plugin for Embulk [![Build Status](https://secure.travis-ci.org/toyama0919/embulk-input-presto.png?branch=master)](http://travis-ci.org/toyama0919/embulk-input-presto)
 
 Facebook Presto input plugin for Embulk.
 [see](https://prestodb.io/).
@@ -18,7 +18,11 @@ Facebook Presto input plugin for Embulk.
 - **catalog**: catalog (string, default: `"native"`)
 - **query**: query (string, required)
 - **user**: user (string, default: `"embulk"`)
-- **columns**: columns (array, required)
+- **columns**(**deprecated**): columns (array, required)
+  - **name**: name (string, required)
+  - **type**: type (string, required)
+
+**Warning** : **columns** deprecated since over v0.2.0. Support auto fetch schema.
 
 ## Example
 
@@ -39,23 +43,17 @@ in:
     group by keyword
     having count(*) >= 10
     order by count(*) desc
-  columns:
-    - {name: keyword, type: string}
-    - {name: count, type: long}
 out:
   type: stdout
 ```
 
-## Limited
-* Only the data type that Embulk supports is possible.
-  * TIMESTAMP
-  * LONG
-  * DOUBLE
-  * BOOLEAN
-  * STRING
-
-* Presto is not support Prepared statement.
-  * Can't fetch schema by sql
+## Support type
+* TIMESTAMP
+* LONG
+* DOUBLE
+* BOOLEAN
+* STRING
+* JSON
 
 ## Build
 

data/embulk-input-presto.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-presto"
-  spec.version       = "0.1.2"
+  spec.version       = "0.2.0"
   spec.authors       = ["toyama0919"]
   spec.summary       = "Facebook Presto input plugin for Embulk"
   spec.description   = "Facebook Presto input plugin for Embulk."

data/lib/embulk/input/presto.rb

@@ -1,3 +1,7 @@
+require_relative 'presto/type_converter'
+require_relative 'presto/explain_parser'
+require_relative 'presto/connection'
+
 module Embulk
   module Input
     class Presto < InputPlugin
@@ -13,11 +17,15 @@ module Embulk
           "catalog" => config.param("catalog", :string, default: "native"),
           "query" => config.param("query", :string),
           "user" => config.param("user", :string, default: "embulk"),
-          "columns" => config.param("columns", :array)
+          "columns" => config.param("columns", :array, default: nil)
         }
 
-        columns = task['columns'].each_with_index.map do |c, i|
-          Column.new(i, c["name"], c["type"].to_sym)
+        columns = if task['columns']
+          task['columns'].each_with_index.map do |c, i|
+            Column.new(i, c["name"], c["type"].to_sym)
+          end
+        else
+          build_output_columns(task)
         end
 
         resume(task, columns, 1, &control)
@@ -30,14 +38,22 @@ module Embulk
         return next_config_diff
       end
 
+      def self.build_output_columns(task)
+        explain_query = "explain (FORMAT TEXT) " + task["query"]
+        Embulk.logger.debug("SQL: #{explain_query}")
+        explain_result = Connection.get_client(task).run("explain (FORMAT TEXT) " + task["query"])
+
+        columns = []
+        ExplainParser.parse(explain_result).each_with_index do |(name, type), i|
+          columns << Column.new(i, name, TypeConverter.get_type(type))
+        end
+        columns
+      end
+
       def init
-        @client = ::Presto::Client.new(
-          server: "#{task['host']}:#{task['port']}",
-          catalog: task['catalog'],
-          user: task['user'],
-          schema: task['schema']
-        )
+        @client = Connection.get_client(task)
         @query = task["query"]
+        @type_converter = TypeConverter.new
 
         Embulk.logger.info "SQL: #{@query}"
       end
@@ -46,7 +62,7 @@ module Embulk
         size = 0
         @client.query(@query) do |q|
           q.each_row {|row|
-            converted_values = row.map.with_index { |value,i| convert_value(value, schema[i]) }
+            converted_values = row.map.with_index { |value,i| @type_converter.convert_value(value, schema[i]) }
             page_builder.add(converted_values)
           }
           size = q.rows.size
@@ -57,37 +73,6 @@ module Embulk
         task_report = { size: size }
         return task_report
       end
-
-      def convert_value(value, field)
-        return nil if value.nil?
-        case field["type"]
-        when :string
-          value
-        when :long
-          value.to_i
-        when :double
-          value.to_f
-        when :boolean
-          if value.is_a?(TrueClass) || value.is_a?(FalseClass)
-            value
-          else
-            downcased_val = value.downcase
-            case downcased_val
-            when 'true' then true
-            when 'false' then false
-            when '1' then true
-            when '0' then false
-            else nil
-            end
-          end
-        when :timestamp
-          Time.parse(value)
-        when :json
-          value
-        else
-          raise "Unsupported type #{field['type']}"
-        end
-      end
     end
   end
 end

data/lib/embulk/input/presto/connection.rb

@@ -0,0 +1,16 @@
+module Embulk
+  module Input
+    class Presto < InputPlugin
+      class Connection
+        def self.get_client(task)
+          ::Presto::Client.new(
+            server: "#{task['host']}:#{task['port']}",
+            catalog: task['catalog'],
+            user: task['user'],
+            schema: task['schema']
+          )
+        end
+      end
+    end
+  end
+end

data/lib/embulk/input/presto/explain_parser.rb

@@ -0,0 +1,15 @@
+module Embulk
+  module Input
+    class Presto < InputPlugin
+      class ExplainParser
+        def self.parse(explain_result)
+          explain_text = explain_result.flatten.last.lines.first
+          column_name_raw, column_type_raw = explain_text.split(' => ')
+          names = column_name_raw.split('[').last.split(']').first.split(',').map{ |name| name.strip }
+          types = column_type_raw.split('[').last.split(']').first.split(',').map{ |info| info.split(':').last }
+          Hash[*names.zip(types).flatten]
+        end
+      end
+    end
+  end
+end

data/lib/embulk/input/presto/type_converter.rb

@@ -0,0 +1,80 @@
+module Embulk
+  module Input
+    class Presto < InputPlugin
+      class TypeConverter
+
+        def initialize
+        end
+
+        def convert_value(value, field)
+          return nil if value.nil?
+          case field["type"]
+          when :string
+            value
+          when :long
+            value.to_i
+          when :double
+            value.to_f
+          when :boolean
+            if value.is_a?(TrueClass) || value.is_a?(FalseClass)
+              value
+            else
+              downcased_val = value.downcase
+              case downcased_val
+              when 'true' then true
+              when 'false' then false
+              when '1' then true
+              when '0' then false
+              else nil
+              end
+            end
+          when :timestamp
+            Time.parse(value)
+          when :json
+            value
+          else
+            raise "Unsupported type #{field['type']}"
+          end
+        end
+
+        def self.get_type(type)
+          if type.start_with?("boolean")
+            :boolean
+          elsif type.start_with?("bigint")
+            :long
+          elsif type.start_with?("double")
+            :double
+          elsif type.start_with?("decimal")
+            :double
+          elsif type.start_with?("varchar")
+            :string
+          elsif type.start_with?("varbinary")
+            :string
+          elsif type.start_with?("json")
+            :json
+          elsif type.start_with?("date")
+            :timestamp
+          elsif type.start_with?("time")
+            :timestamp
+          elsif type.start_with?("time with time zone")
+            :timestamp
+          elsif type.start_with?("timestamp")
+            :timestamp
+          elsif type.start_with?("timestamp with time zone")
+            :timestamp
+          elsif type.start_with?("interval year to month")
+            :timestamp
+          elsif type.start_with?("interval day to second")
+            :timestamp
+          elsif type.start_with?("array")
+            :json
+          elsif type.start_with?("map")
+            :json
+          elsif type.start_with?("row")
+            :json
+          end
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-presto
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - toyama0919
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-06-22 00:00:00.000000000 Z
+date: 2016-06-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -89,12 +89,16 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".ruby-version"
+- ".travis.yml"
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - embulk-input-presto.gemspec
 - lib/embulk/input/presto.rb
+- lib/embulk/input/presto/connection.rb
+- lib/embulk/input/presto/explain_parser.rb
+- lib/embulk/input/presto/type_converter.rb
 - test/helper.rb
 - test/test_transaction.rb
 homepage: https://github.com/toyama0919/embulk-input-presto