embulk-output-vertica 0.5.6 → 0.5.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4fb8733ef4ad44731139ff6f8cb63aa7dfb6aa1c
-  data.tar.gz: b3d80ae9599a03762ac7e65cc2bcfb033f51ecae
+  metadata.gz: daddcdfee4c5483f5844d5d1f5b88e6550d482ab
+  data.tar.gz: 359f2050bb2fc49c6a34d51f4328e17dd17a16cf
 SHA512:
-  metadata.gz: c84bb2d342b70fbbfbbd5656e41e3942115ed25fa2e786bdf7c9d10bf4540cabd0370cfc5d339c107657f71bb8b4b5365eac390d7d104f9f2e905ba3411867e9
-  data.tar.gz: 92df6faa9d4b0b2be4e6085cf6b2f87986c9f6636ce622f4c901d7e691133d691f5f286578ed48a87a40bfffce49653477abeb26849d272f9e48dea4cebfb148
+  metadata.gz: 834eaf89a88c7eface737474dded51f134577b10c3891564f710a9e3f0c86b473b6f7a185ad24b01b5e2a89954e3a7344f0177a7a06ed67c156eb3323eba34ce
+  data.tar.gz: 3a435a2d1f8c0a3c3c4307b38e35ce728e99b5ae9beb78b1ebf208e62f9fd4d6c8f58759adc4b33cb423ffc88b4b3a235ae231aa5ad5a26d66aba7f5ed9c7c40

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.5.7 (2016/01/22)
+
+Enhancements:
+
+* Add json_payload option to avoid construction of JSON in this jruby plugin for performance improvement
+
 # 0.5.6 (2016/01/22)
 
 Enhancements:

data/Gemfile

@@ -5,3 +5,4 @@ gem 'embulk-input-random'
 gem 'embulk-filter-stdout'
 gem 'pry'
 gem 'pry-nav'
+gem 'embulk-parser-none'

data/README.md

@@ -38,6 +38,7 @@
     - timestamp: `boolean`, `long`, `double`, `string`, `timestamp`
   - **timestamp_format**: timestamp format to convert into/from `timestamp` (string, default is "%Y-%m-%d %H:%M:%S %z")
   - **timezone**: timezone to convert into/from `timestamp` (string, default is `default_timezone`).
+- **json_payload**: Assuming first columns of records are json string, COPY each json into vertica directly using fjsonparser. This would improve performance by avoiding construction of json in this jruby plugin. ToDo: auto table creation is not supported for this mode yet (bool, default is false)
 
 ### Modes
 

data/embulk-output-vertica.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-vertica"
-  spec.version       = "0.5.6"
+  spec.version       = "0.5.7"
   spec.authors       = ["eiji.sekiya", "Naotoshi Seo"]
   spec.email         = ["eiji.sekiya.0326@gmail.com", "sonots@gmail.com"]
   spec.summary       = "Vertica output plugin for Embulk"

data/example/example.jsonl

@@ -0,0 +1,8 @@
+{"timestamp_date":"2015-07-13","string_date":"2015-07-13","foo":null,"bar":"bar","id":"90","name":"l6lTsvxdlcTfcqx2c0lQSd9HejVQg40W25f0wGNQViY","score":"903.4"}
+{"timestamp_date":"2015-07-13","string_date":"2015-07-13","foo":null,"bar":"bar","id":"91","name":"XoALSEQg9ycuGqrEWHOb8vdrLbheZSgFO53Wr3mciXY","score":"394.5"}
+{"timestamp_date":"2015-07-13","string_date":"2015-07-13","foo":null,"bar":"bar","id":"92","name":"0hgDRI_mijs5w7rkiLIe__LEayOOLxL0qVT1IHa5QBw","score":"810.9"}
+{"timestamp_date":"2015-07-13","string_date":"2015-07-13","foo":null,"bar":"bar","id":"93","name":"KjCRAc-AVcS-R13toBUR6pK_7d9Y8Gl4TRdYYMaSirc","score":"477.4"}
+{"timestamp_date":"2015-07-13","string_date":"2015-07-13","foo":null,"bar":"bar","id":"94","name":"fyQVGlT8Bqmu_LiajPlgfbmavoNyAqXaBsBP_e4OnN8","score":"725.3"}
+{"timestamp_date":"2015-07-13","string_date":"2015-07-13","foo":null,"bar":"bar","id":"95","name":"FpBYRPWKu6DmLpx5tsB25URWfj3sNCbcydNAXULaiD8","score":"316.6"}
+{"timestamp_date":"2015-07-13","string_date":"2015-07-13","foo":null,"bar":"bar","id":"96","name":"9ikvnUqp1Rf2yVwLvs5bBvxQP-KyqxGi4gZRSZ8c1d4","score":"369.5"}
+{"timestamp_date":"2015-07-13","string_date":"2015-07-13","foo":null,"bar":"bar","id":"97","name":"RRNYDAzKaq4Trtt96Bxgk3N0fXLIV8hXoK0qQ7uw_Wc","score":"506.5"}

data/example/json_payload.yml

@@ -0,0 +1,20 @@
+in:
+  type: file
+  path_prefix: example/example.jsonl
+  parser:
+    type: none
+out:
+  type: vertica 
+  host: 127.0.0.1
+  user: dbadmin
+  password: xxxxxxx
+  database: vdb
+  schema: sandbox
+  table: embulk_test
+  copy_mode: DIRECT
+  abort_on_error: true
+  reject_on_materialized_type_error: true
+  default_timezone: 'Asia/Tokyo'
+  pool: 3
+  compress: GZIP
+  json_payload: true

data/lib/embulk/output/vertica.rb

@@ -30,6 +30,7 @@ module Embulk
           'compress'         => config.param('compress',         :string,  :default => 'UNCOMPRESSED'),
           'default_timezone' => config.param('default_timezone', :string, :default => 'UTC'),
           'column_options'   => config.param('column_options',   :hash,    :default => {}),
+          'json_payload'     => config.param('json_payload',     :bool,    :default => false),
           'reject_on_materialized_type_error' => config.param('reject_on_materialized_type_error', :bool, :default => false),
           'pool'             => config.param('pool',             :integer, :default => processor_count),
         }
@@ -67,12 +68,14 @@ module Embulk
         quoted_table      = ::Jvertica.quote_identifier(task['table'])
         quoted_temp_table = ::Jvertica.quote_identifier(task['temp_table'])
 
-        sql_schema_table = self.sql_schema_from_embulk_schema(schema, task['column_options'])
+        unless task['json_payload'] # ToDo: auto table creation is not supported to json_payload mode yet
+          sql_schema_table = self.sql_schema_from_embulk_schema(schema, task['column_options'])
 
-        # create the target table
-        connect(task) do |jv|
-          query(jv, %[DROP TABLE IF EXISTS #{quoted_schema}.#{quoted_table}]) if task['mode'] == 'REPLACE'
-          query(jv, %[CREATE TABLE IF NOT EXISTS #{quoted_schema}.#{quoted_table} (#{sql_schema_table})])
+          # create the target table
+          connect(task) do |jv|
+            query(jv, %[DROP TABLE IF EXISTS #{quoted_schema}.#{quoted_table}]) if task['mode'] == 'REPLACE'
+            query(jv, %[CREATE TABLE IF NOT EXISTS #{quoted_schema}.#{quoted_table} (#{sql_schema_table})])
+          end
         end
 
         sql_schema_temp_table = self.sql_schema_from_table(task)

data/lib/embulk/output/vertica/output_thread.rb

@@ -5,6 +5,7 @@ module Embulk
     class Vertica < OutputPlugin
       class OutputThreadPool
         def initialize(task, schema, size)
+          @task = task
           @size = size
           @schema = schema
           @converters = ValueConverterFactory.create_converters(schema, task['default_timezone'], task['column_options'])
@@ -33,9 +34,13 @@ module Embulk
         end
 
         def to_json(record)
-          Hash[*(@schema.names.zip(record).map do |column_name, value|
-            [column_name, @converters[column_name].call(value)]
-          end.flatten!(1))].to_json
+          if @task['json_payload']
+            record.first
+          else
+            Hash[*(@schema.names.zip(record).map do |column_name, value|
+              [column_name, @converters[column_name].call(value)]
+            end.flatten!(1))].to_json
+          end
         end
       end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-vertica
 version: !ruby/object:Gem::Version
-  version: 0.5.6
+  version: 0.5.7
 platform: ruby
 authors:
 - eiji.sekiya
@@ -84,7 +84,9 @@ files:
 - embulk-output-vertica.gemspec
 - example/abort.yml
 - example/example.csv
+- example/example.jsonl
 - example/example.yml
+- example/json_payload.yml
 - lib/embulk/output/vertica.rb
 - lib/embulk/output/vertica/output_thread.rb
 - lib/embulk/output/vertica/value_converter_factory.rb