embulk-output-vertica 0.1.9 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 16396802fde10566e9de22321179028ab3401763
-  data.tar.gz: d04b85afa6ce9e401f5c8caa193fcc28ff171470
+  metadata.gz: 4f73299beb951f703ae4b6cab3dee30f85ac7314
+  data.tar.gz: 720bc9331a6dba8460f0053563592fd9e3be5071
 SHA512:
-  metadata.gz: 0a739f9edde7e3166cf9992b11328a4b2a86e6187bd1a6a66ccfa82937aebba5e392f62dd2f2a1e8137fa4f2989b542c149f7e2b8ed10f850ab15d9fa8e3bcd3
-  data.tar.gz: 0e1063afeaad0955cd4811f1558c0ad64d6b0c84848257050cb6327aa6d4143a99ccb90d236086bac9833f8acb3237499da0149b4d2a4be4d65b2bc87a253413
+  metadata.gz: 1ec6d31b3b118e1697aa4e6ead1750f44ce2e1c15fc8be33d3c61fa2b476d991124a32c9a14e4f27cc8f8a674028c172d22a9149c2eddff2ba53a33ac83a58a5
+  data.tar.gz: ede9307074e8d52d932614d0d0c9338dae8abbbc829fe7754ddf4a88074b7475f2d83384f36e1ea4c29ef8f8506f595302ca28996566779555607510b831d842

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.2.0 (2015/07/24)
+
+Enhancements:
+
+* Add `default_timezone` option
+
 # 0.1.9 (2015/07/24)
 
 Enhancements:

data/README.md

@@ -20,6 +20,7 @@
 - **copy_mode**: specifies how data is loaded into the database. (`AUTO`, `DIRECT`, or `TRICKLE`. default: AUTO) See vertica documents for details.
 - **abort_on_error**: Stops the COPY command if a row is rejected and rolls back the command. No data is loaded. (bool, default: false)
 - **reject_on_materialized_type_error**: Use `reject_on_materialized_type_error` option for fjsonparser(). This rejects rows if any of olumn types and value types do not fit. ex) double value into INT column fails. See vertica documents for details. (bool, default: false)
+- **default_timezone**: the default timezone for column_options (string, default is "UTC")
 - **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
   - **type**: type of a column when this plugin creates new tables such as `VARCHAR(255)`, `INTEGER NOT NULL UNIQUE`. This is used on creating intermediate tables (insert and truncate_insert modes) and on creating a new target table. (string, default: depends on input column type, see below)
     - boolean:   `BOOLEAN`
@@ -33,8 +34,8 @@
     - double:    `boolean` (true), `long` (to\_i), `double`, `string` (to\_s), `timestamp` (Time.at)
     - string:    `boolean` (true), `long` (to\_i), `double` (to\_f), `string`, `timestamp` (Time.strptime)
     - timestamp: `boolean` (true), `long` (to\_i), `double` (to\_f), `string` (strftime), `timestamp`
-  - **timestamp_format**: If input column type (embulk type) is string and value_type is timestamp or date, this plugin needs the timestamp format of the string. Also, if input column type (embulk type) is timestamp and value_type is string, this plugin needs the timestamp format of the string. 
-  - **timezone**: Specify timezone to convert into `timestamp` or from `timestamp` (string, default is "UTC").
+  - **timestamp_format**: timestamp format to convert into/from `timestamp` (string, default is "%Y-%m-%d %H:%M:%S %z")
+  - **timezone**: timezone to convert into/from `timestamp` (string, default is `default_timezone`).
 
 ### Modes
 
@@ -59,9 +60,9 @@ out:
   copy_mode: DIRECT
   abort_on_error: true
   column_options:
-    id: {type: INT}
+    id:   {type: INT}
     name: {type: VARCHAR(255)}
-    date: {type: DATE, value_type: Date, timezone: "+09:00"}
+    date: {type: DATE, value_type: timezone, timezone: "+09:00"}
 ```
 
 ## Development

data/embulk-output-vertica.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-vertica"
-  spec.version       = "0.1.9"
+  spec.version       = "0.2.0"
   spec.authors       = ["eiji.sekiya", "Naotoshi Seo"]
   spec.email         = ["eiji.sekiya.0326@gmail.com", "sonots@gmail.com"]
   spec.summary       = "Vertica output plugin for Embulk"

data/example.csv

@@ -1,4 +1,4 @@
-date,test,foo,bar,id,name,score
+timestamp_date,string_date,foo,bar,id,name,score
 2015-07-13,2015-07-13,,bar,90,l6lTsvxdlcTfcqx2c0lQSd9HejVQg40W25f0wGNQViY,903.4
 2015-07-13,2015-07-13,,bar,91,XoALSEQg9ycuGqrEWHOb8vdrLbheZSgFO53Wr3mciXY,394.5
 2015-07-13,2015-07-13,,bar,92,0hgDRI_mijs5w7rkiLIe__LEayOOLxL0qVT1IHa5QBw,810.9

data/example.yml

@@ -35,9 +35,11 @@ out:
   copy_mode: direct
   abort_on_error: true
   reject_on_materialized_type_error: true
+  default_timezone: 'Asia/Tokyo'
   column_options:
+    timestamp_date:  {type: DATE}
+    string_date:     {type: TIMESTAMPTZ, value_type: timestamp, timestamp_format: "%Y-%m-%d", timezone: '+08:00'}
     id:              {type: INT}
     name:            {type: VARCHAR}
-    timestamp_date:  {type: DATE, timezone: "+09:00"}
-    string_date:     {type: DATE, value_type: timestamp, timestamp_format: "%Y-%m-%d", timezone: "Asia/Tokyo"}
     score:           {type: INT, value_type: long}
+

data/lib/embulk/output/vertica.rb

@@ -11,18 +11,19 @@ module Embulk
 
       def self.transaction(config, schema, processor_count, &control)
         task = {
-          'host'           => config.param('host',           :string,  :default => 'localhost'),
-          'port'           => config.param('port',           :integer, :default => 5433),
-          'username'       => config.param('username',       :string),
-          'password'       => config.param('password',       :string,  :default => ''),
-          'database'       => config.param('database',       :string,  :default => 'vdb'),
-          'schema'         => config.param('schema',         :string,  :default => 'public'),
-          'table'          => config.param('table',          :string),
-          'mode'           => config.param('mode',           :string,  :default => 'insert'),
-          'copy_mode'      => config.param('copy_mode',      :string,  :default => 'AUTO'),
-          'abort_on_error' => config.param('abort_on_error', :bool,    :default => false),
+          'host'             => config.param('host',           :string,  :default => 'localhost'),
+          'port'             => config.param('port',           :integer, :default => 5433),
+          'username'         => config.param('username',       :string),
+          'password'         => config.param('password',       :string,  :default => ''),
+          'database'         => config.param('database',       :string,  :default => 'vdb'),
+          'schema'           => config.param('schema',         :string,  :default => 'public'),
+          'table'            => config.param('table',          :string),
+          'mode'             => config.param('mode',           :string,  :default => 'insert'),
+          'copy_mode'        => config.param('copy_mode',      :string,  :default => 'AUTO'),
+          'abort_on_error'   => config.param('abort_on_error', :bool,    :default => false),
+          'default_timezone' => config.param('default_timezone', :string, :default => 'UTC'),
+          'column_options'   => config.param('column_options', :hash,    :default => {}),
           'reject_on_materialized_type_error' => config.param('reject_on_materialized_type_error', :bool, :default => false),
-          'column_options' => config.param('column_options', :hash,    :default => {}),
         }
 
         unless %w[INSERT REPLACE].include?(task['mode'].upcase!)
@@ -69,7 +70,7 @@ module Embulk
 
       def initialize(task, schema, index)
         super
-        @converters = ValueConverterFactory.create_converters(schema, task['column_options'])
+        @converters = ValueConverterFactory.create_converters(schema, task['default_timezone'], task['column_options'])
         Embulk.logger.debug { @converters.to_s }
         @jv = self.class.connect(task)
       end
@@ -84,6 +85,7 @@ module Embulk
           copy(@jv, copy_sql) do |stdin|
             page.each do |record|
               json = to_json(record)
+              Embulk.logger.debug { "embulk-output-vertica: #{json}" }
               stdin << json << "\n"
             end
           end

data/lib/embulk/output/vertica/value_converter_factory.rb

@@ -8,17 +8,16 @@ module Embulk
         attr_reader :schema_type, :value_type, :timestamp_format, :timezone, :zone_offset
 
         DEFAULT_TIMESTAMP_FORMAT = "%Y-%m-%d %H:%M:%S %z"
-        DEFAULT_TIMEZONE         = "+00:00"
 
-        def self.create_converters(schema, column_options)
+        def self.create_converters(schema, default_timezone, column_options)
           # @param [Schema] schema embulk defined column types
           # @param [Hash]   column_options user defined column types
           # @return [Array] value converters (array of Proc)
           Hash[*(schema.names.zip(schema.types).map do |column_name, schema_type|
             if column_options[column_name]
               value_type       = column_options[column_name]['value_type']
-              timestamp_format = column_options[column_name]['timestamp_format']
-              timezone         = column_options[column_name]['timezone']
+              timestamp_format = column_options[column_name]['timestamp_format'] || DEFAULT_TIMESTAMP_FORMAT
+              timezone         = column_options[column_name]['timezone'] || default_timezone
               [column_name, self.new(schema_type, value_type, timestamp_format, timezone).create_converter]
             else
               [column_name, Proc.new {|val| val }]
@@ -29,8 +28,8 @@ module Embulk
         def initialize(schema_type, value_type = nil, timestamp_format = nil, timezone = nil)
           @schema_type = schema_type
           @value_type = value_type || schema_type.to_s
-          @timestamp_format = timestamp_format || DEFAULT_TIMESTAMP_FORMAT
-          @timezone = timezone || DEFAULT_TIMEZONE
+          @timestamp_format = timestamp_format
+          @timezone = timezone
           @zone_offset = get_zone_offset(@timezone)
         end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-vertica
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.2.0
 platform: ruby
 authors:
 - eiji.sekiya