embulk-output-vertica 0.1.7 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: daadcaba0204e6e4c1103f4edd8e5b3c6b70924f
-  data.tar.gz: 339ab9e6350031e1325ac69d2773f45bc6f7e0b0
+  metadata.gz: 9514ee560a9a138d1280efcb9f106d0689c3a3c9
+  data.tar.gz: 02dd9130ce7cc4b0eccd2d6d4af59e7900627fb8
 SHA512:
-  metadata.gz: ecec7a142322fdff6596c3359fb67949235d8082ae626e491df8ed5cce55961777eb7715cb69a7d12243f3305ec97595a97c3e820188f6198f3a68e43595e01f
-  data.tar.gz: 7d17307369af315a30ab296aad4bb94da6ff08806db4f73fef541c1aa710326a7e251804e730004c854be97a68a194059306867b484f607af7864e02210e9bd5
+  metadata.gz: bd6b079069cfd1e5ff9d39362c084c6564690557dc4fd2cb8f179ed794d952a632810bddf07e74bf5597859604ddaf8385768211e01c2a1fcde7417757058ba6
+  data.tar.gz: 8de6ec75256fd77d9346c486957593d46bd983c5bd0302d9d2569759836ebf5c91cfb3d59eb9bc2b6fd53fadeb642d4e82c30b0097d0b9c534786af1ab1f64f3

data/CHANGELOG.md

@@ -1,16 +1,22 @@
-# 0.1.7 (2015/07/25)
+# 0.1.8 (2015/07/24)
+
+Enhancements:
+
+* Support value_type, timezone_format, timezone option for column_options[
+
+# 0.1.7 (2015/07/24)
 
 Enhancements:
 
 * Add reject_on_materialized_type_error option
 
-# 0.1.6 (2015/07/24)
+# 0.1.6 (2015/07/23)
 
 Enhancements:
 
 * Enhancement of debug log
 
-# 0.1.5 (2015/07/24)
+# 0.1.5 (2015/07/23)
 
 Fixes:
 

data/README.md

@@ -22,11 +22,19 @@
 - **reject_on_materialized_type_error**: Use `reject_on_materialized_type_error` option for fjsonparser(). This rejects rows if any of olumn types and value types do not fit. ex) double value into INT column fails. See vertica documents for details. (bool, default: false)
 - **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
   - **type**: type of a column when this plugin creates new tables such as `VARCHAR(255)`, `INTEGER NOT NULL UNIQUE`. This is used on creating intermediate tables (insert and truncate_insert modes) and on creating a new target table. (string, default: depends on input column type, see below)
-    - long: `INT` (same with `BIGINT` in vertica)
-    - boolean: `BOOLEAN`
-    - double: `FLOAT` (same with `DOUBLE PRECISION` in vertica)
-    - string: `VARCHAR`
+    - boolean:   `BOOLEAN`
+    - long:      `INT` (same with `BIGINT` in vertica)
+    - double:    `FLOAT` (same with `DOUBLE PRECISION` in vertica)
+    - string:    `VARCHAR`
     - timestamp: `TIMESTAMP`
+  - **value_type**:  The types (embulk types) of values to convert (string, default: no conversion. See below for available types)
+    - boolean:   `boolean`, `string` (to\_s)
+    - long:      `boolean` (true), `long`, `double` (to\_f), `string` (to\_s), `timestamp` (Time.at)
+    - double:    `boolean` (true), `long` (to\_i), `double`, `string` (to\_s), `timestamp` (Time.at)
+    - string:    `boolean` (true), `long` (to\_i), `double` (to\_f), `string`, `timestamp` (Time.strptime)
+    - timestamp: `boolean` (true), `long` (to\_i), `double` (to\_f), `string` (strftime), `timestamp`
+  - **timestamp_format**: If input column type (embulk type) is string and value_type is timestamp or date, this plugin needs the timestamp format of the string. Also, if input column type (embulk type) is timestamp and value_type is string, this plugin needs the timestamp format of the string. 
+  - **timezone**: With format of "+HH:MM" "-HH:MM". `timestamp` column uses this (string, default is "+00:00").
 
 ### Modes
 
@@ -53,8 +61,12 @@ out:
   column_options:
     id: {type: INT}
     name: {type: VARCHAR(255)}
+    date: {type: DATE, value_type: Date, timezone: "+09:00"}
 ```
 
+## ToDo
+
+* Use timezone for string => timezone conversion
 
 ## Development
 

data/embulk-output-vertica.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-vertica"
-  spec.version       = "0.1.7"
+  spec.version       = "0.1.8"
   spec.authors       = ["eiji.sekiya", "Naotoshi Seo"]
   spec.email         = ["eiji.sekiya.0326@gmail.com", "sonots@gmail.com"]
   spec.summary       = "Vertica output plugin for Embulk"

data/lib/embulk/output/vertica/value_converter_factory.rb

@@ -0,0 +1,98 @@
+module Embulk
+  module Output
+    class Vertica < OutputPlugin
+      class ValueConverterFactory
+        attr_reader :schema_type, :value_type, :timestamp_format, :timezone
+
+        DEFAULT_TIMESTAMP_FORMAT = "%Y-%m-%d %H:%M:%S %z"
+        DEFAULT_TIMEZONE         = "+00:00"
+
+        def self.create_converters(schema, column_options)
+          # @param [Schema] schema embulk defined column types
+          # @param [Hash]   column_options user defined column types
+          # @return [Array] value converters (array of Proc)
+          Hash[*(schema.names.zip(schema.types).map do |column_name, schema_type|
+            if column_options[column_name]
+              value_type       = column_options[column_name]['value_type']
+              timestamp_format = column_options[column_name]['timestamp_format']
+              timezone         = column_options[column_name]['timezone']
+              [column_name, self.new(schema_type, value_type, timestamp_format, timezone).create_converter]
+            else
+              [column_name, Proc.new {|val| val }]
+            end
+          end.flatten!(1))]
+        end
+
+        def initialize(schema_type, value_type = nil, timestamp_format = nil, timezone = nil)
+          @schema_type = schema_type
+          @value_type = value_type || schema_type.to_s
+          @timestampt_format = timestamp_format || DEFAULT_TIMESTAMP_FORMAT
+          @timezone = timezone || DEFAULT_TIMEZONE
+        end
+
+        def create_converter
+          case schema_type
+          when :boolean   then boolean_converter
+          when :long      then long_converter
+          when :double    then double_converter
+          when :string    then string_converter
+          when :timestamp then timestamp_converter
+          else raise NotSupportedType, "embulk-output-vertica cannot take column type #{schema_type}"
+          end
+        end
+
+        def boolean_converter
+          case value_type
+          when 'boolean' then Proc.new {|val| val }
+          when 'string'  then Proc.new {|val| val.to_s }
+          else raise NotSupportedType, "embulk-output-vertica cannot take column value_type #{value_type} for boolean column"
+          end
+        end
+
+        def long_converter
+          case value_type
+          when 'boolean'    then Proc.new {|val| !!val }
+          when 'long'       then Proc.new {|val| val }
+          when 'double'     then Proc.new {|val| val.to_f }
+          when 'string'     then Proc.new {|val| val.to_s }
+          when 'timestamp'  then Proc.new {|val| Time.at(val).localtime(timezone) }
+          else raise NotSupportedType, "embulk-output-vertica cannot take column value_type #{value_type} for long column"
+          end
+        end
+
+        def double_converter
+          case value_type
+          when 'boolean'   then Proc.new {|val| !!val }
+          when 'long'      then Proc.new {|val| val.to_i }
+          when 'double'    then Proc.new {|val| val }
+          when 'string'    then Proc.new {|val| val.to_s }
+          when 'timestamp' then Proc.new {|val| Time.at(val).localtime(timezone) }
+          else raise NotSupportedType, "embulk-output-vertica cannot take column value_type #{value_type} for double column"
+          end
+        end
+
+        def string_converter
+          case value_type
+          when 'boolean'   then Proc.new {|val| !!val }
+          when 'long'      then Proc.new {|val| val.to_i }
+          when 'double'    then Proc.new {|val| val.to_f }
+          when 'string'    then Proc.new {|val| val }
+          when 'timestamp' then Proc.new {|val| Time.strptime(val, timestamp_format) } # ToDo: timezone
+          else raise NotSupportedType, "embulk-output-vertica cannot take column value_type #{value_type} for string column"
+          end
+        end
+
+        def timestamp_converter
+          case value_type
+          when 'boolean'   then Proc.new {|val| !!val }
+          when 'long'      then Proc.new {|val| val.to_i }
+          when 'double'    then Proc.new {|val| val.to_f }
+          when 'string'    then Proc.new {|val| val.localtime(timezone).strftime(timestamp_format) }
+          when 'timestamp' then Proc.new {|val| val.localtime(timezone) }
+          else raise NotSupportedType, "embulk-output-vertica cannot take column value_type #{value_type} for timesatmp column"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/embulk/output/vertica.rb

@@ -1,4 +1,5 @@
 require 'jvertica'
+require_relative 'vertica/value_converter_factory'
 
 module Embulk
   module Output
@@ -68,6 +69,8 @@ module Embulk
 
       def initialize(task, schema, index)
         super
+        @converters = ValueConverterFactory.create_converters(schema, task['column_options'])
+        Embulk.logger.debug { @converters.to_s }
         @jv = self.class.connect(task)
       end
 
@@ -172,7 +175,9 @@ module Embulk
       end
 
       def to_json(record)
-        Hash[*(schema.names.zip(record).flatten!(1))].to_json
+        Hash[*(schema.names.zip(record).map do |column_name, value|
+          [column_name, @converters[column_name].call(value)]
+        end.flatten!(1))].to_json
       end
 
       def quoted_schema

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-vertica
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.8
 platform: ruby
 authors:
 - eiji.sekiya
@@ -71,6 +71,7 @@ files:
 - example.csv
 - example.yml
 - lib/embulk/output/vertica.rb
+- lib/embulk/output/vertica/value_converter_factory.rb
 homepage: https://github.com/eratostennis/embulk-output-vertica
 licenses:
 - MIT