embulk-output-vertica 0.1.8 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9514ee560a9a138d1280efcb9f106d0689c3a3c9
-  data.tar.gz: 02dd9130ce7cc4b0eccd2d6d4af59e7900627fb8
+  metadata.gz: 16396802fde10566e9de22321179028ab3401763
+  data.tar.gz: d04b85afa6ce9e401f5c8caa193fcc28ff171470
 SHA512:
-  metadata.gz: bd6b079069cfd1e5ff9d39362c084c6564690557dc4fd2cb8f179ed794d952a632810bddf07e74bf5597859604ddaf8385768211e01c2a1fcde7417757058ba6
-  data.tar.gz: 8de6ec75256fd77d9346c486957593d46bd983c5bd0302d9d2569759836ebf5c91cfb3d59eb9bc2b6fd53fadeb642d4e82c30b0097d0b9c534786af1ab1f64f3
+  metadata.gz: 0a739f9edde7e3166cf9992b11328a4b2a86e6187bd1a6a66ccfa82937aebba5e392f62dd2f2a1e8137fa4f2989b542c149f7e2b8ed10f850ab15d9fa8e3bcd3
+  data.tar.gz: 0e1063afeaad0955cd4811f1558c0ad64d6b0c84848257050cb6327aa6d4143a99ccb90d236086bac9833f8acb3237499da0149b4d2a4be4d65b2bc87a253413

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.1.9 (2015/07/24)
+
+Enhancements:
+
+* Support timezone for string converter
+
 # 0.1.8 (2015/07/24)
 
 Enhancements:

data/README.md

@@ -34,7 +34,7 @@
     - string:    `boolean` (true), `long` (to\_i), `double` (to\_f), `string`, `timestamp` (Time.strptime)
     - timestamp: `boolean` (true), `long` (to\_i), `double` (to\_f), `string` (strftime), `timestamp`
   - **timestamp_format**: If input column type (embulk type) is string and value_type is timestamp or date, this plugin needs the timestamp format of the string. Also, if input column type (embulk type) is timestamp and value_type is string, this plugin needs the timestamp format of the string. 
-  - **timezone**: With format of "+HH:MM" "-HH:MM". `timestamp` column uses this (string, default is "+00:00").
+  - **timezone**: Specify timezone to convert into `timestamp` or from `timestamp` (string, default is "UTC").
 
 ### Modes
 
@@ -64,10 +64,6 @@ out:
     date: {type: DATE, value_type: Date, timezone: "+09:00"}
 ```
 
-## ToDo
-
-* Use timezone for string => timezone conversion
-
 ## Development
 
 Run example:

data/embulk-output-vertica.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-vertica"
-  spec.version       = "0.1.8"
+  spec.version       = "0.1.9"
   spec.authors       = ["eiji.sekiya", "Naotoshi Seo"]
   spec.email         = ["eiji.sekiya.0326@gmail.com", "sonots@gmail.com"]
   spec.summary       = "Vertica output plugin for Embulk"
@@ -14,6 +14,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "jvertica", "~> 0.2"
+  spec.add_dependency "tzinfo"
   spec.add_development_dependency "bundler", "~> 1.7"
   spec.add_development_dependency "rake", "~> 10.0"
 end

data/example.csv

@@ -1,9 +1,9 @@
-date,foo,bar,id,name,score
-2015-07-13,,bar,90,l6lTsvxdlcTfcqx2c0lQSd9HejVQg40W25f0wGNQViY,903.4
-2015-07-13,,bar,91,XoALSEQg9ycuGqrEWHOb8vdrLbheZSgFO53Wr3mciXY,394.5
-2015-07-13,,bar,92,0hgDRI_mijs5w7rkiLIe__LEayOOLxL0qVT1IHa5QBw,810.9
-2015-07-13,,bar,93,KjCRAc-AVcS-R13toBUR6pK_7d9Y8Gl4TRdYYMaSirc,477.4
-2015-07-13,,bar,94,fyQVGlT8Bqmu_LiajPlgfbmavoNyAqXaBsBP_e4OnN8,725.3
-2015-07-13,,bar,95,FpBYRPWKu6DmLpx5tsB25URWfj3sNCbcydNAXULaiD8,316.6
-2015-07-13,,bar,96,9ikvnUqp1Rf2yVwLvs5bBvxQP-KyqxGi4gZRSZ8c1d4,369.5
-2015-07-13,,bar,97,RRNYDAzKaq4Trtt96Bxgk3N0fXLIV8hXoK0qQ7uw_Wc,506.5
+date,test,foo,bar,id,name,score
+2015-07-13,2015-07-13,,bar,90,l6lTsvxdlcTfcqx2c0lQSd9HejVQg40W25f0wGNQViY,903.4
+2015-07-13,2015-07-13,,bar,91,XoALSEQg9ycuGqrEWHOb8vdrLbheZSgFO53Wr3mciXY,394.5
+2015-07-13,2015-07-13,,bar,92,0hgDRI_mijs5w7rkiLIe__LEayOOLxL0qVT1IHa5QBw,810.9
+2015-07-13,2015-07-13,,bar,93,KjCRAc-AVcS-R13toBUR6pK_7d9Y8Gl4TRdYYMaSirc,477.4
+2015-07-13,2015-07-13,,bar,94,fyQVGlT8Bqmu_LiajPlgfbmavoNyAqXaBsBP_e4OnN8,725.3
+2015-07-13,2015-07-13,,bar,95,FpBYRPWKu6DmLpx5tsB25URWfj3sNCbcydNAXULaiD8,316.6
+2015-07-13,2015-07-13,,bar,96,9ikvnUqp1Rf2yVwLvs5bBvxQP-KyqxGi4gZRSZ8c1d4,369.5
+2015-07-13,2015-07-13,,bar,97,RRNYDAzKaq4Trtt96Bxgk3N0fXLIV8hXoK0qQ7uw_Wc,506.5

data/example.yml

@@ -16,7 +16,8 @@ in:
     skip_header_lines: 1
     comment_line_marker: '#'
     columns:
-      - {name: date,  type: timestamp, format: "%Y-%m-%d"}
+      - {name: timestamp_date, type: timestamp, format: "%Y-%m-%d", timezone: "+09:00"}
+      - {name: string_date,    type: string}
       - {name: foo,   type: string}
       - {name: bar,   type: string}
       - {name: id,    type: long}
@@ -35,6 +36,8 @@ out:
   abort_on_error: true
   reject_on_materialized_type_error: true
   column_options:
-    date: {type: DATE}
-    id: {type: INT}
-    name: {type: VARCHAR}
+    id:              {type: INT}
+    name:            {type: VARCHAR}
+    timestamp_date:  {type: DATE, timezone: "+09:00"}
+    string_date:     {type: DATE, value_type: timestamp, timestamp_format: "%Y-%m-%d", timezone: "Asia/Tokyo"}
+    score:           {type: INT, value_type: long}

data/lib/embulk/output/vertica/value_converter_factory.rb

@@ -1,8 +1,11 @@
+require 'time'
+require 'tzinfo'
+
 module Embulk
   module Output
     class Vertica < OutputPlugin
       class ValueConverterFactory
-        attr_reader :schema_type, :value_type, :timestamp_format, :timezone
+        attr_reader :schema_type, :value_type, :timestamp_format, :timezone, :zone_offset
 
         DEFAULT_TIMESTAMP_FORMAT = "%Y-%m-%d %H:%M:%S %z"
         DEFAULT_TIMEZONE         = "+00:00"
@@ -26,8 +29,9 @@ module Embulk
         def initialize(schema_type, value_type = nil, timestamp_format = nil, timezone = nil)
           @schema_type = schema_type
           @value_type = value_type || schema_type.to_s
-          @timestampt_format = timestamp_format || DEFAULT_TIMESTAMP_FORMAT
+          @timestamp_format = timestamp_format || DEFAULT_TIMESTAMP_FORMAT
           @timezone = timezone || DEFAULT_TIMEZONE
+          @zone_offset = get_zone_offset(@timezone)
         end
 
         def create_converter
@@ -55,7 +59,7 @@ module Embulk
           when 'long'       then Proc.new {|val| val }
           when 'double'     then Proc.new {|val| val.to_f }
           when 'string'     then Proc.new {|val| val.to_s }
-          when 'timestamp'  then Proc.new {|val| Time.at(val).localtime(timezone) }
+          when 'timestamp'  then Proc.new {|val| Time.at(val).localtime(zone_offset) }
           else raise NotSupportedType, "embulk-output-vertica cannot take column value_type #{value_type} for long column"
           end
         end
@@ -66,7 +70,7 @@ module Embulk
           when 'long'      then Proc.new {|val| val.to_i }
           when 'double'    then Proc.new {|val| val }
           when 'string'    then Proc.new {|val| val.to_s }
-          when 'timestamp' then Proc.new {|val| Time.at(val).localtime(timezone) }
+          when 'timestamp' then Proc.new {|val| Time.at(val).localtime(zone_offset) }
           else raise NotSupportedType, "embulk-output-vertica cannot take column value_type #{value_type} for double column"
           end
         end
@@ -77,7 +81,7 @@ module Embulk
           when 'long'      then Proc.new {|val| val.to_i }
           when 'double'    then Proc.new {|val| val.to_f }
           when 'string'    then Proc.new {|val| val }
-          when 'timestamp' then Proc.new {|val| Time.strptime(val, timestamp_format) } # ToDo: timezone
+          when 'timestamp' then Proc.new {|val| strptime_with_zone(val, timestamp_format, zone_offset) }
           else raise NotSupportedType, "embulk-output-vertica cannot take column value_type #{value_type} for string column"
           end
         end
@@ -87,11 +91,36 @@ module Embulk
           when 'boolean'   then Proc.new {|val| !!val }
           when 'long'      then Proc.new {|val| val.to_i }
           when 'double'    then Proc.new {|val| val.to_f }
-          when 'string'    then Proc.new {|val| val.localtime(timezone).strftime(timestamp_format) }
-          when 'timestamp' then Proc.new {|val| val.localtime(timezone) }
+          when 'string'    then Proc.new {|val| val.localtime(zone_offset).strftime(timestamp_format) }
+          when 'timestamp' then Proc.new {|val| val.localtime(zone_offset) }
           else raise NotSupportedType, "embulk-output-vertica cannot take column value_type #{value_type} for timesatmp column"
           end
         end
+
+        private
+        
+        # [+-]HH:MM, [+-]HHMM, [+-]HH
+        NUMERIC_PATTERN = %r{\A[+-]\d\d(:?\d\d)?\z}
+
+        # Region/Zone, Region/Zone/Zone
+        NAME_PATTERN = %r{\A[^/]+/[^/]+(/[^/]+)?\z}
+
+        def strptime_with_zone(date, timestamp_format, zone_offset)
+          time = Time.strptime(date, timestamp_format)
+          utc_offset = time.utc_offset
+          time.localtime(zone_offset) + utc_offset - zone_offset
+        end
+
+        def get_zone_offset(timezone)
+          if NUMERIC_PATTERN === timezone
+            Time.zone_offset(timezone)
+          elsif NAME_PATTERN === timezone
+            tz = TZInfo::Timezone.get(timezone)
+            tz.period_for_utc(Time.now).utc_total_offset
+          else
+            raise ArgumentError, "timezone format is invalid: #{timezone}"
+          end
+        end
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-vertica
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.1.9
 platform: ruby
 authors:
 - eiji.sekiya
@@ -25,6 +25,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.2'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: tzinfo
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: