embulk-output-bigquery 0.6.2 → 0.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2168730943154d9fb8d8ebfce9e4a1c2130b16b5
-  data.tar.gz: 8c4549b91f75d3e7a874f310e0df791bd9c28030
+  metadata.gz: 8b3d7d7d675d8428946f81517d1002f667f4fafe
+  data.tar.gz: 25940b93f70492675869d3c4dd50f83f8b7347cf
 SHA512:
-  metadata.gz: ae7e67855daddce745e3d9a8c5ee659d99c7e84510365bf216f63fb7ed92b9ce166cb2b005b76680020f67354a6f97cb3596d9b15490630cf1b927968e5b8f0e
-  data.tar.gz: 83531a34355dbf3af0ec602db9318ad52e0b6b9ba84b5ea1bef15f4a50b67fa69b664e0bf62deffe8f37ac62d9b995a6c5a135b69e5cf365df06d97f32264cfa
+  metadata.gz: 97a2aff66c765f24289717ac79e0a25a6bf31ee3ec5b84b64c96e8573382b31b0a27c30f06692a296b3bfedd70ea9f34f1a451cea7de27d3fa4c61a7502bab98
+  data.tar.gz: b795d47af337e109dfafb9f41a0a720d0eb314c7ba7219193648505ec9dffa3874215b5d311256f625228a4f3e52b73153ee3d694a3d2f88d4c2fd0dd24960b1

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.6.3 - 2019-10-28
+
+* [enhancement] Add DATE type conveter (thanks to @tksfjt1024)
+
 ## 0.6.2 - 2019-10-16
 
 * [maintenance] Lock signet and google-api-client version (thanks to @hiroyuki-sato)

data/README.md

@@ -307,17 +307,17 @@ Column options are used to aid guessing BigQuery schema, or to define conversion
 
 - **column_options**: advanced: an array of options for columns
   - **name**: column name
-  - **type**: BigQuery type such as `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, and `RECORD`. See belows for supported conversion type.
+  - **type**: BigQuery type such as `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATE`, and `RECORD`. See belows for supported conversion type.
     - boolean:   `BOOLEAN`, `STRING` (default: `BOOLEAN`)
     - long:      `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP` (default: `INTEGER`)
     - double:    `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP` (default: `FLOAT`)
-    - string:    `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `RECORD` (default: `STRING`)
-    - timestamp: `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP` (default: `TIMESTAMP`)
+    - string:    `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATE`, `RECORD` (default: `STRING`)
+    - timestamp: `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATE` (default: `TIMESTAMP`)
     - json:      `STRING`,  `RECORD` (default: `STRING`)
   - **mode**: BigQuery mode such as `NULLABLE`, `REQUIRED`, and `REPEATED` (string, default: `NULLABLE`)
   - **fields**: Describes the nested schema fields if the type property is set to RECORD. Please note that this is **required** for `RECORD` column.
   - **timestamp_format**: timestamp format to convert into/from `timestamp` (string, default is `default_timestamp_format`)
-  - **timezone**: timezone to convert into/from `timestamp` (string, default is `default_timezone`).
+  - **timezone**: timezone to convert into/from `timestamp`, `date` (string, default is `default_timezone`).
 - **default_timestamp_format**: default timestamp format for column_options (string, default is "%Y-%m-%d %H:%M:%S.%6N")
 - **default_timezone**: default timezone for column_options (string, default is "UTC")
 

data/embulk-output-bigquery.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-bigquery"
-  spec.version       = "0.6.2"
+  spec.version       = "0.6.3"
   spec.authors       = ["Satoshi Akama", "Naotoshi Seo"]
   spec.summary       = "Google BigQuery output plugin for Embulk"
   spec.description   = "Embulk plugin that insert records to Google BigQuery."

data/lib/embulk/output/bigquery/value_converter_factory.rb

@@ -203,6 +203,13 @@ module Embulk
                 val # Users must care of BQ timestamp format
               }
             end
+          when 'DATE'
+            Proc.new {|val|
+              next nil if val.nil?
+              with_typecast_error(val) do |val|
+                TimeWithZone.set_zone_offset(Time.parse(val), zone_offset).strftime("%Y-%m-%d")
+              end
+            }
           when 'RECORD'
             Proc.new {|val|
               next nil if val.nil?
@@ -240,6 +247,11 @@ module Embulk
               next nil if val.nil?
               val.strftime("%Y-%m-%d %H:%M:%S.%6N %:z")
             }
+          when 'DATE'
+            Proc.new {|val|
+              next nil if val.nil?
+              val.localtime(zone_offset).strftime("%Y-%m-%d")
+            }
           else
             raise NotSupportedType, "cannot take column type #{type} for timestamp column"
           end

data/test/test_helper.rb

@@ -62,7 +62,8 @@ module Embulk
             Column.new({index: 2, name: 'double', type: :double}),
             Column.new({index: 3, name: 'string', type: :string}),
             Column.new({index: 4, name: 'timestamp', type: :timestamp}),
-            Column.new({index: 5, name: 'json', type: :json}),
+            Column.new({index: 5, name: 'date', type: :timestamp}),
+            Column.new({index: 6, name: 'json', type: :json}),
           ])
           task = {
             'column_options' => [
@@ -71,6 +72,7 @@ module Embulk
               {'name' => 'double',    'type' => 'STRING'},
               {'name' => 'string',    'type' => 'INTEGER'},
               {'name' => 'timestamp', 'type' => 'INTEGER'},
+              {'name' => 'date',      'type' => 'DATE'},
               {'name' => 'json',      'type' => 'RECORD', 'fields' => [
                 { 'name' => 'key1',   'type' => 'STRING' },
               ]},
@@ -82,6 +84,7 @@ module Embulk
             {name: 'double',    type: 'STRING'},
             {name: 'string',    type: 'INTEGER'},
             {name: 'timestamp', type: 'INTEGER'},
+            {name: 'date',      type: 'DATE'},
             {name: 'json',      type: 'RECORD', fields: [
               {name: 'key1',    type: 'STRING'},
             ]},

data/test/test_value_converter_factory.rb

@@ -90,6 +90,10 @@ module Embulk
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'TIMESTAMP').create_converter }
         end
 
+        def test_date
+          assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
+        end
+
         def test_record
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
         end
@@ -130,6 +134,10 @@ module Embulk
           assert_equal 1408452095, converter.call(1408452095)
         end
 
+        def test_date
+          assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
+        end
+
         def test_record
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
         end
@@ -166,6 +174,10 @@ module Embulk
           assert_equal 1408452095.188766, converter.call(1408452095.188766)
         end
 
+        def test_date
+          assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
+        end
+
         def test_record
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
         end
@@ -216,6 +228,14 @@ module Embulk
           assert_equal "2016-02-26 00:00:00", converter.call("2016-02-26 00:00:00")
         end
 
+        def test_date
+          converter = ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter
+          assert_equal nil, converter.call(nil)
+          assert_equal "2016-02-26", converter.call("2016-02-26")
+          assert_equal "2016-02-26", converter.call("2016-02-26 00:00:00")
+          assert_raise { converter.call('foo') }
+        end
+
         def test_record
           converter = ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter
           assert_equal({'foo'=>'foo'}, converter.call(%Q[{"foo":"foo"}]))
@@ -268,6 +288,24 @@ module Embulk
           assert_equal expected, converter.call(Time.at(subject).utc)
         end
 
+        def test_date
+          converter = ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter
+          assert_equal nil, converter.call(nil)
+          timestamp = Time.parse("2016-02-26 00:00:00.500000 +00:00")
+          expected = "2016-02-26"
+          assert_equal expected, converter.call(timestamp)
+          
+          converter = ValueConverterFactory.new(
+            SCHEMA_TYPE, 'DATE', timezone: 'Asia/Tokyo'
+          ).create_converter
+          assert_equal nil, converter.call(nil)
+          timestamp = Time.parse("2016-02-25 15:00:00.500000 +00:00")
+          expected = "2016-02-26"
+          assert_equal expected, converter.call(timestamp)
+
+          assert_raise { converter.call('foo') }
+        end
+
         def test_record
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
         end
@@ -298,6 +336,10 @@ module Embulk
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'TIMESTAMP').create_converter }
         end
 
+        def test_date
+          assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
+        end
+
         def test_record
           converter = ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter
           assert_equal nil, converter.call(nil)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.6.3
 platform: ruby
 authors:
 - Satoshi Akama
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-10-16 00:00:00.000000000 Z
+date: 2019-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement