embulk-output-bigquery 0.5.0 → 0.6.4

data/lib/embulk/output/bigquery/value_converter_factory.rb

@@ -203,6 +203,27 @@ module Embulk
                 val # Users must care of BQ timestamp format
               }
             end
+          when 'DATE'
+            Proc.new {|val|
+              next nil if val.nil?
+              with_typecast_error(val) do |val|
+                TimeWithZone.set_zone_offset(Time.parse(val), zone_offset).strftime("%Y-%m-%d")
+              end
+            }
+          when 'DATETIME'
+            if @timestamp_format
+              Proc.new {|val|
+                next nil if val.nil?
+                with_typecast_error(val) do |val|
+                  Time.strptime(val, @timestamp_format).strftime("%Y-%m-%d %H:%M:%S.%6N")
+                end
+              }
+            else
+              Proc.new {|val|
+                next nil if val.nil?
+                val # Users must care of BQ timestamp format
+              }
+            end
           when 'RECORD'
             Proc.new {|val|
               next nil if val.nil?
@@ -240,6 +261,16 @@ module Embulk
               next nil if val.nil?
               val.strftime("%Y-%m-%d %H:%M:%S.%6N %:z")
             }
+          when 'DATE'
+            Proc.new {|val|
+              next nil if val.nil?
+              val.localtime(zone_offset).strftime("%Y-%m-%d")
+            }
+          when 'DATETIME'
+            Proc.new {|val|
+              next nil if val.nil?
+              val.localtime(zone_offset).strftime("%Y-%m-%d %H:%M:%S.%6N")
+            }
           else
             raise NotSupportedType, "cannot take column type #{type} for timestamp column"
           end

data/test/test_bigquery_client.rb

@@ -32,7 +32,7 @@ else
             'dataset'          => 'your_dataset_name',
             'table'            => 'your_table_name',
             'auth_method'      => 'json_key',
-            'json_keyfile'     => JSON_KEYFILE,
+            'json_keyfile'     => File.read(JSON_KEYFILE),
             'retries'          => 3,
             'timeout_sec'      => 300,
             'open_timeout_sec' => 300,
@@ -61,10 +61,6 @@ else
           def test_json_keyfile
             assert_nothing_raised { BigqueryClient.new(least_task, schema).client }
           end
-
-          def test_p12_keyfile
-            # pending
-          end
         end
 
         sub_test_case "create_dataset" do

data/test/test_configure.rb

@@ -18,10 +18,9 @@ module Embulk
 
       def least_config
         DataSource.new({
-          'project' => 'your_project_name',
-          'dataset' => 'your_dataset_name',
-          'table'   => 'your_table_name',
-          'p12_keyfile' => __FILE__, # fake
+          'project'      => 'your_project_name',
+          'dataset'      => 'your_dataset_name',
+          'table'        => 'your_table_name',
         })
       end
 
@@ -43,9 +42,7 @@ module Embulk
       def test_configure_default
         task = Bigquery.configure(least_config, schema, processor_count)
         assert_equal "append", task['mode']
-        assert_equal "private_key", task['auth_method']
-        assert_equal nil, task['service_account_email']
-        assert_equal __FILE__, task['p12_keyfile']
+        assert_equal "application_default", task['auth_method']
         assert_equal nil, task['json_keyfile']
         assert_equal "your_project_name", task['project']
         assert_equal "your_dataset_name", task['dataset']
@@ -132,11 +129,6 @@ module Embulk
         config = least_config.merge('auth_method' => 'foobar')
         assert_raise { Bigquery.configure(config, schema, processor_count) }
 
-        config = least_config.merge('auth_method' => 'private_key').tap {|h| h.delete('p12_keyfile') }
-        assert_raise { Bigquery.configure(config, schema, processor_count) }
-        config = least_config.merge('auth_method' => 'private_key', 'p12_keyfile' => 'dummy')
-        assert_nothing_raised { Bigquery.configure(config, schema, processor_count) }
-
         config = least_config.merge('auth_method' => 'json_key').tap {|h| h.delete('json_keyfile') }
         assert_raise { Bigquery.configure(config, schema, processor_count) }
         config = least_config.merge('auth_method' => 'json_key', 'json_keyfile' => "#{EXAMPLE_ROOT}/json_key.json")

data/test/test_helper.rb

@@ -62,7 +62,9 @@ module Embulk
             Column.new({index: 2, name: 'double', type: :double}),
             Column.new({index: 3, name: 'string', type: :string}),
             Column.new({index: 4, name: 'timestamp', type: :timestamp}),
-            Column.new({index: 5, name: 'json', type: :json}),
+            Column.new({index: 5, name: 'date', type: :timestamp}),
+            Column.new({index: 6, name: 'datetime', type: :timestamp}),
+            Column.new({index: 7, name: 'json', type: :json}),
           ])
           task = {
             'column_options' => [
@@ -71,6 +73,8 @@ module Embulk
               {'name' => 'double',    'type' => 'STRING'},
               {'name' => 'string',    'type' => 'INTEGER'},
               {'name' => 'timestamp', 'type' => 'INTEGER'},
+              {'name' => 'date',      'type' => 'DATE'},
+              {'name' => 'datetime',  'type' => 'DATETIME'},
               {'name' => 'json',      'type' => 'RECORD', 'fields' => [
                 { 'name' => 'key1',   'type' => 'STRING' },
               ]},
@@ -82,6 +86,8 @@ module Embulk
             {name: 'double',    type: 'STRING'},
             {name: 'string',    type: 'INTEGER'},
             {name: 'timestamp', type: 'INTEGER'},
+            {name: 'date',      type: 'DATE'},
+            {name: 'datetime',  type: 'DATETIME'},
             {name: 'json',      type: 'RECORD', fields: [
               {name: 'key1',    type: 'STRING'},
             ]},

data/test/test_transaction.rb

@@ -8,12 +8,11 @@ module Embulk
     class TestTransaction < Test::Unit::TestCase
       def least_config
         DataSource.new({
-          'project'     => 'your_project_name',
-          'dataset'     => 'your_dataset_name',
-          'table'       => 'your_table_name',
-          'p12_keyfile' => __FILE__, # fake
-          'temp_table'  => 'temp_table', # randomly created is not good for our test
-          'path_prefix' => 'tmp/', # randomly created is not good for our test
+          'project'      => 'your_project_name',
+          'dataset'      => 'your_dataset_name',
+          'table'        => 'your_table_name',
+          'temp_table'   => 'temp_table', # randomly created is not good for our test
+          'path_prefix'  => 'tmp/', # randomly created is not good for our test
         })
       end
 

data/test/test_value_converter_factory.rb

@@ -90,6 +90,14 @@ module Embulk
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'TIMESTAMP').create_converter }
         end
 
+        def test_date
+          assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
+        end
+
+        def test_datetime
+          assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter }
+        end
+
         def test_record
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
         end
@@ -130,6 +138,14 @@ module Embulk
           assert_equal 1408452095, converter.call(1408452095)
         end
 
+        def test_date
+          assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
+        end
+
+        def test_datetime
+          assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter }
+        end
+
         def test_record
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
         end
@@ -166,6 +182,14 @@ module Embulk
           assert_equal 1408452095.188766, converter.call(1408452095.188766)
         end
 
+        def test_date
+          assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
+        end
+
+        def test_datetime
+          assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter }
+        end
+
         def test_record
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
         end
@@ -216,6 +240,28 @@ module Embulk
           assert_equal "2016-02-26 00:00:00", converter.call("2016-02-26 00:00:00")
         end
 
+        def test_date
+          converter = ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter
+          assert_equal nil, converter.call(nil)
+          assert_equal "2016-02-26", converter.call("2016-02-26")
+          assert_equal "2016-02-26", converter.call("2016-02-26 00:00:00")
+          assert_raise { converter.call('foo') }
+        end
+
+        def test_datetime
+          converter = ValueConverterFactory.new(
+            SCHEMA_TYPE, 'DATETIME',
+            timestamp_format: '%Y/%m/%d'
+          ).create_converter
+          assert_equal nil, converter.call(nil)
+          assert_equal "2016-02-26 00:00:00.000000", converter.call("2016/02/26")
+
+          # Users must care of BQ datetime format by themselves with no timestamp_format
+          converter = ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter
+          assert_equal nil, converter.call(nil)
+          assert_equal "2016-02-26 00:00:00", converter.call("2016-02-26 00:00:00")
+        end
+
         def test_record
           converter = ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter
           assert_equal({'foo'=>'foo'}, converter.call(%Q[{"foo":"foo"}]))
@@ -268,6 +314,42 @@ module Embulk
           assert_equal expected, converter.call(Time.at(subject).utc)
         end
 
+        def test_date
+          converter = ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter
+          assert_equal nil, converter.call(nil)
+          timestamp = Time.parse("2016-02-26 00:00:00.500000 +00:00")
+          expected = "2016-02-26"
+          assert_equal expected, converter.call(timestamp)
+
+          converter = ValueConverterFactory.new(
+            SCHEMA_TYPE, 'DATE', timezone: 'Asia/Tokyo'
+          ).create_converter
+          assert_equal nil, converter.call(nil)
+          timestamp = Time.parse("2016-02-25 15:00:00.500000 +00:00")
+          expected = "2016-02-26"
+          assert_equal expected, converter.call(timestamp)
+
+          assert_raise { converter.call('foo') }
+        end
+
+        def test_datetime
+          converter = ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter
+          assert_equal nil, converter.call(nil)
+          timestamp = Time.parse("2016-02-26 00:00:00.500000 +00:00")
+          expected = "2016-02-26 00:00:00.500000"
+          assert_equal expected, converter.call(timestamp)
+
+          converter = ValueConverterFactory.new(
+            SCHEMA_TYPE, 'DATETIME', timezone: 'Asia/Tokyo'
+          ).create_converter
+          assert_equal nil, converter.call(nil)
+          timestamp = Time.parse("2016-02-25 15:00:00.500000 +00:00")
+          expected = "2016-02-26 00:00:00.500000"
+          assert_equal expected, converter.call(timestamp)
+
+          assert_raise { converter.call('foo') }
+        end
+
         def test_record
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
         end
@@ -298,6 +380,10 @@ module Embulk
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'TIMESTAMP').create_converter }
         end
 
+        def test_date
+          assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
+        end
+
         def test_record
           converter = ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter
           assert_equal nil, converter.call(nil)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.4
 platform: ruby
 authors:
 - Satoshi Akama
@@ -9,22 +9,42 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-08-10 00:00:00.000000000 Z
+date: 2019-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.7'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+  name: signet
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.33.0
   name: google-api-client
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.33.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -83,51 +103,8 @@ files:
 - README.md
 - Rakefile
 - embulk-output-bigquery.gemspec
-- example/config_append_direct_schema_update_options.yml
-- example/config_client_options.yml
-- example/config_csv.yml
-- example/config_delete_in_advance.yml
-- example/config_delete_in_advance_field_partitioned_table.yml
-- example/config_delete_in_advance_partitioned_table.yml
-- example/config_expose_errors.yml
-- example/config_gcs.yml
-- example/config_guess_from_embulk_schema.yml
-- example/config_guess_with_column_options.yml
-- example/config_gzip.yml
-- example/config_jsonl.yml
-- example/config_max_threads.yml
-- example/config_min_ouput_tasks.yml
-- example/config_mode_append.yml
-- example/config_mode_append_direct.yml
-- example/config_nested_record.yml
-- example/config_payload_column.yml
-- example/config_payload_column_index.yml
-- example/config_progress_log_interval.yml
-- example/config_replace.yml
-- example/config_replace_backup.yml
-- example/config_replace_backup_field_partitioned_table.yml
-- example/config_replace_backup_partitioned_table.yml
-- example/config_replace_field_partitioned_table.yml
-- example/config_replace_partitioned_table.yml
-- example/config_replace_schema_update_options.yml
-- example/config_skip_file_generation.yml
-- example/config_table_strftime.yml
-- example/config_template_table.yml
-- example/config_uncompressed.yml
-- example/config_with_rehearsal.yml
-- example/example.csv
-- example/example.yml
-- example/example2_1.csv
-- example/example2_2.csv
-- example/example4_1.csv
-- example/example4_2.csv
-- example/example4_3.csv
-- example/example4_4.csv
-- example/json_key.json
-- example/nested_example.jsonl
-- example/schema.json
-- example/schema_expose_errors.json
 - lib/embulk/output/bigquery.rb
+- lib/embulk/output/bigquery/auth.rb
 - lib/embulk/output/bigquery/bigquery_client.rb
 - lib/embulk/output/bigquery/file_writer.rb
 - lib/embulk/output/bigquery/gcs_client.rb
@@ -161,7 +138,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubyforge_project:
+rubygems_version: 2.6.14.1
 signing_key:
 specification_version: 4
 summary: Google BigQuery output plugin for Embulk

data/example/config_append_direct_schema_update_options.yml

@@ -1,31 +0,0 @@
-in:
-  type: file
-  path_prefix: example/example.csv
-  parser:
-    type: csv
-    charset: UTF-8
-    newline: CRLF
-    null_string: 'NULL'
-    skip_header_lines: 1
-    comment_line_marker: '#'
-    columns:
-      - {name: date,        type: string}
-      - {name: timestamp,   type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
-      - {name: "null",      type: string}
-      - {name: long,        type: long}
-      - {name: string,      type: string}
-      - {name: double,     type: double}
-      - {name: boolean,     type: boolean}
-out:
-  type: bigquery
-  mode: append_direct
-  auth_method: json_key
-  json_keyfile: example/your-project-000.json
-  dataset: your_dataset_name
-  table: your_table_name
-  source_format: NEWLINE_DELIMITED_JSON
-  compression: NONE
-  auto_create_dataset: true
-  auto_create_table: true
-  schema_file: example/schema.json
-  schema_update_options: [ALLOW_FIELD_ADDITION, ALLOW_FIELD_RELAXATION]

data/example/config_client_options.yml

@@ -1,33 +0,0 @@
-in:
-  type: file
-  path_prefix: example/example.csv
-  parser:
-    type: csv
-    charset: UTF-8
-    newline: CRLF
-    null_string: 'NULL'
-    skip_header_lines: 1
-    comment_line_marker: '#'
-    columns:
-      - {name: date,        type: string}
-      - {name: timestamp,   type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
-      - {name: "null",      type: string}
-      - {name: long,        type: long}
-      - {name: string,      type: string}
-      - {name: double,      type: double}
-      - {name: boolean,     type: boolean}
-out:
-  type: bigquery
-  mode: replace
-  auth_method: json_key
-  json_keyfile: example/your-project-000.json
-  dataset: your_dataset_name
-  table: your_table_name
-  source_format: NEWLINE_DELIMITED_JSON
-  auto_create_dataset: true
-  auto_create_table: true
-  schema_file: example/schema.json
-  timeout_sec: 400
-  open_timeout_sec: 400
-  retries: 2
-  application_name: "Embulk BigQuery plugin test"