embulk-output-bigquery 0.6.1 → 0.6.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ddfd10c5e85614e1dae0333494333653f1af95b8158dfda8977f8b00d64b3478
-  data.tar.gz: 2cec70eaa49c828d7fe9347bc0d9699b9398f21db96880e997a66bdab23deb89
+  metadata.gz: d48b65d07302466f8f52dadb559ad049a054453db3741d4384209125e7b9e9cd
+  data.tar.gz: 13cd70568cfaebba819a9b7a9a51d1c45ff9f1599893b4a0b451e82dc84e40c9
 SHA512:
-  metadata.gz: 4782a28272da610f8399aca50cc4ddaefea00b8dbf45a37bec24771d7ecdb05bbdcd6de85ff167c5c3745f6689413c215689bb8d420960705cd6cb2026e99932
-  data.tar.gz: 9dbabb787e2f1b5797ccb2a2cd8786ce28d0e0d01310cd522ea4894337a279e809de10abca14b50b836553b6de95df4afd886596d75e7193d4de60a5c6f95781
+  metadata.gz: ee51c9bf570ce2f2a55e43a5ab7842f1669d814b93cdd1395853be8f6b3ff770f6a5a6c9f9a81b6c0eca1e9bc72aff3d1302d37760fe559ecfa33a740e1da724
+  data.tar.gz: 8ada113513a089d786bf93bce1de98ad4bcc900ff73931c164a801b29e0bad9fd4b001bdf85962570998df995209e4ff320ba74e1fae22b61fb389a621121073

data/CHANGELOG.md

@@ -1,3 +1,25 @@
+## 0.6.6 - 2021-06-10
+
+* [maintenance] Fix network retry function (thanks to @case-k-git)
+* [enhancement] Allow to specify the billing project and the project to which the data will be loaded separately (thanks to @ck-fm0211)
+* [enhancement] Include original error message on json parse error (thanks to @k-yomo)
+
+## 0.6.5 - 2021-06-10
+* [maintenance] Fix failed tests (thanks to @kyoshidajp)
+* [maintenance] Lock representable version for avoiding requiring Ruby 2.4 (thanks to @hiroyuki-sato)
+
+## 0.6.4 - 2019-11-06
+
+* [enhancement] Add DATETIME type conveter (thanks to @kekekenta)
+
+## 0.6.3 - 2019-10-28
+
+* [enhancement] Add DATE type conveter (thanks to @tksfjt1024)
+
+## 0.6.2 - 2019-10-16
+
+* [maintenance] Lock signet and google-api-client version (thanks to @hiroyuki-sato)
+
 ## 0.6.1 - 2019-08-28
 
 * [maintenance] Release a new gem not to include symlinks to make it work on Windows.

data/Gemfile

@@ -1,7 +1,7 @@
 source 'https://rubygems.org/'
 
 gemspec
-gem 'embulk'
+gem 'embulk', '< 0.10'
 gem 'liquid', '= 4.0.0' # the version included in embulk.jar
 gem 'embulk-parser-none'
 gem 'embulk-parser-jsonl'

data/README.md

@@ -33,6 +33,7 @@ OAuth flow for installed applications.
 |  auth_method                         | string      | optional   | "application\_default"   | See [Authentication](#authentication) |
 |  json_keyfile                        | string      | optional   |                          | keyfile path or `content` |
 |  project                             | string      | required unless service\_account's `json_keyfile` is given. | | project\_id |
+|  destination_project                 | string      | optional   | `project` value         |  A destination project to which the data will be loaded. Use this if you want to separate a billing project (the `project` value) and a destination project (the `destination_project` value). |
 |  dataset                             | string      | required   |                          | dataset |
 |  location                            | string      | optional   | nil                      | geographic location of dataset. See [Location](#location) |
 |  table                               | string      | required   |                          | table name, or table name with a partition decorator such as `table_name$20160929`|
@@ -307,17 +308,17 @@ Column options are used to aid guessing BigQuery schema, or to define conversion
 
 - **column_options**: advanced: an array of options for columns
   - **name**: column name
-  - **type**: BigQuery type such as `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, and `RECORD`. See belows for supported conversion type.
+  - **type**: BigQuery type such as `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATETIME`, `DATE`, and `RECORD`. See belows for supported conversion type.
     - boolean:   `BOOLEAN`, `STRING` (default: `BOOLEAN`)
     - long:      `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP` (default: `INTEGER`)
     - double:    `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP` (default: `FLOAT`)
-    - string:    `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `RECORD` (default: `STRING`)
-    - timestamp: `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP` (default: `TIMESTAMP`)
+    - string:    `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATETIME`, `DATE`, `RECORD` (default: `STRING`)
+    - timestamp: `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATETIME`, `DATE` (default: `TIMESTAMP`)
     - json:      `STRING`,  `RECORD` (default: `STRING`)
   - **mode**: BigQuery mode such as `NULLABLE`, `REQUIRED`, and `REPEATED` (string, default: `NULLABLE`)
   - **fields**: Describes the nested schema fields if the type property is set to RECORD. Please note that this is **required** for `RECORD` column.
   - **timestamp_format**: timestamp format to convert into/from `timestamp` (string, default is `default_timestamp_format`)
-  - **timezone**: timezone to convert into/from `timestamp` (string, default is `default_timezone`).
+  - **timezone**: timezone to convert into/from `timestamp`, `date` (string, default is `default_timezone`).
 - **default_timestamp_format**: default timestamp format for column_options (string, default is "%Y-%m-%d %H:%M:%S.%6N")
 - **default_timezone**: default timezone for column_options (string, default is "UTC")
 

data/embulk-output-bigquery.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-bigquery"
-  spec.version       = "0.6.1"
+  spec.version       = "0.6.6"
   spec.authors       = ["Satoshi Akama", "Naotoshi Seo"]
   spec.summary       = "Google BigQuery output plugin for Embulk"
   spec.description   = "Embulk plugin that insert records to Google BigQuery."
@@ -14,8 +14,18 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency 'google-api-client'
+  # TODO
+  # signet 0.12.0 and google-api-client 0.33.0 require >= Ruby 2.4.
+  # Embulk 0.9 use JRuby 9.1.X.Y and it's compatible with Ruby 2.3.
+  # So, force install signet < 0.12 and google-api-client < 0.33.0
+  # Also, representable version >= 3.1.0 requires Ruby version >= 2.4
+  spec.add_dependency 'signet', '~> 0.7', '< 0.12.0'
+  spec.add_dependency 'google-api-client','< 0.33.0'
   spec.add_dependency 'time_with_zone'
+  spec.add_dependency "representable", ['~> 3.0.0', '< 3.1']
+  # faraday 1.1.0 require >= Ruby 2.4.
+  # googleauth 0.9.0 requires faraday ~> 0.12
+  spec.add_dependency "faraday", '~> 0.12'
 
   spec.add_development_dependency 'bundler', ['>= 1.10.6']
   spec.add_development_dependency 'rake', ['>= 10.0']

data/lib/embulk/output/bigquery.rb

@@ -36,6 +36,7 @@ module Embulk
           'auth_method'                    => config.param('auth_method',                    :string,  :default => 'application_default'),
           'json_keyfile'                   => config.param('json_keyfile',                  LocalFile, :default => nil),
           'project'                        => config.param('project',                        :string,  :default => nil),
+          'destination_project'            => config.param('destination_project',            :string,  :default => nil),
           'dataset'                        => config.param('dataset',                        :string),
           'location'                       => config.param('location',                       :string,  :default => nil),
           'table'                          => config.param('table',                          :string),
@@ -135,12 +136,13 @@ module Embulk
             json_key = JSON.parse(task['json_keyfile'])
             task['project'] ||= json_key['project_id']
           rescue => e
-            raise ConfigError.new "json_keyfile is not a JSON file"
+            raise ConfigError.new "Parsing 'json_keyfile' failed with error: #{e.class} #{e.message}"
           end
         end
         if task['project'].nil?
           raise ConfigError.new "Required field \"project\" is not set"
         end
+        task['destination_project'] ||= task['project']
 
         if (task['payload_column'] or task['payload_column_index']) and task['auto_create_table']
           if task['schema_file'].nil? and task['template_table'].nil?
@@ -166,7 +168,7 @@ module Embulk
           begin
             JSON.parse(File.read(task['schema_file']))
           rescue => e
-            raise ConfigError.new "schema_file #{task['schema_file']} is not a JSON file"
+            raise ConfigError.new "Parsing 'schema_file' #{task['schema_file']} failed with error: #{e.class} #{e.message}"
           end
         end
 

data/lib/embulk/output/bigquery/bigquery_client.rb

@@ -18,6 +18,7 @@ module Embulk
           @schema = schema
           reset_fields(fields) if fields
           @project = @task['project']
+          @destination_project = @task['destination_project']
           @dataset = @task['dataset']
           @location = @task['location']
           @location_for_log = @location.nil? ? 'us/eu' : @location
@@ -80,7 +81,7 @@ module Embulk
               # As https://cloud.google.com/bigquery/docs/managing_jobs_datasets_projects#managingjobs says,
               # we should generate job_id in client code, otherwise, retrying would cause duplication
               job_id = "embulk_load_job_#{SecureRandom.uuid}"
-              Embulk.logger.info { "embulk-output-bigquery: Load job starting... job_id:[#{job_id}] #{object_uris} => #{@project}:#{@dataset}.#{table} in #{@location_for_log}" }
+              Embulk.logger.info { "embulk-output-bigquery: Load job starting... job_id:[#{job_id}] #{object_uris} => #{@destination_project}:#{@dataset}.#{table} in #{@location_for_log}" }
 
               body = {
                 job_reference: {
@@ -90,7 +91,7 @@ module Embulk
                 configuration: {
                   load: {
                     destination_table: {
-                      project_id: @project,
+                      project_id: @destination_project,
                       dataset_id: @dataset,
                       table_id: table,
                     },
@@ -130,7 +131,7 @@ module Embulk
               Embulk.logger.error {
                 "embulk-output-bigquery: insert_job(#{@project}, #{body}, #{opts}), response:#{response}"
               }
-              raise Error, "failed to load #{object_uris} to #{@project}:#{@dataset}.#{table} in #{@location_for_log}, response:#{response}"
+              raise Error, "failed to load #{object_uris} to #{@destination_project}:#{@dataset}.#{table} in #{@location_for_log}, response:#{response}"
             end
           end
         end
@@ -171,7 +172,7 @@ module Embulk
                 # As https://cloud.google.com/bigquery/docs/managing_jobs_datasets_projects#managingjobs says,
                 # we should generate job_id in client code, otherwise, retrying would cause duplication
                 job_id = "embulk_load_job_#{SecureRandom.uuid}"
-                Embulk.logger.info { "embulk-output-bigquery: Load job starting... job_id:[#{job_id}] #{path} => #{@project}:#{@dataset}.#{table} in #{@location_for_log}" }
+                Embulk.logger.info { "embulk-output-bigquery: Load job starting... job_id:[#{job_id}] #{path} => #{@destination_project}:#{@dataset}.#{table} in #{@location_for_log}" }
               else
                 Embulk.logger.info { "embulk-output-bigquery: Load job starting... #{path} does not exist, skipped" }
                 return
@@ -185,7 +186,7 @@ module Embulk
                 configuration: {
                   load: {
                     destination_table: {
-                      project_id: @project,
+                      project_id: @destination_project,
                       dataset_id: @dataset,
                       table_id: table,
                     },
@@ -232,7 +233,7 @@ module Embulk
               Embulk.logger.error {
                 "embulk-output-bigquery: insert_job(#{@project}, #{body}, #{opts}), response:#{response}"
               }
-              raise Error, "failed to load #{path} to #{@project}:#{@dataset}.#{table} in #{@location_for_log}, response:#{response}"
+              raise Error, "failed to load #{path} to #{@destination_project}:#{@dataset}.#{table} in #{@location_for_log}, response:#{response}"
             end
           end
         end
@@ -245,7 +246,7 @@ module Embulk
 
               Embulk.logger.info {
                 "embulk-output-bigquery: Copy job starting... job_id:[#{job_id}] " \
-                "#{@project}:#{@dataset}.#{source_table} => #{@project}:#{destination_dataset}.#{destination_table}"
+                "#{@destination_project}:#{@dataset}.#{source_table} => #{@destination_project}:#{destination_dataset}.#{destination_table}"
               }
 
               body = {
@@ -258,12 +259,12 @@ module Embulk
                     create_deposition: 'CREATE_IF_NEEDED',
                     write_disposition: write_disposition,
                     source_table: {
-                      project_id: @project,
+                      project_id: @destination_project,
                       dataset_id: @dataset,
                       table_id: source_table,
                     },
                     destination_table: {
-                      project_id: @project,
+                      project_id: @destination_project,
                       dataset_id: destination_dataset,
                       table_id: destination_table,
                     },
@@ -284,8 +285,8 @@ module Embulk
               Embulk.logger.error {
                 "embulk-output-bigquery: insert_job(#{@project}, #{body}, #{opts}), response:#{response}"
               }
-              raise Error, "failed to copy #{@project}:#{@dataset}.#{source_table} " \
-                "to #{@project}:#{destination_dataset}.#{destination_table}, response:#{response}"
+              raise Error, "failed to copy #{@destination_project}:#{@dataset}.#{source_table} " \
+                "to #{@destination_project}:#{destination_dataset}.#{destination_table}, response:#{response}"
             end
           end
         end
@@ -354,7 +355,7 @@ module Embulk
         def create_dataset(dataset = nil, reference: nil)
           dataset ||= @dataset
           begin
-            Embulk.logger.info { "embulk-output-bigquery: Create dataset... #{@project}:#{dataset} in #{@location_for_log}" }
+            Embulk.logger.info { "embulk-output-bigquery: Create dataset... #{@destination_project}:#{dataset} in #{@location_for_log}" }
             hint = {}
             if reference
               response = get_dataset(reference)
@@ -382,25 +383,25 @@ module Embulk
             Embulk.logger.error {
               "embulk-output-bigquery: insert_dataset(#{@project}, #{body}, #{opts}), response:#{response}"
             }
-            raise Error, "failed to create dataset #{@project}:#{dataset} in #{@location_for_log}, response:#{response}"
+            raise Error, "failed to create dataset #{@destination_project}:#{dataset} in #{@location_for_log}, response:#{response}"
           end
         end
 
         def get_dataset(dataset = nil)
           dataset ||= @dataset
           begin
-            Embulk.logger.info { "embulk-output-bigquery: Get dataset... #{@project}:#{dataset}" }
-            with_network_retry { client.get_dataset(@project, dataset) }
+            Embulk.logger.info { "embulk-output-bigquery: Get dataset... #{@destination_project}:#{dataset}" }
+            with_network_retry { client.get_dataset(@destination_project, dataset) }
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             if e.status_code == 404
-              raise NotFoundError, "Dataset #{@project}:#{dataset} is not found"
+              raise NotFoundError, "Dataset #{@destination_project}:#{dataset} is not found"
             end
 
             response = {status_code: e.status_code, message: e.message, error_class: e.class}
             Embulk.logger.error {
-              "embulk-output-bigquery: get_dataset(#{@project}, #{dataset}), response:#{response}"
+              "embulk-output-bigquery: get_dataset(#{@destination_project}, #{dataset}), response:#{response}"
             }
-            raise Error, "failed to get dataset #{@project}:#{dataset}, response:#{response}"
+            raise Error, "failed to get dataset #{@destination_project}:#{dataset}, response:#{response}"
           end
         end
 
@@ -414,7 +415,7 @@ module Embulk
               table = Helper.chomp_partition_decorator(table)
             end
 
-            Embulk.logger.info { "embulk-output-bigquery: Create table... #{@project}:#{dataset}.#{table}" }
+            Embulk.logger.info { "embulk-output-bigquery: Create table... #{@destination_project}:#{dataset}.#{table}" }
             body = {
               table_reference: {
                 table_id: table,
@@ -452,7 +453,7 @@ module Embulk
             Embulk.logger.error {
               "embulk-output-bigquery: insert_table(#{@project}, #{dataset}, #{@location_for_log}, #{body}, #{opts}), response:#{response}"
             }
-            raise Error, "failed to create table #{@project}:#{dataset}.#{table} in #{@location_for_log}, response:#{response}"
+            raise Error, "failed to create table #{@destination_project}:#{dataset}.#{table} in #{@location_for_log}, response:#{response}"
           end
         end
 
@@ -469,8 +470,8 @@ module Embulk
         def delete_table_or_partition(table, dataset: nil)
           begin
             dataset ||= @dataset
-            Embulk.logger.info { "embulk-output-bigquery: Delete table... #{@project}:#{dataset}.#{table}" }
-            with_network_retry { client.delete_table(@project, dataset, table) }
+            Embulk.logger.info { "embulk-output-bigquery: Delete table... #{@destination_project}:#{dataset}.#{table}" }
+            with_network_retry { client.delete_table(@destination_project, dataset, table) }
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             if e.status_code == 404 && /Not found:/ =~ e.message
               # ignore 'Not Found' error
@@ -479,9 +480,9 @@ module Embulk
 
             response = {status_code: e.status_code, message: e.message, error_class: e.class}
             Embulk.logger.error {
-              "embulk-output-bigquery: delete_table(#{@project}, #{dataset}, #{table}), response:#{response}"
+              "embulk-output-bigquery: delete_table(#{@destination_project}, #{dataset}, #{table}), response:#{response}"
             }
-            raise Error, "failed to delete table #{@project}:#{dataset}.#{table}, response:#{response}"
+            raise Error, "failed to delete table #{@destination_project}:#{dataset}.#{table}, response:#{response}"
           end
         end
 
@@ -497,18 +498,18 @@ module Embulk
         def get_table_or_partition(table, dataset: nil)
           begin
             dataset ||= @dataset
-            Embulk.logger.info { "embulk-output-bigquery: Get table... #{@project}:#{dataset}.#{table}" }
-            with_network_retry { client.get_table(@project, dataset, table) }
+            Embulk.logger.info { "embulk-output-bigquery: Get table... #{@destination_project}:#{dataset}.#{table}" }
+            with_network_retry { client.get_table(@destination_project, dataset, table) }
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             if e.status_code == 404
-              raise NotFoundError, "Table #{@project}:#{dataset}.#{table} is not found"
+              raise NotFoundError, "Table #{@destination_project}:#{dataset}.#{table} is not found"
             end
 
             response = {status_code: e.status_code, message: e.message, error_class: e.class}
             Embulk.logger.error {
-              "embulk-output-bigquery: get_table(#{@project}, #{dataset}, #{table}), response:#{response}"
+              "embulk-output-bigquery: get_table(#{@destination_project}, #{dataset}, #{table}), response:#{response}"
             }
-            raise Error, "failed to get table #{@project}:#{dataset}.#{table}, response:#{response}"
+            raise Error, "failed to get table #{@destination_project}:#{dataset}.#{table}, response:#{response}"
           end
         end
       end

data/lib/embulk/output/bigquery/gcs_client.rb

@@ -16,6 +16,7 @@ module Embulk
           super(task, scope, client_class)
 
           @project = @task['project']
+          @destination_project = @task['destination_project']
           @bucket = @task['gcs_bucket']
           @location = @task['location']
         end
@@ -23,7 +24,7 @@ module Embulk
         def insert_temporary_bucket(bucket = nil)
           bucket ||= @bucket
           begin
-            Embulk.logger.info { "embulk-output-bigquery: Insert bucket... #{@project}:#{bucket}" }
+            Embulk.logger.info { "embulk-output-bigquery: Insert bucket... #{@destination_project}:#{bucket}" }
             body = {
               name: bucket,
               lifecycle: {
@@ -57,7 +58,7 @@ module Embulk
             Embulk.logger.error {
               "embulk-output-bigquery: insert_temporary_bucket(#{@project}, #{body}, #{opts}), response:#{response}"
             }
-            raise Error, "failed to insert bucket #{@project}:#{bucket}, response:#{response}"
+            raise Error, "failed to insert bucket #{@destination_project}:#{bucket}, response:#{response}"
           end
         end
 
@@ -69,7 +70,7 @@ module Embulk
 
           started = Time.now
           begin
-            Embulk.logger.info { "embulk-output-bigquery: Insert object... #{path} => #{@project}:#{object_uri}" }
+            Embulk.logger.info { "embulk-output-bigquery: Insert object... #{path} => #{@destination_project}:#{object_uri}" }
             body = {
               name: object,
             }
@@ -86,7 +87,7 @@ module Embulk
             Embulk.logger.error {
               "embulk-output-bigquery: insert_object(#{bucket}, #{body}, #{opts}), response:#{response}"
             }
-            raise Error, "failed to insert object #{@project}:#{object_uri}, response:#{response}"
+            raise Error, "failed to insert object #{@destination_project}:#{object_uri}, response:#{response}"
           end
         end
 
@@ -109,7 +110,7 @@ module Embulk
           object = object.start_with?('/') ? object[1..-1] : object
           object_uri = URI.join("gs://#{bucket}", object).to_s
           begin
-            Embulk.logger.info { "embulk-output-bigquery: Delete object... #{@project}:#{object_uri}" }
+            Embulk.logger.info { "embulk-output-bigquery: Delete object... #{@destination_project}:#{object_uri}" }
             opts = {}
 
             Embulk.logger.debug { "embulk-output-bigquery: delete_object(#{bucket}, #{object}, #{opts})" }
@@ -122,7 +123,7 @@ module Embulk
             Embulk.logger.error {
               "embulk-output-bigquery: delete_object(#{bucket}, #{object}, #{opts}), response:#{response}"
             }
-            raise Error, "failed to delete object #{@project}:#{object_uri}, response:#{response}"
+            raise Error, "failed to delete object #{@destination_project}:#{object_uri}, response:#{response}"
           end
         end
       end

data/lib/embulk/output/bigquery/google_client.rb

@@ -50,7 +50,9 @@ module Embulk
           begin
             yield
           rescue ::Java::Java.net.SocketException, ::Java::Java.net.ConnectException => e
-            if ['Broken pipe', 'Connection reset', 'Connection timed out'].include?(e.message)
+            if ['Broken pipe', 'Connection reset', 'Connection timed out'].select { |x| e.message.include?(x) }.empty?
+              raise e
+            else
               if retries < @task['retries']
                 retries += 1
                 Embulk.logger.warn { "embulk-output-bigquery: retry \##{retries}, #{e.class} #{e.message}" }
@@ -59,8 +61,6 @@ module Embulk
                 Embulk.logger.error { "embulk-output-bigquery: retry exhausted \##{retries}, #{e.class} #{e.message}" }
                 raise e
               end
-            else
-              raise e
             end
           end
         end

data/lib/embulk/output/bigquery/value_converter_factory.rb

@@ -203,6 +203,27 @@ module Embulk
                 val # Users must care of BQ timestamp format
               }
             end
+          when 'DATE'
+            Proc.new {|val|
+              next nil if val.nil?
+              with_typecast_error(val) do |val|
+                TimeWithZone.set_zone_offset(Time.parse(val), zone_offset).strftime("%Y-%m-%d")
+              end
+            }
+          when 'DATETIME'
+            if @timestamp_format
+              Proc.new {|val|
+                next nil if val.nil?
+                with_typecast_error(val) do |val|
+                  Time.strptime(val, @timestamp_format).strftime("%Y-%m-%d %H:%M:%S.%6N")
+                end
+              }
+            else
+              Proc.new {|val|
+                next nil if val.nil?
+                val # Users must care of BQ timestamp format
+              }
+            end
           when 'RECORD'
             Proc.new {|val|
               next nil if val.nil?
@@ -240,6 +261,16 @@ module Embulk
               next nil if val.nil?
               val.strftime("%Y-%m-%d %H:%M:%S.%6N %:z")
             }
+          when 'DATE'
+            Proc.new {|val|
+              next nil if val.nil?
+              val.localtime(zone_offset).strftime("%Y-%m-%d")
+            }
+          when 'DATETIME'
+            Proc.new {|val|
+              next nil if val.nil?
+              val.localtime(zone_offset).strftime("%Y-%m-%d %H:%M:%S.%6N")
+            }
           else
             raise NotSupportedType, "cannot take column type #{type} for timestamp column"
           end

data/test/test_bigquery_client.rb

@@ -29,6 +29,7 @@ else
         def least_task
           {
             'project'          => JSON.parse(File.read(JSON_KEYFILE))['project_id'],
+            'destination_project' => JSON.parse(File.read(JSON_KEYFILE))['project_id'],
             'dataset'          => 'your_dataset_name',
             'table'            => 'your_table_name',
             'auth_method'      => 'json_key',

data/test/test_configure.rb

@@ -45,6 +45,7 @@ module Embulk
         assert_equal "application_default", task['auth_method']
         assert_equal nil, task['json_keyfile']
         assert_equal "your_project_name", task['project']
+        assert_equal "your_project_name", task['destination_project']
         assert_equal "your_dataset_name", task['dataset']
         assert_equal nil, task['location']
         assert_equal "your_table_name", task['table']
@@ -284,6 +285,16 @@ module Embulk
         config = least_config.merge('schema_update_options' => ['FOO'])
         assert_raise { Bigquery.configure(config, schema, processor_count) }
       end
+
+      def test_destination_project
+        config = least_config.merge('destination_project' => 'your_destination_project_name')
+        task = Bigquery.configure(config, schema, processor_count)
+
+        assert_nothing_raised { Bigquery.configure(config, schema, processor_count) }
+        assert_equal 'your_destination_project_name', task['destination_project']
+        assert_equal 'your_project_name', task['project']
+      end
+
     end
   end
 end

data/test/test_helper.rb

@@ -62,7 +62,9 @@ module Embulk
             Column.new({index: 2, name: 'double', type: :double}),
             Column.new({index: 3, name: 'string', type: :string}),
             Column.new({index: 4, name: 'timestamp', type: :timestamp}),
-            Column.new({index: 5, name: 'json', type: :json}),
+            Column.new({index: 5, name: 'date', type: :timestamp}),
+            Column.new({index: 6, name: 'datetime', type: :timestamp}),
+            Column.new({index: 7, name: 'json', type: :json}),
           ])
           task = {
             'column_options' => [
@@ -71,6 +73,8 @@ module Embulk
               {'name' => 'double',    'type' => 'STRING'},
               {'name' => 'string',    'type' => 'INTEGER'},
               {'name' => 'timestamp', 'type' => 'INTEGER'},
+              {'name' => 'date',      'type' => 'DATE'},
+              {'name' => 'datetime',  'type' => 'DATETIME'},
               {'name' => 'json',      'type' => 'RECORD', 'fields' => [
                 { 'name' => 'key1',   'type' => 'STRING' },
               ]},
@@ -82,6 +86,8 @@ module Embulk
             {name: 'double',    type: 'STRING'},
             {name: 'string',    type: 'INTEGER'},
             {name: 'timestamp', type: 'INTEGER'},
+            {name: 'date',      type: 'DATE'},
+            {name: 'datetime',  type: 'DATETIME'},
             {name: 'json',      type: 'RECORD', fields: [
               {name: 'key1',    type: 'STRING'},
             ]},

data/test/test_value_converter_factory.rb

@@ -90,6 +90,14 @@ module Embulk
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'TIMESTAMP').create_converter }
         end
 
+        def test_date
+          assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
+        end
+
+        def test_datetime
+          assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter }
+        end
+
         def test_record
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
         end
@@ -130,6 +138,14 @@ module Embulk
           assert_equal 1408452095, converter.call(1408452095)
         end
 
+        def test_date
+          assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
+        end
+
+        def test_datetime
+          assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter }
+        end
+
         def test_record
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
         end
@@ -166,6 +182,14 @@ module Embulk
           assert_equal 1408452095.188766, converter.call(1408452095.188766)
         end
 
+        def test_date
+          assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
+        end
+
+        def test_datetime
+          assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter }
+        end
+
         def test_record
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
         end
@@ -216,6 +240,28 @@ module Embulk
           assert_equal "2016-02-26 00:00:00", converter.call("2016-02-26 00:00:00")
         end
 
+        def test_date
+          converter = ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter
+          assert_equal nil, converter.call(nil)
+          assert_equal "2016-02-26", converter.call("2016-02-26")
+          assert_equal "2016-02-26", converter.call("2016-02-26 00:00:00")
+          assert_raise { converter.call('foo') }
+        end
+
+        def test_datetime
+          converter = ValueConverterFactory.new(
+            SCHEMA_TYPE, 'DATETIME',
+            timestamp_format: '%Y/%m/%d'
+          ).create_converter
+          assert_equal nil, converter.call(nil)
+          assert_equal "2016-02-26 00:00:00.000000", converter.call("2016/02/26")
+
+          # Users must care of BQ datetime format by themselves with no timestamp_format
+          converter = ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter
+          assert_equal nil, converter.call(nil)
+          assert_equal "2016-02-26 00:00:00", converter.call("2016-02-26 00:00:00")
+        end
+
         def test_record
           converter = ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter
           assert_equal({'foo'=>'foo'}, converter.call(%Q[{"foo":"foo"}]))
@@ -268,6 +314,42 @@ module Embulk
           assert_equal expected, converter.call(Time.at(subject).utc)
         end
 
+        def test_date
+          converter = ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter
+          assert_equal nil, converter.call(nil)
+          timestamp = Time.parse("2016-02-26 00:00:00.500000 +00:00")
+          expected = "2016-02-26"
+          assert_equal expected, converter.call(timestamp)
+
+          converter = ValueConverterFactory.new(
+            SCHEMA_TYPE, 'DATE', timezone: 'Asia/Tokyo'
+          ).create_converter
+          assert_equal nil, converter.call(nil)
+          timestamp = Time.parse("2016-02-25 15:00:00.500000 +00:00")
+          expected = "2016-02-26"
+          assert_equal expected, converter.call(timestamp)
+
+          assert_raise { converter.call('foo') }
+        end
+
+        def test_datetime
+          converter = ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter
+          assert_equal nil, converter.call(nil)
+          timestamp = Time.parse("2016-02-26 00:00:00.500000 +00:00")
+          expected = "2016-02-26 00:00:00.500000"
+          assert_equal expected, converter.call(timestamp)
+
+          converter = ValueConverterFactory.new(
+            SCHEMA_TYPE, 'DATETIME', timezone: 'Asia/Tokyo'
+          ).create_converter
+          assert_equal nil, converter.call(nil)
+          timestamp = Time.parse("2016-02-25 15:00:00.500000 +00:00")
+          expected = "2016-02-26 00:00:00.500000"
+          assert_equal expected, converter.call(timestamp)
+
+          assert_raise { converter.call('foo') }
+        end
+
         def test_record
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
         end
@@ -298,6 +380,10 @@ module Embulk
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'TIMESTAMP').create_converter }
         end
 
+        def test_date
+          assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
+        end
+
         def test_record
           converter = ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter
           assert_equal nil, converter.call(nil)

metadata

@@ -1,67 +1,121 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.6.6
 platform: ruby
 authors:
 - Satoshi Akama
 - Naotoshi Seo
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-08-28 00:00:00.000000000 Z
+date: 2021-06-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: signet
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  name: google-api-client
+        version: '0.7'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+  type: :runtime
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+- !ruby/object:Gem::Dependency
+  name: google-api-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.33.0
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.33.0
 - !ruby/object:Gem::Dependency
+  name: time_with_zone
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: time_with_zone
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: representable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+- !ruby/object:Gem::Dependency
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.10.6
-  name: bundler
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.10.6
 - !ruby/object:Gem::Dependency
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
-  name: rake
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -103,7 +157,7 @@ homepage: https://github.com/embulk/embulk-output-bigquery
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -119,7 +173,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.3
-signing_key:
+signing_key: 
 specification_version: 4
 summary: Google BigQuery output plugin for Embulk
 test_files: