embulk-output-bigquery 0.6.3 → 0.6.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 8b3d7d7d675d8428946f81517d1002f667f4fafe
-  data.tar.gz: 25940b93f70492675869d3c4dd50f83f8b7347cf
+SHA256:
+  metadata.gz: 5ad08405eb0d6f4a7ff867c6252d393628437d55e03fe33ec2cc71f940926153
+  data.tar.gz: 431ce1cf5298d5ec114a66191c9847f34fd554cc5de7c12fca8950a9e59538bf
 SHA512:
-  metadata.gz: 97a2aff66c765f24289717ac79e0a25a6bf31ee3ec5b84b64c96e8573382b31b0a27c30f06692a296b3bfedd70ea9f34f1a451cea7de27d3fa4c61a7502bab98
-  data.tar.gz: b795d47af337e109dfafb9f41a0a720d0eb314c7ba7219193648505ec9dffa3874215b5d311256f625228a4f3e52b73153ee3d694a3d2f88d4c2fd0dd24960b1
+  metadata.gz: c23e79e1b7e6d7b2af8455e97fd58bb4c4bc5e7b7fb1809bd49642294adfe644c27beca55177cff8427f5136532accae3973db1b000660c96f532e6d1cbfbc09
+  data.tar.gz: 2d6ee9d0cf3504683bc2898df3edc246401a2cfd44db2f6311368903e89e09e9b31e746232005fdbbf43f039b91ca1fe1b4f190704ceee3e7a9778962de5be1b

data/CHANGELOG.md

@@ -1,3 +1,20 @@
+## 0.6.7 - 2021-09-10
+* [enhancement] Add an expiration option of temporary table to clean up (thanks to @TKNGUE)
+
+## 0.6.6 - 2021-06-10
+
+* [maintenance] Fix network retry function (thanks to @case-k-git)
+* [enhancement] Allow to specify the billing project and the project to which the data will be loaded separately (thanks to @ck-fm0211)
+* [enhancement] Include original error message on json parse error (thanks to @k-yomo)
+
+## 0.6.5 - 2021-06-10
+* [maintenance] Fix failed tests (thanks to @kyoshidajp)
+* [maintenance] Lock representable version for avoiding requiring Ruby 2.4 (thanks to @hiroyuki-sato)
+
+## 0.6.4 - 2019-11-06
+
+* [enhancement] Add DATETIME type conveter (thanks to @kekekenta)
+
 ## 0.6.3 - 2019-10-28
 
 * [enhancement] Add DATE type conveter (thanks to @tksfjt1024)

data/Gemfile

@@ -1,7 +1,7 @@
 source 'https://rubygems.org/'
 
 gemspec
-gem 'embulk'
+gem 'embulk', '< 0.10'
 gem 'liquid', '= 4.0.0' # the version included in embulk.jar
 gem 'embulk-parser-none'
 gem 'embulk-parser-jsonl'

data/README.md

@@ -33,6 +33,7 @@ OAuth flow for installed applications.
 |  auth_method                         | string      | optional   | "application\_default"   | See [Authentication](#authentication) |
 |  json_keyfile                        | string      | optional   |                          | keyfile path or `content` |
 |  project                             | string      | required unless service\_account's `json_keyfile` is given. | | project\_id |
+|  destination_project                 | string      | optional   | `project` value         |  A destination project to which the data will be loaded. Use this if you want to separate a billing project (the `project` value) and a destination project (the `destination_project` value). |
 |  dataset                             | string      | required   |                          | dataset |
 |  location                            | string      | optional   | nil                      | geographic location of dataset. See [Location](#location) |
 |  table                               | string      | required   |                          | table name, or table name with a partition decorator such as `table_name$20160929`|
@@ -78,6 +79,13 @@ Options for intermediate local files
 |  delete_from_local_when_job_end      | boolean     | optional   | true                     | If set to true, delete generate local files when job is end |
 |  compression                         | string      | optional   | "NONE"                   | Compression of local files (`GZIP` or `NONE`) |
 
+
+Options for intermediate tables on BigQuery
+
+| name                                 | type        | required?  | default                  | description            |
+|:-------------------------------------|:------------|:-----------|:-------------------------|:-----------------------|
+|  temporary_table_expiration          | integer     | optional   |                          | Temporary table's expiration time in seconds |
+
 `source_format` is also used to determine formatter (csv or jsonl).
 
 #### Same options of bq command-line tools or BigQuery job's property
@@ -307,12 +315,12 @@ Column options are used to aid guessing BigQuery schema, or to define conversion
 
 - **column_options**: advanced: an array of options for columns
   - **name**: column name
-  - **type**: BigQuery type such as `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATE`, and `RECORD`. See belows for supported conversion type.
+  - **type**: BigQuery type such as `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATETIME`, `DATE`, and `RECORD`. See belows for supported conversion type.
     - boolean:   `BOOLEAN`, `STRING` (default: `BOOLEAN`)
     - long:      `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP` (default: `INTEGER`)
     - double:    `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP` (default: `FLOAT`)
-    - string:    `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATE`, `RECORD` (default: `STRING`)
-    - timestamp: `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATE` (default: `TIMESTAMP`)
+    - string:    `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATETIME`, `DATE`, `RECORD` (default: `STRING`)
+    - timestamp: `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATETIME`, `DATE` (default: `TIMESTAMP`)
     - json:      `STRING`,  `RECORD` (default: `STRING`)
   - **mode**: BigQuery mode such as `NULLABLE`, `REQUIRED`, and `REPEATED` (string, default: `NULLABLE`)
   - **fields**: Describes the nested schema fields if the type property is set to RECORD. Please note that this is **required** for `RECORD` column.

data/embulk-output-bigquery.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-bigquery"
-  spec.version       = "0.6.3"
+  spec.version       = "0.6.7"
   spec.authors       = ["Satoshi Akama", "Naotoshi Seo"]
   spec.summary       = "Google BigQuery output plugin for Embulk"
   spec.description   = "Embulk plugin that insert records to Google BigQuery."
@@ -16,11 +16,16 @@ Gem::Specification.new do |spec|
 
   # TODO
   # signet 0.12.0 and google-api-client 0.33.0 require >= Ruby 2.4.
-  # Embulk 0.9 use JRuby 9.1.X.Y and It compatible Ruby 2.3.
-  # So, Force install signet < 0.12 and google-api-client < 0.33.0
+  # Embulk 0.9 use JRuby 9.1.X.Y and it's compatible with Ruby 2.3.
+  # So, force install signet < 0.12 and google-api-client < 0.33.0
+  # Also, representable version >= 3.1.0 requires Ruby version >= 2.4
   spec.add_dependency 'signet', '~> 0.7', '< 0.12.0'
   spec.add_dependency 'google-api-client','< 0.33.0'
   spec.add_dependency 'time_with_zone'
+  spec.add_dependency "representable", ['~> 3.0.0', '< 3.1']
+  # faraday 1.1.0 require >= Ruby 2.4.
+  # googleauth 0.9.0 requires faraday ~> 0.12
+  spec.add_dependency "faraday", '~> 0.12'
 
   spec.add_development_dependency 'bundler', ['>= 1.10.6']
   spec.add_development_dependency 'rake', ['>= 10.0']

data/lib/embulk/output/bigquery/bigquery_client.rb

@@ -18,6 +18,7 @@ module Embulk
           @schema = schema
           reset_fields(fields) if fields
           @project = @task['project']
+          @destination_project = @task['destination_project']
           @dataset = @task['dataset']
           @location = @task['location']
           @location_for_log = @location.nil? ? 'us/eu' : @location
@@ -80,7 +81,7 @@ module Embulk
               # As https://cloud.google.com/bigquery/docs/managing_jobs_datasets_projects#managingjobs says,
               # we should generate job_id in client code, otherwise, retrying would cause duplication
               job_id = "embulk_load_job_#{SecureRandom.uuid}"
-              Embulk.logger.info { "embulk-output-bigquery: Load job starting... job_id:[#{job_id}] #{object_uris} => #{@project}:#{@dataset}.#{table} in #{@location_for_log}" }
+              Embulk.logger.info { "embulk-output-bigquery: Load job starting... job_id:[#{job_id}] #{object_uris} => #{@destination_project}:#{@dataset}.#{table} in #{@location_for_log}" }
 
               body = {
                 job_reference: {
@@ -90,7 +91,7 @@ module Embulk
                 configuration: {
                   load: {
                     destination_table: {
-                      project_id: @project,
+                      project_id: @destination_project,
                       dataset_id: @dataset,
                       table_id: table,
                     },
@@ -130,7 +131,7 @@ module Embulk
               Embulk.logger.error {
                 "embulk-output-bigquery: insert_job(#{@project}, #{body}, #{opts}), response:#{response}"
               }
-              raise Error, "failed to load #{object_uris} to #{@project}:#{@dataset}.#{table} in #{@location_for_log}, response:#{response}"
+              raise Error, "failed to load #{object_uris} to #{@destination_project}:#{@dataset}.#{table} in #{@location_for_log}, response:#{response}"
             end
           end
         end
@@ -171,7 +172,7 @@ module Embulk
                 # As https://cloud.google.com/bigquery/docs/managing_jobs_datasets_projects#managingjobs says,
                 # we should generate job_id in client code, otherwise, retrying would cause duplication
                 job_id = "embulk_load_job_#{SecureRandom.uuid}"
-                Embulk.logger.info { "embulk-output-bigquery: Load job starting... job_id:[#{job_id}] #{path} => #{@project}:#{@dataset}.#{table} in #{@location_for_log}" }
+                Embulk.logger.info { "embulk-output-bigquery: Load job starting... job_id:[#{job_id}] #{path} => #{@destination_project}:#{@dataset}.#{table} in #{@location_for_log}" }
               else
                 Embulk.logger.info { "embulk-output-bigquery: Load job starting... #{path} does not exist, skipped" }
                 return
@@ -185,7 +186,7 @@ module Embulk
                 configuration: {
                   load: {
                     destination_table: {
-                      project_id: @project,
+                      project_id: @destination_project,
                       dataset_id: @dataset,
                       table_id: table,
                     },
@@ -232,7 +233,7 @@ module Embulk
               Embulk.logger.error {
                 "embulk-output-bigquery: insert_job(#{@project}, #{body}, #{opts}), response:#{response}"
               }
-              raise Error, "failed to load #{path} to #{@project}:#{@dataset}.#{table} in #{@location_for_log}, response:#{response}"
+              raise Error, "failed to load #{path} to #{@destination_project}:#{@dataset}.#{table} in #{@location_for_log}, response:#{response}"
             end
           end
         end
@@ -245,7 +246,7 @@ module Embulk
 
               Embulk.logger.info {
                 "embulk-output-bigquery: Copy job starting... job_id:[#{job_id}] " \
-                "#{@project}:#{@dataset}.#{source_table} => #{@project}:#{destination_dataset}.#{destination_table}"
+                "#{@destination_project}:#{@dataset}.#{source_table} => #{@destination_project}:#{destination_dataset}.#{destination_table}"
               }
 
               body = {
@@ -258,12 +259,12 @@ module Embulk
                     create_deposition: 'CREATE_IF_NEEDED',
                     write_disposition: write_disposition,
                     source_table: {
-                      project_id: @project,
+                      project_id: @destination_project,
                       dataset_id: @dataset,
                       table_id: source_table,
                     },
                     destination_table: {
-                      project_id: @project,
+                      project_id: @destination_project,
                       dataset_id: destination_dataset,
                       table_id: destination_table,
                     },
@@ -284,8 +285,8 @@ module Embulk
               Embulk.logger.error {
                 "embulk-output-bigquery: insert_job(#{@project}, #{body}, #{opts}), response:#{response}"
               }
-              raise Error, "failed to copy #{@project}:#{@dataset}.#{source_table} " \
-                "to #{@project}:#{destination_dataset}.#{destination_table}, response:#{response}"
+              raise Error, "failed to copy #{@destination_project}:#{@dataset}.#{source_table} " \
+                "to #{@destination_project}:#{destination_dataset}.#{destination_table}, response:#{response}"
             end
           end
         end
@@ -354,7 +355,7 @@ module Embulk
         def create_dataset(dataset = nil, reference: nil)
           dataset ||= @dataset
           begin
-            Embulk.logger.info { "embulk-output-bigquery: Create dataset... #{@project}:#{dataset} in #{@location_for_log}" }
+            Embulk.logger.info { "embulk-output-bigquery: Create dataset... #{@destination_project}:#{dataset} in #{@location_for_log}" }
             hint = {}
             if reference
               response = get_dataset(reference)
@@ -382,25 +383,25 @@ module Embulk
             Embulk.logger.error {
               "embulk-output-bigquery: insert_dataset(#{@project}, #{body}, #{opts}), response:#{response}"
             }
-            raise Error, "failed to create dataset #{@project}:#{dataset} in #{@location_for_log}, response:#{response}"
+            raise Error, "failed to create dataset #{@destination_project}:#{dataset} in #{@location_for_log}, response:#{response}"
           end
         end
 
         def get_dataset(dataset = nil)
           dataset ||= @dataset
           begin
-            Embulk.logger.info { "embulk-output-bigquery: Get dataset... #{@project}:#{dataset}" }
-            with_network_retry { client.get_dataset(@project, dataset) }
+            Embulk.logger.info { "embulk-output-bigquery: Get dataset... #{@destination_project}:#{dataset}" }
+            with_network_retry { client.get_dataset(@destination_project, dataset) }
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             if e.status_code == 404
-              raise NotFoundError, "Dataset #{@project}:#{dataset} is not found"
+              raise NotFoundError, "Dataset #{@destination_project}:#{dataset} is not found"
             end
 
             response = {status_code: e.status_code, message: e.message, error_class: e.class}
             Embulk.logger.error {
-              "embulk-output-bigquery: get_dataset(#{@project}, #{dataset}), response:#{response}"
+              "embulk-output-bigquery: get_dataset(#{@destination_project}, #{dataset}), response:#{response}"
             }
-            raise Error, "failed to get dataset #{@project}:#{dataset}, response:#{response}"
+            raise Error, "failed to get dataset #{@destination_project}:#{dataset}, response:#{response}"
           end
         end
 
@@ -414,7 +415,7 @@ module Embulk
               table = Helper.chomp_partition_decorator(table)
             end
 
-            Embulk.logger.info { "embulk-output-bigquery: Create table... #{@project}:#{dataset}.#{table}" }
+            Embulk.logger.info { "embulk-output-bigquery: Create table... #{@destination_project}:#{dataset}.#{table}" }
             body = {
               table_reference: {
                 table_id: table,
@@ -439,6 +440,11 @@ module Embulk
               }
             end
 
+            if options['expiration_time']
+              # expiration_time is expressed in milliseconds
+              body[:expiration_time] = (Time.now.to_i + options['expiration_time']) * 1000
+            end
+
             opts = {}
             Embulk.logger.debug { "embulk-output-bigquery: insert_table(#{@project}, #{dataset}, #{@location_for_log}, #{body}, #{opts})" }
             with_network_retry { client.insert_table(@project, dataset, body, opts) }
@@ -452,7 +458,7 @@ module Embulk
             Embulk.logger.error {
               "embulk-output-bigquery: insert_table(#{@project}, #{dataset}, #{@location_for_log}, #{body}, #{opts}), response:#{response}"
             }
-            raise Error, "failed to create table #{@project}:#{dataset}.#{table} in #{@location_for_log}, response:#{response}"
+            raise Error, "failed to create table #{@destination_project}:#{dataset}.#{table} in #{@location_for_log}, response:#{response}"
           end
         end
 
@@ -469,8 +475,8 @@ module Embulk
         def delete_table_or_partition(table, dataset: nil)
           begin
             dataset ||= @dataset
-            Embulk.logger.info { "embulk-output-bigquery: Delete table... #{@project}:#{dataset}.#{table}" }
-            with_network_retry { client.delete_table(@project, dataset, table) }
+            Embulk.logger.info { "embulk-output-bigquery: Delete table... #{@destination_project}:#{dataset}.#{table}" }
+            with_network_retry { client.delete_table(@destination_project, dataset, table) }
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             if e.status_code == 404 && /Not found:/ =~ e.message
               # ignore 'Not Found' error
@@ -479,9 +485,9 @@ module Embulk
 
             response = {status_code: e.status_code, message: e.message, error_class: e.class}
             Embulk.logger.error {
-              "embulk-output-bigquery: delete_table(#{@project}, #{dataset}, #{table}), response:#{response}"
+              "embulk-output-bigquery: delete_table(#{@destination_project}, #{dataset}, #{table}), response:#{response}"
             }
-            raise Error, "failed to delete table #{@project}:#{dataset}.#{table}, response:#{response}"
+            raise Error, "failed to delete table #{@destination_project}:#{dataset}.#{table}, response:#{response}"
           end
         end
 
@@ -497,18 +503,18 @@ module Embulk
         def get_table_or_partition(table, dataset: nil)
           begin
             dataset ||= @dataset
-            Embulk.logger.info { "embulk-output-bigquery: Get table... #{@project}:#{dataset}.#{table}" }
-            with_network_retry { client.get_table(@project, dataset, table) }
+            Embulk.logger.info { "embulk-output-bigquery: Get table... #{@destination_project}:#{dataset}.#{table}" }
+            with_network_retry { client.get_table(@destination_project, dataset, table) }
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             if e.status_code == 404
-              raise NotFoundError, "Table #{@project}:#{dataset}.#{table} is not found"
+              raise NotFoundError, "Table #{@destination_project}:#{dataset}.#{table} is not found"
             end
 
             response = {status_code: e.status_code, message: e.message, error_class: e.class}
             Embulk.logger.error {
-              "embulk-output-bigquery: get_table(#{@project}, #{dataset}, #{table}), response:#{response}"
+              "embulk-output-bigquery: get_table(#{@destination_project}, #{dataset}, #{table}), response:#{response}"
             }
-            raise Error, "failed to get table #{@project}:#{dataset}.#{table}, response:#{response}"
+            raise Error, "failed to get table #{@destination_project}:#{dataset}.#{table}, response:#{response}"
           end
         end
       end

data/lib/embulk/output/bigquery/gcs_client.rb

@@ -16,6 +16,7 @@ module Embulk
           super(task, scope, client_class)
 
           @project = @task['project']
+          @destination_project = @task['destination_project']
           @bucket = @task['gcs_bucket']
           @location = @task['location']
         end
@@ -23,7 +24,7 @@ module Embulk
         def insert_temporary_bucket(bucket = nil)
           bucket ||= @bucket
           begin
-            Embulk.logger.info { "embulk-output-bigquery: Insert bucket... #{@project}:#{bucket}" }
+            Embulk.logger.info { "embulk-output-bigquery: Insert bucket... #{@destination_project}:#{bucket}" }
             body = {
               name: bucket,
               lifecycle: {
@@ -57,7 +58,7 @@ module Embulk
             Embulk.logger.error {
               "embulk-output-bigquery: insert_temporary_bucket(#{@project}, #{body}, #{opts}), response:#{response}"
             }
-            raise Error, "failed to insert bucket #{@project}:#{bucket}, response:#{response}"
+            raise Error, "failed to insert bucket #{@destination_project}:#{bucket}, response:#{response}"
           end
         end
 
@@ -69,7 +70,7 @@ module Embulk
 
           started = Time.now
           begin
-            Embulk.logger.info { "embulk-output-bigquery: Insert object... #{path} => #{@project}:#{object_uri}" }
+            Embulk.logger.info { "embulk-output-bigquery: Insert object... #{path} => #{@destination_project}:#{object_uri}" }
             body = {
               name: object,
             }
@@ -86,7 +87,7 @@ module Embulk
             Embulk.logger.error {
               "embulk-output-bigquery: insert_object(#{bucket}, #{body}, #{opts}), response:#{response}"
             }
-            raise Error, "failed to insert object #{@project}:#{object_uri}, response:#{response}"
+            raise Error, "failed to insert object #{@destination_project}:#{object_uri}, response:#{response}"
           end
         end
 
@@ -109,7 +110,7 @@ module Embulk
           object = object.start_with?('/') ? object[1..-1] : object
           object_uri = URI.join("gs://#{bucket}", object).to_s
           begin
-            Embulk.logger.info { "embulk-output-bigquery: Delete object... #{@project}:#{object_uri}" }
+            Embulk.logger.info { "embulk-output-bigquery: Delete object... #{@destination_project}:#{object_uri}" }
             opts = {}
 
             Embulk.logger.debug { "embulk-output-bigquery: delete_object(#{bucket}, #{object}, #{opts})" }
@@ -122,7 +123,7 @@ module Embulk
             Embulk.logger.error {
               "embulk-output-bigquery: delete_object(#{bucket}, #{object}, #{opts}), response:#{response}"
             }
-            raise Error, "failed to delete object #{@project}:#{object_uri}, response:#{response}"
+            raise Error, "failed to delete object #{@destination_project}:#{object_uri}, response:#{response}"
           end
         end
       end

data/lib/embulk/output/bigquery/google_client.rb

@@ -50,7 +50,9 @@ module Embulk
           begin
             yield
           rescue ::Java::Java.net.SocketException, ::Java::Java.net.ConnectException => e
-            if ['Broken pipe', 'Connection reset', 'Connection timed out'].include?(e.message)
+            if ['Broken pipe', 'Connection reset', 'Connection timed out'].select { |x| e.message.include?(x) }.empty?
+              raise e
+            else
               if retries < @task['retries']
                 retries += 1
                 Embulk.logger.warn { "embulk-output-bigquery: retry \##{retries}, #{e.class} #{e.message}" }
@@ -59,8 +61,6 @@ module Embulk
                 Embulk.logger.error { "embulk-output-bigquery: retry exhausted \##{retries}, #{e.class} #{e.message}" }
                 raise e
               end
-            else
-              raise e
             end
           end
         end

data/lib/embulk/output/bigquery/value_converter_factory.rb

@@ -210,6 +210,20 @@ module Embulk
                 TimeWithZone.set_zone_offset(Time.parse(val), zone_offset).strftime("%Y-%m-%d")
               end
             }
+          when 'DATETIME'
+            if @timestamp_format
+              Proc.new {|val|
+                next nil if val.nil?
+                with_typecast_error(val) do |val|
+                  Time.strptime(val, @timestamp_format).strftime("%Y-%m-%d %H:%M:%S.%6N")
+                end
+              }
+            else
+              Proc.new {|val|
+                next nil if val.nil?
+                val # Users must care of BQ timestamp format
+              }
+            end
           when 'RECORD'
             Proc.new {|val|
               next nil if val.nil?
@@ -252,6 +266,11 @@ module Embulk
               next nil if val.nil?
               val.localtime(zone_offset).strftime("%Y-%m-%d")
             }
+          when 'DATETIME'
+            Proc.new {|val|
+              next nil if val.nil?
+              val.localtime(zone_offset).strftime("%Y-%m-%d %H:%M:%S.%6N")
+            }
           else
             raise NotSupportedType, "cannot take column type #{type} for timestamp column"
           end

data/lib/embulk/output/bigquery.rb

@@ -36,6 +36,7 @@ module Embulk
           'auth_method'                    => config.param('auth_method',                    :string,  :default => 'application_default'),
           'json_keyfile'                   => config.param('json_keyfile',                  LocalFile, :default => nil),
           'project'                        => config.param('project',                        :string,  :default => nil),
+          'destination_project'            => config.param('destination_project',            :string,  :default => nil),
           'dataset'                        => config.param('dataset',                        :string),
           'location'                       => config.param('location',                       :string,  :default => nil),
           'table'                          => config.param('table',                          :string),
@@ -89,6 +90,8 @@ module Embulk
           'clustering'                     => config.param('clustering',                     :hash,    :default => nil), # google-api-ruby-client >= v0.21.0
           'schema_update_options'          => config.param('schema_update_options',          :array,   :default => nil),
 
+          'temporary_table_expiration'     => config.param('temporary_table_expiration',     :integer, :default => nil),
+
           # for debug
           'skip_load'                      => config.param('skip_load',                      :bool,    :default => false),
           'temp_table'                     => config.param('temp_table',                     :string,  :default => nil),
@@ -135,12 +138,13 @@ module Embulk
             json_key = JSON.parse(task['json_keyfile'])
             task['project'] ||= json_key['project_id']
           rescue => e
-            raise ConfigError.new "json_keyfile is not a JSON file"
+            raise ConfigError.new "Parsing 'json_keyfile' failed with error: #{e.class} #{e.message}"
           end
         end
         if task['project'].nil?
           raise ConfigError.new "Required field \"project\" is not set"
         end
+        task['destination_project'] ||= task['project']
 
         if (task['payload_column'] or task['payload_column_index']) and task['auto_create_table']
           if task['schema_file'].nil? and task['template_table'].nil?
@@ -166,7 +170,7 @@ module Embulk
           begin
             JSON.parse(File.read(task['schema_file']))
           rescue => e
-            raise ConfigError.new "schema_file #{task['schema_file']} is not a JSON file"
+            raise ConfigError.new "Parsing 'schema_file' #{task['schema_file']} failed with error: #{e.class} #{e.message}"
           end
         end
 
@@ -298,19 +302,23 @@ module Embulk
           end
         end
 
+        temp_table_expiration = task['temporary_table_expiration']
+        temp_options = {'expiration_time' => temp_table_expiration}
+
         case task['mode']
         when 'delete_in_advance'
           bigquery.delete_table_or_partition(task['table'])
           bigquery.create_table_if_not_exists(task['table'])
         when 'replace'
-          bigquery.create_table_if_not_exists(task['temp_table'])
+          bigquery.create_table_if_not_exists(task['temp_table'], options: temp_options)
           bigquery.create_table_if_not_exists(task['table']) # needs for when task['table'] is a partition
         when 'append'
-          bigquery.create_table_if_not_exists(task['temp_table'])
+          bigquery.create_table_if_not_exists(task['temp_table'], options: temp_options)
           bigquery.create_table_if_not_exists(task['table']) # needs for when task['table'] is a partition
         when 'replace_backup'
-          bigquery.create_table_if_not_exists(task['temp_table'])
+          bigquery.create_table_if_not_exists(task['temp_table'], options: temp_options)
           bigquery.create_table_if_not_exists(task['table'])
+
           bigquery.create_table_if_not_exists(task['table_old'], dataset: task['dataset_old']) # needs for when a partition
         else # append_direct
           if task['auto_create_table']

data/test/test_bigquery_client.rb

@@ -29,6 +29,7 @@ else
         def least_task
           {
             'project'          => JSON.parse(File.read(JSON_KEYFILE))['project_id'],
+            'destination_project' => JSON.parse(File.read(JSON_KEYFILE))['project_id'],
             'dataset'          => 'your_dataset_name',
             'table'            => 'your_table_name',
             'auth_method'      => 'json_key',

data/test/test_configure.rb

@@ -45,6 +45,7 @@ module Embulk
         assert_equal "application_default", task['auth_method']
         assert_equal nil, task['json_keyfile']
         assert_equal "your_project_name", task['project']
+        assert_equal "your_project_name", task['destination_project']
         assert_equal "your_dataset_name", task['dataset']
         assert_equal nil, task['location']
         assert_equal "your_table_name", task['table']
@@ -284,6 +285,16 @@ module Embulk
         config = least_config.merge('schema_update_options' => ['FOO'])
         assert_raise { Bigquery.configure(config, schema, processor_count) }
       end
+
+      def test_destination_project
+        config = least_config.merge('destination_project' => 'your_destination_project_name')
+        task = Bigquery.configure(config, schema, processor_count)
+
+        assert_nothing_raised { Bigquery.configure(config, schema, processor_count) }
+        assert_equal 'your_destination_project_name', task['destination_project']
+        assert_equal 'your_project_name', task['project']
+      end
+
     end
   end
 end

data/test/test_helper.rb

@@ -63,7 +63,8 @@ module Embulk
             Column.new({index: 3, name: 'string', type: :string}),
             Column.new({index: 4, name: 'timestamp', type: :timestamp}),
             Column.new({index: 5, name: 'date', type: :timestamp}),
-            Column.new({index: 6, name: 'json', type: :json}),
+            Column.new({index: 6, name: 'datetime', type: :timestamp}),
+            Column.new({index: 7, name: 'json', type: :json}),
           ])
           task = {
             'column_options' => [
@@ -73,6 +74,7 @@ module Embulk
               {'name' => 'string',    'type' => 'INTEGER'},
               {'name' => 'timestamp', 'type' => 'INTEGER'},
               {'name' => 'date',      'type' => 'DATE'},
+              {'name' => 'datetime',  'type' => 'DATETIME'},
               {'name' => 'json',      'type' => 'RECORD', 'fields' => [
                 { 'name' => 'key1',   'type' => 'STRING' },
               ]},
@@ -85,6 +87,7 @@ module Embulk
             {name: 'string',    type: 'INTEGER'},
             {name: 'timestamp', type: 'INTEGER'},
             {name: 'date',      type: 'DATE'},
+            {name: 'datetime',  type: 'DATETIME'},
             {name: 'json',      type: 'RECORD', fields: [
               {name: 'key1',    type: 'STRING'},
             ]},

data/test/test_value_converter_factory.rb

@@ -94,6 +94,10 @@ module Embulk
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
         end
 
+        def test_datetime
+          assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter }
+        end
+
         def test_record
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
         end
@@ -138,6 +142,10 @@ module Embulk
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
         end
 
+        def test_datetime
+          assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter }
+        end
+
         def test_record
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
         end
@@ -178,6 +186,10 @@ module Embulk
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
         end
 
+        def test_datetime
+          assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter }
+        end
+
         def test_record
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
         end
@@ -236,6 +248,20 @@ module Embulk
           assert_raise { converter.call('foo') }
         end
 
+        def test_datetime
+          converter = ValueConverterFactory.new(
+            SCHEMA_TYPE, 'DATETIME',
+            timestamp_format: '%Y/%m/%d'
+          ).create_converter
+          assert_equal nil, converter.call(nil)
+          assert_equal "2016-02-26 00:00:00.000000", converter.call("2016/02/26")
+
+          # Users must care of BQ datetime format by themselves with no timestamp_format
+          converter = ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter
+          assert_equal nil, converter.call(nil)
+          assert_equal "2016-02-26 00:00:00", converter.call("2016-02-26 00:00:00")
+        end
+
         def test_record
           converter = ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter
           assert_equal({'foo'=>'foo'}, converter.call(%Q[{"foo":"foo"}]))
@@ -294,7 +320,7 @@ module Embulk
           timestamp = Time.parse("2016-02-26 00:00:00.500000 +00:00")
           expected = "2016-02-26"
           assert_equal expected, converter.call(timestamp)
-          
+
           converter = ValueConverterFactory.new(
             SCHEMA_TYPE, 'DATE', timezone: 'Asia/Tokyo'
           ).create_converter
@@ -306,6 +332,24 @@ module Embulk
           assert_raise { converter.call('foo') }
         end
 
+        def test_datetime
+          converter = ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter
+          assert_equal nil, converter.call(nil)
+          timestamp = Time.parse("2016-02-26 00:00:00.500000 +00:00")
+          expected = "2016-02-26 00:00:00.500000"
+          assert_equal expected, converter.call(timestamp)
+
+          converter = ValueConverterFactory.new(
+            SCHEMA_TYPE, 'DATETIME', timezone: 'Asia/Tokyo'
+          ).create_converter
+          assert_equal nil, converter.call(nil)
+          timestamp = Time.parse("2016-02-25 15:00:00.500000 +00:00")
+          expected = "2016-02-26 00:00:00.500000"
+          assert_equal expected, converter.call(timestamp)
+
+          assert_raise { converter.call('foo') }
+        end
+
         def test_record
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
         end

metadata

@@ -1,17 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.6.3
+  version: 0.6.7
 platform: ruby
 authors:
 - Satoshi Akama
 - Naotoshi Seo
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-28 00:00:00.000000000 Z
+date: 2021-09-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: signet
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -20,9 +21,8 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 0.12.0
-  name: signet
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -32,56 +32,90 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.12.0
 - !ruby/object:Gem::Dependency
+  name: google-api-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "<"
       - !ruby/object:Gem::Version
         version: 0.33.0
-  name: google-api-client
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "<"
       - !ruby/object:Gem::Version
         version: 0.33.0
 - !ruby/object:Gem::Dependency
+  name: time_with_zone
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: time_with_zone
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: representable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+- !ruby/object:Gem::Dependency
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.10.6
-  name: bundler
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.10.6
 - !ruby/object:Gem::Dependency
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
-  name: rake
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -123,7 +157,7 @@ homepage: https://github.com/embulk/embulk-output-bigquery
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -138,9 +172,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.14.1
-signing_key:
+rubygems_version: 3.0.3
+signing_key: 
 specification_version: 4
 summary: Google BigQuery output plugin for Embulk
 test_files: