embulk-output-bigquery 0.6.6 → 0.6.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d48b65d07302466f8f52dadb559ad049a054453db3741d4384209125e7b9e9cd
-  data.tar.gz: 13cd70568cfaebba819a9b7a9a51d1c45ff9f1599893b4a0b451e82dc84e40c9
+  metadata.gz: c09f158dadb5a3b59f6ffb41d0f9736274e33bcbfbf09898018c0a5055f75010
+  data.tar.gz: b62efb97686156f0fdf8597ed24322f7af3332682b07e5f13a2270f9cf5d8270
 SHA512:
-  metadata.gz: ee51c9bf570ce2f2a55e43a5ab7842f1669d814b93cdd1395853be8f6b3ff770f6a5a6c9f9a81b6c0eca1e9bc72aff3d1302d37760fe559ecfa33a740e1da724
-  data.tar.gz: 8ada113513a089d786bf93bce1de98ad4bcc900ff73931c164a801b29e0bad9fd4b001bdf85962570998df995209e4ff320ba74e1fae22b61fb389a621121073
+  metadata.gz: b108f379dae0cf5e8eedaff9be22de2234963b8fd57dfdc11b9458593cac27a99174e6d95e15919a54208c15cdcabe5f60c4ea7a0f959011db9f5e7c4e75042f
+  data.tar.gz: e3f276763e4efc8f83ef26d641540263f5127c5ad1c941a01b1634a1bdc5d87ac278e0e7172ea5249bfd010a1ddcbe5d97d442ccb35be7222ea87e84255d9f10

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.6.8 - 2022-10-12
+* [enhancement] Support JSON type (thanks to @civitaspo )
+* [maintenance] Add an error message in order to retry (thanks to @mzumi)
+
+## 0.6.7 - 2021-09-10
+* [enhancement] Add an expiration option of temporary table to clean up (thanks to @TKNGUE)
+
 ## 0.6.6 - 2021-06-10
 
 * [maintenance] Fix network retry function (thanks to @case-k-git)

data/README.md

@@ -79,6 +79,13 @@ Options for intermediate local files
 |  delete_from_local_when_job_end      | boolean     | optional   | true                     | If set to true, delete generate local files when job is end |
 |  compression                         | string      | optional   | "NONE"                   | Compression of local files (`GZIP` or `NONE`) |
 
+
+Options for intermediate tables on BigQuery
+
+| name                                 | type        | required?  | default                  | description            |
+|:-------------------------------------|:------------|:-----------|:-------------------------|:-----------------------|
+|  temporary_table_expiration          | integer     | optional   |                          | Temporary table's expiration time in seconds |
+
 `source_format` is also used to determine formatter (csv or jsonl).
 
 #### Same options of bq command-line tools or BigQuery job's property

data/embulk-output-bigquery.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-bigquery"
-  spec.version       = "0.6.6"
+  spec.version       = "0.6.8"
   spec.authors       = ["Satoshi Akama", "Naotoshi Seo"]
   spec.summary       = "Google BigQuery output plugin for Embulk"
   spec.description   = "Embulk plugin that insert records to Google BigQuery."

data/lib/embulk/output/bigquery/bigquery_client.rb

@@ -440,6 +440,11 @@ module Embulk
               }
             end
 
+            if options['expiration_time']
+              # expiration_time is expressed in milliseconds
+              body[:expiration_time] = (Time.now.to_i + options['expiration_time']) * 1000
+            end
+
             opts = {}
             Embulk.logger.debug { "embulk-output-bigquery: insert_table(#{@project}, #{dataset}, #{@location_for_log}, #{body}, #{opts})" }
             with_network_retry { client.insert_table(@project, dataset, body, opts) }

data/lib/embulk/output/bigquery/google_client.rb

@@ -50,7 +50,13 @@ module Embulk
           begin
             yield
           rescue ::Java::Java.net.SocketException, ::Java::Java.net.ConnectException => e
-            if ['Broken pipe', 'Connection reset', 'Connection timed out'].select { |x| e.message.include?(x) }.empty?
+            retry_messages = [
+              'Broken pipe',
+              'Connection reset',
+              'Connection timed out',
+              'Connection or outbound has closed',
+            ]
+            if retry_messages.select { |x| e.message.include?(x) }.empty?
               raise e
             else
               if retries < @task['retries']

data/lib/embulk/output/bigquery/value_converter_factory.rb

@@ -288,6 +288,10 @@ module Embulk
             Proc.new {|val|
               val
             }
+          when 'JSON'
+            Proc.new {|val|
+              val
+            }
           else
             raise NotSupportedType, "cannot take column type #{type} for json column"
           end

data/lib/embulk/output/bigquery.rb

@@ -90,6 +90,8 @@ module Embulk
           'clustering'                     => config.param('clustering',                     :hash,    :default => nil), # google-api-ruby-client >= v0.21.0
           'schema_update_options'          => config.param('schema_update_options',          :array,   :default => nil),
 
+          'temporary_table_expiration'     => config.param('temporary_table_expiration',     :integer, :default => nil),
+
           # for debug
           'skip_load'                      => config.param('skip_load',                      :bool,    :default => false),
           'temp_table'                     => config.param('temp_table',                     :string,  :default => nil),
@@ -300,19 +302,23 @@ module Embulk
           end
         end
 
+        temp_table_expiration = task['temporary_table_expiration']
+        temp_options = {'expiration_time' => temp_table_expiration}
+
         case task['mode']
         when 'delete_in_advance'
           bigquery.delete_table_or_partition(task['table'])
           bigquery.create_table_if_not_exists(task['table'])
         when 'replace'
-          bigquery.create_table_if_not_exists(task['temp_table'])
+          bigquery.create_table_if_not_exists(task['temp_table'], options: temp_options)
           bigquery.create_table_if_not_exists(task['table']) # needs for when task['table'] is a partition
         when 'append'
-          bigquery.create_table_if_not_exists(task['temp_table'])
+          bigquery.create_table_if_not_exists(task['temp_table'], options: temp_options)
           bigquery.create_table_if_not_exists(task['table']) # needs for when task['table'] is a partition
         when 'replace_backup'
-          bigquery.create_table_if_not_exists(task['temp_table'])
+          bigquery.create_table_if_not_exists(task['temp_table'], options: temp_options)
           bigquery.create_table_if_not_exists(task['table'])
+
           bigquery.create_table_if_not_exists(task['table_old'], dataset: task['dataset_old']) # needs for when a partition
         else # append_direct
           if task['auto_create_table']

data/test/test_value_converter_factory.rb

@@ -389,6 +389,12 @@ module Embulk
           assert_equal nil, converter.call(nil)
           assert_equal({'foo'=>'foo'}, converter.call({'foo'=>'foo'}))
         end
+
+        def test_json
+          converter = ValueConverterFactory.new(SCHEMA_TYPE, 'JSON').create_converter
+          assert_equal nil, converter.call(nil)
+          assert_equal({'foo'=>'foo'}, converter.call({'foo'=>'foo'}))
+        end
       end
 
       def test_strict_false

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.6.6
+  version: 0.6.8
 platform: ruby
 authors:
 - Satoshi Akama
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-06-10 00:00:00.000000000 Z
+date: 2022-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: signet
@@ -172,7 +172,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.3.7
 signing_key: 
 specification_version: 4
 summary: Google BigQuery output plugin for Embulk