embulk-output-bigquery 0.6.6 → 0.6.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d48b65d07302466f8f52dadb559ad049a054453db3741d4384209125e7b9e9cd
4
- data.tar.gz: 13cd70568cfaebba819a9b7a9a51d1c45ff9f1599893b4a0b451e82dc84e40c9
3
+ metadata.gz: c09f158dadb5a3b59f6ffb41d0f9736274e33bcbfbf09898018c0a5055f75010
4
+ data.tar.gz: b62efb97686156f0fdf8597ed24322f7af3332682b07e5f13a2270f9cf5d8270
5
5
  SHA512:
6
- metadata.gz: ee51c9bf570ce2f2a55e43a5ab7842f1669d814b93cdd1395853be8f6b3ff770f6a5a6c9f9a81b6c0eca1e9bc72aff3d1302d37760fe559ecfa33a740e1da724
7
- data.tar.gz: 8ada113513a089d786bf93bce1de98ad4bcc900ff73931c164a801b29e0bad9fd4b001bdf85962570998df995209e4ff320ba74e1fae22b61fb389a621121073
6
+ metadata.gz: b108f379dae0cf5e8eedaff9be22de2234963b8fd57dfdc11b9458593cac27a99174e6d95e15919a54208c15cdcabe5f60c4ea7a0f959011db9f5e7c4e75042f
7
+ data.tar.gz: e3f276763e4efc8f83ef26d641540263f5127c5ad1c941a01b1634a1bdc5d87ac278e0e7172ea5249bfd010a1ddcbe5d97d442ccb35be7222ea87e84255d9f10
data/CHANGELOG.md CHANGED
@@ -1,3 +1,10 @@
1
+ ## 0.6.8 - 2022-10-12
2
+ * [enhancement] Support JSON type (thanks to @civitaspo )
3
+ * [maintenance] Add an error message in order to retry (thanks to @mzumi)
4
+
5
+ ## 0.6.7 - 2021-09-10
6
+ * [enhancement] Add an expiration option of temporary table to clean up (thanks to @TKNGUE)
7
+
1
8
  ## 0.6.6 - 2021-06-10
2
9
 
3
10
  * [maintenance] Fix network retry function (thanks to @case-k-git)
data/README.md CHANGED
@@ -79,6 +79,13 @@ Options for intermediate local files
79
79
  | delete_from_local_when_job_end | boolean | optional | true | If set to true, delete generate local files when job is end |
80
80
  | compression | string | optional | "NONE" | Compression of local files (`GZIP` or `NONE`) |
81
81
 
82
+
83
+ Options for intermediate tables on BigQuery
84
+
85
+ | name | type | required? | default | description |
86
+ |:-------------------------------------|:------------|:-----------|:-------------------------|:-----------------------|
87
+ | temporary_table_expiration | integer | optional | | Temporary table's expiration time in seconds |
88
+
82
89
  `source_format` is also used to determine formatter (csv or jsonl).
83
90
 
84
91
  #### Same options of bq command-line tools or BigQuery job's property
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |spec|
2
2
  spec.name = "embulk-output-bigquery"
3
- spec.version = "0.6.6"
3
+ spec.version = "0.6.8"
4
4
  spec.authors = ["Satoshi Akama", "Naotoshi Seo"]
5
5
  spec.summary = "Google BigQuery output plugin for Embulk"
6
6
  spec.description = "Embulk plugin that insert records to Google BigQuery."
@@ -440,6 +440,11 @@ module Embulk
440
440
  }
441
441
  end
442
442
 
443
+ if options['expiration_time']
444
+ # expiration_time is expressed in milliseconds
445
+ body[:expiration_time] = (Time.now.to_i + options['expiration_time']) * 1000
446
+ end
447
+
443
448
  opts = {}
444
449
  Embulk.logger.debug { "embulk-output-bigquery: insert_table(#{@project}, #{dataset}, #{@location_for_log}, #{body}, #{opts})" }
445
450
  with_network_retry { client.insert_table(@project, dataset, body, opts) }
@@ -50,7 +50,13 @@ module Embulk
50
50
  begin
51
51
  yield
52
52
  rescue ::Java::Java.net.SocketException, ::Java::Java.net.ConnectException => e
53
- if ['Broken pipe', 'Connection reset', 'Connection timed out'].select { |x| e.message.include?(x) }.empty?
53
+ retry_messages = [
54
+ 'Broken pipe',
55
+ 'Connection reset',
56
+ 'Connection timed out',
57
+ 'Connection or outbound has closed',
58
+ ]
59
+ if retry_messages.select { |x| e.message.include?(x) }.empty?
54
60
  raise e
55
61
  else
56
62
  if retries < @task['retries']
@@ -288,6 +288,10 @@ module Embulk
288
288
  Proc.new {|val|
289
289
  val
290
290
  }
291
+ when 'JSON'
292
+ Proc.new {|val|
293
+ val
294
+ }
291
295
  else
292
296
  raise NotSupportedType, "cannot take column type #{type} for json column"
293
297
  end
@@ -90,6 +90,8 @@ module Embulk
90
90
  'clustering' => config.param('clustering', :hash, :default => nil), # google-api-ruby-client >= v0.21.0
91
91
  'schema_update_options' => config.param('schema_update_options', :array, :default => nil),
92
92
 
93
+ 'temporary_table_expiration' => config.param('temporary_table_expiration', :integer, :default => nil),
94
+
93
95
  # for debug
94
96
  'skip_load' => config.param('skip_load', :bool, :default => false),
95
97
  'temp_table' => config.param('temp_table', :string, :default => nil),
@@ -300,19 +302,23 @@ module Embulk
300
302
  end
301
303
  end
302
304
 
305
+ temp_table_expiration = task['temporary_table_expiration']
306
+ temp_options = {'expiration_time' => temp_table_expiration}
307
+
303
308
  case task['mode']
304
309
  when 'delete_in_advance'
305
310
  bigquery.delete_table_or_partition(task['table'])
306
311
  bigquery.create_table_if_not_exists(task['table'])
307
312
  when 'replace'
308
- bigquery.create_table_if_not_exists(task['temp_table'])
313
+ bigquery.create_table_if_not_exists(task['temp_table'], options: temp_options)
309
314
  bigquery.create_table_if_not_exists(task['table']) # needs for when task['table'] is a partition
310
315
  when 'append'
311
- bigquery.create_table_if_not_exists(task['temp_table'])
316
+ bigquery.create_table_if_not_exists(task['temp_table'], options: temp_options)
312
317
  bigquery.create_table_if_not_exists(task['table']) # needs for when task['table'] is a partition
313
318
  when 'replace_backup'
314
- bigquery.create_table_if_not_exists(task['temp_table'])
319
+ bigquery.create_table_if_not_exists(task['temp_table'], options: temp_options)
315
320
  bigquery.create_table_if_not_exists(task['table'])
321
+
316
322
  bigquery.create_table_if_not_exists(task['table_old'], dataset: task['dataset_old']) # needs for when a partition
317
323
  else # append_direct
318
324
  if task['auto_create_table']
@@ -389,6 +389,12 @@ module Embulk
389
389
  assert_equal nil, converter.call(nil)
390
390
  assert_equal({'foo'=>'foo'}, converter.call({'foo'=>'foo'}))
391
391
  end
392
+
393
+ def test_json
394
+ converter = ValueConverterFactory.new(SCHEMA_TYPE, 'JSON').create_converter
395
+ assert_equal nil, converter.call(nil)
396
+ assert_equal({'foo'=>'foo'}, converter.call({'foo'=>'foo'}))
397
+ end
392
398
  end
393
399
 
394
400
  def test_strict_false
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-bigquery
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.6
4
+ version: 0.6.8
5
5
  platform: ruby
6
6
  authors:
7
7
  - Satoshi Akama
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2021-06-10 00:00:00.000000000 Z
12
+ date: 2022-10-12 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: signet
@@ -172,7 +172,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
172
172
  - !ruby/object:Gem::Version
173
173
  version: '0'
174
174
  requirements: []
175
- rubygems_version: 3.0.3
175
+ rubygems_version: 3.3.7
176
176
  signing_key:
177
177
  specification_version: 4
178
178
  summary: Google BigQuery output plugin for Embulk