embulk-output-bigquery 0.6.6 → 0.6.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -0
- data/README.md +7 -0
- data/embulk-output-bigquery.gemspec +1 -1
- data/lib/embulk/output/bigquery/bigquery_client.rb +5 -0
- data/lib/embulk/output/bigquery/google_client.rb +7 -1
- data/lib/embulk/output/bigquery/value_converter_factory.rb +4 -0
- data/lib/embulk/output/bigquery.rb +9 -3
- data/test/test_value_converter_factory.rb +6 -0
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c09f158dadb5a3b59f6ffb41d0f9736274e33bcbfbf09898018c0a5055f75010
|
4
|
+
data.tar.gz: b62efb97686156f0fdf8597ed24322f7af3332682b07e5f13a2270f9cf5d8270
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b108f379dae0cf5e8eedaff9be22de2234963b8fd57dfdc11b9458593cac27a99174e6d95e15919a54208c15cdcabe5f60c4ea7a0f959011db9f5e7c4e75042f
|
7
|
+
data.tar.gz: e3f276763e4efc8f83ef26d641540263f5127c5ad1c941a01b1634a1bdc5d87ac278e0e7172ea5249bfd010a1ddcbe5d97d442ccb35be7222ea87e84255d9f10
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,10 @@
|
|
1
|
+
## 0.6.8 - 2022-10-12
|
2
|
+
* [enhancement] Support JSON type (thanks to @civitaspo )
|
3
|
+
* [maintenance] Add an error message in order to retry (thanks to @mzumi)
|
4
|
+
|
5
|
+
## 0.6.7 - 2021-09-10
|
6
|
+
* [enhancement] Add an expiration option of temporary table to clean up (thanks to @TKNGUE)
|
7
|
+
|
1
8
|
## 0.6.6 - 2021-06-10
|
2
9
|
|
3
10
|
* [maintenance] Fix network retry function (thanks to @case-k-git)
|
data/README.md
CHANGED
@@ -79,6 +79,13 @@ Options for intermediate local files
|
|
79
79
|
| delete_from_local_when_job_end | boolean | optional | true | If set to true, delete generate local files when job is end |
|
80
80
|
| compression | string | optional | "NONE" | Compression of local files (`GZIP` or `NONE`) |
|
81
81
|
|
82
|
+
|
83
|
+
Options for intermediate tables on BigQuery
|
84
|
+
|
85
|
+
| name | type | required? | default | description |
|
86
|
+
|:-------------------------------------|:------------|:-----------|:-------------------------|:-----------------------|
|
87
|
+
| temporary_table_expiration | integer | optional | | Temporary table's expiration time in seconds |
|
88
|
+
|
82
89
|
`source_format` is also used to determine formatter (csv or jsonl).
|
83
90
|
|
84
91
|
#### Same options of bq command-line tools or BigQuery job's property
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |spec|
|
2
2
|
spec.name = "embulk-output-bigquery"
|
3
|
-
spec.version = "0.6.
|
3
|
+
spec.version = "0.6.8"
|
4
4
|
spec.authors = ["Satoshi Akama", "Naotoshi Seo"]
|
5
5
|
spec.summary = "Google BigQuery output plugin for Embulk"
|
6
6
|
spec.description = "Embulk plugin that insert records to Google BigQuery."
|
@@ -440,6 +440,11 @@ module Embulk
|
|
440
440
|
}
|
441
441
|
end
|
442
442
|
|
443
|
+
if options['expiration_time']
|
444
|
+
# expiration_time is expressed in milliseconds
|
445
|
+
body[:expiration_time] = (Time.now.to_i + options['expiration_time']) * 1000
|
446
|
+
end
|
447
|
+
|
443
448
|
opts = {}
|
444
449
|
Embulk.logger.debug { "embulk-output-bigquery: insert_table(#{@project}, #{dataset}, #{@location_for_log}, #{body}, #{opts})" }
|
445
450
|
with_network_retry { client.insert_table(@project, dataset, body, opts) }
|
@@ -50,7 +50,13 @@ module Embulk
|
|
50
50
|
begin
|
51
51
|
yield
|
52
52
|
rescue ::Java::Java.net.SocketException, ::Java::Java.net.ConnectException => e
|
53
|
-
|
53
|
+
retry_messages = [
|
54
|
+
'Broken pipe',
|
55
|
+
'Connection reset',
|
56
|
+
'Connection timed out',
|
57
|
+
'Connection or outbound has closed',
|
58
|
+
]
|
59
|
+
if retry_messages.select { |x| e.message.include?(x) }.empty?
|
54
60
|
raise e
|
55
61
|
else
|
56
62
|
if retries < @task['retries']
|
@@ -90,6 +90,8 @@ module Embulk
|
|
90
90
|
'clustering' => config.param('clustering', :hash, :default => nil), # google-api-ruby-client >= v0.21.0
|
91
91
|
'schema_update_options' => config.param('schema_update_options', :array, :default => nil),
|
92
92
|
|
93
|
+
'temporary_table_expiration' => config.param('temporary_table_expiration', :integer, :default => nil),
|
94
|
+
|
93
95
|
# for debug
|
94
96
|
'skip_load' => config.param('skip_load', :bool, :default => false),
|
95
97
|
'temp_table' => config.param('temp_table', :string, :default => nil),
|
@@ -300,19 +302,23 @@ module Embulk
|
|
300
302
|
end
|
301
303
|
end
|
302
304
|
|
305
|
+
temp_table_expiration = task['temporary_table_expiration']
|
306
|
+
temp_options = {'expiration_time' => temp_table_expiration}
|
307
|
+
|
303
308
|
case task['mode']
|
304
309
|
when 'delete_in_advance'
|
305
310
|
bigquery.delete_table_or_partition(task['table'])
|
306
311
|
bigquery.create_table_if_not_exists(task['table'])
|
307
312
|
when 'replace'
|
308
|
-
bigquery.create_table_if_not_exists(task['temp_table'])
|
313
|
+
bigquery.create_table_if_not_exists(task['temp_table'], options: temp_options)
|
309
314
|
bigquery.create_table_if_not_exists(task['table']) # needs for when task['table'] is a partition
|
310
315
|
when 'append'
|
311
|
-
bigquery.create_table_if_not_exists(task['temp_table'])
|
316
|
+
bigquery.create_table_if_not_exists(task['temp_table'], options: temp_options)
|
312
317
|
bigquery.create_table_if_not_exists(task['table']) # needs for when task['table'] is a partition
|
313
318
|
when 'replace_backup'
|
314
|
-
bigquery.create_table_if_not_exists(task['temp_table'])
|
319
|
+
bigquery.create_table_if_not_exists(task['temp_table'], options: temp_options)
|
315
320
|
bigquery.create_table_if_not_exists(task['table'])
|
321
|
+
|
316
322
|
bigquery.create_table_if_not_exists(task['table_old'], dataset: task['dataset_old']) # needs for when a partition
|
317
323
|
else # append_direct
|
318
324
|
if task['auto_create_table']
|
@@ -389,6 +389,12 @@ module Embulk
|
|
389
389
|
assert_equal nil, converter.call(nil)
|
390
390
|
assert_equal({'foo'=>'foo'}, converter.call({'foo'=>'foo'}))
|
391
391
|
end
|
392
|
+
|
393
|
+
def test_json
|
394
|
+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'JSON').create_converter
|
395
|
+
assert_equal nil, converter.call(nil)
|
396
|
+
assert_equal({'foo'=>'foo'}, converter.call({'foo'=>'foo'}))
|
397
|
+
end
|
392
398
|
end
|
393
399
|
|
394
400
|
def test_strict_false
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-bigquery
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Satoshi Akama
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2022-10-12 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: signet
|
@@ -172,7 +172,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
172
172
|
- !ruby/object:Gem::Version
|
173
173
|
version: '0'
|
174
174
|
requirements: []
|
175
|
-
rubygems_version: 3.
|
175
|
+
rubygems_version: 3.3.7
|
176
176
|
signing_key:
|
177
177
|
specification_version: 4
|
178
178
|
summary: Google BigQuery output plugin for Embulk
|