embulk-output-bigquery 0.4.9 → 0.4.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/README.md +17 -2
- data/embulk-output-bigquery.gemspec +1 -1
- data/lib/embulk/output/bigquery/bigquery_client.rb +2 -0
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 546e23b30d21823bbf956e4db3c6fb83bb0584eea8f5bc9796ebe2f34bf20e0c
|
4
|
+
data.tar.gz: 9a7f85a1890cb117153b5dd5089cf7b20788544b929ef19be3f229d21da45d42
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: da50a836f4f23c3d1aba39d9db8c85cdf0781e73da784db4898ce781470ead74b37dc8ae2d1dfa1f8098f0ff73341b4ac9c0c9e23656a9987ae6ca28285abffa
|
7
|
+
data.tar.gz: b4f6e491f85701e095f2d1df739388cd6989ca5349838890aac1deea0d21f1d028ca5ebada8dc82dcf1ecffca94c16107e39caba53d68fccb91c0e7089e15106
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -105,8 +105,10 @@ Following options are same as [bq command-line tools](https://cloud.google.com/b
|
|
105
105
|
| allow_quoted_newlines | boolean | optional | false | Set true, if data contains newline characters. It may cause slow procsssing |
|
106
106
|
| time_partitioning | hash | optional | `{"type":"DAY"}` if `table` parameter has a partition decorator, otherwise nil | See [Time Partitioning](#time-partitioning) |
|
107
107
|
| time_partitioning.type | string | required | nil | The only type supported is DAY, which will generate one partition per day based on data loading time. |
|
108
|
-
| time_partitioning.expiration_ms
|
109
|
-
|
|
108
|
+
| time_partitioning.expiration_ms | int | optional | nil | Number of milliseconds for which to keep the storage for a partition. |
|
109
|
+
| time_partitioning.field | string | optional | nil | `DATE` or `TIMESTAMP` column used for partitioning |
|
110
|
+
| time_partitioning.requirePartitionFilter | boolean | optional | nil | If ture, valid partition filter is required when query |
|
111
|
+
| schema_update_options | array | optional | nil | (Experimental) List of `ALLOW_FIELD_ADDITION` or `ALLOW_FIELD_RELAXATION` or both. See [jobs#configuration.load.schemaUpdateOptions](https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.load.schemaUpdateOptions). NOTE for the current status: `schema_update_options` does not work for `copy` job, that is, is not effective for most of modes such as `append`, `replace` and `replace_backup`. `delete_in_advance` deletes origin table so does not need to update schema. Only `append_direct` can utilize schema update. |
|
110
112
|
|
111
113
|
### Example
|
112
114
|
|
@@ -409,6 +411,19 @@ out:
|
|
409
411
|
expiration_ms: 259200000
|
410
412
|
```
|
411
413
|
|
414
|
+
You can also create column-based partitioning table as:
|
415
|
+
```yaml
|
416
|
+
out:
|
417
|
+
type: bigquery
|
418
|
+
mode: replace
|
419
|
+
auto_create_table: true
|
420
|
+
table: table_name
|
421
|
+
time_partitioning:
|
422
|
+
type: DAY
|
423
|
+
field: timestamp
|
424
|
+
```
|
425
|
+
Note the `time_partitioning.field` should be top-level `DATE` or `TIMESTAMP`.
|
426
|
+
|
412
427
|
Use [Tables: patch](https://cloud.google.com/bigquery/docs/reference/v2/tables/patch) API to update the schema of the partitioned table, embulk-output-bigquery itself does not support it, though.
|
413
428
|
Note that only adding a new column, and relaxing non-necessary columns to be `NULLABLE` are supported now. Deleting columns, and renaming columns are not supported.
|
414
429
|
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |spec|
|
2
2
|
spec.name = "embulk-output-bigquery"
|
3
|
-
spec.version = "0.4.
|
3
|
+
spec.version = "0.4.10"
|
4
4
|
spec.authors = ["Satoshi Akama", "Naotoshi Seo"]
|
5
5
|
spec.summary = "Google BigQuery output plugin for Embulk"
|
6
6
|
spec.description = "Embulk plugin that insert records to Google BigQuery."
|
@@ -429,6 +429,8 @@ module Embulk
|
|
429
429
|
body[:time_partitioning] = {
|
430
430
|
type: options['time_partitioning']['type'],
|
431
431
|
expiration_ms: options['time_partitioning']['expiration_ms'],
|
432
|
+
field: options['time_partitioning']['field'],
|
433
|
+
requirePartitionFilter: options['time_partitioning']['requirePartitionFilter'],
|
432
434
|
}
|
433
435
|
end
|
434
436
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-bigquery
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Satoshi Akama
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2018-
|
12
|
+
date: 2018-11-07 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: google-api-client
|
@@ -175,7 +175,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
175
175
|
version: '0'
|
176
176
|
requirements: []
|
177
177
|
rubyforge_project:
|
178
|
-
rubygems_version: 2.7.
|
178
|
+
rubygems_version: 2.7.6
|
179
179
|
signing_key:
|
180
180
|
specification_version: 4
|
181
181
|
summary: Google BigQuery output plugin for Embulk
|