embulk-output-bigquery 0.4.9 → 0.4.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1b02f82363fcc5b0d7ee79ede178abb05f7ccd261f999fde60c382a86c694620
-  data.tar.gz: 94541e8c719c18b3c1d02a16617cc2daabef545a354c35bdbd2c19b6490d24dc
+  metadata.gz: 546e23b30d21823bbf956e4db3c6fb83bb0584eea8f5bc9796ebe2f34bf20e0c
+  data.tar.gz: 9a7f85a1890cb117153b5dd5089cf7b20788544b929ef19be3f229d21da45d42
 SHA512:
-  metadata.gz: 9cc25e352180cecbe6502d031e1eda10c8bfd566b407467bffacef04b45f993ddcc0976ee09a46314453c661306ba1d2604d1ae5c3bac7633f28ea2faa4eb071
-  data.tar.gz: f11b24264d4ace3576581da99c29a561af6eb5c26a230a9fc6fc11ef897b7f44d1645c2fdfbea9cd347e21addbff6d3340baf9c9a7a1560d00a18b49761242fd
+  metadata.gz: da50a836f4f23c3d1aba39d9db8c85cdf0781e73da784db4898ce781470ead74b37dc8ae2d1dfa1f8098f0ff73341b4ac9c0c9e23656a9987ae6ca28285abffa
+  data.tar.gz: b4f6e491f85701e095f2d1df739388cd6989ca5349838890aac1deea0d21f1d028ca5ebada8dc82dcf1ecffca94c16107e39caba53d68fccb91c0e7089e15106

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.4.10 - 2018-11-08
+* [enhancement] Support column-based partition (thanks to Chi-Ruei Li)
+
 ## 0.4.9 - 2018-09-08
 * [enhancement] Enable object lifecycle management when creating buckets with `auto_create_gcs_bucket` (thanks to @potato2003)
 

data/README.md

@@ -105,8 +105,10 @@ Following options are same as [bq command-line tools](https://cloud.google.com/b
 |  allow_quoted_newlines            | boolean  | optional  | false   | Set true, if data contains newline characters. It may cause slow procsssing |
 |  time_partitioning                | hash     | optional  | `{"type":"DAY"}` if `table` parameter has a partition decorator, otherwise nil | See [Time Partitioning](#time-partitioning) |
 |  time_partitioning.type           | string   | required  | nil     | The only type supported is DAY, which will generate one partition per day based on data loading time. |
-|  time_partitioning.expiration_ms | int      | optional  | nil     | Number of milliseconds for which to keep the storage for a partition. partition |
-|  schema_update_options            | array    | optional  | nil     | (Experimental) List of `ALLOW_FIELD_ADDITION` or `ALLOW_FIELD_RELAXATION` or both. See [jobs#configuration.load.schemaUpdateOptions](https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.load.schemaUpdateOptions). NOTE for the current status: `schema_update_options` does not work for `copy` job, that is, is not effective for most of modes such as `append`, `append_direct`, `replace`, `replace_backup` (except `delete_in_advance`) |
+|  time_partitioning.expiration_ms  | int      | optional  | nil     | Number of milliseconds for which to keep the storage for a partition. |
+|  time_partitioning.field          | string   | optional  | nil     | `DATE` or `TIMESTAMP` column used for partitioning |
+|  time_partitioning.requirePartitionFilter | boolean      | optional  | nil     | If ture, valid partition filter is required when query |
+|  schema_update_options            | array    | optional  | nil     | (Experimental) List of `ALLOW_FIELD_ADDITION` or `ALLOW_FIELD_RELAXATION` or both. See [jobs#configuration.load.schemaUpdateOptions](https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.load.schemaUpdateOptions). NOTE for the current status: `schema_update_options` does not work for `copy` job, that is, is not effective for most of modes such as `append`, `replace` and `replace_backup`. `delete_in_advance` deletes origin table so does not need to update schema. Only `append_direct` can utilize schema update. |
 
 ### Example
 
@@ -409,6 +411,19 @@ out:
     expiration_ms: 259200000
 ```
 
+You can also create column-based partitioning table as:
+```yaml
+out:
+  type: bigquery
+  mode: replace
+  auto_create_table: true
+  table: table_name
+  time_partitioning:
+    type: DAY
+    field: timestamp
+```
+Note the `time_partitioning.field` should be top-level `DATE` or `TIMESTAMP`.
+
 Use [Tables: patch](https://cloud.google.com/bigquery/docs/reference/v2/tables/patch) API to update the schema of the partitioned table, embulk-output-bigquery itself does not support it, though.
 Note that only adding a new column, and relaxing non-necessary columns to be `NULLABLE` are supported now. Deleting columns, and renaming columns are not supported.
 

data/embulk-output-bigquery.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-bigquery"
-  spec.version       = "0.4.9"
+  spec.version       = "0.4.10"
   spec.authors       = ["Satoshi Akama", "Naotoshi Seo"]
   spec.summary       = "Google BigQuery output plugin for Embulk"
   spec.description   = "Embulk plugin that insert records to Google BigQuery."

data/lib/embulk/output/bigquery/bigquery_client.rb

@@ -429,6 +429,8 @@ module Embulk
               body[:time_partitioning] = {
                 type: options['time_partitioning']['type'],
                 expiration_ms: options['time_partitioning']['expiration_ms'],
+                field: options['time_partitioning']['field'],
+                requirePartitionFilter: options['time_partitioning']['requirePartitionFilter'],
               }
             end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.4.9
+  version: 0.4.10
 platform: ruby
 authors:
 - Satoshi Akama
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-08 00:00:00.000000000 Z
+date: 2018-11-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-api-client
@@ -175,7 +175,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.4
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Google BigQuery output plugin for Embulk