embulk-output-bigquery 0.4.12 → 0.4.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: 9fceabdc34780426ea3ceda76c1afe00a4c99115207bb8feaf0151ce9ae48911
-  data.tar.gz: 708649547b8a6693a12722e376c5c923a6e3f3a17ef180ebed56ecc92d6270b3
+SHA1:
+  metadata.gz: 4be15715120c9820fd039cd694a07a09826235d2
+  data.tar.gz: e12dc361afd90c250a19ec186d559fc7eeb6ee85
 SHA512:
-  metadata.gz: 8d31eb9867c7c70b9eb1b01bfb2889afae9a3693b328391d62ddc74cc552452832d50edb2af8d69285b9468cc99292c72de132f86bf48d2cb1920ab6f2be5fcf
-  data.tar.gz: 4bcab8f4bf48962985d9904c64530dd45b71dcd2afbc42388f3640636f0c1a971f0fcebda0db8bde2599ab54355210822bcf5de8d2196d42eceb9bca3d6145ae
+  metadata.gz: 755ed9ddae0b079c84e57f49b5c852138fe4b441c2b49e0ad5ab6343ff6a785356078a8955600a34143716da1c758519d8903a841cb7eec2a989b3eeaa6e51fb
+  data.tar.gz: e659fddc3c350df529845659e28d75a0c856e69ff35d2c78a99035c926f163d01d48b3e1d11ccdf7f46942aef616dac910c58c48c49f56ccb3b2338259c6d61d

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.4.13 - 2019-03-20
+
+* [enhancement] Support clustered table as an experimental feature
+
 ## 0.4.12 - 2019-03-20
 
 * [maintenance] Fix `time_partitioning.requirePartitionFilter` was not working. Use `time_partitioning.require_partition_filter` (thanks to @gitetsu)

data/README.md

@@ -108,6 +108,8 @@ Following options are same as [bq command-line tools](https://cloud.google.com/b
 |  time_partitioning.expiration_ms  | int      | optional  | nil     | Number of milliseconds for which to keep the storage for a partition. |
 |  time_partitioning.field          | string   | optional  | nil     | `DATE` or `TIMESTAMP` column used for partitioning |
 |  time_partitioning.require_partition_filter | boolean      | optional  | nil     | If true, valid partition filter is required when query |
+|  clustering                       | hash     | optional  | nil     | (Experimental) Currently, clustering is supported for partitioned tables, so must be used with `time_partitioning` option. NOTE: **clustered tables** is a beta release. See [clustered tables](https://cloud.google.com/bigquery/docs/clustered-tables) |
+|  clustering.fields                | array    | required  | nil     | One or more fields on which data should be clustered. The order of the specified columns determines the sort order of the data. |
 |  schema_update_options            | array    | optional  | nil     | (Experimental) List of `ALLOW_FIELD_ADDITION` or `ALLOW_FIELD_RELAXATION` or both. See [jobs#configuration.load.schemaUpdateOptions](https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.load.schemaUpdateOptions). NOTE for the current status: `schema_update_options` does not work for `copy` job, that is, is not effective for most of modes such as `append`, `replace` and `replace_backup`. `delete_in_advance` deletes origin table so does not need to update schema. Only `append_direct` can utilize schema update. |
 
 ### Example
@@ -466,7 +468,7 @@ $ bundle exec env RUBYOPT="-r ./embulk.jar" ruby test/test_example.rb
 
 ### Release gem:
 
-Fix gemspec, then
+Change the version of gemspec, and write CHANGELOG.md. Then,
 
 ```
 $ bundle exec rake release

data/embulk-output-bigquery.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-bigquery"
-  spec.version       = "0.4.12"
+  spec.version       = "0.4.13"
   spec.authors       = ["Satoshi Akama", "Naotoshi Seo"]
   spec.summary       = "Google BigQuery output plugin for Embulk"
   spec.description   = "Embulk plugin that insert records to Google BigQuery."

data/lib/embulk/output/bigquery.rb

@@ -89,6 +89,7 @@ module Embulk
           'ignore_unknown_values'          => config.param('ignore_unknown_values',          :bool,    :default => false),
           'allow_quoted_newlines'          => config.param('allow_quoted_newlines',          :bool,    :default => false),
           'time_partitioning'              => config.param('time_partitioning',              :hash,    :default => nil),
+          'clustering'                     => config.param('clustering',                     :hash,    :default => nil), # google-api-ruby-client >= v0.21.0
           'schema_update_options'          => config.param('schema_update_options',          :array,   :default => nil),
 
           # for debug
@@ -234,6 +235,12 @@ module Embulk
           task['time_partitioning'] = {'type' => 'DAY'}
         end
 
+        if task['clustering']
+          unless task['clustering']['fields']
+            raise ConfigError.new "`clustering` must have `fields` key"
+          end
+        end
+
         if task['schema_update_options']
           task['schema_update_options'].each do |schema_update_option|
             unless %w[ALLOW_FIELD_ADDITION ALLOW_FIELD_RELAXATION].include?(schema_update_option)

data/lib/embulk/output/bigquery/bigquery_client.rb

@@ -441,6 +441,13 @@ module Embulk
               }
             end
 
+            options['clustering'] ||= @task['clustering']
+            if options['clustering']
+              body[:clustering] = {
+                fields: options['clustering']['fields'],
+              }
+            end
+
             opts = {}
             Embulk.logger.debug { "embulk-output-bigquery: insert_table(#{@project}, #{dataset}, #{@location_for_log}, #{body}, #{opts})" }
             with_network_retry { client.insert_table(@project, dataset, body, opts) }

data/test/test_configure.rb

@@ -84,6 +84,7 @@ module Embulk
         assert_equal false, task['ignore_unknown_values']
         assert_equal false, task['allow_quoted_newlines']
         assert_equal nil, task['time_partitioning']
+        assert_equal nil, task['clustering']
         assert_equal false, task['skip_load']
       end
 
@@ -277,6 +278,14 @@ module Embulk
         assert_equal 'DAY', task['time_partitioning']['type']
       end
 
+      def test_clustering
+        config = least_config.merge('clustering' => {'fields' => ['field_a']})
+        assert_nothing_raised { Bigquery.configure(config, schema, processor_count) }
+
+        config = least_config.merge('clustering' => {})
+        assert_raise { Bigquery.configure(config, schema, processor_count) }
+      end
+
       def test_schema_update_options
         config = least_config.merge('schema_update_options' => ['ALLOW_FIELD_ADDITION', 'ALLOW_FIELD_RELAXATION'])
         assert_nothing_raised { Bigquery.configure(config, schema, processor_count) }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.4.12
+  version: 0.4.13
 platform: ruby
 authors:
 - Satoshi Akama
@@ -160,7 +160,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubyforge_project:
+rubygems_version: 2.6.14.1
 signing_key:
 specification_version: 4
 summary: Google BigQuery output plugin for Embulk