embulk-output-bigquery 0.4.6 → 0.4.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: 860095ce62db6591bce595ae0dc7687ef821195c4462f9542ac711e1026fe986
-  data.tar.gz: a44f3f2925cb502d2c9a03ca0b8af7931582b6dd0112d4f50bc40a316e6877b9
+SHA1:
+  metadata.gz: 4ff7af5986004058e6bfc02cca1f00e9809f61bc
+  data.tar.gz: 243490b5b6468a9b204b9f7369eea55559fa08b0
 SHA512:
-  metadata.gz: 4e89c528a2151b6a348f5b1d9a8a3f7cf2b2fe7d61a062a9d95c195255e9d3598ac529a401010b260eb5f3821ef4c1b432ac947cbda2445368dcac5a6683726d
-  data.tar.gz: 84f8250e8110add2c98fb2d164b11ad6307293a4172984a62e675579b8e83a5ac214e32a1a7b01c5a1fbf2f167389b10cb728eac4ecc9adb0490fdd6e453418b
+  metadata.gz: 1ace47fce8c23201c9a9cfabfc8f35700c81495eecdf34e255fb463af89f053709a892b9ec0b1603f468d2a09d33b64773bc1ec0909de745efd020a4136504eb
+  data.tar.gz: df1cbc7852e15de10bfd7d9fb88cb4c288926438a64beb7d7f88ebc077c7accf8c02e57778329249192f03b5b5da428eb2380074be4fcafc4d0cdcf640baed5b

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.4.7 - 2017-05-02
+* [enhancement] Support location option to allow to use 'asia-northeast1' region
+
+## 0.4.6 - 2017-04-17
+* [enhancement] Support auth_method 'application_default'
+
 ## 0.4.5 - 2017-04-04
 
 * [maintenance] Fix deprecated warning log condition for `timeout_sec`

data/README.md

@@ -44,6 +44,7 @@ v0.3.x has incompatibility changes with v0.2.x. Please see [CHANGELOG.md](CHANGE
 |  json_keyfile                        | string      | required when auth_method is json_key     |   | Fullpath of json key |
 |  project                             | string      | required if json_keyfile is not given     |   | project_id |
 |  dataset                             | string      | required   |                          | dataset |
+|  location                            | string      | optional   | nil                      | geographic location of dataset. See [Location](#location) |
 |  table                               | string      | required   |                          | table name, or table name with a partition decorator such as `table_name$20160929`|
 |  auto_create_dataset                 | boolean     | optional   | false                    | automatically create dataset |
 |  auto_create_table                   | boolean     | optional   | false                    | See [Dynamic Table Creating](#dynamic-table-creating) |
@@ -123,6 +124,14 @@ out:
   source_format: NEWLINE_DELIMITED_JSON
 ```
 
+### location
+
+The geographic location of the dataset. Required except for US and EU.
+
+`auto_create_table` isn't supported except for US and EU. And GCS bucket should be in same region when you use `gcs_bucket`.
+
+See also [Dataset Locations | BigQuery | Google Cloud](https://cloud.google.com/bigquery/docs/dataset-locations)
+
 ### mode
 
 5 modes are provided.

data/embulk-output-bigquery.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-bigquery"
-  spec.version       = "0.4.6"
+  spec.version       = "0.4.7"
   spec.authors       = ["Satoshi Akama", "Naotoshi Seo"]
   spec.summary       = "Google BigQuery output plugin for Embulk"
   spec.description   = "Embulk plugin that insert records to Google BigQuery."

data/lib/embulk/output/bigquery.rb

@@ -39,6 +39,7 @@ module Embulk
           'json_keyfile'                   => config.param('json_keyfile',                  LocalFile, :default => nil),
           'project'                        => config.param('project',                        :string,  :default => nil),
           'dataset'                        => config.param('dataset',                        :string),
+          'location'                       => config.param('location',                       :string,  :default => nil),
           'table'                          => config.param('table',                          :string),
           'dataset_old'                    => config.param('dataset_old',                    :string,  :default => nil),
           'table_old'                      => config.param('table_old',                      :string,  :default => nil),
@@ -112,6 +113,17 @@ module Embulk
           task['table_old']   ||= task['table']
         end
 
+        unless task['location'].nil?
+          task['location'] = task['location'].downcase
+          # google-api-client doesn't support create bucket with region
+          # We need to use Cloud Storage Client Libraries to support it
+          if task['auto_create_gcs_bucket']
+            unless %w[us eu].include?(task['location'])
+              raise ConfigError.new "`auto_create_gcs_bucket` isn't supported excepts in us/eu"
+            end
+          end
+        end
+
         if task['table_old']
           task['table_old'] = now.strftime(task['table_old'])
         end

data/lib/embulk/output/bigquery/bigquery_client.rb

@@ -19,6 +19,8 @@ module Embulk
           reset_fields(fields) if fields
           @project = @task['project']
           @dataset = @task['dataset']
+          @location = @task['location']
+          @location_for_log = @location.nil? ? 'us/eu' : @location
 
           @task['source_format'] ||= 'CSV'
           @task['max_bad_records'] ||= 0
@@ -82,7 +84,7 @@ module Embulk
               else
                 job_id = "embulk_load_job_#{SecureRandom.uuid}"
               end
-              Embulk.logger.info { "embulk-output-bigquery: Load job starting... job_id:[#{job_id}] #{object_uris} => #{@project}:#{@dataset}.#{table}" }
+              Embulk.logger.info { "embulk-output-bigquery: Load job starting... job_id:[#{job_id}] #{object_uris} => #{@project}:#{@dataset}.#{table} in #{@location_for_log}" }
 
               body = {
                 job_reference: {
@@ -110,6 +112,10 @@ module Embulk
                   }
                 }
               }
+
+              if @location
+                body[:job_reference][:location] = @location
+              end
               
               if @task['schema_update_options']
                 body[:configuration][:load][:schema_update_options] = @task['schema_update_options']
@@ -128,7 +134,7 @@ module Embulk
               Embulk.logger.error {
                 "embulk-output-bigquery: insert_job(#{@project}, #{body}, #{opts}), response:#{response}"
               }
-              raise Error, "failed to load #{object_uris} to #{@project}:#{@dataset}.#{table}, response:#{response}"
+              raise Error, "failed to load #{object_uris} to #{@project}:#{@dataset}.#{table} in #{@location_for_log}, response:#{response}"
             end
           end
         end
@@ -173,7 +179,7 @@ module Embulk
                 else
                   job_id = "embulk_load_job_#{SecureRandom.uuid}"
                 end
-                Embulk.logger.info { "embulk-output-bigquery: Load job starting... job_id:[#{job_id}] #{path} => #{@project}:#{@dataset}.#{table}" }
+                Embulk.logger.info { "embulk-output-bigquery: Load job starting... job_id:[#{job_id}] #{path} => #{@project}:#{@dataset}.#{table} in #{@location_for_log}" }
               else
                 Embulk.logger.info { "embulk-output-bigquery: Load job starting... #{path} does not exist, skipped" }
                 return
@@ -205,6 +211,10 @@ module Embulk
                 }
               }
 
+              if @location
+                body[:job_reference][:location] = @location
+              end
+
               if @task['schema_update_options']
                 body[:configuration][:load][:schema_update_options] = @task['schema_update_options']
               end
@@ -230,7 +240,7 @@ module Embulk
               Embulk.logger.error {
                 "embulk-output-bigquery: insert_job(#{@project}, #{body}, #{opts}), response:#{response}"
               }
-              raise Error, "failed to load #{path} to #{@project}:#{@dataset}.#{table}, response:#{response}"
+              raise Error, "failed to load #{path} to #{@project}:#{@dataset}.#{table} in #{@location_for_log}, response:#{response}"
             end
           end
         end
@@ -269,6 +279,10 @@ module Embulk
                 }
               }
 
+              if @location
+                body[:job_reference][:location] = @location
+              end
+
               opts = {}
               Embulk.logger.debug { "embulk-output-bigquery: insert_job(#{@project}, #{body}, #{opts})" }
               response = with_network_retry { client.insert_job(@project, body, opts) }
@@ -312,7 +326,7 @@ module Embulk
                 "job_id:[#{job_id}] elapsed_time:#{elapsed.to_f}sec status:[#{status}]"
               }
               sleep wait_interval
-              _response = with_network_retry { client.get_job(@project, job_id) }
+              _response = with_network_retry { client.get_job(@project, job_id, location: @location) }
             end
           end
 
@@ -341,7 +355,7 @@ module Embulk
         def create_dataset(dataset = nil, reference: nil)
           dataset ||= @dataset
           begin
-            Embulk.logger.info { "embulk-output-bigquery: Create dataset... #{@project}:#{dataset}" }
+            Embulk.logger.info { "embulk-output-bigquery: Create dataset... #{@project}:#{dataset} in #{@location_for_log}" }
             hint = {}
             if reference
               response = get_dataset(reference)
@@ -353,8 +367,11 @@ module Embulk
                 dataset_id: dataset,
               },
             }.merge(hint)
+            if @location
+              body[:location] = @location
+            end
             opts = {}
-            Embulk.logger.debug { "embulk-output-bigquery: insert_dataset(#{@project}, #{dataset}, #{body}, #{opts})" }
+            Embulk.logger.debug { "embulk-output-bigquery: insert_dataset(#{@project}, #{dataset}, #{@location_for_log}, #{body}, #{opts})" }
             with_network_retry { client.insert_dataset(@project, body, opts) }
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             if e.status_code == 409 && /Already Exists:/ =~ e.message
@@ -366,14 +383,14 @@ module Embulk
             Embulk.logger.error {
               "embulk-output-bigquery: insert_dataset(#{@project}, #{body}, #{opts}), response:#{response}"
             }
-            raise Error, "failed to create dataset #{@project}:#{dataset}, response:#{response}"
+            raise Error, "failed to create dataset #{@project}:#{dataset} in #{@location_for_log}, response:#{response}"
           end
         end
 
         def get_dataset(dataset = nil)
           dataset ||= @dataset
           begin
-            Embulk.logger.info { "embulk-output-bigquery: Get dataset... #{@project}:#{@dataset}" }
+            Embulk.logger.info { "embulk-output-bigquery: Get dataset... #{@project}:#{dataset}" }
             with_network_retry { client.get_dataset(@project, dataset) }
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             if e.status_code == 404
@@ -416,7 +433,7 @@ module Embulk
             end
 
             opts = {}
-            Embulk.logger.debug { "embulk-output-bigquery: insert_table(#{@project}, #{dataset}, #{body}, #{opts})" }
+            Embulk.logger.debug { "embulk-output-bigquery: insert_table(#{@project}, #{dataset}, #{@location_for_log}, #{body}, #{opts})" }
             with_network_retry { client.insert_table(@project, dataset, body, opts) }
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             if e.status_code == 409 && /Already Exists:/ =~ e.message
@@ -426,9 +443,9 @@ module Embulk
 
             response = {status_code: e.status_code, message: e.message, error_class: e.class}
             Embulk.logger.error {
-              "embulk-output-bigquery: insert_table(#{@project}, #{dataset}, #{body}, #{opts}), response:#{response}"
+              "embulk-output-bigquery: insert_table(#{@project}, #{dataset}, #{@location_for_log}, #{body}, #{opts}), response:#{response}"
             }
-            raise Error, "failed to create table #{@project}:#{dataset}.#{table}, response:#{response}"
+            raise Error, "failed to create table #{@project}:#{dataset}.#{table} in #{@location_for_log}, response:#{response}"
           end
         end
 

data/lib/embulk/output/bigquery/helper.rb

@@ -67,6 +67,7 @@ module Embulk
           elements = [
             Digest::MD5.file(path).hexdigest,
             task['dataset'],
+            task['location'],
             task['table'],
             fields,
             task['source_format'],

data/test/test_configure.rb

@@ -49,6 +49,7 @@ module Embulk
         assert_equal nil, task['json_keyfile']
         assert_equal "your_project_name", task['project']
         assert_equal "your_dataset_name", task['dataset']
+        assert_equal nil, task['location']
         assert_equal "your_table_name", task['table']
         assert_equal nil, task['dataset_old']
         assert_equal nil, task['table_old']
@@ -103,6 +104,20 @@ module Embulk
         assert_raise { Bigquery.configure(config, schema, processor_count) }
       end
 
+      def test_location
+        config = least_config.merge('location' => 'us')
+        assert_nothing_raised { Bigquery.configure(config, schema, processor_count) }
+
+        config = least_config.merge('location' => 'eu')
+        assert_nothing_raised { Bigquery.configure(config, schema, processor_count) }
+
+        config = least_config.merge('location' => 'asia-northeast1')
+        assert_nothing_raised { Bigquery.configure(config, schema, processor_count) }
+
+        config = least_config.merge('location' => 'asia-northeast1', 'auto_create_gcs_bucket' => true)
+        assert_raise { Bigquery.configure(config, schema, processor_count) }
+      end
+
       def test_dataset_table_old
         task = nil
         config = least_config.merge('mode' => 'replace_backup', 'table_old' => 'backup')

data/test/test_helper.rb

@@ -94,6 +94,7 @@ module Embulk
       def test_create_load_job_id
         task = {
           'dataset' => 'your_dataset_name',
+          'location' => 'asia-northeast1',
           'table' => 'your_table_name',
           'source_format' => 'CSV',
           'max_bad_records' => nil,
@@ -108,6 +109,7 @@ module Embulk
         File.write("tmp/your_file_name", "foobarbaz")
         job_id = Helper.create_load_job_id(task, 'tmp/your_file_name', fields)
         assert job_id.is_a?(String)
+        assert_equal 'embulk_load_job_2abaf528b69987db0224e52bbd1f0eec', job_id
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.4.6
+  version: 0.4.7
 platform: ruby
 authors:
 - Satoshi Akama
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-17 00:00:00.000000000 Z
+date: 2018-05-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -175,7 +175,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Google BigQuery output plugin for Embulk