embulk-output-bigquery 0.4.5 → 0.4.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 6d60f1c428336993a486d43f7d1afb065503e8d4
-  data.tar.gz: 87a27028e74ec601ae71a951ba9634a01384915a
+SHA256:
+  metadata.gz: 860095ce62db6591bce595ae0dc7687ef821195c4462f9542ac711e1026fe986
+  data.tar.gz: a44f3f2925cb502d2c9a03ca0b8af7931582b6dd0112d4f50bc40a316e6877b9
 SHA512:
-  metadata.gz: c735a7c4665f2fd6daf1a2c13159c15bc68a31e4b8fc917385ef86757e72b6d9185e69b263d808907d6658409887ec91363a7d8462e7da8ca2a09097fbded503
-  data.tar.gz: 5cdda4398d326eb1ad7f2bc1c6f6a73c4db429e59d8f6a8817ac90930b761d07f5f19c141d67db5a2093910172193ad54e3100402584cd39a7827ede33439630
+  metadata.gz: 4e89c528a2151b6a348f5b1d9a8a3f7cf2b2fe7d61a062a9d95c195255e9d3598ac529a401010b260eb5f3821ef4c1b432ac947cbda2445368dcac5a6683726d
+  data.tar.gz: 84f8250e8110add2c98fb2d164b11ad6307293a4172984a62e675579b8e83a5ac214e32a1a7b01c5a1fbf2f167389b10cb728eac4ecc9adb0490fdd6e453418b

data/README.md

@@ -49,7 +49,7 @@ v0.3.x has incompatibility changes with v0.2.x. Please see [CHANGELOG.md](CHANGE
 |  auto_create_table                   | boolean     | optional   | false                    | See [Dynamic Table Creating](#dynamic-table-creating) |
 |  schema_file                         | string      | optional   |                          | /path/to/schema.json |
 |  template_table                      | string      | optional   |                          | template table name. See [Dynamic Table Creating](#dynamic-table-creating) |
-|  prevent_duplicate_insert            | boolean     | optional   | false                    | See [Prevent Duplication] (#prevent-duplication) |
+|  prevent_duplicate_insert            | boolean     | optional   | false                    | See [Prevent Duplication](#prevent-duplication) |
 |  job_status_max_polling_time         | int         | optional   | 3600 sec                 | Max job status polling time |
 |  job_status_polling_interval         | int         | optional   | 10 sec                   | Job status polling interval |
 |  is_skip_job_result_check            | boolean     | optional   | false                    | Skip waiting Load job finishes. Available for append, or delete_in_advance mode | 
@@ -315,7 +315,7 @@ However, this plugin is written in jruby, and jruby plugins are slower than java
 
 Therefore, it is recommended to format records with filter plugins written in Java such as [embulk-filter-to_json](https://github.com/civitaspo/embulk-filter-to_json) as:
 
-```
+```yaml
 filters:
   - type: to_json
     column: {name: payload, type: string}
@@ -327,7 +327,7 @@ out:
 
 Furtheremore, if your files are originally jsonl or csv files, you can even skip a parser with [embulk-parser-none](https://github.com/sonots/embulk-parser-none) as:
 
-```
+```yaml
 in:
   type: file
   path_prefix: example/example.jsonl

data/embulk-output-bigquery.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-bigquery"
-  spec.version       = "0.4.5"
+  spec.version       = "0.4.6"
   spec.authors       = ["Satoshi Akama", "Naotoshi Seo"]
   spec.summary       = "Google BigQuery output plugin for Embulk"
   spec.description   = "Embulk plugin that insert records to Google BigQuery."

data/example/config_gzip.yml

@@ -1 +1 @@
-config_csv.yml
+example/config_csv.yml

data/example/config_jsonl.yml

@@ -1 +1 @@
-config_replace.yml
+example/config_replace.yml

data/example/config_nested_record.yml

@@ -1 +1 @@
-config_guess_with_column_options.yml
+example/config_guess_with_column_options.yml

data/example/config_uncompressed.yml

@@ -1 +1 @@
-config_replace.yml
+example/config_replace.yml

data/example/example.yml

@@ -1 +1 @@
-config_replace.yml
+example/config_replace.yml

data/example/example2_1.csv

@@ -1 +1 @@
-example.csv
+example/example.csv

data/example/example2_2.csv

@@ -1 +1 @@
-example.csv
+example/example.csv

data/example/example4_1.csv

@@ -1 +1 @@
-example.csv
+example/example.csv

data/example/example4_2.csv

@@ -1 +1 @@
-example.csv
+example/example.csv

data/example/example4_3.csv

@@ -1 +1 @@
-example.csv
+example/example.csv

data/example/example4_4.csv

@@ -1 +1 @@
-example.csv
+example/example.csv

data/lib/embulk/output/bigquery.rb

@@ -120,8 +120,8 @@ module Embulk
         end
 
         task['auth_method'] = task['auth_method'].downcase
-        unless %w[private_key json_key compute_engine].include?(task['auth_method'])
-          raise ConfigError.new "`auth_method` must be one of private_key, json_key, compute_engine"
+        unless %w[private_key json_key compute_engine application_default].include?(task['auth_method'])
+          raise ConfigError.new "`auth_method` must be one of private_key, json_key, compute_engine, application_default"
         end
         if task['auth_method'] == 'private_key' and task['p12_keyfile'].nil?
           raise ConfigError.new "`p12_keyfile` is required for auth_method private_key"

data/lib/embulk/output/bigquery/bigquery_client.rb

@@ -8,6 +8,8 @@ module Embulk
   module Output
     class Bigquery < OutputPlugin
       class BigqueryClient < GoogleClient
+        BIGQUERY_TABLE_OPERATION_INTERVAL = 2 # https://cloud.google.com/bigquery/quotas
+
         def initialize(task, schema, fields = nil)
           scope = "https://www.googleapis.com/auth/bigquery"
           client_class = Google::Apis::BigqueryV2::BigqueryService
@@ -52,7 +54,11 @@ module Embulk
           retries = 0
           begin
             yield
-          rescue BackendError, InternalError => e
+          rescue BackendError, InternalError, RateLimitExceeded => e
+            if e.is_a?(RateLimitExceeded)
+              sleep(BIGQUERY_TABLE_OPERATION_INTERVAL)
+            end
+
             if retries < @task['retries']
               retries += 1
               Embulk.logger.warn { "embulk-output-bigquery: retry \##{retries}, #{e.message}" }
@@ -319,6 +325,8 @@ module Embulk
               raise BackendError, msg
             elsif _errors.any? {|error| error.reason == 'internalError' }
               raise InternalError, msg
+            elsif _errors.any? {|error| error.reason == 'rateLimitExceeded' }
+              raise RateLimitExceeded, msg
             else
               Embulk.logger.error { "embulk-output-bigquery: #{msg}" }
               raise Error, msg

data/lib/embulk/output/bigquery/google_client.rb

@@ -8,6 +8,7 @@ module Embulk
       class NotFoundError < Error; end
       class BackendError < Error; end
       class InternalError < Error; end
+      class RateLimitExceeded < Error; end
 
       class GoogleClient
         def initialize(task, scope, client_class)

data/test/test_configure.rb

@@ -69,8 +69,6 @@ module Embulk
         assert_equal "%Y-%m-%d %H:%M:%S.%6N", task['default_timestamp_format']
         assert_equal nil, task['payload_column']
         assert_equal nil, task['payload_column_index']
-        assert_equal 300, task['timeout_sec']
-        assert_equal 300, task['open_timeout_sec']
         assert_equal 5, task['retries']
         assert_equal "Embulk BigQuery plugin", task['application_name']
         # assert_equal "/tmp/embulk_output_bigquery_20160228-27184-pubcn0", task['path_prefix']

metadata

@@ -1,81 +1,81 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.4.5
+  version: 0.4.6
 platform: ruby
 authors:
 - Satoshi Akama
 - Naotoshi Seo
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-04-04 00:00:00.000000000 Z
+date: 2018-04-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: google-api-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  name: google-api-client
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: time_with_zone
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  name: time_with_zone
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: embulk
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.8.2
-  type: :development
+  name: embulk
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.8.2
 - !ruby/object:Gem::Dependency
-  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.10.6
-  type: :development
+  name: bundler
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.10.6
 - !ruby/object:Gem::Dependency
-  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
-  type: :development
+  name: rake
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -159,7 +159,7 @@ homepage: https://github.com/embulk/embulk-output-bigquery
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -174,9 +174,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.6.11
+signing_key:
 specification_version: 4
 summary: Google BigQuery output plugin for Embulk
 test_files: