embulk-output-bigquery 0.4.10 → 0.4.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: 546e23b30d21823bbf956e4db3c6fb83bb0584eea8f5bc9796ebe2f34bf20e0c
-  data.tar.gz: 9a7f85a1890cb117153b5dd5089cf7b20788544b929ef19be3f229d21da45d42
+SHA1:
+  metadata.gz: 17069f10356213861511a440c26f35b1e2bedc2c
+  data.tar.gz: 931cc6a396ca4497c04edf9248d40d4a4a5d5669
 SHA512:
-  metadata.gz: da50a836f4f23c3d1aba39d9db8c85cdf0781e73da784db4898ce781470ead74b37dc8ae2d1dfa1f8098f0ff73341b4ac9c0c9e23656a9987ae6ca28285abffa
-  data.tar.gz: b4f6e491f85701e095f2d1df739388cd6989ca5349838890aac1deea0d21f1d028ca5ebada8dc82dcf1ecffca94c16107e39caba53d68fccb91c0e7089e15106
+  metadata.gz: 216f4ceefdd3e3a6fc3ca8e58f93d32facf66af350b861e065695a4d2acb0ffa888a9705f40ba2a3570fadae91af1e6601b6a191c4e7a5a41144bd08528b7a9f
+  data.tar.gz: ba57e3df7be7a4b8cc320dd3a61ab3db77aece068132ec0238b94da3dd417b2df9e4b2e3f754f75036b628ce583c75935b8881a49d297347312314b4afca0073

data/.gitignore

@@ -7,3 +7,4 @@ vendor/
 .ruby-version
 .tags
 your-project-000.json
+embulk.jar

data/.travis.yml

@@ -1,12 +1,17 @@
 language: ruby
-cache: bundler
-rvm:
-  - jruby-9.0.5.0
-  - jruby-head
-jdk:
-  - openjdk7
-before_install:
-  - gem install bundler
 matrix:
+  include:
+    - env: EMBULK_VERSION=0.8.39
+      rvm: jruby-9.1.5.0 # bundled jruby version
+      jdk: openjdk7 # embulk 0.8.x uses jdk7
+    - env: EMBULK_VERSION=0.9.15
+      rvm: jruby-9.1.5.0 # bundled jruby version
+      jdk: openjdk8 # embulk 0.9.x uses jdk8
+    - env: EMBULK_VERSION=latest
+      rvm: jruby-9.1.5.0 # ?
+      jdk: openjdk8 # ?
   allow_failures:
-    - rvm: jruby-head
+    - env: EMBULK_VERSION=latest
+before_install:
+  - curl -o embulk.jar --create-dirs -L "http://dl.embulk.org/embulk-${EMBULK_VERSION}.jar"
+script: bundle exec env RUBYOPT="-r ./embulk.jar" rake test

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.4.11 - 2019-03-07
+
+* [maintenance] Fix to use `response.status.error_result` instead of `response.status.errors` to check job failure status (thanks to @nownabe)
+
 ## 0.4.10 - 2018-11-08
 * [enhancement] Support column-based partition (thanks to Chi-Ruei Li)
 

data/README.md

@@ -14,7 +14,7 @@ https://developers.google.com/bigquery/loading-data-into-bigquery
 * **Cleanup supported**: no
 * **Dynamic table creating**: yes
 
-### NOT IMPLEMENTED 
+### NOT IMPLEMENTED
 * insert data over streaming inserts
   * for continuous real-time insertions
   * Please use other product, like [fluent-plugin-bigquery](https://github.com/kaizenplatform/fluent-plugin-bigquery)
@@ -35,7 +35,7 @@ v0.3.x has incompatibility changes with v0.2.x. Please see [CHANGELOG.md](CHANGE
 
 #### Original options
 
-| name                                 | type        | required?  | default                  | description            |  
+| name                                 | type        | required?  | default                  | description            |
 |:-------------------------------------|:------------|:-----------|:-------------------------|:-----------------------|
 |  mode                                | string      | optional   | "append"                 | See [Mode](#mode)     |
 |  auth_method                         | string      | optional   | "private_key"            | `private_key` , `json_key` or `compute_engine`
@@ -53,7 +53,7 @@ v0.3.x has incompatibility changes with v0.2.x. Please see [CHANGELOG.md](CHANGE
 |  prevent_duplicate_insert            | boolean     | optional   | false                    | See [Prevent Duplication](#prevent-duplication) |
 |  job_status_max_polling_time         | int         | optional   | 3600 sec                 | Max job status polling time |
 |  job_status_polling_interval         | int         | optional   | 10 sec                   | Job status polling interval |
-|  is_skip_job_result_check            | boolean     | optional   | false                    | Skip waiting Load job finishes. Available for append, or delete_in_advance mode | 
+|  is_skip_job_result_check            | boolean     | optional   | false                    | Skip waiting Load job finishes. Available for append, or delete_in_advance mode |
 |  with_rehearsal                      | boolean     | optional   | false                    | Load `rehearsal_counts` records as a rehearsal. Rehearsal loads into REHEARSAL temporary table, and delete finally. You may use this option to investigate data errors as early stage as possible |
 |  rehearsal_counts                    | integer     | optional   | 1000                     | Specify number of records to load in a rehearsal |
 |  abort_on_error                      | boolean     | optional   | true if max_bad_records is 0, otherwise false | Raise an error if number of input rows and number of output rows does not match |
@@ -80,7 +80,7 @@ Client or request options
 
 Options for intermediate local files
 
-| name                                 | type        | required?  | default                  | description            |  
+| name                                 | type        | required?  | default                  | description            |
 |:-------------------------------------|:------------|:-----------|:-------------------------|:-----------------------|
 |  path_prefix                         | string      | optional   |                          | Path prefix of local files such as "/tmp/prefix_". Default randomly generates with [tempfile](http://ruby-doc.org/stdlib-2.2.3/libdoc/tempfile/rdoc/Tempfile.html) |
 |  sequence_format                     | string      | optional   | .%d.%d                   | Sequence format for pid, thread id |
@@ -107,7 +107,7 @@ Following options are same as [bq command-line tools](https://cloud.google.com/b
 |  time_partitioning.type           | string   | required  | nil     | The only type supported is DAY, which will generate one partition per day based on data loading time. |
 |  time_partitioning.expiration_ms  | int      | optional  | nil     | Number of milliseconds for which to keep the storage for a partition. |
 |  time_partitioning.field          | string   | optional  | nil     | `DATE` or `TIMESTAMP` column used for partitioning |
-|  time_partitioning.requirePartitionFilter | boolean      | optional  | nil     | If ture, valid partition filter is required when query |
+|  time_partitioning.requirePartitionFilter | boolean      | optional  | nil     | If true, valid partition filter is required when query |
 |  schema_update_options            | array    | optional  | nil     | (Experimental) List of `ALLOW_FIELD_ADDITION` or `ALLOW_FIELD_RELAXATION` or both. See [jobs#configuration.load.schemaUpdateOptions](https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.load.schemaUpdateOptions). NOTE for the current status: `schema_update_options` does not work for `copy` job, that is, is not effective for most of modes such as `append`, `replace` and `replace_backup`. `delete_in_advance` deletes origin table so does not need to update schema. Only `append_direct` can utilize schema update. |
 
 ### Example
@@ -372,7 +372,7 @@ This is useful to reduce number of consumed jobs, which is limited by [50,000 jo
 
 This plugin originally loads local files into BigQuery in parallel, that is, consumes a number of jobs, say 24 jobs on 24 CPU core machine for example (this depends on embulk parameters such as `min_output_tasks` and `max_threads`).
 
-BigQuery supports loading multiple files from GCS with one job (but not from local files, sigh), therefore, uploading local files to GCS and then loading from GCS into BigQuery reduces number of consumed jobs.
+BigQuery supports loading multiple files from GCS with one job, therefore, uploading local files to GCS in parallel and then loading from GCS into BigQuery reduces number of consumed jobs to 1.
 
 Using `gcs_bucket` option, such strategy is enabled. You may also use `auto_create_gcs_bucket` to create the specified GCS bucket automatically.
 
@@ -444,16 +444,24 @@ $ embulk run -X page_size=1 -b . -l trace example/example.yml
 
 ### Run test:
 
+Place your embulk with `.jar` extension:
+
+```
+$ cp -a $(which embulk) embulk.jar
+```
+
+Run tests with `env RUBYOPT="-r ./embulk.jar`:
+
 ```
-$ bundle exec rake test
+$ bundle exec env RUBYOPT="-r ./embulk.jar" rake test
 ```
 
 To run tests which actually connects to BigQuery such as test/test\_bigquery\_client.rb,
 prepare a json\_keyfile at example/your-project-000.json, then
 
 ```
-$ bundle exec ruby test/test_bigquery_client.rb
-$ bundle exec ruby test/test_example.rb
+$ bundle exec env RUBYOPT="-r ./embulk.jar" ruby test/test_bigquery_client.rb
+$ bundle exec env RUBYOPT="-r ./embulk.jar" ruby test/test_example.rb
 ```
 
 ### Release gem:

data/embulk-output-bigquery.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-bigquery"
-  spec.version       = "0.4.10"
+  spec.version       = "0.4.11"
   spec.authors       = ["Satoshi Akama", "Naotoshi Seo"]
   spec.summary       = "Google BigQuery output plugin for Embulk"
   spec.description   = "Embulk plugin that insert records to Google BigQuery."
@@ -15,7 +15,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'google-api-client'
   spec.add_dependency 'time_with_zone'
 
-  spec.add_development_dependency 'embulk', ['>= 0.8.2']
   spec.add_development_dependency 'bundler', ['>= 1.10.6']
   spec.add_development_dependency 'rake', ['>= 10.0']
 end

data/lib/embulk/output/bigquery/bigquery_client.rb

@@ -330,10 +330,13 @@ module Embulk
             end
           end
 
-          # cf. http://www.rubydoc.info/github/google/google-api-ruby-client/Google/Apis/BigqueryV2/JobStatus#errors-instance_method
           # `errors` returns Array<Google::Apis::BigqueryV2::ErrorProto> if any error exists.
+          _errors = _response.status.errors
+
+          # cf. http://www.rubydoc.info/github/google/google-api-ruby-client/Google/Apis/BigqueryV2/JobStatus#errors-instance_method
+          # `error_result` returns Google::Apis::BigqueryV2::ErrorProto if job failed.
           # Otherwise, this returns nil.
-          if _errors = _response.status.errors
+          if _response.status.error_result
             msg = "failed during waiting a #{kind} job, get_job(#{@project}, #{job_id}), errors:#{_errors.map(&:to_h)}"
             if _errors.any? {|error| error.reason == 'backendError' }
               raise BackendError, msg
@@ -347,6 +350,10 @@ module Embulk
             end
           end
 
+          if _errors
+            Embulk.logger.warn { "embulk-output-bigquery: #{kind} job errors... job_id:[#{job_id}] errors:#{_errors.map(&:to_h)}" }
+          end
+
           Embulk.logger.info { "embulk-output-bigquery: #{kind} job response... job_id:[#{job_id}] response.statistics:#{_response.statistics.to_h}" }
 
           _response

data/test/helper.rb

@@ -3,9 +3,14 @@
 require 'test/unit'
 require 'test/unit/rr'
 
-# require 'embulk/java/bootstrap'
 require 'embulk'
-Embulk.setup
+begin
+  # Embulk ~> 0.8.x
+  Embulk.setup
+rescue NotImplementedError
+  # Embulk ~> 0.9.x
+  require 'embulk/java/bootstrap'
+end
 Embulk.logger = Embulk::Logger.new('/dev/null')
 
 APP_ROOT = File.expand_path('../', __dir__)

metadata

@@ -1,81 +1,67 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.4.10
+  version: 0.4.11
 platform: ruby
 authors:
 - Satoshi Akama
 - Naotoshi Seo
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-11-07 00:00:00.000000000 Z
+date: 2019-03-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: google-api-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  name: google-api-client
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: time_with_zone
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  name: time_with_zone
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: embulk
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.8.2
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.8.2
-- !ruby/object:Gem::Dependency
-  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.10.6
-  type: :development
+  name: bundler
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.10.6
 - !ruby/object:Gem::Dependency
-  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
-  type: :development
+  name: rake
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -159,7 +145,7 @@ homepage: https://github.com/embulk/embulk-output-bigquery
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -174,9 +160,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.6.14
+signing_key:
 specification_version: 4
 summary: Google BigQuery output plugin for Embulk
 test_files: