embulk-output-bigquery 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f21e4f5989b1aa631de606560ee75591a113c6f5
-  data.tar.gz: da801735b3ad2871a5d78bdde79d4f8e5e87ca30
+  metadata.gz: 14f823d8b06be1f8537c52ae244c4c94bbeb8833
+  data.tar.gz: b419e43610303eea86f1a963bde2bb4db472b26e
 SHA512:
-  metadata.gz: 582b300dacd9a45e39b424c3d0c0c3a887f5edc860430b2f0341df945ee723c0c8c5458619f28f18b2f028fe214fc3dbf58afd2751735bd2c143addb5ba164b3
-  data.tar.gz: 593d02fb4ec66bff1e3095e7e65f4d9b2adc3cb471ec3e998007ccc0fef73cfb48ad1bee6b0ee232d45ad41031affacb941e1d89097520052de007310d769465
+  metadata.gz: 009d542bbcbe0e73b40ccb76a1197c8dcd4185fcc0bc8b1819d799d74d9fc59428935250ffd67df560da14228a6d9526d9b41503eb7b88b23b1da505712a9157
+  data.tar.gz: 6215755f4975f62a949ba52e5766d61449b74ff0e63ca731caca0ceef17a69ff88a2adfe2636f1fe592f6feb51c10d7fa1624976224ff509d9a5c0bf54f8bb2e

data/.travis.yml

@@ -0,0 +1,12 @@
+language: ruby
+cache: bundler
+rvm:
+  - jruby-9.0.5.0
+  - jruby-head
+jdk:
+  - openjdk7
+before_install:
+  - gem install bundler
+matrix:
+  allow_failures:
+    - rvm: jruby-head

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.3.2 - 2016-05-03
+
+* [new feature] Add `abort_on_error` option
+* [maintenance] Use uuid instead of current time for temp_table name
+
 ## 0.3.1 - 2016-04-15
 
 * [new feature] Add `sdk_log_level` option to show log of google-api-client

data/README.md

@@ -21,6 +21,14 @@ https://developers.google.com/bigquery/loading-data-into-bigquery
 Current version of this plugin supports Google API with Service Account Authentication, but does not support
 OAuth flow for installed applications.
 
+### INCOMPATIBILITY CHANGES
+
+v0.3.x has incompatibility changes with v0.2.x. Please see [CHANGELOG.md](CHANGELOG.md) for details.
+
+* `formatter` option (formatter plugin support) is dropped. Use `source_format` option instead. (it already exists in v0.2.x too)
+* `encoders` option (encoder plugin support) is dropped. Use `compression` option instead (it already exists in v0.2.x too).
+* `mode: append` mode now expresses a transactional append, and `mode: append_direct` is one which is not transactional.
+
 ## Configuration
 
 #### Original options
@@ -45,6 +53,7 @@ OAuth flow for installed applications.
 |  is_skip_job_result_check            | boolean     | optional   | false                    | Skip waiting Load job finishes. Available for append, or delete_in_advance mode | 
 |  with_rehearsal                      | boolean     | optional   | false                    | Load `rehearsal_counts` records as a rehearsal. Rehearsal loads into REHEARSAL temporary table, and delete finally. You may use this option to investigate data errors as early stage as possible |
 |  rehearsal_counts                    | integer     | optional   | 1000                     | Specify number of records to load in a rehearsal |
+|  abort_on_error                      | boolean     | optional   | true if max_bad_records is 0, otherwise false | Raise an error if number of input rows and number of output rows does not match |
 |  column_options                      | hash        | optional   |                          | [See below](#column-options) |
 |  default_timezone                    | string      | optional   | UTC                      | |
 |  default_timestamp_format            | string      | optional   | %Y-%m-%d %H:%M:%S.%6N    | |

data/embulk-output-bigquery.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-bigquery"
-  spec.version       = "0.3.1"
+  spec.version       = "0.3.2"
   spec.authors       = ["Satoshi Akama", "Naotoshi Seo"]
   spec.summary       = "Google BigQuery output plugin for Embulk"
   spec.description   = "Embulk plugin that insert records to Google BigQuery."

data/example/config_nested_record.yml

@@ -0,0 +1,40 @@
+# embulk gem install embulk-parser-jsonl
+in:
+  type: file
+  path_prefix: example/nested_example.jsonl
+  parser:
+    type: jsonl
+    columns:
+      - {name: date,        type: string}
+      - {name: timestamp,   type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
+      - {name: "null",      type: string}
+      - {name: long,        type: long}
+      - {name: string,      type: string}
+      - {name: double,      type: double}
+      - {name: json,        type: json}
+      - {name: boolean,     type: boolean}
+out:
+  type: bigquery
+  mode: replace
+  auth_method: json_key
+  json_keyfile: example/your-project-000.json
+  dataset: your_dataset_name
+  table: your_table_name
+  compression: GZIP
+  source_format: NEWLINE_DELIMITED_JSON
+  auto_create_dataset: true
+  auto_create_table: true
+  column_options:
+    - {name: date,        type: TIMESTAMP, timestamp_format: "%Y-%m-%d", timezone: "+09:00"}
+    - {name: timestamp,   type: STRING,    timestamp_format: "%Y-%m-%d", timezone: "+09:00"}
+    - {name: long,        type: STRING}
+    - {name: string,      type: STRING}
+    - {name: double,      type: STRING}
+    - {name: boolean,     type: STRING}
+    - name: json
+      type: RECORD
+      fields:
+        - {name: k1,      type: STRING}
+        - {name: k2,      type: STRING}
+# 2015-07-13
+# 2015-07-12 15:00:00

data/lib/embulk/output/bigquery.rb

@@ -1,6 +1,7 @@
 require 'json'
 require 'tempfile'
 require 'fileutils'
+require 'securerandom'
 require_relative 'bigquery/bigquery_client'
 require_relative 'bigquery/file_writer'
 require_relative 'bigquery/value_converter_factory'
@@ -52,6 +53,7 @@ module Embulk
           'prevent_duplicate_insert'       => config.param('prevent_duplicate_insert',       :bool,    :default => false),
           'with_rehearsal'                 => config.param('with_rehearsal',                 :bool,    :default => false),
           'rehearsal_counts'               => config.param('rehearsal_counts',               :integer, :default => 1000),
+          'abort_on_error'                 => config.param('abort_on_error',                 :bool,    :default => nil),
 
           'column_options'                 => config.param('column_options',                 :array,   :default => []),
           'default_timezone'               => config.param('default_timezone',               :string,  :default => ValueConverterFactory::DEFAULT_TIMEZONE),
@@ -193,7 +195,7 @@ module Embulk
           task['file_ext'] = file_ext
         end
 
-        unique_name = "%08x%08x%08x" % [Process.pid, now.tv_sec, now.tv_nsec]
+        unique_name = SecureRandom.uuid.gsub('-', '_')
 
         if %w[replace replace_backup append].include?(task['mode'])
           task['temp_table'] ||= "LOAD_TEMP_#{unique_name}_#{task['table']}"
@@ -207,6 +209,10 @@ module Embulk
           Google::Apis.logger.level = eval("::Logger::#{task['sdk_log_level'].upcase}")
         end
 
+        if task['abort_on_error'].nil?
+          task['abort_on_error'] = (task['max_bad_records'] == 0)
+        end
+
         task
       end
 
@@ -226,12 +232,16 @@ module Embulk
         @rehearsal_thread = rehearsal_thread
       end
 
-      def self.transaction_report(file_writers, responses, target_table)
+      def self.transaction_report(task, responses)
         num_input_rows = file_writers.empty? ? 0 : file_writers.map(&:num_rows).inject(:+)
         num_response_rows = responses.inject(0) do |sum, response|
           sum + (response ? response.statistics.load.output_rows.to_i : 0)
         end
-        num_output_rows = bigquery.get_table(target_table).num_rows.to_i
+        if task['temp_table']
+          num_output_rows = bigquery.get_table(task['temp_table']).num_rows.to_i
+        else
+          num_output_rows = num_response_rows
+        end
         num_rejected_rows = num_input_rows - num_output_rows
         transaction_report = {
           'num_input_rows' => num_input_rows,
@@ -304,9 +314,16 @@ module Embulk
           else
             target_table = task['temp_table'] ? task['temp_table'] : task['table']
             responses = bigquery.load_in_parallel(paths, target_table)
-            transaction_report = self.transaction_report(file_writers, responses, target_table)
+            transaction_report = self.transaction_report(task, responses)
             Embulk.logger.info { "embulk-output-bigquery: transaction_report: #{transaction_report.to_json}" }
 
+            if task['abort_on_error']
+              if transaction_report['num_input_rows'] != transaction_report['num_output_rows']
+                raise Error, "ABORT: `num_input_rows (#{transaction_report['num_input_rows']})` and " \
+                  "`num_output_rows (#{transaction_report['num_output_rows']})` does not match"
+              end
+            end
+
             if task['mode'] == 'replace_backup'
               bigquery.copy(task['table'], task['table_old'], task['dataset_old'])
             end

data/test/test_transaction.rb

@@ -34,13 +34,16 @@ module Embulk
         Proc.new {|task| task_reports = [] }
       end
 
+      def setup
+        stub(Bigquery).transaction_report { {'num_input_rows' => 1, 'num_output_rows' => 1, 'num_rejected_rows' => 0} }
+      end
+
       def test_append
         config = least_config.merge('mode' => 'append', 'temp_table' => 'temp_table')
         any_instance_of(BigqueryClient) do |obj|
           mock(obj).get_dataset(config['dataset'])
           mock(obj).create_table(config['temp_table'])
           mock(obj).load_in_parallel(anything, config['temp_table']) { [] }
-          mock(obj).get_table(config['temp_table']) { OpenStruct.new(num_rows: 1) }
           mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_APPEND')
           mock(obj).delete_table(config['temp_table'])
         end
@@ -54,7 +57,6 @@ module Embulk
             mock(obj).get_dataset(config['dataset'])
             mock(obj).get_table(config['table'])
             mock(obj).load_in_parallel(anything, config['table']) { [] }
-            mock(obj).get_table(config['table']) { OpenStruct.new(num_rows: 1) }
           end
           Bigquery.transaction(config, schema, processor_count, &control)
         end
@@ -65,7 +67,6 @@ module Embulk
             mock(obj).create_dataset(config['dataset'])
             mock(obj).create_table(config['table'])
             mock(obj).load_in_parallel(anything, config['table']) { [] }
-            mock(obj).get_table(config['table']) { OpenStruct.new(num_rows: 1) }
           end
           Bigquery.transaction(config, schema, processor_count, &control)
         end
@@ -78,7 +79,6 @@ module Embulk
           mock(obj).delete_table(config['table'])
           mock(obj).create_table(config['table'])
           mock(obj).load_in_parallel(anything, config['table']) { [] }
-          mock(obj).get_table(config['table']) { OpenStruct.new(num_rows: 1) }
         end
         Bigquery.transaction(config, schema, processor_count, &control)
       end
@@ -89,7 +89,6 @@ module Embulk
           mock(obj).get_dataset(config['dataset'])
           mock(obj).create_table(config['temp_table'])
           mock(obj).load_in_parallel(anything, config['temp_table']) { [] }
-          mock(obj).get_table(config['temp_table']) { OpenStruct.new(num_rows: 1) }
           mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_TRUNCATE')
           mock(obj).delete_table(config['temp_table'])
         end
@@ -104,7 +103,6 @@ module Embulk
             mock(obj).get_dataset(config['dataset_old'])
             mock(obj).create_table(config['temp_table'])
             mock(obj).load_in_parallel(anything, config['temp_table']) { [] }
-            mock(obj).get_table(config['temp_table']) { OpenStruct.new(num_rows: 1) }
 
             mock(obj).copy(config['table'], config['table_old'], config['dataset_old'])
 
@@ -121,7 +119,6 @@ module Embulk
             mock(obj).create_dataset(config['dataset_old'], reference: config['dataset'])
             mock(obj).create_table(config['temp_table'])
             mock(obj).load_in_parallel(anything, config['temp_table']) { [] }
-            mock(obj).get_table(config['temp_table']) { OpenStruct.new(num_rows: 1) }
 
             mock(obj).copy(config['table'], config['table_old'], config['dataset_old'])
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Satoshi Akama
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-04-15 00:00:00.000000000 Z
+date: 2016-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-api-client
@@ -90,6 +90,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
@@ -108,6 +109,7 @@ files:
 - example/config_min_ouput_tasks.yml
 - example/config_mode_append.yml
 - example/config_mode_append_direct.yml
+- example/config_nested_record.yml
 - example/config_payload_column.yml
 - example/config_payload_column_index.yml
 - example/config_prevent_duplicate_insert.yml