embulk-output-bigquery 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 06dd5ff0d084e46e4e6d3c5b4428b5a0500a69e1
-  data.tar.gz: 974f71b43c073e5f324e27232cac48100ce5c9bf
+  metadata.gz: f21e4f5989b1aa631de606560ee75591a113c6f5
+  data.tar.gz: da801735b3ad2871a5d78bdde79d4f8e5e87ca30
 SHA512:
-  metadata.gz: 55fc22719752768be1f4a45b1e7a4c011f01a75dfd68840f1deadbed8e2aa4cd88071ddbeaf9e3ea63ac0cdbe0b875756a51f9ef024c08f5722545833da6b5e3
-  data.tar.gz: a5d30638f60ad162950219fbc58a525dd11662cc86b05d94cb02b8fd7dbf089a644a20f5a35a06e51e85f8685f5a1adb1fb07e24c46cc7d240f7453355c77953
+  metadata.gz: 582b300dacd9a45e39b424c3d0c0c3a887f5edc860430b2f0341df945ee723c0c8c5458619f28f18b2f028fe214fc3dbf58afd2751735bd2c143addb5ba164b3
+  data.tar.gz: 593d02fb4ec66bff1e3095e7e65f4d9b2adc3cb471ec3e998007ccc0fef73cfb48ad1bee6b0ee232d45ad41031affacb941e1d89097520052de007310d769465

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 0.3.1 - 2016-04-15
+
+* [new feature] Add `sdk_log_level` option to show log of google-api-client
+* [maintenance] Fix `prevent_duplicate_insert` was not working correctly
+* [maintenance] Change to get `num_output_rows` of `transaction_report` from `get_table` API
+* [maintenance] Log response.statistics of load jobs
+* [maintenance] Always create job_id on client side as [google recommends](https://cloud.google.com/bigquery/docs/managing_jobs_datasets_projects#managingjobs) so that duplication not to be occurred
+* [maintenance] Fix a possibility which rehearsal would load 0 rows file
+
 ## 0.3.0 - 2016-04-08
 
 Big change is introduced. Now, embulk-output-bigquery is written in JRuby.

data/README.md

@@ -39,7 +39,7 @@ OAuth flow for installed applications.
 |  auto_create_table                   | boolean     | optional   | false                    | [See below](#dynamic-table-creating) |
 |  schema_file                         | string      | optional   |                          | /path/to/schema.json |
 |  template_table                      | string      | optional   |                          | template table name [See below](#dynamic-table-creating) |
-|  prevent_duplicate_insert            | boolean     | optional   | false                    | [See below](#data-consistency) |
+|  prevent_duplicate_insert            | boolean     | optional   | false                    | [See below](#prevent-duplication) |
 |  job_status_max_polling_time         | int         | optional   | 3600 sec                 | Max job status polling time |
 |  job_status_polling_interval         | int         | optional   | 10 sec                   | Job status polling interval |
 |  is_skip_job_result_check            | boolean     | optional   | false                    | Skip waiting Load job finishes. Available for append, or delete_in_advance mode | 
@@ -59,6 +59,7 @@ Client or request options
 |  open_timeout_sec                    | integer     | optional   | 300                      | Seconds to wait for the connection to open |
 |  retries                             | integer     | optional   | 5                        | Number of retries |
 |  application_name                    | string      | optional   | "Embulk BigQuery plugin" | User-Agent |
+|  sdk_log_level                       | string      | optional   | nil (WARN)               | Log level of google api client library |
 
 Options for intermediate local files
 
@@ -317,15 +318,15 @@ out:
   payload_column_index: 0 # or, payload_column: payload
 ```
 
-### Data Consistency
+### Prevent Duplication
 
-When `prevent_duplicate_insert` is set to true, embulk-output-bigquery generate job ID from md5 hash of file  and other options to prevent duplicate data insertion.
+`prevent_duplicate_insert` option is used to prevent inserting same data for modes `append` or `append_direct`.
 
-`job ID = md5(md5(file) + dataset + table + schema + source_format + file_delimiter + max_bad_records + encoding + ignore_unknown_values + allow_quoted_newlines)`
+When `prevent_duplicate_insert` is set to true, embulk-output-bigquery generate job ID from md5 hash of file and other options.
 
-[job ID must be unique(including failures)](https://cloud.google.com/bigquery/loading-data-into-bigquery#consistency). So same data can't insert with same settings.
+`job ID = md5(md5(file) + dataset + table + schema + source_format + file_delimiter + max_bad_records + encoding + ignore_unknown_values + allow_quoted_newlines)`
 
-In other words, you can retry as many times as you like, in case something bad error(like network error) happens before job insertion.
+[job ID must be unique(including failures)](https://cloud.google.com/bigquery/loading-data-into-bigquery#consistency) so that same data can't be inserted with same settings repeatedly.
 
 ```yaml
 out:

data/embulk-output-bigquery.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-bigquery"
-  spec.version       = "0.3.0"
+  spec.version       = "0.3.1"
   spec.authors       = ["Satoshi Akama", "Naotoshi Seo"]
   spec.summary       = "Google BigQuery output plugin for Embulk"
   spec.description   = "Embulk plugin that insert records to Google BigQuery."

data/example/config_prevent_duplicate_insert.yml

@@ -18,7 +18,7 @@ in:
       - {name: boolean,     type: boolean}
 out:
   type: bigquery
-  mode: replace
+  mode: append
   auth_method: json_key
   json_keyfile: example/your-project-000.json
   dataset: your_dataset_name

data/lib/embulk/output/bigquery.rb

@@ -44,6 +44,7 @@ module Embulk
           'auto_create_table'              => config.param('auto_create_table',              :bool,    :default => false),
           'schema_file'                    => config.param('schema_file',                    :string,  :default => nil),
           'template_table'                 => config.param('template_table',                 :string,  :default => nil),
+
           'delete_from_local_when_job_end' => config.param('delete_from_local_when_job_end', :bool,    :default => true),
           'job_status_max_polling_time'    => config.param('job_status_max_polling_time',    :integer, :default => 3600),
           'job_status_polling_interval'    => config.param('job_status_polling_interval',    :integer, :default => 10),
@@ -62,6 +63,7 @@ module Embulk
           'open_timeout_sec'               => config.param('open_timeout_sec',               :integer, :default => 300),
           'retries'                        => config.param('retries',                        :integer, :default => 5),
           'application_name'               => config.param('application_name',               :string,  :default => 'Embulk BigQuery plugin'),
+          'sdk_log_level'                  => config.param('sdk_log_level',                  :string,  :default => nil),
 
           'path_prefix'                    => config.param('path_prefix',                    :string,  :default => nil),
           'sequence_format'                => config.param('sequence_format',                :string,  :default => '.%d.%d'),
@@ -201,6 +203,10 @@ module Embulk
           task['rehearsal_table'] ||= "LOAD_REHEARSAL_#{unique_name}_#{task['table']}"
         end
 
+        if task['sdk_log_level']
+          Google::Apis.logger.level = eval("::Logger::#{task['sdk_log_level'].upcase}")
+        end
+
         task
       end
 
@@ -220,16 +226,16 @@ module Embulk
         @rehearsal_thread = rehearsal_thread
       end
 
-      def self.transaction_report(task_reports, responses)
-        num_input_rows = task_reports.inject(0) do |sum, task_report|
-          sum + task_report['num_input_rows']
-        end
-        num_output_rows = responses.inject(0) do |sum, response|
+      def self.transaction_report(file_writers, responses, target_table)
+        num_input_rows = file_writers.empty? ? 0 : file_writers.map(&:num_rows).inject(:+)
+        num_response_rows = responses.inject(0) do |sum, response|
           sum + (response ? response.statistics.load.output_rows.to_i : 0)
         end
+        num_output_rows = bigquery.get_table(target_table).num_rows.to_i
         num_rejected_rows = num_input_rows - num_output_rows
         transaction_report = {
           'num_input_rows' => num_input_rows,
+          'num_response_rows' => num_response_rows,
           'num_output_rows' => num_output_rows,
           'num_rejected_rows' => num_rejected_rows,
         }
@@ -278,12 +284,12 @@ module Embulk
             path_pattern = "#{task['path_prefix']}*#{task['file_ext']}"
             Embulk.logger.info { "embulk-output-bigquery: Skip file generation. Get paths from `#{path_pattern}`" }
             paths = Dir.glob(path_pattern)
-            task_reports = paths.map {|path| { 'num_input_rows' => 0 } }
           else
             task_reports = yield(task) # generates local files
-            Embulk.logger.info { "embulk-output-bigquery: task_reports: #{task_reports.to_json}" }
-            paths = FileWriter.paths
-            FileWriter.ios.values.each do |io|
+
+            ios = file_writers.map(&:io)
+            paths = ios.map(&:path)
+            ios.each do |io|
               Embulk.logger.debug { "close #{io.path}" }
               io.close rescue nil
             end
@@ -298,7 +304,7 @@ module Embulk
           else
             target_table = task['temp_table'] ? task['temp_table'] : task['table']
             responses = bigquery.load_in_parallel(paths, target_table)
-            transaction_report = self.transaction_report(task_reports, responses)
+            transaction_report = self.transaction_report(file_writers, responses, target_table)
             Embulk.logger.info { "embulk-output-bigquery: transaction_report: #{transaction_report.to_json}" }
 
             if task['mode'] == 'replace_backup'
@@ -339,37 +345,63 @@ module Embulk
         return next_config_diff
       end
 
-      # instance is created on each thread
+      @file_writers_mutex = Mutex.new
+      @file_writers = Array.new
+
+      def self.reset_file_writers
+        @file_writers = Array.new
+      end
+
+      def self.file_writers
+        @file_writers
+      end
+
+      def self.add_file_writer(file_writer)
+        @file_writers_mutex.synchronize do
+          @file_writers << file_writer
+        end
+      end
+
+      FILE_WRITER_KEY = :embulk_output_bigquery_file_writer
+
+      # Create one FileWriter object for one output thread, that is, share among tasks.
+      # Close theses shared objects in transaction.
+      # This is mainly to suppress (or control by -X max_threads) number of files, which
+      # equals to number of concurrency to load in parallel, when number of input tasks is many
+      #
+      # #file_writer must be called at only #add because threads in other methods
+      # are different (called from non-output threads). Note also that #add method
+      # of the same task instance would be called in different output threads
+      def file_writer
+        return Thread.current[FILE_WRITER_KEY] if Thread.current[FILE_WRITER_KEY]
+        file_writer = FileWriter.new(@task, @schema, @index, self.class.converters)
+        self.class.add_file_writer(file_writer)
+        Thread.current[FILE_WRITER_KEY] = file_writer
+      end
+
+      # instance is created on each task
       def initialize(task, schema, index)
         super
 
         if task['with_rehearsal'] and @index == 0
           @rehearsaled = false
-          @num_rows = 0
-        end
-
-        unless task['skip_file_generation']
-          @file_writer = FileWriter.new(task, schema, index, self.class.converters)
         end
       end
 
-      # called for each page in each thread
+      # called for each page in each task
       def close
       end
 
-      # called for each page in each thread
+      # called for each page in each task
       def add(page)
+        return if task['skip_file_generation']
+        num_rows = file_writer.add(page)
+
         if task['with_rehearsal'] and @index == 0 and !@rehearsaled
-          page = page.to_a # to avoid https://github.com/embulk/embulk/issues/403
-          if @num_rows >= task['rehearsal_counts']
+          if num_rows >= task['rehearsal_counts']
             load_rehearsal
             @rehearsaled = true
           end
-          @num_rows += page.to_a.size
-        end
-
-        unless task['skip_file_generation']
-          @file_writer.add(page)
         end
       end
 
@@ -377,11 +409,11 @@ module Embulk
         bigquery = self.class.bigquery
         Embulk.logger.info { "embulk-output-bigquery: Rehearsal started" }
 
-        io = @file_writer.close # need to close once for gzip
+        io = file_writer.close # need to close once for gzip
         rehearsal_path = "#{io.path}.rehearsal"
         Embulk.logger.debug { "embulk_output_bigquery: cp #{io.path} #{rehearsal_path}" }
         FileUtils.cp(io.path, rehearsal_path)
-        @file_writer.reopen
+        file_writer.reopen
 
         self.class.rehearsal_thread = Thread.new do
           begin
@@ -403,13 +435,9 @@ module Embulk
       def abort
       end
 
-      # called after processing all pages in each thread, returns a task_report
+      # called after processing all pages in each task, returns a task_report
       def commit
-        unless task['skip_file_generation']
-          @file_writer.commit
-        else
-          {}
-        end
+        {}
       end
     end
   end

data/lib/embulk/output/bigquery/bigquery_client.rb

@@ -107,49 +107,46 @@ module Embulk
           #
           # We before had a `max_load_parallels` option, but this was not extensible for map reduce executor
           # So, we dropped it. See https://github.com/embulk/embulk-output-bigquery/pull/35
-          max_load_parallels = paths.size # @task['max_load_parallels'] || paths.size
           responses = []
-          paths.each_with_index.each_slice(max_load_parallels) do |paths_group|
-            Embulk.logger.debug { "embulk-output-bigquery: LOAD IN PARALLEL #{paths_group}" }
-            threads = []
-            paths_group.each do |path, idx|
-              threads << Thread.new do
-                # I am not sure whether google-api-ruby-client is thread-safe,
-                # so let me create new instances for each thread for safe
-                bigquery = self.class.new(@task, @schema, fields)
-                response = bigquery.load(path, table)
-                [idx, response]
-              end
-            end
-            ThreadsWait.all_waits(*threads) do |th|
-              idx, response = th.value # raise errors occurred in threads
-              responses[idx] = response
+          threads = []
+          Embulk.logger.debug { "embulk-output-bigquery: LOAD IN PARALLEL #{paths}" }
+          paths.each_with_index do |path, idx|
+            threads << Thread.new do
+              # I am not sure whether google-api-ruby-client is thread-safe,
+              # so let me create new instances for each thread for safe
+              bigquery = self.class.new(@task, @schema, fields)
+              response = bigquery.load(path, table)
+              [idx, response]
             end
           end
+          ThreadsWait.all_waits(*threads) do |th|
+            idx, response = th.value # raise errors occurred in threads
+            responses[idx] = response
+          end
           responses
         end
 
         def load(path, table)
           begin
             if File.exist?(path)
-              Embulk.logger.info { "embulk-output-bigquery: Load job starting... #{path} => #{@project}:#{@dataset}.#{table}" }
+              # As https://cloud.google.com/bigquery/docs/managing_jobs_datasets_projects#managingjobs says,
+              # we should generate job_id in client code, otherwise, retrying would cause duplication
+              if @task['prevent_duplicate_insert'] and (@task['mode'] == 'append' or @task['mode'] == 'append_direct')
+                job_id = Helper.create_load_job_id(@task, path, fields)
+              else
+                job_id = "embulk_load_job_#{SecureRandom.uuid}"
+              end
+              Embulk.logger.info { "embulk-output-bigquery: Load job starting... job_id:[#{job_id}] #{path} => #{@project}:#{@dataset}.#{table}" }
             else
               Embulk.logger.info { "embulk-output-bigquery: Load job starting... #{path} does not exist, skipped" }
               return
             end
 
-            if @task['prevent_duplicate_insert']
-              job_reference = {
-                job_reference: {
-                  project_id: @project,
-                  job_id: Helper.create_job_id(@task, path, table, fields),
-                }
-              }
-            else
-              job_reference = {}
-            end
-
             body = {
+              job_reference: {
+                project_id: @project,
+                job_id: job_id,
+              },
               configuration: {
                 load: {
                   destination_table: {
@@ -168,8 +165,9 @@ module Embulk
                   ignore_unknown_values: @task['ignore_unknown_values'],
                   allow_quoted_newlines: @task['allow_quoted_newlines'],
                 }
-              }.merge!(job_reference)
+              }
             }
+
             opts = {
               upload_source: path,
               content_type: "application/octet-stream",
@@ -182,7 +180,7 @@ module Embulk
             Embulk.logger.debug { "embulk-output-bigquery: insert_job(#{@project}, #{body}, #{opts})" }
             response = client.insert_job(@project, body, opts)
             unless @task['is_skip_job_result_check']
-              wait_load('Load', response)
+              response = wait_load('Load', response)
             end
           rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
             response = {status_code: e.status_code, message: e.message, error_class: e.class}
@@ -196,11 +194,18 @@ module Embulk
         def copy(source_table, destination_table, destination_dataset = nil, write_disposition: 'WRITE_TRUNCATE')
           begin
             destination_dataset ||= @dataset
+            job_id = "embulk_copy_job_#{SecureRandom.uuid}"
+
             Embulk.logger.info {
-              "embulk-output-bigquery: Copy job starting... " \
+              "embulk-output-bigquery: Copy job starting... job_id:[#{job_id}] " \
               "#{@project}:#{@dataset}.#{source_table} => #{@project}:#{destination_dataset}.#{destination_table}"
             }
+
             body = {
+              job_reference: {
+                project_id: @project,
+                job_id: job_id,
+              },
               configuration: {
                 copy: {
                   create_deposition: 'CREATE_IF_NEEDED',
@@ -218,6 +223,7 @@ module Embulk
                 }
               }
             }
+
             opts = {}
             Embulk.logger.debug { "embulk-output-bigquery: insert_job(#{@project}, #{body}, #{opts})" }
             response = client.insert_job(@project, body, opts)
@@ -246,18 +252,18 @@ module Embulk
             if status == "DONE"
               Embulk.logger.info {
                 "embulk-output-bigquery: #{kind} job completed... " \
-                "job id:[#{job_id}] elapsed_time:#{elapsed.to_f}sec status:[#{status}]"
+                "job_id:[#{job_id}] elapsed_time:#{elapsed.to_f}sec status:[#{status}]"
               }
               break
             elsif elapsed.to_i > max_polling_time
-              message = "embulk-output-bigquery: Checking #{kind} job status... " \
-                "job id:[#{job_id}] elapsed_time:#{elapsed.to_f}sec status:[TIMEOUT]"
+              message = "embulk-output-bigquery: #{kind} job checking... " \
+                "job_id:[#{job_id}] elapsed_time:#{elapsed.to_f}sec status:[TIMEOUT]"
               Embulk.logger.info { message }
               raise JobTimeoutError.new(message)
             else
               Embulk.logger.info {
-                "embulk-output-bigquery: Checking #{kind} job status... " \
-                "job id:[#{job_id}] elapsed_time:#{elapsed.to_f}sec status:[#{status}]"
+                "embulk-output-bigquery: #{kind} job checking... " \
+                "job_id:[#{job_id}] elapsed_time:#{elapsed.to_f}sec status:[#{status}]"
               }
               sleep wait_interval
               _response = client.get_job(@project, job_id)
@@ -275,6 +281,8 @@ module Embulk
             raise Error, "failed during waiting a #{kind} job, errors:#{_errors.map(&:to_h)}"
           end
 
+          Embulk.logger.info { "embulk-output-bigquery: #{kind} job response... job_id:[#{job_id}] response.statistics:#{_response.statistics.to_h}" }
+
           _response
         end
 

data/lib/embulk/output/bigquery/file_writer.rb

@@ -7,15 +7,17 @@ module Embulk
   module Output
     class Bigquery < OutputPlugin
       class FileWriter
+        attr_reader :num_rows
+
         def initialize(task, schema, index, converters = nil)
           @task = task
           @schema = schema
           @index = index
           @converters = converters || ValueConverterFactory.create_converters(task, schema)
 
-          @num_input_rows = 0
+          @num_rows = 0
           @progress_log_timer = Time.now
-          @previous_num_input_rows = 0
+          @previous_num_rows = 0
 
           if @task['payload_column_index']
             @payload_column_index = @task['payload_column_index']
@@ -30,35 +32,8 @@ module Embulk
           end
         end
 
-        @mutex = Mutex.new
-        @ios = Hash.new
-
-        def self.mutex
-          @mutex
-        end
-
-        def self.reset_ios
-          @ios = Hash.new
-        end
-
-        def self.ios
-          @ios
-        end
-
-        def self.paths
-          @ios.keys
-        end
-
-        THREAD_LOCAL_IO_KEY = :embulk_output_bigquery_file_writer_io
-
-        # Create one io object for one output thread, that is, share among tasks
-        # Close theses shared io objects in transaction
-        #
-        # Thread IO must be created at #add because threads in #initialize or #commit
-        # are different (called from non-output threads). Note also that #add of the
-        # same instance would be called in different output threads
-        def thread_io
-          return Thread.current[THREAD_LOCAL_IO_KEY] if Thread.current[THREAD_LOCAL_IO_KEY]
+        def io
+          return @io if @io
 
           path = sprintf(
             "#{@task['path_prefix']}#{@task['sequence_format']}#{@task['file_ext']}",
@@ -70,7 +45,7 @@ module Embulk
           end
           Embulk.logger.info { "embulk-output-bigquery: create #{path}" }
 
-          open(path, 'w')
+          @io = open(path, 'w')
         end
 
         def open(path, mode = 'w')
@@ -81,21 +56,16 @@ module Embulk
           else
             io = file_io
           end
-          self.class.mutex.synchronize do
-            self.class.ios[path] = io
-          end
-          Thread.current[THREAD_LOCAL_IO_KEY] = io
+          io
         end
 
         def close
-          io = thread_io
           io.close rescue nil
           io
         end
 
         def reopen
-          io = thread_io
-          open(io.path, 'a')
+          @io = open(io.path, 'a')
         end
 
         def to_payload(record)
@@ -123,29 +93,24 @@ module Embulk
         end
 
         def add(page)
-          io = thread_io
+          _io = io
           # I once tried to split IO writing into another IO thread using SizedQueue
           # However, it resulted in worse performance, so I removed the codes.
           page.each do |record|
             Embulk.logger.trace { "embulk-output-bigquery: record #{record}" }
             formatted_record = @formatter_proc.call(record)
             Embulk.logger.trace { "embulk-output-bigquery: formatted_record #{formatted_record.chomp}" }
-            io.write formatted_record
-            @num_input_rows += 1
+            _io.write formatted_record
+            @num_rows += 1
           end
           now = Time.now
           if @progress_log_timer < now - 10 # once in 10 seconds
-            speed = ((@num_input_rows - @previous_num_input_rows) / (now - @progress_log_timer).to_f).round(1)
+            speed = ((@num_rows - @previous_num_rows) / (now - @progress_log_timer).to_f).round(1)
             @progress_log_timer = now
-            @previous_num_input_rows = @num_input_rows
-            Embulk.logger.info { "embulk-output-bigquery: num_input_rows #{num_format(@num_input_rows)} (#{num_format(speed)} rows/sec)" }
+            @previous_num_rows = @num_rows
+            Embulk.logger.info { "embulk-output-bigquery: num_rows #{num_format(@num_rows)} (#{num_format(speed)} rows/sec)" }
           end
-        end
-
-        def commit
-          task_report = {
-            'num_input_rows' => @num_input_rows,
-          }
+          @num_rows
         end
       end
     end

data/lib/embulk/output/bigquery/helper.rb

@@ -1,4 +1,5 @@
 require 'digest/md5'
+require 'securerandom'
 
 module Embulk
   module Output
@@ -52,11 +53,11 @@ module Embulk
           end
         end
 
-        def self.create_job_id(task, path, table, fields)
+        def self.create_load_job_id(task, path, fields)
           elements = [
             Digest::MD5.file(path).hexdigest,
             task['dataset'],
-            table,
+            task['table'],
             fields,
             task['source_format'],
             task['max_bad_records'],
@@ -68,9 +69,7 @@ module Embulk
 
           str = elements.map(&:to_s).join('')
           md5 = Digest::MD5.hexdigest(str)
-          job_id = "embulk_job_#{md5}"
-          Embulk.logger.debug { "embulk-output-bigquery: create_job_id(#{path}, #{table}) #=> #{job_id}" }
-          job_id
+          "embulk_load_job_#{md5}"
         end
       end
     end

data/test/test_file_writer.rb

@@ -16,11 +16,6 @@ module Embulk
         end
       end
 
-      def setup
-        Thread.current[FileWriter::THREAD_LOCAL_IO_KEY] = nil
-        FileWriter.reset_ios
-      end
-
       def default_task
         {
           'compression' => 'GZIP',
@@ -65,7 +60,7 @@ module Embulk
           ensure
             io.close rescue nil
           end
-          path = FileWriter.paths.first
+          path = file_writer.io.path
           assert_equal 'tmp/foo.1', path
         end
       end
@@ -108,12 +103,12 @@ module Embulk
 
           begin
             file_writer.add(page)
-            io = FileWriter.ios.values.first
+            io = file_writer.io
             assert_equal Zlib::GzipWriter, io.class
           ensure
             io.close rescue nil
           end
-          path = FileWriter.paths.first
+          path = file_writer.io.path
           assert_true File.exist?(path)
           assert_nothing_raised { Zlib::GzipReader.open(path) {|gz| } }
         end
@@ -124,12 +119,12 @@ module Embulk
 
           begin
             file_writer.add(page)
-            io = FileWriter.ios.values.first
+            io = file_writer.io
             assert_equal File, io.class
           ensure
             io.close rescue nil
           end
-          path = FileWriter.paths.first
+          path = file_writer.io.path
           assert_true File.exist?(path)
           assert_raise { Zlib::GzipReader.open(path) {|gz| } }
         end

data/test/test_helper.rb

@@ -81,9 +81,10 @@ module Embulk
         end
       end
 
-      def test_create_job_id
+      def test_create_load_job_id
         task = {
           'dataset' => 'your_dataset_name',
+          'table' => 'your_table_name',
           'source_format' => 'CSV',
           'max_bad_records' => nil,
           'field_delimiter' => ',',
@@ -95,7 +96,7 @@ module Embulk
           name: 'a', type: 'STRING',
         }
         File.write("tmp/your_file_name", "foobarbaz")
-        job_id = Helper.create_job_id(task, 'tmp/your_file_name', 'your_table_name', fields)
+        job_id = Helper.create_load_job_id(task, 'tmp/your_file_name', fields)
         assert job_id.is_a?(String)
       end
     end

data/test/test_transaction.rb

@@ -40,6 +40,7 @@ module Embulk
           mock(obj).get_dataset(config['dataset'])
           mock(obj).create_table(config['temp_table'])
           mock(obj).load_in_parallel(anything, config['temp_table']) { [] }
+          mock(obj).get_table(config['temp_table']) { OpenStruct.new(num_rows: 1) }
           mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_APPEND')
           mock(obj).delete_table(config['temp_table'])
         end
@@ -53,6 +54,7 @@ module Embulk
             mock(obj).get_dataset(config['dataset'])
             mock(obj).get_table(config['table'])
             mock(obj).load_in_parallel(anything, config['table']) { [] }
+            mock(obj).get_table(config['table']) { OpenStruct.new(num_rows: 1) }
           end
           Bigquery.transaction(config, schema, processor_count, &control)
         end
@@ -63,6 +65,7 @@ module Embulk
             mock(obj).create_dataset(config['dataset'])
             mock(obj).create_table(config['table'])
             mock(obj).load_in_parallel(anything, config['table']) { [] }
+            mock(obj).get_table(config['table']) { OpenStruct.new(num_rows: 1) }
           end
           Bigquery.transaction(config, schema, processor_count, &control)
         end
@@ -75,6 +78,7 @@ module Embulk
           mock(obj).delete_table(config['table'])
           mock(obj).create_table(config['table'])
           mock(obj).load_in_parallel(anything, config['table']) { [] }
+          mock(obj).get_table(config['table']) { OpenStruct.new(num_rows: 1) }
         end
         Bigquery.transaction(config, schema, processor_count, &control)
       end
@@ -85,6 +89,7 @@ module Embulk
           mock(obj).get_dataset(config['dataset'])
           mock(obj).create_table(config['temp_table'])
           mock(obj).load_in_parallel(anything, config['temp_table']) { [] }
+          mock(obj).get_table(config['temp_table']) { OpenStruct.new(num_rows: 1) }
           mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_TRUNCATE')
           mock(obj).delete_table(config['temp_table'])
         end
@@ -99,6 +104,7 @@ module Embulk
             mock(obj).get_dataset(config['dataset_old'])
             mock(obj).create_table(config['temp_table'])
             mock(obj).load_in_parallel(anything, config['temp_table']) { [] }
+            mock(obj).get_table(config['temp_table']) { OpenStruct.new(num_rows: 1) }
 
             mock(obj).copy(config['table'], config['table_old'], config['dataset_old'])
 
@@ -115,6 +121,7 @@ module Embulk
             mock(obj).create_dataset(config['dataset_old'], reference: config['dataset'])
             mock(obj).create_table(config['temp_table'])
             mock(obj).load_in_parallel(anything, config['temp_table']) { [] }
+            mock(obj).get_table(config['temp_table']) { OpenStruct.new(num_rows: 1) }
 
             mock(obj).copy(config['table'], config['table_old'], config['dataset_old'])
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Satoshi Akama
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-04-08 00:00:00.000000000 Z
+date: 2016-04-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-api-client