embulk-output-bigquery 0.3.0.pre2 → 0.3.0.pre3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6d94d238ce16c412168d14a2d598c4858c2517a1
-  data.tar.gz: 98c6a413ce767acc0db2a1b4b6fa8f1f43f1d7db
+  metadata.gz: f1dbd4398ccaccaf5b4d2d812dcbf64d2610c41c
+  data.tar.gz: 3b1ad06791c8b6b65139dfaf7d82e7e1f253c35d
 SHA512:
-  metadata.gz: d5aee1620e2171c7ca84101c196bb85015bd53d1a5816949353e3e921559dcc96211a81c1c9d6a35a563de2c4656ed98c8b495577ba57d1a8e74e046bb9ab594
-  data.tar.gz: 50f32129feea6bb33c7574e0d7aa73e387e37ced9ebe5d96a550225de78bddb365ab603b490a0a8361b008f16332a1092e84c278cf25e21a168700b8a751023a
+  metadata.gz: 5307cce2d9983b7c2710180f28e0e19515d6895dcd3b5d6e0c48503c57b5e555332831ed328a21daf9743e7688d7d3a51e1f81e4436370311e8defdd1dbc6b22
+  data.tar.gz: eeae7ee0e52c2e218478d5d58d3a2eeb76fedb02e5f6d1ba85a393069dbf1c455028658593b3dcedeff7f6dced4de9c31185e33d2fde824bfc9d4b3494acbc09

data/README.md

@@ -65,7 +65,7 @@ Options for intermediate local files
 | name                                 | type        | required?  | default                  | description            |  
 |:-------------------------------------|:------------|:-----------|:-------------------------|:-----------------------|
 |  path_prefix                         | string      | optional   |                          | Path prefix of local files such as "/tmp/prefix_". Default randomly generates with [tempfile](http://ruby-doc.org/stdlib-2.2.3/libdoc/tempfile/rdoc/Tempfile.html) |
-|  sequence_format                     | string      | optional   | .%d.%03d                 | Sequence format for pid, task index |
+|  sequence_format                     | string      | optional   | .%d.%d                   | Sequence format for pid, thread id |
 |  file_ext                            | string      | optional   |                          | The file extension of local files such as ".csv.gz" ".json.gz". Default automatically generates from `source_format` and `compression`|
 |  skip_file_generation                | boolean     | optional   |                          | Load already generated local files into BigQuery if available. Specify correct path_prefix and file_ext. |
 |  delete_from_local_when_job_end      | boolean     | optional   | false                    | If set to true, delete glocal file when job is end |

data/embulk-output-bigquery.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-bigquery"
-  spec.version       = "0.3.0.pre2"
+  spec.version       = "0.3.0.pre3"
   spec.authors       = ["Satoshi Akama", "Naotoshi Seo"]
   spec.summary       = "Google BigQuery output plugin for Embulk"
   spec.description   = "Embulk plugin that insert records to Google BigQuery."

data/example/config_max_threads.yml

@@ -0,0 +1,34 @@
+in:
+  type: file
+  path_prefix: example/example4_
+  parser:
+    type: csv
+    charset: UTF-8
+    newline: CRLF
+    null_string: 'NULL'
+    skip_header_lines: 1
+    comment_line_marker: '#'
+    columns:
+      - {name: date,        type: string}
+      - {name: timestamp,   type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
+      - {name: "null",      type: string}
+      - {name: long,        type: long}
+      - {name: string,      type: string}
+      - {name: double,      type: double}
+      - {name: boolean,     type: boolean}
+out:
+  type: bigquery
+  mode: replace
+  auth_method: json_key
+  json_keyfile: example/your-project-000.json
+  dataset: your_dataset_name
+  table: your_table_name
+  source_format: NEWLINE_DELIMITED_JSON
+  compression: NONE
+  auto_create_dataset: true
+  auto_create_table: true
+  schema_file: example/schema.json
+exec:
+  type: local
+  min_output_tasks: 2
+  max_threads: 2

data/example/config_min_ouput_tasks.yml

@@ -0,0 +1,34 @@
+in:
+  type: file
+  path_prefix: example/example2_
+  parser:
+    type: csv
+    charset: UTF-8
+    newline: CRLF
+    null_string: 'NULL'
+    skip_header_lines: 1
+    comment_line_marker: '#'
+    columns:
+      - {name: date,        type: string}
+      - {name: timestamp,   type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
+      - {name: "null",      type: string}
+      - {name: long,        type: long}
+      - {name: string,      type: string}
+      - {name: double,      type: double}
+      - {name: boolean,     type: boolean}
+out:
+  type: bigquery
+  mode: replace
+  auth_method: json_key
+  json_keyfile: example/your-project-000.json
+  dataset: your_dataset_name
+  table: your_table_name
+  source_format: NEWLINE_DELIMITED_JSON
+  compression: GZIP
+  auto_create_dataset: true
+  auto_create_table: true
+  schema_file: example/schema.json
+exec:
+  type: local
+  min_output_tasks: 8
+  max_threads: 4

data/example/example2_1.csv

@@ -0,0 +1,17 @@
+date,timestamp,null,long,string,double,boolean
+2015-07-13,2015-07-13 00:00:00.100000,,90,l6lTsvxd,903.4,true
+2015-07-13,2015-07-13 00:00:00.100000,,91,XoALSEQg,394.5,true
+2015-07-13,2015-07-13 00:00:00.100000,,92,0hgDRI_m,810.9,true
+2015-07-13,2015-07-13 00:00:00.100000,,93,KjCRAc-A,477.4,true
+2015-07-13,2015-07-13 00:00:00.100000,,94,fyQVGlT8,725.3,true
+2015-07-13,2015-07-13 00:00:00.100000,,95,FpBYRPWK,316.6,true
+2015-07-13,2015-07-13 00:00:00.100000,,96,9ikvnUqp,369.5,true
+2015-07-13,2015-07-13 00:00:00.100000,,97,RRNYDAzK,506.5,true
+2015-07-13,2015-07-13 00:00:00.100000,,90,l6lTsvxd,903.4,false
+2015-07-13,2015-07-13 00:00:00.100000,,91,XoALSEQg,394.5,false
+2015-07-13,2015-07-13 00:00:00.100000,,92,0hgDRI_m,810.9,false
+2015-07-13,2015-07-13 00:00:00.100000,,93,KjCRAc-A,477.4,false
+2015-07-13,2015-07-13 00:00:00.100000,,94,fyQVGlT8,725.3,false
+2015-07-13,2015-07-13 00:00:00.100000,,95,FpBYRPWK,316.6,false
+2015-07-13,2015-07-13 00:00:00.100000,,96,9ikvnUqp,369.5,false
+2015-07-13,2015-07-13 00:00:00.100000,,97,RRNYDAzK,506.5,false

data/example/example2_2.csv

@@ -0,0 +1,17 @@
+date,timestamp,null,long,string,double,boolean
+2015-07-13,2015-07-13 00:00:00.100000,,90,l6lTsvxd,903.4,true
+2015-07-13,2015-07-13 00:00:00.100000,,91,XoALSEQg,394.5,true
+2015-07-13,2015-07-13 00:00:00.100000,,92,0hgDRI_m,810.9,true
+2015-07-13,2015-07-13 00:00:00.100000,,93,KjCRAc-A,477.4,true
+2015-07-13,2015-07-13 00:00:00.100000,,94,fyQVGlT8,725.3,true
+2015-07-13,2015-07-13 00:00:00.100000,,95,FpBYRPWK,316.6,true
+2015-07-13,2015-07-13 00:00:00.100000,,96,9ikvnUqp,369.5,true
+2015-07-13,2015-07-13 00:00:00.100000,,97,RRNYDAzK,506.5,true
+2015-07-13,2015-07-13 00:00:00.100000,,90,l6lTsvxd,903.4,false
+2015-07-13,2015-07-13 00:00:00.100000,,91,XoALSEQg,394.5,false
+2015-07-13,2015-07-13 00:00:00.100000,,92,0hgDRI_m,810.9,false
+2015-07-13,2015-07-13 00:00:00.100000,,93,KjCRAc-A,477.4,false
+2015-07-13,2015-07-13 00:00:00.100000,,94,fyQVGlT8,725.3,false
+2015-07-13,2015-07-13 00:00:00.100000,,95,FpBYRPWK,316.6,false
+2015-07-13,2015-07-13 00:00:00.100000,,96,9ikvnUqp,369.5,false
+2015-07-13,2015-07-13 00:00:00.100000,,97,RRNYDAzK,506.5,false

data/example/example4_1.csv

@@ -0,0 +1,17 @@
+date,timestamp,null,long,string,double,boolean
+2015-07-13,2015-07-13 00:00:00.100000,,90,l6lTsvxd,903.4,true
+2015-07-13,2015-07-13 00:00:00.100000,,91,XoALSEQg,394.5,true
+2015-07-13,2015-07-13 00:00:00.100000,,92,0hgDRI_m,810.9,true
+2015-07-13,2015-07-13 00:00:00.100000,,93,KjCRAc-A,477.4,true
+2015-07-13,2015-07-13 00:00:00.100000,,94,fyQVGlT8,725.3,true
+2015-07-13,2015-07-13 00:00:00.100000,,95,FpBYRPWK,316.6,true
+2015-07-13,2015-07-13 00:00:00.100000,,96,9ikvnUqp,369.5,true
+2015-07-13,2015-07-13 00:00:00.100000,,97,RRNYDAzK,506.5,true
+2015-07-13,2015-07-13 00:00:00.100000,,90,l6lTsvxd,903.4,false
+2015-07-13,2015-07-13 00:00:00.100000,,91,XoALSEQg,394.5,false
+2015-07-13,2015-07-13 00:00:00.100000,,92,0hgDRI_m,810.9,false
+2015-07-13,2015-07-13 00:00:00.100000,,93,KjCRAc-A,477.4,false
+2015-07-13,2015-07-13 00:00:00.100000,,94,fyQVGlT8,725.3,false
+2015-07-13,2015-07-13 00:00:00.100000,,95,FpBYRPWK,316.6,false
+2015-07-13,2015-07-13 00:00:00.100000,,96,9ikvnUqp,369.5,false
+2015-07-13,2015-07-13 00:00:00.100000,,97,RRNYDAzK,506.5,false

data/example/example4_2.csv

@@ -0,0 +1,17 @@
+date,timestamp,null,long,string,double,boolean
+2015-07-13,2015-07-13 00:00:00.100000,,90,l6lTsvxd,903.4,true
+2015-07-13,2015-07-13 00:00:00.100000,,91,XoALSEQg,394.5,true
+2015-07-13,2015-07-13 00:00:00.100000,,92,0hgDRI_m,810.9,true
+2015-07-13,2015-07-13 00:00:00.100000,,93,KjCRAc-A,477.4,true
+2015-07-13,2015-07-13 00:00:00.100000,,94,fyQVGlT8,725.3,true
+2015-07-13,2015-07-13 00:00:00.100000,,95,FpBYRPWK,316.6,true
+2015-07-13,2015-07-13 00:00:00.100000,,96,9ikvnUqp,369.5,true
+2015-07-13,2015-07-13 00:00:00.100000,,97,RRNYDAzK,506.5,true
+2015-07-13,2015-07-13 00:00:00.100000,,90,l6lTsvxd,903.4,false
+2015-07-13,2015-07-13 00:00:00.100000,,91,XoALSEQg,394.5,false
+2015-07-13,2015-07-13 00:00:00.100000,,92,0hgDRI_m,810.9,false
+2015-07-13,2015-07-13 00:00:00.100000,,93,KjCRAc-A,477.4,false
+2015-07-13,2015-07-13 00:00:00.100000,,94,fyQVGlT8,725.3,false
+2015-07-13,2015-07-13 00:00:00.100000,,95,FpBYRPWK,316.6,false
+2015-07-13,2015-07-13 00:00:00.100000,,96,9ikvnUqp,369.5,false
+2015-07-13,2015-07-13 00:00:00.100000,,97,RRNYDAzK,506.5,false

data/example/example4_3.csv

@@ -0,0 +1,17 @@
+date,timestamp,null,long,string,double,boolean
+2015-07-13,2015-07-13 00:00:00.100000,,90,l6lTsvxd,903.4,true
+2015-07-13,2015-07-13 00:00:00.100000,,91,XoALSEQg,394.5,true
+2015-07-13,2015-07-13 00:00:00.100000,,92,0hgDRI_m,810.9,true
+2015-07-13,2015-07-13 00:00:00.100000,,93,KjCRAc-A,477.4,true
+2015-07-13,2015-07-13 00:00:00.100000,,94,fyQVGlT8,725.3,true
+2015-07-13,2015-07-13 00:00:00.100000,,95,FpBYRPWK,316.6,true
+2015-07-13,2015-07-13 00:00:00.100000,,96,9ikvnUqp,369.5,true
+2015-07-13,2015-07-13 00:00:00.100000,,97,RRNYDAzK,506.5,true
+2015-07-13,2015-07-13 00:00:00.100000,,90,l6lTsvxd,903.4,false
+2015-07-13,2015-07-13 00:00:00.100000,,91,XoALSEQg,394.5,false
+2015-07-13,2015-07-13 00:00:00.100000,,92,0hgDRI_m,810.9,false
+2015-07-13,2015-07-13 00:00:00.100000,,93,KjCRAc-A,477.4,false
+2015-07-13,2015-07-13 00:00:00.100000,,94,fyQVGlT8,725.3,false
+2015-07-13,2015-07-13 00:00:00.100000,,95,FpBYRPWK,316.6,false
+2015-07-13,2015-07-13 00:00:00.100000,,96,9ikvnUqp,369.5,false
+2015-07-13,2015-07-13 00:00:00.100000,,97,RRNYDAzK,506.5,false

data/example/example4_4.csv

@@ -0,0 +1,17 @@
+date,timestamp,null,long,string,double,boolean
+2015-07-13,2015-07-13 00:00:00.100000,,90,l6lTsvxd,903.4,true
+2015-07-13,2015-07-13 00:00:00.100000,,91,XoALSEQg,394.5,true
+2015-07-13,2015-07-13 00:00:00.100000,,92,0hgDRI_m,810.9,true
+2015-07-13,2015-07-13 00:00:00.100000,,93,KjCRAc-A,477.4,true
+2015-07-13,2015-07-13 00:00:00.100000,,94,fyQVGlT8,725.3,true
+2015-07-13,2015-07-13 00:00:00.100000,,95,FpBYRPWK,316.6,true
+2015-07-13,2015-07-13 00:00:00.100000,,96,9ikvnUqp,369.5,true
+2015-07-13,2015-07-13 00:00:00.100000,,97,RRNYDAzK,506.5,true
+2015-07-13,2015-07-13 00:00:00.100000,,90,l6lTsvxd,903.4,false
+2015-07-13,2015-07-13 00:00:00.100000,,91,XoALSEQg,394.5,false
+2015-07-13,2015-07-13 00:00:00.100000,,92,0hgDRI_m,810.9,false
+2015-07-13,2015-07-13 00:00:00.100000,,93,KjCRAc-A,477.4,false
+2015-07-13,2015-07-13 00:00:00.100000,,94,fyQVGlT8,725.3,false
+2015-07-13,2015-07-13 00:00:00.100000,,95,FpBYRPWK,316.6,false
+2015-07-13,2015-07-13 00:00:00.100000,,96,9ikvnUqp,369.5,false
+2015-07-13,2015-07-13 00:00:00.100000,,97,RRNYDAzK,506.5,false

data/lib/embulk/output/bigquery.rb

@@ -63,7 +63,7 @@ module Embulk
           'application_name'               => config.param('application_name',               :string,  :default => 'Embulk BigQuery plugin'),
 
           'path_prefix'                    => config.param('path_prefix',                    :string,  :default => nil),
-          'sequence_format'                => config.param('sequence_format',                :string,  :default => '.%d.%03d'),
+          'sequence_format'                => config.param('sequence_format',                :string,  :default => '.%d.%d'),
           'file_ext'                       => config.param('file_ext',                       :string,  :default => nil),
           'skip_file_generation'           => config.param('skip_file_generation',           :bool,    :default => false),
           'compression'                    => config.param('compression',                    :string,  :default => 'NONE'),
@@ -273,11 +273,12 @@ module Embulk
             path_pattern = "#{task['path_prefix']}*#{task['file_ext']}"
             Embulk.logger.info { "embulk-output-bigquery: Skip file generation. Get paths from `#{path_pattern}`" }
             paths = Dir.glob(path_pattern)
-            task_reports = paths.map {|path| { 'path' => path, 'num_input_rows' => 0 } }
+            task_reports = paths.map {|path| { 'num_input_rows' => 0 } }
           else
             task_reports = yield(task) # generates local files
             Embulk.logger.info { "embulk-output-bigquery: task_reports: #{task_reports.to_json}" }
-            paths = task_reports.map {|report| report['path'] }
+            paths = FileWriter.paths
+            FileWriter.ios.each {|io| io.close rescue nil }
           end
 
           if task['skip_load'] # only for debug
@@ -294,11 +295,9 @@ module Embulk
 
             if task['temp_table']
               if task['mode'] == 'append'
-                bigquery.copy(task['temp_table'], task['table'],
-                              write_disposition: 'WRITE_APPEND')
+                bigquery.copy(task['temp_table'], task['table'], write_disposition: 'WRITE_APPEND')
               else # replace or replace_backup
-                bigquery.copy(task['temp_table'], task['table'],
-                              write_disposition: 'WRITE_TRUNCATE')
+                bigquery.copy(task['temp_table'], task['table'], write_disposition: 'WRITE_TRUNCATE')
               end
             end
           end
@@ -351,11 +350,11 @@ module Embulk
       def add(page)
         if task['with_rehearsal'] and @index == 0 and !@rehearsaled
           page = page.to_a # to avoid https://github.com/embulk/embulk/issues/403
-          if @num_rows > task['rehearsal_counts']
+          if @num_rows >= task['rehearsal_counts']
             Embulk.logger.info { "embulk-output-bigquery: Rehearsal started" }
             begin
               @bigquery.create_table(task['rehearsal_table'])
-              @bigquery.load(@file_writer.path, task['rehearsal_table'])
+              @bigquery.load(FileWriter.paths.first, task['rehearsal_table'])
             ensure
               @bigquery.delete_table(task['rehearsal_table'])
             end

data/lib/embulk/output/bigquery/bigquery_client.rb

@@ -245,7 +245,7 @@ module Embulk
             status = _response.status.state
             if status == "DONE"
               Embulk.logger.info {
-                "embulk-output-bigquery: #{kind} job completed successfully... " \
+                "embulk-output-bigquery: #{kind} job completed... " \
                 "job id:[#{job_id}] elapsed_time:#{elapsed.to_f}sec status:[#{status}]"
               }
               break

data/lib/embulk/output/bigquery/file_writer.rb

@@ -7,8 +7,6 @@ module Embulk
   module Output
     class Bigquery < OutputPlugin
       class FileWriter
-        attr_reader :path
-
         def initialize(task, schema, index, converters = nil)
           @task = task
           @schema = schema
@@ -30,21 +28,49 @@ module Embulk
               @formatter_proc = self.method(:to_jsonl)
             end
           end
+        end
+
+        def self.reset_ios
+          @ios = Set.new
+        end
+
+        def self.ios
+          @ios ||= Set.new
+        end
+
+        def self.paths
+          ios.map {|io| io.path }
+        end
+
+        THREAD_LOCAL_IO_KEY = :embulk_output_bigquery_file_writer_io
+
+        # Create one io object for one output thread, that is, share among tasks
+        # Close theses shared io objects in transaction
+        #
+        # Thread IO must be created at #add because threads in #initialize or #commit
+        # are different (called from non-output threads). Note also that #add of the
+        # same instance would be called in different output threads
+        def thread_io
+          return Thread.current[THREAD_LOCAL_IO_KEY] if Thread.current[THREAD_LOCAL_IO_KEY]
 
-          @path = sprintf("#{@task['path_prefix']}#{@task['sequence_format']}#{@task['file_ext']}", Process.pid, index)
-          Embulk.logger.info { "embulk-output-bigquery: will create #{@path}" }
-          if File.exist?(@path)
-            Embulk.logger.warn { "embulk-output-bigquery: unlink already existing #{@path}" }
-            File.unlink(@path) rescue nil
+          path = sprintf(
+            "#{@task['path_prefix']}#{@task['sequence_format']}#{@task['file_ext']}",
+            Process.pid, Thread.current.object_id
+          )
+          if File.exist?(path)
+            Embulk.logger.warn { "embulk-output-bigquery: unlink already existing #{path}" }
+            File.unlink(path) rescue nil
           end
-          @file_io = File.open(@path, 'w')
+          Embulk.logger.info { "embulk-output-bigquery: create #{path}" }
+          file_io = File.open(path, 'w')
 
           case @task['compression'].downcase
           when 'gzip'
-            @io = Zlib::GzipWriter.new(@file_io)
+            io = Zlib::GzipWriter.new(file_io)
           else
-            @io = @file_io
+            io = file_io
           end
+          Thread.current[THREAD_LOCAL_IO_KEY] = io
         end
 
         def to_payload(record)
@@ -72,13 +98,15 @@ module Embulk
         end
 
         def add(page)
+          io = thread_io
+          self.class.ios.add(io)
           # I once tried to split IO writing into another IO thread using SizedQueue
           # However, it resulted in worse performance, so I removed the codes.
           page.each do |record|
             Embulk.logger.trace { "embulk-output-bigquery: record #{record}" }
             formatted_record = @formatter_proc.call(record)
             Embulk.logger.trace { "embulk-output-bigquery: formatted_record #{formatted_record.chomp}" }
-            @io.write formatted_record
+            io.write formatted_record
             @num_input_rows += 1
           end
           now = Time.now
@@ -91,10 +119,8 @@ module Embulk
         end
 
         def commit
-          @io.close rescue nil
           task_report = {
             'num_input_rows' => @num_input_rows,
-            'path' => @path,
           }
         end
       end

data/test/test_configure.rb

@@ -74,7 +74,7 @@ module Embulk
         assert_equal 5, task['retries']
         assert_equal "Embulk BigQuery plugin", task['application_name']
         # assert_equal "/tmp/embulk_output_bigquery_20160228-27184-pubcn0", task['path_prefix']
-        assert_equal ".%d.%03d", task['sequence_format']
+        assert_equal ".%d.%d", task['sequence_format']
         assert_equal ".csv", task['file_ext']
         assert_equal false, task['skip_file_generation']
         assert_equal "NONE", task['compression']

data/test/test_example.rb

@@ -20,6 +20,7 @@ if ENV['CONNECT']
     files = files.reject {|file| File.symlink?(file) }
     # files.shift
     files.each do |config_path|
+      next if File.basename(config_path) == 'config_expose_errors.yml'
       define_method(:"test_#{File.basename(config_path, ".yml")}") do
         success = Bundler.with_clean_env do
           cmd = "#{embulk_path} run -X page_size=1 -b . -l trace #{config_path}"

data/test/test_file_writer.rb

@@ -16,6 +16,11 @@ module Embulk
         end
       end
 
+      def setup
+        Thread.current[FileWriter::THREAD_LOCAL_IO_KEY] = nil
+        FileWriter.reset_ios
+      end
+
       def default_task
         {
           'compression' => 'GZIP',
@@ -42,19 +47,30 @@ module Embulk
         @converters ||= ValueConverterFactory.create_converters(default_task, schema)
       end
 
+      def record
+        [true, 1, 1.1, 'foo', Time.parse("2016-02-26 00:00:00 +09:00"), {"foo"=>"foo"}]
+      end
+
+      def page
+        [record]
+      end
+
       sub_test_case "path" do
         def test_path
           task = default_task.merge('path_prefix' => 'tmp/foo', 'sequence_format' => '', 'file_ext' => '.1')
           file_writer = FileWriter.new(task, schema, 0, converters)
-          assert_equal 'tmp/foo.1', file_writer.instance_variable_get(:@path)
+
+          begin
+            file_writer.add(page)
+          ensure
+            io.close rescue nil
+          end
+          path = FileWriter.paths.first
+          assert_equal 'tmp/foo.1', path
         end
       end
 
       sub_test_case "formatter" do
-        def record
-          [true, 1, 1.1, 'foo', Time.parse("2016-02-26 00:00:00 +09:00"), {"foo"=>"foo"}]
-        end
-
         def test_payload_column_index
           task = default_task.merge('payload_column_index' => 0)
           file_writer = FileWriter.new(task, schema, 0, converters)
@@ -86,42 +102,36 @@ module Embulk
       end
 
       sub_test_case "compression" do
-        def record
-          [true, 1, 1.1, 'foo', Time.parse("2016-02-26 00:00:00 +09:00"), {"foo"=>"foo"}]
-        end
-
-        def page
-          [record]
-        end
-
         def test_gzip
           task = default_task.merge('compression' => 'GZIP')
           file_writer = FileWriter.new(task, schema, 0, converters)
-          io = file_writer.instance_variable_get(:@io)
-          assert_equal Zlib::GzipWriter, io.class
 
           begin
             file_writer.add(page)
+            io = FileWriter.ios.first
+            assert_equal Zlib::GzipWriter, io.class
           ensure
-            file_writer.commit
+            io.close rescue nil
           end
-          assert_true File.exist?(file_writer.path)
-          assert_nothing_raised { Zlib::GzipReader.open(file_writer.path) {|gz| } }
+          path = FileWriter.paths.first
+          assert_true File.exist?(path)
+          assert_nothing_raised { Zlib::GzipReader.open(path) {|gz| } }
         end
 
         def test_uncompressed
           task = default_task.merge('compression' => 'NONE')
           file_writer = FileWriter.new(task, schema, 0, converters)
-          io = file_writer.instance_variable_get(:@io)
-          assert_equal File, io.class
 
           begin
             file_writer.add(page)
+            io = FileWriter.ios.first
+            assert_equal File, io.class
           ensure
-            file_writer.commit
+            io.close rescue nil
           end
-          assert_true File.exist?(file_writer.path)
-          assert_raise { Zlib::GzipReader.open(file_writer.path) {|gz| } }
+          path = FileWriter.paths.first
+          assert_true File.exist?(path)
+          assert_raise { Zlib::GzipReader.open(path) {|gz| } }
         end
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.3.0.pre2
+  version: 0.3.0.pre3
 platform: ruby
 authors:
 - Satoshi Akama
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-18 00:00:00.000000000 Z
+date: 2016-03-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-api-client
@@ -104,6 +104,8 @@ files:
 - example/config_guess_with_column_options.yml
 - example/config_gzip.yml
 - example/config_jsonl.yml
+- example/config_max_threads.yml
+- example/config_min_ouput_tasks.yml
 - example/config_mode_append.yml
 - example/config_mode_append_direct.yml
 - example/config_payload_column.yml
@@ -119,6 +121,12 @@ files:
 - example/example.csv
 - example/example.jsonl
 - example/example.yml
+- example/example2_1.csv
+- example/example2_2.csv
+- example/example4_1.csv
+- example/example4_2.csv
+- example/example4_3.csv
+- example/example4_4.csv
 - example/json_key.json
 - example/nested_example.jsonl
 - example/schema.json