fluent-plugin-bigquery 0.2.14 → 0.2.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 571266d1f383aaeeab090ba11195671a9fcd452f
-  data.tar.gz: 3caf0f7ce01b4396d198241b31cd1c4f65abfec8
+  metadata.gz: 27f275ca7fb430c0576f0358736759c256b18492
+  data.tar.gz: 0f910671b2e06f5f3af370a88122c98fc275fa02
 SHA512:
-  metadata.gz: 489ed6d5a4b10c2f52cf86a14193c4e53bc9cfeb641fb968552fec2e514b5bad708d845c449e8f8d036765f75de330cb01a6fe917a0b1bb79ace6d75361a73d3
-  data.tar.gz: 9148962886487649c2d83f1d211b9c6730f2fa0f66380539a3552eae716c8deaa1142347af50d13cfb7e06ca7bcc5f141b7ceb48ddb362d9eedcb75bb7508a1c
+  metadata.gz: a7014fe6b9f39479a08d8440f6d5a2e4f7524e89c72e07d77008ead605bb430963893ae88ab77934f4a6aff9a9e903af6d7d23820a5885df38c00203248991cd
+  data.tar.gz: 5115561a849c5a5f3150c2254e536fd1b645757853c40860b35ee733a80e16452edb99b630a049f247d39ee56494f64632c1b2891544e7296d1549d63c781d82

data/fluent-plugin-bigquery.gemspec

@@ -23,8 +23,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "test-unit", "~> 3.0.2"
   spec.add_development_dependency "test-unit-rr", "~> 1.0.3"
 
-  spec.add_runtime_dependency "google-api-client", "~> 0.8.0"
+  spec.add_runtime_dependency "google-api-client", "~> 0.9.1"
   spec.add_runtime_dependency "googleauth", ">=  0.5.0"
+  spec.add_runtime_dependency "multi_json"
   spec.add_runtime_dependency "fluentd"
   spec.add_runtime_dependency "fluent-mixin-plaintextformatter", '>= 0.2.1'
   spec.add_runtime_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"

data/lib/fluent/plugin/bigquery/version.rb

@@ -1,6 +1,6 @@
 module Fluent
   module BigQueryPlugin
-    VERSION = "0.2.14"
+    VERSION = "0.2.15"
   end
 end
 

data/lib/fluent/plugin/out_bigquery.rb

@@ -131,9 +131,12 @@ module Fluent
 
     def initialize
       super
-      require 'json'
-      require 'google/api_client'
+      require 'multi_json'
+      require 'google/apis/bigquery_v2'
       require 'googleauth'
+      require 'active_support/json'
+      require 'active_support/core_ext/hash'
+      require 'active_support/core_ext/object/json'
 
       # MEMO: signet-0.6.1 depend on Farady.default_connection
       Faraday.default_connection.options.timeout = 60
@@ -172,7 +175,7 @@ module Fluent
 
       @fields = RecordSchema.new('record')
       if @schema_path
-        @fields.load_schema(JSON.parse(File.read(@schema_path)))
+        @fields.load_schema(MultiJson.load(File.read(@schema_path)))
       end
 
       types = %w(string integer float boolean timestamp)
@@ -221,7 +224,6 @@ module Fluent
     def start
       super
 
-      @bq = client.discovered_api("bigquery", "v2") # TODO: refresh with specified expiration
       @cached_client = nil
       @cached_client_expiration = nil
 
@@ -234,15 +236,13 @@ module Fluent
     def client
       return @cached_client if @cached_client && @cached_client_expiration > Time.now
 
-      client = Google::APIClient.new(
-        application_name: 'Fluentd BigQuery plugin',
-        application_version: Fluent::BigQueryPlugin::VERSION
-      )
+      client = Google::Apis::BigqueryV2::BigqueryService.new
 
       scope = "https://www.googleapis.com/auth/bigquery"
 
       case @auth_method
       when 'private_key'
+        require 'google/api_client/auth/key_utils'
         key = Google::APIClient::KeyUtils.load_from_pkcs12(@private_key_path, @private_key_passphrase)
         auth = Signet::OAuth2::Client.new(
                 token_credential_uri: "https://accounts.google.com/o/oauth2/token",
@@ -271,7 +271,6 @@ module Fluent
         raise ConfigError, "Unknown auth method: #{@auth_method}"
       end
 
-      auth.fetch_access_token!
       client.authorization = auth
 
       @cached_client_expiration = Time.now + 1800
@@ -282,7 +281,7 @@ module Fluent
       format, col = table_id_format.split(/@/)
       time = if col && row
                keys = col.split('.')
-               t = keys.inject(row['json']) {|obj, attr| obj[attr] }
+               t = keys.inject(row[:json]) {|obj, attr| obj[attr.to_sym] }
                Time.at(t)
              else
                current_time
@@ -301,71 +300,43 @@ module Fluent
     end
 
     def create_table(table_id)
-      res = client().execute(
-        :api_method => @bq.tables.insert,
-        :parameters => {
-          'projectId' => @project,
-          'datasetId' => @dataset,
+      client.insert_table(@project, @dataset, {
+        table_reference: {
+          table_id: table_id,
         },
-        :body_object => {
-          'tableReference' => {
-            'tableId' => table_id,
-          },
-          'schema' => {
-            'fields' => @fields.to_a,
-          },
+        schema: {
+          fields: @fields.to_a,
         }
-      )
-      unless res.success?
-        # api_error? -> client cache clear
-        @cached_client = nil
-
-        message = res.body
-        if res.body =~ /^\{/
-          begin
-            res_obj = JSON.parse(res.body)
-            message = res_obj['error']['message'] || res.body
-          rescue => e
-            log.warn "Parse error: google api error response body", :body => res.body
-          end
-          if res_obj and res_obj['error']['code'] == 409 and /Already Exists:/ =~ message
-            # ignore 'Already Exists' error
-            return
-          end
-        end
-        log.error "tables.insert API", :project_id => @project, :dataset => @dataset, :table => table_id, :code => res.status, :message => message
-        raise "failed to create table in bigquery" # TODO: error class
+      }, {})
+    rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
+      # api_error? -> client cache clear
+      @cached_client = nil
+
+      message = e.message
+      if e.status_code == 409 && /Already Exists:/ =~ message
+        # ignore 'Already Exists' error
+        return
       end
+      log.error "tables.insert API", :project_id => @project, :dataset => @dataset, :table => table_id, :code => e.status_code, :message => message
+      raise "failed to create table in bigquery" # TODO: error class
     end
 
     def insert(table_id, rows)
-      res = client().execute(
-        api_method: @bq.tabledata.insert_all,
-        parameters: {
-          'projectId' => @project,
-          'datasetId' => @dataset,
-          'tableId' => table_id,
-        },
-        body_object: {
-          "rows" => rows
-        }
-      )
-      unless res.success?
-        # api_error? -> client cache clear
-        @cached_client = nil
-
-        res_obj = extract_response_obj(res.body)
-        message = res_obj['error']['message'] || res.body
-        if res_obj
-          if @auto_create_table and res_obj and res_obj['error']['code'] == 404 and /Not Found: Table/i =~ message.to_s
-            # Table Not Found: Auto Create Table
-            create_table(table_id)
-            raise "table created. send rows next time."
-          end
-        end
-        log.error "tabledata.insertAll API", project_id: @project, dataset: @dataset, table: table_id, code: res.status, message: message
-        raise "failed to insert into bigquery" # TODO: error class
+      client.insert_all_table_data(@project, @dataset, table_id, {
+        rows: rows
+      }, {})
+    rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
+      # api_error? -> client cache clear
+      @cached_client = nil
+
+      message = e.message
+      if @auto_create_table && e.status_code == 404 && /Not Found: Table/i =~ message.to_s
+        # Table Not Found: Auto Create Table
+        create_table(table_id)
+        raise "table created. send rows next time."
       end
+      log.error "tabledata.insertAll API", project_id: @project, dataset: @dataset, table: table_id, code: e.status_code, message: message
+      raise "failed to insert into bigquery" # TODO: error class
     end
 
     def load
@@ -389,7 +360,7 @@ module Fluent
     def convert_hash_to_json(record)
       record.each do |key, value|
         if value.class == Hash
-          record[key] = value.to_json
+          record[key] = MultiJson.dump(value)
         end
       end
       record
@@ -409,7 +380,7 @@ module Fluent
       row = @fields.format(@add_time_field.call(record, time))
       unless row.empty?
         row = {"json" => row}
-        row['insertId'] = @get_insert_id.call(record) if @get_insert_id
+        row['insert_id'] = @get_insert_id.call(record) if @get_insert_id
         buf << row.to_msgpack
       end
       buf
@@ -419,7 +390,7 @@ module Fluent
       rows = []
       chunk.msgpack_each do |row_object|
         # TODO: row size limit
-        rows << row_object
+        rows << row_object.deep_symbolize_keys
       end
 
       # TODO: method
@@ -438,58 +409,17 @@ module Fluent
     def fetch_schema
       table_id_format = @tablelist[0]
       table_id = generate_table_id(table_id_format, Time.at(Fluent::Engine.now))
-      res = client.execute(
-        api_method: @bq.tables.get,
-        parameters: {
-          'projectId' => @project,
-          'datasetId' => @dataset,
-          'tableId' => table_id,
-        }
-      )
+      res = client.get_table(@project, @dataset, table_id)
 
-      unless res.success?
-        # api_error? -> client cache clear
-        @cached_client = nil
-        message = extract_error_message(res.body)
-        log.error "tables.get API", project_id: @project, dataset: @dataset, table: table_id, code: res.status, message: message
-        raise "failed to fetch schema from bigquery" # TODO: error class
-      end
-
-      res_obj = JSON.parse(res.body)
-      schema = res_obj['schema']['fields']
+      schema = res.schema.fields.as_json
       log.debug "Load schema from BigQuery: #{@project}:#{@dataset}.#{table_id} #{schema}"
       @fields.load_schema(schema, false)
-    end
-
-    # def client_oauth # not implemented
-    #   raise NotImplementedError, "OAuth needs browser authentication..."
-    #
-    #   client = Google::APIClient.new(
-    #     application_name: 'Example Ruby application',
-    #     application_version: '1.0.0'
-    #   )
-    #   bigquery = client.discovered_api('bigquery', 'v2')
-    #   flow = Google::APIClient::InstalledAppFlow.new(
-    #     client_id: @client_id
-    #     client_secret: @client_secret
-    #     scope: ['https://www.googleapis.com/auth/bigquery']
-    #   )
-    #   client.authorization = flow.authorize # browser authentication !
-    #   client
-    # end
-
-    def extract_response_obj(response_body)
-      return nil unless response_body =~ /^\{/
-      JSON.parse(response_body)
-    rescue
-      log.warn "Parse error: google api error response body", body: response_body
-      return nil
-    end
-
-    def extract_error_message(response_body)
-      res_obj = extract_response_obj(response_body)
-      return response_body if res_obj.nil?
-      res_obj['error']['message'] || response_body
+    rescue Google::Apis::ServerError, Google::Apis::ClientError, Google::Apis::AuthorizationError => e
+      # api_error? -> client cache clear
+      @cached_client = nil
+      message = e.message
+      log.error "tables.get API", project_id: @project, dataset: @dataset, table: table_id, code: e.status_code, message: message
+      raise "failed to fetch schema from bigquery" # TODO: error class
     end
 
     class FieldSchema
@@ -531,9 +461,9 @@ module Fluent
 
       def to_h
         {
-          'name' => name,
-          'type' => type.to_s.upcase,
-          'mode' => mode.to_s.upcase,
+          :name => name,
+          :type => type.to_s.upcase,
+          :mode => mode.to_s.upcase,
         }
       end
     end
@@ -619,10 +549,10 @@ module Fluent
 
       def to_h
         {
-          'name' => name,
-          'type' => type.to_s.upcase,
-          'mode' => mode.to_s.upcase,
-          'fields' => self.to_a,
+          :name => name,
+          :type => type.to_s.upcase,
+          :mode => mode.to_s.upcase,
+          :fields => self.to_a,
         }
       end
 

data/test/plugin/test_out_bigquery.rb

@@ -1,7 +1,11 @@
 require 'helper'
-require 'google/api_client'
+require 'google/apis/bigquery_v2'
+require 'google/api_client/auth/key_utils'
 require 'googleauth'
-require 'fluent/plugin/buf_memory'
+require 'active_support/json'
+require 'active_support/core_ext/hash'
+require 'active_support/core_ext/object/json'
+
 
 class BigQueryOutputTest < Test::Unit::TestCase
   def setup
@@ -32,7 +36,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
 
   def stub_client(driver)
     stub(client = Object.new) do |expect|
-      expect.discovered_api("bigquery", "v2") { stub! }
       yield expect if defined?(yield)
     end
     stub(driver.instance).client { client }
@@ -65,7 +68,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
     key = stub!
     mock(Google::APIClient::KeyUtils).load_from_pkcs12('/path/to/key', 'notasecret') { key }
     authorization = Object.new
-    mock(authorization).fetch_access_token!
     stub(Signet::OAuth2::Client).new
     mock(Signet::OAuth2::Client).new(
       token_credential_uri: "https://accounts.google.com/o/oauth2/token",
@@ -74,7 +76,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
       issuer: 'foo@bar.example',
       signing_key: key) { authorization }
 
-    mock.proxy(Google::APIClient).new.with_any_args {
+    mock.proxy(Google::Apis::BigqueryV2::BigqueryService).new.with_any_args {
       mock!.__send__(:authorization=, authorization) {}
     }
 
@@ -84,10 +86,9 @@ class BigQueryOutputTest < Test::Unit::TestCase
 
   def test_configure_auth_compute_engine
     authorization = Object.new
-    mock(authorization).fetch_access_token!
     mock(Google::Auth::GCECredentials).new { authorization }
 
-    mock.proxy(Google::APIClient).new.with_any_args {
+    mock.proxy(Google::Apis::BigqueryV2::BigqueryService).new.with_any_args {
       mock!.__send__(:authorization=, authorization) {}
     }
 
@@ -104,10 +105,9 @@ class BigQueryOutputTest < Test::Unit::TestCase
   def test_configure_auth_json_key_as_file
     json_key_path = 'test/plugin/testdata/json_key.json'
     authorization = Object.new
-    mock(authorization).fetch_access_token!
     mock(Google::Auth::ServiceAccountCredentials).make_creds(json_key_io: File.open(json_key_path), scope: API_SCOPE) { authorization }
 
-    mock.proxy(Google::APIClient).new.with_any_args {
+    mock.proxy(Google::Apis::BigqueryV2::BigqueryService).new.with_any_args {
       mock!.__send__(:authorization=, authorization) {}
     }
 
@@ -127,10 +127,9 @@ class BigQueryOutputTest < Test::Unit::TestCase
     json_key_io = StringIO.new(json_key)
     mock(StringIO).new(json_key) { json_key_io }
     authorization = Object.new
-    mock(authorization).fetch_access_token!
     mock(Google::Auth::ServiceAccountCredentials).make_creds(json_key_io: json_key_io, scope: API_SCOPE) { authorization }
 
-    mock.proxy(Google::APIClient).new.with_any_args {
+    mock.proxy(Google::Apis::BigqueryV2::BigqueryService).new.with_any_args {
       mock!.__send__(:authorization=, authorization) {}
     }
 
@@ -147,10 +146,9 @@ class BigQueryOutputTest < Test::Unit::TestCase
 
   def test_configure_auth_application_default
     authorization = Object.new
-    mock(authorization).fetch_access_token!
     mock(Google::Auth).get_application_default([API_SCOPE]) { authorization }
 
-    mock.proxy(Google::APIClient).new.with_any_args {
+    mock.proxy(Google::Apis::BigqueryV2::BigqueryService).new.with_any_args {
       mock!.__send__(:authorization=, authorization) {}
     }
 
@@ -286,9 +284,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
     }
 
     driver = create_driver(CONFIG)
-    mock_client(driver) do |expect|
-      expect.discovered_api("bigquery", "v2") { stub! }
-    end
     driver.instance.start
     buf = driver.instance.format_stream("my.tag", [input])
     driver.instance.shutdown
@@ -456,9 +451,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
       schema_path #{File.join(File.dirname(__FILE__), "testdata", "apache.schema")}
       field_integer time
     CONFIG
-    mock_client(driver) do |expect|
-      expect.discovered_api("bigquery", "v2") { stub! }
-    end
     driver.instance.start
     buf = driver.instance.format_stream("my.tag", [input])
     driver.instance.shutdown
@@ -499,9 +491,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
       schema_path #{File.join(File.dirname(__FILE__), "testdata", "sudo.schema")}
       field_integer time
     CONFIG
-    mock_client(driver) do |expect|
-      expect.discovered_api("bigquery", "v2") { stub! }
-    end
     driver.instance.start
     buf = driver.instance.format_stream("my.tag", [input])
     driver.instance.shutdown
@@ -543,18 +532,13 @@ class BigQueryOutputTest < Test::Unit::TestCase
       field_integer time
     CONFIG
     mock_client(driver) do |expect|
-      expect.discovered_api("bigquery", "v2") { mock!.tables.mock!.get { Object.new } }
-      expect.execute(
-        :api_method => anything,
-        :parameters => {
-          'projectId' => 'yourproject_id',
-          'datasetId' => 'yourdataset_id',
-          'tableId' => 'foo'
-        }
-      ) {
+      expect.get_table('yourproject_id', 'yourdataset_id', 'foo') {
         s = stub!
-        s.success? { true }
-        s.body { JSON.generate(sudo_schema_response) }
+        schema_stub = stub!
+        fields_stub = stub!
+        s.schema { schema_stub }
+        schema_stub.fields { fields_stub }
+        fields_stub.as_json { sudo_schema_response.deep_stringify_keys["schema"]["fields"] }
         s
       }
     end
@@ -620,18 +604,13 @@ class BigQueryOutputTest < Test::Unit::TestCase
       field_integer time
     CONFIG
     mock_client(driver) do |expect|
-      expect.discovered_api("bigquery", "v2") { mock!.tables.mock!.get { Object.new } }
-      expect.execute(
-        :api_method => anything,
-        :parameters => {
-          'projectId' => 'yourproject_id',
-          'datasetId' => 'yourdataset_id',
-          'tableId' => now.strftime('foo_%Y_%m_%d')
-        }
-      ) {
+      expect.get_table('yourproject_id', 'yourdataset_id', now.strftime('foo_%Y_%m_%d')) {
         s = stub!
-        s.success? { true }
-        s.body { JSON.generate(sudo_schema_response) }
+        schema_stub = stub!
+        fields_stub = stub!
+        s.schema { schema_stub }
+        schema_stub.fields { fields_stub }
+        fields_stub.as_json { sudo_schema_response.deep_stringify_keys["schema"]["fields"] }
         s
       }
     end
@@ -672,7 +651,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
       }
     ]
     expected = {
-      "insertId" => "9ABFF756-0267-4247-847F-0895B65F0938",
+      "insert_id" => "9ABFF756-0267-4247-847F-0895B65F0938",
       "json" => {
         "uuid" => "9ABFF756-0267-4247-847F-0895B65F0938",
       }
@@ -688,9 +667,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
       insert_id_field uuid
       field_string uuid
     CONFIG
-    mock_client(driver) do |expect|
-      expect.discovered_api("bigquery", "v2") { stub! }
-    end
     driver.instance.start
     buf = driver.instance.format_stream("my.tag", [input])
     driver.instance.shutdown
@@ -709,7 +685,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
       }
     ]
     expected = {
-      "insertId" => "809F6BA7-1C16-44CD-9816-4B20E2C7AA2A",
+      "insert_id" => "809F6BA7-1C16-44CD-9816-4B20E2C7AA2A",
       "json" => {
         "data" => {
           "uuid" => "809F6BA7-1C16-44CD-9816-4B20E2C7AA2A",
@@ -727,9 +703,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
       insert_id_field data.uuid
       field_string data.uuid
     CONFIG
-    mock_client(driver) do |expect|
-      expect.discovered_api("bigquery", "v2") { stub! }
-    end
     driver.instance.start
     buf = driver.instance.format_stream("my.tag", [input])
     driver.instance.shutdown
@@ -762,9 +735,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
       schema_path #{File.join(File.dirname(__FILE__), "testdata", "sudo.schema")}
       field_integer time
     CONFIG
-    mock_client(driver) do |expect|
-      expect.discovered_api("bigquery", "v2") { stub! }
-    end
     driver.instance.start
     assert_raises(RuntimeError.new("Required field user cannot be null")) do
       driver.instance.format_stream("my.tag", [input])
@@ -810,9 +780,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
       field_string vhost, referer
       field_boolean bot_access, login_session
     CONFIG
-    mock_client(driver) do |expect|
-      expect.discovered_api("bigquery", "v2") { stub! }
-    end
     driver.instance.start
     buf = driver.instance.format_stream("my.tag", [input])
     driver.instance.shutdown
@@ -864,9 +831,6 @@ class BigQueryOutputTest < Test::Unit::TestCase
       field_string vhost, referer, remote
       field_boolean bot_access, loginsession
     CONFIG
-    mock_client(driver) do |expect|
-      expect.discovered_api("bigquery", "v2") { stub! }
-    end
     driver.instance.start
     buf = driver.instance.format_stream("my.tag", [input])
     driver.instance.shutdown
@@ -875,25 +839,18 @@ class BigQueryOutputTest < Test::Unit::TestCase
   end
 
   def test_write
-    entry = {"json" => {"a" => "b"}}, {"json" => {"b" => "c"}}
+    entry = {json: {a: "b"}}, {json: {b: "c"}}
     driver = create_driver(CONFIG)
     mock_client(driver) do |expect|
-      expect.discovered_api("bigquery", "v2") { mock!.tabledata.mock!.insert_all { Object.new } }
-      expect.execute(
-        :api_method => anything,
-        :parameters => {
-          'projectId' => 'yourproject_id',
-          'datasetId' => 'yourdataset_id',
-          'tableId' => 'foo',
-        },
-        :body_object => {
-          'rows' => [entry]
-        }
-      ) { stub!.success? { true } }
+      expect.insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', {
+        rows: entry
+      }, {}) { stub! }
     end
 
     chunk = Fluent::MemoryBufferChunk.new("my.tag")
-    chunk << entry.to_msgpack
+    entry.each do |e|
+      chunk << e.to_msgpack
+    end
 
     driver.instance.start
     driver.instance.write(chunk)
@@ -902,8 +859,8 @@ class BigQueryOutputTest < Test::Unit::TestCase
 
   def test_write_with_row_based_table_id_formatting
     entry = [
-      {"json" => {"a" => "b", "created_at" => Time.local(2014,8,20,9,0,0).to_i}},
-      {"json" => {"b" => "c", "created_at" => Time.local(2014,8,21,9,0,0).to_i}}
+      {json: {a: "b", created_at: Time.local(2014,8,20,9,0,0).to_i}},
+      {json: {b: "c", created_at: Time.local(2014,8,21,9,0,0).to_i}}
     ]
     driver = create_driver(<<-CONFIG)
       table foo_%Y_%m_%d@created_at
@@ -921,31 +878,13 @@ class BigQueryOutputTest < Test::Unit::TestCase
       field_boolean bot_access,loginsession
     CONFIG
     mock_client(driver) do |expect|
-      expect.discovered_api("bigquery", "v2") { mock!.tabledata.times(2).mock!.insert_all.times(2) { Object.new } }
-
-      expect.execute(
-        :api_method => anything,
-        :parameters => {
-          'projectId' => 'yourproject_id',
-          'datasetId' => 'yourdataset_id',
-          'tableId' => 'foo_2014_08_20',
-        },
-        :body_object => {
-          'rows' => [entry[0]]
-        }
-      ) { stub!.success? { true } }
-
-      expect.execute(
-        :api_method => anything,
-        :parameters => {
-          'projectId' => 'yourproject_id',
-          'datasetId' => 'yourdataset_id',
-          'tableId' => 'foo_2014_08_21',
-        },
-        :body_object => {
-          'rows' => [entry[1]]
-        }
-      ) { stub!.success? { true } }
+      expect.insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo_2014_08_20', {
+        rows: [entry[0]]
+      }, {})
+
+      expect.insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo_2014_08_21', {
+        rows: [entry[1]]
+      }, {})
     end
 
     chunk = Fluent::MemoryBufferChunk.new("my.tag")
@@ -970,7 +909,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
     driver = create_driver
     table_id_format = 'foo_%Y_%m_%d@created_at'
     time = Time.local(2014, 8, 11, 21, 20, 56)
-    row = { "json" => { "created_at" => Time.local(2014,8,10,21,20,57).to_i } }
+    row = { json: { created_at: Time.local(2014,8,10,21,20,57).to_i } }
     table_id = driver.instance.generate_table_id(table_id_format, time, row)
     assert_equal 'foo_2014_08_10', table_id
   end
@@ -979,7 +918,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
     driver = create_driver
     table_id_format = 'foo_%Y_%m_%d@foo.bar.created_at'
     time = Time.local(2014, 8, 11, 21, 20, 56)
-    row = { "json" => { "foo" => { "bar" => { "created_at" => Time.local(2014,8,10,21,20,57).to_i } } } }
+    row = { json: { foo: { bar: { created_at: Time.local(2014,8,10,21,20,57).to_i } } } }
     table_id = driver.instance.generate_table_id(table_id_format, time, row)
     assert_equal 'foo_2014_08_10', table_id
   end
@@ -1022,7 +961,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
           "bytes" => 72,
         },
       }
-    }
+    }.deep_symbolize_keys
 
     driver = create_driver(<<-CONFIG)
       table foo
@@ -1038,55 +977,27 @@ class BigQueryOutputTest < Test::Unit::TestCase
       schema_path #{File.join(File.dirname(__FILE__), "testdata", "apache.schema")}
     CONFIG
     mock_client(driver) do |expect|
-      expect.discovered_api("bigquery", "v2") {
-        mock! {
-          tables.mock!.insert { Object.new }
-          tabledata.mock!.insert_all { Object.new }
-        }
+      expect.insert_all_table_data('yourproject_id', 'yourdataset_id', 'foo', {
+        rows: [message]
+      }, {}) {
+        raise Google::Apis::ServerError.new("Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404, body: "Not found: Table yourproject_id:yourdataset_id.foo")
       }
-      expect.execute(
-        :api_method => anything,
-        :parameters => {
-          'projectId' => 'yourproject_id',
-          'datasetId' => 'yourdataset_id',
-          'tableId' => 'foo'
+      expect.insert_table('yourproject_id', 'yourdataset_id', {
+        table_reference: {
+          table_id: 'foo',
         },
-        :body_object => {
-          "rows" => [ message ]
+        schema: {
+          fields: JSON.parse(File.read(File.join(File.dirname(__FILE__), "testdata", "apache.schema"))).map(&:deep_symbolize_keys),
         }
-      ) {
-        s = stub!
-        s.success? { false }
-        s.body { JSON.generate({
-          'error' => { "code" => 404, "message" => "Not found: Table yourproject_id:yourdataset_id.foo" }
-        }) }
-        s.status { 404 }
-        s
-      }
-      expect.execute(
-        :api_method => anything,
-        :parameters => {
-          'projectId' => 'yourproject_id',
-          'datasetId' => 'yourdataset_id',
-        },
-        :body_object => {
-          'tableReference' => {
-            'tableId' => 'foo',
-          },
-          'schema' => {
-            'fields' => JSON.parse(File.read(File.join(File.dirname(__FILE__), "testdata", "apache.schema")))
-          }
-        }
-      ) {
-        s = stub!
-        s.success? { true }
-        s
+      }, {}) {
+        stub!
       }
     end
     chunk = Fluent::MemoryBufferChunk.new("my.tag")
     chunk << message.to_msgpack
 
     driver.instance.start
+
     assert_raise(RuntimeError) {
       driver.instance.write(chunk)
     }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.2.14
+  version: 0.2.15
 platform: ruby
 authors:
 - Naoya Ito
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-24 00:00:00.000000000 Z
+date: 2016-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -72,14 +72,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: 0.9.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: 0.9.1
 - !ruby/object:Gem::Dependency
   name: googleauth
   requirement: !ruby/object:Gem::Requirement
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.5.0
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement