fluent-plugin-bigquery 0.2.6 → 0.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: eca8d5626366799524377edcdd7ced540cc633ef
-  data.tar.gz: f39debef79fbb52c9c8c7068d4e68f8cc7bbc036
+  metadata.gz: e55e2b00afb7ffd8b32cc9b7fa0a305ab66af863
+  data.tar.gz: b93594427c232d70db4f7ceca1a28554732b5122
 SHA512:
-  metadata.gz: bc6e6af3faf9b1eb9a6dc0fe96a17b8534828fbbcc1133f3e6f9d7c956cca5a65c15319dd2c8df6e4f55a674b561edbb88087ca7c5e298c24ee50db3924e546b
-  data.tar.gz: 6eff9dcacc9e6b956cc2eb0b55b91f6db0b3418474f941bd83a18eb1f7cef33c68541b6a7d650c42f053edb7ae7464a460d737d09b71bb47e19591521758450f
+  metadata.gz: 5662de9f1d839a5ef38fd04df24bd4b90de8bd4a494020ecde58e20191df1317322b903eaef985f7cdaf68f24260ed86b802a6aca53dfb3e3825772547cafaf9
+  data.tar.gz: 03a4f811ccc74d36530a4d341218a31c9bc913f530321565696a6e70227c94067bc61a91c2ea97bd5ba5ebf2c617bc1da51e0c1b8c71e97037c444092f62c18c

data/README.md

@@ -163,6 +163,26 @@ data is inserted into tables `accesslog_2014_08`, `accesslog_2014_09` and so on.
 Note that the timestamp of logs and the date in the table id do not always match,
 because there is a time lag between collection and transmission of logs.
 
+### Dynamic table creating
+
+When `auto_create_table` is set to `true`, try to create the table using BigQuery API when insertion failed with code=404 "Not Found: Table ...".
+Next retry of insertion is expected to be success.
+
+NOTE: `auto_create_table` option cannot be used with `fetch_schema`. You should create the table on ahead to use `fetch_schema`.
+
+```apache
+<match dummy>
+  type bigquery
+  
+  ...
+  
+  auto_create_table true
+  table accesslog_%Y_%m
+  
+  ...
+</match>
+```
+
 ### Table schema
 
 There are three methods to describe the schema of the target table.

data/lib/fluent/plugin/bigquery/version.rb

@@ -1,6 +1,6 @@
 module Fluent
   module BigQueryPlugin
-    VERSION = "0.2.6"
+    VERSION = "0.2.7"
   end
 end
 

data/lib/fluent/plugin/out_bigquery.rb

@@ -40,12 +40,12 @@ module Fluent
     # Available methods are:
     # * private_key -- Use service account credential
     # * compute_engine -- Use access token available in instances of ComputeEngine
-    config_param :auth_method, :string, :default => 'private_key'
+    config_param :auth_method, :string, default: 'private_key'
 
     ### Service Account credential
-    config_param :email, :string, :default => nil
-    config_param :private_key_path, :string, :default => nil
-    config_param :private_key_passphrase, :string, :default => 'notasecret'
+    config_param :email, :string, default: nil
+    config_param :private_key_path, :string, default: nil
+    config_param :private_key_passphrase, :string, default: 'notasecret'
 
     # see as simple reference
     #   https://github.com/abronte/BigQuery/blob/master/lib/bigquery.rb
@@ -58,32 +58,34 @@ module Fluent
 
     # table_id
     #   In Table ID, enter a name for your new table. Naming rules are the same as for your dataset.
-    config_param :table, :string, :default => nil
-    config_param :tables, :string, :default => nil
-
-    config_param :schema_path, :string, :default => nil
-    config_param :fetch_schema, :bool, :default => false
-    config_param :field_string,  :string, :default => nil
-    config_param :field_integer, :string, :default => nil
-    config_param :field_float,   :string, :default => nil
-    config_param :field_boolean, :string, :default => nil
-    config_param :field_timestamp, :string, :default => nil
+    config_param :table, :string, default: nil
+    config_param :tables, :string, default: nil
+
+    config_param :auto_create_table, :bool, default: false
+
+    config_param :schema_path, :string, default: nil
+    config_param :fetch_schema, :bool, default: false
+    config_param :field_string,  :string, default: nil
+    config_param :field_integer, :string, default: nil
+    config_param :field_float,   :string, default: nil
+    config_param :field_boolean, :string, default: nil
+    config_param :field_timestamp, :string, default: nil
     ### TODO: record field stream inserts doesn't works well?
     ###  At table creation, table type json + field type record -> field type validation fails
     ###  At streaming inserts, schema cannot be specified
-    # config_param :field_record,  :string, :defualt => nil
-    # config_param :optional_data_field, :string, :default => nil
+    # config_param :field_record,  :string, defualt: nil
+    # config_param :optional_data_field, :string, default: nil
 
-    config_param :time_format, :string, :default => nil
-    config_param :localtime, :bool, :default => nil
-    config_param :utc, :bool, :default => nil
-    config_param :time_field, :string, :default => nil
+    config_param :time_format, :string, default: nil
+    config_param :localtime, :bool, default: nil
+    config_param :utc, :bool, default: nil
+    config_param :time_field, :string, default: nil
 
-    config_param :insert_id_field, :string, :default => nil
+    config_param :insert_id_field, :string, default: nil
 
-    config_param :method, :string, :default => 'insert' # or 'load' # TODO: not implemented now
+    config_param :method, :string, default: 'insert' # or 'load' # TODO: not implemented now
 
-    config_param :load_size_limit, :integer, :default => 1000**4 # < 1TB (1024^4) # TODO: not implemented now
+    config_param :load_size_limit, :integer, default: 1000**4 # < 1TB (1024^4) # TODO: not implemented now
     ### method: 'load'
     #   https://developers.google.com/bigquery/loading-data-into-bigquery
     # Maximum File Sizes:
@@ -92,9 +94,9 @@ module Fluent
     #                          Without new-lines in strings: 1 TB
     # JSON        1 GB         1 TB
 
-    config_param :row_size_limit, :integer, :default => 100*1000 # < 100KB # configurable in google ?
-    # config_param :insert_size_limit, :integer, :default => 1000**2 # < 1MB
-    # config_param :rows_per_second_limit, :integer, :default => 1000 # spike limit
+    config_param :row_size_limit, :integer, default: 100*1000 # < 100KB # configurable in google ?
+    # config_param :insert_size_limit, :integer, default: 1000**2 # < 1MB
+    # config_param :rows_per_second_limit, :integer, default: 1000 # spike limit
     ### method: ''Streaming data inserts support
     #  https://developers.google.com/bigquery/streaming-data-into-bigquery#usecases
     # Maximum row size: 100 KB
@@ -137,7 +139,7 @@ module Fluent
 
       case @auth_method
       when 'private_key'
-        if !@email || !@private_key_path
+        unless @email && @private_key_path
           raise Fluent::ConfigError, "'email' and 'private_key_path' must be specified if auth_method == 'private_key'"
         end
       when 'compute_engine'
@@ -146,7 +148,7 @@ module Fluent
         raise Fluent::ConfigError, "unrecognized 'auth_method': #{@auth_method}"
       end
 
-      if (!@table && !@tables) || (@table && @tables)
+      unless @table.nil? ^ @tables.nil?
         raise Fluent::ConfigError, "'table' or 'tables' must be specified, and both are invalid"
       end
 
@@ -156,53 +158,34 @@ module Fluent
       if @schema_path
         @fields.load_schema(JSON.parse(File.read(@schema_path)))
       end
-      if @field_string
-        @field_string.split(',').each do |fieldname|
-          @fields.register_field fieldname.strip, :string
-        end
-      end
-      if @field_integer
-        @field_integer.split(',').each do |fieldname|
-          @fields.register_field fieldname.strip, :integer
-        end
-      end
-      if @field_float
-        @field_float.split(',').each do |fieldname|
-          @fields.register_field fieldname.strip, :float
-        end
-      end
-      if @field_boolean
-        @field_boolean.split(',').each do |fieldname|
-          @fields.register_field fieldname.strip, :boolean
-        end
-      end
-      if @field_timestamp
-        @field_timestamp.split(',').each do |fieldname|
-          @fields.register_field fieldname.strip, :timestamp
-        end
-      end
 
-      if @localtime.nil?
-        if @utc
-          @localtime = false
+      types = %w(string integer float boolean timestamp)
+      types.each do |type|
+        raw_fields = instance_variable_get("@field_#{type}")
+        next unless raw_fields
+        raw_fields.split(',').each do |field|
+          @fields.register_field field.strip, type.to_sym
         end
       end
+
+      @localtime = false if @localtime.nil? && @utc
+
       @timef = TimeFormatter.new(@time_format, @localtime)
 
       if @time_field
         keys = @time_field.split('.')
         last_key = keys.pop
-        @add_time_field = lambda {|record, time|
+        @add_time_field = ->(record, time) {
           keys.inject(record) { |h, k| h[k] ||= {} }[last_key] = @timef.format(time)
           record
         }
       else
-        @add_time_field = lambda {|record, time| record }
+        @add_time_field = ->(record, time) { record }
       end
 
       if @insert_id_field
         insert_id_keys = @insert_id_field.split('.')
-        @get_insert_id = lambda {|record|
+        @get_insert_id = ->(record) {
           insert_id_keys.inject(record) {|h, k| h[k] }
         }
       else
@@ -223,17 +206,12 @@ module Fluent
       fetch_schema() if @fetch_schema
     end
 
-    def shutdown
-      super
-      # nothing to do
-    end
-
     def client
       return @cached_client if @cached_client && @cached_client_expiration > Time.now
 
       client = Google::APIClient.new(
-        :application_name => 'Fluentd BigQuery plugin',
-        :application_version => Fluent::BigQueryPlugin::VERSION
+        application_name: 'Fluentd BigQuery plugin',
+        application_version: Fluent::BigQueryPlugin::VERSION
       )
 
       case @auth_method
@@ -264,17 +242,20 @@ module Fluent
       current_time.strftime(table_id_format)
     end
 
-    def insert(table_id_format, rows)
-      table_id = generate_table_id(table_id_format, Time.at(Fluent::Engine.now))
+    def create_table(table_id)
       res = client().execute(
-        :api_method => @bq.tabledata.insert_all,
+        :api_method => @bq.tables.insert,
         :parameters => {
           'projectId' => @project,
           'datasetId' => @dataset,
-          'tableId' => table_id,
         },
         :body_object => {
-          "rows" => rows
+          'tableReference' => {
+            'tableId' => table_id,
+          },
+          'schema' => {
+            'fields' => @fields.to_a,
+          },
         }
       )
       unless res.success?
@@ -289,8 +270,42 @@ module Fluent
           rescue => e
             log.warn "Parse error: google api error response body", :body => res.body
           end
+          if res_obj and res_obj['code'] == 409 and /Already Exists:/ =~ message
+            # ignore 'Already Exists' error
+            return
+          end
         end
-        log.error "tabledata.insertAll API", :project_id => @project, :dataset => @dataset, :table => table_id, :code => res.status, :message => message
+        log.error "tables.insert API", :project_id => @project, :dataset => @dataset, :table => table_id, :code => res.status, :message => message
+        raise "failed to create table in bigquery" # TODO: error class
+      end
+    end
+
+    def insert(table_id_format, rows)
+      table_id = generate_table_id(table_id_format, Time.at(Fluent::Engine.now))
+      res = client().execute(
+        api_method: @bq.tabledata.insert_all,
+        parameters: {
+          'projectId' => @project,
+          'datasetId' => @dataset,
+          'tableId' => table_id,
+        },
+        body_object: {
+          "rows" => rows
+        }
+      )
+      unless res.success?
+        # api_error? -> client cache clear
+        @cached_client = nil
+
+        res_obj = extract_response_obj(res.body)
+        message = res_obj['error']['message'] || res.body
+        if res_obj
+          if @auto_create_table and res_obj and res_obj['error']['code'] == 404 and /Not Found: Table/ =~ message.to_s
+            # Table Not Found: Auto Create Table
+            create_table(table_id)
+          end
+        end
+        log.error "tabledata.insertAll API", project_id: @project, dataset: @dataset, table: table_id, code: res.status, message: message
         raise "failed to insert into bigquery" # TODO: error class
       end
     end
@@ -335,8 +350,8 @@ module Fluent
       table_id_format = @tablelist[0]
       table_id = generate_table_id(table_id_format, Time.at(Fluent::Engine.now))
       res = client.execute(
-        :api_method => @bq.tables.get,
-        :parameters => {
+        api_method: @bq.tables.get,
+        parameters: {
           'projectId' => @project,
           'datasetId' => @dataset,
           'tableId' => table_id,
@@ -346,17 +361,8 @@ module Fluent
       unless res.success?
         # api_error? -> client cache clear
         @cached_client = nil
-
-        message = res.body
-        if res.body =~ /^\{/
-          begin
-            res_obj = JSON.parse(res.body)
-            message = res_obj['error']['message'] || res.body
-          rescue => e
-            log.warn "Parse error: google api error response body", :body => res.body
-          end
-        end
-        log.error "tables.get API", :project_id => @project, :dataset => @dataset, :table => table_id, :code => res.status, :message => message
+        message = extract_error_message(res.body)
+        log.error "tables.get API", project_id: @project, dataset: @dataset, table: table_id, code: res.status, message: message
         raise "failed to fetch schema from bigquery" # TODO: error class
       end
 
@@ -370,19 +376,33 @@ module Fluent
     #   raise NotImplementedError, "OAuth needs browser authentication..."
     #
     #   client = Google::APIClient.new(
-    #     :application_name => 'Example Ruby application',
-    #     :application_version => '1.0.0'
+    #     application_name: 'Example Ruby application',
+    #     application_version: '1.0.0'
     #   )
     #   bigquery = client.discovered_api('bigquery', 'v2')
     #   flow = Google::APIClient::InstalledAppFlow.new(
-    #     :client_id => @client_id
-    #     :client_secret => @client_secret
-    #     :scope => ['https://www.googleapis.com/auth/bigquery']
+    #     client_id: @client_id
+    #     client_secret: @client_secret
+    #     scope: ['https://www.googleapis.com/auth/bigquery']
     #   )
     #   client.authorization = flow.authorize # browser authentication !
     #   client
     # end
 
+    def extract_response_obj(response_body)
+      return nil unless response_body =~ /^\{/
+      JSON.parse(response_body)
+    rescue
+      log.warn "Parse error: google api error response body", body: response_body
+      return nil
+    end
+
+    def extract_error_message(response_body)
+      res_obj = extract_response_obj(response_body)
+      return response_body if res_obj.nil?
+      res_obj['error']['message'] || response_body
+    end
+
     class FieldSchema
       def initialize(name, mode = :nullable)
         unless [:nullable, :required, :repeated].include?(mode)
@@ -419,6 +439,14 @@ module Fluent
       def format_one(value)
         raise NotImplementedError, "Must implement in a subclass" 
       end
+
+      def to_h
+        {
+          'name' => name,
+          'type' => type.to_s.upcase,
+          'mode' => mode.to_s.upcase,
+        }
+      end
     end
 
     class StringFieldSchema < FieldSchema
@@ -473,12 +501,12 @@ module Fluent
 
     class RecordSchema < FieldSchema
       FIELD_TYPES = {
-        :string => StringFieldSchema,
-        :integer => IntegerFieldSchema,
-        :float => FloatFieldSchema,
-        :boolean => BooleanFieldSchema,
-        :timestamp => TimestampFieldSchema,
-        :record => RecordSchema
+        string: StringFieldSchema,
+        integer: IntegerFieldSchema,
+        float: FloatFieldSchema,
+        boolean: BooleanFieldSchema,
+        timestamp: TimestampFieldSchema,
+        record: RecordSchema
       }.freeze
 
       def initialize(name, mode = :nullable)
@@ -494,6 +522,21 @@ module Fluent
         @fields[name]
       end
 
+      def to_a
+        @fields.map do |_, field_schema|
+          field_schema.to_h
+        end
+      end
+
+      def to_h
+        {
+          'name' => name,
+          'type' => type.to_s.upcase,
+          'mode' => mode.to_s.upcase,
+          'fields' => self.to_a,
+        }
+      end
+
       def load_schema(schema, allow_overwrite=true)
         schema.each do |field|
           raise ConfigError, 'field must have type' unless field.key?('type')

data/test/plugin/test_out_bigquery.rb

@@ -717,6 +717,103 @@ class BigQueryOutputTest < Test::Unit::TestCase
     assert_equal 'foo_2014_08_11', table_id
   end
 
+  def test_auto_create_table_by_bigquery_api
+    now = Time.now
+    message = {
+      "json" => {
+        "time" => now.to_i,
+        "request" => {
+          "vhost" => "bar",
+          "path" => "/path/to/baz",
+          "method" => "GET",
+          "protocol" => "HTTP/1.0",
+          "agent" => "libwww",
+          "referer" => "http://referer.example",
+          "time" => (now - 1).to_f,
+          "bot_access" => true,
+          "loginsession" => false,
+        },
+        "remote" => {
+          "host" => "remote.example",
+          "ip" =>  "192.168.1.1",
+          "user" => "nagachika",
+        },
+        "response" => {
+          "status" => 200,
+          "bytes" => 72,
+        },
+      }
+    }
+
+    driver = create_driver(<<-CONFIG)
+      table foo
+      email foo@bar.example
+      private_key_path /path/to/key
+      project yourproject_id
+      dataset yourdataset_id
+
+      time_format %s
+      time_field  time
+
+      auto_create_table true
+      schema_path #{File.join(File.dirname(__FILE__), "testdata", "apache.schema")}
+    CONFIG
+    mock_client(driver) do |expect|
+      expect.discovered_api("bigquery", "v2") {
+        mock! {
+          tables.mock!.insert { Object.new }
+          tabledata.mock!.insert_all { Object.new }
+        }
+      }
+      expect.execute(
+        :api_method => anything,
+        :parameters => {
+          'projectId' => 'yourproject_id',
+          'datasetId' => 'yourdataset_id',
+          'tableId' => 'foo'
+        },
+        :body_object => {
+          "rows" => [ message ]
+        }
+      ) {
+        s = stub!
+        s.success? { false }
+        s.body { JSON.generate({
+          'error' => { "code" => 404, "message" => "Not Found: Table yourproject_id:yourdataset_id.foo" }
+        }) }
+        s.status { 404 }
+        s
+      }
+      expect.execute(
+        :api_method => anything,
+        :parameters => {
+          'projectId' => 'yourproject_id',
+          'datasetId' => 'yourdataset_id',
+        },
+        :body_object => {
+          'tableReference' => {
+            'tableId' => 'foo',
+          },
+          'schema' => {
+            'fields' => JSON.parse(File.read(File.join(File.dirname(__FILE__), "testdata", "apache.schema")))
+          }
+        }
+      ) {
+        s = stub!
+        s.success? { true }
+        s
+      }
+    end
+    chunk = Fluent::MemoryBufferChunk.new("my.tag")
+    chunk << message.to_msgpack
+
+    driver.instance.start
+    assert_raise(RuntimeError) {
+      driver.instance.write(chunk)
+    }
+    driver.instance.shutdown
+  end
+
   private
 
   def sudo_schema_response

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.2.7
 platform: ruby
 authors:
 - Naoya Ito
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-22 00:00:00.000000000 Z
+date: 2015-01-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake