fluent-plugin-bigquery 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0802b40a0e2792d20ca2e92f20cfe398d92aa5db
-  data.tar.gz: a210aa8139c2d82467979adac99f6396096ef6d7
+  metadata.gz: 91a4bfe02680f2079998d36ab098c5498bdaea04
+  data.tar.gz: 234135f11479a6d364d1c5d0f58d90cfe7d72717
 SHA512:
-  metadata.gz: 865a4f9cd3de0d912678768a5a29d3795b62382de2949ac7fc220446d2aaedb2c62616056ad2d44901c64c2782d45c3e2f2f1bb78ca73174394d1b2954291e5f
-  data.tar.gz: 1bde3427c107b3ac396e808f4856daff92f9a50666d07567dfe1b3cdd0fd50a0ca550b1573368b1913cde125dfa4d874ed6aa96862014fdb2bd22e950ce78bbf
+  metadata.gz: a1e3df653dca491e163824f57e6d3d0932314ef59b2cc0cf36caa77f9007df624b183cd2b700caf5c02bd6a477322d136e44c14bed15f44c8c201eb9fb43838e
+  data.tar.gz: d20de3a18a7507fb5532480cd51704e23e79740d12a58bde482824095740681b10ede39a9e0611e878bba6e609078dc406e5e983b5ea5160e83b0632a4c83542

data/.travis.yml

@@ -0,0 +1,8 @@
+language: ruby
+
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.0
+
+script: bundle exec rake test

data/README.md

@@ -1,6 +1,6 @@
 # fluent-plugin-bigquery
 
-Fluentd output plugin to load/insert data into Google BigQuery.
+[Fluentd](http://fluentd.org) output plugin to load/insert data into Google BigQuery.
 
 * insert data over streaming inserts
   * for continuous real-time insertions, under many limitations
@@ -104,6 +104,7 @@ Important options for high rate events are:
 ### Authentication
 
 There are two methods supported to fetch access token for the service account.
+
 1. Public-Private key pair
 2. Predefined access token (Compute Engine only)
 
@@ -135,6 +136,73 @@ Compute Engine instance, then you can configure fluentd like this.
 </match>
 ```
 
+### Table schema
+
+There are two methods to describe the schema of the target table.
+
+1. List fields in fluent.conf
+2. Load a schema file in JSON.
+
+The examples above use the first method.  In this method,
+you can also specify nested fields by prefixing their belonging record fields.
+
+```apache
+<match dummy>
+  type bigquery
+
+  ...
+  
+  time_format %s
+  time_field  time
+
+  field_integer time,response.status,response.bytes
+  field_string  request.vhost,request.path,request.method,request.protocol,request.agent,request.referer,remote.host,remote.ip,remote.user
+  field_float   request.time
+  field_boolean request.bot_access,request.loginsession
+</match>
+```
+
+This schema accepts structured JSON data like:
+
+```json
+{
+  "request":{
+    "time":1391748126.7000976,
+    "vhost":"www.example.com",
+    "path":"/",
+    "method":"GET",
+    "protocol":"HTTP/1.1",
+    "agent":"HotJava",
+    "bot_access":false
+  },
+  "remote":{ "ip": "192.0.2.1" },
+  "response":{
+    "status":200,
+    "bytes":1024
+  }
+}
+```
+
+The second method is to specify a path to a BigQuery schema file instead of listing fields.  In this case, your fluent.conf looks like:
+
+```apache
+<match dummy>
+  type bigquery
+
+  ...
+
+  time_format %s
+  time_field  time
+
+  schema_path /path/to/httpd.schema
+  field_integer time
+</match>
+```
+where /path/to/httpd.schema is a path to the JSON-encoded schema file which you used for creating the table on BigQuery.
+
+NOTE: Since JSON does not define how to encode data of TIMESTAMP type,
+you are still recommended to specify JSON types for TIMESTAMP fields as "time" field does in the example.
+
 
 ### patches
 

data/fluent-plugin-bigquery.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_development_dependency "rake"
+  spec.add_development_dependency "rr"
   spec.add_runtime_dependency "google-api-client", "~> 0.7.1"
   spec.add_runtime_dependency "fluentd"
   spec.add_runtime_dependency "fluent-mixin-plaintextformatter", '>= 0.2.1'

data/lib/fluent/plugin/bigquery/version.rb

@@ -1,6 +1,6 @@
 module Fluent
   module BigQueryPlugin
-    VERSION = "0.0.3"
+    VERSION = "0.0.4"
   end
 end
 

data/lib/fluent/plugin/out_bigquery.rb

@@ -63,6 +63,7 @@ module Fluent
     config_param :table, :string, :default => nil
     config_param :tables, :string, :default => nil
 
+    config_param :schema_path, :string, :default => nil
     config_param :field_string,  :string, :default => nil
     config_param :field_integer, :string, :default => nil
     config_param :field_float,   :string, :default => nil
@@ -117,6 +118,7 @@ module Fluent
 
     def initialize
       super
+      require 'json'
       require 'google/api_client'
       require 'google/api_client/client_secrets'
       require 'google/api_client/auth/installed_app'
@@ -137,13 +139,16 @@ module Fluent
         raise Fluent::ConfigError, "unrecognized 'auth_method': #{@auth_method}"
       end
 
-      if (!@table && !@tables) || (@table && @table)
+      if (!@table && !@tables) || (@table && @tables)
         raise Fluent::ConfigError, "'table' or 'tables' must be specified, and both are invalid"
       end
 
       @tablelist = @tables ? @tables.split(',') : [@table]
 
       @fields = RecordSchema.new
+      if @schema_path
+        @fields.load_schema(JSON.parse(File.read(@schema_path)))
+      end
       if @field_string
         @field_string.split(',').each do |fieldname|
           @fields.register_field fieldname, :string
@@ -171,6 +176,17 @@ module Fluent
         end
       end
       @timef = TimeFormatter.new(@time_format, @localtime)
+
+      if @time_field
+        keys = @time_field.split('.')
+        last_key = keys.pop
+        @add_time_field = lambda {|record, time|
+          keys.inject(record) { |h, k| h[k] ||= {} }[last_key] = @timef.format(time)
+          record
+        }
+      else
+        @add_time_field = lambda {|record, time| record }
+      end
     end
 
     def start
@@ -260,11 +276,7 @@ module Fluent
       super
       buf = ''
       es.each do |time, record|
-        row = if @time_field
-                @fields.format(record.merge({@time_field => @timef.format(time)}))
-              else
-                @fields.format(record)
-              end
+        row = @fields.format(@add_time_field.call(record, time))
         buf << {"json" => row}.to_msgpack unless row.empty?
       end
       buf
@@ -350,12 +362,24 @@ module Fluent
       end
     end
 
+    class TimestampFieldSchema < FieldSchema
+      def type
+        :timestamp
+      end
+
+      def format(value)
+        value
+      end
+    end
+
     class RecordSchema < FieldSchema
       FIELD_TYPES = {
         :string => StringFieldSchema,
         :integer => IntegerFieldSchema,
         :float => FloatFieldSchema,
-        :boolean => BooleanFieldSchema
+        :boolean => BooleanFieldSchema,
+        :timestamp => TimestampFieldSchema,
+        :record => RecordSchema
       }.freeze
 
       def initialize
@@ -370,8 +394,27 @@ module Fluent
         @fields[name]
       end
 
+      def load_schema(schema)
+        schema.each do |field|
+          raise ConfigError, 'field must have type' unless field.key?('type')
+
+          type = field['type'].downcase.to_sym
+          field_schema_class = FIELD_TYPES[type]
+          raise ConfigError, "Invalid field type: #{field['type']}" unless field_schema_class
+
+          field_schema = field_schema_class.new
+          @fields[field['name']] = field_schema
+          if type == :record
+            raise ConfigError, "record field must have fields" unless field.key?('fields')
+            field_schema.load_schema(field['fields'])
+          end
+        end
+      end
+
       def register_field(name, type)
-        raise ConfigError, "field #{name} is registered twice" if @fields.key?(name)
+        if @fields.key?(name) and @fields[name].type != :timestamp
+          raise ConfigError, "field #{name} is registered twice"
+        end
         if name[/\./]
           recordname = $`
           fieldname = $'

data/test/helper.rb

@@ -29,5 +29,7 @@ require 'fluent/plugin/buf_file'
 require 'fluent/plugin/out_bigquery'
 require 'fluent/plugin/bigquery/load_request_body_wrapper'
 
+require 'rr'
+
 class Test::Unit::TestCase
 end

data/test/plugin/test_out_bigquery.rb

@@ -0,0 +1,335 @@
+require 'helper'
+require 'google/api_client'
+require 'fluent/plugin/buf_memory'
+
+class BigQueryOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    table foo
+    email foo@bar.example
+    private_key_path /path/to/key
+    project yourproject_id
+    dataset yourdataset_id
+
+    time_format %s
+    time_field  time
+
+    field_integer time,status,bytes
+    field_string  vhost,path,method,protocol,agent,referer,remote.host,remote.ip,remote.user
+    field_float   requesttime
+    field_boolean bot_access,loginsession
+  ]
+
+  API_SCOPE = "https://www.googleapis.com/auth/bigquery"
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::OutputTestDriver.new(Fluent::BigQueryOutput).configure(conf)
+  end
+
+  def stub_client(driver)
+    stub(client = Object.new) do |expect|
+      expect.discovered_api("bigquery", "v2") { stub! }
+      yield expect if defined?(yield)
+    end
+    stub(driver.instance).client { client }
+    client
+  end
+
+  def mock_client(driver)
+    mock(client = Object.new) do |expect|
+      yield expect
+    end
+    stub(driver.instance).client { client }
+    client
+  end
+
+  def test_configure_table
+    driver = create_driver
+    assert_equal driver.instance.table, 'foo'
+    assert_nil driver.instance.tables
+
+    driver = create_driver(CONFIG.sub(/\btable\s+.*$/,  'tables foo,bar'))
+    assert_nil driver.instance.table
+    assert_equal driver.instance.tables, 'foo,bar'
+
+    assert_raise(Fluent::ConfigError, "'table' or 'tables' must be specified, and both are invalid") {
+      create_driver(CONFIG + "tables foo,bar")
+    }
+  end
+
+  def test_configure_auth
+    key = stub!
+    mock(Google::APIClient::PKCS12).load_key('/path/to/key', 'notasecret') { key }
+    authorization = Object.new
+    asserter = mock!.authorize { authorization }
+    mock(Google::APIClient::JWTAsserter).new('foo@bar.example', API_SCOPE, key) { asserter }
+
+    mock.proxy(Google::APIClient).new.with_any_args { 
+      mock!.__send__(:authorization=, authorization) {}
+    }
+
+    driver = create_driver(CONFIG)
+    driver.instance.client()
+  end
+
+  def test_format_stream
+    now = Time.now
+    input = [
+      now,
+      {
+        "status" => "1",
+        "bytes" => 3.0,
+        "vhost" => :bar,
+        "path" => "/path/to/baz",
+        "method" => "GET",
+        "protocol" => "HTTP/0.9",
+        "agent" => "libwww",
+        "referer" => "http://referer.example",
+        "requesttime" => (now - 1).to_f.to_s,
+        "bot_access" => true,
+        "loginsession" => false,
+        "something-else" => "would be ignored",
+        "yet-another" => {
+          "foo" => "bar",
+          "baz" => 1,
+        },
+        "remote" => {
+          "host" => "remote.example",
+          "ip" =>  "192.0.2.1",
+          "port" => 12345,
+          "user" => "tagomoris",
+        }
+      }
+    ]
+    expected = {
+      "json" => {
+        "time" => now.to_i,
+        "status" => 1,
+        "bytes" => 3,
+        "vhost" => "bar",
+        "path" => "/path/to/baz",
+        "method" => "GET",
+        "protocol" => "HTTP/0.9",
+        "agent" => "libwww",
+        "referer" => "http://referer.example",
+        "requesttime" => (now - 1).to_f.to_s.to_f,
+        "bot_access" => true,
+        "loginsession" => false,
+        "remote" => {
+          "host" => "remote.example",
+          "ip" =>  "192.0.2.1",
+          "user" => "tagomoris",
+        }
+      }
+    }
+
+    driver = create_driver(CONFIG)
+    mock_client(driver) do |expect|
+      expect.discovered_api("bigquery", "v2") { stub! }
+    end
+    driver.instance.start
+    buf = driver.instance.format_stream("my.tag", [input])
+    driver.instance.shutdown
+
+    assert_equal expected, MessagePack.unpack(buf)
+  end
+
+  [
+    # <time_format>, <time field type>, <time expectation generator>, <assertion>
+    [
+      "%s.%6N", "field_float",
+      lambda{|t| t.strftime("%s.%6N").to_f },
+      lambda{|recv, expected, actual|
+        recv.assert_in_delta(expected, actual, Float::EPSILON / 10**3)
+      }
+    ],
+    [
+      "%Y-%m-%dT%H:%M:%SZ", "field_string",
+      lambda{|t| t.iso8601 },
+      :assert_equal.to_proc
+    ],
+    [
+      "%a, %d %b %Y %H:%M:%S GMT", "field_string",
+      lambda{|t| t.httpdate },
+      :assert_equal.to_proc
+    ],
+  ].each do |format, type, expect_time, assert|
+    define_method("test_time_formats_#{format}") do
+      now = Time.now.utc
+      input = [ now, {} ]
+      expected = { "json" => { "time" => expect_time[now], } }
+
+      driver = create_driver(<<-CONFIG)
+        table foo
+        email foo@bar.example
+        private_key_path /path/to/key
+        project yourproject_id
+        dataset yourdataset_id
+
+        time_format #{format}
+        time_field  time
+        #{type}     time
+      CONFIG
+      stub_client(driver)
+
+      driver.instance.start
+      buf = driver.instance.format_stream("my.tag", [input])
+      driver.instance.shutdown
+
+      assert[self, expected["json"]["time"], MessagePack.unpack(buf)["json"]["time"]]
+    end
+  end
+
+  def test_format_nested_time
+    now = Time.now
+    input = [
+      now,
+      {
+        "metadata" => {
+          "node" => "mynode.example",
+        },
+        "log" => "something",
+      }
+    ]
+    expected = {
+      "json" => {
+        "metadata" => {
+          "time" => now.strftime("%s").to_i,
+          "node" => "mynode.example",
+        },
+        "log" => "something",
+      }
+    }
+
+    driver = create_driver(<<-CONFIG)
+      table foo
+      email foo@bar.example
+      private_key_path /path/to/key
+      project yourproject_id
+      dataset yourdataset_id
+
+      time_format %s
+      time_field  metadata.time
+
+      field_integer metadata.time
+      field_string  metadata.node,log
+    CONFIG
+    stub_client(driver)
+    driver.instance.start
+    buf = driver.instance.format_stream("my.tag", [input])
+    driver.instance.shutdown
+
+    assert_equal expected, MessagePack.unpack(buf)
+  end
+
+  def test_format_with_schema
+    now = Time.now
+    input = [
+      now,
+      {
+        "request" => {
+          "vhost" => :bar,
+          "path" => "/path/to/baz",
+          "method" => "GET",
+          "protocol" => "HTTP/0.9",
+          "agent" => "libwww",
+          "referer" => "http://referer.example",
+          "time" => (now - 1).to_f,
+          "bot_access" => true,
+          "loginsession" => false,
+        },
+        "response" => {
+          "status" => "1",
+          "bytes" => 3.0,
+        },
+        "remote" => {
+          "host" => "remote.example",
+          "ip" =>  "192.0.2.1",
+          "port" => 12345,
+          "user" => "tagomoris",
+        },
+        "something-else" => "would be ignored",
+        "yet-another" => {
+          "foo" => "bar",
+          "baz" => 1,
+        },
+      }
+    ]
+    expected = {
+      "json" => {
+        "time" => now.to_i,
+        "request" => {
+          "vhost" => "bar",
+          "path" => "/path/to/baz",
+          "method" => "GET",
+          "protocol" => "HTTP/0.9",
+          "agent" => "libwww",
+          "referer" => "http://referer.example",
+          "time" => (now - 1).to_f,
+          "bot_access" => true,
+          "loginsession" => false,
+        },
+        "remote" => {
+          "host" => "remote.example",
+          "ip" =>  "192.0.2.1",
+          "user" => "tagomoris",
+        },
+        "response" => {
+          "status" => 1,
+          "bytes" => 3,
+        },
+      }
+    }
+
+    driver = create_driver(<<-CONFIG)
+      table foo
+      email foo@bar.example
+      private_key_path /path/to/key
+      project yourproject_id
+      dataset yourdataset_id
+
+      time_format %s
+      time_field  time
+
+      schema_path #{File.join(File.dirname(__FILE__), "testdata", "apache.schema")}
+      field_integer time
+    CONFIG
+    mock_client(driver) do |expect|
+      expect.discovered_api("bigquery", "v2") { stub! }
+    end
+    driver.instance.start
+    buf = driver.instance.format_stream("my.tag", [input])
+    driver.instance.shutdown
+
+    assert_equal expected, MessagePack.unpack(buf)
+  end
+
+  def test_write
+    entry = {"json" => {"a" => "b"}}, {"json" => {"b" => "c"}}
+    driver = create_driver(CONFIG)
+    mock_client(driver) do |expect|
+      expect.discovered_api("bigquery", "v2") { mock!.tabledata.mock!.insert_all { Object.new } }
+      expect.execute(
+        :api_method => anything,
+        :parameters => {
+          'projectId' => 'yourproject_id',
+          'datasetId' => 'yourdataset_id',
+          'tableId' => 'foo',
+        },
+        :body_object => {
+          'rows' => [entry]
+        }
+      ) { stub!.success? { true } }
+    end
+
+    chunk = Fluent::MemoryBufferChunk.new("my.tag")
+    chunk << entry.to_msgpack
+
+    driver.instance.start
+    driver.instance.write(chunk)
+    driver.instance.shutdown
+  end
+end

data/test/plugin/testdata/apache.schema

@@ -0,0 +1,98 @@
+[
+  {
+    "name": "time",
+    "type": "TIMESTAMP",
+    "mode": "REQUIRED"
+  },
+  {
+    "name": "request",
+    "type": "RECORD",
+    "mode": "REQUIRED",
+    "fields": [
+      {
+        "name": "vhost",
+        "type": "STRING",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "path",
+        "type": "STRING",
+        "mode": "REQUIRED"
+      },
+      {
+        "name": "method",
+        "type": "STRING",
+        "mode": "REQUIRED"
+      },
+      {
+        "name": "protocol",
+        "type": "STRING",
+        "mode": "REQUIRED"
+      },
+      {
+        "name": "agent",
+        "type": "STRING",
+        "mode": "REQUIRED"
+      },
+      {
+        "name": "referer",
+        "type": "STRING",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "time",
+        "type": "TIMESTAMP",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "bot_access",
+        "type": "BOOLEAN",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "loginsession",
+        "type": "BOOLEAN",
+        "mode": "NULLABLE"
+      }
+    ]
+  },
+  {
+    "name": "remote",
+    "type": "RECORD",
+    "mode": "NULLABLE",
+    "fields": [
+      {
+        "name": "host",
+        "type": "STRING",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "ip",
+        "type": "STRING",
+        "mode": "REQUIRED"
+      },
+      {
+        "name": "user",
+        "type": "STRING",
+        "mode": "NULLABLE"
+      }
+    ]
+  },
+  {
+    "name": "response",
+    "type": "RECORD",
+    "mode": "REQUIRED",
+    "fields": [
+      {
+        "name": "status",
+        "type": "INTEGER",
+        "mode": "REQUIRED"
+      },
+      {
+        "name": "bytes",
+        "type": "INTEGER",
+        "mode": "REQUIRED"
+      }
+    ]
+  }
+]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-06 00:00:00.000000000 Z
+date: 2014-02-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -24,6 +24,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: google-api-client
   requirement: !ruby/object:Gem::Requirement
@@ -117,6 +131,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -126,6 +141,8 @@ files:
 - lib/fluent/plugin/bigquery/version.rb
 - lib/fluent/plugin/out_bigquery.rb
 - test/helper.rb
+- test/plugin/test_out_bigquery.rb
+- test/plugin/testdata/apache.schema
 - test/test_load_request_body_wrapper.rb
 homepage: https://github.com/tagomoris/fluent-plugin-bigquery
 licenses:
@@ -153,5 +170,7 @@ specification_version: 4
 summary: Fluentd plugin to store data on Google BigQuery
 test_files:
 - test/helper.rb
+- test/plugin/test_out_bigquery.rb
+- test/plugin/testdata/apache.schema
 - test/test_load_request_body_wrapper.rb
 has_rdoc: