fluent-plugin-bigquery 0.2.10 → 0.2.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6c82a1342eb122e1fda82079f62e8d46983fedc5
-  data.tar.gz: ec5fee2a347f6803ee422b55059d0e06241b61c2
+  metadata.gz: e93c58d0a0c1da28a59026a81f9d060ec3b0e532
+  data.tar.gz: ff810e3c6177a362a28a14cd92ea3091294212d4
 SHA512:
-  metadata.gz: b8693580b3dcf0f016562a713add34fa7faf074450c416867d9b3c80e30627233c73d05567972e4ba4ff896ae64bded01f2f0c283d78cbfd1c8602d675433270
-  data.tar.gz: 2c4c919bbd66d5b0c3ce8cdef01ff1bfa65489b78e5440b79594466ca6d841abb25867bce361a44ede0fdab8d8ac2e32b9e0489462a76934c5d29e92e8fc2aa2
+  metadata.gz: 86e853fdad235e8ab2b014e100b4e1f355ff9e11da27605e30d66aa9676de3af888f8ab0499445b42df60c884114d5d380c4fa79f6b92ee226063554c4f1344c
+  data.tar.gz: a8e3d37c866073bfd54d25d15482387bb3454191ab5f6a41718c4a7b324d838e275b3dc2a8ce33eb213f57ee0d9a21b66cd878f5b3aec7f555be179238c4dedd

data/lib/fluent/plugin/bigquery/version.rb

@@ -1,6 +1,6 @@
 module Fluent
   module BigQueryPlugin
-    VERSION = "0.2.10"
+    VERSION = "0.2.11"
   end
 end
 

data/lib/fluent/plugin/out_bigquery.rb

@@ -76,6 +76,10 @@ module Fluent
     # config_param :field_record,  :string, defualt: nil
     # config_param :optional_data_field, :string, default: nil
 
+    REGEXP_MAX_NUM = 10
+    config_param :replace_record_key, :bool, default: false
+    (1..REGEXP_MAX_NUM).each {|i| config_param :"replace_record_key_regexp#{i}", :string, default: nil }
+
     config_param :time_format, :string, default: nil
     config_param :localtime, :bool, default: nil
     config_param :utc, :bool, default: nil
@@ -168,6 +172,15 @@ module Fluent
         end
       end
 
+      @regexps = {}
+      (1..REGEXP_MAX_NUM).each do |i|
+        next unless conf["replace_record_key_regexp#{i}"]
+        regexp, replacement = conf["replace_record_key_regexp#{i}"].split(/ /, 2)
+        raise ConfigError, "replace_record_key_regexp#{i} does not contain 2 parameters" unless replacement
+        raise ConfigError, "replace_record_key_regexp#{i} contains a duplicated key, #{regexp}" if @regexps[regexp]
+        @regexps[regexp] = replacement
+      end
+
       @localtime = false if @localtime.nil? && @utc
 
       @timef = TimeFormatter.new(@time_format, @localtime)
@@ -316,10 +329,27 @@ module Fluent
       raise NotImplementedError # TODO
     end
 
+    def replace_record_key(record)
+      new_record = {}
+      record.each do |key, _|
+        new_key = key
+        @regexps.each do |regexp, replacement|
+          new_key = new_key.gsub(/#{regexp}/, replacement)
+        end
+        new_key = new_key.gsub(/\W/, '')
+        new_record.store(new_key, record[key])
+      end
+      new_record
+    end
+
     def format_stream(tag, es)
       super
       buf = ''
       es.each do |time, record|
+        if @replace_record_key
+          record = replace_record_key(record)
+        end
+        
         row = @fields.format(@add_time_field.call(record, time))
         unless row.empty?
           row = {"json" => row}

data/test/plugin/test_out_bigquery.rb

@@ -683,6 +683,54 @@ class BigQueryOutputTest < Test::Unit::TestCase
     driver.instance.shutdown
   end
 
+  def test_replace_record_key
+    now = Time.now
+    input = [
+      now,
+      {
+        "vhost" => :bar,
+        "@referer" => "http://referer.example",
+        "bot_access" => true,
+        "login-session" => false
+      }
+    ]
+    expected = {
+      "json" => {
+        "time" => now.to_i,
+        "vhost" => "bar",
+        "referer" => "http://referer.example",
+        "bot_access" => true,
+        "login_session" => false
+      }
+    }
+
+    driver = create_driver(<<-CONFIG)
+      table foo
+      email foo@bar.example
+      private_key_path /path/to/key
+      project yourproject_id
+      dataset yourdataset_id
+
+      replace_record_key true
+      replace_record_key_regexp1 - _
+
+      time_format %s
+      time_field time
+
+      field_integer time
+      field_string vhost, referer
+      field_boolean bot_access, login_session
+    CONFIG
+    mock_client(driver) do |expect|
+      expect.discovered_api("bigquery", "v2") { stub! }
+    end
+    driver.instance.start
+    buf = driver.instance.format_stream("my.tag", [input])
+    driver.instance.shutdown
+
+    assert_equal expected, MessagePack.unpack(buf)
+  end
+
   def test_write
     entry = {"json" => {"a" => "b"}}, {"json" => {"b" => "c"}}
     driver = create_driver(CONFIG)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.2.10
+  version: 0.2.11
 platform: ruby
 authors:
 - Naoya Ito
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-29 00:00:00.000000000 Z
+date: 2015-08-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake