fluent-plugin-norikra 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e29c9f16d25507902aaf3dd02a7b43e1596880ca
-  data.tar.gz: bc441ac622bbeb6f9c9b40b1eefcd8fd5efc2277
+  metadata.gz: 17d22de3fa94371d074467e259195bcf675198da
+  data.tar.gz: cfe8d59434ca08f513000b392e95f39af82d3b4c
 SHA512:
-  metadata.gz: 4b95d21010ab1989e74ccbd19407220af2855bdb1333324edbdf4082ed210e5aa6a0deec663391ea386c478480f093103f2083138fdc184645cdce6a4b72b81f
-  data.tar.gz: c57fe2a8e659919719bf457b4b3eb7215b89d9eec56e77841cec3789ac46285dd392dd0149d397a3994c6b2a79b57c53f190609ed799e54a62504c2baa02e29d
+  metadata.gz: 2138b0f8de9901ec4c5d99d49a41e8e5aa3d631dada661767306b829543810d5be9f7c8517f4c55cc3c57b4cc9b344ea3fb4a9fe9b374065eef5f21c071bec6a
+  data.tar.gz: c4437d0720878f816954b31b2e1e523240ce941de4cad94fd0fdc59f869bdd7dc79011604679bfa39d799b755160eb457f28eeaae0d1d6c0a8865b216a8c36e2

data/README.md

@@ -49,6 +49,7 @@ Sends events to remote Norikra server. Minimal configurations are:
 ```
 
 NorikraOutput plugin opens Norikra's target for newly incoming tags. You can specify fields to include/exclude, and specify types of each fields, for each targets (and all targets by `default`). Definitions in `<target TARGET_NAME>` overwrites `<default>` specifications.
+
 ```apache
 <match data.*>
   type    norikra
@@ -82,6 +83,7 @@ NorikraOutput plugin opens Norikra's target for newly incoming tags. You can spe
 With default setting, all fields are defined as 'string', so you must use `field_xxxx` parameters for numerical processing in query (For more details, see Norikra and Esper's documents).
 
 If fluentd's events has so many variations of sets of fields, you can specify not to include fields automatically, with `auto_field` option:
+
 ```apache
 <match data.*>
   type    norikra
@@ -98,6 +100,8 @@ If fluentd's events has so many variations of sets of fields, you can specify no
 
 Fields which are referred in queries are automatically registered on norikra server in spite of `auto_field false`.
 
+Use `time_key FIELDNAME` to include time of Fluentd's event into data field of Norikra (by milliseconds with Norikra/Esper's rule). This is useful for queries with `.win:ext_timed_batch(FIELD, PERIOD)` views.
+
 ** NOTE: <default> and <target> sections in NorikraOutput ignores <query> sections. see NorikraFilterOutput **
 
 ## NorikraInput

data/fluent-plugin-norikra.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-norikra"
-  spec.version       = "0.1.1"
+  spec.version       = "0.1.2"
   spec.authors       = ["TAGOMORI Satoshi"]
   spec.email         = ["tagomoris@gmail.com"]
   spec.description   = %q{process events on fluentd with SQL like query, with built-in Norikra server if needed.}

data/lib/fluent/plugin/norikra/config_section.rb

@@ -1,6 +1,8 @@
 module Fluent::NorikraPlugin
   class ConfigSection
-    attr_accessor :target, :target_matcher, :auto_field, :escape_fieldname, :filter_params, :field_definitions, :query_generators
+    attr_accessor :target, :target_matcher
+    attr_accessor :auto_field, :time_key, :escape_fieldname
+    attr_accessor :filter_params, :field_definitions, :query_generators
 
     def initialize(section, enable_auto_query=true)
       @target = nil
@@ -16,6 +18,7 @@ module Fluent::NorikraPlugin
       end
 
       @auto_field = Fluent::Config.bool_value(section['auto_field'])
+      @time_key = section['time_key']
       @escape_fieldname = Fluent::Config.bool_value(section['escape_fieldname'])
 
       @filter_params = {
@@ -49,6 +52,7 @@ module Fluent::NorikraPlugin
       end
       r = self.class.new(Fluent::Config::Element.new('target', (other.target ? other.target : self.target), {}, []))
       r.auto_field = (other.auto_field.nil? ? self.auto_field : other.auto_field)
+      r.time_key = other.time_key || self.time_key
 
       others_filter = {}
       other.filter_params.keys.each do |k|

data/lib/fluent/plugin/norikra/output.rb

@@ -183,7 +183,7 @@ module Fluent::NorikraPlugin
           t
         end
 
-        event = tgt.filter(record)
+        event = tgt.filter(time, record)
 
         out << [tgt.escaped_name,event].to_msgpack
       end

data/lib/fluent/plugin/norikra/target.rb

@@ -1,6 +1,6 @@
 module Fluent::NorikraPlugin
   class Target
-    attr_accessor :name, :auto_field, :fields, :queries
+    attr_accessor :name, :auto_field, :time_key, :fields, :queries
     attr_reader :escaped_name
 
     def self.escape(src)
@@ -24,15 +24,23 @@ module Fluent::NorikraPlugin
       @name = target
       @escaped_name = self.class.escape(@name)
       @auto_field = config.auto_field.nil? ? true : config.auto_field
+      @time_key = config.time_key
       @escape_fieldname = config.escape_fieldname
 
       @filter = RecordFilter.new(*([:include, :include_regexp, :exclude, :exclude_regexp].map{|s| config.filter_params[s]}))
       @fields = config.field_definitions
+      if @time_key
+        @fields[:integer].push @time_key
+      end
       @queries = config.query_generators.map{|g| g.generate(@name, @escaped_name)}
     end
 
-    def filter(record)
+    def filter(time, record)
       r = @filter.filter(record)
+      if @time_key
+        # Fluentd time (sec) -> Norikra timestamp (milliseconds)
+        r = r.merge({ @time_key => time * 1000 })
+      end
       if @escape_fieldname
         escape_recursive(r)
       else

data/test/test_config_section.rb

@@ -62,6 +62,38 @@ class ConfigSectionTest < Test::Unit::TestCase
     assert_equal 0, s1.query_generators.size
   end
 
+  def test_init_default_with_time_key
+    q1 = Fluent::Config::Element.new('query', nil, {
+        'name' => 'q1_${target}',
+        'expression' => 'SELECT * FROM ${target}.win:time_batch(10 min) WHERE q1',
+        'tag' => 'q1.${target}'
+      }, [])
+    q2 = Fluent::Config::Element.new('query', nil, {
+        'name' => 'q2_${target}',
+        'expression' => 'SELECT * FROM ${target}.win:time_batch(50 min) WHERE q2.length() > 0',
+        'tag' => 'q2.${target}'
+      }, [])
+    c1 = Fluent::Config::Element.new('default', nil, {
+        'time_key' => 'timestamp',
+        'include' => '*',
+        'exclude' => 'flag',
+        'exclude_regexp' => 'f_.*',
+        'field_string' => 's1,s2,s3',
+        'field_boolean' => 'bool1,bool2',
+        'field_integer' => 'i1,i2,i3,i4,num1,num2',
+        'field_float' => 'f1,f2,d',
+      }, [q1,q2])
+    s1 = @this.new(c1, false)
+
+    assert_nil s1.target
+    assert_equal({:include => '*', :include_regexp => nil, :exclude => 'flag', :exclude_regexp => 'f_.*'}, s1.filter_params)
+    assert_equal({
+        :string => %w(s1 s2 s3), :boolean => %w(bool1 bool2), :integer => %w(i1 i2 i3 i4 num1 num2),
+        :float => %w(f1 f2 d),
+      }, s1.field_definitions)
+    assert_equal 'timestamp', s1.time_key
+  end
+
   def test_init_target
     q3 = Fluent::Config::Element.new('query', nil, {
         'name' => 'q3_test2',
@@ -131,6 +163,7 @@ class ConfigSectionTest < Test::Unit::TestCase
         'tag' => 'q2.${target}'
       }, [])
     c1 = Fluent::Config::Element.new('default', nil, {
+        'time_key' => 'ts',
         'include' => '*',
         'exclude' => 'flag',
         'exclude_regexp' => 'f_.*',
@@ -147,6 +180,7 @@ class ConfigSectionTest < Test::Unit::TestCase
         'tag' => 'q3.test'
       }, [])
     c2 = Fluent::Config::Element.new('target', 'test', {
+        'time_key' => 'timestamp',
         'exclude_regexp' => '(f|g)_.*',
         'field_float' => 'd1,d2,d3,d4'
       }, [q3])
@@ -162,6 +196,7 @@ class ConfigSectionTest < Test::Unit::TestCase
       }, s.field_definitions)
     assert_equal 3, s.query_generators.size
     assert_equal (10 * 60 / 5), s.query_generators.map(&:fetch_interval).sort.first
+    assert_equal 'timestamp', s.time_key
   end
 
   def test_join_with_nil
@@ -207,6 +242,7 @@ class ConfigSectionTest < Test::Unit::TestCase
         'tag' => 'q2.${target}'
       }, [])
     c1 = Fluent::Config::Element.new('default', nil, {
+        'time_key' => 'timestamp',
         'include' => '*',
         'exclude' => 'flag',
         'exclude_regexp' => 'f_.*',
@@ -230,5 +266,6 @@ class ConfigSectionTest < Test::Unit::TestCase
 
     s = s1 + s2
     assert_equal 0, s.query_generators.size
+    assert_equal 'timestamp', s.time_key
   end
 end

data/test/test_target.rb

@@ -45,6 +45,7 @@ class TargetTest < Test::Unit::TestCase
       'tag' => 'q3.test'
     }, [])
   C2 = Fluent::Config::Element.new('target', 'test', {
+      'time_key' => 'timestamp',
       'exclude_regexp' => '(f|g)_.*',
       'field_float' => 'd1,d2,d3,d4'
     }, [Q3])
@@ -55,14 +56,16 @@ class TargetTest < Test::Unit::TestCase
 
     assert_equal 'test', t.name
     assert_equal({
-        :string => %w(s1 s2 s3), :boolean => %w(bool1 bool2), :integer => %w(i1 i2 i3 i4 num1 num2),
+        :string => %w(s1 s2 s3), :boolean => %w(bool1 bool2), :integer => %w(i1 i2 i3 i4 num1 num2 timestamp),
         :float => %w(f1 f2 d d1 d2 d3 d4)
       }, t.fields)
     assert_equal 3, t.queries.size
 
-    r = t.filter({'x'=>1,'y'=>'y','z'=>'zett','flag'=>true,'f_x'=>'true','g_1'=>'g'})
-    assert_equal 3, r.size
-    assert_equal({'x'=>1,'y'=>'y','z'=>'zett'}, r)
+    now = Time.now.to_i
+
+    r = t.filter(now, {'x'=>1,'y'=>'y','z'=>'zett','flag'=>true,'f_x'=>'true','g_1'=>'g'})
+    assert_equal 4, r.size
+    assert_equal({'x'=>1,'y'=>'y','z'=>'zett','timestamp'=>(now*1000)}, r)
 
     # reserve_fields
     assert_equal({
@@ -70,7 +73,8 @@ class TargetTest < Test::Unit::TestCase
         'bool1' => 'boolean', 'bool2' => 'boolean',
         'i1' => 'integer', 'i2' => 'integer', 'i3' => 'integer', 'i4' => 'integer', 'num1' => 'integer', 'num2' => 'integer',
         'f1' => 'float', 'f2' => 'float',
-        'd' => 'float', 'd1' => 'float', 'd2' => 'float', 'd3' => 'float', 'd4' => 'float'
+        'd' => 'float', 'd1' => 'float', 'd2' => 'float', 'd3' => 'float', 'd4' => 'float',
+        'timestamp' => 'integer', # time_key
       }, t.reserve_fields)
   end
 
@@ -102,8 +106,10 @@ class TargetTest < Test::Unit::TestCase
   S4 = Fluent::NorikraPlugin::ConfigSection.new(C4)
 
   def test_escape_fieldname
+    now = Time.now.to_i
+
     t = @this.new('test.service', S3)
-    r = t.filter({'a 1' => '1', 'b 2' => 2, 'c-1' => { 'd/1' => '1', 'd 2' => '2' }, 'f' => [1, 2, {'g+1' => 3}] })
+    r = t.filter(now, {'a 1' => '1', 'b 2' => 2, 'c-1' => { 'd/1' => '1', 'd 2' => '2' }, 'f' => [1, 2, {'g+1' => 3}] })
     assert_equal '1', r['a 1']
     assert_equal 2,   r['b 2']
     assert_equal '1', r['c-1']['d/1']
@@ -118,7 +124,7 @@ class TargetTest < Test::Unit::TestCase
     assert_nil r['f'][2]['g_1']
 
     t = @this.new('test.service', S4)
-    r = t.filter({'a 1' => '1', 'b 2' => 2, 'c-1' => { 'd/1' => '1', 'd 2' => '2' }, 'f' => [1, 2, {'g+1' => 3}] })
+    r = t.filter(now, {'a 1' => '1', 'b 2' => 2, 'c-1' => { 'd/1' => '1', 'd 2' => '2' }, 'f' => [1, 2, {'g+1' => 3}] })
     assert_nil r['a 1']
     assert_nil r['b 2']
     assert_nil r['c-1']

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-norikra
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-26 00:00:00.000000000 Z
+date: 2014-04-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: norikra-client