fluent-plugin-forest 0.1.4 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 64c115e1a16dd698ed72620f85092fdf2ffa2df6
-  data.tar.gz: add9f6483e7b64610f4c1b768e385738c2edaf82
+  metadata.gz: 8de96a623bfa6608e043f68ca1dbd0549ed4d315
+  data.tar.gz: e34249c2332e2520d75aba897b852045db5ce983
 SHA512:
-  metadata.gz: 4ec2639b99f43aca4d7d69f669d823e4637d679b3168e3165d130da51c1ac50deca7964ee81390e3d99f05817e099f5563a2f2f322651256ca6979b66310ad25
-  data.tar.gz: 26f9b84fcad4872df358b580ef5deba7cf5705037b1d0931dc42f6b0a13e5706abc135cb4cd9d0d4983130e7c12b8ddbdce8bd2fef65cdb9d11ca26e1ec3df65
+  metadata.gz: 64f32283a2f2cde11e07ce690ed810b492990ee9a48b7a18e91bb643b31140dec89991d9f924998ce0152505df703466adaf01d6bc8f2562c679ff20f7d4aa86
+  data.tar.gz: 35aa6d7e0c3e8dd8df00387569d60a6a583a66511fb0bdaa9fea9d04bba75f9b2742450b39c367f9d0dab1d164083125e99a6144c49fa8cf6165265a15ef2e45

data/README.md

@@ -98,9 +98,59 @@ If you want to place logs /var/archive for `service.search.**` as filename with
       </case>
     </match>
 
+Version 0.2.0 or later, subsections adding/overwriting are supported. About the case below, three `<store>` subsections are defined for `search.**` pattern.
+
+    <match service.*>
+      type forest
+      subtype copy
+      <template>
+        <store>
+          type file
+          path /path/to/copy1
+        </store>
+        <store>
+          type file
+          path /path/to/copy2
+        </store>
+      </template>
+      <case search.**>
+        <store>
+          type file
+          path /path/to/copy3
+        </store>
+      </case>
+    </match>
+
+Subsections with same arguments will be overwritten. See this example:
+
+    <match service.*>
+      type forest
+      subtype route
+      <template>
+        <route {search,admin}.a>
+          add_prefix first
+        </route>
+        <route {search,admin}.b>
+          add_prefix second
+        </route>
+        <route {search,admin}.c>
+          add_prefix third
+        </route>
+        <route {search,admin}.*>
+          add_prefix extra
+        </route>
+      </template>
+      <case admin.*>
+        <route {search,admin}.*>
+          add_prefix other
+        </route>
+      </case>
+    </match>
+
+In this case, `<route {search,admin}.*>` subsection will be overwritten to add prefix 'other' for tag `service.admin.*`.
+
 ## TODO
 
-* consider what to do next
 * patches welcome!
 
 ## Copyright

data/fluent-plugin-forest.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-forest"
-  gem.version       = "0.1.4"
+  gem.version       = "0.2.0"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.description   = %q{create sub-plugin dynamically per tags, with template configuration and parameters}

data/lib/fluent/plugin/out_forest.rb

@@ -28,10 +28,9 @@ class Fluent::ForestOutput < Fluent::MultiOutput
     @cases = []
 
     conf.elements.each do |element|
-      element.keys.each do |k|
-        # read and throw away to supress unread configuration warning
-        element[k]
-      end
+      # read and throw away to supress unread configuration warning
+      touch_recursive(element)
+
       case element.name
       when 'template'
         @template = element
@@ -44,6 +43,15 @@ class Fluent::ForestOutput < Fluent::MultiOutput
     self
   end
 
+  def touch_recursive(e)
+    e.keys.each do |k|
+      e[k]
+    end
+    e.elements.each do |child|
+      touch_recursive(child)
+    end
+  end
+
   def shutdown
     super
     @mapping.values.each do |output|
@@ -51,12 +59,15 @@ class Fluent::ForestOutput < Fluent::MultiOutput
     end
   end
 
-  def parameter(tag, e)
+  def parameter(tag, e, name = 'instance', arg = '')
     pairs = {}
     e.each do |k,v|
       pairs[k] = v.gsub('__TAG__', tag).gsub('${tag}', tag).gsub('__HOSTNAME__', @hostname).gsub('${hostname}', @hostname)
     end
-    Fluent::Config::Element.new('instance', '', pairs, [])
+    elements = e.elements.map do |child|
+      parameter(tag, child, child.name, child.arg)
+    end
+    Fluent::Config::Element.new(name, arg, pairs, elements)
   end
 
   def spec(tag)
@@ -64,7 +75,17 @@ class Fluent::ForestOutput < Fluent::MultiOutput
     conf = parameter(tag, @template) + conf if @template # a + b -> b.merge(a) (see: fluentd/lib/fluent/config.rb)
     @cases.each do |m,e|
       if m.match(tag)
-        conf = parameter(tag, e) + conf
+        matched_case = parameter(tag, e)
+
+        matched_case.elements.each { |case_child|
+          unless case_child.arg.empty?
+            conf.elements.delete_if { |conf_child|
+              conf_child.name == case_child.name && conf_child.arg == case_child.arg
+            }
+          end
+        }
+
+        conf = matched_case + conf
         break
       end
     end

data/test/plugin/test_out_forest.rb

@@ -12,7 +12,7 @@ remove_prefix test
 <template>
   key_name f
   suffix !
-  tag out.__TAG__  
+  tag out.__TAG__
 </template>
 <case foo.bar>
   prefix p1:
@@ -62,7 +62,7 @@ subtype hoge
     assert_equal 'yyyyyy.xx', conf['keyy']
     assert_equal 'z1', conf['keyz']
     assert_equal 'b', conf['alt_key']
-    
+
     conf = d.instance.spec('yy')
     assert_equal 'xxxxxx', conf['keyx']
     assert_equal 'yyyyyy.yy', conf['keyy']
@@ -82,6 +82,155 @@ subtype hoge
     assert_equal 'd.zz', conf['alt_key']
   end
 
+  NESTED_CONF = %[
+subtype hoge
+<template>
+  keyx xxxxxx
+  keyy yyyyyy.__TAG__
+  alt_key a
+  <tagx>
+    key xxx
+  </tagx>
+  <tagy>
+    key yyy.__TAG__
+  </tagy>
+  <tag_with attr.foo>
+    key bar
+  </tag_with>
+  <tag_with attr.foo>
+    key dup
+  </tag_with>
+  <tag_with attr.foo>
+    key dupdup
+  </tag_with>
+</template>
+<case xx>
+  keyz z1
+  alt_key b
+</case>
+<case yy.**>
+  keyz z2
+  alt_key c
+  <tagx>
+    key not_overwrite
+  </tagx>
+  <tag_with attr.foo>
+    key overwrite
+  </tag_with>
+  <tag_with attr.foo>
+    key overwrite_dup
+  </tag_with>
+</case>
+<case *>
+  keyz z3
+  alt_key d.__TAG__
+  <tagz>
+    <tagw>
+      key www
+    </tagw>
+  </tagz>
+  <tag_with attr.*>
+    key not_overwrite
+  </tag_with>
+</case>
+    ]
+
+  def test_spec_nested_overwrite_attributes_only
+    d = create_driver NESTED_CONF
+    conf = d.instance.spec('xx')
+    assert_equal 'xxxxxx', conf['keyx']
+    assert_equal 'yyyyyy.xx', conf['keyy']
+    assert_equal 'z1', conf['keyz']
+    assert_equal 'b', conf['alt_key']
+
+    assert_equal 5, conf.elements.size
+
+    assert_equal 'tagx', conf.elements[0].name
+    assert_equal 'xxx', conf.elements[0]['key']
+
+    assert_equal 'tagy', conf.elements[1].name
+    assert_equal 'yyy.xx', conf.elements[1]['key']
+
+    assert_equal 'tag_with', conf.elements[2].name
+    assert_equal 'attr.foo', conf.elements[2].arg
+    assert_equal 'bar', conf.elements[2]['key']
+
+    assert_equal 'tag_with', conf.elements[3].name
+    assert_equal 'attr.foo', conf.elements[3].arg
+    assert_equal 'dup', conf.elements[3]['key']
+
+    assert_equal 'tag_with', conf.elements[4].name
+    assert_equal 'attr.foo', conf.elements[4].arg
+    assert_equal 'dupdup', conf.elements[4]['key']
+  end
+  def test_spec_nested_overwrite_subsections_with_arg
+    d = create_driver NESTED_CONF
+    conf = d.instance.spec('yy')
+
+    assert_equal 'xxxxxx', conf['keyx']
+    assert_equal 'yyyyyy.yy', conf['keyy']
+    assert_equal 'z2', conf['keyz']
+    assert_equal 'c', conf['alt_key']
+
+    assert_equal 5, conf.elements.size
+
+    assert_equal 'tagx', conf.elements[0].name
+    assert_equal 'not_overwrite', conf.elements[0]['key']
+
+    assert_equal 'tag_with', conf.elements[1].name
+    assert_equal 'attr.foo', conf.elements[1].arg
+    assert_equal 'overwrite', conf.elements[1]['key']
+
+    assert_equal 'tag_with', conf.elements[2].name
+    assert_equal 'attr.foo', conf.elements[2].arg
+    assert_equal 'overwrite_dup', conf.elements[2]['key']
+
+    assert_equal 'tagx', conf.elements[3].name
+    assert_equal 'xxx', conf.elements[3]['key']
+
+    assert_equal 'tagy', conf.elements[4].name
+    assert_equal 'yyy.yy', conf.elements[4]['key']
+  end
+
+  def test_spec_nested_subsections_added
+    d = create_driver NESTED_CONF
+    conf = d.instance.spec('zz')
+
+    assert_equal 'xxxxxx', conf['keyx']
+    assert_equal 'yyyyyy.zz', conf['keyy']
+    assert_equal 'z3', conf['keyz']
+    assert_equal 'd.zz', conf['alt_key']
+
+    assert_equal 7, conf.elements.size
+
+    assert_equal 'tagz', conf.elements[0].name
+    assert_equal 1, conf.elements[0].elements.size
+    assert_equal 'tagw', conf.elements[0].elements[0].name
+    assert_equal 'www', conf.elements[0].elements[0]['key']
+
+    assert_equal 'tag_with', conf.elements[1].name
+    assert_equal 'attr.*', conf.elements[1].arg
+    assert_equal 'not_overwrite', conf.elements[1]['key']
+
+    assert_equal 'tagx', conf.elements[2].name
+    assert_equal 'xxx', conf.elements[2]['key']
+
+    assert_equal 'tagy', conf.elements[3].name
+    assert_equal 'yyy.zz', conf.elements[3]['key']
+
+    assert_equal 'tag_with', conf.elements[4].name
+    assert_equal 'attr.foo', conf.elements[4].arg
+    assert_equal 'bar', conf.elements[4]['key']
+
+    assert_equal 'tag_with', conf.elements[5].name
+    assert_equal 'attr.foo', conf.elements[5].arg
+    assert_equal 'dup', conf.elements[5]['key']
+
+    assert_equal 'tag_with', conf.elements[6].name
+    assert_equal 'attr.foo', conf.elements[6].arg
+    assert_equal 'dupdup', conf.elements[6]['key']
+  end
+
   def test_spec_hostname
     d = create_driver %[
 subtype hoge
@@ -109,7 +258,7 @@ hostname somehost.local
     assert_equal 'yyyyyy.xx', conf['keyy']
     assert_equal 'z1', conf['keyz']
     assert_equal 'b', conf['alt_key']
-    
+
     conf = d.instance.spec('yy')
     assert_equal 'xxxxxx.somehost.local', conf['keyx']
     assert_equal 'yyyyyy.yy', conf['keyy']
@@ -156,7 +305,7 @@ subtype hoge
     assert_equal 'yyyyyy.xx', conf['keyy']
     assert_equal 'z1', conf['keyz']
     assert_equal 'b', conf['alt_key']
-    
+
     conf = d.instance.spec('yy')
     assert_equal 'xxxxxx.' + hostname, conf['keyx']
     assert_equal 'yyyyyy.yy', conf['keyy']
@@ -189,7 +338,7 @@ remove_prefix test
 <template>
   key_name f
   suffix !
-  tag __TAG__  
+  tag __TAG__
 </template>
 <case foo.bar>
   prefix p1:
@@ -229,7 +378,7 @@ remove_prefix test
     assert_equal time, e[1]
     assert_equal "p4:message 1!", e[2]['f']
     assert_nil e[2]['not_started']
-    
+
     e = emits[1]
     assert_equal 'out.second', e[0]
     assert_equal time, e[1]

data/test/plugin/test_out_forest_test.rb

@@ -47,7 +47,7 @@ suffix :end
     assert_equal 'test', emits[1][2]['ttt']
   end
 
-  def test_emit
+  def test_emit1
     d = create_driver
     time = Time.parse("2012-01-02 13:14:15").to_i
     d.run do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-forest
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.2.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-24 00:00:00.000000000 Z
+date: 2013-08-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -76,7 +76,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.0
+rubygems_version: 2.0.2
 signing_key: 
 specification_version: 4
 summary: plugin to create output plugin instances per tags dynamically
@@ -85,3 +85,4 @@ test_files:
 - test/output/out_forest_test.rb
 - test/plugin/test_out_forest.rb
 - test/plugin/test_out_forest_test.rb
+has_rdoc: