fluent-plugin-forest 0.1.0

data/.gitignore

@@ -0,0 +1,25 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+# For TextMate, emacs, vim
+*.tmproj
+tmtags
+*~
+\#*
+.\#*
+*.swp
+

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-forest.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright (c) 2012- TAGOMORI Satoshi
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,103 @@
+# fluent-plugin-forest
+
+## Component
+
+### ForestOutput
+
+ForestOutput creates sub plugin instance of a output plugin dynamically per tag, from template configurations.
+In template configurations, you can write configuration lines for overall tags by <template>, and for specified tags by <case TAG_PATTERN>, and you can use \_\_TAG\_\_ placeholder at anywhere in <template> and <case>.
+
+This plugin helps you if you are writing very long configurations by copy&paste with a little little diff for many tags.
+
+You SHOULD NOT use ForestOutput for tags increasing infinitly. 
+
+## Configuration
+
+### ForestOutput
+
+If you are writing long long configurations like below:
+
+    <match service.blog>
+      type file
+      time_slice_format %Y%m%d%H
+      compress yes
+      path /var/log/blog.*.log
+    </match>
+    <match service.portal>
+      type file
+      time_slice_format %Y%m%d%H
+      compress yes
+      path /var/log/portal.*.log
+    </match>
+    <match service.news>
+      type file
+      time_slice_format %Y%m%d%H
+      compress yes
+      path /var/log/news.*.log
+    </match>
+    <match service.sns>
+      type file
+      time_slice_format %Y%m%d%H
+      compress yes
+      path /var/log/sns.*.log
+    </match>
+    # ...
+
+You can write configuration with ForestOutput like below:
+
+    <match service.*>
+      type forest
+      subtype file
+      remove_prefix service
+      <template>
+        time_slice_format %Y%m%d%H
+        compress yes
+        path /var/log/__TAG__.*.log
+      </template>
+    </match>
+
+If you want to place logs /var/archive for `service.search.**` without compression, `case` directive is useful:
+
+    <match service.*>
+      type forest
+      subtype file
+      remove_prefix service
+      <template>
+        time_slice_format %Y%m%d%H
+      </template>
+      <case search.**>
+        compress no
+        path /var/archive/__TAG__.*.log
+      </case>
+      <case *>
+        compress yes
+        path /var/log/__TAG__.*.log
+      </case>
+    </match>
+
+`case` configuration overwrites `template` configuration, so you can also write like this:
+
+    <match service.*>
+      type forest
+      subtype file
+      remove_prefix service
+      <template>
+        time_slice_format %Y%m%d%H
+        compress yes
+        path /var/log/__TAG__.*.log
+      </template>
+      <case search.**>
+        compress no
+        path /var/archive/__TAG__.*.log
+      </case>
+    </match>
+
+## TODO
+
+* consider what to do next
+* patches welcome!
+
+## Copyright
+
+Copyright:: Copyright (c) 2012- TAGOMORI Satoshi (tagomoris)
+License::   Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-forest.gemspec

@@ -0,0 +1,18 @@
+# -*- encoding: utf-8 -*-
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-forest"
+  gem.version       = "0.1.0"
+  gem.authors       = ["TAGOMORI Satoshi"]
+  gem.email         = ["tagomoris@gmail.com"]
+  gem.description   = %q{create sub-plugin dynamically per tags, with template configuration and parameters}
+  gem.summary       = %q{plugin to create output plugin instances per tags dynamically}
+  gem.homepage      = "https://github.com/tagomoris/fluent-plugin-forest"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_development_dependency "fluentd"
+  gem.add_runtime_dependency "fluentd"
+end

data/lib/fluent/plugin/out_forest.rb

@@ -0,0 +1,120 @@
+class Fluent::ForestOutput < Fluent::Output
+  Fluent::Plugin.register_output('forest', self)
+
+  config_param :subtype, :string
+  config_param :remove_prefix, :string, :default => nil
+  config_param :add_prefix, :string, :default => nil
+
+  def configure(conf)
+    super
+
+    if @remove_prefix
+      @removed_prefix_string = @remove_prefix + '.'
+      @removed_length = @removed_prefix_string.length
+    end
+    if @add_prefix
+      @added_prefix_string = @add_prefix + '.'
+    end
+
+    @mapping = {} # tag => output
+    @mutex = Mutex.new
+
+    @template = nil
+    @parameter = nil
+    @cases = []
+
+    conf.elements.each do |element|
+      element.keys.each do |k|
+        # read and throw away to supress unread configuration warning
+        element[k]
+      end
+      case element.name
+      when 'template'
+        @template = element
+      when 'case'
+        matcher = Fluent::GlobMatchPattern.new(element.arg)
+        @cases.push([matcher, element])
+      end
+    end
+
+    self
+  end
+
+  def shutdown
+    super
+    @mapping.values.each do |output|
+      output.shutdown
+    end
+  end
+
+  def parameter(tag, e)
+    pairs = {}
+    e.each do |k,v|
+      pairs[k] = v.gsub('__TAG__', tag)
+    end
+    Fluent::Config::Element.new('instance', '', pairs, [])
+  end
+
+  def spec(tag)
+    conf = Fluent::Config::Element.new('instance', '', {}, [])
+    conf = parameter(tag, @template) + conf if @template # a + b -> b.merge(a) (see: fluentd/lib/fluent/config.rb)
+    @cases.each do |m,e|
+      if m.match(tag)
+        conf = parameter(tag, e) + conf
+        break
+      end
+    end
+    conf
+  end
+
+  def plant(tag)
+    output = nil
+    begin
+      @mutex.synchronize {
+        output = @mapping[tag]
+        unless output
+          output = Fluent::Plugin.new_output(@subtype)
+          output.configure(spec(tag))
+          output.start
+          @mapping[tag] = output
+        end
+      }
+      $log.info "out_forest plants new output: #{@subtype} for tag '#{tag}'"
+    rescue Fluent::ConfigError => e
+      $log.error "failed to configure sub output #{@subtype}: #{e.message}"
+      $log.error e.backtrace.join("\n")
+      $log.error "Cannot output messages with tag '#{tag}'"
+      output = nil
+    rescue StandardError => e
+      $log.error "failed to configure/start sub output #{@subtype}: #{e.message}"
+      $log.error e.backtrace.join("\n")
+      $log.error "Cannot output messages with tag '#{tag}'"
+      output = nil
+    end
+    output
+  end
+
+  def emit(tag, es, chain)
+    if @remove_prefix and
+        ( (tag.start_with?(@removed_prefix_string) and tag.length > @removed_length) or tag == @remove_prefix)
+      tag = tag[@removed_length..-1]
+    end 
+    if @add_prefix
+      tag = if tag.length > 0
+              @added_prefix_string + tag
+            else
+              @add_prefix
+            end
+    end
+
+    output = @mapping[tag]
+    unless output
+      output = plant(tag)
+    end
+    if output
+      output.emit(tag, es, chain)
+    else
+      chain.next
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,28 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/out_forest'
+
+class Test::Unit::TestCase
+end

data/test/output/out_forest_test.rb

@@ -0,0 +1,43 @@
+class Fluent::ForestTestOutput < Fluent::Output
+  Fluent::Plugin.register_output('forest_test', self)
+
+  config_param :key_name, :string, :default => 'msg'
+  config_param :tag
+  config_param :prefix, :string, :default => ''
+  config_param :suffix, :string, :default => ''
+  config_param :tagfield, :string, :default => nil
+
+  attr_accessor :started, :stopped
+
+  def configure(conf)
+    super
+
+    if @tag == 'raise.error'
+      raise Fluent::ConfigError, "specified to raise.error"
+    end
+  end
+
+  def start
+    super
+    @started = true
+  end
+
+  def shutdown
+    super
+    @stopped = true
+  end
+
+  def emit(tag, es, chain)
+    es.each {|time, record|
+      r = record.merge({@key_name => @prefix + record[@key_name] + @suffix})
+      unless @started
+        r = r.merge({'not_started' => true})
+      end
+      if @tagfield
+        r[@tagfield] = tag
+      end
+      Fluent::Engine.emit(@tag, time, r)
+    }
+    chain.next
+  end
+end

data/test/plugin/test_out_forest.rb

@@ -0,0 +1,181 @@
+require 'helper'
+require_relative '../output/out_forest_test'
+
+class ForestOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+subtype forest_test
+remove_prefix test
+<template>
+  key_name f
+  suffix !
+  tag out.__TAG__  
+</template>
+<case foo.bar>
+  prefix p1:
+</case>
+<case foo.*>
+  prefix p2:
+</case>
+<case bar.**>
+  prefix p3:
+</case>
+<case *>
+  prefix p4:
+</case>
+  ]
+
+  def create_driver(conf = CONFIG, tag='test.default')
+    Fluent::Test::OutputTestDriver.new(Fluent::ForestOutput, tag).configure(conf)
+  end
+
+  def test_configure
+    assert_nothing_raised { d = create_driver }
+  end
+
+  def test_spec
+    d = create_driver %[
+subtype hoge
+<template>
+  keyx xxxxxx
+  keyy yyyyyy.__TAG__
+  alt_key a
+</template>
+<case xx>
+  keyz z1
+  alt_key b
+</case>
+<case yy.**>
+  keyz z2
+  alt_key c
+</case>
+<case *>
+  keyz z3
+  alt_key d.__TAG__
+</case>
+    ]
+    conf = d.instance.spec('xx')
+    assert_equal 'xxxxxx', conf['keyx']
+    assert_equal 'yyyyyy.xx', conf['keyy']
+    assert_equal 'z1', conf['keyz']
+    assert_equal 'b', conf['alt_key']
+    
+    conf = d.instance.spec('yy')
+    assert_equal 'xxxxxx', conf['keyx']
+    assert_equal 'yyyyyy.yy', conf['keyy']
+    assert_equal 'z2', conf['keyz']
+    assert_equal 'c', conf['alt_key']
+
+    conf = d.instance.spec('yy.3')
+    assert_equal 'xxxxxx', conf['keyx']
+    assert_equal 'yyyyyy.yy.3', conf['keyy']
+    assert_equal 'z2', conf['keyz']
+    assert_equal 'c', conf['alt_key']
+
+    conf = d.instance.spec('zz')
+    assert_equal 'xxxxxx', conf['keyx']
+    assert_equal 'yyyyyy.zz', conf['keyy']
+    assert_equal 'z3', conf['keyz']
+    assert_equal 'd.zz', conf['alt_key']
+  end
+
+  def test_faild_plant
+    d = create_driver
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d.tag = 'test.xxxxxx';  d.run { d.emit({'f' => "message 1"}, time) }
+    emits = d.emits
+    assert_equal 1, emits.length
+
+    d = create_driver %[
+subtype forest_test
+remove_prefix test
+<template>
+  key_name f
+  suffix !
+  tag __TAG__  
+</template>
+<case foo.bar>
+  prefix p1:
+</case>
+<case foo.*>
+  prefix p2:
+</case>
+<case bar.**>
+  prefix p3:
+</case>
+<case *>
+  prefix p4:
+</case>
+    ]
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d.tag = 'test.raise.error';  d.run { d.emit({'f' => "message 1"}, time) }
+    emits = d.emits
+    assert_equal 0, emits.length
+  end
+
+  def test_emit
+    d = create_driver
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d.tag = 'test.first';  d.run { d.emit({'f' => "message 1"}, time) }
+    d.tag = 'test.second'; d.run { d.emit({'f' => "message 2"}, time) }
+    d.tag = 'test.foo.bar'; d.run { d.emit({'f' => "message 3"}, time) }
+    d.tag = 'test.foo.baz'; d.run { d.emit({'f' => "message 4"}, time) }
+    d.tag = 'test.bar';     d.run { d.emit({'f' => "message 5"}, time) }
+    d.tag = 'test.baz';     d.run { d.emit({'f' => "message 6"}, time) }
+    d.tag = 'test.foo.bar'; d.run { d.emit({'f' => "message 7"}, time) }
+    d.tag = 'test.bar';     d.run { d.emit({'f' => "message 8"}, time) }
+
+    emits = d.emits
+
+    e = emits[0]
+    assert_equal 'out.first', e[0]
+    assert_equal time, e[1]
+    assert_equal "p4:message 1!", e[2]['f']
+    assert_nil e[2]['not_started']
+    
+    e = emits[1]
+    assert_equal 'out.second', e[0]
+    assert_equal time, e[1]
+    assert_equal "p4:message 2!", e[2]['f']
+    assert_nil e[2]['not_started']
+
+    e = emits[2]
+    assert_equal 'out.foo.bar', e[0]
+    assert_equal time, e[1]
+    assert_equal "p1:message 3!", e[2]['f']
+    assert_nil e[2]['not_started']
+
+    e = emits[3]
+    assert_equal 'out.foo.baz', e[0]
+    assert_equal time, e[1]
+    assert_equal "p2:message 4!", e[2]['f']
+    assert_nil e[2]['not_started']
+
+    e = emits[4]
+    assert_equal 'out.bar', e[0]
+    assert_equal time, e[1]
+    assert_equal "p3:message 5!", e[2]['f']
+    assert_nil e[2]['not_started']
+
+    e = emits[5]
+    assert_equal 'out.baz', e[0]
+    assert_equal time, e[1]
+    assert_equal "p4:message 6!", e[2]['f']
+    assert_nil e[2]['not_started']
+
+    e = emits[6]
+    assert_equal 'out.foo.bar', e[0]
+    assert_equal time, e[1]
+    assert_equal "p1:message 7!", e[2]['f']
+    assert_nil e[2]['not_started']
+
+    e = emits[7]
+    assert_equal 'out.bar', e[0]
+    assert_equal time, e[1]
+    assert_equal "p3:message 8!", e[2]['f']
+    assert_nil e[2]['not_started']
+  end
+end

data/test/plugin/test_out_forest_test.rb

@@ -0,0 +1,103 @@
+require 'helper'
+require_relative '../output/out_forest_test'
+
+class ForestTestOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+tag foo.bar
+tagfield ttt
+prefix fixed:
+suffix :end
+  ]
+
+  def create_driver(conf = CONFIG, tag='test')
+    Fluent::Test::OutputTestDriver.new(Fluent::ForestTestOutput, tag).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal 'foo.bar', d.instance.tag
+    assert_equal 'fixed:', d.instance.prefix
+    assert_equal ':end', d.instance.suffix
+  end
+
+  def test_emit
+    d = create_driver
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d.run do
+      d.emit({'msg' => 'xyz 123', 'alt' => 'aaa bbb ccc'}, time)
+      d.emit({'msg' => 'xyz 456', 'alt' => 'aaa bbb ccc ddd'}, time)
+    end
+    emits = d.emits
+    assert_equal 2, emits.length
+
+    assert_equal 'foo.bar', emits[0][0]
+    assert_equal time, emits[0][1]
+    assert_equal 'fixed:xyz 123:end', emits[0][2]['msg']
+    assert_equal 'aaa bbb ccc', emits[0][2]['alt']
+    assert_equal 'test', emits[0][2]['ttt']
+
+    assert_equal 'foo.bar', emits[1][0]
+    assert_equal time, emits[1][1]
+    assert_equal 'fixed:xyz 456:end', emits[1][2]['msg']
+    assert_equal 'aaa bbb ccc ddd', emits[1][2]['alt']
+    assert_equal 'test', emits[1][2]['ttt']
+  end
+
+  def test_emit
+    d = create_driver
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d.run do
+      d.emit({'msg' => 'xyz 123', 'alt' => 'aaa bbb ccc'}, time)
+      d.tag = 'test2'
+      d.emit({'msg' => 'xyz 456', 'alt' => 'aaa bbb ccc ddd'}, time)
+    end
+    emits = d.emits
+    assert_equal 2, emits.length
+
+    assert_equal 'foo.bar', emits[0][0]
+    assert_equal time, emits[0][1]
+    assert_equal 'fixed:xyz 123:end', emits[0][2]['msg']
+    assert_equal 'aaa bbb ccc', emits[0][2]['alt']
+    assert_equal 'test', emits[0][2]['ttt']
+    assert_nil emits[0][2]['not_started']
+
+    assert_equal 'foo.bar', emits[1][0]
+    assert_equal time, emits[1][1]
+    assert_equal 'fixed:xyz 456:end', emits[1][2]['msg']
+    assert_equal 'aaa bbb ccc ddd', emits[1][2]['alt']
+    assert_equal 'test2', emits[1][2]['ttt']
+    assert_nil emits[1][2]['not_started']
+  end
+
+  def test_emit2
+    d = create_driver
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d.run do
+      d.emit({'msg' => 'xyz 123', 'alt' => 'aaa bbb ccc'}, time)
+    end
+    d.tag = 'test2'
+    d.run do
+      d.emit({'msg' => 'xyz 456', 'alt' => 'aaa bbb ccc ddd'}, time)
+    end
+    emits = d.emits
+    assert_equal 2, emits.length
+
+    assert_equal 'foo.bar', emits[0][0]
+    assert_equal time, emits[0][1]
+    assert_equal 'fixed:xyz 123:end', emits[0][2]['msg']
+    assert_equal 'aaa bbb ccc', emits[0][2]['alt']
+    assert_equal 'test', emits[0][2]['ttt']
+    assert_nil emits[0][2]['not_started']
+
+    assert_equal 'foo.bar', emits[1][0]
+    assert_equal time, emits[1][1]
+    assert_equal 'fixed:xyz 456:end', emits[1][2]['msg']
+    assert_equal 'aaa bbb ccc ddd', emits[1][2]['alt']
+    assert_equal 'test2', emits[1][2]['ttt']
+    assert_nil emits[1][2]['not_started']
+  end
+end

metadata

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-forest
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- TAGOMORI Satoshi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-04-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: create sub-plugin dynamically per tags, with template configuration and
+  parameters
+email:
+- tagomoris@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-forest.gemspec
+- lib/fluent/plugin/out_forest.rb
+- test/helper.rb
+- test/output/out_forest_test.rb
+- test/plugin/test_out_forest.rb
+- test/plugin/test_out_forest_test.rb
+homepage: https://github.com/tagomoris/fluent-plugin-forest
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.21
+signing_key: 
+specification_version: 3
+summary: plugin to create output plugin instances per tags dynamically
+test_files:
+- test/helper.rb
+- test/output/out_forest_test.rb
+- test/plugin/test_out_forest.rb
+- test/plugin/test_out_forest_test.rb