fluent-plugin-config-expander 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-config-expander.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright (c) 2012- TAGOMORI Satoshi
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,73 @@
+# fluent-plugin-config-expander
+
+## ConfigExpanderInput, ConfigExpanderOutput
+
+ConfigExpanderInput and ConfigExpanderOutput plugins provide simple configuration template to write items repeatedly.
+In <config> section, you can write actual configuration for actual input/output plugin, with special directives for loop controls.
+
+## Configuration
+
+For both of input and output (for <source> and <match>), you can use 'config_expander' and its 'for' directive like below:
+
+    <match example.**>
+      type config_expander
+      <config>
+        type forward
+        flush_interval 30s
+        <for x in 01 02 03>
+          <server>
+            host worker__x__.local
+            port 24224
+          </server>
+        </for>
+      </config>
+    </match>
+
+Configuration above is equal to below:
+
+    <match example.**>
+      type forward
+      flush_interval 30s
+      <server>
+        host worker01.local
+        port 24224
+      </server>
+      <server>
+        host worker02.local
+        port 24224
+      </server>
+      <server>
+        host worker03.local
+        port 24224
+      </server>
+    </match>
+
+Nested 'for' directive is valid:
+
+    <match example.**>
+      type config_expander
+      <config>
+        type forward
+        flush_interval 30s
+        <for x in 01 02 03>
+          <for p in 24221 24222 24223 24224
+            <server>
+              host worker__x__.local
+              port __p__
+            </server>
+          </for>
+        </for>
+      </config>
+    </match>
+
+## TODO
+
+* more tests
+* patches welcome!
+
+## Copyright
+
+* Copyright (c) 2012- TAGOMORI Satoshi (tagomoris)
+* License
+  * Apache License, Version 2.0
+

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-config-expander.gemspec

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-config-expander"
+  gem.version       = "0.0.1"
+  gem.authors       = ["TAGOMORI Satoshi"]
+  gem.email         = ["tagomoris@gmail.com"]
+  gem.description   = %q{This plugin provides directives for loop extraction}
+  gem.summary       = %q{Fluentd plugin to serve some DSL directives in configuration}
+  gem.homepage      = "https://github.com/tagomoris/fluent-plugin-config-expander"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "fluentd"
+  gem.add_runtime_dependency "fluentd"
+end

data/lib/fluent/plugin/expander.rb

@@ -0,0 +1,31 @@
+require 'fluent/config'
+
+module Fluent::Config::Expander
+  def self.replace(str, mapping)
+    mapping.reduce(str){|r,p| str.gsub(p[0], p[1])}
+  end
+
+  def self.expand(element, mapping)
+    name = replace(element.name, mapping)
+    arg = replace(element.arg, mapping)
+    attrs = element.reduce({}){|r,p| r[replace(p.first, mapping)] = replace(p.last, mapping); r}
+    elements = []
+    element.elements.each do |e|
+      if e.name == 'for'
+        unless e.arg =~ /^([a-zA-Z0-9]+) in (.+)$/
+          raise Fluent::ConfigError, "invalid for tag syntax: <for NAME in ARG1 ARG2 ...>"
+        end
+        vname = '__' + $1 + '__'
+        vargs = $2.split(/ +/).select{|v| v.size > 0}
+        vargs.each do |v|
+          expanded = expand(e, mapping.merge({vname => v}))
+          attrs.update(expanded)
+          elements += expanded.elements.map{|xe| expand(xe, mapping)}
+        end
+      else
+        elements.push(expand(e, mapping))
+      end
+    end
+    Fluent::Config::Element.new(name, arg, attrs, elements, [])
+  end
+end

data/lib/fluent/plugin/in_config_expander.rb

@@ -0,0 +1,34 @@
+require_relative 'expander'
+
+class Fluent::ConfigExpanderInput < Fluent::Input
+  Fluent::Plugin.register_input('config_expander', self)
+
+  attr_accessor :plugin
+
+  def expand_config(conf)
+    ex = Fluent::Config::Expander.expand(conf, {})
+    ex.name = ''
+    ex.arg = ''
+    ex
+  end
+
+  def configure(conf)
+    super
+    
+    configs = conf.elements.select{|e| e.name == 'config'}
+    if configs.size != 1
+      raise Fluent::ConfigError, "config_expander needs just one <config> ... </config> section"
+    end
+    ex = expand_config(configs.first)
+    @plugin = Fluent::Plugin.new_input(ex['type'])
+    @plugin.configure(ex)
+  end
+
+  def start
+    @plugin.start
+  end
+
+  def shutdown
+    @plugin.shutdown
+  end
+end

data/lib/fluent/plugin/out_config_expander.rb

@@ -0,0 +1,38 @@
+require_relative 'expander'
+
+class Fluent::ConfigExpanderOutput < Fluent::Output
+  Fluent::Plugin.register_output('config_expander', self)
+
+  attr_accessor :plugin
+
+  def expand_config(conf)
+    ex = Fluent::Config::Expander.expand(conf, {})
+    ex.name = ''
+    ex.arg = ''
+    ex
+  end
+
+  def configure(conf)
+    super
+
+    configs = conf.elements.select{|e| e.name == 'config'}
+    if configs.size != 1
+      raise Fluent::ConfigError, "config_expander needs just one <config> ... </config> section"
+    end
+    ex = expand_config(configs.first)
+    @plugin = Fluent::Plugin.new_output(ex['type'])
+    @plugin.configure(ex)
+  end
+
+  def start
+    @plugin.start
+  end
+
+  def shutdown
+    @plugin.shutdown
+  end
+
+  def emit(tag, es, chain)
+    @plugin.emit(tag, es, chain)
+  end
+end

data/test/helper.rb

@@ -0,0 +1,29 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/in_config_expander'
+require 'fluent/plugin/out_config_expander'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_expander.rb

@@ -0,0 +1,132 @@
+require 'helper'
+
+class ConfigExpanderTest < Test::Unit::TestCase
+  def setup
+    @m = Fluent::Config::Expander
+  end
+
+  def test_replace
+    assert_equal nil, @m.replace(nil, {})
+    assert_equal "", @m.replace("", {})
+    assert_equal "", @m.replace("", {'foo' => 'bar'})
+    assert_equal "barbar", @m.replace("foobar", {'foo' => 'bar'})
+    assert_equal "foofoo", @m.replace("foobar", {'bar' => 'foo'})
+    assert_equal "foobar", @m.replace("foobar", {'hoge' => 'moge'})
+    assert_equal "xxbar", @m.replace("foofoobar", {'foo' => 'x'})
+  end
+
+  def test_expand
+    e1 = Fluent::Config::Element.new('config', '', {'attr_first'=>'value_first', 'attr_second'=>'__NUM__'}, [], [])
+    assert_equal "<config>\n  attr_first value_first\n  attr_second __NUM__\n</config>\n", e1.to_s
+    assert_equal "<config>\n  attr_first value_first\n  attr_second 24224\n</config>\n", @m.expand(e1, {'__NUM__' => '24224'}).to_s
+
+    e2 = Fluent::Config::Element.new('server', '', {'host'=> 'test', 'port'=>'__PORT__', 'ext'=>'__PORT__', 'num__PORT__'=>'1'}, [], [])
+    assert_equal "<server>\n  host test\n  port __PORT__\n  ext __PORT__\n  num__PORT__ 1\n</server>\n", e2.to_s
+    assert_equal "<server>\n  host test\n  port 24224\n  ext 24224\n  num24224 1\n</server>\n", @m.expand(e2, {'__PORT__' => '24224'}).to_s
+
+    e3 = Fluent::Config::Element.new('config', 'for __PORT__', {'attr_first'=>'value_first', 'attr_second'=>'__PORT__'}, [e2], [])
+    exconf1 = <<EOL
+<config for 24224>
+  attr_first value_first
+  attr_second 24224
+  <server>
+    host test
+    port 24224
+    ext 24224
+    num24224 1
+  </server>
+</config>
+EOL
+    assert_equal exconf1, @m.expand(e3, {'__PORT__' => '24224'}).to_s
+
+    nonexconf2 = <<EOL
+<config>
+  <for portnum in 24221 24222 24223 24224>
+    <server>
+      host node__nodenum__.local
+      port __portnum__
+    </server>
+  </for>
+</config>
+EOL
+    conf2 = Fluent::Config.parse(nonexconf2, 'hoge').elements.first
+    assert_equal nonexconf2, conf2.to_s
+    exconf2 = <<EOL
+<config>
+  <server>
+    host node__nodenum__.local
+    port 24221
+  </server>
+  <server>
+    host node__nodenum__.local
+    port 24222
+  </server>
+  <server>
+    host node__nodenum__.local
+    port 24223
+  </server>
+  <server>
+    host node__nodenum__.local
+    port 24224
+  </server>
+</config>
+EOL
+    assert_equal exconf2, @m.expand(conf2, {}).to_s
+
+    nonexconf3 = <<EOL
+<config>
+  type forward
+  flush_interval 1s
+  <for nodenum in 01 02 >
+    <for portnum in 24221 24222 24223 24224>
+      <server>
+        host node__nodenum__.local
+        port __portnum__
+      </server>
+    </for>
+  </for>
+</config>
+EOL
+    conf3 = Fluent::Config.parse(nonexconf3, 'hoge').elements.first
+    assert_equal nonexconf3, conf3.to_s
+    exconf3 = <<EOL
+<config>
+  type forward
+  flush_interval 1s
+  <server>
+    host node01.local
+    port 24221
+  </server>
+  <server>
+    host node01.local
+    port 24222
+  </server>
+  <server>
+    host node01.local
+    port 24223
+  </server>
+  <server>
+    host node01.local
+    port 24224
+  </server>
+  <server>
+    host node02.local
+    port 24221
+  </server>
+  <server>
+    host node02.local
+    port 24222
+  </server>
+  <server>
+    host node02.local
+    port 24223
+  </server>
+  <server>
+    host node02.local
+    port 24224
+  </server>
+</config>
+EOL
+    assert_equal exconf3, @m.expand(conf3, {}).to_s
+  end
+end

data/test/plugin/test_in_config_expander.rb

@@ -0,0 +1,42 @@
+require 'helper'
+require_relative '../plugins'
+
+class ConfigExpanderInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+type config_expander
+<config>
+  type config_expander_test
+  tag foobar
+  <for x in 1 2 3>
+    <node>
+      attr1 __x__
+    </node>
+  </for>
+</config>
+]
+  def create_driver(conf=CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::ConfigExpanderInput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal 'foobar', d.instance.plugin.tag
+    assert_equal 3, d.instance.plugin.nodes.size
+    assert_equal ['1','2','3'], d.instance.plugin.nodes.map{|n| n['attr1']}.sort
+
+    assert_equal false, d.instance.plugin.started
+    assert_equal false, d.instance.plugin.stopped
+
+    d.instance.start()
+    assert_equal true, d.instance.plugin.started
+    assert_equal false, d.instance.plugin.stopped
+    
+    d.instance.shutdown()
+    assert_equal true, d.instance.plugin.stopped
+  end
+
+end

data/test/plugin/test_out_config_expander.rb

@@ -0,0 +1,66 @@
+require 'helper'
+require_relative '../plugins'
+
+class ConfigExpanderOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+type config_expander
+<config>
+  type config_expander_test
+  tag foobar
+  <for x in 1 2 3>
+    <node>
+      attr1 __x__
+    </node>
+  </for>
+</config>
+]
+  def create_driver(conf=CONFIG, tag='test.default')
+    Fluent::Test::OutputTestDriver.new(Fluent::ConfigExpanderOutput, tag).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal 'foobar', d.instance.plugin.tag
+    assert_equal 3, d.instance.plugin.nodes.size
+    assert_equal ['1','2','3'], d.instance.plugin.nodes.map{|n| n['attr1']}.sort
+
+    assert_equal false, d.instance.plugin.started
+    assert_equal false, d.instance.plugin.stopped
+
+    d.instance.start()
+    assert_equal true, d.instance.plugin.started
+    assert_equal false, d.instance.plugin.stopped
+    
+    d.instance.shutdown()
+    assert_equal true, d.instance.plugin.stopped
+  end
+
+  def test_emit
+    d = create_driver
+    d.run do
+      d.emit({'field' => 'value1'})
+      d.emit({'field' => 'value2'})
+      d.emit({'field' => 'value3'})
+    end
+
+    emits = d.emits
+    assert_equal 3, emits.size
+
+    assert_equal 'foobar', emits[0][0]
+    assert_equal 'value1', emits[0][2]['field']
+    assert_equal 'expander', emits[0][2]['over']
+
+    assert_equal 'foobar', emits[1][0]
+    assert_equal 'value2', emits[1][2]['field']
+    assert_equal 'expander', emits[1][2]['over']
+
+    assert_equal 'foobar', emits[2][0]
+    assert_equal 'value3', emits[2][2]['field']
+    assert_equal 'expander', emits[2][2]['over']
+  end
+
+end

data/test/plugins.rb

@@ -0,0 +1,58 @@
+class Fluent::ConfigExpanderTestInput < Fluent::Input
+  Fluent::Plugin.register_input('config_expander_test', self)
+
+  config_param :tag, :string
+  attr_accessor :nodes
+  attr_accessor :started, :stopped
+
+  def configure(conf)
+    super
+    @nodes = []
+    conf.elements.each do |e|
+      next if e.name != 'node'
+      @nodes << {}.merge(e)
+    end
+    @started = @stopped = false
+  end
+  def start
+    super
+    @started = true
+  end
+  def shutdown
+    super
+    @stopped = true
+  end
+end
+
+class Fluent::ConfigExpanderTestOutput < Fluent::Output
+  Fluent::Plugin.register_output('config_expander_test', self)
+
+  config_param :tag, :string
+  attr_accessor :nodes
+  attr_accessor :started, :stopped
+
+  def configure(conf)
+    super
+    @nodes = []
+    conf.elements.each do |e|
+      next if e.name != 'node'
+      @nodes << {}.merge(e)
+    end
+    @started = @stopped = false
+  end
+  def start
+    super
+    @started = true
+  end
+  def shutdown
+    super
+    @stopped = true
+  end
+
+  def emit(tag, es, chain)
+    es.each do |time, record|
+      Fluent::Engine.emit(@tag, time, record.merge({'over' => 'expander'}))
+    end
+    chain.next
+  end
+end

metadata

@@ -0,0 +1,113 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-config-expander
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- TAGOMORI Satoshi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-08-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This plugin provides directives for loop extraction
+email:
+- tagomoris@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-config-expander.gemspec
+- lib/fluent/plugin/config_ext.rb
+- lib/fluent/plugin/expander.rb
+- lib/fluent/plugin/in_config_expander.rb
+- lib/fluent/plugin/out_config_expander.rb
+- test/helper.rb
+- test/plugin/test_expander.rb
+- test/plugin/test_in_config_expander.rb
+- test/plugin/test_out_config_expander.rb
+- test/plugins.rb
+homepage: https://github.com/tagomoris/fluent-plugin-config-expander
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.21
+signing_key: 
+specification_version: 3
+summary: Fluentd plugin to serve some DSL directives in configuration
+test_files:
+- test/helper.rb
+- test/plugin/test_expander.rb
+- test/plugin/test_in_config_expander.rb
+- test/plugin/test_out_config_expander.rb
+- test/plugins.rb