fluent-plugin-config-expander 0.2.1 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6b7eccf9f3a0d3b56837e0fec4a558463d5e807c
-  data.tar.gz: 371c9510751723c5f95586f612a5bbb19c5a1fac
+  metadata.gz: f027aae70a497c28fe302e764006f38d4a63b9c9
+  data.tar.gz: a921f29a8cf2689cd975d69bbb728112dac7b97b
 SHA512:
-  metadata.gz: 6955d0fa408b21455f66d265b3177bd51979ce9125488db89b529933298daaa2c528a6530c8bbb0fc4341560baa484fcb44db3534d171f90804b33b48de99259
-  data.tar.gz: eb0f0b181ed65b6f6111ad01460985b8a146c62080517536938ec5e5a07325d105af0aee9be6d4f902ff0be58f6cabe69673b740920a5f246845f071a1c39a3a
+  metadata.gz: 84b3bd584a5378fbaa9a48e2364d6a1b280d93ad248c68c127fd4b9151781526d5a2bfebd8121e87333d0cc80a1e32fbaf09c74f9e213d81b722beaa631b0174
+  data.tar.gz: 7d5adde5590ab2c3b18c6221390cb281585d29434daf8b06ca899151f0825268700d8df4e6b58a96ef09bd5ab1f65a8dc4f0dfc1f259e3661921ed9f2dc9d821

data/.travis.yml

@@ -1,9 +1,9 @@
 language: ruby
 sudo: false
 rvm:
-  - 2.0.0
   - 2.1
   - 2.2
-  - 2.3.0
+  - 2.3.3
+  - 2.4.0
 before_install:
   - gem update bundler

data/README.md

@@ -4,15 +4,15 @@ This is a plugin for [Fluentd](http://fluentd.org).
 
 ## ConfigExpanderInput, ConfigExpanderOutput
 
-ConfigExpanderInput and ConfigExpanderOutput plugins provide simple configuration template to write items repeatedly.
-In <config> section, you can write actual configuration for actual input/output plugin, with special directives for loop controls.
+ConfigExpanderInput, ConfigExpanderFilter and ConfigExpanderOutput plugins provide simple configuration template to write items repeatedly.
+In <config> section, you can write actual configuration for actual input/filter/output plugin, with special directives for loop controls.
 
 And also supports built-in placeholders below:
  * hostname (ex: \_\_HOSTNAME\_\_, \_\_hostname\_\_, ${hostname}, ${HOSTNAME})
 
 ## Configuration
 
-For both of input and output (for <source> and <match>), you can use 'config_expander' and its 'for' directive like below:
+For all of input, filter and output (for <source>, <filter> and <match>), you can use 'config_expander' and its 'for' directive like below:
 
     <match example.**>
       @type config_expander
@@ -87,6 +87,7 @@ Set hostname into tag in 'tail' input plugin:
       @type config_expander
       <config>
         @type tail
+        @label @access_events
         format /..../
         path /var/log/access.log
         tag access.log.${hostname}

data/fluent-plugin-config-expander.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-config-expander"
-  gem.version       = "0.2.1"
+  gem.version       = "1.0.0"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.description   = %q{This plugin provides directives for loop extraction}
@@ -15,7 +15,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.add_runtime_dependency "fluentd", "< 0.14.0"
+  gem.add_runtime_dependency "fluentd", ">= 0.14.0"
   gem.add_development_dependency "rake"
   gem.add_development_dependency "test-unit", "~> 3.1"
 end

data/lib/fluent/plugin/expander.rb

@@ -2,13 +2,13 @@ require 'fluent/config'
 
 module Fluent::Config::Expander
   def self.replace(str, mapping)
-    mapping.reduce(str){|r,p| r.gsub(p[0], p[1])}
+    mapping.reduce(str){|r, pair| r.gsub(pair[0], pair[1])}
   end
 
   def self.expand(element, mapping)
     name = replace(element.name, mapping)
     arg = replace(element.arg, mapping)
-    attrs = element.reduce({}){|r,p| r[replace(p.first, mapping)] = replace(p.last, mapping); r}
+    attrs = element.reduce({}){|r, pair| r[replace(pair[0], mapping)] = replace(pair[1], mapping); r}
     elements = []
     element.elements.each do |e|
       if e.name == 'for'

data/lib/fluent/plugin/filter_config_expander.rb

@@ -0,0 +1,50 @@
+require 'fluent/plugin/filter'
+
+require_relative 'expander'
+require 'forwardable'
+require 'socket'
+
+class Fluent::Plugin::ConfigExpanderFilter < Fluent::Plugin::Filter
+  Fluent::Plugin.register_input('config_expander', self)
+
+  config_param :hostname, :string, default: Socket.gethostname
+  config_section :config, multi: false, required: true, param_name: :config_config do
+    # to raise configuration error for missing section
+  end
+
+  def mark_used(conf)
+    conf.keys.each {|key| conf[key] } # to suppress unread configuration warning
+    conf.elements.each{|e| mark_used(e)}
+  end
+
+  def builtin_mapping
+    {'__hostname__' => @hostname, '__HOSTNAME__' => @hostname, '${hostname}' => @hostname, '${HOSTNAME}' => @hostname}
+  end
+
+  def expand_config(conf)
+    ex = Fluent::Config::Expander.expand(conf, builtin_mapping())
+    ex.name = 'filter' # name/arg will be ignored by Plugin#configure, but anyway
+    ex.arg = conf.arg
+    ex
+  end
+
+  def configure(conf)
+    super
+
+    ex = expand_config(@config_config.corresponding_config_element)
+    type = ex['@type']
+    @plugin = Fluent::Plugin.new_input(type)
+    @plugin.context_router = self.event_emitter_router(conf['@label'])
+    @plugin.configure(ex)
+    mark_used(@config_config.corresponding_config_element)
+
+    self.extend SingleForwardable
+    override_methods = self.methods + @plugin.methods - SingleForwardable.instance_methods - Object.instance_methods
+    override_methods.uniq!
+    def_delegators(:@plugin, *override_methods)
+  end
+
+  def method_missing(name, *args, &block)
+    @plugin.__send__(name, *args, &block)
+  end
+end

data/lib/fluent/plugin/in_config_expander.rb

@@ -1,16 +1,17 @@
+require 'fluent/plugin/input'
+
 require_relative 'expander'
+require 'forwardable'
+require 'socket'
 
-class Fluent::ConfigExpanderInput < Fluent::Input
+class Fluent::Plugin::ConfigExpanderInput < Fluent::Plugin::Input
   Fluent::Plugin.register_input('config_expander', self)
 
-  # Define `log` method for v0.10.42 or earlier
-  unless method_defined?(:log)
-    define_method("log") { $log }
+  config_param :hostname, :string, default: Socket.gethostname
+  config_section :config, multi: false, required: true, param_name: :config_config do
+    # to raise configuration error for missing section
   end
 
-  config_param :hostname, :string, :default => `hostname`.chomp
-  attr_accessor :plugin
-
   def mark_used(conf)
     conf.keys.each {|key| conf[key] } # to suppress unread configuration warning
     conf.elements.each{|e| mark_used(e)}
@@ -22,31 +23,28 @@ class Fluent::ConfigExpanderInput < Fluent::Input
 
   def expand_config(conf)
     ex = Fluent::Config::Expander.expand(conf, builtin_mapping())
-    ex.name = ''
+    ex.name = 'source' # name/arg will be ignored by Plugin#configure, but anyway
     ex.arg = ''
     ex
   end
 
   def configure(conf)
     super
-    
-    configs = conf.elements.select{|e| e.name == 'config'}
-    if configs.size != 1
-      raise Fluent::ConfigError, "config_expander needs just one <config> ... </config> section"
-    end
-    ex = expand_config(configs.first)
-    type = ex['@type'] || ex['type']
+
+    ex = expand_config(@config_config.corresponding_config_element)
+    type = ex['@type']
     @plugin = Fluent::Plugin.new_input(type)
+    @plugin.context_router = self.event_emitter_router(conf['@label'])
     @plugin.configure(ex)
+    mark_used(@config_config.corresponding_config_element)
 
-    mark_used(configs.first)
-  end
-
-  def start
-    @plugin.start
+    self.extend SingleForwardable
+    override_methods = self.methods + @plugin.methods - SingleForwardable.instance_methods - Object.instance_methods
+    override_methods.uniq!
+    def_delegators(:@plugin, *override_methods)
   end
 
-  def shutdown
-    @plugin.shutdown
+  def method_missing(name, *args, &block)
+    @plugin.__send__(name, *args, &block)
   end
 end

data/lib/fluent/plugin/out_config_expander.rb

@@ -1,15 +1,18 @@
+require 'fluent/plugin/bare_output'
+
 require_relative 'expander'
+require 'forwardable'
+require 'socket'
 
-class Fluent::ConfigExpanderOutput < Fluent::MultiOutput
+class Fluent::Plugin::ConfigExpanderOutput < Fluent::Plugin::BareOutput
   Fluent::Plugin.register_output('config_expander', self)
 
-  # Define `log` method for v0.10.42 or earlier
-  unless method_defined?(:log)
-    define_method("log") { $log }
-  end
+  helpers :event_emitter
 
-  config_param :hostname, :string, :default => `hostname`.chomp
-  attr_accessor :plugin
+  config_param :hostname, :string, default: Socket.gethostname
+  config_section :config, multi: false, required: true, param_name: :config_config do
+    # to raise configuration error for missing section
+  end
 
   def mark_used(conf)
     conf.keys.each {|key| conf[key] } # to suppress unread configuration warning
@@ -22,39 +25,28 @@ class Fluent::ConfigExpanderOutput < Fluent::MultiOutput
 
   def expand_config(conf)
     ex = Fluent::Config::Expander.expand(conf, builtin_mapping())
-    ex.name = ''
-    ex.arg = ''
+    ex.name = 'match' # name/arg will be ignored by Plugin#configure, but anyway
+    ex.arg = conf.arg
     ex
   end
 
-  attr_reader :outputs
-
   def configure(conf)
     super
 
-    configs = conf.elements.select{|e| e.name == 'config'}
-    if configs.size != 1
-      raise Fluent::ConfigError, "config_expander needs just one <config> ... </config> section"
-    end
-    ex = expand_config(configs.first)
-    type = ex['@type'] || ex['type']
+    ex = expand_config(@config_config.corresponding_config_element)
+    type = ex['@type']
     @plugin = Fluent::Plugin.new_output(type)
+    @plugin.context_router = self.event_emitter_router(conf['@label'])
     @plugin.configure(ex)
+    mark_used(@config_config.corresponding_config_element)
 
-    @outputs = [@plugin]
-
-    mark_used(configs.first)
-  end
-
-  def start
-    @plugin.start
-  end
-
-  def shutdown
-    @plugin.shutdown
+    self.extend SingleForwardable
+    override_methods = self.methods + @plugin.methods - SingleForwardable.instance_methods - Object.instance_methods
+    override_methods.uniq!
+    def_delegators(:@plugin, *override_methods)
   end
 
-  def emit(tag, es, chain)
-    @plugin.emit(tag, es, chain)
+  def method_missing(name, *args, &block)
+    @plugin.__send__(name, *args, &block)
   end
 end

data/test/helper.rb

@@ -12,7 +12,6 @@ require 'test/unit'
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 require 'fluent/test'
-require 'fluent/input'
 unless ENV.has_key?('VERBOSE')
   nulllogger = Object.new
   nulllogger.instance_eval {|obj|
@@ -23,7 +22,13 @@ unless ENV.has_key?('VERBOSE')
   $log = nulllogger
 end
 
+require 'fluent/test/driver/input'
+require 'fluent/test/driver/filter'
+require 'fluent/test/driver/base_owner' # for output (BareOutput)
+require 'fluent/test/driver/event_feeder'
+
 require 'fluent/plugin/in_config_expander'
+require 'fluent/plugin/filter_config_expander'
 require 'fluent/plugin/out_config_expander'
 
 class Test::Unit::TestCase

data/test/plugin/test_in_config_expander.rb

@@ -20,24 +20,24 @@ type config_expander
 ]
 
   def create_driver(conf=CONFIG)
-    Fluent::Test::InputTestDriver.new(Fluent::ConfigExpanderInput).configure(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::ConfigExpanderInput).configure(conf)
   end
 
   def test_configure
     d = create_driver
-    assert_equal 'foobar', d.instance.plugin.tag
-    assert_equal 3, d.instance.plugin.nodes.size
-    assert_equal ['1','2','3'], d.instance.plugin.nodes.map{|n| n['attr1']}.sort
+    assert_equal 'foobar', d.instance.tag
+    assert_equal 3, d.instance.nodes.size
+    assert_equal ['1','2','3'], d.instance.nodes.map{|n| n.attr1 }.sort
 
-    assert_equal false, d.instance.plugin.started
-    assert_equal false, d.instance.plugin.stopped
+    assert_equal false, d.instance.started
+    assert_equal false, d.instance.stopped
 
     d.instance.start()
-    assert_equal true, d.instance.plugin.started
-    assert_equal false, d.instance.plugin.stopped
-    
+    assert_equal true, d.instance.started
+    assert_equal false, d.instance.stopped
+
     d.instance.shutdown()
-    assert_equal true, d.instance.plugin.stopped
+    assert_equal true, d.instance.stopped
   end
 
   CONFIG2 = %[
@@ -52,14 +52,14 @@ hostname testing.node.local
     attr3 __hostname__
     attr4 __HOSTNAME__
   </node>
+</config>
 ]
   def test_configure_hostname
     d = create_driver CONFIG2
-    assert_equal 1, d.instance.plugin.nodes.size
-    assert_equal 'testing.node.local', d.instance.plugin.nodes.first['attr1']
-    assert_equal 'testing.node.local', d.instance.plugin.nodes.first['attr2']
-    assert_equal 'testing.node.local', d.instance.plugin.nodes.first['attr3']
-    assert_equal 'testing.node.local', d.instance.plugin.nodes.first['attr4']
+    assert_equal 1, d.instance.nodes.size
+    assert_equal 'testing.node.local', d.instance.nodes.first.attr1
+    assert_equal 'testing.node.local', d.instance.nodes.first.attr2
+    assert_equal 'testing.node.local', d.instance.nodes.first.attr3
+    assert_equal 'testing.node.local', d.instance.nodes.first.attr4
   end
-
 end

data/test/plugin/test_out_config_expander.rb

@@ -18,25 +18,28 @@ type config_expander
   </for>
 </config>
 ]
-  def create_driver(conf=CONFIG, tag='test.default')
-    Fluent::Test::OutputTestDriver.new(Fluent::ConfigExpanderOutput, tag).configure(conf)
+
+  def create_driver(conf=CONFIG)
+    d = Fluent::Test::Driver::BaseOwner.new(Fluent::Plugin::ConfigExpanderOutput)
+    d.extend Fluent::Test::Driver::EventFeeder
+    d.configure(conf)
   end
 
   def test_configure
     d = create_driver
-    assert_equal 'foobar', d.instance.plugin.tag
-    assert_equal 3, d.instance.plugin.nodes.size
-    assert_equal ['1','2','3'], d.instance.plugin.nodes.map{|n| n['attr1']}.sort
+    assert_equal 'foobar', d.instance.tag
+    assert_equal 3, d.instance.nodes.size
+    assert_equal ['1','2','3'], d.instance.nodes.map{|n| n.attr1 }.sort
 
-    assert_equal false, d.instance.plugin.started
-    assert_equal false, d.instance.plugin.stopped
+    assert_equal false, d.instance.started
+    assert_equal false, d.instance.stopped
 
     d.instance.start()
-    assert_equal true, d.instance.plugin.started
-    assert_equal false, d.instance.plugin.stopped
-    
+    assert_equal true, d.instance.started
+    assert_equal false, d.instance.stopped
+
     d.instance.shutdown()
-    assert_equal true, d.instance.plugin.stopped
+    assert_equal true, d.instance.stopped
   end
 
   CONFIG2 = %[
@@ -51,38 +54,38 @@ hostname testing.node.local
     attr3 __hostname__
     attr4 __HOSTNAME__
   </node>
+</config>
 ]
   def test_configure_hostname
     d = create_driver CONFIG2
-    assert_equal 1, d.instance.plugin.nodes.size
-    assert_equal 'testing.node.local', d.instance.plugin.nodes.first['attr1']
-    assert_equal 'testing.node.local', d.instance.plugin.nodes.first['attr2']
-    assert_equal 'testing.node.local', d.instance.plugin.nodes.first['attr3']
-    assert_equal 'testing.node.local', d.instance.plugin.nodes.first['attr4']
+    assert_equal 1, d.instance.nodes.size
+    assert_equal 'testing.node.local', d.instance.nodes.first.attr1
+    assert_equal 'testing.node.local', d.instance.nodes.first.attr2
+    assert_equal 'testing.node.local', d.instance.nodes.first.attr3
+    assert_equal 'testing.node.local', d.instance.nodes.first.attr4
   end
 
   def test_emit
     d = create_driver
-    d.run do
-      d.emit({'field' => 'value1'})
-      d.emit({'field' => 'value2'})
-      d.emit({'field' => 'value3'})
+    d.run(default_tag: 'test.default', expect_records: 3) do
+      d.feed({'field' => 'value1'})
+      d.feed({'field' => 'value2'})
+      d.feed({'field' => 'value3'})
     end
 
-    emits = d.emits
-    assert_equal 3, emits.size
+    events = d.events
+    assert_equal 3, events.size
 
-    assert_equal 'foobar', emits[0][0]
-    assert_equal 'value1', emits[0][2]['field']
-    assert_equal 'expander', emits[0][2]['over']
+    assert_equal 'foobar', events[0][0]
+    assert_equal 'value1', events[0][2]['field']
+    assert_equal 'expander', events[0][2]['over']
 
-    assert_equal 'foobar', emits[1][0]
-    assert_equal 'value2', emits[1][2]['field']
-    assert_equal 'expander', emits[1][2]['over']
+    assert_equal 'foobar', events[1][0]
+    assert_equal 'value2', events[1][2]['field']
+    assert_equal 'expander', events[1][2]['over']
 
-    assert_equal 'foobar', emits[2][0]
-    assert_equal 'value3', emits[2][2]['field']
-    assert_equal 'expander', emits[2][2]['over']
+    assert_equal 'foobar', events[2][0]
+    assert_equal 'value3', events[2][2]['field']
+    assert_equal 'expander', events[2][2]['over']
   end
-
 end

data/test/plugins.rb

@@ -1,17 +1,19 @@
-class Fluent::ConfigExpanderTestInput < Fluent::Input
+require 'fluent/plugin/input'
+
+class Fluent::Plugin::ConfigExpanderTestInput < Fluent::Plugin::Input
   Fluent::Plugin.register_input('config_expander_test', self)
 
   config_param :tag, :string
-  attr_accessor :nodes
+  config_section :node, param_name: :nodes, multi: true do
+    config_param :attr1, :string, default: nil
+    config_param :attr2, :string, default: nil
+    config_param :attr3, :string, default: nil
+    config_param :attr4, :string, default: nil
+  end
   attr_accessor :started, :stopped
 
   def configure(conf)
     super
-    @nodes = []
-    conf.elements.each do |e|
-      next if e.name != 'node'
-      @nodes << {}.merge(e)
-    end
     @started = @stopped = false
   end
   def start
@@ -19,25 +21,29 @@ class Fluent::ConfigExpanderTestInput < Fluent::Input
     @started = true
   end
   def shutdown
-    super
     @stopped = true
+    super
   end
 end
 
-class Fluent::ConfigExpanderTestOutput < Fluent::Output
+require 'fluent/plugin/output'
+
+class Fluent::Plugin::ConfigExpanderTestOutput < Fluent::Plugin::Output
   Fluent::Plugin.register_output('config_expander_test', self)
 
+  helpers :event_emitter
+
   config_param :tag, :string
-  attr_accessor :nodes
+  config_section :node, param_name: :nodes, multi: true do
+    config_param :attr1, :string, default: nil
+    config_param :attr2, :string, default: nil
+    config_param :attr3, :string, default: nil
+    config_param :attr4, :string, default: nil
+  end
   attr_accessor :started, :stopped
 
   def configure(conf)
     super
-    @nodes = []
-    conf.elements.each do |e|
-      next if e.name != 'node'
-      @nodes << {}.merge(e)
-    end
     @started = @stopped = false
   end
   def start
@@ -45,19 +51,13 @@ class Fluent::ConfigExpanderTestOutput < Fluent::Output
     @started = true
   end
   def shutdown
-    super
     @stopped = true
+    super
   end
 
-  # Define `router` method of v0.12 to support v0.10 or earlier
-  unless method_defined?(:router)
-    define_method("router") { Fluent::Engine }
-  end
-
-  def emit(tag, es, chain)
+  def process(tag, es)
     es.each do |time, record|
       router.emit(@tag, time, record.merge({'over' => 'expander'}))
     end
-    chain.next
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-config-expander
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 1.0.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
@@ -14,14 +14,14 @@ dependencies:
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.14.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.14.0
 - !ruby/object:Gem::Dependency
@@ -67,6 +67,7 @@ files:
 - Rakefile
 - fluent-plugin-config-expander.gemspec
 - lib/fluent/plugin/expander.rb
+- lib/fluent/plugin/filter_config_expander.rb
 - lib/fluent/plugin/in_config_expander.rb
 - lib/fluent/plugin/out_config_expander.rb
 - test/helper.rb