fluent-plugin-amplifier-filter 0.3.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8a96b9a77928a922347d8f8397c8894f3c59a527
-  data.tar.gz: da7c1ba756b5b8508edcce1ffbefe7bd1085b1e0
+  metadata.gz: 206214436b2ba5533fe99bff1f9b9cb72ba325b1
+  data.tar.gz: 5b39b1c5aaea9698811df778e0ab86454ffc73ae
 SHA512:
-  metadata.gz: c759709df880360790144d80f7b33445c30dee5801fa32557b0f7b8bf22acaf049379f7b0d3dd83b19658c1ffebe624480dbbfb05b7ddf11e4e6d169b6049fea
-  data.tar.gz: 78dadfd3bc126cdd6c45fb22f60f290720f5c17f14ae7d0cc55d78412b0a3680bfc4c1d4c4552926e63bcf6565493fb7501b8c639757dc3746fa508c6e3056ae
+  metadata.gz: 39c06d2e15faedd4820469073d254d3dc094957e3e9b39997811b57bc2c6d46d926d74466523251909cb8b957d747247b9bc355eab68fac80fe54b04c5d80f9f
+  data.tar.gz: da14b083393b81e9150d07cbed550308d9d279a7d49e61fefa2aba2cccf4daff5defb8c586f2402d809f83b115fe6bb4f855e6c23179f5c6f52a126c9453dc94

data/.travis.yml

@@ -1,7 +1,6 @@
 language: ruby
 sudo: false
 rvm:
-  - 2.0
   - 2.1
   - 2.2
   - 2.3.0

data/README.md

@@ -1,15 +1,37 @@
 # fluent-plugin-amplifier-filter
 
-## Component
+[Fluentd](http://fluentd.org) plugin to modify numeric values of specified fields. Useful for counting values of sampled data (by fluent-plugin-sampling-filter or etc).
 
-### AmplifierFilterOutput
+## Configuration
 
-[Fluentd](http://fluentd.org) plugin to modify numeric values of specified fields, and Re-emit with modified tags. Useful for counting values of sampled data (by fluent-plugin-sampling-filter or etc).
+### AmplifierFilter
 
-## Configuration
+To do 10x for count values from messages 1/10 sampled, and to do 100x for 1/100 sampled:
+
+    <label @sampled>
+      <filter big.service> # I know its logs are sampled into 1/10
+        @type amplifier
+        ratio 10
+        key_names count, rate
+      </filter>
+      
+      <filter huge.service> # I know its logs are sampled into 1/100
+        @type amplifier
+        ratio 100
+        key_pattern .*_(count|rate)$
+      </filter>
+      
+      <match **>
+        # output result to visualization tools, or ....
+      </match>
+    </label>
+
+There is an option to `floor`(bool) the result of amplifying numeric values into integer. Its default value is `false`.
 
 ### AmplifierFilterOutput
 
+**NOTE: This output plugin is deprecated. Use 'amplifier' filter plugin instead.**
+
 To do x10 for messages 1/10 sampled, and to do x100 for messages 1/100 sampled:
 
     <match sampled_10.**>
@@ -44,46 +66,8 @@ To do x10 for messages 1/10 sampled, and to do x100 for messages 1/100 sampled:
       # output configurations where to send original/modified messages...
       </match>
 
-### AmplifierFilter
-
-Filter version of AmplifierFilterOutput plugin.
-It depends on Fluentd 0.12 or later.
-
-    <filter sampled_10.**>
-      @type amplifier_filter
-      ratio 10
-      key_names counts,rates
-    </filter>
-
-    <filter sampled_100.**>
-      @type amplifier_filter
-      ratio 100
-      key_names counts,rates
-    </filter>
-
-    <match sampled_10.**>
-      # output configurations where to send original/modified messages...
-    </match>
-
-    <match sampled_100.**>
-      # output configurations where to send original/modified messages...
-    </match>
-
-`key_pattern`(regexp) useful insted of `key_names`, and `add_prefix` is also useful:
-
-    <filter sampled_10.**>
-      @type amplifier_filter
-      ratio 10
-      key_pattern .*_(count|rate)$
-    </filter>
-
-    <match sampled_10.**>
-      # output configurations where to send original/modified messages...
-    </match>
-
 ## TODO
 
-* consider what to do next
 * patches welcome!
 
 ## Copyright

data/fluent-plugin-amplifier-filter.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-amplifier-filter"
-  gem.version       = "0.3.0"
+  gem.version       = "1.0.0"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{plugin to re-emit messages with amplified values}
@@ -14,7 +14,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.add_runtime_dependency "fluentd", "< 0.14.0"
+  gem.add_runtime_dependency "fluentd", ">= 0.14.0"
   gem.add_development_dependency "bundler"
   gem.add_development_dependency "rake"
   gem.add_development_dependency "test-unit", "~> 3.1.0"

data/lib/fluent/plugin/filter_amplifier.rb

@@ -0,0 +1,79 @@
+require 'fluent/plugin/filter'
+
+class Fluent::Plugin::AmplifierFilter < Fluent::Plugin::Filter
+  Fluent::Plugin.register_filter('amplifier', self)
+  Fluent::Plugin.register_filter('amplifier_filter', self)
+
+  config_param :ratio, :float
+
+  config_param :key_names, :array, value_type: :string, default: nil
+  config_param :key_pattern, :string, default: nil
+
+  config_param :floor, :bool, default: false
+
+  def configure(conf)
+    super
+
+    if @key_names.nil? && @key_pattern.nil?
+      raise Fluent::ConfigError, "missing both of key_names and key_pattern"
+    end
+    if @key_names && @key_pattern
+      raise Fluent::ConfigError, "cannot specify both of key_names and key_pattern"
+    end
+    if @key_pattern
+      @key_pattern = Regexp.new(@key_pattern)
+    end
+
+    amp = @floor ? :amp_with_floor : :amp_without_floor
+    self.define_singleton_method(:amp, method(amp))
+
+    filter_method = @key_names ? :filter_with_names : :filter_with_patterns
+    self.define_singleton_method(:filter, method(filter_method))
+  end
+
+  def amp_without_floor(value)
+    value.to_f * @ratio
+  end
+
+  def amp_with_floor(value)
+    (value.to_f * @ratio).floor
+  end
+
+  def filter(tag, time, record)
+    if @key_names
+      filter_with_names(tag, time, record)
+    else
+      filter_with_patterns(tag, time, record)
+    end
+  end
+
+  def filter_with_names(tag, time, record)
+    updated = {}
+    @key_names.each do |key|
+      val = record[key]
+      next unless val
+      updated[key] = amp(val)
+    end
+    log.trace "amplifier", tag: tag, floor: @floor, ratio: @ratio, updated: updated, original: record
+    if updated.size > 0
+      record.merge(updated)
+    else
+      record
+    end
+  end
+
+  def filter_with_patterns(tag, time, record)
+    updated = {}
+    record.each_pair do |key, val|
+      next unless val
+      next unless @key_pattern.match(key)
+      updated[key] = amp(val)
+    end
+    log.trace "amplifier", tag: tag, floor: @floor, ratio: @ratio, updated: updated, original: record
+    if updated.size > 0
+      record.merge(updated)
+    else
+      record
+    end
+  end
+end

data/lib/fluent/plugin/filter_amplifier_filter.rb

@@ -1,86 +1 @@
-class Fluent::AmplifierFilter < Fluent::Filter
-  Fluent::Plugin.register_filter('amplifier_filter', self)
-
-  config_param :ratio, :float
-
-  config_param :key_names, :string, default: nil
-  config_param :key_pattern, :string, default: nil
-
-  config_param :floor, :bool, default: false
-
-  # Define `log` method for v0.10.42 or earlier
-  unless method_defined?(:log)
-    define_method("log") { $log }
-  end
-
-  def configure(conf)
-    super
-
-    if @key_names.nil? and @key_pattern.nil?
-      raise Fluent::ConfigError, "missing both of key_names and key_pattern"
-    end
-    if not @key_names.nil? and not @key_pattern.nil?
-      raise Fluent::ConfigError, "cannot specify both of key_names and key_pattern"
-    end
-    if @key_names
-      @key_names = @key_names.split(',')
-    end
-    if @key_pattern
-      @key_pattern = Regexp.new(@key_pattern)
-    end
-
-    amp = if @floor
-            method(:amp_with_floor)
-          else
-            method(:amp_without_floor)
-          end
-    (class << self; self; end).module_eval do
-      define_method(:amp, amp)
-    end
-  end
-
-  def amp_without_floor(value)
-    value.to_f * @ratio
-  end
-
-  def amp_with_floor(value)
-    (value.to_f * @ratio).floor
-  end
-
-  def filter_stream(tag, es)
-    new_es = Fluent::MultiEventStream.new
-    if @key_names
-      es.each {|time,record|
-        updated = {}
-        @key_names.each {|key|
-          val = record[key]
-          next unless val
-          updated[key] = amp(val)
-        }
-        log.debug "amplifier tag:#{tag} floor:#{@floor} ratio:#{@ratio} updated:#{updated.to_json} record:#{record.to_json}"
-        if updated.size > 0
-          new_es.add(time, record.merge(updated))
-        else
-          new_es.add(time, record.dup)
-        end
-      }
-    else @key_pattern
-      es.each {|time,record|
-        updated = {}
-        record.keys.each {|key|
-          val = record[key]
-          next unless val
-          next unless @key_pattern.match(key)
-          updated[key] = amp(val)
-        }
-        log.debug "amplifier tag:#{tag} floor:#{@floor} ratio:#{@ratio} updated:#{updated.to_json} record:#{record.to_json}"
-        if updated.size > 0
-          new_es.add(time, record.merge(updated))
-        else
-          new_es.add(time, record.dup)
-        end
-      }
-    end
-    new_es
-  end
-end if defined?(Fluent::Filter)
+require_relative 'filter_amplifier'

data/lib/fluent/plugin/out_amplifier_filter.rb

@@ -1,33 +1,31 @@
-class Fluent::AmplifierFilterOutput < Fluent::Output
+require 'fluent/plugin/output'
+
+class Fluent::Output::AmplifierFilterOutput < Fluent::Plugin::Output
   Fluent::Plugin.register_output('amplifier_filter', self)
 
+  helpers :event_emitter
+
   config_param :ratio, :float
 
-  config_param :key_names, :string, :default => nil
-  config_param :key_pattern, :string, :default => nil
+  config_param :key_names, :array, value_type: :string, default: nil
+  config_param :key_pattern, :string, default: nil
 
-  config_param :floor, :bool, :default => false
+  config_param :floor, :bool, default: false
 
-  config_param :remove_prefix, :string, :default => nil
-  config_param :add_prefix, :string, :default => nil
-
-  # Define `log` method for v0.10.42 or earlier
-  unless method_defined?(:log)
-    define_method("log") { $log }
-  end
+  config_param :remove_prefix, :string, default: nil
+  config_param :add_prefix, :string, default: nil
 
   def configure(conf)
     super
 
+    log.warn "'amplifier_filter' output plugin is deprecated. use 'amplifier' filter plugin instead."
+
     if @key_names.nil? and @key_pattern.nil?
       raise Fluent::ConfigError, "missing both of key_names and key_pattern"
     end
-    if not @key_names.nil? and not @key_pattern.nil?
+    if @key_names && @key_pattern
       raise Fluent::ConfigError, "cannot specify both of key_names and key_pattern"
     end
-    if @key_names
-      @key_names = @key_names.split(',')
-    end
     if @key_pattern
       @key_pattern = Regexp.new(@key_pattern)
     end
@@ -37,9 +35,7 @@ class Fluent::AmplifierFilterOutput < Fluent::Output
           else
             method(:amp_without_floor)
           end
-    (class << self; self; end).module_eval do
-      define_method(:amp, amp)
-    end
+    self.define_singleton_method(:amp, amp)
 
     if not @remove_prefix and not @add_prefix
       raise Fluent::ConfigError, "missing both of remove_prefix and add_prefix"
@@ -61,7 +57,7 @@ class Fluent::AmplifierFilterOutput < Fluent::Output
     (value.to_f * @ratio).floor
   end
 
-  def emit(tag, es, chain)
+  def process(tag, es)
     if @remove_prefix and
         ( (tag.start_with?(@removed_prefix_string) and tag.length > @removed_length) or tag == @remove_prefix)
       tag = tag[@removed_length..-1]
@@ -108,7 +104,5 @@ class Fluent::AmplifierFilterOutput < Fluent::Output
       }
     end
     router.emit_array(tag, pairs)
-
-    chain.next
   end
 end

data/test/plugin/test_filter_amplifier_filter.rb

@@ -1,4 +1,5 @@
 require 'helper'
+require 'fluent/test/driver/filter'
 
 class AmplifierFilterTest < Test::Unit::TestCase
   def setup
@@ -26,27 +27,27 @@ class AmplifierFilterTest < Test::Unit::TestCase
     key_pattern field.*
   ]
 
-  def create_driver(conf = CONFIG, tag='test')
-    Fluent::Test::FilterTestDriver.new(Fluent::AmplifierFilter, tag).configure(conf)
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Filter.new(Fluent::Plugin::AmplifierFilter).configure(conf)
   end
 
   def test_configure
     assert_raise(Fluent::ConfigError) {
-      d = create_driver('')
+      create_driver('')
     }
     assert_raise(Fluent::ConfigError) {
-      d = create_driver(%[
+      create_driver(%[
         ratio 1
       ])
     }
     assert_nothing_thrown {
-      d = create_driver(%[
+      create_driver(%[
         ratio 1
         key_names field1
       ])
     }
     assert_nothing_raised {
-      d = create_driver(%[
+      create_driver(%[
         ratio 1
         key_pattern field\d+
       ])
@@ -62,23 +63,22 @@ class AmplifierFilterTest < Test::Unit::TestCase
     #   ratio 1.5
     #   key_names foo,bar,baz
     # ]
-    d1 = create_driver(CONFIG, 'test.service')
-    d1.run do
-      d1.filter({'name' => 'first',  'foo' => 10, 'bar' => 1, 'baz' => 20, 'zap' => 50})
-      d1.filter({'name' => 'second', 'foo' => 10, 'bar' => 2, 'baz' => 40, 'zap' => 50})
+    d1 = create_driver(CONFIG)
+    d1.run(default_tag: 'test.service') do
+      d1.feed({'name' => 'first',  'foo' => 10, 'bar' => 1, 'baz' => 20, 'zap' => 50})
+      d1.feed({'name' => 'second', 'foo' => 10, 'bar' => 2, 'baz' => 40, 'zap' => 50})
     end
-    filtered = d1.filtered_as_array
+    filtered = d1.filtered.map{|e| e.last }
     assert_equal 2, filtered.length
-    assert_equal 'test.service', filtered[0][0] # tag
 
-    first = filtered[0][2]
+    first = filtered[0]
     assert_equal 'first', first['name']
     assert_equal 15     , first['foo']
     assert_equal 1.5    , first['bar']
     assert_equal 30     , first['baz']
     assert_equal 50     , first['zap']
 
-    second = filtered[1][2]
+    second = filtered[1]
     assert_equal 'second', second['name']
     assert_equal 15      , second['foo']
     assert_equal 3       , second['bar']
@@ -92,23 +92,22 @@ class AmplifierFilterTest < Test::Unit::TestCase
     #   floor yes
     #   key_pattern field.*
     # ]
-    d3 = create_driver(CONFIG2, 'test.service')
-    d3.run do
-      d3.filter({'name' => 'first',  'fieldfoo' => 10, 'fieldbar' => 1, 'fieldbaz' => 20, 'zap' => 50})
-      d3.filter({'name' => 'second', 'fieldfoo' => '10', 'fieldbar' => '2', 'fieldbaz' => '40', 'zap' => '50'})
+    d3 = create_driver(CONFIG2)
+    d3.run(default_tag: 'test.service') do
+      d3.feed({'name' => 'first',  'fieldfoo' => 10, 'fieldbar' => 1, 'fieldbaz' => 20, 'zap' => 50})
+      d3.feed({'name' => 'second', 'fieldfoo' => '10', 'fieldbar' => '2', 'fieldbaz' => '40', 'zap' => '50'})
     end
-    filtered = d3.filtered_as_array
+    filtered = d3.filtered.map {|e| e.last }
     assert_equal 2, filtered.length
-    assert_equal 'test.service', filtered[0][0] # tag
 
-    first = filtered[0][2]
+    first = filtered[0]
     assert_equal 'first', first['name']
     assert_equal 7      , first['fieldfoo']
     assert_equal 0      , first['fieldbar']
     assert_equal 15     , first['fieldbaz']
     assert_equal 50     , first['zap']
 
-    second = filtered[1][2]
+    second = filtered[1]
     assert_equal 'second', second['name']
     assert_equal 7       , second['fieldfoo']
     assert_equal 1       , second['fieldbar']

data/test/plugin/test_out_amplifier_filter.rb

@@ -1,4 +1,5 @@
 require 'helper'
+require 'fluent/test/driver/output'
 
 class AmplifierFilterOutputTest < Test::Unit::TestCase
   def setup
@@ -25,40 +26,40 @@ class AmplifierFilterOutputTest < Test::Unit::TestCase
     remove_prefix test
   ]
 
-  def create_driver(conf = CONFIG, tag='test')
-    Fluent::Test::OutputTestDriver.new(Fluent::AmplifierFilterOutput, tag).configure(conf)
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Output.new(Fluent::Output::AmplifierFilterOutput).configure(conf)
   end
 
   def test_configure
     assert_raise(Fluent::ConfigError) {
-      d = create_driver('')
+      create_driver('')
     }
     assert_raise(Fluent::ConfigError) {
-      d = create_driver(%[
+      create_driver(%[
         ratio 1
       ])
     }
     assert_raise(Fluent::ConfigError) {
-      d = create_driver(%[
+      create_driver(%[
         ratio 1
         key_names field1
       ])
     }
     assert_raise(Fluent::ConfigError) {
-      d = create_driver(%[
+      create_driver(%[
         ratio 1
         add_prefix modified
       ])
     }
     assert_nothing_thrown {
-      d = create_driver(%[
+      create_driver(%[
         ratio 1
         key_names field1
         add_prefix modified
       ])
     }
     assert_nothing_raised {
-      d = create_driver(%[
+      create_driver(%[
         ratio 1
         key_pattern field\d+
         remove_prefix sampled
@@ -77,37 +78,37 @@ class AmplifierFilterOutputTest < Test::Unit::TestCase
     #   remove_prefix test
     #   add_prefix    modified
     # ]
-    d1 = create_driver(CONFIG, 'test.service')
-    d1.run do
-      d1.emit({'name' => 'first',  'foo' => 10, 'bar' => 1, 'baz' => 20, 'zap' => 50})
-      d1.emit({'name' => 'second', 'foo' => 10, 'bar' => 2, 'baz' => 40, 'zap' => 50})
+    d1 = create_driver(CONFIG)
+    d1.run(default_tag: 'test.service') do
+      d1.feed({'name' => 'first',  'foo' => 10, 'bar' => 1, 'baz' => 20, 'zap' => 50})
+      d1.feed({'name' => 'second', 'foo' => 10, 'bar' => 2, 'baz' => 40, 'zap' => 50})
     end
-    emits = d1.emits
-    assert_equal 2, emits.length
-    assert_equal 'modified.service', emits[0][0] # tag
+    events = d1.events
+    assert_equal 2, events.length
+    assert_equal 'modified.service', events[0][0] # tag
 
-    first = emits[0][2]
+    first = events[0][2]
     assert_equal 'first', first['name']
     assert_equal 15     , first['foo']
     assert_equal 1.5    , first['bar']
     assert_equal 30     , first['baz']
     assert_equal 50     , first['zap']
 
-    second = emits[1][2]
+    second = events[1][2]
     assert_equal 'second', second['name']
     assert_equal 15      , second['foo']
     assert_equal 3       , second['bar']
     assert_equal 60      , second['baz']
     assert_equal 50      , second['zap']
 
-    d2 = create_driver(CONFIG, 'test')
-    d2.run do
-      d2.emit({'name' => 'first',  'foo' => 10, 'bar' => 1, 'baz' => 20, 'zap' => 50})
-      d2.emit({'name' => 'second', 'foo' => 10, 'bar' => 2, 'baz' => 40, 'zap' => 50})
+    d2 = create_driver(CONFIG)
+    d2.run(default_tag: 'test') do
+      d2.feed({'name' => 'first',  'foo' => 10, 'bar' => 1, 'baz' => 20, 'zap' => 50})
+      d2.feed({'name' => 'second', 'foo' => 10, 'bar' => 2, 'baz' => 40, 'zap' => 50})
     end
-    emits = d2.emits
-    assert_equal 2, emits.length
-    assert_equal 'modified', emits[0][0] # tag
+    events = d2.events
+    assert_equal 2, events.length
+    assert_equal 'modified', events[0][0] # tag
 
     # CONFIG2 = %[
     #   ratio 0.75
@@ -115,23 +116,23 @@ class AmplifierFilterOutputTest < Test::Unit::TestCase
     #   key_pattern field.*
     #   remove_prefix test
     # ]
-    d3 = create_driver(CONFIG2, 'test.service')
-    d3.run do
-      d3.emit({'name' => 'first',  'fieldfoo' => 10, 'fieldbar' => 1, 'fieldbaz' => 20, 'zap' => 50})
-      d3.emit({'name' => 'second', 'fieldfoo' => '10', 'fieldbar' => '2', 'fieldbaz' => '40', 'zap' => '50'})
+    d3 = create_driver(CONFIG2)
+    d3.run(default_tag: 'test.service') do
+      d3.feed({'name' => 'first',  'fieldfoo' => 10, 'fieldbar' => 1, 'fieldbaz' => 20, 'zap' => 50})
+      d3.feed({'name' => 'second', 'fieldfoo' => '10', 'fieldbar' => '2', 'fieldbaz' => '40', 'zap' => '50'})
     end
-    emits = d3.emits
-    assert_equal 2, emits.length
-    assert_equal 'service', emits[0][0] # tag
+    events = d3.events
+    assert_equal 2, events.length
+    assert_equal 'service', events[0][0] # tag
 
-    first = emits[0][2]
+    first = events[0][2]
     assert_equal 'first', first['name']
     assert_equal 7      , first['fieldfoo']
     assert_equal 0      , first['fieldbar']
     assert_equal 15     , first['fieldbaz']
     assert_equal 50     , first['zap']
 
-    second = emits[1][2]
+    second = events[1][2]
     assert_equal 'second', second['name']
     assert_equal 7       , second['fieldfoo']
     assert_equal 1       , second['fieldbar']

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-amplifier-filter
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 1.0.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
@@ -14,14 +14,14 @@ dependencies:
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.14.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.14.0
 - !ruby/object:Gem::Dependency
@@ -80,6 +80,7 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-amplifier-filter.gemspec
+- lib/fluent/plugin/filter_amplifier.rb
 - lib/fluent/plugin/filter_amplifier_filter.rb
 - lib/fluent/plugin/out_amplifier_filter.rb
 - test/helper.rb