fluent-plugin-amplifier-filter 0.1.6 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4c418acc02e375c36620d6731b7d65e95435ee36
-  data.tar.gz: d5958fe64969a439d70019f67af14918a8fe073e
+  metadata.gz: 5c9755678e4245819f34011b36969ea35245d75c
+  data.tar.gz: ace2c6968786f5ec90a9b9579043de1105430430
 SHA512:
-  metadata.gz: 2373ab79ff181e64694f0b2118234a451db8be96bcb4b2e755d2e3a86228e978798072978523d2b70cd05d3d007a49b32d5f23dd0e065d044f626044e56a4774
-  data.tar.gz: 044566f36afe08bfa9e3ada239c863ed7153d638ee1b8d87a1d76395f91d0d9b33b642167f2de96dc8119ce0bafba2a29e96372215c199fa140dbc49547958c0
+  metadata.gz: 3b6916158e1612720492b56b1b570bf7f799255f6ac0ca664d70c8c2199ad23f0a24f15ae31e6372aad2fb11c977458be9c0a91e87a079c756c018898f02eafe
+  data.tar.gz: 8ed7db05d43c4a0c09b5a2dff4a045c4d1e7eac745d8112809a9afbf9ccd80f1cda68fa526864c357384b9e4cffc50dbb5022eb985f4fda74b39ad5f8ceb9624

data/README.md

@@ -39,9 +39,46 @@ To do x10 for messages 1/10 sampled, and to do x100 for messages 1/100 sampled:
       add_prefix summary
       key_pattern .*_(count|rate)$
     </match>
-    
+
     <match summary.**>
       # output configurations where to send original/modified messages...
+      </match>
+
+### AmplifierFilter
+
+Filter version of AmplifierFilterOutput plugin.
+It depends on Fluentd 0.12 or later.
+
+    <filter sampled_10.**>
+      type amplifier_filter
+      ratio 10
+      key_names counts,rates
+    </filter>
+
+    <filter sampled_100.**>
+      type amplifier_filter
+      ratio 100
+      key_names counts,rates
+    </filter>
+
+    <match sampled_10.**>
+      # output configurations where to send original/modified messages...
+    </match>
+
+    <match sampled_100.**>
+      # output configurations where to send original/modified messages...
+    </match>
+
+`key_pattern`(regexp) useful insted of `key_names`, and `add_prefix` is also useful:
+
+    <filter sampled_10.**>
+      type amplifier_filter
+      ratio 10
+      key_pattern .*_(count|rate)$
+    </filter>
+
+    <match sampled_10.**>
+      # output configurations where to send original/modified messages...
     </match>
 
 ## TODO

data/fluent-plugin-amplifier-filter.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-amplifier-filter"
-  gem.version       = "0.1.6"
+  gem.version       = "0.2.0"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.summary       = %q{plugin to re-emit messages with amplified values}
@@ -17,4 +17,5 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "fluentd"
   gem.add_development_dependency "bundler"
   gem.add_development_dependency "rake"
+  gem.add_development_dependency "test-unit", "~> 3.1.0"
 end

data/lib/fluent/plugin/filter_amplifier.rb

@@ -0,0 +1,86 @@
+class Fluent::AmplifierFilter < Fluent::Filter
+  Fluent::Plugin.register_filter('amplifier_filter', self)
+
+  config_param :ratio, :float
+
+  config_param :key_names, :string, default: nil
+  config_param :key_pattern, :string, default: nil
+
+  config_param :floor, :bool, default: false
+
+  # Define `log` method for v0.10.42 or earlier
+  unless method_defined?(:log)
+    define_method("log") { $log }
+  end
+
+  def configure(conf)
+    super
+
+    if @key_names.nil? and @key_pattern.nil?
+      raise Fluent::ConfigError, "missing both of key_names and key_pattern"
+    end
+    if not @key_names.nil? and not @key_pattern.nil?
+      raise Fluent::ConfigError, "cannot specify both of key_names and key_pattern"
+    end
+    if @key_names
+      @key_names = @key_names.split(',')
+    end
+    if @key_pattern
+      @key_pattern = Regexp.new(@key_pattern)
+    end
+
+    amp = if @floor
+            method(:amp_with_floor)
+          else
+            method(:amp_without_floor)
+          end
+    (class << self; self; end).module_eval do
+      define_method(:amp, amp)
+    end
+  end
+
+  def amp_without_floor(value)
+    value.to_f * @ratio
+  end
+
+  def amp_with_floor(value)
+    (value.to_f * @ratio).floor
+  end
+
+  def filter_stream(tag, es)
+    new_es = Fluent::MultiEventStream.new
+    if @key_names
+      es.each {|time,record|
+        updated = {}
+        @key_names.each {|key|
+          val = record[key]
+          next unless val
+          updated[key] = amp(val)
+        }
+        log.debug "amplifier tag:#{tag} floor:#{@floor} ratio:#{@ratio} updated:#{updated.to_json} record:#{record.to_json}"
+        if updated.size > 0
+          new_es.add(time, record.merge(updated))
+        else
+          new_es.add(time, record.dup)
+        end
+      }
+    else @key_pattern
+      es.each {|time,record|
+        updated = {}
+        record.keys.each {|key|
+          val = record[key]
+          next unless val
+          next unless @key_pattern.match(key)
+          updated[key] = amp(val)
+        }
+        log.debug "amplifier tag:#{tag} floor:#{@floor} ratio:#{@ratio} updated:#{updated.to_json} record:#{record.to_json}"
+        if updated.size > 0
+          new_es.add(time, record.merge(updated))
+        else
+          new_es.add(time, record.dup)
+        end
+      }
+    end
+    new_es
+  end
+end if defined?(Fluent::Filter)

data/lib/fluent/plugin/out_amplifier_filter.rb

@@ -107,7 +107,7 @@ class Fluent::AmplifierFilterOutput < Fluent::Output
         end
       }
     end
-    Fluent::Engine.emit_array(tag, pairs)
+    router.emit_array(tag, pairs)
 
     chain.next
   end

data/test/helper.rb

@@ -25,6 +25,7 @@ else
 end
 
 require 'fluent/plugin/out_amplifier_filter'
+require 'fluent/plugin/filter_amplifier'
 
 class Test::Unit::TestCase
 end

data/test/plugin/test_filter_amplifier.rb

@@ -0,0 +1,118 @@
+require 'helper'
+
+class AmplifierFilterTest < Test::Unit::TestCase
+  def setup
+    if not defined?(Fluent::Filter)
+      omit("Fluent::Filter is not defined. Use fluentd 0.12 or later.")
+    end
+
+    Fluent::Test.setup
+  end
+
+  # config_param :ratio, :float
+  # config_param :key_names, :string, :default => nil
+  # config_param :key_pattern, :string, :default => nil
+  # config_param :floor, :bool, :default => false
+  # config_param :remove_prefix, :string, :default => nil
+  # config_param :add_prefix, :string, :default => nil
+
+  CONFIG = %[
+    ratio 1.5
+    key_names foo,bar,baz
+  ]
+  CONFIG2 = %[
+    ratio 0.75
+    floor yes
+    key_pattern field.*
+  ]
+
+  def create_driver(conf = CONFIG, tag='test')
+    Fluent::Test::FilterTestDriver.new(Fluent::AmplifierFilter, tag).configure(conf)
+  end
+
+  def test_configure
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('')
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver(%[
+        ratio 1
+      ])
+    }
+    assert_nothing_thrown {
+      d = create_driver(%[
+        ratio 1
+        key_names field1
+      ])
+    }
+    assert_nothing_raised {
+      d = create_driver(%[
+        ratio 1
+        key_pattern field\d+
+      ])
+    }
+
+    d = create_driver
+    assert_equal false, d.instance.floor
+    assert_equal ['foo', 'bar', 'baz'], d.instance.key_names
+  end
+
+  def test_filter_1
+    # CONFIG = %[
+    #   ratio 1.5
+    #   key_names foo,bar,baz
+    # ]
+    d1 = create_driver(CONFIG, 'test.service')
+    d1.run do
+      d1.filter({'name' => 'first',  'foo' => 10, 'bar' => 1, 'baz' => 20, 'zap' => 50})
+      d1.filter({'name' => 'second', 'foo' => 10, 'bar' => 2, 'baz' => 40, 'zap' => 50})
+    end
+    filtered = d1.filtered_as_array
+    assert_equal 2, filtered.length
+    assert_equal 'test.service', filtered[0][0] # tag
+
+    first = filtered[0][2]
+    assert_equal 'first', first['name']
+    assert_equal 15     , first['foo']
+    assert_equal 1.5    , first['bar']
+    assert_equal 30     , first['baz']
+    assert_equal 50     , first['zap']
+
+    second = filtered[1][2]
+    assert_equal 'second', second['name']
+    assert_equal 15      , second['foo']
+    assert_equal 3       , second['bar']
+    assert_equal 60      , second['baz']
+    assert_equal 50      , second['zap']
+  end
+
+  def test_filter_2
+    # CONFIG2 = %[
+    #   ratio 0.75
+    #   floor yes
+    #   key_pattern field.*
+    # ]
+    d3 = create_driver(CONFIG2, 'test.service')
+    d3.run do
+      d3.filter({'name' => 'first',  'fieldfoo' => 10, 'fieldbar' => 1, 'fieldbaz' => 20, 'zap' => 50})
+      d3.filter({'name' => 'second', 'fieldfoo' => '10', 'fieldbar' => '2', 'fieldbaz' => '40', 'zap' => '50'})
+    end
+    filtered = d3.filtered_as_array
+    assert_equal 2, filtered.length
+    assert_equal 'test.service', filtered[0][0] # tag
+
+    first = filtered[0][2]
+    assert_equal 'first', first['name']
+    assert_equal 7      , first['fieldfoo']
+    assert_equal 0      , first['fieldbar']
+    assert_equal 15     , first['fieldbaz']
+    assert_equal 50     , first['zap']
+
+    second = filtered[1][2]
+    assert_equal 'second', second['name']
+    assert_equal 7       , second['fieldfoo']
+    assert_equal 1       , second['fieldbar']
+    assert_equal 30      , second['fieldbaz']
+    assert_equal '50'    , second['zap']
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-amplifier-filter
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.2.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-16 00:00:00.000000000 Z
+date: 2015-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.0
 description: plugin to increase/decrease values by specified ratio (0-1 or 1-)
 email:
 - tagomoris@gmail.com
@@ -66,8 +80,10 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-amplifier-filter.gemspec
+- lib/fluent/plugin/filter_amplifier.rb
 - lib/fluent/plugin/out_amplifier_filter.rb
 - test/helper.rb
+- test/plugin/test_filter_amplifier.rb
 - test/plugin/test_out_amplifier_filter.rb
 homepage: http://github.com/tagomoris/fluent-plugin-amplifier-filter
 licenses:
@@ -89,10 +105,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: plugin to re-emit messages with amplified values
 test_files:
 - test/helper.rb
+- test/plugin/test_filter_amplifier.rb
 - test/plugin/test_out_amplifier_filter.rb
+has_rdoc: