fluent-plugin-amplifier-filter 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.gitignore ADDED
@@ -0,0 +1,25 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
18
+ # For TextMate, emacs, vim
19
+ *.tmproj
20
+ tmtags
21
+ *~
22
+ \#*
23
+ .\#*
24
+ *.swp
25
+
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-amplifier.gemspec
4
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,13 @@
1
+ Copyright (c) 2012- TAGOMORI Satoshi
2
+
3
+ Licensed under the Apache License, Version 2.0 (the "License");
4
+ you may not use this file except in compliance with the License.
5
+ You may obtain a copy of the License at
6
+
7
+ http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ Unless required by applicable law or agreed to in writing, software
10
+ distributed under the License is distributed on an "AS IS" BASIS,
11
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ See the License for the specific language governing permissions and
13
+ limitations under the License.
data/README.md ADDED
@@ -0,0 +1,55 @@
1
+ # fluent-plugin-amplifier-filter
2
+
3
+ ## Component
4
+
5
+ ### AmplifierFilterOutput
6
+
7
+ Modify numeric values of specified fields, and Re-emit with modified tags. Useful for counting values of sampled data (by fluent-plugin-sampling-filter or etc).
8
+
9
+ ## Configuration
10
+
11
+ ### AmplifierFilterOutput
12
+
13
+ To do x10 for messages 1/10 sampled, and to do x100 for messages 1/100 sampled:
14
+
15
+ <match sampled_10.**>
16
+ type amplifier_filter
17
+ ratio 10
18
+ remove_prefix sampled_10
19
+ key_names counts,rates
20
+ </match>
21
+
22
+ <match sampled_100.**>
23
+ type amplifier_filter
24
+ ratio 100
25
+ remove_prefix sampled_100
26
+ key_names counts,rates
27
+ </match>
28
+
29
+ <match logs.**>
30
+ # output configurations where to send original/modified messages...
31
+ </match>
32
+
33
+ `key_pattern`(regexp) useful insted of `key_names`, and `add_prefix` is also useful:
34
+
35
+ <match sampled_10.**>
36
+ type amplifier_filter
37
+ ratio 10
38
+ remove_prefix sampled_10
39
+ add_prefix summary
40
+ key_pattern .*_(count|rate)$
41
+ </match>
42
+
43
+ <match summary.**>
44
+ # output configurations where to send original/modified messages...
45
+ </match>
46
+
47
+ ## TODO
48
+
49
+ * consider what to do next
50
+ * patches welcome!
51
+
52
+ ## Copyright
53
+
54
+ Copyright:: Copyright (c) 2012- TAGOMORI Satoshi (tagomoris)
55
+ License:: Apache License, Version 2.0
data/Rakefile ADDED
@@ -0,0 +1,11 @@
1
+ #!/usr/bin/env rake
2
+ require "bundler/gem_tasks"
3
+
4
+ require 'rake/testtask'
5
+ Rake::TestTask.new(:test) do |test|
6
+ test.libs << 'lib' << 'test'
7
+ test.pattern = 'test/**/test_*.rb'
8
+ test.verbose = true
9
+ end
10
+
11
+ task :default => :test
@@ -0,0 +1,18 @@
1
+ # -*- encoding: utf-8 -*-
2
+ Gem::Specification.new do |gem|
3
+ gem.name = "fluent-plugin-amplifier-filter"
4
+ gem.version = "0.1.0"
5
+ gem.authors = ["TAGOMORI Satoshi"]
6
+ gem.email = ["tagomoris@gmail.com"]
7
+ gem.summary = %q{plugin to re-emit messages with amplified values}
8
+ gem.description = %q{plugin to increase/decrease values by specified ratio (0-1 or 1-)}
9
+ gem.homepage = "http://github.com/tagomoris/fluent-plugin-amplifier"
10
+
11
+ gem.files = `git ls-files`.split($\)
12
+ gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
13
+ gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
14
+ gem.require_paths = ["lib"]
15
+
16
+ gem.add_development_dependency "fluentd"
17
+ gem.add_runtime_dependency "fluentd"
18
+ end
@@ -0,0 +1,91 @@
1
+ class Fluent::AmplifierFilterOutput < Fluent::Output
2
+ Fluent::Plugin.register_output('amplifier_filter', self)
3
+
4
+ config_param :ratio, :float
5
+
6
+ config_param :key_names, :string, :default => nil
7
+ config_param :key_pattern, :string, :default => nil
8
+
9
+ config_param :floor, :bool, :default => false
10
+
11
+ config_param :remove_prefix, :string, :default => nil
12
+ config_param :add_prefix, :string, :default => nil
13
+
14
+ def configure(conf)
15
+ super
16
+
17
+ if @key_names.nil? and @key_pattern.nil?
18
+ raise Fluent::ConfigError, "missing both of key_names and key_pattern"
19
+ end
20
+ if not @key_names.nil? and not @key_pattern.nil?
21
+ raise Fluent::ConfigError, "cannot specify both of key_names and key_pattern"
22
+ end
23
+ if @key_names
24
+ @key_names = @key_names.split(',')
25
+ end
26
+ if @key_pattern
27
+ @key_pattern = Regexp.new(@key_pattern)
28
+ end
29
+
30
+ amp = if @floor
31
+ method(:amp_with_floor)
32
+ else
33
+ method(:amp_without_floor)
34
+ end
35
+ (class << self; self; end).module_eval do
36
+ define_method(:amp, amp)
37
+ end
38
+
39
+ if not @remove_prefix and not @add_prefix
40
+ raise Fluent::ConfigError, "missing both of remove_prefix and add_prefix"
41
+ end
42
+ if @remove_prefix
43
+ @removed_prefix_string = @remove_prefix + '.'
44
+ @removed_length = @removed_prefix_string.length
45
+ end
46
+ if @add_prefix
47
+ @added_prefix_string = @add_prefix + '.'
48
+ end
49
+ end
50
+
51
+ def amp_without_floor(value)
52
+ value.to_f * @ratio
53
+ end
54
+
55
+ def amp_with_floor(value)
56
+ (value.to_f * @ratio).floor
57
+ end
58
+
59
+ def emit(tag, es, chain)
60
+ if @remove_prefix and
61
+ ( (tag.start_with?(@removed_prefix_string) and tag.length > @removed_length) or tag == @remove_prefix)
62
+ tag = tag[@removed_length..-1]
63
+ end
64
+ if @add_prefix
65
+ tag = if tag and tag.length > 0
66
+ @added_prefix_string + tag
67
+ else
68
+ @add_prefix
69
+ end
70
+ end
71
+
72
+ if @key_names
73
+ es.each {|time,record|
74
+ @key_names.each {|key|
75
+ record[key] = amp(record[key]) if record[key]
76
+ }
77
+ Fluent::Engine.emit(tag, time, record)
78
+ }
79
+ else @key_pattern
80
+ es.each {|time,record|
81
+ record.keys.each {|key|
82
+ next unless record[key] and @key_pattern.match(key)
83
+ record[key] = amp(record[key])
84
+ }
85
+ Fluent::Engine.emit(tag, time, record)
86
+ }
87
+ end
88
+
89
+ chain.next
90
+ end
91
+ end
data/test/helper.rb ADDED
@@ -0,0 +1,28 @@
1
+ require 'rubygems'
2
+ require 'bundler'
3
+ begin
4
+ Bundler.setup(:default, :development)
5
+ rescue Bundler::BundlerError => e
6
+ $stderr.puts e.message
7
+ $stderr.puts "Run `bundle install` to install missing gems"
8
+ exit e.status_code
9
+ end
10
+ require 'test/unit'
11
+
12
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
13
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
14
+ require 'fluent/test'
15
+ unless ENV.has_key?('VERBOSE')
16
+ nulllogger = Object.new
17
+ nulllogger.instance_eval {|obj|
18
+ def method_missing(method, *args)
19
+ # pass
20
+ end
21
+ }
22
+ $log = nulllogger
23
+ end
24
+
25
+ require 'fluent/plugin/out_amplifier_filter'
26
+
27
+ class Test::Unit::TestCase
28
+ end
@@ -0,0 +1,141 @@
1
+ require 'helper'
2
+
3
+ class AmplifierFilterOutputTest < Test::Unit::TestCase
4
+ def setup
5
+ Fluent::Test.setup
6
+ end
7
+
8
+ # config_param :ratio, :float
9
+ # config_param :key_names, :string, :default => nil
10
+ # config_param :key_pattern, :string, :default => nil
11
+ # config_param :floor, :bool, :default => false
12
+ # config_param :remove_prefix, :string, :default => nil
13
+ # config_param :add_prefix, :string, :default => nil
14
+
15
+ CONFIG = %[
16
+ ratio 1.5
17
+ key_names foo,bar,baz
18
+ remove_prefix test
19
+ add_prefix modified
20
+ ]
21
+ CONFIG2 = %[
22
+ ratio 0.75
23
+ floor yes
24
+ key_pattern field.*
25
+ remove_prefix test
26
+ ]
27
+
28
+ def create_driver(conf = CONFIG, tag='test')
29
+ Fluent::Test::OutputTestDriver.new(Fluent::AmplifierFilterOutput, tag).configure(conf)
30
+ end
31
+
32
+ def test_configure
33
+ assert_raise(Fluent::ConfigError) {
34
+ d = create_driver('')
35
+ }
36
+ assert_raise(Fluent::ConfigError) {
37
+ d = create_driver(%[
38
+ ratio 1
39
+ ])
40
+ }
41
+ assert_raise(Fluent::ConfigError) {
42
+ d = create_driver(%[
43
+ ratio 1
44
+ key_names field1
45
+ ])
46
+ }
47
+ assert_raise(Fluent::ConfigError) {
48
+ d = create_driver(%[
49
+ ratio 1
50
+ add_prefix modified
51
+ ])
52
+ }
53
+ assert_nothing_thrown {
54
+ d = create_driver(%[
55
+ ratio 1
56
+ key_names field1
57
+ add_prefix modified
58
+ ])
59
+ }
60
+ assert_nothing_raised {
61
+ d = create_driver(%[
62
+ ratio 1
63
+ key_pattern field\d+
64
+ remove_prefix sampled
65
+ ])
66
+ }
67
+
68
+ d = create_driver
69
+ assert_equal false, d.instance.floor
70
+ assert_equal ['foo', 'bar', 'baz'], d.instance.key_names
71
+ end
72
+
73
+ def test_emit
74
+ # CONFIG = %[
75
+ # ratio 1.5
76
+ # key_names foo,bar,baz
77
+ # remove_prefix test
78
+ # add_prefix modified
79
+ # ]
80
+ d1 = create_driver(CONFIG, 'test.service')
81
+ d1.run do
82
+ d1.emit({'name' => 'first', 'foo' => 10, 'bar' => 1, 'baz' => 20, 'zap' => 50})
83
+ d1.emit({'name' => 'second', 'foo' => 10, 'bar' => 2, 'baz' => 40, 'zap' => 50})
84
+ end
85
+ emits = d1.emits
86
+ assert_equal 2, emits.length
87
+ assert_equal 'modified.service', emits[0][0] # tag
88
+
89
+ first = emits[0][2]
90
+ assert_equal 'first', first['name']
91
+ assert_equal 15 , first['foo']
92
+ assert_equal 1.5 , first['bar']
93
+ assert_equal 30 , first['baz']
94
+ assert_equal 50 , first['zap']
95
+
96
+ second = emits[1][2]
97
+ assert_equal 'second', second['name']
98
+ assert_equal 15 , second['foo']
99
+ assert_equal 3 , second['bar']
100
+ assert_equal 60 , second['baz']
101
+ assert_equal 50 , second['zap']
102
+
103
+ d2 = create_driver(CONFIG, 'test')
104
+ d2.run do
105
+ d2.emit({'name' => 'first', 'foo' => 10, 'bar' => 1, 'baz' => 20, 'zap' => 50})
106
+ d2.emit({'name' => 'second', 'foo' => 10, 'bar' => 2, 'baz' => 40, 'zap' => 50})
107
+ end
108
+ emits = d2.emits
109
+ assert_equal 2, emits.length
110
+ assert_equal 'modified', emits[0][0] # tag
111
+
112
+ # CONFIG2 = %[
113
+ # ratio 0.75
114
+ # floor yes
115
+ # key_pattern field.*
116
+ # remove_prefix test
117
+ # ]
118
+ d3 = create_driver(CONFIG2, 'test.service')
119
+ d3.run do
120
+ d3.emit({'name' => 'first', 'fieldfoo' => 10, 'fieldbar' => 1, 'fieldbaz' => 20, 'zap' => 50})
121
+ d3.emit({'name' => 'second', 'fieldfoo' => '10', 'fieldbar' => '2', 'fieldbaz' => '40', 'zap' => '50'})
122
+ end
123
+ emits = d3.emits
124
+ assert_equal 2, emits.length
125
+ assert_equal 'service', emits[0][0] # tag
126
+
127
+ first = emits[0][2]
128
+ assert_equal 'first', first['name']
129
+ assert_equal 7 , first['fieldfoo']
130
+ assert_equal 0 , first['fieldbar']
131
+ assert_equal 15 , first['fieldbaz']
132
+ assert_equal 50 , first['zap']
133
+
134
+ second = emits[1][2]
135
+ assert_equal 'second', second['name']
136
+ assert_equal 7 , second['fieldfoo']
137
+ assert_equal 1 , second['fieldbar']
138
+ assert_equal 30 , second['fieldbaz']
139
+ assert_equal '50' , second['zap']
140
+ end
141
+ end
metadata ADDED
@@ -0,0 +1,88 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-amplifier-filter
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - TAGOMORI Satoshi
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2012-04-04 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: fluentd
16
+ requirement: !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ! '>='
20
+ - !ruby/object:Gem::Version
21
+ version: '0'
22
+ type: :development
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ! '>='
28
+ - !ruby/object:Gem::Version
29
+ version: '0'
30
+ - !ruby/object:Gem::Dependency
31
+ name: fluentd
32
+ requirement: !ruby/object:Gem::Requirement
33
+ none: false
34
+ requirements:
35
+ - - ! '>='
36
+ - !ruby/object:Gem::Version
37
+ version: '0'
38
+ type: :runtime
39
+ prerelease: false
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ! '>='
44
+ - !ruby/object:Gem::Version
45
+ version: '0'
46
+ description: plugin to increase/decrease values by specified ratio (0-1 or 1-)
47
+ email:
48
+ - tagomoris@gmail.com
49
+ executables: []
50
+ extensions: []
51
+ extra_rdoc_files: []
52
+ files:
53
+ - .gitignore
54
+ - Gemfile
55
+ - LICENSE.txt
56
+ - README.md
57
+ - Rakefile
58
+ - fluent-plugin-amplifier-filter.gemspec
59
+ - lib/fluent/plugin/out_amplifier_filter.rb
60
+ - test/helper.rb
61
+ - test/plugin/test_out_amplifier_filter.rb
62
+ homepage: http://github.com/tagomoris/fluent-plugin-amplifier
63
+ licenses: []
64
+ post_install_message:
65
+ rdoc_options: []
66
+ require_paths:
67
+ - lib
68
+ required_ruby_version: !ruby/object:Gem::Requirement
69
+ none: false
70
+ requirements:
71
+ - - ! '>='
72
+ - !ruby/object:Gem::Version
73
+ version: '0'
74
+ required_rubygems_version: !ruby/object:Gem::Requirement
75
+ none: false
76
+ requirements:
77
+ - - ! '>='
78
+ - !ruby/object:Gem::Version
79
+ version: '0'
80
+ requirements: []
81
+ rubyforge_project:
82
+ rubygems_version: 1.8.21
83
+ signing_key:
84
+ specification_version: 3
85
+ summary: plugin to re-emit messages with amplified values
86
+ test_files:
87
+ - test/helper.rb
88
+ - test/plugin/test_out_amplifier_filter.rb