fluent-plugin-amplifier-filter 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +25 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +13 -0
- data/README.md +55 -0
- data/Rakefile +11 -0
- data/fluent-plugin-amplifier-filter.gemspec +18 -0
- data/lib/fluent/plugin/out_amplifier_filter.rb +91 -0
- data/test/helper.rb +28 -0
- data/test/plugin/test_out_amplifier_filter.rb +141 -0
- metadata +88 -0
data/.gitignore
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
*.gem
|
2
|
+
*.rbc
|
3
|
+
.bundle
|
4
|
+
.config
|
5
|
+
.yardoc
|
6
|
+
Gemfile.lock
|
7
|
+
InstalledFiles
|
8
|
+
_yardoc
|
9
|
+
coverage
|
10
|
+
doc/
|
11
|
+
lib/bundler/man
|
12
|
+
pkg
|
13
|
+
rdoc
|
14
|
+
spec/reports
|
15
|
+
test/tmp
|
16
|
+
test/version_tmp
|
17
|
+
tmp
|
18
|
+
# For TextMate, emacs, vim
|
19
|
+
*.tmproj
|
20
|
+
tmtags
|
21
|
+
*~
|
22
|
+
\#*
|
23
|
+
.\#*
|
24
|
+
*.swp
|
25
|
+
|
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
Copyright (c) 2012- TAGOMORI Satoshi
|
2
|
+
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
you may not use this file except in compliance with the License.
|
5
|
+
You may obtain a copy of the License at
|
6
|
+
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
See the License for the specific language governing permissions and
|
13
|
+
limitations under the License.
|
data/README.md
ADDED
@@ -0,0 +1,55 @@
|
|
1
|
+
# fluent-plugin-amplifier-filter
|
2
|
+
|
3
|
+
## Component
|
4
|
+
|
5
|
+
### AmplifierFilterOutput
|
6
|
+
|
7
|
+
Modify numeric values of specified fields, and Re-emit with modified tags. Useful for counting values of sampled data (by fluent-plugin-sampling-filter or etc).
|
8
|
+
|
9
|
+
## Configuration
|
10
|
+
|
11
|
+
### AmplifierFilterOutput
|
12
|
+
|
13
|
+
To do x10 for messages 1/10 sampled, and to do x100 for messages 1/100 sampled:
|
14
|
+
|
15
|
+
<match sampled_10.**>
|
16
|
+
type amplifier_filter
|
17
|
+
ratio 10
|
18
|
+
remove_prefix sampled_10
|
19
|
+
key_names counts,rates
|
20
|
+
</match>
|
21
|
+
|
22
|
+
<match sampled_100.**>
|
23
|
+
type amplifier_filter
|
24
|
+
ratio 100
|
25
|
+
remove_prefix sampled_100
|
26
|
+
key_names counts,rates
|
27
|
+
</match>
|
28
|
+
|
29
|
+
<match logs.**>
|
30
|
+
# output configurations where to send original/modified messages...
|
31
|
+
</match>
|
32
|
+
|
33
|
+
`key_pattern`(regexp) useful insted of `key_names`, and `add_prefix` is also useful:
|
34
|
+
|
35
|
+
<match sampled_10.**>
|
36
|
+
type amplifier_filter
|
37
|
+
ratio 10
|
38
|
+
remove_prefix sampled_10
|
39
|
+
add_prefix summary
|
40
|
+
key_pattern .*_(count|rate)$
|
41
|
+
</match>
|
42
|
+
|
43
|
+
<match summary.**>
|
44
|
+
# output configurations where to send original/modified messages...
|
45
|
+
</match>
|
46
|
+
|
47
|
+
## TODO
|
48
|
+
|
49
|
+
* consider what to do next
|
50
|
+
* patches welcome!
|
51
|
+
|
52
|
+
## Copyright
|
53
|
+
|
54
|
+
Copyright:: Copyright (c) 2012- TAGOMORI Satoshi (tagomoris)
|
55
|
+
License:: Apache License, Version 2.0
|
data/Rakefile
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
Gem::Specification.new do |gem|
|
3
|
+
gem.name = "fluent-plugin-amplifier-filter"
|
4
|
+
gem.version = "0.1.0"
|
5
|
+
gem.authors = ["TAGOMORI Satoshi"]
|
6
|
+
gem.email = ["tagomoris@gmail.com"]
|
7
|
+
gem.summary = %q{plugin to re-emit messages with amplified values}
|
8
|
+
gem.description = %q{plugin to increase/decrease values by specified ratio (0-1 or 1-)}
|
9
|
+
gem.homepage = "http://github.com/tagomoris/fluent-plugin-amplifier"
|
10
|
+
|
11
|
+
gem.files = `git ls-files`.split($\)
|
12
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
13
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
14
|
+
gem.require_paths = ["lib"]
|
15
|
+
|
16
|
+
gem.add_development_dependency "fluentd"
|
17
|
+
gem.add_runtime_dependency "fluentd"
|
18
|
+
end
|
@@ -0,0 +1,91 @@
|
|
1
|
+
class Fluent::AmplifierFilterOutput < Fluent::Output
|
2
|
+
Fluent::Plugin.register_output('amplifier_filter', self)
|
3
|
+
|
4
|
+
config_param :ratio, :float
|
5
|
+
|
6
|
+
config_param :key_names, :string, :default => nil
|
7
|
+
config_param :key_pattern, :string, :default => nil
|
8
|
+
|
9
|
+
config_param :floor, :bool, :default => false
|
10
|
+
|
11
|
+
config_param :remove_prefix, :string, :default => nil
|
12
|
+
config_param :add_prefix, :string, :default => nil
|
13
|
+
|
14
|
+
def configure(conf)
|
15
|
+
super
|
16
|
+
|
17
|
+
if @key_names.nil? and @key_pattern.nil?
|
18
|
+
raise Fluent::ConfigError, "missing both of key_names and key_pattern"
|
19
|
+
end
|
20
|
+
if not @key_names.nil? and not @key_pattern.nil?
|
21
|
+
raise Fluent::ConfigError, "cannot specify both of key_names and key_pattern"
|
22
|
+
end
|
23
|
+
if @key_names
|
24
|
+
@key_names = @key_names.split(',')
|
25
|
+
end
|
26
|
+
if @key_pattern
|
27
|
+
@key_pattern = Regexp.new(@key_pattern)
|
28
|
+
end
|
29
|
+
|
30
|
+
amp = if @floor
|
31
|
+
method(:amp_with_floor)
|
32
|
+
else
|
33
|
+
method(:amp_without_floor)
|
34
|
+
end
|
35
|
+
(class << self; self; end).module_eval do
|
36
|
+
define_method(:amp, amp)
|
37
|
+
end
|
38
|
+
|
39
|
+
if not @remove_prefix and not @add_prefix
|
40
|
+
raise Fluent::ConfigError, "missing both of remove_prefix and add_prefix"
|
41
|
+
end
|
42
|
+
if @remove_prefix
|
43
|
+
@removed_prefix_string = @remove_prefix + '.'
|
44
|
+
@removed_length = @removed_prefix_string.length
|
45
|
+
end
|
46
|
+
if @add_prefix
|
47
|
+
@added_prefix_string = @add_prefix + '.'
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def amp_without_floor(value)
|
52
|
+
value.to_f * @ratio
|
53
|
+
end
|
54
|
+
|
55
|
+
def amp_with_floor(value)
|
56
|
+
(value.to_f * @ratio).floor
|
57
|
+
end
|
58
|
+
|
59
|
+
def emit(tag, es, chain)
|
60
|
+
if @remove_prefix and
|
61
|
+
( (tag.start_with?(@removed_prefix_string) and tag.length > @removed_length) or tag == @remove_prefix)
|
62
|
+
tag = tag[@removed_length..-1]
|
63
|
+
end
|
64
|
+
if @add_prefix
|
65
|
+
tag = if tag and tag.length > 0
|
66
|
+
@added_prefix_string + tag
|
67
|
+
else
|
68
|
+
@add_prefix
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
if @key_names
|
73
|
+
es.each {|time,record|
|
74
|
+
@key_names.each {|key|
|
75
|
+
record[key] = amp(record[key]) if record[key]
|
76
|
+
}
|
77
|
+
Fluent::Engine.emit(tag, time, record)
|
78
|
+
}
|
79
|
+
else @key_pattern
|
80
|
+
es.each {|time,record|
|
81
|
+
record.keys.each {|key|
|
82
|
+
next unless record[key] and @key_pattern.match(key)
|
83
|
+
record[key] = amp(record[key])
|
84
|
+
}
|
85
|
+
Fluent::Engine.emit(tag, time, record)
|
86
|
+
}
|
87
|
+
end
|
88
|
+
|
89
|
+
chain.next
|
90
|
+
end
|
91
|
+
end
|
data/test/helper.rb
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler'
|
3
|
+
begin
|
4
|
+
Bundler.setup(:default, :development)
|
5
|
+
rescue Bundler::BundlerError => e
|
6
|
+
$stderr.puts e.message
|
7
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
8
|
+
exit e.status_code
|
9
|
+
end
|
10
|
+
require 'test/unit'
|
11
|
+
|
12
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
13
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
14
|
+
require 'fluent/test'
|
15
|
+
unless ENV.has_key?('VERBOSE')
|
16
|
+
nulllogger = Object.new
|
17
|
+
nulllogger.instance_eval {|obj|
|
18
|
+
def method_missing(method, *args)
|
19
|
+
# pass
|
20
|
+
end
|
21
|
+
}
|
22
|
+
$log = nulllogger
|
23
|
+
end
|
24
|
+
|
25
|
+
require 'fluent/plugin/out_amplifier_filter'
|
26
|
+
|
27
|
+
class Test::Unit::TestCase
|
28
|
+
end
|
@@ -0,0 +1,141 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class AmplifierFilterOutputTest < Test::Unit::TestCase
|
4
|
+
def setup
|
5
|
+
Fluent::Test.setup
|
6
|
+
end
|
7
|
+
|
8
|
+
# config_param :ratio, :float
|
9
|
+
# config_param :key_names, :string, :default => nil
|
10
|
+
# config_param :key_pattern, :string, :default => nil
|
11
|
+
# config_param :floor, :bool, :default => false
|
12
|
+
# config_param :remove_prefix, :string, :default => nil
|
13
|
+
# config_param :add_prefix, :string, :default => nil
|
14
|
+
|
15
|
+
CONFIG = %[
|
16
|
+
ratio 1.5
|
17
|
+
key_names foo,bar,baz
|
18
|
+
remove_prefix test
|
19
|
+
add_prefix modified
|
20
|
+
]
|
21
|
+
CONFIG2 = %[
|
22
|
+
ratio 0.75
|
23
|
+
floor yes
|
24
|
+
key_pattern field.*
|
25
|
+
remove_prefix test
|
26
|
+
]
|
27
|
+
|
28
|
+
def create_driver(conf = CONFIG, tag='test')
|
29
|
+
Fluent::Test::OutputTestDriver.new(Fluent::AmplifierFilterOutput, tag).configure(conf)
|
30
|
+
end
|
31
|
+
|
32
|
+
def test_configure
|
33
|
+
assert_raise(Fluent::ConfigError) {
|
34
|
+
d = create_driver('')
|
35
|
+
}
|
36
|
+
assert_raise(Fluent::ConfigError) {
|
37
|
+
d = create_driver(%[
|
38
|
+
ratio 1
|
39
|
+
])
|
40
|
+
}
|
41
|
+
assert_raise(Fluent::ConfigError) {
|
42
|
+
d = create_driver(%[
|
43
|
+
ratio 1
|
44
|
+
key_names field1
|
45
|
+
])
|
46
|
+
}
|
47
|
+
assert_raise(Fluent::ConfigError) {
|
48
|
+
d = create_driver(%[
|
49
|
+
ratio 1
|
50
|
+
add_prefix modified
|
51
|
+
])
|
52
|
+
}
|
53
|
+
assert_nothing_thrown {
|
54
|
+
d = create_driver(%[
|
55
|
+
ratio 1
|
56
|
+
key_names field1
|
57
|
+
add_prefix modified
|
58
|
+
])
|
59
|
+
}
|
60
|
+
assert_nothing_raised {
|
61
|
+
d = create_driver(%[
|
62
|
+
ratio 1
|
63
|
+
key_pattern field\d+
|
64
|
+
remove_prefix sampled
|
65
|
+
])
|
66
|
+
}
|
67
|
+
|
68
|
+
d = create_driver
|
69
|
+
assert_equal false, d.instance.floor
|
70
|
+
assert_equal ['foo', 'bar', 'baz'], d.instance.key_names
|
71
|
+
end
|
72
|
+
|
73
|
+
def test_emit
|
74
|
+
# CONFIG = %[
|
75
|
+
# ratio 1.5
|
76
|
+
# key_names foo,bar,baz
|
77
|
+
# remove_prefix test
|
78
|
+
# add_prefix modified
|
79
|
+
# ]
|
80
|
+
d1 = create_driver(CONFIG, 'test.service')
|
81
|
+
d1.run do
|
82
|
+
d1.emit({'name' => 'first', 'foo' => 10, 'bar' => 1, 'baz' => 20, 'zap' => 50})
|
83
|
+
d1.emit({'name' => 'second', 'foo' => 10, 'bar' => 2, 'baz' => 40, 'zap' => 50})
|
84
|
+
end
|
85
|
+
emits = d1.emits
|
86
|
+
assert_equal 2, emits.length
|
87
|
+
assert_equal 'modified.service', emits[0][0] # tag
|
88
|
+
|
89
|
+
first = emits[0][2]
|
90
|
+
assert_equal 'first', first['name']
|
91
|
+
assert_equal 15 , first['foo']
|
92
|
+
assert_equal 1.5 , first['bar']
|
93
|
+
assert_equal 30 , first['baz']
|
94
|
+
assert_equal 50 , first['zap']
|
95
|
+
|
96
|
+
second = emits[1][2]
|
97
|
+
assert_equal 'second', second['name']
|
98
|
+
assert_equal 15 , second['foo']
|
99
|
+
assert_equal 3 , second['bar']
|
100
|
+
assert_equal 60 , second['baz']
|
101
|
+
assert_equal 50 , second['zap']
|
102
|
+
|
103
|
+
d2 = create_driver(CONFIG, 'test')
|
104
|
+
d2.run do
|
105
|
+
d2.emit({'name' => 'first', 'foo' => 10, 'bar' => 1, 'baz' => 20, 'zap' => 50})
|
106
|
+
d2.emit({'name' => 'second', 'foo' => 10, 'bar' => 2, 'baz' => 40, 'zap' => 50})
|
107
|
+
end
|
108
|
+
emits = d2.emits
|
109
|
+
assert_equal 2, emits.length
|
110
|
+
assert_equal 'modified', emits[0][0] # tag
|
111
|
+
|
112
|
+
# CONFIG2 = %[
|
113
|
+
# ratio 0.75
|
114
|
+
# floor yes
|
115
|
+
# key_pattern field.*
|
116
|
+
# remove_prefix test
|
117
|
+
# ]
|
118
|
+
d3 = create_driver(CONFIG2, 'test.service')
|
119
|
+
d3.run do
|
120
|
+
d3.emit({'name' => 'first', 'fieldfoo' => 10, 'fieldbar' => 1, 'fieldbaz' => 20, 'zap' => 50})
|
121
|
+
d3.emit({'name' => 'second', 'fieldfoo' => '10', 'fieldbar' => '2', 'fieldbaz' => '40', 'zap' => '50'})
|
122
|
+
end
|
123
|
+
emits = d3.emits
|
124
|
+
assert_equal 2, emits.length
|
125
|
+
assert_equal 'service', emits[0][0] # tag
|
126
|
+
|
127
|
+
first = emits[0][2]
|
128
|
+
assert_equal 'first', first['name']
|
129
|
+
assert_equal 7 , first['fieldfoo']
|
130
|
+
assert_equal 0 , first['fieldbar']
|
131
|
+
assert_equal 15 , first['fieldbaz']
|
132
|
+
assert_equal 50 , first['zap']
|
133
|
+
|
134
|
+
second = emits[1][2]
|
135
|
+
assert_equal 'second', second['name']
|
136
|
+
assert_equal 7 , second['fieldfoo']
|
137
|
+
assert_equal 1 , second['fieldbar']
|
138
|
+
assert_equal 30 , second['fieldbaz']
|
139
|
+
assert_equal '50' , second['zap']
|
140
|
+
end
|
141
|
+
end
|
metadata
ADDED
@@ -0,0 +1,88 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-amplifier-filter
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- TAGOMORI Satoshi
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-04-04 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: fluentd
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :development
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: fluentd
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
46
|
+
description: plugin to increase/decrease values by specified ratio (0-1 or 1-)
|
47
|
+
email:
|
48
|
+
- tagomoris@gmail.com
|
49
|
+
executables: []
|
50
|
+
extensions: []
|
51
|
+
extra_rdoc_files: []
|
52
|
+
files:
|
53
|
+
- .gitignore
|
54
|
+
- Gemfile
|
55
|
+
- LICENSE.txt
|
56
|
+
- README.md
|
57
|
+
- Rakefile
|
58
|
+
- fluent-plugin-amplifier-filter.gemspec
|
59
|
+
- lib/fluent/plugin/out_amplifier_filter.rb
|
60
|
+
- test/helper.rb
|
61
|
+
- test/plugin/test_out_amplifier_filter.rb
|
62
|
+
homepage: http://github.com/tagomoris/fluent-plugin-amplifier
|
63
|
+
licenses: []
|
64
|
+
post_install_message:
|
65
|
+
rdoc_options: []
|
66
|
+
require_paths:
|
67
|
+
- lib
|
68
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
69
|
+
none: false
|
70
|
+
requirements:
|
71
|
+
- - ! '>='
|
72
|
+
- !ruby/object:Gem::Version
|
73
|
+
version: '0'
|
74
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
75
|
+
none: false
|
76
|
+
requirements:
|
77
|
+
- - ! '>='
|
78
|
+
- !ruby/object:Gem::Version
|
79
|
+
version: '0'
|
80
|
+
requirements: []
|
81
|
+
rubyforge_project:
|
82
|
+
rubygems_version: 1.8.21
|
83
|
+
signing_key:
|
84
|
+
specification_version: 3
|
85
|
+
summary: plugin to re-emit messages with amplified values
|
86
|
+
test_files:
|
87
|
+
- test/helper.rb
|
88
|
+
- test/plugin/test_out_amplifier_filter.rb
|