fluent-plugin-suppress 1.0.0 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +1 -0
- data/README.md +2 -0
- data/fluent-plugin-suppress.gemspec +1 -1
- data/lib/fluent/plugin/filter_suppress.rb +8 -0
- data/lib/fluent/plugin/out_suppress.rb +8 -0
- data/test/plugin/test_filter_suppress.rb +33 -0
- data/test/plugin/test_out_suppress.rb +31 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 932d867f3ee590af09d05ecbc3ffb5011db6d3b6
|
4
|
+
data.tar.gz: c075ee6fdb2bc95efd4609f82cd50300525010a4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ce6988ea6023d58966295a038864b3e39973f14c456cda0dfde4c96048dd4fdfbce4dc229f44de34daf8eecc19616edd8a6e3464ceeaec654967a08491751e72
|
7
|
+
data.tar.gz: f0b2d1150f429d5a875d420ca3171bb60c312cf69861b83d117fd1e421c2cb104e10a0caf28ee9231564627958dd4d8d818b6ead62feb1acd5fe5a4cf0024585
|
data/.travis.yml
CHANGED
data/README.md
CHANGED
@@ -15,12 +15,14 @@ fluentd.conf
|
|
15
15
|
@type suppress
|
16
16
|
interval 10
|
17
17
|
num 2
|
18
|
+
max_slot_num 100000
|
18
19
|
attr_keys host,message
|
19
20
|
add_tag_prefix sp.
|
20
21
|
</match>
|
21
22
|
```
|
22
23
|
|
23
24
|
In `interval` sec, `num` messages which grouped by `attr_keys` value, add tag prefix "`sp.`" and pass to follow process. Other messages will be removed.
|
25
|
+
`max_slot_num` put an upper limit on the length of the internal variable which contains message occurences.
|
24
26
|
|
25
27
|
Input messages:
|
26
28
|
|
@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
|
|
13
13
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
14
14
|
gem.name = "fluent-plugin-suppress"
|
15
15
|
gem.require_paths = ["lib"]
|
16
|
-
gem.version = "1.
|
16
|
+
gem.version = "1.1.0"
|
17
17
|
|
18
18
|
gem.add_runtime_dependency "fluentd", [">= 0.14.8", "< 2"]
|
19
19
|
gem.add_development_dependency "rake", ">= 0.9.2"
|
@@ -6,6 +6,7 @@ module Fluent::Plugin
|
|
6
6
|
|
7
7
|
config_param :attr_keys, :string, default: nil
|
8
8
|
config_param :num, :integer, default: 3
|
9
|
+
config_param :max_slot_num, :integer, default: 100000
|
9
10
|
config_param :interval, :integer, default: 300
|
10
11
|
|
11
12
|
def configure(conf)
|
@@ -38,6 +39,13 @@ module Fluent::Plugin
|
|
38
39
|
next
|
39
40
|
end
|
40
41
|
|
42
|
+
if @slots.length > @max_slot_num
|
43
|
+
(evict_key, evict_slot) = @slots.shift
|
44
|
+
if evict_slot.last && (evict_slot.last > expired)
|
45
|
+
log.warn "@slots length exceeded @max_slot_num: #{@max_slot_num}. Evicted slot for the key: #{evict_key}"
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
41
49
|
slot.push(time.to_f)
|
42
50
|
new_es.add(time, record)
|
43
51
|
end
|
@@ -10,6 +10,7 @@ module Fluent::Plugin
|
|
10
10
|
|
11
11
|
config_param :attr_keys, :string, default: nil
|
12
12
|
config_param :num, :integer, default: 3
|
13
|
+
config_param :max_slot_num, :integer, default: 100000
|
13
14
|
config_param :interval, :integer, default: 300
|
14
15
|
|
15
16
|
def configure(conf)
|
@@ -52,6 +53,13 @@ module Fluent::Plugin
|
|
52
53
|
next
|
53
54
|
end
|
54
55
|
|
56
|
+
if @slots.length > @max_slot_num
|
57
|
+
(evict_key, evict_slot) = @slots.shift
|
58
|
+
if evict_slot.last && (evict_slot.last > expired)
|
59
|
+
log.warn "@slots length exceeded @max_slot_num: #{@max_slot_num}. Evicted slot for the key: #{evict_key}"
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
55
63
|
slot.push(time.to_f)
|
56
64
|
_tag = tag.clone
|
57
65
|
filter_record(_tag, time, record)
|
@@ -26,6 +26,13 @@ class SuppressFilterTest < Test::Unit::TestCase
|
|
26
26
|
num 2
|
27
27
|
]
|
28
28
|
|
29
|
+
CONFIG_MAX_SLOT_NUM = %[
|
30
|
+
interval 10
|
31
|
+
num 2
|
32
|
+
max_slot_num 3
|
33
|
+
attr_keys host, message
|
34
|
+
]
|
35
|
+
|
29
36
|
def create_driver(conf = CONFIG)
|
30
37
|
Fluent::Test::Driver::Filter.new(Fluent::Plugin::SuppressFilter).configure(conf)
|
31
38
|
end
|
@@ -109,4 +116,30 @@ class SuppressFilterTest < Test::Unit::TestCase
|
|
109
116
|
assert_equal({"id"=>6, "host"=>"web06", "message"=>"6 error!!"}, records[2])
|
110
117
|
assert_equal({"id"=>7, "host"=>"web07", "message"=>"7 error!!"}, records[3])
|
111
118
|
end
|
119
|
+
|
120
|
+
def test_emit_max_slot_num
|
121
|
+
d = create_driver(CONFIG_MAX_SLOT_NUM)
|
122
|
+
es = Fluent::MultiEventStream.new
|
123
|
+
|
124
|
+
time = event_time("2012-11-22 11:22:33 UTC")
|
125
|
+
es.add(time + 1, {"id" => 1, "host" => "web01", "message" => "1 error!!"})
|
126
|
+
es.add(time + 2, {"id" => 2, "host" => "web02", "message" => "2 error!!"})
|
127
|
+
es.add(time + 3, {"id" => 3, "host" => "web03", "message" => "3 error!!"})
|
128
|
+
es.add(time + 4, {"id" => 4, "host" => "web01", "message" => "1 error!!"})
|
129
|
+
es.add(time + 5, {"id" => 5, "host" => "web04", "message" => "4 error!!"})
|
130
|
+
es.add(time + 6, {"id" => 6, "host" => "web01", "message" => "1 error!!"})
|
131
|
+
|
132
|
+
d.run(default_tag: "test.info") do
|
133
|
+
d.feed(es)
|
134
|
+
end
|
135
|
+
records = d.filtered_records
|
136
|
+
|
137
|
+
assert_equal 6, records.length
|
138
|
+
assert_equal({"id"=>1, "host"=>"web01", "message"=>"1 error!!"}, records[0])
|
139
|
+
assert_equal({"id"=>2, "host"=>"web02", "message"=>"2 error!!"}, records[1])
|
140
|
+
assert_equal({"id"=>3, "host"=>"web03", "message"=>"3 error!!"}, records[2])
|
141
|
+
assert_equal({"id"=>4, "host"=>"web01", "message"=>"1 error!!"}, records[3])
|
142
|
+
assert_equal({"id"=>5, "host"=>"web04", "message"=>"4 error!!"}, records[4])
|
143
|
+
assert_equal({"id"=>6, "host"=>"web01", "message"=>"1 error!!"}, records[5])
|
144
|
+
end
|
112
145
|
end
|
@@ -27,6 +27,14 @@ class SuppressOutputTest < Test::Unit::TestCase
|
|
27
27
|
add_tag_prefix sp.
|
28
28
|
]
|
29
29
|
|
30
|
+
CONFIG_MAX_SLOT_NUM = %[
|
31
|
+
interval 10
|
32
|
+
num 2
|
33
|
+
max_slot_num 3
|
34
|
+
attr_keys host, message
|
35
|
+
add_tag_prefix sp.
|
36
|
+
]
|
37
|
+
|
30
38
|
def create_driver(conf = CONFIG)
|
31
39
|
Fluent::Test::Driver::Output.new(Fluent::Plugin::SuppressOutput).configure(conf)
|
32
40
|
end
|
@@ -103,4 +111,27 @@ class SuppressOutputTest < Test::Unit::TestCase
|
|
103
111
|
assert_equal ["sp.test.info", time + 12, {"id"=>6, "host"=>"web06", "message"=>"6 error!!"}], events[2]
|
104
112
|
assert_equal ["sp.test.info", time + 13, {"id"=>7, "host"=>"web07", "message"=>"7 error!!"}], events[3]
|
105
113
|
end
|
114
|
+
|
115
|
+
def test_emit_max_slot_num
|
116
|
+
d = create_driver(CONFIG_MAX_SLOT_NUM)
|
117
|
+
|
118
|
+
time = event_time("2012-11-22 11:22:33 UTC")
|
119
|
+
d.run(default_tag: "test.info") do
|
120
|
+
d.feed(time + 1, {"id" => 1, "host" => "web01", "message" => "1 error!!"})
|
121
|
+
d.feed(time + 2, {"id" => 2, "host" => "web02", "message" => "2 error!!"})
|
122
|
+
d.feed(time + 3, {"id" => 3, "host" => "web03", "message" => "3 error!!"})
|
123
|
+
d.feed(time + 4, {"id" => 4, "host" => "web01", "message" => "1 error!!"})
|
124
|
+
d.feed(time + 5, {"id" => 5, "host" => "web04", "message" => "4 error!!"})
|
125
|
+
d.feed(time + 6, {"id" => 6, "host" => "web01", "message" => "1 error!!"})
|
126
|
+
end
|
127
|
+
|
128
|
+
events = d.events
|
129
|
+
assert_equal 6, events.length
|
130
|
+
assert_equal ["sp.test.info", time + 1, {"id"=>1, "host"=>"web01", "message"=>"1 error!!"}], events[0]
|
131
|
+
assert_equal ["sp.test.info", time + 2, {"id"=>2, "host"=>"web02", "message"=>"2 error!!"}], events[1]
|
132
|
+
assert_equal ["sp.test.info", time + 3, {"id"=>3, "host"=>"web03", "message"=>"3 error!!"}], events[2]
|
133
|
+
assert_equal ["sp.test.info", time + 4, {"id"=>4, "host"=>"web01", "message"=>"1 error!!"}], events[3]
|
134
|
+
assert_equal ["sp.test.info", time + 5, {"id"=>5, "host"=>"web04", "message"=>"4 error!!"}], events[4]
|
135
|
+
assert_equal ["sp.test.info", time + 6, {"id"=>6, "host"=>"web01", "message"=>"1 error!!"}], events[5]
|
136
|
+
end
|
106
137
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-suppress
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- FUJIWARA Shunichiro
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-09-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|