fluent-plugin-suppress 1.0.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.github/workflows/test.yml +18 -0
- data/README.md +2 -0
- data/fluent-plugin-suppress.gemspec +1 -1
- data/lib/fluent/plugin/filter_suppress.rb +9 -1
- data/lib/fluent/plugin/out_suppress.rb +10 -2
- data/test/plugin/test_filter_suppress.rb +33 -0
- data/test/plugin/test_out_suppress.rb +31 -0
- metadata +4 -5
- data/.travis.yml +0 -8
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
|
-
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: abb843eab25b8ca62438b6342b38f189cfa73e500c51d4fd6c51976f6080b970
|
|
4
|
+
data.tar.gz: e5c98d0402bd7402b08d3342bd64babb0da25923e8dcb5be91c8c3883f1020e2
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6d3eb6c899fbee42d3a4464b9188847028c0e9c5bf646dcfe02d876dc1e6a10ea32995c5edf80b5e17d8e94282a3f69fbfa816cab06bf09ba129be41e4bab0ea
|
|
7
|
+
data.tar.gz: 6c7dbed41a8b0a0dcce9c9b36a6b4afd7659ad0695e784ed5bc5b4aafcf9cabcef6e20af81b7c39645a1872f37903be0f569eaf3942700e3d79127bfb7a70ec9
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
name: test
|
|
2
|
+
on: [push]
|
|
3
|
+
jobs:
|
|
4
|
+
build:
|
|
5
|
+
runs-on: ubuntu-latest
|
|
6
|
+
strategy:
|
|
7
|
+
matrix:
|
|
8
|
+
ruby: ["3.1", "3.2", "3.3", "3.4"]
|
|
9
|
+
name: test on Ruby ${{ matrix.ruby }}
|
|
10
|
+
steps:
|
|
11
|
+
- uses: actions/checkout@v4
|
|
12
|
+
- uses: ruby/setup-ruby@v1
|
|
13
|
+
with:
|
|
14
|
+
ruby-version: ${{ matrix.ruby }}
|
|
15
|
+
bundler-cache: true
|
|
16
|
+
- run: bundle install
|
|
17
|
+
- run: |
|
|
18
|
+
bundle exec rake test
|
data/README.md
CHANGED
|
@@ -15,12 +15,14 @@ fluentd.conf
|
|
|
15
15
|
@type suppress
|
|
16
16
|
interval 10
|
|
17
17
|
num 2
|
|
18
|
+
max_slot_num 100000
|
|
18
19
|
attr_keys host,message
|
|
19
20
|
add_tag_prefix sp.
|
|
20
21
|
</match>
|
|
21
22
|
```
|
|
22
23
|
|
|
23
24
|
In `interval` sec, `num` messages which grouped by `attr_keys` value, add tag prefix "`sp.`" and pass to follow process. Other messages will be removed.
|
|
25
|
+
`max_slot_num` put an upper limit on the length of the internal variable which contains message occurences.
|
|
24
26
|
|
|
25
27
|
Input messages:
|
|
26
28
|
|
|
@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
|
|
|
13
13
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
|
14
14
|
gem.name = "fluent-plugin-suppress"
|
|
15
15
|
gem.require_paths = ["lib"]
|
|
16
|
-
gem.version = "1.
|
|
16
|
+
gem.version = "1.1.1"
|
|
17
17
|
|
|
18
18
|
gem.add_runtime_dependency "fluentd", [">= 0.14.8", "< 2"]
|
|
19
19
|
gem.add_development_dependency "rake", ">= 0.9.2"
|
|
@@ -6,6 +6,7 @@ module Fluent::Plugin
|
|
|
6
6
|
|
|
7
7
|
config_param :attr_keys, :string, default: nil
|
|
8
8
|
config_param :num, :integer, default: 3
|
|
9
|
+
config_param :max_slot_num, :integer, default: 100000
|
|
9
10
|
config_param :interval, :integer, default: 300
|
|
10
11
|
|
|
11
12
|
def configure(conf)
|
|
@@ -34,10 +35,17 @@ module Fluent::Plugin
|
|
|
34
35
|
end
|
|
35
36
|
|
|
36
37
|
if slot.length >= @num
|
|
37
|
-
log.debug "suppressed record: #{record.to_json}"
|
|
38
|
+
log.debug { "suppressed record: #{record.to_json}" }
|
|
38
39
|
next
|
|
39
40
|
end
|
|
40
41
|
|
|
42
|
+
if @slots.length > @max_slot_num
|
|
43
|
+
(evict_key, evict_slot) = @slots.shift
|
|
44
|
+
if evict_slot.last && (evict_slot.last > expired)
|
|
45
|
+
log.warn { "@slots length exceeded @max_slot_num: #{@max_slot_num}. Evicted slot for the key: #{evict_key}" }
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
41
49
|
slot.push(time.to_f)
|
|
42
50
|
new_es.add(time, record)
|
|
43
51
|
end
|
|
@@ -10,6 +10,7 @@ module Fluent::Plugin
|
|
|
10
10
|
|
|
11
11
|
config_param :attr_keys, :string, default: nil
|
|
12
12
|
config_param :num, :integer, default: 3
|
|
13
|
+
config_param :max_slot_num, :integer, default: 100000
|
|
13
14
|
config_param :interval, :integer, default: 300
|
|
14
15
|
|
|
15
16
|
def configure(conf)
|
|
@@ -48,17 +49,24 @@ module Fluent::Plugin
|
|
|
48
49
|
end
|
|
49
50
|
|
|
50
51
|
if slot.length >= @num
|
|
51
|
-
log.debug "suppressed record: #{record.to_json}"
|
|
52
|
+
log.debug { "suppressed record: #{record.to_json}" }
|
|
52
53
|
next
|
|
53
54
|
end
|
|
54
55
|
|
|
56
|
+
if @slots.length > @max_slot_num
|
|
57
|
+
(evict_key, evict_slot) = @slots.shift
|
|
58
|
+
if evict_slot.last && (evict_slot.last > expired)
|
|
59
|
+
log.warn { "@slots length exceeded @max_slot_num: #{@max_slot_num}. Evicted slot for the key: #{evict_key}" }
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
55
63
|
slot.push(time.to_f)
|
|
56
64
|
_tag = tag.clone
|
|
57
65
|
filter_record(_tag, time, record)
|
|
58
66
|
if @labelled || tag != _tag
|
|
59
67
|
router.emit(_tag, time, record)
|
|
60
68
|
else
|
|
61
|
-
log.warn "Drop record #{record} tag '#{tag}' was not replaced. Can't emit record, cause infinity looping. Set remove_tag_prefix, remove_tag_suffix, add_tag_prefix or add_tag_suffix correctly."
|
|
69
|
+
log.warn { "Drop record #{record} tag '#{tag}' was not replaced. Can't emit record, cause infinity looping. Set remove_tag_prefix, remove_tag_suffix, add_tag_prefix or add_tag_suffix correctly." }
|
|
62
70
|
end
|
|
63
71
|
end
|
|
64
72
|
end
|
|
@@ -26,6 +26,13 @@ class SuppressFilterTest < Test::Unit::TestCase
|
|
|
26
26
|
num 2
|
|
27
27
|
]
|
|
28
28
|
|
|
29
|
+
CONFIG_MAX_SLOT_NUM = %[
|
|
30
|
+
interval 10
|
|
31
|
+
num 2
|
|
32
|
+
max_slot_num 3
|
|
33
|
+
attr_keys host, message
|
|
34
|
+
]
|
|
35
|
+
|
|
29
36
|
def create_driver(conf = CONFIG)
|
|
30
37
|
Fluent::Test::Driver::Filter.new(Fluent::Plugin::SuppressFilter).configure(conf)
|
|
31
38
|
end
|
|
@@ -109,4 +116,30 @@ class SuppressFilterTest < Test::Unit::TestCase
|
|
|
109
116
|
assert_equal({"id"=>6, "host"=>"web06", "message"=>"6 error!!"}, records[2])
|
|
110
117
|
assert_equal({"id"=>7, "host"=>"web07", "message"=>"7 error!!"}, records[3])
|
|
111
118
|
end
|
|
119
|
+
|
|
120
|
+
def test_emit_max_slot_num
|
|
121
|
+
d = create_driver(CONFIG_MAX_SLOT_NUM)
|
|
122
|
+
es = Fluent::MultiEventStream.new
|
|
123
|
+
|
|
124
|
+
time = event_time("2012-11-22 11:22:33 UTC")
|
|
125
|
+
es.add(time + 1, {"id" => 1, "host" => "web01", "message" => "1 error!!"})
|
|
126
|
+
es.add(time + 2, {"id" => 2, "host" => "web02", "message" => "2 error!!"})
|
|
127
|
+
es.add(time + 3, {"id" => 3, "host" => "web03", "message" => "3 error!!"})
|
|
128
|
+
es.add(time + 4, {"id" => 4, "host" => "web01", "message" => "1 error!!"})
|
|
129
|
+
es.add(time + 5, {"id" => 5, "host" => "web04", "message" => "4 error!!"})
|
|
130
|
+
es.add(time + 6, {"id" => 6, "host" => "web01", "message" => "1 error!!"})
|
|
131
|
+
|
|
132
|
+
d.run(default_tag: "test.info") do
|
|
133
|
+
d.feed(es)
|
|
134
|
+
end
|
|
135
|
+
records = d.filtered_records
|
|
136
|
+
|
|
137
|
+
assert_equal 6, records.length
|
|
138
|
+
assert_equal({"id"=>1, "host"=>"web01", "message"=>"1 error!!"}, records[0])
|
|
139
|
+
assert_equal({"id"=>2, "host"=>"web02", "message"=>"2 error!!"}, records[1])
|
|
140
|
+
assert_equal({"id"=>3, "host"=>"web03", "message"=>"3 error!!"}, records[2])
|
|
141
|
+
assert_equal({"id"=>4, "host"=>"web01", "message"=>"1 error!!"}, records[3])
|
|
142
|
+
assert_equal({"id"=>5, "host"=>"web04", "message"=>"4 error!!"}, records[4])
|
|
143
|
+
assert_equal({"id"=>6, "host"=>"web01", "message"=>"1 error!!"}, records[5])
|
|
144
|
+
end
|
|
112
145
|
end
|
|
@@ -27,6 +27,14 @@ class SuppressOutputTest < Test::Unit::TestCase
|
|
|
27
27
|
add_tag_prefix sp.
|
|
28
28
|
]
|
|
29
29
|
|
|
30
|
+
CONFIG_MAX_SLOT_NUM = %[
|
|
31
|
+
interval 10
|
|
32
|
+
num 2
|
|
33
|
+
max_slot_num 3
|
|
34
|
+
attr_keys host, message
|
|
35
|
+
add_tag_prefix sp.
|
|
36
|
+
]
|
|
37
|
+
|
|
30
38
|
def create_driver(conf = CONFIG)
|
|
31
39
|
Fluent::Test::Driver::Output.new(Fluent::Plugin::SuppressOutput).configure(conf)
|
|
32
40
|
end
|
|
@@ -103,4 +111,27 @@ class SuppressOutputTest < Test::Unit::TestCase
|
|
|
103
111
|
assert_equal ["sp.test.info", time + 12, {"id"=>6, "host"=>"web06", "message"=>"6 error!!"}], events[2]
|
|
104
112
|
assert_equal ["sp.test.info", time + 13, {"id"=>7, "host"=>"web07", "message"=>"7 error!!"}], events[3]
|
|
105
113
|
end
|
|
114
|
+
|
|
115
|
+
def test_emit_max_slot_num
|
|
116
|
+
d = create_driver(CONFIG_MAX_SLOT_NUM)
|
|
117
|
+
|
|
118
|
+
time = event_time("2012-11-22 11:22:33 UTC")
|
|
119
|
+
d.run(default_tag: "test.info") do
|
|
120
|
+
d.feed(time + 1, {"id" => 1, "host" => "web01", "message" => "1 error!!"})
|
|
121
|
+
d.feed(time + 2, {"id" => 2, "host" => "web02", "message" => "2 error!!"})
|
|
122
|
+
d.feed(time + 3, {"id" => 3, "host" => "web03", "message" => "3 error!!"})
|
|
123
|
+
d.feed(time + 4, {"id" => 4, "host" => "web01", "message" => "1 error!!"})
|
|
124
|
+
d.feed(time + 5, {"id" => 5, "host" => "web04", "message" => "4 error!!"})
|
|
125
|
+
d.feed(time + 6, {"id" => 6, "host" => "web01", "message" => "1 error!!"})
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
events = d.events
|
|
129
|
+
assert_equal 6, events.length
|
|
130
|
+
assert_equal ["sp.test.info", time + 1, {"id"=>1, "host"=>"web01", "message"=>"1 error!!"}], events[0]
|
|
131
|
+
assert_equal ["sp.test.info", time + 2, {"id"=>2, "host"=>"web02", "message"=>"2 error!!"}], events[1]
|
|
132
|
+
assert_equal ["sp.test.info", time + 3, {"id"=>3, "host"=>"web03", "message"=>"3 error!!"}], events[2]
|
|
133
|
+
assert_equal ["sp.test.info", time + 4, {"id"=>4, "host"=>"web01", "message"=>"1 error!!"}], events[3]
|
|
134
|
+
assert_equal ["sp.test.info", time + 5, {"id"=>5, "host"=>"web04", "message"=>"4 error!!"}], events[4]
|
|
135
|
+
assert_equal ["sp.test.info", time + 6, {"id"=>6, "host"=>"web01", "message"=>"1 error!!"}], events[5]
|
|
136
|
+
end
|
|
106
137
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-suppress
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.1.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- FUJIWARA Shunichiro
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2025-03-28 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: fluentd
|
|
@@ -65,8 +65,8 @@ executables: []
|
|
|
65
65
|
extensions: []
|
|
66
66
|
extra_rdoc_files: []
|
|
67
67
|
files:
|
|
68
|
+
- ".github/workflows/test.yml"
|
|
68
69
|
- ".gitignore"
|
|
69
|
-
- ".travis.yml"
|
|
70
70
|
- Changes
|
|
71
71
|
- Gemfile
|
|
72
72
|
- LICENSE
|
|
@@ -97,8 +97,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
97
97
|
- !ruby/object:Gem::Version
|
|
98
98
|
version: '0'
|
|
99
99
|
requirements: []
|
|
100
|
-
|
|
101
|
-
rubygems_version: 2.6.11
|
|
100
|
+
rubygems_version: 3.4.10
|
|
102
101
|
signing_key:
|
|
103
102
|
specification_version: 4
|
|
104
103
|
summary: Fluentd filter plugin to suppress same messages
|