fluent-plugin-suppress 0.0.5 → 0.0.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +100 -0
- data/fluent-plugin-supress.gemspec +1 -1
- data/lib/fluent/plugin/filter_suppress.rb +54 -0
- data/test/plugin/test_filter_suppress.rb +113 -0
- metadata +6 -3
- data/README.rdoc +0 -71
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 2aae4d23ad644c4d3108686f8fed825e7f399b16
|
|
4
|
+
data.tar.gz: c5b4aacdd63741a43111a400b032fcf3f8776df1
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: c1df1e8f388f1827a6e2530e15bc35b8636928903f4d89e34b09a8c662237442a02899df4c03251db65fbeda6c308639ca1d78a5edaa9625d5e56ca51549e793
|
|
7
|
+
data.tar.gz: 815337b3468853a34c8171dfe1f43b431698cf34b91633eb3228fac64885d39812e650fb304dce88fb57d9974d3e225f59d742cbe0bf1e7daccc877a38f76cfb
|
data/README.md
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# fluent-plugin-suppress
|
|
2
|
+
|
|
3
|
+
[](https://travis-ci.org/fujiwara/fluent-plugin-suppress)
|
|
4
|
+
|
|
5
|
+
## SuppressOutout / SuppressFilter
|
|
6
|
+
|
|
7
|
+
Fluentd plugin to suppress same messages.
|
|
8
|
+
|
|
9
|
+
## Configuration
|
|
10
|
+
|
|
11
|
+
fluentd.conf
|
|
12
|
+
|
|
13
|
+
```
|
|
14
|
+
<match foo.**>
|
|
15
|
+
type suppress
|
|
16
|
+
interval 10
|
|
17
|
+
num 2
|
|
18
|
+
attr_keys host,message
|
|
19
|
+
add_tag_prefix sp.
|
|
20
|
+
</match>
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
In `interval` sec, `num` messages which grouped by `attr_keys` value, add tag prefix "`sp.`" and pass to follow process. Other messages will be removed.
|
|
24
|
+
|
|
25
|
+
Input messages:
|
|
26
|
+
|
|
27
|
+
```
|
|
28
|
+
2012-11-22T11:22:33 foo.info {"id":1,"host":"web01","message":"error!!"}
|
|
29
|
+
2012-11-22T11:22:34 foo.info {"id":2,"host":"web01","message":"error!!"}
|
|
30
|
+
* 2012-11-22T11:22:35 foo.info {"id":3,"host":"web01","message":"error!!"}
|
|
31
|
+
* 2012-11-22T11:22:36 foo.info {"id":4,"host":"web01","message":"error!!"}
|
|
32
|
+
2012-11-22T11:22:37 foo.info {"id":5,"host":"app01","message":"error!!"}
|
|
33
|
+
* 2012-11-22T11:22:38 foo.info {"id":6,"host":"web01","message":"error!!"}
|
|
34
|
+
* 2012-11-22T11:22:39 foo.info {"id":7,"host":"web01","message":"error!!"}
|
|
35
|
+
* 2012-11-22T11:22:40 foo.info {"id":8,"host":"web01","message":"error!!"}
|
|
36
|
+
2012-11-22T11:22:45 foo.info {"id":9,"host":"web01","message":"error!!"}
|
|
37
|
+
```
|
|
38
|
+
(* = suppressed)
|
|
39
|
+
|
|
40
|
+
Output messages:
|
|
41
|
+
|
|
42
|
+
```
|
|
43
|
+
2012-11-22T11:22:33 sp.foo.info {"id":1,"host":"web01","message":"error!!"}
|
|
44
|
+
2012-11-22T11:22:34 sp.foo.info {"id":2,"host":"web01","message":"error!!"}
|
|
45
|
+
2012-11-22T11:22:37 sp.foo.info {"id":5,"host":"app01","message":"error!!"}
|
|
46
|
+
2012-11-22T11:22:45 sp.foo.info {"id":9,"host":"web01","message":"error!!"}
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
### Filter plugin
|
|
50
|
+
|
|
51
|
+
Fluentd >= v0.12 can use filter plugin.
|
|
52
|
+
|
|
53
|
+
```
|
|
54
|
+
<filter foo.**>
|
|
55
|
+
type suppress
|
|
56
|
+
interval 10
|
|
57
|
+
num 2
|
|
58
|
+
attr_keys host,message
|
|
59
|
+
</filter>
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Filter plugin will not replace a tag.
|
|
63
|
+
|
|
64
|
+
### Group by nested key name
|
|
65
|
+
|
|
66
|
+
`attr_keys` allows nested key name (eg. `foo.bar`).
|
|
67
|
+
|
|
68
|
+
```
|
|
69
|
+
<match foo.**>
|
|
70
|
+
type suppress
|
|
71
|
+
attr_keys data.host, data.message
|
|
72
|
+
</match>
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
Input messages will be suppressed by key data.host and data.message.
|
|
76
|
+
|
|
77
|
+
```
|
|
78
|
+
2012-11-22T11:22:33 foo.info {"id":1,"data":{"host":"web01","message":"error!!"}}
|
|
79
|
+
2012-11-22T11:22:34 foo.info {"id":2,"data":{"host":"web01","message":"error!!"}}
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
### Suppressing by tag only
|
|
83
|
+
|
|
84
|
+
If `attr_keys` is not specified, records will be suppressed by tag only.
|
|
85
|
+
|
|
86
|
+
```
|
|
87
|
+
<match foo.**>
|
|
88
|
+
type suppress
|
|
89
|
+
...
|
|
90
|
+
</match>
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
## TODO
|
|
94
|
+
|
|
95
|
+
patches welcome!
|
|
96
|
+
|
|
97
|
+
## Copyright
|
|
98
|
+
|
|
99
|
+
Copyright (c) 2012- FUJIWARA Shunichiro
|
|
100
|
+
License Apache License, Version 2.0
|
|
@@ -14,7 +14,7 @@ Gem::Specification.new do |gem|
|
|
|
14
14
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
|
15
15
|
gem.name = "fluent-plugin-suppress"
|
|
16
16
|
gem.require_paths = ["lib"]
|
|
17
|
-
gem.version = "0.0.
|
|
17
|
+
gem.version = "0.0.6"
|
|
18
18
|
|
|
19
19
|
gem.add_runtime_dependency "fluentd", ">= 0.10.0"
|
|
20
20
|
gem.add_development_dependency "rake", ">= 0.9.2"
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
module Fluent
|
|
3
|
+
class SuppressFilter < Filter
|
|
4
|
+
Fluent::Plugin.register_filter('suppress', self)
|
|
5
|
+
|
|
6
|
+
config_param :attr_keys, :string, :default => nil
|
|
7
|
+
config_param :num, :integer, :default => 3
|
|
8
|
+
config_param :interval, :integer, :default => 300
|
|
9
|
+
|
|
10
|
+
def configure(conf)
|
|
11
|
+
super
|
|
12
|
+
@keys = @attr_keys ? @attr_keys.split(/ *, */) : nil
|
|
13
|
+
@slots = {}
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def start
|
|
17
|
+
super
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def shutdown
|
|
21
|
+
super
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def filter_stream(tag, es)
|
|
25
|
+
new_es = MultiEventStream.new
|
|
26
|
+
es.each do |time, record|
|
|
27
|
+
if @keys
|
|
28
|
+
keys = @keys.map do |key|
|
|
29
|
+
key.split(/\./).inject(record) {|r, k| r[k] }
|
|
30
|
+
end
|
|
31
|
+
key = tag + "\0" + keys.join("\0")
|
|
32
|
+
else
|
|
33
|
+
key = tag
|
|
34
|
+
end
|
|
35
|
+
slot = @slots[key] ||= []
|
|
36
|
+
|
|
37
|
+
# expire old records time
|
|
38
|
+
expired = time.to_f - @interval
|
|
39
|
+
while slot.first && (slot.first <= expired)
|
|
40
|
+
slot.shift
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
if slot.length >= @num
|
|
44
|
+
log.debug "suppressed record: #{record.to_json}"
|
|
45
|
+
next
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
slot.push(time.to_f)
|
|
49
|
+
new_es.add(time, record)
|
|
50
|
+
end
|
|
51
|
+
return new_es
|
|
52
|
+
end
|
|
53
|
+
end if defined?(Filter) # Support only >= v0.12
|
|
54
|
+
end
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
require 'test/unit'
|
|
2
|
+
require 'fluent/log'
|
|
3
|
+
require 'fluent/test'
|
|
4
|
+
require 'fluent/plugin/filter_suppress'
|
|
5
|
+
|
|
6
|
+
class SuppressFilterTest < Test::Unit::TestCase
|
|
7
|
+
include Fluent
|
|
8
|
+
|
|
9
|
+
def setup
|
|
10
|
+
Fluent::Test.setup
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
CONFIG = %[
|
|
14
|
+
interval 10
|
|
15
|
+
num 2
|
|
16
|
+
attr_keys host, message
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
CONFIG_WITH_NESTED_KEY = %[
|
|
20
|
+
interval 10
|
|
21
|
+
num 2
|
|
22
|
+
attr_keys data.host, data.message
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
CONFIG_TAG_ONLY = %[
|
|
26
|
+
interval 10
|
|
27
|
+
num 2
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
def create_driver(conf = CONFIG, tag='test.info')
|
|
31
|
+
Fluent::Test::FilterTestDriver.new(Fluent::SuppressFilter).configure(conf, tag)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def test_emit
|
|
35
|
+
return unless defined? Fluent::Filter
|
|
36
|
+
|
|
37
|
+
d = create_driver(CONFIG)
|
|
38
|
+
es = Fluent::MultiEventStream.new
|
|
39
|
+
|
|
40
|
+
time = Time.parse("2012-11-22 11:22:33 UTC").to_i
|
|
41
|
+
es.add(time + 1, {"id" => 1, "host" => "web01", "message" => "error!!"})
|
|
42
|
+
es.add(time + 2, {"id" => 2, "host" => "web01", "message" => "error!!"})
|
|
43
|
+
es.add(time + 3, {"id" => 3, "host" => "web01", "message" => "error!!"})
|
|
44
|
+
es.add(time + 4, {"id" => 4, "host" => "web01", "message" => "error!!"})
|
|
45
|
+
es.add(time + 4, {"id" => 5, "host" => "app01", "message" => "error!!"})
|
|
46
|
+
es.add(time + 12, {"id" => 6, "host" => "web01", "message" => "error!!"})
|
|
47
|
+
es.add(time + 13, {"id" => 7, "host" => "web01", "message" => "error!!"})
|
|
48
|
+
es.add(time + 14, {"id" => 8, "host" => "web01", "message" => "error!!"})
|
|
49
|
+
|
|
50
|
+
filtered_es = d.filter_stream('test.info', es)
|
|
51
|
+
records = filtered_es.instance_variable_get(:@record_array)
|
|
52
|
+
assert_equal 5, records.length
|
|
53
|
+
assert_equal({"id" => 1, "host" => "web01", "message" => "error!!"}, records[0])
|
|
54
|
+
assert_equal({"id" => 2, "host" => "web01", "message" => "error!!"}, records[1])
|
|
55
|
+
assert_equal({"id" => 5, "host" => "app01", "message" => "error!!"}, records[2])
|
|
56
|
+
assert_equal({"id" => 6, "host" => "web01", "message" => "error!!"}, records[3])
|
|
57
|
+
assert_equal({"id" => 7, "host" => "web01", "message" => "error!!"}, records[4])
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def test_emit_wtih_nested_key
|
|
61
|
+
return unless defined? Fluent::Filter
|
|
62
|
+
|
|
63
|
+
d = create_driver(CONFIG_WITH_NESTED_KEY)
|
|
64
|
+
es = Fluent::MultiEventStream.new
|
|
65
|
+
|
|
66
|
+
time = Time.parse("2012-11-22 11:22:33 UTC").to_i
|
|
67
|
+
es.add(time + 1, {"id" => 1, "data" => {"host" => "web01", "message" => "error!!"}})
|
|
68
|
+
es.add(time + 2, {"id" => 2, "data" => {"host" => "web01", "message" => "error!!"}})
|
|
69
|
+
es.add(time + 3, {"id" => 3, "data" => {"host" => "web01", "message" => "error!!"}})
|
|
70
|
+
es.add(time + 4, {"id" => 4, "data" => {"host" => "web01", "message" => "error!!"}})
|
|
71
|
+
es.add(time + 4, {"id" => 5, "data" => {"host" => "app01", "message" => "error!!"}})
|
|
72
|
+
es.add(time + 12, {"id" => 6, "data" => {"host" => "web01", "message" => "error!!"}})
|
|
73
|
+
es.add(time + 13, {"id" => 7, "data" => {"host" => "web01", "message" => "error!!"}})
|
|
74
|
+
es.add(time + 14, {"id" => 8, "data" => {"host" => "web01", "message" => "error!!"}})
|
|
75
|
+
|
|
76
|
+
filtered_es = d.filter_stream('test.info', es)
|
|
77
|
+
records = filtered_es.instance_variable_get(:@record_array)
|
|
78
|
+
|
|
79
|
+
assert_equal 5, records.length
|
|
80
|
+
assert_equal({"id"=>1, "data" => {"host"=>"web01", "message"=>"error!!"}}, records[0])
|
|
81
|
+
assert_equal({"id"=>2, "data" => {"host"=>"web01", "message"=>"error!!"}}, records[1])
|
|
82
|
+
assert_equal({"id"=>5, "data" => {"host"=>"app01", "message"=>"error!!"}}, records[2])
|
|
83
|
+
assert_equal({"id"=>6, "data" => {"host"=>"web01", "message"=>"error!!"}}, records[3])
|
|
84
|
+
assert_equal({"id"=>7, "data" => {"host"=>"web01", "message"=>"error!!"}}, records[4])
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def test_emit_tagonly
|
|
88
|
+
return unless defined? Fluent::Filter
|
|
89
|
+
|
|
90
|
+
d = create_driver(CONFIG_TAG_ONLY)
|
|
91
|
+
es = Fluent::MultiEventStream.new
|
|
92
|
+
|
|
93
|
+
time = Time.parse("2012-11-22 11:22:33 UTC").to_i
|
|
94
|
+
es.add(time + 1, {"id" => 1, "host" => "web01", "message" => "1 error!!"})
|
|
95
|
+
es.add(time + 2, {"id" => 2, "host" => "web02", "message" => "2 error!!"})
|
|
96
|
+
es.add(time + 3, {"id" => 3, "host" => "web03", "message" => "3 error!!"})
|
|
97
|
+
es.add(time + 4, {"id" => 4, "host" => "web04", "message" => "4 error!!"})
|
|
98
|
+
es.add(time + 4, {"id" => 5, "host" => "app05", "message" => "5 error!!"})
|
|
99
|
+
es.add(time + 12,{"id" => 6, "host" => "web06", "message" => "6 error!!"})
|
|
100
|
+
es.add(time + 13,{"id" => 7, "host" => "web07", "message" => "7 error!!"})
|
|
101
|
+
es.add(time + 14,{"id" => 8, "host" => "web08", "message" => "8 error!!"})
|
|
102
|
+
|
|
103
|
+
filtered_es = d.filter_stream('test.info', es)
|
|
104
|
+
records = filtered_es.instance_variable_get(:@record_array)
|
|
105
|
+
|
|
106
|
+
assert_equal 4, records.length
|
|
107
|
+
assert_equal({"id"=>1, "host"=>"web01", "message"=>"1 error!!"}, records[0])
|
|
108
|
+
assert_equal({"id"=>2, "host"=>"web02", "message"=>"2 error!!"}, records[1])
|
|
109
|
+
assert_equal({"id"=>6, "host"=>"web06", "message"=>"6 error!!"}, records[2])
|
|
110
|
+
assert_equal({"id"=>7, "host"=>"web07", "message"=>"7 error!!"}, records[3])
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-suppress
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0.
|
|
4
|
+
version: 0.0.6
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- FUJIWARA Shunichiro
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2014-12-
|
|
11
|
+
date: 2014-12-17 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: fluentd
|
|
@@ -51,11 +51,13 @@ files:
|
|
|
51
51
|
- Gemfile
|
|
52
52
|
- Gemfile.fluentd.lt.0.10.43
|
|
53
53
|
- LICENSE
|
|
54
|
-
- README.
|
|
54
|
+
- README.md
|
|
55
55
|
- Rakefile
|
|
56
56
|
- fluent-plugin-supress.gemspec
|
|
57
|
+
- lib/fluent/plugin/filter_suppress.rb
|
|
57
58
|
- lib/fluent/plugin/out_suppress.rb
|
|
58
59
|
- test/helper.rb
|
|
60
|
+
- test/plugin/test_filter_suppress.rb
|
|
59
61
|
- test/plugin/test_out_suppress.rb
|
|
60
62
|
homepage: https://github.com/fujiwara/fluent-plugin-suppress
|
|
61
63
|
licenses:
|
|
@@ -83,4 +85,5 @@ specification_version: 4
|
|
|
83
85
|
summary: Fluentd plugin to suppress same messages
|
|
84
86
|
test_files:
|
|
85
87
|
- test/helper.rb
|
|
88
|
+
- test/plugin/test_filter_suppress.rb
|
|
86
89
|
- test/plugin/test_out_suppress.rb
|
data/README.rdoc
DELETED
|
@@ -1,71 +0,0 @@
|
|
|
1
|
-
= fluent-plugin-suppress
|
|
2
|
-
|
|
3
|
-
== Component
|
|
4
|
-
|
|
5
|
-
=== SuppressOutout
|
|
6
|
-
|
|
7
|
-
Fluentd plugin to suppress same messages.
|
|
8
|
-
|
|
9
|
-
== Configuration
|
|
10
|
-
|
|
11
|
-
=== SuppressOutput
|
|
12
|
-
|
|
13
|
-
fluentd.conf
|
|
14
|
-
|
|
15
|
-
<match foo.**>
|
|
16
|
-
type suppress
|
|
17
|
-
interval 10
|
|
18
|
-
num 2
|
|
19
|
-
attr_keys host,message
|
|
20
|
-
add_tag_prefix sp.
|
|
21
|
-
</match>
|
|
22
|
-
|
|
23
|
-
In [interval] sec, [num] messages which grouped by attr_keys value, add tag prefix "sp" and pass. Other messages will be removed.
|
|
24
|
-
|
|
25
|
-
Input messages:
|
|
26
|
-
|
|
27
|
-
2012-11-22T11:22:33 foo.info {"id":1,"host":"web01","message":"error!!"}
|
|
28
|
-
2012-11-22T11:22:34 foo.info {"id":2,"host":"web01","message":"error!!"}
|
|
29
|
-
* 2012-11-22T11:22:35 foo.info {"id":3,"host":"web01","message":"error!!"}
|
|
30
|
-
* 2012-11-22T11:22:36 foo.info {"id":4,"host":"web01","message":"error!!"}
|
|
31
|
-
2012-11-22T11:22:37 foo.info {"id":5,"host":"app01","message":"error!!"}
|
|
32
|
-
* 2012-11-22T11:22:38 foo.info {"id":6,"host":"web01","message":"error!!"}
|
|
33
|
-
* 2012-11-22T11:22:39 foo.info {"id":7,"host":"web01","message":"error!!"}
|
|
34
|
-
* 2012-11-22T11:22:40 foo.info {"id":8,"host":"web01","message":"error!!"}
|
|
35
|
-
2012-11-22T11:22:45 foo.info {"id":9,"host":"web01","message":"error!!"}
|
|
36
|
-
(* = suppressed)
|
|
37
|
-
|
|
38
|
-
Output messages:
|
|
39
|
-
|
|
40
|
-
2012-11-22T11:22:33 sp.foo.info {"id":1,"host":"web01","message":"error!!"}
|
|
41
|
-
2012-11-22T11:22:34 sp.foo.info {"id":2,"host":"web01","message":"error!!"}
|
|
42
|
-
2012-11-22T11:22:37 sp.foo.info {"id":5,"host":"app01","message":"error!!"}
|
|
43
|
-
2012-11-22T11:22:45 sp.foo.info {"id":9,"host":"web01","message":"error!!"}
|
|
44
|
-
|
|
45
|
-
attr_keys allows nested key name (eg. foo.bar).
|
|
46
|
-
|
|
47
|
-
<match foo.**>
|
|
48
|
-
type suppress
|
|
49
|
-
attr_keys data.host, data.message
|
|
50
|
-
</match>
|
|
51
|
-
|
|
52
|
-
Input messages will be suppressed by key data.host and data.message.
|
|
53
|
-
|
|
54
|
-
2012-11-22T11:22:33 foo.info {"id":1,"data":{"host":"web01","message":"error!!"}}
|
|
55
|
-
2012-11-22T11:22:34 foo.info {"id":2,"data":{"host":"web01","message":"error!!"}}
|
|
56
|
-
|
|
57
|
-
If attr_keys is not specified, suppressed by tag only.
|
|
58
|
-
|
|
59
|
-
<match foo.**>
|
|
60
|
-
type suppress
|
|
61
|
-
</match>
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
== TODO
|
|
65
|
-
|
|
66
|
-
- patches welcome!
|
|
67
|
-
|
|
68
|
-
== Copyright
|
|
69
|
-
|
|
70
|
-
Copyright:: Copyright (c) 2012- FUJIWARA Shunichiro
|
|
71
|
-
License:: Apache License, Version 2.0
|