fluent-plugin-filter-list 0.2.4 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +43 -4
- data/fluent-plugin-out_filter_list.gemspec +2 -2
- data/lib/fluent/plugin/filter_filter_list.rb +16 -14
- data/lib/fluent/plugin/out_filter_list.rb +57 -48
- data/lib/fluent/plugin/out_filter_list/version.rb +4 -2
- metadata +6 -6
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 9ea3dab5080825c71ef4c5809764f0a677d46b2f
|
|
4
|
+
data.tar.gz: 1d3bee1125fba97deee08dff1f7337ed268b1a3d
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: c1325ec3086b46309c0e183b6059753ac9b298e9efb2a3c3ca55dcded59d6d0783f7b97c1f842a2116751c2e991b8a25ee92af8df1f2ffa5b22c30dfadc5a8de
|
|
7
|
+
data.tar.gz: bef22cada3e0ee5c7fc909c0e05b004fb7f2a724c5a7f1b6120fc86dcee2fb9a1e4f747caca1b0e20a070c889180ef6626c6f1e94525a9da2bada360d9df760c
|
data/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
[](https://travis-ci.org/yanana/fluent-plugin-filter-list)
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
Want to filter fluentd messages containing black-listed words in the list effectively? Use the _fluent-plugin-filter-list_ plugin. The plugin enables you to filter messages in the list of words you provide. You can either discard such messages simply, or process them in a different flow by retagging them.
|
|
6
6
|
|
|
7
7
|
## Installation
|
|
8
8
|
|
|
@@ -22,13 +22,52 @@ Or install it yourself as:
|
|
|
22
22
|
|
|
23
23
|
## Usage
|
|
24
24
|
|
|
25
|
+
This repository contains two plugins: _Filter_ and _Output_, and expects two main use cases.
|
|
26
|
+
|
|
27
|
+
### Filter plugin
|
|
28
|
+
|
|
29
|
+
Use the `filter_list` filter. Configure fluentd as follows.
|
|
30
|
+
|
|
31
|
+
```
|
|
32
|
+
<filter pattern>
|
|
33
|
+
@type filter_list
|
|
34
|
+
|
|
35
|
+
key_to_filter xyz
|
|
36
|
+
patterns_file_path blacklist.txt
|
|
37
|
+
</filter>
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
Given the `blacklist.txt` is as follows.
|
|
41
|
+
|
|
42
|
+
```
|
|
43
|
+
foo
|
|
44
|
+
bar
|
|
45
|
+
buzz
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
The following message is discarded since its `x` field contains the sequence of characters _bar_, contained in the list.
|
|
49
|
+
|
|
50
|
+
```json
|
|
51
|
+
{"x":"halbart","y":1}
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
While the following message is passed through as the target field specified in the config is not _y_ but _x_ .
|
|
55
|
+
|
|
56
|
+
```json
|
|
57
|
+
{"x":1,"y":"halbart"}
|
|
25
58
|
```
|
|
26
|
-
|
|
59
|
+
|
|
60
|
+
### Output plugin
|
|
61
|
+
|
|
62
|
+
The other use case is to filter messages likewise, but process the filtered messages in a different tag. You need to configure the plugin to tell it how to retag both non-filtered messages and filtered messages. We provide two mutually-exclusive parameters: `tag` and `add_prefix`. THe `tag` parameter tells the plugin to retag the message with the value exactly provided by the parameter. The `add_prefix` parameter tells the plugin to retag the messages with the original tag prepended with the value you provide. So if the original message had a tag _foo_ and you set the `add_prefix` parameter _filtered_, then the processed message would have the tag _filtered.foo_ (note that the period before the original tag value is also prepended).
|
|
63
|
+
|
|
64
|
+
```
|
|
65
|
+
<match pattern>
|
|
27
66
|
@type filter_list
|
|
28
|
-
|
|
67
|
+
|
|
29
68
|
key_to_filter field_name_you_want_to_filter
|
|
30
69
|
patterns_file_path file_including_patterns_separated_by_new_line
|
|
31
|
-
|
|
70
|
+
|
|
32
71
|
<retag>
|
|
33
72
|
add_prefix x # retag non-filtered messages whose tag will be "x.your_tag"
|
|
34
73
|
</retag>
|
|
@@ -7,7 +7,7 @@ require 'fluent/plugin/out_filter_list/version'
|
|
|
7
7
|
|
|
8
8
|
Gem::Specification.new do |spec|
|
|
9
9
|
spec.name = 'fluent-plugin-filter-list'
|
|
10
|
-
spec.version = Fluent::
|
|
10
|
+
spec.version = Fluent::Plugin::FilterList::VERSION
|
|
11
11
|
spec.authors = ['Shun Yanaura']
|
|
12
12
|
spec.email = ['metroplexity@gmail.com']
|
|
13
13
|
|
|
@@ -27,5 +27,5 @@ Gem::Specification.new do |spec|
|
|
|
27
27
|
spec.add_development_dependency 'rake', '~> 12.0'
|
|
28
28
|
spec.add_development_dependency 'minitest', '~> 5.0'
|
|
29
29
|
spec.add_development_dependency 'test-unit', '~> 3.2'
|
|
30
|
-
spec.add_runtime_dependency 'fluentd', '~> 0.
|
|
30
|
+
spec.add_runtime_dependency 'fluentd', '~> 0.14', '>= 0.14.0'
|
|
31
31
|
end
|
|
@@ -2,24 +2,26 @@ require 'fluent/plugin/out_filter_list/version'
|
|
|
2
2
|
require 'aho_corasick'
|
|
3
3
|
|
|
4
4
|
module Fluent
|
|
5
|
-
|
|
6
|
-
|
|
5
|
+
module Plugin
|
|
6
|
+
class FilterListFilter < Filter
|
|
7
|
+
include Matchers
|
|
7
8
|
|
|
8
|
-
|
|
9
|
+
Plugin.register_filter('filter_list', self)
|
|
9
10
|
|
|
10
|
-
|
|
11
|
-
|
|
11
|
+
config_param :key_to_filter, :string, default: nil
|
|
12
|
+
config_param :patterns_file_path, :string, default: ''
|
|
12
13
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
14
|
+
def configure(conf)
|
|
15
|
+
super
|
|
16
|
+
patterns = @patterns_file_path.empty? ? [] : File.readlines(@patterns_file_path).map(&:chomp).reject(&:empty?)
|
|
17
|
+
@matcher = ACMatcher.new(patterns)
|
|
18
|
+
end
|
|
18
19
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
20
|
+
def filter(_tag, _time, record)
|
|
21
|
+
target = record[@key_to_filter]
|
|
22
|
+
return nil if target && @matcher.matches?(target)
|
|
23
|
+
record
|
|
24
|
+
end
|
|
23
25
|
end
|
|
24
26
|
end
|
|
25
27
|
end
|
|
@@ -1,66 +1,75 @@
|
|
|
1
|
+
require 'fluent/plugin/output'
|
|
1
2
|
require 'fluent/plugin/out_filter_list/version'
|
|
2
3
|
require 'aho_corasick'
|
|
3
4
|
|
|
4
5
|
module Fluent
|
|
5
|
-
|
|
6
|
-
|
|
6
|
+
module Plugin
|
|
7
|
+
class FilterListOutput < Output
|
|
8
|
+
include Matchers
|
|
7
9
|
|
|
8
|
-
|
|
10
|
+
Plugin.register_output('filter_list', self)
|
|
9
11
|
|
|
10
|
-
|
|
11
|
-
config_param :patterns_file_path, :string, default: ''
|
|
12
|
+
helpers :event_emitter
|
|
12
13
|
|
|
13
|
-
|
|
14
|
-
config_param :
|
|
15
|
-
config_param :add_prefix, :string, default: nil
|
|
16
|
-
end
|
|
14
|
+
config_param :key_to_filter, :string, default: nil
|
|
15
|
+
config_param :patterns_file_path, :string, default: ''
|
|
17
16
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
17
|
+
config_section :retag, required: true, multi: false do
|
|
18
|
+
config_param :tag, :string, default: nil
|
|
19
|
+
config_param :add_prefix, :string, default: nil
|
|
20
|
+
end
|
|
22
21
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
22
|
+
config_section :retag_filtered, param_name: :retag_for_filtered, required: false, multi: false do
|
|
23
|
+
config_param :tag, :string, default: nil
|
|
24
|
+
config_param :add_prefix, :string, default: nil
|
|
25
|
+
end
|
|
26
26
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
raise Fluent::ConfigError, "tag and add_prefix are mutually exclusive" if retag.tag && retag.add_prefix
|
|
31
|
-
end
|
|
27
|
+
def initialize
|
|
28
|
+
super
|
|
29
|
+
end
|
|
32
30
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
31
|
+
def validate(retag)
|
|
32
|
+
return unless retag
|
|
33
|
+
raise Fluent::ConfigError, "missing tag and add_prefix" unless retag.tag || retag.add_prefix
|
|
34
|
+
raise Fluent::ConfigError, "tag and add_prefix are mutually exclusive" if retag.tag && retag.add_prefix
|
|
35
|
+
end
|
|
38
36
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
37
|
+
def configure_prefixes
|
|
38
|
+
@prefix_for_filtered_tag = @retag_for_filtered.add_prefix + '.' if @retag_for_filtered && @retag_for_filtered.add_prefix
|
|
39
|
+
@prefix_for_filtered_tag = @retag_for_filtered && @retag_for_filtered.add_prefix ? @retag_for_filtered.add_prefix + '.' : ''
|
|
40
|
+
@prefix = @retag && @retag.add_prefix ? @retag.add_prefix + '.' : ''
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def configure(conf)
|
|
44
|
+
super
|
|
45
|
+
[@retag, @retag_for_filtered].each { |c| validate c }
|
|
46
|
+
patterns = @patterns_file_path.empty? ? [] : File.readlines(@patterns_file_path).map(&:chomp).reject(&:empty?)
|
|
47
|
+
@matcher = ACMatcher.new(patterns)
|
|
48
|
+
configure_prefixes
|
|
49
|
+
log.debug "prefix: #{@prefix}, prefix_for_filtered_tag: #{@prefix_for_filtered_tag || ''}"
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def multi_workers_ready?
|
|
53
|
+
true
|
|
54
|
+
end
|
|
47
55
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
56
|
+
def process(tag, es)
|
|
57
|
+
es.each do |time, record|
|
|
58
|
+
target = record[@key_to_filter]
|
|
59
|
+
log.debug "target: #{target}"
|
|
60
|
+
# Do filter
|
|
61
|
+
if target && @matcher.matches?(target)
|
|
62
|
+
if @retag_for_filtered
|
|
63
|
+
tag = @retag_for_filtered.tag || ((tag && !tag.empty?) ? @prefix_for_filtered_tag + tag : @retag_for_filtered.add_prefix)
|
|
64
|
+
log.debug "re-emit with tag: #{tag}"
|
|
65
|
+
router.emit(tag, time, record)
|
|
66
|
+
end
|
|
67
|
+
next
|
|
58
68
|
end
|
|
59
|
-
|
|
69
|
+
tag = @retag.tag || ((tag && !tag.empty?) ? @prefix + tag : @retag.add_prefix)
|
|
70
|
+
log.debug "re-emit with tag: #{tag}"
|
|
71
|
+
router.emit(tag, time, record)
|
|
60
72
|
end
|
|
61
|
-
tag = @retag.tag || ((tag && !tag.empty?) ? @prefix + tag : @retag.add_prefix)
|
|
62
|
-
log.debug "re-emit with tag: #{tag}"
|
|
63
|
-
router.emit(tag, time, record)
|
|
64
73
|
end
|
|
65
74
|
end
|
|
66
75
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-filter-list
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.3.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Shun Yanaura
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2017-08-
|
|
11
|
+
date: 2017-08-25 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -72,20 +72,20 @@ dependencies:
|
|
|
72
72
|
requirements:
|
|
73
73
|
- - "~>"
|
|
74
74
|
- !ruby/object:Gem::Version
|
|
75
|
-
version: '0.
|
|
75
|
+
version: '0.14'
|
|
76
76
|
- - ">="
|
|
77
77
|
- !ruby/object:Gem::Version
|
|
78
|
-
version: 0.
|
|
78
|
+
version: 0.14.0
|
|
79
79
|
type: :runtime
|
|
80
80
|
prerelease: false
|
|
81
81
|
version_requirements: !ruby/object:Gem::Requirement
|
|
82
82
|
requirements:
|
|
83
83
|
- - "~>"
|
|
84
84
|
- !ruby/object:Gem::Version
|
|
85
|
-
version: '0.
|
|
85
|
+
version: '0.14'
|
|
86
86
|
- - ">="
|
|
87
87
|
- !ruby/object:Gem::Version
|
|
88
|
-
version: 0.
|
|
88
|
+
version: 0.14.0
|
|
89
89
|
description: A fluentd output plugin to filter keywords from messages
|
|
90
90
|
email:
|
|
91
91
|
- metroplexity@gmail.com
|