fluent-plugin-filter-list 0.3.6 → 0.4.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +4 -0
- data/.travis.yml +4 -3
- data/Gemfile +1 -1
- data/README.md +39 -0
- data/fluent-plugin-out_filter_list.gemspec +4 -6
- data/lib/fluent/plugin/filter_filter_list.rb +5 -2
- data/lib/fluent/plugin/out_filter_list.rb +5 -2
- data/lib/fluent/plugin/out_filter_list/version.rb +1 -1
- data/lib/ip.rb +17 -0
- data/lib/{aho_corasick.rb → matcher.rb} +31 -3
- metadata +15 -12
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e2c3677869c408f56ed9c9aefc31cedc9a152e34872e1ae9e2ec0e6d8a138af2
|
4
|
+
data.tar.gz: 92986f0067646a44c583bc46dee304320ae128035680175d2c1de43decde02c4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d084ca17a78eac9cc3c8faa0be7f2e4a0e49a2f0bfea39d8de8a0a3d6bb52a7335ded2d9bc8104ee6a8adde964db140d1df10c1365c064caf7fd7e1ded586a21
|
7
|
+
data.tar.gz: 3d9541d21edc8a1a1ea84759e53459d5b8d99be6d979986e537c5be8660c3077db3d580ebbac622f69193d18fbfa17fdcf98bc1416d3d309936533ac2fb1f3dc
|
data/.rubocop.yml
CHANGED
data/.travis.yml
CHANGED
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -28,10 +28,12 @@ This repository contains two plugins: _Filter_ and _Output_, and expects two mai
|
|
28
28
|
|
29
29
|
Use the `filter_list` filter. Configure fluentd as follows.
|
30
30
|
|
31
|
+
#### ACMatcher
|
31
32
|
```
|
32
33
|
<filter pattern>
|
33
34
|
@type filter_list
|
34
35
|
|
36
|
+
filter AC
|
35
37
|
key_to_filter xyz
|
36
38
|
patterns_file_path blacklist.txt
|
37
39
|
</filter>
|
@@ -57,6 +59,43 @@ While the following message is passed through as the target field specified in t
|
|
57
59
|
{"x":1,"y":"halbart"}
|
58
60
|
```
|
59
61
|
|
62
|
+
#### IPMatcher
|
63
|
+
```
|
64
|
+
<filter pattern>
|
65
|
+
@type filter_list
|
66
|
+
|
67
|
+
filter IP
|
68
|
+
key_to_filter ip
|
69
|
+
patterns_file_path blacklist.txt
|
70
|
+
</filter>
|
71
|
+
```
|
72
|
+
|
73
|
+
Given the `blacklist.txt` is as follows.
|
74
|
+
|
75
|
+
```
|
76
|
+
192.168.1.0/24
|
77
|
+
127.0.0.1/24
|
78
|
+
255.255.0.0
|
79
|
+
```
|
80
|
+
|
81
|
+
The following message is discarded since its `ip` field is the IP address in the list (exact IP).
|
82
|
+
|
83
|
+
```json
|
84
|
+
{"ip":"255.255.0.0","y":1}
|
85
|
+
```
|
86
|
+
|
87
|
+
Also the following message is discarded since its `ip` field is the IP address in the list (CIDR-notated IP).
|
88
|
+
|
89
|
+
```json
|
90
|
+
{"ip":"192.168.1.255","y":1}
|
91
|
+
```
|
92
|
+
|
93
|
+
While the following message is passed through.
|
94
|
+
|
95
|
+
```json
|
96
|
+
{"ip":"192.168.2.0","y":1}
|
97
|
+
```
|
98
|
+
|
60
99
|
### Output plugin
|
61
100
|
|
62
101
|
The other use case is to filter messages likewise, but process the filtered messages in a different tag. You need to configure the plugin to tell it how to retag both non-filtered messages and filtered messages. We provide two mutually-exclusive parameters: `tag` and `add_prefix`. THe `tag` parameter tells the plugin to retag the message with the value exactly provided by the parameter. The `add_prefix` parameter tells the plugin to retag the messages with the original tag prepended with the value you provide. So if the original message had a tag _foo_ and you set the `add_prefix` parameter _filtered_, then the processed message would have the tag _filtered.foo_ (note that the period before the original tag value is also prepended).
|
@@ -1,6 +1,4 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
lib = File.expand_path('../lib', __FILE__)
|
1
|
+
lib = File.expand_path('lib', __dir__)
|
4
2
|
|
5
3
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
6
4
|
require 'fluent/plugin/out_filter_list/version'
|
@@ -19,13 +17,13 @@ Gem::Specification.new do |spec|
|
|
19
17
|
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
20
18
|
f.match(%r{^(test|spec|features)/})
|
21
19
|
end
|
22
|
-
spec.bindir = '
|
23
|
-
spec.executables = spec.files.grep(%r{^
|
20
|
+
spec.bindir = 'bin'
|
21
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
24
22
|
spec.require_paths = ['lib']
|
25
23
|
|
26
24
|
spec.add_development_dependency 'bundler', '~> 1.16'
|
25
|
+
spec.add_development_dependency 'minitest', '~> 5.11'
|
27
26
|
spec.add_development_dependency 'rake', '~> 12.3'
|
28
|
-
spec.add_development_dependency 'minitest', '~> 5.0'
|
29
27
|
spec.add_development_dependency 'test-unit', '~> 3.2'
|
30
28
|
spec.add_runtime_dependency 'fluentd', '>= 0.14.0', '< 2.0.0'
|
31
29
|
end
|
@@ -1,21 +1,24 @@
|
|
1
1
|
require 'fluent/plugin/out_filter_list/version'
|
2
|
-
require '
|
2
|
+
require 'matcher'
|
3
3
|
require 'fluent/plugin/filter'
|
4
|
+
require 'ip'
|
4
5
|
|
5
6
|
module Fluent
|
6
7
|
module Plugin
|
7
8
|
class FilterListFilter < Filter
|
8
9
|
include Matchers
|
10
|
+
include IP
|
9
11
|
|
10
12
|
Plugin.register_filter('filter_list', self)
|
11
13
|
|
14
|
+
config_param :filter, :string, default: 'AC'
|
12
15
|
config_param :key_to_filter, :string, default: nil
|
13
16
|
config_param :patterns_file_path, :string, default: ''
|
14
17
|
|
15
18
|
def configure(conf)
|
16
19
|
super
|
17
20
|
patterns = @patterns_file_path.empty? ? [] : File.readlines(@patterns_file_path).map(&:chomp).reject(&:empty?)
|
18
|
-
@matcher = ACMatcher.new(patterns)
|
21
|
+
@matcher = (@filter == 'IP') ? IPMatcher.new(patterns) : ACMatcher.new(patterns)
|
19
22
|
end
|
20
23
|
|
21
24
|
def filter(_tag, _time, record)
|
@@ -1,16 +1,19 @@
|
|
1
1
|
require 'fluent/plugin/output'
|
2
2
|
require 'fluent/plugin/out_filter_list/version'
|
3
|
-
require '
|
3
|
+
require 'matcher'
|
4
|
+
require 'ip'
|
4
5
|
|
5
6
|
module Fluent
|
6
7
|
module Plugin
|
7
8
|
class FilterListOutput < Output
|
8
9
|
include Matchers
|
10
|
+
include IP
|
9
11
|
|
10
12
|
Plugin.register_output('filter_list', self)
|
11
13
|
|
12
14
|
helpers :event_emitter
|
13
15
|
|
16
|
+
config_param :filter, :string, default: 'AC'
|
14
17
|
config_param :key_to_filter, :string, default: nil
|
15
18
|
config_param :patterns_file_path, :string, default: ''
|
16
19
|
|
@@ -44,7 +47,7 @@ module Fluent
|
|
44
47
|
super
|
45
48
|
[@retag, @retag_for_filtered].each { |c| validate c }
|
46
49
|
patterns = @patterns_file_path.empty? ? [] : File.readlines(@patterns_file_path).map(&:chomp).reject(&:empty?)
|
47
|
-
@matcher = ACMatcher.new(patterns)
|
50
|
+
@matcher = (@filter == 'IP') ? IPMatcher.new(patterns) : ACMatcher.new(patterns)
|
48
51
|
configure_prefixes
|
49
52
|
end
|
50
53
|
|
data/lib/ip.rb
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
module IP
|
2
|
+
require 'ipaddr'
|
3
|
+
class IP
|
4
|
+
attr_reader :subnet
|
5
|
+
|
6
|
+
# This constructor accepts both CIDR-notated IP and also exact IP.
|
7
|
+
def initialize(ip)
|
8
|
+
@address = IPAddr.new(ip)
|
9
|
+
@subnet = 32 - Math.log2(@address.to_range.to_a.size).round
|
10
|
+
end
|
11
|
+
|
12
|
+
def to_binary
|
13
|
+
binary = @address.to_i.to_s(2)
|
14
|
+
binary.slice(0, @subnet)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -1,3 +1,6 @@
|
|
1
|
+
require 'ipaddr'
|
2
|
+
require 'ip'
|
3
|
+
|
1
4
|
module Matchers
|
2
5
|
class ACMatcher
|
3
6
|
attr_reader :trie
|
@@ -18,6 +21,22 @@ module Matchers
|
|
18
21
|
end
|
19
22
|
end
|
20
23
|
|
24
|
+
class IPMatcher
|
25
|
+
attr_reader :trie
|
26
|
+
include IP
|
27
|
+
|
28
|
+
def initialize(patterns)
|
29
|
+
patterns = (patterns || []).compact.reject(&:empty?).map { |ip| IP.new(ip) }.map(&:to_binary)
|
30
|
+
@trie = Trie.new patterns
|
31
|
+
end
|
32
|
+
|
33
|
+
def matches?(text)
|
34
|
+
return false if text.nil?
|
35
|
+
ip = IPAddr.new(text).to_i.to_s(2)
|
36
|
+
trie.forward_match(ip)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
21
40
|
class Trie
|
22
41
|
attr_reader :root
|
23
42
|
def initialize(patterns)
|
@@ -55,13 +74,22 @@ module Matchers
|
|
55
74
|
cur_node.children.each do |char, child|
|
56
75
|
q.push(child)
|
57
76
|
detect_node = cur_node.failure || @root
|
58
|
-
while detect_node.children[char].nil?
|
59
|
-
detect_node = detect_node.failure
|
60
|
-
end
|
77
|
+
detect_node = detect_node.failure while detect_node.children[char].nil?
|
61
78
|
child.failure = detect_node.children[char]
|
62
79
|
end
|
63
80
|
end
|
64
81
|
end
|
82
|
+
|
83
|
+
def forward_match(pattern)
|
84
|
+
return false if @root.children.empty?
|
85
|
+
cur_node = @root
|
86
|
+
pattern.split('').each do |char|
|
87
|
+
return true if cur_node.children.empty?
|
88
|
+
return false unless cur_node.children.key?(char)
|
89
|
+
cur_node = cur_node.children[char]
|
90
|
+
end
|
91
|
+
true
|
92
|
+
end
|
65
93
|
end
|
66
94
|
|
67
95
|
class Node
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-filter-list
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shun Yanaura
|
8
8
|
autorequire:
|
9
|
-
bindir:
|
9
|
+
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-04-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -25,33 +25,33 @@ dependencies:
|
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '1.16'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
28
|
+
name: minitest
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
33
|
+
version: '5.11'
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
40
|
+
version: '5.11'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: rake
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '
|
47
|
+
version: '12.3'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: '
|
54
|
+
version: '12.3'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: test-unit
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -89,7 +89,9 @@ dependencies:
|
|
89
89
|
description: A fluentd output plugin to filter keywords from messages
|
90
90
|
email:
|
91
91
|
- metroplexity@gmail.com
|
92
|
-
executables:
|
92
|
+
executables:
|
93
|
+
- console
|
94
|
+
- setup
|
93
95
|
extensions: []
|
94
96
|
extra_rdoc_files: []
|
95
97
|
files:
|
@@ -104,10 +106,11 @@ files:
|
|
104
106
|
- bin/console
|
105
107
|
- bin/setup
|
106
108
|
- fluent-plugin-out_filter_list.gemspec
|
107
|
-
- lib/aho_corasick.rb
|
108
109
|
- lib/fluent/plugin/filter_filter_list.rb
|
109
110
|
- lib/fluent/plugin/out_filter_list.rb
|
110
111
|
- lib/fluent/plugin/out_filter_list/version.rb
|
112
|
+
- lib/ip.rb
|
113
|
+
- lib/matcher.rb
|
111
114
|
homepage: https://github.com/yanana/fluent-plugin-filter-list
|
112
115
|
licenses:
|
113
116
|
- MIT
|
@@ -128,7 +131,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
128
131
|
version: '0'
|
129
132
|
requirements: []
|
130
133
|
rubyforge_project:
|
131
|
-
rubygems_version: 2.7.
|
134
|
+
rubygems_version: 2.7.6
|
132
135
|
signing_key:
|
133
136
|
specification_version: 4
|
134
137
|
summary: A fluentd output plugin to filter keywords from messages
|