fluent-plugin-label-router 0.1.3 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/README.md +101 -25
- data/fluent-plugin-label-router.gemspec +1 -1
- data/lib/fluent/plugin/out_label_router.rb +66 -14
- data/test/plugin/test_out_label_router.rb +115 -1
- metadata +3 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 07025a5148e5985c287b5d07e4a8f05c642d650734db78c995be11c2ee478a5e
|
4
|
+
data.tar.gz: 8e787eacb12b8e12a1e2895f919128d1c755e4c553c9e5b218a7970e89f3564b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8e116c6ace26a31641f467eed943f5a643c1f4c6737ea602106c43af69f40cac89fe54533b08e596d3c32f9630c5a2d1a9532f37314bdf24081c6b4aeb70a120
|
7
|
+
data.tar.gz: e8d0c1c3653c047a65fc0642602a7e5e4aef3e3a341653fca64bd7e15323233c31583451591f9b744becd3f54122b55abd0d936ae48d60a63a5fc4dc640695ca
|
data/README.md
CHANGED
@@ -32,7 +32,10 @@ $ bundle
|
|
32
32
|
|
33
33
|
## Configuration
|
34
34
|
|
35
|
-
The configuration builds from `<route>` sections.
|
35
|
+
The configuration builds from `<route>` sections. Each `route` section
|
36
|
+
can have several `<match>` statement. These statements computed in order and
|
37
|
+
positive (or in case of *negate true* negative) results break the evaluation.
|
38
|
+
We can say that the sections are coupled in a **lazy evaluation OR**.
|
36
39
|
|
37
40
|
```
|
38
41
|
<match example.tag**>
|
@@ -41,32 +44,86 @@ The configuration builds from `<route>` sections.
|
|
41
44
|
...
|
42
45
|
</route>
|
43
46
|
<route>
|
44
|
-
|
47
|
+
<match>
|
48
|
+
...
|
49
|
+
</match>
|
50
|
+
<match> #Exclude
|
51
|
+
negate true
|
52
|
+
...
|
53
|
+
</match>
|
45
54
|
</route>
|
46
55
|
</match>
|
47
56
|
```
|
48
57
|
|
49
|
-
|
50
|
-
|
51
|
-
|
|
52
|
-
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
58
|
+
Configuration reference
|
59
|
+
|
60
|
+
| Parameter | Description | Type | Default |
|
61
|
+
|---------------|--------------------------------------------------------------------------------------------------------|---------|---------|
|
62
|
+
| emit_mode | Emit mode. If `batch`, the plugin will emit events per labels matched. Enum: record, batch | enum | batch |
|
63
|
+
| sticky_tags | Sticky tags will match only one record from an event stream. The same tag will be treated the same way | bool | true |
|
64
|
+
| default_route | If defined all non-matching record passes to this label. | string | "" |
|
65
|
+
| default_tag | If defined all non-matching record rewrited to this tag. (Can be used with label simoultanesly) | string | "" |
|
66
|
+
| \<route\> | Route the log if match with parameters defined | []route | nil |
|
67
|
+
|
68
|
+
#### \<route\>
|
69
|
+
| Parameter | Description | Type | Default |
|
70
|
+
|---------------|--------------------------------------------------------------------------------------------------------|---------|---------|
|
71
|
+
| @label | Route the matching record to the given `label` | string | "" |
|
72
|
+
| tag | Tag the matching record to the given `tag` | string | "" |
|
73
|
+
| \<match\> | List of match statements. Repeatable. | []match | nil |
|
74
|
+
|
75
|
+
|
76
|
+
#### \<match\>
|
77
|
+
| Parameter | Description | Type | Default |
|
78
|
+
|------------|-------------------------------------------------------------------------------|----------|----------|
|
79
|
+
| labels | Label definition to match record. Example: `app:nginx` | Hash | nil |
|
80
|
+
| namespaces | Comma separated list of namespaces. Ignored if left empty. | []string | nil |
|
81
|
+
| negate | Negate the selector meaning to exclude matches | bool | false |
|
82
|
+
|
83
|
+
## Rules of thumb
|
84
|
+
|
85
|
+
1. Defining more than one namespace in `namespaces` inside a `match` statement
|
86
|
+
will check whether any of that namespaces matches.
|
87
|
+
|
88
|
+
2. Using `sticky_tags` means that only the **first** record will be analysed per `tag`.
|
89
|
+
Keep that in mind if you are ingesting traffic that is not unique on a per tag bases.
|
90
|
+
Fluentd and fluent-bit tail logs from Kubernetes are unique per container.
|
91
|
+
|
92
|
+
3. The plugin does not check if the configuration is valid so be careful to not define
|
93
|
+
statements like identical `match` statement with negate because the negate rule will never
|
94
|
+
be evaluated.
|
57
95
|
|
58
96
|
## Examples
|
59
97
|
|
60
|
-
### 1. Route specific `labels` and `
|
98
|
+
### 1. Route specific `labels` and `namespaces` to `@label` and new `tag`
|
61
99
|
Configuration to re-tag and re-label all logs from `default` namespace with label `app=nginx` and `env=dev`.
|
62
100
|
```
|
63
101
|
<match example.tag**>
|
64
102
|
@type label_router
|
65
103
|
<route>
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
104
|
+
@label @NGINX
|
105
|
+
tag new_tag
|
106
|
+
<match>
|
107
|
+
labels app:nginx,env:dev
|
108
|
+
namespaces default
|
109
|
+
</match>
|
110
|
+
</route>
|
111
|
+
</match>
|
112
|
+
```
|
113
|
+
|
114
|
+
### 2. Exclude specific `labels` and `namespaces`
|
115
|
+
Configuration to re-tag and re-label all logs that **not** from `default` namespace **and not** have labels `ap=nginx` and `env=dev`
|
116
|
+
```
|
117
|
+
<match example.tag**>
|
118
|
+
@type label_router
|
119
|
+
<route>
|
120
|
+
@label @NGINX
|
121
|
+
tag new_tag
|
122
|
+
<match>
|
123
|
+
negate true
|
124
|
+
labels app:nginx,env:dev
|
125
|
+
namespaces default
|
126
|
+
</match>
|
70
127
|
</route>
|
71
128
|
</match>
|
72
129
|
```
|
@@ -91,9 +148,11 @@ Only `labels`
|
|
91
148
|
<match example.tag**>
|
92
149
|
@type label_router
|
93
150
|
<route>
|
94
|
-
|
95
|
-
|
96
|
-
|
151
|
+
@label @NGINX
|
152
|
+
tag new_tag
|
153
|
+
<match>
|
154
|
+
labels app:nginx
|
155
|
+
</match>
|
97
156
|
</route>
|
98
157
|
</match>
|
99
158
|
```
|
@@ -102,9 +161,11 @@ Only `namespace`
|
|
102
161
|
<match example.tag**>
|
103
162
|
@type label_router
|
104
163
|
<route>
|
105
|
-
|
106
|
-
|
107
|
-
|
164
|
+
@label @NGINX
|
165
|
+
tag new_tag
|
166
|
+
<match>
|
167
|
+
namespaces default
|
168
|
+
</match>
|
108
169
|
</route>
|
109
170
|
</match>
|
110
171
|
```
|
@@ -112,16 +173,31 @@ Rewrite all
|
|
112
173
|
```
|
113
174
|
<match example.tag**>
|
114
175
|
@type label_router
|
115
|
-
<
|
116
|
-
|
117
|
-
|
118
|
-
</
|
176
|
+
<match>
|
177
|
+
@label @NGINX
|
178
|
+
tag new_tag
|
179
|
+
</match>
|
119
180
|
</match>
|
120
181
|
```
|
121
182
|
|
122
183
|
### 3. One of `@label` ot `tag` configuration should be specified
|
123
184
|
If you don't rewrite either of them fluent will **likely to crash** because it will reprocess the same messages again.
|
124
185
|
|
186
|
+
### 4. Default route/tag
|
187
|
+
|
188
|
+
Use `default_label` and/or `default_tag` to route non matching records.
|
189
|
+
|
190
|
+
```
|
191
|
+
<match example.tag**>
|
192
|
+
@type label_router
|
193
|
+
default_label @default_sink
|
194
|
+
<route>
|
195
|
+
...
|
196
|
+
</route>
|
197
|
+
</match>
|
198
|
+
```
|
199
|
+
|
200
|
+
|
125
201
|
## Copyright
|
126
202
|
|
127
203
|
* Copyright(c) 2019- Banzai Cloud
|
@@ -26,33 +26,67 @@ module Fluent
|
|
26
26
|
#record_accessor_create("log")
|
27
27
|
#record_accessor_create("$.key1.key2")
|
28
28
|
#record_accessor_create("$['key1'][0]['key2']")
|
29
|
+
desc "Emit mode. If `batch`, the plugin will emit events per labels matched."
|
30
|
+
config_param :emit_mode, :enum, list: [:record, :batch], default: :batch
|
31
|
+
desc "Sticky tags will match only one record from an event stream. The same tag will be treated the same way"
|
32
|
+
config_param :sticky_tags, :bool, default: true
|
33
|
+
desc "Default label to drain unmatched patterns"
|
34
|
+
config_param :default_route, :string, :default => ""
|
35
|
+
desc "Default tag to drain unmatched patterns"
|
36
|
+
config_param :default_tag, :string, :default => ""
|
29
37
|
|
30
38
|
config_section :route, param_name: :routes, multi: true do
|
31
|
-
desc "Label definition to match record. Example: app:nginx. You can specify more values as comma separated list: key1:value1,key2:value2"
|
32
|
-
config_param :labels, :hash, :default => {}
|
33
|
-
desc "Namespaces definition to filter the record. Ignored if left empty."
|
34
|
-
config_param :namespace, :string, :default => ""
|
35
39
|
desc "New @LABEL if selectors matched"
|
36
40
|
config_param :@label, :string, :default => nil
|
37
41
|
desc "New tag if selectors matched"
|
38
42
|
config_param :tag, :string, :default => ""
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
+
|
44
|
+
config_section :match, param_name: :selectors, multi: true do
|
45
|
+
desc "Label definition to match record. Example: app:nginx. You can specify more values as comma separated list: key1:value1,key2:value2"
|
46
|
+
config_param :labels, :hash, :default => {}
|
47
|
+
desc "List of namespace definition to filter the record. Ignored if left empty."
|
48
|
+
config_param :namespaces, :array, :default => [], value_type: :string
|
49
|
+
desc "Negate the selection making it an exclude"
|
50
|
+
config_param :negate, :bool, :default => false
|
51
|
+
end
|
52
|
+
|
43
53
|
end
|
44
54
|
|
55
|
+
|
56
|
+
|
45
57
|
class Route
|
46
|
-
def initialize(
|
58
|
+
def initialize(selectors, tag, router)
|
47
59
|
@router = router
|
48
|
-
@
|
49
|
-
@namespace = namespace
|
60
|
+
@selectors = selectors
|
50
61
|
@tag = tag
|
51
62
|
end
|
52
63
|
|
64
|
+
# Evaluate selectors
|
65
|
+
# We evaluate <match> statements in order:
|
66
|
+
# 1. If match == true and negate == false -> return true
|
67
|
+
# 2. If match == true and negate == true -> return false
|
68
|
+
# 3. If match == false and negate == false -> continue
|
69
|
+
# 4. If match == false and negate == true -> continue
|
70
|
+
# There is no match at all -> return false
|
53
71
|
def match?(labels, namespace)
|
54
|
-
|
55
|
-
|
72
|
+
@selectors.each do |selector|
|
73
|
+
if (filter_select(selector, labels, namespace) and !selector.negate)
|
74
|
+
return true
|
75
|
+
end
|
76
|
+
if (filter_select(selector, labels, namespace) and selector.negate)
|
77
|
+
return false
|
78
|
+
end
|
79
|
+
end
|
80
|
+
false
|
81
|
+
end
|
82
|
+
|
83
|
+
# Returns true if filter passes (filter match)
|
84
|
+
def filter_select(selector, labels, namespace)
|
85
|
+
# Break if list of namespaces is not empty and does not include actual namespace
|
86
|
+
unless selector.namespaces.empty? or selector.namespaces.include?(namespace)
|
87
|
+
return false
|
88
|
+
end
|
89
|
+
match_labels(labels, selector.labels)
|
56
90
|
end
|
57
91
|
|
58
92
|
def emit(tag, time, record)
|
@@ -89,8 +123,10 @@ module Fluent
|
|
89
123
|
es.each do |time, record|
|
90
124
|
input_labels = @access_to_labels.call(record).to_h
|
91
125
|
input_namespace = @access_to_namespace.call(record).to_s
|
126
|
+
orphan_record = true
|
92
127
|
@routers.each do |r|
|
93
128
|
if r.match?(input_labels, input_namespace)
|
129
|
+
orphan_record = false
|
94
130
|
if @sticky_tags
|
95
131
|
@route_map[tag].push(r)
|
96
132
|
end
|
@@ -101,6 +137,16 @@ module Fluent
|
|
101
137
|
end
|
102
138
|
end
|
103
139
|
end
|
140
|
+
if !@default_router.nil? && orphan_record
|
141
|
+
if @sticky_tags
|
142
|
+
@route_map[tag].push(@default_router)
|
143
|
+
end
|
144
|
+
if @batch
|
145
|
+
event_stream[@default_router].add(time, record)
|
146
|
+
else
|
147
|
+
@default_router.emit(tag, time, record.dup)
|
148
|
+
end
|
149
|
+
end
|
104
150
|
if @batch
|
105
151
|
event_stream.each do |r, es|
|
106
152
|
r.emit_es(tag, es.dup)
|
@@ -113,9 +159,15 @@ module Fluent
|
|
113
159
|
super
|
114
160
|
@route_map = Hash.new { |h, k| h[k] = Array.new }
|
115
161
|
@routers = []
|
162
|
+
@default_router = nil
|
116
163
|
@routes.each do |rule|
|
117
164
|
route_router = event_emitter_router(rule['@label'])
|
118
|
-
|
165
|
+
puts rule
|
166
|
+
@routers << Route.new(rule.selectors, rule.tag.to_s, route_router)
|
167
|
+
end
|
168
|
+
|
169
|
+
if @default_route != '' or @default_tag != ''
|
170
|
+
@default_router = Route.new(nil, @default_tag, event_emitter_router(@default_route))
|
119
171
|
end
|
120
172
|
|
121
173
|
@access_to_labels = record_accessor_create("$.kubernetes.labels")
|
@@ -34,11 +34,70 @@ class LabelRouterOutputTest < Test::Unit::TestCase
|
|
34
34
|
d.configure(conf)
|
35
35
|
end
|
36
36
|
|
37
|
+
sub_test_case 'test_routing' do
|
38
|
+
test 'basic configuration' do
|
39
|
+
routing_conf = %(
|
40
|
+
<route>
|
41
|
+
<match>
|
42
|
+
labels app:app1
|
43
|
+
</match>
|
44
|
+
<match>
|
45
|
+
labels app2:app2
|
46
|
+
negate true
|
47
|
+
</match>
|
48
|
+
tag new_app_tag
|
49
|
+
</route>
|
50
|
+
<route>
|
51
|
+
<match>
|
52
|
+
labels app:app1
|
53
|
+
namespaces default,test
|
54
|
+
</match>
|
55
|
+
<match>
|
56
|
+
labels app:app2
|
57
|
+
namespaces system
|
58
|
+
negate true
|
59
|
+
</match>
|
60
|
+
tag new_app_tag
|
61
|
+
</route>
|
62
|
+
<route>
|
63
|
+
<match>
|
64
|
+
labels app:nginx
|
65
|
+
namespaces dev,sandbox
|
66
|
+
</match>
|
67
|
+
</route>
|
68
|
+
)
|
69
|
+
d = Fluent::Test::Driver::BaseOwner.new(Fluent::Plugin::LabelRouterOutput)
|
70
|
+
d.configure(routing_conf)
|
71
|
+
|
72
|
+
r1 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[0].selectors, "",nil)
|
73
|
+
# Selector matched: GO
|
74
|
+
assert_equal(true, r1.match?({'app' => 'app1'},""))
|
75
|
+
# Exclude match: NO GO
|
76
|
+
assert_equal(false, r1.match?({"app" => "app2"},""))
|
77
|
+
# Nothing matched: NO GO
|
78
|
+
assert_equal(false, r1.match?({"app3" => "app2"},""))
|
79
|
+
|
80
|
+
r2 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[1].selectors, "",nil)
|
81
|
+
# Match selector and namespace: GO
|
82
|
+
assert_equal(true, r2.match?({"app" => "app1"},"test"))
|
83
|
+
# Exclude via namespace
|
84
|
+
assert_equal(false, r2.match?({"app" => "app2"},"system"))
|
85
|
+
# Nothing matched: NO GO
|
86
|
+
assert_equal(false, r2.match?({"app3" => "app"},"system"))
|
87
|
+
|
88
|
+
r3 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[2].selectors, "",nil)
|
89
|
+
assert_equal(true, r3.match?({"app" => "nginx"},"dev"))
|
90
|
+
assert_equal(true, r3.match?({"app" => "nginx"},"sandbox"))
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
37
94
|
sub_test_case 'test_tag' do
|
38
95
|
test 'normal' do
|
39
96
|
CONFIG = %[
|
40
97
|
<route>
|
41
|
-
|
98
|
+
<match>
|
99
|
+
labels app:app1
|
100
|
+
</match>
|
42
101
|
tag new_app_tag
|
43
102
|
</route>
|
44
103
|
]
|
@@ -56,4 +115,59 @@ class LabelRouterOutputTest < Test::Unit::TestCase
|
|
56
115
|
assert_equal ["new_app_tag", event_time, {"kubernetes" => {"labels" => {"app" => "app1"} } }], events[0]
|
57
116
|
end
|
58
117
|
end
|
118
|
+
|
119
|
+
sub_test_case 'test_default_router' do
|
120
|
+
test 'normal' do
|
121
|
+
CONFIG2 = %[
|
122
|
+
<route>
|
123
|
+
<match>
|
124
|
+
labels app:app1
|
125
|
+
</match>
|
126
|
+
tag new_app_tag
|
127
|
+
</route>
|
128
|
+
default_route @default
|
129
|
+
default_tag "new_tag"
|
130
|
+
]
|
131
|
+
event_time = event_time("2019-07-17 11:11:11 UTC")
|
132
|
+
d = create_driver(CONFIG2)
|
133
|
+
d.run(default_tag: 'test') do
|
134
|
+
d.feed(event_time, {"kubernetes" => {"labels" => {"app" => "app1"} } } )
|
135
|
+
end
|
136
|
+
d.run(default_tag: 'test2') do
|
137
|
+
d.feed(event_time, {"kubernetes" => {"labels" => {"app" => "app2"} } } )
|
138
|
+
end
|
139
|
+
events = d.events
|
140
|
+
|
141
|
+
assert_equal(2, events.size)
|
142
|
+
assert_equal ["new_app_tag", event_time, {"kubernetes" => {"labels" => {"app" => "app1"} } }], events[0]
|
143
|
+
assert_equal ["new_tag", event_time, {"kubernetes" => {"labels" => {"app" => "app2"} } }], events[1]
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
sub_test_case 'test_empty_router' do
|
148
|
+
test 'normal' do
|
149
|
+
CONFIG2 = %[
|
150
|
+
<route>
|
151
|
+
tag new_app_tag
|
152
|
+
<match>
|
153
|
+
labels
|
154
|
+
namespaces
|
155
|
+
</match>
|
156
|
+
</route>
|
157
|
+
]
|
158
|
+
event_time = event_time("2019-07-17 11:11:11 UTC")
|
159
|
+
d = create_driver(CONFIG2)
|
160
|
+
d.run(default_tag: 'test') do
|
161
|
+
d.feed(event_time, {"kubernetes" => {"labels" => {"app" => "app1"} } } )
|
162
|
+
end
|
163
|
+
d.run(default_tag: 'test2') do
|
164
|
+
d.feed(event_time, {"kubernetes" => {"labels" => {"app" => "app2"} } } )
|
165
|
+
end
|
166
|
+
events = d.events
|
167
|
+
|
168
|
+
assert_equal(2, events.size)
|
169
|
+
assert_equal ["new_app_tag", event_time, {"kubernetes" => {"labels" => {"app" => "app1"} } }], events[0]
|
170
|
+
assert_equal ["new_app_tag", event_time, {"kubernetes" => {"labels" => {"app" => "app2"} } }], events[1]
|
171
|
+
end
|
172
|
+
end
|
59
173
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-label-router
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Banzai Cloud
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-02-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -107,8 +107,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
107
107
|
- !ruby/object:Gem::Version
|
108
108
|
version: '0'
|
109
109
|
requirements: []
|
110
|
-
|
111
|
-
rubygems_version: 2.5.2.3
|
110
|
+
rubygems_version: 3.0.3
|
112
111
|
signing_key:
|
113
112
|
specification_version: 4
|
114
113
|
summary: Routing records based on Kubernetes labels.
|