fluent-plugin-label-router 0.2.1 → 0.2.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/fluent-plugin-label-router.gemspec +2 -1
- data/lib/fluent/plugin/out_label_router.rb +49 -25
- data/test/plugin/test_out_label_router.rb +31 -11
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8db0a645ba254d2ab4e728cae01d96f8e0b4d10d299b69321740d0a183475aa3
|
4
|
+
data.tar.gz: ce2821e9db352d06d402acc1a1962711a4256e40f754fc5fb01b7a444fa69e96
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 453fdada4c6b1d6d06102203cde03030fdb97e9ae3d4a3cbddd78eb5fa45c61b650c537e2e7333f7d775527512adf92247c795d24dffe945209887d7e7539562
|
7
|
+
data.tar.gz: bad2d5a50a3b6a78bde32b3820690cfce2c2c97a56649ac6a319a7610d81a9634223ac078013145b6c04a39868712a0820fdd667d6820ef87ee273a825d65a86
|
data/README.md
CHANGED
@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |spec|
|
5
5
|
spec.name = "fluent-plugin-label-router"
|
6
|
-
spec.version = "0.2.
|
6
|
+
spec.version = "0.2.6"
|
7
7
|
spec.authors = ["Banzai Cloud"]
|
8
8
|
spec.email = ["info@banzaicloud.com"]
|
9
9
|
|
@@ -23,5 +23,6 @@ Gem::Specification.new do |spec|
|
|
23
23
|
spec.add_development_dependency "bundler", "~> 1.14"
|
24
24
|
spec.add_development_dependency "rake", "~> 12.0"
|
25
25
|
spec.add_development_dependency "test-unit", "~> 3.0"
|
26
|
+
spec.add_dependency "prometheus-client", ">= 2.1.0"
|
26
27
|
spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
|
27
28
|
end
|
@@ -11,10 +11,11 @@
|
|
11
11
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
12
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
13
|
# See the License for the specific language governing permissions and
|
14
|
-
# limitations under the License
|
14
|
+
# limitations under the License
|
15
15
|
|
16
16
|
require "fluent/plugin/output"
|
17
|
-
require '
|
17
|
+
require 'prometheus/client'
|
18
|
+
|
18
19
|
|
19
20
|
module Fluent
|
20
21
|
module Plugin
|
@@ -34,12 +35,16 @@ module Fluent
|
|
34
35
|
config_param :default_route, :string, :default => ""
|
35
36
|
desc "Default tag to drain unmatched patterns"
|
36
37
|
config_param :default_tag, :string, :default => ""
|
38
|
+
desc "Enable metrics for the router"
|
39
|
+
config_param :metrics, :bool, :default => false
|
37
40
|
|
38
41
|
config_section :route, param_name: :routes, multi: true do
|
39
42
|
desc "New @LABEL if selectors matched"
|
40
43
|
config_param :@label, :string, :default => nil
|
41
44
|
desc "New tag if selectors matched"
|
42
45
|
config_param :tag, :string, :default => ""
|
46
|
+
desc "Extra labels for metrics"
|
47
|
+
config_param :metrics_labels, :hash, :default => {}
|
43
48
|
|
44
49
|
config_section :match, param_name: :matches, multi: true do
|
45
50
|
desc "Label definition to match record. Example: app:nginx. You can specify more values as comma separated list: key1:value1,key2:value2"
|
@@ -53,16 +58,25 @@ module Fluent
|
|
53
58
|
desc "Negate the selection making it an exclude"
|
54
59
|
config_param :negate, :bool, :default => false
|
55
60
|
end
|
56
|
-
|
57
61
|
end
|
58
62
|
|
59
|
-
|
60
|
-
|
61
63
|
class Route
|
62
|
-
def initialize(
|
64
|
+
def initialize(rule, router, registry)
|
63
65
|
@router = router
|
64
|
-
@matches = matches
|
65
|
-
@tag = tag
|
66
|
+
@matches = rule['matches']
|
67
|
+
@tag = rule['tag'].to_s
|
68
|
+
@label = rule['@label']
|
69
|
+
@metrics_labels = (rule['metrics_labels'].map { |k, v| [k.to_sym, v] }.to_h if rule['metrics_labels'])
|
70
|
+
@counter = nil
|
71
|
+
unless registry.nil?
|
72
|
+
@counter = registry.counter(:fluentd_router_records_total, "Total number of events router for the flow")
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def get_labels
|
77
|
+
default = { 'flow': @label }
|
78
|
+
labels = default.merge(@metrics_labels)
|
79
|
+
labels
|
66
80
|
end
|
67
81
|
|
68
82
|
# Evaluate selectors
|
@@ -91,7 +105,7 @@ module Fluent
|
|
91
105
|
return false
|
92
106
|
end
|
93
107
|
# Break on host mismatch
|
94
|
-
unless match.container_names.empty? || match.container_names.include?(metadata[:
|
108
|
+
unless match.container_names.empty? || match.container_names.include?(metadata[:container])
|
95
109
|
return false
|
96
110
|
end
|
97
111
|
# Break if list of namespaces is not empty and does not include actual namespace
|
@@ -116,6 +130,8 @@ module Fluent
|
|
116
130
|
else
|
117
131
|
@router.emit_stream(@tag, es)
|
118
132
|
end
|
133
|
+
# increment the counter for a given label set
|
134
|
+
@counter&.increment(by: es.size, labels: get_labels)
|
119
135
|
end
|
120
136
|
|
121
137
|
def match_labels(input, match)
|
@@ -125,13 +141,15 @@ module Fluent
|
|
125
141
|
|
126
142
|
def process(tag, es)
|
127
143
|
if @sticky_tags
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
144
|
+
@mutex.synchronize {
|
145
|
+
if @route_map.has_key?(tag)
|
146
|
+
# We already matched with this tag send events to the routers
|
147
|
+
@route_map[tag].each do |r|
|
148
|
+
r.emit_es(tag, es.dup)
|
149
|
+
end
|
150
|
+
return
|
132
151
|
end
|
133
|
-
|
134
|
-
end
|
152
|
+
}
|
135
153
|
end
|
136
154
|
event_stream = Hash.new {|h, k| h[k] = Fluent::MultiEventStream.new }
|
137
155
|
es.each do |time, record|
|
@@ -144,7 +162,9 @@ module Fluent
|
|
144
162
|
if r.match?(input_metadata)
|
145
163
|
orphan_record = false
|
146
164
|
if @sticky_tags
|
147
|
-
@
|
165
|
+
@mutex.synchronize {
|
166
|
+
@route_map[tag].add(r)
|
167
|
+
}
|
148
168
|
end
|
149
169
|
if @batch
|
150
170
|
event_stream[r].add(time, record)
|
@@ -155,7 +175,9 @@ module Fluent
|
|
155
175
|
end
|
156
176
|
if !@default_router.nil? && orphan_record
|
157
177
|
if @sticky_tags
|
158
|
-
@
|
178
|
+
@mutex.synchronize {
|
179
|
+
@route_map[tag].add(@default_router)
|
180
|
+
}
|
159
181
|
end
|
160
182
|
if @batch
|
161
183
|
event_stream[@default_router].add(time, record)
|
@@ -163,27 +185,29 @@ module Fluent
|
|
163
185
|
@default_router.emit(tag, time, record.dup)
|
164
186
|
end
|
165
187
|
end
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
188
|
+
end
|
189
|
+
if @batch
|
190
|
+
event_stream.each do |r, es|
|
191
|
+
r.emit_es(tag, es.dup)
|
170
192
|
end
|
171
193
|
end
|
172
194
|
end
|
173
195
|
|
174
196
|
def configure(conf)
|
175
197
|
super
|
176
|
-
@
|
198
|
+
@registry = (::Prometheus::Client.registry if @metrics)
|
199
|
+
@route_map = Hash.new { |h, k| h[k] = Set.new }
|
200
|
+
@mutex = Mutex.new
|
177
201
|
@routers = []
|
178
202
|
@default_router = nil
|
179
203
|
@routes.each do |rule|
|
180
204
|
route_router = event_emitter_router(rule['@label'])
|
181
|
-
|
182
|
-
@routers << Route.new(rule.matches, rule.tag.to_s, route_router)
|
205
|
+
@routers << Route.new(rule, route_router, @registry)
|
183
206
|
end
|
184
207
|
|
185
208
|
if @default_route != '' or @default_tag != ''
|
186
|
-
|
209
|
+
default_rule = { 'matches' => nil, 'tag' => @default_tag, '@label' => @default_route}
|
210
|
+
@default_router = Route.new(default_rule, event_emitter_router(@default_route), @registry)
|
187
211
|
end
|
188
212
|
|
189
213
|
@access_to_labels = record_accessor_create("$.kubernetes.labels")
|
@@ -76,7 +76,7 @@ class LabelRouterOutputTest < Test::Unit::TestCase
|
|
76
76
|
d = Fluent::Test::Driver::BaseOwner.new(Fluent::Plugin::LabelRouterOutput)
|
77
77
|
d.configure(routing_conf)
|
78
78
|
|
79
|
-
r1 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[0]
|
79
|
+
r1 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[0], nil,nil)
|
80
80
|
# Selector matched: GO
|
81
81
|
assert_equal(true, r1.match?(labels: { 'app' => 'app1' }, namespace: ''))
|
82
82
|
# Exclude match: NO GO
|
@@ -84,7 +84,7 @@ class LabelRouterOutputTest < Test::Unit::TestCase
|
|
84
84
|
# Nothing matched: NO GO
|
85
85
|
assert_equal(false, r1.match?(labels: { 'app3' => 'app2' }, namespace: ''))
|
86
86
|
|
87
|
-
r2 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[1]
|
87
|
+
r2 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[1], nil,nil)
|
88
88
|
# Match selector and namespace: GO
|
89
89
|
assert_equal(true, r2.match?(labels: { 'app' => 'app1' }, namespace: 'test'))
|
90
90
|
# Exclude via namespace
|
@@ -92,18 +92,18 @@ class LabelRouterOutputTest < Test::Unit::TestCase
|
|
92
92
|
# Nothing matched: NO GO
|
93
93
|
assert_equal(false, r2.match?(labels: { 'app3' => 'app' }, namespace: 'system'))
|
94
94
|
|
95
|
-
r3 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[2]
|
95
|
+
r3 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[2], nil,nil)
|
96
96
|
assert_equal(true, r3.match?(labels: { 'app' => 'nginx' }, namespace: 'dev'))
|
97
97
|
assert_equal(true, r3.match?(labels: { 'app' => 'nginx' }, namespace: 'sandbox'))
|
98
98
|
assert_equal(false, r3.match?(labels: { 'app' => 'nginx2' }, namespace: 'sandbox'))
|
99
99
|
|
100
|
-
r4 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[3]
|
100
|
+
r4 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[3], nil,nil)
|
101
101
|
# Matching container name
|
102
|
-
assert_equal(true, r4.match?(labels: { 'app' => 'nginx' }, namespace: 'dev',
|
102
|
+
assert_equal(true, r4.match?(labels: { 'app' => 'nginx' }, namespace: 'dev', container: 'mycontainer'))
|
103
103
|
# Missing container name is equal to wrong container
|
104
104
|
assert_equal(false, r4.match?(labels: { 'app' => 'nginx' }, namespace: 'sandbox'))
|
105
105
|
# Wrong container name
|
106
|
-
assert_equal(false, r4.match?(labels: { 'app' => 'nginx' }, namespace: 'dev',
|
106
|
+
assert_equal(false, r4.match?(labels: { 'app' => 'nginx' }, namespace: 'dev', container: 'mycontainer2'))
|
107
107
|
# Wrong label but good namespace and container_name
|
108
108
|
assert_equal(false, r4.match?(labels: { 'app' => 'nginx2' }, namespace: 'sandbox', container_name: 'mycontainer2'))
|
109
109
|
end
|
@@ -134,6 +134,26 @@ class LabelRouterOutputTest < Test::Unit::TestCase
|
|
134
134
|
end
|
135
135
|
end
|
136
136
|
|
137
|
+
|
138
|
+
sub_test_case 'test_multiple_events_batched' do
|
139
|
+
test 'normal' do
|
140
|
+
conf = %[
|
141
|
+
<route>
|
142
|
+
<match>
|
143
|
+
</match>
|
144
|
+
</route>
|
145
|
+
]
|
146
|
+
event_time = event_time("2019-07-17 11:11:11 UTC")
|
147
|
+
d = create_driver(conf)
|
148
|
+
d.run(expect_emits: 1, expect_records: 2) do
|
149
|
+
d.feed("test", [
|
150
|
+
[event_time, {"kubernetes" => {} } ],
|
151
|
+
[event_time, {"kubernetes" => {} } ],
|
152
|
+
])
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
137
157
|
sub_test_case 'test_default_router' do
|
138
158
|
test 'normal' do
|
139
159
|
CONFIG2 = %[
|
@@ -148,11 +168,11 @@ default_tag "new_tag"
|
|
148
168
|
]
|
149
169
|
event_time = event_time("2019-07-17 11:11:11 UTC")
|
150
170
|
d = create_driver(CONFIG2)
|
151
|
-
d.run(
|
152
|
-
d.feed(
|
153
|
-
|
154
|
-
|
155
|
-
|
171
|
+
d.run() do
|
172
|
+
d.feed("test", [
|
173
|
+
[event_time, {"kubernetes" => {"labels" => {"app" => "app1"} } } ],
|
174
|
+
[event_time, {"kubernetes" => {"labels" => {"app" => "app2"} } } ],
|
175
|
+
])
|
156
176
|
end
|
157
177
|
events = d.events
|
158
178
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-label-router
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Banzai Cloud
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-05-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -52,6 +52,20 @@ dependencies:
|
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '3.0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: prometheus-client
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 2.1.0
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 2.1.0
|
55
69
|
- !ruby/object:Gem::Dependency
|
56
70
|
name: fluentd
|
57
71
|
requirement: !ruby/object:Gem::Requirement
|