fluent-plugin-label-router 0.2.1 → 0.2.6

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d2c9fe1a41256dc29a1a7ff87833cfb6f7f4ca6e4ab330b185aa85f66e0e35b3
4
- data.tar.gz: 2440249e93baffc3de72dd31fe70ef6d83ce5f4832b9af615a338bb855489cb4
3
+ metadata.gz: 8db0a645ba254d2ab4e728cae01d96f8e0b4d10d299b69321740d0a183475aa3
4
+ data.tar.gz: ce2821e9db352d06d402acc1a1962711a4256e40f754fc5fb01b7a444fa69e96
5
5
  SHA512:
6
- metadata.gz: 0fc6928ca8716774f8291eb013b737f31d2e6d02b34dee5d473ae644c7898aa8145aca73b3691c9ed4520165d3ee19f63da7dee72d48dc1b416c4fd54e45e79f
7
- data.tar.gz: cc9afbd8db7ac5c1f10001c6bc91403eb9961b529773f210f3c9b2ac7bf877c792a10e18f6b04cec5b1526762238b3f081f51cce08b4d4a867ebba63f28030d7
6
+ metadata.gz: 453fdada4c6b1d6d06102203cde03030fdb97e9ae3d4a3cbddd78eb5fa45c61b650c537e2e7333f7d775527512adf92247c795d24dffe945209887d7e7539562
7
+ data.tar.gz: bad2d5a50a3b6a78bde32b3820690cfce2c2c97a56649ac6a319a7610d81a9634223ac078013145b6c04a39868712a0820fdd667d6820ef87ee273a825d65a86
data/README.md CHANGED
@@ -192,7 +192,7 @@ Use `default_label` and/or `default_tag` to route non matching records.
192
192
  ```
193
193
  <match example.tag**>
194
194
  @type label_router
195
- default_label @default_sink
195
+ default_route @default_sink
196
196
  <route>
197
197
  ...
198
198
  </route>
@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3
3
 
4
4
  Gem::Specification.new do |spec|
5
5
  spec.name = "fluent-plugin-label-router"
6
- spec.version = "0.2.1"
6
+ spec.version = "0.2.6"
7
7
  spec.authors = ["Banzai Cloud"]
8
8
  spec.email = ["info@banzaicloud.com"]
9
9
 
@@ -23,5 +23,6 @@ Gem::Specification.new do |spec|
23
23
  spec.add_development_dependency "bundler", "~> 1.14"
24
24
  spec.add_development_dependency "rake", "~> 12.0"
25
25
  spec.add_development_dependency "test-unit", "~> 3.0"
26
+ spec.add_dependency "prometheus-client", ">= 2.1.0"
26
27
  spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
27
28
  end
@@ -11,10 +11,11 @@
11
11
  # distributed under the License is distributed on an "AS IS" BASIS,
12
12
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
13
  # See the License for the specific language governing permissions and
14
- # limitations under the License.
14
+ # limitations under the License
15
15
 
16
16
  require "fluent/plugin/output"
17
- require 'digest/md5'
17
+ require 'prometheus/client'
18
+
18
19
 
19
20
  module Fluent
20
21
  module Plugin
@@ -34,12 +35,16 @@ module Fluent
34
35
  config_param :default_route, :string, :default => ""
35
36
  desc "Default tag to drain unmatched patterns"
36
37
  config_param :default_tag, :string, :default => ""
38
+ desc "Enable metrics for the router"
39
+ config_param :metrics, :bool, :default => false
37
40
 
38
41
  config_section :route, param_name: :routes, multi: true do
39
42
  desc "New @LABEL if selectors matched"
40
43
  config_param :@label, :string, :default => nil
41
44
  desc "New tag if selectors matched"
42
45
  config_param :tag, :string, :default => ""
46
+ desc "Extra labels for metrics"
47
+ config_param :metrics_labels, :hash, :default => {}
43
48
 
44
49
  config_section :match, param_name: :matches, multi: true do
45
50
  desc "Label definition to match record. Example: app:nginx. You can specify more values as comma separated list: key1:value1,key2:value2"
@@ -53,16 +58,25 @@ module Fluent
53
58
  desc "Negate the selection making it an exclude"
54
59
  config_param :negate, :bool, :default => false
55
60
  end
56
-
57
61
  end
58
62
 
59
-
60
-
61
63
  class Route
62
- def initialize(matches, tag, router)
64
+ def initialize(rule, router, registry)
63
65
  @router = router
64
- @matches = matches
65
- @tag = tag
66
+ @matches = rule['matches']
67
+ @tag = rule['tag'].to_s
68
+ @label = rule['@label']
69
+ @metrics_labels = (rule['metrics_labels'].map { |k, v| [k.to_sym, v] }.to_h if rule['metrics_labels'])
70
+ @counter = nil
71
+ unless registry.nil?
72
+ @counter = registry.counter(:fluentd_router_records_total, "Total number of events router for the flow")
73
+ end
74
+ end
75
+
76
+ def get_labels
77
+ default = { 'flow': @label }
78
+ labels = default.merge(@metrics_labels)
79
+ labels
66
80
  end
67
81
 
68
82
  # Evaluate selectors
@@ -91,7 +105,7 @@ module Fluent
91
105
  return false
92
106
  end
93
107
  # Break on host mismatch
94
- unless match.container_names.empty? || match.container_names.include?(metadata[:container_name])
108
+ unless match.container_names.empty? || match.container_names.include?(metadata[:container])
95
109
  return false
96
110
  end
97
111
  # Break if list of namespaces is not empty and does not include actual namespace
@@ -116,6 +130,8 @@ module Fluent
116
130
  else
117
131
  @router.emit_stream(@tag, es)
118
132
  end
133
+ # increment the counter for a given label set
134
+ @counter&.increment(by: es.size, labels: get_labels)
119
135
  end
120
136
 
121
137
  def match_labels(input, match)
@@ -125,13 +141,15 @@ module Fluent
125
141
 
126
142
  def process(tag, es)
127
143
  if @sticky_tags
128
- if @route_map.has_key?(tag)
129
- # We already matched with this tag send events to the routers
130
- @route_map[tag].each do |r|
131
- r.emit_es(tag, es.dup)
144
+ @mutex.synchronize {
145
+ if @route_map.has_key?(tag)
146
+ # We already matched with this tag send events to the routers
147
+ @route_map[tag].each do |r|
148
+ r.emit_es(tag, es.dup)
149
+ end
150
+ return
132
151
  end
133
- return
134
- end
152
+ }
135
153
  end
136
154
  event_stream = Hash.new {|h, k| h[k] = Fluent::MultiEventStream.new }
137
155
  es.each do |time, record|
@@ -144,7 +162,9 @@ module Fluent
144
162
  if r.match?(input_metadata)
145
163
  orphan_record = false
146
164
  if @sticky_tags
147
- @route_map[tag].push(r)
165
+ @mutex.synchronize {
166
+ @route_map[tag].add(r)
167
+ }
148
168
  end
149
169
  if @batch
150
170
  event_stream[r].add(time, record)
@@ -155,7 +175,9 @@ module Fluent
155
175
  end
156
176
  if !@default_router.nil? && orphan_record
157
177
  if @sticky_tags
158
- @route_map[tag].push(@default_router)
178
+ @mutex.synchronize {
179
+ @route_map[tag].add(@default_router)
180
+ }
159
181
  end
160
182
  if @batch
161
183
  event_stream[@default_router].add(time, record)
@@ -163,27 +185,29 @@ module Fluent
163
185
  @default_router.emit(tag, time, record.dup)
164
186
  end
165
187
  end
166
- if @batch
167
- event_stream.each do |r, es|
168
- r.emit_es(tag, es.dup)
169
- end
188
+ end
189
+ if @batch
190
+ event_stream.each do |r, es|
191
+ r.emit_es(tag, es.dup)
170
192
  end
171
193
  end
172
194
  end
173
195
 
174
196
  def configure(conf)
175
197
  super
176
- @route_map = Hash.new { |h, k| h[k] = Array.new }
198
+ @registry = (::Prometheus::Client.registry if @metrics)
199
+ @route_map = Hash.new { |h, k| h[k] = Set.new }
200
+ @mutex = Mutex.new
177
201
  @routers = []
178
202
  @default_router = nil
179
203
  @routes.each do |rule|
180
204
  route_router = event_emitter_router(rule['@label'])
181
- puts rule
182
- @routers << Route.new(rule.matches, rule.tag.to_s, route_router)
205
+ @routers << Route.new(rule, route_router, @registry)
183
206
  end
184
207
 
185
208
  if @default_route != '' or @default_tag != ''
186
- @default_router = Route.new(nil, @default_tag, event_emitter_router(@default_route))
209
+ default_rule = { 'matches' => nil, 'tag' => @default_tag, '@label' => @default_route}
210
+ @default_router = Route.new(default_rule, event_emitter_router(@default_route), @registry)
187
211
  end
188
212
 
189
213
  @access_to_labels = record_accessor_create("$.kubernetes.labels")
@@ -76,7 +76,7 @@ class LabelRouterOutputTest < Test::Unit::TestCase
76
76
  d = Fluent::Test::Driver::BaseOwner.new(Fluent::Plugin::LabelRouterOutput)
77
77
  d.configure(routing_conf)
78
78
 
79
- r1 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[0].matches, d.instance.routes[0].tag,nil)
79
+ r1 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[0], nil,nil)
80
80
  # Selector matched: GO
81
81
  assert_equal(true, r1.match?(labels: { 'app' => 'app1' }, namespace: ''))
82
82
  # Exclude match: NO GO
@@ -84,7 +84,7 @@ class LabelRouterOutputTest < Test::Unit::TestCase
84
84
  # Nothing matched: NO GO
85
85
  assert_equal(false, r1.match?(labels: { 'app3' => 'app2' }, namespace: ''))
86
86
 
87
- r2 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[1].matches, d.instance.routes[1].tag,nil)
87
+ r2 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[1], nil,nil)
88
88
  # Match selector and namespace: GO
89
89
  assert_equal(true, r2.match?(labels: { 'app' => 'app1' }, namespace: 'test'))
90
90
  # Exclude via namespace
@@ -92,18 +92,18 @@ class LabelRouterOutputTest < Test::Unit::TestCase
92
92
  # Nothing matched: NO GO
93
93
  assert_equal(false, r2.match?(labels: { 'app3' => 'app' }, namespace: 'system'))
94
94
 
95
- r3 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[2].matches, d.instance.routes[2].tag,nil)
95
+ r3 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[2], nil,nil)
96
96
  assert_equal(true, r3.match?(labels: { 'app' => 'nginx' }, namespace: 'dev'))
97
97
  assert_equal(true, r3.match?(labels: { 'app' => 'nginx' }, namespace: 'sandbox'))
98
98
  assert_equal(false, r3.match?(labels: { 'app' => 'nginx2' }, namespace: 'sandbox'))
99
99
 
100
- r4 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[3].matches, d.instance.routes[3].tag,nil)
100
+ r4 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[3], nil,nil)
101
101
  # Matching container name
102
- assert_equal(true, r4.match?(labels: { 'app' => 'nginx' }, namespace: 'dev', container_name: 'mycontainer'))
102
+ assert_equal(true, r4.match?(labels: { 'app' => 'nginx' }, namespace: 'dev', container: 'mycontainer'))
103
103
  # Missing container name is equal to wrong container
104
104
  assert_equal(false, r4.match?(labels: { 'app' => 'nginx' }, namespace: 'sandbox'))
105
105
  # Wrong container name
106
- assert_equal(false, r4.match?(labels: { 'app' => 'nginx' }, namespace: 'dev', container_name: 'mycontainer2'))
106
+ assert_equal(false, r4.match?(labels: { 'app' => 'nginx' }, namespace: 'dev', container: 'mycontainer2'))
107
107
  # Wrong label but good namespace and container_name
108
108
  assert_equal(false, r4.match?(labels: { 'app' => 'nginx2' }, namespace: 'sandbox', container_name: 'mycontainer2'))
109
109
  end
@@ -134,6 +134,26 @@ class LabelRouterOutputTest < Test::Unit::TestCase
134
134
  end
135
135
  end
136
136
 
137
+
138
+ sub_test_case 'test_multiple_events_batched' do
139
+ test 'normal' do
140
+ conf = %[
141
+ <route>
142
+ <match>
143
+ </match>
144
+ </route>
145
+ ]
146
+ event_time = event_time("2019-07-17 11:11:11 UTC")
147
+ d = create_driver(conf)
148
+ d.run(expect_emits: 1, expect_records: 2) do
149
+ d.feed("test", [
150
+ [event_time, {"kubernetes" => {} } ],
151
+ [event_time, {"kubernetes" => {} } ],
152
+ ])
153
+ end
154
+ end
155
+ end
156
+
137
157
  sub_test_case 'test_default_router' do
138
158
  test 'normal' do
139
159
  CONFIG2 = %[
@@ -148,11 +168,11 @@ default_tag "new_tag"
148
168
  ]
149
169
  event_time = event_time("2019-07-17 11:11:11 UTC")
150
170
  d = create_driver(CONFIG2)
151
- d.run(default_tag: 'test') do
152
- d.feed(event_time, {"kubernetes" => {"labels" => {"app" => "app1"} } } )
153
- end
154
- d.run(default_tag: 'test2') do
155
- d.feed(event_time, {"kubernetes" => {"labels" => {"app" => "app2"} } } )
171
+ d.run() do
172
+ d.feed("test", [
173
+ [event_time, {"kubernetes" => {"labels" => {"app" => "app1"} } } ],
174
+ [event_time, {"kubernetes" => {"labels" => {"app" => "app2"} } } ],
175
+ ])
156
176
  end
157
177
  events = d.events
158
178
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-label-router
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.1
4
+ version: 0.2.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Banzai Cloud
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-02-28 00:00:00.000000000 Z
11
+ date: 2021-05-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -52,6 +52,20 @@ dependencies:
52
52
  - - "~>"
53
53
  - !ruby/object:Gem::Version
54
54
  version: '3.0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: prometheus-client
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: 2.1.0
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: 2.1.0
55
69
  - !ruby/object:Gem::Dependency
56
70
  name: fluentd
57
71
  requirement: !ruby/object:Gem::Requirement