fluent-plugin-label-router 0.2.3 → 0.2.8

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: bd6b78faaf79aa156a111780a7dde808623edb45059e75412e839a8f899121ff
4
- data.tar.gz: 52498b7e26d5d3d9d69edb4c2dc4ae497729c73785ba07b4ffbad474e6ae96ae
3
+ metadata.gz: 214b0ab1c791ea384c2bc1f1f2e44f8f27ee23b0ba5b35c9d357e535f0ac4bda
4
+ data.tar.gz: 9dc61783f297966d51469a9716465a4cf37c62d2c1f7059c9175b3686709a694
5
5
  SHA512:
6
- metadata.gz: d89cca1fad117cea4b1a40fbf31ff54d811562165929ba9c1603e40923f694221e2407262c24b867d11edf6d6d9ef452446fbb51d72da63aede210016d3b5a87
7
- data.tar.gz: 91aa7edd22925ecdae9f05cf1fc1f8f0e7e9fb5cf89b15437db50269112aaf1d44738bde764acb60d3840c9b36566fc819e99d9fff366ed180019711bc2847af
6
+ metadata.gz: b55e88561ee31fe9dbdf6481c293be9337a8aab482031c00717a972a212785f263a382e3e05384b45efed6258c39f30a71099e80556e9f55b001933a735d58b3
7
+ data.tar.gz: d0e80aeba02b97c9a2bc2d17f991eac2baf8e53fb56d1a7f24dee60d528f97c5da417a4233e1f9d65c3ec5a9f71d749cfaa1da269db947ed262c4becfb65769d
data/README.md CHANGED
@@ -192,7 +192,7 @@ Use `default_label` and/or `default_tag` to route non matching records.
192
192
  ```
193
193
  <match example.tag**>
194
194
  @type label_router
195
- default_label @default_sink
195
+ default_route @default_sink
196
196
  <route>
197
197
  ...
198
198
  </route>
@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3
3
 
4
4
  Gem::Specification.new do |spec|
5
5
  spec.name = "fluent-plugin-label-router"
6
- spec.version = "0.2.3"
6
+ spec.version = "0.2.8"
7
7
  spec.authors = ["Banzai Cloud"]
8
8
  spec.email = ["info@banzaicloud.com"]
9
9
 
@@ -20,8 +20,8 @@ Gem::Specification.new do |spec|
20
20
  spec.test_files = test_files
21
21
  spec.require_paths = ["lib"]
22
22
 
23
- spec.add_development_dependency "bundler", "~> 1.14"
24
23
  spec.add_development_dependency "rake", "~> 12.0"
25
24
  spec.add_development_dependency "test-unit", "~> 3.0"
25
+ spec.add_dependency "prometheus-client", ">= 2.1.0"
26
26
  spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
27
27
  end
@@ -11,10 +11,11 @@
11
11
  # distributed under the License is distributed on an "AS IS" BASIS,
12
12
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
13
  # See the License for the specific language governing permissions and
14
- # limitations under the License.
14
+ # limitations under the License
15
15
 
16
16
  require "fluent/plugin/output"
17
- require 'digest/md5'
17
+ require 'prometheus/client'
18
+
18
19
 
19
20
  module Fluent
20
21
  module Plugin
@@ -34,12 +35,16 @@ module Fluent
34
35
  config_param :default_route, :string, :default => ""
35
36
  desc "Default tag to drain unmatched patterns"
36
37
  config_param :default_tag, :string, :default => ""
38
+ desc "Enable metrics for the router"
39
+ config_param :metrics, :bool, :default => false
37
40
 
38
41
  config_section :route, param_name: :routes, multi: true do
39
42
  desc "New @LABEL if selectors matched"
40
43
  config_param :@label, :string, :default => nil
41
44
  desc "New tag if selectors matched"
42
45
  config_param :tag, :string, :default => ""
46
+ desc "Extra labels for metrics"
47
+ config_param :metrics_labels, :hash, :default => {}
43
48
 
44
49
  config_section :match, param_name: :matches, multi: true do
45
50
  desc "Label definition to match record. Example: app:nginx. You can specify more values as comma separated list: key1:value1,key2:value2"
@@ -56,10 +61,22 @@ module Fluent
56
61
  end
57
62
 
58
63
  class Route
59
- def initialize(matches, tag, router)
64
+ def initialize(rule, router, registry)
60
65
  @router = router
61
- @matches = matches
62
- @tag = tag
66
+ @matches = rule['matches']
67
+ @tag = rule['tag'].to_s
68
+ @label = rule['@label']
69
+ @metrics_labels = (rule['metrics_labels'].map { |k, v| [k.to_sym, v] }.to_h if rule['metrics_labels'])
70
+ @counter = nil
71
+ unless registry.nil?
72
+ @counter = registry.counter(:fluentd_router_records_total, docstring: "Total number of events router for the flow", labels: [:flow, :id])
73
+ end
74
+ end
75
+
76
+ def get_labels
77
+ default = { 'flow': @label }
78
+ labels = default.merge(@metrics_labels)
79
+ labels
63
80
  end
64
81
 
65
82
  # Evaluate selectors
@@ -88,7 +105,7 @@ module Fluent
88
105
  return false
89
106
  end
90
107
  # Break on host mismatch
91
- unless match.container_names.empty? || match.container_names.include?(metadata[:container_name])
108
+ unless match.container_names.empty? || match.container_names.include?(metadata[:container])
92
109
  return false
93
110
  end
94
111
  # Break if list of namespaces is not empty and does not include actual namespace
@@ -105,6 +122,7 @@ module Fluent
105
122
  else
106
123
  @router.emit(@tag, time, record)
107
124
  end
125
+ @counter&.increment(by: 1, labels: get_labels)
108
126
  end
109
127
 
110
128
  def emit_es(tag, es)
@@ -113,6 +131,8 @@ module Fluent
113
131
  else
114
132
  @router.emit_stream(@tag, es)
115
133
  end
134
+ # increment the counter for a given label set
135
+ @counter&.increment(by: es.size, labels: get_labels)
116
136
  end
117
137
 
118
138
  def match_labels(input, match)
@@ -176,18 +196,19 @@ module Fluent
176
196
 
177
197
  def configure(conf)
178
198
  super
199
+ @registry = (::Prometheus::Client.registry if @metrics)
179
200
  @route_map = Hash.new { |h, k| h[k] = Set.new }
180
201
  @mutex = Mutex.new
181
202
  @routers = []
182
203
  @default_router = nil
183
204
  @routes.each do |rule|
184
205
  route_router = event_emitter_router(rule['@label'])
185
- puts rule
186
- @routers << Route.new(rule.matches, rule.tag.to_s, route_router)
206
+ @routers << Route.new(rule, route_router, @registry)
187
207
  end
188
208
 
189
209
  if @default_route != '' or @default_tag != ''
190
- @default_router = Route.new(nil, @default_tag, event_emitter_router(@default_route))
210
+ default_rule = { 'matches' => nil, 'tag' => @default_tag, '@label' => @default_route}
211
+ @default_router = Route.new(default_rule, event_emitter_router(@default_route), @registry)
191
212
  end
192
213
 
193
214
  @access_to_labels = record_accessor_create("$.kubernetes.labels")
@@ -76,7 +76,7 @@ class LabelRouterOutputTest < Test::Unit::TestCase
76
76
  d = Fluent::Test::Driver::BaseOwner.new(Fluent::Plugin::LabelRouterOutput)
77
77
  d.configure(routing_conf)
78
78
 
79
- r1 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[0].matches, d.instance.routes[0].tag,nil)
79
+ r1 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[0], nil,nil)
80
80
  # Selector matched: GO
81
81
  assert_equal(true, r1.match?(labels: { 'app' => 'app1' }, namespace: ''))
82
82
  # Exclude match: NO GO
@@ -84,7 +84,7 @@ class LabelRouterOutputTest < Test::Unit::TestCase
84
84
  # Nothing matched: NO GO
85
85
  assert_equal(false, r1.match?(labels: { 'app3' => 'app2' }, namespace: ''))
86
86
 
87
- r2 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[1].matches, d.instance.routes[1].tag,nil)
87
+ r2 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[1], nil,nil)
88
88
  # Match selector and namespace: GO
89
89
  assert_equal(true, r2.match?(labels: { 'app' => 'app1' }, namespace: 'test'))
90
90
  # Exclude via namespace
@@ -92,18 +92,18 @@ class LabelRouterOutputTest < Test::Unit::TestCase
92
92
  # Nothing matched: NO GO
93
93
  assert_equal(false, r2.match?(labels: { 'app3' => 'app' }, namespace: 'system'))
94
94
 
95
- r3 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[2].matches, d.instance.routes[2].tag,nil)
95
+ r3 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[2], nil,nil)
96
96
  assert_equal(true, r3.match?(labels: { 'app' => 'nginx' }, namespace: 'dev'))
97
97
  assert_equal(true, r3.match?(labels: { 'app' => 'nginx' }, namespace: 'sandbox'))
98
98
  assert_equal(false, r3.match?(labels: { 'app' => 'nginx2' }, namespace: 'sandbox'))
99
99
 
100
- r4 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[3].matches, d.instance.routes[3].tag,nil)
100
+ r4 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[3], nil,nil)
101
101
  # Matching container name
102
- assert_equal(true, r4.match?(labels: { 'app' => 'nginx' }, namespace: 'dev', container_name: 'mycontainer'))
102
+ assert_equal(true, r4.match?(labels: { 'app' => 'nginx' }, namespace: 'dev', container: 'mycontainer'))
103
103
  # Missing container name is equal to wrong container
104
104
  assert_equal(false, r4.match?(labels: { 'app' => 'nginx' }, namespace: 'sandbox'))
105
105
  # Wrong container name
106
- assert_equal(false, r4.match?(labels: { 'app' => 'nginx' }, namespace: 'dev', container_name: 'mycontainer2'))
106
+ assert_equal(false, r4.match?(labels: { 'app' => 'nginx' }, namespace: 'dev', container: 'mycontainer2'))
107
107
  # Wrong label but good namespace and container_name
108
108
  assert_equal(false, r4.match?(labels: { 'app' => 'nginx2' }, namespace: 'sandbox', container_name: 'mycontainer2'))
109
109
  end
@@ -208,4 +208,35 @@ default_tag "new_tag"
208
208
  assert_equal ["new_app_tag", event_time, {"kubernetes" => {"labels" => {"app" => "app2"} } }], events[1]
209
209
  end
210
210
  end
211
+
212
+ sub_test_case 'test_metrics' do
213
+ test 'normal' do
214
+ CONFIG4 = %[
215
+ @id xxx
216
+ metrics true
217
+ <route>
218
+ metrics_labels {"id": "test"}
219
+ tag new_app_tag
220
+ <match>
221
+ labels
222
+ namespaces
223
+ </match>
224
+ </route>
225
+ ]
226
+ event_time = event_time("2019-07-17 11:11:11 UTC")
227
+ d = create_driver(CONFIG4)
228
+ d.run(default_tag: 'test') do
229
+ d.feed(event_time, {"kubernetes" => {"labels" => {"app" => "app1"} } } )
230
+ end
231
+ d.run(default_tag: 'test2') do
232
+ d.feed(event_time, {"kubernetes" => {"labels" => {"app" => "app2"} } } )
233
+ end
234
+ events = d.events
235
+
236
+ assert_equal(2, events.size)
237
+ assert_equal ["new_app_tag", event_time, {"kubernetes" => {"labels" => {"app" => "app1"} } }], events[0]
238
+ assert_equal ["new_app_tag", event_time, {"kubernetes" => {"labels" => {"app" => "app2"} } }], events[1]
239
+ end
240
+ end
241
+
211
242
  end
metadata CHANGED
@@ -1,57 +1,57 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-label-router
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.3
4
+ version: 0.2.8
5
5
  platform: ruby
6
6
  authors:
7
7
  - Banzai Cloud
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-03-25 00:00:00.000000000 Z
11
+ date: 2021-06-16 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: bundler
14
+ name: rake
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '1.14'
19
+ version: '12.0'
20
20
  type: :development
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '1.14'
26
+ version: '12.0'
27
27
  - !ruby/object:Gem::Dependency
28
- name: rake
28
+ name: test-unit
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: '12.0'
33
+ version: '3.0'
34
34
  type: :development
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: '12.0'
40
+ version: '3.0'
41
41
  - !ruby/object:Gem::Dependency
42
- name: test-unit
42
+ name: prometheus-client
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - "~>"
45
+ - - ">="
46
46
  - !ruby/object:Gem::Version
47
- version: '3.0'
48
- type: :development
47
+ version: 2.1.0
48
+ type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - "~>"
52
+ - - ">="
53
53
  - !ruby/object:Gem::Version
54
- version: '3.0'
54
+ version: 2.1.0
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: fluentd
57
57
  requirement: !ruby/object:Gem::Requirement
@@ -92,7 +92,7 @@ homepage: https://github.com/banzaicloud/fluent-plugin-label-router
92
92
  licenses:
93
93
  - Apache-2.0
94
94
  metadata: {}
95
- post_install_message:
95
+ post_install_message:
96
96
  rdoc_options: []
97
97
  require_paths:
98
98
  - lib
@@ -107,8 +107,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
107
107
  - !ruby/object:Gem::Version
108
108
  version: '0'
109
109
  requirements: []
110
- rubygems_version: 3.0.3
111
- signing_key:
110
+ rubygems_version: 3.1.6
111
+ signing_key:
112
112
  specification_version: 4
113
113
  summary: Routing records based on Kubernetes labels.
114
114
  test_files: