fluent-plugin-label-router 0.2.2 → 0.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 323aeb0b86fedafdcca48546cb3eed9aa7523efa01df41c3f9a46cd069cd76bb
-  data.tar.gz: b42d4410cdbacabcf49a8eb2f5d8f8be01d3761efb9b436370ba3166a64eaeb3
+  metadata.gz: 9fcacfb0c98686f99facd3bd94da5fa8065b4446cf1d2e5850a3e1456e5721d1
+  data.tar.gz: 65f9d48cbf0636faa0544b052d16465760cc74c9a8fe3fb8aa378f122f7d0da6
 SHA512:
-  metadata.gz: 3e0989c5509dda968b509bb1ec2b01f125b83353aaa7347a68364b2412b41eec93187d320d5005dde9d9e9fadda1afb9ce8b6416dcb86771f6646dd0c8ade2ce
-  data.tar.gz: 5c53945c123e3271223dff6e04d9eb41bbb674c0876ce62f48af49f70b9adb7c90218c8e70899557b9a20215d999434f595e47dfb1f34a55bf817233b5ddf955
+  metadata.gz: 6b1f0a77e69be61a6ade1329b4e1caa58dde56fdd0d2313bbfa54476342d5d04cacdbd71dac21f5b04ce191f5839624f25ff4d3c765f6d627a48e8c692b86bb8
+  data.tar.gz: 3129f7a8a90f4d175941b9a719c96796245ed58bae0c0d60dbb42a608c310079b512aafb2385aebc7cee35b9df51c4bf781f3831696085f4438cce633c9e095a

data/README.md

@@ -192,7 +192,7 @@ Use `default_label` and/or `default_tag` to route non matching records.
 ```
 <match example.tag**>
   @type label_router
-  default_label @default_sink
+  default_route @default_sink
   <route>
      ...
   </route>

data/fluent-plugin-label-router.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-label-router"
-  spec.version = "0.2.2"
+  spec.version = "0.2.7"
   spec.authors = ["Banzai Cloud"]
   spec.email   = ["info@banzaicloud.com"]
 
@@ -20,8 +20,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = test_files
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.14"
   spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "test-unit", "~> 3.0"
+  spec.add_dependency "prometheus-client", ">= 2.1.0"
   spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
 end

data/lib/fluent/plugin/out_label_router.rb

@@ -11,10 +11,11 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations under the License.
+# limitations under the License
 
 require "fluent/plugin/output"
-require 'digest/md5'
+require 'prometheus/client'
+
 
 module Fluent
   module Plugin
@@ -34,12 +35,16 @@ module Fluent
       config_param :default_route, :string, :default => ""
       desc "Default tag to drain unmatched patterns"
       config_param :default_tag, :string, :default => ""
+      desc "Enable metrics for the router"
+      config_param :metrics, :bool, :default => false
 
       config_section :route, param_name: :routes, multi: true do
         desc "New @LABEL if selectors matched"
         config_param :@label, :string, :default => nil
         desc "New tag if selectors matched"
         config_param :tag, :string, :default => ""
+        desc "Extra labels for metrics"
+        config_param :metrics_labels, :hash, :default => {}
 
         config_section :match, param_name: :matches, multi: true do
           desc "Label definition to match record. Example: app:nginx. You can specify more values as comma separated list: key1:value1,key2:value2"
@@ -56,10 +61,22 @@ module Fluent
       end
 
       class Route
-        def initialize(matches, tag, router)
+        def initialize(rule, router, registry)
           @router = router
-          @matches = matches
-          @tag = tag
+          @matches = rule['matches']
+          @tag = rule['tag'].to_s
+          @label = rule['@label']
+          @metrics_labels = (rule['metrics_labels'].map { |k, v| [k.to_sym, v] }.to_h if rule['metrics_labels'])
+          @counter = nil
+          unless registry.nil?
+              @counter = registry.counter(:fluentd_router_records_total, docstring: "Total number of events router for the flow", labels: [:flow])
+          end
+        end
+
+        def get_labels
+          default = { 'flow': @label }
+          labels = default.merge(@metrics_labels)
+          labels
         end
 
         # Evaluate selectors
@@ -88,7 +105,7 @@ module Fluent
             return false
           end
           # Break on host mismatch
-          unless match.container_names.empty? || match.container_names.include?(metadata[:container_name])
+          unless match.container_names.empty? || match.container_names.include?(metadata[:container])
             return false
           end
           # Break if list of namespaces is not empty and does not include actual namespace
@@ -113,6 +130,8 @@ module Fluent
           else
             @router.emit_stream(@tag, es)
           end
+          # increment the counter for a given label set
+          @counter&.increment(by: es.size, labels: get_labels)
         end
 
         def match_labels(input, match)
@@ -122,13 +141,15 @@ module Fluent
 
       def process(tag, es)
         if @sticky_tags
-          if @route_map.has_key?(tag)
-            # We already matched with this tag send events to the routers
-            @route_map[tag].each do |r|
-              r.emit_es(tag, es.dup)
+          @mutex.synchronize {
+            if @route_map.has_key?(tag)
+              # We already matched with this tag send events to the routers
+              @route_map[tag].each do |r|
+                r.emit_es(tag, es.dup)
+              end
+              return
             end
-            return
-          end
+          }
         end
         event_stream = Hash.new {|h, k| h[k] = Fluent::MultiEventStream.new }
         es.each do |time, record|
@@ -141,7 +162,9 @@ module Fluent
             if r.match?(input_metadata)
               orphan_record = false
               if @sticky_tags
-                @route_map[tag].push(r)
+                @mutex.synchronize {
+                  @route_map[tag].add(r)
+                }
               end
               if @batch
                 event_stream[r].add(time, record)
@@ -152,7 +175,9 @@ module Fluent
           end
           if !@default_router.nil? && orphan_record
             if @sticky_tags
-              @route_map[tag].push(@default_router)
+              @mutex.synchronize {
+                @route_map[tag].add(@default_router)
+              }
             end
             if @batch
               event_stream[@default_router].add(time, record)
@@ -170,17 +195,19 @@ module Fluent
 
       def configure(conf)
         super
-        @route_map = Hash.new { |h, k| h[k] = Array.new }
+        @registry = (::Prometheus::Client.registry if @metrics)
+        @route_map = Hash.new { |h, k| h[k] = Set.new }
+        @mutex = Mutex.new
         @routers = []
         @default_router = nil
         @routes.each do |rule|
           route_router = event_emitter_router(rule['@label'])
-          puts rule
-          @routers << Route.new(rule.matches, rule.tag.to_s, route_router)
+          @routers << Route.new(rule, route_router, @registry)
         end
 
         if @default_route != '' or @default_tag != ''
-          @default_router = Route.new(nil, @default_tag, event_emitter_router(@default_route))
+          default_rule = { 'matches' => nil, 'tag' => @default_tag, '@label' => @default_route}
+          @default_router = Route.new(default_rule, event_emitter_router(@default_route), @registry)
         end
 
         @access_to_labels = record_accessor_create("$.kubernetes.labels")

data/test/plugin/test_out_label_router.rb

@@ -76,7 +76,7 @@ class LabelRouterOutputTest < Test::Unit::TestCase
       d = Fluent::Test::Driver::BaseOwner.new(Fluent::Plugin::LabelRouterOutput)
       d.configure(routing_conf)
 
-      r1 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[0].matches, d.instance.routes[0].tag,nil)
+      r1 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[0], nil,nil)
       # Selector matched: GO
       assert_equal(true, r1.match?(labels: { 'app' => 'app1' }, namespace: ''))
       # Exclude match: NO GO
@@ -84,7 +84,7 @@ class LabelRouterOutputTest < Test::Unit::TestCase
       # Nothing matched: NO GO
       assert_equal(false, r1.match?(labels: { 'app3' => 'app2' }, namespace: ''))
 
-      r2 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[1].matches, d.instance.routes[1].tag,nil)
+      r2 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[1], nil,nil)
       # Match selector and namespace: GO
       assert_equal(true, r2.match?(labels: { 'app' => 'app1' }, namespace: 'test'))
       # Exclude via namespace
@@ -92,18 +92,18 @@ class LabelRouterOutputTest < Test::Unit::TestCase
       # Nothing matched: NO GO
       assert_equal(false, r2.match?(labels: { 'app3' => 'app' }, namespace: 'system'))
 
-      r3 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[2].matches, d.instance.routes[2].tag,nil)
+      r3 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[2], nil,nil)
       assert_equal(true, r3.match?(labels: { 'app' => 'nginx' }, namespace: 'dev'))
       assert_equal(true, r3.match?(labels: { 'app' => 'nginx' }, namespace: 'sandbox'))
       assert_equal(false, r3.match?(labels: { 'app' => 'nginx2' }, namespace: 'sandbox'))
 
-      r4 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[3].matches, d.instance.routes[3].tag,nil)
+      r4 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[3], nil,nil)
       # Matching container name
-      assert_equal(true, r4.match?(labels: { 'app' => 'nginx' }, namespace: 'dev', container_name: 'mycontainer'))
+      assert_equal(true, r4.match?(labels: { 'app' => 'nginx' }, namespace: 'dev', container: 'mycontainer'))
       # Missing container name is equal to wrong container
       assert_equal(false, r4.match?(labels: { 'app' => 'nginx' }, namespace: 'sandbox'))
       # Wrong container name
-      assert_equal(false, r4.match?(labels: { 'app' => 'nginx' }, namespace: 'dev', container_name: 'mycontainer2'))
+      assert_equal(false, r4.match?(labels: { 'app' => 'nginx' }, namespace: 'dev', container: 'mycontainer2'))
       # Wrong label but good namespace and container_name
       assert_equal(false, r4.match?(labels: { 'app' => 'nginx2' }, namespace: 'sandbox',  container_name: 'mycontainer2'))
     end
@@ -208,4 +208,33 @@ default_tag "new_tag"
       assert_equal ["new_app_tag", event_time, {"kubernetes" => {"labels" => {"app" => "app2"} } }], events[1]
     end
   end
+
+    sub_test_case 'test_metrics' do
+      test 'normal' do
+        CONFIG4 = %[
+  metrics true
+  <route>
+    tag new_app_tag
+    <match>
+      labels
+      namespaces
+    </match>
+  </route>
+  ]
+        event_time = event_time("2019-07-17 11:11:11 UTC")
+        d = create_driver(CONFIG4)
+        d.run(default_tag: 'test') do
+          d.feed(event_time, {"kubernetes" => {"labels" => {"app" => "app1"} } } )
+        end
+        d.run(default_tag: 'test2') do
+          d.feed(event_time, {"kubernetes" => {"labels" => {"app" => "app2"} } } )
+        end
+        events = d.events
+
+        assert_equal(2, events.size)
+        assert_equal ["new_app_tag", event_time, {"kubernetes" => {"labels" => {"app" => "app1"} } }], events[0]
+        assert_equal ["new_app_tag", event_time, {"kubernetes" => {"labels" => {"app" => "app2"} } }], events[1]
+      end
+    end
+
 end

metadata

@@ -1,57 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-label-router
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.7
 platform: ruby
 authors:
 - Banzai Cloud
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-03-24 00:00:00.000000000 Z
+date: 2021-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: '12.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: '12.0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
-  name: test-unit
+  name: prometheus-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :development
+        version: 2.1.0
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: 2.1.0
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -92,7 +92,7 @@ homepage: https://github.com/banzaicloud/fluent-plugin-label-router
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -107,8 +107,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.6
+signing_key:
 specification_version: 4
 summary: Routing records based on Kubernetes labels.
 test_files: