fluent-plugin-label-router 0.1.3 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 752b0fee18d27194a9e3173eaa789626d9a27e43
-  data.tar.gz: c99654113f3b085822b50ca9491cfb99e5e55b55
+SHA256:
+  metadata.gz: e7a22624dbabbce0daa1bcf3d6845bf15e377d9453a5e733ad6c91b1c9d886d5
+  data.tar.gz: 5cb0f6eed3e37706bd2f09c0f399acd4fb6dc714aadb7d9879309575904dccfe
 SHA512:
-  metadata.gz: b28bb46f96513f43f029e956c06b4262ffc0afe50ab7e70a380acf87751081a827a42b0060349c31811bf1bf71c188bc21b6f726c4c0ee1b9f7d7cebb7705c53
-  data.tar.gz: 91a600e6105e278407b9d423ea7baf4547c2da741ecd044c9b4be0b7db436a032f6916e89688230c1376945db0c3379df73e8761cdc50b1577c4b3bd97369270
+  metadata.gz: f1240df83181e1e6188121c48dbffc4fbe226c756064155e1457dd2478aae79d6740dcb5b403202fd2ef0d55b7850f1937e6aed9f414dc73615f24f5ae3e0f2a
+  data.tar.gz: 13093eb11d980427a8bc5540612da4bca7a199a6aa848c2054ec157a930b4db372cd6c6ceed4ec854f783e165bfe7448f6ad53add6409f150460b49f522ea1e4

data/README.md

@@ -32,7 +32,10 @@ $ bundle
 
 ## Configuration
 
-The configuration builds from `<route>` sections.
+The configuration builds from `<route>` sections. Each `route` section
+can have several `<match>` statement. These statements computed in order and
+positive (or in case of *negate true* negative) results break the evaluation.
+We can say that the sections are coupled in a **lazy evaluation OR**. 
 
 ```
 <match example.tag**>
@@ -41,32 +44,88 @@ The configuration builds from `` sections.
      ...
   </route>
   <route>
-     ...
+    <match>
+      ...
+    </match>
+    <match> #Exclude
+      negate true
+      ...
+    </match>
   </route>
 </match>
 ```
 
-| Parameter | Description | Default |
-|-----------|-------------|---------|
-| labels | Label definition to match record. Example: app:nginx  | nil |
-| namespace | Namespaces definition to filter the record. Ignored if left empty. | "" |
-| @label | New @LABEL if selectors matched | nil |
-| tag | New tag if selectors matched | "" |
-| emit_mode | Emit mode. If `batch`, the plugin will emit events per labels matched. Enum: record, batch | batch |
-| sticky_tags | Sticky tags will match only one record from an event stream. The same tag will be treated the same way | true |
+Configuration reference
+
+| Parameter     | Description                                                                                            | Type    | Default |
+|---------------|--------------------------------------------------------------------------------------------------------|---------|---------|
+| emit_mode     | Emit mode. If `batch`, the plugin will emit events per labels matched. Enum: record, batch             | enum    | batch   |
+| sticky_tags   | Sticky tags will match only one record from an event stream. The same tag will be treated the same way | bool    | true    |
+| default_route | If defined all non-matching record passes to this label.                                               | string  |  ""     |
+| default_tag   | If defined all non-matching record rewrited to this tag. (Can be used with label simoultanesly)        | string  |  ""     |
+| \<route\>     | Route the log if match with parameters defined                                                         | []route | nil     |
+
+#### \<route\>
+| Parameter     | Description                                                                                            | Type    | Default |
+|---------------|--------------------------------------------------------------------------------------------------------|---------|---------|
+| @label        | Route the matching record to the given `label`                                                         | string  | ""      |
+| tag           | Tag the matching record to the given `tag`                                                             | string  | ""      |
+| \<match\>     | List of match statements. Repeatable.                                                                  | []match | nil     |
+
+
+#### \<match\>
+| Parameter       | Description                                                                   | Type     | Default  |
+|-----------------|-------------------------------------------------------------------------------|----------|----------|
+| labels          | Label definition to match record. Example: `app:nginx`                        | Hash     | nil      |
+| namespaces      | Comma separated list of namespaces. Ignored if left empty.                    | []string | nil      |
+| hosts           | Comma separated list of hosts. Ignored if left empty.                         | []string | nil      |
+| container_names | Comma separated list of container names. Ignored if left empty.               | []string | nil      |
+| negate          | Negate the selector meaning to exclude matches                                | bool     | false    |
+
+## Rules of thumb
+
+1. Defining more than one namespace in `namespaces` inside a `match` statement
+will check whether any of that namespaces matches.
+
+2. Using `sticky_tags` means that only the **first** record will be analysed per `tag`.
+Keep that in mind if you are ingesting traffic that is not unique on a per tag bases.
+Fluentd and fluent-bit tail logs from Kubernetes are unique per container.
+
+3. The plugin does not check if the configuration is valid so be careful to not define
+ statements like identical `match` statement with negate because the negate rule will never
+ be evaluated.
 
 ## Examples
 
-### 1. Route specific `labels` and `namespace` to `@label` and new `tag`
+### 1. Route specific `labels` and `namespaces` to `@label` and new `tag`
 Configuration to re-tag and re-label all logs from `default` namespace with label `app=nginx` and `env=dev`.
 ```
 <match example.tag**>
   @type label_router
   <route>
-     labels app:nginx,env:dev
-     namespace default
-     @label @NGINX
-     tag new_tag
+    @label @NGINX
+    tag new_tag
+    <match>
+      labels app:nginx,env:dev
+      namespaces default
+    </match>
+  </route>
+</match>
+```
+
+### 2. Exclude specific `labels` and `namespaces`
+Configuration to re-tag and re-label all logs that **not** from `default` namespace **and not** have labels `ap=nginx` and `env=dev`
+```
+<match example.tag**>
+  @type label_router
+  <route>
+    @label @NGINX
+    tag new_tag
+    <match>
+      negate true
+      labels app:nginx,env:dev
+      namespaces default
+    </match>
   </route>
 </match>
 ```
@@ -91,9 +150,11 @@ Only `labels`
 <match example.tag**>
   @type label_router
   <route>
-     labels app:nginx
-     @label @NGINX
-     tag new_tag
+    @label @NGINX
+    tag new_tag
+    <match>
+      labels app:nginx
+    </match>
   </route>
 </match>
 ```
@@ -102,9 +163,11 @@ Only `namespace`
 <match example.tag**>
   @type label_router
   <route>
-     namespace default
-     @label @NGINX
-     tag new_tag
+    @label @NGINX
+    tag new_tag
+    <match>
+      namespaces default
+    </match>
   </route>
 </match>
 ```
@@ -112,16 +175,31 @@ Rewrite all
 ```
 <match example.tag**>
   @type label_router
-  <route>
-     @label @NGINX
-     tag new_tag
-  </route>
+  <match>
+    @label @NGINX
+    tag new_tag
+  </match>
 </match>
 ```
 
 ### 3. One of `@label` ot `tag` configuration should be specified
 If you don't rewrite either of them fluent will **likely to crash** because it will reprocess the same messages again.
 
+### 4. Default route/tag
+
+Use `default_label` and/or `default_tag` to route non matching records.
+
+```
+<match example.tag**>
+  @type label_router
+  default_route @default_sink
+  <route>
+     ...
+  </route>
+</match>
+```
+
+
 ## Copyright
 
 * Copyright(c) 2019- Banzai Cloud

data/fluent-plugin-label-router.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-label-router"
-  spec.version = "0.1.3"
+  spec.version = "0.2.4"
   spec.authors = ["Banzai Cloud"]
   spec.email   = ["info@banzaicloud.com"]
 

data/lib/fluent/plugin/out_label_router.rb

@@ -26,33 +26,77 @@ module Fluent
       #record_accessor_create("log")
       #record_accessor_create("$.key1.key2")
       #record_accessor_create("$['key1'][0]['key2']")
+      desc "Emit mode. If `batch`, the plugin will emit events per labels matched."
+      config_param :emit_mode, :enum, list: [:record, :batch], default: :batch
+      desc "Sticky tags will match only one record from an event stream. The same tag will be treated the same way"
+      config_param :sticky_tags, :bool, default: true
+      desc "Default label to drain unmatched patterns"
+      config_param :default_route, :string, :default => ""
+      desc "Default tag to drain unmatched patterns"
+      config_param :default_tag, :string, :default => ""
 
       config_section :route, param_name: :routes, multi: true do
-        desc "Label definition to match record. Example: app:nginx. You can specify more values as comma separated list: key1:value1,key2:value2"
-        config_param :labels, :hash, :default => {}
-        desc "Namespaces definition to filter the record. Ignored if left empty."
-        config_param :namespace, :string, :default => ""
         desc "New @LABEL if selectors matched"
         config_param :@label, :string, :default => nil
         desc "New tag if selectors matched"
         config_param :tag, :string, :default => ""
-        desc "Emit mode. If `batch`, the plugin will emit events per labels matched."
-        config_param :emit_mode, :enum, list: [:record, :batch], default: :batch
-        desc "Sticky tags will match only one record from an event stream. The same tag will be treated the same way"
-        config_param :sticky_tags, :bool, default: true
+
+        config_section :match, param_name: :matches, multi: true do
+          desc "Label definition to match record. Example: app:nginx. You can specify more values as comma separated list: key1:value1,key2:value2"
+          config_param :labels, :hash, :default => {}
+          desc "List of namespace definition to filter the record. Ignored if left empty."
+          config_param :namespaces, :array, :default => [], value_type: :string
+          desc "List of hosts definition to filter the record. Ignored if left empty."
+          config_param :hosts, :array, :default => [], value_type: :string
+          desc "List of container names definition to filter the record. Ignored if left empty."
+          config_param :container_names, :array, :default => [], value_type: :string
+          desc "Negate the selection making it an exclude"
+          config_param :negate, :bool, :default => false
+        end
       end
 
       class Route
-        def initialize(selector, namespace, tag, router)
+        def initialize(matches, tag, router)
           @router = router
-          @selector = selector
-          @namespace = namespace
+          @matches = matches
           @tag = tag
         end
 
-        def match?(labels, namespace)
-          # Match labels and namespace if defined
-          return (match_labels(labels, @selector) and (@namespace ==  "" or namespace == @namespace))
+        # Evaluate selectors
+        # We evaluate <match> statements in order:
+        # 1. If match == true and negate == false  -> return true
+        # 2. If match == true and negate == true   -> return false
+        # 3. If match == false and negate == false -> continue
+        # 4. If match == false and negate == true  -> continue
+        # There is no match at all                 -> return false
+        def match?(metadata)
+          @matches.each do |match|
+            if filter_select(match, metadata) and !match.negate
+              return true
+            end
+            if filter_select(match, metadata) and match.negate
+              return false
+            end
+          end
+          false
+        end
+
+        # Returns true if filter passes (filter match)
+        def filter_select(match, metadata)
+          # Break on container_name mismatch
+          unless match.hosts.empty? || match.hosts.include?(metadata[:host])
+            return false
+          end
+          # Break on host mismatch
+          unless match.container_names.empty? || match.container_names.include?(metadata[:container])
+            return false
+          end
+          # Break if list of namespaces is not empty and does not include actual namespace
+          unless match.namespaces.empty? || match.namespaces.include?(metadata[:namespace])
+            return false
+          end
+
+          match_labels(metadata[:labels], match.labels)
         end
 
         def emit(tag, time, record)
@@ -70,29 +114,38 @@ module Fluent
             @router.emit_stream(@tag, es)
           end
         end
+
         def match_labels(input, match)
-          return (match.to_a - input.to_a).empty?
+          (match.to_a - input.to_a).empty?
         end
       end
 
       def process(tag, es)
         if @sticky_tags
-          if @route_map.has_key?(tag)
-            # We already matched with this tag send events to the routers
-            @route_map[tag].each do |r|
-              r.emit_es(tag, es.dup)
+          @mutex.synchronize {
+            if @route_map.has_key?(tag)
+              # We already matched with this tag send events to the routers
+              @route_map[tag].each do |r|
+                r.emit_es(tag, es.dup)
+              end
+              return
             end
-            return
-          end
+          }
         end
         event_stream = Hash.new {|h, k| h[k] = Fluent::MultiEventStream.new }
         es.each do |time, record|
-          input_labels = @access_to_labels.call(record).to_h
-          input_namespace = @access_to_namespace.call(record).to_s
+          input_metadata = { labels: @access_to_labels.call(record).to_h,
+                             namespace: @access_to_namespace.call(record).to_s,
+                             container: @access_to_container_name.call(record).to_s,
+                             host: @access_to_host.call(record).to_s}
+          orphan_record = true
           @routers.each do |r|
-            if r.match?(input_labels, input_namespace)
+            if r.match?(input_metadata)
+              orphan_record = false
               if @sticky_tags
-                @route_map[tag].push(r)
+                @mutex.synchronize {
+                  @route_map[tag].add(r)
+                }
               end
               if @batch
                 event_stream[r].add(time, record)
@@ -101,25 +154,46 @@ module Fluent
               end
             end
           end
-          if @batch
-            event_stream.each do |r, es|
-              r.emit_es(tag, es.dup)
+          if !@default_router.nil? && orphan_record
+            if @sticky_tags
+              @mutex.synchronize {
+                @route_map[tag].add(@default_router)
+              }
+            end
+            if @batch
+              event_stream[@default_router].add(time, record)
+            else
+              @default_router.emit(tag, time, record.dup)
             end
           end
         end
+        if @batch
+          event_stream.each do |r, es|
+            r.emit_es(tag, es.dup)
+          end
+        end
       end
 
       def configure(conf)
         super
-        @route_map = Hash.new { |h, k| h[k] = Array.new }
+        @route_map = Hash.new { |h, k| h[k] = Set.new }
+        @mutex = Mutex.new
         @routers = []
+        @default_router = nil
         @routes.each do |rule|
           route_router = event_emitter_router(rule['@label'])
-          @routers << Route.new(rule.labels, rule.namespace.to_s, rule.tag.to_s, route_router)
+          puts rule
+          @routers << Route.new(rule.matches, rule.tag.to_s, route_router)
+        end
+
+        if @default_route != '' or @default_tag != ''
+          @default_router = Route.new(nil, @default_tag, event_emitter_router(@default_route))
         end
 
         @access_to_labels = record_accessor_create("$.kubernetes.labels")
         @access_to_namespace = record_accessor_create("$.kubernetes.namespace_name")
+        @access_to_host = record_accessor_create("$.kubernetes.host")
+        @access_to_container_name = record_accessor_create("$.kubernetes.container_name")
 
         @batch = @emit_mode == :batch
       end

data/test/plugin/test_out_label_router.rb

@@ -34,11 +34,88 @@ class LabelRouterOutputTest < Test::Unit::TestCase
     d.configure(conf)
   end
 
+  sub_test_case 'test_routing' do
+    test 'basic configuration' do
+      routing_conf = %(
+<route>
+  <match>
+    labels app:app1
+  </match>
+  <match>
+    labels app2:app2
+    negate true
+  </match>
+  tag new_app_tag
+</route>
+<route>
+  <match>
+    labels app:app1
+    namespaces default,test
+  </match>
+  <match>
+    labels app:app2
+    namespaces system
+    negate true
+  </match>
+  tag new_app_tag
+</route>
+<route>
+  <match>
+    labels app:nginx
+    namespaces dev,sandbox
+  </match>
+</route>
+<route>
+  <match>
+    labels app:nginx
+    namespaces dev,sandbox
+    container_names mycontainer
+  </match>
+</route>
+)
+      d = Fluent::Test::Driver::BaseOwner.new(Fluent::Plugin::LabelRouterOutput)
+      d.configure(routing_conf)
+
+      r1 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[0].matches, d.instance.routes[0].tag,nil)
+      # Selector matched: GO
+      assert_equal(true, r1.match?(labels: { 'app' => 'app1' }, namespace: ''))
+      # Exclude match: NO GO
+      assert_equal(false, r1.match?(labels: { 'app' => 'app2' }, namespace: ''))
+      # Nothing matched: NO GO
+      assert_equal(false, r1.match?(labels: { 'app3' => 'app2' }, namespace: ''))
+
+      r2 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[1].matches, d.instance.routes[1].tag,nil)
+      # Match selector and namespace: GO
+      assert_equal(true, r2.match?(labels: { 'app' => 'app1' }, namespace: 'test'))
+      # Exclude via namespace
+      assert_equal(false, r2.match?(labels: { 'app' => 'app2' }, namespace: 'system'))
+      # Nothing matched: NO GO
+      assert_equal(false, r2.match?(labels: { 'app3' => 'app' }, namespace: 'system'))
+
+      r3 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[2].matches, d.instance.routes[2].tag,nil)
+      assert_equal(true, r3.match?(labels: { 'app' => 'nginx' }, namespace: 'dev'))
+      assert_equal(true, r3.match?(labels: { 'app' => 'nginx' }, namespace: 'sandbox'))
+      assert_equal(false, r3.match?(labels: { 'app' => 'nginx2' }, namespace: 'sandbox'))
+
+      r4 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[3].matches, d.instance.routes[3].tag,nil)
+      # Matching container name
+      assert_equal(true, r4.match?(labels: { 'app' => 'nginx' }, namespace: 'dev', container: 'mycontainer'))
+      # Missing container name is equal to wrong container
+      assert_equal(false, r4.match?(labels: { 'app' => 'nginx' }, namespace: 'sandbox'))
+      # Wrong container name
+      assert_equal(false, r4.match?(labels: { 'app' => 'nginx' }, namespace: 'dev', container: 'mycontainer2'))
+      # Wrong label but good namespace and container_name
+      assert_equal(false, r4.match?(labels: { 'app' => 'nginx2' }, namespace: 'sandbox',  container_name: 'mycontainer2'))
+    end
+  end
+
   sub_test_case 'test_tag' do
     test 'normal' do
       CONFIG = %[
 <route>
-  labels app:app1
+  <match>
+    labels app:app1
+  </match>
   tag new_app_tag
 </route>
 ]
@@ -56,4 +133,79 @@ class LabelRouterOutputTest < Test::Unit::TestCase
       assert_equal ["new_app_tag", event_time, {"kubernetes" => {"labels" => {"app" => "app1"} } }], events[0]
     end
   end
+
+
+  sub_test_case 'test_multiple_events_batched' do
+    test 'normal' do
+      conf = %[
+<route>
+  <match>
+  </match>
+</route>
+]
+      event_time = event_time("2019-07-17 11:11:11 UTC")
+      d = create_driver(conf)
+      d.run(expect_emits: 1, expect_records: 2) do
+        d.feed("test", [
+          [event_time, {"kubernetes" => {} } ],
+          [event_time, {"kubernetes" => {} } ],
+        ])
+      end
+    end
+  end
+
+  sub_test_case 'test_default_router' do
+    test 'normal' do
+      CONFIG2 = %[
+<route>
+  <match>
+    labels app:app1
+  </match>
+  tag new_app_tag
+</route>
+default_route @default
+default_tag "new_tag"
+]
+      event_time = event_time("2019-07-17 11:11:11 UTC")
+      d = create_driver(CONFIG2)
+      d.run() do
+        d.feed("test", [
+          [event_time, {"kubernetes" => {"labels" => {"app" => "app1"} } } ],
+          [event_time, {"kubernetes" => {"labels" => {"app" => "app2"} } } ],
+        ])
+      end
+      events = d.events
+
+      assert_equal(2, events.size)
+      assert_equal ["new_app_tag", event_time, {"kubernetes" => {"labels" => {"app" => "app1"} } }], events[0]
+      assert_equal ["new_tag", event_time, {"kubernetes" => {"labels" => {"app" => "app2"} } }], events[1]
+    end
+  end
+
+  sub_test_case 'test_empty_router' do
+    test 'normal' do
+      CONFIG3 = %[
+<route>
+  tag new_app_tag
+  <match>
+    labels
+    namespaces
+  </match>
+</route>
+]
+      event_time = event_time("2019-07-17 11:11:11 UTC")
+      d = create_driver(CONFIG3)
+      d.run(default_tag: 'test') do
+        d.feed(event_time, {"kubernetes" => {"labels" => {"app" => "app1"} } } )
+      end
+      d.run(default_tag: 'test2') do
+        d.feed(event_time, {"kubernetes" => {"labels" => {"app" => "app2"} } } )
+      end
+      events = d.events
+
+      assert_equal(2, events.size)
+      assert_equal ["new_app_tag", event_time, {"kubernetes" => {"labels" => {"app" => "app1"} } }], events[0]
+      assert_equal ["new_app_tag", event_time, {"kubernetes" => {"labels" => {"app" => "app2"} } }], events[1]
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-label-router
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.4
 platform: ruby
 authors:
 - Banzai Cloud
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-18 00:00:00.000000000 Z
+date: 2020-07-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -107,8 +107,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2.3
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Routing records based on Kubernetes labels.