fluent-plugin-label-router 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 752b0fee18d27194a9e3173eaa789626d9a27e43
-  data.tar.gz: c99654113f3b085822b50ca9491cfb99e5e55b55
+SHA256:
+  metadata.gz: 07025a5148e5985c287b5d07e4a8f05c642d650734db78c995be11c2ee478a5e
+  data.tar.gz: 8e787eacb12b8e12a1e2895f919128d1c755e4c553c9e5b218a7970e89f3564b
 SHA512:
-  metadata.gz: b28bb46f96513f43f029e956c06b4262ffc0afe50ab7e70a380acf87751081a827a42b0060349c31811bf1bf71c188bc21b6f726c4c0ee1b9f7d7cebb7705c53
-  data.tar.gz: 91a600e6105e278407b9d423ea7baf4547c2da741ecd044c9b4be0b7db436a032f6916e89688230c1376945db0c3379df73e8761cdc50b1577c4b3bd97369270
+  metadata.gz: 8e116c6ace26a31641f467eed943f5a643c1f4c6737ea602106c43af69f40cac89fe54533b08e596d3c32f9630c5a2d1a9532f37314bdf24081c6b4aeb70a120
+  data.tar.gz: e8d0c1c3653c047a65fc0642602a7e5e4aef3e3a341653fca64bd7e15323233c31583451591f9b744becd3f54122b55abd0d936ae48d60a63a5fc4dc640695ca

data/README.md

@@ -32,7 +32,10 @@ $ bundle
 
 ## Configuration
 
-The configuration builds from `<route>` sections.
+The configuration builds from `<route>` sections. Each `route` section
+can have several `<match>` statement. These statements computed in order and
+positive (or in case of *negate true* negative) results break the evaluation.
+We can say that the sections are coupled in a **lazy evaluation OR**. 
 
 ```
 <match example.tag**>
@@ -41,32 +44,86 @@ The configuration builds from `<route>` sections.
      ...
   </route>
   <route>
-     ...
+    <match>
+      ...
+    </match>
+    <match> #Exclude
+      negate true
+      ...
+    </match>
   </route>
 </match>
 ```
 
-| Parameter | Description | Default |
-|-----------|-------------|---------|
-| labels | Label definition to match record. Example: app:nginx  | nil |
-| namespace | Namespaces definition to filter the record. Ignored if left empty. | "" |
-| @label | New @LABEL if selectors matched | nil |
-| tag | New tag if selectors matched | "" |
-| emit_mode | Emit mode. If `batch`, the plugin will emit events per labels matched. Enum: record, batch | batch |
-| sticky_tags | Sticky tags will match only one record from an event stream. The same tag will be treated the same way | true |
+Configuration reference
+
+| Parameter     | Description                                                                                            | Type    | Default |
+|---------------|--------------------------------------------------------------------------------------------------------|---------|---------|
+| emit_mode     | Emit mode. If `batch`, the plugin will emit events per labels matched. Enum: record, batch             | enum    | batch   |
+| sticky_tags   | Sticky tags will match only one record from an event stream. The same tag will be treated the same way | bool    | true    |
+| default_route | If defined all non-matching record passes to this label.                                               | string  |  ""     |
+| default_tag   | If defined all non-matching record rewrited to this tag. (Can be used with label simoultanesly)        | string  |  ""     |
+| \<route\>     | Route the log if match with parameters defined                                                         | []route | nil     |
+
+#### \<route\>
+| Parameter     | Description                                                                                            | Type    | Default |
+|---------------|--------------------------------------------------------------------------------------------------------|---------|---------|
+| @label        | Route the matching record to the given `label`                                                         | string  | ""      |
+| tag           | Tag the matching record to the given `tag`                                                             | string  | ""      |
+| \<match\>     | List of match statements. Repeatable.                                                                  | []match | nil     |
+
+
+#### \<match\>
+| Parameter  | Description                                                                   | Type     | Default  |
+|------------|-------------------------------------------------------------------------------|----------|----------|
+| labels     | Label definition to match record. Example: `app:nginx`                        | Hash     | nil      |
+| namespaces | Comma separated list of namespaces. Ignored if left empty.                    | []string | nil      |
+| negate     | Negate the selector meaning to exclude matches                                | bool     | false    |
+
+## Rules of thumb
+
+1. Defining more than one namespace in `namespaces` inside a `match` statement
+will check whether any of that namespaces matches.
+
+2. Using `sticky_tags` means that only the **first** record will be analysed per `tag`.
+Keep that in mind if you are ingesting traffic that is not unique on a per tag bases.
+Fluentd and fluent-bit tail logs from Kubernetes are unique per container.
+
+3. The plugin does not check if the configuration is valid so be careful to not define
+ statements like identical `match` statement with negate because the negate rule will never
+ be evaluated.
 
 ## Examples
 
-### 1. Route specific `labels` and `namespace` to `@label` and new `tag`
+### 1. Route specific `labels` and `namespaces` to `@label` and new `tag`
 Configuration to re-tag and re-label all logs from `default` namespace with label `app=nginx` and `env=dev`.
 ```
 <match example.tag**>
   @type label_router
   <route>
-     labels app:nginx,env:dev
-     namespace default
-     @label @NGINX
-     tag new_tag
+    @label @NGINX
+    tag new_tag
+    <match>
+      labels app:nginx,env:dev
+      namespaces default
+    </match>
+  </route>
+</match>
+```
+
+### 2. Exclude specific `labels` and `namespaces`
+Configuration to re-tag and re-label all logs that **not** from `default` namespace **and not** have labels `ap=nginx` and `env=dev`
+```
+<match example.tag**>
+  @type label_router
+  <route>
+    @label @NGINX
+    tag new_tag
+    <match>
+      negate true
+      labels app:nginx,env:dev
+      namespaces default
+    </match>
   </route>
 </match>
 ```
@@ -91,9 +148,11 @@ Only `labels`
 <match example.tag**>
   @type label_router
   <route>
-     labels app:nginx
-     @label @NGINX
-     tag new_tag
+    @label @NGINX
+    tag new_tag
+    <match>
+      labels app:nginx
+    </match>
   </route>
 </match>
 ```
@@ -102,9 +161,11 @@ Only `namespace`
 <match example.tag**>
   @type label_router
   <route>
-     namespace default
-     @label @NGINX
-     tag new_tag
+    @label @NGINX
+    tag new_tag
+    <match>
+      namespaces default
+    </match>
   </route>
 </match>
 ```
@@ -112,16 +173,31 @@ Rewrite all
 ```
 <match example.tag**>
   @type label_router
-  <route>
-     @label @NGINX
-     tag new_tag
-  </route>
+  <match>
+    @label @NGINX
+    tag new_tag
+  </match>
 </match>
 ```
 
 ### 3. One of `@label` ot `tag` configuration should be specified
 If you don't rewrite either of them fluent will **likely to crash** because it will reprocess the same messages again.
 
+### 4. Default route/tag
+
+Use `default_label` and/or `default_tag` to route non matching records.
+
+```
+<match example.tag**>
+  @type label_router
+  default_label @default_sink
+  <route>
+     ...
+  </route>
+</match>
+```
+
+
 ## Copyright
 
 * Copyright(c) 2019- Banzai Cloud

data/fluent-plugin-label-router.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-label-router"
-  spec.version = "0.1.3"
+  spec.version = "0.2.0"
   spec.authors = ["Banzai Cloud"]
   spec.email   = ["info@banzaicloud.com"]
 

data/lib/fluent/plugin/out_label_router.rb

@@ -26,33 +26,67 @@ module Fluent
       #record_accessor_create("log")
       #record_accessor_create("$.key1.key2")
       #record_accessor_create("$['key1'][0]['key2']")
+      desc "Emit mode. If `batch`, the plugin will emit events per labels matched."
+      config_param :emit_mode, :enum, list: [:record, :batch], default: :batch
+      desc "Sticky tags will match only one record from an event stream. The same tag will be treated the same way"
+      config_param :sticky_tags, :bool, default: true
+      desc "Default label to drain unmatched patterns"
+      config_param :default_route, :string, :default => ""
+      desc "Default tag to drain unmatched patterns"
+      config_param :default_tag, :string, :default => ""
 
       config_section :route, param_name: :routes, multi: true do
-        desc "Label definition to match record. Example: app:nginx. You can specify more values as comma separated list: key1:value1,key2:value2"
-        config_param :labels, :hash, :default => {}
-        desc "Namespaces definition to filter the record. Ignored if left empty."
-        config_param :namespace, :string, :default => ""
         desc "New @LABEL if selectors matched"
         config_param :@label, :string, :default => nil
         desc "New tag if selectors matched"
         config_param :tag, :string, :default => ""
-        desc "Emit mode. If `batch`, the plugin will emit events per labels matched."
-        config_param :emit_mode, :enum, list: [:record, :batch], default: :batch
-        desc "Sticky tags will match only one record from an event stream. The same tag will be treated the same way"
-        config_param :sticky_tags, :bool, default: true
+
+        config_section :match, param_name: :selectors, multi: true do
+          desc "Label definition to match record. Example: app:nginx. You can specify more values as comma separated list: key1:value1,key2:value2"
+          config_param :labels, :hash, :default => {}
+          desc "List of namespace definition to filter the record. Ignored if left empty."
+          config_param :namespaces, :array, :default => [], value_type: :string
+          desc "Negate the selection making it an exclude"
+          config_param :negate, :bool, :default => false
+        end
+
       end
 
+
+
       class Route
-        def initialize(selector, namespace, tag, router)
+        def initialize(selectors, tag, router)
           @router = router
-          @selector = selector
-          @namespace = namespace
+          @selectors = selectors
           @tag = tag
         end
 
+        # Evaluate selectors
+        # We evaluate <match> statements in order:
+        # 1. If match == true and negate == false  -> return true
+        # 2. If match == true and negate == true   -> return false
+        # 3. If match == false and negate == false -> continue
+        # 4. If match == false and negate == true  -> continue
+        # There is no match at all                 -> return false
         def match?(labels, namespace)
-          # Match labels and namespace if defined
-          return (match_labels(labels, @selector) and (@namespace ==  "" or namespace == @namespace))
+          @selectors.each do |selector|
+            if (filter_select(selector, labels, namespace) and !selector.negate)
+              return true
+            end
+            if (filter_select(selector, labels, namespace) and selector.negate)
+              return false
+            end
+          end
+          false
+        end
+
+        # Returns true if filter passes (filter match)
+        def filter_select(selector, labels, namespace)
+          # Break if list of namespaces is not empty and does not include actual namespace
+          unless selector.namespaces.empty? or selector.namespaces.include?(namespace)
+            return false
+          end
+          match_labels(labels, selector.labels)
         end
 
         def emit(tag, time, record)
@@ -89,8 +123,10 @@ module Fluent
         es.each do |time, record|
           input_labels = @access_to_labels.call(record).to_h
           input_namespace = @access_to_namespace.call(record).to_s
+          orphan_record = true
           @routers.each do |r|
             if r.match?(input_labels, input_namespace)
+              orphan_record = false
               if @sticky_tags
                 @route_map[tag].push(r)
               end
@@ -101,6 +137,16 @@ module Fluent
               end
             end
           end
+          if !@default_router.nil? && orphan_record
+            if @sticky_tags
+              @route_map[tag].push(@default_router)
+            end
+            if @batch
+              event_stream[@default_router].add(time, record)
+            else
+              @default_router.emit(tag, time, record.dup)
+            end
+          end
           if @batch
             event_stream.each do |r, es|
               r.emit_es(tag, es.dup)
@@ -113,9 +159,15 @@ module Fluent
         super
         @route_map = Hash.new { |h, k| h[k] = Array.new }
         @routers = []
+        @default_router = nil
         @routes.each do |rule|
           route_router = event_emitter_router(rule['@label'])
-          @routers << Route.new(rule.labels, rule.namespace.to_s, rule.tag.to_s, route_router)
+          puts rule
+          @routers << Route.new(rule.selectors, rule.tag.to_s, route_router)
+        end
+
+        if @default_route != '' or @default_tag != ''
+          @default_router = Route.new(nil, @default_tag, event_emitter_router(@default_route))
         end
 
         @access_to_labels = record_accessor_create("$.kubernetes.labels")

data/test/plugin/test_out_label_router.rb

@@ -34,11 +34,70 @@ class LabelRouterOutputTest < Test::Unit::TestCase
     d.configure(conf)
   end
 
+  sub_test_case 'test_routing' do
+    test 'basic configuration' do
+      routing_conf = %(
+<route>
+  <match>
+    labels app:app1
+  </match>
+  <match>
+    labels app2:app2
+    negate true
+  </match>
+  tag new_app_tag
+</route>
+<route>
+  <match>
+    labels app:app1
+    namespaces default,test
+  </match>
+  <match>
+    labels app:app2
+    namespaces system
+    negate true
+  </match>
+  tag new_app_tag
+</route>
+<route>
+  <match>
+    labels app:nginx
+    namespaces dev,sandbox
+  </match>
+</route>
+)
+      d = Fluent::Test::Driver::BaseOwner.new(Fluent::Plugin::LabelRouterOutput)
+      d.configure(routing_conf)
+
+      r1 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[0].selectors, "",nil)
+      # Selector matched: GO
+      assert_equal(true, r1.match?({'app' => 'app1'},""))
+      # Exclude match: NO GO
+      assert_equal(false, r1.match?({"app" => "app2"},""))
+      # Nothing matched: NO GO
+      assert_equal(false, r1.match?({"app3" => "app2"},""))
+
+      r2 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[1].selectors, "",nil)
+      # Match selector and namespace: GO
+      assert_equal(true, r2.match?({"app" => "app1"},"test"))
+      # Exclude via namespace
+      assert_equal(false, r2.match?({"app" => "app2"},"system"))
+      # Nothing matched: NO GO
+      assert_equal(false, r2.match?({"app3" => "app"},"system"))
+
+      r3 = Fluent::Plugin::LabelRouterOutput::Route.new(d.instance.routes[2].selectors, "",nil)
+      assert_equal(true, r3.match?({"app" => "nginx"},"dev"))
+      assert_equal(true, r3.match?({"app" => "nginx"},"sandbox"))
+    end
+  end
+
   sub_test_case 'test_tag' do
     test 'normal' do
       CONFIG = %[
 <route>
-  labels app:app1
+  <match>
+    labels app:app1
+  </match>
   tag new_app_tag
 </route>
 ]
@@ -56,4 +115,59 @@ class LabelRouterOutputTest < Test::Unit::TestCase
       assert_equal ["new_app_tag", event_time, {"kubernetes" => {"labels" => {"app" => "app1"} } }], events[0]
     end
   end
+
+  sub_test_case 'test_default_router' do
+    test 'normal' do
+      CONFIG2 = %[
+<route>
+  <match>
+    labels app:app1
+  </match>
+  tag new_app_tag
+</route>
+default_route @default
+default_tag "new_tag"
+]
+      event_time = event_time("2019-07-17 11:11:11 UTC")
+      d = create_driver(CONFIG2)
+      d.run(default_tag: 'test') do
+        d.feed(event_time, {"kubernetes" => {"labels" => {"app" => "app1"} } } )
+      end
+      d.run(default_tag: 'test2') do
+        d.feed(event_time, {"kubernetes" => {"labels" => {"app" => "app2"} } } )
+      end
+      events = d.events
+
+      assert_equal(2, events.size)
+      assert_equal ["new_app_tag", event_time, {"kubernetes" => {"labels" => {"app" => "app1"} } }], events[0]
+      assert_equal ["new_tag", event_time, {"kubernetes" => {"labels" => {"app" => "app2"} } }], events[1]
+    end
+  end
+
+  sub_test_case 'test_empty_router' do
+    test 'normal' do
+      CONFIG2 = %[
+<route>
+  tag new_app_tag
+  <match>
+    labels
+    namespaces
+  </match>
+</route>
+]
+      event_time = event_time("2019-07-17 11:11:11 UTC")
+      d = create_driver(CONFIG2)
+      d.run(default_tag: 'test') do
+        d.feed(event_time, {"kubernetes" => {"labels" => {"app" => "app1"} } } )
+      end
+      d.run(default_tag: 'test2') do
+        d.feed(event_time, {"kubernetes" => {"labels" => {"app" => "app2"} } } )
+      end
+      events = d.events
+
+      assert_equal(2, events.size)
+      assert_equal ["new_app_tag", event_time, {"kubernetes" => {"labels" => {"app" => "app1"} } }], events[0]
+      assert_equal ["new_app_tag", event_time, {"kubernetes" => {"labels" => {"app" => "app2"} } }], events[1]
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-label-router
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - Banzai Cloud
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-18 00:00:00.000000000 Z
+date: 2020-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -107,8 +107,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2.3
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Routing records based on Kubernetes labels.