fluentd-plugin-annotation-filter 1.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 1768b27d7704200041610300fc039585b4375ca418d561f0396fe6283aac37d8
+  data.tar.gz: ca5091f841efcc2e01d9d25f2ddd72b624d974da1e1f7dfd8c568bf8dfad7d51
+SHA512:
+  metadata.gz: 1c61d281421542d367272a98ede9480f9cb5ac9d8cb21d333ea4e598c662bd4721a5a0e8dc061c3c6c0c31d3773289abf5069fe37eb4fcccdbe534668482238f
+  data.tar.gz: f04505e998d45e85952aa84979245accf314f6c11a7d6a2319033949021c940afef018fc74e69ba10f604b3a245c3bbed6f27270c204a68d8244e4e70da92f37

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/README.md

@@ -0,0 +1,124 @@
+# Fluentd Plugin: Kubernetes Annotation Filter
+
+Checks whether the container name is included in an annotation on the pod that is the source of a log entry.
+
+## Development
+
+### Running Locally
+
+It can be a bit of a challenge to get fluentd running locally for "poke-it-and-see" style testing. One approach is to:
+
+1. Get a docker container with fluentd in it
+1. Create an `app/plugins` directory
+1. Copy `lib/fluent/plugin/*` from this repository into that directory
+1. Create a fluentd configuration that uses this plugin, taking input from a file in the `/app` directory. An example
+   configuration can be found at [example/fluentd-test.conf](example/fluentd-test.conf).
+1. Run the container
+1. Put your test lines into the file
+1. Watch the output
+
+This is not a particularly nice way to develop anything, but it does _work_. You'll then need to copy the modified
+source code _back_ into this repository.
+
+### Tests
+
+You can run tests locally with:
+
+```
+> rspec
+```
+
+Or in docker with:
+
+```
+> make test
+```
+
+The tests are written using Fluent's [Test Driver](https://docs.fluentd.org/plugin-development/plugin-test-code); this
+requires the `test-unit` gem be included (for some helper methods we're not using) in addition to `rspec`, which is
+actually used to run the tests (in accordance with what all the rest of DRE's Ruby code uses).
+
+## Operations
+
+### Installation
+
+Add the following to your Gemfile:
+
+```ruby
+gem 'fluent-plugin-kubernetes-annotations-filter
+```
+
+### Dependencies
+
+This plugin relies on _Kubernetes metadata_ being available on log records. This metadata can be obtained by use of the
+[kuberenetes_metadata_filter](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter) plugin.
+
+Specifically, this plugin relies on the following fields:
+
+| Field                       | Content                                                        |
+|-----------------------------|----------------------------------------------------------------|
+| `kubernetes.container_name` | The name of the container the record was emitted from          |
+| `kubernetes.annotations`    | The annotations applied to the pod the record was emitted from |
+
+### Configuration
+
+An example configuration could look like:
+
+```
+<filter camelid>
+@type kubernetes_annotation
+
+<contains_container_name>
+annotation dromedary
+</contains_container_name>
+</filter>
+```
+
+This would look for the value of `kubernetes.container_name` in the `kubernetes.annotations.dromedary` field of records
+tagged "camelid"; for example, these records would be allowed to pass through the filter:
+
+```
+{"message": "hi", "kubernetes": {"containter_name": "beluga", ..., "annotations": { "dromedary": "[\"beluga\"]" } } }
+{"message": "wat", "kubernetes": {"containter_name": "humpback", ..., "annotations": { "dromedary": "[\"minke\", \"humpback\"]" } } }
+```
+
+but these ones would not:
+
+```
+{"message": "hi", "kubernetes": {"containter_name": "pelican", ..., "annotations": { "dromedary": "[\"minke\", \"humpback\"]" } } }
+{"message": "wat", "kubernetes": {"containter_name": "gannet", ..., "annotations": { } } }
+```
+
+Note that the value of the provided annotation is expected to be a _JSON array_ inside a string (this appears to be the
+way people suggest encoding data of this kind).
+
+The `pass_through_events_without_kubernetes_tags` option can be used to control whether objects without Kubernetes tags
+are passed through the filter or dropped. It defaults to `false`.
+
+## Maintainers
+
+delivery-engineers@redbubble.com
+
+    fluentd-plugin-annotation-filter
+
+    Copyright (c) Redbubble
+
+    Permission is hereby granted, free of charge, to any person obtaining
+    a copy of this software and associated documentation files (the
+    "Software"), to deal in the Software without restriction, including
+    without limitation the rights to use, copy, modify, merge, publish,
+    distribute, sublicense, and/or sell copies of the Software, and to
+    permit persons to whom the Software is furnished to do so, subject to
+    the following conditions:
+
+    The above copyright notice and this permission notice shall be included
+    in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+    IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+    CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+    TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+    SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

data/fluentd-plugin-annotation-filter.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluentd-plugin-annotation-filter"
+  gem.version       = ENV["VERSION"]
+  gem.authors       = ["Delivery Engineering"]
+  gem.email         = ["delivery-engineers@redbubble.com"]
+  gem.description   = %q{Fluentd plugin to filter based on Kubernetes annotations}
+  gem.summary       = %q{A filter plugin to drop log entries without the right set of Kubernetes annotations}
+  gem.homepage      = "https://github.com/redbubble/fluentd-plugin-annotation-filter"
+  gem.license       = "MIT"
+
+  gem.files = Dir['lib/**/*'] + %w(Gemfile README.md fluentd-plugin-annotation-filter.gemspec)
+  gem.executables   = gem.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.required_ruby_version = '>= 2.5.0'
+
+  gem.add_runtime_dependency "fluentd", '~> 1.3.3', '>= 1.3.3'
+
+  gem.add_development_dependency "bundler", "~> 1.3"
+  gem.add_development_dependency "rspec", "~> 3.0"     # Like all our other Ruby projects, our tests are in RSpec
+  gem.add_development_dependency "test-unit", "~> 3.3" # TestUnit is, however, used by fluentd's test driver and must be included
+end

data/lib/fluent/plugin/filter-kubernetes-annotation.rb

@@ -0,0 +1,63 @@
+require "json"
+require "fluent/plugin/filter"
+
+module Fluent
+  module Plugin
+    class KubernetesAnnotationFilter < Filter
+      Fluent::Plugin.register_filter("kubernetes_annotation", self)
+
+      attr_reader :containing_annotations
+
+      def initialize
+        super
+      end
+
+      config_param :pass_through_events_without_kubernetes_tags, :bool, default: false
+
+      config_section :contains_container_name, param_name: :contains_sections, multi: true do
+        desc "The name of the Kuberntes annotation that must contain the value in the value field"
+        config_param :annotation, :string
+      end
+
+      def configure(conf)
+        super
+
+        @containing_annotations = @contains_sections.map(&:annotation)
+      end
+
+      def filter(_, _, record)
+        begin
+          if(record.has_key?("kubernetes") && record["kubernetes"].has_key?("annotations"))
+            unless annotations_contain_container_name?(record)
+              return nil
+            end
+          elsif(!@pass_through_events_without_kubernetes_tags)
+            return nil
+          end
+
+        rescue => e
+          log.warn "failed to filter by kubernetes annotation", error: e
+          log.warn_backtrace
+        end
+
+        return record
+      end
+
+      private
+
+      def annotations_contain_container_name?(record)
+        return false if containing_annotations.size == 0
+
+        container_name = record['kubernetes']['container_name']
+
+        containing_annotations.all? { |annotation|
+          content = record["kubernetes"]["annotations"][annotation]
+
+          if(content)
+            JSON.parse(content)&.include?(container_name)
+          end
+        }
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,109 @@
+--- !ruby/object:Gem::Specification
+name: fluentd-plugin-annotation-filter
+version: !ruby/object:Gem::Version
+  version: 1.0.1
+platform: ruby
+authors:
+- Delivery Engineering
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-11-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.3
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.3
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.3
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+description: Fluentd plugin to filter based on Kubernetes annotations
+email:
+- delivery-engineers@redbubble.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- README.md
+- fluentd-plugin-annotation-filter.gemspec
+- lib/fluent/plugin/filter-kubernetes-annotation.rb
+homepage: https://github.com/redbubble/fluentd-plugin-annotation-filter
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.5.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: A filter plugin to drop log entries without the right set of Kubernetes annotations
+test_files: []