fluent-plugin-kubernetes_metadata_filter 2.9.3 → 2.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b6344b1f6ff0602b9d78a5cf317ba5b44e54d0040d6172d102edd002a364d1e7
-  data.tar.gz: 421689d196ae7a0c307ac7e8d4d066755464b76357921e7f6c55ad122ca744b0
+  metadata.gz: 8dcf9979d36d0b08161765e73d3ae244efba378ac3dd0d4d10f582be6abcc488
+  data.tar.gz: 9ea29129e636535052a9183119b61647ae419773e8174ca86adb9a9ebbbdebd8
 SHA512:
-  metadata.gz: 63b03d82fec8888d13f4f752b1ea2519a7c3e594ac45eb0363109d323a35a56174fdd0fa401b787069e5bae84168878ff08baefa899810776b0d710bd55d3c04
-  data.tar.gz: 6368f491a96f30482ded9e12bdc0e80232aebfc26a3a98be75133312c96b1311ff04ce2de5d818a06ba67d2d75274b658471668bfa846db2251202ea92f4b96e
+  metadata.gz: 1dc73fe9ef457fdd9bf424fe935ac99060df59d7c67273dc2c1320ae891cb70c98483c427613a675603557a7bc675d48029d4eefa69eb29b8c2b1f9536cccad2
+  data.tar.gz: fb160f1bc82037cc0ed3ddd7fae884aeab816c8d246256a40d7e5831b5ff64ee93512955709830028180017df4889421c9920f9364d9282029e8ceda5fe000c6

data/.circleci/config.yml

@@ -4,7 +4,8 @@ install: &install
   name: Install bundle
   command: |
             gem install bundler
-            bundle install --path vendor/bundle
+            bundle config set --local path vendor/bundle
+            bundle install
 
 missingdeps: &missingdeps
   name: Install missing dependecies
@@ -20,9 +21,6 @@ test: &test
   command: bundle exec rake test --trace
 
 executors:
-  ruby-2-5:
-    docker:
-      - image: circleci/ruby:2.5.5
   ruby-2-6:
     docker:
       - image: circleci/ruby:2.6.3
@@ -49,8 +47,6 @@ jobs:
 workflows:
   "test_multiple_ruby_versions":
     jobs:
-      - ruby-test:
-          ruby-version: ruby-2-5
       - ruby-test:
           ruby-version: ruby-2-6
       - ruby-test:

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-kubernetes_metadata_filter (2.9.3)
+    fluent-plugin-kubernetes_metadata_filter (2.10.0)
       fluentd (>= 0.14.0, < 1.15)
       kubeclient (>= 4.0.0, < 5.0.0)
       lru_redux
@@ -16,7 +16,7 @@ GEM
     charlock_holmes (0.7.7)
     codeclimate-test-reporter (0.6.0)
       simplecov (>= 0.7.1, < 1.0.0)
-    concurrent-ruby (1.1.9)
+    concurrent-ruby (1.1.10)
     cool.io (1.7.1)
     copyright-header (1.0.22)
       github-linguist
@@ -26,16 +26,16 @@ GEM
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     escape_utils (1.2.1)
-    ffi (1.15.4)
+    ffi (1.15.5)
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
       rake
-    fluentd (1.14.3)
+    fluentd (1.14.6)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.9.0)
       msgpack (>= 1.3.1, < 2.0.0)
-      serverengine (>= 2.2.2, < 3.0.0)
+      serverengine (>= 2.2.5, < 3.0.0)
       sigdump (~> 0.2.2)
       strptime (>= 0.2.4, < 1.0.0)
       tzinfo (>= 1.0, < 3.0)
@@ -62,7 +62,7 @@ GEM
     http_parser.rb (0.8.0)
     jsonpath (1.1.0)
       multi_json
-    kubeclient (4.9.2)
+    kubeclient (4.9.3)
       http (>= 3.0, < 5.0)
       jsonpath (~> 1.0)
       recursive-open-struct (~> 1.1, >= 1.1.1)
@@ -70,10 +70,10 @@ GEM
     lru_redux (1.1.0)
     mime-types (3.4.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2021.1115)
+    mime-types-data (3.2022.0105)
     mini_mime (1.1.2)
     minitest (4.7.5)
-    msgpack (1.4.2)
+    msgpack (1.5.1)
     multi_json (1.15.0)
     netrc (0.11.0)
     parallel (1.21.0)
@@ -105,7 +105,7 @@ GEM
       parser (>= 3.0.1.1)
     ruby-progressbar (1.11.0)
     rugged (1.2.0)
-    serverengine (2.2.4)
+    serverengine (2.2.5)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
     simplecov (0.21.2)
@@ -122,11 +122,11 @@ GEM
       test-unit (>= 2.5.2)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2021.5)
+    tzinfo-data (1.2022.1)
       tzinfo (>= 1.0.0)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.8)
+    unf_ext (0.0.8.1)
     unicode-display_width (2.1.0)
     vcr (6.0.0)
     webmock (3.14.0)
@@ -134,7 +134,7 @@ GEM
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
     webrick (1.7.0)
-    yajl-ruby (1.4.1)
+    yajl-ruby (1.4.2)
 
 PLATFORMS
   ruby
@@ -155,4 +155,4 @@ DEPENDENCIES
   yajl-ruby
 
 BUNDLED WITH
-   2.2.19
+   2.3.4

data/README.md

@@ -17,6 +17,7 @@ that rely on the authenticity of the namespace for proper log isolation.
 
 | fluent-plugin-kubernetes_metadata_filter  | fluentd | ruby |
 |-------------------|---------|------|
+| >= 2.10.0 | >= v1.10.0 | >= 2.6 |
 | >= 2.5.0 | >= v1.10.0 | >= 2.5 |
 | >= 2.0.0 | >= v0.14.20 | >= 2.1 |
 |  < 2.0.0 | >= v0.12.0 | >= 1.9 |
@@ -41,14 +42,14 @@ Configuration options for fluent.conf are:
 * `client_key` - path to a client key file to authenticate to the API server
 * `bearer_token_file` - path to a file containing the bearer token to use for authentication
 * `tag_to_kubernetes_name_regexp` - the regular expression used to extract kubernetes metadata (pod name, container name, namespace) from the current fluentd tag.
-This must used named capture groups for `container_name`, `pod_name` & `namespace` default: See [code](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/blob/master/lib/fluent/plugin/filter_kubernetes_metadata.rb#L52)
+This must use named capture groups for `container_name`, `pod_name`, `namespace`, and either `pod_uuid (/var/log/pods)` or `docker_id (/var/log/containers)`
 * `cache_size` - size of the cache of Kubernetes metadata to reduce requests to the API server (default: `1000`)
 * `cache_ttl` - TTL in seconds of each cached element. Set to negative value to disable TTL eviction (default: `3600` - 1 hour)
 * `watch` - set up a watch on pods on the API server for updates to metadata (default: `true`)
-* `de_dot` - replace dots in labels and annotations with configured `de_dot_separator`, required for Datadog and ElasticSearch 2.x compatibility (default: `true`)
-* `de_dot_separator` - separator to use if `de_dot` is enabled (default: `_`)
-* `de_slash` - replace slashes in labels and annotations with configured `de_slash_separator`, required for Datadog compatibility (default: `false`)
-* `de_slash_separator` - separator to use if `de_slash` is enabled (default: `__`)
+* *DEPRECATED*`de_dot` - replace dots in labels and annotations with configured `de_dot_separator`, required for Datadog and ElasticSearch 2.x compatibility (default: `true`)
+* *DEPRECATED*`de_dot_separator` - separator to use if `de_dot` is enabled (default: `_`)
+* *DEPRECATED*`de_slash` - replace slashes in labels and annotations with configured `de_slash_separator`, required for Datadog compatibility (default: `false`)
+* *DEPRECATED*`de_slash_separator` - separator to use if `de_slash` is enabled (default: `__`)
 * *DEPRECATED* `use_journal` - If false, messages are expected to be formatted and tagged as if read by the fluentd in\_tail plugin with wildcard filename.  If true, messages are expected to be formatted as if read from the systemd journal.  The `MESSAGE` field has the full message.  The `CONTAINER_NAME` field has the encoded k8s metadata (see below).  The `CONTAINER_ID_FULL` field has the full container uuid.  This requires docker to use the `--log-driver=journald` log driver.  If unset (the default), the plugin will use the `CONTAINER_NAME` and `CONTAINER_ID_FULL` fields
 if available, otherwise, will use the tag in the `tag_to_kubernetes_name_regexp` format.
 * `container_name_to_kubernetes_regexp` - The regular expression used to extract the k8s metadata encoded in the journal `CONTAINER_NAME` field default: See [code](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/blob/master/lib/fluent/plugin/filter_kubernetes_metadata.rb#L68)
@@ -68,23 +69,6 @@ when true (default: `true`)
 * `skip_namespace_metadata` - Skip the namespace_id field from the metadata. The fetch_namespace_metadata function will be skipped. The plugin will be faster and cpu consumption will be less.
 * `watch_retry_interval` - The time interval in seconds for retry backoffs when watch connections fail. (default: `10`)
 
-**NOTE:** As of the release 2.1.x of this plugin, it no longer supports parsing the source message into JSON and attaching it to the
-payload.  The following configuration options are removed:
-
-* `merge_json_log`
-* `preserve_json_log`
-
-One way of preserving JSON logs can be through the [parser plugin](https://docs.fluentd.org/filter/parser)
-
-**NOTE** As of release v2.1.4, the use of `use_journal` is **DEPRECATED**.  If this setting is not present, the plugin will
-attempt to figure out the source of the metadata fields from the following:
-- If `lookup_from_k8s_field true` (the default) and the following fields are present in the record:
-`docker.container_id`, `kubernetes.namespace_name`, `kubernetes.pod_name`, `kubernetes.container_name`,
-then the plugin will use those values as the source to use to lookup the metadata
-- If `use_journal true`, or `use_journal` is unset, and the fields `CONTAINER_NAME` and `CONTAINER_ID_FULL` are present in the record,
-then the plugin will parse those values using `container_name_to_kubernetes_regexp` and use those as the source to lookup the metadata
-- Otherwise, if the tag matches `tag_to_kubernetes_name_regexp`, the plugin will parse the tag and use those values to
-lookup the metdata
 
 Reading from the JSON formatted log files with `in_tail` and wildcard filenames while respecting the CRI-o log format with the same config you need the fluent-plugin "multi-format-parser":
 
@@ -154,22 +138,7 @@ Reading from the systemd journal (requires the fluentd `fluent-plugin-systemd` a
   @type stdout
 </match>
 ```
-## Log content as JSON
-In former versions this plugin parsed the value of the key log as JSON. In the current version this feature was removed, to avoid duplicate features in the fluentd plugin ecosystem. It can parsed with the parser plugin like this:
-```
-<filter kubernetes.**>
-  @type parser
-  key_name log
-  <parse>
-    @type json
-    json_parser json
-  </parse>
-  replace_invalid_sequence true
-  reserve_data true # this preserves unparsable log lines
-  emit_invalid_record_to_error false # In case of unparsable log lines keep the error log clean
-  reserve_time # the time was already parsed in the source, we don't want to overwrite it with current time.
-</filter>
-```
+
 
 ## Environment variables for Kubernetes
 

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -5,9 +5,9 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = 'fluent-plugin-kubernetes_metadata_filter'
-  gem.version       = '2.9.3'
-  gem.authors       = ['Jimmi Dyson']
-  gem.email         = ['jimmidyson@gmail.com']
+  gem.version       = '2.10.0'
+  gem.authors       = ['OpenShift Cluster Logging','Jimmi Dyson']
+  gem.email         = ['team-logging@redhat.com','jimmidyson@gmail.com']
   gem.description   = 'Filter plugin to add Kubernetes metadata'
   gem.summary       = 'Fluentd filter plugin to add Kubernetes metadata'
   gem.homepage      = 'https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter'

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -52,8 +52,8 @@ module Fluent::Plugin
     config_param :ca_file, :string, default: nil
     config_param :verify_ssl, :bool, default: true
 
-    REGEX_VAR_LOG_PODS = '(?<prefix>var\.log\.pods)\.(?<namespace>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<pod_uuid>[a-z0-9-]*)\.(?<container_name>.+)\..*\.log$'
-    REGEX_VAR_LOG_CONTAINERS = '(?<prefix>var\.log\.containers)\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
+    REGEX_VAR_LOG_PODS = '(var\.log\.pods)\.(?<namespace>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<pod_uuid>[a-z0-9-]*)\.(?<container_name>.+)\..*\.log$'
+    REGEX_VAR_LOG_CONTAINERS = '(var\.log\.containers)\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
 
     #tag_to_kubernetes_name_regexp which must include named capture groups:
     #  namespace            - The namespace in which the pod is deployed
@@ -305,7 +305,7 @@ module Fluent::Plugin
       end
     end
 
-    def get_metadata_for_record(source, namespace_name, pod_name, container_name, cache_key, create_time, batch_miss_cache)
+    def get_metadata_for_record(namespace_name, pod_name, container_name, cache_key, create_time, batch_miss_cache, docker_id)
       metadata = {
         'docker' => { 'container_id' => "" },
         'kubernetes' => {
@@ -314,7 +314,7 @@ module Fluent::Plugin
           'pod_name' => pod_name
         }
       }
-      metadata['docker']['container_id'] = cache_key unless source == 'var.log.pods'
+      metadata['docker']['container_id'] = docker_id unless docker_id.nil?
       container_cache_key = container_name
       if present?(@kubernetes_url)
         pod_metadata = get_pod_metadata(cache_key, namespace_name, pod_name, create_time, batch_miss_cache)
@@ -344,8 +344,9 @@ module Fluent::Plugin
           else
             tag_match_data['docker_id']
           end 
-          tag_metadata = get_metadata_for_record(tag_match_data['prefix'], tag_match_data['namespace'], tag_match_data['pod_name'], tag_match_data['container_name'],
-                                                 cache_key, create_time_from_record(record, time), batch_miss_cache)
+          docker_id = tag_match_data.names.include?('docker_id') ? tag_match_data['docker_id'] : nil
+          tag_metadata = get_metadata_for_record(tag_match_data['namespace'], tag_match_data['pod_name'], tag_match_data['container_name'],
+                                                 cache_key, create_time_from_record(record, time), batch_miss_cache, docker_id)
         end
         metadata = Marshal.load(Marshal.dump(tag_metadata)) if tag_metadata
         if (@use_journal || @use_journal.nil?) &&
@@ -358,9 +359,9 @@ module Fluent::Plugin
            record['kubernetes'].key?('pod_name') &&
            record['kubernetes'].key?('container_name') &&
            record['docker'].key?('container_id') &&
-           (k_metadata = get_metadata_for_record(tag_match_data['prefix'], record['kubernetes']['namespace_name'], record['kubernetes']['pod_name'],
+           (k_metadata = get_metadata_for_record(record['kubernetes']['namespace_name'], record['kubernetes']['pod_name'],
                                                  record['kubernetes']['container_name'], record['docker']['container_id'],
-                                                 create_time_from_record(record, time), batch_miss_cache))
+                                                 create_time_from_record(record, time), batch_miss_cache, record['docker']['container_id']))
           metadata = k_metadata
         end
         record = record.merge(metadata) if metadata
@@ -374,8 +375,8 @@ module Fluent::Plugin
       metadata = nil
       if record.key?('CONTAINER_NAME') && record.key?('CONTAINER_ID_FULL')
         metadata = record['CONTAINER_NAME'].match(@container_name_to_kubernetes_regexp_compiled) do |match_data|
-          get_metadata_for_record(match_data['name_prefix'], match_data['namespace'], match_data['pod_name'], match_data['container_name'],
-            record['CONTAINER_ID_FULL'], create_time_from_record(record, time), batch_miss_cache)
+          get_metadata_for_record(match_data['namespace'], match_data['pod_name'], match_data['container_name'],
+            record['CONTAINER_ID_FULL'], create_time_from_record(record, time), batch_miss_cache, record['CONTAINER_ID_FULL'])
         end
         unless metadata
           log.debug "Error: could not match CONTAINER_NAME from record #{record}"

data/lib/fluent/plugin/kubernetes_metadata_test_api_adapter.rb

@@ -26,7 +26,7 @@ module KubernetesMetadata
       def api_valid?
         true
       end
-      def get_namespace(namespace_name)
+      def get_namespace(namespace_name, unused, options)
         return {
           metadata: {
             name: namespace_name,
@@ -38,7 +38,7 @@ module KubernetesMetadata
         }
       end
 
-      def get_pod(pod_name, namespace_name)
+      def get_pod(pod_name, namespace_name, options)
         return {
           metadata: {
             name: pod_name,

data/release_notes.md

@@ -0,0 +1,42 @@
+# Release Notes
+
+## 2.9.4
+As of this release, the 'de_dot' functionality is depricated and will be removed in future releases. 
+Ref: https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/issues/320
+
+## v2.1.4
+The use of `use_journal` is **DEPRECATED**.  If this setting is not present, the plugin will
+attempt to figure out the source of the metadata fields from the following:
+- If `lookup_from_k8s_field true` (the default) and the following fields are present in the record:
+`docker.container_id`, `kubernetes.namespace_name`, `kubernetes.pod_name`, `kubernetes.container_name`,
+then the plugin will use those values as the source to use to lookup the metadata
+- If `use_journal true`, or `use_journal` is unset, and the fields `CONTAINER_NAME` and `CONTAINER_ID_FULL` are present in the record,
+then the plugin will parse those values using `container_name_to_kubernetes_regexp` and use those as the source to lookup the metadata
+- Otherwise, if the tag matches `tag_to_kubernetes_name_regexp`, the plugin will parse the tag and use those values to
+lookup the metdata
+
+## v2.1.x
+
+As of the release 2.1.x of this plugin, it no longer supports parsing the source message into JSON and attaching it to the
+payload.  The following configuration options are removed:
+
+* `merge_json_log`
+* `preserve_json_log`
+
+One way of preserving JSON logs can be through the [parser plugin](https://docs.fluentd.org/filter/parser).
+It can parsed with the parser plugin like this:
+
+```
+<filter kubernetes.**>
+  @type parser
+  key_name log
+  <parse>
+    @type json
+    json_parser json
+  </parse>
+  replace_invalid_sequence true
+  reserve_data true # this preserves unparsable log lines
+  emit_invalid_record_to_error false # In case of unparsable log lines keep the error log clean
+  reserve_time # the time was already parsed in the source, we don't want to overwrite it with current time.
+</filter>
+```

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 2.9.3
+  version: 2.10.0
 platform: ruby
 authors:
+- OpenShift Cluster Logging
 - Jimmi Dyson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-06 00:00:00.000000000 Z
+date: 2022-04-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -206,6 +207,7 @@ dependencies:
         version: '0'
 description: Filter plugin to add Kubernetes metadata
 email:
+- team-logging@redhat.com
 - jimmidyson@gmail.com
 executables: []
 extensions: []
@@ -228,6 +230,7 @@ files:
 - lib/fluent/plugin/kubernetes_metadata_util.rb
 - lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb
 - lib/fluent/plugin/kubernetes_metadata_watch_pods.rb
+- release_notes.md
 - test/cassettes/invalid_api_server_config.yml
 - test/cassettes/kubernetes_docker_metadata_annotations.yml
 - test/cassettes/kubernetes_docker_metadata_dotted_slashed_labels.yml