RubyGems - fluent-plugin-kubernetes_metadata_filter - Versions diffs - 2.8.1 → 2.9.3 - Mend

fluent-plugin-kubernetes_metadata_filter 2.8.1 → 2.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (12) hide show

checksums.yaml +4 -4
data/Gemfile.lock +27 -27
data/README.md +4 -2
data/fluent-plugin-kubernetes_metadata_filter.gemspec +1 -1
data/lib/fluent/plugin/filter_kubernetes_metadata.rb +54 -27
data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb +1 -3
data/lib/fluent/plugin/kubernetes_metadata_common.rb +14 -5
data/test/cassettes/{kubernetes_docker_metadata_dotted_labels.yml → kubernetes_docker_metadata_dotted_slashed_labels.yml} +0 -0
data/test/cassettes/kubernetes_get_pod_container_init.yml +145 -0
data/test/plugin/test_cache_strategy.rb +1 -1
data/test/plugin/test_filter_kubernetes_metadata.rb +108 -24
metadata +4 -3

checksums.yaml CHANGED Viewed

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 71fd60517a92c89e8c35bc91a3147487ff0637ae41378ecf82b047424cdb203f
-  data.tar.gz: 310b9a7bbb7e5bd59d1aa3deedb699114f561295b4b3a97fdf54fadcf6d7ee08
+  metadata.gz: b6344b1f6ff0602b9d78a5cf317ba5b44e54d0040d6172d102edd002a364d1e7
+  data.tar.gz: 421689d196ae7a0c307ac7e8d4d066755464b76357921e7f6c55ad122ca744b0
 SHA512:
-  metadata.gz: 76ccb8d8b2a2d26ca1e2823a93c38fef974e76526f402f11a10c8ea4697cbda765370eb202177490e9a4aadb10fef4f9980ba4e38cb39953c55ae3bf49ac3bf4
-  data.tar.gz: 8ea9696e6df02bf264a7f0e68ab36732adae9a007379e6bbd841633b6ad6dd830d9241a30bdcd0744183151898fdcc71711be14656973a47d8afa7a1f273abb1
+  metadata.gz: 63b03d82fec8888d13f4f752b1ea2519a7c3e594ac45eb0363109d323a35a56174fdd0fa401b787069e5bae84168878ff08baefa899810776b0d710bd55d3c04
+  data.tar.gz: 6368f491a96f30482ded9e12bdc0e80232aebfc26a3a98be75133312c96b1311ff04ce2de5d818a06ba67d2d75274b658471668bfa846db2251202ea92f4b96e

data/Gemfile.lock CHANGED Viewed

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-kubernetes_metadata_filter (2.8.1)
+    fluent-plugin-kubernetes_metadata_filter (2.9.3)
       fluentd (>= 0.14.0, < 1.15)
       kubeclient (>= 4.0.0, < 5.0.0)
       lru_redux
@@ -11,7 +11,7 @@ GEM
   specs:
     addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
-    ast (2.4.1)
+    ast (2.4.2)
     bump (0.10.0)
     charlock_holmes (0.7.7)
     codeclimate-test-reporter (0.6.0)
@@ -22,7 +22,7 @@ GEM
       github-linguist
     crack (0.4.5)
       rexml
-    docile (1.3.5)
+    docile (1.4.0)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     escape_utils (1.2.1)
@@ -30,19 +30,19 @@ GEM
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
       rake
-    fluentd (1.14.0)
+    fluentd (1.14.3)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)
-      http_parser.rb (>= 0.5.1, < 0.8.0)
+      http_parser.rb (>= 0.5.1, < 0.9.0)
       msgpack (>= 1.3.1, < 2.0.0)
       serverengine (>= 2.2.2, < 3.0.0)
       sigdump (~> 0.2.2)
-      strptime (>= 0.2.2, < 1.0.0)
+      strptime (>= 0.2.4, < 1.0.0)
       tzinfo (>= 1.0, < 3.0)
       tzinfo-data (~> 1.0)
       webrick (>= 1.4.2, < 1.8.0)
       yajl-ruby (~> 1.0)
-    github-linguist (7.12.2)
+    github-linguist (7.17.0)
       charlock_holmes (~> 0.7.7)
       escape_utils (~> 1.2.0)
       mini_mime (~> 1.0)
@@ -59,7 +59,7 @@ GEM
     http-form_data (2.3.0)
     http-parser (1.2.3)
       ffi-compiler (>= 1.0, < 2.0)
-    http_parser.rb (0.7.0)
+    http_parser.rb (0.8.0)
     jsonpath (1.1.0)
       multi_json
     kubeclient (4.9.2)
@@ -68,43 +68,43 @@ GEM
       recursive-open-struct (~> 1.1, >= 1.1.1)
       rest-client (~> 2.0)
     lru_redux (1.1.0)
-    mime-types (3.3.1)
+    mime-types (3.4.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2021.0901)
-    mini_mime (1.0.2)
+    mime-types-data (3.2021.1115)
+    mini_mime (1.1.2)
     minitest (4.7.5)
     msgpack (1.4.2)
     multi_json (1.15.0)
     netrc (0.11.0)
-    parallel (1.20.1)
-    parser (3.0.0.0)
+    parallel (1.21.0)
+    parser (3.0.2.0)
       ast (~> 2.4.1)
-    power_assert (1.2.0)
+    power_assert (2.0.1)
     public_suffix (4.0.6)
     rainbow (3.0.0)
-    rake (13.0.3)
+    rake (13.0.6)
     recursive-open-struct (1.1.3)
-    regexp_parser (2.0.3)
+    regexp_parser (2.1.1)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
     rexml (3.2.5)
-    rr (1.2.1)
-    rubocop (1.8.1)
+    rr (3.0.8)
+    rubocop (1.22.3)
       parallel (~> 1.10)
       parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 1.2.0, < 2.0)
+      rubocop-ast (>= 1.12.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.4.0)
-      parser (>= 2.7.1.5)
+    rubocop-ast (1.12.0)
+      parser (>= 3.0.1.1)
     ruby-progressbar (1.11.0)
-    rugged (1.1.0)
+    rugged (1.2.0)
     serverengine (2.2.4)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
@@ -113,7 +113,7 @@ GEM
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.2)
+    simplecov_json_formatter (0.1.3)
     strptime (0.2.5)
     test-unit (3.0.9)
       power_assert
@@ -122,15 +122,15 @@ GEM
       test-unit (>= 2.5.2)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2021.1)
+    tzinfo-data (1.2021.5)
       tzinfo (>= 1.0.0)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.8)
-    unicode-display_width (2.0.0)
+    unicode-display_width (2.1.0)
     vcr (6.0.0)
-    webmock (3.11.1)
-      addressable (>= 2.3.6)
+    webmock (3.14.0)
+      addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
     webrick (1.7.0)

data/README.md CHANGED Viewed

@@ -45,8 +45,10 @@ This must used named capture groups for `container_name`, `pod_name` & `namespac
 * `cache_size` - size of the cache of Kubernetes metadata to reduce requests to the API server (default: `1000`)
 * `cache_ttl` - TTL in seconds of each cached element. Set to negative value to disable TTL eviction (default: `3600` - 1 hour)
 * `watch` - set up a watch on pods on the API server for updates to metadata (default: `true`)
-* `de_dot` - replace dots in labels and annotations with configured `de_dot_separator`, required for ElasticSearch 2.x compatibility (default: `true`)
+* `de_dot` - replace dots in labels and annotations with configured `de_dot_separator`, required for Datadog and ElasticSearch 2.x compatibility (default: `true`)
 * `de_dot_separator` - separator to use if `de_dot` is enabled (default: `_`)
+* `de_slash` - replace slashes in labels and annotations with configured `de_slash_separator`, required for Datadog compatibility (default: `false`)
+* `de_slash_separator` - separator to use if `de_slash` is enabled (default: `__`)
 * *DEPRECATED* `use_journal` - If false, messages are expected to be formatted and tagged as if read by the fluentd in\_tail plugin with wildcard filename.  If true, messages are expected to be formatted as if read from the systemd journal.  The `MESSAGE` field has the full message.  The `CONTAINER_NAME` field has the encoded k8s metadata (see below).  The `CONTAINER_ID_FULL` field has the full container uuid.  This requires docker to use the `--log-driver=journald` log driver.  If unset (the default), the plugin will use the `CONTAINER_NAME` and `CONTAINER_ID_FULL` fields
 if available, otherwise, will use the tag in the `tag_to_kubernetes_name_regexp` format.
 * `container_name_to_kubernetes_regexp` - The regular expression used to extract the k8s metadata encoded in the journal `CONTAINER_NAME` field default: See [code](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/blob/master/lib/fluent/plugin/filter_kubernetes_metadata.rb#L68)
@@ -74,7 +76,7 @@ payload.  The following configuration options are removed:
 One way of preserving JSON logs can be through the [parser plugin](https://docs.fluentd.org/filter/parser)
-**NOTE** As of this release, the use of `use_journal` is **DEPRECATED**.  If this setting is not present, the plugin will
+**NOTE** As of release v2.1.4, the use of `use_journal` is **DEPRECATED**.  If this setting is not present, the plugin will
 attempt to figure out the source of the metadata fields from the following:
 - If `lookup_from_k8s_field true` (the default) and the following fields are present in the record:
 `docker.container_id`, `kubernetes.namespace_name`, `kubernetes.pod_name`, `kubernetes.container_name`,

data/fluent-plugin-kubernetes_metadata_filter.gemspec CHANGED Viewed

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 Gem::Specification.new do |gem|
   gem.name          = 'fluent-plugin-kubernetes_metadata_filter'
-  gem.version       = '2.8.1'
+  gem.version       = '2.9.3'
   gem.authors       = ['Jimmi Dyson']
   gem.email         = ['jimmidyson@gmail.com']
   gem.description   = 'Filter plugin to add Kubernetes metadata'

data/lib/fluent/plugin/filter_kubernetes_metadata.rb CHANGED Viewed

@@ -51,13 +51,23 @@ module Fluent::Plugin
     config_param :client_key, :string, default: nil
     config_param :ca_file, :string, default: nil
     config_param :verify_ssl, :bool, default: true
-    config_param :tag_to_kubernetes_name_regexp,
-                 :string,
-                 default: 'var\.log\.containers\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
+    REGEX_VAR_LOG_PODS = '(?<prefix>var\.log\.pods)\.(?<namespace>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<pod_uuid>[a-z0-9-]*)\.(?<container_name>.+)\..*\.log$'
+    REGEX_VAR_LOG_CONTAINERS = '(?<prefix>var\.log\.containers)\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
+    #tag_to_kubernetes_name_regexp which must include named capture groups:
+    #  namespace            - The namespace in which the pod is deployed
+    #  pod_name             - The pod name
+    #  container_name       - The name of the container
+    #  pod_uuid (/var/log/pods) | docker_id (/var/log/containers) - Unique identifier used in caching of either pod_uuid or the container hash
+    config_param :tag_to_kubernetes_name_regexp, :string, default: "(#{REGEX_VAR_LOG_PODS}|#{REGEX_VAR_LOG_CONTAINERS})"
     config_param :bearer_token_file, :string, default: nil
     config_param :secret_dir, :string, default: '/var/run/secrets/kubernetes.io/serviceaccount'
     config_param :de_dot, :bool, default: true
     config_param :de_dot_separator, :string, default: '_'
+    config_param :de_slash, :bool, default: false
+    config_param :de_slash_separator, :string, default: '__'
     # if reading from the journal, the record will contain the following fields in the following
     # format:
     # CONTAINER_NAME=k8s_$containername.$containerhash_$podname_$namespacename_$poduuid_$rand32bitashex
@@ -98,15 +108,15 @@ module Fluent::Plugin
     config_param :watch_retry_max_times, :integer, default: 10
     def fetch_pod_metadata(namespace_name, pod_name)
-      log.trace("fetching pod metadata: #{namespace_name}/#{pod_name}") if log.trace?
+      log.trace("fetching pod metadata: #{namespace_name}/#{pod_name}")
       options = {
         resource_version: '0' # Fetch from API server cache instead of etcd quorum read
       }
       pod_object = @client.get_pod(pod_name, namespace_name, options)
-      log.trace("raw metadata for #{namespace_name}/#{pod_name}: #{pod_object}") if log.trace?
+      log.trace("raw metadata for #{namespace_name}/#{pod_name}: #{pod_object}")
       metadata = parse_pod_metadata(pod_object)
       @stats.bump(:pod_cache_api_updates)
-      log.trace("parsed metadata for #{namespace_name}/#{pod_name}: #{metadata}") if log.trace?
+      log.trace("parsed metadata for #{namespace_name}/#{pod_name}: #{metadata}")
       @cache[metadata['pod_id']] = metadata
     rescue StandardError => e
       @stats.bump(:pod_cache_api_nil_error)
@@ -130,15 +140,15 @@ module Fluent::Plugin
     end
     def fetch_namespace_metadata(namespace_name)
-      log.trace("fetching namespace metadata: #{namespace_name}") if log.trace?
+      log.trace("fetching namespace metadata: #{namespace_name}")
       options = {
         resource_version: '0' # Fetch from API server cache instead of etcd quorum read
       }
       namespace_object = @client.get_namespace(namespace_name, nil, options)
-      log.trace("raw metadata for #{namespace_name}: #{namespace_object}") if log.trace?
+      log.trace("raw metadata for #{namespace_name}: #{namespace_object}")
       metadata = parse_namespace_metadata(namespace_object)
       @stats.bump(:namespace_cache_api_updates)
-      log.trace("parsed metadata for #{namespace_name}: #{metadata}") if log.trace?
+      log.trace("parsed metadata for #{namespace_name}: #{metadata}")
       @namespace_cache[metadata['namespace_id']] = metadata
     rescue StandardError => e
       @stats.bump(:namespace_cache_api_nil_error)
@@ -154,10 +164,6 @@ module Fluent::Plugin
     def configure(conf)
       super
-      def log.trace?
-        level == Fluent::Log::LEVEL_TRACE
-      end
       require 'kubeclient'
       require 'lru_redux'
       @stats = KubernetesMetadata::Stats.new
@@ -166,6 +172,10 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "Invalid de_dot_separator: cannot be or contain '.'"
       end
+      if @de_slash && @de_slash_separator.include?('/')
+        raise Fluent::ConfigError, "Invalid de_slash_separator: cannot be or contain '/'"
+      end
       if @cache_ttl < 0
         log.info 'Setting the cache TTL to :none because it was <= 0'
         @cache_ttl = :none
@@ -181,6 +191,7 @@ module Fluent::Plugin
       @namespace_cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl)
       @tag_to_kubernetes_name_regexp_compiled = Regexp.compile(@tag_to_kubernetes_name_regexp)
       @container_name_to_kubernetes_regexp_compiled = Regexp.compile(@container_name_to_kubernetes_regexp)
       # Use Kubernetes default service account if we're in a pod.
@@ -294,40 +305,47 @@ module Fluent::Plugin
       end
     end
-    def get_metadata_for_record(namespace_name, pod_name, container_name, container_id, create_time, batch_miss_cache)
+    def get_metadata_for_record(source, namespace_name, pod_name, container_name, cache_key, create_time, batch_miss_cache)
       metadata = {
-        'docker' => { 'container_id' => container_id },
+        'docker' => { 'container_id' => "" },
         'kubernetes' => {
           'container_name' => container_name,
           'namespace_name' => namespace_name,
           'pod_name' => pod_name
         }
       }
+      metadata['docker']['container_id'] = cache_key unless source == 'var.log.pods'
+      container_cache_key = container_name
       if present?(@kubernetes_url)
-        pod_metadata = get_pod_metadata(container_id, namespace_name, pod_name, create_time, batch_miss_cache)
-        if (pod_metadata.include? 'containers') && (pod_metadata['containers'].include? container_id) && !@skip_container_metadata
-          metadata['kubernetes']['container_image'] = pod_metadata['containers'][container_id]['image']
-          metadata['kubernetes']['container_image_id'] = pod_metadata['containers'][container_id]['image_id']
+        pod_metadata = get_pod_metadata(cache_key, namespace_name, pod_name, create_time, batch_miss_cache)
+        if (pod_metadata.include? 'containers') && (pod_metadata['containers'].include? container_cache_key) && !@skip_container_metadata
+          metadata['kubernetes']['container_image'] = pod_metadata['containers'][container_cache_key]['image']
+          metadata['kubernetes']['container_image_id'] = pod_metadata['containers'][container_cache_key]['image_id'] unless pod_metadata['containers'][container_cache_key]['image_id'].empty?
+          metadata['docker']['container_id'] = pod_metadata['containers'][container_cache_key]['containerID'] unless pod_metadata['containers'][container_cache_key]['containerID'].empty?
         end
         metadata['kubernetes'].merge!(pod_metadata) if pod_metadata
         metadata['kubernetes'].delete('containers')
       end
+      metadata.delete('docker') if metadata['docker'] && (metadata['docker']['container_id'].nil? || metadata['docker']['container_id'].empty?)
       metadata
     end
     def filter_stream(tag, es)
       return es if (es.respond_to?(:empty?) && es.empty?) || !es.is_a?(Fluent::EventStream)
       new_es = Fluent::MultiEventStream.new
       tag_match_data = tag.match(@tag_to_kubernetes_name_regexp_compiled) unless @use_journal
       tag_metadata = nil
       batch_miss_cache = {}
       es.each do |time, record|
         if tag_match_data && tag_metadata.nil?
-          tag_metadata = get_metadata_for_record(tag_match_data['namespace'], tag_match_data['pod_name'], tag_match_data['container_name'],
-                                                 tag_match_data['docker_id'], create_time_from_record(record, time), batch_miss_cache)
+          cache_key =  if tag_match_data.names.include?('pod_uuid') && !tag_match_data['pod_uuid'].nil?
+            tag_match_data['pod_uuid']
+          else
+            tag_match_data['docker_id']
+          end
+          tag_metadata = get_metadata_for_record(tag_match_data['prefix'], tag_match_data['namespace'], tag_match_data['pod_name'], tag_match_data['container_name'],
+                                                 cache_key, create_time_from_record(record, time), batch_miss_cache)
         end
         metadata = Marshal.load(Marshal.dump(tag_metadata)) if tag_metadata
         if (@use_journal || @use_journal.nil?) &&
@@ -340,12 +358,11 @@ module Fluent::Plugin
            record['kubernetes'].key?('pod_name') &&
            record['kubernetes'].key?('container_name') &&
            record['docker'].key?('container_id') &&
-           (k_metadata = get_metadata_for_record(record['kubernetes']['namespace_name'], record['kubernetes']['pod_name'],
+           (k_metadata = get_metadata_for_record(tag_match_data['prefix'], record['kubernetes']['namespace_name'], record['kubernetes']['pod_name'],
                                                  record['kubernetes']['container_name'], record['docker']['container_id'],
                                                  create_time_from_record(record, time), batch_miss_cache))
           metadata = k_metadata
         end
         record = record.merge(metadata) if metadata
         new_es.add(time, record)
       end
@@ -357,8 +374,8 @@ module Fluent::Plugin
       metadata = nil
       if record.key?('CONTAINER_NAME') && record.key?('CONTAINER_ID_FULL')
         metadata = record['CONTAINER_NAME'].match(@container_name_to_kubernetes_regexp_compiled) do |match_data|
-          get_metadata_for_record(match_data['namespace'], match_data['pod_name'], match_data['container_name'],
-                                  record['CONTAINER_ID_FULL'], create_time_from_record(record, time), batch_miss_cache)
+          get_metadata_for_record(match_data['name_prefix'], match_data['namespace'], match_data['pod_name'], match_data['container_name'],
+            record['CONTAINER_ID_FULL'], create_time_from_record(record, time), batch_miss_cache)
         end
         unless metadata
           log.debug "Error: could not match CONTAINER_NAME from record #{record}"
@@ -381,6 +398,16 @@ module Fluent::Plugin
       end
     end
+    def de_slash!(h)
+      h.keys.each do |ref|
+        next unless h[ref] && ref =~ /\//
+        v = h.delete(ref)
+        newref = ref.to_s.gsub('/', @de_slash_separator)
+        h[newref] = v
+      end
+    end
     # copied from activesupport
     def present?(object)
       object.respond_to?(:empty?) ? !object.empty? : !!object

data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb CHANGED Viewed

@@ -24,8 +24,6 @@ module KubernetesMetadata
       metadata = {}
       ids = @id_cache[key]
       if ids.nil?
-        # FAST PATH
-        # Cache hit, fetch metadata from the cache
         @stats.bump(:id_cache_miss)
         return batch_miss_cache["#{namespace_name}_#{pod_name}"] if batch_miss_cache.key?("#{namespace_name}_#{pod_name}")
@@ -65,7 +63,7 @@ module KubernetesMetadata
               @stats.bump(:id_cache_orphaned_record)
             end
             if @allow_orphans
-              log.trace("orphaning message for: #{namespace_name}/#{pod_name} ") if log.trace?
+              log.trace("orphaning message for: #{namespace_name}/#{pod_name} ")
               metadata = {
                 'orphaned_namespace' => namespace_name,
                 'namespace_name' => @orphaned_namespace_name,

data/lib/fluent/plugin/kubernetes_metadata_common.rb CHANGED Viewed

@@ -47,6 +47,10 @@ module KubernetesMetadata
         de_dot!(labels) unless @skip_labels
         de_dot!(annotations)
       end
+      if @de_slash
+        de_slash!(labels) unless @skip_labels
+        de_slash!(annotations)
+      end
       kubernetes_metadata = {
         'namespace_id' => namespace_object[:metadata][:uid],
         'creation_timestamp' => namespace_object[:metadata][:creationTimestamp]
@@ -65,14 +69,18 @@ module KubernetesMetadata
         de_dot!(labels) unless @skip_labels
         de_dot!(annotations)
       end
+      if @de_slash
+        de_slash!(labels) unless @skip_labels
+        de_slash!(annotations)
+      end
       # collect container information
       container_meta = {}
       begin
         pod_object[:status][:containerStatuses].each do |container_status|
-          # get plain container id (eg. docker://hash -> hash)
-          container_id = container_status[:containerID].sub(%r{^[-_a-zA-Z0-9]+://}, '')
-          container_meta[container_id] = if @skip_container_metadata
+          container_id = (container_status[:containerID]||"").sub(%r{^[-_a-zA-Z0-9]+://}, '')
+          key = container_status[:name]
+          container_meta[key] = if @skip_container_metadata
                                            {
                                              'name' => container_status[:name]
                                            }
@@ -80,10 +88,11 @@ module KubernetesMetadata
                                            {
                                              'name' => container_status[:name],
                                              'image' => container_status[:image],
-                                             'image_id' => container_status[:imageID]
+                                             'image_id' => container_status[:imageID],
+                                             :containerID => container_id
                                            }
                                          end
-        end
+        end if pod_object[:status] && pod_object[:status][:containerStatuses]
       rescue StandardError=>e
         log.warn("parsing container meta information failed for: #{pod_object[:metadata][:namespace]}/#{pod_object[:metadata][:name]}: #{e}")
       end

data/test/cassettes/{kubernetes_docker_metadata_dotted_labels.yml → kubernetes_docker_metadata_dotted_slashed_labels.yml} RENAMED Viewed

File without changes

data/test/cassettes/kubernetes_get_pod_container_init.yml ADDED Viewed

@@ -0,0 +1,145 @@
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Fri, 08 May 2015 10:35:37 GMT
+      Transfer-Encoding:
+      - chunked
+    body:
+      encoding: UTF-8
+      string: |-
+        {
+          "kind": "Pod",
+          "apiVersion": "v1",
+          "metadata": {
+            "name": "fabric8-console-controller-98rqc",
+            "generateName": "fabric8-console-controller-",
+            "namespace": "default",
+            "selfLink": "/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc",
+            "uid": "c76927af-f563-11e4-b32d-54ee7527188d",
+            "resourceVersion": "122",
+            "creationTimestamp": "2015-05-08T09:22:42Z",
+            "labels": {
+              "component": "fabric8Console"
+            }
+          },
+          "spec": {
+            "volumes": [
+              {
+                "name": "openshift-cert-secrets",
+                "hostPath": null,
+                "emptyDir": null,
+                "gcePersistentDisk": null,
+                "gitRepo": null,
+                "secret": {
+                  "secretName": "openshift-cert-secrets"
+                },
+                "nfs": null,
+                "iscsi": null,
+                "glusterfs": null
+              }
+            ],
+            "containers": [
+              {
+                "name": "fabric8-console-container",
+                "image": "fabric8/hawtio-kubernetes:latest",
+                "ports": [
+                  {
+                    "containerPort": 9090,
+                    "protocol": "TCP"
+                  }
+                ],
+                "env": [
+                  {
+                    "name": "OAUTH_CLIENT_ID",
+                    "value": "fabric8-console"
+                  },
+                  {
+                    "name": "OAUTH_AUTHORIZE_URI",
+                    "value": "https://localhost:8443/oauth/authorize"
+                  }
+                ],
+                "resources": {},
+                "volumeMounts": [
+                  {
+                    "name": "openshift-cert-secrets",
+                    "readOnly": true,
+                    "mountPath": "/etc/secret-volume"
+                  }
+                ],
+                "terminationMessagePath": "/dev/termination-log",
+                "imagePullPolicy": "IfNotPresent",
+                "capabilities": {}
+              }
+            ],
+            "restartPolicy": "Always",
+            "dnsPolicy": "ClusterFirst",
+            "nodeName": "jimmi-redhat.localnet"
+          },
+          "status": {
+            "phase": "Running",
+            "Condition": [
+              {
+                "type": "Ready",
+                "status": "True"
+              }
+            ],
+            "hostIP": "172.17.42.1",
+            "podIP": "172.17.0.8",
+            "containerStatuses": [
+              {
+                "name": "fabric8-console-container",
+                "state": {
+                  "waiting": {
+                    "reason": "ContainerCreating"
+                  }
+                },
+                "lastState": {},
+                "ready": true,
+                "restartCount": 0,
+                "image": "fabric8/hawtio-kubernetes:latest",
+                "imageID": ""
+              }
+            ]
+          }
+        }
+    http_version:
+  recorded_at: Fri, 08 May 2015 10:35:37 GMT
+recorded_with: VCR 2.9.3

data/test/plugin/test_cache_strategy.rb CHANGED Viewed

@@ -44,7 +44,7 @@ class TestCacheStrategy
   def log
     logger = {}
-    def logger.trace?
+    def logger.on_trace
       true
     end

data/test/plugin/test_filter_kubernetes_metadata.rb CHANGED Viewed

@@ -28,7 +28,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
     @time = Fluent::Engine.now
   end
-  DEFAULT_TAG = 'var.log.containers.fabric8-console-controller-98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log'
+  VAR_LOG_CONTAINER_TAG = 'var.log.containers.fabric8-console-controller-98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log'
+  VAR_LOG_POD_TAG = 'var.log.pods.default_fabric8-console-controller-98rqc_c76927af-f563-11e4-b32d-54ee7527188d.fabric8-console-container.0.log'
   def create_driver(conf = '')
     Test::Driver::Filter.new(Plugin::KubernetesMetadataFilter).configure(conf)
@@ -45,7 +46,6 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       assert_equal('KubernetesMetadata::TestApiAdapter', d.instance.test_api_adapter)
     end
     test 'kubernetes url' do
       VCR.use_cassette('valid_kubernetes_api_server') do
         d = create_driver('
@@ -138,7 +138,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
           cache_size 1
         ', d: nil)
       d = create_driver(config) if d.nil?
-      d.run(default_tag: DEFAULT_TAG) do
+      d.run(default_tag: VAR_LOG_CONTAINER_TAG) do
         d.feed(@time, msg)
       end
       d.filtered.map(&:last)
@@ -165,6 +165,76 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       plugin.filter_stream('tag', Fluent::MultiEventStream.new)
     end
+    sub_test_case 'parsing_pod_metadata when container_status is missing from the pod status' do
+      test 'using the tag_to_kubernetes_name_regexp for /var/log/containers ' do
+        VCR.use_cassettes(
+          [
+            { name: 'valid_kubernetes_api_server' },
+            { name: 'kubernetes_get_api_v1' },
+            { name: 'kubernetes_get_namespace_default' },
+            { name: 'kubernetes_get_pod_container_init' }
+          ]) do
+          filtered = emit({}, "
+            kubernetes_url https://localhost:8443
+            watch false
+            cache_size 1
+          ")
+          expected_kube_metadata = {
+            'docker' => {
+              'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+            },
+            'kubernetes' => {
+              'container_image'=>'fabric8/hawtio-kubernetes:latest',
+              'container_name'=>'fabric8-console-container',
+              'host' => 'jimmi-redhat.localnet',
+              'pod_name' => 'fabric8-console-controller-98rqc',
+              'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
+              'namespace_name' => 'default',
+              'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
+              'pod_ip' => '172.17.0.8',
+              'master_url' => 'https://localhost:8443',
+              'labels' => {
+                'component' => 'fabric8Console'
+              }
+            }
+          }
+          assert_equal(expected_kube_metadata, filtered[0])
+        end
+      end
+      test 'using the tag_to_kubernetes_name_regexp for /var/log/pods' do
+        VCR.use_cassettes(
+          [
+            { name: 'valid_kubernetes_api_server' },
+            { name: 'kubernetes_get_api_v1' },
+            { name: 'kubernetes_get_namespace_default' },
+            { name: 'kubernetes_get_pod_container_init' }
+          ]) do
+          filtered = emit_with_tag(VAR_LOG_POD_TAG,{}, "
+            kubernetes_url https://localhost:8443
+            watch false
+            cache_size 1
+          ")
+          expected_kube_metadata = {
+            'kubernetes' => {
+              'container_image'=>'fabric8/hawtio-kubernetes:latest',
+              'container_name'=>'fabric8-console-container',
+              'host' => 'jimmi-redhat.localnet',
+              'pod_name' => 'fabric8-console-controller-98rqc',
+              'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
+              'namespace_name' => 'default',
+              'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
+              'pod_ip' => '172.17.0.8',
+              'master_url' => 'https://localhost:8443',
+              'labels' => {
+                'component' => 'fabric8Console'
+              }
+            }
+          }
+          assert_equal(expected_kube_metadata, filtered[0])
+        end
+      end
+    end
     test 'inability to connect to the api server handles exception and doensnt block pipeline' do
       VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' }]) do
         driver = create_driver('
@@ -177,8 +247,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
         filtered = emit({ 'time' => '2015-05-08T09:22:01Z' }, '', d: driver)
         expected_kube_metadata = {
           'time' => '2015-05-08T09:22:01Z',
-          'docker' => {
-            'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+          'docker'=>{
+            'container_id'=>'49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
           },
           'kubernetes' => {
             'pod_name' => 'fabric8-console-controller-98rqc',
@@ -265,6 +335,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
     end
     test 'with docker & kubernetes metadata' do
       VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' }, { name: 'kubernetes_get_pod' }, { name: 'kubernetes_get_namespace_default' }]) do
         filtered = emit({ 'time' => '2015-05-08T09:22:01Z' })
         expected_kube_metadata = {
@@ -361,8 +432,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
     test 'with docker & kubernetes metadata but no configured api server' do
       filtered = emit({}, '')
       expected_kube_metadata = {
-        'docker' => {
-          'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+        'docker'=>{
+          'container_id'=>'49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
         },
         'kubernetes' => {
           'pod_name' => 'fabric8-console-controller-98rqc',
@@ -383,8 +454,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       stub_request(:any, 'https://localhost:8443/api/v1/namespaces/default').to_timeout
       filtered = emit
       expected_kube_metadata = {
-        'docker' => {
-          'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+        'docker'=>{
+          'container_id'=>'49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
         },
         'kubernetes' => {
           'pod_name' => 'fabric8-console-controller-98rqc',
@@ -406,8 +477,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       stub_request(:any, 'https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller.98rqc').to_timeout
       filtered = emit_with_tag('var.log.containers.fabric8-console-controller.98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log', {}, '')
       expected_kube_metadata = {
-        'docker' => {
-          'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+        'docker'=>{
+          'container_id'=>'49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
         },
         'kubernetes' => {
           'pod_name' => 'fabric8-console-controller.98rqc',
@@ -432,13 +503,15 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       assert_equal(msg, filtered[0])
     end
-    test 'with kubernetes dotted labels, de_dot enabled' do
+    test 'with kubernetes dotted and slashed labels, de_dot and de_slash enabled' do
       VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' },
-                         { name: 'kubernetes_docker_metadata_dotted_labels' }]) do
+                         { name: 'kubernetes_docker_metadata_dotted_slashed_labels' }]) do
         filtered = emit({}, '
           kubernetes_url https://localhost:8443
           watch false
           cache_size 1
+          de_dot true
+          de_slash true
         ')
         expected_kube_metadata = {
           'docker' => {
@@ -452,14 +525,14 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
             'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
             'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
             'namespace_labels' => {
-              'kubernetes_io/namespacetest' => 'somevalue'
+              'kubernetes_io__namespacetest' => 'somevalue'
             },
             'namespace_name' => 'default',
             'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
             'pod_ip' => '172.17.0.8',
             'master_url' => 'https://localhost:8443',
             'labels' => {
-              'kubernetes_io/test' => 'somevalue'
+              'kubernetes_io__test' => 'somevalue'
             }
           }
         }
@@ -467,14 +540,15 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       end
     end
-    test 'with kubernetes dotted labels, de_dot disabled' do
+    test 'with kubernetes dotted and slashed labels, de_dot and de_slash disabled' do
       VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' },
-                         { name: 'kubernetes_docker_metadata_dotted_labels' }]) do
+                         { name: 'kubernetes_docker_metadata_dotted_slashed_labels' }]) do
         filtered = emit({}, '
           kubernetes_url https://localhost:8443
           watch false
           cache_size 1
           de_dot false
+          de_slash false
         ')
         expected_kube_metadata = {
           'docker' => {
@@ -506,11 +580,21 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
     test 'invalid de_dot_separator config' do
       assert_raise Fluent::ConfigError do
         create_driver('
+          de_dot true
           de_dot_separator contains.
         ')
       end
     end
+    test 'invalid de_slash_separator config' do
+      assert_raise Fluent::ConfigError do
+        create_driver('
+          de_slash true
+          de_slash_separator contains/
+        ')
+      end
+    end
     test 'with records from journald and docker & kubernetes metadata' do
       # with use_journal true should ignore tags and use CONTAINER_NAME and CONTAINER_ID_FULL
       tag = 'var.log.containers.junk1_junk2_junk3-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed450.log'
@@ -686,7 +770,6 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
     test 'uses metadata from tag if use_journal false and lookup_from_k8s_field false' do
       # with use_journal unset, should still use the journal fields instead of tag fields
-      tag = 'var.log.containers.fabric8-console-controller-98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log'
       msg = {
         'CONTAINER_NAME' => 'k8s_journald-container-name.db89db89_journald-pod-name_journald-namespace-name_c76927af-f563-11e4-b32d-54ee7527188d_89db89db',
         'CONTAINER_ID_FULL' => '838350c64bacba968d39a30a50789b2795291fceca6ccbff55298671d46b0e3b',
@@ -701,13 +784,13 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' }, { name: 'kubernetes_get_pod' },
                          { name: 'kubernetes_get_namespace_default', options: { allow_playback_repeats: true } },
                          { name: 'metadata_from_tag_and_journald_fields' }]) do
-        es = emit_with_tag(tag, msg, '
+        es = emit_with_tag(VAR_LOG_CONTAINER_TAG, msg, "
           kubernetes_url https://localhost:8443
           watch false
           cache_size 1
           lookup_from_k8s_field false
           use_journal false
-        ')
+        ")
         expected_kube_metadata = {
           'docker' => {
             'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
@@ -941,7 +1024,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
     end
     test 'processes all events when reading from MessagePackEventStream' do
-      VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' },
+      VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' },
+                         { name: 'kubernetes_get_api_v1' },
                          { name: 'kubernetes_get_pod' },
                          { name: 'kubernetes_get_namespace_default' }]) do
         entries = [[@time, { 'time' => '2015-05-08T09:22:01Z' }], [@time, { 'time' => '2015-05-08T09:22:01Z' }]]
@@ -954,7 +1038,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
           cache_size 1
         ')
         d.run do
-          d.feed(DEFAULT_TAG, msgpack_stream)
+          d.feed(VAR_LOG_CONTAINER_TAG, msgpack_stream)
         end
         filtered = d.filtered.map(&:last)
@@ -998,8 +1082,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
           skip_namespace_metadata true
         ')
         expected_kube_metadata = {
-          'docker' => {
-            'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+          'docker'=>{
+            'container_id'=>'49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
           },
           'kubernetes' => {
             'host' => 'jimmi-redhat.localnet',

metadata CHANGED Viewed

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 2.8.1
+  version: 2.9.3
 platform: ruby
 authors:
 - Jimmi Dyson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-09-17 00:00:00.000000000 Z
+date: 2021-12-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -230,12 +230,13 @@ files:
 - lib/fluent/plugin/kubernetes_metadata_watch_pods.rb
 - test/cassettes/invalid_api_server_config.yml
 - test/cassettes/kubernetes_docker_metadata_annotations.yml
-- test/cassettes/kubernetes_docker_metadata_dotted_labels.yml
+- test/cassettes/kubernetes_docker_metadata_dotted_slashed_labels.yml
 - test/cassettes/kubernetes_get_api_v1.yml
 - test/cassettes/kubernetes_get_api_v1_using_token.yml
 - test/cassettes/kubernetes_get_namespace_default.yml
 - test/cassettes/kubernetes_get_namespace_default_using_token.yml
 - test/cassettes/kubernetes_get_pod.yml
+- test/cassettes/kubernetes_get_pod_container_init.yml
 - test/cassettes/kubernetes_get_pod_using_token.yml
 - test/cassettes/metadata_from_tag_and_journald_fields.yml
 - test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml