fluent-plugin-kubernetes_metadata_filter 2.8.1 → 2.9.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 71fd60517a92c89e8c35bc91a3147487ff0637ae41378ecf82b047424cdb203f
-  data.tar.gz: 310b9a7bbb7e5bd59d1aa3deedb699114f561295b4b3a97fdf54fadcf6d7ee08
+  metadata.gz: b6344b1f6ff0602b9d78a5cf317ba5b44e54d0040d6172d102edd002a364d1e7
+  data.tar.gz: 421689d196ae7a0c307ac7e8d4d066755464b76357921e7f6c55ad122ca744b0
 SHA512:
-  metadata.gz: 76ccb8d8b2a2d26ca1e2823a93c38fef974e76526f402f11a10c8ea4697cbda765370eb202177490e9a4aadb10fef4f9980ba4e38cb39953c55ae3bf49ac3bf4
-  data.tar.gz: 8ea9696e6df02bf264a7f0e68ab36732adae9a007379e6bbd841633b6ad6dd830d9241a30bdcd0744183151898fdcc71711be14656973a47d8afa7a1f273abb1
+  metadata.gz: 63b03d82fec8888d13f4f752b1ea2519a7c3e594ac45eb0363109d323a35a56174fdd0fa401b787069e5bae84168878ff08baefa899810776b0d710bd55d3c04
+  data.tar.gz: 6368f491a96f30482ded9e12bdc0e80232aebfc26a3a98be75133312c96b1311ff04ce2de5d818a06ba67d2d75274b658471668bfa846db2251202ea92f4b96e

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-kubernetes_metadata_filter (2.8.1)
+    fluent-plugin-kubernetes_metadata_filter (2.9.3)
       fluentd (>= 0.14.0, < 1.15)
       kubeclient (>= 4.0.0, < 5.0.0)
       lru_redux
@@ -11,7 +11,7 @@ GEM
   specs:
     addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
-    ast (2.4.1)
+    ast (2.4.2)
     bump (0.10.0)
     charlock_holmes (0.7.7)
     codeclimate-test-reporter (0.6.0)
@@ -22,7 +22,7 @@ GEM
       github-linguist
     crack (0.4.5)
       rexml
-    docile (1.3.5)
+    docile (1.4.0)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     escape_utils (1.2.1)
@@ -30,19 +30,19 @@ GEM
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
       rake
-    fluentd (1.14.0)
+    fluentd (1.14.3)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)
-      http_parser.rb (>= 0.5.1, < 0.8.0)
+      http_parser.rb (>= 0.5.1, < 0.9.0)
       msgpack (>= 1.3.1, < 2.0.0)
       serverengine (>= 2.2.2, < 3.0.0)
       sigdump (~> 0.2.2)
-      strptime (>= 0.2.2, < 1.0.0)
+      strptime (>= 0.2.4, < 1.0.0)
       tzinfo (>= 1.0, < 3.0)
       tzinfo-data (~> 1.0)
       webrick (>= 1.4.2, < 1.8.0)
       yajl-ruby (~> 1.0)
-    github-linguist (7.12.2)
+    github-linguist (7.17.0)
       charlock_holmes (~> 0.7.7)
       escape_utils (~> 1.2.0)
       mini_mime (~> 1.0)
@@ -59,7 +59,7 @@ GEM
     http-form_data (2.3.0)
     http-parser (1.2.3)
       ffi-compiler (>= 1.0, < 2.0)
-    http_parser.rb (0.7.0)
+    http_parser.rb (0.8.0)
     jsonpath (1.1.0)
       multi_json
     kubeclient (4.9.2)
@@ -68,43 +68,43 @@ GEM
       recursive-open-struct (~> 1.1, >= 1.1.1)
       rest-client (~> 2.0)
     lru_redux (1.1.0)
-    mime-types (3.3.1)
+    mime-types (3.4.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2021.0901)
-    mini_mime (1.0.2)
+    mime-types-data (3.2021.1115)
+    mini_mime (1.1.2)
     minitest (4.7.5)
     msgpack (1.4.2)
     multi_json (1.15.0)
     netrc (0.11.0)
-    parallel (1.20.1)
-    parser (3.0.0.0)
+    parallel (1.21.0)
+    parser (3.0.2.0)
       ast (~> 2.4.1)
-    power_assert (1.2.0)
+    power_assert (2.0.1)
     public_suffix (4.0.6)
     rainbow (3.0.0)
-    rake (13.0.3)
+    rake (13.0.6)
     recursive-open-struct (1.1.3)
-    regexp_parser (2.0.3)
+    regexp_parser (2.1.1)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
     rexml (3.2.5)
-    rr (1.2.1)
-    rubocop (1.8.1)
+    rr (3.0.8)
+    rubocop (1.22.3)
       parallel (~> 1.10)
       parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 1.2.0, < 2.0)
+      rubocop-ast (>= 1.12.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.4.0)
-      parser (>= 2.7.1.5)
+    rubocop-ast (1.12.0)
+      parser (>= 3.0.1.1)
     ruby-progressbar (1.11.0)
-    rugged (1.1.0)
+    rugged (1.2.0)
     serverengine (2.2.4)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
@@ -113,7 +113,7 @@ GEM
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.2)
+    simplecov_json_formatter (0.1.3)
     strptime (0.2.5)
     test-unit (3.0.9)
       power_assert
@@ -122,15 +122,15 @@ GEM
       test-unit (>= 2.5.2)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2021.1)
+    tzinfo-data (1.2021.5)
       tzinfo (>= 1.0.0)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.8)
-    unicode-display_width (2.0.0)
+    unicode-display_width (2.1.0)
     vcr (6.0.0)
-    webmock (3.11.1)
-      addressable (>= 2.3.6)
+    webmock (3.14.0)
+      addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
     webrick (1.7.0)

data/README.md

@@ -45,8 +45,10 @@ This must used named capture groups for `container_name`, `pod_name` & `namespac
 * `cache_size` - size of the cache of Kubernetes metadata to reduce requests to the API server (default: `1000`)
 * `cache_ttl` - TTL in seconds of each cached element. Set to negative value to disable TTL eviction (default: `3600` - 1 hour)
 * `watch` - set up a watch on pods on the API server for updates to metadata (default: `true`)
-* `de_dot` - replace dots in labels and annotations with configured `de_dot_separator`, required for ElasticSearch 2.x compatibility (default: `true`)
+* `de_dot` - replace dots in labels and annotations with configured `de_dot_separator`, required for Datadog and ElasticSearch 2.x compatibility (default: `true`)
 * `de_dot_separator` - separator to use if `de_dot` is enabled (default: `_`)
+* `de_slash` - replace slashes in labels and annotations with configured `de_slash_separator`, required for Datadog compatibility (default: `false`)
+* `de_slash_separator` - separator to use if `de_slash` is enabled (default: `__`)
 * *DEPRECATED* `use_journal` - If false, messages are expected to be formatted and tagged as if read by the fluentd in\_tail plugin with wildcard filename.  If true, messages are expected to be formatted as if read from the systemd journal.  The `MESSAGE` field has the full message.  The `CONTAINER_NAME` field has the encoded k8s metadata (see below).  The `CONTAINER_ID_FULL` field has the full container uuid.  This requires docker to use the `--log-driver=journald` log driver.  If unset (the default), the plugin will use the `CONTAINER_NAME` and `CONTAINER_ID_FULL` fields
 if available, otherwise, will use the tag in the `tag_to_kubernetes_name_regexp` format.
 * `container_name_to_kubernetes_regexp` - The regular expression used to extract the k8s metadata encoded in the journal `CONTAINER_NAME` field default: See [code](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/blob/master/lib/fluent/plugin/filter_kubernetes_metadata.rb#L68)
@@ -74,7 +76,7 @@ payload.  The following configuration options are removed:
 
 One way of preserving JSON logs can be through the [parser plugin](https://docs.fluentd.org/filter/parser)
 
-**NOTE** As of this release, the use of `use_journal` is **DEPRECATED**.  If this setting is not present, the plugin will
+**NOTE** As of release v2.1.4, the use of `use_journal` is **DEPRECATED**.  If this setting is not present, the plugin will
 attempt to figure out the source of the metadata fields from the following:
 - If `lookup_from_k8s_field true` (the default) and the following fields are present in the record:
 `docker.container_id`, `kubernetes.namespace_name`, `kubernetes.pod_name`, `kubernetes.container_name`,

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = 'fluent-plugin-kubernetes_metadata_filter'
-  gem.version       = '2.8.1'
+  gem.version       = '2.9.3'
   gem.authors       = ['Jimmi Dyson']
   gem.email         = ['jimmidyson@gmail.com']
   gem.description   = 'Filter plugin to add Kubernetes metadata'

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -51,13 +51,23 @@ module Fluent::Plugin
     config_param :client_key, :string, default: nil
     config_param :ca_file, :string, default: nil
     config_param :verify_ssl, :bool, default: true
-    config_param :tag_to_kubernetes_name_regexp,
-                 :string,
-                 default: 'var\.log\.containers\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
+
+    REGEX_VAR_LOG_PODS = '(?<prefix>var\.log\.pods)\.(?<namespace>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<pod_uuid>[a-z0-9-]*)\.(?<container_name>.+)\..*\.log$'
+    REGEX_VAR_LOG_CONTAINERS = '(?<prefix>var\.log\.containers)\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
+
+    #tag_to_kubernetes_name_regexp which must include named capture groups:
+    #  namespace            - The namespace in which the pod is deployed
+    #  pod_name             - The pod name
+    #  container_name       - The name of the container
+    #  pod_uuid (/var/log/pods) | docker_id (/var/log/containers) - Unique identifier used in caching of either pod_uuid or the container hash
+    config_param :tag_to_kubernetes_name_regexp, :string, default: "(#{REGEX_VAR_LOG_PODS}|#{REGEX_VAR_LOG_CONTAINERS})"
+
     config_param :bearer_token_file, :string, default: nil
     config_param :secret_dir, :string, default: '/var/run/secrets/kubernetes.io/serviceaccount'
     config_param :de_dot, :bool, default: true
     config_param :de_dot_separator, :string, default: '_'
+    config_param :de_slash, :bool, default: false
+    config_param :de_slash_separator, :string, default: '__'
     # if reading from the journal, the record will contain the following fields in the following
     # format:
     # CONTAINER_NAME=k8s_$containername.$containerhash_$podname_$namespacename_$poduuid_$rand32bitashex
@@ -98,15 +108,15 @@ module Fluent::Plugin
     config_param :watch_retry_max_times, :integer, default: 10
 
     def fetch_pod_metadata(namespace_name, pod_name)
-      log.trace("fetching pod metadata: #{namespace_name}/#{pod_name}") if log.trace?
+      log.trace("fetching pod metadata: #{namespace_name}/#{pod_name}")
       options = {
         resource_version: '0' # Fetch from API server cache instead of etcd quorum read
       }
       pod_object = @client.get_pod(pod_name, namespace_name, options)
-      log.trace("raw metadata for #{namespace_name}/#{pod_name}: #{pod_object}") if log.trace?
+      log.trace("raw metadata for #{namespace_name}/#{pod_name}: #{pod_object}")
       metadata = parse_pod_metadata(pod_object)
       @stats.bump(:pod_cache_api_updates)
-      log.trace("parsed metadata for #{namespace_name}/#{pod_name}: #{metadata}") if log.trace?
+      log.trace("parsed metadata for #{namespace_name}/#{pod_name}: #{metadata}")
       @cache[metadata['pod_id']] = metadata
     rescue StandardError => e
       @stats.bump(:pod_cache_api_nil_error)
@@ -130,15 +140,15 @@ module Fluent::Plugin
     end
 
     def fetch_namespace_metadata(namespace_name)
-      log.trace("fetching namespace metadata: #{namespace_name}") if log.trace?
+      log.trace("fetching namespace metadata: #{namespace_name}")
       options = {
         resource_version: '0' # Fetch from API server cache instead of etcd quorum read
       }
       namespace_object = @client.get_namespace(namespace_name, nil, options)
-      log.trace("raw metadata for #{namespace_name}: #{namespace_object}") if log.trace?
+      log.trace("raw metadata for #{namespace_name}: #{namespace_object}")
       metadata = parse_namespace_metadata(namespace_object)
       @stats.bump(:namespace_cache_api_updates)
-      log.trace("parsed metadata for #{namespace_name}: #{metadata}") if log.trace?
+      log.trace("parsed metadata for #{namespace_name}: #{metadata}")
       @namespace_cache[metadata['namespace_id']] = metadata
     rescue StandardError => e
       @stats.bump(:namespace_cache_api_nil_error)
@@ -154,10 +164,6 @@ module Fluent::Plugin
     def configure(conf)
       super
 
-      def log.trace?
-        level == Fluent::Log::LEVEL_TRACE
-      end
-
       require 'kubeclient'
       require 'lru_redux'
       @stats = KubernetesMetadata::Stats.new
@@ -166,6 +172,10 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "Invalid de_dot_separator: cannot be or contain '.'"
       end
 
+      if @de_slash && @de_slash_separator.include?('/')
+        raise Fluent::ConfigError, "Invalid de_slash_separator: cannot be or contain '/'"
+      end
+
       if @cache_ttl < 0
         log.info 'Setting the cache TTL to :none because it was <= 0'
         @cache_ttl = :none
@@ -181,6 +191,7 @@ module Fluent::Plugin
       @namespace_cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl)
 
       @tag_to_kubernetes_name_regexp_compiled = Regexp.compile(@tag_to_kubernetes_name_regexp)
+      
       @container_name_to_kubernetes_regexp_compiled = Regexp.compile(@container_name_to_kubernetes_regexp)
 
       # Use Kubernetes default service account if we're in a pod.
@@ -294,40 +305,47 @@ module Fluent::Plugin
       end
     end
 
-    def get_metadata_for_record(namespace_name, pod_name, container_name, container_id, create_time, batch_miss_cache)
+    def get_metadata_for_record(source, namespace_name, pod_name, container_name, cache_key, create_time, batch_miss_cache)
       metadata = {
-        'docker' => { 'container_id' => container_id },
+        'docker' => { 'container_id' => "" },
         'kubernetes' => {
           'container_name' => container_name,
           'namespace_name' => namespace_name,
           'pod_name' => pod_name
         }
       }
+      metadata['docker']['container_id'] = cache_key unless source == 'var.log.pods'
+      container_cache_key = container_name
       if present?(@kubernetes_url)
-        pod_metadata = get_pod_metadata(container_id, namespace_name, pod_name, create_time, batch_miss_cache)
-
-        if (pod_metadata.include? 'containers') && (pod_metadata['containers'].include? container_id) && !@skip_container_metadata
-          metadata['kubernetes']['container_image'] = pod_metadata['containers'][container_id]['image']
-          metadata['kubernetes']['container_image_id'] = pod_metadata['containers'][container_id]['image_id']
+        pod_metadata = get_pod_metadata(cache_key, namespace_name, pod_name, create_time, batch_miss_cache)
+        if (pod_metadata.include? 'containers') && (pod_metadata['containers'].include? container_cache_key) && !@skip_container_metadata
+          metadata['kubernetes']['container_image'] = pod_metadata['containers'][container_cache_key]['image']
+          metadata['kubernetes']['container_image_id'] = pod_metadata['containers'][container_cache_key]['image_id'] unless pod_metadata['containers'][container_cache_key]['image_id'].empty?
+          metadata['docker']['container_id'] = pod_metadata['containers'][container_cache_key]['containerID'] unless pod_metadata['containers'][container_cache_key]['containerID'].empty?
         end
 
         metadata['kubernetes'].merge!(pod_metadata) if pod_metadata
         metadata['kubernetes'].delete('containers')
       end
+      metadata.delete('docker') if metadata['docker'] && (metadata['docker']['container_id'].nil? || metadata['docker']['container_id'].empty?)
       metadata
     end
 
     def filter_stream(tag, es)
       return es if (es.respond_to?(:empty?) && es.empty?) || !es.is_a?(Fluent::EventStream)
-
       new_es = Fluent::MultiEventStream.new
       tag_match_data = tag.match(@tag_to_kubernetes_name_regexp_compiled) unless @use_journal
       tag_metadata = nil
       batch_miss_cache = {}
       es.each do |time, record|
         if tag_match_data && tag_metadata.nil?
-          tag_metadata = get_metadata_for_record(tag_match_data['namespace'], tag_match_data['pod_name'], tag_match_data['container_name'],
-                                                 tag_match_data['docker_id'], create_time_from_record(record, time), batch_miss_cache)
+          cache_key =  if tag_match_data.names.include?('pod_uuid') && !tag_match_data['pod_uuid'].nil?
+            tag_match_data['pod_uuid']
+          else
+            tag_match_data['docker_id']
+          end 
+          tag_metadata = get_metadata_for_record(tag_match_data['prefix'], tag_match_data['namespace'], tag_match_data['pod_name'], tag_match_data['container_name'],
+                                                 cache_key, create_time_from_record(record, time), batch_miss_cache)
         end
         metadata = Marshal.load(Marshal.dump(tag_metadata)) if tag_metadata
         if (@use_journal || @use_journal.nil?) &&
@@ -340,12 +358,11 @@ module Fluent::Plugin
            record['kubernetes'].key?('pod_name') &&
            record['kubernetes'].key?('container_name') &&
            record['docker'].key?('container_id') &&
-           (k_metadata = get_metadata_for_record(record['kubernetes']['namespace_name'], record['kubernetes']['pod_name'],
+           (k_metadata = get_metadata_for_record(tag_match_data['prefix'], record['kubernetes']['namespace_name'], record['kubernetes']['pod_name'],
                                                  record['kubernetes']['container_name'], record['docker']['container_id'],
                                                  create_time_from_record(record, time), batch_miss_cache))
           metadata = k_metadata
         end
-
         record = record.merge(metadata) if metadata
         new_es.add(time, record)
       end
@@ -357,8 +374,8 @@ module Fluent::Plugin
       metadata = nil
       if record.key?('CONTAINER_NAME') && record.key?('CONTAINER_ID_FULL')
         metadata = record['CONTAINER_NAME'].match(@container_name_to_kubernetes_regexp_compiled) do |match_data|
-          get_metadata_for_record(match_data['namespace'], match_data['pod_name'], match_data['container_name'],
-                                  record['CONTAINER_ID_FULL'], create_time_from_record(record, time), batch_miss_cache)
+          get_metadata_for_record(match_data['name_prefix'], match_data['namespace'], match_data['pod_name'], match_data['container_name'],
+            record['CONTAINER_ID_FULL'], create_time_from_record(record, time), batch_miss_cache)
         end
         unless metadata
           log.debug "Error: could not match CONTAINER_NAME from record #{record}"
@@ -381,6 +398,16 @@ module Fluent::Plugin
       end
     end
 
+    def de_slash!(h)
+      h.keys.each do |ref|
+        next unless h[ref] && ref =~ /\//
+
+        v = h.delete(ref)
+        newref = ref.to_s.gsub('/', @de_slash_separator)
+        h[newref] = v
+      end
+    end
+
     # copied from activesupport
     def present?(object)
       object.respond_to?(:empty?) ? !object.empty? : !!object

data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb

@@ -24,8 +24,6 @@ module KubernetesMetadata
       metadata = {}
       ids = @id_cache[key]
       if ids.nil?
-        # FAST PATH
-        # Cache hit, fetch metadata from the cache
         @stats.bump(:id_cache_miss)
         return batch_miss_cache["#{namespace_name}_#{pod_name}"] if batch_miss_cache.key?("#{namespace_name}_#{pod_name}")
 
@@ -65,7 +63,7 @@ module KubernetesMetadata
               @stats.bump(:id_cache_orphaned_record)
             end
             if @allow_orphans
-              log.trace("orphaning message for: #{namespace_name}/#{pod_name} ") if log.trace?
+              log.trace("orphaning message for: #{namespace_name}/#{pod_name} ")
               metadata = {
                 'orphaned_namespace' => namespace_name,
                 'namespace_name' => @orphaned_namespace_name,

data/lib/fluent/plugin/kubernetes_metadata_common.rb

@@ -47,6 +47,10 @@ module KubernetesMetadata
         de_dot!(labels) unless @skip_labels
         de_dot!(annotations)
       end
+      if @de_slash
+        de_slash!(labels) unless @skip_labels
+        de_slash!(annotations)
+      end
       kubernetes_metadata = {
         'namespace_id' => namespace_object[:metadata][:uid],
         'creation_timestamp' => namespace_object[:metadata][:creationTimestamp]
@@ -65,14 +69,18 @@ module KubernetesMetadata
         de_dot!(labels) unless @skip_labels
         de_dot!(annotations)
       end
+      if @de_slash
+        de_slash!(labels) unless @skip_labels
+        de_slash!(annotations)
+      end
 
       # collect container information
       container_meta = {}
       begin
         pod_object[:status][:containerStatuses].each do |container_status|
-          # get plain container id (eg. docker://hash -> hash)
-          container_id = container_status[:containerID].sub(%r{^[-_a-zA-Z0-9]+://}, '')
-          container_meta[container_id] = if @skip_container_metadata
+          container_id = (container_status[:containerID]||"").sub(%r{^[-_a-zA-Z0-9]+://}, '')
+          key = container_status[:name]
+          container_meta[key] = if @skip_container_metadata
                                            {
                                              'name' => container_status[:name]
                                            }
@@ -80,10 +88,11 @@ module KubernetesMetadata
                                            {
                                              'name' => container_status[:name],
                                              'image' => container_status[:image],
-                                             'image_id' => container_status[:imageID]
+                                             'image_id' => container_status[:imageID],
+                                             :containerID => container_id
                                            }
                                          end
-        end
+        end if pod_object[:status] && pod_object[:status][:containerStatuses]
       rescue StandardError=>e
         log.warn("parsing container meta information failed for: #{pod_object[:metadata][:namespace]}/#{pod_object[:metadata][:name]}: #{e}")
       end

data/test/cassettes/kubernetes_get_pod_container_init.yml

@@ -0,0 +1,145 @@
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Fri, 08 May 2015 10:35:37 GMT
+      Transfer-Encoding:
+      - chunked
+    body:
+      encoding: UTF-8
+      string: |-
+        {
+          "kind": "Pod",
+          "apiVersion": "v1",
+          "metadata": {
+            "name": "fabric8-console-controller-98rqc",
+            "generateName": "fabric8-console-controller-",
+            "namespace": "default",
+            "selfLink": "/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc",
+            "uid": "c76927af-f563-11e4-b32d-54ee7527188d",
+            "resourceVersion": "122",
+            "creationTimestamp": "2015-05-08T09:22:42Z",
+            "labels": {
+              "component": "fabric8Console"
+            }
+          },
+          "spec": {
+            "volumes": [
+              {
+                "name": "openshift-cert-secrets",
+                "hostPath": null,
+                "emptyDir": null,
+                "gcePersistentDisk": null,
+                "gitRepo": null,
+                "secret": {
+                  "secretName": "openshift-cert-secrets"
+                },
+                "nfs": null,
+                "iscsi": null,
+                "glusterfs": null
+              }
+            ],
+            "containers": [
+              {
+                "name": "fabric8-console-container",
+                "image": "fabric8/hawtio-kubernetes:latest",
+                "ports": [
+                  {
+                    "containerPort": 9090,
+                    "protocol": "TCP"
+                  }
+                ],
+                "env": [
+                  {
+                    "name": "OAUTH_CLIENT_ID",
+                    "value": "fabric8-console"
+                  },
+                  {
+                    "name": "OAUTH_AUTHORIZE_URI",
+                    "value": "https://localhost:8443/oauth/authorize"
+                  }
+                ],
+                "resources": {},
+                "volumeMounts": [
+                  {
+                    "name": "openshift-cert-secrets",
+                    "readOnly": true,
+                    "mountPath": "/etc/secret-volume"
+                  }
+                ],
+                "terminationMessagePath": "/dev/termination-log",
+                "imagePullPolicy": "IfNotPresent",
+                "capabilities": {}
+              }
+            ],
+            "restartPolicy": "Always",
+            "dnsPolicy": "ClusterFirst",
+            "nodeName": "jimmi-redhat.localnet"
+          },
+          "status": {
+            "phase": "Running",
+            "Condition": [
+              {
+                "type": "Ready",
+                "status": "True"
+              }
+            ],
+            "hostIP": "172.17.42.1",
+            "podIP": "172.17.0.8",
+            "containerStatuses": [
+              {
+                "name": "fabric8-console-container",
+                "state": {
+                  "waiting": {
+                    "reason": "ContainerCreating"
+                  }
+                },
+                "lastState": {},
+                "ready": true,
+                "restartCount": 0,
+                "image": "fabric8/hawtio-kubernetes:latest",
+                "imageID": ""
+              }
+            ]
+          }
+        }
+    http_version: 
+  recorded_at: Fri, 08 May 2015 10:35:37 GMT
+recorded_with: VCR 2.9.3

data/test/plugin/test_cache_strategy.rb

@@ -44,7 +44,7 @@ class TestCacheStrategy
 
   def log
     logger = {}
-    def logger.trace?
+    def logger.on_trace
       true
     end
 

data/test/plugin/test_filter_kubernetes_metadata.rb

@@ -28,7 +28,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
     @time = Fluent::Engine.now
   end
 
-  DEFAULT_TAG = 'var.log.containers.fabric8-console-controller-98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log'
+  VAR_LOG_CONTAINER_TAG = 'var.log.containers.fabric8-console-controller-98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log'
+  VAR_LOG_POD_TAG = 'var.log.pods.default_fabric8-console-controller-98rqc_c76927af-f563-11e4-b32d-54ee7527188d.fabric8-console-container.0.log'
 
   def create_driver(conf = '')
     Test::Driver::Filter.new(Plugin::KubernetesMetadataFilter).configure(conf)
@@ -45,7 +46,6 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       assert_equal('KubernetesMetadata::TestApiAdapter', d.instance.test_api_adapter)
     end
 
-
     test 'kubernetes url' do
       VCR.use_cassette('valid_kubernetes_api_server') do
         d = create_driver('
@@ -138,7 +138,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
           cache_size 1
         ', d: nil)
       d = create_driver(config) if d.nil?
-      d.run(default_tag: DEFAULT_TAG) do
+      d.run(default_tag: VAR_LOG_CONTAINER_TAG) do
         d.feed(@time, msg)
       end
       d.filtered.map(&:last)
@@ -165,6 +165,76 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       plugin.filter_stream('tag', Fluent::MultiEventStream.new)
     end
 
+    sub_test_case 'parsing_pod_metadata when container_status is missing from the pod status' do
+      test 'using the tag_to_kubernetes_name_regexp for /var/log/containers ' do
+        VCR.use_cassettes(
+          [
+            { name: 'valid_kubernetes_api_server' },
+            { name: 'kubernetes_get_api_v1' },
+            { name: 'kubernetes_get_namespace_default' },
+            { name: 'kubernetes_get_pod_container_init' }
+          ]) do
+          filtered = emit({}, "
+            kubernetes_url https://localhost:8443
+            watch false
+            cache_size 1
+          ")
+          expected_kube_metadata = {
+            'docker' => {
+              'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+            },
+            'kubernetes' => {
+              'container_image'=>'fabric8/hawtio-kubernetes:latest',
+              'container_name'=>'fabric8-console-container',
+              'host' => 'jimmi-redhat.localnet',
+              'pod_name' => 'fabric8-console-controller-98rqc',
+              'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
+              'namespace_name' => 'default',
+              'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
+              'pod_ip' => '172.17.0.8',
+              'master_url' => 'https://localhost:8443',
+              'labels' => {
+                'component' => 'fabric8Console'
+              }
+            }
+          }
+          assert_equal(expected_kube_metadata, filtered[0])
+        end
+      end
+      test 'using the tag_to_kubernetes_name_regexp for /var/log/pods' do
+        VCR.use_cassettes(
+          [
+            { name: 'valid_kubernetes_api_server' },
+            { name: 'kubernetes_get_api_v1' },
+            { name: 'kubernetes_get_namespace_default' },
+            { name: 'kubernetes_get_pod_container_init' }
+          ]) do
+          filtered = emit_with_tag(VAR_LOG_POD_TAG,{}, "
+            kubernetes_url https://localhost:8443
+            watch false
+            cache_size 1
+          ")
+          expected_kube_metadata = {
+            'kubernetes' => {
+              'container_image'=>'fabric8/hawtio-kubernetes:latest',
+              'container_name'=>'fabric8-console-container',
+              'host' => 'jimmi-redhat.localnet',
+              'pod_name' => 'fabric8-console-controller-98rqc',
+              'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
+              'namespace_name' => 'default',
+              'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
+              'pod_ip' => '172.17.0.8',
+              'master_url' => 'https://localhost:8443',
+              'labels' => {
+                'component' => 'fabric8Console'
+              }
+            }
+          }
+          assert_equal(expected_kube_metadata, filtered[0])
+        end
+      end
+    end
+
     test 'inability to connect to the api server handles exception and doensnt block pipeline' do
       VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' }]) do
         driver = create_driver('
@@ -177,8 +247,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
         filtered = emit({ 'time' => '2015-05-08T09:22:01Z' }, '', d: driver)
         expected_kube_metadata = {
           'time' => '2015-05-08T09:22:01Z',
-          'docker' => {
-            'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+          'docker'=>{
+            'container_id'=>'49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
           },
           'kubernetes' => {
             'pod_name' => 'fabric8-console-controller-98rqc',
@@ -265,6 +335,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
     end
 
     test 'with docker & kubernetes metadata' do
+
       VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' }, { name: 'kubernetes_get_pod' }, { name: 'kubernetes_get_namespace_default' }]) do
         filtered = emit({ 'time' => '2015-05-08T09:22:01Z' })
         expected_kube_metadata = {
@@ -361,8 +432,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
     test 'with docker & kubernetes metadata but no configured api server' do
       filtered = emit({}, '')
       expected_kube_metadata = {
-        'docker' => {
-          'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+        'docker'=>{
+          'container_id'=>'49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
         },
         'kubernetes' => {
           'pod_name' => 'fabric8-console-controller-98rqc',
@@ -383,8 +454,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       stub_request(:any, 'https://localhost:8443/api/v1/namespaces/default').to_timeout
       filtered = emit
       expected_kube_metadata = {
-        'docker' => {
-          'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+        'docker'=>{
+          'container_id'=>'49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
         },
         'kubernetes' => {
           'pod_name' => 'fabric8-console-controller-98rqc',
@@ -406,8 +477,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       stub_request(:any, 'https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller.98rqc').to_timeout
       filtered = emit_with_tag('var.log.containers.fabric8-console-controller.98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log', {}, '')
       expected_kube_metadata = {
-        'docker' => {
-          'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+        'docker'=>{
+          'container_id'=>'49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
         },
         'kubernetes' => {
           'pod_name' => 'fabric8-console-controller.98rqc',
@@ -432,13 +503,15 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       assert_equal(msg, filtered[0])
     end
 
-    test 'with kubernetes dotted labels, de_dot enabled' do
+    test 'with kubernetes dotted and slashed labels, de_dot and de_slash enabled' do
       VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' },
-                         { name: 'kubernetes_docker_metadata_dotted_labels' }]) do
+                         { name: 'kubernetes_docker_metadata_dotted_slashed_labels' }]) do
         filtered = emit({}, '
           kubernetes_url https://localhost:8443
           watch false
           cache_size 1
+          de_dot true
+          de_slash true
         ')
         expected_kube_metadata = {
           'docker' => {
@@ -452,14 +525,14 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
             'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
             'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
             'namespace_labels' => {
-              'kubernetes_io/namespacetest' => 'somevalue'
+              'kubernetes_io__namespacetest' => 'somevalue'
             },
             'namespace_name' => 'default',
             'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
             'pod_ip' => '172.17.0.8',
             'master_url' => 'https://localhost:8443',
             'labels' => {
-              'kubernetes_io/test' => 'somevalue'
+              'kubernetes_io__test' => 'somevalue'
             }
           }
         }
@@ -467,14 +540,15 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       end
     end
 
-    test 'with kubernetes dotted labels, de_dot disabled' do
+    test 'with kubernetes dotted and slashed labels, de_dot and de_slash disabled' do
       VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' },
-                         { name: 'kubernetes_docker_metadata_dotted_labels' }]) do
+                         { name: 'kubernetes_docker_metadata_dotted_slashed_labels' }]) do
         filtered = emit({}, '
           kubernetes_url https://localhost:8443
           watch false
           cache_size 1
           de_dot false
+          de_slash false
         ')
         expected_kube_metadata = {
           'docker' => {
@@ -506,11 +580,21 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
     test 'invalid de_dot_separator config' do
       assert_raise Fluent::ConfigError do
         create_driver('
+          de_dot true
           de_dot_separator contains.
         ')
       end
     end
 
+    test 'invalid de_slash_separator config' do
+      assert_raise Fluent::ConfigError do
+        create_driver('
+          de_slash true
+          de_slash_separator contains/
+        ')
+      end
+    end
+
     test 'with records from journald and docker & kubernetes metadata' do
       # with use_journal true should ignore tags and use CONTAINER_NAME and CONTAINER_ID_FULL
       tag = 'var.log.containers.junk1_junk2_junk3-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed450.log'
@@ -686,7 +770,6 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
 
     test 'uses metadata from tag if use_journal false and lookup_from_k8s_field false' do
       # with use_journal unset, should still use the journal fields instead of tag fields
-      tag = 'var.log.containers.fabric8-console-controller-98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log'
       msg = {
         'CONTAINER_NAME' => 'k8s_journald-container-name.db89db89_journald-pod-name_journald-namespace-name_c76927af-f563-11e4-b32d-54ee7527188d_89db89db',
         'CONTAINER_ID_FULL' => '838350c64bacba968d39a30a50789b2795291fceca6ccbff55298671d46b0e3b',
@@ -701,13 +784,13 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' }, { name: 'kubernetes_get_pod' },
                          { name: 'kubernetes_get_namespace_default', options: { allow_playback_repeats: true } },
                          { name: 'metadata_from_tag_and_journald_fields' }]) do
-        es = emit_with_tag(tag, msg, '
+        es = emit_with_tag(VAR_LOG_CONTAINER_TAG, msg, "
           kubernetes_url https://localhost:8443
           watch false
           cache_size 1
           lookup_from_k8s_field false
           use_journal false
-        ')
+        ")
         expected_kube_metadata = {
           'docker' => {
             'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
@@ -941,7 +1024,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
     end
 
     test 'processes all events when reading from MessagePackEventStream' do
-      VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' },
+      VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, 
+                         { name: 'kubernetes_get_api_v1' },
                          { name: 'kubernetes_get_pod' },
                          { name: 'kubernetes_get_namespace_default' }]) do
         entries = [[@time, { 'time' => '2015-05-08T09:22:01Z' }], [@time, { 'time' => '2015-05-08T09:22:01Z' }]]
@@ -954,7 +1038,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
           cache_size 1
         ')
         d.run do
-          d.feed(DEFAULT_TAG, msgpack_stream)
+          d.feed(VAR_LOG_CONTAINER_TAG, msgpack_stream)
         end
         filtered = d.filtered.map(&:last)
 
@@ -998,8 +1082,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
           skip_namespace_metadata true
         ')
         expected_kube_metadata = {
-          'docker' => {
-            'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+          'docker'=>{
+            'container_id'=>'49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
           },
           'kubernetes' => {
             'host' => 'jimmi-redhat.localnet',

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 2.8.1
+  version: 2.9.3
 platform: ruby
 authors:
 - Jimmi Dyson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-09-17 00:00:00.000000000 Z
+date: 2021-12-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -230,12 +230,13 @@ files:
 - lib/fluent/plugin/kubernetes_metadata_watch_pods.rb
 - test/cassettes/invalid_api_server_config.yml
 - test/cassettes/kubernetes_docker_metadata_annotations.yml
-- test/cassettes/kubernetes_docker_metadata_dotted_labels.yml
+- test/cassettes/kubernetes_docker_metadata_dotted_slashed_labels.yml
 - test/cassettes/kubernetes_get_api_v1.yml
 - test/cassettes/kubernetes_get_api_v1_using_token.yml
 - test/cassettes/kubernetes_get_namespace_default.yml
 - test/cassettes/kubernetes_get_namespace_default_using_token.yml
 - test/cassettes/kubernetes_get_pod.yml
+- test/cassettes/kubernetes_get_pod_container_init.yml
 - test/cassettes/kubernetes_get_pod_using_token.yml
 - test/cassettes/metadata_from_tag_and_journald_fields.yml
 - test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml