fluent-plugin-kubernetes_metadata_filter 2.4.1 → 2.4.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 636af222c31e7e4db80d96594f76d3f4918f981120e9a4006e414ddd6b94df15
-  data.tar.gz: a4edbb9c153da167210ee0fc879d38e37428839ed640264ce81ad824e7991004
+  metadata.gz: b79c9bdf0078c411c16855d5447963bf0cf112545dbdd9ce482636ef5628b731
+  data.tar.gz: 2b1289c54f2ea7926d077d2ce1176b0470ed869e024577172e01b42a8f17c3e5
 SHA512:
-  metadata.gz: a7b667937578e95e259752e9ad64391774103d1ad2ea7f569259ee6f0d6addfbb0e0960fe95379300052d8863342bc6cb0235f95af8f0346a03f6dba3dffffb2
-  data.tar.gz: b8e0a412669d5fe57ce3c9cb8ed641e75c0c67d42305cdf88b91838621e781d172a19e079a4e312117e3e1dfd411f5c91c92a0e1626ed6a9c152e0633f64bcb2
+  metadata.gz: e354cffb2942f9da443d7219d0cbf4b93463207ec188471b6c6e8a06b180655a2befd2500b9a55c62fb96a524025a6a77fb1fdd6a031295cad6394e767104439
+  data.tar.gz: 344d0cdd184e0bdab9c072367ead78b23a16ee69fd4c938ba460aa6213179a091c14fdc6dd3829d74a470307f6bd5dc680f9fc7226f066daaa1ac21bdfab9b09

data/.circleci/config.yml

@@ -2,13 +2,15 @@ version: 2.1
 
 install: &install
   name: Install bundle
-  command: bundle install --path vendor/bundle
+  command: |
+            gem install bundler
+            bundle install --path vendor/bundle
 
 missingdeps: &missingdeps
   name: Install missing dependecies
   command: |
             cat /etc/os-release
-            printf "deb http://archive.debian.org/debian/ jessie main\ndeb-src http://archive.debian.org/debian/ jessie main\ndeb http://security.debian.org jessie/updates main\ndeb-src http://security.debian.org jessie/updates main" > /tmp/sources.list
+            printf "deb http://deb.debian.org/debian buster main\ndeb http://security.debian.org buster/updates main\ndeb-src http://security.debian.org buster/updates main" > /tmp/sources.list
             sudo cp /tmp/sources.list /etc/apt/sources.list
             sudo apt-get update
             sudo apt-get install cmake libicu-dev libssl-dev
@@ -53,4 +55,3 @@ workflows:
           ruby-version: ruby-2-5
       - ruby-test:
           ruby-version: ruby-2-6
-      

data/.gitignore

@@ -4,7 +4,6 @@
 .config
 .yardoc
 vendor/
-Gemfile.lock
 InstalledFiles
 _yardoc
 coverage

data/Gemfile.lock

@@ -0,0 +1,153 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-kubernetes_metadata_filter (2.4.7)
+      fluentd (>= 0.14.0, < 1.10)
+      kubeclient (< 5)
+      lru_redux
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    ast (2.4.0)
+    bump (0.9.0)
+    charlock_holmes (0.7.7)
+    codeclimate-test-reporter (0.6.0)
+      simplecov (>= 0.7.1, < 1.0.0)
+    concurrent-ruby (1.1.6)
+    cool.io (1.6.0)
+    copyright-header (1.0.22)
+      github-linguist
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    docile (1.3.2)
+    domain_name (0.5.20190701)
+      unf (>= 0.0.5, < 1.0.0)
+    escape_utils (1.2.1)
+    ffi (1.13.1)
+    ffi-compiler (1.0.1)
+      ffi (>= 1.0.0)
+      rake
+    fluentd (1.9.3)
+      cool.io (>= 1.4.5, < 2.0.0)
+      http_parser.rb (>= 0.5.1, < 0.7.0)
+      msgpack (>= 1.3.1, < 2.0.0)
+      serverengine (>= 2.0.4, < 3.0.0)
+      sigdump (~> 0.2.2)
+      strptime (>= 0.2.2, < 1.0.0)
+      tzinfo (>= 1.0, < 3.0)
+      tzinfo-data (~> 1.0)
+      yajl-ruby (~> 1.0)
+    github-linguist (7.8.0)
+      charlock_holmes (~> 0.7.6)
+      escape_utils (~> 1.2.0)
+      mini_mime (~> 1.0)
+      rugged (>= 0.25.1)
+    hashdiff (1.0.1)
+    http (4.4.1)
+      addressable (~> 2.3)
+      http-cookie (~> 1.0)
+      http-form_data (~> 2.2)
+      http-parser (~> 1.2.0)
+    http-accept (1.7.0)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    http-form_data (2.3.0)
+    http-parser (1.2.1)
+      ffi-compiler (>= 1.0, < 2.0)
+    http_parser.rb (0.6.0)
+    jaro_winkler (1.5.4)
+    jsonpath (1.0.5)
+      multi_json
+      to_regexp (~> 0.2.1)
+    kubeclient (4.8.0)
+      http (>= 3.0, < 5.0)
+      jsonpath (~> 1.0)
+      recursive-open-struct (~> 1.1, >= 1.1.1)
+      rest-client (~> 2.0)
+    lru_redux (1.1.0)
+    mime-types (3.3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2020.0512)
+    mini_mime (1.0.2)
+    minitest (4.7.5)
+    msgpack (1.3.3)
+    multi_json (1.15.0)
+    netrc (0.11.0)
+    parallel (1.19.1)
+    parser (2.7.0.4)
+      ast (~> 2.4.0)
+    power_assert (1.1.6)
+    public_suffix (4.0.3)
+    rainbow (3.0.0)
+    rake (13.0.1)
+    recursive-open-struct (1.1.2)
+    rest-client (2.1.0)
+      http-accept (>= 1.7.0, < 2.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 4.0)
+      netrc (~> 0.8)
+    rexml (3.2.4)
+    rr (1.2.1)
+    rubocop (0.80.1)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.7.0.1)
+      rainbow (>= 2.2.2, < 4.0)
+      rexml
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 1.7)
+    ruby-progressbar (1.10.1)
+    rugged (0.99.0)
+    safe_yaml (1.0.5)
+    serverengine (2.2.1)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.4)
+    simplecov (0.18.5)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+    simplecov-html (0.12.2)
+    strptime (0.2.4)
+    test-unit (3.0.9)
+      power_assert
+    test-unit-rr (1.0.5)
+      rr (>= 1.1.1)
+      test-unit (>= 2.5.2)
+    to_regexp (0.2.1)
+    tzinfo (2.0.2)
+      concurrent-ruby (~> 1.0)
+    tzinfo-data (1.2020.1)
+      tzinfo (>= 1.0.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.7)
+    unicode-display_width (1.6.1)
+    vcr (5.1.0)
+    webmock (3.8.3)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
+    yajl-ruby (1.4.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bump
+  bundler (~> 2.0)
+  codeclimate-test-reporter (< 1.0.0)
+  copyright-header
+  fluent-plugin-kubernetes_metadata_filter!
+  minitest (~> 4.0)
+  rake
+  rubocop
+  test-unit (~> 3.0.2)
+  test-unit-rr (~> 1.0.3)
+  vcr
+  webmock
+  yajl-ruby
+
+BUNDLED WITH
+   2.1.4

data/README.md

@@ -47,7 +47,7 @@ This must used named capture groups for `container_name`, `pod_name` & `namespac
 * *DEPRECATED* `use_journal` - If false, messages are expected to be formatted and tagged as if read by the fluentd in\_tail plugin with wildcard filename.  If true, messages are expected to be formatted as if read from the systemd journal.  The `MESSAGE` field has the full message.  The `CONTAINER_NAME` field has the encoded k8s metadata (see below).  The `CONTAINER_ID_FULL` field has the full container uuid.  This requires docker to use the `--log-driver=journald` log driver.  If unset (the default), the plugin will use the `CONTAINER_NAME` and `CONTAINER_ID_FULL` fields
 if available, otherwise, will use the tag in the `tag_to_kubernetes_name_regexp` format.
 * `container_name_to_kubernetes_regexp` - The regular expression used to extract the k8s metadata encoded in the journal `CONTAINER_NAME` field (default: `'^(?<name_prefix>[^_]+)_(?<container_name>[^\._]+)(\.(?<container_hash>[^_]+))?_(?<pod_name>[^_]+)_(?<namespace>[^_]+)_[^_]+_[^_]+$'`
-  * This corresponds to the definition [in the source](https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/dockertools/docker.go#L317)
+  * This corresponds to the definition [in the source](https://github.com/kubernetes/kubernetes/blob/release-1.6/pkg/kubelet/dockertools/docker.go#L317)
 * `annotation_match` - Array of regular expressions matching annotation field names. Matched annotations are added to a log record.
 * `allow_orphans` - Modify the namespace and namespace id to the values of `orphaned_namespace_name` and `orphaned_namespace_id`
 when true (default: `true`)
@@ -61,6 +61,7 @@ when true (default: `true`)
 * `skip_container_metadata` - Skip some of the container data of the metadata. The metadata will not contain the container_image and container_image_id fields.
 * `skip_master_url` - Skip the master_url field from the metadata.
 * `skip_namespace_metadata` - Skip the namespace_id field from the metadata. The fetch_namespace_metadata function will be skipped. The plugin will be faster and cpu consumption will be less.
+* `watch_retry_interval` - The time interval in seconds for retry backoffs when watch connections fail. (default: `10`)
 
 **NOTE:** As of the release 2.1.x of this plugin, it no longer supports parsing the source message into JSON and attaching it to the
 payload.  The following configuration options are removed:
@@ -80,16 +81,36 @@ then the plugin will parse those values using `container_name_to_kubernetes_rege
 - Otherwise, if the tag matches `tag_to_kubernetes_name_regexp`, the plugin will parse the tag and use those values to
 lookup the metdata
 
-Reading from the JSON formatted log files with `in_tail` and wildcard filenames:
+Reading from the JSON formatted log files with `in_tail` and wildcard filenames while respecting the CRI-o log format with the same config you need the fluent-plugin "multi-format-parser":
+
+```
+fluent-gem install fluent-plugin-multi-format-parser
+```
+
+The config block could look like this:
 ```
 <source>
   @type tail
   path /var/log/containers/*.log
   pos_file fluentd-docker.pos
-  time_format %Y-%m-%dT%H:%M:%S
-  tag kubernetes.*
-  format json
   read_from_head true
+  tag kubernetes.*
+  <parse>
+    @type multi_format
+    <pattern>
+      format json
+      time_key time
+      time_type string
+      time_format "%Y-%m-%dT%H:%M:%S.%NZ"
+      keep_time_key false
+    </pattern>
+    <pattern>
+      format regexp
+      expression /^(?<time>.+) (?<stream>stdout|stderr)( (?<logtag>.))? (?<log>.*)$/
+      time_format '%Y-%m-%dT%H:%M:%S.%N%:z'
+      keep_time_key false
+    </pattern>
+  </parse>
 </source>
 
 <filter kubernetes.var.log.containers.**.log>
@@ -128,6 +149,22 @@ Reading from the systemd journal (requires the fluentd `fluent-plugin-systemd` a
   @type stdout
 </match>
 ```
+## Log content as JSON
+In former versions this plugin parsed the value of the key log as JSON. In the current version this feature was removed, to avoid duplicate features in the fluentd plugin ecosystem. It can parsed with the parser plugin like this:
+```
+<filter kubernetes.**>
+  @type parser
+  key_name log
+  <parse>
+    @type json
+    json_parser json
+  </parse>
+  replace_invalid_sequence true
+  reserve_data true # this preserves unparsable log lines
+  emit_invalid_record_to_error false # In case of unparsable log lines keep the error log clean
+  reserve_time # the time was already parsed in the source, we don't want to overwrite it with current time.
+</filter>
+```
 
 ## Environment variables for Kubernetes
 
@@ -171,6 +208,7 @@ Then output becomes as belows
     "host": "jimmi-redhat.localnet",
     "pod_name":"fabric8-console-controller-98rqc",
     "pod_id": "c76927af-f563-11e4-b32d-54ee7527188d",
+    "pod_ip": "172.17.0.8",
     "container_name": "fabric8-console-container",
     "namespace_name": "default",
     "namespace_id": "23437884-8e08-4d95-850b-e94378c9b2fd",

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-kubernetes_metadata_filter"
-  gem.version       = "2.4.1"
+  gem.version       = "2.4.7"
   gem.authors       = ["Jimmi Dyson"]
   gem.email         = ["jimmidyson@gmail.com"]
   gem.description   = %q{Filter plugin to add Kubernetes metadata}
@@ -19,11 +19,11 @@ Gem::Specification.new do |gem|
 
   gem.required_ruby_version = '>= 2.1.0'
 
-  gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 2']
+  gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 1.10']
   gem.add_runtime_dependency "lru_redux"
   gem.add_runtime_dependency "kubeclient", '< 5'
 
-  gem.add_development_dependency "bundler", "~> 2.0.2"
+  gem.add_development_dependency "bundler", "~> 2.0"
   gem.add_development_dependency "rake"
   gem.add_development_dependency "minitest", "~> 4.0"
   gem.add_development_dependency "test-unit", "~> 3.0.2"

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -22,6 +22,7 @@ require_relative 'kubernetes_metadata_common'
 require_relative 'kubernetes_metadata_stats'
 require_relative 'kubernetes_metadata_watch_namespaces'
 require_relative 'kubernetes_metadata_watch_pods'
+
 require 'fluent/plugin/filter'
 require 'resolv'
 
@@ -61,7 +62,7 @@ module Fluent::Plugin
     # Field 2 is the container_hash, field 5 is the pod_id, and field 6 is the pod_randhex
     # I would have included them as named groups, but you can't have named groups that are
     # non-capturing :P
-    # parse format is defined here: https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/dockertools/docker.go#L317
+    # parse format is defined here: https://github.com/kubernetes/kubernetes/blob/release-1.6/pkg/kubelet/dockertools/docker.go#L317
     config_param :container_name_to_kubernetes_regexp,
                  :string,
                  :default => '^(?<name_prefix>[^_]+)_(?<container_name>[^\._]+)(\.(?<container_hash>[^_]+))?_(?<pod_name>[^_]+)_(?<namespace>[^_]+)_[^_]+_[^_]+$'
@@ -80,6 +81,12 @@ module Fluent::Plugin
     config_param :skip_container_metadata, :bool, default: false
     config_param :skip_master_url, :bool, default: false
     config_param :skip_namespace_metadata, :bool, default: false
+    # The time interval in seconds for retry backoffs when watch connections fail.
+    config_param :watch_retry_interval, :integer, default: 1
+    # The base number of exponential backoff for retries.
+    config_param :watch_retry_exponential_backoff_base, :integer, default: 2
+    # The maximum number of times to retry pod and namespace watches.
+    config_param :watch_retry_max_times, :integer, default: 10
 
     def fetch_pod_metadata(namespace_name, pod_name)
       log.trace("fetching pod metadata: #{namespace_name}/#{pod_name}") if log.trace?
@@ -201,6 +208,8 @@ module Fluent::Plugin
           end
           @kubernetes_url = "https://#{env_host}:#{env_port}/api"
           log.debug "Kubernetes URL is now '#{@kubernetes_url}'"
+        else
+          log.debug "No Kubernetes URL could be found in config or environ"
         end
       end
 
@@ -264,9 +273,10 @@ module Fluent::Plugin
         end
 
         if @watch
-          thread = Thread.new(self) { |this| this.start_pod_watch }
-          thread.abort_on_exception = true
-          namespace_thread = Thread.new(self) { |this| this.start_namespace_watch }
+          pod_thread = Thread.new(self) { |this| this.set_up_pod_thread }
+          pod_thread.abort_on_exception = true
+
+          namespace_thread = Thread.new(self) { |this| this.set_up_namespace_thread }
           namespace_thread.abort_on_exception = true
         end
       end

data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb

@@ -23,19 +23,92 @@ module KubernetesMetadata
 
     include ::KubernetesMetadata::Common
 
+    def set_up_namespace_thread
+      # Any failures / exceptions in the initial setup should raise
+      # Fluent:ConfigError, so that users can inspect potential errors in
+      # the configuration.
+      namespace_watcher = start_namespace_watch
+      Thread.current[:namespace_watch_retry_backoff_interval] = @watch_retry_interval
+      Thread.current[:namespace_watch_retry_count] = 0
+
+      # Any failures / exceptions in the followup watcher notice
+      # processing will be swallowed and retried. These failures /
+      # exceptions could be caused by Kubernetes API being temporarily
+      # down. We assume the configuration is correct at this point.
+      while true
+        begin
+          namespace_watcher ||= get_namespaces_and_start_watcher
+          process_namespace_watcher_notices(namespace_watcher)
+        rescue Exception => e
+          @stats.bump(:namespace_watch_failures)
+          if Thread.current[:namespace_watch_retry_count] < @watch_retry_max_times
+            # Instead of raising exceptions and crashing Fluentd, swallow
+            # the exception and reset the watcher.
+            log.info(
+              "Exception encountered parsing namespace watch event. " \
+              "The connection might have been closed. Sleeping for " \
+              "#{Thread.current[:namespace_watch_retry_backoff_interval]} " \
+              "seconds and resetting the namespace watcher.", e)
+            sleep(Thread.current[:namespace_watch_retry_backoff_interval])
+            Thread.current[:namespace_watch_retry_count] += 1
+            Thread.current[:namespace_watch_retry_backoff_interval] *= @watch_retry_exponential_backoff_base
+            namespace_watcher = nil
+          else
+            # Since retries failed for many times, log as errors instead
+            # of info and raise exceptions and trigger Fluentd to restart.
+            message =
+              "Exception encountered parsing namespace watch event. The " \
+              "connection might have been closed. Retried " \
+              "#{@watch_retry_max_times} times yet still failing. Restarting."
+            log.error(message, e)
+            raise Fluent::UnrecoverableError.new(message)
+          end
+        end
+      end
+    end
+
     def start_namespace_watch
-      begin
-        resource_version = @client.get_namespaces.resourceVersion
-        watcher          = @client.watch_namespaces(resource_version)
-      rescue Exception=>e
-        message = "start_namespace_watch: Exception encountered setting up namespace watch from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}: #{e.message}"
-        message += " (#{e.response})" if e.respond_to?(:response)
-        log.debug(message)
-        raise Fluent::ConfigError, message
+      return get_namespaces_and_start_watcher
+    rescue Exception => e
+      message = "start_namespace_watch: Exception encountered setting up " \
+                "namespace watch from Kubernetes API #{@apiVersion} endpoint " \
+                "#{@kubernetes_url}: #{e.message}"
+      message += " (#{e.response})" if e.respond_to?(:response)
+      log.debug(message)
+
+      raise Fluent::ConfigError, message
+    end
+
+    # List all namespaces, record the resourceVersion and return a watcher
+    # starting from that resourceVersion.
+    def get_namespaces_and_start_watcher
+      options = {
+        resource_version: '0'  # Fetch from API server.
+      }
+      namespaces = @client.get_namespaces(options)
+      namespaces.each do |namespace|
+        cache_key = namespace.metadata['uid']
+        @namespace_cache[cache_key] = parse_namespace_metadata(namespace)
+        @stats.bump(:namespace_cache_host_updates)
       end
+      options[:resource_version] = namespaces.resourceVersion
+      watcher = @client.watch_namespaces(options)
+      watcher
+    end
+
+    # Reset namespace watch retry count and backoff interval as there is a
+    # successful watch notice.
+    def reset_namespace_watch_retry_stats
+      Thread.current[:namespace_watch_retry_count] = 0
+      Thread.current[:namespace_watch_retry_backoff_interval] = @watch_retry_interval
+    end
+
+    # Process a watcher notice and potentially raise an exception.
+    def process_namespace_watcher_notices(watcher)
       watcher.each do |notice|
         case notice.type
           when 'MODIFIED'
+            reset_namespace_watch_retry_stats
             cache_key = notice.object['metadata']['uid']
             cached    = @namespace_cache[cache_key]
             if cached
@@ -45,16 +118,21 @@ module KubernetesMetadata
               @stats.bump(:namespace_cache_watch_misses)
             end
           when 'DELETED'
-            # ignore and let age out for cases where 
+            reset_namespace_watch_retry_stats
+            # ignore and let age out for cases where
             # deleted but still processing logs
             @stats.bump(:namespace_cache_watch_deletes_ignored)
+          when 'ERROR'
+            @stats.bump(:namespace_watch_error_type_notices)
+            message = notice['object']['message'] if notice['object'] && notice['object']['message']
+            raise "Error while watching namespaces: #{message}"
           else
+            reset_namespace_watch_retry_stats
             # Don't pay attention to creations, since the created namespace may not
-            # be used by any pod on this node.
+            # be used by any namespace on this node.
             @stats.bump(:namespace_cache_watch_ignored)
         end
       end
     end
-
   end
 end

data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb

@@ -19,36 +19,111 @@
 require_relative 'kubernetes_metadata_common'
 
 module KubernetesMetadata
+
+  class GoneError < StandardError
+    def initialize(msg="410 Gone")
+      super
+    end
+  end
+
   module WatchPods
 
     include ::KubernetesMetadata::Common
 
-    def start_pod_watch
-      begin
-        options = {
-          resource_version: '0'  # Fetch from API server.
-        }
-        if ENV['K8S_NODE_NAME']
-          options[:field_selector] = 'spec.nodeName=' + ENV['K8S_NODE_NAME']
-        end
-        pods = @client.get_pods(options)
-        pods.each do |pod|
-          cache_key = pod.metadata['uid']
-          @cache[cache_key] = parse_pod_metadata(pod)
-          @stats.bump(:pod_cache_host_updates)
+    def set_up_pod_thread
+      # Any failures / exceptions in the initial setup should raise
+      # Fluent:ConfigError, so that users can inspect potential errors in
+      # the configuration.
+      pod_watcher = start_pod_watch
+      Thread.current[:pod_watch_retry_backoff_interval] = @watch_retry_interval
+      Thread.current[:pod_watch_retry_count] = 0
+
+      # Any failures / exceptions in the followup watcher notice
+      # processing will be swallowed and retried. These failures /
+      # exceptions could be caused by Kubernetes API being temporarily
+      # down. We assume the configuration is correct at this point.
+      while true
+        begin
+          pod_watcher ||= get_pods_and_start_watcher
+          process_pod_watcher_notices(pod_watcher)
+        rescue GoneError
+          # Expected error. Quietly go back through the loop in order to
+          # start watching from the latest resource versions
+          log.debug("410 Gone encountered. Restarting watch to reset resource versions.")
+          pod_watcher = nil
+        rescue Exception => e
+          @stats.bump(:pod_watch_failures)
+          if Thread.current[:pod_watch_retry_count] < @watch_retry_max_times
+            # Instead of raising exceptions and crashing Fluentd, swallow
+            # the exception and reset the watcher.
+            log.info(
+              "Exception encountered parsing pod watch event. The " \
+              "connection might have been closed. Sleeping for " \
+              "#{Thread.current[:pod_watch_retry_backoff_interval]} " \
+              "seconds and resetting the pod watcher.", e)
+            sleep(Thread.current[:pod_watch_retry_backoff_interval])
+            Thread.current[:pod_watch_retry_count] += 1
+            Thread.current[:pod_watch_retry_backoff_interval] *= @watch_retry_exponential_backoff_base
+            pod_watcher = nil
+          else
+            # Since retries failed for many times, log as errors instead
+            # of info and raise exceptions and trigger Fluentd to restart.
+            message =
+              "Exception encountered parsing pod watch event. The " \
+              "connection might have been closed. Retried " \
+              "#{@watch_retry_max_times} times yet still failing. Restarting."
+            log.error(message, e)
+            raise Fluent::UnrecoverableError.new(message)
+          end
         end
-        options[:resource_version] = pods.resourceVersion
-        watcher = @client.watch_pods(options)
-      rescue Exception => e
-        message = "Exception encountered fetching metadata from Kubernetes API endpoint: #{e.message}"
-        message += " (#{e.response})" if e.respond_to?(:response)
+      end
+    end
 
-        raise Fluent::ConfigError, message
+    def start_pod_watch
+      get_pods_and_start_watcher
+    rescue Exception => e
+      message = "start_pod_watch: Exception encountered setting up pod watch " \
+                "from Kubernetes API #{@apiVersion} endpoint " \
+                "#{@kubernetes_url}: #{e.message}"
+      message += " (#{e.response})" if e.respond_to?(:response)
+      log.debug(message)
+
+      raise Fluent::ConfigError, message
+    end
+
+    # List all pods, record the resourceVersion and return a watcher starting
+    # from that resourceVersion.
+    def get_pods_and_start_watcher
+      options = {
+        resource_version: '0'  # Fetch from API server.
+      }
+      if ENV['K8S_NODE_NAME']
+        options[:field_selector] = 'spec.nodeName=' + ENV['K8S_NODE_NAME']
+      end
+      pods = @client.get_pods(options)
+      pods.each do |pod|
+        cache_key = pod.metadata['uid']
+        @cache[cache_key] = parse_pod_metadata(pod)
+        @stats.bump(:pod_cache_host_updates)
       end
+      options[:resource_version] = pods.resourceVersion
+      watcher = @client.watch_pods(options)
+      watcher
+    end
+
+    # Reset pod watch retry count and backoff interval as there is a
+    # successful watch notice.
+    def reset_pod_watch_retry_stats
+      Thread.current[:pod_watch_retry_count] = 0
+      Thread.current[:pod_watch_retry_backoff_interval] = @watch_retry_interval
+    end
 
+    # Process a watcher notice and potentially raise an exception.
+    def process_pod_watcher_notices(watcher)
       watcher.each do |notice|
         case notice.type
           when 'MODIFIED'
+            reset_pod_watch_retry_stats
             cache_key = notice.object['metadata']['uid']
             cached    = @cache[cache_key]
             if cached
@@ -61,10 +136,21 @@ module KubernetesMetadata
               @stats.bump(:pod_cache_watch_misses)
             end
           when 'DELETED'
+            reset_pod_watch_retry_stats
             # ignore and let age out for cases where pods
             # deleted but still processing logs
             @stats.bump(:pod_cache_watch_delete_ignored)
+          when 'ERROR'
+            if notice.object && notice.object['code'] == 410
+              @stats.bump(:pod_watch_gone_notices)
+              raise GoneError
+            else
+              @stats.bump(:pod_watch_error_type_notices)
+              message = notice['object']['message'] if notice['object'] && notice['object']['message']
+              raise "Error while watching pods: #{message}"
+            end
           else
+            reset_pod_watch_retry_stats
             # Don't pay attention to creations, since the created pod may not
             # end up on this node.
             @stats.bump(:pod_cache_watch_ignored)

data/test/plugin/test_watch_namespaces.rb

@@ -25,6 +25,24 @@ class WatchNamespacesTestTest < WatchTest
      include KubernetesMetadata::WatchNamespaces
 
      setup do
+       @initial = Kubeclient::Common::EntityList.new(
+         'NamespaceList',
+         '123',
+         [
+           Kubeclient::Resource.new({
+                                      'metadata' => {
+                                        'name' => 'initial',
+                                        'uid' => 'initial_uid'
+                                      }
+                                    }),
+           Kubeclient::Resource.new({
+                                      'metadata' => {
+                                        'name' => 'modified',
+                                        'uid' => 'modified_uid'
+                                      }
+                                    })
+         ])
+
        @created = OpenStruct.new(
          type: 'CREATED',
          object: {
@@ -52,11 +70,39 @@ class WatchNamespacesTestTest < WatchTest
             }
          }
        )
+       @error = OpenStruct.new(
+         type: 'ERROR',
+         object: {
+           'message' => 'some error message'
+         }
+       )
      end
 
+    test 'namespace list caches namespaces' do
+      @client.stub :get_namespaces, @initial do
+        process_namespace_watcher_notices(start_namespace_watch)
+        assert_equal(true, @namespace_cache.key?('initial_uid'))
+        assert_equal(true, @namespace_cache.key?('modified_uid'))
+        assert_equal(2, @stats[:namespace_cache_host_updates])
+      end
+    end
+
+    test 'namespace list caches namespaces and watch updates' do
+      orig_env_val = ENV['K8S_NODE_NAME']
+      ENV['K8S_NODE_NAME'] = 'aNodeName'
+      @client.stub :get_namespaces, @initial do
+        @client.stub :watch_namespaces, [@modified] do
+          process_namespace_watcher_notices(start_namespace_watch)
+          assert_equal(2, @stats[:namespace_cache_host_updates])
+          assert_equal(1, @stats[:namespace_cache_watch_updates])
+        end
+      end
+      ENV['K8S_NODE_NAME'] = orig_env_val
+    end
+
     test 'namespace watch ignores CREATED' do
       @client.stub :watch_namespaces, [@created] do
-       start_namespace_watch
+       process_namespace_watcher_notices(start_namespace_watch)
        assert_equal(false, @namespace_cache.key?('created_uid'))
        assert_equal(1, @stats[:namespace_cache_watch_ignored])
       end
@@ -64,7 +110,7 @@ class WatchNamespacesTestTest < WatchTest
 
     test 'namespace watch ignores MODIFIED when info not in cache' do
       @client.stub :watch_namespaces, [@modified] do
-       start_namespace_watch
+       process_namespace_watcher_notices(start_namespace_watch)
        assert_equal(false, @namespace_cache.key?('modified_uid'))
        assert_equal(1, @stats[:namespace_cache_watch_misses])
       end
@@ -73,7 +119,7 @@ class WatchNamespacesTestTest < WatchTest
     test 'namespace watch updates cache when MODIFIED is received and info is cached' do
       @namespace_cache['modified_uid'] = {}
       @client.stub :watch_namespaces, [@modified] do
-       start_namespace_watch
+       process_namespace_watcher_notices(start_namespace_watch)
        assert_equal(true, @namespace_cache.key?('modified_uid'))
        assert_equal(1, @stats[:namespace_cache_watch_updates])
       end
@@ -82,10 +128,55 @@ class WatchNamespacesTestTest < WatchTest
     test 'namespace watch ignores DELETED' do
       @namespace_cache['deleted_uid'] = {}
       @client.stub :watch_namespaces, [@deleted] do
-       start_namespace_watch
+       process_namespace_watcher_notices(start_namespace_watch)
        assert_equal(true, @namespace_cache.key?('deleted_uid'))
        assert_equal(1, @stats[:namespace_cache_watch_deletes_ignored])
       end
     end
 
+    test 'namespace watch retries when exceptions are encountered' do
+      @client.stub :get_namespaces, @initial do
+        @client.stub :watch_namespaces, [[@created, @exception_raised]] do
+          assert_raise Fluent::UnrecoverableError do
+            set_up_namespace_thread
+          end
+          assert_equal(3, @stats[:namespace_watch_failures])
+          assert_equal(2, Thread.current[:namespace_watch_retry_count])
+          assert_equal(4, Thread.current[:namespace_watch_retry_backoff_interval])
+          assert_nil(@stats[:namespace_watch_error_type_notices])
+        end
+      end
+    end
+
+    test 'namespace watch retries when error is received' do
+      @client.stub :get_namespaces, @initial do
+        @client.stub :watch_namespaces, [@error] do
+          assert_raise Fluent::UnrecoverableError do
+            set_up_namespace_thread
+          end
+          assert_equal(3, @stats[:namespace_watch_failures])
+          assert_equal(2, Thread.current[:namespace_watch_retry_count])
+          assert_equal(4, Thread.current[:namespace_watch_retry_backoff_interval])
+          assert_equal(3, @stats[:namespace_watch_error_type_notices])
+        end
+      end
+    end
+
+    test 'namespace watch continues after retries succeed' do
+      @client.stub :get_namespaces, @initial do
+        @client.stub :watch_namespaces, [@modified, @error, @modified] do
+          # Force the infinite watch loop to exit after 3 seconds. Verifies that
+          # no unrecoverable error was thrown during this period of time.
+          assert_raise Timeout::Error.new('execution expired') do
+            Timeout.timeout(3) do
+              set_up_namespace_thread
+            end
+          end
+          assert_operator(@stats[:namespace_watch_failures], :>=, 3)
+          assert_operator(Thread.current[:namespace_watch_retry_count], :<=, 1)
+          assert_operator(Thread.current[:namespace_watch_retry_backoff_interval], :<=, 1)
+          assert_operator(@stats[:namespace_watch_error_type_notices], :>=, 3)
+        end
+      end
+    end
 end

data/test/plugin/test_watch_pods.rb

@@ -136,13 +136,33 @@ class DefaultPodWatchStrategyTest < WatchTest
             }
          }
        )
+       @error = OpenStruct.new(
+         type: 'ERROR',
+         object: {
+           'message' => 'some error message'
+         }
+       )
+       @gone = OpenStruct.new(
+           type: 'ERROR',
+           object: {
+               'code' => 410,
+               'kind' => 'Status',
+               'message' => 'too old resource version: 123 (391079)',
+               'metadata' => {
+                   'name' => 'gone',
+                   'namespace' => 'gone',
+                   'uid' => 'gone_uid'
+               },
+               'reason' => 'Gone'
+           }
+       )
      end
 
     test 'pod list caches pods' do
       orig_env_val = ENV['K8S_NODE_NAME']
       ENV['K8S_NODE_NAME'] = 'aNodeName'
       @client.stub :get_pods, @initial do
-        start_pod_watch
+        process_pod_watcher_notices(start_pod_watch)
         assert_equal(true, @cache.key?('initial_uid'))
         assert_equal(true, @cache.key?('modified_uid'))
         assert_equal(2, @stats[:pod_cache_host_updates])
@@ -155,7 +175,7 @@ class DefaultPodWatchStrategyTest < WatchTest
       ENV['K8S_NODE_NAME'] = 'aNodeName'
       @client.stub :get_pods, @initial do
         @client.stub :watch_pods, [@modified] do
-          start_pod_watch
+          process_pod_watcher_notices(start_pod_watch)
           assert_equal(2, @stats[:pod_cache_host_updates])
           assert_equal(1, @stats[:pod_cache_watch_updates])
         end
@@ -166,7 +186,7 @@ class DefaultPodWatchStrategyTest < WatchTest
     test 'pod watch notice ignores CREATED' do
       @client.stub :get_pods, @initial do
         @client.stub :watch_pods, [@created] do
-          start_pod_watch
+          process_pod_watcher_notices(start_pod_watch)
           assert_equal(false, @cache.key?('created_uid'))
           assert_equal(1, @stats[:pod_cache_watch_ignored])
         end
@@ -175,7 +195,7 @@ class DefaultPodWatchStrategyTest < WatchTest
 
     test 'pod watch notice is ignored when info not cached and MODIFIED is received' do
       @client.stub :watch_pods, [@modified] do
-       start_pod_watch
+       process_pod_watcher_notices(start_pod_watch)
        assert_equal(false, @cache.key?('modified_uid'))
        assert_equal(1, @stats[:pod_cache_watch_misses])
       end
@@ -185,7 +205,7 @@ class DefaultPodWatchStrategyTest < WatchTest
       orig_env_val = ENV['K8S_NODE_NAME']
       ENV['K8S_NODE_NAME'] = 'aNodeName'
       @client.stub :watch_pods, [@modified] do
-       start_pod_watch
+       process_pod_watcher_notices(start_pod_watch)
        assert_equal(true, @cache.key?('modified_uid'))
        assert_equal(1, @stats[:pod_cache_host_updates])
       end
@@ -195,7 +215,7 @@ class DefaultPodWatchStrategyTest < WatchTest
     test 'pod watch notice is updated when MODIFIED is received' do
       @cache['modified_uid'] = {}
       @client.stub :watch_pods, [@modified] do
-       start_pod_watch
+       process_pod_watcher_notices(start_pod_watch)
        assert_equal(true, @cache.key?('modified_uid'))
        assert_equal(1, @stats[:pod_cache_watch_updates])
       end
@@ -204,10 +224,65 @@ class DefaultPodWatchStrategyTest < WatchTest
     test 'pod watch notice is ignored when delete is received' do
       @cache['deleted_uid'] = {}
       @client.stub :watch_pods, [@deleted] do
-       start_pod_watch
+       process_pod_watcher_notices(start_pod_watch)
        assert_equal(true, @cache.key?('deleted_uid'))
        assert_equal(1, @stats[:pod_cache_watch_delete_ignored])
       end
     end
 
+    test 'pod watch retries when exceptions are encountered' do
+      @client.stub :get_pods, @initial do
+        @client.stub :watch_pods, [[@created, @exception_raised]] do
+          assert_raise Fluent::UnrecoverableError do
+            set_up_pod_thread
+          end
+          assert_equal(3, @stats[:pod_watch_failures])
+          assert_equal(2, Thread.current[:pod_watch_retry_count])
+          assert_equal(4, Thread.current[:pod_watch_retry_backoff_interval])
+          assert_nil(@stats[:pod_watch_error_type_notices])
+        end
+      end
+    end
+
+     test 'pod watch raises a GoneError when a 410 Gone error is received' do
+       @cache['gone_uid'] = {}
+       @client.stub :watch_pods, [@gone] do
+         assert_raise KubernetesMetadata::GoneError do
+           process_pod_watcher_notices(start_pod_watch)
+         end
+         assert_equal(1, @stats[:pod_watch_gone_notices])
+       end
+     end
+
+    test 'pod watch retries when error is received' do
+      @client.stub :get_pods, @initial do
+        @client.stub :watch_pods, [@error] do
+          assert_raise Fluent::UnrecoverableError do
+            set_up_pod_thread
+          end
+          assert_equal(3, @stats[:pod_watch_failures])
+          assert_equal(2, Thread.current[:pod_watch_retry_count])
+          assert_equal(4, Thread.current[:pod_watch_retry_backoff_interval])
+          assert_equal(3, @stats[:pod_watch_error_type_notices])
+        end
+      end
+    end
+
+    test 'pod watch continues after retries succeed' do
+      @client.stub :get_pods, @initial do
+        @client.stub :watch_pods, [@modified, @error, @modified] do
+          # Force the infinite watch loop to exit after 3 seconds. Verifies that
+          # no unrecoverable error was thrown during this period of time.
+          assert_raise Timeout::Error.new('execution expired') do
+            Timeout.timeout(3) do
+              set_up_pod_thread
+            end
+          end
+          assert_operator(@stats[:pod_watch_failures], :>=, 3)
+          assert_operator(Thread.current[:pod_watch_retry_count], :<=, 1)
+          assert_operator(Thread.current[:pod_watch_retry_backoff_interval], :<=, 1)
+          assert_operator(@stats[:pod_watch_error_type_notices], :>=, 3)
+        end
+      end
+    end
 end

data/test/plugin/watch_test.rb

@@ -20,38 +20,56 @@ require_relative '../helper'
 require 'ostruct'
 
 class WatchTest < Test::Unit::TestCase
-   
-    setup do
-      @annotations_regexps = []
-      @namespace_cache = {}
-      @cache = {}
-      @stats = KubernetesMetadata::Stats.new
-      @client = OpenStruct.new
-      def @client.resourceVersion
-        '12345'
-      end
-      def @client.watch_pods(options = {})
-         []
-      end
-      def @client.watch_namespaces(options = {})
-         []
-      end
-      def @client.get_namespaces(options = {})
-          self
-      end
-      def @client.get_pods(options = {})
-          self
-      end
-    end
 
-    def watcher=(value)
+  def thread_current_running?
+    true
+  end
+
+  setup do
+    @annotations_regexps = []
+    @namespace_cache = {}
+    @watch_retry_max_times = 2
+    @watch_retry_interval = 1
+    @watch_retry_exponential_backoff_base = 2
+    @cache = {}
+    @stats = KubernetesMetadata::Stats.new
+    Thread.current[:pod_watch_retry_count] = 0
+    Thread.current[:namespace_watch_retry_count] = 0
+
+    @client = OpenStruct.new
+    def @client.resourceVersion
+      '12345'
+    end
+    def @client.watch_pods(options = {})
+      []
+    end
+    def @client.watch_namespaces(options = {})
+      []
+    end
+    def @client.get_namespaces(options = {})
+      self
+    end
+    def @client.get_pods(options = {})
+      self
     end
 
-    def log
-        logger = {}
-        def logger.debug(message)
-        end
-        logger
+    @exception_raised = OpenStruct.new
+    def @exception_raised.each
+      raise Exception
     end
+  end
+
+  def watcher=(value)
+  end
 
+  def log
+    logger = {}
+    def logger.debug(message)
+    end
+    def logger.info(message, error)
+    end
+    def logger.error(message, error)
+    end
+    logger
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 2.4.1
+  version: 2.4.7
 platform: ruby
 authors:
 - Jimmi Dyson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-09 00:00:00.000000000 Z
+date: 2020-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -19,7 +19,7 @@ dependencies:
         version: 0.14.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '1.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -29,7 +29,7 @@ dependencies:
         version: 0.14.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '1.10'
 - !ruby/object:Gem::Dependency
   name: lru_redux
   requirement: !ruby/object:Gem::Requirement
@@ -64,14 +64,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.2
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.2
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -208,6 +208,7 @@ files:
 - ".circleci/config.yml"
 - ".gitignore"
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -258,7 +259,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Fluentd filter plugin to add Kubernetes metadata