fluent-plugin-kubernetes_metadata_filter 2.1.0 → 3.4.0

data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #
 # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
 # Kubernetes metadata
@@ -16,46 +18,186 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+# TODO: this is mostly copy-paste from kubernetes_metadata_watch_namespaces.rb unify them
 require_relative 'kubernetes_metadata_common'
 
 module KubernetesMetadata
   module WatchPods
-
     include ::KubernetesMetadata::Common
 
+    def set_up_pod_thread
+      # Any failures / exceptions in the initial setup should raise
+      # Fluent:ConfigError, so that users can inspect potential errors in
+      # the configuration.
+      pod_watcher = start_pod_watch
+
+      Thread.current[:pod_watch_retry_backoff_interval] = @watch_retry_interval
+      Thread.current[:pod_watch_retry_count] = 0
+
+      # Any failures / exceptions in the followup watcher notice
+      # processing will be swallowed and retried. These failures /
+      # exceptions could be caused by Kubernetes API being temporarily
+      # down. We assume the configuration is correct at this point.
+      loop do
+        pod_watcher ||= get_pods_and_start_watcher
+        process_pod_watcher_notices(pod_watcher)
+      rescue GoneError => e
+        # Expected error. Quietly go back through the loop in order to
+        # start watching from the latest resource versions
+        @stats.bump(:pod_watch_gone_errors)
+        log.info('410 Gone encountered. Restarting pod watch to reset resource versions.', e)
+        pod_watcher = nil
+      rescue KubeException => e
+        if e.error_code == 401
+          # recreate client to refresh token
+          log.info("Encountered '401 Unauthorized' exception in watch, recreating client to refresh token")
+          create_client()
+          pod_watcher = nil
+        else
+          # treat all other errors the same as StandardError, log, swallow and reset
+          @stats.bump(:pod_watch_failures)
+          if Thread.current[:pod_watch_retry_count] < @watch_retry_max_times
+            # Instead of raising exceptions and crashing Fluentd, swallow
+            # the exception and reset the watcher.
+            log.info(
+              'Exception encountered parsing pod watch event. The ' \
+              'connection might have been closed. Sleeping for ' \
+              "#{Thread.current[:pod_watch_retry_backoff_interval]} " \
+              'seconds and resetting the pod watcher.', e
+            )
+            sleep(Thread.current[:pod_watch_retry_backoff_interval])
+            Thread.current[:pod_watch_retry_count] += 1
+            Thread.current[:pod_watch_retry_backoff_interval] *= @watch_retry_exponential_backoff_base
+            pod_watcher = nil
+          else
+            # Since retries failed for many times, log as errors instead
+            # of info and raise exceptions and trigger Fluentd to restart.
+            message =
+              'Exception encountered parsing pod watch event. The ' \
+              'connection might have been closed. Retried ' \
+              "#{@watch_retry_max_times} times yet still failing. Restarting."
+            log.error(message, e)
+            raise Fluent::UnrecoverableError, message
+          end
+        end
+      rescue StandardError => e
+        @stats.bump(:pod_watch_failures)
+        if Thread.current[:pod_watch_retry_count] < @watch_retry_max_times
+          # Instead of raising exceptions and crashing Fluentd, swallow
+          # the exception and reset the watcher.
+          log.info(
+            'Exception encountered parsing pod watch event. The ' \
+            'connection might have been closed. Sleeping for ' \
+            "#{Thread.current[:pod_watch_retry_backoff_interval]} " \
+            'seconds and resetting the pod watcher.', e
+          )
+          sleep(Thread.current[:pod_watch_retry_backoff_interval])
+          Thread.current[:pod_watch_retry_count] += 1
+          Thread.current[:pod_watch_retry_backoff_interval] *= @watch_retry_exponential_backoff_base
+          pod_watcher = nil
+        else
+          # Since retries failed for many times, log as errors instead
+          # of info and raise exceptions and trigger Fluentd to restart.
+          message =
+            'Exception encountered parsing pod watch event. The ' \
+            'connection might have been closed. Retried ' \
+            "#{@watch_retry_max_times} times yet still failing. Restarting."
+          log.error(message, e)
+          raise Fluent::UnrecoverableError, message
+        end
+      end
+    end
+
     def start_pod_watch
-      begin
-        resource_version = @client.get_pods.resourceVersion
-        watcher          = @client.watch_pods(resource_version)
-      rescue Exception => e
-        message = "Exception encountered fetching metadata from Kubernetes API endpoint: #{e.message}"
-        message += " (#{e.response})" if e.respond_to?(:response)
-
-        raise Fluent::ConfigError, message
+      get_pods_and_start_watcher
+    rescue StandardError => e
+      message = 'start_pod_watch: Exception encountered setting up pod watch ' \
+                "from Kubernetes API #{@apiVersion} endpoint " \
+                "#{@kubernetes_url}: #{e.message}"
+      message += " (#{e.response})" if e.respond_to?(:response)
+      log.debug(message)
+
+      raise Fluent::ConfigError, message
+    end
+
+    # List all pods, record the resourceVersion and return a watcher starting
+    # from that resourceVersion.
+    def get_pods_and_start_watcher
+      options = {
+        resource_version: '0' # Fetch from API server cache instead of etcd quorum read
+      }
+      if ENV['K8S_NODE_NAME']
+        options[:field_selector] = 'spec.nodeName=' + ENV['K8S_NODE_NAME']
+      end
+      if @last_seen_resource_version
+        options[:resource_version] = @last_seen_resource_version
+      else
+        pods = @client.get_pods(options)
+        pods[:items].each do |pod|
+          cache_key = pod[:metadata][:uid]
+          @cache[cache_key] = parse_pod_metadata(pod)
+          @stats.bump(:pod_cache_host_updates)
+        end
+
+        # continue watching from most recent resourceVersion
+        options[:resource_version] = pods[:metadata][:resourceVersion]
       end
 
+      watcher = @client.watch_pods(options)
+      reset_pod_watch_retry_stats
+      watcher
+    end
+
+    # Reset pod watch retry count and backoff interval as there is a
+    # successful watch notice.
+    def reset_pod_watch_retry_stats
+      Thread.current[:pod_watch_retry_count] = 0
+      Thread.current[:pod_watch_retry_backoff_interval] = @watch_retry_interval
+    end
+
+    # Process a watcher notice and potentially raise an exception.
+    def process_pod_watcher_notices(watcher)
       watcher.each do |notice|
-        case notice.type
-          when 'MODIFIED'
-            cache_key = notice.object['metadata']['uid']
-            cached    = @cache[cache_key]
-            if cached
-              @cache[cache_key] = parse_pod_metadata(notice.object)
-              @stats.bump(:pod_cache_watch_updates)
-            elsif ENV['K8S_NODE_NAME'] == notice.object['spec']['nodeName'] then
-              @cache[cache_key] = parse_pod_metadata(notice.object)
-              @stats.bump(:pod_cache_host_updates)
-            else
-              @stats.bump(:pod_cache_watch_misses)
-            end
-          when 'DELETED'
-            # ignore and let age out for cases where pods
-            # deleted but still processing logs
-            @stats.bump(:pod_cache_watch_delete_ignored)
+        # store version we processed to not reprocess it ... do not unset when there is no version in response
+        version = ( # TODO: replace with &.dig once we are on ruby 2.5+
+          notice[:object] && notice[:object][:metadata] && notice[:object][:metadata][:resourceVersion]
+        )
+        @last_seen_resource_version = version if version
+
+        case notice[:type]
+        when 'MODIFIED'
+          reset_pod_watch_retry_stats
+          cache_key = notice.dig(:object, :metadata, :uid)
+          cached    = @cache[cache_key]
+          if cached
+            @cache[cache_key] = parse_pod_metadata(notice[:object])
+            @stats.bump(:pod_cache_watch_updates)
+          elsif ENV['K8S_NODE_NAME'] == notice[:object][:spec][:nodeName]
+            @cache[cache_key] = parse_pod_metadata(notice[:object])
+            @stats.bump(:pod_cache_host_updates)
+          else
+            @stats.bump(:pod_cache_watch_misses)
+          end
+        when 'DELETED'
+          reset_pod_watch_retry_stats
+          # ignore and let age out for cases where pods
+          # deleted but still processing logs
+          @stats.bump(:pod_cache_watch_delete_ignored)
+        when 'ERROR'
+          if notice[:object] && notice[:object][:code] == 410
+            @last_seen_resource_version = nil # requested resourceVersion was too old, need to reset
+            @stats.bump(:pod_watch_gone_notices)
+            raise GoneError
           else
-            # Don't pay attention to creations, since the created pod may not
-            # end up on this node.
-            @stats.bump(:pod_cache_watch_ignored)
+            @stats.bump(:pod_watch_error_type_notices)
+            message = notice[:object][:message] if notice[:object] && notice[:object][:message]
+            raise "Error while watching pods: #{message}"
+          end
+        else
+          reset_pod_watch_retry_stats
+          # Don't pay attention to creations, since the created pod may not
+          # end up on this node.
+          @stats.bump(:pod_cache_watch_ignored)
         end
       end
     end

data/release_notes.md

@@ -0,0 +1,42 @@
+# Release Notes
+
+## 2.9.4
+As of this release, the 'de_dot' functionality is depricated and will be removed in future releases. 
+Ref: https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/issues/320
+
+## v2.1.4
+The use of `use_journal` is **DEPRECATED**.  If this setting is not present, the plugin will
+attempt to figure out the source of the metadata fields from the following:
+- If `lookup_from_k8s_field true` (the default) and the following fields are present in the record:
+`docker.container_id`, `kubernetes.namespace_name`, `kubernetes.pod_name`, `kubernetes.container_name`,
+then the plugin will use those values as the source to use to lookup the metadata
+- If `use_journal true`, or `use_journal` is unset, and the fields `CONTAINER_NAME` and `CONTAINER_ID_FULL` are present in the record,
+then the plugin will parse those values using `container_name_to_kubernetes_regexp` and use those as the source to lookup the metadata
+- Otherwise, if the tag matches `tag_to_kubernetes_name_regexp`, the plugin will parse the tag and use those values to
+lookup the metdata
+
+## v2.1.x
+
+As of the release 2.1.x of this plugin, it no longer supports parsing the source message into JSON and attaching it to the
+payload.  The following configuration options are removed:
+
+* `merge_json_log`
+* `preserve_json_log`
+
+One way of preserving JSON logs can be through the [parser plugin](https://docs.fluentd.org/filter/parser).
+It can parsed with the parser plugin like this:
+
+```
+<filter kubernetes.**>
+  @type parser
+  key_name log
+  <parse>
+    @type json
+    json_parser json
+  </parse>
+  replace_invalid_sequence true
+  reserve_data true # this preserves unparsable log lines
+  emit_invalid_record_to_error false # In case of unparsable log lines keep the error log clean
+  reserve_time # the time was already parsed in the source, we don't want to overwrite it with current time.
+</filter>
+```

data/test/cassettes/kubernetes_docker_metadata_annotations.yml

@@ -18,40 +18,6 @@
 #
 ---
 http_interactions:
-- request:
-    method: get
-    uri: https://localhost:8443/api
-    body:
-      encoding: US-ASCII
-      string: ''
-    headers:
-      Accept:
-      - "*/*; q=0.5, application/xml"
-      Accept-Encoding:
-      - gzip, deflate
-      User-Agent:
-      - Ruby
-  response:
-    status:
-      code: 200
-      message: OK
-    headers:
-      Content-Type:
-      - application/json
-      Date:
-      - Fri, 08 May 2015 10:35:37 GMT
-      Content-Length:
-      - '67'
-    body:
-      encoding: UTF-8
-      string: |-
-        {
-          "versions": [
-            "v1"
-          ]
-        }
-    http_version: 
-  recorded_at: Fri, 08 May 2015 10:35:37 GMT
 - request:
     method: get
     uri: https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc

data/test/cassettes/{kubernetes_docker_metadata_dotted_labels.yml → kubernetes_docker_metadata_dotted_slashed_labels.yml}

@@ -18,40 +18,6 @@
 #
 ---
 http_interactions:
-- request:
-    method: get
-    uri: https://localhost:8443/api
-    body:
-      encoding: US-ASCII
-      string: ''
-    headers:
-      Accept:
-      - "*/*; q=0.5, application/xml"
-      Accept-Encoding:
-      - gzip, deflate
-      User-Agent:
-      - Ruby
-  response:
-    status:
-      code: 200
-      message: OK
-    headers:
-      Content-Type:
-      - application/json
-      Date:
-      - Fri, 08 May 2015 10:35:37 GMT
-      Content-Length:
-      - '67'
-    body:
-      encoding: UTF-8
-      string: |-
-        {
-          "versions": [
-            "v1"
-          ]
-        }
-    http_version: 
-  recorded_at: Fri, 08 May 2015 10:35:37 GMT
 - request:
     method: get
     uri: https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc

data/test/cassettes/kubernetes_get_api_v1.yml

@@ -0,0 +1,193 @@
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://localhost:8443/api/v1
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Fri, 08 May 2015 10:35:37 GMT
+      Transfer-Encoding:
+      - chunked
+    body:
+      encoding: UTF-8
+      string: |-
+        {
+          "kind": "APIResourceList",
+          "groupVersion": "v1",
+          "resources": [
+            {
+              "name": "bindings",
+              "singularName": "",
+              "namespaced": true,
+              "kind": "Binding",
+              "verbs": [
+                "create"
+              ]
+            },
+            {
+              "name": "namespaces",
+              "singularName": "",
+              "namespaced": false,
+              "kind": "Namespace",
+              "verbs": [
+                "create",
+                "delete",
+                "get",
+                "list",
+                "patch",
+                "update",
+                "watch"
+              ],
+              "shortNames": [
+                "ns"
+              ]
+            },
+            {
+              "name": "namespaces/finalize",
+              "singularName": "",
+              "namespaced": false,
+              "kind": "Namespace",
+              "verbs": [
+                "update"
+              ]
+            },
+            {
+              "name": "namespaces/status",
+              "singularName": "",
+              "namespaced": false,
+              "kind": "Namespace",
+              "verbs": [
+                "get",
+                "patch",
+                "update"
+              ]
+            },
+            {
+              "name": "pods",
+              "singularName": "",
+              "namespaced": true,
+              "kind": "Pod",
+              "verbs": [
+                "create",
+                "delete",
+                "deletecollection",
+                "get",
+                "list",
+                "patch",
+                "update",
+                "watch"
+              ],
+              "shortNames": [
+                "po"
+              ],
+              "categories": [
+                "all"
+              ]
+            },
+            {
+              "name": "pods/attach",
+              "singularName": "",
+              "namespaced": true,
+              "kind": "Pod",
+              "verbs": []
+            },
+            {
+              "name": "pods/binding",
+              "singularName": "",
+              "namespaced": true,
+              "kind": "Binding",
+              "verbs": [
+                "create"
+              ]
+            },
+            {
+              "name": "pods/eviction",
+              "singularName": "",
+              "namespaced": true,
+              "group": "policy",
+              "version": "v1beta1",
+              "kind": "Eviction",
+              "verbs": [
+                "create"
+              ]
+            },
+            {
+              "name": "pods/exec",
+              "singularName": "",
+              "namespaced": true,
+              "kind": "Pod",
+              "verbs": []
+            },
+            {
+              "name": "pods/log",
+              "singularName": "",
+              "namespaced": true,
+              "kind": "Pod",
+              "verbs": [
+                "get"
+              ]
+            },
+            {
+              "name": "pods/portforward",
+              "singularName": "",
+              "namespaced": true,
+              "kind": "Pod",
+              "verbs": []
+            },
+            {
+              "name": "pods/proxy",
+              "singularName": "",
+              "namespaced": true,
+              "kind": "Pod",
+              "verbs": []
+            },
+            {
+              "name": "pods/status",
+              "singularName": "",
+              "namespaced": true,
+              "kind": "Pod",
+              "verbs": [
+                "get",
+                "patch",
+                "update"
+              ]
+            }
+          ]
+        }
+    http_version: 
+  recorded_at: Fri, 08 May 2015 10:35:37 GMT
+recorded_with: VCR 2.9.3