fluent-plugin-kubernetes_metadata_filter 2.5.0 → 3.2.0

data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #
 # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
 # Kubernetes metadata
@@ -16,11 +18,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+# TODO: this is mostly copy-paste from kubernetes_metadata_watch_namespaces.rb unify them
 require_relative 'kubernetes_metadata_common'
 
 module KubernetesMetadata
   module WatchPods
-
     include ::KubernetesMetadata::Common
 
     def set_up_pod_thread
@@ -28,6 +30,7 @@ module KubernetesMetadata
       # Fluent:ConfigError, so that users can inspect potential errors in
       # the configuration.
       pod_watcher = start_pod_watch
+
       Thread.current[:pod_watch_retry_backoff_interval] = @watch_retry_interval
       Thread.current[:pod_watch_retry_count] = 0
 
@@ -35,20 +38,33 @@ module KubernetesMetadata
       # processing will be swallowed and retried. These failures /
       # exceptions could be caused by Kubernetes API being temporarily
       # down. We assume the configuration is correct at this point.
-      while true
-        begin
-          pod_watcher ||= get_pods_and_start_watcher
-          process_pod_watcher_notices(pod_watcher)
-        rescue Exception => e
+      loop do
+        pod_watcher ||= get_pods_and_start_watcher
+        process_pod_watcher_notices(pod_watcher)
+      rescue GoneError => e
+        # Expected error. Quietly go back through the loop in order to
+        # start watching from the latest resource versions
+        @stats.bump(:pod_watch_gone_errors)
+        log.info('410 Gone encountered. Restarting pod watch to reset resource versions.', e)
+        pod_watcher = nil
+      rescue KubeException => e
+        if e.error_code == 401
+          # recreate client to refresh token
+          log.info("Encountered '401 Unauthorized' exception in watch, recreating client to refresh token")
+          create_client()
+          pod_watcher = nil
+        else
+          # treat all other errors the same as StandardError, log, swallow and reset
           @stats.bump(:pod_watch_failures)
           if Thread.current[:pod_watch_retry_count] < @watch_retry_max_times
             # Instead of raising exceptions and crashing Fluentd, swallow
             # the exception and reset the watcher.
             log.info(
-              "Exception encountered parsing pod watch event. The " \
-              "connection might have been closed. Sleeping for " \
+              'Exception encountered parsing pod watch event. The ' \
+              'connection might have been closed. Sleeping for ' \
               "#{Thread.current[:pod_watch_retry_backoff_interval]} " \
-              "seconds and resetting the pod watcher.", e)
+              'seconds and resetting the pod watcher.', e
+            )
             sleep(Thread.current[:pod_watch_retry_backoff_interval])
             Thread.current[:pod_watch_retry_count] += 1
             Thread.current[:pod_watch_retry_backoff_interval] *= @watch_retry_exponential_backoff_base
@@ -57,20 +73,45 @@ module KubernetesMetadata
             # Since retries failed for many times, log as errors instead
             # of info and raise exceptions and trigger Fluentd to restart.
             message =
-              "Exception encountered parsing pod watch event. The " \
-              "connection might have been closed. Retried " \
+              'Exception encountered parsing pod watch event. The ' \
+              'connection might have been closed. Retried ' \
               "#{@watch_retry_max_times} times yet still failing. Restarting."
             log.error(message, e)
-            raise Fluent::UnrecoverableError.new(message)
+            raise Fluent::UnrecoverableError, message
           end
         end
+      rescue StandardError => e
+        @stats.bump(:pod_watch_failures)
+        if Thread.current[:pod_watch_retry_count] < @watch_retry_max_times
+          # Instead of raising exceptions and crashing Fluentd, swallow
+          # the exception and reset the watcher.
+          log.info(
+            'Exception encountered parsing pod watch event. The ' \
+            'connection might have been closed. Sleeping for ' \
+            "#{Thread.current[:pod_watch_retry_backoff_interval]} " \
+            'seconds and resetting the pod watcher.', e
+          )
+          sleep(Thread.current[:pod_watch_retry_backoff_interval])
+          Thread.current[:pod_watch_retry_count] += 1
+          Thread.current[:pod_watch_retry_backoff_interval] *= @watch_retry_exponential_backoff_base
+          pod_watcher = nil
+        else
+          # Since retries failed for many times, log as errors instead
+          # of info and raise exceptions and trigger Fluentd to restart.
+          message =
+            'Exception encountered parsing pod watch event. The ' \
+            'connection might have been closed. Retried ' \
+            "#{@watch_retry_max_times} times yet still failing. Restarting."
+          log.error(message, e)
+          raise Fluent::UnrecoverableError, message
+        end
       end
     end
 
     def start_pod_watch
       get_pods_and_start_watcher
-    rescue Exception => e
-      message = "start_pod_watch: Exception encountered setting up pod watch " \
+    rescue StandardError => e
+      message = 'start_pod_watch: Exception encountered setting up pod watch ' \
                 "from Kubernetes API #{@apiVersion} endpoint " \
                 "#{@kubernetes_url}: #{e.message}"
       message += " (#{e.response})" if e.respond_to?(:response)
@@ -83,19 +124,27 @@ module KubernetesMetadata
     # from that resourceVersion.
     def get_pods_and_start_watcher
       options = {
-        resource_version: '0'  # Fetch from API server.
+        resource_version: '0' # Fetch from API server cache instead of etcd quorum read
       }
       if ENV['K8S_NODE_NAME']
         options[:field_selector] = 'spec.nodeName=' + ENV['K8S_NODE_NAME']
       end
-      pods = @client.get_pods(options)
-      pods.each do |pod|
-        cache_key = pod.metadata['uid']
-        @cache[cache_key] = parse_pod_metadata(pod)
-        @stats.bump(:pod_cache_host_updates)
+      if @last_seen_resource_version
+        options[:resource_version] = @last_seen_resource_version
+      else
+        pods = @client.get_pods(options)
+        pods[:items].each do |pod|
+          cache_key = pod[:metadata][:uid]
+          @cache[cache_key] = parse_pod_metadata(pod)
+          @stats.bump(:pod_cache_host_updates)
+        end
+
+        # continue watching from most recent resourceVersion
+        options[:resource_version] = pods[:metadata][:resourceVersion]
       end
-      options[:resource_version] = pods.resourceVersion
+
       watcher = @client.watch_pods(options)
+      reset_pod_watch_retry_stats
       watcher
     end
 
@@ -109,34 +158,46 @@ module KubernetesMetadata
     # Process a watcher notice and potentially raise an exception.
     def process_pod_watcher_notices(watcher)
       watcher.each do |notice|
-        case notice.type
-          when 'MODIFIED'
-            reset_pod_watch_retry_stats
-            cache_key = notice.object['metadata']['uid']
-            cached    = @cache[cache_key]
-            if cached
-              @cache[cache_key] = parse_pod_metadata(notice.object)
-              @stats.bump(:pod_cache_watch_updates)
-            elsif ENV['K8S_NODE_NAME'] == notice.object['spec']['nodeName'] then
-              @cache[cache_key] = parse_pod_metadata(notice.object)
-              @stats.bump(:pod_cache_host_updates)
-            else
-              @stats.bump(:pod_cache_watch_misses)
-            end
-          when 'DELETED'
-            reset_pod_watch_retry_stats
-            # ignore and let age out for cases where pods
-            # deleted but still processing logs
-            @stats.bump(:pod_cache_watch_delete_ignored)
-          when 'ERROR'
+        # store version we processed to not reprocess it ... do not unset when there is no version in response
+        version = ( # TODO: replace with &.dig once we are on ruby 2.5+
+          notice[:object] && notice[:object][:metadata] && notice[:object][:metadata][:resourceVersion]
+        )
+        @last_seen_resource_version = version if version
+
+        case notice[:type]
+        when 'MODIFIED'
+          reset_pod_watch_retry_stats
+          cache_key = notice.dig(:object, :metadata, :uid)
+          cached    = @cache[cache_key]
+          if cached
+            @cache[cache_key] = parse_pod_metadata(notice[:object])
+            @stats.bump(:pod_cache_watch_updates)
+          elsif ENV['K8S_NODE_NAME'] == notice[:object][:spec][:nodeName]
+            @cache[cache_key] = parse_pod_metadata(notice[:object])
+            @stats.bump(:pod_cache_host_updates)
+          else
+            @stats.bump(:pod_cache_watch_misses)
+          end
+        when 'DELETED'
+          reset_pod_watch_retry_stats
+          # ignore and let age out for cases where pods
+          # deleted but still processing logs
+          @stats.bump(:pod_cache_watch_delete_ignored)
+        when 'ERROR'
+          if notice[:object] && notice[:object][:code] == 410
+            @last_seen_resource_version = nil # requested resourceVersion was too old, need to reset
+            @stats.bump(:pod_watch_gone_notices)
+            raise GoneError
+          else
             @stats.bump(:pod_watch_error_type_notices)
-            message = notice['object']['message'] if notice['object'] && notice['object']['message']
+            message = notice[:object][:message] if notice[:object] && notice[:object][:message]
             raise "Error while watching pods: #{message}"
-          else
-            reset_pod_watch_retry_stats
-            # Don't pay attention to creations, since the created pod may not
-            # end up on this node.
-            @stats.bump(:pod_cache_watch_ignored)
+          end
+        else
+          reset_pod_watch_retry_stats
+          # Don't pay attention to creations, since the created pod may not
+          # end up on this node.
+          @stats.bump(:pod_cache_watch_ignored)
         end
       end
     end

data/release_notes.md

@@ -0,0 +1,42 @@
+# Release Notes
+
+## 2.9.4
+As of this release, the 'de_dot' functionality is depricated and will be removed in future releases. 
+Ref: https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/issues/320
+
+## v2.1.4
+The use of `use_journal` is **DEPRECATED**.  If this setting is not present, the plugin will
+attempt to figure out the source of the metadata fields from the following:
+- If `lookup_from_k8s_field true` (the default) and the following fields are present in the record:
+`docker.container_id`, `kubernetes.namespace_name`, `kubernetes.pod_name`, `kubernetes.container_name`,
+then the plugin will use those values as the source to use to lookup the metadata
+- If `use_journal true`, or `use_journal` is unset, and the fields `CONTAINER_NAME` and `CONTAINER_ID_FULL` are present in the record,
+then the plugin will parse those values using `container_name_to_kubernetes_regexp` and use those as the source to lookup the metadata
+- Otherwise, if the tag matches `tag_to_kubernetes_name_regexp`, the plugin will parse the tag and use those values to
+lookup the metdata
+
+## v2.1.x
+
+As of the release 2.1.x of this plugin, it no longer supports parsing the source message into JSON and attaching it to the
+payload.  The following configuration options are removed:
+
+* `merge_json_log`
+* `preserve_json_log`
+
+One way of preserving JSON logs can be through the [parser plugin](https://docs.fluentd.org/filter/parser).
+It can parsed with the parser plugin like this:
+
+```
+<filter kubernetes.**>
+  @type parser
+  key_name log
+  <parse>
+    @type json
+    json_parser json
+  </parse>
+  replace_invalid_sequence true
+  reserve_data true # this preserves unparsable log lines
+  emit_invalid_record_to_error false # In case of unparsable log lines keep the error log clean
+  reserve_time # the time was already parsed in the source, we don't want to overwrite it with current time.
+</filter>
+```

data/test/cassettes/kubernetes_get_pod_container_init.yml

@@ -0,0 +1,145 @@
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Fri, 08 May 2015 10:35:37 GMT
+      Transfer-Encoding:
+      - chunked
+    body:
+      encoding: UTF-8
+      string: |-
+        {
+          "kind": "Pod",
+          "apiVersion": "v1",
+          "metadata": {
+            "name": "fabric8-console-controller-98rqc",
+            "generateName": "fabric8-console-controller-",
+            "namespace": "default",
+            "selfLink": "/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc",
+            "uid": "c76927af-f563-11e4-b32d-54ee7527188d",
+            "resourceVersion": "122",
+            "creationTimestamp": "2015-05-08T09:22:42Z",
+            "labels": {
+              "component": "fabric8Console"
+            }
+          },
+          "spec": {
+            "volumes": [
+              {
+                "name": "openshift-cert-secrets",
+                "hostPath": null,
+                "emptyDir": null,
+                "gcePersistentDisk": null,
+                "gitRepo": null,
+                "secret": {
+                  "secretName": "openshift-cert-secrets"
+                },
+                "nfs": null,
+                "iscsi": null,
+                "glusterfs": null
+              }
+            ],
+            "containers": [
+              {
+                "name": "fabric8-console-container",
+                "image": "fabric8/hawtio-kubernetes:latest",
+                "ports": [
+                  {
+                    "containerPort": 9090,
+                    "protocol": "TCP"
+                  }
+                ],
+                "env": [
+                  {
+                    "name": "OAUTH_CLIENT_ID",
+                    "value": "fabric8-console"
+                  },
+                  {
+                    "name": "OAUTH_AUTHORIZE_URI",
+                    "value": "https://localhost:8443/oauth/authorize"
+                  }
+                ],
+                "resources": {},
+                "volumeMounts": [
+                  {
+                    "name": "openshift-cert-secrets",
+                    "readOnly": true,
+                    "mountPath": "/etc/secret-volume"
+                  }
+                ],
+                "terminationMessagePath": "/dev/termination-log",
+                "imagePullPolicy": "IfNotPresent",
+                "capabilities": {}
+              }
+            ],
+            "restartPolicy": "Always",
+            "dnsPolicy": "ClusterFirst",
+            "nodeName": "jimmi-redhat.localnet"
+          },
+          "status": {
+            "phase": "Running",
+            "Condition": [
+              {
+                "type": "Ready",
+                "status": "True"
+              }
+            ],
+            "hostIP": "172.17.42.1",
+            "podIP": "172.17.0.8",
+            "containerStatuses": [
+              {
+                "name": "fabric8-console-container",
+                "state": {
+                  "waiting": {
+                    "reason": "ContainerCreating"
+                  }
+                },
+                "lastState": {},
+                "ready": true,
+                "restartCount": 0,
+                "image": "fabric8/hawtio-kubernetes:latest",
+                "imageID": ""
+              }
+            ]
+          }
+        }
+    http_version: 
+  recorded_at: Fri, 08 May 2015 10:35:37 GMT
+recorded_with: VCR 2.9.3

data/test/helper.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #
 # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
 # Kubernetes metadata
@@ -16,6 +18,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+require 'bundler/setup'
 require 'codeclimate-test-reporter'
 SimpleCov.start do
   formatter SimpleCov::Formatter::MultiFormatter.new [
@@ -31,8 +34,14 @@ require 'fileutils'
 require 'fluent/log'
 require 'fluent/test'
 require 'minitest/autorun'
-require 'webmock/test_unit'
 require 'vcr'
+require 'ostruct'
+require 'fluent/plugin/filter_kubernetes_metadata'
+require 'fluent/test/driver/filter'
+require 'kubeclient'
+
+require 'webmock/test_unit'
+WebMock.disable_net_connect!
 
 VCR.configure do |config|
   config.cassette_library_dir = 'test/cassettes'
@@ -58,7 +67,16 @@ def ipv6_enabled?
   begin
     TCPServer.open('::1', 0)
     true
-  rescue
+  rescue StandardError
     false
   end
 end
+
+# TEST_NAME='foo' ruby test_file.rb to run a single test case
+if ENV['TEST_NAME']
+  (class << Test::Unit::TestCase; self; end).prepend(Module.new do
+    def test(name)
+      super if name == ENV['TEST_NAME']
+    end
+  end)
+end

data/test/plugin/test_cache_stats.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #
 # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
 # Kubernetes metadata
@@ -17,20 +19,15 @@
 # limitations under the License.
 #
 require_relative '../helper'
-require 'fluent/plugin/kubernetes_metadata_stats'
-require 'webmock/test_unit'
-WebMock.disable_net_connect!
 
 class KubernetesMetadataCacheStatsTest < Test::Unit::TestCase
-  
-    test 'watch stats' do
-      require 'lru_redux'
-      stats = KubernetesMetadata::Stats.new
-      stats.bump(:missed)
-      stats.bump(:deleted)
-      stats.bump(:deleted)
+  test 'watch stats' do
+    require 'lru_redux'
+    stats = KubernetesMetadata::Stats.new
+    stats.bump(:missed)
+    stats.bump(:deleted)
+    stats.bump(:deleted)
 
-      assert_equal("stats - deleted: 2, missed: 1", stats.to_s)
-    end
-    
+    assert_equal('stats - deleted: 2, missed: 1', stats.to_s)
+  end
 end