fluent-plugin-kubernetes_metadata_filter_fix 2.2.0.pre.1

data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb

@@ -0,0 +1,98 @@
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module KubernetesMetadata
+  module CacheStrategy
+    def get_pod_metadata(key, namespace_name, pod_name, record_create_time, batch_miss_cache)
+      metadata = {}
+      ids = @id_cache[key]
+      if !ids.nil?
+        # FAST PATH
+        # Cache hit, fetch metadata from the cache
+        metadata = @cache.fetch(ids[:pod_id]) do
+          @stats.bump(:pod_cache_miss)
+          m = fetch_pod_metadata(namespace_name, pod_name)
+          (m.nil? || m.empty?) ? {'pod_id'=>ids[:pod_id]} : m
+        end
+        metadata.merge!(@namespace_cache.fetch(ids[:namespace_id]) do
+          @stats.bump(:namespace_cache_miss)
+          m = fetch_namespace_metadata(namespace_name) unless @skip_namespace_metadata
+          (m.nil? || m.empty?) ?  {'namespace_id'=>ids[:namespace_id]} : m
+        end)
+      else
+        # SLOW PATH
+        @stats.bump(:id_cache_miss)
+        return batch_miss_cache["#{namespace_name}_#{pod_name}"] if batch_miss_cache.key?("#{namespace_name}_#{pod_name}")
+        pod_metadata = fetch_pod_metadata(namespace_name, pod_name)
+        if @skip_namespace_metadata
+          ids = { :pod_id=> pod_metadata['pod_id'] }
+          @id_cache[key] = ids
+          return pod_metadata
+        end
+        namespace_metadata = fetch_namespace_metadata(namespace_name)
+        ids = { :pod_id=> pod_metadata['pod_id'], :namespace_id => namespace_metadata['namespace_id'] }
+        if !ids[:pod_id].nil? && !ids[:namespace_id].nil?
+          # pod found and namespace found
+          metadata = pod_metadata
+          metadata.merge!(namespace_metadata)
+        else
+          if ids[:pod_id].nil? && !ids[:namespace_id].nil?
+            # pod not found, but namespace found
+            @stats.bump(:id_cache_pod_not_found_namespace)
+            ns_time = Time.parse(namespace_metadata['creation_timestamp'])
+            if ns_time <= record_create_time
+              # namespace is older then record for pod
+              ids[:pod_id] = key
+              metadata = @cache.fetch(ids[:pod_id]) do
+                m = { 'pod_id' => ids[:pod_id] }
+              end
+            end
+            metadata.merge!(namespace_metadata)
+          else
+            if !ids[:pod_id].nil? && ids[:namespace_id].nil?
+              # pod found, but namespace NOT found
+              # this should NEVER be possible since pod meta can
+              # only be retrieved with a namespace
+              @stats.bump(:id_cache_namespace_not_found_pod)
+            else
+              # nothing found
+              @stats.bump(:id_cache_orphaned_record)
+            end
+            if @allow_orphans
+              log.trace("orphaning message for: #{namespace_name}/#{pod_name} ") if log.trace?
+              metadata = {
+                'orphaned_namespace' => namespace_name,
+                'namespace_name' => @orphaned_namespace_name,
+                'namespace_id' => @orphaned_namespace_id
+              }
+            else
+              metadata = {}
+            end
+            batch_miss_cache["#{namespace_name}_#{pod_name}"] = metadata
+          end
+        end
+        @id_cache[key] = ids unless batch_miss_cache.key?("#{namespace_name}_#{pod_name}")
+      end
+
+      # remove namespace info that is only used for comparison
+      metadata.delete('creation_timestamp')
+      metadata.delete_if{|k,v| v.nil?}
+    end
+
+  end
+end

data/lib/fluent/plugin/kubernetes_metadata_common.rb

@@ -0,0 +1,113 @@
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module KubernetesMetadata
+  module Common
+
+    def match_annotations(annotations)
+      result = {}
+      @annotations_regexps.each do |regexp|
+        annotations.each do |key, value|
+          if ::Fluent::StringUtil.match_regexp(regexp, key.to_s)
+            result[key] = value
+          end
+        end
+      end
+      result
+    end
+
+    def parse_namespace_metadata(namespace_object)
+      labels = String.new
+      labels = syms_to_strs(namespace_object['metadata']['labels'].to_h) unless @skip_labels
+
+      annotations = match_annotations(syms_to_strs(namespace_object['metadata']['annotations'].to_h))
+      if @de_dot
+        self.de_dot!(labels) unless @skip_labels
+        self.de_dot!(annotations)
+      end
+      kubernetes_metadata = {
+        'namespace_id' => namespace_object['metadata']['uid'],
+        'creation_timestamp' => namespace_object['metadata']['creationTimestamp']
+      }
+      kubernetes_metadata['namespace_labels'] = labels unless labels.empty?
+      kubernetes_metadata['namespace_annotations'] = annotations unless annotations.empty?
+      return kubernetes_metadata
+    end
+
+    def parse_pod_metadata(pod_object)
+      labels = String.new
+      labels = syms_to_strs(pod_object['metadata']['labels'].to_h) unless @skip_labels
+
+      annotations = match_annotations(syms_to_strs(pod_object['metadata']['annotations'].to_h))
+      if @de_dot
+        self.de_dot!(labels) unless @skip_labels
+        self.de_dot!(annotations)
+      end
+
+      # collect container informations
+      container_meta = {}
+      begin
+        pod_object['status']['containerStatuses'].each do|container_status|
+          # get plain container id (eg. docker://hash -> hash)
+          container_id = container_status['containerID'].sub /^[-_a-zA-Z0-9]+:\/\//, ''
+          unless @skip_container_metadata
+            container_meta[container_id] = {
+                'name' => container_status['name'],
+                'image' => container_status['image'],
+                'image_id' => container_status['imageID']
+            }
+          else
+            container_meta[container_id] = {
+                'name' => container_status['name']
+            }
+          end
+        end
+      rescue => e
+        log.debug("parsing container meta information failed for: #{pod_object['metadata']['namespace']}/#{pod_object['metadata']['name']} adfs #{e}")
+      end
+
+      kubernetes_metadata = {
+          'namespace_name' => pod_object['metadata']['namespace'],
+          'pod_id'         => pod_object['metadata']['uid'],
+          'pod_name'       => pod_object['metadata']['name'],
+          'containers'     => syms_to_strs(container_meta),
+          'host'           => pod_object['spec']['nodeName']
+      }
+      kubernetes_metadata['annotations'] = annotations unless annotations.empty?
+      kubernetes_metadata['labels'] = labels unless labels.empty?
+      kubernetes_metadata['master_url'] = @kubernetes_url unless @skip_master_url
+      return kubernetes_metadata
+    end
+
+    def syms_to_strs(hsh)
+      newhsh = {}
+      hsh.each_pair do |kk,vv|
+        if vv.is_a?(Hash)
+          vv = syms_to_strs(vv)
+        end
+        if kk.is_a?(Symbol)
+          newhsh[kk.to_s] = vv
+        else
+          newhsh[kk] = vv
+        end
+      end
+      newhsh
+    end
+
+  end
+end

data/lib/fluent/plugin/kubernetes_metadata_stats.rb

@@ -0,0 +1,46 @@
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require 'lru_redux'
+module KubernetesMetadata
+  class Stats
+
+    def initialize
+      @stats = ::LruRedux::TTL::ThreadSafeCache.new(1000, 3600)
+    end
+
+    def bump(key)
+        @stats[key] = @stats.getset(key) { 0 } + 1
+    end
+
+    def set(key, value)
+       @stats[key] = value
+    end
+
+    def [](key)
+      @stats[key]
+    end
+
+    def to_s
+      "stats - " + [].tap do |a|
+          @stats.each {|k,v| a << "#{k.to_s}: #{v}"}
+      end.join(', ')
+    end
+
+  end
+end

data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb

@@ -0,0 +1,60 @@
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require_relative 'kubernetes_metadata_common'
+
+module KubernetesMetadata
+  module WatchNamespaces
+
+    include ::KubernetesMetadata::Common
+
+    def start_namespace_watch
+      begin
+        resource_version = @client.get_namespaces.resourceVersion
+        watcher          = @client.watch_namespaces(resource_version)
+      rescue Exception=>e
+        message = "start_namespace_watch: Exception encountered setting up namespace watch from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}: #{e.message}"
+        message += " (#{e.response})" if e.respond_to?(:response)
+        log.debug(message)
+        raise Fluent::ConfigError, message
+      end
+      watcher.each do |notice|
+        case notice.type
+          when 'MODIFIED'
+            cache_key = notice.object['metadata']['uid']
+            cached    = @namespace_cache[cache_key]
+            if cached
+              @namespace_cache[cache_key] = parse_namespace_metadata(notice.object)
+              @stats.bump(:namespace_cache_watch_updates)
+            else
+              @stats.bump(:namespace_cache_watch_misses)
+            end
+          when 'DELETED'
+            # ignore and let age out for cases where 
+            # deleted but still processing logs
+            @stats.bump(:namespace_cache_watch_deletes_ignored)
+          else
+            # Don't pay attention to creations, since the created namespace may not
+            # be used by any pod on this node.
+            @stats.bump(:namespace_cache_watch_ignored)
+        end
+      end
+    end
+
+  end
+end

data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb

@@ -0,0 +1,63 @@
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require_relative 'kubernetes_metadata_common'
+
+module KubernetesMetadata
+  module WatchPods
+
+    include ::KubernetesMetadata::Common
+
+    def start_pod_watch
+      begin
+        resource_version = @client.get_pods.resourceVersion
+        watcher          = @client.watch_pods(resource_version)
+      rescue Exception => e
+        message = "Exception encountered fetching metadata from Kubernetes API endpoint: #{e.message}"
+        message += " (#{e.response})" if e.respond_to?(:response)
+
+        raise Fluent::ConfigError, message
+      end
+
+      watcher.each do |notice|
+        case notice.type
+          when 'MODIFIED'
+            cache_key = notice.object['metadata']['uid']
+            cached    = @cache[cache_key]
+            if cached
+              @cache[cache_key] = parse_pod_metadata(notice.object)
+              @stats.bump(:pod_cache_watch_updates)
+            elsif ENV['K8S_NODE_NAME'] == notice.object['spec']['nodeName'] then
+              @cache[cache_key] = parse_pod_metadata(notice.object)
+              @stats.bump(:pod_cache_host_updates)
+            else
+              @stats.bump(:pod_cache_watch_misses)
+            end
+          when 'DELETED'
+            # ignore and let age out for cases where pods
+            # deleted but still processing logs
+            @stats.bump(:pod_cache_watch_delete_ignored)
+          else
+            # Don't pay attention to creations, since the created pod may not
+            # end up on this node.
+            @stats.bump(:pod_cache_watch_ignored)
+        end
+      end
+    end
+  end
+end

data/test/cassettes/invalid_api_server_config.yml

@@ -0,0 +1,53 @@
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://localhost:8443/api
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+      Authorization:
+      - Bearer YzYyYzFlODMtODdhNS00ZTMyLWIzMmItNmY4NDc4OTI1ZWF
+  response:
+    status:
+      code: 401
+      message: Unauthorized
+    headers:
+      Content-Type:
+      - text/plain; charset=utf-8
+      Date:
+      - Sat, 09 May 2015 14:04:39 GMT
+      Content-Length:
+      - '13'
+    body:
+      encoding: UTF-8
+      string: |
+        Unauthorized
+    http_version: 
+  recorded_at: Sat, 09 May 2015 14:04:39 GMT
+recorded_with: VCR 2.9.3

data/test/cassettes/kubernetes_docker_metadata.yml

@@ -0,0 +1,228 @@
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://localhost:8443/api
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Fri, 08 May 2015 10:35:37 GMT
+      Content-Length:
+      - '67'
+    body:
+      encoding: UTF-8
+      string: |-
+        {
+          "versions": [
+            "v1"
+          ]
+        }
+    http_version: 
+  recorded_at: Fri, 08 May 2015 10:35:37 GMT
+- request:
+    method: get
+    uri: https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Fri, 08 May 2015 10:35:37 GMT
+      Transfer-Encoding:
+      - chunked
+    body:
+      encoding: UTF-8
+      string: |-
+        {
+          "kind": "Pod",
+          "apiVersion": "v1",
+          "metadata": {
+            "name": "fabric8-console-controller-98rqc",
+            "generateName": "fabric8-console-controller-",
+            "namespace": "default",
+            "selfLink": "/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc",
+            "uid": "c76927af-f563-11e4-b32d-54ee7527188d",
+            "resourceVersion": "122",
+            "creationTimestamp": "2015-05-08T09:22:42Z",
+            "labels": {
+              "component": "fabric8Console"
+            }
+          },
+          "spec": {
+            "volumes": [
+              {
+                "name": "openshift-cert-secrets",
+                "hostPath": null,
+                "emptyDir": null,
+                "gcePersistentDisk": null,
+                "gitRepo": null,
+                "secret": {
+                  "secretName": "openshift-cert-secrets"
+                },
+                "nfs": null,
+                "iscsi": null,
+                "glusterfs": null
+              }
+            ],
+            "containers": [
+              {
+                "name": "fabric8-console-container",
+                "image": "fabric8/hawtio-kubernetes:latest",
+                "ports": [
+                  {
+                    "containerPort": 9090,
+                    "protocol": "TCP"
+                  }
+                ],
+                "env": [
+                  {
+                    "name": "OAUTH_CLIENT_ID",
+                    "value": "fabric8-console"
+                  },
+                  {
+                    "name": "OAUTH_AUTHORIZE_URI",
+                    "value": "https://localhost:8443/oauth/authorize"
+                  }
+                ],
+                "resources": {},
+                "volumeMounts": [
+                  {
+                    "name": "openshift-cert-secrets",
+                    "readOnly": true,
+                    "mountPath": "/etc/secret-volume"
+                  }
+                ],
+                "terminationMessagePath": "/dev/termination-log",
+                "imagePullPolicy": "IfNotPresent",
+                "capabilities": {}
+              }
+            ],
+            "restartPolicy": "Always",
+            "dnsPolicy": "ClusterFirst",
+            "nodeName": "jimmi-redhat.localnet"
+          },
+          "status": {
+            "phase": "Running",
+            "Condition": [
+              {
+                "type": "Ready",
+                "status": "True"
+              }
+            ],
+            "hostIP": "172.17.42.1",
+            "podIP": "172.17.0.8",
+            "containerStatuses": [
+              {
+                "name": "fabric8-console-container",
+                "state": {
+                  "running": {
+                    "startedAt": "2015-05-08T09:22:44Z"
+                  }
+                },
+                "lastState": {},
+                "ready": true,
+                "restartCount": 0,
+                "image": "fabric8/hawtio-kubernetes:latest",
+                "imageID": "docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303",
+                "containerID": "docker://49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459"
+              }
+            ]
+          }
+        }
+    http_version: 
+  recorded_at: Fri, 08 May 2015 10:35:37 GMT
+- request:
+    method: get
+    uri: https://localhost:8443/api/v1/namespaces/default
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Fri, 08 May 2015 10:35:37 GMT
+      Transfer-Encoding:
+      - chunked
+    body:
+      encoding: UTF-8
+      string: |-
+        {
+          "kind": "Namespace",
+          "apiVersion": "v1",
+          "metadata": {
+            "name": "default",
+            "selfLink": "/api/v1/namespaces/default",
+            "uid": "898268c8-4a36-11e5-9d81-42010af0194c",
+            "resourceVersion": "6",
+            "creationTimestamp": "2015-05-08T09:22:01Z"
+          },
+          "spec": {
+            "finalizers": [
+                "kubernetes"
+            ]
+          },
+          "status": {
+            "phase": "Active"
+          }
+        }
+    http_version:
+  recorded_at: Fri, 08 May 2015 10:35:37 GMT
+recorded_with: VCR 2.9.3