fluent-plugin-kubernetes_metadata_filter 0.10.0 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f2e0964802476e178c9905c6e0cbaba9ffadb7c1
-  data.tar.gz: 5ccaeb987a53deacc7c93726804737a459582852
+  metadata.gz: 4e0fcd32fc3504d09147c16350c4596822151394
+  data.tar.gz: d6032d7ec7cd42cf46fcf101e3d8579883502caf
 SHA512:
-  metadata.gz: 641df616bdc5284f88d3e363693de7ff996301aef62397d0f44fce6907b7ca602bb9c4b13451be058e8d47c944bb49c9fc22a994c0688520e4db463749ffd213
-  data.tar.gz: c6054cb7f0184b34ce9c4bee0c665cfda2bfb012e87ac5744eb4664cffe43c2589af046a18de640b86beb2762d05997d1d27a87c0dafce4fb01b291f6e6a6256
+  metadata.gz: 606ac89791ceec0de9f1f46d80ced6b126dff9325513c1bc177dcb1cb7f35896f9cdedea4a9aff657f5b0eab1569b0a0eb0143c38c51678cc86cbc5d47c6e1e7
+  data.tar.gz: 98c9aa0d292a7dc9d679ab46961136986ecfbfcfdd2c9276ded1bca00c67fa6ca11a52581942ff3a3d10c32267b259d0267efc0db462f2d8331454f3af6f869c

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-kubernetes_metadata_filter"
-  gem.version       = "0.10.0"
+  gem.version       = "0.11.0"
   gem.authors       = ["Jimmi Dyson"]
   gem.email         = ["jimmidyson@gmail.com"]
   gem.description   = %q{Filter plugin to add Kubernetes metadata}

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -34,20 +34,20 @@ module Fluent
                  :default => '\.(?<pod_name>[^\._]+)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
     config_param :bearer_token_file, :string, default: ''
     config_param :merge_json_log, :bool, default: true
+    config_param :include_namespace_id, :bool, default: false
 
-    def get_metadata(namespace, pod_name, container_name)
+    def get_metadata(namespace_name, pod_name, container_name)
       begin
-        metadata = @client.get_pod(pod_name, namespace)
-        if metadata
-          return {
-              uid:            metadata['metadata']['uid'],
-              namespace:      metadata['metadata']['namespace'],
-              pod_name:       metadata['metadata']['name'],
-              container_name: container_name,
-              labels:         metadata['metadata']['labels'].to_h,
-              host:           metadata['spec']['host']
-          }
-        end
+        metadata = @client.get_pod(pod_name, namespace_name)
+        return if !metadata
+        return {
+            namespace_name: namespace_name,
+            pod_id:         metadata['metadata']['uid'],
+            pod_name:       pod_name,
+            container_name: container_name,
+            labels:         metadata['metadata']['labels'].to_h,
+            host:           metadata['spec']['host']
+        }
       rescue KubeException
         nil
       end
@@ -67,7 +67,10 @@ module Fluent
       if @cache_ttl < 0
         @cache_ttl = :none
       end
-      @cache                                  = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl)
+      @cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl)
+      if @include_namespace_id
+        @namespace_cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl)
+      end
       @tag_to_kubernetes_name_regexp_compiled = Regexp.compile(@tag_to_kubernetes_name_regexp)
 
       if @kubernetes_url.present?
@@ -97,10 +100,12 @@ module Fluent
         end
 
         if @watch
-          thread                    = Thread.new(self) { |this|
-            this.start_watch
-          }
+          thread = Thread.new(self) { |this| this.start_watch }
           thread.abort_on_exception = true
+          if @include_namespace_id
+            namespace_thread = Thread.new(self) { |this| this.start_namespace_watch }
+            namespace_thread.abort_on_exception = true
+          end
         end
       end
     end
@@ -116,20 +121,20 @@ module Fluent
                 container_id: match_data['docker_id']
             },
             kubernetes: {
-                namespace: match_data['namespace'],
+                namespace_name: match_data['namespace'],
                 pod_name: match_data['pod_name'],
                 container_name: match_data['container_name']
             }
         }
 
         if @kubernetes_url.present?
-          cache_key = "#{metadata[:kubernetes][:namespace]}_#{metadata[:kubernetes][:pod_name]}_#{metadata[:kubernetes][:container_name]}"
+          cache_key = "#{metadata[:kubernetes][:namespace_name]}_#{metadata[:kubernetes][:pod_name]}_#{metadata[:kubernetes][:container_name]}"
 
           this     = self
           metadata = @cache.getset(cache_key) {
             if metadata
               kubernetes_metadata = this.get_metadata(
-                metadata[:kubernetes][:namespace],
+                metadata[:kubernetes][:namespace_name],
                 metadata[:kubernetes][:pod_name],
                 metadata[:kubernetes][:container_name]
               )
@@ -137,6 +142,14 @@ module Fluent
               metadata
             end
           }
+          if @include_namespace_id
+            namespace_name = metadata[:kubernetes][:namespace_name]
+            namespace_id = @namespace_cache.getset(namespace_name) {
+              namespace = @client.get_namespace(namespace_name)
+              namespace['metadata']['uid'] if namespace
+            }
+            metadata[:kubernetes][:namespace_id] = namespace_id if namespace_id
+          end
         end
       end
 
@@ -196,7 +209,23 @@ module Fluent
               }
             end
           else
-            # ignoring...
+            # Don't pay attention to creations, since the created pod may not
+            # end up on this node.
+        end
+      end
+    end
+
+    def start_namespace_watch
+      resource_version = @client.get_namespaces.resourceVersion
+      watcher          = @client.watch_namespaces(resource_version)
+      watcher.each do |notice|
+        puts notice
+        case notice.type
+          when 'DELETED'
+            @namespace_cache.delete(notice.object['metadata']['uid'])
+          else
+            # We only care about each namespace's name and UID, neither of which
+            # is modifiable, so we only have to care about deletions.
         end
       end
     end

data/test/cassettes/metadata_with_namespace_id.yml

@@ -0,0 +1,228 @@
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://localhost:8443/api
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Fri, 08 May 2015 10:35:37 GMT
+      Content-Length:
+      - '67'
+    body:
+      encoding: UTF-8
+      string: |-
+        {
+          "versions": [
+            "v1"
+          ]
+        }
+    http_version: 
+  recorded_at: Fri, 08 May 2015 10:35:37 GMT
+- request:
+    method: get
+    uri: https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Fri, 08 May 2015 10:35:37 GMT
+      Transfer-Encoding:
+      - chunked
+    body:
+      encoding: UTF-8
+      string: |-
+        {
+          "kind": "Pod",
+          "apiVersion": "v1",
+          "metadata": {
+            "name": "fabric8-console-controller-98rqc",
+            "generateName": "fabric8-console-controller-",
+            "namespace": "default",
+            "selfLink": "/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc",
+            "uid": "c76927af-f563-11e4-b32d-54ee7527188d",
+            "resourceVersion": "122",
+            "creationTimestamp": "2015-05-08T09:22:42Z",
+            "labels": {
+              "component": "fabric8Console"
+            }
+          },
+          "spec": {
+            "volumes": [
+              {
+                "name": "openshift-cert-secrets",
+                "hostPath": null,
+                "emptyDir": null,
+                "gcePersistentDisk": null,
+                "gitRepo": null,
+                "secret": {
+                  "secretName": "openshift-cert-secrets"
+                },
+                "nfs": null,
+                "iscsi": null,
+                "glusterfs": null
+              }
+            ],
+            "containers": [
+              {
+                "name": "fabric8-console-container",
+                "image": "fabric8/hawtio-kubernetes:latest",
+                "ports": [
+                  {
+                    "containerPort": 9090,
+                    "protocol": "TCP"
+                  }
+                ],
+                "env": [
+                  {
+                    "name": "OAUTH_CLIENT_ID",
+                    "value": "fabric8-console"
+                  },
+                  {
+                    "name": "OAUTH_AUTHORIZE_URI",
+                    "value": "https://localhost:8443/oauth/authorize"
+                  }
+                ],
+                "resources": {},
+                "volumeMounts": [
+                  {
+                    "name": "openshift-cert-secrets",
+                    "readOnly": true,
+                    "mountPath": "/etc/secret-volume"
+                  }
+                ],
+                "terminationMessagePath": "/dev/termination-log",
+                "imagePullPolicy": "IfNotPresent",
+                "capabilities": {}
+              }
+            ],
+            "restartPolicy": "Always",
+            "dnsPolicy": "ClusterFirst",
+            "host": "jimmi-redhat.localnet"
+          },
+          "status": {
+            "phase": "Running",
+            "Condition": [
+              {
+                "type": "Ready",
+                "status": "True"
+              }
+            ],
+            "hostIP": "172.17.42.1",
+            "podIP": "172.17.0.8",
+            "containerStatuses": [
+              {
+                "name": "fabric8-console-container",
+                "state": {
+                  "running": {
+                    "startedAt": "2015-05-08T09:22:44Z"
+                  }
+                },
+                "lastState": {},
+                "ready": true,
+                "restartCount": 0,
+                "image": "fabric8/hawtio-kubernetes:latest",
+                "imageID": "docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303",
+                "containerID": "docker://49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459"
+              }
+            ]
+          }
+        }
+    http_version: 
+  recorded_at: Fri, 08 May 2015 10:35:37 GMT
+- request:
+    method: get
+    uri: https://localhost:8443/api/v1/namespaces/default
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Fri, 08 May 2015 10:35:37 GMT
+      Transfer-Encoding:
+      - chunked
+    body:
+      encoding: UTF-8
+      string: |-
+        {
+          "kind": "Namespace",
+          "apiVersion": "v1",
+          "metadata": {
+            "name": "default",
+            "selfLink": "/api/v1/namespaces/default",
+            "uid": "898268c8-4a36-11e5-9d81-42010af0194c",
+            "resourceVersion": "6",
+            "creationTimestamp": "2015-05-08T09:22:01Z"
+          },
+          "spec": {
+            "finalizers": [
+                "kubernetes"
+            ]
+          },
+          "status": {
+            "phase": "Active"
+          }
+        }
+    http_version: 
+  recorded_at: Fri, 08 May 2015 10:35:37 GMT
+recorded_with: VCR 2.9.3

data/test/plugin/test_filter_kubernetes_metadata.rb

@@ -112,8 +112,36 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
             host:           'jimmi-redhat.localnet',
             pod_name:       'fabric8-console-controller-98rqc',
             container_name: 'fabric8-console-container',
-            namespace:      'default',
-            uid:            'c76927af-f563-11e4-b32d-54ee7527188d',
+            namespace_name:      'default',
+            pod_id:            'c76927af-f563-11e4-b32d-54ee7527188d',
+            labels: {
+              component: 'fabric8Console'
+            }
+          }
+        }
+        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
+      end
+    end
+
+    test 'with docker & kubernetes metadata & namespace_id enabled' do
+      VCR.use_cassette('metadata_with_namespace_id') do
+        es = emit({}, '
+          kubernetes_url https://localhost:8443
+          watch false
+          cache_size 1
+          include_namespace_id true
+        ')
+        expected_kube_metadata = {
+          docker: {
+              container_id: '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+          },
+          kubernetes: {
+            host:           'jimmi-redhat.localnet',
+            pod_name:       'fabric8-console-controller-98rqc',
+            container_name: 'fabric8-console-container',
+            namespace_name: 'default',
+            namespace_id:   '898268c8-4a36-11e5-9d81-42010af0194c',
+            pod_id:         'c76927af-f563-11e4-b32d-54ee7527188d',
             labels: {
               component: 'fabric8Console'
             }
@@ -139,8 +167,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
             host:           'jimmi-redhat.localnet',
             pod_name:       'fabric8-console-controller-98rqc',
             container_name: 'fabric8-console-container',
-            namespace:      'default',
-            uid:            'c76927af-f563-11e4-b32d-54ee7527188d',
+            namespace_name:      'default',
+            pod_id:            'c76927af-f563-11e4-b32d-54ee7527188d',
             labels: {
               component: 'fabric8Console'
             }
@@ -159,7 +187,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
           kubernetes: {
               pod_name:       'fabric8-console-controller-98rqc',
               container_name: 'fabric8-console-container',
-              namespace:      'default',
+              namespace_name:      'default',
           }
       }
       assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
@@ -180,7 +208,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
         kubernetes: {
           pod_name:       'fabric8-console-controller-98rqc',
           container_name: 'fabric8-console-container',
-          namespace:      'default'
+          namespace_name:      'default'
         }
       }
       assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.11.0
 platform: ruby
 authors:
 - Jimmi Dyson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-01 00:00:00.000000000 Z
+date: 2015-09-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -225,6 +225,7 @@ files:
 - test/cassettes/invalid_api_server_config.yml
 - test/cassettes/kubernetes_docker_metadata.yml
 - test/cassettes/kubernetes_docker_metadata_using_bearer_token.yml
+- test/cassettes/metadata_with_namespace_id.yml
 - test/cassettes/non_kubernetes_docker_metadata.yml
 - test/cassettes/valid_kubernetes_api_server.yml
 - test/helper.rb
@@ -258,6 +259,7 @@ test_files:
 - test/cassettes/invalid_api_server_config.yml
 - test/cassettes/kubernetes_docker_metadata.yml
 - test/cassettes/kubernetes_docker_metadata_using_bearer_token.yml
+- test/cassettes/metadata_with_namespace_id.yml
 - test/cassettes/non_kubernetes_docker_metadata.yml
 - test/cassettes/valid_kubernetes_api_server.yml
 - test/helper.rb