RubyGems - fluent-plugin-kubernetes_metadata_filter - Versions diffs - 0.10.0 → 0.11.0 - Mend

fluent-plugin-kubernetes_metadata_filter 0.10.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (6) hide show

checksums.yaml +4 -4
data/fluent-plugin-kubernetes_metadata_filter.gemspec +1 -1
data/lib/fluent/plugin/filter_kubernetes_metadata.rb +49 -20
data/test/cassettes/metadata_with_namespace_id.yml +228 -0
data/test/plugin/test_filter_kubernetes_metadata.rb +34 -6
metadata +4 -2

checksums.yaml CHANGED

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f2e0964802476e178c9905c6e0cbaba9ffadb7c1
-  data.tar.gz: 5ccaeb987a53deacc7c93726804737a459582852
+  metadata.gz: 4e0fcd32fc3504d09147c16350c4596822151394
+  data.tar.gz: d6032d7ec7cd42cf46fcf101e3d8579883502caf
 SHA512:
-  metadata.gz: 641df616bdc5284f88d3e363693de7ff996301aef62397d0f44fce6907b7ca602bb9c4b13451be058e8d47c944bb49c9fc22a994c0688520e4db463749ffd213
-  data.tar.gz: c6054cb7f0184b34ce9c4bee0c665cfda2bfb012e87ac5744eb4664cffe43c2589af046a18de640b86beb2762d05997d1d27a87c0dafce4fb01b291f6e6a6256
+  metadata.gz: 606ac89791ceec0de9f1f46d80ced6b126dff9325513c1bc177dcb1cb7f35896f9cdedea4a9aff657f5b0eab1569b0a0eb0143c38c51678cc86cbc5d47c6e1e7
+  data.tar.gz: 98c9aa0d292a7dc9d679ab46961136986ecfbfcfdd2c9276ded1bca00c67fa6ca11a52581942ff3a3d10c32267b259d0267efc0db462f2d8331454f3af6f869c

data/fluent-plugin-kubernetes_metadata_filter.gemspec CHANGED

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-kubernetes_metadata_filter"
-  gem.version       = "0.10.0"
+  gem.version       = "0.11.0"
   gem.authors       = ["Jimmi Dyson"]
   gem.email         = ["jimmidyson@gmail.com"]
   gem.description   = %q{Filter plugin to add Kubernetes metadata}

data/lib/fluent/plugin/filter_kubernetes_metadata.rb CHANGED

@@ -34,20 +34,20 @@ module Fluent
                  :default => '\.(?<pod_name>[^\._]+)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
     config_param :bearer_token_file, :string, default: ''
     config_param :merge_json_log, :bool, default: true
+    config_param :include_namespace_id, :bool, default: false
-    def get_metadata(namespace, pod_name, container_name)
+    def get_metadata(namespace_name, pod_name, container_name)
       begin
-        metadata = @client.get_pod(pod_name, namespace)
-        if metadata
-          return {
-              uid:            metadata['metadata']['uid'],
-              namespace:      metadata['metadata']['namespace'],
-              pod_name:       metadata['metadata']['name'],
-              container_name: container_name,
-              labels:         metadata['metadata']['labels'].to_h,
-              host:           metadata['spec']['host']
-          }
-        end
+        metadata = @client.get_pod(pod_name, namespace_name)
+        return if !metadata
+        return {
+            namespace_name: namespace_name,
+            pod_id:         metadata['metadata']['uid'],
+            pod_name:       pod_name,
+            container_name: container_name,
+            labels:         metadata['metadata']['labels'].to_h,
+            host:           metadata['spec']['host']
+        }
       rescue KubeException
         nil
       end
@@ -67,7 +67,10 @@ module Fluent
       if @cache_ttl < 0
         @cache_ttl = :none
       end
-      @cache                                  = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl)
+      @cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl)
+      if @include_namespace_id
+        @namespace_cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl)
+      end
       @tag_to_kubernetes_name_regexp_compiled = Regexp.compile(@tag_to_kubernetes_name_regexp)
       if @kubernetes_url.present?
@@ -97,10 +100,12 @@ module Fluent
         end
         if @watch
-          thread                    = Thread.new(self) { |this|
-            this.start_watch
-          }
+          thread = Thread.new(self) { |this| this.start_watch }
           thread.abort_on_exception = true
+          if @include_namespace_id
+            namespace_thread = Thread.new(self) { |this| this.start_namespace_watch }
+            namespace_thread.abort_on_exception = true
+          end
         end
       end
     end
@@ -116,20 +121,20 @@ module Fluent
                 container_id: match_data['docker_id']
             },
             kubernetes: {
-                namespace: match_data['namespace'],
+                namespace_name: match_data['namespace'],
                 pod_name: match_data['pod_name'],
                 container_name: match_data['container_name']
             }
         }
         if @kubernetes_url.present?
-          cache_key = "#{metadata[:kubernetes][:namespace]}_#{metadata[:kubernetes][:pod_name]}_#{metadata[:kubernetes][:container_name]}"
+          cache_key = "#{metadata[:kubernetes][:namespace_name]}_#{metadata[:kubernetes][:pod_name]}_#{metadata[:kubernetes][:container_name]}"
           this     = self
           metadata = @cache.getset(cache_key) {
             if metadata
               kubernetes_metadata = this.get_metadata(
-                metadata[:kubernetes][:namespace],
+                metadata[:kubernetes][:namespace_name],
                 metadata[:kubernetes][:pod_name],
                 metadata[:kubernetes][:container_name]
               )
@@ -137,6 +142,14 @@ module Fluent
               metadata
             end
           }
+          if @include_namespace_id
+            namespace_name = metadata[:kubernetes][:namespace_name]
+            namespace_id = @namespace_cache.getset(namespace_name) {
+              namespace = @client.get_namespace(namespace_name)
+              namespace['metadata']['uid'] if namespace
+            }
+            metadata[:kubernetes][:namespace_id] = namespace_id if namespace_id
+          end
         end
       end
@@ -196,7 +209,23 @@ module Fluent
               }
             end
           else
-            # ignoring...
+            # Don't pay attention to creations, since the created pod may not
+            # end up on this node.
+        end
+      end
+    end
+    def start_namespace_watch
+      resource_version = @client.get_namespaces.resourceVersion
+      watcher          = @client.watch_namespaces(resource_version)
+      watcher.each do |notice|
+        puts notice
+        case notice.type
+          when 'DELETED'
+            @namespace_cache.delete(notice.object['metadata']['uid'])
+          else
+            # We only care about each namespace's name and UID, neither of which
+            # is modifiable, so we only have to care about deletions.
         end
       end
     end

data/test/cassettes/metadata_with_namespace_id.yml ADDED

@@ -0,0 +1,228 @@
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://localhost:8443/api
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Fri, 08 May 2015 10:35:37 GMT
+      Content-Length:
+      - '67'
+    body:
+      encoding: UTF-8
+      string: |-
+        {
+          "versions": [
+            "v1"
+          ]
+        }
+    http_version:
+  recorded_at: Fri, 08 May 2015 10:35:37 GMT
+- request:
+    method: get
+    uri: https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Fri, 08 May 2015 10:35:37 GMT
+      Transfer-Encoding:
+      - chunked
+    body:
+      encoding: UTF-8
+      string: |-
+        {
+          "kind": "Pod",
+          "apiVersion": "v1",
+          "metadata": {
+            "name": "fabric8-console-controller-98rqc",
+            "generateName": "fabric8-console-controller-",
+            "namespace": "default",
+            "selfLink": "/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc",
+            "uid": "c76927af-f563-11e4-b32d-54ee7527188d",
+            "resourceVersion": "122",
+            "creationTimestamp": "2015-05-08T09:22:42Z",
+            "labels": {
+              "component": "fabric8Console"
+            }
+          },
+          "spec": {
+            "volumes": [
+              {
+                "name": "openshift-cert-secrets",
+                "hostPath": null,
+                "emptyDir": null,
+                "gcePersistentDisk": null,
+                "gitRepo": null,
+                "secret": {
+                  "secretName": "openshift-cert-secrets"
+                },
+                "nfs": null,
+                "iscsi": null,
+                "glusterfs": null
+              }
+            ],
+            "containers": [
+              {
+                "name": "fabric8-console-container",
+                "image": "fabric8/hawtio-kubernetes:latest",
+                "ports": [
+                  {
+                    "containerPort": 9090,
+                    "protocol": "TCP"
+                  }
+                ],
+                "env": [
+                  {
+                    "name": "OAUTH_CLIENT_ID",
+                    "value": "fabric8-console"
+                  },
+                  {
+                    "name": "OAUTH_AUTHORIZE_URI",
+                    "value": "https://localhost:8443/oauth/authorize"
+                  }
+                ],
+                "resources": {},
+                "volumeMounts": [
+                  {
+                    "name": "openshift-cert-secrets",
+                    "readOnly": true,
+                    "mountPath": "/etc/secret-volume"
+                  }
+                ],
+                "terminationMessagePath": "/dev/termination-log",
+                "imagePullPolicy": "IfNotPresent",
+                "capabilities": {}
+              }
+            ],
+            "restartPolicy": "Always",
+            "dnsPolicy": "ClusterFirst",
+            "host": "jimmi-redhat.localnet"
+          },
+          "status": {
+            "phase": "Running",
+            "Condition": [
+              {
+                "type": "Ready",
+                "status": "True"
+              }
+            ],
+            "hostIP": "172.17.42.1",
+            "podIP": "172.17.0.8",
+            "containerStatuses": [
+              {
+                "name": "fabric8-console-container",
+                "state": {
+                  "running": {
+                    "startedAt": "2015-05-08T09:22:44Z"
+                  }
+                },
+                "lastState": {},
+                "ready": true,
+                "restartCount": 0,
+                "image": "fabric8/hawtio-kubernetes:latest",
+                "imageID": "docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303",
+                "containerID": "docker://49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459"
+              }
+            ]
+          }
+        }
+    http_version:
+  recorded_at: Fri, 08 May 2015 10:35:37 GMT
+- request:
+    method: get
+    uri: https://localhost:8443/api/v1/namespaces/default
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Fri, 08 May 2015 10:35:37 GMT
+      Transfer-Encoding:
+      - chunked
+    body:
+      encoding: UTF-8
+      string: |-
+        {
+          "kind": "Namespace",
+          "apiVersion": "v1",
+          "metadata": {
+            "name": "default",
+            "selfLink": "/api/v1/namespaces/default",
+            "uid": "898268c8-4a36-11e5-9d81-42010af0194c",
+            "resourceVersion": "6",
+            "creationTimestamp": "2015-05-08T09:22:01Z"
+          },
+          "spec": {
+            "finalizers": [
+                "kubernetes"
+            ]
+          },
+          "status": {
+            "phase": "Active"
+          }
+        }
+    http_version:
+  recorded_at: Fri, 08 May 2015 10:35:37 GMT
+recorded_with: VCR 2.9.3

data/test/plugin/test_filter_kubernetes_metadata.rb CHANGED

@@ -112,8 +112,36 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
             host:           'jimmi-redhat.localnet',
             pod_name:       'fabric8-console-controller-98rqc',
             container_name: 'fabric8-console-container',
-            namespace:      'default',
-            uid:            'c76927af-f563-11e4-b32d-54ee7527188d',
+            namespace_name:      'default',
+            pod_id:            'c76927af-f563-11e4-b32d-54ee7527188d',
+            labels: {
+              component: 'fabric8Console'
+            }
+          }
+        }
+        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
+      end
+    end
+    test 'with docker & kubernetes metadata & namespace_id enabled' do
+      VCR.use_cassette('metadata_with_namespace_id') do
+        es = emit({}, '
+          kubernetes_url https://localhost:8443
+          watch false
+          cache_size 1
+          include_namespace_id true
+        ')
+        expected_kube_metadata = {
+          docker: {
+              container_id: '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+          },
+          kubernetes: {
+            host:           'jimmi-redhat.localnet',
+            pod_name:       'fabric8-console-controller-98rqc',
+            container_name: 'fabric8-console-container',
+            namespace_name: 'default',
+            namespace_id:   '898268c8-4a36-11e5-9d81-42010af0194c',
+            pod_id:         'c76927af-f563-11e4-b32d-54ee7527188d',
             labels: {
               component: 'fabric8Console'
             }
@@ -139,8 +167,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
             host:           'jimmi-redhat.localnet',
             pod_name:       'fabric8-console-controller-98rqc',
             container_name: 'fabric8-console-container',
-            namespace:      'default',
-            uid:            'c76927af-f563-11e4-b32d-54ee7527188d',
+            namespace_name:      'default',
+            pod_id:            'c76927af-f563-11e4-b32d-54ee7527188d',
             labels: {
               component: 'fabric8Console'
             }
@@ -159,7 +187,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
           kubernetes: {
               pod_name:       'fabric8-console-controller-98rqc',
               container_name: 'fabric8-console-container',
-              namespace:      'default',
+              namespace_name:      'default',
           }
       }
       assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
@@ -180,7 +208,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
         kubernetes: {
           pod_name:       'fabric8-console-controller-98rqc',
           container_name: 'fabric8-console-container',
-          namespace:      'default'
+          namespace_name:      'default'
         }
       }
       assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])

metadata CHANGED

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.11.0
 platform: ruby
 authors:
 - Jimmi Dyson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-09-01 00:00:00.000000000 Z
+date: 2015-09-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -225,6 +225,7 @@ files:
 - test/cassettes/invalid_api_server_config.yml
 - test/cassettes/kubernetes_docker_metadata.yml
 - test/cassettes/kubernetes_docker_metadata_using_bearer_token.yml
+- test/cassettes/metadata_with_namespace_id.yml
 - test/cassettes/non_kubernetes_docker_metadata.yml
 - test/cassettes/valid_kubernetes_api_server.yml
 - test/helper.rb
@@ -258,6 +259,7 @@ test_files:
 - test/cassettes/invalid_api_server_config.yml
 - test/cassettes/kubernetes_docker_metadata.yml
 - test/cassettes/kubernetes_docker_metadata_using_bearer_token.yml
+- test/cassettes/metadata_with_namespace_id.yml
 - test/cassettes/non_kubernetes_docker_metadata.yml
 - test/cassettes/valid_kubernetes_api_server.yml
 - test/helper.rb