fluent-plugin-kubernetes_metadata_filter 0.15.0 → 0.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 247710a5a22705fe46717e612a13998857661dc9
-  data.tar.gz: 419da7d1297da104e6802110ad95df22608e5c41
+  metadata.gz: e3cac3ef5c79d4a00c3344458a5f341759319b70
+  data.tar.gz: d8770e74580e828b42fd450515f92d8fb50712db
 SHA512:
-  metadata.gz: e4417767dddc3bfabb183a32d4f79cd3622541c531fb65205dacd0678280351476bcd2165c927ba524b41f7f1dd33ddfcbb4aafc33da302cb4a13247ae16d772
-  data.tar.gz: 5e1dd6ba75d5ba6965b31bbbd98df670280bf07164680067439b662135d97fcecd96decd6a129b1dbd1a2114e5078ea7384a26f97c1a5792f6e8d7ff7578fc11
+  metadata.gz: b555868c660ac76fcb05c4ca0579a2c94d26166f0d49c0b70b7fd50c9b8e13c562e2caa76d4305fcc8f355a233a8eb0477b88514a13c3a8889e1ac2ff3a648a7
+  data.tar.gz: c5d175766caeeee49fc8896cb46e957cf8bba60964a6b0e9164b4efdbba8492469bf6de62e55364fb88bfccaa9ab7b00e183d4a46c219c4f3332f35d8c821b91

data/README.md

@@ -14,7 +14,7 @@ Configuration options for fluent.conf are:
 * `kubernetes_url` - URL to the API server. Set this to retrieve further kubernetes metadata for logs from kubernetes API server
 * `apiVersion` - API version to use (default: `v1`)
 * `ca_file` - path to CA file for Kubernetes server certificate validation
-* `verify_ssl` - validate SSL certificates (default: true)
+* `verify_ssl` - validate SSL certificates (default: `true`)
 * `client_cert` - path to a client cert file to authenticate to the API server
 * `client_key` - path to a client key file to authenticate to the API server
 * `bearer_token_file` - path to a file containing the bearer token to use for authentication
@@ -24,6 +24,8 @@ This must used named capture groups for `container_name`, `pod_name` & `namespac
 * `cache_ttl` - TTL in seconds of each cached element. Set to negative value to disable TTL eviction (default: `3600` - 1 hour)
 * `watch` - set up a watch on pods on the API server for updates to metadata (default: `true`)
 * `merge_json_log` - merge logs in JSON format as top level keys (default: `true`)
+* `de_dot` - replace dots in labels with configured `de_dot_separator`, required for ElasticSearch 2.x compatibility (default: `true`)
+* `de_dot_separator` - separator to use if `de_dot` is enabled (default: `_`)
 
 ```
 <source>

data/circle.yml

@@ -1,3 +1,7 @@
+machine:
+  ruby:
+    version: 2.2.3
+
 test:
   override:
     - bundle exec rake test

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-kubernetes_metadata_filter"
-  gem.version       = "0.15.0"
+  gem.version       = "0.16.0"
   gem.authors       = ["Jimmi Dyson"]
   gem.email         = ["jimmidyson@gmail.com"]
   gem.description   = %q{Filter plugin to add Kubernetes metadata}

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -23,33 +23,39 @@ module Fluent
 
     Fluent::Plugin.register_filter('kubernetes_metadata', self)
 
-    config_param :kubernetes_url, :string, default: ''
+    config_param :kubernetes_url, :string, default: nil
     config_param :cache_size, :integer, default: 1000
     config_param :cache_ttl, :integer, default: 60 * 60
     config_param :watch, :bool, default: true
     config_param :apiVersion, :string, default: 'v1'
-    config_param :client_cert, :string, default: ''
-    config_param :client_key, :string, default: ''
-    config_param :ca_file, :string, default: ''
+    config_param :client_cert, :string, default: nil
+    config_param :client_key, :string, default: nil
+    config_param :ca_file, :string, default: nil
     config_param :verify_ssl, :bool, default: true
     config_param :tag_to_kubernetes_name_regexp,
                  :string,
                  :default => 'var\.log\.containers\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
-    config_param :bearer_token_file, :string, default: ''
+    config_param :bearer_token_file, :string, default: nil
     config_param :merge_json_log, :bool, default: true
     config_param :include_namespace_id, :bool, default: false
     config_param :secret_dir, :string, default: '/var/run/secrets/kubernetes.io/serviceaccount'
+    config_param :de_dot, :bool, default: true
+    config_param :de_dot_separator, :string, default: '_'
 
     def get_metadata(namespace_name, pod_name, container_name)
       begin
         metadata = @client.get_pod(pod_name, namespace_name)
         return if !metadata
+        labels = metadata['metadata']['labels'].to_h
+        if @de_dot
+          self.de_dot!(labels)
+        end
         return {
             namespace_name: namespace_name,
             pod_id:         metadata['metadata']['uid'],
             pod_name:       pod_name,
             container_name: container_name,
-            labels:         metadata['metadata']['labels'].to_h,
+            labels:         labels,
             host:           metadata['spec']['nodeName']
         }
       rescue KubeException
@@ -68,6 +74,10 @@ module Fluent
       require 'active_support/core_ext/object/blank'
       require 'lru_redux'
 
+      if @de_dot && (@de_dot_separator =~ /\./).present?
+        raise Fluent::ConfigError, "Invalid de_dot_separator: cannot be or contain '.'"
+      end
+
       if @cache_ttl < 0
         @cache_ttl = :none
       end
@@ -78,10 +88,10 @@ module Fluent
       @tag_to_kubernetes_name_regexp_compiled = Regexp.compile(@tag_to_kubernetes_name_regexp)
 
       # Use Kubernetes default service account if we're in a pod.
-      if !@kubernetes_url.present?
+      if @kubernetes_url.nil?
         env_host = ENV['KUBERNETES_SERVICE_HOST']
         env_port = ENV['KUBERNETES_SERVICE_PORT']
-        if !env_host.nil? and !env_port.nil?
+        if env_host.present? && env_port.present?
           @kubernetes_url = "https://#{env_host}:#{env_port}/api"
         end
       end
@@ -204,6 +214,16 @@ module Fluent
       record
     end
 
+    def de_dot!(h)
+      h.keys.each do |ref|
+        if h[ref] && ref =~ /\./
+          v = h.delete(ref)
+          newref = ref.to_s.gsub('.', @de_dot_separator)
+          h[newref.to_sym] = v
+        end
+      end
+    end
+
     def start_watch
       resource_version = @client.get_pods.resourceVersion
       watcher          = @client.watch_pods(resource_version)
@@ -211,24 +231,27 @@ module Fluent
         case notice.type
           when 'MODIFIED'
             if notice.object.status.containerStatuses
+              pod_cache_key = "#{notice.object['metadata']['namespace']}_#{notice.object['metadata']['name']}"
               notice.object.status.containerStatuses.each { |container_status|
-                if container_status['containerId']
-                  containerId = container_status['containerId'].sub(/^docker:\/\//, '')
-                  cached      = @cache[containerId]
-                  if cached
-                    # Only thing that can be modified is labels
-                    cached[:labels]     = v.object.metadata.labels.to_h
-                    @cache[containerId] = cached
+                cache_key = "#{pod_cache_key}_#{container_status['name']}"
+                cached    = @cache[cache_key]
+                if cached
+                  # Only thing that can be modified is labels
+                  labels = notice.object.metadata.labels.to_h
+                  if @de_dot
+                    self.de_dot!(labels)
                   end
+                  cached[:kubernetes][:labels] = labels
+                  @cache[cache_key]            = cached
                 end
               }
             end
           when 'DELETED'
             if notice.object.status.containerStatuses
+              pod_cache_key = "#{notice.object['metadata']['namespace']}_#{notice.object['metadata']['name']}"
               notice.object.status.containerStatuses.each { |container_status|
-                if container_status['containerId']
-                  @cache.delete(container_status['containerId'].sub(/^docker:\/\//, ''))
-                end
+                cache_key = "#{pod_cache_key}_#{container_status['name']}"
+                @cache.delete(cache_key)
               }
             end
           else
@@ -245,7 +268,7 @@ module Fluent
         puts notice
         case notice.type
           when 'DELETED'
-            @namespace_cache.delete(notice.object['metadata']['uid'])
+            @namespace_cache.delete(notice.object['metadata']['name'])
           else
             # We only care about each namespace's name and UID, neither of which
             # is modifiable, so we only have to care about deletions.

data/test/cassettes/kubernetes_docker_metadata_dotted_labels.yml

@@ -0,0 +1,180 @@
+#
+# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
+# Kubernetes metadata
+#
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://localhost:8443/api
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Fri, 08 May 2015 10:35:37 GMT
+      Content-Length:
+      - '67'
+    body:
+      encoding: UTF-8
+      string: |-
+        {
+          "versions": [
+            "v1"
+          ]
+        }
+    http_version: 
+  recorded_at: Fri, 08 May 2015 10:35:37 GMT
+- request:
+    method: get
+    uri: https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept:
+      - "*/*; q=0.5, application/xml"
+      Accept-Encoding:
+      - gzip, deflate
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json
+      Date:
+      - Fri, 08 May 2015 10:35:37 GMT
+      Transfer-Encoding:
+      - chunked
+    body:
+      encoding: UTF-8
+      string: |-
+        {
+          "kind": "Pod",
+          "apiVersion": "v1",
+          "metadata": {
+            "name": "fabric8-console-controller-98rqc",
+            "generateName": "fabric8-console-controller-",
+            "namespace": "default",
+            "selfLink": "/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc",
+            "uid": "c76927af-f563-11e4-b32d-54ee7527188d",
+            "resourceVersion": "122",
+            "creationTimestamp": "2015-05-08T09:22:42Z",
+            "labels": {
+              "kubernetes.io/test": "somevalue"
+            }
+          },
+          "spec": {
+            "volumes": [
+              {
+                "name": "openshift-cert-secrets",
+                "hostPath": null,
+                "emptyDir": null,
+                "gcePersistentDisk": null,
+                "gitRepo": null,
+                "secret": {
+                  "secretName": "openshift-cert-secrets"
+                },
+                "nfs": null,
+                "iscsi": null,
+                "glusterfs": null
+              }
+            ],
+            "containers": [
+              {
+                "name": "fabric8-console-container",
+                "image": "fabric8/hawtio-kubernetes:latest",
+                "ports": [
+                  {
+                    "containerPort": 9090,
+                    "protocol": "TCP"
+                  }
+                ],
+                "env": [
+                  {
+                    "name": "OAUTH_CLIENT_ID",
+                    "value": "fabric8-console"
+                  },
+                  {
+                    "name": "OAUTH_AUTHORIZE_URI",
+                    "value": "https://localhost:8443/oauth/authorize"
+                  }
+                ],
+                "resources": {},
+                "volumeMounts": [
+                  {
+                    "name": "openshift-cert-secrets",
+                    "readOnly": true,
+                    "mountPath": "/etc/secret-volume"
+                  }
+                ],
+                "terminationMessagePath": "/dev/termination-log",
+                "imagePullPolicy": "IfNotPresent",
+                "capabilities": {}
+              }
+            ],
+            "restartPolicy": "Always",
+            "dnsPolicy": "ClusterFirst",
+            "nodeName": "jimmi-redhat.localnet"
+          },
+          "status": {
+            "phase": "Running",
+            "Condition": [
+              {
+                "type": "Ready",
+                "status": "True"
+              }
+            ],
+            "hostIP": "172.17.42.1",
+            "podIP": "172.17.0.8",
+            "containerStatuses": [
+              {
+                "name": "fabric8-console-container",
+                "state": {
+                  "running": {
+                    "startedAt": "2015-05-08T09:22:44Z"
+                  }
+                },
+                "lastState": {},
+                "ready": true,
+                "restartCount": 0,
+                "image": "fabric8/hawtio-kubernetes:latest",
+                "imageID": "docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303",
+                "containerID": "docker://49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459"
+              }
+            ]
+          }
+        }
+    http_version: 
+  recorded_at: Fri, 08 May 2015 10:35:37 GMT
+recorded_with: VCR 2.9.3

data/test/plugin/test_filter_kubernetes_metadata.rb

@@ -112,10 +112,15 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
           ENV['KUBERNETES_SERVICE_HOST'] = 'localhost'
           ENV['KUBERNETES_SERVICE_PORT'] = '8443'
 
-          d = create_driver('watch false')
-          assert_equal(d.instance.kubernetes_url, "https://localhost:8443/api")
-          assert_false(d.instance.ca_file.present?)
-          assert_false(d.instance.bearer_token_file.present?)
+          Dir.mktmpdir { |dir|
+            d = create_driver("
+              watch false
+              secret_dir #{dir}
+            ")
+            assert_equal(d.instance.kubernetes_url, "https://localhost:8443/api")
+            assert_false(d.instance.ca_file.present?)
+            assert_false(d.instance.bearer_token_file.present?)
+          }
         ensure
           ENV['KUBERNETES_SERVICE_HOST'] = nil
           ENV['KUBERNETES_SERVICE_PORT'] = nil
@@ -297,5 +302,62 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       es = emit_with_tag('non-kubernetes', msg, '')
       assert_equal(msg.merge(json_log), es.instance_variable_get(:@record_array)[0])
     end
+
+    test 'with kubernetes dotted labels, de_dot enabled' do
+      VCR.use_cassette('kubernetes_docker_metadata_dotted_labels') do
+        es = emit()
+        expected_kube_metadata = {
+          docker: {
+              container_id: '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+          },
+          kubernetes: {
+            host:           'jimmi-redhat.localnet',
+            pod_name:       'fabric8-console-controller-98rqc',
+            container_name: 'fabric8-console-container',
+            namespace_name: 'default',
+            pod_id:         'c76927af-f563-11e4-b32d-54ee7527188d',
+            labels: {
+              'kubernetes_io/test': 'somevalue'
+            }
+          }
+        }
+        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
+      end
+    end
+
+    test 'with kubernetes dotted labels, de_dot disabled' do
+      VCR.use_cassette('kubernetes_docker_metadata_dotted_labels') do
+        es = emit({}, '
+          kubernetes_url https://localhost:8443
+          watch false
+          cache_size 1
+          de_dot false
+        ')
+        expected_kube_metadata = {
+          docker: {
+              container_id: '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+          },
+          kubernetes: {
+            host:           'jimmi-redhat.localnet',
+            pod_name:       'fabric8-console-controller-98rqc',
+            container_name: 'fabric8-console-container',
+            namespace_name: 'default',
+            pod_id:         'c76927af-f563-11e4-b32d-54ee7527188d',
+            labels: {
+              'kubernetes.io/test': 'somevalue'
+            }
+          }
+        }
+        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
+      end
+    end
+
+    test 'invalid de_dot_separator config' do
+      assert_raise Fluent::ConfigError do
+        create_driver('
+          de_dot_separator contains.
+        ')
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 0.15.0
+  version: 0.16.0
 platform: ruby
 authors:
 - Jimmi Dyson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-27 00:00:00.000000000 Z
+date: 2016-02-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -224,6 +224,7 @@ files:
 - lib/fluent/plugin/filter_kubernetes_metadata.rb
 - test/cassettes/invalid_api_server_config.yml
 - test/cassettes/kubernetes_docker_metadata.yml
+- test/cassettes/kubernetes_docker_metadata_dotted_labels.yml
 - test/cassettes/kubernetes_docker_metadata_using_bearer_token.yml
 - test/cassettes/metadata_with_namespace_id.yml
 - test/cassettes/non_kubernetes_docker_metadata.yml
@@ -258,6 +259,7 @@ summary: Filter plugin to add Kubernetes metadata
 test_files:
 - test/cassettes/invalid_api_server_config.yml
 - test/cassettes/kubernetes_docker_metadata.yml
+- test/cassettes/kubernetes_docker_metadata_dotted_labels.yml
 - test/cassettes/kubernetes_docker_metadata_using_bearer_token.yml
 - test/cassettes/metadata_with_namespace_id.yml
 - test/cassettes/non_kubernetes_docker_metadata.yml