fluent-plugin-kubernetes_metadata_filter 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fc794d4eb88ae0226aaeaa73ed83ca71149db1a3
-  data.tar.gz: 9c69ce241a593c562f4cccb244dc940111a6fc8c
+  metadata.gz: 3c5cc732f8758c1f6d82d606134748d36d7b2b04
+  data.tar.gz: dafc960f1353bc761e535098d3c24f3ed30f078d
 SHA512:
-  metadata.gz: 24575557e613c96da0c79d9592c740f3ced4c0f6c36ff18b9af7aac88483af6b1e790f04b71425aa04a13e9390c629c7ce3a2332fd94dde6b41bf4461a8f47f3
-  data.tar.gz: 2f9053e6a5775b2d4a857b6d3ad67821b64d9007c93f7019d58f0326b59324752fdee04f2305612ee79a8c973653c0e95cdae7d6b3df4d23a3f14118cb0116c3
+  metadata.gz: d4aae90186c4aab8bab96449d6ecf31bb46beed4d7d223571fb217879d3a89f1fde36a211a1ce0585f4174a29a6a8a248bb719e050a0b689706cdb845c74886c
+  data.tar.gz: a0e303e2e1d14b8a1b09db8235d6258b97f7d48d0e56a928232f27ed4f4915a3031742c6ca7188f5e8bd87aa0fc9731e76392f2ee5ec93f4add5fd65f3b28230

data/README.md

@@ -11,35 +11,33 @@
 
 Configuration options for fluent.conf are:
 
-* `kubernetes_url` - URL to the API server. *This is required*
+* `kubernetes_url` - URL to the API server. Set this to retrieve further kubernetes metadata for logs from kubernetes API server
 * `apiVersion` - API version to use (default: `v1beta3`)
 * `ca_file` - path to CA file for Kubernetes server certificate validation
 * `verify_ssl` - validate SSL certificates (default: true)
 * `client_cert` - path to a client cert file to authenticate to the API server
 * `client_key` - path to a client key file to authenticate to the API server
 * `bearer_token_file` - path to a file containing the bearer token to use for authentication
-* `container_name_to_kubernetes_name_regexp` - the regular expression used to extract kubernetes metadata (pod name, container name, namespace) from the Docker container name. This must used named capture groups for `pod_container_name`, `pod_name` & `namespace` (default: `'\/?[^_]+_(?<pod_container_name>[^\.]+)[^_]+_(?<pod_name>[^_]+)_(?<namespace>[^_]+)'`)
+* `tag_to_kubernetes_name_regexp` - the regular expression used to extract kubernetes metadata (pod name, container name, namespace) from the current fluentd tag.
+This must used named capture groups for `container_name`, `pod_name` & `namespace` (default: `\.(?<pod_name>[^\._]+)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$</pod>)`)
 * `cache_size` - size of the cache of Kubernetes metadata to reduce requests to the API server (default: `1000`)
 * `cache_ttl` - TTL in seconds of each cached element. Set to negative value to disable TTL eviction (default: `3600` - 1 hour)
+* `watch` - set up a watch on pods on the API server for updates to metadata (default: `true`)
+* `merge_json_log` - merge logs in JSON format as top level keys (default: `true`)
 
 ```
 <source>
   type tail
-  path /var/lib/docker/containers/*/*-json.log
+  path /var/log/containers/*.log
   pos_file fluentd-docker.pos
   time_format %Y-%m-%dT%H:%M:%S
-  tag docker.*
+  tag kubernetes.*
   format json
   read_from_head true
 </source>
 
-<filter docker.var.lib.docker.containers.*.*.log>
-  type docker_metadata
-</filter>
-
-<filter docker.var.lib.docker.containers.*.*.log>
+<filter kubernetes.var.lib.docker.containers.*.*.log>
   type kubernetes_metadata
-  kubernetes_url https://localhost:8443
 </filter>
 
 <match **>
@@ -49,12 +47,10 @@ Configuration options for fluent.conf are:
 
 ## Example input/output
 
-Docker logs in JSON format. Log files are normally in
-`/var/lib/docker/containers/*/*-json.log`, depending on what your Docker
-data directory is.
+Kubernetes creates symlinks to Docker log files in `/var/log/containers/*.log`. Docker logs in JSON format.
 
 Assuming following inputs are coming from a log file:
-df14e0d5ae4c07284fa636d739c8fc2e6b52bc344658de7d3f08c36a2e804115-json.log:
+
 ```
 {
   "log": "2015/05/05 19:54:41 \n",
@@ -70,11 +66,6 @@ Then output becomes as belows
   "stream": "stderr",
   "docker": {
     "id": "df14e0d5ae4c07284fa636d739c8fc2e6b52bc344658de7d3f08c36a2e804115",
-    "name": "/k8s_fabric8-console-container.efbd6e64_fabric8-console-controller-9knhj_default_8ae2f621-f360-11e4-8d12-54ee7527188d_7ec9aa3e",
-    "container_hostname": "fabric8-console-controller-9knhj",
-    "image": "fabric8/hawtio-kubernetes:latest",
-    "image_id": "b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303",
-    "labels": {}
   }
   "kubernetes": {
     "host": "jimmi-redhat.localnet",

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-kubernetes_metadata_filter"
-  gem.version       = "0.5.0"
+  gem.version       = "0.6.0"
   gem.authors       = ["Jimmi Dyson"]
   gem.email         = ["jimmidyson@gmail.com"]
   gem.description   = %q{Filter plugin to add Kubernetes metadata}

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -20,7 +20,7 @@ module Fluent
   class KubernetesMetadataFilter < Fluent::Filter
     Fluent::Plugin.register_filter('kubernetes_metadata', self)
 
-    config_param :kubernetes_url, :string
+    config_param :kubernetes_url, :string, default: ''
     config_param :cache_size, :integer, default: 1000
     config_param :cache_ttl, :integer, default: 60 * 60
     config_param :watch, :bool, default: true
@@ -29,9 +29,9 @@ module Fluent
     config_param :client_key, :string, default: ''
     config_param :ca_file, :string, default: ''
     config_param :verify_ssl, :bool, default: true
-    config_param :container_name_to_kubernetes_name_regexp,
+    config_param :tag_to_kubernetes_name_regexp,
                  :string,
-                 :default => '\/?[^_]+_(?<pod_container_name>[^\.]+)[^_]+_(?<pod_name>[^_]+)_(?<namespace>[^_]+)'
+                 :default => '\.(?<pod_name>[^\._]+)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
     config_param :bearer_token_file, :string, default: ''
     config_param :merge_json_log, :bool, default: true
 
@@ -40,12 +40,12 @@ module Fluent
         metadata = @client.get_pod(pod_name, namespace)
         if metadata
           return {
-            uid:            metadata['metadata']['uid'],
-            namespace:      metadata['metadata']['namespace'],
-            pod_name:       metadata['metadata']['name'],
-            container_name: container_name,
-            labels:         metadata['metadata']['labels'].to_h,
-            host:           metadata['spec']['host']
+              uid:            metadata['metadata']['uid'],
+              namespace:      metadata['metadata']['namespace'],
+              pod_name:       metadata['metadata']['name'],
+              container_name: container_name,
+              labels:         metadata['metadata']['labels'].to_h,
+              host:           metadata['spec']['host']
           }
         end
       rescue KubeException
@@ -64,59 +64,80 @@ module Fluent
       require 'active_support/core_ext/object/blank'
       require 'lru_redux'
 
-      @client = Kubeclient::Client.new @kubernetes_url, @apiVersion
-
-      @client.ssl_options(
-        client_cert: @client_cert.present? ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil,
-        client_key:  @client_key.present? ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
-        ca_file:     @ca_file,
-        verify_ssl:  @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
-      )
-
-      if @bearer_token_file.present?
-        bearer_token = File.read(@bearer_token_file)
-        @client.bearer_token(bearer_token)
-      end
-
-      begin
-        @client.api_valid?
-      rescue KubeException => kube_error
-        raise Fluent::ConfigError, "Invalid Kubernetes API endpoint: #{kube_error.message}"
-      end
-
       if @cache_ttl < 0
         @cache_ttl = :none
       end
-      @cache                                             = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl)
-      @container_name_to_kubernetes_name_regexp_compiled = Regexp.compile(@container_name_to_kubernetes_name_regexp)
+      @cache                                  = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl)
+      @tag_to_kubernetes_name_regexp_compiled = Regexp.compile(@tag_to_kubernetes_name_regexp)
+
+      if @kubernetes_url.present?
+        @client = Kubeclient::Client.new @kubernetes_url, @apiVersion
+
+        @client.ssl_options(
+            client_cert: @client_cert.present? ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil,
+            client_key:  @client_key.present? ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
+            ca_file:     @ca_file,
+            verify_ssl:  @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
+        )
+
+        if @bearer_token_file.present?
+          bearer_token = File.read(@bearer_token_file)
+          @client.bearer_token(bearer_token)
+        end
 
-      if @watch
-        thread                    = Thread.new(self) { |this|
-          this.start_watch
-        }
-        thread.abort_on_exception = true
+        begin
+          @client.api_valid?
+        rescue KubeException => kube_error
+          raise Fluent::ConfigError, "Invalid Kubernetes API endpoint: #{kube_error.message}"
+        end
+
+        if @watch
+          thread                    = Thread.new(self) { |this|
+            this.start_watch
+          }
+          thread.abort_on_exception = true
+        end
       end
     end
 
     def filter_stream(tag, es)
       new_es = MultiEventStream.new
 
-      es.each { |time, record|
-        if record.has_key?(:docker) && record[:docker].has_key?(:id) && record[:docker].has_key?(:name)
-          this                = self
-          metadata            = @cache.getset(record[:docker][:id]) {
-            match_data = record[:docker][:name].match(@container_name_to_kubernetes_name_regexp_compiled)
-            if match_data
-              this.get_metadata(
-                match_data[:pod_name],
-                match_data[:pod_container_name],
-                match_data[:namespace]
-              )
-            end
-          }
+      match_data = tag.match(@tag_to_kubernetes_name_regexp_compiled)
+
+      if match_data
+        metadata = {
+            docker: {
+                container_id: match_data['docker_id']
+            },
+            kubernetes: {
+                namespace: match_data['namespace'],
+                pod_name: match_data['pod_name'],
+                container_name: match_data['container_name']
+            }
+        }
 
-          record[:kubernetes] = metadata if metadata
+        if @kubernetes_url.present?
+          cache_key = "#{metadata['namespace']}_#{metadata['pod_name']}_#{metadata['pod_container_name']}"
+
+          if cache_key.present?
+            this                = self
+            metadata            = @cache.getset(cache_key) {
+              if metadata
+                metadata[:kubernetes] = this.get_metadata(
+                    metadata[:kubernetes][:pod_name],
+                    metadata[:kubernetes][:container_name],
+                    metadata[:kubernetes][:namespace]
+                )
+                metadata
+              end
+            }
+          end
         end
+      end
+
+      es.each { |time, record|
+        record = record.merge(metadata) if metadata
 
         if @merge_json_log
           record = merge_json_log(record)
@@ -134,7 +155,7 @@ module Fluent
         if log[0].eql?('{') && log[-1].eql?('}')
           begin
             parsed_log = JSON.parse(log)
-            record = record.merge(parsed_log)
+            record     = record.merge(parsed_log)
             unless parsed_log.has_key?('log')
               record.delete('log')
             end

data/test/plugin/test_filter_kubernetes_metadata.rb

@@ -31,14 +31,13 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
   end
 
   def create_driver(conf = '')
-    Test::FilterTestDriver.new(KubernetesMetadataFilter).configure(conf, true)
+    Test::FilterTestDriver.new(KubernetesMetadataFilter, 'var.log.containers.fabric8-console-controller-98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log').configure(conf, true)
   end
 
   sub_test_case 'configure' do
     test 'check default' do
-      assert_raise Fluent::ConfigError do
-        create_driver
-      end
+      d = create_driver
+      assert_equal(1000, d.instance.cache_size)
     end
 
     test 'kubernetes url' do
@@ -80,7 +79,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
 
   sub_test_case 'filter_stream' do
 
-    def emit(msg, config='
+    def emit(msg={}, config='
           kubernetes_url https://localhost:8443
           watch false
           cache_size 1
@@ -91,16 +90,24 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       }.filtered
     end
 
+    def emit_with_tag(tag, msg={}, config='
+          kubernetes_url https://localhost:8443
+          watch false
+          cache_size 1
+        ')
+      d = create_driver(config)
+      d.run {
+        d.emit_with_tag(tag, msg, @time)
+      }.filtered
+    end
+
     test 'with docker & kubernetes metadata' do
       VCR.use_cassette('kubernetes_docker_metadata') do
-        msg = {
-          docker: {
-            id: '/49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459',
-            name: '/k8s_fabric8-console-container.efbd6e64_fabric8-console-controller-98rqc_default_c76927af-f563-11e4-b32d-54ee7527188d_42cbc279'
-          }
-        }
-        es = emit(msg)
+        es = emit()
         expected_kube_metadata = {
+          docker: {
+              container_id: '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+          },
           kubernetes: {
             host:           'jimmi-redhat.localnet',
             pod_name:       'fabric8-console-controller-98rqc',
@@ -112,25 +119,22 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
             }
           }
         }
-        assert_equal(msg.merge(expected_kube_metadata), es.instance_variable_get(:@record_array)[0])
+        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
       end
     end
 
     test 'with docker & kubernetes metadata using bearer token' do
       VCR.use_cassette('kubernetes_docker_metadata_using_bearer_token') do
-        msg = {
-          docker: {
-            id: '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459',
-            name: '/k8s_fabric8-console-container.efbd6e64_fabric8-console-controller-98rqc_default_c76927af-f563-11e4-b32d-54ee7527188d_42cbc279'
-          }
-        }
-        es = emit(msg, '
+        es = emit({}, '
           kubernetes_url https://localhost:8443
           verify_ssl false
           watch false
           bearer_token_file test/plugin/test.token
         ')
         expected_kube_metadata = {
+          docker: {
+            container_id: '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+          },
           kubernetes: {
             host:           'jimmi-redhat.localnet',
             pod_name:       'fabric8-console-controller-98rqc',
@@ -142,47 +146,41 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
             }
           }
         }
-        assert_equal(msg.merge(expected_kube_metadata), es.instance_variable_get(:@record_array)[0])
+        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
       end
     end
 
-    test 'with docker metadata, non-kubernetes' do
-      VCR.use_cassette('non_kubernetes_docker_metadata') do
-        msg = {
+    test 'with docker & kubernetes metadata but no configured api server' do
+      es = emit({}, '')
+      expected_kube_metadata = {
           docker: {
-            id: '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459',
-            name: '/k8s_POD.c0b903ca_fabric8-forge-controller-ymkew_default_bcde9961-f4b7-11e4-bdbf-54ee7527188d_e1f00705'
+              container_id: '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+          },
+          kubernetes: {
+              pod_name:       'fabric8-console-controller-98rqc',
+              container_name: 'fabric8-console-container',
+              namespace:      'default',
           }
-        }
-        es = emit(msg)
-        assert_equal(msg, es.instance_variable_get(:@record_array)[0])
-        assert_false(es.instance_variable_get(:@record_array)[0].has_key?(:kubernetes))
-      end
+      }
+      assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
     end
 
-    test 'without docker metadata' do
-      VCR.use_cassette('valid_kubernetes_api_server') do
-        msg = {'foo' => 'bar'}
-        es = emit(msg)
-        assert_equal(msg, es.instance_variable_get(:@record_array)[0])
-      end
+
+
+    test 'with docker metadata, non-kubernetes' do
+      es = emit_with_tag('non-kubernetes', {}, '')
+      assert_false(es.instance_variable_get(:@record_array)[0].has_key?(:kubernetes))
     end
 
     test 'merges json log data' do
-      VCR.use_cassette('valid_kubernetes_api_server') do
-        json_log = {
-          'hello' => 'world'
-        }
-        msg = {
-          'log' => "#{json_log.to_json}"
-        }
-        es = emit(msg, '
-            kubernetes_url https://localhost:8443
-            verify_ssl false
-            watch false
-          ')
-        assert_equal(json_log, es.instance_variable_get(:@record_array)[0])
-      end
+      json_log = {
+        'hello' => 'world'
+      }
+      msg = {
+        'log' => "#{json_log.to_json}"
+      }
+      es = emit_with_tag('non-kubernetes', msg, '')
+      assert_equal(json_log, es.instance_variable_get(:@record_array)[0])
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Jimmi Dyson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-18 00:00:00.000000000 Z
+date: 2015-05-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd