fluent-plugin-kubernetes_metadata_filter 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7c85f8cb8e0e93bb71aaafe430cbeacedb7877a8
-  data.tar.gz: e7148afb3e76d346a5b1bda9a5952021682b71cb
+  metadata.gz: fc794d4eb88ae0226aaeaa73ed83ca71149db1a3
+  data.tar.gz: 9c69ce241a593c562f4cccb244dc940111a6fc8c
 SHA512:
-  metadata.gz: 06fe87cf491110b44c962ce7f092b4dd55aaeb90db9d229c37f673ccdfd981a38af47670feb2eec7749521a149e0512bfa485340d0513ca084b477ea565869de
-  data.tar.gz: b477f9416509a823f2a400d67784c6dc570158088a21cf65c27847e4db1495dd2e730d6f942694ce95d53844d11026abac7aa69b77ed3bdce86664f7528b5a36
+  metadata.gz: 24575557e613c96da0c79d9592c740f3ced4c0f6c36ff18b9af7aac88483af6b1e790f04b71425aa04a13e9390c629c7ce3a2332fd94dde6b41bf4461a8f47f3
+  data.tar.gz: 2f9053e6a5775b2d4a857b6d3ad67821b64d9007c93f7019d58f0326b59324752fdee04f2305612ee79a8c973653c0e95cdae7d6b3df4d23a3f14118cb0116c3

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-kubernetes_metadata_filter"
-  gem.version       = "0.4.0"
+  gem.version       = "0.5.0"
   gem.authors       = ["Jimmi Dyson"]
   gem.email         = ["jimmidyson@gmail.com"]
   gem.description   = %q{Filter plugin to add Kubernetes metadata}

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -21,30 +21,31 @@ module Fluent
     Fluent::Plugin.register_filter('kubernetes_metadata', self)
 
     config_param :kubernetes_url, :string
-    config_param :cache_size, :integer, :default => 1000
-    config_param :cache_ttl, :integer, :default => 60 * 60
-    config_param :watch, :bool, :default => true
-    config_param :apiVersion, :string, :default => 'v1beta3'
-    config_param :client_cert, :string, :default => ''
-    config_param :client_key, :string, :default => ''
-    config_param :ca_file, :string, :default => ''
-    config_param :verify_ssl, :bool, :default => true
+    config_param :cache_size, :integer, default: 1000
+    config_param :cache_ttl, :integer, default: 60 * 60
+    config_param :watch, :bool, default: true
+    config_param :apiVersion, :string, default: 'v1beta3'
+    config_param :client_cert, :string, default: ''
+    config_param :client_key, :string, default: ''
+    config_param :ca_file, :string, default: ''
+    config_param :verify_ssl, :bool, default: true
     config_param :container_name_to_kubernetes_name_regexp,
                  :string,
                  :default => '\/?[^_]+_(?<pod_container_name>[^\.]+)[^_]+_(?<pod_name>[^_]+)_(?<namespace>[^_]+)'
-    config_param :bearer_token_file, :string, :default => ''
+    config_param :bearer_token_file, :string, default: ''
+    config_param :merge_json_log, :bool, default: true
 
     def get_metadata(pod_name, container_name, namespace)
       begin
         metadata = @client.get_pod(pod_name, namespace)
         if metadata
           return {
-            uid: metadata['metadata']['uid'],
-            namespace: metadata['metadata']['namespace'],
-            pod_name: metadata['metadata']['name'],
+            uid:            metadata['metadata']['uid'],
+            namespace:      metadata['metadata']['namespace'],
+            pod_name:       metadata['metadata']['name'],
             container_name: container_name,
-            labels: metadata['metadata']['labels'].to_h,
-            host: metadata['spec']['host']
+            labels:         metadata['metadata']['labels'].to_h,
+            host:           metadata['spec']['host']
           }
         end
       rescue KubeException
@@ -67,9 +68,9 @@ module Fluent
 
       @client.ssl_options(
         client_cert: @client_cert.present? ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil,
-        client_key: @client_key.present? ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
-        ca_file: @ca_file,
-        verify_ssl: @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
+        client_key:  @client_key.present? ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
+        ca_file:     @ca_file,
+        verify_ssl:  @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
       )
 
       if @bearer_token_file.present?
@@ -86,11 +87,11 @@ module Fluent
       if @cache_ttl < 0
         @cache_ttl = :none
       end
-      @cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl)
+      @cache                                             = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl)
       @container_name_to_kubernetes_name_regexp_compiled = Regexp.compile(@container_name_to_kubernetes_name_regexp)
 
       if @watch
-        thread = Thread.new(self) { |this|
+        thread                    = Thread.new(self) { |this|
           this.start_watch
         }
         thread.abort_on_exception = true
@@ -100,10 +101,10 @@ module Fluent
     def filter_stream(tag, es)
       new_es = MultiEventStream.new
 
-      es.each {|time, record|
+      es.each { |time, record|
         if record.has_key?(:docker) && record[:docker].has_key?(:id) && record[:docker].has_key?(:name)
-          this = self
-          metadata = @cache.getset(record[:docker][:id]){
+          this                = self
+          metadata            = @cache.getset(record[:docker][:id]) {
             match_data = record[:docker][:name].match(@container_name_to_kubernetes_name_regexp_compiled)
             if match_data
               this.get_metadata(
@@ -117,27 +118,47 @@ module Fluent
           record[:kubernetes] = metadata if metadata
         end
 
+        if @merge_json_log
+          record = merge_json_log(record)
+        end
+
         new_es.add(time, record)
       }
 
       new_es
     end
 
+    def merge_json_log(record)
+      if record.has_key?('log')
+        log = record['log'].strip
+        if log[0].eql?('{') && log[-1].eql?('}')
+          begin
+            parsed_log = JSON.parse(log)
+            record = record.merge(parsed_log)
+            unless parsed_log.has_key?('log')
+              record.delete('log')
+            end
+          rescue JSON::ParserError
+          end
+        end
+      end
+      record
+    end
+
     def start_watch
       resource_version = @client.get_pods.resourceVersion
-      watcher = @client.watch_pods(resource_version)
+      watcher          = @client.watch_pods(resource_version)
       watcher.each do |notice|
-        puts notice
         case notice.type
           when 'MODIFIED'
             if notice.object.status.containerStatuses
               notice.object.status.containerStatuses.each { |container_status|
                 if container_status['containerId']
                   containerId = container_status['containerId'].sub(/^docker:\/\//, '')
-                  cached = @cache[containerId]
+                  cached      = @cache[containerId]
                   if cached
                     # Only thing that can be modified is labels
-                    cached[:labels] = v.object.metadata.labels.to_h
+                    cached[:labels]     = v.object.metadata.labels.to_h
                     @cache[containerId] = cached
                   end
                 end

data/test/plugin/test_filter_kubernetes_metadata.rb

@@ -168,5 +168,21 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       end
     end
 
+    test 'merges json log data' do
+      VCR.use_cassette('valid_kubernetes_api_server') do
+        json_log = {
+          'hello' => 'world'
+        }
+        msg = {
+          'log' => "#{json_log.to_json}"
+        }
+        es = emit(msg, '
+            kubernetes_url https://localhost:8443
+            verify_ssl false
+            watch false
+          ')
+        assert_equal(json_log, es.instance_variable_get(:@record_array)[0])
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Jimmi Dyson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-15 00:00:00.000000000 Z
+date: 2015-05-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd