fluent-plugin-kubernetes_metadata_filter 1.0.3 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 51343e9798a88739c3959e9cf6e350ec1b213ac3
-  data.tar.gz: d361fbbaa5f7f43832f4ae261746c5715c2e3aa0
+  metadata.gz: b6b445ba2f0982bd35de2efb14c3c63018929a88
+  data.tar.gz: ca6ca38cc0dc1c02617b5656249b81ea3079c463
 SHA512:
-  metadata.gz: 4307c9fc6195ed933cca79e12dbedb6698266943632814c9f25ad669cb909ebab2957317694ce764cc5abed0eabd4366622f2446e0e45f68566ad15f4f3082ce
-  data.tar.gz: 62f89e376091e5d4f856d034a8f47f0b15839e346912840cd3fafa2e7654e91bb4aa990030b22d87cc9056496b9c2aa4f893cbebbf8379993d9616d9aecb634a
+  metadata.gz: 9a6494885196744461c705ee8216ce6dc2471a30819c9346ee4661860b613ca3795e079aa3b80b6a00ebea3fc184bf8eedacfca60d11d24ea5be3a24ccd7a5ab
+  data.tar.gz: da5ed5376cc30f925cbef088aeb8c1aea78cd687964b12b399df835f11de2fb495daac661b69a077343612e544163a51f6fd9674430fe2b9c6dc1a62a724ef5c

data/README.md

@@ -42,8 +42,6 @@ This must used named capture groups for `container_name`, `pod_name` & `namespac
 * `cache_size` - size of the cache of Kubernetes metadata to reduce requests to the API server (default: `1000`)
 * `cache_ttl` - TTL in seconds of each cached element. Set to negative value to disable TTL eviction (default: `3600` - 1 hour)
 * `watch` - set up a watch on pods on the API server for updates to metadata (default: `true`)
-* `merge_json_log` - merge logs in JSON format as top level keys (default: `true`)
-* `preserve_json_log` - preserve JSON logs in raw form in the `log` key, only used if the previous option is true (default: `true`)
 * `de_dot` - replace dots in labels and annotations with configured `de_dot_separator`, required for ElasticSearch 2.x compatibility (default: `true`)
 * `de_dot_separator` - separator to use if `de_dot` is enabled (default: `_`)
 * `use_journal` - If false (default), messages are expected to be formatted and tagged as if read by the fluentd in\_tail plugin with wildcard filename.  If true, messages are expected to be formatted as if read from the systemd journal.  The `MESSAGE` field has the full message.  The `CONTAINER_NAME` field has the encoded k8s metadata (see below).  The `CONTAINER_ID_FULL` field has the full container uuid.  This requires docker to use the `--log-driver=journald` log driver.
@@ -55,6 +53,12 @@ when true (default: `true`)
 * `orphaned_namespace_name` - The namespace to associate with records where the namespace can not be determined (default: `.orphaned`)
 * `orphaned_namespace_id` - The namespace id to associate with records where the namespace can not be determined (default: `orphaned`)
 
+**NOTE:** As of the release 1.1.x of this plugin, it no longer supports parsing the source message into JSON and attaching it to the
+payload.  The following configuration options are removed:
+
+* `merge_json_log`
+* `preserve_json_log`
+
 Reading from the JSON formatted log files with `in_tail` and wildcard filenames:
 ```
 <source>

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-kubernetes_metadata_filter"
-  gem.version       = "1.0.3"
+  gem.version       = "1.1.0"
   gem.authors       = ["Jimmi Dyson"]
   gem.email         = ["jimmidyson@gmail.com"]
   gem.description   = %q{Filter plugin to add Kubernetes metadata}

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -48,8 +48,6 @@ module Fluent
                  :string,
                  :default => 'var\.log\.containers\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
     config_param :bearer_token_file, :string, default: nil
-    config_param :merge_json_log, :bool, default: true
-    config_param :preserve_json_log, :bool, default: true
     config_param :secret_dir, :string, default: '/var/run/secrets/kubernetes.io/serviceaccount'
     config_param :de_dot, :bool, default: true
     config_param :de_dot_separator, :string, default: '_'
@@ -237,10 +235,10 @@ module Fluent
         end
       end
       if @use_journal
-        @merge_json_log_key = 'MESSAGE'
+        log.debug "Will stream from the journal"
         self.class.class_eval { alias_method :filter_stream, :filter_stream_from_journal }
       else
-        @merge_json_log_key = 'log'
+        log.debug "Will stream from the files"
         self.class.class_eval { alias_method :filter_stream, :filter_stream_from_files }
       end
 
@@ -299,13 +297,10 @@ module Fluent
         }
       end
 
-      es.each { |time, record|
-        record = merge_json_log(record) if @merge_json_log
-
+      es.each do |time, record|
         record = record.merge(Marshal.load(Marshal.dump(metadata))) if metadata
-
         new_es.add(time, record)
-      }
+      end
       dump_stats
       new_es
     end
@@ -313,8 +308,7 @@ module Fluent
     def filter_stream_from_journal(tag, es)
       new_es = MultiEventStream.new
       batch_miss_cache = {}
-      es.each { |time, record|
-        record = merge_json_log(record) if @merge_json_log
+      es.each do |time, record|
         metadata = nil
         if record.has_key?('CONTAINER_NAME') && record.has_key?('CONTAINER_ID_FULL')
           metadata = record['CONTAINER_NAME'].match(@container_name_to_kubernetes_regexp_compiled) do |match_data|
@@ -337,35 +331,15 @@ module Fluent
           @stats.bump(:container_name_id_missing)
         end
 
-        if metadata
-          record = record.merge(metadata)
-        end
+        record = record.merge(metadata) if metadata
 
         new_es.add(time, record)
-      }
+      end
 
       dump_stats
       new_es
     end
 
-    def merge_json_log(record)
-      if record.has_key?(@merge_json_log_key)
-        value = record[@merge_json_log_key].strip
-        if value[0].eql?('{') && value[-1].eql?('}')
-          begin
-            record = JSON.parse(value).merge(record)
-            unless @preserve_json_log
-              record.delete(@merge_json_log_key)
-            end
-          rescue JSON::ParserError=>e
-            @stats.bump(:merge_json_parse_errors)
-            log.debug(e)
-          end
-        end
-      end
-      record
-    end
-
     def de_dot!(h)
       h.keys.each do |ref|
         if h[ref] && ref =~ /\./

data/test/plugin/test_filter_kubernetes_metadata.rb

@@ -397,45 +397,6 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       assert_false(es.instance_variable_get(:@record_array)[0].has_key?(:kubernetes))
     end
 
-    test 'merges json log data' do
-      json_log = {
-        'hello' => 'world'
-      }
-      msg = {
-        'log' => "#{json_log.to_json}"
-      }
-      es = emit_with_tag('non-kubernetes', msg, '')
-      assert_equal(msg.merge(json_log), es.instance_variable_get(:@record_array)[0])
-    end
-
-    test 'merges json log data in MESSAGE' do
-      json_log = {
-        'hello' => 'world'
-      }
-      msg = {
-        'MESSAGE' => "#{json_log.to_json}"
-      }
-      es = emit_with_tag('non-kubernetes', msg, 'use_journal true')
-      assert_equal(msg.merge(json_log), es.instance_variable_get(:@record_array)[0])
-    end
-
-    test 'merges json log data with message field' do
-      json_log = {
-        'timeMillis' => 1459853347608,
-        'thread' => 'main',
-        'level' => 'INFO',
-        'loggerName' => 'org.apache.camel.spring.SpringCamelContext',
-        'message' => 'Total 1 routes, of which 1 is started.',
-        'endOfBatch' => false,
-        'loggerFqcn' => 'org.apache.logging.slf4j.Log4jLogger'
-      }
-      msg = {
-        'log' => "#{json_log.to_json}"
-      }
-      es = emit_with_tag('non-kubernetes', msg, '')
-      assert_equal(msg.merge(json_log), es.instance_variable_get(:@record_array)[0])
-    end
-
     test 'ignores invalid json in log field' do
       json_log = "{'foo':123}"
       msg = {
@@ -445,50 +406,6 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       assert_equal(msg, es.instance_variable_get(:@record_array)[0])
     end
 
-    test 'merges json log data with message field in MESSAGE' do
-      json_log = {
-        'timeMillis' => 1459853347608,
-        'thread' => 'main',
-        'level' => 'INFO',
-        'loggerName' => 'org.apache.camel.spring.SpringCamelContext',
-        'message' => 'Total 1 routes, of which 1 is started.',
-        'endOfBatch' => false,
-        'loggerFqcn' => 'org.apache.logging.slf4j.Log4jLogger'
-      }
-      msg = {
-        'MESSAGE' => "#{json_log.to_json}"
-      }
-      es = emit_with_tag('non-kubernetes', msg, 'use_journal true')
-      assert_equal(msg.merge(json_log), es.instance_variable_get(:@record_array)[0])
-    end
-
-    test 'emit individual fields from json, throw out whole original string' do
-      json_log = {
-        'hello' => 'world',
-        'more' => 'data'
-      }
-      msg = {
-        'log' => "#{json_log.to_json}"
-      }
-      es = emit_with_tag('non-kubernetes', msg, 'preserve_json_log false')
-      assert_equal(json_log, es.instance_variable_get(:@record_array)[0])
-    end
-
-    test 'emit individual fields from json, throw out whole original string in MESSAGE' do
-      json_log = {
-        'hello' => 'world',
-        'more' => 'data'
-      }
-      msg = {
-        'MESSAGE' => "#{json_log.to_json}"
-      }
-      es = emit_with_tag('non-kubernetes', msg, '
-preserve_json_log false
-use_journal true
-')
-      assert_equal(json_log, es.instance_variable_get(:@record_array)[0])
-    end
-
     test 'with kubernetes dotted labels, de_dot enabled' do
       VCR.use_cassette('kubernetes_docker_metadata_dotted_labels') do
         es = emit({}, '

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.1.0
 platform: ruby
 authors:
 - Jimmi Dyson