fluent-plugin-kubernetes_metadata_filter 2.0.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3e76bf94f6dd2d7d375b01379c3a0533670291e7
-  data.tar.gz: 0cf265d4423a322313a6233a9360c0438081f878
+  metadata.gz: 58bdc9ed97a93eb46eeec306ad5851b94d3b5656
+  data.tar.gz: 56806c61750f533d2190c0a0915fb1fed0b4d46e
 SHA512:
-  metadata.gz: c52356d4f5edc444bfb26b9f812257cafc250328f7de81708bb7f3e64711c1c85895a46877537d7fd5d17e486f75328900a848d80625e6450922672005b7ce6a
-  data.tar.gz: 4d0e4b4cdfd0734736419d32fe4237130e81fd04cc6612a5ea5f1eca22c69b8300b8ea4baeeca943b63dad93f256aeb7f35bd160f753a1a28ad650d395ba43b4
+  metadata.gz: e38271b6f13d95781df47a4c99474ec9d54bd8926a48a55d2613086079997728902d74b1207986325e36f9ea647b65786b0602f280a8ae5d4c37f3ae4737973a
+  data.tar.gz: 90a46813a80eec7928f35387adc4778d3c07e1e6f61eea0cf3ca1725ef8195bf73816f733bb7c5cb0464cd69269981a8fb7562bc2a3e17ffaf00384624821b74

data/README.md

@@ -42,8 +42,6 @@ This must used named capture groups for `container_name`, `pod_name` & `namespac
 * `cache_size` - size of the cache of Kubernetes metadata to reduce requests to the API server (default: `1000`)
 * `cache_ttl` - TTL in seconds of each cached element. Set to negative value to disable TTL eviction (default: `3600` - 1 hour)
 * `watch` - set up a watch on pods on the API server for updates to metadata (default: `true`)
-* `merge_json_log` - merge logs in JSON format as top level keys (default: `true`)
-* `preserve_json_log` - preserve JSON logs in raw form in the `log` key, only used if the previous option is true (default: `true`)
 * `de_dot` - replace dots in labels and annotations with configured `de_dot_separator`, required for ElasticSearch 2.x compatibility (default: `true`)
 * `de_dot_separator` - separator to use if `de_dot` is enabled (default: `_`)
 * `use_journal` - If false (default), messages are expected to be formatted and tagged as if read by the fluentd in\_tail plugin with wildcard filename.  If true, messages are expected to be formatted as if read from the systemd journal.  The `MESSAGE` field has the full message.  The `CONTAINER_NAME` field has the encoded k8s metadata (see below).  The `CONTAINER_ID_FULL` field has the full container uuid.  This requires docker to use the `--log-driver=journald` log driver.
@@ -55,6 +53,12 @@ when true (default: `true`)
 * `orphaned_namespace_name` - The namespace to associate with records where the namespace can not be determined (default: `.orphaned`)
 * `orphaned_namespace_id` - The namespace id to associate with records where the namespace can not be determined (default: `orphaned`)
 
+**NOTE:** As of the release 1.1.x of this plugin, it no longer supports parsing the source message into JSON and attaching it to the
+payload.  The following configuration options are removed:
+
+* `merge_json_log`
+* `preserve_json_log`
+
 Reading from the JSON formatted log files with `in_tail` and wildcard filenames:
 ```
 <source>
@@ -104,6 +108,22 @@ Reading from the systemd journal (requires the fluentd `fluent-plugin-systemd` a
 </match>
 ```
 
+## Environment variables for Kubernetes
+
+If the name of the Kubernetes node the plugin is running on is set as
+an environment variable with the name `K8S_NODE_NAME`, it will reduce cache
+misses and needless calls to the Kubernetes API.
+
+In the Kubernetes container definition, this is easily accomplished by:
+
+```yaml
+env:
+- name: K8S_NODE_NAME
+  valueFrom:
+    fieldRef:
+      fieldPath: spec.nodeName
+```
+
 ## Example input/output
 
 Kubernetes creates symlinks to Docker log files in `/var/log/containers/*.log`. Docker logs in JSON format.

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-kubernetes_metadata_filter"
-  gem.version       = "2.0.0"
+  gem.version       = "2.1.0"
   gem.authors       = ["Jimmi Dyson"]
   gem.email         = ["jimmidyson@gmail.com"]
   gem.description   = %q{Filter plugin to add Kubernetes metadata}

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -50,8 +50,6 @@ module Fluent::Plugin
                  :string,
                  :default => 'var\.log\.containers\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
     config_param :bearer_token_file, :string, default: nil
-    config_param :merge_json_log, :bool, default: true
-    config_param :preserve_json_log, :bool, default: true
     config_param :secret_dir, :string, default: '/var/run/secrets/kubernetes.io/serviceaccount'
     config_param :de_dot, :bool, default: true
     config_param :de_dot_separator, :string, default: '_'
@@ -247,11 +245,9 @@ module Fluent::Plugin
       end
       if @use_journal
         log.debug "Will stream from the journal"
-        @merge_json_log_key = 'MESSAGE'
         self.class.class_eval { alias_method :filter_stream, :filter_stream_from_journal }
       else
         log.debug "Will stream from the files"
-        @merge_json_log_key = 'log'
         self.class.class_eval { alias_method :filter_stream, :filter_stream_from_files }
       end
 
@@ -309,13 +305,10 @@ module Fluent::Plugin
         }
       end
 
-      es.each { |time, record|
-        record = merge_json_log(record) if @merge_json_log
-
+      es.each do |time, record|
         record = record.merge(Marshal.load(Marshal.dump(metadata))) if metadata
-
         new_es.add(time, record)
-      }
+      end
       dump_stats
       new_es
     end
@@ -323,8 +316,7 @@ module Fluent::Plugin
     def filter_stream_from_journal(tag, es)
       new_es = Fluent::MultiEventStream.new
       batch_miss_cache = {}
-      es.each { |time, record|
-        record = merge_json_log(record) if @merge_json_log
+      es.each do |time, record|
         metadata = nil
         if record.has_key?('CONTAINER_NAME') && record.has_key?('CONTAINER_ID_FULL')
           metadata = record['CONTAINER_NAME'].match(@container_name_to_kubernetes_regexp_compiled) do |match_data|
@@ -347,35 +339,15 @@ module Fluent::Plugin
           @stats.bump(:container_name_id_missing)
         end
 
-        if metadata
-          record = record.merge(metadata)
-        end
+        record = record.merge(metadata) if metadata
 
         new_es.add(time, record)
-      }
+      end
 
       dump_stats
       new_es
     end
 
-    def merge_json_log(record)
-      if record.has_key?(@merge_json_log_key)
-        value = record[@merge_json_log_key].strip
-        if value[0].eql?('{') && value[-1].eql?('}')
-          begin
-            record = JSON.parse(value).merge(record)
-            unless @preserve_json_log
-              record.delete(@merge_json_log_key)
-            end
-          rescue JSON::ParserError=>e
-            @stats.bump(:merge_json_parse_errors)
-            log.debug(e)
-          end
-        end
-      end
-      record
-    end
-
     def de_dot!(h)
       h.keys.each do |ref|
         if h[ref] && ref =~ /\./

data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb

@@ -42,6 +42,9 @@ module KubernetesMetadata
             if cached
               @cache[cache_key] = parse_pod_metadata(notice.object)
               @stats.bump(:pod_cache_watch_updates)
+            elsif ENV['K8S_NODE_NAME'] == notice.object['spec']['nodeName'] then
+              @cache[cache_key] = parse_pod_metadata(notice.object)
+              @stats.bump(:pod_cache_host_updates)
             else
               @stats.bump(:pod_cache_watch_misses)
             end

data/test/plugin/test_filter_kubernetes_metadata.rb

@@ -381,45 +381,6 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       assert_false(filtered[0].has_key?(:kubernetes))
     end
 
-    test 'merges json log data' do
-      json_log = {
-        'hello' => 'world'
-      }
-      msg = {
-        'log' => "#{json_log.to_json}"
-      }
-      filtered = emit_with_tag('non-kubernetes', msg, '')
-      assert_equal(msg.merge(json_log), filtered[0])
-    end
-
-    test 'merges json log data in MESSAGE' do
-      json_log = {
-        'hello' => 'world'
-      }
-      msg = {
-        'MESSAGE' => "#{json_log.to_json}"
-      }
-      filtered = emit_with_tag('non-kubernetes', msg, 'use_journal true')
-      assert_equal(msg.merge(json_log), filtered[0])
-    end
-
-    test 'merges json log data with message field' do
-      json_log = {
-        'timeMillis' => 1459853347608,
-        'thread' => 'main',
-        'level' => 'INFO',
-        'loggerName' => 'org.apache.camel.spring.SpringCamelContext',
-        'message' => 'Total 1 routes, of which 1 is started.',
-        'endOfBatch' => false,
-        'loggerFqcn' => 'org.apache.logging.slf4j.Log4jLogger'
-      }
-      msg = {
-        'log' => "#{json_log.to_json}"
-      }
-      filtered = emit_with_tag('non-kubernetes', msg, '')
-      assert_equal(msg.merge(json_log), filtered[0])
-    end
-
     test 'ignores invalid json in log field' do
       json_log = "{'foo':123}"
       msg = {
@@ -429,50 +390,6 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       assert_equal(msg, filtered[0])
     end
 
-    test 'merges json log data with message field in MESSAGE' do
-      json_log = {
-        'timeMillis' => 1459853347608,
-        'thread' => 'main',
-        'level' => 'INFO',
-        'loggerName' => 'org.apache.camel.spring.SpringCamelContext',
-        'message' => 'Total 1 routes, of which 1 is started.',
-        'endOfBatch' => false,
-        'loggerFqcn' => 'org.apache.logging.slf4j.Log4jLogger'
-      }
-      msg = {
-        'MESSAGE' => "#{json_log.to_json}"
-      }
-      filtered = emit_with_tag('non-kubernetes', msg, 'use_journal true')
-      assert_equal(msg.merge(json_log), filtered[0])
-    end
-
-    test 'emit individual fields from json, throw out whole original string' do
-      json_log = {
-        'hello' => 'world',
-        'more' => 'data'
-      }
-      msg = {
-        'log' => "#{json_log.to_json}"
-      }
-      filtered = emit_with_tag('non-kubernetes', msg, 'preserve_json_log false')
-      assert_equal(json_log, filtered[0])
-    end
-
-    test 'emit individual fields from json, throw out whole original string in MESSAGE' do
-      json_log = {
-        'hello' => 'world',
-        'more' => 'data'
-      }
-      msg = {
-        'MESSAGE' => "#{json_log.to_json}"
-      }
-      filtered = emit_with_tag('non-kubernetes', msg, '
-preserve_json_log false
-use_journal true
-')
-      assert_equal(json_log, filtered[0])
-    end
-
     test 'with kubernetes dotted labels, de_dot enabled' do
       VCR.use_cassette('kubernetes_docker_metadata_dotted_labels') do
         filtered = emit({}, '

data/test/plugin/test_watch_pods.rb

@@ -81,6 +81,17 @@ class DefaultPodWatchStrategyTest < WatchTest
       end
     end
 
+    test 'pod MODIFIED cached when hostname matches' do
+      orig_env_val = ENV['K8S_NODE_NAME']
+      ENV['K8S_NODE_NAME'] = 'aNodeName'
+      @client.stub :watch_pods, [@modified] do
+       start_pod_watch
+       assert_equal(true, @cache.key?('modified_uid'))
+       assert_equal(1, @stats[:pod_cache_host_updates])
+      end
+      ENV['K8S_NODE_NAME'] = orig_env_val
+    end
+
     test 'pod watch notice is updated when MODIFIED is received' do
       @cache['modified_uid'] = {}
       @client.stub :watch_pods, [@modified] do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.1.0
 platform: ruby
 authors:
 - Jimmi Dyson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-19 00:00:00.000000000 Z
+date: 2018-05-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd