fluent-plugin-kubernetes_metadata_filter 0.21.0 → 0.22.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b20a61e09187754c82416670f602ee2588399078
-  data.tar.gz: ff91664e2431bf6d419b9a9a4ef04367ce0d8dae
+  metadata.gz: 300f1d004ac647a2b8ecbbd255b25ccc37865794
+  data.tar.gz: 871d3ccd62bc3d9129fd3b9b78858022aa576d40
 SHA512:
-  metadata.gz: 10d00af65128a6b779fb8aab70a39e0ac20d8cf6b3030514eec564d66e3297e04ea43689cf7296e55ab2174332c0d9aac11425bcc506459798c0820cc45ec4fa
-  data.tar.gz: 2bbd2911b66686211fefcf31f62a076b24b37b3784cf4998f1d5f7e5c9238adc2a1752dd8d9a3645762fbf912f787da3941d84b30d8056a9220ce04d24ed35b6
+  metadata.gz: a64f8fc1a0511df3df1afe86c33150b99f3636e95d91e2f385d49fdea82ec7e2d446cc2fe4f1e7cd2d26eb7d47466214c79d3ad6b6f874f0b5ca85d875972e1e
+  data.tar.gz: d6a913cd9c1a0ee7def06ef7ef67b008a3a5c3086e92a65d89b5f3fe9e36e26d4507b17484cc3cf421b7d26d56324fd34a7ee6a9d1aaef1b7ca4d0074d5d82d7

data/README.md

@@ -27,7 +27,10 @@ This must used named capture groups for `container_name`, `pod_name` & `namespac
 * `preserve_json_log` - preserve JSON logs in raw form in the `log` key, only used if the previous option is true (default: `true`)
 * `de_dot` - replace dots in labels with configured `de_dot_separator`, required for ElasticSearch 2.x compatibility (default: `true`)
 * `de_dot_separator` - separator to use if `de_dot` is enabled (default: `_`)
+* `use_journal` - If false (default), messages are expected to be formatted and tagged as if read by the fluentd in\_tail plugin with wildcard filename.  If true, messages are expected to be formatted as if read from the systemd journal.  The `MESSAGE` field has the full message.  The `CONTAINER_NAME` field has the encoded k8s metadata (see below).  The `CONTAINER_ID_FULL` field has the full container uuid.  This requires docker to use the `--log-driver=journald` log driver.
+* `container_name_to_kubernetes_regexp` - The regular expression used to extract the k8s metadata encoded in the journal `CONTAINER_NAME` field (default: `'^k8s_(?<container_name>[^\.]+)\.(?<container_hash>[a-z0-9]{8})_(?<pod_name>[^_]+)_(?<namespace>[^_]+)_(?<pod_id>[^_]+)_(?<pod_randhex>[a-z0-9]{8})$'`)
 
+Reading from the JSON formatted log files with `in_tail` and wildcard filenames:
 ```
 <source>
   type tail
@@ -48,11 +51,39 @@ This must used named capture groups for `container_name`, `pod_name` & `namespac
 </match>
 ```
 
+Reading from the systemd journal (requires the fluentd `fluent-plugin-systemd` and `systemd-journal` plugins, and requires docker to use the `--log-driver=journald` log driver):
+```
+<source>
+  type systemd
+  path /run/log/journal
+  pos_file journal.pos
+  tag journal
+  read_from_head true
+</source>
+
+# probably want to use something like fluent-plugin-rewrite-tag-filter to
+# retag entries from k8s
+<match journal>
+  @type rewrite_tag_filter
+  rewriterule1 CONTAINER_NAME ^k8s_ kubernetes.journal.container
+  ...
+</match>
+
+<filter kubernetes.**>
+  type kubernetes_metadata
+  use_journal true
+</filter>
+
+<match **>
+  type stdout
+</match>
+```
+
 ## Example input/output
 
 Kubernetes creates symlinks to Docker log files in `/var/log/containers/*.log`. Docker logs in JSON format.
 
-Assuming following inputs are coming from a log file:
+Assuming following inputs are coming from a log file named `/var/log/containers/fabric8-console-controller-98rqc_default_fabric8-console-container-df14e0d5ae4c07284fa636d739c8fc2e6b52bc344658de7d3f08c36a2e804115.log`:
 
 ```
 {
@@ -73,9 +104,10 @@ Then output becomes as belows
   "kubernetes": {
     "host": "jimmi-redhat.localnet",
     "pod_name":"fabric8-console-controller-98rqc",
+    "pod_id": "c76927af-f563-11e4-b32d-54ee7527188d",
     "container_name": "fabric8-console-container",
-    "namespace": "default",
-    "uid": "c76927af-f563-11e4-b32d-54ee7527188d",
+    "namespace_name": "default",
+    "namespace_id": "23437884-8e08-4d95-850b-e94378c9b2fd",
     "labels": {
       "component": "fabric8Console"
     }
@@ -83,6 +115,18 @@ Then output becomes as belows
 }
 ```
 
+If using journal input, from docker configured with `--log-driver=journald`, the input looks like the `journalctl -o export` format:
+```
+# The stream identification is encoded into the PRIORITY field as an
+# integer: 6, or github.com/coreos/go-systemd/journal.Info, marks stdout,
+# while 3, or github.com/coreos/go-systemd/journal.Err, marks stderr.
+PRIORITY=6
+CONTAINER_ID=b6cbb6e73c0a
+CONTAINER_ID_FULL=b6cbb6e73c0ad63ab820e4baa97cdc77cec729930e38a714826764ac0491341a
+CONTAINER_NAME=k8s_registry.a49f5318_docker-registry-1-hhoj0_default_ae3a9bdc-1f66-11e6-80a2-fa163e2fff3a_799e4035
+MESSAGE=172.17.0.1 - - [21/May/2016:16:52:05 +0000] "GET /healthz HTTP/1.1" 200 0 "" "Go-http-client/1.1"
+```
+
 ## Contributing
 
 1. Fork it

data/Rakefile

@@ -13,8 +13,8 @@ Rake::TestTask.new(:base_test) do |t|
   #  $ bundle exec rake base_test TEST=test/test_*.rb
   t.libs << 'test'
   t.test_files = Dir['test/**/test_*.rb'].sort
-  t.verbose = true
-  #t.warning = true
+  #t.verbose = true
+  t.warning = false
 end
 
 desc 'Add copyright headers'

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-kubernetes_metadata_filter"
-  gem.version       = "0.21.0"
+  gem.version       = "0.22.0"
   gem.authors       = ["Jimmi Dyson"]
   gem.email         = ["jimmidyson@gmail.com"]
   gem.description   = %q{Filter plugin to add Kubernetes metadata}

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -42,6 +42,14 @@ module Fluent
     config_param :secret_dir, :string, default: '/var/run/secrets/kubernetes.io/serviceaccount'
     config_param :de_dot, :bool, default: true
     config_param :de_dot_separator, :string, default: '_'
+    # if reading from the journal, the record will contain the following fields in the following
+    # format:
+    # CONTAINER_NAME=k8s_$containername.$containerhash_$podname_$namespacename_$poduuid_$rand32bitashex
+    # CONTAINER_FULL_ID=dockeridassha256hexvalue
+    config_param :use_journal, :bool, default: false
+    config_param :container_name_to_kubernetes_regexp,
+                 :string,
+                 :default => '^k8s_(?<container_name>[^\.]+)\.(?<container_hash>[a-z0-9]{8})_(?<pod_name>[^_]+)_(?<namespace>[^_]+)_(?<pod_id>[^_]+)_(?<pod_randhex>[a-z0-9]{8})$'
 
     def syms_to_strs(hsh)
       newhsh = {}
@@ -102,6 +110,7 @@ module Fluent
         @namespace_cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl)
       end
       @tag_to_kubernetes_name_regexp_compiled = Regexp.compile(@tag_to_kubernetes_name_regexp)
+      @container_name_to_kubernetes_regexp_compiled = Regexp.compile(@container_name_to_kubernetes_regexp)
 
       # Use Kubernetes default service account if we're in a pod.
       if @kubernetes_url.nil?
@@ -161,25 +170,30 @@ module Fluent
           end
         end
       end
+      if @use_journal
+        @merge_json_log_key = 'MESSAGE'
+        self.class.class_eval { alias_method :filter_stream, :filter_stream_from_journal }
+      else
+        @merge_json_log_key = 'log'
+        self.class.class_eval { alias_method :filter_stream, :filter_stream_from_files }
+      end
     end
 
-    # NOTE: fluentd requires that records/hashes have string keys, not symbol keys
-    # http://docs.fluentd.org/articles/plugin-development#record-format
-    def filter_stream(tag, es)
+    def filter_stream_from_files(tag, es)
       new_es = MultiEventStream.new
 
       match_data = tag.match(@tag_to_kubernetes_name_regexp_compiled)
 
       if match_data
         metadata = {
-            'docker' => {
-                'container_id' => match_data['docker_id']
-            },
-            'kubernetes' => {
-                'namespace_name' => match_data['namespace'],
-                'pod_name'       => match_data['pod_name'],
-                'container_name' => match_data['container_name']
-            }
+          'docker' => {
+            'container_id' => match_data['docker_id']
+          },
+          'kubernetes' => {
+            'namespace_name' => match_data['namespace'],
+            'pod_name'       => match_data['pod_name'],
+            'container_name' => match_data['container_name']
+          }
         }
 
         if @kubernetes_url.present?
@@ -219,14 +233,79 @@ module Fluent
       new_es
     end
 
+    def filter_stream_from_journal(tag, es)
+      new_es = MultiEventStream.new
+
+      es.each { |time, record|
+        record = merge_json_log(record) if @merge_json_log
+
+        metadata = nil
+        if record.has_key?('CONTAINER_NAME') && record.has_key?('CONTAINER_ID_FULL')
+          metadata = record['CONTAINER_NAME'].match(@container_name_to_kubernetes_regexp_compiled) do |match_data|
+            metadata = {
+              'docker' => {
+                'container_id' => record['CONTAINER_ID_FULL']
+              },
+              'kubernetes' => {
+                'namespace_name' => match_data['namespace'],
+                'pod_name'       => match_data['pod_name'],
+                'container_name' => match_data['container_name']
+              }
+            }
+            if @kubernetes_url.present?
+              cache_key = "#{metadata['kubernetes']['namespace_name']}_#{metadata['kubernetes']['pod_name']}_#{metadata['kubernetes']['container_name']}"
+
+              this     = self
+              metadata = @cache.getset(cache_key) {
+                if metadata
+                  kubernetes_metadata = this.get_metadata(
+                    metadata['kubernetes']['namespace_name'],
+                    metadata['kubernetes']['pod_name'],
+                    metadata['kubernetes']['container_name']
+                  )
+                  metadata['kubernetes'] = kubernetes_metadata if kubernetes_metadata
+                  metadata
+                end
+              }
+              if match_data['pod_id'] && (match_data['pod_id'] != metadata['kubernetes']['pod_id'])
+                log.debug("pod_id #{match_data['pod_id']} from log not equal to pod_id #{metadata['kubernetes']['pod_id']} from kubernetes for #{cache_key}")
+              end
+              if @include_namespace_id
+                namespace_name = metadata['kubernetes']['namespace_name']
+                namespace_id = @namespace_cache.getset(namespace_name) {
+                  namespace = @client.get_namespace(namespace_name)
+                  namespace['metadata']['uid'] if namespace
+                }
+                metadata['kubernetes']['namespace_id'] = namespace_id if namespace_id
+              end
+            end
+            metadata
+          end
+          unless metadata
+            log.debug "Error: could not match CONTAINER_NAME from record #{record}"
+          end
+        elsif record.has_key?('CONTAINER_NAME') && record['CONTAINER_NAME'].start_with?('k8s_')
+          log.debug "Error: no container name and id in record #{record}"
+        end
+
+        if metadata
+          record = record.merge(metadata)
+        end
+
+        new_es.add(time, record)
+      }
+
+      new_es
+    end
+
     def merge_json_log(record)
-      if record.has_key?('log')
-        log = record['log'].strip
+      if record.has_key?(@merge_json_log_key)
+        log = record[@merge_json_log_key].strip
         if log[0].eql?('{') && log[-1].eql?('}')
           begin
             record = JSON.parse(log).merge(record)
             unless @preserve_json_log
-              record.delete('log')
+              record.delete(@merge_json_log_key)
             end
           rescue JSON::ParserError
           end

data/test/plugin/test_filter_kubernetes_metadata.rb

@@ -303,6 +303,17 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       assert_equal(msg.merge(json_log), es.instance_variable_get(:@record_array)[0])
     end
 
+    test 'merges json log data in MESSAGE' do
+      json_log = {
+        'hello' => 'world'
+      }
+      msg = {
+        'MESSAGE' => "#{json_log.to_json}"
+      }
+      es = emit_with_tag('non-kubernetes', msg, 'use_journal true')
+      assert_equal(msg.merge(json_log), es.instance_variable_get(:@record_array)[0])
+    end
+
     test 'merges json log data with message field' do
       json_log = {
         'timeMillis' => 1459853347608,
@@ -320,6 +331,23 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       assert_equal(msg.merge(json_log), es.instance_variable_get(:@record_array)[0])
     end
 
+    test 'merges json log data with message field in MESSAGE' do
+      json_log = {
+        'timeMillis' => 1459853347608,
+        'thread' => 'main',
+        'level' => 'INFO',
+        'loggerName' => 'org.apache.camel.spring.SpringCamelContext',
+        'message' => 'Total 1 routes, of which 1 is started.',
+        'endOfBatch' => false,
+        'loggerFqcn' => 'org.apache.logging.slf4j.Log4jLogger'
+      }
+      msg = {
+        'MESSAGE' => "#{json_log.to_json}"
+      }
+      es = emit_with_tag('non-kubernetes', msg, 'use_journal true')
+      assert_equal(msg.merge(json_log), es.instance_variable_get(:@record_array)[0])
+    end
+
     test 'emit individual fields from json, throw out whole original string' do
       json_log = {
         'hello' => 'world',
@@ -332,6 +360,21 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
       assert_equal(json_log, es.instance_variable_get(:@record_array)[0])
     end
 
+    test 'emit individual fields from json, throw out whole original string in MESSAGE' do
+      json_log = {
+        'hello' => 'world',
+        'more' => 'data'
+      }
+      msg = {
+        'MESSAGE' => "#{json_log.to_json}"
+      }
+      es = emit_with_tag('non-kubernetes', msg, '
+preserve_json_log false
+use_journal true
+')
+      assert_equal(json_log, es.instance_variable_get(:@record_array)[0])
+    end
+
     test 'with kubernetes dotted labels, de_dot enabled' do
       VCR.use_cassette('kubernetes_docker_metadata_dotted_labels') do
         es = emit()
@@ -388,5 +431,76 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
         ')
       end
     end
+
+    test 'with records from journald and docker & kubernetes metadata' do
+      # with use_journal true should ignore tags and use CONTAINER_NAME and CONTAINER_ID_FULL
+      tag = 'var.log.containers.junk1_junk2_junk3-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed450.log'
+      msg = {
+        'CONTAINER_NAME' => 'k8s_fabric8-console-container.db89db89_fabric8-console-controller-98rqc_default_c76927af-f563-11e4-b32d-54ee7527188d_89db89db',
+        'CONTAINER_ID_FULL' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459',
+        'randomfield' => 'randomvalue'
+      }
+      VCR.use_cassette('kubernetes_docker_metadata') do
+        es = emit_with_tag(tag, msg, '
+          kubernetes_url https://localhost:8443
+          watch false
+          cache_size 1
+          use_journal true
+        ')
+        expected_kube_metadata = {
+          'docker' => {
+              'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+          },
+          'kubernetes' => {
+            'host'           => 'jimmi-redhat.localnet',
+            'pod_name'       => 'fabric8-console-controller-98rqc',
+            'container_name' => 'fabric8-console-container',
+            'namespace_name' => 'default',
+            'pod_id'         => 'c76927af-f563-11e4-b32d-54ee7527188d',
+            'labels' => {
+              'component' => 'fabric8Console'
+            }
+          }
+        }.merge(msg)
+        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
+      end
+    end
+
+    test 'with records from journald and docker & kubernetes metadata & namespace_id enabled' do
+      # with use_journal true should ignore tags and use CONTAINER_NAME and CONTAINER_ID_FULL
+      tag = 'var.log.containers.junk1_junk2_junk3-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed450.log'
+      msg = {
+        'CONTAINER_NAME' => 'k8s_fabric8-console-container.db89db89_fabric8-console-controller-98rqc_default_c76927af-f563-11e4-b32d-54ee7527188d_89db89db',
+        'CONTAINER_ID_FULL' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459',
+        'randomfield' => 'randomvalue'
+      }
+      VCR.use_cassette('metadata_with_namespace_id') do
+        es = emit_with_tag(tag, msg, '
+          kubernetes_url https://localhost:8443
+          watch false
+          cache_size 1
+          include_namespace_id true
+          use_journal true
+        ')
+        expected_kube_metadata = {
+          'docker' => {
+              'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
+          },
+          'kubernetes' => {
+            'host'           => 'jimmi-redhat.localnet',
+            'pod_name'       => 'fabric8-console-controller-98rqc',
+            'container_name' => 'fabric8-console-container',
+            'namespace_name' => 'default',
+            'namespace_id'   => '898268c8-4a36-11e5-9d81-42010af0194c',
+            'pod_id'         => 'c76927af-f563-11e4-b32d-54ee7527188d',
+            'labels' => {
+              'component' => 'fabric8Console'
+            }
+          }
+        }.merge(msg)
+        assert_equal(expected_kube_metadata, es.instance_variable_get(:@record_array)[0])
+      end
+    end
+
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_metadata_filter
 version: !ruby/object:Gem::Version
-  version: 0.21.0
+  version: 0.22.0
 platform: ruby
 authors:
 - Jimmi Dyson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-22 00:00:00.000000000 Z
+date: 2016-06-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd