fluent-plugin-kubernetes_metadata_filter 1.2.2 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: d31ac807f14fb0b2022253ea499c950cf3a549fa805b4145cc68704bf89f94c2
-  data.tar.gz: e4b1065aaa81240212171a9c661da46ec7df01ee6696531cf2a54a4b5c27864a
+SHA1:
+  metadata.gz: 3e76bf94f6dd2d7d375b01379c3a0533670291e7
+  data.tar.gz: 0cf265d4423a322313a6233a9360c0438081f878
 SHA512:
-  metadata.gz: 1161bed66cd4da96c0baf942ad17fbb7fa67de144d2aa53a0c3984c3114adffcd67e47a91602869a491056faecb44beffa5ca04cec12df5493b6b55870b3378b
-  data.tar.gz: '0822363e158a8f0e970a068f72357bc1e2137c24dac17ba01be94463c6e111bb39f9f508187af6a08fddb1c265bced97411a511fadd7988b6d0e5804853b5647'
+  metadata.gz: c52356d4f5edc444bfb26b9f812257cafc250328f7de81708bb7f3e64711c1c85895a46877537d7fd5d17e486f75328900a848d80625e6450922672005b7ce6a
+  data.tar.gz: 4d0e4b4cdfd0734736419d32fe4237130e81fd04cc6612a5ea5f1eca22c69b8300b8ea4baeeca943b63dad93f256aeb7f35bd160f753a1a28ad650d395ba43b4

data/Gemfile

@@ -1,11 +1,7 @@
 source 'https://rubygems.org'
 
 gem 'codeclimate-test-reporter', '<1.0.0', :group => :test, :require => nil
-if RUBY_VERSION == "2.0.0"
-  gem 'rubocop', '<0.51.0', require: false
-else
-  gem 'rubocop', require: false
-end
+gem 'rubocop', require: false
 
 # Specify your gem's dependencies in fluent-plugin-add.gemspec
 gemspec

data/README.md

@@ -42,10 +42,11 @@ This must used named capture groups for `container_name`, `pod_name` & `namespac
 * `cache_size` - size of the cache of Kubernetes metadata to reduce requests to the API server (default: `1000`)
 * `cache_ttl` - TTL in seconds of each cached element. Set to negative value to disable TTL eviction (default: `3600` - 1 hour)
 * `watch` - set up a watch on pods on the API server for updates to metadata (default: `true`)
+* `merge_json_log` - merge logs in JSON format as top level keys (default: `true`)
+* `preserve_json_log` - preserve JSON logs in raw form in the `log` key, only used if the previous option is true (default: `true`)
 * `de_dot` - replace dots in labels and annotations with configured `de_dot_separator`, required for ElasticSearch 2.x compatibility (default: `true`)
 * `de_dot_separator` - separator to use if `de_dot` is enabled (default: `_`)
-* *DEPRECATED* `use_journal` - If false, messages are expected to be formatted and tagged as if read by the fluentd in\_tail plugin with wildcard filename.  If true, messages are expected to be formatted as if read from the systemd journal.  The `MESSAGE` field has the full message.  The `CONTAINER_NAME` field has the encoded k8s metadata (see below).  The `CONTAINER_ID_FULL` field has the full container uuid.  This requires docker to use the `--log-driver=journald` log driver.  If unset (the default), the plugin will use the `CONTAINER_NAME` and `CONTAINER_ID_FULL` fields
-if available, otherwise, will use the tag in the `tag_to_kubernetes_name_regexp` format.
+* `use_journal` - If false (default), messages are expected to be formatted and tagged as if read by the fluentd in\_tail plugin with wildcard filename.  If true, messages are expected to be formatted as if read from the systemd journal.  The `MESSAGE` field has the full message.  The `CONTAINER_NAME` field has the encoded k8s metadata (see below).  The `CONTAINER_ID_FULL` field has the full container uuid.  This requires docker to use the `--log-driver=journald` log driver.
 * `container_name_to_kubernetes_regexp` - The regular expression used to extract the k8s metadata encoded in the journal `CONTAINER_NAME` field (default: `'^(?<name_prefix>[^_]+)_(?<container_name>[^\._]+)(\.(?<container_hash>[^_]+))?_(?<pod_name>[^_]+)_(?<namespace>[^_]+)_[^_]+_[^_]+$'`
   * This corresponds to the definition [in the source](https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/dockertools/docker.go#L317)
 * `annotation_match` - Array of regular expressions matching annotation field names. Matched annotations are added to a log record.
@@ -53,23 +54,6 @@ if available, otherwise, will use the tag in the `tag_to_kubernetes_name_regexp`
 when true (default: `true`)
 * `orphaned_namespace_name` - The namespace to associate with records where the namespace can not be determined (default: `.orphaned`)
 * `orphaned_namespace_id` - The namespace id to associate with records where the namespace can not be determined (default: `orphaned`)
-* `lookup_from_k8s_field` - If the field `kubernetes` is present, lookup the metadata from the given subfields such as `kubernetes.namespace_name`, `kubernetes.pod_name`, etc.  This allows you to avoid having to pass in metadata to lookup in an explicitly formatted tag name or in an explicitly formatted `CONTAINER_NAME` value.  For example, set `kubernetes.namespace_name`, `kubernetes.pod_name`, `kubernetes.container_name`, and `docker.id` in the record, and the filter will fill in the rest. (default: `true`)
-
-**NOTE:** As of the release 1.1.x of this plugin, it no longer supports parsing the source message into JSON and attaching it to the
-payload.  The following configuration options are removed:
-
-* `merge_json_log`
-* `preserve_json_log`
-
-**NOTE** As of this release, the use of `use_journal` is **DEPRECATED**.  If this setting is not present, the plugin will
-attempt to figure out the source of the metadata fields from the following:
-- If `lookup_from_k8s_field true` (the default) and the following fields are present in the record:
-`docker.container_id`, `kubernetes.namespace_name`, `kubernetes.pod_name`, `kubernetes.container_name`,
-then the plugin will use those values as the source to use to lookup the metadata
-- If `use_journal true`, or `use_journal` is unset, and the fields `CONTAINER_NAME` and `CONTAINER_ID_FULL` are present in the record,
-then the plugin will parse those values using `container_name_to_kubernetes_regexp` and use those as the source to lookup the metadata
-- Otherwise, if the tag matches `tag_to_kubernetes_name_regexp`, the plugin will parse the tag and use those values to
-lookup the metdata
 
 Reading from the JSON formatted log files with `in_tail` and wildcard filenames:
 ```
@@ -120,22 +104,6 @@ Reading from the systemd journal (requires the fluentd `fluent-plugin-systemd` a
 </match>
 ```
 
-## Environment variables for Kubernetes
-
-If the name of the Kubernetes node the plugin is running on is set as
-an environment variable with the name `K8S_NODE_NAME`, it will reduce cache
-misses and needless calls to the Kubernetes API.
-
-In the Kubernetes container definition, this is easily accomplished by:
-
-```yaml
-env:
-- name: K8S_NODE_NAME
-  valueFrom:
-    fieldRef:
-      fieldPath: spec.nodeName
-```
-
 ## Example input/output
 
 Kubernetes creates symlinks to Docker log files in `/var/log/containers/*.log`. Docker logs in JSON format.

data/circle.yml

@@ -7,9 +7,11 @@ dependencies:
     - 'rvm-exec 2.1.9 bundle install'
     - 'rvm-exec 2.2.5 bundle install'
     - 'rvm-exec 2.3.1 bundle install'
+    - 'rvm-exec 2.4.3 bundle install'
 
 test:
   override:
     - 'rvm-exec 2.1.9 bundle exec rake test'
     - 'rvm-exec 2.2.5 bundle exec rake test'
     - 'rvm-exec 2.3.1 bundle exec rake test'
+    - 'rvm-exec 2.4.3 bundle exec rake test'

data/fluent-plugin-kubernetes_metadata_filter.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-kubernetes_metadata_filter"
-  gem.version       = "1.2.2"
+  gem.version       = "2.0.0"
   gem.authors       = ["Jimmi Dyson"]
   gem.email         = ["jimmidyson@gmail.com"]
   gem.description   = %q{Filter plugin to add Kubernetes metadata}
@@ -18,23 +18,14 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.has_rdoc      = false
 
-  gem.required_ruby_version = '>= 2.0.0'
+  gem.required_ruby_version = '>= 2.1.0'
 
-  gem.add_runtime_dependency "fluentd", ">= 0.12.0"
+  gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 2']
   gem.add_runtime_dependency "lru_redux"
   gem.add_runtime_dependency "kubeclient", "~> 1.1.4"
-  if RUBY_VERSION == "2.0.0"
-    gem.add_runtime_dependency "public_suffix", "< 3"
-    gem.add_runtime_dependency "parallel", "< 1.14"
-    gem.add_runtime_dependency "rainbow", "< 3"
-  end
 
   gem.add_development_dependency "bundler", "~> 1.3"
-  if RUBY_VERSION == "2.0.0"
-    gem.add_development_dependency "rake", "< 13"
-  else
-    gem.add_development_dependency "rake"
-  end
+  gem.add_development_dependency "rake"
   gem.add_development_dependency "minitest", "~> 4.0"
   gem.add_development_dependency "test-unit", "~> 3.0.2"
   gem.add_development_dependency "test-unit-rr", "~> 1.0.3"

data/lib/fluent/plugin/filter_kubernetes_metadata.rb

@@ -23,8 +23,10 @@ require_relative 'kubernetes_metadata_stats'
 require_relative 'kubernetes_metadata_watch_namespaces'
 require_relative 'kubernetes_metadata_watch_pods'
 
-module Fluent
-  class KubernetesMetadataFilter < Fluent::Filter
+require 'fluent/plugin/filter'
+
+module Fluent::Plugin
+  class KubernetesMetadataFilter < Fluent::Plugin::Filter
     K8_POD_CA_CERT = 'ca.crt'
     K8_POD_TOKEN = 'token'
 
@@ -48,6 +50,8 @@ module Fluent
                  :string,
                  :default => 'var\.log\.containers\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
     config_param :bearer_token_file, :string, default: nil
+    config_param :merge_json_log, :bool, default: true
+    config_param :preserve_json_log, :bool, default: true
     config_param :secret_dir, :string, default: '/var/run/secrets/kubernetes.io/serviceaccount'
     config_param :de_dot, :bool, default: true
     config_param :de_dot_separator, :string, default: '_'
@@ -55,7 +59,7 @@ module Fluent
     # format:
     # CONTAINER_NAME=k8s_$containername.$containerhash_$podname_$namespacename_$poduuid_$rand32bitashex
     # CONTAINER_FULL_ID=dockeridassha256hexvalue
-    config_param :use_journal, :bool, default: nil
+    config_param :use_journal, :bool, default: false
     # Field 2 is the container_hash, field 5 is the pod_id, and field 6 is the pod_randhex
     # I would have included them as named groups, but you can't have named groups that are
     # non-capturing :P
@@ -69,7 +73,6 @@ module Fluent
     config_param :allow_orphans, :bool, default: true
     config_param :orphaned_namespace_name, :string, default: '.orphaned'
     config_param :orphaned_namespace_id, :string, default: 'orphaned'
-    config_param :lookup_from_k8s_field, :bool, default: true
 
     def fetch_pod_metadata(namespace_name, pod_name)
       log.trace("fetching pod metadata: #{namespace_name}/#{pod_name}") if log.trace?
@@ -89,12 +92,12 @@ module Fluent
           rescue Exception=>e
             log.debug(e)
             @stats.bump(:pod_cache_api_nil_bad_resp_payload)
-            log.trace("returning empty metadata for #{namespace_name}/#{pod_name} due to error '#{e}'") if log.trace?
+            log.trace("returning empty metadata for #{namespace_name}/#{pod_name} due to error") if log.trace?
           end
         end
-      rescue Exception=>e
+      rescue KubeException=>e
         @stats.bump(:pod_cache_api_nil_error)
-        log.debug "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
+        log.debug "Exception encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}: #{e.message}"
       end
       {}
     end
@@ -126,17 +129,17 @@ module Fluent
             metadata = parse_namespace_metadata(metadata)
             @stats.bump(:namespace_cache_api_updates)
             log.trace("parsed metadata for #{namespace_name}: #{metadata}") if log.trace?
-            @namespace_cache[metadata['namespace_id']] = metadata
+             @namespace_cache[metadata['namespace_id']] = metadata
             return metadata
           rescue Exception => e
             log.debug(e)
             @stats.bump(:namespace_cache_api_nil_bad_resp_payload)
-            log.trace("returning empty metadata for #{namespace_name} due to error '#{e}'") if log.trace?
+            log.trace("returning empty metadata for #{namespace_name} due to error") if log.trace?
           end
         end
-      rescue Exception => kube_error
+      rescue KubeException => kube_error
         @stats.bump(:namespace_cache_api_nil_error)
-        log.debug "Exception '#{kube_error}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
+        log.debug "Exception encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}: #{kube_error.message}"
       end
       {}
     end
@@ -154,6 +157,7 @@ module Fluent
       end
 
       require 'kubeclient'
+      require 'active_support/core_ext/object/blank'
       require 'lru_redux'
       @stats = KubernetesMetadata::Stats.new
 
@@ -180,23 +184,29 @@ module Fluent
 
       # Use Kubernetes default service account if we're in a pod.
       if @kubernetes_url.nil?
+        log.debug "Kubernetes URL is not set - inspecting environ"
+
         env_host = ENV['KUBERNETES_SERVICE_HOST']
         env_port = ENV['KUBERNETES_SERVICE_PORT']
         if env_host.present? && env_port.present?
           @kubernetes_url = "https://#{env_host}:#{env_port}/api"
+          log.debug "Kubernetes URL is now '#{@kubernetes_url}'"
         end
       end
 
       # Use SSL certificate and bearer token from Kubernetes service account.
       if Dir.exist?(@secret_dir)
+        log.debug "Found directory with secrets: #{@secret_dir}"
         ca_cert = File.join(@secret_dir, K8_POD_CA_CERT)
         pod_token = File.join(@secret_dir, K8_POD_TOKEN)
 
         if !@ca_file.present? and File.exist?(ca_cert)
+          log.debug "Found CA certificate: #{ca_cert}"
           @ca_file = ca_cert
         end
 
         if !@bearer_token_file.present? and File.exist?(pod_token)
+          log.debug "Found pod token: #{pod_token}"
           @bearer_token_file = pod_token
         end
       end
@@ -217,6 +227,7 @@ module Fluent
           auth_options[:bearer_token] = bearer_token
         end
 
+        log.debug "Creating K8S client"
         @client = Kubeclient::Client.new @kubernetes_url, @apiVersion,
                                          ssl_options: ssl_options,
                                          auth_options: auth_options
@@ -234,10 +245,15 @@ module Fluent
           namespace_thread.abort_on_exception = true
         end
       end
-      @time_fields = []
-      @time_fields.push('_SOURCE_REALTIME_TIMESTAMP', '__REALTIME_TIMESTAMP') if @use_journal || @use_journal.nil?
-      @time_fields.push('time') unless @use_journal
-      @time_fields.push('@timestamp') if @lookup_from_k8s_field
+      if @use_journal
+        log.debug "Will stream from the journal"
+        @merge_json_log_key = 'MESSAGE'
+        self.class.class_eval { alias_method :filter_stream, :filter_stream_from_journal }
+      else
+        log.debug "Will stream from the files"
+        @merge_json_log_key = 'log'
+        self.class.class_eval { alias_method :filter_stream, :filter_stream_from_files }
+      end
 
       @annotations_regexps = []
       @annotation_match.each do |regexp|
@@ -250,93 +266,114 @@ module Fluent
 
     end
 
-    def get_metadata_for_record(namespace_name, pod_name, container_name, container_id, create_time, batch_miss_cache)
+    def get_metadata_for_record(match_data, cache_key, create_time, batch_miss_cache)
+      namespace_name = match_data['namespace']
+      pod_name = match_data['pod_name']
       metadata = {
-        'docker' => {'container_id' => container_id},
-        'kubernetes' => {
-          'container_name'  => container_name,
-          'namespace_name'  => namespace_name,
-          'pod_name'        => pod_name
-        }
+        'container_name' => match_data['container_name'],
+        'namespace_name' => namespace_name,
+        'pod_name'       => pod_name
       }
       if @kubernetes_url.present?
-        pod_metadata = get_pod_metadata(container_id, namespace_name, pod_name, create_time, batch_miss_cache)
+        pod_metadata = get_pod_metadata(cache_key, namespace_name, pod_name, create_time, batch_miss_cache)
+        metadata.merge!(pod_metadata) if pod_metadata
+      end
+      metadata
+    end
 
-        if (pod_metadata.include? 'containers') && (pod_metadata['containers'].include? container_id)
-          metadata['kubernetes']['container_image'] = pod_metadata['containers'][container_id]['image']
-          metadata['kubernetes']['container_image_id'] = pod_metadata['containers'][container_id]['image_id']
+    def create_time_from_record(record)
+        time = if @use_journal
+           record['_SOURCE_REALTIME_TIMESTAMP'].nil? ? record['_SOURCE_REALTIME_TIMESTAMP'] : record['__REALTIME_TIMESTAMP']
+        else
+          record['time']
         end
+        (time.nil? || time.chop.empty?) ? Time.now : Time.parse(time)
+    end
 
-        metadata['kubernetes'].merge!(pod_metadata) if pod_metadata
-        metadata['kubernetes'].delete('containers')
-      end
-      metadata
+    def filter_stream(tag, es)
+      es
     end
 
-    def create_time_from_record(record, internal_time)
-      time_key = @time_fields.detect{ |ii| record.has_key?(ii) }
-      time = record[time_key]
-      if time.nil? || time.chop.empty?
-        return internal_time
-      end
-      if ['_SOURCE_REALTIME_TIMESTAMP', '__REALTIME_TIMESTAMP'].include?(time_key)
-        timei= time.to_i
-        return Time.at(timei / 1000000, timei % 1000000)
+    def filter_stream_from_files(tag, es)
+      new_es = Fluent::MultiEventStream.new
+
+      match_data = tag.match(@tag_to_kubernetes_name_regexp_compiled)
+      batch_miss_cache = {}
+      if match_data
+        container_id = match_data['docker_id']
+        metadata = {
+          'docker' => {
+            'container_id' => container_id
+          },
+          'kubernetes' => get_metadata_for_record(match_data, container_id, create_time_from_record(es.first[1]), batch_miss_cache)
+        }
       end
-      return Time.parse(time)
+
+      es.each { |time, record|
+        record = merge_json_log(record) if @merge_json_log
+
+        record = record.merge(Marshal.load(Marshal.dump(metadata))) if metadata
+
+        new_es.add(time, record)
+      }
+      dump_stats
+      new_es
     end
 
-    def filter_stream(tag, es)
-      return es if (es.respond_to?(:empty?) && es.empty?) || !es.is_a?(Fluent::EventStream)
+    def filter_stream_from_journal(tag, es)
       new_es = Fluent::MultiEventStream.new
-      tag_match_data = tag.match(@tag_to_kubernetes_name_regexp_compiled) unless @use_journal
-      tag_metadata = nil
       batch_miss_cache = {}
-      es.each do |time, record|
-        if tag_match_data && tag_metadata.nil?
-          tag_metadata = get_metadata_for_record(tag_match_data['namespace'], tag_match_data['pod_name'], tag_match_data['container_name'],
-            tag_match_data['docker_id'], create_time_from_record(record, time), batch_miss_cache)
-        end
-        metadata = Marshal.load(Marshal.dump(tag_metadata)) if tag_metadata
-        if (@use_journal || @use_journal.nil?) &&
-          (j_metadata = get_metadata_for_journal_record(record, time, batch_miss_cache))
-          metadata = j_metadata
+      es.each { |time, record|
+        record = merge_json_log(record) if @merge_json_log
+        metadata = nil
+        if record.has_key?('CONTAINER_NAME') && record.has_key?('CONTAINER_ID_FULL')
+          metadata = record['CONTAINER_NAME'].match(@container_name_to_kubernetes_regexp_compiled) do |match_data|
+           container_id = record['CONTAINER_ID_FULL']
+            metadata = {
+              'docker' => {
+                'container_id' => container_id
+              },
+              'kubernetes' => get_metadata_for_record(match_data, container_id, create_time_from_record(record), batch_miss_cache)
+            }
+
+            metadata
+          end
+          unless metadata
+            log.debug "Error: could not match CONTAINER_NAME from record #{record}"
+            @stats.bump(:container_name_match_failed)
+          end
+        elsif record.has_key?('CONTAINER_NAME') && record['CONTAINER_NAME'].start_with?('k8s_')
+          log.debug "Error: no container name and id in record #{record}"
+          @stats.bump(:container_name_id_missing)
         end
-        if @lookup_from_k8s_field && record.has_key?('kubernetes') && record.has_key?('docker') &&
-          record['kubernetes'].respond_to?(:has_key?) && record['docker'].respond_to?(:has_key?) &&
-          record['kubernetes'].has_key?('namespace_name') &&
-          record['kubernetes'].has_key?('pod_name') &&
-          record['kubernetes'].has_key?('container_name') &&
-          record['docker'].has_key?('container_id') &&
-          (k_metadata = get_metadata_for_record(record['kubernetes']['namespace_name'], record['kubernetes']['pod_name'],
-            record['kubernetes']['container_name'], record['docker']['container_id'],
-            create_time_from_record(record, time), batch_miss_cache))
-            metadata = k_metadata
+
+        if metadata
+          record = record.merge(metadata)
         end
 
-        record = record.merge(metadata) if metadata
         new_es.add(time, record)
-      end
+      }
+
       dump_stats
       new_es
     end
 
-    def get_metadata_for_journal_record(record, time, batch_miss_cache)
-      metadata = nil
-      if record.has_key?('CONTAINER_NAME') && record.has_key?('CONTAINER_ID_FULL')
-        metadata = record['CONTAINER_NAME'].match(@container_name_to_kubernetes_regexp_compiled) do |match_data|
-          get_metadata_for_record(match_data['namespace'], match_data['pod_name'], match_data['container_name'],
-            record['CONTAINER_ID_FULL'], create_time_from_record(record, time), batch_miss_cache)
-        end
-        unless metadata
-          log.debug "Error: could not match CONTAINER_NAME from record #{record}"
-          @stats.bump(:container_name_match_failed)
+    def merge_json_log(record)
+      if record.has_key?(@merge_json_log_key)
+        value = record[@merge_json_log_key].strip
+        if value[0].eql?('{') && value[-1].eql?('}')
+          begin
+            record = JSON.parse(value).merge(record)
+            unless @preserve_json_log
+              record.delete(@merge_json_log_key)
+            end
+          rescue JSON::ParserError=>e
+            @stats.bump(:merge_json_parse_errors)
+            log.debug(e)
+          end
         end
-      elsif record.has_key?('CONTAINER_NAME') && record['CONTAINER_NAME'].start_with?('k8s_')
-        log.debug "Error: no container name and id in record #{record}"
-        @stats.bump(:container_name_id_missing)
       end
-      metadata
+      record
     end
 
     def de_dot!(h)