fluent-plugin-viaq_data_model 0.0.20 → 0.0.21

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cf1d5470d55d205da4f4509713986c34ac7e10ca6f8b6fda68bdf2d810a34f06
-  data.tar.gz: 3e09f85cacd9c778b92dced23fa27115b44c4695454f730022e13751e07987f4
+  metadata.gz: cbe4139e57813f44bb6a18d68ff720ce1f6ea5faac0a67d20e001210e01d8c80
+  data.tar.gz: 617ebea9745c86549a26db46a88576fde34ef08e43f165ca0da2864d133854f7
 SHA512:
-  metadata.gz: 40cb5a22fc967944b0c1127caa757396c813a5da47e90184c723c322c1c682bf0f96cdb250daa4ada4cdc7195a66c59119a3194d928cd3d35c969b92d6d6deaf
-  data.tar.gz: 74aae5ecb6cf4128daf3cca879a93a1d6e7d6bf7417e6a1017fd7e67e92c8301b2e9d05add95aec7b5315e9bbe9cf4e81c8259d18e39b69dcba0731dbc9843d2
+  metadata.gz: 93a8c658581f71116b9bc053d45ecad028092b210c569877bde0e5c395f091503028ccdb3600865e5e0dcdfcf12ebb2cde25c8945210089e7bbe566bd2935c81
+  data.tar.gz: '0831689e1814494e0c6e4eccb25169cef00f7b5cdee3a38c6eb59de8344b6ede98ac38a68bc528c28ca3f5892ccfe99ef45e54f02d9faf943f609faab1f8a8d2'

data/README.md

@@ -89,6 +89,13 @@ See `filter-viaq_data_model.conf` for an example filter configuration.
 * `keep_empty_fields` - comma delimited string - default `''`
   * Always keep these top level fields, even if they are empty
   * `keep_empty_fields message` - keep the `message` field, even if empty
+* `process_kubernetes_events` - boolean - default `true`
+  * If a record looks like a kubernetes event that has been added by the
+  * eventrouter, add its fields under `kubernetes.event`
+  * A record is considered to be a kubernetes eventrouter event if the
+  * record has a top level field called `event` and it is a Hash.  If you
+  * are not sure if this is always true, then define a specific tag or tags
+  * and use a per-formatter `process_kubernetes_events` setting, below
 * `use_undefined` - boolean - default `false`
   * If `true`, move fields not specified in `default_keep_fields` and
   `extra_keep_fields` to the `undefined` top level field.  If you use
@@ -140,8 +147,16 @@ See `filter-viaq_data_model.conf` for an example filter configuration.
     * `sys_var_log` - a record read from `/var/log/messages`
     * `k8s_json_file` - a record read from a `/var/log/containers/*.log` JSON
       formatted container log file
-    * `tag` - the Fluentd tag pattern to match for these records
-    * `remove_keys` - comma delimited list of keys to remove from the record
+  * `tag` - the Fluentd tag pattern to match for these records
+  * `remove_keys` - comma delimited list of keys to remove from the record
+  * `process_kubernetes_events` - boolean - defaults to the global setting above
+    * If a record looks like a kubernetes event that has been added by the
+    * eventrouter, add its fields under `kubernetes.event`
+    * A record is considered to be a kubernetes eventrouter event if the
+    * record has a top level field called `event` and it is a Hash.
+    * This per-formatter setting will override the global setting (see above)
+    * This allows you to set the global setting to `false`, then only process
+    * kubernetes events for this formatter matching this set of `tag` patterns.
 * `pipeline_type` - which part of the pipeline is this? `collector` or
   `normalizer` - the default is `collector`
 * `elasticsearch_index_name` - how to construct Elasticsearch index names or

data/fluent-plugin-viaq_data_model.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-viaq_data_model"
-  gem.version       = "0.0.20"
+  gem.version       = "0.0.21"
   gem.authors       = ["Rich Megginson"]
   gem.email         = ["rmeggins@redhat.com"]
   gem.description   = %q{Filter plugin to ensure data is in the ViaQ common data model}

data/lib/fluent/plugin/filter_viaq_data_model.rb

@@ -119,6 +119,8 @@ module Fluent
       config_param :tag, :string
       desc 'remove these keys from the record - same as record_transformer "remove_keys" field'
       config_param :remove_keys, :string, default: nil
+      desc 'enable/disable processing of kubernetes events'
+      config_param :process_kubernetes_events, :bool, default: nil
     end
 
     desc 'Which part of the pipeline is this - collector, normalizer, etc. for pipeline_metadata'
@@ -178,7 +180,6 @@ module Fluent
         @formatters.each do |fmtr|
           matcher = ViaqMatchClass.new(fmtr.tag, nil)
           fmtr.instance_eval{ @params[:matcher] = matcher }
-          fmtr.instance_eval{ @params[:fmtr_type] = fmtr.type }
           if fmtr.remove_keys
             fmtr.instance_eval{ @params[:fmtr_remove_keys] = fmtr.remove_keys.split(',') }
           else
@@ -193,6 +194,8 @@ module Fluent
             fmtr_func = method(:process_k8s_json_file_fields)
           end
           fmtr.instance_eval{ @params[:fmtr_func] = fmtr_func }
+          proc_k8s_ev = fmtr.process_kubernetes_events.nil? ? @process_kubernetes_events : fmtr.process_kubernetes_events
+          fmtr.instance_eval{ @params[:process_kubernetes_events] = proc_k8s_ev }
         end
         @formatter_cache = {}
         @formatter_cache_nomatch = {}
@@ -200,7 +203,7 @@ module Fluent
       begin
         @docker_hostname = File.open('/etc/docker-hostname') { |f| f.readline }.rstrip
       rescue
-        @docker_hostname = nil
+        @docker_hostname = ENV['NODE_NAME'] || nil
       end
       @ipaddr4 = ENV['IPADDR4'] || '127.0.0.1'
       @ipaddr6 = nil
@@ -303,7 +306,7 @@ module Fluent
       retlevel || 'unknown'
     end
 
-    def process_sys_var_log_fields(tag, time, record, fmtr_type = nil)
+    def process_sys_var_log_fields(tag, time, record, fmtr = nil)
       record['systemd'] = {"t" => {"PID" => record['pid']}, "u" => {"SYSLOG_IDENTIFIER" => record['ident']}}
       if record[@dest_time_name].nil? # e.g. already has @timestamp
         # handle the case where the time reported in /var/log/messages is for a previous year
@@ -320,7 +323,7 @@ module Fluent
       end
     end
 
-    def process_k8s_json_file_fields(tag, time, record, fmtr_type = nil)
+    def process_k8s_json_file_fields(tag, time, record, fmtr = nil)
       record['message'] = record['message'] || record['log']
       record['level'] = normalize_level(record['level'], nil)
       if record.key?('kubernetes') && record['kubernetes'].respond_to?(:fetch) && \
@@ -339,7 +342,7 @@ module Fluent
         end
         record['time'] = rectime.utc.to_datetime.rfc3339(6)
       end
-      transform_eventrouter(tag, record)
+      transform_eventrouter(tag, record, fmtr)
     end
 
     def check_for_match_and_format(tag, time, record)
@@ -355,7 +358,7 @@ module Fluent
           return
         end
       end
-      fmtr.fmtr_func.call(tag, time, record, fmtr.fmtr_type)
+      fmtr.fmtr_func.call(tag, time, record, fmtr)
 
       if record[@dest_time_name].nil? && record['time'].nil?
         record['time'] = Time.at(time).utc.to_datetime.rfc3339(6)
@@ -449,8 +452,8 @@ module Fluent
       end
     end
 
-    def transform_eventrouter(tag, record)
-      return unless @process_kubernetes_events
+    def transform_eventrouter(tag, record, fmtr)
+      return if fmtr.nil? || !fmtr.process_kubernetes_events
       if record.key?("event") && record["event"].respond_to?(:key?)
         if record.key?("verb")
           record["event"]["verb"] = record.delete("verb")

data/lib/fluent/plugin/filter_viaq_data_model_systemd.rb

@@ -68,7 +68,7 @@ module ViaqDataModelFilterSystemd
 
   JOURNAL_TIME_FIELDS = ['_SOURCE_REALTIME_TIMESTAMP', '__REALTIME_TIMESTAMP']
 
-  def process_journal_fields(tag, time, record, fmtr_type)
+  def process_journal_fields(tag, time, record, fmtr)
     systemd_t = {}
     JOURNAL_FIELD_MAP_SYSTEMD_T.each do |jkey, key|
       if record.key?(jkey)
@@ -104,7 +104,7 @@ module ViaqDataModelFilterSystemd
         break
       end
     end
-    case fmtr_type
+    case fmtr.type
     when :sys_journal
       record['message'] = record['MESSAGE']
       if record['_HOSTNAME'].eql?('localhost') && @docker_hostname
@@ -136,7 +136,7 @@ module ViaqDataModelFilterSystemd
       else
         record['hostname'] = record['_HOSTNAME']
       end
-      transform_eventrouter(tag, record)
+      transform_eventrouter(tag, record, fmtr)
     end
   end
 end

data/test/test_filter_viaq_data_model.rb

@@ -884,6 +884,37 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
       dellist = 'host,pid,ident,event,verb'.split(',')
       dellist.each{|field| assert_nil(rec[field])}
     end
+    test 'process a k8s json-file record with event from eventrouter, per formatter setting' do
+      ENV['IPADDR4'] = '127.0.0.1'
+      ENV['IPADDR6'] = '::1'
+      ENV['FLUENTD_VERSION'] = 'fversion'
+      ENV['DATA_VERSION'] = 'dversion'
+      input = {'kubernetes'=>{'host'=>'k8shost'},'stream'=>'stderr','time'=>@timestamp_str,'log'=>'mymessage'}
+      input = add_event(input)
+      rec = emit_with_tag('kubernetes.var.log.containers.name.name_this_that_other_log', input, '
+        <formatter>
+          tag "kubernetes.var.log.containers**"
+          type k8s_json_file
+          remove_keys log,stream
+          process_kubernetes_events true
+        </formatter>
+        pipeline_type collector
+        process_kubernetes_events false
+      ')
+      assert_equal('ADDED', rec['kubernetes']['event']['verb'])
+      assert_equal('event message', rec['message'])
+      assert_equal('mymessage', rec['pipeline_metadata']['collector']['original_raw_message'])
+      assert_equal('unknown', rec['level'])
+      assert_equal('2017-07-27T17:23:46.216527+00:00', rec['@timestamp'])
+      assert_equal('127.0.0.1', rec['pipeline_metadata']['collector']['ipaddr4'])
+      assert_equal('::1', rec['pipeline_metadata']['collector']['ipaddr6'])
+      assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['collector']['inputname'])
+      assert_equal('fluentd', rec['pipeline_metadata']['collector']['name'])
+      assert_equal('fversion dversion', rec['pipeline_metadata']['collector']['version'])
+      assert_equal(@timestamp_str, rec['pipeline_metadata']['collector']['received_at'])
+      dellist = 'host,pid,ident,event,verb'.split(',')
+      dellist.each{|field| assert_nil(rec[field])}
+    end
     test 'process a k8s json-file record with a string valued timestamp' do
       ENV['IPADDR4'] = '127.0.0.1'
       ENV['IPADDR6'] = '::1'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-viaq_data_model
 version: !ruby/object:Gem::Version
-  version: 0.0.20
+  version: 0.0.21
 platform: ruby
 authors:
 - Rich Megginson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-08-06 00:00:00.000000000 Z
+date: 2019-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd