fluent-plugin-viaq_data_model 0.0.11 → 0.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b9e299245761d3debf06c0c4ae48413be713f5be
-  data.tar.gz: aa684c6a42046d65849734c7989e3ab4f5435029
+  metadata.gz: 5d9843f79f5f9c77f2d7e846d2db2f9603ef97bb
+  data.tar.gz: c2c3d05a6f76e75274246a11af13d870a6bdf84f
 SHA512:
-  metadata.gz: 150204ed3baf62ed86df87117935753d9f99a94cabcd8f19f75233a5f29ed353f71f0245d93f129c67b2b46c753013edba80a77bf9106d2595065502d7fd66c8
-  data.tar.gz: 8513d7f9af3799646c8e670c45716ab288e6f128af022d78757c36298fbce3d7c62a3702e6c8ea4b0b0b7b33bce352cf030c575d12c41b04db1e2d1f84feb669
+  metadata.gz: 9799e82962f80305db119f3134f36ad9040e3a72515c56c82872e3d72326ca427f54d902a6f765c9c189f0b28d5c1935d83e922971de2d4ae805f2f2cb3a98cd
+  data.tar.gz: dbdff538da0efa79bb3a07891364984c77c78712230fbf6d5460dc164458b35d194a6a79f8cf4911c0e8d19a5f0c730fdc9c2eac184399315e594c453efebf56

data/fluent-plugin-viaq_data_model.gemspec

@@ -7,7 +7,7 @@ FLUENTD_VERSION = ENV['FLUENTD_VERSION'] || "0.12.0"
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-viaq_data_model"
-  gem.version       = "0.0.11"
+  gem.version       = "0.0.12"
   gem.authors       = ["Rich Megginson"]
   gem.email         = ["rmeggins@redhat.com"]
   gem.description   = %q{Filter plugin to ensure data is in the ViaQ common data model}

data/lib/fluent/plugin/filter_viaq_data_model.rb

@@ -225,6 +225,68 @@ module Fluent
       thing
     end
 
+    # https://github.com/ViaQ/elasticsearch-templates/blob/master/namespaces/_default_.yml#L63
+    NORMAL_LEVELS = {
+      'emerg'    => 'emerg',
+      'panic'    => 'emerg',
+      'alert'    => 'alert',
+      'crit'     => 'crit',
+      'critical' => 'crit',
+      'err'      => 'err',
+      'error'    => 'err',
+      'warning'  => 'warning',
+      'warn'     => 'warning',
+      'notice'   => 'notice',
+      'info'     => 'info',
+      'debug'    => 'debug',
+      'trace'    => 'trace',
+      'unknown'  => 'unknown',
+    }
+    # numeric levels for the PRIORITY field
+    PRIORITY_LEVELS = {
+      0 => 'emerg',
+      1 => 'alert',
+      2 => 'crit',
+      3 => 'err',
+      4 => 'warning',
+      5 => 'notice',
+      6 => 'info',
+      7 => 'debug',
+      8 => 'trace',
+      9 => 'unknown',
+      '0' => 'emerg',
+      '1' => 'alert',
+      '2' => 'crit',
+      '3' => 'err',
+      '4' => 'warning',
+      '5' => 'notice',
+      '6' => 'info',
+      '7' => 'debug',
+      '8' => 'trace',
+      '9' => 'unknown',
+    }
+    def normalize_level(level, newlevel, stream=nil, priority=nil)
+      # if the record already has a level field, and it looks like one of our well
+      # known values, convert it to the canonical normalized form - otherwise,
+      # preserve the value in string format
+      retlevel = nil
+      if !level.nil?
+        unless (retlevel = NORMAL_LEVELS[level]) ||
+               (level.respond_to?(:downcase) && (retlevel = NORMAL_LEVELS[level.downcase]))
+          retlevel = level.to_s # don't know what it is - just convert to string
+        end
+      elsif stream == 'stdout'
+        retlevel = 'info'
+      elsif stream == 'stderr'
+        retlevel = 'err'
+      elsif !priority.nil?
+        retlevel = PRIORITY_LEVELS[priority]
+      else
+        retlevel = NORMAL_LEVELS[newlevel]
+      end
+      retlevel || 'unknown'
+    end
+
     def process_sys_var_log_fields(tag, time, record, fmtr_type = nil)
       record['systemd'] = {"t" => {"PID" => record['pid']}, "u" => {"SYSLOG_IDENTIFIER" => record['ident']}}
       if record[@dest_time_name].nil? # e.g. already has @timestamp
@@ -244,7 +306,7 @@ module Fluent
 
     def process_k8s_json_file_fields(tag, time, record, fmtr_type = nil)
       record['message'] = record['message'] || record['log']
-      record['level'] = (record['stream'] == 'stdout') ? 'info' : 'err'
+      record['level'] = normalize_level(record['level'], nil, record['stream'])
       if record.key?('kubernetes') && record['kubernetes'].respond_to?(:fetch) && \
          (k8shost = record['kubernetes'].fetch('host', nil))
         record['hostname'] = k8shost

data/lib/fluent/plugin/filter_viaq_data_model_systemd.rb

@@ -93,18 +93,7 @@ module ViaqDataModelFilterSystemd
     unless systemd_k.empty?
       (record['systemd'] ||= {})['k'] = systemd_k
     end
-    begin
-      pri_index = ('%d' % record['PRIORITY'] || 9).to_i
-      case
-      when pri_index < 0
-        pri_index = 9
-      when pri_index > 9
-        pri_index = 9
-      end
-    rescue
-      pri_index = 9
-    end
-    record['level'] = ["emerg", "alert", "crit", "err", "warning", "notice", "info", "debug", "trace", "unknown"][pri_index]
+    record['level'] = normalize_level(record['level'], nil, nil, record['PRIORITY'])
     JOURNAL_TIME_FIELDS.each do |field|
       if (val = record[field])
         vali = val.to_i

data/test/test_filter_viaq_data_model.rb

@@ -471,6 +471,16 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
       assert_equal('b', rec['a'])
       assert_equal('unknown', rec['level'])
     end
+    test 'try a PRIORITY value that is a number' do
+      rec = emit_with_tag('journal.system', {'a'=>'b', 'PRIORITY'=>4}, '
+        <formatter>
+          tag "journal.system**"
+          type sys_journal
+        </formatter>
+      ')
+      assert_equal('b', rec['a'])
+      assert_equal('warning', rec['level'])
+    end
     test 'test with fallback to __REALTIME_TIMESTAMP' do
       input = normal_input.reject{|k,v| k == '_SOURCE_REALTIME_TIMESTAMP'}
       rec = emit_with_tag('journal.system', input, '
@@ -934,6 +944,87 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
       dellist = 'host,pid,ident'.split(',')
       dellist.each{|field| assert_nil(rec[field])}
     end
+    test 'process a k8s json-file record, stdout stream' do
+      ENV['IPADDR4'] = '127.0.0.1'
+      ENV['IPADDR6'] = '::1'
+      ENV['FLUENTD_VERSION'] = 'fversion'
+      ENV['DATA_VERSION'] = 'dversion'
+      input = {'kubernetes'=>{'host'=>'k8shost'},'stream'=>'stdout','time'=>@timestamp_str,'log'=>'mymessage'}
+      rec = emit_with_tag('kubernetes.var.log.containers.name.name_this_that_other_log', input, '
+        <formatter>
+          tag "kubernetes.var.log.containers**"
+          type k8s_json_file
+          remove_keys log,stream
+        </formatter>
+        pipeline_type normalizer
+      ')
+      assert_equal('mymessage', rec['message'])
+      assert_equal('k8shost', rec['hostname'])
+      assert_equal('info', rec['level'])
+      assert_equal(@timestamp_str, rec['@timestamp'])
+      assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
+      assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
+      assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
+      assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
+      assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
+      assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
+      dellist = 'host,pid,ident'.split(',')
+      dellist.each{|field| assert_nil(rec[field])}
+    end
+    test 'process a k8s json-file record, existing level field' do
+      ENV['IPADDR4'] = '127.0.0.1'
+      ENV['IPADDR6'] = '::1'
+      ENV['FLUENTD_VERSION'] = 'fversion'
+      ENV['DATA_VERSION'] = 'dversion'
+      input = {'kubernetes'=>{'host'=>'k8shost'},'stream'=>'stderr','time'=>@timestamp_str,'log'=>'mymessage','level'=>0}
+      rec = emit_with_tag('kubernetes.var.log.containers.name.name_this_that_other_log', input, '
+        <formatter>
+          tag "kubernetes.var.log.containers**"
+          type k8s_json_file
+          remove_keys log,stream
+        </formatter>
+        pipeline_type normalizer
+      ')
+      assert_equal('mymessage', rec['message'])
+      assert_equal('k8shost', rec['hostname'])
+      assert_equal('0', rec['level'])
+      assert_equal(@timestamp_str, rec['@timestamp'])
+      assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
+      assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
+      assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
+      assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
+      assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
+      assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
+      dellist = 'host,pid,ident'.split(',')
+      dellist.each{|field| assert_nil(rec[field])}
+    end
+    test 'process a k8s json-file record, already normalized existing level field' do
+      ENV['IPADDR4'] = '127.0.0.1'
+      ENV['IPADDR6'] = '::1'
+      ENV['FLUENTD_VERSION'] = 'fversion'
+      ENV['DATA_VERSION'] = 'dversion'
+      input = {'kubernetes'=>{'host'=>'k8shost'},'stream'=>'stderr','time'=>@timestamp_str,'log'=>'mymessage','level'=>'debug'}
+      rec = emit_with_tag('kubernetes.var.log.containers.name.name_this_that_other_log', input, '
+        <formatter>
+          tag "kubernetes.var.log.containers**"
+          type k8s_json_file
+          remove_keys log,stream
+        </formatter>
+        pipeline_type normalizer
+      ')
+      assert_equal('mymessage', rec['message'])
+      assert_equal('k8shost', rec['hostname'])
+      assert_equal('debug', rec['level'])
+      assert_equal(@timestamp_str, rec['@timestamp'])
+      assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
+      assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
+      assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
+      assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
+      assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
+      assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
+      dellist = 'host,pid,ident'.split(',')
+      dellist.each{|field| assert_nil(rec[field])}
+    end
     # tests for elasticsearch index functionality
     test 'construct an operations index prefix' do
       rec = emit_with_tag('journal.system', normal_input, '
@@ -1253,5 +1344,66 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
       ')
       assert_equal('.operations.2017.07.27', rec['viaq_index_name'])
     end
+    # level field processing
+    test 'see if existing level is preserved for journald log' do
+      rec = emit_with_tag('journal.system', {'a'=>'b', 'level'=>'this is my level'}, '
+        <formatter>
+          tag "journal.system**"
+          type sys_journal
+        </formatter>
+      ')
+      assert_equal('b', rec['a'])
+      assert_equal('this is my level', rec['level'])
+    end
+    test 'see if existing level is preserved and overrides PRIORITY' do
+      rec = emit_with_tag('journal.system', {'a'=>'b', 'PRIORITY'=>'3', 'level'=>'this is my level'}, '
+        <formatter>
+          tag "journal.system**"
+          type sys_journal
+        </formatter>
+      ')
+      assert_equal('b', rec['a'])
+      assert_equal('this is my level', rec['level'])
+    end
+    test 'see if existing level is preserved and converted to string' do
+      rec = emit_with_tag('journal.system', {'a'=>'b', 'level'=>1}, '
+        <formatter>
+          tag "journal.system**"
+          type sys_journal
+        </formatter>
+      ')
+      assert_equal('b', rec['a'])
+      assert_equal('1', rec['level'])
+    end
+    test 'see if existing level is preserved and converted to string with an odd type' do
+      rec = emit_with_tag('journal.system', {'a'=>'b', 'level'=>{}}, '
+        <formatter>
+          tag "journal.system**"
+          type sys_journal
+        </formatter>
+      ')
+      assert_equal('b', rec['a'])
+      assert_equal('{}', rec['level'])
+    end
+    test 'see if existing level is normalized to canonical form' do
+      rec = emit_with_tag('journal.system', {'a'=>'b', 'level'=>'CRITICAL'}, '
+        <formatter>
+          tag "journal.system**"
+          type sys_journal
+        </formatter>
+      ')
+      assert_equal('b', rec['a'])
+      assert_equal('crit', rec['level'])
+    end
+    test 'see if existing level already normalized is preserved' do
+      rec = emit_with_tag('journal.system', {'a'=>'b', 'level'=>'crit'}, '
+        <formatter>
+          tag "journal.system**"
+          type sys_journal
+        </formatter>
+      ')
+      assert_equal('b', rec['a'])
+      assert_equal('crit', rec['level'])
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-viaq_data_model
 version: !ruby/object:Gem::Version
-  version: 0.0.11
+  version: 0.0.12
 platform: ruby
 authors:
 - Rich Megginson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-26 00:00:00.000000000 Z
+date: 2017-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd