fluent-plugin-viaq_data_model 0.0.8 → 0.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 533e2b71ea3c79ce83a75a7c157c6603188bc5d2
-  data.tar.gz: 77ad9e855ddf94ca5d87cfe4815be363cf8d6ec1
+  metadata.gz: a18e41903af1963acc60e14cfa67a97e1150c2e4
+  data.tar.gz: 16197edf63812241935d2c2d29e806865bfbe7eb
 SHA512:
-  metadata.gz: d78d938a5e2daf26c447175776b8e03f4e2d47b6fad8c779bb797f6e77211956b7e47a9c8090c2ae9478ad035832d11d7644aa74c8893fa0ddfc06a8c136b38a
-  data.tar.gz: 5613cb33d7c645a607a4e2f90032fb7c18bd0d01f3e27f16af5fc2337d41d9afd27eb7083f9adc0b9fde6681d783d969e8b23b81fcca34cb490e872b904e637b
+  metadata.gz: 67b5c1ac90d3b88aa6e0d7bf27a613fe9480adec00ae22d4e7ab1ecf5287ff292744799dfafb7abdd43165d6fa7d8f0ab4ea4fe93997781e804853261ad541fa
+  data.tar.gz: 8a459c3fa776e944a397a0bb8443ec7d245696f86ba3df9c670d4d9604855142fee65b2061f433d862ad147f3981909a2a65a5ef7578f16327a6606112fa7de5

data/fluent-plugin-viaq_data_model.gemspec

@@ -7,7 +7,7 @@ FLUENTD_VERSION = ENV['FLUENTD_VERSION'] || "0.12.0"
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-viaq_data_model"
-  gem.version       = "0.0.8"
+  gem.version       = "0.0.9"
   gem.authors       = ["Rich Megginson"]
   gem.email         = ["rmeggins@redhat.com"]
   gem.description   = %q{Filter plugin to ensure data is in the ViaQ common data model}

data/lib/fluent/plugin/filter_viaq_data_model.rb

@@ -225,14 +225,13 @@ module Fluent
 
     def process_sys_var_log_fields(tag, time, record, fmtr_type = nil)
       record['systemd'] = {"t" => {"PID" => record['pid']}, "u" => {"SYSLOG_IDENTIFIER" => record['ident']}}
-      unless record[@dest_time_name] # e.g. already has @timestamp
-        rectime = record['time'] || time
+      if record[@dest_time_name].nil? # e.g. already has @timestamp
         # handle the case where the time reported in /var/log/messages is for a previous year
-        if Time.at(rectime) > Time.now
-          record['time'] = Time.new((rectime.year - 1), rectime.month, rectime.day, rectime.hour, rectime.min, rectime.sec, rectime.utc_offset).utc.to_datetime.rfc3339(6)
-        else
-          record['time'] = rectime.utc.to_datetime.rfc3339(6)
+        timeobj = Time.at(time)
+        if timeobj > Time.now
+          timeobj = Time.new((timeobj.year - 1), timeobj.month, timeobj.day, timeobj.hour, timeobj.min, timeobj.sec, timeobj.utc_offset)
         end
+        record['time'] = timeobj.utc.to_datetime.rfc3339(6)
       end
       if record['host'].eql?('localhost') && @docker_hostname
         record['hostname'] = @docker_hostname
@@ -244,13 +243,21 @@ module Fluent
     def process_k8s_json_file_fields(tag, time, record, fmtr_type = nil)
       record['message'] = record['message'] || record['log']
       record['level'] = (record['stream'] == 'stdout') ? 'info' : 'err'
-      if record['kubernetes'] && record['kubernetes']['host']
-        record['hostname'] = record['kubernetes']['host']
+      if record.key?('kubernetes') && record['kubernetes'].respond_to?(:fetch) && \
+         (k8shost = record['kubernetes'].fetch('host', nil))
+        record['hostname'] = k8shost
       elsif @docker_hostname
         record['hostname'] = @docker_hostname
       end
-      unless record[@dest_time_name] # e.g. already has @timestamp
-        record['time'] = record['time'].utc.to_datetime.rfc3339(6)
+      if record[@dest_time_name].nil? # e.g. already has @timestamp
+        unless record['time'].nil?
+          # convert from string - parses a wide variety of formats
+          rectime = Time.parse(record['time'])
+        else
+          # convert from time_t
+          rectime = Time.at(time)
+        end
+        record['time'] = rectime.utc.to_datetime.rfc3339(6)
       end
     end
 

data/lib/fluent/plugin/filter_viaq_data_model_systemd.rb

@@ -68,19 +68,19 @@ module ViaqDataModelFilterSystemd
   def process_journal_fields(tag, time, record, fmtr_type)
     systemd_t = {}
     JOURNAL_FIELD_MAP_SYSTEMD_T.each do |jkey, key|
-      if record[jkey]
+      if record.key?(jkey)
         systemd_t[key] = record[jkey]
       end
     end
     systemd_u = {}
     JOURNAL_FIELD_MAP_SYSTEMD_U.each do |jkey, key|
-      if record[jkey]
+      if record.key?(jkey)
         systemd_u[key] = record[jkey]
       end
     end
     systemd_k = {}
     JOURNAL_FIELD_MAP_SYSTEMD_K.each do |jkey, key|
-      if record[jkey]
+      if record.key?(jkey)
         systemd_k[key] = record[jkey]
       end
     end
@@ -106,8 +106,8 @@ module ViaqDataModelFilterSystemd
     end
     record['level'] = ["emerg", "alert", "crit", "err", "warning", "notice", "info", "debug", "trace", "unknown"][pri_index]
     JOURNAL_TIME_FIELDS.each do |field|
-      if record[field]
-        record['time'] = Time.at(record[field].to_f / 1000000.0).utc.to_datetime.rfc3339(6)
+      if (val = record[field])
+        record['time'] = Time.at(val.to_f / 1000000.0).utc.to_datetime.rfc3339(6)
         break
       end
     end
@@ -121,8 +121,9 @@ module ViaqDataModelFilterSystemd
       end
     when :k8s_journal
       record['message'] = record['message'] || record['MESSAGE'] || record['log']
-      if record['kubernetes'] && record['kubernetes']['host']
-        record['hostname'] = record['kubernetes']['host']
+      if record.key?('kubernetes') && record['kubernetes'].respond_to?(:fetch) && \
+         (k8shost = record['kubernetes'].fetch('host', nil))
+        record['hostname'] = k8shost
       elsif @docker_hostname
         record['hostname'] = @docker_hostname
       else

data/test/test_filter_viaq_data_model.rb

@@ -601,8 +601,9 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
       ENV['IPADDR6'] = '::1'
       ENV['FLUENTD_VERSION'] = 'fversion'
       ENV['DATA_VERSION'] = 'dversion'
-      now = Time.now
-      input = {"pid"=>12345,"ident"=>"service","host"=>"myhost","time"=>now,"message"=>"mymessage"}
+      timestr = Time.at(@time).strftime('%b %d %H:%M:%S')
+      expectedtime = Time.at(@time).utc.to_datetime.rfc3339(6)
+      input = {"pid"=>12345,"ident"=>"service","host"=>"myhost","time"=>timestr,"message"=>"mymessage"}
       rec = emit_with_tag('system.var.log.messages', input, '
         <formatter>
           tag "system.var.log**"
@@ -615,7 +616,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
       assert_equal("service", rec['systemd']['u']['SYSLOG_IDENTIFIER'])
       assert_equal('mymessage', rec['message'])
       assert_equal('myhost', rec['hostname'])
-      assert_equal(now.utc.to_datetime.rfc3339(6), rec['@timestamp'])
+      assert_equal(expectedtime, rec['@timestamp'])
       assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
       assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
       assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
@@ -630,10 +631,12 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
       ENV['IPADDR6'] = '::1'
       ENV['FLUENTD_VERSION'] = 'fversion'
       ENV['DATA_VERSION'] = 'dversion'
-      now = DateTime.strptime('Dec 31 23:59:59', '%b %d %H:%M:%S').to_time.utc
+      timestr = 'Dec 31 23:59:59'
+      future = DateTime.strptime(timestr, '%b %d %H:%M:%S').to_time
+      @time = future.to_i
       # subtract 1 from year
-      expected = Time.new(now.year-1, now.month, now.day, now.hour, now.min, now.sec, now.utc_offset)
-      input = {"pid"=>12345,"ident"=>"service","host"=>"myhost","time"=>now,"message"=>"mymessage"}
+      expected = Time.new(future.year-1, future.month, future.day, future.hour, future.min, future.sec, future.utc_offset)
+      input = {"pid"=>12345,"ident"=>"service","host"=>"myhost","time"=>timestr,"message"=>"mymessage"}
       rec = emit_with_tag('system.var.log.messages', input, '
         <formatter>
           tag "system.var.log**"
@@ -656,13 +659,72 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
       dellist = 'host,pid,ident'.split(',')
       dellist.each{|field| assert_nil(rec[field])}
     end
+    test 'process a /var/log/messages record, already has @timestamp' do
+      ENV['IPADDR4'] = '127.0.0.1'
+      ENV['IPADDR6'] = '::1'
+      ENV['FLUENTD_VERSION'] = 'fversion'
+      ENV['DATA_VERSION'] = 'dversion'
+      timestr = Time.at(@time).strftime('%b %d %H:%M:%S')
+      input = {"pid"=>12345,"ident"=>"service","host"=>"myhost","time"=>timestr,
+               "message"=>"mymessage","@timestamp"=>@timestamp_str}
+      rec = emit_with_tag('system.var.log.messages', input, '
+        rename_time_if_missing true
+        <formatter>
+          tag "system.var.log**"
+          type sys_var_log
+          remove_keys host,pid,ident
+        </formatter>
+        pipeline_type normalizer
+      ')
+      assert_equal(12345, rec['systemd']['t']['PID'])
+      assert_equal("service", rec['systemd']['u']['SYSLOG_IDENTIFIER'])
+      assert_equal('mymessage', rec['message'])
+      assert_equal('myhost', rec['hostname'])
+      assert_equal(@timestamp_str, rec['@timestamp'])
+      assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
+      assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
+      assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
+      assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
+      assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
+      assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
+      dellist = 'host,pid,ident'.split(',')
+      dellist.each{|field| assert_nil(rec[field])}
+    end
     test 'process a k8s json-file record, default settings' do
+      ENV['IPADDR4'] = '127.0.0.1'
+      ENV['IPADDR6'] = '::1'
+      ENV['FLUENTD_VERSION'] = 'fversion'
+      ENV['DATA_VERSION'] = 'dversion'
+      input = {'kubernetes'=>{'host'=>'k8shost'},'stream'=>'stderr','time'=>@timestamp_str,'log'=>'mymessage'}
+      rec = emit_with_tag('kubernetes.var.log.containers.name.name_this_that_other_log', input, '
+        <formatter>
+          tag "kubernetes.var.log.containers**"
+          type k8s_json_file
+          remove_keys log,stream
+        </formatter>
+        pipeline_type normalizer
+      ')
+      assert_equal('mymessage', rec['message'])
+      assert_equal('k8shost', rec['hostname'])
+      assert_equal('err', rec['level'])
+      assert_equal(@timestamp_str, rec['@timestamp'])
+      assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
+      assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
+      assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
+      assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
+      assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
+      assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
+      dellist = 'host,pid,ident'.split(',')
+      dellist.each{|field| assert_nil(rec[field])}
+    end
+    test 'process a k8s json-file record with a string valued timestamp' do
       ENV['IPADDR4'] = '127.0.0.1'
       ENV['IPADDR6'] = '::1'
       ENV['FLUENTD_VERSION'] = 'fversion'
       ENV['DATA_VERSION'] = 'dversion'
       now = Time.now
-      input = {'kubernetes'=>{'host'=>'k8shost'},'stream'=>'stderr','time'=>now,'log'=>'mymessage'}
+      input = {'kubernetes'=>{'host'=>'k8shost'},
+               'stream'=>'stderr','time'=>now.to_datetime.rfc3339(9),'log'=>'mymessage'}
       rec = emit_with_tag('kubernetes.var.log.containers.name.name_this_that_other_log', input, '
         <formatter>
           tag "kubernetes.var.log.containers**"
@@ -684,6 +746,93 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
       dellist = 'host,pid,ident'.split(',')
       dellist.each{|field| assert_nil(rec[field])}
     end
+    test 'process a k8s json-file record with a string valued timestamp, alternate format' do
+      ENV['IPADDR4'] = '127.0.0.1'
+      ENV['IPADDR6'] = '::1'
+      ENV['FLUENTD_VERSION'] = 'fversion'
+      ENV['DATA_VERSION'] = 'dversion'
+      now = Time.now
+      expectedtime = Time.parse(now.to_datetime.ctime).utc.to_datetime.rfc3339(6)
+      input = {'kubernetes'=>{'host'=>'k8shost'},
+               'stream'=>'stderr','time'=>now.to_datetime.ctime,'log'=>'mymessage'}
+      rec = emit_with_tag('kubernetes.var.log.containers.name.name_this_that_other_log', input, '
+        <formatter>
+          tag "kubernetes.var.log.containers**"
+          type k8s_json_file
+          remove_keys log,stream
+        </formatter>
+        pipeline_type normalizer
+      ')
+      assert_equal('mymessage', rec['message'])
+      assert_equal('k8shost', rec['hostname'])
+      assert_equal('err', rec['level'])
+      assert_equal(expectedtime, rec['@timestamp'])
+      assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
+      assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
+      assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
+      assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
+      assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
+      assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
+      dellist = 'host,pid,ident'.split(',')
+      dellist.each{|field| assert_nil(rec[field])}
+    end
+    test 'process a k8s json-file record, already has @timestamp' do
+      ENV['IPADDR4'] = '127.0.0.1'
+      ENV['IPADDR6'] = '::1'
+      ENV['FLUENTD_VERSION'] = 'fversion'
+      ENV['DATA_VERSION'] = 'dversion'
+      input = {'kubernetes'=>{'host'=>'k8shost'},'@timestamp'=>@timestamp_str,
+               'stream'=>'stderr','time'=>'ignored','log'=>'mymessage'}
+      rec = emit_with_tag('kubernetes.var.log.containers.name.name_this_that_other_log', input, '
+        rename_time_if_missing true
+        <formatter>
+          tag "kubernetes.var.log.containers**"
+          type k8s_json_file
+          remove_keys log,stream
+        </formatter>
+        pipeline_type normalizer
+      ')
+      assert_equal('mymessage', rec['message'])
+      assert_equal('k8shost', rec['hostname'])
+      assert_equal('err', rec['level'])
+      assert_equal(@timestamp_str, rec['@timestamp'])
+      assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
+      assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
+      assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
+      assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
+      assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
+      assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
+      dellist = 'host,pid,ident'.split(',')
+      dellist.each{|field| assert_nil(rec[field])}
+    end
+    test 'process a k8s json-file record, no time field' do
+      ENV['IPADDR4'] = '127.0.0.1'
+      ENV['IPADDR6'] = '::1'
+      ENV['FLUENTD_VERSION'] = 'fversion'
+      ENV['DATA_VERSION'] = 'dversion'
+      input = {'kubernetes'=>{'host'=>'k8shost'},
+               'stream'=>'stderr','log'=>'mymessage'}
+      rec = emit_with_tag('kubernetes.var.log.containers.name.name_this_that_other_log', input, '
+        <formatter>
+          tag "kubernetes.var.log.containers**"
+          type k8s_json_file
+          remove_keys log,stream
+        </formatter>
+        pipeline_type normalizer
+      ')
+      assert_equal('mymessage', rec['message'])
+      assert_equal('k8shost', rec['hostname'])
+      assert_equal('err', rec['level'])
+      assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['@timestamp'])
+      assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
+      assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
+      assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
+      assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
+      assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
+      assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
+      dellist = 'host,pid,ident'.split(',')
+      dellist.each{|field| assert_nil(rec[field])}
+    end
     # tests for elasticsearch index functionality
     test 'construct an operations index prefix' do
       rec = emit_with_tag('journal.system', normal_input, '

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-viaq_data_model
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
 platform: ruby
 authors:
 - Rich Megginson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-05 00:00:00.000000000 Z
+date: 2017-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd