fluent-plugin-viaq_data_model 0.0.8 → 0.0.9
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a18e41903af1963acc60e14cfa67a97e1150c2e4
|
4
|
+
data.tar.gz: 16197edf63812241935d2c2d29e806865bfbe7eb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 67b5c1ac90d3b88aa6e0d7bf27a613fe9480adec00ae22d4e7ab1ecf5287ff292744799dfafb7abdd43165d6fa7d8f0ab4ea4fe93997781e804853261ad541fa
|
7
|
+
data.tar.gz: 8a459c3fa776e944a397a0bb8443ec7d245696f86ba3df9c670d4d9604855142fee65b2061f433d862ad147f3981909a2a65a5ef7578f16327a6606112fa7de5
|
@@ -7,7 +7,7 @@ FLUENTD_VERSION = ENV['FLUENTD_VERSION'] || "0.12.0"
|
|
7
7
|
|
8
8
|
Gem::Specification.new do |gem|
|
9
9
|
gem.name = "fluent-plugin-viaq_data_model"
|
10
|
-
gem.version = "0.0.
|
10
|
+
gem.version = "0.0.9"
|
11
11
|
gem.authors = ["Rich Megginson"]
|
12
12
|
gem.email = ["rmeggins@redhat.com"]
|
13
13
|
gem.description = %q{Filter plugin to ensure data is in the ViaQ common data model}
|
@@ -225,14 +225,13 @@ module Fluent
|
|
225
225
|
|
226
226
|
def process_sys_var_log_fields(tag, time, record, fmtr_type = nil)
|
227
227
|
record['systemd'] = {"t" => {"PID" => record['pid']}, "u" => {"SYSLOG_IDENTIFIER" => record['ident']}}
|
228
|
-
|
229
|
-
rectime = record['time'] || time
|
228
|
+
if record[@dest_time_name].nil? # e.g. already has @timestamp
|
230
229
|
# handle the case where the time reported in /var/log/messages is for a previous year
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
record['time'] = rectime.utc.to_datetime.rfc3339(6)
|
230
|
+
timeobj = Time.at(time)
|
231
|
+
if timeobj > Time.now
|
232
|
+
timeobj = Time.new((timeobj.year - 1), timeobj.month, timeobj.day, timeobj.hour, timeobj.min, timeobj.sec, timeobj.utc_offset)
|
235
233
|
end
|
234
|
+
record['time'] = timeobj.utc.to_datetime.rfc3339(6)
|
236
235
|
end
|
237
236
|
if record['host'].eql?('localhost') && @docker_hostname
|
238
237
|
record['hostname'] = @docker_hostname
|
@@ -244,13 +243,21 @@ module Fluent
|
|
244
243
|
def process_k8s_json_file_fields(tag, time, record, fmtr_type = nil)
|
245
244
|
record['message'] = record['message'] || record['log']
|
246
245
|
record['level'] = (record['stream'] == 'stdout') ? 'info' : 'err'
|
247
|
-
if record
|
248
|
-
|
246
|
+
if record.key?('kubernetes') && record['kubernetes'].respond_to?(:fetch) && \
|
247
|
+
(k8shost = record['kubernetes'].fetch('host', nil))
|
248
|
+
record['hostname'] = k8shost
|
249
249
|
elsif @docker_hostname
|
250
250
|
record['hostname'] = @docker_hostname
|
251
251
|
end
|
252
|
-
|
253
|
-
|
252
|
+
if record[@dest_time_name].nil? # e.g. already has @timestamp
|
253
|
+
unless record['time'].nil?
|
254
|
+
# convert from string - parses a wide variety of formats
|
255
|
+
rectime = Time.parse(record['time'])
|
256
|
+
else
|
257
|
+
# convert from time_t
|
258
|
+
rectime = Time.at(time)
|
259
|
+
end
|
260
|
+
record['time'] = rectime.utc.to_datetime.rfc3339(6)
|
254
261
|
end
|
255
262
|
end
|
256
263
|
|
@@ -68,19 +68,19 @@ module ViaqDataModelFilterSystemd
|
|
68
68
|
def process_journal_fields(tag, time, record, fmtr_type)
|
69
69
|
systemd_t = {}
|
70
70
|
JOURNAL_FIELD_MAP_SYSTEMD_T.each do |jkey, key|
|
71
|
-
if record
|
71
|
+
if record.key?(jkey)
|
72
72
|
systemd_t[key] = record[jkey]
|
73
73
|
end
|
74
74
|
end
|
75
75
|
systemd_u = {}
|
76
76
|
JOURNAL_FIELD_MAP_SYSTEMD_U.each do |jkey, key|
|
77
|
-
if record
|
77
|
+
if record.key?(jkey)
|
78
78
|
systemd_u[key] = record[jkey]
|
79
79
|
end
|
80
80
|
end
|
81
81
|
systemd_k = {}
|
82
82
|
JOURNAL_FIELD_MAP_SYSTEMD_K.each do |jkey, key|
|
83
|
-
if record
|
83
|
+
if record.key?(jkey)
|
84
84
|
systemd_k[key] = record[jkey]
|
85
85
|
end
|
86
86
|
end
|
@@ -106,8 +106,8 @@ module ViaqDataModelFilterSystemd
|
|
106
106
|
end
|
107
107
|
record['level'] = ["emerg", "alert", "crit", "err", "warning", "notice", "info", "debug", "trace", "unknown"][pri_index]
|
108
108
|
JOURNAL_TIME_FIELDS.each do |field|
|
109
|
-
if record[field]
|
110
|
-
record['time'] = Time.at(
|
109
|
+
if (val = record[field])
|
110
|
+
record['time'] = Time.at(val.to_f / 1000000.0).utc.to_datetime.rfc3339(6)
|
111
111
|
break
|
112
112
|
end
|
113
113
|
end
|
@@ -121,8 +121,9 @@ module ViaqDataModelFilterSystemd
|
|
121
121
|
end
|
122
122
|
when :k8s_journal
|
123
123
|
record['message'] = record['message'] || record['MESSAGE'] || record['log']
|
124
|
-
if record
|
125
|
-
|
124
|
+
if record.key?('kubernetes') && record['kubernetes'].respond_to?(:fetch) && \
|
125
|
+
(k8shost = record['kubernetes'].fetch('host', nil))
|
126
|
+
record['hostname'] = k8shost
|
126
127
|
elsif @docker_hostname
|
127
128
|
record['hostname'] = @docker_hostname
|
128
129
|
else
|
@@ -601,8 +601,9 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
601
601
|
ENV['IPADDR6'] = '::1'
|
602
602
|
ENV['FLUENTD_VERSION'] = 'fversion'
|
603
603
|
ENV['DATA_VERSION'] = 'dversion'
|
604
|
-
|
605
|
-
|
604
|
+
timestr = Time.at(@time).strftime('%b %d %H:%M:%S')
|
605
|
+
expectedtime = Time.at(@time).utc.to_datetime.rfc3339(6)
|
606
|
+
input = {"pid"=>12345,"ident"=>"service","host"=>"myhost","time"=>timestr,"message"=>"mymessage"}
|
606
607
|
rec = emit_with_tag('system.var.log.messages', input, '
|
607
608
|
<formatter>
|
608
609
|
tag "system.var.log**"
|
@@ -615,7 +616,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
615
616
|
assert_equal("service", rec['systemd']['u']['SYSLOG_IDENTIFIER'])
|
616
617
|
assert_equal('mymessage', rec['message'])
|
617
618
|
assert_equal('myhost', rec['hostname'])
|
618
|
-
assert_equal(
|
619
|
+
assert_equal(expectedtime, rec['@timestamp'])
|
619
620
|
assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
|
620
621
|
assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
|
621
622
|
assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
|
@@ -630,10 +631,12 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
630
631
|
ENV['IPADDR6'] = '::1'
|
631
632
|
ENV['FLUENTD_VERSION'] = 'fversion'
|
632
633
|
ENV['DATA_VERSION'] = 'dversion'
|
633
|
-
|
634
|
+
timestr = 'Dec 31 23:59:59'
|
635
|
+
future = DateTime.strptime(timestr, '%b %d %H:%M:%S').to_time
|
636
|
+
@time = future.to_i
|
634
637
|
# subtract 1 from year
|
635
|
-
expected = Time.new(
|
636
|
-
input = {"pid"=>12345,"ident"=>"service","host"=>"myhost","time"=>
|
638
|
+
expected = Time.new(future.year-1, future.month, future.day, future.hour, future.min, future.sec, future.utc_offset)
|
639
|
+
input = {"pid"=>12345,"ident"=>"service","host"=>"myhost","time"=>timestr,"message"=>"mymessage"}
|
637
640
|
rec = emit_with_tag('system.var.log.messages', input, '
|
638
641
|
<formatter>
|
639
642
|
tag "system.var.log**"
|
@@ -656,13 +659,72 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
656
659
|
dellist = 'host,pid,ident'.split(',')
|
657
660
|
dellist.each{|field| assert_nil(rec[field])}
|
658
661
|
end
|
662
|
+
test 'process a /var/log/messages record, already has @timestamp' do
|
663
|
+
ENV['IPADDR4'] = '127.0.0.1'
|
664
|
+
ENV['IPADDR6'] = '::1'
|
665
|
+
ENV['FLUENTD_VERSION'] = 'fversion'
|
666
|
+
ENV['DATA_VERSION'] = 'dversion'
|
667
|
+
timestr = Time.at(@time).strftime('%b %d %H:%M:%S')
|
668
|
+
input = {"pid"=>12345,"ident"=>"service","host"=>"myhost","time"=>timestr,
|
669
|
+
"message"=>"mymessage","@timestamp"=>@timestamp_str}
|
670
|
+
rec = emit_with_tag('system.var.log.messages', input, '
|
671
|
+
rename_time_if_missing true
|
672
|
+
<formatter>
|
673
|
+
tag "system.var.log**"
|
674
|
+
type sys_var_log
|
675
|
+
remove_keys host,pid,ident
|
676
|
+
</formatter>
|
677
|
+
pipeline_type normalizer
|
678
|
+
')
|
679
|
+
assert_equal(12345, rec['systemd']['t']['PID'])
|
680
|
+
assert_equal("service", rec['systemd']['u']['SYSLOG_IDENTIFIER'])
|
681
|
+
assert_equal('mymessage', rec['message'])
|
682
|
+
assert_equal('myhost', rec['hostname'])
|
683
|
+
assert_equal(@timestamp_str, rec['@timestamp'])
|
684
|
+
assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
|
685
|
+
assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
|
686
|
+
assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
|
687
|
+
assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
|
688
|
+
assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
|
689
|
+
assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
|
690
|
+
dellist = 'host,pid,ident'.split(',')
|
691
|
+
dellist.each{|field| assert_nil(rec[field])}
|
692
|
+
end
|
659
693
|
test 'process a k8s json-file record, default settings' do
|
694
|
+
ENV['IPADDR4'] = '127.0.0.1'
|
695
|
+
ENV['IPADDR6'] = '::1'
|
696
|
+
ENV['FLUENTD_VERSION'] = 'fversion'
|
697
|
+
ENV['DATA_VERSION'] = 'dversion'
|
698
|
+
input = {'kubernetes'=>{'host'=>'k8shost'},'stream'=>'stderr','time'=>@timestamp_str,'log'=>'mymessage'}
|
699
|
+
rec = emit_with_tag('kubernetes.var.log.containers.name.name_this_that_other_log', input, '
|
700
|
+
<formatter>
|
701
|
+
tag "kubernetes.var.log.containers**"
|
702
|
+
type k8s_json_file
|
703
|
+
remove_keys log,stream
|
704
|
+
</formatter>
|
705
|
+
pipeline_type normalizer
|
706
|
+
')
|
707
|
+
assert_equal('mymessage', rec['message'])
|
708
|
+
assert_equal('k8shost', rec['hostname'])
|
709
|
+
assert_equal('err', rec['level'])
|
710
|
+
assert_equal(@timestamp_str, rec['@timestamp'])
|
711
|
+
assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
|
712
|
+
assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
|
713
|
+
assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
|
714
|
+
assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
|
715
|
+
assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
|
716
|
+
assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
|
717
|
+
dellist = 'host,pid,ident'.split(',')
|
718
|
+
dellist.each{|field| assert_nil(rec[field])}
|
719
|
+
end
|
720
|
+
test 'process a k8s json-file record with a string valued timestamp' do
|
660
721
|
ENV['IPADDR4'] = '127.0.0.1'
|
661
722
|
ENV['IPADDR6'] = '::1'
|
662
723
|
ENV['FLUENTD_VERSION'] = 'fversion'
|
663
724
|
ENV['DATA_VERSION'] = 'dversion'
|
664
725
|
now = Time.now
|
665
|
-
input = {'kubernetes'=>{'host'=>'k8shost'},
|
726
|
+
input = {'kubernetes'=>{'host'=>'k8shost'},
|
727
|
+
'stream'=>'stderr','time'=>now.to_datetime.rfc3339(9),'log'=>'mymessage'}
|
666
728
|
rec = emit_with_tag('kubernetes.var.log.containers.name.name_this_that_other_log', input, '
|
667
729
|
<formatter>
|
668
730
|
tag "kubernetes.var.log.containers**"
|
@@ -684,6 +746,93 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
684
746
|
dellist = 'host,pid,ident'.split(',')
|
685
747
|
dellist.each{|field| assert_nil(rec[field])}
|
686
748
|
end
|
749
|
+
test 'process a k8s json-file record with a string valued timestamp, alternate format' do
|
750
|
+
ENV['IPADDR4'] = '127.0.0.1'
|
751
|
+
ENV['IPADDR6'] = '::1'
|
752
|
+
ENV['FLUENTD_VERSION'] = 'fversion'
|
753
|
+
ENV['DATA_VERSION'] = 'dversion'
|
754
|
+
now = Time.now
|
755
|
+
expectedtime = Time.parse(now.to_datetime.ctime).utc.to_datetime.rfc3339(6)
|
756
|
+
input = {'kubernetes'=>{'host'=>'k8shost'},
|
757
|
+
'stream'=>'stderr','time'=>now.to_datetime.ctime,'log'=>'mymessage'}
|
758
|
+
rec = emit_with_tag('kubernetes.var.log.containers.name.name_this_that_other_log', input, '
|
759
|
+
<formatter>
|
760
|
+
tag "kubernetes.var.log.containers**"
|
761
|
+
type k8s_json_file
|
762
|
+
remove_keys log,stream
|
763
|
+
</formatter>
|
764
|
+
pipeline_type normalizer
|
765
|
+
')
|
766
|
+
assert_equal('mymessage', rec['message'])
|
767
|
+
assert_equal('k8shost', rec['hostname'])
|
768
|
+
assert_equal('err', rec['level'])
|
769
|
+
assert_equal(expectedtime, rec['@timestamp'])
|
770
|
+
assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
|
771
|
+
assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
|
772
|
+
assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
|
773
|
+
assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
|
774
|
+
assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
|
775
|
+
assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
|
776
|
+
dellist = 'host,pid,ident'.split(',')
|
777
|
+
dellist.each{|field| assert_nil(rec[field])}
|
778
|
+
end
|
779
|
+
test 'process a k8s json-file record, already has @timestamp' do
|
780
|
+
ENV['IPADDR4'] = '127.0.0.1'
|
781
|
+
ENV['IPADDR6'] = '::1'
|
782
|
+
ENV['FLUENTD_VERSION'] = 'fversion'
|
783
|
+
ENV['DATA_VERSION'] = 'dversion'
|
784
|
+
input = {'kubernetes'=>{'host'=>'k8shost'},'@timestamp'=>@timestamp_str,
|
785
|
+
'stream'=>'stderr','time'=>'ignored','log'=>'mymessage'}
|
786
|
+
rec = emit_with_tag('kubernetes.var.log.containers.name.name_this_that_other_log', input, '
|
787
|
+
rename_time_if_missing true
|
788
|
+
<formatter>
|
789
|
+
tag "kubernetes.var.log.containers**"
|
790
|
+
type k8s_json_file
|
791
|
+
remove_keys log,stream
|
792
|
+
</formatter>
|
793
|
+
pipeline_type normalizer
|
794
|
+
')
|
795
|
+
assert_equal('mymessage', rec['message'])
|
796
|
+
assert_equal('k8shost', rec['hostname'])
|
797
|
+
assert_equal('err', rec['level'])
|
798
|
+
assert_equal(@timestamp_str, rec['@timestamp'])
|
799
|
+
assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
|
800
|
+
assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
|
801
|
+
assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
|
802
|
+
assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
|
803
|
+
assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
|
804
|
+
assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
|
805
|
+
dellist = 'host,pid,ident'.split(',')
|
806
|
+
dellist.each{|field| assert_nil(rec[field])}
|
807
|
+
end
|
808
|
+
test 'process a k8s json-file record, no time field' do
|
809
|
+
ENV['IPADDR4'] = '127.0.0.1'
|
810
|
+
ENV['IPADDR6'] = '::1'
|
811
|
+
ENV['FLUENTD_VERSION'] = 'fversion'
|
812
|
+
ENV['DATA_VERSION'] = 'dversion'
|
813
|
+
input = {'kubernetes'=>{'host'=>'k8shost'},
|
814
|
+
'stream'=>'stderr','log'=>'mymessage'}
|
815
|
+
rec = emit_with_tag('kubernetes.var.log.containers.name.name_this_that_other_log', input, '
|
816
|
+
<formatter>
|
817
|
+
tag "kubernetes.var.log.containers**"
|
818
|
+
type k8s_json_file
|
819
|
+
remove_keys log,stream
|
820
|
+
</formatter>
|
821
|
+
pipeline_type normalizer
|
822
|
+
')
|
823
|
+
assert_equal('mymessage', rec['message'])
|
824
|
+
assert_equal('k8shost', rec['hostname'])
|
825
|
+
assert_equal('err', rec['level'])
|
826
|
+
assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['@timestamp'])
|
827
|
+
assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
|
828
|
+
assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
|
829
|
+
assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
|
830
|
+
assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
|
831
|
+
assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
|
832
|
+
assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
|
833
|
+
dellist = 'host,pid,ident'.split(',')
|
834
|
+
dellist.each{|field| assert_nil(rec[field])}
|
835
|
+
end
|
687
836
|
# tests for elasticsearch index functionality
|
688
837
|
test 'construct an operations index prefix' do
|
689
838
|
rec = emit_with_tag('journal.system', normal_input, '
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-viaq_data_model
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Rich Megginson
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-09-
|
11
|
+
date: 2017-09-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|