fluent-plugin-viaq_data_model 0.0.18 → 0.0.19
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b20471334c7756a6393385dfe17987751ea233c8cac638f592459ea5a3a8692f
|
4
|
+
data.tar.gz: 0d1738ff46f83bba941f415616c374ece1268a89c59f7df333c0311106809e67
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c77bfd1197a5d06b994668034dcd4d662ddb955fd4c3a803fad71a47124f86e28f29f0a063eecfe86d6bf23c9791159ca693405be9e0c4744288345ee3f0d3e0
|
7
|
+
data.tar.gz: 902a1adc2eca199a51b7cb6ce37600c2ec9764eeacbda1e2313410bcde774f1184513d4b87af91b7efd15651c35e2f7b97caf872e6e5f124a9d56232e2e5f433
|
data/README.md
CHANGED
@@ -162,10 +162,10 @@ See `filter-viaq_data_model.conf` for an example filter configuration.
|
|
162
162
|
create the full index name.
|
163
163
|
* `elasticsearch_index_name_field` - name of the field in the record which stores
|
164
164
|
the index name - you should remove this field in the elasticsearch output
|
165
|
-
plugin using the `remove_keys` config parameter - default is `
|
165
|
+
plugin using the `remove_keys` config parameter - default is `viaq_index_name`
|
166
166
|
* `elasticsearch_index_prefix_field` - name of the field in the record which stores
|
167
167
|
the index prefix - you should remove this field in the elasticsearch output
|
168
|
-
plugin using the `remove_keys` config parameter - default is `
|
168
|
+
plugin using the `remove_keys` config parameter - default is `viaq_index_prefix`
|
169
169
|
|
170
170
|
**NOTE** The `formatter` blocks are matched in the given order in the file.
|
171
171
|
This means, don't use `tag "**"` as the first formatter or none of your
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |gem|
|
6
6
|
gem.name = "fluent-plugin-viaq_data_model"
|
7
|
-
gem.version = "0.0.
|
7
|
+
gem.version = "0.0.19"
|
8
8
|
gem.authors = ["Rich Megginson"]
|
9
9
|
gem.email = ["rmeggins@redhat.com"]
|
10
10
|
gem.description = %q{Filter plugin to ensure data is in the ViaQ common data model}
|
@@ -280,7 +280,7 @@ module Fluent
|
|
280
280
|
'8' => 'trace',
|
281
281
|
'9' => 'unknown',
|
282
282
|
}
|
283
|
-
def normalize_level(level, newlevel,
|
283
|
+
def normalize_level(level, newlevel, priority=nil)
|
284
284
|
# if the record already has a level field, and it looks like one of our well
|
285
285
|
# known values, convert it to the canonical normalized form - otherwise,
|
286
286
|
# preserve the value in string format
|
@@ -290,10 +290,6 @@ module Fluent
|
|
290
290
|
(level.respond_to?(:downcase) && (retlevel = NORMAL_LEVELS[level.downcase]))
|
291
291
|
retlevel = level.to_s # don't know what it is - just convert to string
|
292
292
|
end
|
293
|
-
elsif stream == 'stdout'
|
294
|
-
retlevel = 'info'
|
295
|
-
elsif stream == 'stderr'
|
296
|
-
retlevel = 'err'
|
297
293
|
elsif !priority.nil?
|
298
294
|
retlevel = PRIORITY_LEVELS[priority]
|
299
295
|
else
|
@@ -321,7 +317,7 @@ module Fluent
|
|
321
317
|
|
322
318
|
def process_k8s_json_file_fields(tag, time, record, fmtr_type = nil)
|
323
319
|
record['message'] = record['message'] || record['log']
|
324
|
-
record['level'] = normalize_level(record['level'], nil
|
320
|
+
record['level'] = normalize_level(record['level'], nil)
|
325
321
|
if record.key?('kubernetes') && record['kubernetes'].respond_to?(:fetch) && \
|
326
322
|
(k8shost = record['kubernetes'].fetch('host', nil))
|
327
323
|
record['hostname'] = k8shost
|
@@ -21,25 +21,28 @@ require 'date'
|
|
21
21
|
module ViaqDataModelFilterSystemd
|
22
22
|
# map of journal fields to viaq data model field
|
23
23
|
JOURNAL_FIELD_MAP_SYSTEMD_T = {
|
24
|
-
"_AUDIT_LOGINUID"
|
25
|
-
"_AUDIT_SESSION"
|
26
|
-
"_BOOT_ID"
|
27
|
-
"_CAP_EFFECTIVE"
|
28
|
-
"_CMDLINE"
|
29
|
-
"_COMM"
|
30
|
-
"_EXE"
|
31
|
-
"_GID"
|
32
|
-
"
|
33
|
-
"
|
34
|
-
"
|
35
|
-
"
|
36
|
-
"
|
37
|
-
"
|
38
|
-
"
|
39
|
-
"
|
40
|
-
"
|
41
|
-
"
|
42
|
-
"
|
24
|
+
"_AUDIT_LOGINUID" => "AUDIT_LOGINUID",
|
25
|
+
"_AUDIT_SESSION" => "AUDIT_SESSION",
|
26
|
+
"_BOOT_ID" => "BOOT_ID",
|
27
|
+
"_CAP_EFFECTIVE" => "CAP_EFFECTIVE",
|
28
|
+
"_CMDLINE" => "CMDLINE",
|
29
|
+
"_COMM" => "COMM",
|
30
|
+
"_EXE" => "EXE",
|
31
|
+
"_GID" => "GID",
|
32
|
+
"_LINE_BREAK" => "LINE_BREAK",
|
33
|
+
"_MACHINE_ID" => "MACHINE_ID",
|
34
|
+
"_PID" => "PID",
|
35
|
+
"_SELINUX_CONTEXT" => "SELINUX_CONTEXT",
|
36
|
+
"_STREAM_ID" => "STREAM_ID",
|
37
|
+
"_SYSTEMD_CGROUP" => "SYSTEMD_CGROUP",
|
38
|
+
"_SYSTEMD_INVOCATION_ID" => "SYSTEMD_INVOCATION_ID",
|
39
|
+
"_SYSTEMD_OWNER_UID" => "SYSTEMD_OWNER_UID",
|
40
|
+
"_SYSTEMD_SESSION" => "SYSTEMD_SESSION",
|
41
|
+
"_SYSTEMD_SLICE" => "SYSTEMD_SLICE",
|
42
|
+
"_SYSTEMD_UNIT" => "SYSTEMD_UNIT",
|
43
|
+
"_SYSTEMD_USER_UNIT" => "SYSTEMD_USER_UNIT",
|
44
|
+
"_TRANSPORT" => "TRANSPORT",
|
45
|
+
"_UID" => "UID"
|
43
46
|
}
|
44
47
|
|
45
48
|
JOURNAL_FIELD_MAP_SYSTEMD_U = {
|
@@ -93,7 +96,7 @@ module ViaqDataModelFilterSystemd
|
|
93
96
|
unless systemd_k.empty?
|
94
97
|
(record['systemd'] ||= {})['k'] = systemd_k
|
95
98
|
end
|
96
|
-
record['level'] = normalize_level(record['level'], nil,
|
99
|
+
record['level'] = normalize_level(record['level'], nil, record['PRIORITY'])
|
97
100
|
JOURNAL_TIME_FIELDS.each do |field|
|
98
101
|
if (val = record[field])
|
99
102
|
vali = val.to_i
|
@@ -239,9 +239,12 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
239
239
|
"_EXE" => "EXE",
|
240
240
|
"_GID" => "GID",
|
241
241
|
"_MACHINE_ID" => "MACHINE_ID",
|
242
|
+
"_LINE_BREAK" => "LINE_BREAK",
|
242
243
|
"_PID" => "PID",
|
243
244
|
"_SELINUX_CONTEXT" => "SELINUX_CONTEXT",
|
245
|
+
"_STREAM_ID" => "STREAM_ID",
|
244
246
|
"_SYSTEMD_CGROUP" => "SYSTEMD_CGROUP",
|
247
|
+
"_SYSTEMD_INVOCATION_ID" => "SYSTEMD_INVOCATION_ID",
|
245
248
|
"_SYSTEMD_OWNER_UID" => "SYSTEMD_OWNER_UID",
|
246
249
|
"_SYSTEMD_SESSION" => "SYSTEMD_SESSION",
|
247
250
|
"_SYSTEMD_SLICE" => "SYSTEMD_SLICE",
|
@@ -273,25 +276,28 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
273
276
|
end
|
274
277
|
def normal_output_t
|
275
278
|
{
|
276
|
-
"AUDIT_LOGINUID"
|
277
|
-
"AUDIT_SESSION"
|
278
|
-
"BOOT_ID"
|
279
|
-
"CAP_EFFECTIVE"
|
280
|
-
"CMDLINE"
|
281
|
-
"COMM"
|
282
|
-
"EXE"
|
283
|
-
"GID"
|
284
|
-
"MACHINE_ID"
|
285
|
-
"
|
286
|
-
"
|
287
|
-
"
|
288
|
-
"
|
289
|
-
"
|
290
|
-
"
|
291
|
-
"
|
292
|
-
"
|
293
|
-
"
|
294
|
-
"
|
279
|
+
"AUDIT_LOGINUID" =>"AUDIT_LOGINUID",
|
280
|
+
"AUDIT_SESSION" =>"AUDIT_SESSION",
|
281
|
+
"BOOT_ID" =>"BOOT_ID",
|
282
|
+
"CAP_EFFECTIVE" =>"CAP_EFFECTIVE",
|
283
|
+
"CMDLINE" =>"CMDLINE",
|
284
|
+
"COMM" =>"COMM",
|
285
|
+
"EXE" =>"EXE",
|
286
|
+
"GID" =>"GID",
|
287
|
+
"MACHINE_ID" =>"MACHINE_ID",
|
288
|
+
"LINE_BREAK" =>"LINE_BREAK",
|
289
|
+
"PID" =>"PID",
|
290
|
+
"SELINUX_CONTEXT" =>"SELINUX_CONTEXT",
|
291
|
+
"STREAM_ID" =>"STREAM_ID",
|
292
|
+
"SYSTEMD_INVOCATION_ID" =>"SYSTEMD_INVOCATION_ID",
|
293
|
+
"SYSTEMD_CGROUP" =>"SYSTEMD_CGROUP",
|
294
|
+
"SYSTEMD_OWNER_UID" =>"SYSTEMD_OWNER_UID",
|
295
|
+
"SYSTEMD_SESSION" =>"SYSTEMD_SESSION",
|
296
|
+
"SYSTEMD_SLICE" =>"SYSTEMD_SLICE",
|
297
|
+
"SYSTEMD_UNIT" =>"SYSTEMD_UNIT",
|
298
|
+
"SYSTEMD_USER_UNIT" =>"SYSTEMD_USER_UNIT",
|
299
|
+
"TRANSPORT" =>"TRANSPORT",
|
300
|
+
"UID" =>"UID"
|
295
301
|
}
|
296
302
|
end
|
297
303
|
def normal_output_u
|
@@ -838,7 +844,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
838
844
|
')
|
839
845
|
assert_equal('mymessage', rec['message'])
|
840
846
|
assert_equal('k8shost', rec['hostname'])
|
841
|
-
assert_equal('
|
847
|
+
assert_equal('unknown', rec['level'])
|
842
848
|
assert_equal(@timestamp_str, rec['@timestamp'])
|
843
849
|
assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
|
844
850
|
assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
|
@@ -867,7 +873,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
867
873
|
assert_equal('ADDED', rec['kubernetes']['event']['verb'])
|
868
874
|
assert_equal('event message', rec['message'])
|
869
875
|
assert_equal('mymessage', rec['pipeline_metadata']['collector']['original_raw_message'])
|
870
|
-
assert_equal('
|
876
|
+
assert_equal('unknown', rec['level'])
|
871
877
|
assert_equal('2017-07-27T17:23:46.216527+00:00', rec['@timestamp'])
|
872
878
|
assert_equal('127.0.0.1', rec['pipeline_metadata']['collector']['ipaddr4'])
|
873
879
|
assert_equal('::1', rec['pipeline_metadata']['collector']['ipaddr6'])
|
@@ -896,7 +902,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
896
902
|
')
|
897
903
|
assert_equal('mymessage', rec['message'])
|
898
904
|
assert_equal('k8shost', rec['hostname'])
|
899
|
-
assert_equal('
|
905
|
+
assert_equal('unknown', rec['level'])
|
900
906
|
assert_equal(now.utc.to_datetime.rfc3339(6), rec['@timestamp'])
|
901
907
|
assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
|
902
908
|
assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
|
@@ -926,7 +932,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
926
932
|
')
|
927
933
|
assert_equal('mymessage', rec['message'])
|
928
934
|
assert_equal('k8shost', rec['hostname'])
|
929
|
-
assert_equal('
|
935
|
+
assert_equal('unknown', rec['level'])
|
930
936
|
assert_equal(expectedtime, rec['@timestamp'])
|
931
937
|
assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
|
932
938
|
assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
|
@@ -955,7 +961,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
955
961
|
')
|
956
962
|
assert_equal('mymessage', rec['message'])
|
957
963
|
assert_equal('k8shost', rec['hostname'])
|
958
|
-
assert_equal('
|
964
|
+
assert_equal('unknown', rec['level'])
|
959
965
|
assert_equal(@timestamp_str, rec['@timestamp'])
|
960
966
|
assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
|
961
967
|
assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
|
@@ -983,7 +989,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
983
989
|
')
|
984
990
|
assert_equal('mymessage', rec['message'])
|
985
991
|
assert_equal('k8shost', rec['hostname'])
|
986
|
-
assert_equal('
|
992
|
+
assert_equal('unknown', rec['level'])
|
987
993
|
assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['@timestamp'])
|
988
994
|
assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
|
989
995
|
assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
|
@@ -1010,7 +1016,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
1010
1016
|
')
|
1011
1017
|
assert_equal('mymessage', rec['message'])
|
1012
1018
|
assert_equal('k8shost', rec['hostname'])
|
1013
|
-
assert_equal('
|
1019
|
+
assert_equal('unknown', rec['level'])
|
1014
1020
|
assert_equal(@timestamp_str, rec['@timestamp'])
|
1015
1021
|
assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
|
1016
1022
|
assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-viaq_data_model
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.19
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Rich Megginson
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-07-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -146,8 +146,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
146
146
|
- !ruby/object:Gem::Version
|
147
147
|
version: '0'
|
148
148
|
requirements: []
|
149
|
-
|
150
|
-
rubygems_version: 2.7.6
|
149
|
+
rubygems_version: 3.0.3
|
151
150
|
signing_key:
|
152
151
|
specification_version: 4
|
153
152
|
summary: Filter plugin to ensure data is in the ViaQ common data model
|