fluent-plugin-viaq_data_model 0.0.18 → 0.0.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b20471334c7756a6393385dfe17987751ea233c8cac638f592459ea5a3a8692f
|
4
|
+
data.tar.gz: 0d1738ff46f83bba941f415616c374ece1268a89c59f7df333c0311106809e67
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c77bfd1197a5d06b994668034dcd4d662ddb955fd4c3a803fad71a47124f86e28f29f0a063eecfe86d6bf23c9791159ca693405be9e0c4744288345ee3f0d3e0
|
7
|
+
data.tar.gz: 902a1adc2eca199a51b7cb6ce37600c2ec9764eeacbda1e2313410bcde774f1184513d4b87af91b7efd15651c35e2f7b97caf872e6e5f124a9d56232e2e5f433
|
data/README.md
CHANGED
@@ -162,10 +162,10 @@ See `filter-viaq_data_model.conf` for an example filter configuration.
|
|
162
162
|
create the full index name.
|
163
163
|
* `elasticsearch_index_name_field` - name of the field in the record which stores
|
164
164
|
the index name - you should remove this field in the elasticsearch output
|
165
|
-
plugin using the `remove_keys` config parameter - default is `
|
165
|
+
plugin using the `remove_keys` config parameter - default is `viaq_index_name`
|
166
166
|
* `elasticsearch_index_prefix_field` - name of the field in the record which stores
|
167
167
|
the index prefix - you should remove this field in the elasticsearch output
|
168
|
-
plugin using the `remove_keys` config parameter - default is `
|
168
|
+
plugin using the `remove_keys` config parameter - default is `viaq_index_prefix`
|
169
169
|
|
170
170
|
**NOTE** The `formatter` blocks are matched in the given order in the file.
|
171
171
|
This means, don't use `tag "**"` as the first formatter or none of your
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |gem|
|
6
6
|
gem.name = "fluent-plugin-viaq_data_model"
|
7
|
-
gem.version = "0.0.
|
7
|
+
gem.version = "0.0.19"
|
8
8
|
gem.authors = ["Rich Megginson"]
|
9
9
|
gem.email = ["rmeggins@redhat.com"]
|
10
10
|
gem.description = %q{Filter plugin to ensure data is in the ViaQ common data model}
|
@@ -280,7 +280,7 @@ module Fluent
|
|
280
280
|
'8' => 'trace',
|
281
281
|
'9' => 'unknown',
|
282
282
|
}
|
283
|
-
def normalize_level(level, newlevel,
|
283
|
+
def normalize_level(level, newlevel, priority=nil)
|
284
284
|
# if the record already has a level field, and it looks like one of our well
|
285
285
|
# known values, convert it to the canonical normalized form - otherwise,
|
286
286
|
# preserve the value in string format
|
@@ -290,10 +290,6 @@ module Fluent
|
|
290
290
|
(level.respond_to?(:downcase) && (retlevel = NORMAL_LEVELS[level.downcase]))
|
291
291
|
retlevel = level.to_s # don't know what it is - just convert to string
|
292
292
|
end
|
293
|
-
elsif stream == 'stdout'
|
294
|
-
retlevel = 'info'
|
295
|
-
elsif stream == 'stderr'
|
296
|
-
retlevel = 'err'
|
297
293
|
elsif !priority.nil?
|
298
294
|
retlevel = PRIORITY_LEVELS[priority]
|
299
295
|
else
|
@@ -321,7 +317,7 @@ module Fluent
|
|
321
317
|
|
322
318
|
def process_k8s_json_file_fields(tag, time, record, fmtr_type = nil)
|
323
319
|
record['message'] = record['message'] || record['log']
|
324
|
-
record['level'] = normalize_level(record['level'], nil
|
320
|
+
record['level'] = normalize_level(record['level'], nil)
|
325
321
|
if record.key?('kubernetes') && record['kubernetes'].respond_to?(:fetch) && \
|
326
322
|
(k8shost = record['kubernetes'].fetch('host', nil))
|
327
323
|
record['hostname'] = k8shost
|
@@ -21,25 +21,28 @@ require 'date'
|
|
21
21
|
module ViaqDataModelFilterSystemd
|
22
22
|
# map of journal fields to viaq data model field
|
23
23
|
JOURNAL_FIELD_MAP_SYSTEMD_T = {
|
24
|
-
"_AUDIT_LOGINUID"
|
25
|
-
"_AUDIT_SESSION"
|
26
|
-
"_BOOT_ID"
|
27
|
-
"_CAP_EFFECTIVE"
|
28
|
-
"_CMDLINE"
|
29
|
-
"_COMM"
|
30
|
-
"_EXE"
|
31
|
-
"_GID"
|
32
|
-
"
|
33
|
-
"
|
34
|
-
"
|
35
|
-
"
|
36
|
-
"
|
37
|
-
"
|
38
|
-
"
|
39
|
-
"
|
40
|
-
"
|
41
|
-
"
|
42
|
-
"
|
24
|
+
"_AUDIT_LOGINUID" => "AUDIT_LOGINUID",
|
25
|
+
"_AUDIT_SESSION" => "AUDIT_SESSION",
|
26
|
+
"_BOOT_ID" => "BOOT_ID",
|
27
|
+
"_CAP_EFFECTIVE" => "CAP_EFFECTIVE",
|
28
|
+
"_CMDLINE" => "CMDLINE",
|
29
|
+
"_COMM" => "COMM",
|
30
|
+
"_EXE" => "EXE",
|
31
|
+
"_GID" => "GID",
|
32
|
+
"_LINE_BREAK" => "LINE_BREAK",
|
33
|
+
"_MACHINE_ID" => "MACHINE_ID",
|
34
|
+
"_PID" => "PID",
|
35
|
+
"_SELINUX_CONTEXT" => "SELINUX_CONTEXT",
|
36
|
+
"_STREAM_ID" => "STREAM_ID",
|
37
|
+
"_SYSTEMD_CGROUP" => "SYSTEMD_CGROUP",
|
38
|
+
"_SYSTEMD_INVOCATION_ID" => "SYSTEMD_INVOCATION_ID",
|
39
|
+
"_SYSTEMD_OWNER_UID" => "SYSTEMD_OWNER_UID",
|
40
|
+
"_SYSTEMD_SESSION" => "SYSTEMD_SESSION",
|
41
|
+
"_SYSTEMD_SLICE" => "SYSTEMD_SLICE",
|
42
|
+
"_SYSTEMD_UNIT" => "SYSTEMD_UNIT",
|
43
|
+
"_SYSTEMD_USER_UNIT" => "SYSTEMD_USER_UNIT",
|
44
|
+
"_TRANSPORT" => "TRANSPORT",
|
45
|
+
"_UID" => "UID"
|
43
46
|
}
|
44
47
|
|
45
48
|
JOURNAL_FIELD_MAP_SYSTEMD_U = {
|
@@ -93,7 +96,7 @@ module ViaqDataModelFilterSystemd
|
|
93
96
|
unless systemd_k.empty?
|
94
97
|
(record['systemd'] ||= {})['k'] = systemd_k
|
95
98
|
end
|
96
|
-
record['level'] = normalize_level(record['level'], nil,
|
99
|
+
record['level'] = normalize_level(record['level'], nil, record['PRIORITY'])
|
97
100
|
JOURNAL_TIME_FIELDS.each do |field|
|
98
101
|
if (val = record[field])
|
99
102
|
vali = val.to_i
|
@@ -239,9 +239,12 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
239
239
|
"_EXE" => "EXE",
|
240
240
|
"_GID" => "GID",
|
241
241
|
"_MACHINE_ID" => "MACHINE_ID",
|
242
|
+
"_LINE_BREAK" => "LINE_BREAK",
|
242
243
|
"_PID" => "PID",
|
243
244
|
"_SELINUX_CONTEXT" => "SELINUX_CONTEXT",
|
245
|
+
"_STREAM_ID" => "STREAM_ID",
|
244
246
|
"_SYSTEMD_CGROUP" => "SYSTEMD_CGROUP",
|
247
|
+
"_SYSTEMD_INVOCATION_ID" => "SYSTEMD_INVOCATION_ID",
|
245
248
|
"_SYSTEMD_OWNER_UID" => "SYSTEMD_OWNER_UID",
|
246
249
|
"_SYSTEMD_SESSION" => "SYSTEMD_SESSION",
|
247
250
|
"_SYSTEMD_SLICE" => "SYSTEMD_SLICE",
|
@@ -273,25 +276,28 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
273
276
|
end
|
274
277
|
def normal_output_t
|
275
278
|
{
|
276
|
-
"AUDIT_LOGINUID"
|
277
|
-
"AUDIT_SESSION"
|
278
|
-
"BOOT_ID"
|
279
|
-
"CAP_EFFECTIVE"
|
280
|
-
"CMDLINE"
|
281
|
-
"COMM"
|
282
|
-
"EXE"
|
283
|
-
"GID"
|
284
|
-
"MACHINE_ID"
|
285
|
-
"
|
286
|
-
"
|
287
|
-
"
|
288
|
-
"
|
289
|
-
"
|
290
|
-
"
|
291
|
-
"
|
292
|
-
"
|
293
|
-
"
|
294
|
-
"
|
279
|
+
"AUDIT_LOGINUID" =>"AUDIT_LOGINUID",
|
280
|
+
"AUDIT_SESSION" =>"AUDIT_SESSION",
|
281
|
+
"BOOT_ID" =>"BOOT_ID",
|
282
|
+
"CAP_EFFECTIVE" =>"CAP_EFFECTIVE",
|
283
|
+
"CMDLINE" =>"CMDLINE",
|
284
|
+
"COMM" =>"COMM",
|
285
|
+
"EXE" =>"EXE",
|
286
|
+
"GID" =>"GID",
|
287
|
+
"MACHINE_ID" =>"MACHINE_ID",
|
288
|
+
"LINE_BREAK" =>"LINE_BREAK",
|
289
|
+
"PID" =>"PID",
|
290
|
+
"SELINUX_CONTEXT" =>"SELINUX_CONTEXT",
|
291
|
+
"STREAM_ID" =>"STREAM_ID",
|
292
|
+
"SYSTEMD_INVOCATION_ID" =>"SYSTEMD_INVOCATION_ID",
|
293
|
+
"SYSTEMD_CGROUP" =>"SYSTEMD_CGROUP",
|
294
|
+
"SYSTEMD_OWNER_UID" =>"SYSTEMD_OWNER_UID",
|
295
|
+
"SYSTEMD_SESSION" =>"SYSTEMD_SESSION",
|
296
|
+
"SYSTEMD_SLICE" =>"SYSTEMD_SLICE",
|
297
|
+
"SYSTEMD_UNIT" =>"SYSTEMD_UNIT",
|
298
|
+
"SYSTEMD_USER_UNIT" =>"SYSTEMD_USER_UNIT",
|
299
|
+
"TRANSPORT" =>"TRANSPORT",
|
300
|
+
"UID" =>"UID"
|
295
301
|
}
|
296
302
|
end
|
297
303
|
def normal_output_u
|
@@ -838,7 +844,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
838
844
|
')
|
839
845
|
assert_equal('mymessage', rec['message'])
|
840
846
|
assert_equal('k8shost', rec['hostname'])
|
841
|
-
assert_equal('
|
847
|
+
assert_equal('unknown', rec['level'])
|
842
848
|
assert_equal(@timestamp_str, rec['@timestamp'])
|
843
849
|
assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
|
844
850
|
assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
|
@@ -867,7 +873,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
867
873
|
assert_equal('ADDED', rec['kubernetes']['event']['verb'])
|
868
874
|
assert_equal('event message', rec['message'])
|
869
875
|
assert_equal('mymessage', rec['pipeline_metadata']['collector']['original_raw_message'])
|
870
|
-
assert_equal('
|
876
|
+
assert_equal('unknown', rec['level'])
|
871
877
|
assert_equal('2017-07-27T17:23:46.216527+00:00', rec['@timestamp'])
|
872
878
|
assert_equal('127.0.0.1', rec['pipeline_metadata']['collector']['ipaddr4'])
|
873
879
|
assert_equal('::1', rec['pipeline_metadata']['collector']['ipaddr6'])
|
@@ -896,7 +902,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
896
902
|
')
|
897
903
|
assert_equal('mymessage', rec['message'])
|
898
904
|
assert_equal('k8shost', rec['hostname'])
|
899
|
-
assert_equal('
|
905
|
+
assert_equal('unknown', rec['level'])
|
900
906
|
assert_equal(now.utc.to_datetime.rfc3339(6), rec['@timestamp'])
|
901
907
|
assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
|
902
908
|
assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
|
@@ -926,7 +932,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
926
932
|
')
|
927
933
|
assert_equal('mymessage', rec['message'])
|
928
934
|
assert_equal('k8shost', rec['hostname'])
|
929
|
-
assert_equal('
|
935
|
+
assert_equal('unknown', rec['level'])
|
930
936
|
assert_equal(expectedtime, rec['@timestamp'])
|
931
937
|
assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
|
932
938
|
assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
|
@@ -955,7 +961,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
955
961
|
')
|
956
962
|
assert_equal('mymessage', rec['message'])
|
957
963
|
assert_equal('k8shost', rec['hostname'])
|
958
|
-
assert_equal('
|
964
|
+
assert_equal('unknown', rec['level'])
|
959
965
|
assert_equal(@timestamp_str, rec['@timestamp'])
|
960
966
|
assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
|
961
967
|
assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
|
@@ -983,7 +989,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
983
989
|
')
|
984
990
|
assert_equal('mymessage', rec['message'])
|
985
991
|
assert_equal('k8shost', rec['hostname'])
|
986
|
-
assert_equal('
|
992
|
+
assert_equal('unknown', rec['level'])
|
987
993
|
assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['@timestamp'])
|
988
994
|
assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
|
989
995
|
assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
|
@@ -1010,7 +1016,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
1010
1016
|
')
|
1011
1017
|
assert_equal('mymessage', rec['message'])
|
1012
1018
|
assert_equal('k8shost', rec['hostname'])
|
1013
|
-
assert_equal('
|
1019
|
+
assert_equal('unknown', rec['level'])
|
1014
1020
|
assert_equal(@timestamp_str, rec['@timestamp'])
|
1015
1021
|
assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
|
1016
1022
|
assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-viaq_data_model
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.19
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Rich Megginson
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-07-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -146,8 +146,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
146
146
|
- !ruby/object:Gem::Version
|
147
147
|
version: '0'
|
148
148
|
requirements: []
|
149
|
-
|
150
|
-
rubygems_version: 2.7.6
|
149
|
+
rubygems_version: 3.0.3
|
151
150
|
signing_key:
|
152
151
|
specification_version: 4
|
153
152
|
summary: Filter plugin to ensure data is in the ViaQ common data model
|