fluent-plugin-viaq_data_model 0.0.11 → 0.0.12
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5d9843f79f5f9c77f2d7e846d2db2f9603ef97bb
|
4
|
+
data.tar.gz: c2c3d05a6f76e75274246a11af13d870a6bdf84f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9799e82962f80305db119f3134f36ad9040e3a72515c56c82872e3d72326ca427f54d902a6f765c9c189f0b28d5c1935d83e922971de2d4ae805f2f2cb3a98cd
|
7
|
+
data.tar.gz: dbdff538da0efa79bb3a07891364984c77c78712230fbf6d5460dc164458b35d194a6a79f8cf4911c0e8d19a5f0c730fdc9c2eac184399315e594c453efebf56
|
@@ -7,7 +7,7 @@ FLUENTD_VERSION = ENV['FLUENTD_VERSION'] || "0.12.0"
|
|
7
7
|
|
8
8
|
Gem::Specification.new do |gem|
|
9
9
|
gem.name = "fluent-plugin-viaq_data_model"
|
10
|
-
gem.version = "0.0.
|
10
|
+
gem.version = "0.0.12"
|
11
11
|
gem.authors = ["Rich Megginson"]
|
12
12
|
gem.email = ["rmeggins@redhat.com"]
|
13
13
|
gem.description = %q{Filter plugin to ensure data is in the ViaQ common data model}
|
@@ -225,6 +225,68 @@ module Fluent
|
|
225
225
|
thing
|
226
226
|
end
|
227
227
|
|
228
|
+
# https://github.com/ViaQ/elasticsearch-templates/blob/master/namespaces/_default_.yml#L63
|
229
|
+
NORMAL_LEVELS = {
|
230
|
+
'emerg' => 'emerg',
|
231
|
+
'panic' => 'emerg',
|
232
|
+
'alert' => 'alert',
|
233
|
+
'crit' => 'crit',
|
234
|
+
'critical' => 'crit',
|
235
|
+
'err' => 'err',
|
236
|
+
'error' => 'err',
|
237
|
+
'warning' => 'warning',
|
238
|
+
'warn' => 'warning',
|
239
|
+
'notice' => 'notice',
|
240
|
+
'info' => 'info',
|
241
|
+
'debug' => 'debug',
|
242
|
+
'trace' => 'trace',
|
243
|
+
'unknown' => 'unknown',
|
244
|
+
}
|
245
|
+
# numeric levels for the PRIORITY field
|
246
|
+
PRIORITY_LEVELS = {
|
247
|
+
0 => 'emerg',
|
248
|
+
1 => 'alert',
|
249
|
+
2 => 'crit',
|
250
|
+
3 => 'err',
|
251
|
+
4 => 'warning',
|
252
|
+
5 => 'notice',
|
253
|
+
6 => 'info',
|
254
|
+
7 => 'debug',
|
255
|
+
8 => 'trace',
|
256
|
+
9 => 'unknown',
|
257
|
+
'0' => 'emerg',
|
258
|
+
'1' => 'alert',
|
259
|
+
'2' => 'crit',
|
260
|
+
'3' => 'err',
|
261
|
+
'4' => 'warning',
|
262
|
+
'5' => 'notice',
|
263
|
+
'6' => 'info',
|
264
|
+
'7' => 'debug',
|
265
|
+
'8' => 'trace',
|
266
|
+
'9' => 'unknown',
|
267
|
+
}
|
268
|
+
def normalize_level(level, newlevel, stream=nil, priority=nil)
|
269
|
+
# if the record already has a level field, and it looks like one of our well
|
270
|
+
# known values, convert it to the canonical normalized form - otherwise,
|
271
|
+
# preserve the value in string format
|
272
|
+
retlevel = nil
|
273
|
+
if !level.nil?
|
274
|
+
unless (retlevel = NORMAL_LEVELS[level]) ||
|
275
|
+
(level.respond_to?(:downcase) && (retlevel = NORMAL_LEVELS[level.downcase]))
|
276
|
+
retlevel = level.to_s # don't know what it is - just convert to string
|
277
|
+
end
|
278
|
+
elsif stream == 'stdout'
|
279
|
+
retlevel = 'info'
|
280
|
+
elsif stream == 'stderr'
|
281
|
+
retlevel = 'err'
|
282
|
+
elsif !priority.nil?
|
283
|
+
retlevel = PRIORITY_LEVELS[priority]
|
284
|
+
else
|
285
|
+
retlevel = NORMAL_LEVELS[newlevel]
|
286
|
+
end
|
287
|
+
retlevel || 'unknown'
|
288
|
+
end
|
289
|
+
|
228
290
|
def process_sys_var_log_fields(tag, time, record, fmtr_type = nil)
|
229
291
|
record['systemd'] = {"t" => {"PID" => record['pid']}, "u" => {"SYSLOG_IDENTIFIER" => record['ident']}}
|
230
292
|
if record[@dest_time_name].nil? # e.g. already has @timestamp
|
@@ -244,7 +306,7 @@ module Fluent
|
|
244
306
|
|
245
307
|
def process_k8s_json_file_fields(tag, time, record, fmtr_type = nil)
|
246
308
|
record['message'] = record['message'] || record['log']
|
247
|
-
record['level'] = (record['
|
309
|
+
record['level'] = normalize_level(record['level'], nil, record['stream'])
|
248
310
|
if record.key?('kubernetes') && record['kubernetes'].respond_to?(:fetch) && \
|
249
311
|
(k8shost = record['kubernetes'].fetch('host', nil))
|
250
312
|
record['hostname'] = k8shost
|
@@ -93,18 +93,7 @@ module ViaqDataModelFilterSystemd
|
|
93
93
|
unless systemd_k.empty?
|
94
94
|
(record['systemd'] ||= {})['k'] = systemd_k
|
95
95
|
end
|
96
|
-
|
97
|
-
pri_index = ('%d' % record['PRIORITY'] || 9).to_i
|
98
|
-
case
|
99
|
-
when pri_index < 0
|
100
|
-
pri_index = 9
|
101
|
-
when pri_index > 9
|
102
|
-
pri_index = 9
|
103
|
-
end
|
104
|
-
rescue
|
105
|
-
pri_index = 9
|
106
|
-
end
|
107
|
-
record['level'] = ["emerg", "alert", "crit", "err", "warning", "notice", "info", "debug", "trace", "unknown"][pri_index]
|
96
|
+
record['level'] = normalize_level(record['level'], nil, nil, record['PRIORITY'])
|
108
97
|
JOURNAL_TIME_FIELDS.each do |field|
|
109
98
|
if (val = record[field])
|
110
99
|
vali = val.to_i
|
@@ -471,6 +471,16 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
471
471
|
assert_equal('b', rec['a'])
|
472
472
|
assert_equal('unknown', rec['level'])
|
473
473
|
end
|
474
|
+
test 'try a PRIORITY value that is a number' do
|
475
|
+
rec = emit_with_tag('journal.system', {'a'=>'b', 'PRIORITY'=>4}, '
|
476
|
+
<formatter>
|
477
|
+
tag "journal.system**"
|
478
|
+
type sys_journal
|
479
|
+
</formatter>
|
480
|
+
')
|
481
|
+
assert_equal('b', rec['a'])
|
482
|
+
assert_equal('warning', rec['level'])
|
483
|
+
end
|
474
484
|
test 'test with fallback to __REALTIME_TIMESTAMP' do
|
475
485
|
input = normal_input.reject{|k,v| k == '_SOURCE_REALTIME_TIMESTAMP'}
|
476
486
|
rec = emit_with_tag('journal.system', input, '
|
@@ -934,6 +944,87 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
934
944
|
dellist = 'host,pid,ident'.split(',')
|
935
945
|
dellist.each{|field| assert_nil(rec[field])}
|
936
946
|
end
|
947
|
+
test 'process a k8s json-file record, stdout stream' do
|
948
|
+
ENV['IPADDR4'] = '127.0.0.1'
|
949
|
+
ENV['IPADDR6'] = '::1'
|
950
|
+
ENV['FLUENTD_VERSION'] = 'fversion'
|
951
|
+
ENV['DATA_VERSION'] = 'dversion'
|
952
|
+
input = {'kubernetes'=>{'host'=>'k8shost'},'stream'=>'stdout','time'=>@timestamp_str,'log'=>'mymessage'}
|
953
|
+
rec = emit_with_tag('kubernetes.var.log.containers.name.name_this_that_other_log', input, '
|
954
|
+
<formatter>
|
955
|
+
tag "kubernetes.var.log.containers**"
|
956
|
+
type k8s_json_file
|
957
|
+
remove_keys log,stream
|
958
|
+
</formatter>
|
959
|
+
pipeline_type normalizer
|
960
|
+
')
|
961
|
+
assert_equal('mymessage', rec['message'])
|
962
|
+
assert_equal('k8shost', rec['hostname'])
|
963
|
+
assert_equal('info', rec['level'])
|
964
|
+
assert_equal(@timestamp_str, rec['@timestamp'])
|
965
|
+
assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
|
966
|
+
assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
|
967
|
+
assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
|
968
|
+
assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
|
969
|
+
assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
|
970
|
+
assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
|
971
|
+
dellist = 'host,pid,ident'.split(',')
|
972
|
+
dellist.each{|field| assert_nil(rec[field])}
|
973
|
+
end
|
974
|
+
test 'process a k8s json-file record, existing level field' do
|
975
|
+
ENV['IPADDR4'] = '127.0.0.1'
|
976
|
+
ENV['IPADDR6'] = '::1'
|
977
|
+
ENV['FLUENTD_VERSION'] = 'fversion'
|
978
|
+
ENV['DATA_VERSION'] = 'dversion'
|
979
|
+
input = {'kubernetes'=>{'host'=>'k8shost'},'stream'=>'stderr','time'=>@timestamp_str,'log'=>'mymessage','level'=>0}
|
980
|
+
rec = emit_with_tag('kubernetes.var.log.containers.name.name_this_that_other_log', input, '
|
981
|
+
<formatter>
|
982
|
+
tag "kubernetes.var.log.containers**"
|
983
|
+
type k8s_json_file
|
984
|
+
remove_keys log,stream
|
985
|
+
</formatter>
|
986
|
+
pipeline_type normalizer
|
987
|
+
')
|
988
|
+
assert_equal('mymessage', rec['message'])
|
989
|
+
assert_equal('k8shost', rec['hostname'])
|
990
|
+
assert_equal('0', rec['level'])
|
991
|
+
assert_equal(@timestamp_str, rec['@timestamp'])
|
992
|
+
assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
|
993
|
+
assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
|
994
|
+
assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
|
995
|
+
assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
|
996
|
+
assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
|
997
|
+
assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
|
998
|
+
dellist = 'host,pid,ident'.split(',')
|
999
|
+
dellist.each{|field| assert_nil(rec[field])}
|
1000
|
+
end
|
1001
|
+
test 'process a k8s json-file record, already normalized existing level field' do
|
1002
|
+
ENV['IPADDR4'] = '127.0.0.1'
|
1003
|
+
ENV['IPADDR6'] = '::1'
|
1004
|
+
ENV['FLUENTD_VERSION'] = 'fversion'
|
1005
|
+
ENV['DATA_VERSION'] = 'dversion'
|
1006
|
+
input = {'kubernetes'=>{'host'=>'k8shost'},'stream'=>'stderr','time'=>@timestamp_str,'log'=>'mymessage','level'=>'debug'}
|
1007
|
+
rec = emit_with_tag('kubernetes.var.log.containers.name.name_this_that_other_log', input, '
|
1008
|
+
<formatter>
|
1009
|
+
tag "kubernetes.var.log.containers**"
|
1010
|
+
type k8s_json_file
|
1011
|
+
remove_keys log,stream
|
1012
|
+
</formatter>
|
1013
|
+
pipeline_type normalizer
|
1014
|
+
')
|
1015
|
+
assert_equal('mymessage', rec['message'])
|
1016
|
+
assert_equal('k8shost', rec['hostname'])
|
1017
|
+
assert_equal('debug', rec['level'])
|
1018
|
+
assert_equal(@timestamp_str, rec['@timestamp'])
|
1019
|
+
assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
|
1020
|
+
assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
|
1021
|
+
assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
|
1022
|
+
assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
|
1023
|
+
assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
|
1024
|
+
assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
|
1025
|
+
dellist = 'host,pid,ident'.split(',')
|
1026
|
+
dellist.each{|field| assert_nil(rec[field])}
|
1027
|
+
end
|
937
1028
|
# tests for elasticsearch index functionality
|
938
1029
|
test 'construct an operations index prefix' do
|
939
1030
|
rec = emit_with_tag('journal.system', normal_input, '
|
@@ -1253,5 +1344,66 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
1253
1344
|
')
|
1254
1345
|
assert_equal('.operations.2017.07.27', rec['viaq_index_name'])
|
1255
1346
|
end
|
1347
|
+
# level field processing
|
1348
|
+
test 'see if existing level is preserved for journald log' do
|
1349
|
+
rec = emit_with_tag('journal.system', {'a'=>'b', 'level'=>'this is my level'}, '
|
1350
|
+
<formatter>
|
1351
|
+
tag "journal.system**"
|
1352
|
+
type sys_journal
|
1353
|
+
</formatter>
|
1354
|
+
')
|
1355
|
+
assert_equal('b', rec['a'])
|
1356
|
+
assert_equal('this is my level', rec['level'])
|
1357
|
+
end
|
1358
|
+
test 'see if existing level is preserved and overrides PRIORITY' do
|
1359
|
+
rec = emit_with_tag('journal.system', {'a'=>'b', 'PRIORITY'=>'3', 'level'=>'this is my level'}, '
|
1360
|
+
<formatter>
|
1361
|
+
tag "journal.system**"
|
1362
|
+
type sys_journal
|
1363
|
+
</formatter>
|
1364
|
+
')
|
1365
|
+
assert_equal('b', rec['a'])
|
1366
|
+
assert_equal('this is my level', rec['level'])
|
1367
|
+
end
|
1368
|
+
test 'see if existing level is preserved and converted to string' do
|
1369
|
+
rec = emit_with_tag('journal.system', {'a'=>'b', 'level'=>1}, '
|
1370
|
+
<formatter>
|
1371
|
+
tag "journal.system**"
|
1372
|
+
type sys_journal
|
1373
|
+
</formatter>
|
1374
|
+
')
|
1375
|
+
assert_equal('b', rec['a'])
|
1376
|
+
assert_equal('1', rec['level'])
|
1377
|
+
end
|
1378
|
+
test 'see if existing level is preserved and converted to string with an odd type' do
|
1379
|
+
rec = emit_with_tag('journal.system', {'a'=>'b', 'level'=>{}}, '
|
1380
|
+
<formatter>
|
1381
|
+
tag "journal.system**"
|
1382
|
+
type sys_journal
|
1383
|
+
</formatter>
|
1384
|
+
')
|
1385
|
+
assert_equal('b', rec['a'])
|
1386
|
+
assert_equal('{}', rec['level'])
|
1387
|
+
end
|
1388
|
+
test 'see if existing level is normalized to canonical form' do
|
1389
|
+
rec = emit_with_tag('journal.system', {'a'=>'b', 'level'=>'CRITICAL'}, '
|
1390
|
+
<formatter>
|
1391
|
+
tag "journal.system**"
|
1392
|
+
type sys_journal
|
1393
|
+
</formatter>
|
1394
|
+
')
|
1395
|
+
assert_equal('b', rec['a'])
|
1396
|
+
assert_equal('crit', rec['level'])
|
1397
|
+
end
|
1398
|
+
test 'see if existing level already normalized is preserved' do
|
1399
|
+
rec = emit_with_tag('journal.system', {'a'=>'b', 'level'=>'crit'}, '
|
1400
|
+
<formatter>
|
1401
|
+
tag "journal.system**"
|
1402
|
+
type sys_journal
|
1403
|
+
</formatter>
|
1404
|
+
')
|
1405
|
+
assert_equal('b', rec['a'])
|
1406
|
+
assert_equal('crit', rec['level'])
|
1407
|
+
end
|
1256
1408
|
end
|
1257
1409
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-viaq_data_model
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.12
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Rich Megginson
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-11-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|