fluent-plugin-viaq_data_model 0.0.5 → 0.0.6

Sign up to get free protection for your applications and to get access to all the features.
Files changed (8) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +5 -1
  3. data/README.md +157 -0
  4. data/fluent-plugin-viaq_data_model.gemspec +7 -3
  5. data/lib/fluent/plugin/filter_viaq_data_model.rb +244 -0
  6. data/lib/fluent/plugin/filter_viaq_data_model_systemd.rb +133 -0
  7. data/test/test_filter_viaq_data_model.rb +654 -1
  8. metadata +21 -6
data/lib/fluent/plugin/filter_viaq_data_model_systemd.rb ADDED
@@ -0,0 +1,133 @@
1
+ #
2
+ # Fluentd ViaQ data model Filter Plugin
3
+ #
4
+ # Copyright 2017 Red Hat, Inc.
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+ require 'time'
19
+ require 'date'
20
+
21
+ module ViaqDataModelFilterSystemd
22
+ # map of journal fields to viaq data model field
23
+ JOURNAL_FIELD_MAP_SYSTEMD_T = {
24
+ "_AUDIT_LOGINUID" => "AUDIT_LOGINUID",
25
+ "_AUDIT_SESSION" => "AUDIT_SESSION",
26
+ "_BOOT_ID" => "BOOT_ID",
27
+ "_CAP_EFFECTIVE" => "CAP_EFFECTIVE",
28
+ "_CMDLINE" => "CMDLINE",
29
+ "_COMM" => "COMM",
30
+ "_EXE" => "EXE",
31
+ "_GID" => "GID",
32
+ "_MACHINE_ID" => "MACHINE_ID",
33
+ "_PID" => "PID",
34
+ "_SELINUX_CONTEXT" => "SELINUX_CONTEXT",
35
+ "_SYSTEMD_CGROUP" => "SYSTEMD_CGROUP",
36
+ "_SYSTEMD_OWNER_UID" => "SYSTEMD_OWNER_UID",
37
+ "_SYSTEMD_SESSION" => "SYSTEMD_SESSION",
38
+ "_SYSTEMD_SLICE" => "SYSTEMD_SLICE",
39
+ "_SYSTEMD_UNIT" => "SYSTEMD_UNIT",
40
+ "_SYSTEMD_USER_UNIT" => "SYSTEMD_USER_UNIT",
41
+ "_TRANSPORT" => "TRANSPORT",
42
+ "_UID" => "UID"
43
+ }
44
+
45
+ JOURNAL_FIELD_MAP_SYSTEMD_U = {
46
+ "CODE_FILE" => "CODE_FILE",
47
+ "CODE_FUNCTION" => "CODE_FUNCTION",
48
+ "CODE_LINE" => "CODE_LINE",
49
+ "ERRNO" => "ERRNO",
50
+ "MESSAGE_ID" => "MESSAGE_ID",
51
+ "RESULT" => "RESULT",
52
+ "UNIT" => "UNIT",
53
+ "SYSLOG_FACILITY" => "SYSLOG_FACILITY",
54
+ "SYSLOG_IDENTIFIER" => "SYSLOG_IDENTIFIER",
55
+ "SYSLOG_PID" => "SYSLOG_PID"
56
+ }
57
+
58
+ JOURNAL_FIELD_MAP_SYSTEMD_K = {
59
+ "_KERNEL_DEVICE" => "KERNEL_DEVICE",
60
+ "_KERNEL_SUBSYSTEM" => "KERNEL_SUBSYSTEM",
61
+ "_UDEV_SYSNAME" => "UDEV_SYSNAME",
62
+ "_UDEV_DEVNODE" => "UDEV_DEVNODE",
63
+ "_UDEV_DEVLINK" => "UDEV_DEVLINK",
64
+ }
65
+
66
+ JOURNAL_TIME_FIELDS = ['_SOURCE_REALTIME_TIMESTAMP', '__REALTIME_TIMESTAMP']
67
+
68
+ def process_journal_fields(tag, time, record, fmtr_type)
69
+ systemd_t = {}
70
+ JOURNAL_FIELD_MAP_SYSTEMD_T.each do |jkey, key|
71
+ if record[jkey]
72
+ systemd_t[key] = record[jkey]
73
+ end
74
+ end
75
+ systemd_u = {}
76
+ JOURNAL_FIELD_MAP_SYSTEMD_U.each do |jkey, key|
77
+ if record[jkey]
78
+ systemd_u[key] = record[jkey]
79
+ end
80
+ end
81
+ systemd_k = {}
82
+ JOURNAL_FIELD_MAP_SYSTEMD_K.each do |jkey, key|
83
+ if record[jkey]
84
+ systemd_k[key] = record[jkey]
85
+ end
86
+ end
87
+ unless systemd_t.empty?
88
+ (record['systemd'] ||= {})['t'] = systemd_t
89
+ end
90
+ unless systemd_u.empty?
91
+ (record['systemd'] ||= {})['u'] = systemd_u
92
+ end
93
+ unless systemd_k.empty?
94
+ (record['systemd'] ||= {})['k'] = systemd_k
95
+ end
96
+ begin
97
+ pri_index = ('%d' % record['PRIORITY'] || 9).to_i
98
+ case
99
+ when pri_index < 0
100
+ pri_index = 9
101
+ when pri_index > 9
102
+ pri_index = 9
103
+ end
104
+ rescue
105
+ pri_index = 9
106
+ end
107
+ record['level'] = ["emerg", "alert", "crit", "err", "warning", "notice", "info", "debug", "trace", "unknown"][pri_index]
108
+ JOURNAL_TIME_FIELDS.each do |field|
109
+ if record[field]
110
+ record['time'] = Time.at(record[field].to_f / 1000000.0).utc.to_datetime.rfc3339(6)
111
+ break
112
+ end
113
+ end
114
+ case fmtr_type
115
+ when :sys_journal
116
+ record['message'] = record['MESSAGE']
117
+ if record['_HOSTNAME'].eql?('localhost') && @docker_hostname
118
+ record['hostname'] = @docker_hostname
119
+ else
120
+ record['hostname'] = record['_HOSTNAME']
121
+ end
122
+ when :k8s_journal
123
+ record['message'] = record['message'] || record['MESSAGE'] || record['log']
124
+ if record['kubernetes'] && record['kubernetes']['host']
125
+ record['hostname'] = record['kubernetes']['host']
126
+ elsif @docker_hostname
127
+ record['hostname'] = @docker_hostname
128
+ else
129
+ record['hostname'] = record['_HOSTNAME']
130
+ end
131
+ end
132
+ end
133
+ end
data/test/test_filter_viaq_data_model.rb CHANGED
@@ -31,7 +31,9 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
31
31
  end
32
32
 
33
33
  def create_driver(conf = '')
34
- Test::FilterTestDriver.new(ViaqDataModelFilter, 'this.is.a.tag').configure(conf, true)
34
+ d = Test::FilterTestDriver.new(ViaqDataModelFilter, 'this.is.a.tag').configure(conf, true)
35
+ @dlog = d.instance.log
36
+ d
35
37
  end
36
38
 
37
39
  sub_test_case 'configure' do
@@ -212,6 +214,657 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
212
214
  assert_equal([88], rec['h']['i']['m'])
213
215
  assert_true(rec['h']['i']['n'])
214
216
  end
217
+ end
215
218
 
219
+ sub_test_case 'formatters and elasticsearch index names' do
220
+ def emit_with_tag(tag, msg={}, conf='')
221
+ d = create_driver(conf)
222
+ d.run {
223
+ d.emit_with_tag(tag, msg, @time)
224
+ }.filtered.instance_variable_get(:@record_array)[0]
225
+ end
226
+
227
+ def normal_input
228
+ {
229
+ "_AUDIT_LOGINUID" => "AUDIT_LOGINUID",
230
+ "_AUDIT_SESSION" => "AUDIT_SESSION",
231
+ "_BOOT_ID" => "BOOT_ID",
232
+ "_CAP_EFFECTIVE" => "CAP_EFFECTIVE",
233
+ "_CMDLINE" => "CMDLINE",
234
+ "_COMM" => "COMM",
235
+ "_EXE" => "EXE",
236
+ "_GID" => "GID",
237
+ "_MACHINE_ID" => "MACHINE_ID",
238
+ "_PID" => "PID",
239
+ "_SELINUX_CONTEXT" => "SELINUX_CONTEXT",
240
+ "_SYSTEMD_CGROUP" => "SYSTEMD_CGROUP",
241
+ "_SYSTEMD_OWNER_UID" => "SYSTEMD_OWNER_UID",
242
+ "_SYSTEMD_SESSION" => "SYSTEMD_SESSION",
243
+ "_SYSTEMD_SLICE" => "SYSTEMD_SLICE",
244
+ "_SYSTEMD_UNIT" => "SYSTEMD_UNIT",
245
+ "_SYSTEMD_USER_UNIT" => "SYSTEMD_USER_UNIT",
246
+ "_TRANSPORT" => "TRANSPORT",
247
+ "_UID" => "UID",
248
+ "CODE_FILE" => "CODE_FILE",
249
+ "CODE_FUNCTION" => "CODE_FUNCTION",
250
+ "CODE_LINE" => "CODE_LINE",
251
+ "ERRNO" => "ERRNO",
252
+ "MESSAGE_ID" => "MESSAGE_ID",
253
+ "RESULT" => "RESULT",
254
+ "UNIT" => "UNIT",
255
+ "SYSLOG_FACILITY" => "SYSLOG_FACILITY",
256
+ "SYSLOG_IDENTIFIER" => "SYSLOG_IDENTIFIER",
257
+ "SYSLOG_PID" => "SYSLOG_PID",
258
+ "_KERNEL_DEVICE" => "KERNEL_DEVICE",
259
+ "_KERNEL_SUBSYSTEM" => "KERNEL_SUBSYSTEM",
260
+ "_UDEV_SYSNAME" => "UDEV_SYSNAME",
261
+ "_UDEV_DEVNODE" => "UDEV_DEVNODE",
262
+ "_UDEV_DEVLINK" => "UDEV_DEVLINK",
263
+ "_SOURCE_REALTIME_TIMESTAMP" => "1501176466216527",
264
+ "__REALTIME_TIMESTAMP" => "1501176466216527",
265
+ "MESSAGE" => "hello world",
266
+ "PRIORITY" => "6",
267
+ "_HOSTNAME" => "myhost"
268
+ }
269
+ end
270
+ def normal_output_t
271
+ {
272
+ "AUDIT_LOGINUID" =>"AUDIT_LOGINUID",
273
+ "AUDIT_SESSION" =>"AUDIT_SESSION",
274
+ "BOOT_ID" =>"BOOT_ID",
275
+ "CAP_EFFECTIVE" =>"CAP_EFFECTIVE",
276
+ "CMDLINE" =>"CMDLINE",
277
+ "COMM" =>"COMM",
278
+ "EXE" =>"EXE",
279
+ "GID" =>"GID",
280
+ "MACHINE_ID" =>"MACHINE_ID",
281
+ "PID" =>"PID",
282
+ "SELINUX_CONTEXT" =>"SELINUX_CONTEXT",
283
+ "SYSTEMD_CGROUP" =>"SYSTEMD_CGROUP",
284
+ "SYSTEMD_OWNER_UID" =>"SYSTEMD_OWNER_UID",
285
+ "SYSTEMD_SESSION" =>"SYSTEMD_SESSION",
286
+ "SYSTEMD_SLICE" =>"SYSTEMD_SLICE",
287
+ "SYSTEMD_UNIT" =>"SYSTEMD_UNIT",
288
+ "SYSTEMD_USER_UNIT" =>"SYSTEMD_USER_UNIT",
289
+ "TRANSPORT" =>"TRANSPORT",
290
+ "UID" =>"UID"
291
+ }
292
+ end
293
+ def normal_output_u
294
+ {
295
+ "CODE_FILE" =>"CODE_FILE",
296
+ "CODE_FUNCTION" =>"CODE_FUNCTION",
297
+ "CODE_LINE" =>"CODE_LINE",
298
+ "ERRNO" =>"ERRNO",
299
+ "MESSAGE_ID" =>"MESSAGE_ID",
300
+ "RESULT" =>"RESULT",
301
+ "UNIT" =>"UNIT",
302
+ "SYSLOG_FACILITY" =>"SYSLOG_FACILITY",
303
+ "SYSLOG_IDENTIFIER" =>"SYSLOG_IDENTIFIER",
304
+ "SYSLOG_PID" =>"SYSLOG_PID"
305
+ }
306
+ end
307
+ def normal_output_k
308
+ {
309
+ "KERNEL_DEVICE" =>"KERNEL_DEVICE",
310
+ "KERNEL_SUBSYSTEM" =>"KERNEL_SUBSYSTEM",
311
+ "UDEV_SYSNAME" =>"UDEV_SYSNAME",
312
+ "UDEV_DEVNODE" =>"UDEV_DEVNODE",
313
+ "UDEV_DEVLINK" =>"UDEV_DEVLINK"
314
+ }
315
+ end
316
+ test 'match records with journal_system_record_tag' do
317
+ rec = emit_with_tag('journal.system', {'a'=>'b', 'MESSAGE'=>'here'}, '
318
+ <formatter>
319
+ tag "**do_not_match**"
320
+ type sys_journal
321
+ remove_keys a,message
322
+ </formatter>
323
+ <formatter>
324
+ tag "journal.system**"
325
+ type sys_journal
326
+ </formatter>
327
+ <formatter>
328
+ tag "**"
329
+ type sys_journal
330
+ remove_keys a,message
331
+ </formatter>
332
+ ')
333
+ assert_equal('b', rec['a'])
334
+ assert_equal('here', rec['message'])
335
+ end
336
+ test 'do not match records without journal_system_record_tag' do
337
+ rec = emit_with_tag('journal.systm', {'a'=>'b', 'MESSAGE'=>'here'}, '
338
+ <formatter>
339
+ tag "journal.system**"
340
+ type sys_journal
341
+ </formatter>
342
+ ')
343
+ assert_equal('b', rec['a'])
344
+ assert_equal('here', rec['MESSAGE'])
345
+ end
346
+ test 'process a journal record, default settings' do
347
+ ENV['IPADDR4'] = '127.0.0.1'
348
+ ENV['IPADDR6'] = '::1'
349
+ ENV['FLUENTD_VERSION'] = 'fversion'
350
+ ENV['DATA_VERSION'] = 'dversion'
351
+ rec = emit_with_tag('journal.system', normal_input, '
352
+ <formatter>
353
+ tag "journal.system**"
354
+ type sys_journal
355
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
356
+ </formatter>
357
+ pipeline_type normalizer
358
+ ')
359
+ assert_equal(normal_output_t, rec['systemd']['t'])
360
+ assert_equal(normal_output_u, rec['systemd']['u'])
361
+ assert_equal(normal_output_k, rec['systemd']['k'])
362
+ assert_equal('hello world', rec['message'])
363
+ assert_equal('info', rec['level'])
364
+ assert_equal('myhost', rec['hostname'])
365
+ assert_equal('2017-07-27T17:27:46.216527+00:00', rec['@timestamp'])
366
+ assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
367
+ assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
368
+ assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
369
+ assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
370
+ assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
371
+ assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['pipeline_metadata']['normalizer']['received_at'])
372
+ dellist = 'log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID'.split(',')
373
+ dellist.each{|field| assert_nil(rec[field])}
374
+ end
375
+ test 'process a journal record, override remove_keys' do
376
+ ENV['IPADDR4'] = '127.0.0.1'
377
+ ENV['IPADDR6'] = '::1'
378
+ ENV['FLUENTD_VERSION'] = 'fversion'
379
+ ENV['DATA_VERSION'] = 'dversion'
380
+ rec = emit_with_tag('journal.system', normal_input, '
381
+ <formatter>
382
+ tag "journal.system**"
383
+ type sys_journal
384
+ remove_keys CONTAINER_NAME,PRIORITY
385
+ </formatter>
386
+ pipeline_type normalizer
387
+ ')
388
+ assert_equal(normal_output_t, rec['systemd']['t'])
389
+ assert_equal(normal_output_u, rec['systemd']['u'])
390
+ assert_equal(normal_output_k, rec['systemd']['k'])
391
+ assert_equal('hello world', rec['message'])
392
+ assert_equal('info', rec['level'])
393
+ assert_equal('myhost', rec['hostname'])
394
+ assert_equal('2017-07-27T17:27:46.216527+00:00', rec['@timestamp'])
395
+ assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
396
+ assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
397
+ assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
398
+ assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
399
+ assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
400
+ assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['pipeline_metadata']['normalizer']['received_at'])
401
+ keeplist = 'log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID'.split(',')
402
+ keeplist.each{|field| normal_input[field] && assert_not_nil(rec[field])}
403
+ dellist = 'CONTAINER_NAME,PRIORITY'.split(',')
404
+ dellist.each{|field| assert_nil(rec[field])}
405
+ end
406
+ test 'try a PRIORITY value that is too large' do
407
+ rec = emit_with_tag('journal.system', {'a'=>'b', 'PRIORITY'=>'10'}, '
408
+ <formatter>
409
+ tag "journal.system**"
410
+ type sys_journal
411
+ </formatter>
412
+ ')
413
+ assert_equal('b', rec['a'])
414
+ assert_equal('unknown', rec['level'])
415
+ end
416
+ test 'try a PRIORITY value that is too small' do
417
+ rec = emit_with_tag('journal.system', {'a'=>'b', 'PRIORITY'=>'-1'}, '
418
+ <formatter>
419
+ tag "journal.system**"
420
+ type sys_journal
421
+ </formatter>
422
+ ')
423
+ assert_equal('b', rec['a'])
424
+ assert_equal('unknown', rec['level'])
425
+ end
426
+ test 'try a PRIORITY value that is not a number' do
427
+ rec = emit_with_tag('journal.system', {'a'=>'b', 'PRIORITY'=>'NaN'}, '
428
+ <formatter>
429
+ tag "journal.system**"
430
+ type sys_journal
431
+ </formatter>
432
+ ')
433
+ assert_equal('b', rec['a'])
434
+ assert_equal('unknown', rec['level'])
435
+ end
436
+ test 'try a PRIORITY value that is a floating point number' do
437
+ rec = emit_with_tag('journal.system', {'a'=>'b', 'PRIORITY'=>'1.0'}, '
438
+ <formatter>
439
+ tag "journal.system**"
440
+ type sys_journal
441
+ </formatter>
442
+ ')
443
+ assert_equal('b', rec['a'])
444
+ assert_equal('unknown', rec['level'])
445
+ end
446
+ test 'test with fallback to __REALTIME_TIMESTAMP' do
447
+ input = normal_input.reject{|k,v| k == '_SOURCE_REALTIME_TIMESTAMP'}
448
+ rec = emit_with_tag('journal.system', input, '
449
+ <formatter>
450
+ tag "journal.system**"
451
+ type sys_journal
452
+ </formatter>
453
+ ')
454
+ assert_equal('2017-07-27T17:27:46.216527+00:00', rec['@timestamp'])
455
+ end
456
+ test 'test using internal time if no timestamp given' do
457
+ input = normal_input.reject do |k,v|
458
+ k == '_SOURCE_REALTIME_TIMESTAMP' || k == '__REALTIME_TIMESTAMP'
459
+ end
460
+ rec = emit_with_tag('journal.system', input, '
461
+ <formatter>
462
+ tag "journal.system**"
463
+ type sys_journal
464
+ </formatter>
465
+ ')
466
+ assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['@timestamp'])
467
+ end
468
+ test 'process a kubernetes journal record, default settings' do
469
+ ENV['IPADDR4'] = '127.0.0.1'
470
+ ENV['IPADDR6'] = '::1'
471
+ ENV['FLUENTD_VERSION'] = 'fversion'
472
+ ENV['DATA_VERSION'] = 'dversion'
473
+ rec = emit_with_tag('kubernetes.journal.container', normal_input, '
474
+ <formatter>
475
+ tag "kubernetes.journal.container**"
476
+ type k8s_journal
477
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
478
+ </formatter>
479
+ pipeline_type normalizer
480
+ ')
481
+ assert_equal(normal_output_t, rec['systemd']['t'])
482
+ assert_equal(normal_output_u, rec['systemd']['u'])
483
+ assert_equal(normal_output_k, rec['systemd']['k'])
484
+ assert_equal('hello world', rec['message'])
485
+ assert_equal('info', rec['level'])
486
+ assert_equal('myhost', rec['hostname'])
487
+ assert_equal('2017-07-27T17:27:46.216527+00:00', rec['@timestamp'])
488
+ assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
489
+ assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
490
+ assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
491
+ assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
492
+ assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
493
+ assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['pipeline_metadata']['normalizer']['received_at'])
494
+ dellist = 'log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID'.split(',')
495
+ dellist.each{|field| assert_nil(rec[field])}
496
+ end
497
+ test 'process a kubernetes journal record, given kubernetes.host' do
498
+ input = normal_input.merge({})
499
+ input['kubernetes'] = {'host' => 'k8shost'}
500
+ ENV['IPADDR4'] = '127.0.0.1'
501
+ ENV['IPADDR6'] = '::1'
502
+ ENV['FLUENTD_VERSION'] = 'fversion'
503
+ ENV['DATA_VERSION'] = 'dversion'
504
+ rec = emit_with_tag('kubernetes.journal.container', input, '
505
+ <formatter>
506
+ tag "kubernetes.journal.container**"
507
+ type k8s_journal
508
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
509
+ </formatter>
510
+ pipeline_type normalizer
511
+ ')
512
+ assert_equal(normal_output_t, rec['systemd']['t'])
513
+ assert_equal(normal_output_u, rec['systemd']['u'])
514
+ assert_equal(normal_output_k, rec['systemd']['k'])
515
+ assert_equal('hello world', rec['message'])
516
+ assert_equal('info', rec['level'])
517
+ assert_equal('k8shost', rec['hostname'])
518
+ assert_equal('2017-07-27T17:27:46.216527+00:00', rec['@timestamp'])
519
+ assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
520
+ assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
521
+ assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
522
+ assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
523
+ assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
524
+ assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['pipeline_metadata']['normalizer']['received_at'])
525
+ dellist = 'log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID'.split(',')
526
+ dellist.each{|field| assert_nil(rec[field])}
527
+ end
528
+ test 'process a kubernetes journal record, preserve message field' do
529
+ input = normal_input.merge({})
530
+ input['message'] = 'my message'
531
+ ENV['IPADDR4'] = '127.0.0.1'
532
+ ENV['IPADDR6'] = '::1'
533
+ ENV['FLUENTD_VERSION'] = 'fversion'
534
+ ENV['DATA_VERSION'] = 'dversion'
535
+ rec = emit_with_tag('kubernetes.journal.container', input, '
536
+ <formatter>
537
+ tag "kubernetes.journal.container**"
538
+ type k8s_journal
539
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
540
+ </formatter>
541
+ pipeline_type normalizer
542
+ ')
543
+ assert_equal(normal_output_t, rec['systemd']['t'])
544
+ assert_equal(normal_output_u, rec['systemd']['u'])
545
+ assert_equal(normal_output_k, rec['systemd']['k'])
546
+ assert_equal('my message', rec['message'])
547
+ assert_equal('info', rec['level'])
548
+ assert_equal('myhost', rec['hostname'])
549
+ assert_equal('2017-07-27T17:27:46.216527+00:00', rec['@timestamp'])
550
+ assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
551
+ assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
552
+ assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
553
+ assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
554
+ assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
555
+ assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['pipeline_metadata']['normalizer']['received_at'])
556
+ dellist = 'log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID'.split(',')
557
+ dellist.each{|field| assert_nil(rec[field])}
558
+ end
559
+ test 'process a /var/log/messages record, default settings' do
560
+ ENV['IPADDR4'] = '127.0.0.1'
561
+ ENV['IPADDR6'] = '::1'
562
+ ENV['FLUENTD_VERSION'] = 'fversion'
563
+ ENV['DATA_VERSION'] = 'dversion'
564
+ now = Time.now
565
+ input = {"pid"=>12345,"ident"=>"service","host"=>"myhost","time"=>now,"message"=>"mymessage"}
566
+ rec = emit_with_tag('system.var.log.messages', input, '
567
+ <formatter>
568
+ tag "system.var.log**"
569
+ type sys_var_log
570
+ remove_keys host,pid,ident
571
+ </formatter>
572
+ pipeline_type normalizer
573
+ ')
574
+ assert_equal(12345, rec['systemd']['t']['PID'])
575
+ assert_equal("service", rec['systemd']['u']['SYSLOG_IDENTIFIER'])
576
+ assert_equal('mymessage', rec['message'])
577
+ assert_equal('myhost', rec['hostname'])
578
+ assert_equal(now.utc.to_datetime.rfc3339(6), rec['@timestamp'])
579
+ assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
580
+ assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
581
+ assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
582
+ assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
583
+ assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
584
+ assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['pipeline_metadata']['normalizer']['received_at'])
585
+ dellist = 'host,pid,ident'.split(',')
586
+ dellist.each{|field| assert_nil(rec[field])}
587
+ end
588
+ test 'process a /var/log/messages record, future date' do
589
+ ENV['IPADDR4'] = '127.0.0.1'
590
+ ENV['IPADDR6'] = '::1'
591
+ ENV['FLUENTD_VERSION'] = 'fversion'
592
+ ENV['DATA_VERSION'] = 'dversion'
593
+ now = DateTime.strptime('Dec 31 23:59:59', '%b %d %H:%M:%S').to_time.utc
594
+ # subtract 1 from year
595
+ expected = Time.new(now.year-1, now.month, now.day, now.hour, now.min, now.sec, now.utc_offset)
596
+ input = {"pid"=>12345,"ident"=>"service","host"=>"myhost","time"=>now,"message"=>"mymessage"}
597
+ rec = emit_with_tag('system.var.log.messages', input, '
598
+ <formatter>
599
+ tag "system.var.log**"
600
+ type sys_var_log
601
+ remove_keys host,pid,ident
602
+ </formatter>
603
+ pipeline_type normalizer
604
+ ')
605
+ assert_equal(12345, rec['systemd']['t']['PID'])
606
+ assert_equal("service", rec['systemd']['u']['SYSLOG_IDENTIFIER'])
607
+ assert_equal('mymessage', rec['message'])
608
+ assert_equal('myhost', rec['hostname'])
609
+ assert_equal(expected.utc.to_datetime.rfc3339(6), rec['@timestamp'])
610
+ assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
611
+ assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
612
+ assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
613
+ assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
614
+ assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
615
+ assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['pipeline_metadata']['normalizer']['received_at'])
616
+ dellist = 'host,pid,ident'.split(',')
617
+ dellist.each{|field| assert_nil(rec[field])}
618
+ end
619
+ test 'process a k8s json-file record, default settings' do
620
+ ENV['IPADDR4'] = '127.0.0.1'
621
+ ENV['IPADDR6'] = '::1'
622
+ ENV['FLUENTD_VERSION'] = 'fversion'
623
+ ENV['DATA_VERSION'] = 'dversion'
624
+ now = Time.now
625
+ input = {'kubernetes'=>{'host'=>'k8shost'},'stream'=>'stderr','time'=>now,'log'=>'mymessage'}
626
+ rec = emit_with_tag('kubernetes.var.log.containers.name.name_this_that_other_log', input, '
627
+ <formatter>
628
+ tag "kubernetes.var.log.containers**"
629
+ type k8s_json_file
630
+ remove_keys log,stream
631
+ </formatter>
632
+ pipeline_type normalizer
633
+ ')
634
+ assert_equal('mymessage', rec['message'])
635
+ assert_equal('k8shost', rec['hostname'])
636
+ assert_equal('err', rec['level'])
637
+ assert_equal(now.utc.to_datetime.rfc3339(6), rec['@timestamp'])
638
+ assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
639
+ assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
640
+ assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
641
+ assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
642
+ assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
643
+ assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['pipeline_metadata']['normalizer']['received_at'])
644
+ dellist = 'host,pid,ident'.split(',')
645
+ dellist.each{|field| assert_nil(rec[field])}
646
+ end
647
+ # tests for elasticsearch index functionality
648
+ test 'construct an operations index prefix' do
649
+ rec = emit_with_tag('journal.system', normal_input, '
650
+ <formatter>
651
+ tag "journal.system**"
652
+ type sys_journal
653
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
654
+ </formatter>
655
+ <formatter>
656
+ tag "kubernetes.journal.container**"
657
+ type k8s_journal
658
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
659
+ </formatter>
660
+ <elasticsearch_index_name>
661
+ tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
662
+ name_type operations_prefix
663
+ </elasticsearch_index_name>
664
+ <elasticsearch_index_name>
665
+ tag "**"
666
+ name_type project_prefix
667
+ </elasticsearch_index_name>
668
+ ')
669
+ assert_equal('.operations', rec['viaq_index_prefix'])
670
+ end
671
+ test 'construct an operations index prefix with named field' do
672
+ rec = emit_with_tag('journal.system', normal_input, '
673
+ <formatter>
674
+ tag "journal.system**"
675
+ type sys_journal
676
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
677
+ </formatter>
678
+ <formatter>
679
+ tag "kubernetes.journal.container**"
680
+ type k8s_journal
681
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
682
+ </formatter>
683
+ <elasticsearch_index_name>
684
+ tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
685
+ name_type operations_prefix
686
+ </elasticsearch_index_name>
687
+ <elasticsearch_index_name>
688
+ tag "**"
689
+ name_type project_prefix
690
+ </elasticsearch_index_name>
691
+ elasticsearch_index_prefix_field my_index_prefix
692
+ ')
693
+ assert_equal('.operations', rec['my_index_prefix'])
694
+ end
695
+ test 'construct an operations index name with named field' do
696
+ rec = emit_with_tag('journal.system', normal_input, '
697
+ <formatter>
698
+ tag "journal.system**"
699
+ type sys_journal
700
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
701
+ </formatter>
702
+ <formatter>
703
+ tag "kubernetes.journal.container**"
704
+ type k8s_journal
705
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
706
+ </formatter>
707
+ <elasticsearch_index_name>
708
+ tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
709
+ name_type operations_full
710
+ </elasticsearch_index_name>
711
+ <elasticsearch_index_name>
712
+ tag "**"
713
+ name_type project_full
714
+ </elasticsearch_index_name>
715
+ elasticsearch_index_name_field my_index_name
716
+ ')
717
+ assert_equal('.operations.2017.07.27', rec['my_index_name'])
718
+ end
719
+ test 'log error if missing kubernetes field' do
720
+ rec = emit_with_tag('kubernetes.journal.container.something', normal_input, '
721
+ <formatter>
722
+ tag "journal.system**"
723
+ type sys_journal
724
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
725
+ </formatter>
726
+ <formatter>
727
+ tag "kubernetes.journal.container**"
728
+ type k8s_journal
729
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
730
+ </formatter>
731
+ <elasticsearch_index_name>
732
+ tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
733
+ name_type operations_prefix
734
+ </elasticsearch_index_name>
735
+ <elasticsearch_index_name>
736
+ tag "**"
737
+ name_type project_prefix
738
+ </elasticsearch_index_name>
739
+ ')
740
+ assert_match /record is missing kubernetes field/, @dlog.logs[0]
741
+ end
742
+ test 'log error if missing kubernetes.namespace_name field' do
743
+ input = normal_input.merge({})
744
+ input['kubernetes'] = 'junk'
745
+ rec = emit_with_tag('kubernetes.journal.container.something', input, '
746
+ <formatter>
747
+ tag "journal.system**"
748
+ type sys_journal
749
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
750
+ </formatter>
751
+ <formatter>
752
+ tag "kubernetes.journal.container**"
753
+ type k8s_journal
754
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
755
+ </formatter>
756
+ <elasticsearch_index_name>
757
+ tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
758
+ name_type operations_prefix
759
+ </elasticsearch_index_name>
760
+ <elasticsearch_index_name>
761
+ tag "**"
762
+ name_type project_prefix
763
+ </elasticsearch_index_name>
764
+ ')
765
+ assert_match /record is missing kubernetes.namespace_name field/, @dlog.logs[0]
766
+ end
767
+ test 'log error if missing kubernetes.namespace_id field' do
768
+ input = normal_input.merge({})
769
+ input['kubernetes'] = {'namespace_name'=>'junk'}
770
+ rec = emit_with_tag('kubernetes.journal.container.something', input, '
771
+ <formatter>
772
+ tag "journal.system**"
773
+ type sys_journal
774
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
775
+ </formatter>
776
+ <formatter>
777
+ tag "kubernetes.journal.container**"
778
+ type k8s_journal
779
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
780
+ </formatter>
781
+ <elasticsearch_index_name>
782
+ tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
783
+ name_type operations_prefix
784
+ </elasticsearch_index_name>
785
+ <elasticsearch_index_name>
786
+ tag "**"
787
+ name_type project_prefix
788
+ </elasticsearch_index_name>
789
+ ')
790
+ assert_match /record is missing kubernetes.namespace_id field/, @dlog.logs[0]
791
+ end
792
+ test 'construct a kubernetes index prefix' do
793
+ input = normal_input.merge({})
794
+ input['kubernetes'] = {'namespace_name'=>'name', 'namespace_id'=>'uuid'}
795
+ rec = emit_with_tag('kubernetes.journal.container.something', input, '
796
+ <formatter>
797
+ tag "journal.system**"
798
+ type sys_journal
799
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
800
+ </formatter>
801
+ <formatter>
802
+ tag "kubernetes.journal.container**"
803
+ type k8s_journal
804
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
805
+ </formatter>
806
+ <elasticsearch_index_name>
807
+ tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
808
+ name_type operations_prefix
809
+ </elasticsearch_index_name>
810
+ <elasticsearch_index_name>
811
+ tag "**"
812
+ name_type project_prefix
813
+ </elasticsearch_index_name>
814
+ ')
815
+ assert_equal('project.name.uuid', rec['viaq_index_prefix'])
816
+ end
817
+ test 'construct a kubernetes index prefix with named field' do
818
+ input = normal_input.merge({})
819
+ input['kubernetes'] = {'namespace_name'=>'name', 'namespace_id'=>'uuid'}
820
+ rec = emit_with_tag('kubernetes.journal.container.something', input, '
821
+ <formatter>
822
+ tag "journal.system**"
823
+ type sys_journal
824
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
825
+ </formatter>
826
+ <formatter>
827
+ tag "kubernetes.journal.container**"
828
+ type k8s_journal
829
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
830
+ </formatter>
831
+ <elasticsearch_index_name>
832
+ tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
833
+ name_type operations_prefix
834
+ </elasticsearch_index_name>
835
+ <elasticsearch_index_name>
836
+ tag "**"
837
+ name_type project_prefix
838
+ </elasticsearch_index_name>
839
+ elasticsearch_index_prefix_field my_index_prefix
840
+ ')
841
+ assert_equal('project.name.uuid', rec['my_index_prefix'])
842
+ end
843
+ test 'construct a kubernetes index name with named field' do
844
+ input = normal_input.merge({})
845
+ input['kubernetes'] = {'namespace_name'=>'name', 'namespace_id'=>'uuid'}
846
+ rec = emit_with_tag('kubernetes.journal.container.something', input, '
847
+ <formatter>
848
+ tag "journal.system**"
849
+ type sys_journal
850
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
851
+ </formatter>
852
+ <formatter>
853
+ tag "kubernetes.journal.container**"
854
+ type k8s_journal
855
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
856
+ </formatter>
857
+ <elasticsearch_index_name>
858
+ tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
859
+ name_type operations_full
860
+ </elasticsearch_index_name>
861
+ <elasticsearch_index_name>
862
+ tag "**"
863
+ name_type project_full
864
+ </elasticsearch_index_name>
865
+ elasticsearch_index_name_field my_index_name
866
+ ')
867
+ assert_equal('project.name.uuid.2017.07.27', rec['my_index_name'])
868
+ end
216
869
  end
217
870
  end