fluent-plugin-viaq_data_model 0.0.5 → 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +5 -1
  3. data/README.md +157 -0
  4. data/fluent-plugin-viaq_data_model.gemspec +7 -3
  5. data/lib/fluent/plugin/filter_viaq_data_model.rb +244 -0
  6. data/lib/fluent/plugin/filter_viaq_data_model_systemd.rb +133 -0
  7. data/test/test_filter_viaq_data_model.rb +654 -1
  8. metadata +21 -6
data/lib/fluent/plugin/filter_viaq_data_model_systemd.rb ADDED
@@ -0,0 +1,133 @@
1
+ #
2
+ # Fluentd ViaQ data model Filter Plugin
3
+ #
4
+ # Copyright 2017 Red Hat, Inc.
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+ require 'time'
19
+ require 'date'
20
+
21
+ module ViaqDataModelFilterSystemd
22
+ # map of journal fields to viaq data model field
23
+ JOURNAL_FIELD_MAP_SYSTEMD_T = {
24
+ "_AUDIT_LOGINUID" => "AUDIT_LOGINUID",
25
+ "_AUDIT_SESSION" => "AUDIT_SESSION",
26
+ "_BOOT_ID" => "BOOT_ID",
27
+ "_CAP_EFFECTIVE" => "CAP_EFFECTIVE",
28
+ "_CMDLINE" => "CMDLINE",
29
+ "_COMM" => "COMM",
30
+ "_EXE" => "EXE",
31
+ "_GID" => "GID",
32
+ "_MACHINE_ID" => "MACHINE_ID",
33
+ "_PID" => "PID",
34
+ "_SELINUX_CONTEXT" => "SELINUX_CONTEXT",
35
+ "_SYSTEMD_CGROUP" => "SYSTEMD_CGROUP",
36
+ "_SYSTEMD_OWNER_UID" => "SYSTEMD_OWNER_UID",
37
+ "_SYSTEMD_SESSION" => "SYSTEMD_SESSION",
38
+ "_SYSTEMD_SLICE" => "SYSTEMD_SLICE",
39
+ "_SYSTEMD_UNIT" => "SYSTEMD_UNIT",
40
+ "_SYSTEMD_USER_UNIT" => "SYSTEMD_USER_UNIT",
41
+ "_TRANSPORT" => "TRANSPORT",
42
+ "_UID" => "UID"
43
+ }
44
+
45
+ JOURNAL_FIELD_MAP_SYSTEMD_U = {
46
+ "CODE_FILE" => "CODE_FILE",
47
+ "CODE_FUNCTION" => "CODE_FUNCTION",
48
+ "CODE_LINE" => "CODE_LINE",
49
+ "ERRNO" => "ERRNO",
50
+ "MESSAGE_ID" => "MESSAGE_ID",
51
+ "RESULT" => "RESULT",
52
+ "UNIT" => "UNIT",
53
+ "SYSLOG_FACILITY" => "SYSLOG_FACILITY",
54
+ "SYSLOG_IDENTIFIER" => "SYSLOG_IDENTIFIER",
55
+ "SYSLOG_PID" => "SYSLOG_PID"
56
+ }
57
+
58
+ JOURNAL_FIELD_MAP_SYSTEMD_K = {
59
+ "_KERNEL_DEVICE" => "KERNEL_DEVICE",
60
+ "_KERNEL_SUBSYSTEM" => "KERNEL_SUBSYSTEM",
61
+ "_UDEV_SYSNAME" => "UDEV_SYSNAME",
62
+ "_UDEV_DEVNODE" => "UDEV_DEVNODE",
63
+ "_UDEV_DEVLINK" => "UDEV_DEVLINK",
64
+ }
65
+
66
+ JOURNAL_TIME_FIELDS = ['_SOURCE_REALTIME_TIMESTAMP', '__REALTIME_TIMESTAMP']
67
+
68
+ def process_journal_fields(tag, time, record, fmtr_type)
69
+ systemd_t = {}
70
+ JOURNAL_FIELD_MAP_SYSTEMD_T.each do |jkey, key|
71
+ if record[jkey]
72
+ systemd_t[key] = record[jkey]
73
+ end
74
+ end
75
+ systemd_u = {}
76
+ JOURNAL_FIELD_MAP_SYSTEMD_U.each do |jkey, key|
77
+ if record[jkey]
78
+ systemd_u[key] = record[jkey]
79
+ end
80
+ end
81
+ systemd_k = {}
82
+ JOURNAL_FIELD_MAP_SYSTEMD_K.each do |jkey, key|
83
+ if record[jkey]
84
+ systemd_k[key] = record[jkey]
85
+ end
86
+ end
87
+ unless systemd_t.empty?
88
+ (record['systemd'] ||= {})['t'] = systemd_t
89
+ end
90
+ unless systemd_u.empty?
91
+ (record['systemd'] ||= {})['u'] = systemd_u
92
+ end
93
+ unless systemd_k.empty?
94
+ (record['systemd'] ||= {})['k'] = systemd_k
95
+ end
96
+ begin
97
+ pri_index = ('%d' % record['PRIORITY'] || 9).to_i
98
+ case
99
+ when pri_index < 0
100
+ pri_index = 9
101
+ when pri_index > 9
102
+ pri_index = 9
103
+ end
104
+ rescue
105
+ pri_index = 9
106
+ end
107
+ record['level'] = ["emerg", "alert", "crit", "err", "warning", "notice", "info", "debug", "trace", "unknown"][pri_index]
108
+ JOURNAL_TIME_FIELDS.each do |field|
109
+ if record[field]
110
+ record['time'] = Time.at(record[field].to_f / 1000000.0).utc.to_datetime.rfc3339(6)
111
+ break
112
+ end
113
+ end
114
+ case fmtr_type
115
+ when :sys_journal
116
+ record['message'] = record['MESSAGE']
117
+ if record['_HOSTNAME'].eql?('localhost') && @docker_hostname
118
+ record['hostname'] = @docker_hostname
119
+ else
120
+ record['hostname'] = record['_HOSTNAME']
121
+ end
122
+ when :k8s_journal
123
+ record['message'] = record['message'] || record['MESSAGE'] || record['log']
124
+ if record['kubernetes'] && record['kubernetes']['host']
125
+ record['hostname'] = record['kubernetes']['host']
126
+ elsif @docker_hostname
127
+ record['hostname'] = @docker_hostname
128
+ else
129
+ record['hostname'] = record['_HOSTNAME']
130
+ end
131
+ end
132
+ end
133
+ end
data/test/test_filter_viaq_data_model.rb CHANGED
@@ -31,7 +31,9 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
31
31
  end
32
32
 
33
33
  def create_driver(conf = '')
34
- Test::FilterTestDriver.new(ViaqDataModelFilter, 'this.is.a.tag').configure(conf, true)
34
+ d = Test::FilterTestDriver.new(ViaqDataModelFilter, 'this.is.a.tag').configure(conf, true)
35
+ @dlog = d.instance.log
36
+ d
35
37
  end
36
38
 
37
39
  sub_test_case 'configure' do
@@ -212,6 +214,657 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
212
214
  assert_equal([88], rec['h']['i']['m'])
213
215
  assert_true(rec['h']['i']['n'])
214
216
  end
217
+ end
215
218
 
219
+ sub_test_case 'formatters and elasticsearch index names' do
220
+ def emit_with_tag(tag, msg={}, conf='')
221
+ d = create_driver(conf)
222
+ d.run {
223
+ d.emit_with_tag(tag, msg, @time)
224
+ }.filtered.instance_variable_get(:@record_array)[0]
225
+ end
226
+
227
+ def normal_input
228
+ {
229
+ "_AUDIT_LOGINUID" => "AUDIT_LOGINUID",
230
+ "_AUDIT_SESSION" => "AUDIT_SESSION",
231
+ "_BOOT_ID" => "BOOT_ID",
232
+ "_CAP_EFFECTIVE" => "CAP_EFFECTIVE",
233
+ "_CMDLINE" => "CMDLINE",
234
+ "_COMM" => "COMM",
235
+ "_EXE" => "EXE",
236
+ "_GID" => "GID",
237
+ "_MACHINE_ID" => "MACHINE_ID",
238
+ "_PID" => "PID",
239
+ "_SELINUX_CONTEXT" => "SELINUX_CONTEXT",
240
+ "_SYSTEMD_CGROUP" => "SYSTEMD_CGROUP",
241
+ "_SYSTEMD_OWNER_UID" => "SYSTEMD_OWNER_UID",
242
+ "_SYSTEMD_SESSION" => "SYSTEMD_SESSION",
243
+ "_SYSTEMD_SLICE" => "SYSTEMD_SLICE",
244
+ "_SYSTEMD_UNIT" => "SYSTEMD_UNIT",
245
+ "_SYSTEMD_USER_UNIT" => "SYSTEMD_USER_UNIT",
246
+ "_TRANSPORT" => "TRANSPORT",
247
+ "_UID" => "UID",
248
+ "CODE_FILE" => "CODE_FILE",
249
+ "CODE_FUNCTION" => "CODE_FUNCTION",
250
+ "CODE_LINE" => "CODE_LINE",
251
+ "ERRNO" => "ERRNO",
252
+ "MESSAGE_ID" => "MESSAGE_ID",
253
+ "RESULT" => "RESULT",
254
+ "UNIT" => "UNIT",
255
+ "SYSLOG_FACILITY" => "SYSLOG_FACILITY",
256
+ "SYSLOG_IDENTIFIER" => "SYSLOG_IDENTIFIER",
257
+ "SYSLOG_PID" => "SYSLOG_PID",
258
+ "_KERNEL_DEVICE" => "KERNEL_DEVICE",
259
+ "_KERNEL_SUBSYSTEM" => "KERNEL_SUBSYSTEM",
260
+ "_UDEV_SYSNAME" => "UDEV_SYSNAME",
261
+ "_UDEV_DEVNODE" => "UDEV_DEVNODE",
262
+ "_UDEV_DEVLINK" => "UDEV_DEVLINK",
263
+ "_SOURCE_REALTIME_TIMESTAMP" => "1501176466216527",
264
+ "__REALTIME_TIMESTAMP" => "1501176466216527",
265
+ "MESSAGE" => "hello world",
266
+ "PRIORITY" => "6",
267
+ "_HOSTNAME" => "myhost"
268
+ }
269
+ end
270
+ def normal_output_t
271
+ {
272
+ "AUDIT_LOGINUID" =>"AUDIT_LOGINUID",
273
+ "AUDIT_SESSION" =>"AUDIT_SESSION",
274
+ "BOOT_ID" =>"BOOT_ID",
275
+ "CAP_EFFECTIVE" =>"CAP_EFFECTIVE",
276
+ "CMDLINE" =>"CMDLINE",
277
+ "COMM" =>"COMM",
278
+ "EXE" =>"EXE",
279
+ "GID" =>"GID",
280
+ "MACHINE_ID" =>"MACHINE_ID",
281
+ "PID" =>"PID",
282
+ "SELINUX_CONTEXT" =>"SELINUX_CONTEXT",
283
+ "SYSTEMD_CGROUP" =>"SYSTEMD_CGROUP",
284
+ "SYSTEMD_OWNER_UID" =>"SYSTEMD_OWNER_UID",
285
+ "SYSTEMD_SESSION" =>"SYSTEMD_SESSION",
286
+ "SYSTEMD_SLICE" =>"SYSTEMD_SLICE",
287
+ "SYSTEMD_UNIT" =>"SYSTEMD_UNIT",
288
+ "SYSTEMD_USER_UNIT" =>"SYSTEMD_USER_UNIT",
289
+ "TRANSPORT" =>"TRANSPORT",
290
+ "UID" =>"UID"
291
+ }
292
+ end
293
+ def normal_output_u
294
+ {
295
+ "CODE_FILE" =>"CODE_FILE",
296
+ "CODE_FUNCTION" =>"CODE_FUNCTION",
297
+ "CODE_LINE" =>"CODE_LINE",
298
+ "ERRNO" =>"ERRNO",
299
+ "MESSAGE_ID" =>"MESSAGE_ID",
300
+ "RESULT" =>"RESULT",
301
+ "UNIT" =>"UNIT",
302
+ "SYSLOG_FACILITY" =>"SYSLOG_FACILITY",
303
+ "SYSLOG_IDENTIFIER" =>"SYSLOG_IDENTIFIER",
304
+ "SYSLOG_PID" =>"SYSLOG_PID"
305
+ }
306
+ end
307
+ def normal_output_k
308
+ {
309
+ "KERNEL_DEVICE" =>"KERNEL_DEVICE",
310
+ "KERNEL_SUBSYSTEM" =>"KERNEL_SUBSYSTEM",
311
+ "UDEV_SYSNAME" =>"UDEV_SYSNAME",
312
+ "UDEV_DEVNODE" =>"UDEV_DEVNODE",
313
+ "UDEV_DEVLINK" =>"UDEV_DEVLINK"
314
+ }
315
+ end
316
+ test 'match records with journal_system_record_tag' do
317
+ rec = emit_with_tag('journal.system', {'a'=>'b', 'MESSAGE'=>'here'}, '
318
+ <formatter>
319
+ tag "**do_not_match**"
320
+ type sys_journal
321
+ remove_keys a,message
322
+ </formatter>
323
+ <formatter>
324
+ tag "journal.system**"
325
+ type sys_journal
326
+ </formatter>
327
+ <formatter>
328
+ tag "**"
329
+ type sys_journal
330
+ remove_keys a,message
331
+ </formatter>
332
+ ')
333
+ assert_equal('b', rec['a'])
334
+ assert_equal('here', rec['message'])
335
+ end
336
+ test 'do not match records without journal_system_record_tag' do
337
+ rec = emit_with_tag('journal.systm', {'a'=>'b', 'MESSAGE'=>'here'}, '
338
+ <formatter>
339
+ tag "journal.system**"
340
+ type sys_journal
341
+ </formatter>
342
+ ')
343
+ assert_equal('b', rec['a'])
344
+ assert_equal('here', rec['MESSAGE'])
345
+ end
346
+ test 'process a journal record, default settings' do
347
+ ENV['IPADDR4'] = '127.0.0.1'
348
+ ENV['IPADDR6'] = '::1'
349
+ ENV['FLUENTD_VERSION'] = 'fversion'
350
+ ENV['DATA_VERSION'] = 'dversion'
351
+ rec = emit_with_tag('journal.system', normal_input, '
352
+ <formatter>
353
+ tag "journal.system**"
354
+ type sys_journal
355
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
356
+ </formatter>
357
+ pipeline_type normalizer
358
+ ')
359
+ assert_equal(normal_output_t, rec['systemd']['t'])
360
+ assert_equal(normal_output_u, rec['systemd']['u'])
361
+ assert_equal(normal_output_k, rec['systemd']['k'])
362
+ assert_equal('hello world', rec['message'])
363
+ assert_equal('info', rec['level'])
364
+ assert_equal('myhost', rec['hostname'])
365
+ assert_equal('2017-07-27T17:27:46.216527+00:00', rec['@timestamp'])
366
+ assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
367
+ assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
368
+ assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
369
+ assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
370
+ assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
371
+ assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['pipeline_metadata']['normalizer']['received_at'])
372
+ dellist = 'log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID'.split(',')
373
+ dellist.each{|field| assert_nil(rec[field])}
374
+ end
375
+ test 'process a journal record, override remove_keys' do
376
+ ENV['IPADDR4'] = '127.0.0.1'
377
+ ENV['IPADDR6'] = '::1'
378
+ ENV['FLUENTD_VERSION'] = 'fversion'
379
+ ENV['DATA_VERSION'] = 'dversion'
380
+ rec = emit_with_tag('journal.system', normal_input, '
381
+ <formatter>
382
+ tag "journal.system**"
383
+ type sys_journal
384
+ remove_keys CONTAINER_NAME,PRIORITY
385
+ </formatter>
386
+ pipeline_type normalizer
387
+ ')
388
+ assert_equal(normal_output_t, rec['systemd']['t'])
389
+ assert_equal(normal_output_u, rec['systemd']['u'])
390
+ assert_equal(normal_output_k, rec['systemd']['k'])
391
+ assert_equal('hello world', rec['message'])
392
+ assert_equal('info', rec['level'])
393
+ assert_equal('myhost', rec['hostname'])
394
+ assert_equal('2017-07-27T17:27:46.216527+00:00', rec['@timestamp'])
395
+ assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
396
+ assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
397
+ assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
398
+ assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
399
+ assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
400
+ assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['pipeline_metadata']['normalizer']['received_at'])
401
+ keeplist = 'log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID'.split(',')
402
+ keeplist.each{|field| normal_input[field] && assert_not_nil(rec[field])}
403
+ dellist = 'CONTAINER_NAME,PRIORITY'.split(',')
404
+ dellist.each{|field| assert_nil(rec[field])}
405
+ end
406
+ test 'try a PRIORITY value that is too large' do
407
+ rec = emit_with_tag('journal.system', {'a'=>'b', 'PRIORITY'=>'10'}, '
408
+ <formatter>
409
+ tag "journal.system**"
410
+ type sys_journal
411
+ </formatter>
412
+ ')
413
+ assert_equal('b', rec['a'])
414
+ assert_equal('unknown', rec['level'])
415
+ end
416
+ test 'try a PRIORITY value that is too small' do
417
+ rec = emit_with_tag('journal.system', {'a'=>'b', 'PRIORITY'=>'-1'}, '
418
+ <formatter>
419
+ tag "journal.system**"
420
+ type sys_journal
421
+ </formatter>
422
+ ')
423
+ assert_equal('b', rec['a'])
424
+ assert_equal('unknown', rec['level'])
425
+ end
426
+ test 'try a PRIORITY value that is not a number' do
427
+ rec = emit_with_tag('journal.system', {'a'=>'b', 'PRIORITY'=>'NaN'}, '
428
+ <formatter>
429
+ tag "journal.system**"
430
+ type sys_journal
431
+ </formatter>
432
+ ')
433
+ assert_equal('b', rec['a'])
434
+ assert_equal('unknown', rec['level'])
435
+ end
436
+ test 'try a PRIORITY value that is a floating point number' do
437
+ rec = emit_with_tag('journal.system', {'a'=>'b', 'PRIORITY'=>'1.0'}, '
438
+ <formatter>
439
+ tag "journal.system**"
440
+ type sys_journal
441
+ </formatter>
442
+ ')
443
+ assert_equal('b', rec['a'])
444
+ assert_equal('unknown', rec['level'])
445
+ end
446
+ test 'test with fallback to __REALTIME_TIMESTAMP' do
447
+ input = normal_input.reject{|k,v| k == '_SOURCE_REALTIME_TIMESTAMP'}
448
+ rec = emit_with_tag('journal.system', input, '
449
+ <formatter>
450
+ tag "journal.system**"
451
+ type sys_journal
452
+ </formatter>
453
+ ')
454
+ assert_equal('2017-07-27T17:27:46.216527+00:00', rec['@timestamp'])
455
+ end
456
+ test 'test using internal time if no timestamp given' do
457
+ input = normal_input.reject do |k,v|
458
+ k == '_SOURCE_REALTIME_TIMESTAMP' || k == '__REALTIME_TIMESTAMP'
459
+ end
460
+ rec = emit_with_tag('journal.system', input, '
461
+ <formatter>
462
+ tag "journal.system**"
463
+ type sys_journal
464
+ </formatter>
465
+ ')
466
+ assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['@timestamp'])
467
+ end
468
+ test 'process a kubernetes journal record, default settings' do
469
+ ENV['IPADDR4'] = '127.0.0.1'
470
+ ENV['IPADDR6'] = '::1'
471
+ ENV['FLUENTD_VERSION'] = 'fversion'
472
+ ENV['DATA_VERSION'] = 'dversion'
473
+ rec = emit_with_tag('kubernetes.journal.container', normal_input, '
474
+ <formatter>
475
+ tag "kubernetes.journal.container**"
476
+ type k8s_journal
477
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
478
+ </formatter>
479
+ pipeline_type normalizer
480
+ ')
481
+ assert_equal(normal_output_t, rec['systemd']['t'])
482
+ assert_equal(normal_output_u, rec['systemd']['u'])
483
+ assert_equal(normal_output_k, rec['systemd']['k'])
484
+ assert_equal('hello world', rec['message'])
485
+ assert_equal('info', rec['level'])
486
+ assert_equal('myhost', rec['hostname'])
487
+ assert_equal('2017-07-27T17:27:46.216527+00:00', rec['@timestamp'])
488
+ assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
489
+ assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
490
+ assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
491
+ assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
492
+ assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
493
+ assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['pipeline_metadata']['normalizer']['received_at'])
494
+ dellist = 'log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID'.split(',')
495
+ dellist.each{|field| assert_nil(rec[field])}
496
+ end
497
+ test 'process a kubernetes journal record, given kubernetes.host' do
498
+ input = normal_input.merge({})
499
+ input['kubernetes'] = {'host' => 'k8shost'}
500
+ ENV['IPADDR4'] = '127.0.0.1'
501
+ ENV['IPADDR6'] = '::1'
502
+ ENV['FLUENTD_VERSION'] = 'fversion'
503
+ ENV['DATA_VERSION'] = 'dversion'
504
+ rec = emit_with_tag('kubernetes.journal.container', input, '
505
+ <formatter>
506
+ tag "kubernetes.journal.container**"
507
+ type k8s_journal
508
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
509
+ </formatter>
510
+ pipeline_type normalizer
511
+ ')
512
+ assert_equal(normal_output_t, rec['systemd']['t'])
513
+ assert_equal(normal_output_u, rec['systemd']['u'])
514
+ assert_equal(normal_output_k, rec['systemd']['k'])
515
+ assert_equal('hello world', rec['message'])
516
+ assert_equal('info', rec['level'])
517
+ assert_equal('k8shost', rec['hostname'])
518
+ assert_equal('2017-07-27T17:27:46.216527+00:00', rec['@timestamp'])
519
+ assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
520
+ assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
521
+ assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
522
+ assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
523
+ assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
524
+ assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['pipeline_metadata']['normalizer']['received_at'])
525
+ dellist = 'log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID'.split(',')
526
+ dellist.each{|field| assert_nil(rec[field])}
527
+ end
528
+ test 'process a kubernetes journal record, preserve message field' do
529
+ input = normal_input.merge({})
530
+ input['message'] = 'my message'
531
+ ENV['IPADDR4'] = '127.0.0.1'
532
+ ENV['IPADDR6'] = '::1'
533
+ ENV['FLUENTD_VERSION'] = 'fversion'
534
+ ENV['DATA_VERSION'] = 'dversion'
535
+ rec = emit_with_tag('kubernetes.journal.container', input, '
536
+ <formatter>
537
+ tag "kubernetes.journal.container**"
538
+ type k8s_journal
539
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
540
+ </formatter>
541
+ pipeline_type normalizer
542
+ ')
543
+ assert_equal(normal_output_t, rec['systemd']['t'])
544
+ assert_equal(normal_output_u, rec['systemd']['u'])
545
+ assert_equal(normal_output_k, rec['systemd']['k'])
546
+ assert_equal('my message', rec['message'])
547
+ assert_equal('info', rec['level'])
548
+ assert_equal('myhost', rec['hostname'])
549
+ assert_equal('2017-07-27T17:27:46.216527+00:00', rec['@timestamp'])
550
+ assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
551
+ assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
552
+ assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
553
+ assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
554
+ assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
555
+ assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['pipeline_metadata']['normalizer']['received_at'])
556
+ dellist = 'log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID'.split(',')
557
+ dellist.each{|field| assert_nil(rec[field])}
558
+ end
559
+ test 'process a /var/log/messages record, default settings' do
560
+ ENV['IPADDR4'] = '127.0.0.1'
561
+ ENV['IPADDR6'] = '::1'
562
+ ENV['FLUENTD_VERSION'] = 'fversion'
563
+ ENV['DATA_VERSION'] = 'dversion'
564
+ now = Time.now
565
+ input = {"pid"=>12345,"ident"=>"service","host"=>"myhost","time"=>now,"message"=>"mymessage"}
566
+ rec = emit_with_tag('system.var.log.messages', input, '
567
+ <formatter>
568
+ tag "system.var.log**"
569
+ type sys_var_log
570
+ remove_keys host,pid,ident
571
+ </formatter>
572
+ pipeline_type normalizer
573
+ ')
574
+ assert_equal(12345, rec['systemd']['t']['PID'])
575
+ assert_equal("service", rec['systemd']['u']['SYSLOG_IDENTIFIER'])
576
+ assert_equal('mymessage', rec['message'])
577
+ assert_equal('myhost', rec['hostname'])
578
+ assert_equal(now.utc.to_datetime.rfc3339(6), rec['@timestamp'])
579
+ assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
580
+ assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
581
+ assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
582
+ assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
583
+ assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
584
+ assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['pipeline_metadata']['normalizer']['received_at'])
585
+ dellist = 'host,pid,ident'.split(',')
586
+ dellist.each{|field| assert_nil(rec[field])}
587
+ end
588
+ test 'process a /var/log/messages record, future date' do
589
+ ENV['IPADDR4'] = '127.0.0.1'
590
+ ENV['IPADDR6'] = '::1'
591
+ ENV['FLUENTD_VERSION'] = 'fversion'
592
+ ENV['DATA_VERSION'] = 'dversion'
593
+ now = DateTime.strptime('Dec 31 23:59:59', '%b %d %H:%M:%S').to_time.utc
594
+ # subtract 1 from year
595
+ expected = Time.new(now.year-1, now.month, now.day, now.hour, now.min, now.sec, now.utc_offset)
596
+ input = {"pid"=>12345,"ident"=>"service","host"=>"myhost","time"=>now,"message"=>"mymessage"}
597
+ rec = emit_with_tag('system.var.log.messages', input, '
598
+ <formatter>
599
+ tag "system.var.log**"
600
+ type sys_var_log
601
+ remove_keys host,pid,ident
602
+ </formatter>
603
+ pipeline_type normalizer
604
+ ')
605
+ assert_equal(12345, rec['systemd']['t']['PID'])
606
+ assert_equal("service", rec['systemd']['u']['SYSLOG_IDENTIFIER'])
607
+ assert_equal('mymessage', rec['message'])
608
+ assert_equal('myhost', rec['hostname'])
609
+ assert_equal(expected.utc.to_datetime.rfc3339(6), rec['@timestamp'])
610
+ assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
611
+ assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
612
+ assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
613
+ assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
614
+ assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
615
+ assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['pipeline_metadata']['normalizer']['received_at'])
616
+ dellist = 'host,pid,ident'.split(',')
617
+ dellist.each{|field| assert_nil(rec[field])}
618
+ end
619
+ test 'process a k8s json-file record, default settings' do
620
+ ENV['IPADDR4'] = '127.0.0.1'
621
+ ENV['IPADDR6'] = '::1'
622
+ ENV['FLUENTD_VERSION'] = 'fversion'
623
+ ENV['DATA_VERSION'] = 'dversion'
624
+ now = Time.now
625
+ input = {'kubernetes'=>{'host'=>'k8shost'},'stream'=>'stderr','time'=>now,'log'=>'mymessage'}
626
+ rec = emit_with_tag('kubernetes.var.log.containers.name.name_this_that_other_log', input, '
627
+ <formatter>
628
+ tag "kubernetes.var.log.containers**"
629
+ type k8s_json_file
630
+ remove_keys log,stream
631
+ </formatter>
632
+ pipeline_type normalizer
633
+ ')
634
+ assert_equal('mymessage', rec['message'])
635
+ assert_equal('k8shost', rec['hostname'])
636
+ assert_equal('err', rec['level'])
637
+ assert_equal(now.utc.to_datetime.rfc3339(6), rec['@timestamp'])
638
+ assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
639
+ assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
640
+ assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
641
+ assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
642
+ assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
643
+ assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['pipeline_metadata']['normalizer']['received_at'])
644
+ dellist = 'host,pid,ident'.split(',')
645
+ dellist.each{|field| assert_nil(rec[field])}
646
+ end
647
+ # tests for elasticsearch index functionality
648
+ test 'construct an operations index prefix' do
649
+ rec = emit_with_tag('journal.system', normal_input, '
650
+ <formatter>
651
+ tag "journal.system**"
652
+ type sys_journal
653
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
654
+ </formatter>
655
+ <formatter>
656
+ tag "kubernetes.journal.container**"
657
+ type k8s_journal
658
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
659
+ </formatter>
660
+ <elasticsearch_index_name>
661
+ tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
662
+ name_type operations_prefix
663
+ </elasticsearch_index_name>
664
+ <elasticsearch_index_name>
665
+ tag "**"
666
+ name_type project_prefix
667
+ </elasticsearch_index_name>
668
+ ')
669
+ assert_equal('.operations', rec['viaq_index_prefix'])
670
+ end
671
+ test 'construct an operations index prefix with named field' do
672
+ rec = emit_with_tag('journal.system', normal_input, '
673
+ <formatter>
674
+ tag "journal.system**"
675
+ type sys_journal
676
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
677
+ </formatter>
678
+ <formatter>
679
+ tag "kubernetes.journal.container**"
680
+ type k8s_journal
681
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
682
+ </formatter>
683
+ <elasticsearch_index_name>
684
+ tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
685
+ name_type operations_prefix
686
+ </elasticsearch_index_name>
687
+ <elasticsearch_index_name>
688
+ tag "**"
689
+ name_type project_prefix
690
+ </elasticsearch_index_name>
691
+ elasticsearch_index_prefix_field my_index_prefix
692
+ ')
693
+ assert_equal('.operations', rec['my_index_prefix'])
694
+ end
695
+ test 'construct an operations index name with named field' do
696
+ rec = emit_with_tag('journal.system', normal_input, '
697
+ <formatter>
698
+ tag "journal.system**"
699
+ type sys_journal
700
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
701
+ </formatter>
702
+ <formatter>
703
+ tag "kubernetes.journal.container**"
704
+ type k8s_journal
705
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
706
+ </formatter>
707
+ <elasticsearch_index_name>
708
+ tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
709
+ name_type operations_full
710
+ </elasticsearch_index_name>
711
+ <elasticsearch_index_name>
712
+ tag "**"
713
+ name_type project_full
714
+ </elasticsearch_index_name>
715
+ elasticsearch_index_name_field my_index_name
716
+ ')
717
+ assert_equal('.operations.2017.07.27', rec['my_index_name'])
718
+ end
719
+ test 'log error if missing kubernetes field' do
720
+ rec = emit_with_tag('kubernetes.journal.container.something', normal_input, '
721
+ <formatter>
722
+ tag "journal.system**"
723
+ type sys_journal
724
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
725
+ </formatter>
726
+ <formatter>
727
+ tag "kubernetes.journal.container**"
728
+ type k8s_journal
729
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
730
+ </formatter>
731
+ <elasticsearch_index_name>
732
+ tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
733
+ name_type operations_prefix
734
+ </elasticsearch_index_name>
735
+ <elasticsearch_index_name>
736
+ tag "**"
737
+ name_type project_prefix
738
+ </elasticsearch_index_name>
739
+ ')
740
+ assert_match /record is missing kubernetes field/, @dlog.logs[0]
741
+ end
742
+ test 'log error if missing kubernetes.namespace_name field' do
743
+ input = normal_input.merge({})
744
+ input['kubernetes'] = 'junk'
745
+ rec = emit_with_tag('kubernetes.journal.container.something', input, '
746
+ <formatter>
747
+ tag "journal.system**"
748
+ type sys_journal
749
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
750
+ </formatter>
751
+ <formatter>
752
+ tag "kubernetes.journal.container**"
753
+ type k8s_journal
754
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
755
+ </formatter>
756
+ <elasticsearch_index_name>
757
+ tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
758
+ name_type operations_prefix
759
+ </elasticsearch_index_name>
760
+ <elasticsearch_index_name>
761
+ tag "**"
762
+ name_type project_prefix
763
+ </elasticsearch_index_name>
764
+ ')
765
+ assert_match /record is missing kubernetes.namespace_name field/, @dlog.logs[0]
766
+ end
767
+ test 'log error if missing kubernetes.namespace_id field' do
768
+ input = normal_input.merge({})
769
+ input['kubernetes'] = {'namespace_name'=>'junk'}
770
+ rec = emit_with_tag('kubernetes.journal.container.something', input, '
771
+ <formatter>
772
+ tag "journal.system**"
773
+ type sys_journal
774
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
775
+ </formatter>
776
+ <formatter>
777
+ tag "kubernetes.journal.container**"
778
+ type k8s_journal
779
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
780
+ </formatter>
781
+ <elasticsearch_index_name>
782
+ tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
783
+ name_type operations_prefix
784
+ </elasticsearch_index_name>
785
+ <elasticsearch_index_name>
786
+ tag "**"
787
+ name_type project_prefix
788
+ </elasticsearch_index_name>
789
+ ')
790
+ assert_match /record is missing kubernetes.namespace_id field/, @dlog.logs[0]
791
+ end
792
+ test 'construct a kubernetes index prefix' do
793
+ input = normal_input.merge({})
794
+ input['kubernetes'] = {'namespace_name'=>'name', 'namespace_id'=>'uuid'}
795
+ rec = emit_with_tag('kubernetes.journal.container.something', input, '
796
+ <formatter>
797
+ tag "journal.system**"
798
+ type sys_journal
799
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
800
+ </formatter>
801
+ <formatter>
802
+ tag "kubernetes.journal.container**"
803
+ type k8s_journal
804
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
805
+ </formatter>
806
+ <elasticsearch_index_name>
807
+ tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
808
+ name_type operations_prefix
809
+ </elasticsearch_index_name>
810
+ <elasticsearch_index_name>
811
+ tag "**"
812
+ name_type project_prefix
813
+ </elasticsearch_index_name>
814
+ ')
815
+ assert_equal('project.name.uuid', rec['viaq_index_prefix'])
816
+ end
817
+ test 'construct a kubernetes index prefix with named field' do
818
+ input = normal_input.merge({})
819
+ input['kubernetes'] = {'namespace_name'=>'name', 'namespace_id'=>'uuid'}
820
+ rec = emit_with_tag('kubernetes.journal.container.something', input, '
821
+ <formatter>
822
+ tag "journal.system**"
823
+ type sys_journal
824
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
825
+ </formatter>
826
+ <formatter>
827
+ tag "kubernetes.journal.container**"
828
+ type k8s_journal
829
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
830
+ </formatter>
831
+ <elasticsearch_index_name>
832
+ tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
833
+ name_type operations_prefix
834
+ </elasticsearch_index_name>
835
+ <elasticsearch_index_name>
836
+ tag "**"
837
+ name_type project_prefix
838
+ </elasticsearch_index_name>
839
+ elasticsearch_index_prefix_field my_index_prefix
840
+ ')
841
+ assert_equal('project.name.uuid', rec['my_index_prefix'])
842
+ end
843
+ test 'construct a kubernetes index name with named field' do
844
+ input = normal_input.merge({})
845
+ input['kubernetes'] = {'namespace_name'=>'name', 'namespace_id'=>'uuid'}
846
+ rec = emit_with_tag('kubernetes.journal.container.something', input, '
847
+ <formatter>
848
+ tag "journal.system**"
849
+ type sys_journal
850
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
851
+ </formatter>
852
+ <formatter>
853
+ tag "kubernetes.journal.container**"
854
+ type k8s_journal
855
+ remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
856
+ </formatter>
857
+ <elasticsearch_index_name>
858
+ tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
859
+ name_type operations_full
860
+ </elasticsearch_index_name>
861
+ <elasticsearch_index_name>
862
+ tag "**"
863
+ name_type project_full
864
+ </elasticsearch_index_name>
865
+ elasticsearch_index_name_field my_index_name
866
+ ')
867
+ assert_equal('project.name.uuid.2017.07.27', rec['my_index_name'])
868
+ end
216
869
  end
217
870
  end