fluent-plugin-viaq_data_model 0.0.6 → 0.0.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +54 -1
- data/fluent-plugin-viaq_data_model.gemspec +1 -1
- data/lib/fluent/plugin/filter_viaq_data_model.rb +26 -12
- data/test/test_filter_viaq_data_model.rb +120 -8
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: '05957e3bf287a122835a57a9c8664d689a618d64'
|
4
|
+
data.tar.gz: 26d5f3dca6e874e40553f1965d096f0669954e41
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c25c5635ac44f09da3b6eacd863b7753a9fa3f1ed460c676d85eb293d80766177f0d7e32edec23598bc54fc980188cca51e0d2a63ff7bbade9c4e92fac06e917
|
7
|
+
data.tar.gz: 8e17b6ed4ea137598c1831f1de606fcfe3f679550204547bf5194f70c2e0f04f438f255e826298addcd9e16aa24de22cb2fb474b2b74c67afe644c68ca154c8f
|
data/README.md
CHANGED
@@ -96,6 +96,9 @@ See `filter-viaq_data_model.conf` for an example filter configuration.
|
|
96
96
|
* This is the name of the top level field to hold the time value. The value
|
97
97
|
is taken from the value of the `src_time_name` field.
|
98
98
|
* `formatter` - a formatter for a well known common data model source
|
99
|
+
* `enabled` - default `true` - is this formatter enabled? **NOTE** if the
|
100
|
+
formatter is disabled, it will still match, it just won't do anything, and
|
101
|
+
it will skip the other formatters.
|
99
102
|
* `type` - one of the well known sources
|
100
103
|
* `sys_journal` - a record read from the systemd journal
|
101
104
|
* `k8s_journal` - a Kubernetes container record read from the systemd
|
@@ -109,6 +112,9 @@ See `filter-viaq_data_model.conf` for an example filter configuration.
|
|
109
112
|
`normalizer` - the default is `collector`
|
110
113
|
* `elasticsearch_index_name` - how to construct Elasticsearch index names or
|
111
114
|
prefixes for given tags
|
115
|
+
* `enabled` - default `true` - is this item enabled? **NOTE** if the
|
116
|
+
item is disabled, it will still match, it just won't do anything, and
|
117
|
+
it will skip the other index name items.
|
112
118
|
* `tag` - the Fluentd tag pattern to match for these records
|
113
119
|
* `name_type` - the well known type of index name or prefix to create -
|
114
120
|
`operations_full, project_full, operations_prefix, project_prefix` - The
|
@@ -231,7 +237,6 @@ Given a configuration like this:
|
|
231
237
|
tag "**"
|
232
238
|
name_type project_full
|
233
239
|
</elasticsearch_index_name>
|
234
|
-
elasticsearch_index_field viaq_index_name
|
235
240
|
|
236
241
|
A record with tag `journal.system` like this:
|
237
242
|
|
@@ -261,6 +266,54 @@ will end up looking like this:
|
|
261
266
|
"viaq_index_name":"project.myproject.000000.2017.07.07"
|
262
267
|
}
|
263
268
|
|
269
|
+
|
270
|
+
### Note about using enabled false
|
271
|
+
|
272
|
+
Given a configuration like this:
|
273
|
+
|
274
|
+
<elasticsearch_index_name>
|
275
|
+
enabled false
|
276
|
+
tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
|
277
|
+
name_type operations_full
|
278
|
+
</elasticsearch_index_name>
|
279
|
+
<elasticsearch_index_name>
|
280
|
+
tag "**"
|
281
|
+
name_type project_full
|
282
|
+
</elasticsearch_index_name>
|
283
|
+
|
284
|
+
A record with tag `journal.system` like this:
|
285
|
+
|
286
|
+
{
|
287
|
+
"@timestamp":"2017-07-27T17:27:46.216527+00:00"
|
288
|
+
}
|
289
|
+
|
290
|
+
will end up looking like this:
|
291
|
+
|
292
|
+
{
|
293
|
+
"@timestamp":"2017-07-27T17:27:46.216527+00:00",
|
294
|
+
}
|
295
|
+
|
296
|
+
That is, the tag will match the first `elasticsearch_index_name`, but since it
|
297
|
+
is disabled, no index name will be created, and it will _not_ fall through to
|
298
|
+
the `**` match below. Using `enabled false` in this case allows you to not
|
299
|
+
generate index names for operations indices, but still continue to generate
|
300
|
+
index names for project indices.
|
301
|
+
|
302
|
+
A record with tag `kubernetes.journal.container` like this:
|
303
|
+
|
304
|
+
{
|
305
|
+
"@timestamp":"2017-07-27T17:27:46.216527+00:00",
|
306
|
+
"kubernetes":{"namespace_name":"myproject","namespace_id":"000000"}
|
307
|
+
}
|
308
|
+
|
309
|
+
will end up looking like this:
|
310
|
+
|
311
|
+
{
|
312
|
+
"@timestamp":"2017-07-27T17:27:46.216527+00:00",
|
313
|
+
"kubernetes":{"namespace_name":"myproject","namespace_id":"000000"}
|
314
|
+
"viaq_index_name":"project.myproject.000000.2017.07.07"
|
315
|
+
}
|
316
|
+
|
264
317
|
## Installation
|
265
318
|
|
266
319
|
gem install fluent-plugin-viaq_data_model
|
@@ -7,7 +7,7 @@ FLUENTD_VERSION = ENV['FLUENTD_VERSION'] || "0.12.0"
|
|
7
7
|
|
8
8
|
Gem::Specification.new do |gem|
|
9
9
|
gem.name = "fluent-plugin-viaq_data_model"
|
10
|
-
gem.version = "0.0.
|
10
|
+
gem.version = "0.0.7"
|
11
11
|
gem.authors = ["Rich Megginson"]
|
12
12
|
gem.email = ["rmeggins@redhat.com"]
|
13
13
|
gem.description = %q{Filter plugin to ensure data is in the ViaQ common data model}
|
@@ -99,6 +99,8 @@ module Fluent
|
|
99
99
|
# come before more general matches
|
100
100
|
desc 'Formatters for common data model, for well known record types'
|
101
101
|
config_section :formatter, param_name: :formatters do
|
102
|
+
desc 'is this formatter enabled?'
|
103
|
+
config_param :enabled, :bool, default: true
|
102
104
|
desc 'one of the well known formatter types'
|
103
105
|
config_param :type, :enum, list: [:sys_journal, :k8s_journal, :sys_var_log, :k8s_json_file]
|
104
106
|
desc 'process records with this tag pattern'
|
@@ -127,7 +129,11 @@ module Fluent
|
|
127
129
|
# come before more general matches e.g. make sure tag "**" is last
|
128
130
|
desc 'Construct Elasticsearch index names or prefixes based on the matching tags pattern and type'
|
129
131
|
config_section :elasticsearch_index_name, param_name: :elasticsearch_index_names do
|
132
|
+
desc 'is this index name enabled?'
|
133
|
+
config_param :enabled, :bool, default: true
|
134
|
+
desc 'create index names for records with this tag pattern'
|
130
135
|
config_param :tag, :string
|
136
|
+
desc 'type of index name to create'
|
131
137
|
config_param :name_type, :enum, list: [:operations_full, :project_full, :operations_prefix, :project_prefix]
|
132
138
|
end
|
133
139
|
desc 'Store the Elasticsearch index name in this field'
|
@@ -219,12 +225,14 @@ module Fluent
|
|
219
225
|
|
220
226
|
def process_sys_var_log_fields(tag, time, record, fmtr_type = nil)
|
221
227
|
record['systemd'] = {"t" => {"PID" => record['pid']}, "u" => {"SYSLOG_IDENTIFIER" => record['ident']}}
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
+
unless record[@dest_time_name] # e.g. already has @timestamp
|
229
|
+
rectime = record['time'] || time
|
230
|
+
# handle the case where the time reported in /var/log/messages is for a previous year
|
231
|
+
if Time.at(rectime) > Time.now
|
232
|
+
record['time'] = Time.new((rectime.year - 1), rectime.month, rectime.day, rectime.hour, rectime.min, rectime.sec, rectime.utc_offset).utc.to_datetime.rfc3339(6)
|
233
|
+
else
|
234
|
+
record['time'] = rectime.utc.to_datetime.rfc3339(6)
|
235
|
+
end
|
228
236
|
end
|
229
237
|
if record['host'].eql?('localhost') && @docker_hostname
|
230
238
|
record['hostname'] = @docker_hostname
|
@@ -241,7 +249,9 @@ module Fluent
|
|
241
249
|
elsif @docker_hostname
|
242
250
|
record['hostname'] = @docker_hostname
|
243
251
|
end
|
244
|
-
record[
|
252
|
+
unless record[@dest_time_name] # e.g. already has @timestamp
|
253
|
+
record['time'] = record['time'].utc.to_datetime.rfc3339(6)
|
254
|
+
end
|
245
255
|
end
|
246
256
|
|
247
257
|
def check_for_match_and_format(tag, time, record)
|
@@ -250,8 +260,7 @@ module Fluent
|
|
250
260
|
fmtr = @formatter_cache[tag]
|
251
261
|
unless fmtr
|
252
262
|
idx = @formatters.index{|fmtr| fmtr.matcher.match(tag)}
|
253
|
-
if idx
|
254
|
-
fmtr = @formatters[idx]
|
263
|
+
if idx && (fmtr = @formatters[idx]).enabled
|
255
264
|
@formatter_cache[tag] = fmtr
|
256
265
|
else
|
257
266
|
@formatter_cache_nomatch[tag] = true
|
@@ -260,7 +269,7 @@ module Fluent
|
|
260
269
|
end
|
261
270
|
fmtr.fmtr_func.call(tag, time, record, fmtr.fmtr_type)
|
262
271
|
|
263
|
-
if record['time'].nil?
|
272
|
+
if record[@dest_time_name].nil? && record['time'].nil?
|
264
273
|
record['time'] = Time.at(time).utc.to_datetime.rfc3339(6)
|
265
274
|
end
|
266
275
|
|
@@ -275,7 +284,7 @@ module Fluent
|
|
275
284
|
"ipaddr6" => @ipaddr6,
|
276
285
|
"inputname" => "fluent-plugin-systemd",
|
277
286
|
"name" => "fluentd",
|
278
|
-
"received_at" => Time.
|
287
|
+
"received_at" => Time.now.utc.to_datetime.rfc3339(6),
|
279
288
|
"version" => @pipeline_version
|
280
289
|
}
|
281
290
|
end
|
@@ -285,6 +294,7 @@ module Fluent
|
|
285
294
|
@elasticsearch_index_names.each do |ein|
|
286
295
|
if ein.matcher.match(tag)
|
287
296
|
found = true
|
297
|
+
return unless ein.enabled
|
288
298
|
if ein.name_type == :operations_full || ein.name_type == :project_full
|
289
299
|
field_name = @elasticsearch_index_name_field
|
290
300
|
need_time = true
|
@@ -333,7 +343,11 @@ module Fluent
|
|
333
343
|
end
|
334
344
|
end
|
335
345
|
unless found
|
336
|
-
|
346
|
+
if ENV['CDM_DEBUG']
|
347
|
+
unless tag == ENV['CDM_DEBUG_IGNORE_TAG']
|
348
|
+
log.error("no match for tag #{tag}")
|
349
|
+
end
|
350
|
+
end
|
337
351
|
end
|
338
352
|
end
|
339
353
|
|
@@ -18,6 +18,7 @@
|
|
18
18
|
#
|
19
19
|
#require_relative '../helper'
|
20
20
|
require 'fluent/test'
|
21
|
+
require 'flexmock/test_unit'
|
21
22
|
|
22
23
|
require 'fluent/plugin/filter_viaq_data_model'
|
23
24
|
|
@@ -28,6 +29,9 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
28
29
|
Fluent::Test.setup
|
29
30
|
@time = Fluent::Engine.now
|
30
31
|
log = Fluent::Engine.log
|
32
|
+
@timestamp = Time.now
|
33
|
+
@timestamp_str = @timestamp.utc.to_datetime.rfc3339(6)
|
34
|
+
flexmock(Time).should_receive(:now).and_return(@timestamp)
|
31
35
|
end
|
32
36
|
|
33
37
|
def create_driver(conf = '')
|
@@ -368,10 +372,28 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
368
372
|
assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
|
369
373
|
assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
|
370
374
|
assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
|
371
|
-
assert_equal(
|
375
|
+
assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
|
372
376
|
dellist = 'log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID'.split(',')
|
373
377
|
dellist.each{|field| assert_nil(rec[field])}
|
374
378
|
end
|
379
|
+
test 'disable journal record processing' do
|
380
|
+
ENV['IPADDR4'] = '127.0.0.1'
|
381
|
+
ENV['IPADDR6'] = '::1'
|
382
|
+
ENV['FLUENTD_VERSION'] = 'fversion'
|
383
|
+
ENV['DATA_VERSION'] = 'dversion'
|
384
|
+
rec = emit_with_tag('journal.system', normal_input, '
|
385
|
+
<formatter>
|
386
|
+
enabled false
|
387
|
+
tag "journal.system**"
|
388
|
+
type sys_journal
|
389
|
+
remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
|
390
|
+
</formatter>
|
391
|
+
pipeline_type normalizer
|
392
|
+
')
|
393
|
+
assert_nil(rec['systemd'])
|
394
|
+
notdellist = 'log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID'.split(',')
|
395
|
+
notdellist.each{|field| assert_equal(normal_input[field], rec[field])}
|
396
|
+
end
|
375
397
|
test 'process a journal record, override remove_keys' do
|
376
398
|
ENV['IPADDR4'] = '127.0.0.1'
|
377
399
|
ENV['IPADDR6'] = '::1'
|
@@ -397,7 +419,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
397
419
|
assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
|
398
420
|
assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
|
399
421
|
assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
|
400
|
-
assert_equal(
|
422
|
+
assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
|
401
423
|
keeplist = 'log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID'.split(',')
|
402
424
|
keeplist.each{|field| normal_input[field] && assert_not_nil(rec[field])}
|
403
425
|
dellist = 'CONTAINER_NAME,PRIORITY'.split(',')
|
@@ -490,10 +512,28 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
490
512
|
assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
|
491
513
|
assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
|
492
514
|
assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
|
493
|
-
assert_equal(
|
515
|
+
assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
|
494
516
|
dellist = 'log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID'.split(',')
|
495
517
|
dellist.each{|field| assert_nil(rec[field])}
|
496
518
|
end
|
519
|
+
test 'disable kubernetes journal record processing' do
|
520
|
+
ENV['IPADDR4'] = '127.0.0.1'
|
521
|
+
ENV['IPADDR6'] = '::1'
|
522
|
+
ENV['FLUENTD_VERSION'] = 'fversion'
|
523
|
+
ENV['DATA_VERSION'] = 'dversion'
|
524
|
+
rec = emit_with_tag('kubernetes.journal.container', normal_input, '
|
525
|
+
<formatter>
|
526
|
+
enabled false
|
527
|
+
tag "kubernetes.journal.container**"
|
528
|
+
type k8s_journal
|
529
|
+
remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
|
530
|
+
</formatter>
|
531
|
+
pipeline_type normalizer
|
532
|
+
')
|
533
|
+
assert_nil(rec['systemd'])
|
534
|
+
notdellist = 'log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID'.split(',')
|
535
|
+
notdellist.each{|field| assert_equal(normal_input[field], rec[field])}
|
536
|
+
end
|
497
537
|
test 'process a kubernetes journal record, given kubernetes.host' do
|
498
538
|
input = normal_input.merge({})
|
499
539
|
input['kubernetes'] = {'host' => 'k8shost'}
|
@@ -521,7 +561,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
521
561
|
assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
|
522
562
|
assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
|
523
563
|
assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
|
524
|
-
assert_equal(
|
564
|
+
assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
|
525
565
|
dellist = 'log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID'.split(',')
|
526
566
|
dellist.each{|field| assert_nil(rec[field])}
|
527
567
|
end
|
@@ -552,7 +592,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
552
592
|
assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
|
553
593
|
assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
|
554
594
|
assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
|
555
|
-
assert_equal(
|
595
|
+
assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
|
556
596
|
dellist = 'log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID'.split(',')
|
557
597
|
dellist.each{|field| assert_nil(rec[field])}
|
558
598
|
end
|
@@ -581,7 +621,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
581
621
|
assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
|
582
622
|
assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
|
583
623
|
assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
|
584
|
-
assert_equal(
|
624
|
+
assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
|
585
625
|
dellist = 'host,pid,ident'.split(',')
|
586
626
|
dellist.each{|field| assert_nil(rec[field])}
|
587
627
|
end
|
@@ -612,7 +652,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
612
652
|
assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
|
613
653
|
assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
|
614
654
|
assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
|
615
|
-
assert_equal(
|
655
|
+
assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
|
616
656
|
dellist = 'host,pid,ident'.split(',')
|
617
657
|
dellist.each{|field| assert_nil(rec[field])}
|
618
658
|
end
|
@@ -640,7 +680,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
640
680
|
assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['normalizer']['inputname'])
|
641
681
|
assert_equal('fluentd', rec['pipeline_metadata']['normalizer']['name'])
|
642
682
|
assert_equal('fversion dversion', rec['pipeline_metadata']['normalizer']['version'])
|
643
|
-
assert_equal(
|
683
|
+
assert_equal(@timestamp_str, rec['pipeline_metadata']['normalizer']['received_at'])
|
644
684
|
dellist = 'host,pid,ident'.split(',')
|
645
685
|
dellist.each{|field| assert_nil(rec[field])}
|
646
686
|
end
|
@@ -716,6 +756,30 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
716
756
|
')
|
717
757
|
assert_equal('.operations.2017.07.27', rec['my_index_name'])
|
718
758
|
end
|
759
|
+
test 'disable operations index name' do
|
760
|
+
rec = emit_with_tag('journal.system', normal_input, '
|
761
|
+
<formatter>
|
762
|
+
tag "journal.system**"
|
763
|
+
type sys_journal
|
764
|
+
remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
|
765
|
+
</formatter>
|
766
|
+
<formatter>
|
767
|
+
tag "kubernetes.journal.container**"
|
768
|
+
type k8s_journal
|
769
|
+
remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
|
770
|
+
</formatter>
|
771
|
+
<elasticsearch_index_name>
|
772
|
+
enabled false
|
773
|
+
tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
|
774
|
+
name_type operations_full
|
775
|
+
</elasticsearch_index_name>
|
776
|
+
<elasticsearch_index_name>
|
777
|
+
tag "**"
|
778
|
+
name_type project_full
|
779
|
+
</elasticsearch_index_name>
|
780
|
+
')
|
781
|
+
assert_nil(rec['viaq_index_name'])
|
782
|
+
end
|
719
783
|
test 'log error if missing kubernetes field' do
|
720
784
|
rec = emit_with_tag('kubernetes.journal.container.something', normal_input, '
|
721
785
|
<formatter>
|
@@ -866,5 +930,53 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
866
930
|
')
|
867
931
|
assert_equal('project.name.uuid.2017.07.27', rec['my_index_name'])
|
868
932
|
end
|
933
|
+
test 'disable kubernetes index names but allow operations index names' do
|
934
|
+
input = normal_input.merge({})
|
935
|
+
input['kubernetes'] = {'namespace_name'=>'name', 'namespace_id'=>'uuid'}
|
936
|
+
rec = emit_with_tag('kubernetes.journal.container.something', input, '
|
937
|
+
<formatter>
|
938
|
+
tag "journal.system**"
|
939
|
+
type sys_journal
|
940
|
+
remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
|
941
|
+
</formatter>
|
942
|
+
<formatter>
|
943
|
+
tag "kubernetes.journal.container**"
|
944
|
+
type k8s_journal
|
945
|
+
remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
|
946
|
+
</formatter>
|
947
|
+
<elasticsearch_index_name>
|
948
|
+
tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
|
949
|
+
name_type operations_full
|
950
|
+
</elasticsearch_index_name>
|
951
|
+
<elasticsearch_index_name>
|
952
|
+
enabled false
|
953
|
+
tag "**"
|
954
|
+
name_type project_full
|
955
|
+
</elasticsearch_index_name>
|
956
|
+
')
|
957
|
+
assert_nil(rec['viaq_index_name'])
|
958
|
+
rec = emit_with_tag('journal.system.something', normal_input, '
|
959
|
+
<formatter>
|
960
|
+
tag "journal.system**"
|
961
|
+
type sys_journal
|
962
|
+
remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
|
963
|
+
</formatter>
|
964
|
+
<formatter>
|
965
|
+
tag "kubernetes.journal.container**"
|
966
|
+
type k8s_journal
|
967
|
+
remove_keys log,stream,MESSAGE,_SOURCE_REALTIME_TIMESTAMP,__REALTIME_TIMESTAMP,CONTAINER_ID,CONTAINER_ID_FULL,CONTAINER_NAME,PRIORITY,_BOOT_ID,_CAP_EFFECTIVE,_CMDLINE,_COMM,_EXE,_GID,_HOSTNAME,_MACHINE_ID,_PID,_SELINUX_CONTEXT,_SYSTEMD_CGROUP,_SYSTEMD_SLICE,_SYSTEMD_UNIT,_TRANSPORT,_UID,_AUDIT_LOGINUID,_AUDIT_SESSION,_SYSTEMD_OWNER_UID,_SYSTEMD_SESSION,_SYSTEMD_USER_UNIT,CODE_FILE,CODE_FUNCTION,CODE_LINE,ERRNO,MESSAGE_ID,RESULT,UNIT,_KERNEL_DEVICE,_KERNEL_SUBSYSTEM,_UDEV_SYSNAME,_UDEV_DEVNODE,_UDEV_DEVLINK,SYSLOG_FACILITY,SYSLOG_IDENTIFIER,SYSLOG_PID
|
968
|
+
</formatter>
|
969
|
+
<elasticsearch_index_name>
|
970
|
+
tag "journal.system** system.var.log** **_default_** **_openshift_** **_openshift-infra_** mux.ops"
|
971
|
+
name_type operations_full
|
972
|
+
</elasticsearch_index_name>
|
973
|
+
<elasticsearch_index_name>
|
974
|
+
enabled false
|
975
|
+
tag "**"
|
976
|
+
name_type project_full
|
977
|
+
</elasticsearch_index_name>
|
978
|
+
')
|
979
|
+
assert_equal('.operations.2017.07.27', rec['viaq_index_name'])
|
980
|
+
end
|
869
981
|
end
|
870
982
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-viaq_data_model
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Rich Megginson
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-09-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|