fluent-plugin-viaq_data_model 0.0.18 → 0.0.19

Sign up to get free protection for your applications and to get access to all the features.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2 -2
  3. data/fluent-plugin-viaq_data_model.gemspec +1 -1
  4. data/lib/fluent/plugin/filter_viaq_data_model.rb +2 -6
  5. data/lib/fluent/plugin/filter_viaq_data_model_systemd.rb +23 -20
  6. data/test/test_filter_viaq_data_model.rb +32 -26
  7. metadata +3 -4
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 1d809d8421925d6bf93ccaf62c93b6c2bbe03aaec5dd70b5e8ad243bcdd4d71c
4
- data.tar.gz: c1b0b023782d77bc6ab3095b3e151a5b8dc29416d8581615c99951e935fa834d
3
+ metadata.gz: b20471334c7756a6393385dfe17987751ea233c8cac638f592459ea5a3a8692f
4
+ data.tar.gz: 0d1738ff46f83bba941f415616c374ece1268a89c59f7df333c0311106809e67
5
5
  SHA512:
6
- metadata.gz: 0000a8e8fc5e586ff4d589db024402ea640cb36d8ceacac6b8df3f3696702ad491939ce0e7b627e11fe5a5abe0ae86de146baacee90d2cc4f1e1c43c122b8c3b
7
- data.tar.gz: 2687cd962c4ec754bf133d0b939174275c1c4c794ef75a31467971be3e9bc16119aa9d6ea5028c36f5666714d9c3a03d2d277c5ec9ce40becdbf72fa36698191
6
+ metadata.gz: c77bfd1197a5d06b994668034dcd4d662ddb955fd4c3a803fad71a47124f86e28f29f0a063eecfe86d6bf23c9791159ca693405be9e0c4744288345ee3f0d3e0
7
+ data.tar.gz: 902a1adc2eca199a51b7cb6ce37600c2ec9764eeacbda1e2313410bcde774f1184513d4b87af91b7efd15651c35e2f7b97caf872e6e5f124a9d56232e2e5f433
data/README.md CHANGED
@@ -162,10 +162,10 @@ See `filter-viaq_data_model.conf` for an example filter configuration.
162
162
  create the full index name.
163
163
  * `elasticsearch_index_name_field` - name of the field in the record which stores
164
164
  the index name - you should remove this field in the elasticsearch output
165
- plugin using the `remove_keys` config parameter - default is `viaq_idnex_name`
165
+ plugin using the `remove_keys` config parameter - default is `viaq_index_name`
166
166
  * `elasticsearch_index_prefix_field` - name of the field in the record which stores
167
167
  the index prefix - you should remove this field in the elasticsearch output
168
- plugin using the `remove_keys` config parameter - default is `viaq_idnex_prefix`
168
+ plugin using the `remove_keys` config parameter - default is `viaq_index_prefix`
169
169
 
170
170
  **NOTE** The `formatter` blocks are matched in the given order in the file.
171
171
  This means, don't use `tag "**"` as the first formatter or none of your
data/fluent-plugin-viaq_data_model.gemspec CHANGED
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |gem|
6
6
  gem.name = "fluent-plugin-viaq_data_model"
7
- gem.version = "0.0.18"
7
+ gem.version = "0.0.19"
8
8
  gem.authors = ["Rich Megginson"]
9
9
  gem.email = ["rmeggins@redhat.com"]
10
10
  gem.description = %q{Filter plugin to ensure data is in the ViaQ common data model}
data/lib/fluent/plugin/filter_viaq_data_model.rb CHANGED
@@ -280,7 +280,7 @@ module Fluent
280
280
  '8' => 'trace',
281
281
  '9' => 'unknown',
282
282
  }
283
- def normalize_level(level, newlevel, stream=nil, priority=nil)
283
+ def normalize_level(level, newlevel, priority=nil)
284
284
  # if the record already has a level field, and it looks like one of our well
285
285
  # known values, convert it to the canonical normalized form - otherwise,
286
286
  # preserve the value in string format
@@ -290,10 +290,6 @@ module Fluent
290
290
  (level.respond_to?(:downcase) && (retlevel = NORMAL_LEVELS[level.downcase]))
291
291
  retlevel = level.to_s # don't know what it is - just convert to string
292
292
  end
293
- elsif stream == 'stdout'
294
- retlevel = 'info'
295
- elsif stream == 'stderr'
296
- retlevel = 'err'
297
293
  elsif !priority.nil?
298
294
  retlevel = PRIORITY_LEVELS[priority]
299
295
  else
@@ -321,7 +317,7 @@ module Fluent
321
317
 
322
318
  def process_k8s_json_file_fields(tag, time, record, fmtr_type = nil)
323
319
  record['message'] = record['message'] || record['log']
324
- record['level'] = normalize_level(record['level'], nil, record['stream'])
320
+ record['level'] = normalize_level(record['level'], nil)
325
321
  if record.key?('kubernetes') && record['kubernetes'].respond_to?(:fetch) && \
326
322
  (k8shost = record['kubernetes'].fetch('host', nil))
327
323
  record['hostname'] = k8shost
data/lib/fluent/plugin/filter_viaq_data_model_systemd.rb CHANGED
@@ -21,25 +21,28 @@ require 'date'
21
21
  module ViaqDataModelFilterSystemd
22
22
  # map of journal fields to viaq data model field
23
23
  JOURNAL_FIELD_MAP_SYSTEMD_T = {
24
- "_AUDIT_LOGINUID" => "AUDIT_LOGINUID",
25
- "_AUDIT_SESSION" => "AUDIT_SESSION",
26
- "_BOOT_ID" => "BOOT_ID",
27
- "_CAP_EFFECTIVE" => "CAP_EFFECTIVE",
28
- "_CMDLINE" => "CMDLINE",
29
- "_COMM" => "COMM",
30
- "_EXE" => "EXE",
31
- "_GID" => "GID",
32
- "_MACHINE_ID" => "MACHINE_ID",
33
- "_PID" => "PID",
34
- "_SELINUX_CONTEXT" => "SELINUX_CONTEXT",
35
- "_SYSTEMD_CGROUP" => "SYSTEMD_CGROUP",
36
- "_SYSTEMD_OWNER_UID" => "SYSTEMD_OWNER_UID",
37
- "_SYSTEMD_SESSION" => "SYSTEMD_SESSION",
38
- "_SYSTEMD_SLICE" => "SYSTEMD_SLICE",
39
- "_SYSTEMD_UNIT" => "SYSTEMD_UNIT",
40
- "_SYSTEMD_USER_UNIT" => "SYSTEMD_USER_UNIT",
41
- "_TRANSPORT" => "TRANSPORT",
42
- "_UID" => "UID"
24
+ "_AUDIT_LOGINUID" => "AUDIT_LOGINUID",
25
+ "_AUDIT_SESSION" => "AUDIT_SESSION",
26
+ "_BOOT_ID" => "BOOT_ID",
27
+ "_CAP_EFFECTIVE" => "CAP_EFFECTIVE",
28
+ "_CMDLINE" => "CMDLINE",
29
+ "_COMM" => "COMM",
30
+ "_EXE" => "EXE",
31
+ "_GID" => "GID",
32
+ "_LINE_BREAK" => "LINE_BREAK",
33
+ "_MACHINE_ID" => "MACHINE_ID",
34
+ "_PID" => "PID",
35
+ "_SELINUX_CONTEXT" => "SELINUX_CONTEXT",
36
+ "_STREAM_ID" => "STREAM_ID",
37
+ "_SYSTEMD_CGROUP" => "SYSTEMD_CGROUP",
38
+ "_SYSTEMD_INVOCATION_ID" => "SYSTEMD_INVOCATION_ID",
39
+ "_SYSTEMD_OWNER_UID" => "SYSTEMD_OWNER_UID",
40
+ "_SYSTEMD_SESSION" => "SYSTEMD_SESSION",
41
+ "_SYSTEMD_SLICE" => "SYSTEMD_SLICE",
42
+ "_SYSTEMD_UNIT" => "SYSTEMD_UNIT",
43
+ "_SYSTEMD_USER_UNIT" => "SYSTEMD_USER_UNIT",
44
+ "_TRANSPORT" => "TRANSPORT",
45
+ "_UID" => "UID"
43
46
  }
44
47
 
45
48
  JOURNAL_FIELD_MAP_SYSTEMD_U = {
@@ -93,7 +96,7 @@ module ViaqDataModelFilterSystemd
93
96
  unless systemd_k.empty?
94
97
  (record['systemd'] ||= {})['k'] = systemd_k
95
98
  end
96
- record['level'] = normalize_level(record['level'], nil, nil, record['PRIORITY'])
99
+ record['level'] = normalize_level(record['level'], nil, record['PRIORITY'])
97
100
  JOURNAL_TIME_FIELDS.each do |field|
98
101
  if (val = record[field])
99
102
  vali = val.to_i
data/test/test_filter_viaq_data_model.rb CHANGED
@@ -239,9 +239,12 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
239
239
  "_EXE" => "EXE",
240
240
  "_GID" => "GID",
241
241
  "_MACHINE_ID" => "MACHINE_ID",
242
+ "_LINE_BREAK" => "LINE_BREAK",
242
243
  "_PID" => "PID",
243
244
  "_SELINUX_CONTEXT" => "SELINUX_CONTEXT",
245
+ "_STREAM_ID" => "STREAM_ID",
244
246
  "_SYSTEMD_CGROUP" => "SYSTEMD_CGROUP",
247
+ "_SYSTEMD_INVOCATION_ID" => "SYSTEMD_INVOCATION_ID",
245
248
  "_SYSTEMD_OWNER_UID" => "SYSTEMD_OWNER_UID",
246
249
  "_SYSTEMD_SESSION" => "SYSTEMD_SESSION",
247
250
  "_SYSTEMD_SLICE" => "SYSTEMD_SLICE",
@@ -273,25 +276,28 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
273
276
  end
274
277
  def normal_output_t
275
278
  {
276
- "AUDIT_LOGINUID" =>"AUDIT_LOGINUID",
277
- "AUDIT_SESSION" =>"AUDIT_SESSION",
278
- "BOOT_ID" =>"BOOT_ID",
279
- "CAP_EFFECTIVE" =>"CAP_EFFECTIVE",
280
- "CMDLINE" =>"CMDLINE",
281
- "COMM" =>"COMM",
282
- "EXE" =>"EXE",
283
- "GID" =>"GID",
284
- "MACHINE_ID" =>"MACHINE_ID",
285
- "PID" =>"PID",
286
- "SELINUX_CONTEXT" =>"SELINUX_CONTEXT",
287
- "SYSTEMD_CGROUP" =>"SYSTEMD_CGROUP",
288
- "SYSTEMD_OWNER_UID" =>"SYSTEMD_OWNER_UID",
289
- "SYSTEMD_SESSION" =>"SYSTEMD_SESSION",
290
- "SYSTEMD_SLICE" =>"SYSTEMD_SLICE",
291
- "SYSTEMD_UNIT" =>"SYSTEMD_UNIT",
292
- "SYSTEMD_USER_UNIT" =>"SYSTEMD_USER_UNIT",
293
- "TRANSPORT" =>"TRANSPORT",
294
- "UID" =>"UID"
279
+ "AUDIT_LOGINUID" =>"AUDIT_LOGINUID",
280
+ "AUDIT_SESSION" =>"AUDIT_SESSION",
281
+ "BOOT_ID" =>"BOOT_ID",
282
+ "CAP_EFFECTIVE" =>"CAP_EFFECTIVE",
283
+ "CMDLINE" =>"CMDLINE",
284
+ "COMM" =>"COMM",
285
+ "EXE" =>"EXE",
286
+ "GID" =>"GID",
287
+ "MACHINE_ID" =>"MACHINE_ID",
288
+ "LINE_BREAK" =>"LINE_BREAK",
289
+ "PID" =>"PID",
290
+ "SELINUX_CONTEXT" =>"SELINUX_CONTEXT",
291
+ "STREAM_ID" =>"STREAM_ID",
292
+ "SYSTEMD_INVOCATION_ID" =>"SYSTEMD_INVOCATION_ID",
293
+ "SYSTEMD_CGROUP" =>"SYSTEMD_CGROUP",
294
+ "SYSTEMD_OWNER_UID" =>"SYSTEMD_OWNER_UID",
295
+ "SYSTEMD_SESSION" =>"SYSTEMD_SESSION",
296
+ "SYSTEMD_SLICE" =>"SYSTEMD_SLICE",
297
+ "SYSTEMD_UNIT" =>"SYSTEMD_UNIT",
298
+ "SYSTEMD_USER_UNIT" =>"SYSTEMD_USER_UNIT",
299
+ "TRANSPORT" =>"TRANSPORT",
300
+ "UID" =>"UID"
295
301
  }
296
302
  end
297
303
  def normal_output_u
@@ -838,7 +844,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
838
844
  ')
839
845
  assert_equal('mymessage', rec['message'])
840
846
  assert_equal('k8shost', rec['hostname'])
841
- assert_equal('err', rec['level'])
847
+ assert_equal('unknown', rec['level'])
842
848
  assert_equal(@timestamp_str, rec['@timestamp'])
843
849
  assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
844
850
  assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
@@ -867,7 +873,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
867
873
  assert_equal('ADDED', rec['kubernetes']['event']['verb'])
868
874
  assert_equal('event message', rec['message'])
869
875
  assert_equal('mymessage', rec['pipeline_metadata']['collector']['original_raw_message'])
870
- assert_equal('err', rec['level'])
876
+ assert_equal('unknown', rec['level'])
871
877
  assert_equal('2017-07-27T17:23:46.216527+00:00', rec['@timestamp'])
872
878
  assert_equal('127.0.0.1', rec['pipeline_metadata']['collector']['ipaddr4'])
873
879
  assert_equal('::1', rec['pipeline_metadata']['collector']['ipaddr6'])
@@ -896,7 +902,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
896
902
  ')
897
903
  assert_equal('mymessage', rec['message'])
898
904
  assert_equal('k8shost', rec['hostname'])
899
- assert_equal('err', rec['level'])
905
+ assert_equal('unknown', rec['level'])
900
906
  assert_equal(now.utc.to_datetime.rfc3339(6), rec['@timestamp'])
901
907
  assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
902
908
  assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
@@ -926,7 +932,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
926
932
  ')
927
933
  assert_equal('mymessage', rec['message'])
928
934
  assert_equal('k8shost', rec['hostname'])
929
- assert_equal('err', rec['level'])
935
+ assert_equal('unknown', rec['level'])
930
936
  assert_equal(expectedtime, rec['@timestamp'])
931
937
  assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
932
938
  assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
@@ -955,7 +961,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
955
961
  ')
956
962
  assert_equal('mymessage', rec['message'])
957
963
  assert_equal('k8shost', rec['hostname'])
958
- assert_equal('err', rec['level'])
964
+ assert_equal('unknown', rec['level'])
959
965
  assert_equal(@timestamp_str, rec['@timestamp'])
960
966
  assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
961
967
  assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
@@ -983,7 +989,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
983
989
  ')
984
990
  assert_equal('mymessage', rec['message'])
985
991
  assert_equal('k8shost', rec['hostname'])
986
- assert_equal('err', rec['level'])
992
+ assert_equal('unknown', rec['level'])
987
993
  assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['@timestamp'])
988
994
  assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
989
995
  assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
@@ -1010,7 +1016,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
1010
1016
  ')
1011
1017
  assert_equal('mymessage', rec['message'])
1012
1018
  assert_equal('k8shost', rec['hostname'])
1013
- assert_equal('info', rec['level'])
1019
+ assert_equal('unknown', rec['level'])
1014
1020
  assert_equal(@timestamp_str, rec['@timestamp'])
1015
1021
  assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
1016
1022
  assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-viaq_data_model
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.18
4
+ version: 0.0.19
5
5
  platform: ruby
6
6
  authors:
7
7
  - Rich Megginson
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-02-27 00:00:00.000000000 Z
11
+ date: 2019-07-18 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -146,8 +146,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
146
146
  - !ruby/object:Gem::Version
147
147
  version: '0'
148
148
  requirements: []
149
- rubyforge_project:
150
- rubygems_version: 2.7.6
149
+ rubygems_version: 3.0.3
151
150
  signing_key:
152
151
  specification_version: 4
153
152
  summary: Filter plugin to ensure data is in the ViaQ common data model