fluent-plugin-viaq_data_model 0.0.18 → 0.0.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2 -2
  3. data/fluent-plugin-viaq_data_model.gemspec +1 -1
  4. data/lib/fluent/plugin/filter_viaq_data_model.rb +2 -6
  5. data/lib/fluent/plugin/filter_viaq_data_model_systemd.rb +23 -20
  6. data/test/test_filter_viaq_data_model.rb +32 -26
  7. metadata +3 -4
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 1d809d8421925d6bf93ccaf62c93b6c2bbe03aaec5dd70b5e8ad243bcdd4d71c
4
- data.tar.gz: c1b0b023782d77bc6ab3095b3e151a5b8dc29416d8581615c99951e935fa834d
3
+ metadata.gz: b20471334c7756a6393385dfe17987751ea233c8cac638f592459ea5a3a8692f
4
+ data.tar.gz: 0d1738ff46f83bba941f415616c374ece1268a89c59f7df333c0311106809e67
5
5
  SHA512:
6
- metadata.gz: 0000a8e8fc5e586ff4d589db024402ea640cb36d8ceacac6b8df3f3696702ad491939ce0e7b627e11fe5a5abe0ae86de146baacee90d2cc4f1e1c43c122b8c3b
7
- data.tar.gz: 2687cd962c4ec754bf133d0b939174275c1c4c794ef75a31467971be3e9bc16119aa9d6ea5028c36f5666714d9c3a03d2d277c5ec9ce40becdbf72fa36698191
6
+ metadata.gz: c77bfd1197a5d06b994668034dcd4d662ddb955fd4c3a803fad71a47124f86e28f29f0a063eecfe86d6bf23c9791159ca693405be9e0c4744288345ee3f0d3e0
7
+ data.tar.gz: 902a1adc2eca199a51b7cb6ce37600c2ec9764eeacbda1e2313410bcde774f1184513d4b87af91b7efd15651c35e2f7b97caf872e6e5f124a9d56232e2e5f433
data/README.md CHANGED
@@ -162,10 +162,10 @@ See `filter-viaq_data_model.conf` for an example filter configuration.
162
162
  create the full index name.
163
163
  * `elasticsearch_index_name_field` - name of the field in the record which stores
164
164
  the index name - you should remove this field in the elasticsearch output
165
- plugin using the `remove_keys` config parameter - default is `viaq_idnex_name`
165
+ plugin using the `remove_keys` config parameter - default is `viaq_index_name`
166
166
  * `elasticsearch_index_prefix_field` - name of the field in the record which stores
167
167
  the index prefix - you should remove this field in the elasticsearch output
168
- plugin using the `remove_keys` config parameter - default is `viaq_idnex_prefix`
168
+ plugin using the `remove_keys` config parameter - default is `viaq_index_prefix`
169
169
 
170
170
  **NOTE** The `formatter` blocks are matched in the given order in the file.
171
171
  This means, don't use `tag "**"` as the first formatter or none of your
data/fluent-plugin-viaq_data_model.gemspec CHANGED
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |gem|
6
6
  gem.name = "fluent-plugin-viaq_data_model"
7
- gem.version = "0.0.18"
7
+ gem.version = "0.0.19"
8
8
  gem.authors = ["Rich Megginson"]
9
9
  gem.email = ["rmeggins@redhat.com"]
10
10
  gem.description = %q{Filter plugin to ensure data is in the ViaQ common data model}
data/lib/fluent/plugin/filter_viaq_data_model.rb CHANGED
@@ -280,7 +280,7 @@ module Fluent
280
280
  '8' => 'trace',
281
281
  '9' => 'unknown',
282
282
  }
283
- def normalize_level(level, newlevel, stream=nil, priority=nil)
283
+ def normalize_level(level, newlevel, priority=nil)
284
284
  # if the record already has a level field, and it looks like one of our well
285
285
  # known values, convert it to the canonical normalized form - otherwise,
286
286
  # preserve the value in string format
@@ -290,10 +290,6 @@ module Fluent
290
290
  (level.respond_to?(:downcase) && (retlevel = NORMAL_LEVELS[level.downcase]))
291
291
  retlevel = level.to_s # don't know what it is - just convert to string
292
292
  end
293
- elsif stream == 'stdout'
294
- retlevel = 'info'
295
- elsif stream == 'stderr'
296
- retlevel = 'err'
297
293
  elsif !priority.nil?
298
294
  retlevel = PRIORITY_LEVELS[priority]
299
295
  else
@@ -321,7 +317,7 @@ module Fluent
321
317
 
322
318
  def process_k8s_json_file_fields(tag, time, record, fmtr_type = nil)
323
319
  record['message'] = record['message'] || record['log']
324
- record['level'] = normalize_level(record['level'], nil, record['stream'])
320
+ record['level'] = normalize_level(record['level'], nil)
325
321
  if record.key?('kubernetes') && record['kubernetes'].respond_to?(:fetch) && \
326
322
  (k8shost = record['kubernetes'].fetch('host', nil))
327
323
  record['hostname'] = k8shost
data/lib/fluent/plugin/filter_viaq_data_model_systemd.rb CHANGED
@@ -21,25 +21,28 @@ require 'date'
21
21
  module ViaqDataModelFilterSystemd
22
22
  # map of journal fields to viaq data model field
23
23
  JOURNAL_FIELD_MAP_SYSTEMD_T = {
24
- "_AUDIT_LOGINUID" => "AUDIT_LOGINUID",
25
- "_AUDIT_SESSION" => "AUDIT_SESSION",
26
- "_BOOT_ID" => "BOOT_ID",
27
- "_CAP_EFFECTIVE" => "CAP_EFFECTIVE",
28
- "_CMDLINE" => "CMDLINE",
29
- "_COMM" => "COMM",
30
- "_EXE" => "EXE",
31
- "_GID" => "GID",
32
- "_MACHINE_ID" => "MACHINE_ID",
33
- "_PID" => "PID",
34
- "_SELINUX_CONTEXT" => "SELINUX_CONTEXT",
35
- "_SYSTEMD_CGROUP" => "SYSTEMD_CGROUP",
36
- "_SYSTEMD_OWNER_UID" => "SYSTEMD_OWNER_UID",
37
- "_SYSTEMD_SESSION" => "SYSTEMD_SESSION",
38
- "_SYSTEMD_SLICE" => "SYSTEMD_SLICE",
39
- "_SYSTEMD_UNIT" => "SYSTEMD_UNIT",
40
- "_SYSTEMD_USER_UNIT" => "SYSTEMD_USER_UNIT",
41
- "_TRANSPORT" => "TRANSPORT",
42
- "_UID" => "UID"
24
+ "_AUDIT_LOGINUID" => "AUDIT_LOGINUID",
25
+ "_AUDIT_SESSION" => "AUDIT_SESSION",
26
+ "_BOOT_ID" => "BOOT_ID",
27
+ "_CAP_EFFECTIVE" => "CAP_EFFECTIVE",
28
+ "_CMDLINE" => "CMDLINE",
29
+ "_COMM" => "COMM",
30
+ "_EXE" => "EXE",
31
+ "_GID" => "GID",
32
+ "_LINE_BREAK" => "LINE_BREAK",
33
+ "_MACHINE_ID" => "MACHINE_ID",
34
+ "_PID" => "PID",
35
+ "_SELINUX_CONTEXT" => "SELINUX_CONTEXT",
36
+ "_STREAM_ID" => "STREAM_ID",
37
+ "_SYSTEMD_CGROUP" => "SYSTEMD_CGROUP",
38
+ "_SYSTEMD_INVOCATION_ID" => "SYSTEMD_INVOCATION_ID",
39
+ "_SYSTEMD_OWNER_UID" => "SYSTEMD_OWNER_UID",
40
+ "_SYSTEMD_SESSION" => "SYSTEMD_SESSION",
41
+ "_SYSTEMD_SLICE" => "SYSTEMD_SLICE",
42
+ "_SYSTEMD_UNIT" => "SYSTEMD_UNIT",
43
+ "_SYSTEMD_USER_UNIT" => "SYSTEMD_USER_UNIT",
44
+ "_TRANSPORT" => "TRANSPORT",
45
+ "_UID" => "UID"
43
46
  }
44
47
 
45
48
  JOURNAL_FIELD_MAP_SYSTEMD_U = {
@@ -93,7 +96,7 @@ module ViaqDataModelFilterSystemd
93
96
  unless systemd_k.empty?
94
97
  (record['systemd'] ||= {})['k'] = systemd_k
95
98
  end
96
- record['level'] = normalize_level(record['level'], nil, nil, record['PRIORITY'])
99
+ record['level'] = normalize_level(record['level'], nil, record['PRIORITY'])
97
100
  JOURNAL_TIME_FIELDS.each do |field|
98
101
  if (val = record[field])
99
102
  vali = val.to_i
data/test/test_filter_viaq_data_model.rb CHANGED
@@ -239,9 +239,12 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
239
239
  "_EXE" => "EXE",
240
240
  "_GID" => "GID",
241
241
  "_MACHINE_ID" => "MACHINE_ID",
242
+ "_LINE_BREAK" => "LINE_BREAK",
242
243
  "_PID" => "PID",
243
244
  "_SELINUX_CONTEXT" => "SELINUX_CONTEXT",
245
+ "_STREAM_ID" => "STREAM_ID",
244
246
  "_SYSTEMD_CGROUP" => "SYSTEMD_CGROUP",
247
+ "_SYSTEMD_INVOCATION_ID" => "SYSTEMD_INVOCATION_ID",
245
248
  "_SYSTEMD_OWNER_UID" => "SYSTEMD_OWNER_UID",
246
249
  "_SYSTEMD_SESSION" => "SYSTEMD_SESSION",
247
250
  "_SYSTEMD_SLICE" => "SYSTEMD_SLICE",
@@ -273,25 +276,28 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
273
276
  end
274
277
  def normal_output_t
275
278
  {
276
- "AUDIT_LOGINUID" =>"AUDIT_LOGINUID",
277
- "AUDIT_SESSION" =>"AUDIT_SESSION",
278
- "BOOT_ID" =>"BOOT_ID",
279
- "CAP_EFFECTIVE" =>"CAP_EFFECTIVE",
280
- "CMDLINE" =>"CMDLINE",
281
- "COMM" =>"COMM",
282
- "EXE" =>"EXE",
283
- "GID" =>"GID",
284
- "MACHINE_ID" =>"MACHINE_ID",
285
- "PID" =>"PID",
286
- "SELINUX_CONTEXT" =>"SELINUX_CONTEXT",
287
- "SYSTEMD_CGROUP" =>"SYSTEMD_CGROUP",
288
- "SYSTEMD_OWNER_UID" =>"SYSTEMD_OWNER_UID",
289
- "SYSTEMD_SESSION" =>"SYSTEMD_SESSION",
290
- "SYSTEMD_SLICE" =>"SYSTEMD_SLICE",
291
- "SYSTEMD_UNIT" =>"SYSTEMD_UNIT",
292
- "SYSTEMD_USER_UNIT" =>"SYSTEMD_USER_UNIT",
293
- "TRANSPORT" =>"TRANSPORT",
294
- "UID" =>"UID"
279
+ "AUDIT_LOGINUID" =>"AUDIT_LOGINUID",
280
+ "AUDIT_SESSION" =>"AUDIT_SESSION",
281
+ "BOOT_ID" =>"BOOT_ID",
282
+ "CAP_EFFECTIVE" =>"CAP_EFFECTIVE",
283
+ "CMDLINE" =>"CMDLINE",
284
+ "COMM" =>"COMM",
285
+ "EXE" =>"EXE",
286
+ "GID" =>"GID",
287
+ "MACHINE_ID" =>"MACHINE_ID",
288
+ "LINE_BREAK" =>"LINE_BREAK",
289
+ "PID" =>"PID",
290
+ "SELINUX_CONTEXT" =>"SELINUX_CONTEXT",
291
+ "STREAM_ID" =>"STREAM_ID",
292
+ "SYSTEMD_INVOCATION_ID" =>"SYSTEMD_INVOCATION_ID",
293
+ "SYSTEMD_CGROUP" =>"SYSTEMD_CGROUP",
294
+ "SYSTEMD_OWNER_UID" =>"SYSTEMD_OWNER_UID",
295
+ "SYSTEMD_SESSION" =>"SYSTEMD_SESSION",
296
+ "SYSTEMD_SLICE" =>"SYSTEMD_SLICE",
297
+ "SYSTEMD_UNIT" =>"SYSTEMD_UNIT",
298
+ "SYSTEMD_USER_UNIT" =>"SYSTEMD_USER_UNIT",
299
+ "TRANSPORT" =>"TRANSPORT",
300
+ "UID" =>"UID"
295
301
  }
296
302
  end
297
303
  def normal_output_u
@@ -838,7 +844,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
838
844
  ')
839
845
  assert_equal('mymessage', rec['message'])
840
846
  assert_equal('k8shost', rec['hostname'])
841
- assert_equal('err', rec['level'])
847
+ assert_equal('unknown', rec['level'])
842
848
  assert_equal(@timestamp_str, rec['@timestamp'])
843
849
  assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
844
850
  assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
@@ -867,7 +873,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
867
873
  assert_equal('ADDED', rec['kubernetes']['event']['verb'])
868
874
  assert_equal('event message', rec['message'])
869
875
  assert_equal('mymessage', rec['pipeline_metadata']['collector']['original_raw_message'])
870
- assert_equal('err', rec['level'])
876
+ assert_equal('unknown', rec['level'])
871
877
  assert_equal('2017-07-27T17:23:46.216527+00:00', rec['@timestamp'])
872
878
  assert_equal('127.0.0.1', rec['pipeline_metadata']['collector']['ipaddr4'])
873
879
  assert_equal('::1', rec['pipeline_metadata']['collector']['ipaddr6'])
@@ -896,7 +902,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
896
902
  ')
897
903
  assert_equal('mymessage', rec['message'])
898
904
  assert_equal('k8shost', rec['hostname'])
899
- assert_equal('err', rec['level'])
905
+ assert_equal('unknown', rec['level'])
900
906
  assert_equal(now.utc.to_datetime.rfc3339(6), rec['@timestamp'])
901
907
  assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
902
908
  assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
@@ -926,7 +932,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
926
932
  ')
927
933
  assert_equal('mymessage', rec['message'])
928
934
  assert_equal('k8shost', rec['hostname'])
929
- assert_equal('err', rec['level'])
935
+ assert_equal('unknown', rec['level'])
930
936
  assert_equal(expectedtime, rec['@timestamp'])
931
937
  assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
932
938
  assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
@@ -955,7 +961,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
955
961
  ')
956
962
  assert_equal('mymessage', rec['message'])
957
963
  assert_equal('k8shost', rec['hostname'])
958
- assert_equal('err', rec['level'])
964
+ assert_equal('unknown', rec['level'])
959
965
  assert_equal(@timestamp_str, rec['@timestamp'])
960
966
  assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
961
967
  assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
@@ -983,7 +989,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
983
989
  ')
984
990
  assert_equal('mymessage', rec['message'])
985
991
  assert_equal('k8shost', rec['hostname'])
986
- assert_equal('err', rec['level'])
992
+ assert_equal('unknown', rec['level'])
987
993
  assert_equal(Time.at(@time).utc.to_datetime.rfc3339(6), rec['@timestamp'])
988
994
  assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
989
995
  assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
@@ -1010,7 +1016,7 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
1010
1016
  ')
1011
1017
  assert_equal('mymessage', rec['message'])
1012
1018
  assert_equal('k8shost', rec['hostname'])
1013
- assert_equal('info', rec['level'])
1019
+ assert_equal('unknown', rec['level'])
1014
1020
  assert_equal(@timestamp_str, rec['@timestamp'])
1015
1021
  assert_equal('127.0.0.1', rec['pipeline_metadata']['normalizer']['ipaddr4'])
1016
1022
  assert_equal('::1', rec['pipeline_metadata']['normalizer']['ipaddr6'])
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-viaq_data_model
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.18
4
+ version: 0.0.19
5
5
  platform: ruby
6
6
  authors:
7
7
  - Rich Megginson
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-02-27 00:00:00.000000000 Z
11
+ date: 2019-07-18 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -146,8 +146,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
146
146
  - !ruby/object:Gem::Version
147
147
  version: '0'
148
148
  requirements: []
149
- rubyforge_project:
150
- rubygems_version: 2.7.6
149
+ rubygems_version: 3.0.3
151
150
  signing_key:
152
151
  specification_version: 4
153
152
  summary: Filter plugin to ensure data is in the ViaQ common data model