fluent-plugin-viaq_data_model 0.0.20 → 0.0.21
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cbe4139e57813f44bb6a18d68ff720ce1f6ea5faac0a67d20e001210e01d8c80
|
4
|
+
data.tar.gz: 617ebea9745c86549a26db46a88576fde34ef08e43f165ca0da2864d133854f7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 93a8c658581f71116b9bc053d45ecad028092b210c569877bde0e5c395f091503028ccdb3600865e5e0dcdfcf12ebb2cde25c8945210089e7bbe566bd2935c81
|
7
|
+
data.tar.gz: '0831689e1814494e0c6e4eccb25169cef00f7b5cdee3a38c6eb59de8344b6ede98ac38a68bc528c28ca3f5892ccfe99ef45e54f02d9faf943f609faab1f8a8d2'
|
data/README.md
CHANGED
@@ -89,6 +89,13 @@ See `filter-viaq_data_model.conf` for an example filter configuration.
|
|
89
89
|
* `keep_empty_fields` - comma delimited string - default `''`
|
90
90
|
* Always keep these top level fields, even if they are empty
|
91
91
|
* `keep_empty_fields message` - keep the `message` field, even if empty
|
92
|
+
* `process_kubernetes_events` - boolean - default `true`
|
93
|
+
* If a record looks like a kubernetes event that has been added by the
|
94
|
+
* eventrouter, add its fields under `kubernetes.event`
|
95
|
+
* A record is considered to be a kubernetes eventrouter event if the
|
96
|
+
* record has a top level field called `event` and it is a Hash. If you
|
97
|
+
* are not sure if this is always true, then define a specific tag or tags
|
98
|
+
* and use a per-formatter `process_kubernetes_events` setting, below
|
92
99
|
* `use_undefined` - boolean - default `false`
|
93
100
|
* If `true`, move fields not specified in `default_keep_fields` and
|
94
101
|
`extra_keep_fields` to the `undefined` top level field. If you use
|
@@ -140,8 +147,16 @@ See `filter-viaq_data_model.conf` for an example filter configuration.
|
|
140
147
|
* `sys_var_log` - a record read from `/var/log/messages`
|
141
148
|
* `k8s_json_file` - a record read from a `/var/log/containers/*.log` JSON
|
142
149
|
formatted container log file
|
143
|
-
|
144
|
-
|
150
|
+
* `tag` - the Fluentd tag pattern to match for these records
|
151
|
+
* `remove_keys` - comma delimited list of keys to remove from the record
|
152
|
+
* `process_kubernetes_events` - boolean - defaults to the global setting above
|
153
|
+
* If a record looks like a kubernetes event that has been added by the
|
154
|
+
* eventrouter, add its fields under `kubernetes.event`
|
155
|
+
* A record is considered to be a kubernetes eventrouter event if the
|
156
|
+
* record has a top level field called `event` and it is a Hash.
|
157
|
+
* This per-formatter setting will override the global setting (see above)
|
158
|
+
* This allows you to set the global setting to `false`, then only process
|
159
|
+
* kubernetes events for this formatter matching this set of `tag` patterns.
|
145
160
|
* `pipeline_type` - which part of the pipeline is this? `collector` or
|
146
161
|
`normalizer` - the default is `collector`
|
147
162
|
* `elasticsearch_index_name` - how to construct Elasticsearch index names or
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |gem|
|
6
6
|
gem.name = "fluent-plugin-viaq_data_model"
|
7
|
-
gem.version = "0.0.
|
7
|
+
gem.version = "0.0.21"
|
8
8
|
gem.authors = ["Rich Megginson"]
|
9
9
|
gem.email = ["rmeggins@redhat.com"]
|
10
10
|
gem.description = %q{Filter plugin to ensure data is in the ViaQ common data model}
|
@@ -119,6 +119,8 @@ module Fluent
|
|
119
119
|
config_param :tag, :string
|
120
120
|
desc 'remove these keys from the record - same as record_transformer "remove_keys" field'
|
121
121
|
config_param :remove_keys, :string, default: nil
|
122
|
+
desc 'enable/disable processing of kubernetes events'
|
123
|
+
config_param :process_kubernetes_events, :bool, default: nil
|
122
124
|
end
|
123
125
|
|
124
126
|
desc 'Which part of the pipeline is this - collector, normalizer, etc. for pipeline_metadata'
|
@@ -178,7 +180,6 @@ module Fluent
|
|
178
180
|
@formatters.each do |fmtr|
|
179
181
|
matcher = ViaqMatchClass.new(fmtr.tag, nil)
|
180
182
|
fmtr.instance_eval{ @params[:matcher] = matcher }
|
181
|
-
fmtr.instance_eval{ @params[:fmtr_type] = fmtr.type }
|
182
183
|
if fmtr.remove_keys
|
183
184
|
fmtr.instance_eval{ @params[:fmtr_remove_keys] = fmtr.remove_keys.split(',') }
|
184
185
|
else
|
@@ -193,6 +194,8 @@ module Fluent
|
|
193
194
|
fmtr_func = method(:process_k8s_json_file_fields)
|
194
195
|
end
|
195
196
|
fmtr.instance_eval{ @params[:fmtr_func] = fmtr_func }
|
197
|
+
proc_k8s_ev = fmtr.process_kubernetes_events.nil? ? @process_kubernetes_events : fmtr.process_kubernetes_events
|
198
|
+
fmtr.instance_eval{ @params[:process_kubernetes_events] = proc_k8s_ev }
|
196
199
|
end
|
197
200
|
@formatter_cache = {}
|
198
201
|
@formatter_cache_nomatch = {}
|
@@ -200,7 +203,7 @@ module Fluent
|
|
200
203
|
begin
|
201
204
|
@docker_hostname = File.open('/etc/docker-hostname') { |f| f.readline }.rstrip
|
202
205
|
rescue
|
203
|
-
@docker_hostname = nil
|
206
|
+
@docker_hostname = ENV['NODE_NAME'] || nil
|
204
207
|
end
|
205
208
|
@ipaddr4 = ENV['IPADDR4'] || '127.0.0.1'
|
206
209
|
@ipaddr6 = nil
|
@@ -303,7 +306,7 @@ module Fluent
|
|
303
306
|
retlevel || 'unknown'
|
304
307
|
end
|
305
308
|
|
306
|
-
def process_sys_var_log_fields(tag, time, record,
|
309
|
+
def process_sys_var_log_fields(tag, time, record, fmtr = nil)
|
307
310
|
record['systemd'] = {"t" => {"PID" => record['pid']}, "u" => {"SYSLOG_IDENTIFIER" => record['ident']}}
|
308
311
|
if record[@dest_time_name].nil? # e.g. already has @timestamp
|
309
312
|
# handle the case where the time reported in /var/log/messages is for a previous year
|
@@ -320,7 +323,7 @@ module Fluent
|
|
320
323
|
end
|
321
324
|
end
|
322
325
|
|
323
|
-
def process_k8s_json_file_fields(tag, time, record,
|
326
|
+
def process_k8s_json_file_fields(tag, time, record, fmtr = nil)
|
324
327
|
record['message'] = record['message'] || record['log']
|
325
328
|
record['level'] = normalize_level(record['level'], nil)
|
326
329
|
if record.key?('kubernetes') && record['kubernetes'].respond_to?(:fetch) && \
|
@@ -339,7 +342,7 @@ module Fluent
|
|
339
342
|
end
|
340
343
|
record['time'] = rectime.utc.to_datetime.rfc3339(6)
|
341
344
|
end
|
342
|
-
transform_eventrouter(tag, record)
|
345
|
+
transform_eventrouter(tag, record, fmtr)
|
343
346
|
end
|
344
347
|
|
345
348
|
def check_for_match_and_format(tag, time, record)
|
@@ -355,7 +358,7 @@ module Fluent
|
|
355
358
|
return
|
356
359
|
end
|
357
360
|
end
|
358
|
-
fmtr.fmtr_func.call(tag, time, record, fmtr
|
361
|
+
fmtr.fmtr_func.call(tag, time, record, fmtr)
|
359
362
|
|
360
363
|
if record[@dest_time_name].nil? && record['time'].nil?
|
361
364
|
record['time'] = Time.at(time).utc.to_datetime.rfc3339(6)
|
@@ -449,8 +452,8 @@ module Fluent
|
|
449
452
|
end
|
450
453
|
end
|
451
454
|
|
452
|
-
def transform_eventrouter(tag, record)
|
453
|
-
return
|
455
|
+
def transform_eventrouter(tag, record, fmtr)
|
456
|
+
return if fmtr.nil? || !fmtr.process_kubernetes_events
|
454
457
|
if record.key?("event") && record["event"].respond_to?(:key?)
|
455
458
|
if record.key?("verb")
|
456
459
|
record["event"]["verb"] = record.delete("verb")
|
@@ -68,7 +68,7 @@ module ViaqDataModelFilterSystemd
|
|
68
68
|
|
69
69
|
JOURNAL_TIME_FIELDS = ['_SOURCE_REALTIME_TIMESTAMP', '__REALTIME_TIMESTAMP']
|
70
70
|
|
71
|
-
def process_journal_fields(tag, time, record,
|
71
|
+
def process_journal_fields(tag, time, record, fmtr)
|
72
72
|
systemd_t = {}
|
73
73
|
JOURNAL_FIELD_MAP_SYSTEMD_T.each do |jkey, key|
|
74
74
|
if record.key?(jkey)
|
@@ -104,7 +104,7 @@ module ViaqDataModelFilterSystemd
|
|
104
104
|
break
|
105
105
|
end
|
106
106
|
end
|
107
|
-
case
|
107
|
+
case fmtr.type
|
108
108
|
when :sys_journal
|
109
109
|
record['message'] = record['MESSAGE']
|
110
110
|
if record['_HOSTNAME'].eql?('localhost') && @docker_hostname
|
@@ -136,7 +136,7 @@ module ViaqDataModelFilterSystemd
|
|
136
136
|
else
|
137
137
|
record['hostname'] = record['_HOSTNAME']
|
138
138
|
end
|
139
|
-
transform_eventrouter(tag, record)
|
139
|
+
transform_eventrouter(tag, record, fmtr)
|
140
140
|
end
|
141
141
|
end
|
142
142
|
end
|
@@ -884,6 +884,37 @@ class ViaqDataModelFilterTest < Test::Unit::TestCase
|
|
884
884
|
dellist = 'host,pid,ident,event,verb'.split(',')
|
885
885
|
dellist.each{|field| assert_nil(rec[field])}
|
886
886
|
end
|
887
|
+
test 'process a k8s json-file record with event from eventrouter, per formatter setting' do
|
888
|
+
ENV['IPADDR4'] = '127.0.0.1'
|
889
|
+
ENV['IPADDR6'] = '::1'
|
890
|
+
ENV['FLUENTD_VERSION'] = 'fversion'
|
891
|
+
ENV['DATA_VERSION'] = 'dversion'
|
892
|
+
input = {'kubernetes'=>{'host'=>'k8shost'},'stream'=>'stderr','time'=>@timestamp_str,'log'=>'mymessage'}
|
893
|
+
input = add_event(input)
|
894
|
+
rec = emit_with_tag('kubernetes.var.log.containers.name.name_this_that_other_log', input, '
|
895
|
+
<formatter>
|
896
|
+
tag "kubernetes.var.log.containers**"
|
897
|
+
type k8s_json_file
|
898
|
+
remove_keys log,stream
|
899
|
+
process_kubernetes_events true
|
900
|
+
</formatter>
|
901
|
+
pipeline_type collector
|
902
|
+
process_kubernetes_events false
|
903
|
+
')
|
904
|
+
assert_equal('ADDED', rec['kubernetes']['event']['verb'])
|
905
|
+
assert_equal('event message', rec['message'])
|
906
|
+
assert_equal('mymessage', rec['pipeline_metadata']['collector']['original_raw_message'])
|
907
|
+
assert_equal('unknown', rec['level'])
|
908
|
+
assert_equal('2017-07-27T17:23:46.216527+00:00', rec['@timestamp'])
|
909
|
+
assert_equal('127.0.0.1', rec['pipeline_metadata']['collector']['ipaddr4'])
|
910
|
+
assert_equal('::1', rec['pipeline_metadata']['collector']['ipaddr6'])
|
911
|
+
assert_equal('fluent-plugin-systemd', rec['pipeline_metadata']['collector']['inputname'])
|
912
|
+
assert_equal('fluentd', rec['pipeline_metadata']['collector']['name'])
|
913
|
+
assert_equal('fversion dversion', rec['pipeline_metadata']['collector']['version'])
|
914
|
+
assert_equal(@timestamp_str, rec['pipeline_metadata']['collector']['received_at'])
|
915
|
+
dellist = 'host,pid,ident,event,verb'.split(',')
|
916
|
+
dellist.each{|field| assert_nil(rec[field])}
|
917
|
+
end
|
887
918
|
test 'process a k8s json-file record with a string valued timestamp' do
|
888
919
|
ENV['IPADDR4'] = '127.0.0.1'
|
889
920
|
ENV['IPADDR6'] = '::1'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-viaq_data_model
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.21
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Rich Megginson
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-10-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|