fluent-plugin-kubernetes_metadata_filter 2.9.2 → 2.9.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +11 -11
- data/README.md +6 -38
- data/fluent-plugin-kubernetes_metadata_filter.gemspec +1 -1
- data/lib/fluent/plugin/filter_kubernetes_metadata.rb +37 -25
- data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb +1 -3
- data/lib/fluent/plugin/kubernetes_metadata_common.rb +5 -4
- data/lib/fluent/plugin/kubernetes_metadata_test_api_adapter.rb +2 -2
- data/release_notes.md +42 -0
- data/test/plugin/test_cache_strategy.rb +1 -1
- data/test/plugin/test_filter_kubernetes_metadata.rb +56 -22
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c2025b09847baf151d711708ca4ce178697f5fbedd1a73dedf4e3ea9e15f3095
|
4
|
+
data.tar.gz: f0b2e2c6c990daeac2b7e75c29bc63b30b534d0a765e659568ec5f029122aeec
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: '09648e794c92b134fba3c332bafc72974f66ba7fc189e5f4949dfa8ab94e79b0918624697760b0047c426cefc8a1aa59fb80736e4c2ea0281e481c79e2b32db6'
|
7
|
+
data.tar.gz: daf83c7fa7e56d0f5833f9b5f7e3efd9796943ad8f7d69fd8ce9755a54a0a9622574ba8b2536e95d6f324afa4a396bbba37e5824d3d3f08f97efa2bda7148f4e
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
fluent-plugin-kubernetes_metadata_filter (2.9.
|
4
|
+
fluent-plugin-kubernetes_metadata_filter (2.9.5)
|
5
5
|
fluentd (>= 0.14.0, < 1.15)
|
6
6
|
kubeclient (>= 4.0.0, < 5.0.0)
|
7
7
|
lru_redux
|
@@ -26,16 +26,16 @@ GEM
|
|
26
26
|
domain_name (0.5.20190701)
|
27
27
|
unf (>= 0.0.5, < 1.0.0)
|
28
28
|
escape_utils (1.2.1)
|
29
|
-
ffi (1.15.
|
29
|
+
ffi (1.15.5)
|
30
30
|
ffi-compiler (1.0.1)
|
31
31
|
ffi (>= 1.0.0)
|
32
32
|
rake
|
33
|
-
fluentd (1.14.
|
33
|
+
fluentd (1.14.5)
|
34
34
|
bundler
|
35
35
|
cool.io (>= 1.4.5, < 2.0.0)
|
36
|
-
http_parser.rb (>= 0.5.1, < 0.
|
36
|
+
http_parser.rb (>= 0.5.1, < 0.9.0)
|
37
37
|
msgpack (>= 1.3.1, < 2.0.0)
|
38
|
-
serverengine (>= 2.2.
|
38
|
+
serverengine (>= 2.2.5, < 3.0.0)
|
39
39
|
sigdump (~> 0.2.2)
|
40
40
|
strptime (>= 0.2.4, < 1.0.0)
|
41
41
|
tzinfo (>= 1.0, < 3.0)
|
@@ -59,7 +59,7 @@ GEM
|
|
59
59
|
http-form_data (2.3.0)
|
60
60
|
http-parser (1.2.3)
|
61
61
|
ffi-compiler (>= 1.0, < 2.0)
|
62
|
-
http_parser.rb (0.
|
62
|
+
http_parser.rb (0.8.0)
|
63
63
|
jsonpath (1.1.0)
|
64
64
|
multi_json
|
65
65
|
kubeclient (4.9.2)
|
@@ -68,12 +68,12 @@ GEM
|
|
68
68
|
recursive-open-struct (~> 1.1, >= 1.1.1)
|
69
69
|
rest-client (~> 2.0)
|
70
70
|
lru_redux (1.1.0)
|
71
|
-
mime-types (3.
|
71
|
+
mime-types (3.4.1)
|
72
72
|
mime-types-data (~> 3.2015)
|
73
|
-
mime-types-data (3.
|
73
|
+
mime-types-data (3.2022.0105)
|
74
74
|
mini_mime (1.1.2)
|
75
75
|
minitest (4.7.5)
|
76
|
-
msgpack (1.4.
|
76
|
+
msgpack (1.4.5)
|
77
77
|
multi_json (1.15.0)
|
78
78
|
netrc (0.11.0)
|
79
79
|
parallel (1.21.0)
|
@@ -105,7 +105,7 @@ GEM
|
|
105
105
|
parser (>= 3.0.1.1)
|
106
106
|
ruby-progressbar (1.11.0)
|
107
107
|
rugged (1.2.0)
|
108
|
-
serverengine (2.2.
|
108
|
+
serverengine (2.2.5)
|
109
109
|
sigdump (~> 0.2.2)
|
110
110
|
sigdump (0.2.4)
|
111
111
|
simplecov (0.21.2)
|
@@ -155,4 +155,4 @@ DEPENDENCIES
|
|
155
155
|
yajl-ruby
|
156
156
|
|
157
157
|
BUNDLED WITH
|
158
|
-
2.
|
158
|
+
2.3.4
|
data/README.md
CHANGED
@@ -41,14 +41,14 @@ Configuration options for fluent.conf are:
|
|
41
41
|
* `client_key` - path to a client key file to authenticate to the API server
|
42
42
|
* `bearer_token_file` - path to a file containing the bearer token to use for authentication
|
43
43
|
* `tag_to_kubernetes_name_regexp` - the regular expression used to extract kubernetes metadata (pod name, container name, namespace) from the current fluentd tag.
|
44
|
-
This must
|
44
|
+
This must use named capture groups for `container_name`, `pod_name`, `namespace`, and either `pod_uuid (/var/log/pods)` or `docker_id (/var/log/containers)`
|
45
45
|
* `cache_size` - size of the cache of Kubernetes metadata to reduce requests to the API server (default: `1000`)
|
46
46
|
* `cache_ttl` - TTL in seconds of each cached element. Set to negative value to disable TTL eviction (default: `3600` - 1 hour)
|
47
47
|
* `watch` - set up a watch on pods on the API server for updates to metadata (default: `true`)
|
48
|
-
*
|
49
|
-
*
|
50
|
-
*
|
51
|
-
*
|
48
|
+
* *DEPRECATED*`de_dot` - replace dots in labels and annotations with configured `de_dot_separator`, required for Datadog and ElasticSearch 2.x compatibility (default: `true`)
|
49
|
+
* *DEPRECATED*`de_dot_separator` - separator to use if `de_dot` is enabled (default: `_`)
|
50
|
+
* *DEPRECATED*`de_slash` - replace slashes in labels and annotations with configured `de_slash_separator`, required for Datadog compatibility (default: `false`)
|
51
|
+
* *DEPRECATED*`de_slash_separator` - separator to use if `de_slash` is enabled (default: `__`)
|
52
52
|
* *DEPRECATED* `use_journal` - If false, messages are expected to be formatted and tagged as if read by the fluentd in\_tail plugin with wildcard filename. If true, messages are expected to be formatted as if read from the systemd journal. The `MESSAGE` field has the full message. The `CONTAINER_NAME` field has the encoded k8s metadata (see below). The `CONTAINER_ID_FULL` field has the full container uuid. This requires docker to use the `--log-driver=journald` log driver. If unset (the default), the plugin will use the `CONTAINER_NAME` and `CONTAINER_ID_FULL` fields
|
53
53
|
if available, otherwise, will use the tag in the `tag_to_kubernetes_name_regexp` format.
|
54
54
|
* `container_name_to_kubernetes_regexp` - The regular expression used to extract the k8s metadata encoded in the journal `CONTAINER_NAME` field default: See [code](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/blob/master/lib/fluent/plugin/filter_kubernetes_metadata.rb#L68)
|
@@ -68,23 +68,6 @@ when true (default: `true`)
|
|
68
68
|
* `skip_namespace_metadata` - Skip the namespace_id field from the metadata. The fetch_namespace_metadata function will be skipped. The plugin will be faster and cpu consumption will be less.
|
69
69
|
* `watch_retry_interval` - The time interval in seconds for retry backoffs when watch connections fail. (default: `10`)
|
70
70
|
|
71
|
-
**NOTE:** As of the release 2.1.x of this plugin, it no longer supports parsing the source message into JSON and attaching it to the
|
72
|
-
payload. The following configuration options are removed:
|
73
|
-
|
74
|
-
* `merge_json_log`
|
75
|
-
* `preserve_json_log`
|
76
|
-
|
77
|
-
One way of preserving JSON logs can be through the [parser plugin](https://docs.fluentd.org/filter/parser)
|
78
|
-
|
79
|
-
**NOTE** As of this release, the use of `use_journal` is **DEPRECATED**. If this setting is not present, the plugin will
|
80
|
-
attempt to figure out the source of the metadata fields from the following:
|
81
|
-
- If `lookup_from_k8s_field true` (the default) and the following fields are present in the record:
|
82
|
-
`docker.container_id`, `kubernetes.namespace_name`, `kubernetes.pod_name`, `kubernetes.container_name`,
|
83
|
-
then the plugin will use those values as the source to use to lookup the metadata
|
84
|
-
- If `use_journal true`, or `use_journal` is unset, and the fields `CONTAINER_NAME` and `CONTAINER_ID_FULL` are present in the record,
|
85
|
-
then the plugin will parse those values using `container_name_to_kubernetes_regexp` and use those as the source to lookup the metadata
|
86
|
-
- Otherwise, if the tag matches `tag_to_kubernetes_name_regexp`, the plugin will parse the tag and use those values to
|
87
|
-
lookup the metdata
|
88
71
|
|
89
72
|
Reading from the JSON formatted log files with `in_tail` and wildcard filenames while respecting the CRI-o log format with the same config you need the fluent-plugin "multi-format-parser":
|
90
73
|
|
@@ -154,22 +137,7 @@ Reading from the systemd journal (requires the fluentd `fluent-plugin-systemd` a
|
|
154
137
|
@type stdout
|
155
138
|
</match>
|
156
139
|
```
|
157
|
-
|
158
|
-
In former versions this plugin parsed the value of the key log as JSON. In the current version this feature was removed, to avoid duplicate features in the fluentd plugin ecosystem. It can parsed with the parser plugin like this:
|
159
|
-
```
|
160
|
-
<filter kubernetes.**>
|
161
|
-
@type parser
|
162
|
-
key_name log
|
163
|
-
<parse>
|
164
|
-
@type json
|
165
|
-
json_parser json
|
166
|
-
</parse>
|
167
|
-
replace_invalid_sequence true
|
168
|
-
reserve_data true # this preserves unparsable log lines
|
169
|
-
emit_invalid_record_to_error false # In case of unparsable log lines keep the error log clean
|
170
|
-
reserve_time # the time was already parsed in the source, we don't want to overwrite it with current time.
|
171
|
-
</filter>
|
172
|
-
```
|
140
|
+
|
173
141
|
|
174
142
|
## Environment variables for Kubernetes
|
175
143
|
|
@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |gem|
|
7
7
|
gem.name = 'fluent-plugin-kubernetes_metadata_filter'
|
8
|
-
gem.version = '2.9.
|
8
|
+
gem.version = '2.9.5'
|
9
9
|
gem.authors = ['Jimmi Dyson']
|
10
10
|
gem.email = ['jimmidyson@gmail.com']
|
11
11
|
gem.description = 'Filter plugin to add Kubernetes metadata'
|
@@ -51,9 +51,17 @@ module Fluent::Plugin
|
|
51
51
|
config_param :client_key, :string, default: nil
|
52
52
|
config_param :ca_file, :string, default: nil
|
53
53
|
config_param :verify_ssl, :bool, default: true
|
54
|
-
|
55
|
-
|
56
|
-
|
54
|
+
|
55
|
+
REGEX_VAR_LOG_PODS = '(var\.log\.pods)\.(?<namespace>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<pod_uuid>[a-z0-9-]*)\.(?<container_name>.+)\..*\.log$'
|
56
|
+
REGEX_VAR_LOG_CONTAINERS = '(var\.log\.containers)\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
|
57
|
+
|
58
|
+
#tag_to_kubernetes_name_regexp which must include named capture groups:
|
59
|
+
# namespace - The namespace in which the pod is deployed
|
60
|
+
# pod_name - The pod name
|
61
|
+
# container_name - The name of the container
|
62
|
+
# pod_uuid (/var/log/pods) | docker_id (/var/log/containers) - Unique identifier used in caching of either pod_uuid or the container hash
|
63
|
+
config_param :tag_to_kubernetes_name_regexp, :string, default: "(#{REGEX_VAR_LOG_PODS}|#{REGEX_VAR_LOG_CONTAINERS})"
|
64
|
+
|
57
65
|
config_param :bearer_token_file, :string, default: nil
|
58
66
|
config_param :secret_dir, :string, default: '/var/run/secrets/kubernetes.io/serviceaccount'
|
59
67
|
config_param :de_dot, :bool, default: true
|
@@ -100,15 +108,15 @@ module Fluent::Plugin
|
|
100
108
|
config_param :watch_retry_max_times, :integer, default: 10
|
101
109
|
|
102
110
|
def fetch_pod_metadata(namespace_name, pod_name)
|
103
|
-
log.trace("fetching pod metadata: #{namespace_name}/#{pod_name}")
|
111
|
+
log.trace("fetching pod metadata: #{namespace_name}/#{pod_name}")
|
104
112
|
options = {
|
105
113
|
resource_version: '0' # Fetch from API server cache instead of etcd quorum read
|
106
114
|
}
|
107
115
|
pod_object = @client.get_pod(pod_name, namespace_name, options)
|
108
|
-
log.trace("raw metadata for #{namespace_name}/#{pod_name}: #{pod_object}")
|
116
|
+
log.trace("raw metadata for #{namespace_name}/#{pod_name}: #{pod_object}")
|
109
117
|
metadata = parse_pod_metadata(pod_object)
|
110
118
|
@stats.bump(:pod_cache_api_updates)
|
111
|
-
log.trace("parsed metadata for #{namespace_name}/#{pod_name}: #{metadata}")
|
119
|
+
log.trace("parsed metadata for #{namespace_name}/#{pod_name}: #{metadata}")
|
112
120
|
@cache[metadata['pod_id']] = metadata
|
113
121
|
rescue StandardError => e
|
114
122
|
@stats.bump(:pod_cache_api_nil_error)
|
@@ -132,15 +140,15 @@ module Fluent::Plugin
|
|
132
140
|
end
|
133
141
|
|
134
142
|
def fetch_namespace_metadata(namespace_name)
|
135
|
-
log.trace("fetching namespace metadata: #{namespace_name}")
|
143
|
+
log.trace("fetching namespace metadata: #{namespace_name}")
|
136
144
|
options = {
|
137
145
|
resource_version: '0' # Fetch from API server cache instead of etcd quorum read
|
138
146
|
}
|
139
147
|
namespace_object = @client.get_namespace(namespace_name, nil, options)
|
140
|
-
log.trace("raw metadata for #{namespace_name}: #{namespace_object}")
|
148
|
+
log.trace("raw metadata for #{namespace_name}: #{namespace_object}")
|
141
149
|
metadata = parse_namespace_metadata(namespace_object)
|
142
150
|
@stats.bump(:namespace_cache_api_updates)
|
143
|
-
log.trace("parsed metadata for #{namespace_name}: #{metadata}")
|
151
|
+
log.trace("parsed metadata for #{namespace_name}: #{metadata}")
|
144
152
|
@namespace_cache[metadata['namespace_id']] = metadata
|
145
153
|
rescue StandardError => e
|
146
154
|
@stats.bump(:namespace_cache_api_nil_error)
|
@@ -156,10 +164,6 @@ module Fluent::Plugin
|
|
156
164
|
def configure(conf)
|
157
165
|
super
|
158
166
|
|
159
|
-
def log.trace?
|
160
|
-
level == Fluent::Log::LEVEL_TRACE
|
161
|
-
end
|
162
|
-
|
163
167
|
require 'kubeclient'
|
164
168
|
require 'lru_redux'
|
165
169
|
@stats = KubernetesMetadata::Stats.new
|
@@ -187,6 +191,7 @@ module Fluent::Plugin
|
|
187
191
|
@namespace_cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl)
|
188
192
|
|
189
193
|
@tag_to_kubernetes_name_regexp_compiled = Regexp.compile(@tag_to_kubernetes_name_regexp)
|
194
|
+
|
190
195
|
@container_name_to_kubernetes_regexp_compiled = Regexp.compile(@container_name_to_kubernetes_regexp)
|
191
196
|
|
192
197
|
# Use Kubernetes default service account if we're in a pod.
|
@@ -300,40 +305,48 @@ module Fluent::Plugin
|
|
300
305
|
end
|
301
306
|
end
|
302
307
|
|
303
|
-
def get_metadata_for_record(namespace_name, pod_name, container_name,
|
308
|
+
def get_metadata_for_record(namespace_name, pod_name, container_name, cache_key, create_time, batch_miss_cache, docker_id)
|
304
309
|
metadata = {
|
305
|
-
'docker' => { 'container_id' =>
|
310
|
+
'docker' => { 'container_id' => "" },
|
306
311
|
'kubernetes' => {
|
307
312
|
'container_name' => container_name,
|
308
313
|
'namespace_name' => namespace_name,
|
309
314
|
'pod_name' => pod_name
|
310
315
|
}
|
311
316
|
}
|
317
|
+
metadata['docker']['container_id'] = docker_id unless docker_id.nil?
|
318
|
+
container_cache_key = container_name
|
312
319
|
if present?(@kubernetes_url)
|
313
|
-
pod_metadata = get_pod_metadata(
|
314
|
-
|
315
|
-
|
316
|
-
metadata['kubernetes']['
|
317
|
-
metadata['
|
320
|
+
pod_metadata = get_pod_metadata(cache_key, namespace_name, pod_name, create_time, batch_miss_cache)
|
321
|
+
if (pod_metadata.include? 'containers') && (pod_metadata['containers'].include? container_cache_key) && !@skip_container_metadata
|
322
|
+
metadata['kubernetes']['container_image'] = pod_metadata['containers'][container_cache_key]['image']
|
323
|
+
metadata['kubernetes']['container_image_id'] = pod_metadata['containers'][container_cache_key]['image_id'] unless pod_metadata['containers'][container_cache_key]['image_id'].empty?
|
324
|
+
metadata['docker']['container_id'] = pod_metadata['containers'][container_cache_key]['containerID'] unless pod_metadata['containers'][container_cache_key]['containerID'].empty?
|
318
325
|
end
|
319
326
|
|
320
327
|
metadata['kubernetes'].merge!(pod_metadata) if pod_metadata
|
321
328
|
metadata['kubernetes'].delete('containers')
|
322
329
|
end
|
330
|
+
metadata.delete('docker') if metadata['docker'] && (metadata['docker']['container_id'].nil? || metadata['docker']['container_id'].empty?)
|
323
331
|
metadata
|
324
332
|
end
|
325
333
|
|
326
334
|
def filter_stream(tag, es)
|
327
335
|
return es if (es.respond_to?(:empty?) && es.empty?) || !es.is_a?(Fluent::EventStream)
|
328
|
-
|
329
336
|
new_es = Fluent::MultiEventStream.new
|
330
337
|
tag_match_data = tag.match(@tag_to_kubernetes_name_regexp_compiled) unless @use_journal
|
331
338
|
tag_metadata = nil
|
332
339
|
batch_miss_cache = {}
|
333
340
|
es.each do |time, record|
|
334
341
|
if tag_match_data && tag_metadata.nil?
|
342
|
+
cache_key = if tag_match_data.names.include?('pod_uuid') && !tag_match_data['pod_uuid'].nil?
|
343
|
+
tag_match_data['pod_uuid']
|
344
|
+
else
|
345
|
+
tag_match_data['docker_id']
|
346
|
+
end
|
347
|
+
docker_id = tag_match_data.names.include?('docker_id') ? tag_match_data['docker_id'] : nil
|
335
348
|
tag_metadata = get_metadata_for_record(tag_match_data['namespace'], tag_match_data['pod_name'], tag_match_data['container_name'],
|
336
|
-
|
349
|
+
cache_key, create_time_from_record(record, time), batch_miss_cache, docker_id)
|
337
350
|
end
|
338
351
|
metadata = Marshal.load(Marshal.dump(tag_metadata)) if tag_metadata
|
339
352
|
if (@use_journal || @use_journal.nil?) &&
|
@@ -348,10 +361,9 @@ module Fluent::Plugin
|
|
348
361
|
record['docker'].key?('container_id') &&
|
349
362
|
(k_metadata = get_metadata_for_record(record['kubernetes']['namespace_name'], record['kubernetes']['pod_name'],
|
350
363
|
record['kubernetes']['container_name'], record['docker']['container_id'],
|
351
|
-
create_time_from_record(record, time), batch_miss_cache))
|
364
|
+
create_time_from_record(record, time), batch_miss_cache, record['docker']['container_id']))
|
352
365
|
metadata = k_metadata
|
353
366
|
end
|
354
|
-
|
355
367
|
record = record.merge(metadata) if metadata
|
356
368
|
new_es.add(time, record)
|
357
369
|
end
|
@@ -364,7 +376,7 @@ module Fluent::Plugin
|
|
364
376
|
if record.key?('CONTAINER_NAME') && record.key?('CONTAINER_ID_FULL')
|
365
377
|
metadata = record['CONTAINER_NAME'].match(@container_name_to_kubernetes_regexp_compiled) do |match_data|
|
366
378
|
get_metadata_for_record(match_data['namespace'], match_data['pod_name'], match_data['container_name'],
|
367
|
-
|
379
|
+
record['CONTAINER_ID_FULL'], create_time_from_record(record, time), batch_miss_cache, record['CONTAINER_ID_FULL'])
|
368
380
|
end
|
369
381
|
unless metadata
|
370
382
|
log.debug "Error: could not match CONTAINER_NAME from record #{record}"
|
@@ -24,8 +24,6 @@ module KubernetesMetadata
|
|
24
24
|
metadata = {}
|
25
25
|
ids = @id_cache[key]
|
26
26
|
if ids.nil?
|
27
|
-
# FAST PATH
|
28
|
-
# Cache hit, fetch metadata from the cache
|
29
27
|
@stats.bump(:id_cache_miss)
|
30
28
|
return batch_miss_cache["#{namespace_name}_#{pod_name}"] if batch_miss_cache.key?("#{namespace_name}_#{pod_name}")
|
31
29
|
|
@@ -65,7 +63,7 @@ module KubernetesMetadata
|
|
65
63
|
@stats.bump(:id_cache_orphaned_record)
|
66
64
|
end
|
67
65
|
if @allow_orphans
|
68
|
-
log.trace("orphaning message for: #{namespace_name}/#{pod_name} ")
|
66
|
+
log.trace("orphaning message for: #{namespace_name}/#{pod_name} ")
|
69
67
|
metadata = {
|
70
68
|
'orphaned_namespace' => namespace_name,
|
71
69
|
'namespace_name' => @orphaned_namespace_name,
|
@@ -78,9 +78,9 @@ module KubernetesMetadata
|
|
78
78
|
container_meta = {}
|
79
79
|
begin
|
80
80
|
pod_object[:status][:containerStatuses].each do |container_status|
|
81
|
-
|
82
|
-
|
83
|
-
container_meta[
|
81
|
+
container_id = (container_status[:containerID]||"").sub(%r{^[-_a-zA-Z0-9]+://}, '')
|
82
|
+
key = container_status[:name]
|
83
|
+
container_meta[key] = if @skip_container_metadata
|
84
84
|
{
|
85
85
|
'name' => container_status[:name]
|
86
86
|
}
|
@@ -88,7 +88,8 @@ module KubernetesMetadata
|
|
88
88
|
{
|
89
89
|
'name' => container_status[:name],
|
90
90
|
'image' => container_status[:image],
|
91
|
-
'image_id' => container_status[:imageID]
|
91
|
+
'image_id' => container_status[:imageID],
|
92
|
+
:containerID => container_id
|
92
93
|
}
|
93
94
|
end
|
94
95
|
end if pod_object[:status] && pod_object[:status][:containerStatuses]
|
@@ -26,7 +26,7 @@ module KubernetesMetadata
|
|
26
26
|
def api_valid?
|
27
27
|
true
|
28
28
|
end
|
29
|
-
def get_namespace(namespace_name)
|
29
|
+
def get_namespace(namespace_name, unused, options)
|
30
30
|
return {
|
31
31
|
metadata: {
|
32
32
|
name: namespace_name,
|
@@ -38,7 +38,7 @@ module KubernetesMetadata
|
|
38
38
|
}
|
39
39
|
end
|
40
40
|
|
41
|
-
def get_pod(pod_name, namespace_name)
|
41
|
+
def get_pod(pod_name, namespace_name, options)
|
42
42
|
return {
|
43
43
|
metadata: {
|
44
44
|
name: pod_name,
|
data/release_notes.md
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
# Release Notes
|
2
|
+
|
3
|
+
## 2.9.4
|
4
|
+
As of this release, the 'de_dot' functionality is depricated and will be removed in future releases.
|
5
|
+
Ref: https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/issues/320
|
6
|
+
|
7
|
+
## v2.1.4
|
8
|
+
The use of `use_journal` is **DEPRECATED**. If this setting is not present, the plugin will
|
9
|
+
attempt to figure out the source of the metadata fields from the following:
|
10
|
+
- If `lookup_from_k8s_field true` (the default) and the following fields are present in the record:
|
11
|
+
`docker.container_id`, `kubernetes.namespace_name`, `kubernetes.pod_name`, `kubernetes.container_name`,
|
12
|
+
then the plugin will use those values as the source to use to lookup the metadata
|
13
|
+
- If `use_journal true`, or `use_journal` is unset, and the fields `CONTAINER_NAME` and `CONTAINER_ID_FULL` are present in the record,
|
14
|
+
then the plugin will parse those values using `container_name_to_kubernetes_regexp` and use those as the source to lookup the metadata
|
15
|
+
- Otherwise, if the tag matches `tag_to_kubernetes_name_regexp`, the plugin will parse the tag and use those values to
|
16
|
+
lookup the metdata
|
17
|
+
|
18
|
+
## v2.1.x
|
19
|
+
|
20
|
+
As of the release 2.1.x of this plugin, it no longer supports parsing the source message into JSON and attaching it to the
|
21
|
+
payload. The following configuration options are removed:
|
22
|
+
|
23
|
+
* `merge_json_log`
|
24
|
+
* `preserve_json_log`
|
25
|
+
|
26
|
+
One way of preserving JSON logs can be through the [parser plugin](https://docs.fluentd.org/filter/parser).
|
27
|
+
It can parsed with the parser plugin like this:
|
28
|
+
|
29
|
+
```
|
30
|
+
<filter kubernetes.**>
|
31
|
+
@type parser
|
32
|
+
key_name log
|
33
|
+
<parse>
|
34
|
+
@type json
|
35
|
+
json_parser json
|
36
|
+
</parse>
|
37
|
+
replace_invalid_sequence true
|
38
|
+
reserve_data true # this preserves unparsable log lines
|
39
|
+
emit_invalid_record_to_error false # In case of unparsable log lines keep the error log clean
|
40
|
+
reserve_time # the time was already parsed in the source, we don't want to overwrite it with current time.
|
41
|
+
</filter>
|
42
|
+
```
|
@@ -28,7 +28,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
28
28
|
@time = Fluent::Engine.now
|
29
29
|
end
|
30
30
|
|
31
|
-
|
31
|
+
VAR_LOG_CONTAINER_TAG = 'var.log.containers.fabric8-console-controller-98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log'
|
32
|
+
VAR_LOG_POD_TAG = 'var.log.pods.default_fabric8-console-controller-98rqc_c76927af-f563-11e4-b32d-54ee7527188d.fabric8-console-container.0.log'
|
32
33
|
|
33
34
|
def create_driver(conf = '')
|
34
35
|
Test::Driver::Filter.new(Plugin::KubernetesMetadataFilter).configure(conf)
|
@@ -45,7 +46,6 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
45
46
|
assert_equal('KubernetesMetadata::TestApiAdapter', d.instance.test_api_adapter)
|
46
47
|
end
|
47
48
|
|
48
|
-
|
49
49
|
test 'kubernetes url' do
|
50
50
|
VCR.use_cassette('valid_kubernetes_api_server') do
|
51
51
|
d = create_driver('
|
@@ -138,7 +138,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
138
138
|
cache_size 1
|
139
139
|
', d: nil)
|
140
140
|
d = create_driver(config) if d.nil?
|
141
|
-
d.run(default_tag:
|
141
|
+
d.run(default_tag: VAR_LOG_CONTAINER_TAG) do
|
142
142
|
d.feed(@time, msg)
|
143
143
|
end
|
144
144
|
d.filtered.map(&:last)
|
@@ -165,8 +165,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
165
165
|
plugin.filter_stream('tag', Fluent::MultiEventStream.new)
|
166
166
|
end
|
167
167
|
|
168
|
-
sub_test_case 'parsing_pod_metadata' do
|
169
|
-
test '
|
168
|
+
sub_test_case 'parsing_pod_metadata when container_status is missing from the pod status' do
|
169
|
+
test 'using the tag_to_kubernetes_name_regexp for /var/log/containers ' do
|
170
170
|
VCR.use_cassettes(
|
171
171
|
[
|
172
172
|
{ name: 'valid_kubernetes_api_server' },
|
@@ -174,16 +174,49 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
174
174
|
{ name: 'kubernetes_get_namespace_default' },
|
175
175
|
{ name: 'kubernetes_get_pod_container_init' }
|
176
176
|
]) do
|
177
|
-
filtered = emit({},
|
177
|
+
filtered = emit({}, "
|
178
178
|
kubernetes_url https://localhost:8443
|
179
179
|
watch false
|
180
180
|
cache_size 1
|
181
|
-
|
181
|
+
")
|
182
182
|
expected_kube_metadata = {
|
183
183
|
'docker' => {
|
184
184
|
'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
|
185
185
|
},
|
186
186
|
'kubernetes' => {
|
187
|
+
'container_image'=>'fabric8/hawtio-kubernetes:latest',
|
188
|
+
'container_name'=>'fabric8-console-container',
|
189
|
+
'host' => 'jimmi-redhat.localnet',
|
190
|
+
'pod_name' => 'fabric8-console-controller-98rqc',
|
191
|
+
'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
|
192
|
+
'namespace_name' => 'default',
|
193
|
+
'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
|
194
|
+
'pod_ip' => '172.17.0.8',
|
195
|
+
'master_url' => 'https://localhost:8443',
|
196
|
+
'labels' => {
|
197
|
+
'component' => 'fabric8Console'
|
198
|
+
}
|
199
|
+
}
|
200
|
+
}
|
201
|
+
assert_equal(expected_kube_metadata, filtered[0])
|
202
|
+
end
|
203
|
+
end
|
204
|
+
test 'using the tag_to_kubernetes_name_regexp for /var/log/pods' do
|
205
|
+
VCR.use_cassettes(
|
206
|
+
[
|
207
|
+
{ name: 'valid_kubernetes_api_server' },
|
208
|
+
{ name: 'kubernetes_get_api_v1' },
|
209
|
+
{ name: 'kubernetes_get_namespace_default' },
|
210
|
+
{ name: 'kubernetes_get_pod_container_init' }
|
211
|
+
]) do
|
212
|
+
filtered = emit_with_tag(VAR_LOG_POD_TAG,{}, "
|
213
|
+
kubernetes_url https://localhost:8443
|
214
|
+
watch false
|
215
|
+
cache_size 1
|
216
|
+
")
|
217
|
+
expected_kube_metadata = {
|
218
|
+
'kubernetes' => {
|
219
|
+
'container_image'=>'fabric8/hawtio-kubernetes:latest',
|
187
220
|
'container_name'=>'fabric8-console-container',
|
188
221
|
'host' => 'jimmi-redhat.localnet',
|
189
222
|
'pod_name' => 'fabric8-console-controller-98rqc',
|
@@ -214,8 +247,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
214
247
|
filtered = emit({ 'time' => '2015-05-08T09:22:01Z' }, '', d: driver)
|
215
248
|
expected_kube_metadata = {
|
216
249
|
'time' => '2015-05-08T09:22:01Z',
|
217
|
-
'docker'
|
218
|
-
'container_id'
|
250
|
+
'docker'=>{
|
251
|
+
'container_id'=>'49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
|
219
252
|
},
|
220
253
|
'kubernetes' => {
|
221
254
|
'pod_name' => 'fabric8-console-controller-98rqc',
|
@@ -302,6 +335,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
302
335
|
end
|
303
336
|
|
304
337
|
test 'with docker & kubernetes metadata' do
|
338
|
+
|
305
339
|
VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' }, { name: 'kubernetes_get_pod' }, { name: 'kubernetes_get_namespace_default' }]) do
|
306
340
|
filtered = emit({ 'time' => '2015-05-08T09:22:01Z' })
|
307
341
|
expected_kube_metadata = {
|
@@ -398,8 +432,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
398
432
|
test 'with docker & kubernetes metadata but no configured api server' do
|
399
433
|
filtered = emit({}, '')
|
400
434
|
expected_kube_metadata = {
|
401
|
-
'docker'
|
402
|
-
'container_id'
|
435
|
+
'docker'=>{
|
436
|
+
'container_id'=>'49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
|
403
437
|
},
|
404
438
|
'kubernetes' => {
|
405
439
|
'pod_name' => 'fabric8-console-controller-98rqc',
|
@@ -420,8 +454,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
420
454
|
stub_request(:any, 'https://localhost:8443/api/v1/namespaces/default').to_timeout
|
421
455
|
filtered = emit
|
422
456
|
expected_kube_metadata = {
|
423
|
-
'docker'
|
424
|
-
'container_id'
|
457
|
+
'docker'=>{
|
458
|
+
'container_id'=>'49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
|
425
459
|
},
|
426
460
|
'kubernetes' => {
|
427
461
|
'pod_name' => 'fabric8-console-controller-98rqc',
|
@@ -443,8 +477,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
443
477
|
stub_request(:any, 'https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller.98rqc').to_timeout
|
444
478
|
filtered = emit_with_tag('var.log.containers.fabric8-console-controller.98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log', {}, '')
|
445
479
|
expected_kube_metadata = {
|
446
|
-
'docker'
|
447
|
-
'container_id'
|
480
|
+
'docker'=>{
|
481
|
+
'container_id'=>'49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
|
448
482
|
},
|
449
483
|
'kubernetes' => {
|
450
484
|
'pod_name' => 'fabric8-console-controller.98rqc',
|
@@ -736,7 +770,6 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
736
770
|
|
737
771
|
test 'uses metadata from tag if use_journal false and lookup_from_k8s_field false' do
|
738
772
|
# with use_journal unset, should still use the journal fields instead of tag fields
|
739
|
-
tag = 'var.log.containers.fabric8-console-controller-98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log'
|
740
773
|
msg = {
|
741
774
|
'CONTAINER_NAME' => 'k8s_journald-container-name.db89db89_journald-pod-name_journald-namespace-name_c76927af-f563-11e4-b32d-54ee7527188d_89db89db',
|
742
775
|
'CONTAINER_ID_FULL' => '838350c64bacba968d39a30a50789b2795291fceca6ccbff55298671d46b0e3b',
|
@@ -751,13 +784,13 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
751
784
|
VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' }, { name: 'kubernetes_get_pod' },
|
752
785
|
{ name: 'kubernetes_get_namespace_default', options: { allow_playback_repeats: true } },
|
753
786
|
{ name: 'metadata_from_tag_and_journald_fields' }]) do
|
754
|
-
es = emit_with_tag(
|
787
|
+
es = emit_with_tag(VAR_LOG_CONTAINER_TAG, msg, "
|
755
788
|
kubernetes_url https://localhost:8443
|
756
789
|
watch false
|
757
790
|
cache_size 1
|
758
791
|
lookup_from_k8s_field false
|
759
792
|
use_journal false
|
760
|
-
|
793
|
+
")
|
761
794
|
expected_kube_metadata = {
|
762
795
|
'docker' => {
|
763
796
|
'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
|
@@ -991,7 +1024,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
991
1024
|
end
|
992
1025
|
|
993
1026
|
test 'processes all events when reading from MessagePackEventStream' do
|
994
|
-
VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' },
|
1027
|
+
VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' },
|
1028
|
+
{ name: 'kubernetes_get_api_v1' },
|
995
1029
|
{ name: 'kubernetes_get_pod' },
|
996
1030
|
{ name: 'kubernetes_get_namespace_default' }]) do
|
997
1031
|
entries = [[@time, { 'time' => '2015-05-08T09:22:01Z' }], [@time, { 'time' => '2015-05-08T09:22:01Z' }]]
|
@@ -1004,7 +1038,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
1004
1038
|
cache_size 1
|
1005
1039
|
')
|
1006
1040
|
d.run do
|
1007
|
-
d.feed(
|
1041
|
+
d.feed(VAR_LOG_CONTAINER_TAG, msgpack_stream)
|
1008
1042
|
end
|
1009
1043
|
filtered = d.filtered.map(&:last)
|
1010
1044
|
|
@@ -1048,8 +1082,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
1048
1082
|
skip_namespace_metadata true
|
1049
1083
|
')
|
1050
1084
|
expected_kube_metadata = {
|
1051
|
-
'docker'
|
1052
|
-
'container_id'
|
1085
|
+
'docker'=>{
|
1086
|
+
'container_id'=>'49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
|
1053
1087
|
},
|
1054
1088
|
'kubernetes' => {
|
1055
1089
|
'host' => 'jimmi-redhat.localnet',
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kubernetes_metadata_filter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.9.
|
4
|
+
version: 2.9.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jimmi Dyson
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-03-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -228,6 +228,7 @@ files:
|
|
228
228
|
- lib/fluent/plugin/kubernetes_metadata_util.rb
|
229
229
|
- lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb
|
230
230
|
- lib/fluent/plugin/kubernetes_metadata_watch_pods.rb
|
231
|
+
- release_notes.md
|
231
232
|
- test/cassettes/invalid_api_server_config.yml
|
232
233
|
- test/cassettes/kubernetes_docker_metadata_annotations.yml
|
233
234
|
- test/cassettes/kubernetes_docker_metadata_dotted_slashed_labels.yml
|
@@ -270,7 +271,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
270
271
|
- !ruby/object:Gem::Version
|
271
272
|
version: '0'
|
272
273
|
requirements: []
|
273
|
-
rubygems_version: 3.
|
274
|
+
rubygems_version: 3.2.3
|
274
275
|
signing_key:
|
275
276
|
specification_version: 4
|
276
277
|
summary: Fluentd filter plugin to add Kubernetes metadata
|