fluent-plugin-kubernetes_metadata_filter 2.7.0 → 2.9.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +1 -1
- data/Gemfile.lock +37 -37
- data/README.md +6 -36
- data/fluent-plugin-kubernetes_metadata_filter.gemspec +3 -3
- data/lib/fluent/plugin/filter_kubernetes_metadata.rb +53 -25
- data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb +1 -3
- data/lib/fluent/plugin/kubernetes_metadata_common.rb +16 -7
- data/release_notes.md +42 -0
- data/test/cassettes/{kubernetes_docker_metadata_dotted_labels.yml → kubernetes_docker_metadata_dotted_slashed_labels.yml} +0 -0
- data/test/cassettes/kubernetes_get_pod_container_init.yml +145 -0
- data/test/plugin/test_cache_strategy.rb +1 -1
- data/test/plugin/test_filter_kubernetes_metadata.rb +108 -24
- data/test/plugin/watch_test.rb +2 -0
- metadata +16 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 98c4002df3754262c547e0b22acf366e62bca72f0142a20430f6a0f6c91dadea
|
4
|
+
data.tar.gz: 3f9e5d2d19b505c7cad15d113b69dea7ccb0943a3d4605d121fdf2234952c4a7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5474457d9a35d7951a8a380c0099deeb16af6d849a9f036c467ba47e0fcc47ce2ff7f77bbd75ad7d6487c9f172897e63406a91fd9f08dc6494eb32332e2b9373
|
7
|
+
data.tar.gz: 91a14c8edabcdd1eaa7b915c14bfd643b7829ae0cc8951ed8f9e77819483638207a5c6264094974971dcf4d7684c50c62a249cee4de36d23574b72cbe4e10589
|
data/.circleci/config.yml
CHANGED
@@ -12,7 +12,7 @@ missingdeps: &missingdeps
|
|
12
12
|
cat /etc/os-release
|
13
13
|
printf "deb http://deb.debian.org/debian buster main\ndeb http://security.debian.org buster/updates main\ndeb-src http://security.debian.org buster/updates main" > /tmp/sources.list
|
14
14
|
sudo cp /tmp/sources.list /etc/apt/sources.list
|
15
|
-
sudo apt-get update
|
15
|
+
sudo apt-get --allow-releaseinfo-change update
|
16
16
|
sudo apt-get install cmake libicu-dev libssl-dev
|
17
17
|
|
18
18
|
test: &test
|
data/Gemfile.lock
CHANGED
@@ -1,48 +1,48 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
fluent-plugin-kubernetes_metadata_filter (2.
|
5
|
-
fluentd (>= 0.14.0, < 1.
|
6
|
-
kubeclient (< 5)
|
4
|
+
fluent-plugin-kubernetes_metadata_filter (2.9.4)
|
5
|
+
fluentd (>= 0.14.0, < 1.15)
|
6
|
+
kubeclient (>= 4.0.0, < 5.0.0)
|
7
7
|
lru_redux
|
8
8
|
|
9
9
|
GEM
|
10
10
|
remote: https://rubygems.org/
|
11
11
|
specs:
|
12
|
-
addressable (2.
|
12
|
+
addressable (2.8.0)
|
13
13
|
public_suffix (>= 2.0.2, < 5.0)
|
14
|
-
ast (2.4.
|
14
|
+
ast (2.4.2)
|
15
15
|
bump (0.10.0)
|
16
16
|
charlock_holmes (0.7.7)
|
17
17
|
codeclimate-test-reporter (0.6.0)
|
18
18
|
simplecov (>= 0.7.1, < 1.0.0)
|
19
|
-
concurrent-ruby (1.1.
|
19
|
+
concurrent-ruby (1.1.9)
|
20
20
|
cool.io (1.7.1)
|
21
21
|
copyright-header (1.0.22)
|
22
22
|
github-linguist
|
23
23
|
crack (0.4.5)
|
24
24
|
rexml
|
25
|
-
docile (1.
|
25
|
+
docile (1.4.0)
|
26
26
|
domain_name (0.5.20190701)
|
27
27
|
unf (>= 0.0.5, < 1.0.0)
|
28
28
|
escape_utils (1.2.1)
|
29
|
-
ffi (1.15.
|
29
|
+
ffi (1.15.4)
|
30
30
|
ffi-compiler (1.0.1)
|
31
31
|
ffi (>= 1.0.0)
|
32
32
|
rake
|
33
|
-
fluentd (1.
|
33
|
+
fluentd (1.14.3)
|
34
34
|
bundler
|
35
35
|
cool.io (>= 1.4.5, < 2.0.0)
|
36
|
-
http_parser.rb (>= 0.5.1, < 0.
|
36
|
+
http_parser.rb (>= 0.5.1, < 0.9.0)
|
37
37
|
msgpack (>= 1.3.1, < 2.0.0)
|
38
38
|
serverengine (>= 2.2.2, < 3.0.0)
|
39
39
|
sigdump (~> 0.2.2)
|
40
|
-
strptime (>= 0.2.
|
40
|
+
strptime (>= 0.2.4, < 1.0.0)
|
41
41
|
tzinfo (>= 1.0, < 3.0)
|
42
42
|
tzinfo-data (~> 1.0)
|
43
43
|
webrick (>= 1.4.2, < 1.8.0)
|
44
44
|
yajl-ruby (~> 1.0)
|
45
|
-
github-linguist (7.
|
45
|
+
github-linguist (7.17.0)
|
46
46
|
charlock_holmes (~> 0.7.7)
|
47
47
|
escape_utils (~> 1.2.0)
|
48
48
|
mini_mime (~> 1.0)
|
@@ -54,58 +54,58 @@ GEM
|
|
54
54
|
http-form_data (~> 2.2)
|
55
55
|
http-parser (~> 1.2.0)
|
56
56
|
http-accept (1.7.0)
|
57
|
-
http-cookie (1.0.
|
57
|
+
http-cookie (1.0.4)
|
58
58
|
domain_name (~> 0.5)
|
59
59
|
http-form_data (2.3.0)
|
60
60
|
http-parser (1.2.3)
|
61
61
|
ffi-compiler (>= 1.0, < 2.0)
|
62
|
-
http_parser.rb (0.
|
62
|
+
http_parser.rb (0.8.0)
|
63
63
|
jsonpath (1.1.0)
|
64
64
|
multi_json
|
65
|
-
kubeclient (4.9.
|
65
|
+
kubeclient (4.9.2)
|
66
66
|
http (>= 3.0, < 5.0)
|
67
67
|
jsonpath (~> 1.0)
|
68
68
|
recursive-open-struct (~> 1.1, >= 1.1.1)
|
69
69
|
rest-client (~> 2.0)
|
70
70
|
lru_redux (1.1.0)
|
71
|
-
mime-types (3.
|
71
|
+
mime-types (3.4.1)
|
72
72
|
mime-types-data (~> 3.2015)
|
73
|
-
mime-types-data (3.2021.
|
74
|
-
mini_mime (1.
|
73
|
+
mime-types-data (3.2021.1115)
|
74
|
+
mini_mime (1.1.2)
|
75
75
|
minitest (4.7.5)
|
76
76
|
msgpack (1.4.2)
|
77
77
|
multi_json (1.15.0)
|
78
78
|
netrc (0.11.0)
|
79
|
-
parallel (1.
|
80
|
-
parser (3.0.
|
79
|
+
parallel (1.21.0)
|
80
|
+
parser (3.0.2.0)
|
81
81
|
ast (~> 2.4.1)
|
82
|
-
power_assert (
|
82
|
+
power_assert (2.0.1)
|
83
83
|
public_suffix (4.0.6)
|
84
84
|
rainbow (3.0.0)
|
85
|
-
rake (13.0.
|
85
|
+
rake (13.0.6)
|
86
86
|
recursive-open-struct (1.1.3)
|
87
|
-
regexp_parser (2.
|
87
|
+
regexp_parser (2.1.1)
|
88
88
|
rest-client (2.1.0)
|
89
89
|
http-accept (>= 1.7.0, < 2.0)
|
90
90
|
http-cookie (>= 1.0.2, < 2.0)
|
91
91
|
mime-types (>= 1.16, < 4.0)
|
92
92
|
netrc (~> 0.8)
|
93
93
|
rexml (3.2.5)
|
94
|
-
rr (
|
95
|
-
rubocop (1.
|
94
|
+
rr (3.0.8)
|
95
|
+
rubocop (1.22.3)
|
96
96
|
parallel (~> 1.10)
|
97
97
|
parser (>= 3.0.0.0)
|
98
98
|
rainbow (>= 2.2.2, < 4.0)
|
99
99
|
regexp_parser (>= 1.8, < 3.0)
|
100
100
|
rexml
|
101
|
-
rubocop-ast (>= 1.
|
101
|
+
rubocop-ast (>= 1.12.0, < 2.0)
|
102
102
|
ruby-progressbar (~> 1.7)
|
103
103
|
unicode-display_width (>= 1.4.0, < 3.0)
|
104
|
-
rubocop-ast (1.
|
105
|
-
parser (>=
|
104
|
+
rubocop-ast (1.12.0)
|
105
|
+
parser (>= 3.0.1.1)
|
106
106
|
ruby-progressbar (1.11.0)
|
107
|
-
rugged (1.
|
108
|
-
serverengine (2.2.
|
107
|
+
rugged (1.2.0)
|
108
|
+
serverengine (2.2.4)
|
109
109
|
sigdump (~> 0.2.2)
|
110
110
|
sigdump (0.2.4)
|
111
111
|
simplecov (0.21.2)
|
@@ -113,7 +113,7 @@ GEM
|
|
113
113
|
simplecov-html (~> 0.11)
|
114
114
|
simplecov_json_formatter (~> 0.1)
|
115
115
|
simplecov-html (0.12.3)
|
116
|
-
simplecov_json_formatter (0.1.
|
116
|
+
simplecov_json_formatter (0.1.3)
|
117
117
|
strptime (0.2.5)
|
118
118
|
test-unit (3.0.9)
|
119
119
|
power_assert
|
@@ -122,15 +122,15 @@ GEM
|
|
122
122
|
test-unit (>= 2.5.2)
|
123
123
|
tzinfo (2.0.4)
|
124
124
|
concurrent-ruby (~> 1.0)
|
125
|
-
tzinfo-data (1.2021.
|
125
|
+
tzinfo-data (1.2021.5)
|
126
126
|
tzinfo (>= 1.0.0)
|
127
127
|
unf (0.1.4)
|
128
128
|
unf_ext
|
129
|
-
unf_ext (0.0.
|
130
|
-
unicode-display_width (2.
|
129
|
+
unf_ext (0.0.8)
|
130
|
+
unicode-display_width (2.1.0)
|
131
131
|
vcr (6.0.0)
|
132
|
-
webmock (3.
|
133
|
-
addressable (>= 2.
|
132
|
+
webmock (3.14.0)
|
133
|
+
addressable (>= 2.8.0)
|
134
134
|
crack (>= 0.3.2)
|
135
135
|
hashdiff (>= 0.4.0, < 2.0.0)
|
136
136
|
webrick (1.7.0)
|
@@ -155,4 +155,4 @@ DEPENDENCIES
|
|
155
155
|
yajl-ruby
|
156
156
|
|
157
157
|
BUNDLED WITH
|
158
|
-
2.
|
158
|
+
2.3.4
|
data/README.md
CHANGED
@@ -41,12 +41,14 @@ Configuration options for fluent.conf are:
|
|
41
41
|
* `client_key` - path to a client key file to authenticate to the API server
|
42
42
|
* `bearer_token_file` - path to a file containing the bearer token to use for authentication
|
43
43
|
* `tag_to_kubernetes_name_regexp` - the regular expression used to extract kubernetes metadata (pod name, container name, namespace) from the current fluentd tag.
|
44
|
-
This must
|
44
|
+
This must use named capture groups for `container_name`, `pod_name`, `namespace`, and either `pod_uuid (/var/log/pods)` or `docker_id (/var/log/containers)`
|
45
45
|
* `cache_size` - size of the cache of Kubernetes metadata to reduce requests to the API server (default: `1000`)
|
46
46
|
* `cache_ttl` - TTL in seconds of each cached element. Set to negative value to disable TTL eviction (default: `3600` - 1 hour)
|
47
47
|
* `watch` - set up a watch on pods on the API server for updates to metadata (default: `true`)
|
48
|
-
*
|
49
|
-
*
|
48
|
+
* *DEPRECATED*`de_dot` - replace dots in labels and annotations with configured `de_dot_separator`, required for Datadog and ElasticSearch 2.x compatibility (default: `true`)
|
49
|
+
* *DEPRECATED*`de_dot_separator` - separator to use if `de_dot` is enabled (default: `_`)
|
50
|
+
* *DEPRECATED*`de_slash` - replace slashes in labels and annotations with configured `de_slash_separator`, required for Datadog compatibility (default: `false`)
|
51
|
+
* *DEPRECATED*`de_slash_separator` - separator to use if `de_slash` is enabled (default: `__`)
|
50
52
|
* *DEPRECATED* `use_journal` - If false, messages are expected to be formatted and tagged as if read by the fluentd in\_tail plugin with wildcard filename. If true, messages are expected to be formatted as if read from the systemd journal. The `MESSAGE` field has the full message. The `CONTAINER_NAME` field has the encoded k8s metadata (see below). The `CONTAINER_ID_FULL` field has the full container uuid. This requires docker to use the `--log-driver=journald` log driver. If unset (the default), the plugin will use the `CONTAINER_NAME` and `CONTAINER_ID_FULL` fields
|
51
53
|
if available, otherwise, will use the tag in the `tag_to_kubernetes_name_regexp` format.
|
52
54
|
* `container_name_to_kubernetes_regexp` - The regular expression used to extract the k8s metadata encoded in the journal `CONTAINER_NAME` field default: See [code](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/blob/master/lib/fluent/plugin/filter_kubernetes_metadata.rb#L68)
|
@@ -66,23 +68,6 @@ when true (default: `true`)
|
|
66
68
|
* `skip_namespace_metadata` - Skip the namespace_id field from the metadata. The fetch_namespace_metadata function will be skipped. The plugin will be faster and cpu consumption will be less.
|
67
69
|
* `watch_retry_interval` - The time interval in seconds for retry backoffs when watch connections fail. (default: `10`)
|
68
70
|
|
69
|
-
**NOTE:** As of the release 2.1.x of this plugin, it no longer supports parsing the source message into JSON and attaching it to the
|
70
|
-
payload. The following configuration options are removed:
|
71
|
-
|
72
|
-
* `merge_json_log`
|
73
|
-
* `preserve_json_log`
|
74
|
-
|
75
|
-
One way of preserving JSON logs can be through the [parser plugin](https://docs.fluentd.org/filter/parser)
|
76
|
-
|
77
|
-
**NOTE** As of this release, the use of `use_journal` is **DEPRECATED**. If this setting is not present, the plugin will
|
78
|
-
attempt to figure out the source of the metadata fields from the following:
|
79
|
-
- If `lookup_from_k8s_field true` (the default) and the following fields are present in the record:
|
80
|
-
`docker.container_id`, `kubernetes.namespace_name`, `kubernetes.pod_name`, `kubernetes.container_name`,
|
81
|
-
then the plugin will use those values as the source to use to lookup the metadata
|
82
|
-
- If `use_journal true`, or `use_journal` is unset, and the fields `CONTAINER_NAME` and `CONTAINER_ID_FULL` are present in the record,
|
83
|
-
then the plugin will parse those values using `container_name_to_kubernetes_regexp` and use those as the source to lookup the metadata
|
84
|
-
- Otherwise, if the tag matches `tag_to_kubernetes_name_regexp`, the plugin will parse the tag and use those values to
|
85
|
-
lookup the metdata
|
86
71
|
|
87
72
|
Reading from the JSON formatted log files with `in_tail` and wildcard filenames while respecting the CRI-o log format with the same config you need the fluent-plugin "multi-format-parser":
|
88
73
|
|
@@ -152,22 +137,7 @@ Reading from the systemd journal (requires the fluentd `fluent-plugin-systemd` a
|
|
152
137
|
@type stdout
|
153
138
|
</match>
|
154
139
|
```
|
155
|
-
|
156
|
-
In former versions this plugin parsed the value of the key log as JSON. In the current version this feature was removed, to avoid duplicate features in the fluentd plugin ecosystem. It can parsed with the parser plugin like this:
|
157
|
-
```
|
158
|
-
<filter kubernetes.**>
|
159
|
-
@type parser
|
160
|
-
key_name log
|
161
|
-
<parse>
|
162
|
-
@type json
|
163
|
-
json_parser json
|
164
|
-
</parse>
|
165
|
-
replace_invalid_sequence true
|
166
|
-
reserve_data true # this preserves unparsable log lines
|
167
|
-
emit_invalid_record_to_error false # In case of unparsable log lines keep the error log clean
|
168
|
-
reserve_time # the time was already parsed in the source, we don't want to overwrite it with current time.
|
169
|
-
</filter>
|
170
|
-
```
|
140
|
+
|
171
141
|
|
172
142
|
## Environment variables for Kubernetes
|
173
143
|
|
@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |gem|
|
7
7
|
gem.name = 'fluent-plugin-kubernetes_metadata_filter'
|
8
|
-
gem.version = '2.
|
8
|
+
gem.version = '2.9.4'
|
9
9
|
gem.authors = ['Jimmi Dyson']
|
10
10
|
gem.email = ['jimmidyson@gmail.com']
|
11
11
|
gem.description = 'Filter plugin to add Kubernetes metadata'
|
@@ -17,8 +17,8 @@ Gem::Specification.new do |gem|
|
|
17
17
|
|
18
18
|
gem.required_ruby_version = '>= 2.5.0'
|
19
19
|
|
20
|
-
gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 1.
|
21
|
-
gem.add_runtime_dependency 'kubeclient', '< 5'
|
20
|
+
gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 1.15']
|
21
|
+
gem.add_runtime_dependency 'kubeclient', ['>= 4.0.0', '< 5.0.0']
|
22
22
|
gem.add_runtime_dependency 'lru_redux'
|
23
23
|
|
24
24
|
gem.add_development_dependency 'bump'
|
@@ -51,13 +51,23 @@ module Fluent::Plugin
|
|
51
51
|
config_param :client_key, :string, default: nil
|
52
52
|
config_param :ca_file, :string, default: nil
|
53
53
|
config_param :verify_ssl, :bool, default: true
|
54
|
-
|
55
|
-
|
56
|
-
|
54
|
+
|
55
|
+
REGEX_VAR_LOG_PODS = '(var\.log\.pods)\.(?<namespace>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<pod_uuid>[a-z0-9-]*)\.(?<container_name>.+)\..*\.log$'
|
56
|
+
REGEX_VAR_LOG_CONTAINERS = '(var\.log\.containers)\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
|
57
|
+
|
58
|
+
#tag_to_kubernetes_name_regexp which must include named capture groups:
|
59
|
+
# namespace - The namespace in which the pod is deployed
|
60
|
+
# pod_name - The pod name
|
61
|
+
# container_name - The name of the container
|
62
|
+
# pod_uuid (/var/log/pods) | docker_id (/var/log/containers) - Unique identifier used in caching of either pod_uuid or the container hash
|
63
|
+
config_param :tag_to_kubernetes_name_regexp, :string, default: "(#{REGEX_VAR_LOG_PODS}|#{REGEX_VAR_LOG_CONTAINERS})"
|
64
|
+
|
57
65
|
config_param :bearer_token_file, :string, default: nil
|
58
66
|
config_param :secret_dir, :string, default: '/var/run/secrets/kubernetes.io/serviceaccount'
|
59
67
|
config_param :de_dot, :bool, default: true
|
60
68
|
config_param :de_dot_separator, :string, default: '_'
|
69
|
+
config_param :de_slash, :bool, default: false
|
70
|
+
config_param :de_slash_separator, :string, default: '__'
|
61
71
|
# if reading from the journal, the record will contain the following fields in the following
|
62
72
|
# format:
|
63
73
|
# CONTAINER_NAME=k8s_$containername.$containerhash_$podname_$namespacename_$poduuid_$rand32bitashex
|
@@ -98,15 +108,15 @@ module Fluent::Plugin
|
|
98
108
|
config_param :watch_retry_max_times, :integer, default: 10
|
99
109
|
|
100
110
|
def fetch_pod_metadata(namespace_name, pod_name)
|
101
|
-
log.trace("fetching pod metadata: #{namespace_name}/#{pod_name}")
|
111
|
+
log.trace("fetching pod metadata: #{namespace_name}/#{pod_name}")
|
102
112
|
options = {
|
103
113
|
resource_version: '0' # Fetch from API server cache instead of etcd quorum read
|
104
114
|
}
|
105
115
|
pod_object = @client.get_pod(pod_name, namespace_name, options)
|
106
|
-
log.trace("raw metadata for #{namespace_name}/#{pod_name}: #{pod_object}")
|
116
|
+
log.trace("raw metadata for #{namespace_name}/#{pod_name}: #{pod_object}")
|
107
117
|
metadata = parse_pod_metadata(pod_object)
|
108
118
|
@stats.bump(:pod_cache_api_updates)
|
109
|
-
log.trace("parsed metadata for #{namespace_name}/#{pod_name}: #{metadata}")
|
119
|
+
log.trace("parsed metadata for #{namespace_name}/#{pod_name}: #{metadata}")
|
110
120
|
@cache[metadata['pod_id']] = metadata
|
111
121
|
rescue StandardError => e
|
112
122
|
@stats.bump(:pod_cache_api_nil_error)
|
@@ -130,15 +140,15 @@ module Fluent::Plugin
|
|
130
140
|
end
|
131
141
|
|
132
142
|
def fetch_namespace_metadata(namespace_name)
|
133
|
-
log.trace("fetching namespace metadata: #{namespace_name}")
|
143
|
+
log.trace("fetching namespace metadata: #{namespace_name}")
|
134
144
|
options = {
|
135
145
|
resource_version: '0' # Fetch from API server cache instead of etcd quorum read
|
136
146
|
}
|
137
147
|
namespace_object = @client.get_namespace(namespace_name, nil, options)
|
138
|
-
log.trace("raw metadata for #{namespace_name}: #{namespace_object}")
|
148
|
+
log.trace("raw metadata for #{namespace_name}: #{namespace_object}")
|
139
149
|
metadata = parse_namespace_metadata(namespace_object)
|
140
150
|
@stats.bump(:namespace_cache_api_updates)
|
141
|
-
log.trace("parsed metadata for #{namespace_name}: #{metadata}")
|
151
|
+
log.trace("parsed metadata for #{namespace_name}: #{metadata}")
|
142
152
|
@namespace_cache[metadata['namespace_id']] = metadata
|
143
153
|
rescue StandardError => e
|
144
154
|
@stats.bump(:namespace_cache_api_nil_error)
|
@@ -154,10 +164,6 @@ module Fluent::Plugin
|
|
154
164
|
def configure(conf)
|
155
165
|
super
|
156
166
|
|
157
|
-
def log.trace?
|
158
|
-
level == Fluent::Log::LEVEL_TRACE
|
159
|
-
end
|
160
|
-
|
161
167
|
require 'kubeclient'
|
162
168
|
require 'lru_redux'
|
163
169
|
@stats = KubernetesMetadata::Stats.new
|
@@ -166,6 +172,10 @@ module Fluent::Plugin
|
|
166
172
|
raise Fluent::ConfigError, "Invalid de_dot_separator: cannot be or contain '.'"
|
167
173
|
end
|
168
174
|
|
175
|
+
if @de_slash && @de_slash_separator.include?('/')
|
176
|
+
raise Fluent::ConfigError, "Invalid de_slash_separator: cannot be or contain '/'"
|
177
|
+
end
|
178
|
+
|
169
179
|
if @cache_ttl < 0
|
170
180
|
log.info 'Setting the cache TTL to :none because it was <= 0'
|
171
181
|
@cache_ttl = :none
|
@@ -181,6 +191,7 @@ module Fluent::Plugin
|
|
181
191
|
@namespace_cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl)
|
182
192
|
|
183
193
|
@tag_to_kubernetes_name_regexp_compiled = Regexp.compile(@tag_to_kubernetes_name_regexp)
|
194
|
+
|
184
195
|
@container_name_to_kubernetes_regexp_compiled = Regexp.compile(@container_name_to_kubernetes_regexp)
|
185
196
|
|
186
197
|
# Use Kubernetes default service account if we're in a pod.
|
@@ -294,40 +305,48 @@ module Fluent::Plugin
|
|
294
305
|
end
|
295
306
|
end
|
296
307
|
|
297
|
-
def get_metadata_for_record(namespace_name, pod_name, container_name,
|
308
|
+
def get_metadata_for_record(namespace_name, pod_name, container_name, cache_key, create_time, batch_miss_cache, docker_id)
|
298
309
|
metadata = {
|
299
|
-
'docker' => { 'container_id' =>
|
310
|
+
'docker' => { 'container_id' => "" },
|
300
311
|
'kubernetes' => {
|
301
312
|
'container_name' => container_name,
|
302
313
|
'namespace_name' => namespace_name,
|
303
314
|
'pod_name' => pod_name
|
304
315
|
}
|
305
316
|
}
|
317
|
+
metadata['docker']['container_id'] = docker_id unless docker_id.nil?
|
318
|
+
container_cache_key = container_name
|
306
319
|
if present?(@kubernetes_url)
|
307
|
-
pod_metadata = get_pod_metadata(
|
308
|
-
|
309
|
-
|
310
|
-
metadata['kubernetes']['
|
311
|
-
metadata['
|
320
|
+
pod_metadata = get_pod_metadata(cache_key, namespace_name, pod_name, create_time, batch_miss_cache)
|
321
|
+
if (pod_metadata.include? 'containers') && (pod_metadata['containers'].include? container_cache_key) && !@skip_container_metadata
|
322
|
+
metadata['kubernetes']['container_image'] = pod_metadata['containers'][container_cache_key]['image']
|
323
|
+
metadata['kubernetes']['container_image_id'] = pod_metadata['containers'][container_cache_key]['image_id'] unless pod_metadata['containers'][container_cache_key]['image_id'].empty?
|
324
|
+
metadata['docker']['container_id'] = pod_metadata['containers'][container_cache_key]['containerID'] unless pod_metadata['containers'][container_cache_key]['containerID'].empty?
|
312
325
|
end
|
313
326
|
|
314
327
|
metadata['kubernetes'].merge!(pod_metadata) if pod_metadata
|
315
328
|
metadata['kubernetes'].delete('containers')
|
316
329
|
end
|
330
|
+
metadata.delete('docker') if metadata['docker'] && (metadata['docker']['container_id'].nil? || metadata['docker']['container_id'].empty?)
|
317
331
|
metadata
|
318
332
|
end
|
319
333
|
|
320
334
|
def filter_stream(tag, es)
|
321
335
|
return es if (es.respond_to?(:empty?) && es.empty?) || !es.is_a?(Fluent::EventStream)
|
322
|
-
|
323
336
|
new_es = Fluent::MultiEventStream.new
|
324
337
|
tag_match_data = tag.match(@tag_to_kubernetes_name_regexp_compiled) unless @use_journal
|
325
338
|
tag_metadata = nil
|
326
339
|
batch_miss_cache = {}
|
327
340
|
es.each do |time, record|
|
328
341
|
if tag_match_data && tag_metadata.nil?
|
342
|
+
cache_key = if tag_match_data.names.include?('pod_uuid') && !tag_match_data['pod_uuid'].nil?
|
343
|
+
tag_match_data['pod_uuid']
|
344
|
+
else
|
345
|
+
tag_match_data['docker_id']
|
346
|
+
end
|
347
|
+
docker_id = tag_match_data.names.include?('docker_id') ? tag_match_data['docker_id'] : nil
|
329
348
|
tag_metadata = get_metadata_for_record(tag_match_data['namespace'], tag_match_data['pod_name'], tag_match_data['container_name'],
|
330
|
-
|
349
|
+
cache_key, create_time_from_record(record, time), batch_miss_cache, docker_id)
|
331
350
|
end
|
332
351
|
metadata = Marshal.load(Marshal.dump(tag_metadata)) if tag_metadata
|
333
352
|
if (@use_journal || @use_journal.nil?) &&
|
@@ -342,10 +361,9 @@ module Fluent::Plugin
|
|
342
361
|
record['docker'].key?('container_id') &&
|
343
362
|
(k_metadata = get_metadata_for_record(record['kubernetes']['namespace_name'], record['kubernetes']['pod_name'],
|
344
363
|
record['kubernetes']['container_name'], record['docker']['container_id'],
|
345
|
-
create_time_from_record(record, time), batch_miss_cache))
|
364
|
+
create_time_from_record(record, time), batch_miss_cache, record['docker']['container_id']))
|
346
365
|
metadata = k_metadata
|
347
366
|
end
|
348
|
-
|
349
367
|
record = record.merge(metadata) if metadata
|
350
368
|
new_es.add(time, record)
|
351
369
|
end
|
@@ -358,7 +376,7 @@ module Fluent::Plugin
|
|
358
376
|
if record.key?('CONTAINER_NAME') && record.key?('CONTAINER_ID_FULL')
|
359
377
|
metadata = record['CONTAINER_NAME'].match(@container_name_to_kubernetes_regexp_compiled) do |match_data|
|
360
378
|
get_metadata_for_record(match_data['namespace'], match_data['pod_name'], match_data['container_name'],
|
361
|
-
|
379
|
+
record['CONTAINER_ID_FULL'], create_time_from_record(record, time), batch_miss_cache, record['CONTAINER_ID_FULL'])
|
362
380
|
end
|
363
381
|
unless metadata
|
364
382
|
log.debug "Error: could not match CONTAINER_NAME from record #{record}"
|
@@ -381,6 +399,16 @@ module Fluent::Plugin
|
|
381
399
|
end
|
382
400
|
end
|
383
401
|
|
402
|
+
def de_slash!(h)
|
403
|
+
h.keys.each do |ref|
|
404
|
+
next unless h[ref] && ref =~ /\//
|
405
|
+
|
406
|
+
v = h.delete(ref)
|
407
|
+
newref = ref.to_s.gsub('/', @de_slash_separator)
|
408
|
+
h[newref] = v
|
409
|
+
end
|
410
|
+
end
|
411
|
+
|
384
412
|
# copied from activesupport
|
385
413
|
def present?(object)
|
386
414
|
object.respond_to?(:empty?) ? !object.empty? : !!object
|
@@ -24,8 +24,6 @@ module KubernetesMetadata
|
|
24
24
|
metadata = {}
|
25
25
|
ids = @id_cache[key]
|
26
26
|
if ids.nil?
|
27
|
-
# FAST PATH
|
28
|
-
# Cache hit, fetch metadata from the cache
|
29
27
|
@stats.bump(:id_cache_miss)
|
30
28
|
return batch_miss_cache["#{namespace_name}_#{pod_name}"] if batch_miss_cache.key?("#{namespace_name}_#{pod_name}")
|
31
29
|
|
@@ -65,7 +63,7 @@ module KubernetesMetadata
|
|
65
63
|
@stats.bump(:id_cache_orphaned_record)
|
66
64
|
end
|
67
65
|
if @allow_orphans
|
68
|
-
log.trace("orphaning message for: #{namespace_name}/#{pod_name} ")
|
66
|
+
log.trace("orphaning message for: #{namespace_name}/#{pod_name} ")
|
69
67
|
metadata = {
|
70
68
|
'orphaned_namespace' => namespace_name,
|
71
69
|
'namespace_name' => @orphaned_namespace_name,
|
@@ -47,6 +47,10 @@ module KubernetesMetadata
|
|
47
47
|
de_dot!(labels) unless @skip_labels
|
48
48
|
de_dot!(annotations)
|
49
49
|
end
|
50
|
+
if @de_slash
|
51
|
+
de_slash!(labels) unless @skip_labels
|
52
|
+
de_slash!(annotations)
|
53
|
+
end
|
50
54
|
kubernetes_metadata = {
|
51
55
|
'namespace_id' => namespace_object[:metadata][:uid],
|
52
56
|
'creation_timestamp' => namespace_object[:metadata][:creationTimestamp]
|
@@ -65,14 +69,18 @@ module KubernetesMetadata
|
|
65
69
|
de_dot!(labels) unless @skip_labels
|
66
70
|
de_dot!(annotations)
|
67
71
|
end
|
72
|
+
if @de_slash
|
73
|
+
de_slash!(labels) unless @skip_labels
|
74
|
+
de_slash!(annotations)
|
75
|
+
end
|
68
76
|
|
69
77
|
# collect container information
|
70
78
|
container_meta = {}
|
71
79
|
begin
|
72
80
|
pod_object[:status][:containerStatuses].each do |container_status|
|
73
|
-
|
74
|
-
|
75
|
-
container_meta[
|
81
|
+
container_id = (container_status[:containerID]||"").sub(%r{^[-_a-zA-Z0-9]+://}, '')
|
82
|
+
key = container_status[:name]
|
83
|
+
container_meta[key] = if @skip_container_metadata
|
76
84
|
{
|
77
85
|
'name' => container_status[:name]
|
78
86
|
}
|
@@ -80,12 +88,13 @@ module KubernetesMetadata
|
|
80
88
|
{
|
81
89
|
'name' => container_status[:name],
|
82
90
|
'image' => container_status[:image],
|
83
|
-
'image_id' => container_status[:imageID]
|
91
|
+
'image_id' => container_status[:imageID],
|
92
|
+
:containerID => container_id
|
84
93
|
}
|
85
94
|
end
|
86
|
-
end
|
87
|
-
rescue StandardError
|
88
|
-
log.
|
95
|
+
end if pod_object[:status] && pod_object[:status][:containerStatuses]
|
96
|
+
rescue StandardError=>e
|
97
|
+
log.warn("parsing container meta information failed for: #{pod_object[:metadata][:namespace]}/#{pod_object[:metadata][:name]}: #{e}")
|
89
98
|
end
|
90
99
|
|
91
100
|
kubernetes_metadata = {
|
data/release_notes.md
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
# Release Notes
|
2
|
+
|
3
|
+
## 2.9.4
|
4
|
+
As of this release, the 'de_dot' functionality is depricated and will be removed in future releases.
|
5
|
+
Ref: https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/issues/320
|
6
|
+
|
7
|
+
## v2.1.4
|
8
|
+
The use of `use_journal` is **DEPRECATED**. If this setting is not present, the plugin will
|
9
|
+
attempt to figure out the source of the metadata fields from the following:
|
10
|
+
- If `lookup_from_k8s_field true` (the default) and the following fields are present in the record:
|
11
|
+
`docker.container_id`, `kubernetes.namespace_name`, `kubernetes.pod_name`, `kubernetes.container_name`,
|
12
|
+
then the plugin will use those values as the source to use to lookup the metadata
|
13
|
+
- If `use_journal true`, or `use_journal` is unset, and the fields `CONTAINER_NAME` and `CONTAINER_ID_FULL` are present in the record,
|
14
|
+
then the plugin will parse those values using `container_name_to_kubernetes_regexp` and use those as the source to lookup the metadata
|
15
|
+
- Otherwise, if the tag matches `tag_to_kubernetes_name_regexp`, the plugin will parse the tag and use those values to
|
16
|
+
lookup the metdata
|
17
|
+
|
18
|
+
## v2.1.x
|
19
|
+
|
20
|
+
As of the release 2.1.x of this plugin, it no longer supports parsing the source message into JSON and attaching it to the
|
21
|
+
payload. The following configuration options are removed:
|
22
|
+
|
23
|
+
* `merge_json_log`
|
24
|
+
* `preserve_json_log`
|
25
|
+
|
26
|
+
One way of preserving JSON logs can be through the [parser plugin](https://docs.fluentd.org/filter/parser).
|
27
|
+
It can parsed with the parser plugin like this:
|
28
|
+
|
29
|
+
```
|
30
|
+
<filter kubernetes.**>
|
31
|
+
@type parser
|
32
|
+
key_name log
|
33
|
+
<parse>
|
34
|
+
@type json
|
35
|
+
json_parser json
|
36
|
+
</parse>
|
37
|
+
replace_invalid_sequence true
|
38
|
+
reserve_data true # this preserves unparsable log lines
|
39
|
+
emit_invalid_record_to_error false # In case of unparsable log lines keep the error log clean
|
40
|
+
reserve_time # the time was already parsed in the source, we don't want to overwrite it with current time.
|
41
|
+
</filter>
|
42
|
+
```
|
File without changes
|
@@ -0,0 +1,145 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
|
3
|
+
# Kubernetes metadata
|
4
|
+
#
|
5
|
+
# Copyright 2015 Red Hat, Inc.
|
6
|
+
#
|
7
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
8
|
+
# you may not use this file except in compliance with the License.
|
9
|
+
# You may obtain a copy of the License at
|
10
|
+
#
|
11
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
12
|
+
#
|
13
|
+
# Unless required by applicable law or agreed to in writing, software
|
14
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
15
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
16
|
+
# See the License for the specific language governing permissions and
|
17
|
+
# limitations under the License.
|
18
|
+
#
|
19
|
+
---
|
20
|
+
http_interactions:
|
21
|
+
- request:
|
22
|
+
method: get
|
23
|
+
uri: https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc
|
24
|
+
body:
|
25
|
+
encoding: US-ASCII
|
26
|
+
string: ''
|
27
|
+
headers:
|
28
|
+
Accept:
|
29
|
+
- "*/*; q=0.5, application/xml"
|
30
|
+
Accept-Encoding:
|
31
|
+
- gzip, deflate
|
32
|
+
User-Agent:
|
33
|
+
- Ruby
|
34
|
+
response:
|
35
|
+
status:
|
36
|
+
code: 200
|
37
|
+
message: OK
|
38
|
+
headers:
|
39
|
+
Content-Type:
|
40
|
+
- application/json
|
41
|
+
Date:
|
42
|
+
- Fri, 08 May 2015 10:35:37 GMT
|
43
|
+
Transfer-Encoding:
|
44
|
+
- chunked
|
45
|
+
body:
|
46
|
+
encoding: UTF-8
|
47
|
+
string: |-
|
48
|
+
{
|
49
|
+
"kind": "Pod",
|
50
|
+
"apiVersion": "v1",
|
51
|
+
"metadata": {
|
52
|
+
"name": "fabric8-console-controller-98rqc",
|
53
|
+
"generateName": "fabric8-console-controller-",
|
54
|
+
"namespace": "default",
|
55
|
+
"selfLink": "/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc",
|
56
|
+
"uid": "c76927af-f563-11e4-b32d-54ee7527188d",
|
57
|
+
"resourceVersion": "122",
|
58
|
+
"creationTimestamp": "2015-05-08T09:22:42Z",
|
59
|
+
"labels": {
|
60
|
+
"component": "fabric8Console"
|
61
|
+
}
|
62
|
+
},
|
63
|
+
"spec": {
|
64
|
+
"volumes": [
|
65
|
+
{
|
66
|
+
"name": "openshift-cert-secrets",
|
67
|
+
"hostPath": null,
|
68
|
+
"emptyDir": null,
|
69
|
+
"gcePersistentDisk": null,
|
70
|
+
"gitRepo": null,
|
71
|
+
"secret": {
|
72
|
+
"secretName": "openshift-cert-secrets"
|
73
|
+
},
|
74
|
+
"nfs": null,
|
75
|
+
"iscsi": null,
|
76
|
+
"glusterfs": null
|
77
|
+
}
|
78
|
+
],
|
79
|
+
"containers": [
|
80
|
+
{
|
81
|
+
"name": "fabric8-console-container",
|
82
|
+
"image": "fabric8/hawtio-kubernetes:latest",
|
83
|
+
"ports": [
|
84
|
+
{
|
85
|
+
"containerPort": 9090,
|
86
|
+
"protocol": "TCP"
|
87
|
+
}
|
88
|
+
],
|
89
|
+
"env": [
|
90
|
+
{
|
91
|
+
"name": "OAUTH_CLIENT_ID",
|
92
|
+
"value": "fabric8-console"
|
93
|
+
},
|
94
|
+
{
|
95
|
+
"name": "OAUTH_AUTHORIZE_URI",
|
96
|
+
"value": "https://localhost:8443/oauth/authorize"
|
97
|
+
}
|
98
|
+
],
|
99
|
+
"resources": {},
|
100
|
+
"volumeMounts": [
|
101
|
+
{
|
102
|
+
"name": "openshift-cert-secrets",
|
103
|
+
"readOnly": true,
|
104
|
+
"mountPath": "/etc/secret-volume"
|
105
|
+
}
|
106
|
+
],
|
107
|
+
"terminationMessagePath": "/dev/termination-log",
|
108
|
+
"imagePullPolicy": "IfNotPresent",
|
109
|
+
"capabilities": {}
|
110
|
+
}
|
111
|
+
],
|
112
|
+
"restartPolicy": "Always",
|
113
|
+
"dnsPolicy": "ClusterFirst",
|
114
|
+
"nodeName": "jimmi-redhat.localnet"
|
115
|
+
},
|
116
|
+
"status": {
|
117
|
+
"phase": "Running",
|
118
|
+
"Condition": [
|
119
|
+
{
|
120
|
+
"type": "Ready",
|
121
|
+
"status": "True"
|
122
|
+
}
|
123
|
+
],
|
124
|
+
"hostIP": "172.17.42.1",
|
125
|
+
"podIP": "172.17.0.8",
|
126
|
+
"containerStatuses": [
|
127
|
+
{
|
128
|
+
"name": "fabric8-console-container",
|
129
|
+
"state": {
|
130
|
+
"waiting": {
|
131
|
+
"reason": "ContainerCreating"
|
132
|
+
}
|
133
|
+
},
|
134
|
+
"lastState": {},
|
135
|
+
"ready": true,
|
136
|
+
"restartCount": 0,
|
137
|
+
"image": "fabric8/hawtio-kubernetes:latest",
|
138
|
+
"imageID": ""
|
139
|
+
}
|
140
|
+
]
|
141
|
+
}
|
142
|
+
}
|
143
|
+
http_version:
|
144
|
+
recorded_at: Fri, 08 May 2015 10:35:37 GMT
|
145
|
+
recorded_with: VCR 2.9.3
|
@@ -28,7 +28,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
28
28
|
@time = Fluent::Engine.now
|
29
29
|
end
|
30
30
|
|
31
|
-
|
31
|
+
VAR_LOG_CONTAINER_TAG = 'var.log.containers.fabric8-console-controller-98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log'
|
32
|
+
VAR_LOG_POD_TAG = 'var.log.pods.default_fabric8-console-controller-98rqc_c76927af-f563-11e4-b32d-54ee7527188d.fabric8-console-container.0.log'
|
32
33
|
|
33
34
|
def create_driver(conf = '')
|
34
35
|
Test::Driver::Filter.new(Plugin::KubernetesMetadataFilter).configure(conf)
|
@@ -45,7 +46,6 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
45
46
|
assert_equal('KubernetesMetadata::TestApiAdapter', d.instance.test_api_adapter)
|
46
47
|
end
|
47
48
|
|
48
|
-
|
49
49
|
test 'kubernetes url' do
|
50
50
|
VCR.use_cassette('valid_kubernetes_api_server') do
|
51
51
|
d = create_driver('
|
@@ -138,7 +138,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
138
138
|
cache_size 1
|
139
139
|
', d: nil)
|
140
140
|
d = create_driver(config) if d.nil?
|
141
|
-
d.run(default_tag:
|
141
|
+
d.run(default_tag: VAR_LOG_CONTAINER_TAG) do
|
142
142
|
d.feed(@time, msg)
|
143
143
|
end
|
144
144
|
d.filtered.map(&:last)
|
@@ -165,6 +165,76 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
165
165
|
plugin.filter_stream('tag', Fluent::MultiEventStream.new)
|
166
166
|
end
|
167
167
|
|
168
|
+
sub_test_case 'parsing_pod_metadata when container_status is missing from the pod status' do
|
169
|
+
test 'using the tag_to_kubernetes_name_regexp for /var/log/containers ' do
|
170
|
+
VCR.use_cassettes(
|
171
|
+
[
|
172
|
+
{ name: 'valid_kubernetes_api_server' },
|
173
|
+
{ name: 'kubernetes_get_api_v1' },
|
174
|
+
{ name: 'kubernetes_get_namespace_default' },
|
175
|
+
{ name: 'kubernetes_get_pod_container_init' }
|
176
|
+
]) do
|
177
|
+
filtered = emit({}, "
|
178
|
+
kubernetes_url https://localhost:8443
|
179
|
+
watch false
|
180
|
+
cache_size 1
|
181
|
+
")
|
182
|
+
expected_kube_metadata = {
|
183
|
+
'docker' => {
|
184
|
+
'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
|
185
|
+
},
|
186
|
+
'kubernetes' => {
|
187
|
+
'container_image'=>'fabric8/hawtio-kubernetes:latest',
|
188
|
+
'container_name'=>'fabric8-console-container',
|
189
|
+
'host' => 'jimmi-redhat.localnet',
|
190
|
+
'pod_name' => 'fabric8-console-controller-98rqc',
|
191
|
+
'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
|
192
|
+
'namespace_name' => 'default',
|
193
|
+
'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
|
194
|
+
'pod_ip' => '172.17.0.8',
|
195
|
+
'master_url' => 'https://localhost:8443',
|
196
|
+
'labels' => {
|
197
|
+
'component' => 'fabric8Console'
|
198
|
+
}
|
199
|
+
}
|
200
|
+
}
|
201
|
+
assert_equal(expected_kube_metadata, filtered[0])
|
202
|
+
end
|
203
|
+
end
|
204
|
+
test 'using the tag_to_kubernetes_name_regexp for /var/log/pods' do
|
205
|
+
VCR.use_cassettes(
|
206
|
+
[
|
207
|
+
{ name: 'valid_kubernetes_api_server' },
|
208
|
+
{ name: 'kubernetes_get_api_v1' },
|
209
|
+
{ name: 'kubernetes_get_namespace_default' },
|
210
|
+
{ name: 'kubernetes_get_pod_container_init' }
|
211
|
+
]) do
|
212
|
+
filtered = emit_with_tag(VAR_LOG_POD_TAG,{}, "
|
213
|
+
kubernetes_url https://localhost:8443
|
214
|
+
watch false
|
215
|
+
cache_size 1
|
216
|
+
")
|
217
|
+
expected_kube_metadata = {
|
218
|
+
'kubernetes' => {
|
219
|
+
'container_image'=>'fabric8/hawtio-kubernetes:latest',
|
220
|
+
'container_name'=>'fabric8-console-container',
|
221
|
+
'host' => 'jimmi-redhat.localnet',
|
222
|
+
'pod_name' => 'fabric8-console-controller-98rqc',
|
223
|
+
'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
|
224
|
+
'namespace_name' => 'default',
|
225
|
+
'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
|
226
|
+
'pod_ip' => '172.17.0.8',
|
227
|
+
'master_url' => 'https://localhost:8443',
|
228
|
+
'labels' => {
|
229
|
+
'component' => 'fabric8Console'
|
230
|
+
}
|
231
|
+
}
|
232
|
+
}
|
233
|
+
assert_equal(expected_kube_metadata, filtered[0])
|
234
|
+
end
|
235
|
+
end
|
236
|
+
end
|
237
|
+
|
168
238
|
test 'inability to connect to the api server handles exception and doensnt block pipeline' do
|
169
239
|
VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' }]) do
|
170
240
|
driver = create_driver('
|
@@ -177,8 +247,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
177
247
|
filtered = emit({ 'time' => '2015-05-08T09:22:01Z' }, '', d: driver)
|
178
248
|
expected_kube_metadata = {
|
179
249
|
'time' => '2015-05-08T09:22:01Z',
|
180
|
-
'docker'
|
181
|
-
'container_id'
|
250
|
+
'docker'=>{
|
251
|
+
'container_id'=>'49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
|
182
252
|
},
|
183
253
|
'kubernetes' => {
|
184
254
|
'pod_name' => 'fabric8-console-controller-98rqc',
|
@@ -265,6 +335,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
265
335
|
end
|
266
336
|
|
267
337
|
test 'with docker & kubernetes metadata' do
|
338
|
+
|
268
339
|
VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' }, { name: 'kubernetes_get_pod' }, { name: 'kubernetes_get_namespace_default' }]) do
|
269
340
|
filtered = emit({ 'time' => '2015-05-08T09:22:01Z' })
|
270
341
|
expected_kube_metadata = {
|
@@ -361,8 +432,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
361
432
|
test 'with docker & kubernetes metadata but no configured api server' do
|
362
433
|
filtered = emit({}, '')
|
363
434
|
expected_kube_metadata = {
|
364
|
-
'docker'
|
365
|
-
'container_id'
|
435
|
+
'docker'=>{
|
436
|
+
'container_id'=>'49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
|
366
437
|
},
|
367
438
|
'kubernetes' => {
|
368
439
|
'pod_name' => 'fabric8-console-controller-98rqc',
|
@@ -383,8 +454,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
383
454
|
stub_request(:any, 'https://localhost:8443/api/v1/namespaces/default').to_timeout
|
384
455
|
filtered = emit
|
385
456
|
expected_kube_metadata = {
|
386
|
-
'docker'
|
387
|
-
'container_id'
|
457
|
+
'docker'=>{
|
458
|
+
'container_id'=>'49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
|
388
459
|
},
|
389
460
|
'kubernetes' => {
|
390
461
|
'pod_name' => 'fabric8-console-controller-98rqc',
|
@@ -406,8 +477,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
406
477
|
stub_request(:any, 'https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller.98rqc').to_timeout
|
407
478
|
filtered = emit_with_tag('var.log.containers.fabric8-console-controller.98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log', {}, '')
|
408
479
|
expected_kube_metadata = {
|
409
|
-
'docker'
|
410
|
-
'container_id'
|
480
|
+
'docker'=>{
|
481
|
+
'container_id'=>'49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
|
411
482
|
},
|
412
483
|
'kubernetes' => {
|
413
484
|
'pod_name' => 'fabric8-console-controller.98rqc',
|
@@ -432,13 +503,15 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
432
503
|
assert_equal(msg, filtered[0])
|
433
504
|
end
|
434
505
|
|
435
|
-
test 'with kubernetes dotted labels, de_dot enabled' do
|
506
|
+
test 'with kubernetes dotted and slashed labels, de_dot and de_slash enabled' do
|
436
507
|
VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' },
|
437
|
-
{ name: '
|
508
|
+
{ name: 'kubernetes_docker_metadata_dotted_slashed_labels' }]) do
|
438
509
|
filtered = emit({}, '
|
439
510
|
kubernetes_url https://localhost:8443
|
440
511
|
watch false
|
441
512
|
cache_size 1
|
513
|
+
de_dot true
|
514
|
+
de_slash true
|
442
515
|
')
|
443
516
|
expected_kube_metadata = {
|
444
517
|
'docker' => {
|
@@ -452,14 +525,14 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
452
525
|
'container_image_id' => 'docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303',
|
453
526
|
'namespace_id' => '898268c8-4a36-11e5-9d81-42010af0194c',
|
454
527
|
'namespace_labels' => {
|
455
|
-
'
|
528
|
+
'kubernetes_io__namespacetest' => 'somevalue'
|
456
529
|
},
|
457
530
|
'namespace_name' => 'default',
|
458
531
|
'pod_id' => 'c76927af-f563-11e4-b32d-54ee7527188d',
|
459
532
|
'pod_ip' => '172.17.0.8',
|
460
533
|
'master_url' => 'https://localhost:8443',
|
461
534
|
'labels' => {
|
462
|
-
'
|
535
|
+
'kubernetes_io__test' => 'somevalue'
|
463
536
|
}
|
464
537
|
}
|
465
538
|
}
|
@@ -467,14 +540,15 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
467
540
|
end
|
468
541
|
end
|
469
542
|
|
470
|
-
test 'with kubernetes dotted labels, de_dot disabled' do
|
543
|
+
test 'with kubernetes dotted and slashed labels, de_dot and de_slash disabled' do
|
471
544
|
VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' },
|
472
|
-
{ name: '
|
545
|
+
{ name: 'kubernetes_docker_metadata_dotted_slashed_labels' }]) do
|
473
546
|
filtered = emit({}, '
|
474
547
|
kubernetes_url https://localhost:8443
|
475
548
|
watch false
|
476
549
|
cache_size 1
|
477
550
|
de_dot false
|
551
|
+
de_slash false
|
478
552
|
')
|
479
553
|
expected_kube_metadata = {
|
480
554
|
'docker' => {
|
@@ -506,11 +580,21 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
506
580
|
test 'invalid de_dot_separator config' do
|
507
581
|
assert_raise Fluent::ConfigError do
|
508
582
|
create_driver('
|
583
|
+
de_dot true
|
509
584
|
de_dot_separator contains.
|
510
585
|
')
|
511
586
|
end
|
512
587
|
end
|
513
588
|
|
589
|
+
test 'invalid de_slash_separator config' do
|
590
|
+
assert_raise Fluent::ConfigError do
|
591
|
+
create_driver('
|
592
|
+
de_slash true
|
593
|
+
de_slash_separator contains/
|
594
|
+
')
|
595
|
+
end
|
596
|
+
end
|
597
|
+
|
514
598
|
test 'with records from journald and docker & kubernetes metadata' do
|
515
599
|
# with use_journal true should ignore tags and use CONTAINER_NAME and CONTAINER_ID_FULL
|
516
600
|
tag = 'var.log.containers.junk1_junk2_junk3-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed450.log'
|
@@ -686,7 +770,6 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
686
770
|
|
687
771
|
test 'uses metadata from tag if use_journal false and lookup_from_k8s_field false' do
|
688
772
|
# with use_journal unset, should still use the journal fields instead of tag fields
|
689
|
-
tag = 'var.log.containers.fabric8-console-controller-98rqc_default_fabric8-console-container-49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459.log'
|
690
773
|
msg = {
|
691
774
|
'CONTAINER_NAME' => 'k8s_journald-container-name.db89db89_journald-pod-name_journald-namespace-name_c76927af-f563-11e4-b32d-54ee7527188d_89db89db',
|
692
775
|
'CONTAINER_ID_FULL' => '838350c64bacba968d39a30a50789b2795291fceca6ccbff55298671d46b0e3b',
|
@@ -701,13 +784,13 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
701
784
|
VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' }, { name: 'kubernetes_get_api_v1' }, { name: 'kubernetes_get_pod' },
|
702
785
|
{ name: 'kubernetes_get_namespace_default', options: { allow_playback_repeats: true } },
|
703
786
|
{ name: 'metadata_from_tag_and_journald_fields' }]) do
|
704
|
-
es = emit_with_tag(
|
787
|
+
es = emit_with_tag(VAR_LOG_CONTAINER_TAG, msg, "
|
705
788
|
kubernetes_url https://localhost:8443
|
706
789
|
watch false
|
707
790
|
cache_size 1
|
708
791
|
lookup_from_k8s_field false
|
709
792
|
use_journal false
|
710
|
-
|
793
|
+
")
|
711
794
|
expected_kube_metadata = {
|
712
795
|
'docker' => {
|
713
796
|
'container_id' => '49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
|
@@ -941,7 +1024,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
941
1024
|
end
|
942
1025
|
|
943
1026
|
test 'processes all events when reading from MessagePackEventStream' do
|
944
|
-
VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' },
|
1027
|
+
VCR.use_cassettes([{ name: 'valid_kubernetes_api_server' },
|
1028
|
+
{ name: 'kubernetes_get_api_v1' },
|
945
1029
|
{ name: 'kubernetes_get_pod' },
|
946
1030
|
{ name: 'kubernetes_get_namespace_default' }]) do
|
947
1031
|
entries = [[@time, { 'time' => '2015-05-08T09:22:01Z' }], [@time, { 'time' => '2015-05-08T09:22:01Z' }]]
|
@@ -954,7 +1038,7 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
954
1038
|
cache_size 1
|
955
1039
|
')
|
956
1040
|
d.run do
|
957
|
-
d.feed(
|
1041
|
+
d.feed(VAR_LOG_CONTAINER_TAG, msgpack_stream)
|
958
1042
|
end
|
959
1043
|
filtered = d.filtered.map(&:last)
|
960
1044
|
|
@@ -998,8 +1082,8 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
998
1082
|
skip_namespace_metadata true
|
999
1083
|
')
|
1000
1084
|
expected_kube_metadata = {
|
1001
|
-
'docker'
|
1002
|
-
'container_id'
|
1085
|
+
'docker'=>{
|
1086
|
+
'container_id'=>'49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459'
|
1003
1087
|
},
|
1004
1088
|
'kubernetes' => {
|
1005
1089
|
'host' => 'jimmi-redhat.localnet',
|
data/test/plugin/watch_test.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kubernetes_metadata_filter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.9.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jimmi Dyson
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-01-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -19,7 +19,7 @@ dependencies:
|
|
19
19
|
version: 0.14.0
|
20
20
|
- - "<"
|
21
21
|
- !ruby/object:Gem::Version
|
22
|
-
version: '1.
|
22
|
+
version: '1.15'
|
23
23
|
type: :runtime
|
24
24
|
prerelease: false
|
25
25
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -29,21 +29,27 @@ dependencies:
|
|
29
29
|
version: 0.14.0
|
30
30
|
- - "<"
|
31
31
|
- !ruby/object:Gem::Version
|
32
|
-
version: '1.
|
32
|
+
version: '1.15'
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
34
|
name: kubeclient
|
35
35
|
requirement: !ruby/object:Gem::Requirement
|
36
36
|
requirements:
|
37
|
+
- - ">="
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: 4.0.0
|
37
40
|
- - "<"
|
38
41
|
- !ruby/object:Gem::Version
|
39
|
-
version:
|
42
|
+
version: 5.0.0
|
40
43
|
type: :runtime
|
41
44
|
prerelease: false
|
42
45
|
version_requirements: !ruby/object:Gem::Requirement
|
43
46
|
requirements:
|
47
|
+
- - ">="
|
48
|
+
- !ruby/object:Gem::Version
|
49
|
+
version: 4.0.0
|
44
50
|
- - "<"
|
45
51
|
- !ruby/object:Gem::Version
|
46
|
-
version:
|
52
|
+
version: 5.0.0
|
47
53
|
- !ruby/object:Gem::Dependency
|
48
54
|
name: lru_redux
|
49
55
|
requirement: !ruby/object:Gem::Requirement
|
@@ -222,14 +228,16 @@ files:
|
|
222
228
|
- lib/fluent/plugin/kubernetes_metadata_util.rb
|
223
229
|
- lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb
|
224
230
|
- lib/fluent/plugin/kubernetes_metadata_watch_pods.rb
|
231
|
+
- release_notes.md
|
225
232
|
- test/cassettes/invalid_api_server_config.yml
|
226
233
|
- test/cassettes/kubernetes_docker_metadata_annotations.yml
|
227
|
-
- test/cassettes/
|
234
|
+
- test/cassettes/kubernetes_docker_metadata_dotted_slashed_labels.yml
|
228
235
|
- test/cassettes/kubernetes_get_api_v1.yml
|
229
236
|
- test/cassettes/kubernetes_get_api_v1_using_token.yml
|
230
237
|
- test/cassettes/kubernetes_get_namespace_default.yml
|
231
238
|
- test/cassettes/kubernetes_get_namespace_default_using_token.yml
|
232
239
|
- test/cassettes/kubernetes_get_pod.yml
|
240
|
+
- test/cassettes/kubernetes_get_pod_container_init.yml
|
233
241
|
- test/cassettes/kubernetes_get_pod_using_token.yml
|
234
242
|
- test/cassettes/metadata_from_tag_and_journald_fields.yml
|
235
243
|
- test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml
|
@@ -263,7 +271,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
263
271
|
- !ruby/object:Gem::Version
|
264
272
|
version: '0'
|
265
273
|
requirements: []
|
266
|
-
rubygems_version: 3.
|
274
|
+
rubygems_version: 3.0.9
|
267
275
|
signing_key:
|
268
276
|
specification_version: 4
|
269
277
|
summary: Fluentd filter plugin to add Kubernetes metadata
|