fluent-plugin-kubernetes_metadata_filter 2.1.4 → 2.9.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (40) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +57 -0
  3. data/.gitignore +0 -1
  4. data/.rubocop.yml +57 -0
  5. data/Gemfile +4 -2
  6. data/Gemfile.lock +158 -0
  7. data/README.md +48 -28
  8. data/Rakefile +15 -11
  9. data/fluent-plugin-kubernetes_metadata_filter.gemspec +25 -28
  10. data/lib/fluent/plugin/filter_kubernetes_metadata.rb +185 -131
  11. data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb +27 -20
  12. data/lib/fluent/plugin/kubernetes_metadata_common.rb +59 -33
  13. data/lib/fluent/plugin/kubernetes_metadata_stats.rb +6 -6
  14. data/lib/fluent/plugin/kubernetes_metadata_test_api_adapter.rb +68 -0
  15. data/lib/fluent/plugin/kubernetes_metadata_util.rb +53 -0
  16. data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +121 -27
  17. data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +138 -29
  18. data/release_notes.md +42 -0
  19. data/test/cassettes/kubernetes_docker_metadata_annotations.yml +0 -34
  20. data/test/cassettes/{kubernetes_docker_metadata_dotted_labels.yml → kubernetes_docker_metadata_dotted_slashed_labels.yml} +0 -34
  21. data/test/cassettes/kubernetes_get_api_v1.yml +193 -0
  22. data/test/cassettes/kubernetes_get_api_v1_using_token.yml +195 -0
  23. data/test/cassettes/kubernetes_get_namespace_default.yml +69 -0
  24. data/test/cassettes/kubernetes_get_namespace_default_using_token.yml +71 -0
  25. data/test/cassettes/{kubernetes_docker_metadata.yml → kubernetes_get_pod.yml} +0 -82
  26. data/test/cassettes/{metadata_with_namespace_id.yml → kubernetes_get_pod_container_init.yml} +3 -134
  27. data/test/cassettes/{kubernetes_docker_metadata_using_bearer_token.yml → kubernetes_get_pod_using_token.yml} +5 -105
  28. data/test/cassettes/metadata_from_tag_and_journald_fields.yml +0 -255
  29. data/test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml +0 -255
  30. data/test/cassettes/{non_kubernetes_docker_metadata.yml → valid_kubernetes_api_server_using_token.yml} +4 -44
  31. data/test/helper.rb +20 -2
  32. data/test/plugin/test_cache_stats.rb +10 -13
  33. data/test/plugin/test_cache_strategy.rb +158 -160
  34. data/test/plugin/test_filter_kubernetes_metadata.rb +480 -320
  35. data/test/plugin/test_utils.rb +56 -0
  36. data/test/plugin/test_watch_namespaces.rb +209 -55
  37. data/test/plugin/test_watch_pods.rb +302 -103
  38. data/test/plugin/watch_test.rb +52 -33
  39. metadata +69 -72
  40. data/circle.yml +0 -17
data/release_notes.md ADDED
@@ -0,0 +1,42 @@
1
+ # Release Notes
2
+
3
+ ## 2.9.4
4
+ As of this release, the 'de_dot' functionality is depricated and will be removed in future releases.
5
+ Ref: https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/issues/320
6
+
7
+ ## v2.1.4
8
+ The use of `use_journal` is **DEPRECATED**. If this setting is not present, the plugin will
9
+ attempt to figure out the source of the metadata fields from the following:
10
+ - If `lookup_from_k8s_field true` (the default) and the following fields are present in the record:
11
+ `docker.container_id`, `kubernetes.namespace_name`, `kubernetes.pod_name`, `kubernetes.container_name`,
12
+ then the plugin will use those values as the source to use to lookup the metadata
13
+ - If `use_journal true`, or `use_journal` is unset, and the fields `CONTAINER_NAME` and `CONTAINER_ID_FULL` are present in the record,
14
+ then the plugin will parse those values using `container_name_to_kubernetes_regexp` and use those as the source to lookup the metadata
15
+ - Otherwise, if the tag matches `tag_to_kubernetes_name_regexp`, the plugin will parse the tag and use those values to
16
+ lookup the metdata
17
+
18
+ ## v2.1.x
19
+
20
+ As of the release 2.1.x of this plugin, it no longer supports parsing the source message into JSON and attaching it to the
21
+ payload. The following configuration options are removed:
22
+
23
+ * `merge_json_log`
24
+ * `preserve_json_log`
25
+
26
+ One way of preserving JSON logs can be through the [parser plugin](https://docs.fluentd.org/filter/parser).
27
+ It can parsed with the parser plugin like this:
28
+
29
+ ```
30
+ <filter kubernetes.**>
31
+ @type parser
32
+ key_name log
33
+ <parse>
34
+ @type json
35
+ json_parser json
36
+ </parse>
37
+ replace_invalid_sequence true
38
+ reserve_data true # this preserves unparsable log lines
39
+ emit_invalid_record_to_error false # In case of unparsable log lines keep the error log clean
40
+ reserve_time # the time was already parsed in the source, we don't want to overwrite it with current time.
41
+ </filter>
42
+ ```
@@ -18,40 +18,6 @@
18
18
  #
19
19
  ---
20
20
  http_interactions:
21
- - request:
22
- method: get
23
- uri: https://localhost:8443/api
24
- body:
25
- encoding: US-ASCII
26
- string: ''
27
- headers:
28
- Accept:
29
- - "*/*; q=0.5, application/xml"
30
- Accept-Encoding:
31
- - gzip, deflate
32
- User-Agent:
33
- - Ruby
34
- response:
35
- status:
36
- code: 200
37
- message: OK
38
- headers:
39
- Content-Type:
40
- - application/json
41
- Date:
42
- - Fri, 08 May 2015 10:35:37 GMT
43
- Content-Length:
44
- - '67'
45
- body:
46
- encoding: UTF-8
47
- string: |-
48
- {
49
- "versions": [
50
- "v1"
51
- ]
52
- }
53
- http_version:
54
- recorded_at: Fri, 08 May 2015 10:35:37 GMT
55
21
  - request:
56
22
  method: get
57
23
  uri: https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc
@@ -18,40 +18,6 @@
18
18
  #
19
19
  ---
20
20
  http_interactions:
21
- - request:
22
- method: get
23
- uri: https://localhost:8443/api
24
- body:
25
- encoding: US-ASCII
26
- string: ''
27
- headers:
28
- Accept:
29
- - "*/*; q=0.5, application/xml"
30
- Accept-Encoding:
31
- - gzip, deflate
32
- User-Agent:
33
- - Ruby
34
- response:
35
- status:
36
- code: 200
37
- message: OK
38
- headers:
39
- Content-Type:
40
- - application/json
41
- Date:
42
- - Fri, 08 May 2015 10:35:37 GMT
43
- Content-Length:
44
- - '67'
45
- body:
46
- encoding: UTF-8
47
- string: |-
48
- {
49
- "versions": [
50
- "v1"
51
- ]
52
- }
53
- http_version:
54
- recorded_at: Fri, 08 May 2015 10:35:37 GMT
55
21
  - request:
56
22
  method: get
57
23
  uri: https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc
@@ -0,0 +1,193 @@
1
+ #
2
+ # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
3
+ # Kubernetes metadata
4
+ #
5
+ # Copyright 2015 Red Hat, Inc.
6
+ #
7
+ # Licensed under the Apache License, Version 2.0 (the "License");
8
+ # you may not use this file except in compliance with the License.
9
+ # You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing, software
14
+ # distributed under the License is distributed on an "AS IS" BASIS,
15
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ # See the License for the specific language governing permissions and
17
+ # limitations under the License.
18
+ #
19
+ ---
20
+ http_interactions:
21
+ - request:
22
+ method: get
23
+ uri: https://localhost:8443/api/v1
24
+ body:
25
+ encoding: US-ASCII
26
+ string: ''
27
+ headers:
28
+ Accept:
29
+ - "*/*; q=0.5, application/xml"
30
+ Accept-Encoding:
31
+ - gzip, deflate
32
+ User-Agent:
33
+ - Ruby
34
+ response:
35
+ status:
36
+ code: 200
37
+ message: OK
38
+ headers:
39
+ Content-Type:
40
+ - application/json
41
+ Date:
42
+ - Fri, 08 May 2015 10:35:37 GMT
43
+ Transfer-Encoding:
44
+ - chunked
45
+ body:
46
+ encoding: UTF-8
47
+ string: |-
48
+ {
49
+ "kind": "APIResourceList",
50
+ "groupVersion": "v1",
51
+ "resources": [
52
+ {
53
+ "name": "bindings",
54
+ "singularName": "",
55
+ "namespaced": true,
56
+ "kind": "Binding",
57
+ "verbs": [
58
+ "create"
59
+ ]
60
+ },
61
+ {
62
+ "name": "namespaces",
63
+ "singularName": "",
64
+ "namespaced": false,
65
+ "kind": "Namespace",
66
+ "verbs": [
67
+ "create",
68
+ "delete",
69
+ "get",
70
+ "list",
71
+ "patch",
72
+ "update",
73
+ "watch"
74
+ ],
75
+ "shortNames": [
76
+ "ns"
77
+ ]
78
+ },
79
+ {
80
+ "name": "namespaces/finalize",
81
+ "singularName": "",
82
+ "namespaced": false,
83
+ "kind": "Namespace",
84
+ "verbs": [
85
+ "update"
86
+ ]
87
+ },
88
+ {
89
+ "name": "namespaces/status",
90
+ "singularName": "",
91
+ "namespaced": false,
92
+ "kind": "Namespace",
93
+ "verbs": [
94
+ "get",
95
+ "patch",
96
+ "update"
97
+ ]
98
+ },
99
+ {
100
+ "name": "pods",
101
+ "singularName": "",
102
+ "namespaced": true,
103
+ "kind": "Pod",
104
+ "verbs": [
105
+ "create",
106
+ "delete",
107
+ "deletecollection",
108
+ "get",
109
+ "list",
110
+ "patch",
111
+ "update",
112
+ "watch"
113
+ ],
114
+ "shortNames": [
115
+ "po"
116
+ ],
117
+ "categories": [
118
+ "all"
119
+ ]
120
+ },
121
+ {
122
+ "name": "pods/attach",
123
+ "singularName": "",
124
+ "namespaced": true,
125
+ "kind": "Pod",
126
+ "verbs": []
127
+ },
128
+ {
129
+ "name": "pods/binding",
130
+ "singularName": "",
131
+ "namespaced": true,
132
+ "kind": "Binding",
133
+ "verbs": [
134
+ "create"
135
+ ]
136
+ },
137
+ {
138
+ "name": "pods/eviction",
139
+ "singularName": "",
140
+ "namespaced": true,
141
+ "group": "policy",
142
+ "version": "v1beta1",
143
+ "kind": "Eviction",
144
+ "verbs": [
145
+ "create"
146
+ ]
147
+ },
148
+ {
149
+ "name": "pods/exec",
150
+ "singularName": "",
151
+ "namespaced": true,
152
+ "kind": "Pod",
153
+ "verbs": []
154
+ },
155
+ {
156
+ "name": "pods/log",
157
+ "singularName": "",
158
+ "namespaced": true,
159
+ "kind": "Pod",
160
+ "verbs": [
161
+ "get"
162
+ ]
163
+ },
164
+ {
165
+ "name": "pods/portforward",
166
+ "singularName": "",
167
+ "namespaced": true,
168
+ "kind": "Pod",
169
+ "verbs": []
170
+ },
171
+ {
172
+ "name": "pods/proxy",
173
+ "singularName": "",
174
+ "namespaced": true,
175
+ "kind": "Pod",
176
+ "verbs": []
177
+ },
178
+ {
179
+ "name": "pods/status",
180
+ "singularName": "",
181
+ "namespaced": true,
182
+ "kind": "Pod",
183
+ "verbs": [
184
+ "get",
185
+ "patch",
186
+ "update"
187
+ ]
188
+ }
189
+ ]
190
+ }
191
+ http_version:
192
+ recorded_at: Fri, 08 May 2015 10:35:37 GMT
193
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,195 @@
1
+ #
2
+ # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
3
+ # Kubernetes metadata
4
+ #
5
+ # Copyright 2015 Red Hat, Inc.
6
+ #
7
+ # Licensed under the Apache License, Version 2.0 (the "License");
8
+ # you may not use this file except in compliance with the License.
9
+ # You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing, software
14
+ # distributed under the License is distributed on an "AS IS" BASIS,
15
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ # See the License for the specific language governing permissions and
17
+ # limitations under the License.
18
+ #
19
+ ---
20
+ http_interactions:
21
+ - request:
22
+ method: get
23
+ uri: https://localhost:8443/api/v1
24
+ body:
25
+ encoding: US-ASCII
26
+ string: ''
27
+ headers:
28
+ Accept:
29
+ - "*/*; q=0.5, application/xml"
30
+ Accept-Encoding:
31
+ - gzip, deflate
32
+ User-Agent:
33
+ - Ruby
34
+ Authorization:
35
+ - Bearer YzYyYzFlODMtODdhNS00ZTMyLWIzMmItNmY4NDc4OTI1ZWFh
36
+ response:
37
+ status:
38
+ code: 200
39
+ message: OK
40
+ headers:
41
+ Content-Type:
42
+ - application/json
43
+ Date:
44
+ - Fri, 08 May 2015 10:35:37 GMT
45
+ Transfer-Encoding:
46
+ - chunked
47
+ body:
48
+ encoding: UTF-8
49
+ string: |-
50
+ {
51
+ "kind": "APIResourceList",
52
+ "groupVersion": "v1",
53
+ "resources": [
54
+ {
55
+ "name": "bindings",
56
+ "singularName": "",
57
+ "namespaced": true,
58
+ "kind": "Binding",
59
+ "verbs": [
60
+ "create"
61
+ ]
62
+ },
63
+ {
64
+ "name": "namespaces",
65
+ "singularName": "",
66
+ "namespaced": false,
67
+ "kind": "Namespace",
68
+ "verbs": [
69
+ "create",
70
+ "delete",
71
+ "get",
72
+ "list",
73
+ "patch",
74
+ "update",
75
+ "watch"
76
+ ],
77
+ "shortNames": [
78
+ "ns"
79
+ ]
80
+ },
81
+ {
82
+ "name": "namespaces/finalize",
83
+ "singularName": "",
84
+ "namespaced": false,
85
+ "kind": "Namespace",
86
+ "verbs": [
87
+ "update"
88
+ ]
89
+ },
90
+ {
91
+ "name": "namespaces/status",
92
+ "singularName": "",
93
+ "namespaced": false,
94
+ "kind": "Namespace",
95
+ "verbs": [
96
+ "get",
97
+ "patch",
98
+ "update"
99
+ ]
100
+ },
101
+ {
102
+ "name": "pods",
103
+ "singularName": "",
104
+ "namespaced": true,
105
+ "kind": "Pod",
106
+ "verbs": [
107
+ "create",
108
+ "delete",
109
+ "deletecollection",
110
+ "get",
111
+ "list",
112
+ "patch",
113
+ "update",
114
+ "watch"
115
+ ],
116
+ "shortNames": [
117
+ "po"
118
+ ],
119
+ "categories": [
120
+ "all"
121
+ ]
122
+ },
123
+ {
124
+ "name": "pods/attach",
125
+ "singularName": "",
126
+ "namespaced": true,
127
+ "kind": "Pod",
128
+ "verbs": []
129
+ },
130
+ {
131
+ "name": "pods/binding",
132
+ "singularName": "",
133
+ "namespaced": true,
134
+ "kind": "Binding",
135
+ "verbs": [
136
+ "create"
137
+ ]
138
+ },
139
+ {
140
+ "name": "pods/eviction",
141
+ "singularName": "",
142
+ "namespaced": true,
143
+ "group": "policy",
144
+ "version": "v1beta1",
145
+ "kind": "Eviction",
146
+ "verbs": [
147
+ "create"
148
+ ]
149
+ },
150
+ {
151
+ "name": "pods/exec",
152
+ "singularName": "",
153
+ "namespaced": true,
154
+ "kind": "Pod",
155
+ "verbs": []
156
+ },
157
+ {
158
+ "name": "pods/log",
159
+ "singularName": "",
160
+ "namespaced": true,
161
+ "kind": "Pod",
162
+ "verbs": [
163
+ "get"
164
+ ]
165
+ },
166
+ {
167
+ "name": "pods/portforward",
168
+ "singularName": "",
169
+ "namespaced": true,
170
+ "kind": "Pod",
171
+ "verbs": []
172
+ },
173
+ {
174
+ "name": "pods/proxy",
175
+ "singularName": "",
176
+ "namespaced": true,
177
+ "kind": "Pod",
178
+ "verbs": []
179
+ },
180
+ {
181
+ "name": "pods/status",
182
+ "singularName": "",
183
+ "namespaced": true,
184
+ "kind": "Pod",
185
+ "verbs": [
186
+ "get",
187
+ "patch",
188
+ "update"
189
+ ]
190
+ }
191
+ ]
192
+ }
193
+ http_version:
194
+ recorded_at: Fri, 08 May 2015 10:35:37 GMT
195
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,69 @@
1
+ #
2
+ # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
3
+ # Kubernetes metadata
4
+ #
5
+ # Copyright 2015 Red Hat, Inc.
6
+ #
7
+ # Licensed under the Apache License, Version 2.0 (the "License");
8
+ # you may not use this file except in compliance with the License.
9
+ # You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing, software
14
+ # distributed under the License is distributed on an "AS IS" BASIS,
15
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ # See the License for the specific language governing permissions and
17
+ # limitations under the License.
18
+ #
19
+ ---
20
+ http_interactions:
21
+ - request:
22
+ method: get
23
+ uri: https://localhost:8443/api/v1/namespaces/default
24
+ body:
25
+ encoding: US-ASCII
26
+ string: ''
27
+ headers:
28
+ Accept:
29
+ - "*/*; q=0.5, application/xml"
30
+ Accept-Encoding:
31
+ - gzip, deflate
32
+ User-Agent:
33
+ - Ruby
34
+ response:
35
+ status:
36
+ code: 200
37
+ message: OK
38
+ headers:
39
+ Content-Type:
40
+ - application/json
41
+ Date:
42
+ - Fri, 08 May 2015 10:35:37 GMT
43
+ Transfer-Encoding:
44
+ - chunked
45
+ body:
46
+ encoding: UTF-8
47
+ string: |-
48
+ {
49
+ "kind": "Namespace",
50
+ "apiVersion": "v1",
51
+ "metadata": {
52
+ "name": "default",
53
+ "selfLink": "/api/v1/namespaces/default",
54
+ "uid": "898268c8-4a36-11e5-9d81-42010af0194c",
55
+ "resourceVersion": "6",
56
+ "creationTimestamp": "2015-05-08T09:22:01Z"
57
+ },
58
+ "spec": {
59
+ "finalizers": [
60
+ "kubernetes"
61
+ ]
62
+ },
63
+ "status": {
64
+ "phase": "Active"
65
+ }
66
+ }
67
+ http_version:
68
+ recorded_at: Fri, 08 May 2015 10:35:37 GMT
69
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,71 @@
1
+ #
2
+ # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
3
+ # Kubernetes metadata
4
+ #
5
+ # Copyright 2015 Red Hat, Inc.
6
+ #
7
+ # Licensed under the Apache License, Version 2.0 (the "License");
8
+ # you may not use this file except in compliance with the License.
9
+ # You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing, software
14
+ # distributed under the License is distributed on an "AS IS" BASIS,
15
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ # See the License for the specific language governing permissions and
17
+ # limitations under the License.
18
+ #
19
+ ---
20
+ http_interactions:
21
+ - request:
22
+ method: get
23
+ uri: https://localhost:8443/api/v1/namespaces/default
24
+ body:
25
+ encoding: US-ASCII
26
+ string: ''
27
+ headers:
28
+ Accept:
29
+ - "*/*; q=0.5, application/xml"
30
+ Accept-Encoding:
31
+ - gzip, deflate
32
+ User-Agent:
33
+ - Ruby
34
+ Authorization:
35
+ - Bearer YzYyYzFlODMtODdhNS00ZTMyLWIzMmItNmY4NDc4OTI1ZWFh
36
+ response:
37
+ status:
38
+ code: 200
39
+ message: OK
40
+ headers:
41
+ Content-Type:
42
+ - application/json
43
+ Date:
44
+ - Fri, 08 May 2015 10:35:37 GMT
45
+ Transfer-Encoding:
46
+ - chunked
47
+ body:
48
+ encoding: UTF-8
49
+ string: |-
50
+ {
51
+ "kind": "Namespace",
52
+ "apiVersion": "v1",
53
+ "metadata": {
54
+ "name": "default",
55
+ "selfLink": "/api/v1/namespaces/default",
56
+ "uid": "898268c8-4a36-11e5-9d81-42010af0194c",
57
+ "resourceVersion": "6",
58
+ "creationTimestamp": "2015-05-08T09:22:01Z"
59
+ },
60
+ "spec": {
61
+ "finalizers": [
62
+ "kubernetes"
63
+ ]
64
+ },
65
+ "status": {
66
+ "phase": "Active"
67
+ }
68
+ }
69
+ http_version:
70
+ recorded_at: Fri, 08 May 2015 10:35:37 GMT
71
+ recorded_with: VCR 2.9.3