fluent-plugin-kubernetes_metadata_filter 2.1.0 → 3.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +5 -5
  2. data/.circleci/config.yml +53 -0
  3. data/.gitignore +0 -2
  4. data/.rubocop.yml +57 -0
  5. data/Gemfile +4 -2
  6. data/Gemfile.lock +159 -0
  7. data/README.md +49 -60
  8. data/Rakefile +15 -11
  9. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/cpu.png +0 -0
  10. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/latency.png +0 -0
  11. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/loss.png +0 -0
  12. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/mem.png +0 -0
  13. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/readme.md +88 -0
  14. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/results.html +127 -0
  15. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/cpu.png +0 -0
  16. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/latency.png +0 -0
  17. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/loss.png +0 -0
  18. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/mem.png +0 -0
  19. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/readme.md +97 -0
  20. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/results.html +136 -0
  21. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/cpu.png +0 -0
  22. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/latency.png +0 -0
  23. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/loss.png +0 -0
  24. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/mem.png +0 -0
  25. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/readme.md +97 -0
  26. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/results.html +136 -0
  27. data/fluent-plugin-kubernetes_metadata_filter.gemspec +25 -28
  28. data/lib/fluent/plugin/filter_kubernetes_metadata.rb +207 -187
  29. data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb +30 -23
  30. data/lib/fluent/plugin/kubernetes_metadata_common.rb +66 -24
  31. data/lib/fluent/plugin/kubernetes_metadata_stats.rb +21 -5
  32. data/lib/fluent/plugin/kubernetes_metadata_test_api_adapter.rb +68 -0
  33. data/lib/fluent/plugin/kubernetes_metadata_util.rb +33 -0
  34. data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +154 -27
  35. data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +171 -29
  36. data/release_notes.md +42 -0
  37. data/test/cassettes/kubernetes_docker_metadata_annotations.yml +0 -34
  38. data/test/cassettes/{kubernetes_docker_metadata_dotted_labels.yml → kubernetes_docker_metadata_dotted_slashed_labels.yml} +0 -34
  39. data/test/cassettes/kubernetes_get_api_v1.yml +193 -0
  40. data/test/cassettes/kubernetes_get_api_v1_using_token.yml +195 -0
  41. data/test/cassettes/kubernetes_get_namespace_default.yml +72 -0
  42. data/test/cassettes/kubernetes_get_namespace_default_using_token.yml +71 -0
  43. data/test/cassettes/{kubernetes_docker_metadata.yml → kubernetes_get_pod.yml} +0 -82
  44. data/test/cassettes/kubernetes_get_pod_container_init.yml +145 -0
  45. data/test/cassettes/{metadata_with_namespace_id.yml → kubernetes_get_pod_using_token.yml} +2 -130
  46. data/test/cassettes/{kubernetes_docker_metadata_using_bearer_token.yml → kubernetes_get_pod_with_ownerrefs.yml} +17 -109
  47. data/test/cassettes/metadata_from_tag_and_journald_fields.yml +153 -0
  48. data/test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml +285 -0
  49. data/test/cassettes/{non_kubernetes_docker_metadata.yml → valid_kubernetes_api_server_using_token.yml} +4 -44
  50. data/test/helper.rb +20 -2
  51. data/test/plugin/test_cache_stats.rb +10 -13
  52. data/test/plugin/test_cache_strategy.rb +158 -160
  53. data/test/plugin/test_filter_kubernetes_metadata.rb +451 -314
  54. data/test/plugin/test_watch_namespaces.rb +209 -55
  55. data/test/plugin/test_watch_pods.rb +302 -71
  56. data/test/plugin/watch_test.rb +52 -33
  57. metadata +91 -70
  58. data/circle.yml +0 -17
@@ -0,0 +1,285 @@
1
+ #
2
+ # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
3
+ # Kubernetes metadata
4
+ #
5
+ # Copyright 2015 Red Hat, Inc.
6
+ #
7
+ # Licensed under the Apache License, Version 2.0 (the "License");
8
+ # you may not use this file except in compliance with the License.
9
+ # You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing, software
14
+ # distributed under the License is distributed on an "AS IS" BASIS,
15
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ # See the License for the specific language governing permissions and
17
+ # limitations under the License.
18
+ #
19
+ ---
20
+ http_interactions:
21
+ - request:
22
+ method: get
23
+ uri: https://localhost:8443/api/v1/namespaces/journald-namespace-name/pods/journald-pod-name
24
+ body:
25
+ encoding: US-ASCII
26
+ string: ''
27
+ headers:
28
+ Accept:
29
+ - "*/*; q=0.5, application/xml"
30
+ Accept-Encoding:
31
+ - gzip, deflate
32
+ User-Agent:
33
+ - Ruby
34
+ response:
35
+ status:
36
+ code: 200
37
+ message: OK
38
+ headers:
39
+ Content-Type:
40
+ - application/json
41
+ Date:
42
+ - Fri, 08 May 2015 10:35:37 GMT
43
+ Transfer-Encoding:
44
+ - chunked
45
+ body:
46
+ encoding: UTF-8
47
+ string: |-
48
+ {
49
+ "kind": "Pod",
50
+ "apiVersion": "v1",
51
+ "metadata": {
52
+ "name": "journald-pod-name",
53
+ "generateName": "journald-pod-name-",
54
+ "namespace": "journald-namespace-name",
55
+ "selfLink": "/api/v1/namespaces/journald-namespace-name/pods/journald-pod-name",
56
+ "uid": "5e1c1e27-b637-4e81-80b6-6d8a8c404d3b",
57
+ "resourceVersion": "122",
58
+ "creationTimestamp": "2015-05-08T09:22:42Z",
59
+ "labels": {
60
+ "component": "journald-test"
61
+ }
62
+ },
63
+ "spec": {
64
+ "containers": [
65
+ {
66
+ "name": "journald-container-name",
67
+ "image": "journald-container-image:latest",
68
+ "terminationMessagePath": "/dev/termination-log",
69
+ "imagePullPolicy": "IfNotPresent",
70
+ "capabilities": {}
71
+ }
72
+ ],
73
+ "nodeName": "jimmi-redhat.localnet"
74
+ },
75
+ "status": {
76
+ "phase": "Running",
77
+ "Condition": [
78
+ {
79
+ "type": "Ready",
80
+ "status": "True"
81
+ }
82
+ ],
83
+ "hostIP": "172.17.42.1",
84
+ "podIP": "172.17.0.8",
85
+ "containerStatuses": [
86
+ {
87
+ "name": "journald-container-name",
88
+ "state": {
89
+ "running": {
90
+ "startedAt": "2015-05-08T09:22:44Z"
91
+ }
92
+ },
93
+ "lastState": {},
94
+ "ready": true,
95
+ "restartCount": 0,
96
+ "image": "journald-container-image:latest",
97
+ "imageID": "docker://dda4c95705fb7050b701b10a7fe928ca5bc971a1dcb521ae3c339194cbf36b47",
98
+ "containerID": "docker://838350c64bacba968d39a30a50789b2795291fceca6ccbff55298671d46b0e3b"
99
+ }
100
+ ]
101
+ }
102
+ }
103
+ http_version:
104
+ recorded_at: Fri, 08 May 2015 10:35:37 GMT
105
+ - request:
106
+ method: get
107
+ uri: https://localhost:8443/api/v1/namespaces/journald-namespace-name
108
+ body:
109
+ encoding: US-ASCII
110
+ string: ''
111
+ headers:
112
+ Accept:
113
+ - "*/*; q=0.5, application/xml"
114
+ Accept-Encoding:
115
+ - gzip, deflate
116
+ User-Agent:
117
+ - Ruby
118
+ response:
119
+ status:
120
+ code: 200
121
+ message: OK
122
+ headers:
123
+ Content-Type:
124
+ - application/json
125
+ Date:
126
+ - Fri, 08 May 2015 10:35:37 GMT
127
+ Transfer-Encoding:
128
+ - chunked
129
+ body:
130
+ encoding: UTF-8
131
+ string: |-
132
+ {
133
+ "kind": "Namespace",
134
+ "apiVersion": "v1",
135
+ "metadata": {
136
+ "name": "journald-namespace-name",
137
+ "selfLink": "/api/v1/namespaces/journald-namespace-name",
138
+ "uid": "8282888f-733f-4f23-a3d3-1fdfa3bdacf2",
139
+ "resourceVersion": "6",
140
+ "creationTimestamp": "2015-05-08T09:22:01Z"
141
+ },
142
+ "spec": {
143
+ "finalizers": [
144
+ "kubernetes"
145
+ ]
146
+ },
147
+ "status": {
148
+ "phase": "Active"
149
+ }
150
+ }
151
+ http_version:
152
+ recorded_at: Fri, 08 May 2015 10:35:37 GMT
153
+ - request:
154
+ method: get
155
+ uri: https://localhost:8443/api/v1/namespaces/k8s-namespace-name/pods/k8s-pod-name
156
+ body:
157
+ encoding: US-ASCII
158
+ string: ''
159
+ headers:
160
+ Accept:
161
+ - "*/*; q=0.5, application/xml"
162
+ Accept-Encoding:
163
+ - gzip, deflate
164
+ User-Agent:
165
+ - Ruby
166
+ response:
167
+ status:
168
+ code: 200
169
+ message: OK
170
+ headers:
171
+ Content-Type:
172
+ - application/json
173
+ Date:
174
+ - Fri, 08 May 2015 10:35:37 GMT
175
+ Transfer-Encoding:
176
+ - chunked
177
+ body:
178
+ encoding: UTF-8
179
+ string: |-
180
+ {
181
+ "kind": "Pod",
182
+ "apiVersion": "v1",
183
+ "metadata": {
184
+ "name": "k8s-pod-name",
185
+ "generateName": "k8s-pod-name-",
186
+ "namespace": "k8s-namespace-name",
187
+ "selfLink": "/api/v1/namespaces/k8s-namespace-name/pods/k8s-pod-name",
188
+ "uid": "ebabf749-5fcd-4750-a3f0-aedd89476da8",
189
+ "resourceVersion": "122",
190
+ "creationTimestamp": "2015-05-08T09:22:42Z",
191
+ "labels": {
192
+ "component": "k8s-test"
193
+ }
194
+ },
195
+ "spec": {
196
+ "containers": [
197
+ {
198
+ "name": "k8s-container-name",
199
+ "image": "k8s-container-image:latest",
200
+ "terminationMessagePath": "/dev/termination-log",
201
+ "imagePullPolicy": "IfNotPresent",
202
+ "capabilities": {}
203
+ }
204
+ ],
205
+ "nodeName": "jimmi-redhat.localnet"
206
+ },
207
+ "status": {
208
+ "phase": "Running",
209
+ "Condition": [
210
+ {
211
+ "type": "Ready",
212
+ "status": "True"
213
+ }
214
+ ],
215
+ "hostIP": "172.17.42.1",
216
+ "podIP": "172.17.0.8",
217
+ "containerStatuses": [
218
+ {
219
+ "name": "k8s-container-name",
220
+ "state": {
221
+ "running": {
222
+ "startedAt": "2015-05-08T09:22:44Z"
223
+ }
224
+ },
225
+ "lastState": {},
226
+ "ready": true,
227
+ "restartCount": 0,
228
+ "image": "k8s-container-image:latest",
229
+ "imageID": "docker://d78c5217c41e9af08d37d9ae2cb070afa1fe3da6bc77bfb18879a8b4bfdf8a34",
230
+ "containerID": "docker://e463bc0d3ae38f5c89d92dca49b30e049e899799920b79d4d5f705acbe82ba95"
231
+ }
232
+ ]
233
+ }
234
+ }
235
+ http_version:
236
+ recorded_at: Fri, 08 May 2015 10:35:37 GMT
237
+ - request:
238
+ method: get
239
+ uri: https://localhost:8443/api/v1/namespaces/k8s-namespace-name
240
+ body:
241
+ encoding: US-ASCII
242
+ string: ''
243
+ headers:
244
+ Accept:
245
+ - "*/*; q=0.5, application/xml"
246
+ Accept-Encoding:
247
+ - gzip, deflate
248
+ User-Agent:
249
+ - Ruby
250
+ response:
251
+ status:
252
+ code: 200
253
+ message: OK
254
+ headers:
255
+ Content-Type:
256
+ - application/json
257
+ Date:
258
+ - Fri, 08 May 2015 10:35:37 GMT
259
+ Transfer-Encoding:
260
+ - chunked
261
+ body:
262
+ encoding: UTF-8
263
+ string: |-
264
+ {
265
+ "kind": "Namespace",
266
+ "apiVersion": "v1",
267
+ "metadata": {
268
+ "name": "k8s-namespace-name",
269
+ "selfLink": "/api/v1/namespaces/k8s-namespace-name",
270
+ "uid": "8e0dc8fc-59f2-49f7-a3e2-ed0913e19d9f",
271
+ "resourceVersion": "6",
272
+ "creationTimestamp": "2015-05-08T09:22:01Z"
273
+ },
274
+ "spec": {
275
+ "finalizers": [
276
+ "kubernetes"
277
+ ]
278
+ },
279
+ "status": {
280
+ "phase": "Active"
281
+ }
282
+ }
283
+ http_version:
284
+ recorded_at: Fri, 08 May 2015 10:35:37 GMT
285
+ recorded_with: VCR 2.9.3
@@ -31,6 +31,8 @@ http_interactions:
31
31
  - gzip, deflate
32
32
  User-Agent:
33
33
  - Ruby
34
+ Authorization:
35
+ - Bearer YzYyYzFlODMtODdhNS00ZTMyLWIzMmItNmY4NDc4OTI1ZWFh
34
36
  response:
35
37
  status:
36
38
  code: 200
@@ -39,7 +41,7 @@ http_interactions:
39
41
  Content-Type:
40
42
  - application/json
41
43
  Date:
42
- - Fri, 08 May 2015 10:35:37 GMT
44
+ - Fri, 08 May 2015 10:13:54 GMT
43
45
  Content-Length:
44
46
  - '67'
45
47
  body:
@@ -51,47 +53,5 @@ http_interactions:
51
53
  ]
52
54
  }
53
55
  http_version:
54
- recorded_at: Fri, 08 May 2015 10:35:37 GMT
55
- - request:
56
- method: get
57
- uri: https://localhost:8443/api/v1/namespaces/default/pods/fabric8-forge-controller-ymkew
58
- body:
59
- encoding: US-ASCII
60
- string: ''
61
- headers:
62
- Accept:
63
- - "*/*; q=0.5, application/xml"
64
- Accept-Encoding:
65
- - gzip, deflate
66
- User-Agent:
67
- - Ruby
68
- response:
69
- status:
70
- code: 404
71
- message: Not Found
72
- headers:
73
- Content-Type:
74
- - application/json
75
- Date:
76
- - Fri, 08 May 2015 10:35:37 GMT
77
- Content-Length:
78
- - '278'
79
- body:
80
- encoding: UTF-8
81
- string: |-
82
- {
83
- "kind": "Status",
84
- "apiVersion": "v1",
85
- "metadata": {},
86
- "status": "Failure",
87
- "message": "pods \"fabric8-forge-controller-ymkew\" not found",
88
- "reason": "NotFound",
89
- "details": {
90
- "id": "fabric8-forge-controller-ymkew",
91
- "kind": "pods"
92
- },
93
- "code": 404
94
- }
95
- http_version:
96
- recorded_at: Fri, 08 May 2015 10:35:37 GMT
56
+ recorded_at: Fri, 08 May 2015 10:13:54 GMT
97
57
  recorded_with: VCR 2.9.3
data/test/helper.rb CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  #
2
4
  # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
3
5
  # Kubernetes metadata
@@ -16,6 +18,7 @@
16
18
  # See the License for the specific language governing permissions and
17
19
  # limitations under the License.
18
20
  #
21
+ require 'bundler/setup'
19
22
  require 'codeclimate-test-reporter'
20
23
  SimpleCov.start do
21
24
  formatter SimpleCov::Formatter::MultiFormatter.new [
@@ -31,8 +34,14 @@ require 'fileutils'
31
34
  require 'fluent/log'
32
35
  require 'fluent/test'
33
36
  require 'minitest/autorun'
34
- require 'webmock/test_unit'
35
37
  require 'vcr'
38
+ require 'ostruct'
39
+ require 'fluent/plugin/filter_kubernetes_metadata'
40
+ require 'fluent/test/driver/filter'
41
+ require 'kubeclient'
42
+
43
+ require 'webmock/test_unit'
44
+ WebMock.disable_net_connect!
36
45
 
37
46
  VCR.configure do |config|
38
47
  config.cassette_library_dir = 'test/cassettes'
@@ -58,7 +67,16 @@ def ipv6_enabled?
58
67
  begin
59
68
  TCPServer.open('::1', 0)
60
69
  true
61
- rescue
70
+ rescue StandardError
62
71
  false
63
72
  end
64
73
  end
74
+
75
+ # TEST_NAME='foo' ruby test_file.rb to run a single test case
76
+ if ENV['TEST_NAME']
77
+ (class << Test::Unit::TestCase; self; end).prepend(Module.new do
78
+ def test(name)
79
+ super if name == ENV['TEST_NAME']
80
+ end
81
+ end)
82
+ end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  #
2
4
  # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
3
5
  # Kubernetes metadata
@@ -17,20 +19,15 @@
17
19
  # limitations under the License.
18
20
  #
19
21
  require_relative '../helper'
20
- require 'fluent/plugin/kubernetes_metadata_stats'
21
- require 'webmock/test_unit'
22
- WebMock.disable_net_connect!
23
22
 
24
23
  class KubernetesMetadataCacheStatsTest < Test::Unit::TestCase
25
-
26
- test 'watch stats' do
27
- require 'lru_redux'
28
- stats = KubernetesMetadata::Stats.new
29
- stats.bump(:missed)
30
- stats.bump(:deleted)
31
- stats.bump(:deleted)
24
+ test 'watch stats' do
25
+ require 'lru_redux'
26
+ stats = KubernetesMetadata::Stats.new
27
+ stats.bump(:missed)
28
+ stats.bump(:deleted)
29
+ stats.bump(:deleted)
32
30
 
33
- assert_equal("stats - deleted: 2, missed: 1", stats.to_s)
34
- end
35
-
31
+ assert_equal('stats - deleted: 2, missed: 1', stats.to_s)
32
+ end
36
33
  end