fluent-plugin-wh-kubernetes_metadata_filter 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (40) hide show
  1. checksums.yaml +7 -0
  2. data/.circleci/config.yml +53 -0
  3. data/.gitignore +18 -0
  4. data/.rubocop.yml +57 -0
  5. data/Gemfile +9 -0
  6. data/LICENSE +201 -0
  7. data/README.md +179 -0
  8. data/Rakefile +41 -0
  9. data/fluent-plugin-wh-kubernetes_metadata_filter.gemspec +33 -0
  10. data/lib/fluent/plugin/filter_kubernetes_metadata.rb +381 -0
  11. data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb +100 -0
  12. data/lib/fluent/plugin/kubernetes_metadata_common.rb +130 -0
  13. data/lib/fluent/plugin/kubernetes_metadata_stats.rb +62 -0
  14. data/lib/fluent/plugin/kubernetes_metadata_test_api_adapter.rb +68 -0
  15. data/lib/fluent/plugin/kubernetes_metadata_util.rb +33 -0
  16. data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +187 -0
  17. data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +205 -0
  18. data/test/cassettes/invalid_api_server_config.yml +53 -0
  19. data/test/cassettes/kubernetes_docker_metadata_annotations.yml +205 -0
  20. data/test/cassettes/kubernetes_docker_metadata_dotted_slashed_labels.yml +197 -0
  21. data/test/cassettes/kubernetes_get_api_v1.yml +193 -0
  22. data/test/cassettes/kubernetes_get_api_v1_using_token.yml +195 -0
  23. data/test/cassettes/kubernetes_get_namespace_default.yml +72 -0
  24. data/test/cassettes/kubernetes_get_namespace_default_using_token.yml +71 -0
  25. data/test/cassettes/kubernetes_get_pod.yml +146 -0
  26. data/test/cassettes/kubernetes_get_pod_container_init.yml +145 -0
  27. data/test/cassettes/kubernetes_get_pod_using_token.yml +148 -0
  28. data/test/cassettes/metadata_from_tag_and_journald_fields.yml +153 -0
  29. data/test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml +285 -0
  30. data/test/cassettes/valid_kubernetes_api_server.yml +55 -0
  31. data/test/cassettes/valid_kubernetes_api_server_using_token.yml +57 -0
  32. data/test/helper.rb +82 -0
  33. data/test/plugin/test.token +1 -0
  34. data/test/plugin/test_cache_stats.rb +33 -0
  35. data/test/plugin/test_cache_strategy.rb +194 -0
  36. data/test/plugin/test_filter_kubernetes_metadata.rb +810 -0
  37. data/test/plugin/test_watch_namespaces.rb +245 -0
  38. data/test/plugin/test_watch_pods.rb +344 -0
  39. data/test/plugin/watch_test.rb +76 -0
  40. metadata +276 -0
data/test/cassettes/kubernetes_get_api_v1_using_token.yml ADDED
@@ -0,0 +1,195 @@
1
+ #
2
+ # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
3
+ # Kubernetes metadata
4
+ #
5
+ # Copyright 2015 Red Hat, Inc.
6
+ #
7
+ # Licensed under the Apache License, Version 2.0 (the "License");
8
+ # you may not use this file except in compliance with the License.
9
+ # You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing, software
14
+ # distributed under the License is distributed on an "AS IS" BASIS,
15
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ # See the License for the specific language governing permissions and
17
+ # limitations under the License.
18
+ #
19
+ ---
20
+ http_interactions:
21
+ - request:
22
+ method: get
23
+ uri: https://localhost:8443/api/v1
24
+ body:
25
+ encoding: US-ASCII
26
+ string: ''
27
+ headers:
28
+ Accept:
29
+ - "*/*; q=0.5, application/xml"
30
+ Accept-Encoding:
31
+ - gzip, deflate
32
+ User-Agent:
33
+ - Ruby
34
+ Authorization:
35
+ - Bearer YzYyYzFlODMtODdhNS00ZTMyLWIzMmItNmY4NDc4OTI1ZWFh
36
+ response:
37
+ status:
38
+ code: 200
39
+ message: OK
40
+ headers:
41
+ Content-Type:
42
+ - application/json
43
+ Date:
44
+ - Fri, 08 May 2015 10:35:37 GMT
45
+ Transfer-Encoding:
46
+ - chunked
47
+ body:
48
+ encoding: UTF-8
49
+ string: |-
50
+ {
51
+ "kind": "APIResourceList",
52
+ "groupVersion": "v1",
53
+ "resources": [
54
+ {
55
+ "name": "bindings",
56
+ "singularName": "",
57
+ "namespaced": true,
58
+ "kind": "Binding",
59
+ "verbs": [
60
+ "create"
61
+ ]
62
+ },
63
+ {
64
+ "name": "namespaces",
65
+ "singularName": "",
66
+ "namespaced": false,
67
+ "kind": "Namespace",
68
+ "verbs": [
69
+ "create",
70
+ "delete",
71
+ "get",
72
+ "list",
73
+ "patch",
74
+ "update",
75
+ "watch"
76
+ ],
77
+ "shortNames": [
78
+ "ns"
79
+ ]
80
+ },
81
+ {
82
+ "name": "namespaces/finalize",
83
+ "singularName": "",
84
+ "namespaced": false,
85
+ "kind": "Namespace",
86
+ "verbs": [
87
+ "update"
88
+ ]
89
+ },
90
+ {
91
+ "name": "namespaces/status",
92
+ "singularName": "",
93
+ "namespaced": false,
94
+ "kind": "Namespace",
95
+ "verbs": [
96
+ "get",
97
+ "patch",
98
+ "update"
99
+ ]
100
+ },
101
+ {
102
+ "name": "pods",
103
+ "singularName": "",
104
+ "namespaced": true,
105
+ "kind": "Pod",
106
+ "verbs": [
107
+ "create",
108
+ "delete",
109
+ "deletecollection",
110
+ "get",
111
+ "list",
112
+ "patch",
113
+ "update",
114
+ "watch"
115
+ ],
116
+ "shortNames": [
117
+ "po"
118
+ ],
119
+ "categories": [
120
+ "all"
121
+ ]
122
+ },
123
+ {
124
+ "name": "pods/attach",
125
+ "singularName": "",
126
+ "namespaced": true,
127
+ "kind": "Pod",
128
+ "verbs": []
129
+ },
130
+ {
131
+ "name": "pods/binding",
132
+ "singularName": "",
133
+ "namespaced": true,
134
+ "kind": "Binding",
135
+ "verbs": [
136
+ "create"
137
+ ]
138
+ },
139
+ {
140
+ "name": "pods/eviction",
141
+ "singularName": "",
142
+ "namespaced": true,
143
+ "group": "policy",
144
+ "version": "v1beta1",
145
+ "kind": "Eviction",
146
+ "verbs": [
147
+ "create"
148
+ ]
149
+ },
150
+ {
151
+ "name": "pods/exec",
152
+ "singularName": "",
153
+ "namespaced": true,
154
+ "kind": "Pod",
155
+ "verbs": []
156
+ },
157
+ {
158
+ "name": "pods/log",
159
+ "singularName": "",
160
+ "namespaced": true,
161
+ "kind": "Pod",
162
+ "verbs": [
163
+ "get"
164
+ ]
165
+ },
166
+ {
167
+ "name": "pods/portforward",
168
+ "singularName": "",
169
+ "namespaced": true,
170
+ "kind": "Pod",
171
+ "verbs": []
172
+ },
173
+ {
174
+ "name": "pods/proxy",
175
+ "singularName": "",
176
+ "namespaced": true,
177
+ "kind": "Pod",
178
+ "verbs": []
179
+ },
180
+ {
181
+ "name": "pods/status",
182
+ "singularName": "",
183
+ "namespaced": true,
184
+ "kind": "Pod",
185
+ "verbs": [
186
+ "get",
187
+ "patch",
188
+ "update"
189
+ ]
190
+ }
191
+ ]
192
+ }
193
+ http_version:
194
+ recorded_at: Fri, 08 May 2015 10:35:37 GMT
195
+ recorded_with: VCR 2.9.3
data/test/cassettes/kubernetes_get_namespace_default.yml ADDED
@@ -0,0 +1,72 @@
1
+ #
2
+ # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
3
+ # Kubernetes metadata
4
+ #
5
+ # Copyright 2015 Red Hat, Inc.
6
+ #
7
+ # Licensed under the Apache License, Version 2.0 (the "License");
8
+ # you may not use this file except in compliance with the License.
9
+ # You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing, software
14
+ # distributed under the License is distributed on an "AS IS" BASIS,
15
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ # See the License for the specific language governing permissions and
17
+ # limitations under the License.
18
+ #
19
+ ---
20
+ http_interactions:
21
+ - request:
22
+ method: get
23
+ uri: https://localhost:8443/api/v1/namespaces/default
24
+ body:
25
+ encoding: US-ASCII
26
+ string: ''
27
+ headers:
28
+ Accept:
29
+ - "*/*; q=0.5, application/xml"
30
+ Accept-Encoding:
31
+ - gzip, deflate
32
+ User-Agent:
33
+ - Ruby
34
+ response:
35
+ status:
36
+ code: 200
37
+ message: OK
38
+ headers:
39
+ Content-Type:
40
+ - application/json
41
+ Date:
42
+ - Fri, 08 May 2015 10:35:37 GMT
43
+ Transfer-Encoding:
44
+ - chunked
45
+ body:
46
+ encoding: UTF-8
47
+ string: |-
48
+ {
49
+ "kind": "Namespace",
50
+ "apiVersion": "v1",
51
+ "metadata": {
52
+ "name": "default",
53
+ "selfLink": "/api/v1/namespaces/default",
54
+ "uid": "898268c8-4a36-11e5-9d81-42010af0194c",
55
+ "resourceVersion": "6",
56
+ "creationTimestamp": "2015-05-08T09:22:01Z",
57
+ "labels": {
58
+ "tenant": "test"
59
+ }
60
+ },
61
+ "spec": {
62
+ "finalizers": [
63
+ "kubernetes"
64
+ ]
65
+ },
66
+ "status": {
67
+ "phase": "Active"
68
+ }
69
+ }
70
+ http_version:
71
+ recorded_at: Fri, 08 May 2015 10:35:37 GMT
72
+ recorded_with: VCR 2.9.3
data/test/cassettes/kubernetes_get_namespace_default_using_token.yml ADDED
@@ -0,0 +1,71 @@
1
+ #
2
+ # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
3
+ # Kubernetes metadata
4
+ #
5
+ # Copyright 2015 Red Hat, Inc.
6
+ #
7
+ # Licensed under the Apache License, Version 2.0 (the "License");
8
+ # you may not use this file except in compliance with the License.
9
+ # You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing, software
14
+ # distributed under the License is distributed on an "AS IS" BASIS,
15
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ # See the License for the specific language governing permissions and
17
+ # limitations under the License.
18
+ #
19
+ ---
20
+ http_interactions:
21
+ - request:
22
+ method: get
23
+ uri: https://localhost:8443/api/v1/namespaces/default
24
+ body:
25
+ encoding: US-ASCII
26
+ string: ''
27
+ headers:
28
+ Accept:
29
+ - "*/*; q=0.5, application/xml"
30
+ Accept-Encoding:
31
+ - gzip, deflate
32
+ User-Agent:
33
+ - Ruby
34
+ Authorization:
35
+ - Bearer YzYyYzFlODMtODdhNS00ZTMyLWIzMmItNmY4NDc4OTI1ZWFh
36
+ response:
37
+ status:
38
+ code: 200
39
+ message: OK
40
+ headers:
41
+ Content-Type:
42
+ - application/json
43
+ Date:
44
+ - Fri, 08 May 2015 10:35:37 GMT
45
+ Transfer-Encoding:
46
+ - chunked
47
+ body:
48
+ encoding: UTF-8
49
+ string: |-
50
+ {
51
+ "kind": "Namespace",
52
+ "apiVersion": "v1",
53
+ "metadata": {
54
+ "name": "default",
55
+ "selfLink": "/api/v1/namespaces/default",
56
+ "uid": "898268c8-4a36-11e5-9d81-42010af0194c",
57
+ "resourceVersion": "6",
58
+ "creationTimestamp": "2015-05-08T09:22:01Z"
59
+ },
60
+ "spec": {
61
+ "finalizers": [
62
+ "kubernetes"
63
+ ]
64
+ },
65
+ "status": {
66
+ "phase": "Active"
67
+ }
68
+ }
69
+ http_version:
70
+ recorded_at: Fri, 08 May 2015 10:35:37 GMT
71
+ recorded_with: VCR 2.9.3
data/test/cassettes/kubernetes_get_pod.yml ADDED
@@ -0,0 +1,146 @@
1
+ #
2
+ # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
3
+ # Kubernetes metadata
4
+ #
5
+ # Copyright 2015 Red Hat, Inc.
6
+ #
7
+ # Licensed under the Apache License, Version 2.0 (the "License");
8
+ # you may not use this file except in compliance with the License.
9
+ # You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing, software
14
+ # distributed under the License is distributed on an "AS IS" BASIS,
15
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ # See the License for the specific language governing permissions and
17
+ # limitations under the License.
18
+ #
19
+ ---
20
+ http_interactions:
21
+ - request:
22
+ method: get
23
+ uri: https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc
24
+ body:
25
+ encoding: US-ASCII
26
+ string: ''
27
+ headers:
28
+ Accept:
29
+ - "*/*; q=0.5, application/xml"
30
+ Accept-Encoding:
31
+ - gzip, deflate
32
+ User-Agent:
33
+ - Ruby
34
+ response:
35
+ status:
36
+ code: 200
37
+ message: OK
38
+ headers:
39
+ Content-Type:
40
+ - application/json
41
+ Date:
42
+ - Fri, 08 May 2015 10:35:37 GMT
43
+ Transfer-Encoding:
44
+ - chunked
45
+ body:
46
+ encoding: UTF-8
47
+ string: |-
48
+ {
49
+ "kind": "Pod",
50
+ "apiVersion": "v1",
51
+ "metadata": {
52
+ "name": "fabric8-console-controller-98rqc",
53
+ "generateName": "fabric8-console-controller-",
54
+ "namespace": "default",
55
+ "selfLink": "/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc",
56
+ "uid": "c76927af-f563-11e4-b32d-54ee7527188d",
57
+ "resourceVersion": "122",
58
+ "creationTimestamp": "2015-05-08T09:22:42Z",
59
+ "labels": {
60
+ "component": "fabric8Console"
61
+ }
62
+ },
63
+ "spec": {
64
+ "volumes": [
65
+ {
66
+ "name": "openshift-cert-secrets",
67
+ "hostPath": null,
68
+ "emptyDir": null,
69
+ "gcePersistentDisk": null,
70
+ "gitRepo": null,
71
+ "secret": {
72
+ "secretName": "openshift-cert-secrets"
73
+ },
74
+ "nfs": null,
75
+ "iscsi": null,
76
+ "glusterfs": null
77
+ }
78
+ ],
79
+ "containers": [
80
+ {
81
+ "name": "fabric8-console-container",
82
+ "image": "fabric8/hawtio-kubernetes:latest",
83
+ "ports": [
84
+ {
85
+ "containerPort": 9090,
86
+ "protocol": "TCP"
87
+ }
88
+ ],
89
+ "env": [
90
+ {
91
+ "name": "OAUTH_CLIENT_ID",
92
+ "value": "fabric8-console"
93
+ },
94
+ {
95
+ "name": "OAUTH_AUTHORIZE_URI",
96
+ "value": "https://localhost:8443/oauth/authorize"
97
+ }
98
+ ],
99
+ "resources": {},
100
+ "volumeMounts": [
101
+ {
102
+ "name": "openshift-cert-secrets",
103
+ "readOnly": true,
104
+ "mountPath": "/etc/secret-volume"
105
+ }
106
+ ],
107
+ "terminationMessagePath": "/dev/termination-log",
108
+ "imagePullPolicy": "IfNotPresent",
109
+ "capabilities": {}
110
+ }
111
+ ],
112
+ "restartPolicy": "Always",
113
+ "dnsPolicy": "ClusterFirst",
114
+ "nodeName": "jimmi-redhat.localnet"
115
+ },
116
+ "status": {
117
+ "phase": "Running",
118
+ "Condition": [
119
+ {
120
+ "type": "Ready",
121
+ "status": "True"
122
+ }
123
+ ],
124
+ "hostIP": "172.17.42.1",
125
+ "podIP": "172.17.0.8",
126
+ "containerStatuses": [
127
+ {
128
+ "name": "fabric8-console-container",
129
+ "state": {
130
+ "running": {
131
+ "startedAt": "2015-05-08T09:22:44Z"
132
+ }
133
+ },
134
+ "lastState": {},
135
+ "ready": true,
136
+ "restartCount": 0,
137
+ "image": "fabric8/hawtio-kubernetes:latest",
138
+ "imageID": "docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303",
139
+ "containerID": "docker://49095a2894da899d3b327c5fde1e056a81376cc9a8f8b09a195f2a92bceed459"
140
+ }
141
+ ]
142
+ }
143
+ }
144
+ http_version:
145
+ recorded_at: Fri, 08 May 2015 10:35:37 GMT
146
+ recorded_with: VCR 2.9.3
data/test/cassettes/kubernetes_get_pod_container_init.yml ADDED
@@ -0,0 +1,145 @@
1
+ #
2
+ # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
3
+ # Kubernetes metadata
4
+ #
5
+ # Copyright 2015 Red Hat, Inc.
6
+ #
7
+ # Licensed under the Apache License, Version 2.0 (the "License");
8
+ # you may not use this file except in compliance with the License.
9
+ # You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing, software
14
+ # distributed under the License is distributed on an "AS IS" BASIS,
15
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ # See the License for the specific language governing permissions and
17
+ # limitations under the License.
18
+ #
19
+ ---
20
+ http_interactions:
21
+ - request:
22
+ method: get
23
+ uri: https://localhost:8443/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc
24
+ body:
25
+ encoding: US-ASCII
26
+ string: ''
27
+ headers:
28
+ Accept:
29
+ - "*/*; q=0.5, application/xml"
30
+ Accept-Encoding:
31
+ - gzip, deflate
32
+ User-Agent:
33
+ - Ruby
34
+ response:
35
+ status:
36
+ code: 200
37
+ message: OK
38
+ headers:
39
+ Content-Type:
40
+ - application/json
41
+ Date:
42
+ - Fri, 08 May 2015 10:35:37 GMT
43
+ Transfer-Encoding:
44
+ - chunked
45
+ body:
46
+ encoding: UTF-8
47
+ string: |-
48
+ {
49
+ "kind": "Pod",
50
+ "apiVersion": "v1",
51
+ "metadata": {
52
+ "name": "fabric8-console-controller-98rqc",
53
+ "generateName": "fabric8-console-controller-",
54
+ "namespace": "default",
55
+ "selfLink": "/api/v1/namespaces/default/pods/fabric8-console-controller-98rqc",
56
+ "uid": "c76927af-f563-11e4-b32d-54ee7527188d",
57
+ "resourceVersion": "122",
58
+ "creationTimestamp": "2015-05-08T09:22:42Z",
59
+ "labels": {
60
+ "component": "fabric8Console"
61
+ }
62
+ },
63
+ "spec": {
64
+ "volumes": [
65
+ {
66
+ "name": "openshift-cert-secrets",
67
+ "hostPath": null,
68
+ "emptyDir": null,
69
+ "gcePersistentDisk": null,
70
+ "gitRepo": null,
71
+ "secret": {
72
+ "secretName": "openshift-cert-secrets"
73
+ },
74
+ "nfs": null,
75
+ "iscsi": null,
76
+ "glusterfs": null
77
+ }
78
+ ],
79
+ "containers": [
80
+ {
81
+ "name": "fabric8-console-container",
82
+ "image": "fabric8/hawtio-kubernetes:latest",
83
+ "ports": [
84
+ {
85
+ "containerPort": 9090,
86
+ "protocol": "TCP"
87
+ }
88
+ ],
89
+ "env": [
90
+ {
91
+ "name": "OAUTH_CLIENT_ID",
92
+ "value": "fabric8-console"
93
+ },
94
+ {
95
+ "name": "OAUTH_AUTHORIZE_URI",
96
+ "value": "https://localhost:8443/oauth/authorize"
97
+ }
98
+ ],
99
+ "resources": {},
100
+ "volumeMounts": [
101
+ {
102
+ "name": "openshift-cert-secrets",
103
+ "readOnly": true,
104
+ "mountPath": "/etc/secret-volume"
105
+ }
106
+ ],
107
+ "terminationMessagePath": "/dev/termination-log",
108
+ "imagePullPolicy": "IfNotPresent",
109
+ "capabilities": {}
110
+ }
111
+ ],
112
+ "restartPolicy": "Always",
113
+ "dnsPolicy": "ClusterFirst",
114
+ "nodeName": "jimmi-redhat.localnet"
115
+ },
116
+ "status": {
117
+ "phase": "Running",
118
+ "Condition": [
119
+ {
120
+ "type": "Ready",
121
+ "status": "True"
122
+ }
123
+ ],
124
+ "hostIP": "172.17.42.1",
125
+ "podIP": "172.17.0.8",
126
+ "containerStatuses": [
127
+ {
128
+ "name": "fabric8-console-container",
129
+ "state": {
130
+ "waiting": {
131
+ "reason": "ContainerCreating"
132
+ }
133
+ },
134
+ "lastState": {},
135
+ "ready": true,
136
+ "restartCount": 0,
137
+ "image": "fabric8/hawtio-kubernetes:latest",
138
+ "imageID": ""
139
+ }
140
+ ]
141
+ }
142
+ }
143
+ http_version:
144
+ recorded_at: Fri, 08 May 2015 10:35:37 GMT
145
+ recorded_with: VCR 2.9.3