fluent-plugin-kubernetes_metadata_filter 0.1.0 → 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: b09f5217a8e64acff136a114d222513bc0dbc6e9
4
- data.tar.gz: 2d5d1e8cc206a9290f74e3428c3aa01f2c2fd46c
3
+ metadata.gz: caf7caa0a18360f227fb306947d74f4d099a89e9
4
+ data.tar.gz: 5a91b4e9e256f711eddcd774d4929b339ba76c88
5
5
  SHA512:
6
- metadata.gz: a1e354e6e49a5c3461a9b038992d6f3a74cd961405672974f8127387aa00ab22d2cefa0d82722bc7245eb3a7f17a8729224b8af8581895e4556a58ac3a323f5a
7
- data.tar.gz: 8b8845065dc989f610e31e72f4daaad7c0806d4163a2c257564bda38f96ce2edcaedfc1204877732da0a6e38e340d25015671fc1edcda5be8910cd9204e67d0f
6
+ metadata.gz: bce6f30ed8df527f77411295005c41e0fba08f4dfafd261a93f325bcb6bf53cbae95194e5cfcaa47f7ee697d25a520df164bfccbdfbb56e39453b9b8b3e39a0f
7
+ data.tar.gz: b91c0ccac088c23c741effcb9616d89ca36ee439a184134ef793a92a5fc296224851ff0c13ae899735d22b919da5b3bd61d1faaddbb8feb87abde097fcbc3bf5
data/README.md CHANGED
@@ -9,14 +9,15 @@
9
9
 
10
10
  ## Configuration
11
11
 
12
- Configuration options are:
12
+ Configuration options for fluent.conf are:
13
13
 
14
14
  * `kubernetes_url` - URL to the API server. *This is required*
15
15
  * `apiVersion` - API version to use (default: `v1beta3`)
16
- * `client_cert` - path to a client cert file to authenticate to the API server
17
- * `client_key` - path to a client key file to authenticate to the API server
18
16
  * `ca_file` - path to CA file for Kubernetes server certificate validation
19
17
  * `verify_ssl` - validate SSL certificates (default: true)
18
+ * `client_cert` - path to a client cert file to authenticate to the API server
19
+ * `client_key` - path to a client key file to authenticate to the API server
20
+ * `bearer_token_file` - path to a file containing the bearer token to use for authentication
20
21
  * `container_name_to_kubernetes_name_regexp` - the regular expression used to extract kubernetes metadata (pod name, container name, namespace) from the Docker container name. This must used named capture groups for `pod_container_name`, `pod_name` & `namespace` (default: `'\/?[^_]+_(?<pod_container_name>[^\.]+)[^_]+_(?<pod_name>[^_]+)_(?<namespace>[^_]+)'`)
21
22
  * `cache_size` - size of the cache of Kubernetes metadata to reduce requests to the API server (default: `1000`)
22
23
 
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |gem|
6
6
  gem.name = "fluent-plugin-kubernetes_metadata_filter"
7
- gem.version = "0.1.0"
7
+ gem.version = "0.2.0"
8
8
  gem.authors = ["Jimmi Dyson"]
9
9
  gem.email = ["jimmidyson@gmail.com"]
10
10
  gem.description = %q{Filter plugin to add Kubernetes metadata}
@@ -30,10 +30,11 @@ module Fluent
30
30
  config_param :container_name_to_kubernetes_name_regexp,
31
31
  :string,
32
32
  :default => '\/?[^_]+_(?<pod_container_name>[^\.]+)[^_]+_(?<pod_name>[^_]+)_(?<namespace>[^_]+)'
33
+ config_param :bearer_token_file, :string, :default => ''
33
34
 
34
- def self.get_metadata(pod_name, container_name, namespace)
35
+ def get_metadata(pod_name, container_name, namespace)
35
36
  begin
36
- metadata = @@client.get_pod(pod_name, namespace)
37
+ metadata = @client.get_pod(pod_name, namespace)
37
38
  if metadata
38
39
  return {
39
40
  :uid => metadata['metadata']['uid'],
@@ -60,19 +61,27 @@ module Fluent
60
61
  require 'active_support/core_ext/object/blank'
61
62
  require 'lru_redux'
62
63
 
63
- @@client = Kubeclient::Client.new @kubernetes_url, @apiVersion
64
+ @client = Kubeclient::Client.new @kubernetes_url, @apiVersion
64
65
 
65
- if @client_cert.present? && @client_key.present? && @ca_file.present?
66
+ @client.ssl_options(
67
+ client_cert: @client_cert.present? ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil,
68
+ client_key: @client_key.present? ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
69
+ ca_file: @ca_file,
70
+ verify_ssl: @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
71
+ )
66
72
 
67
- @@client.ssl_options(
68
- client_cert: OpenSSL::X509::Certificate.new(File.read(@client_cert)),
69
- client_key: OpenSSL::PKey::RSA.new(File.read(@client_key)),
70
- ca_file: @ca_file,
71
- verify_ssl: @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
72
- )
73
+ if @bearer_token_file.present?
74
+ bearer_token = File.read(@bearer_token_file)
75
+ RestClient.add_before_execution_proc do |req, params|
76
+ req['authorization'] ||= "Bearer #{bearer_token}"
77
+ end
73
78
  end
74
79
 
75
- raise Fluent::ConfigError, 'Invalid Kubernetes API endpoint' unless @@client.api_valid?
80
+ begin
81
+ @client.api_valid?
82
+ rescue KubeException => kube_error
83
+ raise Fluent::ConfigError, "Invalid Kubernetes API endpoint: #{kube_error.message}"
84
+ end
76
85
 
77
86
  @cache = LruRedux::ThreadSafeCache.new(@cache_size)
78
87
  @container_name_to_kubernetes_name_regexp_compiled = Regexp.compile(@container_name_to_kubernetes_name_regexp)
@@ -83,10 +92,11 @@ module Fluent
83
92
 
84
93
  es.each {|time, record|
85
94
  if record.has_key?(:docker) && record[:docker].has_key?(:name)
95
+ this = self
86
96
  metadata = @cache.getset(record[:docker][:name]){
87
97
  match_data = record[:docker][:name].match(@container_name_to_kubernetes_name_regexp_compiled)
88
98
  if match_data
89
- KubernetesMetadataFilter.get_metadata(
99
+ this.get_metadata(
90
100
  match_data[:pod_name],
91
101
  match_data[:pod_container_name],
92
102
  match_data[:namespace]
@@ -104,4 +114,4 @@ module Fluent
104
114
  end
105
115
  end
106
116
 
107
- end
117
+ end
@@ -0,0 +1,53 @@
1
+ #
2
+ # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
3
+ # Kubernetes metadata
4
+ #
5
+ # Copyright 2015 Red Hat, Inc.
6
+ #
7
+ # Licensed under the Apache License, Version 2.0 (the "License");
8
+ # you may not use this file except in compliance with the License.
9
+ # You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing, software
14
+ # distributed under the License is distributed on an "AS IS" BASIS,
15
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ # See the License for the specific language governing permissions and
17
+ # limitations under the License.
18
+ #
19
+ ---
20
+ http_interactions:
21
+ - request:
22
+ method: get
23
+ uri: https://localhost:8443/api
24
+ body:
25
+ encoding: US-ASCII
26
+ string: ''
27
+ headers:
28
+ Accept:
29
+ - "*/*; q=0.5, application/xml"
30
+ Accept-Encoding:
31
+ - gzip, deflate
32
+ User-Agent:
33
+ - Ruby
34
+ Authorization:
35
+ - Bearer YzYyYzFlODMtODdhNS00ZTMyLWIzMmItNmY4NDc4OTI1ZWF
36
+ response:
37
+ status:
38
+ code: 401
39
+ message: Unauthorized
40
+ headers:
41
+ Content-Type:
42
+ - text/plain; charset=utf-8
43
+ Date:
44
+ - Sat, 09 May 2015 14:04:39 GMT
45
+ Content-Length:
46
+ - '13'
47
+ body:
48
+ encoding: UTF-8
49
+ string: |
50
+ Unauthorized
51
+ http_version:
52
+ recorded_at: Sat, 09 May 2015 14:04:39 GMT
53
+ recorded_with: VCR 2.9.3
@@ -0,0 +1,202 @@
1
+ #
2
+ # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
3
+ # Kubernetes metadata
4
+ #
5
+ # Copyright 2015 Red Hat, Inc.
6
+ #
7
+ # Licensed under the Apache License, Version 2.0 (the "License");
8
+ # you may not use this file except in compliance with the License.
9
+ # You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing, software
14
+ # distributed under the License is distributed on an "AS IS" BASIS,
15
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ # See the License for the specific language governing permissions and
17
+ # limitations under the License.
18
+ #
19
+ ---
20
+ http_interactions:
21
+ - request:
22
+ method: get
23
+ uri: https://localhost:8443/api
24
+ body:
25
+ encoding: US-ASCII
26
+ string: ''
27
+ headers:
28
+ Accept:
29
+ - "*/*; q=0.5, application/xml"
30
+ Accept-Encoding:
31
+ - gzip, deflate
32
+ User-Agent:
33
+ - Ruby
34
+ Authorization:
35
+ - Bearer YzYyYzFlODMtODdhNS00ZTMyLWIzMmItNmY4NDc4OTI1ZWFh
36
+ response:
37
+ status:
38
+ code: 200
39
+ message: OK
40
+ headers:
41
+ Content-Type:
42
+ - application/json
43
+ Date:
44
+ - Sat, 09 May 2015 13:51:07 GMT
45
+ Content-Length:
46
+ - '67'
47
+ body:
48
+ encoding: UTF-8
49
+ string: |-
50
+ {
51
+ "versions": [
52
+ "v1beta1",
53
+ "v1beta2",
54
+ "v1beta3"
55
+ ]
56
+ }
57
+ http_version:
58
+ recorded_at: Sat, 09 May 2015 13:51:07 GMT
59
+ - request:
60
+ method: get
61
+ uri: https://localhost:8443/api/v1beta3/namespaces/default/pods/fabric8-console-controller-98rqc
62
+ body:
63
+ encoding: US-ASCII
64
+ string: ''
65
+ headers:
66
+ Accept:
67
+ - "*/*; q=0.5, application/xml"
68
+ Accept-Encoding:
69
+ - gzip, deflate
70
+ User-Agent:
71
+ - Ruby
72
+ Authorization:
73
+ - Bearer YzYyYzFlODMtODdhNS00ZTMyLWIzMmItNmY4NDc4OTI1ZWFh
74
+ response:
75
+ status:
76
+ code: 200
77
+ message: OK
78
+ headers:
79
+ Content-Type:
80
+ - application/json
81
+ Date:
82
+ - Sat, 09 May 2015 13:51:07 GMT
83
+ Transfer-Encoding:
84
+ - chunked
85
+ body:
86
+ encoding: UTF-8
87
+ string: |-
88
+ {
89
+ "kind": "Pod",
90
+ "apiVersion": "v1beta3",
91
+ "metadata": {
92
+ "name": "fabric8-console-controller-98rqc",
93
+ "generateName": "fabric8-console-controller-",
94
+ "namespace": "default",
95
+ "selfLink": "/api/v1beta3/namespaces/default/pods/fabric8-console-controller-98rqc",
96
+ "uid": "c76927af-f563-11e4-b32d-54ee7527188d",
97
+ "resourceVersion": "3556",
98
+ "creationTimestamp": "2015-05-08T09:22:42Z",
99
+ "labels": {
100
+ "component": "fabric8Console"
101
+ }
102
+ },
103
+ "spec": {
104
+ "volumes": [
105
+ {
106
+ "name": "openshift-cert-secrets",
107
+ "hostPath": null,
108
+ "emptyDir": null,
109
+ "gcePersistentDisk": null,
110
+ "gitRepo": null,
111
+ "secret": {
112
+ "secretName": "openshift-cert-secrets"
113
+ },
114
+ "nfs": null,
115
+ "iscsi": null,
116
+ "glusterfs": null
117
+ }
118
+ ],
119
+ "containers": [
120
+ {
121
+ "name": "fabric8-console-container",
122
+ "image": "fabric8/hawtio-kubernetes:latest",
123
+ "ports": [
124
+ {
125
+ "containerPort": 9090,
126
+ "protocol": "TCP"
127
+ }
128
+ ],
129
+ "env": [
130
+ {
131
+ "name": "OAUTH_CLIENT_ID",
132
+ "value": "fabric8-console"
133
+ },
134
+ {
135
+ "name": "OAUTH_AUTHORIZE_URI",
136
+ "value": "https://localhost:8443/oauth/authorize"
137
+ }
138
+ ],
139
+ "resources": {},
140
+ "volumeMounts": [
141
+ {
142
+ "name": "openshift-cert-secrets",
143
+ "readOnly": true,
144
+ "mountPath": "/etc/secret-volume"
145
+ }
146
+ ],
147
+ "terminationMessagePath": "/dev/termination-log",
148
+ "imagePullPolicy": "IfNotPresent",
149
+ "capabilities": {}
150
+ }
151
+ ],
152
+ "restartPolicy": "Always",
153
+ "dnsPolicy": "ClusterFirst",
154
+ "host": "jimmi-redhat.localnet"
155
+ },
156
+ "status": {
157
+ "phase": "Running",
158
+ "Condition": [
159
+ {
160
+ "type": "Ready",
161
+ "status": "True"
162
+ }
163
+ ],
164
+ "hostIP": "172.17.42.1",
165
+ "podIP": "172.17.0.4",
166
+ "containerStatuses": [
167
+ {
168
+ "name": "fabric8-console-container",
169
+ "state": {
170
+ "running": {
171
+ "startedAt": "2015-05-09T13:33:38Z"
172
+ }
173
+ },
174
+ "lastState": {},
175
+ "ready": true,
176
+ "restartCount": 2,
177
+ "image": "fabric8/hawtio-kubernetes:latest",
178
+ "imageID": "docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303",
179
+ "containerID": "docker://1b1d1f61c1205fe73328c75b2945e2ce05acfba2fde16299a8103fb22e9ec58a"
180
+ },
181
+ {
182
+ "name": "POD",
183
+ "state": {
184
+ "termination": {
185
+ "exitCode": 0,
186
+ "startedAt": "2015-05-09T09:45:08Z",
187
+ "finishedAt": "2015-05-09T09:47:00Z"
188
+ }
189
+ },
190
+ "lastState": {},
191
+ "ready": false,
192
+ "restartCount": 1,
193
+ "image": "openshift/origin-pod:v0.4.4",
194
+ "imageID": "docker://81fa37a83ec46fed9dddf30a9c3c43f54bfe4b65dea74a4c152fd515a0bf92a8",
195
+ "containerID": "docker://9c8ed1a9bbb6d9804b12379654461dcd3b447438c02c52ca7f7d2504e14ae96b"
196
+ }
197
+ ]
198
+ }
199
+ }
200
+ http_version:
201
+ recorded_at: Sat, 09 May 2015 13:51:07 GMT
202
+ recorded_with: VCR 2.9.3
@@ -0,0 +1 @@
1
+ 12345
@@ -61,6 +61,18 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
61
61
  assert_equal(1, d.instance.cache_size)
62
62
  end
63
63
  end
64
+
65
+ test 'invalid API server config' do
66
+ VCR.use_cassette('invalid_api_server_config') do
67
+ assert_raise Fluent::ConfigError do
68
+ d = create_driver(%[
69
+ kubernetes_url https://localhost:8443
70
+ bearer_token_file test/plugin/test.token
71
+ verify_ssl false
72
+ ])
73
+ end
74
+ end
75
+ end
64
76
  end
65
77
 
66
78
  sub_test_case 'filter_stream' do
@@ -99,6 +111,34 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
99
111
  end
100
112
  end
101
113
 
114
+ test 'with docker & kubernetes metadata using bearer token' do
115
+ VCR.use_cassette('kubernetes_docker_metadata_using_bearer_token') do
116
+ msg = {
117
+ :docker => {
118
+ :name => '/k8s_fabric8-console-container.efbd6e64_fabric8-console-controller-98rqc_default_c76927af-f563-11e4-b32d-54ee7527188d_42cbc279'
119
+ }
120
+ }
121
+ es = emit(msg, %[
122
+ kubernetes_url https://localhost:8443
123
+ verify_ssl false
124
+ bearer_token_file test/plugin/test.token
125
+ ])
126
+ expected_kube_metadata = {
127
+ :kubernetes => {
128
+ :host => "jimmi-redhat.localnet",
129
+ :pod_name =>"fabric8-console-controller-98rqc",
130
+ :container_name => "fabric8-console-container",
131
+ :namespace => "default",
132
+ :uid => "c76927af-f563-11e4-b32d-54ee7527188d",
133
+ :labels => {
134
+ :component => "fabric8Console"
135
+ }
136
+ }
137
+ }
138
+ assert_equal(msg.merge(expected_kube_metadata), es.instance_variable_get(:@record_array)[0])
139
+ end
140
+ end
141
+
102
142
  test 'with docker metadata, non-kubernetes' do
103
143
  VCR.use_cassette('non_kubernetes_docker_metadata') do
104
144
  msg = {
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-kubernetes_metadata_filter
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jimmi Dyson
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-05-08 00:00:00.000000000 Z
11
+ date: 2015-05-09 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -208,10 +208,13 @@ files:
208
208
  - circle.yml
209
209
  - fluent-plugin-kubernetes_metadata_filter.gemspec
210
210
  - lib/fluent/plugin/filter_kubernetes_metadata.rb
211
+ - test/cassettes/invalid_api_server_config.yml
211
212
  - test/cassettes/kubernetes_docker_metadata.yml
213
+ - test/cassettes/kubernetes_docker_metadata_using_bearer_token.yml
212
214
  - test/cassettes/non_kubernetes_docker_metadata.yml
213
215
  - test/cassettes/valid_kubernetes_api_server.yml
214
216
  - test/helper.rb
217
+ - test/plugin/test.token
215
218
  - test/plugin/test_filter_kubernetes_metadata.rb
216
219
  homepage: https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter
217
220
  licenses:
@@ -238,8 +241,11 @@ signing_key:
238
241
  specification_version: 4
239
242
  summary: Filter plugin to add Kubernetes metadata
240
243
  test_files:
244
+ - test/cassettes/invalid_api_server_config.yml
241
245
  - test/cassettes/kubernetes_docker_metadata.yml
246
+ - test/cassettes/kubernetes_docker_metadata_using_bearer_token.yml
242
247
  - test/cassettes/non_kubernetes_docker_metadata.yml
243
248
  - test/cassettes/valid_kubernetes_api_server.yml
244
249
  - test/helper.rb
250
+ - test/plugin/test.token
245
251
  - test/plugin/test_filter_kubernetes_metadata.rb