fluent-plugin-kubernetes_metadata_filter 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +4 -3
- data/fluent-plugin-kubernetes_metadata_filter.gemspec +1 -1
- data/lib/fluent/plugin/filter_kubernetes_metadata.rb +23 -13
- data/test/cassettes/invalid_api_server_config.yml +53 -0
- data/test/cassettes/kubernetes_docker_metadata_using_bearer_token.yml +202 -0
- data/test/plugin/test.token +1 -0
- data/test/plugin/test_filter_kubernetes_metadata.rb +40 -0
- metadata +8 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: caf7caa0a18360f227fb306947d74f4d099a89e9
|
4
|
+
data.tar.gz: 5a91b4e9e256f711eddcd774d4929b339ba76c88
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: bce6f30ed8df527f77411295005c41e0fba08f4dfafd261a93f325bcb6bf53cbae95194e5cfcaa47f7ee697d25a520df164bfccbdfbb56e39453b9b8b3e39a0f
|
7
|
+
data.tar.gz: b91c0ccac088c23c741effcb9616d89ca36ee439a184134ef793a92a5fc296224851ff0c13ae899735d22b919da5b3bd61d1faaddbb8feb87abde097fcbc3bf5
|
data/README.md
CHANGED
@@ -9,14 +9,15 @@
|
|
9
9
|
|
10
10
|
## Configuration
|
11
11
|
|
12
|
-
Configuration options are:
|
12
|
+
Configuration options for fluent.conf are:
|
13
13
|
|
14
14
|
* `kubernetes_url` - URL to the API server. *This is required*
|
15
15
|
* `apiVersion` - API version to use (default: `v1beta3`)
|
16
|
-
* `client_cert` - path to a client cert file to authenticate to the API server
|
17
|
-
* `client_key` - path to a client key file to authenticate to the API server
|
18
16
|
* `ca_file` - path to CA file for Kubernetes server certificate validation
|
19
17
|
* `verify_ssl` - validate SSL certificates (default: true)
|
18
|
+
* `client_cert` - path to a client cert file to authenticate to the API server
|
19
|
+
* `client_key` - path to a client key file to authenticate to the API server
|
20
|
+
* `bearer_token_file` - path to a file containing the bearer token to use for authentication
|
20
21
|
* `container_name_to_kubernetes_name_regexp` - the regular expression used to extract kubernetes metadata (pod name, container name, namespace) from the Docker container name. This must used named capture groups for `pod_container_name`, `pod_name` & `namespace` (default: `'\/?[^_]+_(?<pod_container_name>[^\.]+)[^_]+_(?<pod_name>[^_]+)_(?<namespace>[^_]+)'`)
|
21
22
|
* `cache_size` - size of the cache of Kubernetes metadata to reduce requests to the API server (default: `1000`)
|
22
23
|
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |gem|
|
6
6
|
gem.name = "fluent-plugin-kubernetes_metadata_filter"
|
7
|
-
gem.version = "0.
|
7
|
+
gem.version = "0.2.0"
|
8
8
|
gem.authors = ["Jimmi Dyson"]
|
9
9
|
gem.email = ["jimmidyson@gmail.com"]
|
10
10
|
gem.description = %q{Filter plugin to add Kubernetes metadata}
|
@@ -30,10 +30,11 @@ module Fluent
|
|
30
30
|
config_param :container_name_to_kubernetes_name_regexp,
|
31
31
|
:string,
|
32
32
|
:default => '\/?[^_]+_(?<pod_container_name>[^\.]+)[^_]+_(?<pod_name>[^_]+)_(?<namespace>[^_]+)'
|
33
|
+
config_param :bearer_token_file, :string, :default => ''
|
33
34
|
|
34
|
-
def
|
35
|
+
def get_metadata(pod_name, container_name, namespace)
|
35
36
|
begin
|
36
|
-
metadata =
|
37
|
+
metadata = @client.get_pod(pod_name, namespace)
|
37
38
|
if metadata
|
38
39
|
return {
|
39
40
|
:uid => metadata['metadata']['uid'],
|
@@ -60,19 +61,27 @@ module Fluent
|
|
60
61
|
require 'active_support/core_ext/object/blank'
|
61
62
|
require 'lru_redux'
|
62
63
|
|
63
|
-
|
64
|
+
@client = Kubeclient::Client.new @kubernetes_url, @apiVersion
|
64
65
|
|
65
|
-
|
66
|
+
@client.ssl_options(
|
67
|
+
client_cert: @client_cert.present? ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil,
|
68
|
+
client_key: @client_key.present? ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
|
69
|
+
ca_file: @ca_file,
|
70
|
+
verify_ssl: @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
|
71
|
+
)
|
66
72
|
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
)
|
73
|
+
if @bearer_token_file.present?
|
74
|
+
bearer_token = File.read(@bearer_token_file)
|
75
|
+
RestClient.add_before_execution_proc do |req, params|
|
76
|
+
req['authorization'] ||= "Bearer #{bearer_token}"
|
77
|
+
end
|
73
78
|
end
|
74
79
|
|
75
|
-
|
80
|
+
begin
|
81
|
+
@client.api_valid?
|
82
|
+
rescue KubeException => kube_error
|
83
|
+
raise Fluent::ConfigError, "Invalid Kubernetes API endpoint: #{kube_error.message}"
|
84
|
+
end
|
76
85
|
|
77
86
|
@cache = LruRedux::ThreadSafeCache.new(@cache_size)
|
78
87
|
@container_name_to_kubernetes_name_regexp_compiled = Regexp.compile(@container_name_to_kubernetes_name_regexp)
|
@@ -83,10 +92,11 @@ module Fluent
|
|
83
92
|
|
84
93
|
es.each {|time, record|
|
85
94
|
if record.has_key?(:docker) && record[:docker].has_key?(:name)
|
95
|
+
this = self
|
86
96
|
metadata = @cache.getset(record[:docker][:name]){
|
87
97
|
match_data = record[:docker][:name].match(@container_name_to_kubernetes_name_regexp_compiled)
|
88
98
|
if match_data
|
89
|
-
|
99
|
+
this.get_metadata(
|
90
100
|
match_data[:pod_name],
|
91
101
|
match_data[:pod_container_name],
|
92
102
|
match_data[:namespace]
|
@@ -104,4 +114,4 @@ module Fluent
|
|
104
114
|
end
|
105
115
|
end
|
106
116
|
|
107
|
-
end
|
117
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
|
3
|
+
# Kubernetes metadata
|
4
|
+
#
|
5
|
+
# Copyright 2015 Red Hat, Inc.
|
6
|
+
#
|
7
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
8
|
+
# you may not use this file except in compliance with the License.
|
9
|
+
# You may obtain a copy of the License at
|
10
|
+
#
|
11
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
12
|
+
#
|
13
|
+
# Unless required by applicable law or agreed to in writing, software
|
14
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
15
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
16
|
+
# See the License for the specific language governing permissions and
|
17
|
+
# limitations under the License.
|
18
|
+
#
|
19
|
+
---
|
20
|
+
http_interactions:
|
21
|
+
- request:
|
22
|
+
method: get
|
23
|
+
uri: https://localhost:8443/api
|
24
|
+
body:
|
25
|
+
encoding: US-ASCII
|
26
|
+
string: ''
|
27
|
+
headers:
|
28
|
+
Accept:
|
29
|
+
- "*/*; q=0.5, application/xml"
|
30
|
+
Accept-Encoding:
|
31
|
+
- gzip, deflate
|
32
|
+
User-Agent:
|
33
|
+
- Ruby
|
34
|
+
Authorization:
|
35
|
+
- Bearer YzYyYzFlODMtODdhNS00ZTMyLWIzMmItNmY4NDc4OTI1ZWF
|
36
|
+
response:
|
37
|
+
status:
|
38
|
+
code: 401
|
39
|
+
message: Unauthorized
|
40
|
+
headers:
|
41
|
+
Content-Type:
|
42
|
+
- text/plain; charset=utf-8
|
43
|
+
Date:
|
44
|
+
- Sat, 09 May 2015 14:04:39 GMT
|
45
|
+
Content-Length:
|
46
|
+
- '13'
|
47
|
+
body:
|
48
|
+
encoding: UTF-8
|
49
|
+
string: |
|
50
|
+
Unauthorized
|
51
|
+
http_version:
|
52
|
+
recorded_at: Sat, 09 May 2015 14:04:39 GMT
|
53
|
+
recorded_with: VCR 2.9.3
|
@@ -0,0 +1,202 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
|
3
|
+
# Kubernetes metadata
|
4
|
+
#
|
5
|
+
# Copyright 2015 Red Hat, Inc.
|
6
|
+
#
|
7
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
8
|
+
# you may not use this file except in compliance with the License.
|
9
|
+
# You may obtain a copy of the License at
|
10
|
+
#
|
11
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
12
|
+
#
|
13
|
+
# Unless required by applicable law or agreed to in writing, software
|
14
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
15
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
16
|
+
# See the License for the specific language governing permissions and
|
17
|
+
# limitations under the License.
|
18
|
+
#
|
19
|
+
---
|
20
|
+
http_interactions:
|
21
|
+
- request:
|
22
|
+
method: get
|
23
|
+
uri: https://localhost:8443/api
|
24
|
+
body:
|
25
|
+
encoding: US-ASCII
|
26
|
+
string: ''
|
27
|
+
headers:
|
28
|
+
Accept:
|
29
|
+
- "*/*; q=0.5, application/xml"
|
30
|
+
Accept-Encoding:
|
31
|
+
- gzip, deflate
|
32
|
+
User-Agent:
|
33
|
+
- Ruby
|
34
|
+
Authorization:
|
35
|
+
- Bearer YzYyYzFlODMtODdhNS00ZTMyLWIzMmItNmY4NDc4OTI1ZWFh
|
36
|
+
response:
|
37
|
+
status:
|
38
|
+
code: 200
|
39
|
+
message: OK
|
40
|
+
headers:
|
41
|
+
Content-Type:
|
42
|
+
- application/json
|
43
|
+
Date:
|
44
|
+
- Sat, 09 May 2015 13:51:07 GMT
|
45
|
+
Content-Length:
|
46
|
+
- '67'
|
47
|
+
body:
|
48
|
+
encoding: UTF-8
|
49
|
+
string: |-
|
50
|
+
{
|
51
|
+
"versions": [
|
52
|
+
"v1beta1",
|
53
|
+
"v1beta2",
|
54
|
+
"v1beta3"
|
55
|
+
]
|
56
|
+
}
|
57
|
+
http_version:
|
58
|
+
recorded_at: Sat, 09 May 2015 13:51:07 GMT
|
59
|
+
- request:
|
60
|
+
method: get
|
61
|
+
uri: https://localhost:8443/api/v1beta3/namespaces/default/pods/fabric8-console-controller-98rqc
|
62
|
+
body:
|
63
|
+
encoding: US-ASCII
|
64
|
+
string: ''
|
65
|
+
headers:
|
66
|
+
Accept:
|
67
|
+
- "*/*; q=0.5, application/xml"
|
68
|
+
Accept-Encoding:
|
69
|
+
- gzip, deflate
|
70
|
+
User-Agent:
|
71
|
+
- Ruby
|
72
|
+
Authorization:
|
73
|
+
- Bearer YzYyYzFlODMtODdhNS00ZTMyLWIzMmItNmY4NDc4OTI1ZWFh
|
74
|
+
response:
|
75
|
+
status:
|
76
|
+
code: 200
|
77
|
+
message: OK
|
78
|
+
headers:
|
79
|
+
Content-Type:
|
80
|
+
- application/json
|
81
|
+
Date:
|
82
|
+
- Sat, 09 May 2015 13:51:07 GMT
|
83
|
+
Transfer-Encoding:
|
84
|
+
- chunked
|
85
|
+
body:
|
86
|
+
encoding: UTF-8
|
87
|
+
string: |-
|
88
|
+
{
|
89
|
+
"kind": "Pod",
|
90
|
+
"apiVersion": "v1beta3",
|
91
|
+
"metadata": {
|
92
|
+
"name": "fabric8-console-controller-98rqc",
|
93
|
+
"generateName": "fabric8-console-controller-",
|
94
|
+
"namespace": "default",
|
95
|
+
"selfLink": "/api/v1beta3/namespaces/default/pods/fabric8-console-controller-98rqc",
|
96
|
+
"uid": "c76927af-f563-11e4-b32d-54ee7527188d",
|
97
|
+
"resourceVersion": "3556",
|
98
|
+
"creationTimestamp": "2015-05-08T09:22:42Z",
|
99
|
+
"labels": {
|
100
|
+
"component": "fabric8Console"
|
101
|
+
}
|
102
|
+
},
|
103
|
+
"spec": {
|
104
|
+
"volumes": [
|
105
|
+
{
|
106
|
+
"name": "openshift-cert-secrets",
|
107
|
+
"hostPath": null,
|
108
|
+
"emptyDir": null,
|
109
|
+
"gcePersistentDisk": null,
|
110
|
+
"gitRepo": null,
|
111
|
+
"secret": {
|
112
|
+
"secretName": "openshift-cert-secrets"
|
113
|
+
},
|
114
|
+
"nfs": null,
|
115
|
+
"iscsi": null,
|
116
|
+
"glusterfs": null
|
117
|
+
}
|
118
|
+
],
|
119
|
+
"containers": [
|
120
|
+
{
|
121
|
+
"name": "fabric8-console-container",
|
122
|
+
"image": "fabric8/hawtio-kubernetes:latest",
|
123
|
+
"ports": [
|
124
|
+
{
|
125
|
+
"containerPort": 9090,
|
126
|
+
"protocol": "TCP"
|
127
|
+
}
|
128
|
+
],
|
129
|
+
"env": [
|
130
|
+
{
|
131
|
+
"name": "OAUTH_CLIENT_ID",
|
132
|
+
"value": "fabric8-console"
|
133
|
+
},
|
134
|
+
{
|
135
|
+
"name": "OAUTH_AUTHORIZE_URI",
|
136
|
+
"value": "https://localhost:8443/oauth/authorize"
|
137
|
+
}
|
138
|
+
],
|
139
|
+
"resources": {},
|
140
|
+
"volumeMounts": [
|
141
|
+
{
|
142
|
+
"name": "openshift-cert-secrets",
|
143
|
+
"readOnly": true,
|
144
|
+
"mountPath": "/etc/secret-volume"
|
145
|
+
}
|
146
|
+
],
|
147
|
+
"terminationMessagePath": "/dev/termination-log",
|
148
|
+
"imagePullPolicy": "IfNotPresent",
|
149
|
+
"capabilities": {}
|
150
|
+
}
|
151
|
+
],
|
152
|
+
"restartPolicy": "Always",
|
153
|
+
"dnsPolicy": "ClusterFirst",
|
154
|
+
"host": "jimmi-redhat.localnet"
|
155
|
+
},
|
156
|
+
"status": {
|
157
|
+
"phase": "Running",
|
158
|
+
"Condition": [
|
159
|
+
{
|
160
|
+
"type": "Ready",
|
161
|
+
"status": "True"
|
162
|
+
}
|
163
|
+
],
|
164
|
+
"hostIP": "172.17.42.1",
|
165
|
+
"podIP": "172.17.0.4",
|
166
|
+
"containerStatuses": [
|
167
|
+
{
|
168
|
+
"name": "fabric8-console-container",
|
169
|
+
"state": {
|
170
|
+
"running": {
|
171
|
+
"startedAt": "2015-05-09T13:33:38Z"
|
172
|
+
}
|
173
|
+
},
|
174
|
+
"lastState": {},
|
175
|
+
"ready": true,
|
176
|
+
"restartCount": 2,
|
177
|
+
"image": "fabric8/hawtio-kubernetes:latest",
|
178
|
+
"imageID": "docker://b2bd1a24a68356b2f30128e6e28e672c1ef92df0d9ec01ec0c7faea5d77d2303",
|
179
|
+
"containerID": "docker://1b1d1f61c1205fe73328c75b2945e2ce05acfba2fde16299a8103fb22e9ec58a"
|
180
|
+
},
|
181
|
+
{
|
182
|
+
"name": "POD",
|
183
|
+
"state": {
|
184
|
+
"termination": {
|
185
|
+
"exitCode": 0,
|
186
|
+
"startedAt": "2015-05-09T09:45:08Z",
|
187
|
+
"finishedAt": "2015-05-09T09:47:00Z"
|
188
|
+
}
|
189
|
+
},
|
190
|
+
"lastState": {},
|
191
|
+
"ready": false,
|
192
|
+
"restartCount": 1,
|
193
|
+
"image": "openshift/origin-pod:v0.4.4",
|
194
|
+
"imageID": "docker://81fa37a83ec46fed9dddf30a9c3c43f54bfe4b65dea74a4c152fd515a0bf92a8",
|
195
|
+
"containerID": "docker://9c8ed1a9bbb6d9804b12379654461dcd3b447438c02c52ca7f7d2504e14ae96b"
|
196
|
+
}
|
197
|
+
]
|
198
|
+
}
|
199
|
+
}
|
200
|
+
http_version:
|
201
|
+
recorded_at: Sat, 09 May 2015 13:51:07 GMT
|
202
|
+
recorded_with: VCR 2.9.3
|
@@ -0,0 +1 @@
|
|
1
|
+
12345
|
@@ -61,6 +61,18 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
61
61
|
assert_equal(1, d.instance.cache_size)
|
62
62
|
end
|
63
63
|
end
|
64
|
+
|
65
|
+
test 'invalid API server config' do
|
66
|
+
VCR.use_cassette('invalid_api_server_config') do
|
67
|
+
assert_raise Fluent::ConfigError do
|
68
|
+
d = create_driver(%[
|
69
|
+
kubernetes_url https://localhost:8443
|
70
|
+
bearer_token_file test/plugin/test.token
|
71
|
+
verify_ssl false
|
72
|
+
])
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
64
76
|
end
|
65
77
|
|
66
78
|
sub_test_case 'filter_stream' do
|
@@ -99,6 +111,34 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
99
111
|
end
|
100
112
|
end
|
101
113
|
|
114
|
+
test 'with docker & kubernetes metadata using bearer token' do
|
115
|
+
VCR.use_cassette('kubernetes_docker_metadata_using_bearer_token') do
|
116
|
+
msg = {
|
117
|
+
:docker => {
|
118
|
+
:name => '/k8s_fabric8-console-container.efbd6e64_fabric8-console-controller-98rqc_default_c76927af-f563-11e4-b32d-54ee7527188d_42cbc279'
|
119
|
+
}
|
120
|
+
}
|
121
|
+
es = emit(msg, %[
|
122
|
+
kubernetes_url https://localhost:8443
|
123
|
+
verify_ssl false
|
124
|
+
bearer_token_file test/plugin/test.token
|
125
|
+
])
|
126
|
+
expected_kube_metadata = {
|
127
|
+
:kubernetes => {
|
128
|
+
:host => "jimmi-redhat.localnet",
|
129
|
+
:pod_name =>"fabric8-console-controller-98rqc",
|
130
|
+
:container_name => "fabric8-console-container",
|
131
|
+
:namespace => "default",
|
132
|
+
:uid => "c76927af-f563-11e4-b32d-54ee7527188d",
|
133
|
+
:labels => {
|
134
|
+
:component => "fabric8Console"
|
135
|
+
}
|
136
|
+
}
|
137
|
+
}
|
138
|
+
assert_equal(msg.merge(expected_kube_metadata), es.instance_variable_get(:@record_array)[0])
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
102
142
|
test 'with docker metadata, non-kubernetes' do
|
103
143
|
VCR.use_cassette('non_kubernetes_docker_metadata') do
|
104
144
|
msg = {
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kubernetes_metadata_filter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jimmi Dyson
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-05-
|
11
|
+
date: 2015-05-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -208,10 +208,13 @@ files:
|
|
208
208
|
- circle.yml
|
209
209
|
- fluent-plugin-kubernetes_metadata_filter.gemspec
|
210
210
|
- lib/fluent/plugin/filter_kubernetes_metadata.rb
|
211
|
+
- test/cassettes/invalid_api_server_config.yml
|
211
212
|
- test/cassettes/kubernetes_docker_metadata.yml
|
213
|
+
- test/cassettes/kubernetes_docker_metadata_using_bearer_token.yml
|
212
214
|
- test/cassettes/non_kubernetes_docker_metadata.yml
|
213
215
|
- test/cassettes/valid_kubernetes_api_server.yml
|
214
216
|
- test/helper.rb
|
217
|
+
- test/plugin/test.token
|
215
218
|
- test/plugin/test_filter_kubernetes_metadata.rb
|
216
219
|
homepage: https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter
|
217
220
|
licenses:
|
@@ -238,8 +241,11 @@ signing_key:
|
|
238
241
|
specification_version: 4
|
239
242
|
summary: Filter plugin to add Kubernetes metadata
|
240
243
|
test_files:
|
244
|
+
- test/cassettes/invalid_api_server_config.yml
|
241
245
|
- test/cassettes/kubernetes_docker_metadata.yml
|
246
|
+
- test/cassettes/kubernetes_docker_metadata_using_bearer_token.yml
|
242
247
|
- test/cassettes/non_kubernetes_docker_metadata.yml
|
243
248
|
- test/cassettes/valid_kubernetes_api_server.yml
|
244
249
|
- test/helper.rb
|
250
|
+
- test/plugin/test.token
|
245
251
|
- test/plugin/test_filter_kubernetes_metadata.rb
|