fluent-plugin-kubernetes_metadata_filter 0.4.0 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fc794d4eb88ae0226aaeaa73ed83ca71149db1a3
|
4
|
+
data.tar.gz: 9c69ce241a593c562f4cccb244dc940111a6fc8c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 24575557e613c96da0c79d9592c740f3ced4c0f6c36ff18b9af7aac88483af6b1e790f04b71425aa04a13e9390c629c7ce3a2332fd94dde6b41bf4461a8f47f3
|
7
|
+
data.tar.gz: 2f9053e6a5775b2d4a857b6d3ad67821b64d9007c93f7019d58f0326b59324752fdee04f2305612ee79a8c973653c0e95cdae7d6b3df4d23a3f14118cb0116c3
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |gem|
|
6
6
|
gem.name = "fluent-plugin-kubernetes_metadata_filter"
|
7
|
-
gem.version = "0.
|
7
|
+
gem.version = "0.5.0"
|
8
8
|
gem.authors = ["Jimmi Dyson"]
|
9
9
|
gem.email = ["jimmidyson@gmail.com"]
|
10
10
|
gem.description = %q{Filter plugin to add Kubernetes metadata}
|
@@ -21,30 +21,31 @@ module Fluent
|
|
21
21
|
Fluent::Plugin.register_filter('kubernetes_metadata', self)
|
22
22
|
|
23
23
|
config_param :kubernetes_url, :string
|
24
|
-
config_param :cache_size, :integer, :
|
25
|
-
config_param :cache_ttl, :integer, :
|
26
|
-
config_param :watch, :bool, :
|
27
|
-
config_param :apiVersion, :string, :
|
28
|
-
config_param :client_cert, :string, :
|
29
|
-
config_param :client_key, :string, :
|
30
|
-
config_param :ca_file, :string, :
|
31
|
-
config_param :verify_ssl, :bool, :
|
24
|
+
config_param :cache_size, :integer, default: 1000
|
25
|
+
config_param :cache_ttl, :integer, default: 60 * 60
|
26
|
+
config_param :watch, :bool, default: true
|
27
|
+
config_param :apiVersion, :string, default: 'v1beta3'
|
28
|
+
config_param :client_cert, :string, default: ''
|
29
|
+
config_param :client_key, :string, default: ''
|
30
|
+
config_param :ca_file, :string, default: ''
|
31
|
+
config_param :verify_ssl, :bool, default: true
|
32
32
|
config_param :container_name_to_kubernetes_name_regexp,
|
33
33
|
:string,
|
34
34
|
:default => '\/?[^_]+_(?<pod_container_name>[^\.]+)[^_]+_(?<pod_name>[^_]+)_(?<namespace>[^_]+)'
|
35
|
-
config_param :bearer_token_file, :string, :
|
35
|
+
config_param :bearer_token_file, :string, default: ''
|
36
|
+
config_param :merge_json_log, :bool, default: true
|
36
37
|
|
37
38
|
def get_metadata(pod_name, container_name, namespace)
|
38
39
|
begin
|
39
40
|
metadata = @client.get_pod(pod_name, namespace)
|
40
41
|
if metadata
|
41
42
|
return {
|
42
|
-
uid:
|
43
|
-
namespace:
|
44
|
-
pod_name:
|
43
|
+
uid: metadata['metadata']['uid'],
|
44
|
+
namespace: metadata['metadata']['namespace'],
|
45
|
+
pod_name: metadata['metadata']['name'],
|
45
46
|
container_name: container_name,
|
46
|
-
labels:
|
47
|
-
host:
|
47
|
+
labels: metadata['metadata']['labels'].to_h,
|
48
|
+
host: metadata['spec']['host']
|
48
49
|
}
|
49
50
|
end
|
50
51
|
rescue KubeException
|
@@ -67,9 +68,9 @@ module Fluent
|
|
67
68
|
|
68
69
|
@client.ssl_options(
|
69
70
|
client_cert: @client_cert.present? ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil,
|
70
|
-
client_key:
|
71
|
-
ca_file:
|
72
|
-
verify_ssl:
|
71
|
+
client_key: @client_key.present? ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
|
72
|
+
ca_file: @ca_file,
|
73
|
+
verify_ssl: @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
|
73
74
|
)
|
74
75
|
|
75
76
|
if @bearer_token_file.present?
|
@@ -86,11 +87,11 @@ module Fluent
|
|
86
87
|
if @cache_ttl < 0
|
87
88
|
@cache_ttl = :none
|
88
89
|
end
|
89
|
-
@cache
|
90
|
+
@cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl)
|
90
91
|
@container_name_to_kubernetes_name_regexp_compiled = Regexp.compile(@container_name_to_kubernetes_name_regexp)
|
91
92
|
|
92
93
|
if @watch
|
93
|
-
thread
|
94
|
+
thread = Thread.new(self) { |this|
|
94
95
|
this.start_watch
|
95
96
|
}
|
96
97
|
thread.abort_on_exception = true
|
@@ -100,10 +101,10 @@ module Fluent
|
|
100
101
|
def filter_stream(tag, es)
|
101
102
|
new_es = MultiEventStream.new
|
102
103
|
|
103
|
-
es.each {|time, record|
|
104
|
+
es.each { |time, record|
|
104
105
|
if record.has_key?(:docker) && record[:docker].has_key?(:id) && record[:docker].has_key?(:name)
|
105
|
-
this
|
106
|
-
metadata
|
106
|
+
this = self
|
107
|
+
metadata = @cache.getset(record[:docker][:id]) {
|
107
108
|
match_data = record[:docker][:name].match(@container_name_to_kubernetes_name_regexp_compiled)
|
108
109
|
if match_data
|
109
110
|
this.get_metadata(
|
@@ -117,27 +118,47 @@ module Fluent
|
|
117
118
|
record[:kubernetes] = metadata if metadata
|
118
119
|
end
|
119
120
|
|
121
|
+
if @merge_json_log
|
122
|
+
record = merge_json_log(record)
|
123
|
+
end
|
124
|
+
|
120
125
|
new_es.add(time, record)
|
121
126
|
}
|
122
127
|
|
123
128
|
new_es
|
124
129
|
end
|
125
130
|
|
131
|
+
def merge_json_log(record)
|
132
|
+
if record.has_key?('log')
|
133
|
+
log = record['log'].strip
|
134
|
+
if log[0].eql?('{') && log[-1].eql?('}')
|
135
|
+
begin
|
136
|
+
parsed_log = JSON.parse(log)
|
137
|
+
record = record.merge(parsed_log)
|
138
|
+
unless parsed_log.has_key?('log')
|
139
|
+
record.delete('log')
|
140
|
+
end
|
141
|
+
rescue JSON::ParserError
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
145
|
+
record
|
146
|
+
end
|
147
|
+
|
126
148
|
def start_watch
|
127
149
|
resource_version = @client.get_pods.resourceVersion
|
128
|
-
watcher
|
150
|
+
watcher = @client.watch_pods(resource_version)
|
129
151
|
watcher.each do |notice|
|
130
|
-
puts notice
|
131
152
|
case notice.type
|
132
153
|
when 'MODIFIED'
|
133
154
|
if notice.object.status.containerStatuses
|
134
155
|
notice.object.status.containerStatuses.each { |container_status|
|
135
156
|
if container_status['containerId']
|
136
157
|
containerId = container_status['containerId'].sub(/^docker:\/\//, '')
|
137
|
-
cached
|
158
|
+
cached = @cache[containerId]
|
138
159
|
if cached
|
139
160
|
# Only thing that can be modified is labels
|
140
|
-
cached[:labels]
|
161
|
+
cached[:labels] = v.object.metadata.labels.to_h
|
141
162
|
@cache[containerId] = cached
|
142
163
|
end
|
143
164
|
end
|
@@ -168,5 +168,21 @@ class KubernetesMetadataFilterTest < Test::Unit::TestCase
|
|
168
168
|
end
|
169
169
|
end
|
170
170
|
|
171
|
+
test 'merges json log data' do
|
172
|
+
VCR.use_cassette('valid_kubernetes_api_server') do
|
173
|
+
json_log = {
|
174
|
+
'hello' => 'world'
|
175
|
+
}
|
176
|
+
msg = {
|
177
|
+
'log' => "#{json_log.to_json}"
|
178
|
+
}
|
179
|
+
es = emit(msg, '
|
180
|
+
kubernetes_url https://localhost:8443
|
181
|
+
verify_ssl false
|
182
|
+
watch false
|
183
|
+
')
|
184
|
+
assert_equal(json_log, es.instance_variable_get(:@record_array)[0])
|
185
|
+
end
|
186
|
+
end
|
171
187
|
end
|
172
188
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kubernetes_metadata_filter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jimmi Dyson
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-05-
|
11
|
+
date: 2015-05-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|