fluent-plugin-kubernetes_metadata_filter 1.0.0 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: b64ea97cd4df7be745b06c0f46e82f152e0503ea
4
- data.tar.gz: b026a7a6039cd7f15aa938be5969fa7e5a818c7b
3
+ metadata.gz: ef179686ef1088fb46084410bdeb0a2c4b07c3f6
4
+ data.tar.gz: 02b2dfe0880adb892922876f5dfd123a54922d76
5
5
  SHA512:
6
- metadata.gz: 53f23402783e6a0b7857698b99a3a7dfd0e343f4ccd5b7ea2c440d8c3fc624d61c4ef89846aa2c91b90995e8df7feb408df7cd5f933d577b3bcfe76e5ced5174
7
- data.tar.gz: 912ffc8b2ba2e376bb0ffffe9bd9a881cf2ce6f535328e2c76a53941558d0c7e2209a2359ba319d3d8deb0278252673c15b84c169464bfd8dfc8aa5101adacf8
6
+ metadata.gz: 5972e28a29443b1e11fd7071ee1b7df5ee5b93be0f406a3bac51979058c49651777515acdc12c63ea2d4f9f84f632668a7d2fef592b9aa4d0249def6b54bdf2b
7
+ data.tar.gz: b079681eceb4fd3e1e182c133fd529e9820137cb7b139ffcf63a85b082b8fdeda0ee735ff271e28fa081f348451f2684279aeee8565928d5bd33ba4fc178b851
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |gem|
6
6
  gem.name = "fluent-plugin-kubernetes_metadata_filter"
7
- gem.version = "1.0.0"
7
+ gem.version = "1.0.1"
8
8
  gem.authors = ["Jimmi Dyson"]
9
9
  gem.email = ["jimmidyson@gmail.com"]
10
10
  gem.description = %q{Filter plugin to add Kubernetes metadata}
@@ -17,6 +17,8 @@
17
17
  # limitations under the License.
18
18
  #
19
19
 
20
+ require_relative 'kubernetes_metadata_cache_strategy'
21
+ require_relative 'kubernetes_metadata_common'
20
22
  require_relative 'kubernetes_metadata_stats'
21
23
  require_relative 'kubernetes_metadata_watch_namespaces'
22
24
  require_relative 'kubernetes_metadata_watch_pods'
@@ -26,6 +28,7 @@ module Fluent
26
28
  K8_POD_CA_CERT = 'ca.crt'
27
29
  K8_POD_TOKEN = 'token'
28
30
 
31
+ include KubernetesMetadata::CacheStrategy
29
32
  include KubernetesMetadata::Common
30
33
  include KubernetesMetadata::WatchNamespaces
31
34
  include KubernetesMetadata::WatchPods
@@ -252,7 +255,7 @@ module Fluent
252
255
 
253
256
  end
254
257
 
255
- def get_metadata_for_record(match_data, cache_key, create_time)
258
+ def get_metadata_for_record(match_data, cache_key, create_time, batch_miss_cache)
256
259
  namespace_name = match_data['namespace']
257
260
  pod_name = match_data['pod_name']
258
261
  metadata = {
@@ -261,58 +264,12 @@ module Fluent
261
264
  'pod_name' => pod_name
262
265
  }
263
266
  if @kubernetes_url.present?
264
- pod_metadata = get_pod_metadata(cache_key, namespace_name, pod_name, create_time)
267
+ pod_metadata = get_pod_metadata(cache_key, namespace_name, pod_name, create_time, batch_miss_cache)
265
268
  metadata.merge!(pod_metadata) if pod_metadata
266
269
  end
267
270
  metadata
268
271
  end
269
272
 
270
- def get_pod_metadata(key, namespace_name, pod_name, record_create_time)
271
- ids = @id_cache.fetch(key) do
272
- #no metadata found
273
- @stats.bump(:id_cache_miss)
274
- pod_metadata = fetch_pod_metadata(namespace_name, pod_name)
275
- namespace_metadata = fetch_namespace_metadata(namespace_name)
276
- ids = {:pod_id=> pod_metadata['pod_id'], :namespace_id => namespace_metadata['namespace_id'] }
277
- # pod not found or namespace not found
278
- if ids[:pod_id].nil? || ids[:namespace_id].nil?
279
- # pod notfound & namespace found
280
- if ids[:pod_id].nil? && !ids[:namespace_id].nil?
281
- ns_time = Time.parse(namespace_metadata['creation_timestamp'])
282
- ids[:pod_id] = key if ns_time <= record_create_time #ns is older then record
283
- ids
284
- else
285
- # nothing found
286
- @stats.bump(:pod_cache_orphaned_record)
287
- ids = :orphaned
288
- end
289
- end
290
- ids
291
- end
292
-
293
- if ids == :orphaned
294
- return {} unless @allow_orphans
295
- log.trace("orphaning message for: #{namespace_name}/#{pod_name} ") if log.trace?
296
- return {
297
- 'orphaned_namespace' => namespace_name,
298
- 'namespace_name' => @orphaned_namespace_name,
299
- 'namespace_id' => @orphaned_namespace_id
300
- }
301
- end
302
- metadata = @cache.fetch(ids[:pod_id]) do
303
- @stats.bump(:pod_cache_miss)
304
- m = fetch_pod_metadata(namespace_name, pod_name)
305
- (m.nil? || m.empty?) ? {'pod_id'=>ids[:pod_id]} : m
306
- end
307
- metadata.merge!(@namespace_cache.fetch(ids[:namespace_id]) do
308
- @stats.bump(:namespace_cache_miss)
309
- m = fetch_namespace_metadata(namespace_name)
310
- (m.nil? || m.empty?) ? {'namespace_id'=>ids[:namespace_id]} : m
311
- end)
312
- metadata.delete('creation_timestamp') #remove namespace info thats only used for comparison
313
- metadata
314
- end
315
-
316
273
  def create_time_from_record(record)
317
274
  time = if @use_journal
318
275
  record['_SOURCE_REALTIME_TIMESTAMP'].nil? ? record['_SOURCE_REALTIME_TIMESTAMP'] : record['__REALTIME_TIMESTAMP']
@@ -330,14 +287,14 @@ module Fluent
330
287
  new_es = MultiEventStream.new
331
288
 
332
289
  match_data = tag.match(@tag_to_kubernetes_name_regexp_compiled)
333
-
290
+ batch_miss_cache = {}
334
291
  if match_data
335
292
  container_id = match_data['docker_id']
336
293
  metadata = {
337
294
  'docker' => {
338
295
  'container_id' => container_id
339
296
  },
340
- 'kubernetes' => get_metadata_for_record(match_data, container_id, create_time_from_record(es.first[1]))
297
+ 'kubernetes' => get_metadata_for_record(match_data, container_id, create_time_from_record(es.first[1]), batch_miss_cache)
341
298
  }
342
299
  end
343
300
 
@@ -354,7 +311,7 @@ module Fluent
354
311
 
355
312
  def filter_stream_from_journal(tag, es)
356
313
  new_es = MultiEventStream.new
357
-
314
+ batch_miss_cache = {}
358
315
  es.each { |time, record|
359
316
  record = merge_json_log(record) if @merge_json_log
360
317
  metadata = nil
@@ -365,7 +322,7 @@ module Fluent
365
322
  'docker' => {
366
323
  'container_id' => container_id
367
324
  },
368
- 'kubernetes' => get_metadata_for_record(match_data, container_id, create_time_from_record(record))
325
+ 'kubernetes' => get_metadata_for_record(match_data, container_id, create_time_from_record(record), batch_miss_cache)
369
326
  }
370
327
 
371
328
  metadata
@@ -0,0 +1,93 @@
1
+ #
2
+ # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
3
+ # Kubernetes metadata
4
+ #
5
+ # Copyright 2017 Red Hat, Inc.
6
+ #
7
+ # Licensed under the Apache License, Version 2.0 (the "License");
8
+ # you may not use this file except in compliance with the License.
9
+ # You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing, software
14
+ # distributed under the License is distributed on an "AS IS" BASIS,
15
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ # See the License for the specific language governing permissions and
17
+ # limitations under the License.
18
+ #
19
+ module KubernetesMetadata
20
+ module CacheStrategy
21
+
22
+ def get_pod_metadata(key, namespace_name, pod_name, record_create_time, batch_miss_cache)
23
+ metadata = {}
24
+ ids = @id_cache[key]
25
+ if !ids.nil?
26
+ # FAST PATH
27
+ # Cache hit, fetch metadata from the cache
28
+ metadata = @cache.fetch(ids[:pod_id]) do
29
+ @stats.bump(:pod_cache_miss)
30
+ m = fetch_pod_metadata(namespace_name, pod_name)
31
+ (m.nil? || m.empty?) ? {'pod_id'=>ids[:pod_id]} : m
32
+ end
33
+ metadata.merge!(@namespace_cache.fetch(ids[:namespace_id]) do
34
+ @stats.bump(:namespace_cache_miss)
35
+ m = fetch_namespace_metadata(namespace_name)
36
+ (m.nil? || m.empty?) ? {'namespace_id'=>ids[:namespace_id]} : m
37
+ end)
38
+ else
39
+ # SLOW PATH
40
+ @stats.bump(:id_cache_miss)
41
+ return batch_miss_cache["#{namespace_name}_#{pod_name}"] if batch_miss_cache.key?("#{namespace_name}_#{pod_name}")
42
+ pod_metadata = fetch_pod_metadata(namespace_name, pod_name)
43
+ namespace_metadata = fetch_namespace_metadata(namespace_name)
44
+ ids = { :pod_id=> pod_metadata['pod_id'], :namespace_id => namespace_metadata['namespace_id'] }
45
+ if !ids[:pod_id].nil? && !ids[:namespace_id].nil?
46
+ # pod found and namespace found
47
+ metadata = pod_metadata
48
+ metadata.merge!(namespace_metadata)
49
+ else
50
+ if ids[:pod_id].nil? && !ids[:namespace_id].nil?
51
+ # pod not found, but namespace found
52
+ @stats.bump(:id_cache_pod_not_found_namespace)
53
+ ns_time = Time.parse(namespace_metadata['creation_timestamp'])
54
+ if ns_time <= record_create_time
55
+ # namespace is older then record for pod
56
+ ids[:pod_id] = key
57
+ metadata = @cache.fetch(ids[:pod_id]) do
58
+ m = { 'pod_id' => ids[:pod_id] }
59
+ end
60
+ end
61
+ metadata.merge!(namespace_metadata)
62
+ else
63
+ if !ids[:pod_id].nil? && ids[:namespace_id].nil?
64
+ # pod found, but namespace NOT found
65
+ # this should NEVER be possible since pod meta can
66
+ # only be retrieved with a namespace
67
+ @stats.bump(:id_cache_namespace_not_found_pod)
68
+ else
69
+ # nothing found
70
+ @stats.bump(:id_cache_orphaned_record)
71
+ end
72
+ if @allow_orphans
73
+ log.trace("orphaning message for: #{namespace_name}/#{pod_name} ") if log.trace?
74
+ metadata = {
75
+ 'orphaned_namespace' => namespace_name,
76
+ 'namespace_name' => @orphaned_namespace_name,
77
+ 'namespace_id' => @orphaned_namespace_id
78
+ }
79
+ else
80
+ metadata = {}
81
+ end
82
+ batch_miss_cache["#{namespace_name}_#{pod_name}"] = metadata
83
+ end
84
+ end
85
+ @id_cache[key] = ids unless batch_miss_cache.key?("#{namespace_name}_#{pod_name}")
86
+ end
87
+ # remove namespace info that is only used for comparison
88
+ metadata.delete('creation_timestamp')
89
+ metadata.delete_if{|k,v| v.nil?}
90
+ end
91
+
92
+ end
93
+ end
@@ -0,0 +1,196 @@
1
+ #
2
+ # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
3
+ # Kubernetes metadata
4
+ #
5
+ # Copyright 2015 Red Hat, Inc.
6
+ #
7
+ # Licensed under the Apache License, Version 2.0 (the "License");
8
+ # you may not use this file except in compliance with the License.
9
+ # You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing, software
14
+ # distributed under the License is distributed on an "AS IS" BASIS,
15
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ # See the License for the specific language governing permissions and
17
+ # limitations under the License.
18
+ #
19
+ require_relative '../helper'
20
+ require_relative '../../lib/fluent/plugin/kubernetes_metadata_cache_strategy'
21
+ require_relative '../../lib/fluent/plugin/kubernetes_metadata_stats'
22
+ require 'lru_redux'
23
+ require 'webmock/test_unit'
24
+ WebMock.disable_net_connect!
25
+
26
+ class TestCacheStrategy
27
+ include KubernetesMetadata::CacheStrategy
28
+
29
+ def initialize
30
+ @stats = KubernetesMetadata::Stats.new
31
+ @cache = LruRedux::TTL::ThreadSafeCache.new(100,3600)
32
+ @id_cache = LruRedux::TTL::ThreadSafeCache.new(100,3600)
33
+ @namespace_cache = LruRedux::TTL::ThreadSafeCache.new(100,3600)
34
+ @orphaned_namespace_name = '.orphaned'
35
+ @orphaned_namespace_id = 'orphaned'
36
+ end
37
+
38
+ attr_accessor :stats, :cache, :id_cache, :namespace_cache, :allow_orphans
39
+
40
+ def fetch_pod_metadata(namespace_name, pod_name)
41
+ end
42
+
43
+ def fetch_namespace_metadata(namespace_name)
44
+ end
45
+
46
+ def log
47
+ logger = {}
48
+ def logger.trace?
49
+ true
50
+ end
51
+ def logger.trace(message)
52
+ end
53
+ logger
54
+ end
55
+
56
+ end
57
+
58
+ class KubernetesMetadataCacheStrategyTest < Test::Unit::TestCase
59
+
60
+ def setup
61
+ @strategy = TestCacheStrategy.new
62
+ @cache_key = 'some_long_container_id'
63
+ @namespace_name = 'some_namespace_name'
64
+ @namespace_uuid = 'some_namespace_uuid'
65
+ @pod_name = 'some_pod_name'
66
+ @pod_uuid = 'some_pod_uuid'
67
+ @time = Time.now
68
+ @pod_meta = {'pod_id'=> @pod_uuid, 'labels'=> {'meta'=>'pod'}}
69
+ @namespace_meta = {'namespace_id'=> @namespace_uuid, 'creation_timestamp'=>@time.to_s}
70
+ end
71
+
72
+ test 'when cached metadata is found' do
73
+ exp = @pod_meta.merge(@namespace_meta)
74
+ exp.delete('creation_timestamp')
75
+ @strategy.id_cache[@cache_key] = {
76
+ pod_id: @pod_uuid,
77
+ namespace_id: @namespace_uuid
78
+ }
79
+ @strategy.cache[@pod_uuid] = @pod_meta
80
+ @strategy.namespace_cache[@namespace_uuid] = @namespace_meta
81
+ assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, {}))
82
+ end
83
+
84
+ test 'when previously processed record for pod but metadata is not cached and can not be fetched' do
85
+ exp = {
86
+ 'pod_id'=> @pod_uuid,
87
+ 'namespace_id'=> @namespace_uuid
88
+ }
89
+ @strategy.id_cache[@cache_key] = {
90
+ pod_id: @pod_uuid,
91
+ namespace_id: @namespace_uuid
92
+ }
93
+ @strategy.stub :fetch_pod_metadata, {} do
94
+ @strategy.stub :fetch_namespace_metadata, nil do
95
+ assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, {}))
96
+ end
97
+ end
98
+ end
99
+
100
+ test 'when metadata is not cached and is fetched' do
101
+ exp = @pod_meta.merge(@namespace_meta)
102
+ exp.delete('creation_timestamp')
103
+ @strategy.stub :fetch_pod_metadata, @pod_meta do
104
+ @strategy.stub :fetch_namespace_metadata, @namespace_meta do
105
+ assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, {}))
106
+ assert_true(@strategy.id_cache.key?(@cache_key))
107
+ end
108
+ end
109
+ end
110
+
111
+ test 'when metadata is not cached and pod is deleted and namespace metadata is fetched' do
112
+ # this is the case for a record from a deleted pod where no other
113
+ # records were read. using the container hash since that is all
114
+ # we ever will have and should allow us to process all the deleted
115
+ # pod records
116
+ exp = {
117
+ 'pod_id'=> @cache_key,
118
+ 'namespace_id'=> @namespace_uuid
119
+ }
120
+ @strategy.stub :fetch_pod_metadata, {} do
121
+ @strategy.stub :fetch_namespace_metadata, @namespace_meta do
122
+ assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, {}))
123
+ assert_true(@strategy.id_cache.key?(@cache_key))
124
+ end
125
+ end
126
+ end
127
+
128
+ test 'when metadata is not cached and pod is deleted and namespace is for a different namespace with the same name' do
129
+ # this is the case for a record from a deleted pod from a deleted namespace
130
+ # where new namespace was created with the same name
131
+ exp = {
132
+ 'namespace_id'=> @namespace_uuid
133
+ }
134
+ @strategy.stub :fetch_pod_metadata, {} do
135
+ @strategy.stub :fetch_namespace_metadata, @namespace_meta do
136
+ assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time - 1*86400, {}))
137
+ assert_true(@strategy.id_cache.key?(@cache_key))
138
+ end
139
+ end
140
+ end
141
+
142
+ test 'when metadata is not cached and no metadata can be fetched and not allowing orphans' do
143
+ # we should never see this since pod meta should not be retrievable
144
+ # unless the namespace exists
145
+ @strategy.stub :fetch_pod_metadata, @pod_meta do
146
+ @strategy.stub :fetch_namespace_metadata, {} do
147
+ assert_equal({}, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time - 1*86400, {}))
148
+ end
149
+ end
150
+ end
151
+
152
+ test 'when metadata is not cached and no metadata can be fetched and allowing orphans' do
153
+ # we should never see this since pod meta should not be retrievable
154
+ # unless the namespace exists
155
+ @strategy.allow_orphans = true
156
+ exp = {
157
+ 'orphaned_namespace' => 'namespace',
158
+ 'namespace_name' => '.orphaned',
159
+ 'namespace_id' => 'orphaned'
160
+ }
161
+ @strategy.stub :fetch_pod_metadata, @pod_meta do
162
+ @strategy.stub :fetch_namespace_metadata, {} do
163
+ assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time - 1*86400, {}))
164
+ end
165
+ end
166
+ end
167
+
168
+ test 'when metadata is not cached and no metadata can be fetched and not allowing orphans for multiple records' do
169
+ # processing a batch of records with no meta. ideally we only hit the api server once
170
+ batch_miss_cache = {}
171
+ @strategy.stub :fetch_pod_metadata, {} do
172
+ @strategy.stub :fetch_namespace_metadata, {} do
173
+ assert_equal({}, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, batch_miss_cache))
174
+ end
175
+ end
176
+ assert_equal({}, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, batch_miss_cache))
177
+ end
178
+
179
+ test 'when metadata is not cached and no metadata can be fetched and allowing orphans for multiple records' do
180
+ # we should never see this since pod meta should not be retrievable
181
+ # unless the namespace exists
182
+ @strategy.allow_orphans = true
183
+ exp = {
184
+ 'orphaned_namespace' => 'namespace',
185
+ 'namespace_name' => '.orphaned',
186
+ 'namespace_id' => 'orphaned'
187
+ }
188
+ batch_miss_cache = {}
189
+ @strategy.stub :fetch_pod_metadata, {} do
190
+ @strategy.stub :fetch_namespace_metadata, {} do
191
+ assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, batch_miss_cache))
192
+ end
193
+ end
194
+ assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, batch_miss_cache))
195
+ end
196
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-kubernetes_metadata_filter
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.0
4
+ version: 1.0.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jimmi Dyson
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-12-04 00:00:00.000000000 Z
11
+ date: 2018-01-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -207,6 +207,7 @@ files:
207
207
  - circle.yml
208
208
  - fluent-plugin-kubernetes_metadata_filter.gemspec
209
209
  - lib/fluent/plugin/filter_kubernetes_metadata.rb
210
+ - lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb
210
211
  - lib/fluent/plugin/kubernetes_metadata_common.rb
211
212
  - lib/fluent/plugin/kubernetes_metadata_stats.rb
212
213
  - lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb
@@ -222,6 +223,7 @@ files:
222
223
  - test/helper.rb
223
224
  - test/plugin/test.token
224
225
  - test/plugin/test_cache_stats.rb
226
+ - test/plugin/test_cache_strategy.rb
225
227
  - test/plugin/test_filter_kubernetes_metadata.rb
226
228
  - test/plugin/test_watch_namespaces.rb
227
229
  - test/plugin/test_watch_pods.rb
@@ -262,6 +264,7 @@ test_files:
262
264
  - test/helper.rb
263
265
  - test/plugin/test.token
264
266
  - test/plugin/test_cache_stats.rb
267
+ - test/plugin/test_cache_strategy.rb
265
268
  - test/plugin/test_filter_kubernetes_metadata.rb
266
269
  - test/plugin/test_watch_namespaces.rb
267
270
  - test/plugin/test_watch_pods.rb