fluent-plugin-kubernetes_metadata_filter 2.5.0 → 3.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +10 -14
  3. data/.gitignore +0 -1
  4. data/.rubocop.yml +57 -0
  5. data/Gemfile +4 -2
  6. data/Gemfile.lock +76 -67
  7. data/README.md +9 -83
  8. data/Rakefile +15 -11
  9. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/cpu.png +0 -0
  10. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/latency.png +0 -0
  11. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/loss.png +0 -0
  12. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/mem.png +0 -0
  13. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/readme.md +88 -0
  14. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/results.html +127 -0
  15. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/cpu.png +0 -0
  16. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/latency.png +0 -0
  17. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/loss.png +0 -0
  18. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/mem.png +0 -0
  19. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/readme.md +97 -0
  20. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/results.html +136 -0
  21. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/cpu.png +0 -0
  22. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/latency.png +0 -0
  23. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/loss.png +0 -0
  24. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/mem.png +0 -0
  25. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/readme.md +97 -0
  26. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/results.html +136 -0
  27. data/fluent-plugin-kubernetes_metadata_filter.gemspec +25 -27
  28. data/lib/fluent/plugin/filter_kubernetes_metadata.rb +171 -192
  29. data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb +25 -23
  30. data/lib/fluent/plugin/kubernetes_metadata_common.rb +44 -69
  31. data/lib/fluent/plugin/kubernetes_metadata_stats.rb +21 -5
  32. data/lib/fluent/plugin/kubernetes_metadata_test_api_adapter.rb +68 -0
  33. data/lib/fluent/plugin/kubernetes_metadata_util.rb +33 -0
  34. data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +91 -42
  35. data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +108 -47
  36. data/release_notes.md +42 -0
  37. data/test/cassettes/kubernetes_get_pod_container_init.yml +145 -0
  38. data/test/helper.rb +20 -2
  39. data/test/plugin/test_cache_stats.rb +10 -13
  40. data/test/plugin/test_cache_strategy.rb +158 -160
  41. data/test/plugin/test_filter_kubernetes_metadata.rb +340 -616
  42. data/test/plugin/test_watch_namespaces.rb +188 -125
  43. data/test/plugin/test_watch_pods.rb +282 -202
  44. data/test/plugin/watch_test.rb +16 -15
  45. metadata +77 -67
  46. /data/test/cassettes/{kubernetes_docker_metadata_dotted_labels.yml → kubernetes_docker_metadata_dotted_slashed_labels.yml} +0 -0
data/test/plugin/test_cache_strategy.rb CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  #
2
4
  # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
3
5
  # Kubernetes metadata
@@ -17,180 +19,176 @@
17
19
  # limitations under the License.
18
20
  #
19
21
  require_relative '../helper'
20
- require_relative '../../lib/fluent/plugin/kubernetes_metadata_cache_strategy'
21
- require_relative '../../lib/fluent/plugin/kubernetes_metadata_stats'
22
- require 'lru_redux'
23
- require 'webmock/test_unit'
24
- WebMock.disable_net_connect!
25
22
 
26
23
  class TestCacheStrategy
27
- include KubernetesMetadata::CacheStrategy
28
-
29
- def initialize
30
- @stats = KubernetesMetadata::Stats.new
31
- @cache = LruRedux::TTL::ThreadSafeCache.new(100,3600)
32
- @id_cache = LruRedux::TTL::ThreadSafeCache.new(100,3600)
33
- @namespace_cache = LruRedux::TTL::ThreadSafeCache.new(100,3600)
34
- @orphaned_namespace_name = '.orphaned'
35
- @orphaned_namespace_id = 'orphaned'
24
+ include KubernetesMetadata::CacheStrategy
25
+
26
+ def initialize
27
+ @stats = KubernetesMetadata::Stats.new
28
+ @cache = LruRedux::TTL::ThreadSafeCache.new(100, 3600)
29
+ @id_cache = LruRedux::TTL::ThreadSafeCache.new(100, 3600)
30
+ @namespace_cache = LruRedux::TTL::ThreadSafeCache.new(100, 3600)
31
+ @orphaned_namespace_name = '.orphaned'
32
+ @orphaned_namespace_id = 'orphaned'
33
+ end
34
+
35
+ attr_accessor :stats, :cache, :id_cache, :namespace_cache, :allow_orphans
36
+
37
+ def fetch_pod_metadata(_namespace_name, _pod_name)
38
+ {}
39
+ end
40
+
41
+ def fetch_namespace_metadata(_namespace_name)
42
+ {}
43
+ end
44
+
45
+ def log
46
+ logger = {}
47
+ def logger.on_trace
48
+ true
36
49
  end
37
50
 
38
- attr_accessor :stats, :cache, :id_cache, :namespace_cache, :allow_orphans
39
-
40
- def fetch_pod_metadata(namespace_name, pod_name)
51
+ def logger.trace(message)
41
52
  end
42
-
43
- def fetch_namespace_metadata(namespace_name)
44
- end
45
-
46
- def log
47
- logger = {}
48
- def logger.trace?
49
- true
50
- end
51
- def logger.trace(message)
52
- end
53
- logger
54
- end
55
-
53
+ logger
54
+ end
56
55
  end
57
56
 
58
57
  class KubernetesMetadataCacheStrategyTest < Test::Unit::TestCase
59
-
60
- def setup
61
- @strategy = TestCacheStrategy.new
62
- @cache_key = 'some_long_container_id'
63
- @namespace_name = 'some_namespace_name'
64
- @namespace_uuid = 'some_namespace_uuid'
65
- @pod_name = 'some_pod_name'
66
- @pod_uuid = 'some_pod_uuid'
67
- @time = Time.now
68
- @pod_meta = {'pod_id'=> @pod_uuid, 'labels'=> {'meta'=>'pod'}}
69
- @namespace_meta = {'namespace_id'=> @namespace_uuid, 'creation_timestamp'=>@time.to_s}
58
+ def setup
59
+ @strategy = TestCacheStrategy.new
60
+ @cache_key = 'some_long_container_id'
61
+ @namespace_name = 'some_namespace_name'
62
+ @namespace_uuid = 'some_namespace_uuid'
63
+ @pod_name = 'some_pod_name'
64
+ @pod_uuid = 'some_pod_uuid'
65
+ @time = Time.now
66
+ @pod_meta = { 'pod_id' => @pod_uuid, 'labels' => { 'meta' => 'pod' } }
67
+ @namespace_meta = { 'namespace_id' => @namespace_uuid, 'creation_timestamp' => @time.to_s }
68
+ end
69
+
70
+ test 'when cached metadata is found' do
71
+ exp = @pod_meta.merge(@namespace_meta)
72
+ exp.delete('creation_timestamp')
73
+ @strategy.id_cache[@cache_key] = {
74
+ pod_id: @pod_uuid,
75
+ namespace_id: @namespace_uuid
76
+ }
77
+ @strategy.cache[@pod_uuid] = @pod_meta
78
+ @strategy.namespace_cache[@namespace_uuid] = @namespace_meta
79
+ assert_equal(exp, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time, {}))
80
+ end
81
+
82
+ test 'when previously processed record for pod but metadata is not cached and can not be fetched' do
83
+ exp = {
84
+ 'pod_id' => @pod_uuid,
85
+ 'namespace_id' => @namespace_uuid
86
+ }
87
+ @strategy.id_cache[@cache_key] = {
88
+ pod_id: @pod_uuid,
89
+ namespace_id: @namespace_uuid
90
+ }
91
+ @strategy.stub :fetch_pod_metadata, {} do
92
+ @strategy.stub :fetch_namespace_metadata, nil do
93
+ assert_equal(exp, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time, {}))
94
+ end
70
95
  end
71
-
72
- test 'when cached metadata is found' do
73
- exp = @pod_meta.merge(@namespace_meta)
74
- exp.delete('creation_timestamp')
75
- @strategy.id_cache[@cache_key] = {
76
- pod_id: @pod_uuid,
77
- namespace_id: @namespace_uuid
78
- }
79
- @strategy.cache[@pod_uuid] = @pod_meta
80
- @strategy.namespace_cache[@namespace_uuid] = @namespace_meta
81
- assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, {}))
96
+ end
97
+
98
+ test 'when metadata is not cached and is fetched' do
99
+ exp = @pod_meta.merge(@namespace_meta)
100
+ exp.delete('creation_timestamp')
101
+ @strategy.stub :fetch_pod_metadata, @pod_meta do
102
+ @strategy.stub :fetch_namespace_metadata, @namespace_meta do
103
+ assert_equal(exp, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time, {}))
104
+ assert_true(@strategy.id_cache.key?(@cache_key))
105
+ end
82
106
  end
83
-
84
- test 'when previously processed record for pod but metadata is not cached and can not be fetched' do
85
- exp = {
86
- 'pod_id'=> @pod_uuid,
87
- 'namespace_id'=> @namespace_uuid
88
- }
89
- @strategy.id_cache[@cache_key] = {
90
- pod_id: @pod_uuid,
91
- namespace_id: @namespace_uuid
92
- }
93
- @strategy.stub :fetch_pod_metadata, {} do
94
- @strategy.stub :fetch_namespace_metadata, nil do
95
- assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, {}))
96
- end
97
- end
107
+ end
108
+
109
+ test 'when metadata is not cached and pod is deleted and namespace metadata is fetched' do
110
+ # this is the case for a record from a deleted pod where no other
111
+ # records were read. using the container hash since that is all
112
+ # we ever will have and should allow us to process all the deleted
113
+ # pod records
114
+ exp = {
115
+ 'pod_id' => @cache_key,
116
+ 'namespace_id' => @namespace_uuid
117
+ }
118
+ @strategy.stub :fetch_pod_metadata, {} do
119
+ @strategy.stub :fetch_namespace_metadata, @namespace_meta do
120
+ assert_equal(exp, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time, {}))
121
+ assert_true(@strategy.id_cache.key?(@cache_key))
122
+ end
98
123
  end
99
-
100
- test 'when metadata is not cached and is fetched' do
101
- exp = @pod_meta.merge(@namespace_meta)
102
- exp.delete('creation_timestamp')
103
- @strategy.stub :fetch_pod_metadata, @pod_meta do
104
- @strategy.stub :fetch_namespace_metadata, @namespace_meta do
105
- assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, {}))
106
- assert_true(@strategy.id_cache.key?(@cache_key))
107
- end
108
- end
109
- end
110
-
111
- test 'when metadata is not cached and pod is deleted and namespace metadata is fetched' do
112
- # this is the case for a record from a deleted pod where no other
113
- # records were read. using the container hash since that is all
114
- # we ever will have and should allow us to process all the deleted
115
- # pod records
116
- exp = {
117
- 'pod_id'=> @cache_key,
118
- 'namespace_id'=> @namespace_uuid
119
- }
120
- @strategy.stub :fetch_pod_metadata, {} do
121
- @strategy.stub :fetch_namespace_metadata, @namespace_meta do
122
- assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, {}))
123
- assert_true(@strategy.id_cache.key?(@cache_key))
124
- end
125
- end
126
- end
127
-
128
- test 'when metadata is not cached and pod is deleted and namespace is for a different namespace with the same name' do
129
- # this is the case for a record from a deleted pod from a deleted namespace
130
- # where new namespace was created with the same name
131
- exp = {
132
- 'namespace_id'=> @namespace_uuid
133
- }
134
- @strategy.stub :fetch_pod_metadata, {} do
135
- @strategy.stub :fetch_namespace_metadata, @namespace_meta do
136
- assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time - 1*86400, {}))
137
- assert_true(@strategy.id_cache.key?(@cache_key))
138
- end
139
- end
124
+ end
125
+
126
+ test 'when metadata is not cached and pod is deleted and namespace is for a different namespace with the same name' do
127
+ # this is the case for a record from a deleted pod from a deleted namespace
128
+ # where new namespace was created with the same name
129
+ exp = {
130
+ 'namespace_id' => @namespace_uuid
131
+ }
132
+ @strategy.stub :fetch_pod_metadata, {} do
133
+ @strategy.stub :fetch_namespace_metadata, @namespace_meta do
134
+ assert_equal(exp, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time - 1 * 86_400, {}))
135
+ assert_true(@strategy.id_cache.key?(@cache_key))
136
+ end
140
137
  end
141
-
142
- test 'when metadata is not cached and no metadata can be fetched and not allowing orphans' do
143
- # we should never see this since pod meta should not be retrievable
144
- # unless the namespace exists
145
- @strategy.stub :fetch_pod_metadata, @pod_meta do
146
- @strategy.stub :fetch_namespace_metadata, {} do
147
- assert_equal({}, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time - 1*86400, {}))
148
- end
149
- end
138
+ end
139
+
140
+ test 'when metadata is not cached and no metadata can be fetched and not allowing orphans' do
141
+ # we should never see this since pod meta should not be retrievable
142
+ # unless the namespace exists
143
+ @strategy.stub :fetch_pod_metadata, @pod_meta do
144
+ @strategy.stub :fetch_namespace_metadata, {} do
145
+ assert_equal({}, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time - 1 * 86_400, {}))
146
+ end
150
147
  end
151
-
152
- test 'when metadata is not cached and no metadata can be fetched and allowing orphans' do
153
- # we should never see this since pod meta should not be retrievable
154
- # unless the namespace exists
155
- @strategy.allow_orphans = true
156
- exp = {
157
- 'orphaned_namespace' => 'namespace',
158
- 'namespace_name' => '.orphaned',
159
- 'namespace_id' => 'orphaned'
160
- }
161
- @strategy.stub :fetch_pod_metadata, @pod_meta do
162
- @strategy.stub :fetch_namespace_metadata, {} do
163
- assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time - 1*86400, {}))
164
- end
165
- end
148
+ end
149
+
150
+ test 'when metadata is not cached and no metadata can be fetched and allowing orphans' do
151
+ # we should never see this since pod meta should not be retrievable
152
+ # unless the namespace exists
153
+ @strategy.allow_orphans = true
154
+ exp = {
155
+ 'orphaned_namespace' => 'namespace',
156
+ 'namespace_name' => '.orphaned',
157
+ 'namespace_id' => 'orphaned'
158
+ }
159
+ @strategy.stub :fetch_pod_metadata, @pod_meta do
160
+ @strategy.stub :fetch_namespace_metadata, {} do
161
+ assert_equal(exp, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time - 1 * 86_400, {}))
162
+ end
166
163
  end
167
-
168
- test 'when metadata is not cached and no metadata can be fetched and not allowing orphans for multiple records' do
169
- # processing a batch of records with no meta. ideally we only hit the api server once
170
- batch_miss_cache = {}
171
- @strategy.stub :fetch_pod_metadata, {} do
172
- @strategy.stub :fetch_namespace_metadata, {} do
173
- assert_equal({}, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, batch_miss_cache))
174
- end
175
- end
176
- assert_equal({}, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, batch_miss_cache))
164
+ end
165
+
166
+ test 'when metadata is not cached and no metadata can be fetched and not allowing orphans for multiple records' do
167
+ # processing a batch of records with no meta. ideally we only hit the api server once
168
+ batch_miss_cache = {}
169
+ @strategy.stub :fetch_pod_metadata, {} do
170
+ @strategy.stub :fetch_namespace_metadata, {} do
171
+ assert_equal({}, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time, batch_miss_cache))
172
+ end
177
173
  end
178
-
179
- test 'when metadata is not cached and no metadata can be fetched and allowing orphans for multiple records' do
180
- # we should never see this since pod meta should not be retrievable
181
- # unless the namespace exists
182
- @strategy.allow_orphans = true
183
- exp = {
184
- 'orphaned_namespace' => 'namespace',
185
- 'namespace_name' => '.orphaned',
186
- 'namespace_id' => 'orphaned'
187
- }
188
- batch_miss_cache = {}
189
- @strategy.stub :fetch_pod_metadata, {} do
190
- @strategy.stub :fetch_namespace_metadata, {} do
191
- assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, batch_miss_cache))
192
- end
193
- end
194
- assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, batch_miss_cache))
174
+ assert_equal({}, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time, batch_miss_cache))
175
+ end
176
+
177
+ test 'when metadata is not cached and no metadata can be fetched and allowing orphans for multiple records' do
178
+ # we should never see this since pod meta should not be retrievable
179
+ # unless the namespace exists
180
+ @strategy.allow_orphans = true
181
+ exp = {
182
+ 'orphaned_namespace' => 'namespace',
183
+ 'namespace_name' => '.orphaned',
184
+ 'namespace_id' => 'orphaned'
185
+ }
186
+ batch_miss_cache = {}
187
+ @strategy.stub :fetch_pod_metadata, {} do
188
+ @strategy.stub :fetch_namespace_metadata, {} do
189
+ assert_equal(exp, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time, batch_miss_cache))
190
+ end
195
191
  end
192
+ assert_equal(exp, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time, batch_miss_cache))
193
+ end
196
194
  end