fluent-plugin-kubernetes_metadata_filter 2.1.0 → 3.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +5 -5
  2. data/.circleci/config.yml +53 -0
  3. data/.gitignore +0 -2
  4. data/.rubocop.yml +57 -0
  5. data/Gemfile +4 -2
  6. data/Gemfile.lock +159 -0
  7. data/README.md +49 -60
  8. data/Rakefile +15 -11
  9. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/cpu.png +0 -0
  10. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/latency.png +0 -0
  11. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/loss.png +0 -0
  12. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/mem.png +0 -0
  13. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/readme.md +88 -0
  14. data/doc/benchmark/5m-1-2500lps-256b-baseline-01/results.html +127 -0
  15. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/cpu.png +0 -0
  16. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/latency.png +0 -0
  17. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/loss.png +0 -0
  18. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/mem.png +0 -0
  19. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/readme.md +97 -0
  20. data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/results.html +136 -0
  21. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/cpu.png +0 -0
  22. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/latency.png +0 -0
  23. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/loss.png +0 -0
  24. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/mem.png +0 -0
  25. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/readme.md +97 -0
  26. data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/results.html +136 -0
  27. data/fluent-plugin-kubernetes_metadata_filter.gemspec +25 -28
  28. data/lib/fluent/plugin/filter_kubernetes_metadata.rb +207 -187
  29. data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb +30 -23
  30. data/lib/fluent/plugin/kubernetes_metadata_common.rb +66 -24
  31. data/lib/fluent/plugin/kubernetes_metadata_stats.rb +21 -5
  32. data/lib/fluent/plugin/kubernetes_metadata_test_api_adapter.rb +68 -0
  33. data/lib/fluent/plugin/kubernetes_metadata_util.rb +33 -0
  34. data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +154 -27
  35. data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +171 -29
  36. data/release_notes.md +42 -0
  37. data/test/cassettes/kubernetes_docker_metadata_annotations.yml +0 -34
  38. data/test/cassettes/{kubernetes_docker_metadata_dotted_labels.yml → kubernetes_docker_metadata_dotted_slashed_labels.yml} +0 -34
  39. data/test/cassettes/kubernetes_get_api_v1.yml +193 -0
  40. data/test/cassettes/kubernetes_get_api_v1_using_token.yml +195 -0
  41. data/test/cassettes/kubernetes_get_namespace_default.yml +72 -0
  42. data/test/cassettes/kubernetes_get_namespace_default_using_token.yml +71 -0
  43. data/test/cassettes/{kubernetes_docker_metadata.yml → kubernetes_get_pod.yml} +0 -82
  44. data/test/cassettes/kubernetes_get_pod_container_init.yml +145 -0
  45. data/test/cassettes/{metadata_with_namespace_id.yml → kubernetes_get_pod_using_token.yml} +2 -130
  46. data/test/cassettes/{kubernetes_docker_metadata_using_bearer_token.yml → kubernetes_get_pod_with_ownerrefs.yml} +17 -109
  47. data/test/cassettes/metadata_from_tag_and_journald_fields.yml +153 -0
  48. data/test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml +285 -0
  49. data/test/cassettes/{non_kubernetes_docker_metadata.yml → valid_kubernetes_api_server_using_token.yml} +4 -44
  50. data/test/helper.rb +20 -2
  51. data/test/plugin/test_cache_stats.rb +10 -13
  52. data/test/plugin/test_cache_strategy.rb +158 -160
  53. data/test/plugin/test_filter_kubernetes_metadata.rb +451 -314
  54. data/test/plugin/test_watch_namespaces.rb +209 -55
  55. data/test/plugin/test_watch_pods.rb +302 -71
  56. data/test/plugin/watch_test.rb +52 -33
  57. metadata +91 -70
  58. data/circle.yml +0 -17
data/test/plugin/test_cache_strategy.rb CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  #
2
4
  # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
3
5
  # Kubernetes metadata
@@ -17,180 +19,176 @@
17
19
  # limitations under the License.
18
20
  #
19
21
  require_relative '../helper'
20
- require_relative '../../lib/fluent/plugin/kubernetes_metadata_cache_strategy'
21
- require_relative '../../lib/fluent/plugin/kubernetes_metadata_stats'
22
- require 'lru_redux'
23
- require 'webmock/test_unit'
24
- WebMock.disable_net_connect!
25
22
 
26
23
  class TestCacheStrategy
27
- include KubernetesMetadata::CacheStrategy
28
-
29
- def initialize
30
- @stats = KubernetesMetadata::Stats.new
31
- @cache = LruRedux::TTL::ThreadSafeCache.new(100,3600)
32
- @id_cache = LruRedux::TTL::ThreadSafeCache.new(100,3600)
33
- @namespace_cache = LruRedux::TTL::ThreadSafeCache.new(100,3600)
34
- @orphaned_namespace_name = '.orphaned'
35
- @orphaned_namespace_id = 'orphaned'
24
+ include KubernetesMetadata::CacheStrategy
25
+
26
+ def initialize
27
+ @stats = KubernetesMetadata::Stats.new
28
+ @cache = LruRedux::TTL::ThreadSafeCache.new(100, 3600)
29
+ @id_cache = LruRedux::TTL::ThreadSafeCache.new(100, 3600)
30
+ @namespace_cache = LruRedux::TTL::ThreadSafeCache.new(100, 3600)
31
+ @orphaned_namespace_name = '.orphaned'
32
+ @orphaned_namespace_id = 'orphaned'
33
+ end
34
+
35
+ attr_accessor :stats, :cache, :id_cache, :namespace_cache, :allow_orphans
36
+
37
+ def fetch_pod_metadata(_namespace_name, _pod_name)
38
+ {}
39
+ end
40
+
41
+ def fetch_namespace_metadata(_namespace_name)
42
+ {}
43
+ end
44
+
45
+ def log
46
+ logger = {}
47
+ def logger.on_trace
48
+ true
36
49
  end
37
50
 
38
- attr_accessor :stats, :cache, :id_cache, :namespace_cache, :allow_orphans
39
-
40
- def fetch_pod_metadata(namespace_name, pod_name)
51
+ def logger.trace(message)
41
52
  end
42
-
43
- def fetch_namespace_metadata(namespace_name)
44
- end
45
-
46
- def log
47
- logger = {}
48
- def logger.trace?
49
- true
50
- end
51
- def logger.trace(message)
52
- end
53
- logger
54
- end
55
-
53
+ logger
54
+ end
56
55
  end
57
56
 
58
57
  class KubernetesMetadataCacheStrategyTest < Test::Unit::TestCase
59
-
60
- def setup
61
- @strategy = TestCacheStrategy.new
62
- @cache_key = 'some_long_container_id'
63
- @namespace_name = 'some_namespace_name'
64
- @namespace_uuid = 'some_namespace_uuid'
65
- @pod_name = 'some_pod_name'
66
- @pod_uuid = 'some_pod_uuid'
67
- @time = Time.now
68
- @pod_meta = {'pod_id'=> @pod_uuid, 'labels'=> {'meta'=>'pod'}}
69
- @namespace_meta = {'namespace_id'=> @namespace_uuid, 'creation_timestamp'=>@time.to_s}
58
+ def setup
59
+ @strategy = TestCacheStrategy.new
60
+ @cache_key = 'some_long_container_id'
61
+ @namespace_name = 'some_namespace_name'
62
+ @namespace_uuid = 'some_namespace_uuid'
63
+ @pod_name = 'some_pod_name'
64
+ @pod_uuid = 'some_pod_uuid'
65
+ @time = Time.now
66
+ @pod_meta = { 'pod_id' => @pod_uuid, 'labels' => { 'meta' => 'pod' } }
67
+ @namespace_meta = { 'namespace_id' => @namespace_uuid, 'creation_timestamp' => @time.to_s }
68
+ end
69
+
70
+ test 'when cached metadata is found' do
71
+ exp = @pod_meta.merge(@namespace_meta)
72
+ exp.delete('creation_timestamp')
73
+ @strategy.id_cache[@cache_key] = {
74
+ pod_id: @pod_uuid,
75
+ namespace_id: @namespace_uuid
76
+ }
77
+ @strategy.cache[@pod_uuid] = @pod_meta
78
+ @strategy.namespace_cache[@namespace_uuid] = @namespace_meta
79
+ assert_equal(exp, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time, {}))
80
+ end
81
+
82
+ test 'when previously processed record for pod but metadata is not cached and can not be fetched' do
83
+ exp = {
84
+ 'pod_id' => @pod_uuid,
85
+ 'namespace_id' => @namespace_uuid
86
+ }
87
+ @strategy.id_cache[@cache_key] = {
88
+ pod_id: @pod_uuid,
89
+ namespace_id: @namespace_uuid
90
+ }
91
+ @strategy.stub :fetch_pod_metadata, {} do
92
+ @strategy.stub :fetch_namespace_metadata, nil do
93
+ assert_equal(exp, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time, {}))
94
+ end
70
95
  end
71
-
72
- test 'when cached metadata is found' do
73
- exp = @pod_meta.merge(@namespace_meta)
74
- exp.delete('creation_timestamp')
75
- @strategy.id_cache[@cache_key] = {
76
- pod_id: @pod_uuid,
77
- namespace_id: @namespace_uuid
78
- }
79
- @strategy.cache[@pod_uuid] = @pod_meta
80
- @strategy.namespace_cache[@namespace_uuid] = @namespace_meta
81
- assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, {}))
96
+ end
97
+
98
+ test 'when metadata is not cached and is fetched' do
99
+ exp = @pod_meta.merge(@namespace_meta)
100
+ exp.delete('creation_timestamp')
101
+ @strategy.stub :fetch_pod_metadata, @pod_meta do
102
+ @strategy.stub :fetch_namespace_metadata, @namespace_meta do
103
+ assert_equal(exp, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time, {}))
104
+ assert_true(@strategy.id_cache.key?(@cache_key))
105
+ end
82
106
  end
83
-
84
- test 'when previously processed record for pod but metadata is not cached and can not be fetched' do
85
- exp = {
86
- 'pod_id'=> @pod_uuid,
87
- 'namespace_id'=> @namespace_uuid
88
- }
89
- @strategy.id_cache[@cache_key] = {
90
- pod_id: @pod_uuid,
91
- namespace_id: @namespace_uuid
92
- }
93
- @strategy.stub :fetch_pod_metadata, {} do
94
- @strategy.stub :fetch_namespace_metadata, nil do
95
- assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, {}))
96
- end
97
- end
107
+ end
108
+
109
+ test 'when metadata is not cached and pod is deleted and namespace metadata is fetched' do
110
+ # this is the case for a record from a deleted pod where no other
111
+ # records were read. using the container hash since that is all
112
+ # we ever will have and should allow us to process all the deleted
113
+ # pod records
114
+ exp = {
115
+ 'pod_id' => @cache_key,
116
+ 'namespace_id' => @namespace_uuid
117
+ }
118
+ @strategy.stub :fetch_pod_metadata, {} do
119
+ @strategy.stub :fetch_namespace_metadata, @namespace_meta do
120
+ assert_equal(exp, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time, {}))
121
+ assert_true(@strategy.id_cache.key?(@cache_key))
122
+ end
98
123
  end
99
-
100
- test 'when metadata is not cached and is fetched' do
101
- exp = @pod_meta.merge(@namespace_meta)
102
- exp.delete('creation_timestamp')
103
- @strategy.stub :fetch_pod_metadata, @pod_meta do
104
- @strategy.stub :fetch_namespace_metadata, @namespace_meta do
105
- assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, {}))
106
- assert_true(@strategy.id_cache.key?(@cache_key))
107
- end
108
- end
109
- end
110
-
111
- test 'when metadata is not cached and pod is deleted and namespace metadata is fetched' do
112
- # this is the case for a record from a deleted pod where no other
113
- # records were read. using the container hash since that is all
114
- # we ever will have and should allow us to process all the deleted
115
- # pod records
116
- exp = {
117
- 'pod_id'=> @cache_key,
118
- 'namespace_id'=> @namespace_uuid
119
- }
120
- @strategy.stub :fetch_pod_metadata, {} do
121
- @strategy.stub :fetch_namespace_metadata, @namespace_meta do
122
- assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, {}))
123
- assert_true(@strategy.id_cache.key?(@cache_key))
124
- end
125
- end
126
- end
127
-
128
- test 'when metadata is not cached and pod is deleted and namespace is for a different namespace with the same name' do
129
- # this is the case for a record from a deleted pod from a deleted namespace
130
- # where new namespace was created with the same name
131
- exp = {
132
- 'namespace_id'=> @namespace_uuid
133
- }
134
- @strategy.stub :fetch_pod_metadata, {} do
135
- @strategy.stub :fetch_namespace_metadata, @namespace_meta do
136
- assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time - 1*86400, {}))
137
- assert_true(@strategy.id_cache.key?(@cache_key))
138
- end
139
- end
124
+ end
125
+
126
+ test 'when metadata is not cached and pod is deleted and namespace is for a different namespace with the same name' do
127
+ # this is the case for a record from a deleted pod from a deleted namespace
128
+ # where new namespace was created with the same name
129
+ exp = {
130
+ 'namespace_id' => @namespace_uuid
131
+ }
132
+ @strategy.stub :fetch_pod_metadata, {} do
133
+ @strategy.stub :fetch_namespace_metadata, @namespace_meta do
134
+ assert_equal(exp, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time - 1 * 86_400, {}))
135
+ assert_true(@strategy.id_cache.key?(@cache_key))
136
+ end
140
137
  end
141
-
142
- test 'when metadata is not cached and no metadata can be fetched and not allowing orphans' do
143
- # we should never see this since pod meta should not be retrievable
144
- # unless the namespace exists
145
- @strategy.stub :fetch_pod_metadata, @pod_meta do
146
- @strategy.stub :fetch_namespace_metadata, {} do
147
- assert_equal({}, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time - 1*86400, {}))
148
- end
149
- end
138
+ end
139
+
140
+ test 'when metadata is not cached and no metadata can be fetched and not allowing orphans' do
141
+ # we should never see this since pod meta should not be retrievable
142
+ # unless the namespace exists
143
+ @strategy.stub :fetch_pod_metadata, @pod_meta do
144
+ @strategy.stub :fetch_namespace_metadata, {} do
145
+ assert_equal({}, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time - 1 * 86_400, {}))
146
+ end
150
147
  end
151
-
152
- test 'when metadata is not cached and no metadata can be fetched and allowing orphans' do
153
- # we should never see this since pod meta should not be retrievable
154
- # unless the namespace exists
155
- @strategy.allow_orphans = true
156
- exp = {
157
- 'orphaned_namespace' => 'namespace',
158
- 'namespace_name' => '.orphaned',
159
- 'namespace_id' => 'orphaned'
160
- }
161
- @strategy.stub :fetch_pod_metadata, @pod_meta do
162
- @strategy.stub :fetch_namespace_metadata, {} do
163
- assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time - 1*86400, {}))
164
- end
165
- end
148
+ end
149
+
150
+ test 'when metadata is not cached and no metadata can be fetched and allowing orphans' do
151
+ # we should never see this since pod meta should not be retrievable
152
+ # unless the namespace exists
153
+ @strategy.allow_orphans = true
154
+ exp = {
155
+ 'orphaned_namespace' => 'namespace',
156
+ 'namespace_name' => '.orphaned',
157
+ 'namespace_id' => 'orphaned'
158
+ }
159
+ @strategy.stub :fetch_pod_metadata, @pod_meta do
160
+ @strategy.stub :fetch_namespace_metadata, {} do
161
+ assert_equal(exp, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time - 1 * 86_400, {}))
162
+ end
166
163
  end
167
-
168
- test 'when metadata is not cached and no metadata can be fetched and not allowing orphans for multiple records' do
169
- # processing a batch of records with no meta. ideally we only hit the api server once
170
- batch_miss_cache = {}
171
- @strategy.stub :fetch_pod_metadata, {} do
172
- @strategy.stub :fetch_namespace_metadata, {} do
173
- assert_equal({}, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, batch_miss_cache))
174
- end
175
- end
176
- assert_equal({}, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, batch_miss_cache))
164
+ end
165
+
166
+ test 'when metadata is not cached and no metadata can be fetched and not allowing orphans for multiple records' do
167
+ # processing a batch of records with no meta. ideally we only hit the api server once
168
+ batch_miss_cache = {}
169
+ @strategy.stub :fetch_pod_metadata, {} do
170
+ @strategy.stub :fetch_namespace_metadata, {} do
171
+ assert_equal({}, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time, batch_miss_cache))
172
+ end
177
173
  end
178
-
179
- test 'when metadata is not cached and no metadata can be fetched and allowing orphans for multiple records' do
180
- # we should never see this since pod meta should not be retrievable
181
- # unless the namespace exists
182
- @strategy.allow_orphans = true
183
- exp = {
184
- 'orphaned_namespace' => 'namespace',
185
- 'namespace_name' => '.orphaned',
186
- 'namespace_id' => 'orphaned'
187
- }
188
- batch_miss_cache = {}
189
- @strategy.stub :fetch_pod_metadata, {} do
190
- @strategy.stub :fetch_namespace_metadata, {} do
191
- assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, batch_miss_cache))
192
- end
193
- end
194
- assert_equal(exp, @strategy.get_pod_metadata(@cache_key,'namespace', 'pod', @time, batch_miss_cache))
174
+ assert_equal({}, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time, batch_miss_cache))
175
+ end
176
+
177
+ test 'when metadata is not cached and no metadata can be fetched and allowing orphans for multiple records' do
178
+ # we should never see this since pod meta should not be retrievable
179
+ # unless the namespace exists
180
+ @strategy.allow_orphans = true
181
+ exp = {
182
+ 'orphaned_namespace' => 'namespace',
183
+ 'namespace_name' => '.orphaned',
184
+ 'namespace_id' => 'orphaned'
185
+ }
186
+ batch_miss_cache = {}
187
+ @strategy.stub :fetch_pod_metadata, {} do
188
+ @strategy.stub :fetch_namespace_metadata, {} do
189
+ assert_equal(exp, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time, batch_miss_cache))
190
+ end
195
191
  end
192
+ assert_equal(exp, @strategy.get_pod_metadata(@cache_key, 'namespace', 'pod', @time, batch_miss_cache))
193
+ end
196
194
  end