fluent-plugin-kubernetes_metadata_filter 2.1.4 → 2.9.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +57 -0
  3. data/.gitignore +0 -1
  4. data/.rubocop.yml +57 -0
  5. data/Gemfile +4 -2
  6. data/Gemfile.lock +158 -0
  7. data/README.md +48 -28
  8. data/Rakefile +15 -11
  9. data/fluent-plugin-kubernetes_metadata_filter.gemspec +25 -28
  10. data/lib/fluent/plugin/filter_kubernetes_metadata.rb +185 -131
  11. data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb +27 -20
  12. data/lib/fluent/plugin/kubernetes_metadata_common.rb +59 -33
  13. data/lib/fluent/plugin/kubernetes_metadata_stats.rb +6 -6
  14. data/lib/fluent/plugin/kubernetes_metadata_test_api_adapter.rb +68 -0
  15. data/lib/fluent/plugin/kubernetes_metadata_util.rb +53 -0
  16. data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +121 -27
  17. data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +138 -29
  18. data/release_notes.md +42 -0
  19. data/test/cassettes/kubernetes_docker_metadata_annotations.yml +0 -34
  20. data/test/cassettes/{kubernetes_docker_metadata_dotted_labels.yml → kubernetes_docker_metadata_dotted_slashed_labels.yml} +0 -34
  21. data/test/cassettes/kubernetes_get_api_v1.yml +193 -0
  22. data/test/cassettes/kubernetes_get_api_v1_using_token.yml +195 -0
  23. data/test/cassettes/kubernetes_get_namespace_default.yml +69 -0
  24. data/test/cassettes/kubernetes_get_namespace_default_using_token.yml +71 -0
  25. data/test/cassettes/{kubernetes_docker_metadata.yml → kubernetes_get_pod.yml} +0 -82
  26. data/test/cassettes/{metadata_with_namespace_id.yml → kubernetes_get_pod_container_init.yml} +3 -134
  27. data/test/cassettes/{kubernetes_docker_metadata_using_bearer_token.yml → kubernetes_get_pod_using_token.yml} +5 -105
  28. data/test/cassettes/metadata_from_tag_and_journald_fields.yml +0 -255
  29. data/test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml +0 -255
  30. data/test/cassettes/{non_kubernetes_docker_metadata.yml → valid_kubernetes_api_server_using_token.yml} +4 -44
  31. data/test/helper.rb +20 -2
  32. data/test/plugin/test_cache_stats.rb +10 -13
  33. data/test/plugin/test_cache_strategy.rb +158 -160
  34. data/test/plugin/test_filter_kubernetes_metadata.rb +480 -320
  35. data/test/plugin/test_utils.rb +56 -0
  36. data/test/plugin/test_watch_namespaces.rb +209 -55
  37. data/test/plugin/test_watch_pods.rb +302 -103
  38. data/test/plugin/watch_test.rb +52 -33
  39. metadata +69 -72
  40. data/circle.yml +0 -17
@@ -0,0 +1,56 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
5
+ # Kubernetes metadata
6
+ #
7
+ # Copyright 2015 Red Hat, Inc.
8
+ #
9
+ # Licensed under the Apache License, Version 2.0 (the "License");
10
+ # you may not use this file except in compliance with the License.
11
+ # You may obtain a copy of the License at
12
+ #
13
+ # http://www.apache.org/licenses/LICENSE-2.0
14
+ #
15
+ # Unless required by applicable law or agreed to in writing, software
16
+ # distributed under the License is distributed on an "AS IS" BASIS,
17
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18
+ # See the License for the specific language governing permissions and
19
+ # limitations under the License.
20
+ #
21
+ class KubernetesMetadataCacheStatsTest < Test::Unit::TestCase
22
+ include KubernetesMetadata::Util
23
+
24
+ def setup
25
+ @time_fields = ['time']
26
+ @internal_time = Time.now
27
+ end
28
+
29
+ test '#create_time_from_record when time is empty' do
30
+ record = { 'time' => ' ' }
31
+ assert_equal(@internal_time.to_i, create_time_from_record(record, @internal_time).to_i)
32
+ end
33
+ test '#create_time_from_record when time is nil' do
34
+ record = {}
35
+ assert_equal(@internal_time.to_i, create_time_from_record(record, @internal_time).to_i)
36
+ end
37
+
38
+ test '#create_time_from_record when time is an integer' do
39
+ exp_time = Time.now
40
+ record = { 'time' => exp_time.to_i }
41
+ assert_equal(exp_time.to_i, create_time_from_record(record, @internal_time).to_i)
42
+ end
43
+
44
+ test '#create_time_from_record when time is a string' do
45
+ exp_time = Time.now
46
+ record = { 'time' => exp_time.to_s }
47
+ assert_equal(exp_time.to_i, create_time_from_record(record, @internal_time).to_i)
48
+ end
49
+
50
+ test '#create_time_from_record when timefields include journal time fields' do
51
+ @time_fields = ['_SOURCE_REALTIME_TIMESTAMP']
52
+ exp_time = Time.now
53
+ record = { '_SOURCE_REALTIME_TIMESTAMP' => exp_time.to_i.to_s }
54
+ assert_equal(Time.at(exp_time.to_i / 1_000_000, exp_time.to_i % 1_000_000).to_i, create_time_from_record(record, @internal_time).to_i)
55
+ end
56
+ end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  #
2
4
  # Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
3
5
  # Kubernetes metadata
@@ -17,75 +19,227 @@
17
19
  # limitations under the License.
18
20
  #
19
21
  require_relative '../helper'
20
- require 'ostruct'
21
22
  require_relative 'watch_test'
22
23
 
23
24
  class WatchNamespacesTestTest < WatchTest
25
+ include KubernetesMetadata::WatchNamespaces
24
26
 
25
- include KubernetesMetadata::WatchNamespaces
26
-
27
- setup do
28
- @created = OpenStruct.new(
29
- type: 'CREATED',
30
- object: {
31
- 'metadata' => {
32
- 'name' => 'created',
33
- 'uid' => 'created_uid'
34
- }
35
- }
36
- )
37
- @modified = OpenStruct.new(
38
- type: 'MODIFIED',
39
- object: {
40
- 'metadata' => {
41
- 'name' => 'foo',
42
- 'uid' => 'modified_uid'
43
- }
44
- }
45
- )
46
- @deleted = OpenStruct.new(
47
- type: 'DELETED',
48
- object: {
49
- 'metadata' => {
50
- 'name' => 'deleteme',
51
- 'uid' => 'deleted_uid'
52
- }
53
- }
54
- )
55
- end
56
-
57
- test 'namespace watch ignores CREATED' do
58
- @client.stub :watch_namespaces, [@created] do
59
- start_namespace_watch
60
- assert_equal(false, @namespace_cache.key?('created_uid'))
61
- assert_equal(1, @stats[:namespace_cache_watch_ignored])
62
- end
27
+ setup do
28
+ @initial = {
29
+ kind: 'NamespaceList',
30
+ metadata: { resourceVersion: '123' },
31
+ items: [
32
+ {
33
+ metadata: {
34
+ name: 'initial',
35
+ uid: 'initial_uid'
36
+ }
37
+ },
38
+ {
39
+ metadata: {
40
+ name: 'modified',
41
+ uid: 'modified_uid'
42
+ }
43
+ }
44
+ ]
45
+ }
46
+
47
+ @created = {
48
+ type: 'CREATED',
49
+ object: {
50
+ metadata: {
51
+ name: 'created',
52
+ uid: 'created_uid'
53
+ }
54
+ }
55
+ }
56
+ @modified = {
57
+ type: 'MODIFIED',
58
+ object: {
59
+ metadata: {
60
+ name: 'foo',
61
+ uid: 'modified_uid'
62
+ }
63
+ }
64
+ }
65
+ @deleted = {
66
+ type: 'DELETED',
67
+ object: {
68
+ metadata: {
69
+ name: 'deleteme',
70
+ uid: 'deleted_uid'
71
+ }
72
+ }
73
+ }
74
+ @error = {
75
+ type: 'ERROR',
76
+ object: {
77
+ message: 'some error message'
78
+ }
79
+ }
80
+ @gone = {
81
+ type: 'ERROR',
82
+ object: {
83
+ code: 410,
84
+ kind: 'Status',
85
+ message: 'too old resource version: 123 (391079)',
86
+ metadata: {
87
+ name: 'gone',
88
+ namespace: 'gone',
89
+ uid: 'gone_uid'
90
+ },
91
+ reason: 'Gone'
92
+ }
93
+ }
94
+ end
95
+
96
+ test 'namespace list caches namespaces' do
97
+ @client.stub :get_namespaces, @initial do
98
+ process_namespace_watcher_notices(start_namespace_watch)
99
+ assert_equal(true, @namespace_cache.key?('initial_uid'))
100
+ assert_equal(true, @namespace_cache.key?('modified_uid'))
101
+ assert_equal(2, @stats[:namespace_cache_host_updates])
63
102
  end
103
+ end
64
104
 
65
- test 'namespace watch ignores MODIFIED when info not in cache' do
105
+ test 'namespace list caches namespaces and watch updates' do
106
+ orig_env_val = ENV['K8S_NODE_NAME']
107
+ ENV['K8S_NODE_NAME'] = 'aNodeName'
108
+ @client.stub :get_namespaces, @initial do
66
109
  @client.stub :watch_namespaces, [@modified] do
67
- start_namespace_watch
68
- assert_equal(false, @namespace_cache.key?('modified_uid'))
69
- assert_equal(1, @stats[:namespace_cache_watch_misses])
110
+ process_namespace_watcher_notices(start_namespace_watch)
111
+ assert_equal(2, @stats[:namespace_cache_host_updates])
112
+ assert_equal(1, @stats[:namespace_cache_watch_updates])
70
113
  end
71
114
  end
115
+ ENV['K8S_NODE_NAME'] = orig_env_val
116
+ end
72
117
 
73
- test 'namespace watch updates cache when MODIFIED is received and info is cached' do
74
- @namespace_cache['modified_uid'] = {}
75
- @client.stub :watch_namespaces, [@modified] do
76
- start_namespace_watch
77
- assert_equal(true, @namespace_cache.key?('modified_uid'))
78
- assert_equal(1, @stats[:namespace_cache_watch_updates])
118
+ test 'namespace watch ignores CREATED' do
119
+ @client.stub :watch_namespaces, [@created] do
120
+ process_namespace_watcher_notices(start_namespace_watch)
121
+ assert_equal(false, @namespace_cache.key?('created_uid'))
122
+ assert_equal(1, @stats[:namespace_cache_watch_ignored])
123
+ end
124
+ end
125
+
126
+ test 'namespace watch ignores MODIFIED when info not in cache' do
127
+ @client.stub :watch_namespaces, [@modified] do
128
+ process_namespace_watcher_notices(start_namespace_watch)
129
+ assert_equal(false, @namespace_cache.key?('modified_uid'))
130
+ assert_equal(1, @stats[:namespace_cache_watch_misses])
131
+ end
132
+ end
133
+
134
+ test 'namespace watch updates cache when MODIFIED is received and info is cached' do
135
+ @namespace_cache['modified_uid'] = {}
136
+ @client.stub :watch_namespaces, [@modified] do
137
+ process_namespace_watcher_notices(start_namespace_watch)
138
+ assert_equal(true, @namespace_cache.key?('modified_uid'))
139
+ assert_equal(1, @stats[:namespace_cache_watch_updates])
140
+ end
141
+ end
142
+
143
+ test 'namespace watch ignores DELETED' do
144
+ @namespace_cache['deleted_uid'] = {}
145
+ @client.stub :watch_namespaces, [@deleted] do
146
+ process_namespace_watcher_notices(start_namespace_watch)
147
+ assert_equal(true, @namespace_cache.key?('deleted_uid'))
148
+ assert_equal(1, @stats[:namespace_cache_watch_deletes_ignored])
149
+ end
150
+ end
151
+
152
+ test 'namespace watch raises Fluent::UnrecoverableError when cannot re-establish connection to k8s API server' do
153
+ # Stub start_namespace_watch to simulate initial successful connection to API server
154
+ stub(self).start_namespace_watch
155
+ # Stub watch_namespaces to simluate not being able to set up watch connection to API server
156
+ stub(@client).watch_namespaces { raise }
157
+ @client.stub :get_namespaces, @initial do
158
+ assert_raise Fluent::UnrecoverableError do
159
+ set_up_namespace_thread
160
+ end
161
+ end
162
+ assert_equal(3, @stats[:namespace_watch_failures])
163
+ assert_equal(2, Thread.current[:namespace_watch_retry_count])
164
+ assert_equal(4, Thread.current[:namespace_watch_retry_backoff_interval])
165
+ assert_nil(@stats[:namespace_watch_error_type_notices])
166
+ end
167
+
168
+ test 'namespace watch resets watch retry count when exceptions are encountered and connection to k8s API server is re-established' do
169
+ @client.stub :get_namespaces, @initial do
170
+ @client.stub :watch_namespaces, [[@created, @exception_raised]] do
171
+ # Force the infinite watch loop to exit after 3 seconds. Verifies that
172
+ # no unrecoverable error was thrown during this period of time.
173
+ assert_raise Timeout::Error.new('execution expired') do
174
+ Timeout.timeout(3) do
175
+ set_up_namespace_thread
176
+ end
177
+ end
178
+ assert_operator(@stats[:namespace_watch_failures], :>=, 3)
179
+ assert_operator(Thread.current[:namespace_watch_retry_count], :<=, 1)
180
+ assert_operator(Thread.current[:namespace_watch_retry_backoff_interval], :<=, 1)
181
+ end
182
+ end
183
+ end
184
+
185
+ test 'namespace watch resets watch retry count when error is received and connection to k8s API server is re-established' do
186
+ @client.stub :get_namespaces, @initial do
187
+ @client.stub :watch_namespaces, [@error] do
188
+ # Force the infinite watch loop to exit after 3 seconds. Verifies that
189
+ # no unrecoverable error was thrown during this period of time.
190
+ assert_raise Timeout::Error.new('execution expired') do
191
+ Timeout.timeout(3) do
192
+ set_up_namespace_thread
193
+ end
194
+ end
195
+ assert_operator(@stats[:namespace_watch_failures], :>=, 3)
196
+ assert_operator(Thread.current[:namespace_watch_retry_count], :<=, 1)
197
+ assert_operator(Thread.current[:namespace_watch_retry_backoff_interval], :<=, 1)
198
+ end
199
+ end
200
+ end
201
+
202
+ test 'namespace watch continues after retries succeed' do
203
+ @client.stub :get_namespaces, @initial do
204
+ @client.stub :watch_namespaces, [@modified, @error, @modified] do
205
+ # Force the infinite watch loop to exit after 3 seconds. Verifies that
206
+ # no unrecoverable error was thrown during this period of time.
207
+ assert_raise Timeout::Error.new('execution expired') do
208
+ Timeout.timeout(3) do
209
+ set_up_namespace_thread
210
+ end
211
+ end
212
+ assert_operator(@stats[:namespace_watch_failures], :>=, 3)
213
+ assert_operator(Thread.current[:namespace_watch_retry_count], :<=, 1)
214
+ assert_operator(Thread.current[:namespace_watch_retry_backoff_interval], :<=, 1)
215
+ assert_operator(@stats[:namespace_watch_error_type_notices], :>=, 3)
79
216
  end
80
217
  end
218
+ end
81
219
 
82
- test 'namespace watch ignores DELETED' do
83
- @namespace_cache['deleted_uid'] = {}
84
- @client.stub :watch_namespaces, [@deleted] do
85
- start_namespace_watch
86
- assert_equal(true, @namespace_cache.key?('deleted_uid'))
87
- assert_equal(1, @stats[:namespace_cache_watch_deletes_ignored])
220
+ test 'namespace watch raises a GoneError when a 410 Gone error is received' do
221
+ @cache['gone_uid'] = {}
222
+ @client.stub :watch_namespaces, [@gone] do
223
+ assert_raise KubernetesMetadata::Common::GoneError do
224
+ process_namespace_watcher_notices(start_namespace_watch)
88
225
  end
226
+ assert_equal(1, @stats[:namespace_watch_gone_notices])
89
227
  end
228
+ end
90
229
 
230
+ test 'namespace watch retries when 410 Gone errors are encountered' do
231
+ @client.stub :get_namespaces, @initial do
232
+ @client.stub :watch_namespaces, [@created, @gone, @modified] do
233
+ # Force the infinite watch loop to exit after 3 seconds. Verifies that
234
+ # no unrecoverable error was thrown during this period of time.
235
+ assert_raise Timeout::Error.new('execution expired') do
236
+ Timeout.timeout(3) do
237
+ set_up_namespace_thread
238
+ end
239
+ end
240
+ assert_operator(@stats[:namespace_watch_gone_errors], :>=, 3)
241
+ assert_operator(@stats[:namespace_watch_gone_notices], :>=, 3)
242
+ end
243
+ end
244
+ end
91
245
  end