fluent-plugin-kubernetes_metadata_filter 2.10.0 → 2.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +8 -8
- data/fluent-plugin-kubernetes_metadata_filter.gemspec +1 -1
- data/lib/fluent/plugin/filter_kubernetes_metadata.rb +47 -23
- data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +33 -0
- data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +33 -0
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f8ed17c405e64607d3fee4b581345f32388a7b1f52db9633d83e6863f6da5358
|
4
|
+
data.tar.gz: e5d6002813b1ffb266b43eda059a2f7a6b050bb69162b52a48cf68e5bf116e7d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c39196e160ffbf56e328f1b4baf2ce61cafbb3152e1c1bfc109a445a78083ed4bd25f28f81356472dffb975a2905cd90ef951f33ba1777572e1b3c86810323cd
|
7
|
+
data.tar.gz: 99377e8dbf5b6f95259b02ea923d22c1195fb582ac89ede2734fe29200305543e6916113ee49ec7c1fe6a4a839882038925925c724d662d27b5ef34f7bdeaf9e
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
fluent-plugin-kubernetes_metadata_filter (2.
|
4
|
+
fluent-plugin-kubernetes_metadata_filter (2.11.1)
|
5
5
|
fluentd (>= 0.14.0, < 1.15)
|
6
6
|
kubeclient (>= 4.0.0, < 5.0.0)
|
7
7
|
lru_redux
|
@@ -54,13 +54,13 @@ GEM
|
|
54
54
|
http-form_data (~> 2.2)
|
55
55
|
http-parser (~> 1.2.0)
|
56
56
|
http-accept (1.7.0)
|
57
|
-
http-cookie (1.0.
|
57
|
+
http-cookie (1.0.5)
|
58
58
|
domain_name (~> 0.5)
|
59
59
|
http-form_data (2.3.0)
|
60
60
|
http-parser (1.2.3)
|
61
61
|
ffi-compiler (>= 1.0, < 2.0)
|
62
62
|
http_parser.rb (0.8.0)
|
63
|
-
jsonpath (1.1.
|
63
|
+
jsonpath (1.1.2)
|
64
64
|
multi_json
|
65
65
|
kubeclient (4.9.3)
|
66
66
|
http (>= 3.0, < 5.0)
|
@@ -73,7 +73,7 @@ GEM
|
|
73
73
|
mime-types-data (3.2022.0105)
|
74
74
|
mini_mime (1.1.2)
|
75
75
|
minitest (4.7.5)
|
76
|
-
msgpack (1.5.
|
76
|
+
msgpack (1.5.2)
|
77
77
|
multi_json (1.15.0)
|
78
78
|
netrc (0.11.0)
|
79
79
|
parallel (1.21.0)
|
@@ -105,7 +105,7 @@ GEM
|
|
105
105
|
parser (>= 3.0.1.1)
|
106
106
|
ruby-progressbar (1.11.0)
|
107
107
|
rugged (1.2.0)
|
108
|
-
serverengine (2.
|
108
|
+
serverengine (2.3.0)
|
109
109
|
sigdump (~> 0.2.2)
|
110
110
|
sigdump (0.2.4)
|
111
111
|
simplecov (0.21.2)
|
@@ -126,7 +126,7 @@ GEM
|
|
126
126
|
tzinfo (>= 1.0.0)
|
127
127
|
unf (0.1.4)
|
128
128
|
unf_ext
|
129
|
-
unf_ext (0.0.8.
|
129
|
+
unf_ext (0.0.8.2)
|
130
130
|
unicode-display_width (2.1.0)
|
131
131
|
vcr (6.0.0)
|
132
132
|
webmock (3.14.0)
|
@@ -134,7 +134,7 @@ GEM
|
|
134
134
|
crack (>= 0.3.2)
|
135
135
|
hashdiff (>= 0.4.0, < 2.0.0)
|
136
136
|
webrick (1.7.0)
|
137
|
-
yajl-ruby (1.4.
|
137
|
+
yajl-ruby (1.4.3)
|
138
138
|
|
139
139
|
PLATFORMS
|
140
140
|
ruby
|
@@ -155,4 +155,4 @@ DEPENDENCIES
|
|
155
155
|
yajl-ruby
|
156
156
|
|
157
157
|
BUNDLED WITH
|
158
|
-
2.3.
|
158
|
+
2.3.16
|
@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |gem|
|
7
7
|
gem.name = 'fluent-plugin-kubernetes_metadata_filter'
|
8
|
-
gem.version = '2.
|
8
|
+
gem.version = '2.11.1'
|
9
9
|
gem.authors = ['OpenShift Cluster Logging','Jimmi Dyson']
|
10
10
|
gem.email = ['team-logging@redhat.com','jimmidyson@gmail.com']
|
11
11
|
gem.description = 'Filter plugin to add Kubernetes metadata'
|
@@ -118,11 +118,21 @@ module Fluent::Plugin
|
|
118
118
|
@stats.bump(:pod_cache_api_updates)
|
119
119
|
log.trace("parsed metadata for #{namespace_name}/#{pod_name}: #{metadata}")
|
120
120
|
@cache[metadata['pod_id']] = metadata
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
121
|
+
rescue KubeException => e
|
122
|
+
if e.error_code == 401
|
123
|
+
# recreate client to refresh token
|
124
|
+
log.info("Encountered '401 Unauthorized' exception, recreating client to refresh token")
|
125
|
+
create_client()
|
126
|
+
else
|
127
|
+
log.error "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
|
128
|
+
@stats.bump(:pod_cache_api_nil_error)
|
129
|
+
end
|
130
|
+
{}
|
131
|
+
rescue StandardError => e
|
132
|
+
@stats.bump(:pod_cache_api_nil_error)
|
133
|
+
log.error "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
|
134
|
+
{}
|
135
|
+
end
|
126
136
|
|
127
137
|
def dump_stats
|
128
138
|
@curr_time = Time.now
|
@@ -150,15 +160,27 @@ module Fluent::Plugin
|
|
150
160
|
@stats.bump(:namespace_cache_api_updates)
|
151
161
|
log.trace("parsed metadata for #{namespace_name}: #{metadata}")
|
152
162
|
@namespace_cache[metadata['namespace_id']] = metadata
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
163
|
+
rescue KubeException => e
|
164
|
+
if e.error_code == 401
|
165
|
+
# recreate client to refresh token
|
166
|
+
log.info("Encountered '401 Unauthorized' exception, recreating client to refresh token")
|
167
|
+
create_client()
|
168
|
+
else
|
169
|
+
log.error "Exception '#{e}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
|
170
|
+
@stats.bump(:namespace_cache_api_nil_error)
|
171
|
+
end
|
172
|
+
{}
|
173
|
+
rescue StandardError => e
|
174
|
+
@stats.bump(:namespace_cache_api_nil_error)
|
175
|
+
log.error "Exception '#{e}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
|
176
|
+
{}
|
157
177
|
end
|
158
178
|
|
159
179
|
def initialize
|
160
180
|
super
|
161
181
|
@prev_time = Time.now
|
182
|
+
@ssl_options = {}
|
183
|
+
@auth_options = {}
|
162
184
|
end
|
163
185
|
|
164
186
|
def configure(conf)
|
@@ -230,7 +252,7 @@ module Fluent::Plugin
|
|
230
252
|
end
|
231
253
|
|
232
254
|
if present?(@kubernetes_url)
|
233
|
-
ssl_options = {
|
255
|
+
@ssl_options = {
|
234
256
|
client_cert: present?(@client_cert) ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil,
|
235
257
|
client_key: present?(@client_key) ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
|
236
258
|
ca_file: @ca_file,
|
@@ -249,24 +271,14 @@ module Fluent::Plugin
|
|
249
271
|
0x80000
|
250
272
|
end
|
251
273
|
ssl_store.flags = OpenSSL::X509::V_FLAG_CRL_CHECK_ALL | flagval
|
252
|
-
ssl_options[:cert_store] = ssl_store
|
274
|
+
@ssl_options[:cert_store] = ssl_store
|
253
275
|
end
|
254
276
|
|
255
|
-
auth_options = {}
|
256
|
-
|
257
277
|
if present?(@bearer_token_file)
|
258
|
-
|
259
|
-
auth_options[:bearer_token] = bearer_token
|
278
|
+
@auth_options[:bearer_token_file] = @bearer_token_file
|
260
279
|
end
|
261
280
|
|
262
|
-
|
263
|
-
@client = Kubeclient::Client.new(
|
264
|
-
@kubernetes_url,
|
265
|
-
@apiVersion,
|
266
|
-
ssl_options: ssl_options,
|
267
|
-
auth_options: auth_options,
|
268
|
-
as: :parsed_symbolized
|
269
|
-
)
|
281
|
+
create_client()
|
270
282
|
|
271
283
|
if @test_api_adapter
|
272
284
|
log.info "Extending client with test api adaper #{@test_api_adapter}"
|
@@ -305,6 +317,18 @@ module Fluent::Plugin
|
|
305
317
|
end
|
306
318
|
end
|
307
319
|
|
320
|
+
def create_client()
|
321
|
+
log.debug 'Creating K8S client'
|
322
|
+
@client = nil
|
323
|
+
@client = Kubeclient::Client.new(
|
324
|
+
@kubernetes_url,
|
325
|
+
@apiVersion,
|
326
|
+
ssl_options: @ssl_options,
|
327
|
+
auth_options: @auth_options,
|
328
|
+
as: :parsed_symbolized
|
329
|
+
)
|
330
|
+
end
|
331
|
+
|
308
332
|
def get_metadata_for_record(namespace_name, pod_name, container_name, cache_key, create_time, batch_miss_cache, docker_id)
|
309
333
|
metadata = {
|
310
334
|
'docker' => { 'container_id' => "" },
|
@@ -46,6 +46,39 @@ module KubernetesMetadata
|
|
46
46
|
@stats.bump(:namespace_watch_gone_errors)
|
47
47
|
log.info('410 Gone encountered. Restarting namespace watch to reset resource versions.', e)
|
48
48
|
namespace_watcher = nil
|
49
|
+
rescue KubeException => e
|
50
|
+
if e.error_code == 401
|
51
|
+
# recreate client to refresh token
|
52
|
+
log.info("Encountered '401 Unauthorized' exception in watch, recreating client to refresh token")
|
53
|
+
create_client()
|
54
|
+
namespace_watcher = nil
|
55
|
+
else
|
56
|
+
# treat all other errors the same as StandardError, log, swallow and reset
|
57
|
+
@stats.bump(:namespace_watch_failures)
|
58
|
+
if Thread.current[:namespace_watch_retry_count] < @watch_retry_max_times
|
59
|
+
# Instead of raising exceptions and crashing Fluentd, swallow
|
60
|
+
# the exception and reset the watcher.
|
61
|
+
log.info(
|
62
|
+
'Exception encountered parsing namespace watch event. ' \
|
63
|
+
'The connection might have been closed. Sleeping for ' \
|
64
|
+
"#{Thread.current[:namespace_watch_retry_backoff_interval]} " \
|
65
|
+
'seconds and resetting the namespace watcher.', e
|
66
|
+
)
|
67
|
+
sleep(Thread.current[:namespace_watch_retry_backoff_interval])
|
68
|
+
Thread.current[:namespace_watch_retry_count] += 1
|
69
|
+
Thread.current[:namespace_watch_retry_backoff_interval] *= @watch_retry_exponential_backoff_base
|
70
|
+
namespace_watcher = nil
|
71
|
+
else
|
72
|
+
# Since retries failed for many times, log as errors instead
|
73
|
+
# of info and raise exceptions and trigger Fluentd to restart.
|
74
|
+
message =
|
75
|
+
'Exception encountered parsing namespace watch event. The ' \
|
76
|
+
'connection might have been closed. Retried ' \
|
77
|
+
"#{@watch_retry_max_times} times yet still failing. Restarting."
|
78
|
+
log.error(message, e)
|
79
|
+
raise Fluent::UnrecoverableError, message
|
80
|
+
end
|
81
|
+
end
|
49
82
|
rescue StandardError => e
|
50
83
|
@stats.bump(:namespace_watch_failures)
|
51
84
|
if Thread.current[:namespace_watch_retry_count] < @watch_retry_max_times
|
@@ -47,6 +47,39 @@ module KubernetesMetadata
|
|
47
47
|
@stats.bump(:pod_watch_gone_errors)
|
48
48
|
log.info('410 Gone encountered. Restarting pod watch to reset resource versions.', e)
|
49
49
|
pod_watcher = nil
|
50
|
+
rescue KubeException => e
|
51
|
+
if e.error_code == 401
|
52
|
+
# recreate client to refresh token
|
53
|
+
log.info("Encountered '401 Unauthorized' exception in watch, recreating client to refresh token")
|
54
|
+
create_client()
|
55
|
+
namespace_watcher = nil
|
56
|
+
else
|
57
|
+
# treat all other errors the same as StandardError, log, swallow and reset
|
58
|
+
@stats.bump(:pod_watch_failures)
|
59
|
+
if Thread.current[:pod_watch_retry_count] < @watch_retry_max_times
|
60
|
+
# Instead of raising exceptions and crashing Fluentd, swallow
|
61
|
+
# the exception and reset the watcher.
|
62
|
+
log.info(
|
63
|
+
'Exception encountered parsing pod watch event. The ' \
|
64
|
+
'connection might have been closed. Sleeping for ' \
|
65
|
+
"#{Thread.current[:pod_watch_retry_backoff_interval]} " \
|
66
|
+
'seconds and resetting the pod watcher.', e
|
67
|
+
)
|
68
|
+
sleep(Thread.current[:pod_watch_retry_backoff_interval])
|
69
|
+
Thread.current[:pod_watch_retry_count] += 1
|
70
|
+
Thread.current[:pod_watch_retry_backoff_interval] *= @watch_retry_exponential_backoff_base
|
71
|
+
pod_watcher = nil
|
72
|
+
else
|
73
|
+
# Since retries failed for many times, log as errors instead
|
74
|
+
# of info and raise exceptions and trigger Fluentd to restart.
|
75
|
+
message =
|
76
|
+
'Exception encountered parsing pod watch event. The ' \
|
77
|
+
'connection might have been closed. Retried ' \
|
78
|
+
"#{@watch_retry_max_times} times yet still failing. Restarting."
|
79
|
+
log.error(message, e)
|
80
|
+
raise Fluent::UnrecoverableError, message
|
81
|
+
end
|
82
|
+
end
|
50
83
|
rescue StandardError => e
|
51
84
|
@stats.bump(:pod_watch_failures)
|
52
85
|
if Thread.current[:pod_watch_retry_count] < @watch_retry_max_times
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kubernetes_metadata_filter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.11.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- OpenShift Cluster Logging
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2022-
|
12
|
+
date: 2022-06-24 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
@@ -273,7 +273,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
273
273
|
- !ruby/object:Gem::Version
|
274
274
|
version: '0'
|
275
275
|
requirements: []
|
276
|
-
rubygems_version: 3.
|
276
|
+
rubygems_version: 3.1.4
|
277
277
|
signing_key:
|
278
278
|
specification_version: 4
|
279
279
|
summary: Fluentd filter plugin to add Kubernetes metadata
|