fluent-plugin-kubernetes_metadata_filter 2.10.0 → 2.11.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +8 -8
- data/fluent-plugin-kubernetes_metadata_filter.gemspec +1 -1
- data/lib/fluent/plugin/filter_kubernetes_metadata.rb +47 -23
- data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +33 -0
- data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +33 -0
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f8ed17c405e64607d3fee4b581345f32388a7b1f52db9633d83e6863f6da5358
|
4
|
+
data.tar.gz: e5d6002813b1ffb266b43eda059a2f7a6b050bb69162b52a48cf68e5bf116e7d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c39196e160ffbf56e328f1b4baf2ce61cafbb3152e1c1bfc109a445a78083ed4bd25f28f81356472dffb975a2905cd90ef951f33ba1777572e1b3c86810323cd
|
7
|
+
data.tar.gz: 99377e8dbf5b6f95259b02ea923d22c1195fb582ac89ede2734fe29200305543e6916113ee49ec7c1fe6a4a839882038925925c724d662d27b5ef34f7bdeaf9e
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
fluent-plugin-kubernetes_metadata_filter (2.
|
4
|
+
fluent-plugin-kubernetes_metadata_filter (2.11.1)
|
5
5
|
fluentd (>= 0.14.0, < 1.15)
|
6
6
|
kubeclient (>= 4.0.0, < 5.0.0)
|
7
7
|
lru_redux
|
@@ -54,13 +54,13 @@ GEM
|
|
54
54
|
http-form_data (~> 2.2)
|
55
55
|
http-parser (~> 1.2.0)
|
56
56
|
http-accept (1.7.0)
|
57
|
-
http-cookie (1.0.
|
57
|
+
http-cookie (1.0.5)
|
58
58
|
domain_name (~> 0.5)
|
59
59
|
http-form_data (2.3.0)
|
60
60
|
http-parser (1.2.3)
|
61
61
|
ffi-compiler (>= 1.0, < 2.0)
|
62
62
|
http_parser.rb (0.8.0)
|
63
|
-
jsonpath (1.1.
|
63
|
+
jsonpath (1.1.2)
|
64
64
|
multi_json
|
65
65
|
kubeclient (4.9.3)
|
66
66
|
http (>= 3.0, < 5.0)
|
@@ -73,7 +73,7 @@ GEM
|
|
73
73
|
mime-types-data (3.2022.0105)
|
74
74
|
mini_mime (1.1.2)
|
75
75
|
minitest (4.7.5)
|
76
|
-
msgpack (1.5.
|
76
|
+
msgpack (1.5.2)
|
77
77
|
multi_json (1.15.0)
|
78
78
|
netrc (0.11.0)
|
79
79
|
parallel (1.21.0)
|
@@ -105,7 +105,7 @@ GEM
|
|
105
105
|
parser (>= 3.0.1.1)
|
106
106
|
ruby-progressbar (1.11.0)
|
107
107
|
rugged (1.2.0)
|
108
|
-
serverengine (2.
|
108
|
+
serverengine (2.3.0)
|
109
109
|
sigdump (~> 0.2.2)
|
110
110
|
sigdump (0.2.4)
|
111
111
|
simplecov (0.21.2)
|
@@ -126,7 +126,7 @@ GEM
|
|
126
126
|
tzinfo (>= 1.0.0)
|
127
127
|
unf (0.1.4)
|
128
128
|
unf_ext
|
129
|
-
unf_ext (0.0.8.
|
129
|
+
unf_ext (0.0.8.2)
|
130
130
|
unicode-display_width (2.1.0)
|
131
131
|
vcr (6.0.0)
|
132
132
|
webmock (3.14.0)
|
@@ -134,7 +134,7 @@ GEM
|
|
134
134
|
crack (>= 0.3.2)
|
135
135
|
hashdiff (>= 0.4.0, < 2.0.0)
|
136
136
|
webrick (1.7.0)
|
137
|
-
yajl-ruby (1.4.
|
137
|
+
yajl-ruby (1.4.3)
|
138
138
|
|
139
139
|
PLATFORMS
|
140
140
|
ruby
|
@@ -155,4 +155,4 @@ DEPENDENCIES
|
|
155
155
|
yajl-ruby
|
156
156
|
|
157
157
|
BUNDLED WITH
|
158
|
-
2.3.
|
158
|
+
2.3.16
|
@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |gem|
|
7
7
|
gem.name = 'fluent-plugin-kubernetes_metadata_filter'
|
8
|
-
gem.version = '2.
|
8
|
+
gem.version = '2.11.1'
|
9
9
|
gem.authors = ['OpenShift Cluster Logging','Jimmi Dyson']
|
10
10
|
gem.email = ['team-logging@redhat.com','jimmidyson@gmail.com']
|
11
11
|
gem.description = 'Filter plugin to add Kubernetes metadata'
|
@@ -118,11 +118,21 @@ module Fluent::Plugin
|
|
118
118
|
@stats.bump(:pod_cache_api_updates)
|
119
119
|
log.trace("parsed metadata for #{namespace_name}/#{pod_name}: #{metadata}")
|
120
120
|
@cache[metadata['pod_id']] = metadata
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
121
|
+
rescue KubeException => e
|
122
|
+
if e.error_code == 401
|
123
|
+
# recreate client to refresh token
|
124
|
+
log.info("Encountered '401 Unauthorized' exception, recreating client to refresh token")
|
125
|
+
create_client()
|
126
|
+
else
|
127
|
+
log.error "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
|
128
|
+
@stats.bump(:pod_cache_api_nil_error)
|
129
|
+
end
|
130
|
+
{}
|
131
|
+
rescue StandardError => e
|
132
|
+
@stats.bump(:pod_cache_api_nil_error)
|
133
|
+
log.error "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
|
134
|
+
{}
|
135
|
+
end
|
126
136
|
|
127
137
|
def dump_stats
|
128
138
|
@curr_time = Time.now
|
@@ -150,15 +160,27 @@ module Fluent::Plugin
|
|
150
160
|
@stats.bump(:namespace_cache_api_updates)
|
151
161
|
log.trace("parsed metadata for #{namespace_name}: #{metadata}")
|
152
162
|
@namespace_cache[metadata['namespace_id']] = metadata
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
163
|
+
rescue KubeException => e
|
164
|
+
if e.error_code == 401
|
165
|
+
# recreate client to refresh token
|
166
|
+
log.info("Encountered '401 Unauthorized' exception, recreating client to refresh token")
|
167
|
+
create_client()
|
168
|
+
else
|
169
|
+
log.error "Exception '#{e}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
|
170
|
+
@stats.bump(:namespace_cache_api_nil_error)
|
171
|
+
end
|
172
|
+
{}
|
173
|
+
rescue StandardError => e
|
174
|
+
@stats.bump(:namespace_cache_api_nil_error)
|
175
|
+
log.error "Exception '#{e}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
|
176
|
+
{}
|
157
177
|
end
|
158
178
|
|
159
179
|
def initialize
|
160
180
|
super
|
161
181
|
@prev_time = Time.now
|
182
|
+
@ssl_options = {}
|
183
|
+
@auth_options = {}
|
162
184
|
end
|
163
185
|
|
164
186
|
def configure(conf)
|
@@ -230,7 +252,7 @@ module Fluent::Plugin
|
|
230
252
|
end
|
231
253
|
|
232
254
|
if present?(@kubernetes_url)
|
233
|
-
ssl_options = {
|
255
|
+
@ssl_options = {
|
234
256
|
client_cert: present?(@client_cert) ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil,
|
235
257
|
client_key: present?(@client_key) ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
|
236
258
|
ca_file: @ca_file,
|
@@ -249,24 +271,14 @@ module Fluent::Plugin
|
|
249
271
|
0x80000
|
250
272
|
end
|
251
273
|
ssl_store.flags = OpenSSL::X509::V_FLAG_CRL_CHECK_ALL | flagval
|
252
|
-
ssl_options[:cert_store] = ssl_store
|
274
|
+
@ssl_options[:cert_store] = ssl_store
|
253
275
|
end
|
254
276
|
|
255
|
-
auth_options = {}
|
256
|
-
|
257
277
|
if present?(@bearer_token_file)
|
258
|
-
|
259
|
-
auth_options[:bearer_token] = bearer_token
|
278
|
+
@auth_options[:bearer_token_file] = @bearer_token_file
|
260
279
|
end
|
261
280
|
|
262
|
-
|
263
|
-
@client = Kubeclient::Client.new(
|
264
|
-
@kubernetes_url,
|
265
|
-
@apiVersion,
|
266
|
-
ssl_options: ssl_options,
|
267
|
-
auth_options: auth_options,
|
268
|
-
as: :parsed_symbolized
|
269
|
-
)
|
281
|
+
create_client()
|
270
282
|
|
271
283
|
if @test_api_adapter
|
272
284
|
log.info "Extending client with test api adaper #{@test_api_adapter}"
|
@@ -305,6 +317,18 @@ module Fluent::Plugin
|
|
305
317
|
end
|
306
318
|
end
|
307
319
|
|
320
|
+
def create_client()
|
321
|
+
log.debug 'Creating K8S client'
|
322
|
+
@client = nil
|
323
|
+
@client = Kubeclient::Client.new(
|
324
|
+
@kubernetes_url,
|
325
|
+
@apiVersion,
|
326
|
+
ssl_options: @ssl_options,
|
327
|
+
auth_options: @auth_options,
|
328
|
+
as: :parsed_symbolized
|
329
|
+
)
|
330
|
+
end
|
331
|
+
|
308
332
|
def get_metadata_for_record(namespace_name, pod_name, container_name, cache_key, create_time, batch_miss_cache, docker_id)
|
309
333
|
metadata = {
|
310
334
|
'docker' => { 'container_id' => "" },
|
@@ -46,6 +46,39 @@ module KubernetesMetadata
|
|
46
46
|
@stats.bump(:namespace_watch_gone_errors)
|
47
47
|
log.info('410 Gone encountered. Restarting namespace watch to reset resource versions.', e)
|
48
48
|
namespace_watcher = nil
|
49
|
+
rescue KubeException => e
|
50
|
+
if e.error_code == 401
|
51
|
+
# recreate client to refresh token
|
52
|
+
log.info("Encountered '401 Unauthorized' exception in watch, recreating client to refresh token")
|
53
|
+
create_client()
|
54
|
+
namespace_watcher = nil
|
55
|
+
else
|
56
|
+
# treat all other errors the same as StandardError, log, swallow and reset
|
57
|
+
@stats.bump(:namespace_watch_failures)
|
58
|
+
if Thread.current[:namespace_watch_retry_count] < @watch_retry_max_times
|
59
|
+
# Instead of raising exceptions and crashing Fluentd, swallow
|
60
|
+
# the exception and reset the watcher.
|
61
|
+
log.info(
|
62
|
+
'Exception encountered parsing namespace watch event. ' \
|
63
|
+
'The connection might have been closed. Sleeping for ' \
|
64
|
+
"#{Thread.current[:namespace_watch_retry_backoff_interval]} " \
|
65
|
+
'seconds and resetting the namespace watcher.', e
|
66
|
+
)
|
67
|
+
sleep(Thread.current[:namespace_watch_retry_backoff_interval])
|
68
|
+
Thread.current[:namespace_watch_retry_count] += 1
|
69
|
+
Thread.current[:namespace_watch_retry_backoff_interval] *= @watch_retry_exponential_backoff_base
|
70
|
+
namespace_watcher = nil
|
71
|
+
else
|
72
|
+
# Since retries failed for many times, log as errors instead
|
73
|
+
# of info and raise exceptions and trigger Fluentd to restart.
|
74
|
+
message =
|
75
|
+
'Exception encountered parsing namespace watch event. The ' \
|
76
|
+
'connection might have been closed. Retried ' \
|
77
|
+
"#{@watch_retry_max_times} times yet still failing. Restarting."
|
78
|
+
log.error(message, e)
|
79
|
+
raise Fluent::UnrecoverableError, message
|
80
|
+
end
|
81
|
+
end
|
49
82
|
rescue StandardError => e
|
50
83
|
@stats.bump(:namespace_watch_failures)
|
51
84
|
if Thread.current[:namespace_watch_retry_count] < @watch_retry_max_times
|
@@ -47,6 +47,39 @@ module KubernetesMetadata
|
|
47
47
|
@stats.bump(:pod_watch_gone_errors)
|
48
48
|
log.info('410 Gone encountered. Restarting pod watch to reset resource versions.', e)
|
49
49
|
pod_watcher = nil
|
50
|
+
rescue KubeException => e
|
51
|
+
if e.error_code == 401
|
52
|
+
# recreate client to refresh token
|
53
|
+
log.info("Encountered '401 Unauthorized' exception in watch, recreating client to refresh token")
|
54
|
+
create_client()
|
55
|
+
namespace_watcher = nil
|
56
|
+
else
|
57
|
+
# treat all other errors the same as StandardError, log, swallow and reset
|
58
|
+
@stats.bump(:pod_watch_failures)
|
59
|
+
if Thread.current[:pod_watch_retry_count] < @watch_retry_max_times
|
60
|
+
# Instead of raising exceptions and crashing Fluentd, swallow
|
61
|
+
# the exception and reset the watcher.
|
62
|
+
log.info(
|
63
|
+
'Exception encountered parsing pod watch event. The ' \
|
64
|
+
'connection might have been closed. Sleeping for ' \
|
65
|
+
"#{Thread.current[:pod_watch_retry_backoff_interval]} " \
|
66
|
+
'seconds and resetting the pod watcher.', e
|
67
|
+
)
|
68
|
+
sleep(Thread.current[:pod_watch_retry_backoff_interval])
|
69
|
+
Thread.current[:pod_watch_retry_count] += 1
|
70
|
+
Thread.current[:pod_watch_retry_backoff_interval] *= @watch_retry_exponential_backoff_base
|
71
|
+
pod_watcher = nil
|
72
|
+
else
|
73
|
+
# Since retries failed for many times, log as errors instead
|
74
|
+
# of info and raise exceptions and trigger Fluentd to restart.
|
75
|
+
message =
|
76
|
+
'Exception encountered parsing pod watch event. The ' \
|
77
|
+
'connection might have been closed. Retried ' \
|
78
|
+
"#{@watch_retry_max_times} times yet still failing. Restarting."
|
79
|
+
log.error(message, e)
|
80
|
+
raise Fluent::UnrecoverableError, message
|
81
|
+
end
|
82
|
+
end
|
50
83
|
rescue StandardError => e
|
51
84
|
@stats.bump(:pod_watch_failures)
|
52
85
|
if Thread.current[:pod_watch_retry_count] < @watch_retry_max_times
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kubernetes_metadata_filter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.11.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- OpenShift Cluster Logging
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2022-
|
12
|
+
date: 2022-06-24 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
@@ -273,7 +273,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
273
273
|
- !ruby/object:Gem::Version
|
274
274
|
version: '0'
|
275
275
|
requirements: []
|
276
|
-
rubygems_version: 3.
|
276
|
+
rubygems_version: 3.1.4
|
277
277
|
signing_key:
|
278
278
|
specification_version: 4
|
279
279
|
summary: Fluentd filter plugin to add Kubernetes metadata
|