fluent-plugin-kubernetes_metadata_filter 2.1.4 → 2.9.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +57 -0
- data/.gitignore +0 -1
- data/.rubocop.yml +57 -0
- data/Gemfile +4 -2
- data/Gemfile.lock +158 -0
- data/README.md +48 -28
- data/Rakefile +15 -11
- data/fluent-plugin-kubernetes_metadata_filter.gemspec +25 -28
- data/lib/fluent/plugin/filter_kubernetes_metadata.rb +185 -131
- data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb +27 -20
- data/lib/fluent/plugin/kubernetes_metadata_common.rb +59 -33
- data/lib/fluent/plugin/kubernetes_metadata_stats.rb +6 -6
- data/lib/fluent/plugin/kubernetes_metadata_test_api_adapter.rb +68 -0
- data/lib/fluent/plugin/kubernetes_metadata_util.rb +53 -0
- data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +121 -27
- data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +138 -29
- data/release_notes.md +42 -0
- data/test/cassettes/kubernetes_docker_metadata_annotations.yml +0 -34
- data/test/cassettes/{kubernetes_docker_metadata_dotted_labels.yml → kubernetes_docker_metadata_dotted_slashed_labels.yml} +0 -34
- data/test/cassettes/kubernetes_get_api_v1.yml +193 -0
- data/test/cassettes/kubernetes_get_api_v1_using_token.yml +195 -0
- data/test/cassettes/kubernetes_get_namespace_default.yml +69 -0
- data/test/cassettes/kubernetes_get_namespace_default_using_token.yml +71 -0
- data/test/cassettes/{kubernetes_docker_metadata.yml → kubernetes_get_pod.yml} +0 -82
- data/test/cassettes/{metadata_with_namespace_id.yml → kubernetes_get_pod_container_init.yml} +3 -134
- data/test/cassettes/{kubernetes_docker_metadata_using_bearer_token.yml → kubernetes_get_pod_using_token.yml} +5 -105
- data/test/cassettes/metadata_from_tag_and_journald_fields.yml +0 -255
- data/test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml +0 -255
- data/test/cassettes/{non_kubernetes_docker_metadata.yml → valid_kubernetes_api_server_using_token.yml} +4 -44
- data/test/helper.rb +20 -2
- data/test/plugin/test_cache_stats.rb +10 -13
- data/test/plugin/test_cache_strategy.rb +158 -160
- data/test/plugin/test_filter_kubernetes_metadata.rb +480 -320
- data/test/plugin/test_utils.rb +56 -0
- data/test/plugin/test_watch_namespaces.rb +209 -55
- data/test/plugin/test_watch_pods.rb +302 -103
- data/test/plugin/watch_test.rb +52 -33
- metadata +69 -72
- data/circle.yml +0 -17
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
#
|
2
4
|
# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
|
3
5
|
# Kubernetes metadata
|
@@ -20,10 +22,12 @@
|
|
20
22
|
require_relative 'kubernetes_metadata_cache_strategy'
|
21
23
|
require_relative 'kubernetes_metadata_common'
|
22
24
|
require_relative 'kubernetes_metadata_stats'
|
25
|
+
require_relative 'kubernetes_metadata_util'
|
23
26
|
require_relative 'kubernetes_metadata_watch_namespaces'
|
24
27
|
require_relative 'kubernetes_metadata_watch_pods'
|
25
28
|
|
26
29
|
require 'fluent/plugin/filter'
|
30
|
+
require 'resolv'
|
27
31
|
|
28
32
|
module Fluent::Plugin
|
29
33
|
class KubernetesMetadataFilter < Fluent::Plugin::Filter
|
@@ -32,6 +36,7 @@ module Fluent::Plugin
|
|
32
36
|
|
33
37
|
include KubernetesMetadata::CacheStrategy
|
34
38
|
include KubernetesMetadata::Common
|
39
|
+
include KubernetesMetadata::Util
|
35
40
|
include KubernetesMetadata::WatchNamespaces
|
36
41
|
include KubernetesMetadata::WatchPods
|
37
42
|
|
@@ -46,13 +51,23 @@ module Fluent::Plugin
|
|
46
51
|
config_param :client_key, :string, default: nil
|
47
52
|
config_param :ca_file, :string, default: nil
|
48
53
|
config_param :verify_ssl, :bool, default: true
|
49
|
-
|
50
|
-
|
51
|
-
|
54
|
+
|
55
|
+
REGEX_VAR_LOG_PODS = '(var\.log\.pods)\.(?<namespace>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<pod_uuid>[a-z0-9-]*)\.(?<container_name>.+)\..*\.log$'
|
56
|
+
REGEX_VAR_LOG_CONTAINERS = '(var\.log\.containers)\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
|
57
|
+
|
58
|
+
#tag_to_kubernetes_name_regexp which must include named capture groups:
|
59
|
+
# namespace - The namespace in which the pod is deployed
|
60
|
+
# pod_name - The pod name
|
61
|
+
# container_name - The name of the container
|
62
|
+
# pod_uuid (/var/log/pods) | docker_id (/var/log/containers) - Unique identifier used in caching of either pod_uuid or the container hash
|
63
|
+
config_param :tag_to_kubernetes_name_regexp, :string, default: "(#{REGEX_VAR_LOG_PODS}|#{REGEX_VAR_LOG_CONTAINERS})"
|
64
|
+
|
52
65
|
config_param :bearer_token_file, :string, default: nil
|
53
66
|
config_param :secret_dir, :string, default: '/var/run/secrets/kubernetes.io/serviceaccount'
|
54
67
|
config_param :de_dot, :bool, default: true
|
55
68
|
config_param :de_dot_separator, :string, default: '_'
|
69
|
+
config_param :de_slash, :bool, default: false
|
70
|
+
config_param :de_slash_separator, :string, default: '__'
|
56
71
|
# if reading from the journal, the record will contain the following fields in the following
|
57
72
|
# format:
|
58
73
|
# CONTAINER_NAME=k8s_$containername.$containerhash_$podname_$namespacename_$poduuid_$rand32bitashex
|
@@ -61,10 +76,10 @@ module Fluent::Plugin
|
|
61
76
|
# Field 2 is the container_hash, field 5 is the pod_id, and field 6 is the pod_randhex
|
62
77
|
# I would have included them as named groups, but you can't have named groups that are
|
63
78
|
# non-capturing :P
|
64
|
-
# parse format is defined here: https://github.com/kubernetes/kubernetes/blob/
|
79
|
+
# parse format is defined here: https://github.com/kubernetes/kubernetes/blob/release-1.6/pkg/kubelet/dockertools/docker.go#L317
|
65
80
|
config_param :container_name_to_kubernetes_regexp,
|
66
81
|
:string,
|
67
|
-
:
|
82
|
+
default: '^(?<name_prefix>[^_]+)_(?<container_name>[^\._]+)(\.(?<container_hash>[^_]+))?_(?<pod_name>[^_]+)_(?<namespace>[^_]+)_[^_]+_[^_]+$'
|
68
83
|
|
69
84
|
config_param :annotation_match, :array, default: []
|
70
85
|
config_param :stats_interval, :integer, default: 30
|
@@ -72,38 +87,47 @@ module Fluent::Plugin
|
|
72
87
|
config_param :orphaned_namespace_name, :string, default: '.orphaned'
|
73
88
|
config_param :orphaned_namespace_id, :string, default: 'orphaned'
|
74
89
|
config_param :lookup_from_k8s_field, :bool, default: true
|
90
|
+
# if `ca_file` is for an intermediate CA, or otherwise we do not have the root CA and want
|
91
|
+
# to trust the intermediate CA certs we do have, set this to `true` - this corresponds to
|
92
|
+
# the openssl s_client -partial_chain flag and X509_V_FLAG_PARTIAL_CHAIN
|
93
|
+
config_param :ssl_partial_chain, :bool, default: false
|
94
|
+
config_param :skip_labels, :bool, default: false
|
95
|
+
config_param :skip_container_metadata, :bool, default: false
|
96
|
+
config_param :skip_master_url, :bool, default: false
|
97
|
+
config_param :skip_namespace_metadata, :bool, default: false
|
98
|
+
|
99
|
+
# A classname in the form of Test::APIAdapter which will try
|
100
|
+
# to be resolved from a relative named file 'test_api_adapter'
|
101
|
+
config_param :test_api_adapter, :string, default: nil
|
102
|
+
|
103
|
+
# The time interval in seconds for retry backoffs when watch connections fail.
|
104
|
+
config_param :watch_retry_interval, :integer, default: 1
|
105
|
+
# The base number of exponential backoff for retries.
|
106
|
+
config_param :watch_retry_exponential_backoff_base, :integer, default: 2
|
107
|
+
# The maximum number of times to retry pod and namespace watches.
|
108
|
+
config_param :watch_retry_max_times, :integer, default: 10
|
75
109
|
|
76
110
|
def fetch_pod_metadata(namespace_name, pod_name)
|
77
|
-
log.trace("fetching pod metadata: #{namespace_name}/#{pod_name}")
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
return metadata
|
91
|
-
rescue Exception=>e
|
92
|
-
log.debug(e)
|
93
|
-
@stats.bump(:pod_cache_api_nil_bad_resp_payload)
|
94
|
-
log.trace("returning empty metadata for #{namespace_name}/#{pod_name} due to error '#{e}'") if log.trace?
|
95
|
-
end
|
96
|
-
end
|
97
|
-
rescue Exception=>e
|
98
|
-
@stats.bump(:pod_cache_api_nil_error)
|
99
|
-
log.debug "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
|
100
|
-
end
|
111
|
+
log.trace("fetching pod metadata: #{namespace_name}/#{pod_name}")
|
112
|
+
options = {
|
113
|
+
resource_version: '0' # Fetch from API server cache instead of etcd quorum read
|
114
|
+
}
|
115
|
+
pod_object = @client.get_pod(pod_name, namespace_name, options)
|
116
|
+
log.trace("raw metadata for #{namespace_name}/#{pod_name}: #{pod_object}")
|
117
|
+
metadata = parse_pod_metadata(pod_object)
|
118
|
+
@stats.bump(:pod_cache_api_updates)
|
119
|
+
log.trace("parsed metadata for #{namespace_name}/#{pod_name}: #{metadata}")
|
120
|
+
@cache[metadata['pod_id']] = metadata
|
121
|
+
rescue StandardError => e
|
122
|
+
@stats.bump(:pod_cache_api_nil_error)
|
123
|
+
log.debug "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
|
101
124
|
{}
|
102
125
|
end
|
103
126
|
|
104
127
|
def dump_stats
|
105
128
|
@curr_time = Time.now
|
106
129
|
return if @curr_time.to_i - @prev_time.to_i < @stats_interval
|
130
|
+
|
107
131
|
@prev_time = @curr_time
|
108
132
|
@stats.set(:pod_cache_size, @cache.count)
|
109
133
|
@stats.set(:namespace_cache_size, @namespace_cache.count) if @namespace_cache
|
@@ -116,30 +140,19 @@ module Fluent::Plugin
|
|
116
140
|
end
|
117
141
|
|
118
142
|
def fetch_namespace_metadata(namespace_name)
|
119
|
-
log.trace("fetching namespace metadata: #{namespace_name}")
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
return metadata
|
133
|
-
rescue Exception => e
|
134
|
-
log.debug(e)
|
135
|
-
@stats.bump(:namespace_cache_api_nil_bad_resp_payload)
|
136
|
-
log.trace("returning empty metadata for #{namespace_name} due to error '#{e}'") if log.trace?
|
137
|
-
end
|
138
|
-
end
|
139
|
-
rescue Exception => kube_error
|
140
|
-
@stats.bump(:namespace_cache_api_nil_error)
|
141
|
-
log.debug "Exception '#{kube_error}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
|
142
|
-
end
|
143
|
+
log.trace("fetching namespace metadata: #{namespace_name}")
|
144
|
+
options = {
|
145
|
+
resource_version: '0' # Fetch from API server cache instead of etcd quorum read
|
146
|
+
}
|
147
|
+
namespace_object = @client.get_namespace(namespace_name, nil, options)
|
148
|
+
log.trace("raw metadata for #{namespace_name}: #{namespace_object}")
|
149
|
+
metadata = parse_namespace_metadata(namespace_object)
|
150
|
+
@stats.bump(:namespace_cache_api_updates)
|
151
|
+
log.trace("parsed metadata for #{namespace_name}: #{metadata}")
|
152
|
+
@namespace_cache[metadata['namespace_id']] = metadata
|
153
|
+
rescue StandardError => e
|
154
|
+
@stats.bump(:namespace_cache_api_nil_error)
|
155
|
+
log.debug "Exception '#{e}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
|
143
156
|
{}
|
144
157
|
end
|
145
158
|
|
@@ -151,21 +164,20 @@ module Fluent::Plugin
|
|
151
164
|
def configure(conf)
|
152
165
|
super
|
153
166
|
|
154
|
-
def log.trace?
|
155
|
-
level == Fluent::Log::LEVEL_TRACE
|
156
|
-
end
|
157
|
-
|
158
167
|
require 'kubeclient'
|
159
|
-
require 'active_support/core_ext/object/blank'
|
160
168
|
require 'lru_redux'
|
161
169
|
@stats = KubernetesMetadata::Stats.new
|
162
170
|
|
163
|
-
if @de_dot &&
|
171
|
+
if @de_dot && @de_dot_separator.include?('.')
|
164
172
|
raise Fluent::ConfigError, "Invalid de_dot_separator: cannot be or contain '.'"
|
165
173
|
end
|
166
174
|
|
175
|
+
if @de_slash && @de_slash_separator.include?('/')
|
176
|
+
raise Fluent::ConfigError, "Invalid de_slash_separator: cannot be or contain '/'"
|
177
|
+
end
|
178
|
+
|
167
179
|
if @cache_ttl < 0
|
168
|
-
log.info
|
180
|
+
log.info 'Setting the cache TTL to :none because it was <= 0'
|
169
181
|
@cache_ttl = :none
|
170
182
|
end
|
171
183
|
|
@@ -179,17 +191,24 @@ module Fluent::Plugin
|
|
179
191
|
@namespace_cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl)
|
180
192
|
|
181
193
|
@tag_to_kubernetes_name_regexp_compiled = Regexp.compile(@tag_to_kubernetes_name_regexp)
|
194
|
+
|
182
195
|
@container_name_to_kubernetes_regexp_compiled = Regexp.compile(@container_name_to_kubernetes_regexp)
|
183
196
|
|
184
197
|
# Use Kubernetes default service account if we're in a pod.
|
185
198
|
if @kubernetes_url.nil?
|
186
|
-
log.debug
|
199
|
+
log.debug 'Kubernetes URL is not set - inspecting environ'
|
187
200
|
|
188
201
|
env_host = ENV['KUBERNETES_SERVICE_HOST']
|
189
202
|
env_port = ENV['KUBERNETES_SERVICE_PORT']
|
190
|
-
if
|
203
|
+
if present?(env_host) && present?(env_port)
|
204
|
+
if env_host =~ Resolv::IPv6::Regex
|
205
|
+
# Brackets are needed around IPv6 addresses
|
206
|
+
env_host = "[#{env_host}]"
|
207
|
+
end
|
191
208
|
@kubernetes_url = "https://#{env_host}:#{env_port}/api"
|
192
209
|
log.debug "Kubernetes URL is now '#{@kubernetes_url}'"
|
210
|
+
else
|
211
|
+
log.debug 'No Kubernetes URL could be found in config or environ'
|
193
212
|
end
|
194
213
|
end
|
195
214
|
|
@@ -199,48 +218,77 @@ module Fluent::Plugin
|
|
199
218
|
ca_cert = File.join(@secret_dir, K8_POD_CA_CERT)
|
200
219
|
pod_token = File.join(@secret_dir, K8_POD_TOKEN)
|
201
220
|
|
202
|
-
if
|
221
|
+
if !present?(@ca_file) && File.exist?(ca_cert)
|
203
222
|
log.debug "Found CA certificate: #{ca_cert}"
|
204
223
|
@ca_file = ca_cert
|
205
224
|
end
|
206
225
|
|
207
|
-
if
|
226
|
+
if !present?(@bearer_token_file) && File.exist?(pod_token)
|
208
227
|
log.debug "Found pod token: #{pod_token}"
|
209
228
|
@bearer_token_file = pod_token
|
210
229
|
end
|
211
230
|
end
|
212
231
|
|
213
|
-
if @kubernetes_url
|
214
|
-
|
232
|
+
if present?(@kubernetes_url)
|
215
233
|
ssl_options = {
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
234
|
+
client_cert: present?(@client_cert) ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil,
|
235
|
+
client_key: present?(@client_key) ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
|
236
|
+
ca_file: @ca_file,
|
237
|
+
verify_ssl: @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
|
220
238
|
}
|
221
239
|
|
240
|
+
if @ssl_partial_chain
|
241
|
+
# taken from the ssl.rb OpenSSL::SSL::SSLContext code for DEFAULT_CERT_STORE
|
242
|
+
require 'openssl'
|
243
|
+
ssl_store = OpenSSL::X509::Store.new
|
244
|
+
ssl_store.set_default_paths
|
245
|
+
flagval = if defined? OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
|
246
|
+
OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
|
247
|
+
else
|
248
|
+
# this version of ruby does not define OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
|
249
|
+
0x80000
|
250
|
+
end
|
251
|
+
ssl_store.flags = OpenSSL::X509::V_FLAG_CRL_CHECK_ALL | flagval
|
252
|
+
ssl_options[:cert_store] = ssl_store
|
253
|
+
end
|
254
|
+
|
222
255
|
auth_options = {}
|
223
256
|
|
224
|
-
if @bearer_token_file
|
257
|
+
if present?(@bearer_token_file)
|
225
258
|
bearer_token = File.read(@bearer_token_file)
|
226
259
|
auth_options[:bearer_token] = bearer_token
|
227
260
|
end
|
228
261
|
|
229
|
-
log.debug
|
230
|
-
@client = Kubeclient::Client.new
|
231
|
-
|
232
|
-
|
262
|
+
log.debug 'Creating K8S client'
|
263
|
+
@client = Kubeclient::Client.new(
|
264
|
+
@kubernetes_url,
|
265
|
+
@apiVersion,
|
266
|
+
ssl_options: ssl_options,
|
267
|
+
auth_options: auth_options,
|
268
|
+
as: :parsed_symbolized
|
269
|
+
)
|
270
|
+
|
271
|
+
if @test_api_adapter
|
272
|
+
log.info "Extending client with test api adaper #{@test_api_adapter}"
|
273
|
+
require_relative @test_api_adapter.underscore
|
274
|
+
@client.extend(eval(@test_api_adapter))
|
275
|
+
end
|
233
276
|
|
234
277
|
begin
|
235
278
|
@client.api_valid?
|
236
|
-
rescue KubeException =>
|
237
|
-
raise Fluent::ConfigError, "Invalid Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}: #{
|
279
|
+
rescue KubeException => e
|
280
|
+
raise Fluent::ConfigError, "Invalid Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}: #{e.message}"
|
238
281
|
end
|
239
282
|
|
240
283
|
if @watch
|
241
|
-
|
242
|
-
|
243
|
-
|
284
|
+
if ENV['K8S_NODE_NAME'].nil? || ENV['K8S_NODE_NAME'].strip.empty?
|
285
|
+
log.warn("!! The environment variable 'K8S_NODE_NAME' is not set to the node name which can affect the API server and watch efficiency !!")
|
286
|
+
end
|
287
|
+
|
288
|
+
pod_thread = Thread.new(self, &:set_up_pod_thread)
|
289
|
+
pod_thread.abort_on_exception = true
|
290
|
+
|
291
|
+
namespace_thread = Thread.new(self, &:set_up_namespace_thread)
|
244
292
|
namespace_thread.abort_on_exception = true
|
245
293
|
end
|
246
294
|
end
|
@@ -251,51 +299,38 @@ module Fluent::Plugin
|
|
251
299
|
|
252
300
|
@annotations_regexps = []
|
253
301
|
@annotation_match.each do |regexp|
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
log.error "Error: invalid regular expression in annotation_match: #{e}"
|
258
|
-
end
|
302
|
+
@annotations_regexps << Regexp.compile(regexp)
|
303
|
+
rescue RegexpError => e
|
304
|
+
log.error "Error: invalid regular expression in annotation_match: #{e}"
|
259
305
|
end
|
260
|
-
|
261
306
|
end
|
262
307
|
|
263
|
-
def get_metadata_for_record(namespace_name, pod_name, container_name,
|
308
|
+
def get_metadata_for_record(namespace_name, pod_name, container_name, cache_key, create_time, batch_miss_cache, docker_id)
|
264
309
|
metadata = {
|
265
|
-
'docker' => {'container_id' =>
|
310
|
+
'docker' => { 'container_id' => "" },
|
266
311
|
'kubernetes' => {
|
267
|
-
'container_name'
|
268
|
-
'namespace_name'
|
269
|
-
'pod_name'
|
312
|
+
'container_name' => container_name,
|
313
|
+
'namespace_name' => namespace_name,
|
314
|
+
'pod_name' => pod_name
|
270
315
|
}
|
271
316
|
}
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
metadata['kubernetes']['
|
317
|
+
metadata['docker']['container_id'] = docker_id unless docker_id.nil?
|
318
|
+
container_cache_key = container_name
|
319
|
+
if present?(@kubernetes_url)
|
320
|
+
pod_metadata = get_pod_metadata(cache_key, namespace_name, pod_name, create_time, batch_miss_cache)
|
321
|
+
if (pod_metadata.include? 'containers') && (pod_metadata['containers'].include? container_cache_key) && !@skip_container_metadata
|
322
|
+
metadata['kubernetes']['container_image'] = pod_metadata['containers'][container_cache_key]['image']
|
323
|
+
metadata['kubernetes']['container_image_id'] = pod_metadata['containers'][container_cache_key]['image_id'] unless pod_metadata['containers'][container_cache_key]['image_id'].empty?
|
324
|
+
metadata['docker']['container_id'] = pod_metadata['containers'][container_cache_key]['containerID'] unless pod_metadata['containers'][container_cache_key]['containerID'].empty?
|
278
325
|
end
|
279
326
|
|
280
327
|
metadata['kubernetes'].merge!(pod_metadata) if pod_metadata
|
281
328
|
metadata['kubernetes'].delete('containers')
|
282
329
|
end
|
330
|
+
metadata.delete('docker') if metadata['docker'] && (metadata['docker']['container_id'].nil? || metadata['docker']['container_id'].empty?)
|
283
331
|
metadata
|
284
332
|
end
|
285
333
|
|
286
|
-
def create_time_from_record(record, internal_time)
|
287
|
-
time_key = @time_fields.detect{ |ii| record.has_key?(ii) }
|
288
|
-
time = record[time_key]
|
289
|
-
if time.nil? || time.chop.empty?
|
290
|
-
return internal_time
|
291
|
-
end
|
292
|
-
if ['_SOURCE_REALTIME_TIMESTAMP', '__REALTIME_TIMESTAMP'].include?(time_key)
|
293
|
-
timei= time.to_i
|
294
|
-
return Time.at(timei / 1000000, timei % 1000000)
|
295
|
-
end
|
296
|
-
return Time.parse(time)
|
297
|
-
end
|
298
|
-
|
299
334
|
def filter_stream(tag, es)
|
300
335
|
return es if (es.respond_to?(:empty?) && es.empty?) || !es.is_a?(Fluent::EventStream)
|
301
336
|
new_es = Fluent::MultiEventStream.new
|
@@ -304,26 +339,31 @@ module Fluent::Plugin
|
|
304
339
|
batch_miss_cache = {}
|
305
340
|
es.each do |time, record|
|
306
341
|
if tag_match_data && tag_metadata.nil?
|
342
|
+
cache_key = if tag_match_data.names.include?('pod_uuid') && !tag_match_data['pod_uuid'].nil?
|
343
|
+
tag_match_data['pod_uuid']
|
344
|
+
else
|
345
|
+
tag_match_data['docker_id']
|
346
|
+
end
|
347
|
+
docker_id = tag_match_data.names.include?('docker_id') ? tag_match_data['docker_id'] : nil
|
307
348
|
tag_metadata = get_metadata_for_record(tag_match_data['namespace'], tag_match_data['pod_name'], tag_match_data['container_name'],
|
308
|
-
|
349
|
+
cache_key, create_time_from_record(record, time), batch_miss_cache, docker_id)
|
309
350
|
end
|
310
351
|
metadata = Marshal.load(Marshal.dump(tag_metadata)) if tag_metadata
|
311
352
|
if (@use_journal || @use_journal.nil?) &&
|
312
|
-
|
353
|
+
(j_metadata = get_metadata_for_journal_record(record, time, batch_miss_cache))
|
313
354
|
metadata = j_metadata
|
314
355
|
end
|
315
|
-
if @lookup_from_k8s_field && record.
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
356
|
+
if @lookup_from_k8s_field && record.key?('kubernetes') && record.key?('docker') &&
|
357
|
+
record['kubernetes'].respond_to?(:has_key?) && record['docker'].respond_to?(:has_key?) &&
|
358
|
+
record['kubernetes'].key?('namespace_name') &&
|
359
|
+
record['kubernetes'].key?('pod_name') &&
|
360
|
+
record['kubernetes'].key?('container_name') &&
|
361
|
+
record['docker'].key?('container_id') &&
|
362
|
+
(k_metadata = get_metadata_for_record(record['kubernetes']['namespace_name'], record['kubernetes']['pod_name'],
|
363
|
+
record['kubernetes']['container_name'], record['docker']['container_id'],
|
364
|
+
create_time_from_record(record, time), batch_miss_cache, record['docker']['container_id']))
|
365
|
+
metadata = k_metadata
|
325
366
|
end
|
326
|
-
|
327
367
|
record = record.merge(metadata) if metadata
|
328
368
|
new_es.add(time, record)
|
329
369
|
end
|
@@ -333,16 +373,16 @@ module Fluent::Plugin
|
|
333
373
|
|
334
374
|
def get_metadata_for_journal_record(record, time, batch_miss_cache)
|
335
375
|
metadata = nil
|
336
|
-
if record.
|
376
|
+
if record.key?('CONTAINER_NAME') && record.key?('CONTAINER_ID_FULL')
|
337
377
|
metadata = record['CONTAINER_NAME'].match(@container_name_to_kubernetes_regexp_compiled) do |match_data|
|
338
378
|
get_metadata_for_record(match_data['namespace'], match_data['pod_name'], match_data['container_name'],
|
339
|
-
record['CONTAINER_ID_FULL'], create_time_from_record(record, time), batch_miss_cache)
|
379
|
+
record['CONTAINER_ID_FULL'], create_time_from_record(record, time), batch_miss_cache, record['CONTAINER_ID_FULL'])
|
340
380
|
end
|
341
381
|
unless metadata
|
342
382
|
log.debug "Error: could not match CONTAINER_NAME from record #{record}"
|
343
383
|
@stats.bump(:container_name_match_failed)
|
344
384
|
end
|
345
|
-
elsif record.
|
385
|
+
elsif record.key?('CONTAINER_NAME') && record['CONTAINER_NAME'].start_with?('k8s_')
|
346
386
|
log.debug "Error: no container name and id in record #{record}"
|
347
387
|
@stats.bump(:container_name_id_missing)
|
348
388
|
end
|
@@ -351,13 +391,27 @@ module Fluent::Plugin
|
|
351
391
|
|
352
392
|
def de_dot!(h)
|
353
393
|
h.keys.each do |ref|
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
394
|
+
next unless h[ref] && ref =~ /\./
|
395
|
+
|
396
|
+
v = h.delete(ref)
|
397
|
+
newref = ref.to_s.gsub('.', @de_dot_separator)
|
398
|
+
h[newref] = v
|
359
399
|
end
|
360
400
|
end
|
361
401
|
|
402
|
+
def de_slash!(h)
|
403
|
+
h.keys.each do |ref|
|
404
|
+
next unless h[ref] && ref =~ /\//
|
405
|
+
|
406
|
+
v = h.delete(ref)
|
407
|
+
newref = ref.to_s.gsub('/', @de_slash_separator)
|
408
|
+
h[newref] = v
|
409
|
+
end
|
410
|
+
end
|
411
|
+
|
412
|
+
# copied from activesupport
|
413
|
+
def present?(object)
|
414
|
+
object.respond_to?(:empty?) ? !object.empty? : !!object
|
415
|
+
end
|
362
416
|
end
|
363
417
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
#
|
2
4
|
# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
|
3
5
|
# Kubernetes metadata
|
@@ -21,26 +23,18 @@ module KubernetesMetadata
|
|
21
23
|
def get_pod_metadata(key, namespace_name, pod_name, record_create_time, batch_miss_cache)
|
22
24
|
metadata = {}
|
23
25
|
ids = @id_cache[key]
|
24
|
-
if
|
25
|
-
# FAST PATH
|
26
|
-
# Cache hit, fetch metadata from the cache
|
27
|
-
metadata = @cache.fetch(ids[:pod_id]) do
|
28
|
-
@stats.bump(:pod_cache_miss)
|
29
|
-
m = fetch_pod_metadata(namespace_name, pod_name)
|
30
|
-
(m.nil? || m.empty?) ? {'pod_id'=>ids[:pod_id]} : m
|
31
|
-
end
|
32
|
-
metadata.merge!(@namespace_cache.fetch(ids[:namespace_id]) do
|
33
|
-
@stats.bump(:namespace_cache_miss)
|
34
|
-
m = fetch_namespace_metadata(namespace_name)
|
35
|
-
(m.nil? || m.empty?) ? {'namespace_id'=>ids[:namespace_id]} : m
|
36
|
-
end)
|
37
|
-
else
|
38
|
-
# SLOW PATH
|
26
|
+
if ids.nil?
|
39
27
|
@stats.bump(:id_cache_miss)
|
40
28
|
return batch_miss_cache["#{namespace_name}_#{pod_name}"] if batch_miss_cache.key?("#{namespace_name}_#{pod_name}")
|
29
|
+
|
41
30
|
pod_metadata = fetch_pod_metadata(namespace_name, pod_name)
|
31
|
+
if @skip_namespace_metadata
|
32
|
+
ids = { pod_id: pod_metadata['pod_id'] }
|
33
|
+
@id_cache[key] = ids
|
34
|
+
return pod_metadata
|
35
|
+
end
|
42
36
|
namespace_metadata = fetch_namespace_metadata(namespace_name)
|
43
|
-
ids = { :
|
37
|
+
ids = { pod_id: pod_metadata['pod_id'], namespace_id: namespace_metadata['namespace_id'] }
|
44
38
|
if !ids[:pod_id].nil? && !ids[:namespace_id].nil?
|
45
39
|
# pod found and namespace found
|
46
40
|
metadata = pod_metadata
|
@@ -54,7 +48,7 @@ module KubernetesMetadata
|
|
54
48
|
# namespace is older then record for pod
|
55
49
|
ids[:pod_id] = key
|
56
50
|
metadata = @cache.fetch(ids[:pod_id]) do
|
57
|
-
|
51
|
+
{ 'pod_id' => ids[:pod_id] }
|
58
52
|
end
|
59
53
|
end
|
60
54
|
metadata.merge!(namespace_metadata)
|
@@ -69,7 +63,7 @@ module KubernetesMetadata
|
|
69
63
|
@stats.bump(:id_cache_orphaned_record)
|
70
64
|
end
|
71
65
|
if @allow_orphans
|
72
|
-
log.trace("orphaning message for: #{namespace_name}/#{pod_name} ")
|
66
|
+
log.trace("orphaning message for: #{namespace_name}/#{pod_name} ")
|
73
67
|
metadata = {
|
74
68
|
'orphaned_namespace' => namespace_name,
|
75
69
|
'namespace_name' => @orphaned_namespace_name,
|
@@ -82,12 +76,25 @@ module KubernetesMetadata
|
|
82
76
|
end
|
83
77
|
end
|
84
78
|
@id_cache[key] = ids unless batch_miss_cache.key?("#{namespace_name}_#{pod_name}")
|
79
|
+
else
|
80
|
+
# SLOW PATH
|
81
|
+
metadata = @cache.fetch(ids[:pod_id]) do
|
82
|
+
@stats.bump(:pod_cache_miss)
|
83
|
+
m = fetch_pod_metadata(namespace_name, pod_name)
|
84
|
+
m.nil? || m.empty? ? { 'pod_id' => ids[:pod_id] } : m
|
85
|
+
end
|
86
|
+
metadata.merge!(@namespace_cache.fetch(ids[:namespace_id]) do
|
87
|
+
m = unless @skip_namespace_metadata
|
88
|
+
@stats.bump(:namespace_cache_miss)
|
89
|
+
fetch_namespace_metadata(namespace_name)
|
90
|
+
end
|
91
|
+
m.nil? || m.empty? ? { 'namespace_id' => ids[:namespace_id] } : m
|
92
|
+
end)
|
85
93
|
end
|
86
94
|
|
87
95
|
# remove namespace info that is only used for comparison
|
88
96
|
metadata.delete('creation_timestamp')
|
89
|
-
metadata.delete_if{|
|
97
|
+
metadata.delete_if { |_k, v| v.nil? }
|
90
98
|
end
|
91
|
-
|
92
99
|
end
|
93
100
|
end
|