fluent-plugin-kubernetes_metadata_filter 3.7.0 → 3.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2 -1
- data/Rakefile +3 -4
- data/lib/fluent/plugin/filter_kubernetes_metadata.rb +307 -279
- data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb +15 -11
- data/lib/fluent/plugin/kubernetes_metadata_common.rb +38 -40
- data/lib/fluent/plugin/kubernetes_metadata_stats.rb +4 -6
- data/lib/fluent/plugin/kubernetes_metadata_test_api_adapter.rb +39 -36
- data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +15 -12
- data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +18 -17
- metadata +2 -172
- data/.circleci/config.yml +0 -48
- data/.gitignore +0 -18
- data/.rubocop.yml +0 -57
- data/Gemfile +0 -9
- data/Gemfile.lock +0 -172
- data/fluent-plugin-kubernetes_metadata_filter.gemspec +0 -34
- data/lib/fluent/plugin/kubernetes_metadata_util.rb +0 -33
- data/test/cassettes/invalid_api_server_config.yml +0 -53
- data/test/cassettes/kubernetes_docker_metadata_annotations.yml +0 -205
- data/test/cassettes/kubernetes_docker_metadata_dotted_slashed_labels.yml +0 -197
- data/test/cassettes/kubernetes_get_api_v1.yml +0 -193
- data/test/cassettes/kubernetes_get_api_v1_using_token.yml +0 -195
- data/test/cassettes/kubernetes_get_namespace_default.yml +0 -72
- data/test/cassettes/kubernetes_get_namespace_default_using_token.yml +0 -71
- data/test/cassettes/kubernetes_get_pod.yml +0 -146
- data/test/cassettes/kubernetes_get_pod_container_init.yml +0 -145
- data/test/cassettes/kubernetes_get_pod_using_token.yml +0 -148
- data/test/cassettes/kubernetes_get_pod_with_ownerrefs.yml +0 -156
- data/test/cassettes/metadata_from_tag_and_journald_fields.yml +0 -153
- data/test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml +0 -285
- data/test/cassettes/valid_kubernetes_api_server.yml +0 -55
- data/test/cassettes/valid_kubernetes_api_server_using_token.yml +0 -57
- data/test/helper.rb +0 -82
- data/test/plugin/test.token +0 -1
- data/test/plugin/test_cache_stats.rb +0 -33
- data/test/plugin/test_cache_strategy.rb +0 -194
- data/test/plugin/test_filter_kubernetes_metadata.rb +0 -851
- data/test/plugin/test_watch_namespaces.rb +0 -245
- data/test/plugin/test_watch_pods.rb +0 -344
- data/test/plugin/watch_test.rb +0 -76
@@ -22,362 +22,390 @@
|
|
22
22
|
require_relative 'kubernetes_metadata_cache_strategy'
|
23
23
|
require_relative 'kubernetes_metadata_common'
|
24
24
|
require_relative 'kubernetes_metadata_stats'
|
25
|
-
require_relative 'kubernetes_metadata_util'
|
26
25
|
require_relative 'kubernetes_metadata_watch_namespaces'
|
27
26
|
require_relative 'kubernetes_metadata_watch_pods'
|
28
27
|
|
29
28
|
require 'fluent/plugin/filter'
|
30
29
|
require 'resolv'
|
31
30
|
|
32
|
-
module Fluent
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
31
|
+
module Fluent
|
32
|
+
module Plugin
|
33
|
+
class KubernetesMetadataFilter < Fluent::Plugin::Filter # rubocop:disable Metrics/ClassLength
|
34
|
+
K8_POD_CA_CERT = 'ca.crt'
|
35
|
+
K8_POD_TOKEN = 'token'
|
36
|
+
|
37
|
+
include KubernetesMetadata::CacheStrategy
|
38
|
+
include KubernetesMetadata::Common
|
39
|
+
include KubernetesMetadata::WatchNamespaces
|
40
|
+
include KubernetesMetadata::WatchPods
|
41
|
+
|
42
|
+
Fluent::Plugin.register_filter('kubernetes_metadata', self)
|
43
|
+
|
44
|
+
config_param :kubernetes_url, :string, default: nil
|
45
|
+
config_param :cache_size, :integer, default: 1000
|
46
|
+
config_param :cache_ttl, :integer, default: 60 * 60
|
47
|
+
config_param :ignore_nil, :integer, default: true
|
48
|
+
config_param :watch, :bool, default: true
|
49
|
+
config_param :apiVersion, :string, default: 'v1'
|
50
|
+
config_param :client_cert, :string, default: nil
|
51
|
+
config_param :client_key, :string, default: nil
|
52
|
+
config_param :ca_file, :string, default: nil
|
53
|
+
config_param :verify_ssl, :bool, default: true
|
54
|
+
config_param :open_timeout, :integer, default: 3
|
55
|
+
config_param :read_timeout, :integer, default: 10
|
56
|
+
|
57
|
+
REGEX_VAR_LOG_PODS = '(var\.log\.pods)\.(?<namespace>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<pod_uuid>[a-z0-9-]*)\.(?<container_name>.+)\..*\.log$' # rubocop:disable Layout/LineLength
|
58
|
+
REGEX_VAR_LOG_CONTAINERS = '(var\.log\.containers)\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$' # rubocop:disable Layout/LineLength
|
59
|
+
|
60
|
+
# tag_to_kubernetes_name_regexp which must include named capture groups:
|
61
|
+
# namespace - The namespace in which the pod is deployed
|
62
|
+
# pod_name - The pod name
|
63
|
+
# container_name - The name of the container
|
64
|
+
# pod_uuid (/var/log/pods) | docker_id (/var/log/containers) - Unique identifier used in caching of either
|
65
|
+
# pod_uuid or the container hash
|
66
|
+
config_param :tag_to_kubernetes_name_regexp, :string,
|
67
|
+
default: "(#{REGEX_VAR_LOG_PODS}|#{REGEX_VAR_LOG_CONTAINERS})"
|
68
|
+
|
69
|
+
config_param :bearer_token_file, :string, default: nil
|
70
|
+
config_param :secret_dir, :string, default: '/var/run/secrets/kubernetes.io/serviceaccount'
|
71
|
+
|
72
|
+
config_param :annotation_match, :array, default: []
|
73
|
+
config_param :stats_interval, :integer, default: 30
|
74
|
+
config_param :allow_orphans, :bool, default: true
|
75
|
+
config_param :orphaned_namespace_name, :string, default: '.orphaned'
|
76
|
+
config_param :orphaned_namespace_id, :string, default: 'orphaned'
|
77
|
+
config_param :lookup_from_k8s_field, :bool, default: true
|
78
|
+
# if `ca_file` is for an intermediate CA, or otherwise we do not have the root CA and want
|
79
|
+
# to trust the intermediate CA certs we do have, set this to `true` - this corresponds to
|
80
|
+
# the openssl s_client -partial_chain flag and X509_V_FLAG_PARTIAL_CHAIN
|
81
|
+
config_param :ssl_partial_chain, :bool, default: false
|
82
|
+
config_param :skip_labels, :bool, default: false
|
83
|
+
config_param :skip_pod_labels, :bool, default: false
|
84
|
+
config_param :skip_namespace_labels, :bool, default: false
|
85
|
+
config_param :skip_container_metadata, :bool, default: false
|
86
|
+
config_param :skip_master_url, :bool, default: false
|
87
|
+
config_param :skip_namespace_metadata, :bool, default: false
|
88
|
+
config_param :include_ownerrefs_metadata, :bool, default: false
|
89
|
+
|
90
|
+
# A classname in the form of Test::APIAdapter which will try
|
91
|
+
# to be resolved from a relative named file 'test_api_adapter'
|
92
|
+
config_param :test_api_adapter, :string, default: nil
|
93
|
+
|
94
|
+
# The time interval in seconds for retry backoffs when watch connections fail.
|
95
|
+
config_param :watch_retry_interval, :integer, default: 1
|
96
|
+
# The base number of exponential backoff for retries.
|
97
|
+
config_param :watch_retry_exponential_backoff_base, :integer, default: 2
|
98
|
+
# The maximum number of times to retry pod and namespace watches.
|
99
|
+
config_param :watch_retry_max_times, :integer, default: 10
|
100
|
+
|
101
|
+
def fetch_pod_metadata(namespace_name, pod_name) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
102
|
+
log.trace("fetching pod metadata: #{namespace_name}/#{pod_name}")
|
103
|
+
options = {
|
104
|
+
resource_version: '0' # Fetch from API server cache instead of etcd quorum read
|
105
|
+
}
|
106
|
+
pod_object = @client.get_pod(pod_name, namespace_name, options)
|
107
|
+
log.trace("raw metadata for #{namespace_name}/#{pod_name}: #{pod_object}")
|
108
|
+
metadata = parse_pod_metadata(pod_object)
|
109
|
+
@stats.bump(:pod_cache_api_updates)
|
110
|
+
log.trace("parsed metadata for #{namespace_name}/#{pod_name}: #{metadata}")
|
111
|
+
@cache[metadata['pod_id']] = metadata
|
110
112
|
rescue KubeException => e
|
111
113
|
if e.error_code == 401
|
112
114
|
# recreate client to refresh token
|
113
115
|
log.info("Encountered '401 Unauthorized' exception, recreating client to refresh token")
|
114
|
-
create_client
|
116
|
+
create_client
|
115
117
|
elsif e.error_code == 404
|
116
|
-
log.debug
|
118
|
+
log.debug("Encountered '404 Not Found' exception, pod not found")
|
117
119
|
@stats.bump(:pod_cache_api_nil_error)
|
118
120
|
else
|
119
|
-
log.error
|
121
|
+
log.error("Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}") # rubocop:disable Layout/LineLength
|
120
122
|
@stats.bump(:pod_cache_api_nil_error)
|
121
123
|
end
|
122
124
|
{}
|
123
125
|
rescue StandardError => e
|
124
126
|
@stats.bump(:pod_cache_api_nil_error)
|
125
|
-
log.error
|
127
|
+
log.error("Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}") # rubocop:disable Layout/LineLength
|
126
128
|
{}
|
127
129
|
end
|
128
130
|
|
129
|
-
|
130
|
-
|
131
|
-
|
131
|
+
def dump_stats # rubocop:disable Metrics/AbcSize
|
132
|
+
@curr_time = Time.now
|
133
|
+
return if @curr_time.to_i - @prev_time.to_i < @stats_interval
|
134
|
+
|
135
|
+
@prev_time = @curr_time
|
136
|
+
@stats.set(:pod_cache_size, @cache.count)
|
137
|
+
@stats.set(:namespace_cache_size, @namespace_cache.count) if @namespace_cache
|
138
|
+
log.info(@stats)
|
139
|
+
return unless log.level == Fluent::Log::LEVEL_TRACE
|
132
140
|
|
133
|
-
@prev_time = @curr_time
|
134
|
-
@stats.set(:pod_cache_size, @cache.count)
|
135
|
-
@stats.set(:namespace_cache_size, @namespace_cache.count) if @namespace_cache
|
136
|
-
log.info(@stats)
|
137
|
-
if log.level == Fluent::Log::LEVEL_TRACE
|
138
141
|
log.trace(" id cache: #{@id_cache.to_a}")
|
139
142
|
log.trace(" pod cache: #{@cache.to_a}")
|
140
143
|
log.trace("namespace cache: #{@namespace_cache.to_a}")
|
141
144
|
end
|
142
|
-
end
|
143
145
|
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
146
|
+
def fetch_namespace_metadata(namespace_name) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
147
|
+
log.trace("fetching namespace metadata: #{namespace_name}")
|
148
|
+
options = {
|
149
|
+
resource_version: '0' # Fetch from API server cache instead of etcd quorum read
|
150
|
+
}
|
151
|
+
namespace_object = @client.get_namespace(namespace_name, nil, options)
|
152
|
+
log.trace("raw metadata for #{namespace_name}: #{namespace_object}")
|
153
|
+
metadata = parse_namespace_metadata(namespace_object)
|
154
|
+
@stats.bump(:namespace_cache_api_updates)
|
155
|
+
log.trace("parsed metadata for #{namespace_name}: #{metadata}")
|
156
|
+
@namespace_cache[metadata['namespace_id']] = metadata
|
155
157
|
rescue KubeException => e
|
156
158
|
if e.error_code == 401
|
157
159
|
# recreate client to refresh token
|
158
160
|
log.info("Encountered '401 Unauthorized' exception, recreating client to refresh token")
|
159
|
-
create_client
|
161
|
+
create_client
|
160
162
|
else
|
161
|
-
log.error
|
163
|
+
log.error("Exception '#{e}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}") # rubocop:disable Layout/LineLength
|
162
164
|
@stats.bump(:namespace_cache_api_nil_error)
|
163
165
|
end
|
164
166
|
{}
|
165
167
|
rescue StandardError => e
|
166
168
|
@stats.bump(:namespace_cache_api_nil_error)
|
167
|
-
log.error
|
169
|
+
log.error("Exception '#{e}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}") # rubocop:disable Layout/LineLength
|
168
170
|
{}
|
169
|
-
|
171
|
+
end
|
170
172
|
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
173
|
+
def initialize
|
174
|
+
super
|
175
|
+
@prev_time = Time.now
|
176
|
+
@ssl_options = {}
|
177
|
+
@auth_options = {}
|
178
|
+
end
|
177
179
|
|
178
|
-
|
179
|
-
|
180
|
+
def configure(conf) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
|
181
|
+
super
|
180
182
|
|
181
|
-
|
182
|
-
|
183
|
+
require 'kubeclient'
|
184
|
+
require 'lru_redux'
|
183
185
|
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
186
|
+
@stats = KubernetesMetadata::Stats.new
|
187
|
+
if @stats_interval <= 0
|
188
|
+
@stats = KubernetesMetadata::NoOpStats.new
|
189
|
+
define_singleton_method(:dump_stats) {} # rubocop:disable Lint/EmptyBlock
|
190
|
+
end
|
189
191
|
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
192
|
+
if @cache_ttl < 0
|
193
|
+
log.info 'Setting the cache TTL to :none because it was <= 0'
|
194
|
+
@cache_ttl = :none
|
195
|
+
end
|
194
196
|
|
195
|
-
|
196
|
-
|
197
|
+
# Caches pod/namespace UID tuples for a given container UID.
|
198
|
+
@id_cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl, @ignore_nil)
|
197
199
|
|
198
|
-
|
199
|
-
|
200
|
+
# Use the container UID as the key to fetch a hash containing pod metadata
|
201
|
+
@cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl, @ignore_nil)
|
200
202
|
|
201
|
-
|
202
|
-
|
203
|
+
# Use the namespace UID as the key to fetch a hash containing namespace metadata
|
204
|
+
@namespace_cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl, @ignore_nil)
|
203
205
|
|
204
|
-
|
206
|
+
@tag_to_kubernetes_name_regexp_compiled = Regexp.compile(@tag_to_kubernetes_name_regexp)
|
205
207
|
|
206
|
-
|
207
|
-
|
208
|
-
|
208
|
+
# Use Kubernetes default service account if we're in a pod.
|
209
|
+
if @kubernetes_url.nil?
|
210
|
+
log.debug('Kubernetes URL is not set - inspecting environ')
|
211
|
+
env_host = ENV['KUBERNETES_SERVICE_HOST']
|
212
|
+
env_port = ENV['KUBERNETES_SERVICE_PORT']
|
209
213
|
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
214
|
+
if present?(env_host) && present?(env_port)
|
215
|
+
if Resolv::IPv6::Regex.match?(env_host)
|
216
|
+
# Brackets are needed around IPv6 addresses
|
217
|
+
env_host = "[#{env_host}]"
|
218
|
+
end
|
219
|
+
@kubernetes_url = "https://#{env_host}:#{env_port}/api"
|
220
|
+
log.debug("Kubernetes URL is now '#{@kubernetes_url}'")
|
221
|
+
else
|
222
|
+
log.debug('No Kubernetes URL could be found in config or environ')
|
216
223
|
end
|
217
|
-
@kubernetes_url = "https://#{env_host}:#{env_port}/api"
|
218
|
-
log.debug "Kubernetes URL is now '#{@kubernetes_url}'"
|
219
|
-
else
|
220
|
-
log.debug 'No Kubernetes URL could be found in config or environ'
|
221
224
|
end
|
222
|
-
end
|
223
225
|
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
226
|
+
# Use SSL certificate and bearer token from Kubernetes service account.
|
227
|
+
if Dir.exist?(@secret_dir)
|
228
|
+
log.debug("Found directory with secrets: #{@secret_dir}")
|
229
|
+
ca_cert = File.join(@secret_dir, K8_POD_CA_CERT)
|
230
|
+
pod_token = File.join(@secret_dir, K8_POD_TOKEN)
|
229
231
|
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
232
|
+
if !present?(@ca_file) && File.exist?(ca_cert)
|
233
|
+
log.debug("Found CA certificate: #{ca_cert}")
|
234
|
+
@ca_file = ca_cert
|
235
|
+
end
|
234
236
|
|
235
|
-
|
236
|
-
|
237
|
-
|
237
|
+
if !present?(@bearer_token_file) && File.exist?(pod_token)
|
238
|
+
log.debug("Found pod token: #{pod_token}")
|
239
|
+
@bearer_token_file = pod_token
|
240
|
+
end
|
238
241
|
end
|
239
|
-
end
|
240
242
|
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
243
|
+
if present?(@kubernetes_url)
|
244
|
+
@ssl_options = {
|
245
|
+
client_cert: present?(@client_cert) ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil,
|
246
|
+
client_key: present?(@client_key) ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
|
247
|
+
ca_file: @ca_file,
|
248
|
+
verify_ssl: @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
|
249
|
+
}
|
250
|
+
|
251
|
+
if @ssl_partial_chain
|
252
|
+
# taken from the ssl.rb OpenSSL::SSL::SSLContext code for DEFAULT_CERT_STORE
|
253
|
+
require 'openssl'
|
254
|
+
|
255
|
+
ssl_store = OpenSSL::X509::Store.new
|
256
|
+
ssl_store.set_default_paths
|
257
|
+
flagval = if defined? OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
|
258
|
+
OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
|
259
|
+
else
|
260
|
+
# this version of ruby does not define OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
|
261
|
+
0x80000
|
262
|
+
end
|
263
|
+
ssl_store.flags = OpenSSL::X509::V_FLAG_CRL_CHECK_ALL | flagval
|
264
|
+
@ssl_options[:cert_store] = ssl_store
|
265
|
+
end
|
248
266
|
|
249
|
-
|
250
|
-
# taken from the ssl.rb OpenSSL::SSL::SSLContext code for DEFAULT_CERT_STORE
|
251
|
-
require 'openssl'
|
252
|
-
ssl_store = OpenSSL::X509::Store.new
|
253
|
-
ssl_store.set_default_paths
|
254
|
-
flagval = if defined? OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
|
255
|
-
OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
|
256
|
-
else
|
257
|
-
# this version of ruby does not define OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
|
258
|
-
0x80000
|
259
|
-
end
|
260
|
-
ssl_store.flags = OpenSSL::X509::V_FLAG_CRL_CHECK_ALL | flagval
|
261
|
-
@ssl_options[:cert_store] = ssl_store
|
262
|
-
end
|
267
|
+
@auth_options[:bearer_token_file] = @bearer_token_file if present?(@bearer_token_file)
|
263
268
|
|
264
|
-
|
265
|
-
@auth_options[:bearer_token_file] = @bearer_token_file
|
266
|
-
end
|
269
|
+
create_client
|
267
270
|
|
268
|
-
|
271
|
+
if @test_api_adapter
|
272
|
+
log.info "Extending client with test api adapter #{@test_api_adapter}"
|
273
|
+
@test_api_adapter = @test_api_adapter.gsub('::', '_')
|
274
|
+
.gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
|
275
|
+
.gsub(/([a-z\d])([A-Z])/, '\1_\2')
|
276
|
+
.tr('-', '_')
|
277
|
+
.downcase
|
278
|
+
require_relative @test_api_adapter
|
279
|
+
@client.extend(eval(@test_api_adapter)) # rubocop:disable Security/Eval
|
280
|
+
end
|
269
281
|
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
282
|
+
begin
|
283
|
+
@client.api_valid?
|
284
|
+
rescue KubeException => e
|
285
|
+
raise Fluent::ConfigError, "Invalid Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}: #{e.message}"
|
286
|
+
end
|
275
287
|
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
end
|
288
|
+
if @watch
|
289
|
+
if ENV['K8S_NODE_NAME'].nil? || ENV['K8S_NODE_NAME'].strip.empty?
|
290
|
+
log.warn("!! The environment variable 'K8S_NODE_NAME' is not set to the node name which can affect the API server and watch efficiency !!") # rubocop:disable Layout/LineLength
|
291
|
+
end
|
281
292
|
|
282
|
-
|
283
|
-
|
284
|
-
log.warn("!! The environment variable 'K8S_NODE_NAME' is not set to the node name which can affect the API server and watch efficiency !!")
|
285
|
-
end
|
293
|
+
pod_thread = Thread.new(self, &:set_up_pod_thread)
|
294
|
+
pod_thread.abort_on_exception = true
|
286
295
|
|
287
|
-
|
288
|
-
|
296
|
+
namespace_thread = Thread.new(self, &:set_up_namespace_thread)
|
297
|
+
namespace_thread.abort_on_exception = true
|
298
|
+
end
|
299
|
+
end
|
289
300
|
|
290
|
-
|
291
|
-
|
301
|
+
@annotations_regexps = []
|
302
|
+
@annotation_match.each do |regexp|
|
303
|
+
@annotations_regexps << Regexp.compile(regexp)
|
304
|
+
rescue RegexpError => e
|
305
|
+
log.error("Error: invalid regular expression in annotation_match: #{e}")
|
292
306
|
end
|
293
307
|
end
|
294
308
|
|
295
|
-
|
296
|
-
|
297
|
-
@
|
298
|
-
|
299
|
-
|
309
|
+
def create_client # rubocop:disable Metrics/MethodLength
|
310
|
+
log.debug('Creating K8S client')
|
311
|
+
@client = nil
|
312
|
+
@client = Kubeclient::Client.new(
|
313
|
+
@kubernetes_url,
|
314
|
+
@apiVersion,
|
315
|
+
ssl_options: @ssl_options,
|
316
|
+
auth_options: @auth_options,
|
317
|
+
timeouts: {
|
318
|
+
open: @open_timeout,
|
319
|
+
read: @read_timeout
|
320
|
+
},
|
321
|
+
as: :parsed_symbolized
|
322
|
+
)
|
300
323
|
end
|
301
|
-
end
|
302
324
|
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
open: @open_timeout,
|
313
|
-
read: @read_timeout
|
314
|
-
},
|
315
|
-
as: :parsed_symbolized
|
316
|
-
)
|
317
|
-
end
|
318
|
-
|
319
|
-
def get_metadata_for_record(namespace_name, pod_name, container_name, cache_key, create_time, batch_miss_cache, docker_id)
|
320
|
-
metadata = {
|
321
|
-
'docker' => { 'container_id' => "" },
|
322
|
-
'kubernetes' => {
|
323
|
-
'container_name' => container_name,
|
324
|
-
'namespace_name' => namespace_name,
|
325
|
-
'pod_name' => pod_name
|
325
|
+
def get_metadata_for_record(namespace_name, pod_name, container_name, cache_key, create_time, batch_miss_cache, # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/ParameterLists, Metrics/PerceivedComplexity
|
326
|
+
docker_id)
|
327
|
+
metadata = {
|
328
|
+
'docker' => { 'container_id' => '' },
|
329
|
+
'kubernetes' => {
|
330
|
+
'container_name' => container_name,
|
331
|
+
'namespace_name' => namespace_name,
|
332
|
+
'pod_name' => pod_name
|
333
|
+
}
|
326
334
|
}
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
335
|
+
metadata['docker']['container_id'] = docker_id unless docker_id.nil?
|
336
|
+
container_cache_key = container_name
|
337
|
+
if present?(@kubernetes_url)
|
338
|
+
pod_metadata = get_pod_metadata(cache_key, namespace_name, pod_name, create_time, batch_miss_cache)
|
339
|
+
if (pod_metadata.include? 'containers') && (pod_metadata['containers'].include? container_cache_key) && !@skip_container_metadata # rubocop:disable Layout/LineLength
|
340
|
+
metadata['kubernetes']['container_image'] = pod_metadata['containers'][container_cache_key]['image']
|
341
|
+
unless pod_metadata['containers'][container_cache_key]['image_id'].empty?
|
342
|
+
metadata['kubernetes']['container_image_id'] =
|
343
|
+
pod_metadata['containers'][container_cache_key]['image_id']
|
344
|
+
end
|
345
|
+
unless pod_metadata['containers'][container_cache_key]['containerID'].empty?
|
346
|
+
metadata['docker']['container_id'] =
|
347
|
+
pod_metadata['containers'][container_cache_key]['containerID']
|
348
|
+
end
|
349
|
+
end
|
350
|
+
metadata['kubernetes'].merge!(pod_metadata) if pod_metadata
|
351
|
+
metadata['kubernetes'].delete('containers')
|
336
352
|
end
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
metadata['kubernetes'].tap do |kube|
|
342
|
-
kube.each_pair do |k,v|
|
343
|
-
kube[k.dup] = v.dup
|
353
|
+
metadata['kubernetes'].tap do |kube|
|
354
|
+
kube.each_pair do |k, v|
|
355
|
+
kube[k.dup] = v.dup
|
356
|
+
end
|
344
357
|
end
|
358
|
+
if metadata['docker'] && (metadata['docker']['container_id'].nil? || metadata['docker']['container_id'].empty?)
|
359
|
+
metadata.delete('docker')
|
360
|
+
end
|
361
|
+
metadata
|
345
362
|
end
|
346
|
-
metadata.delete('docker') if metadata['docker'] && (metadata['docker']['container_id'].nil? || metadata['docker']['container_id'].empty?)
|
347
|
-
metadata
|
348
|
-
end
|
349
363
|
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
364
|
+
def filter(tag, time, record) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
|
365
|
+
tag_match_data = tag.match(@tag_to_kubernetes_name_regexp_compiled)
|
366
|
+
batch_miss_cache = {}
|
367
|
+
if tag_match_data
|
368
|
+
cache_key = if tag_match_data.names.include?('pod_uuid') && !tag_match_data['pod_uuid'].nil?
|
369
|
+
tag_match_data['pod_uuid']
|
370
|
+
else
|
371
|
+
tag_match_data['docker_id']
|
372
|
+
end
|
373
|
+
docker_id = tag_match_data.names.include?('docker_id') ? tag_match_data['docker_id'] : nil
|
374
|
+
metadata = get_metadata_for_record(
|
375
|
+
tag_match_data['namespace'],
|
376
|
+
tag_match_data['pod_name'],
|
377
|
+
tag_match_data['container_name'],
|
378
|
+
cache_key,
|
379
|
+
time,
|
380
|
+
batch_miss_cache,
|
381
|
+
docker_id
|
382
|
+
)
|
358
383
|
end
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
384
|
+
if @lookup_from_k8s_field && record.key?('kubernetes') && record.key?('docker') &&
|
385
|
+
record['kubernetes'].respond_to?(:has_key?) && record['docker'].respond_to?(:has_key?) &&
|
386
|
+
record['kubernetes'].key?('namespace_name') &&
|
387
|
+
record['kubernetes'].key?('pod_name') &&
|
388
|
+
record['kubernetes'].key?('container_name') &&
|
389
|
+
record['docker'].key?('container_id')
|
390
|
+
k_metadata = get_metadata_for_record(
|
391
|
+
record['kubernetes']['namespace_name'],
|
392
|
+
record['kubernetes']['pod_name'],
|
393
|
+
record['kubernetes']['container_name'],
|
394
|
+
record['docker']['container_id'],
|
395
|
+
time,
|
396
|
+
batch_miss_cache,
|
397
|
+
record['docker']['container_id']
|
398
|
+
)
|
399
|
+
metadata = k_metadata if k_metadata
|
400
|
+
end
|
401
|
+
dump_stats
|
402
|
+
metadata ? record.merge(metadata) : record
|
373
403
|
end
|
374
|
-
dump_stats
|
375
|
-
metadata ? record.merge(metadata) : record
|
376
|
-
end
|
377
404
|
|
378
|
-
|
379
|
-
|
380
|
-
|
405
|
+
# copied from activesupport
|
406
|
+
def present?(object)
|
407
|
+
object.respond_to?(:empty?) ? !object.empty? : !!object
|
408
|
+
end
|
381
409
|
end
|
382
410
|
end
|
383
411
|
end
|