fluent-plugin-kubernetes_metadata_filter 3.7.0 → 3.7.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2 -1
  3. data/Rakefile +3 -4
  4. data/lib/fluent/plugin/filter_kubernetes_metadata.rb +307 -279
  5. data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb +15 -11
  6. data/lib/fluent/plugin/kubernetes_metadata_common.rb +38 -40
  7. data/lib/fluent/plugin/kubernetes_metadata_stats.rb +4 -6
  8. data/lib/fluent/plugin/kubernetes_metadata_test_api_adapter.rb +39 -36
  9. data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +15 -12
  10. data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +18 -17
  11. metadata +2 -172
  12. data/.circleci/config.yml +0 -48
  13. data/.gitignore +0 -18
  14. data/.rubocop.yml +0 -57
  15. data/Gemfile +0 -9
  16. data/Gemfile.lock +0 -172
  17. data/fluent-plugin-kubernetes_metadata_filter.gemspec +0 -34
  18. data/lib/fluent/plugin/kubernetes_metadata_util.rb +0 -33
  19. data/test/cassettes/invalid_api_server_config.yml +0 -53
  20. data/test/cassettes/kubernetes_docker_metadata_annotations.yml +0 -205
  21. data/test/cassettes/kubernetes_docker_metadata_dotted_slashed_labels.yml +0 -197
  22. data/test/cassettes/kubernetes_get_api_v1.yml +0 -193
  23. data/test/cassettes/kubernetes_get_api_v1_using_token.yml +0 -195
  24. data/test/cassettes/kubernetes_get_namespace_default.yml +0 -72
  25. data/test/cassettes/kubernetes_get_namespace_default_using_token.yml +0 -71
  26. data/test/cassettes/kubernetes_get_pod.yml +0 -146
  27. data/test/cassettes/kubernetes_get_pod_container_init.yml +0 -145
  28. data/test/cassettes/kubernetes_get_pod_using_token.yml +0 -148
  29. data/test/cassettes/kubernetes_get_pod_with_ownerrefs.yml +0 -156
  30. data/test/cassettes/metadata_from_tag_and_journald_fields.yml +0 -153
  31. data/test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml +0 -285
  32. data/test/cassettes/valid_kubernetes_api_server.yml +0 -55
  33. data/test/cassettes/valid_kubernetes_api_server_using_token.yml +0 -57
  34. data/test/helper.rb +0 -82
  35. data/test/plugin/test.token +0 -1
  36. data/test/plugin/test_cache_stats.rb +0 -33
  37. data/test/plugin/test_cache_strategy.rb +0 -194
  38. data/test/plugin/test_filter_kubernetes_metadata.rb +0 -851
  39. data/test/plugin/test_watch_namespaces.rb +0 -245
  40. data/test/plugin/test_watch_pods.rb +0 -344
  41. data/test/plugin/watch_test.rb +0 -76
@@ -22,362 +22,390 @@
22
22
  require_relative 'kubernetes_metadata_cache_strategy'
23
23
  require_relative 'kubernetes_metadata_common'
24
24
  require_relative 'kubernetes_metadata_stats'
25
- require_relative 'kubernetes_metadata_util'
26
25
  require_relative 'kubernetes_metadata_watch_namespaces'
27
26
  require_relative 'kubernetes_metadata_watch_pods'
28
27
 
29
28
  require 'fluent/plugin/filter'
30
29
  require 'resolv'
31
30
 
32
- module Fluent::Plugin
33
- class KubernetesMetadataFilter < Fluent::Plugin::Filter
34
- K8_POD_CA_CERT = 'ca.crt'
35
- K8_POD_TOKEN = 'token'
36
-
37
- include KubernetesMetadata::CacheStrategy
38
- include KubernetesMetadata::Common
39
- include KubernetesMetadata::WatchNamespaces
40
- include KubernetesMetadata::WatchPods
41
-
42
- Fluent::Plugin.register_filter('kubernetes_metadata', self)
43
-
44
- config_param :kubernetes_url, :string, default: nil
45
- config_param :cache_size, :integer, default: 1000
46
- config_param :cache_ttl, :integer, default: 60 * 60
47
- config_param :ignore_nil, :integer, default: true
48
- config_param :watch, :bool, default: true
49
- config_param :apiVersion, :string, default: 'v1'
50
- config_param :client_cert, :string, default: nil
51
- config_param :client_key, :string, default: nil
52
- config_param :ca_file, :string, default: nil
53
- config_param :verify_ssl, :bool, default: true
54
- config_param :open_timeout, :integer, default: 3
55
- config_param :read_timeout, :integer, default: 10
56
-
57
- REGEX_VAR_LOG_PODS = '(var\.log\.pods)\.(?<namespace>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<pod_uuid>[a-z0-9-]*)\.(?<container_name>.+)\..*\.log$'
58
- REGEX_VAR_LOG_CONTAINERS = '(var\.log\.containers)\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
59
-
60
- #tag_to_kubernetes_name_regexp which must include named capture groups:
61
- # namespace - The namespace in which the pod is deployed
62
- # pod_name - The pod name
63
- # container_name - The name of the container
64
- # pod_uuid (/var/log/pods) | docker_id (/var/log/containers) - Unique identifier used in caching of either pod_uuid or the container hash
65
- config_param :tag_to_kubernetes_name_regexp, :string, default: "(#{REGEX_VAR_LOG_PODS}|#{REGEX_VAR_LOG_CONTAINERS})"
66
-
67
- config_param :bearer_token_file, :string, default: nil
68
- config_param :secret_dir, :string, default: '/var/run/secrets/kubernetes.io/serviceaccount'
69
-
70
- config_param :annotation_match, :array, default: []
71
- config_param :stats_interval, :integer, default: 30
72
- config_param :allow_orphans, :bool, default: true
73
- config_param :orphaned_namespace_name, :string, default: '.orphaned'
74
- config_param :orphaned_namespace_id, :string, default: 'orphaned'
75
- config_param :lookup_from_k8s_field, :bool, default: true
76
- # if `ca_file` is for an intermediate CA, or otherwise we do not have the root CA and want
77
- # to trust the intermediate CA certs we do have, set this to `true` - this corresponds to
78
- # the openssl s_client -partial_chain flag and X509_V_FLAG_PARTIAL_CHAIN
79
- config_param :ssl_partial_chain, :bool, default: false
80
- config_param :skip_labels, :bool, default: false
81
- config_param :skip_pod_labels, :bool, default: false
82
- config_param :skip_namespace_labels, :bool, default: false
83
- config_param :skip_container_metadata, :bool, default: false
84
- config_param :skip_master_url, :bool, default: false
85
- config_param :skip_namespace_metadata, :bool, default: false
86
- config_param :include_ownerrefs_metadata, :bool, default: false
87
-
88
- # A classname in the form of Test::APIAdapter which will try
89
- # to be resolved from a relative named file 'test_api_adapter'
90
- config_param :test_api_adapter, :string, default: nil
91
-
92
- # The time interval in seconds for retry backoffs when watch connections fail.
93
- config_param :watch_retry_interval, :integer, default: 1
94
- # The base number of exponential backoff for retries.
95
- config_param :watch_retry_exponential_backoff_base, :integer, default: 2
96
- # The maximum number of times to retry pod and namespace watches.
97
- config_param :watch_retry_max_times, :integer, default: 10
98
-
99
- def fetch_pod_metadata(namespace_name, pod_name)
100
- log.trace("fetching pod metadata: #{namespace_name}/#{pod_name}")
101
- options = {
102
- resource_version: '0' # Fetch from API server cache instead of etcd quorum read
103
- }
104
- pod_object = @client.get_pod(pod_name, namespace_name, options)
105
- log.trace("raw metadata for #{namespace_name}/#{pod_name}: #{pod_object}")
106
- metadata = parse_pod_metadata(pod_object)
107
- @stats.bump(:pod_cache_api_updates)
108
- log.trace("parsed metadata for #{namespace_name}/#{pod_name}: #{metadata}")
109
- @cache[metadata['pod_id']] = metadata
31
+ module Fluent
32
+ module Plugin
33
+ class KubernetesMetadataFilter < Fluent::Plugin::Filter # rubocop:disable Metrics/ClassLength
34
+ K8_POD_CA_CERT = 'ca.crt'
35
+ K8_POD_TOKEN = 'token'
36
+
37
+ include KubernetesMetadata::CacheStrategy
38
+ include KubernetesMetadata::Common
39
+ include KubernetesMetadata::WatchNamespaces
40
+ include KubernetesMetadata::WatchPods
41
+
42
+ Fluent::Plugin.register_filter('kubernetes_metadata', self)
43
+
44
+ config_param :kubernetes_url, :string, default: nil
45
+ config_param :cache_size, :integer, default: 1000
46
+ config_param :cache_ttl, :integer, default: 60 * 60
47
+ config_param :ignore_nil, :integer, default: true
48
+ config_param :watch, :bool, default: true
49
+ config_param :apiVersion, :string, default: 'v1'
50
+ config_param :client_cert, :string, default: nil
51
+ config_param :client_key, :string, default: nil
52
+ config_param :ca_file, :string, default: nil
53
+ config_param :verify_ssl, :bool, default: true
54
+ config_param :open_timeout, :integer, default: 3
55
+ config_param :read_timeout, :integer, default: 10
56
+
57
+ REGEX_VAR_LOG_PODS = '(var\.log\.pods)\.(?<namespace>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<pod_uuid>[a-z0-9-]*)\.(?<container_name>.+)\..*\.log$' # rubocop:disable Layout/LineLength
58
+ REGEX_VAR_LOG_CONTAINERS = '(var\.log\.containers)\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$' # rubocop:disable Layout/LineLength
59
+
60
+ # tag_to_kubernetes_name_regexp which must include named capture groups:
61
+ # namespace - The namespace in which the pod is deployed
62
+ # pod_name - The pod name
63
+ # container_name - The name of the container
64
+ # pod_uuid (/var/log/pods) | docker_id (/var/log/containers) - Unique identifier used in caching of either
65
+ # pod_uuid or the container hash
66
+ config_param :tag_to_kubernetes_name_regexp, :string,
67
+ default: "(#{REGEX_VAR_LOG_PODS}|#{REGEX_VAR_LOG_CONTAINERS})"
68
+
69
+ config_param :bearer_token_file, :string, default: nil
70
+ config_param :secret_dir, :string, default: '/var/run/secrets/kubernetes.io/serviceaccount'
71
+
72
+ config_param :annotation_match, :array, default: []
73
+ config_param :stats_interval, :integer, default: 30
74
+ config_param :allow_orphans, :bool, default: true
75
+ config_param :orphaned_namespace_name, :string, default: '.orphaned'
76
+ config_param :orphaned_namespace_id, :string, default: 'orphaned'
77
+ config_param :lookup_from_k8s_field, :bool, default: true
78
+ # if `ca_file` is for an intermediate CA, or otherwise we do not have the root CA and want
79
+ # to trust the intermediate CA certs we do have, set this to `true` - this corresponds to
80
+ # the openssl s_client -partial_chain flag and X509_V_FLAG_PARTIAL_CHAIN
81
+ config_param :ssl_partial_chain, :bool, default: false
82
+ config_param :skip_labels, :bool, default: false
83
+ config_param :skip_pod_labels, :bool, default: false
84
+ config_param :skip_namespace_labels, :bool, default: false
85
+ config_param :skip_container_metadata, :bool, default: false
86
+ config_param :skip_master_url, :bool, default: false
87
+ config_param :skip_namespace_metadata, :bool, default: false
88
+ config_param :include_ownerrefs_metadata, :bool, default: false
89
+
90
+ # A classname in the form of Test::APIAdapter which will try
91
+ # to be resolved from a relative named file 'test_api_adapter'
92
+ config_param :test_api_adapter, :string, default: nil
93
+
94
+ # The time interval in seconds for retry backoffs when watch connections fail.
95
+ config_param :watch_retry_interval, :integer, default: 1
96
+ # The base number of exponential backoff for retries.
97
+ config_param :watch_retry_exponential_backoff_base, :integer, default: 2
98
+ # The maximum number of times to retry pod and namespace watches.
99
+ config_param :watch_retry_max_times, :integer, default: 10
100
+
101
+ def fetch_pod_metadata(namespace_name, pod_name) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
102
+ log.trace("fetching pod metadata: #{namespace_name}/#{pod_name}")
103
+ options = {
104
+ resource_version: '0' # Fetch from API server cache instead of etcd quorum read
105
+ }
106
+ pod_object = @client.get_pod(pod_name, namespace_name, options)
107
+ log.trace("raw metadata for #{namespace_name}/#{pod_name}: #{pod_object}")
108
+ metadata = parse_pod_metadata(pod_object)
109
+ @stats.bump(:pod_cache_api_updates)
110
+ log.trace("parsed metadata for #{namespace_name}/#{pod_name}: #{metadata}")
111
+ @cache[metadata['pod_id']] = metadata
110
112
  rescue KubeException => e
111
113
  if e.error_code == 401
112
114
  # recreate client to refresh token
113
115
  log.info("Encountered '401 Unauthorized' exception, recreating client to refresh token")
114
- create_client()
116
+ create_client
115
117
  elsif e.error_code == 404
116
- log.debug "Encountered '404 Not Found' exception, pod not found"
118
+ log.debug("Encountered '404 Not Found' exception, pod not found")
117
119
  @stats.bump(:pod_cache_api_nil_error)
118
120
  else
119
- log.error "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
121
+ log.error("Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}") # rubocop:disable Layout/LineLength
120
122
  @stats.bump(:pod_cache_api_nil_error)
121
123
  end
122
124
  {}
123
125
  rescue StandardError => e
124
126
  @stats.bump(:pod_cache_api_nil_error)
125
- log.error "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
127
+ log.error("Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}") # rubocop:disable Layout/LineLength
126
128
  {}
127
129
  end
128
130
 
129
- def dump_stats
130
- @curr_time = Time.now
131
- return if @curr_time.to_i - @prev_time.to_i < @stats_interval
131
+ def dump_stats # rubocop:disable Metrics/AbcSize
132
+ @curr_time = Time.now
133
+ return if @curr_time.to_i - @prev_time.to_i < @stats_interval
134
+
135
+ @prev_time = @curr_time
136
+ @stats.set(:pod_cache_size, @cache.count)
137
+ @stats.set(:namespace_cache_size, @namespace_cache.count) if @namespace_cache
138
+ log.info(@stats)
139
+ return unless log.level == Fluent::Log::LEVEL_TRACE
132
140
 
133
- @prev_time = @curr_time
134
- @stats.set(:pod_cache_size, @cache.count)
135
- @stats.set(:namespace_cache_size, @namespace_cache.count) if @namespace_cache
136
- log.info(@stats)
137
- if log.level == Fluent::Log::LEVEL_TRACE
138
141
  log.trace(" id cache: #{@id_cache.to_a}")
139
142
  log.trace(" pod cache: #{@cache.to_a}")
140
143
  log.trace("namespace cache: #{@namespace_cache.to_a}")
141
144
  end
142
- end
143
145
 
144
- def fetch_namespace_metadata(namespace_name)
145
- log.trace("fetching namespace metadata: #{namespace_name}")
146
- options = {
147
- resource_version: '0' # Fetch from API server cache instead of etcd quorum read
148
- }
149
- namespace_object = @client.get_namespace(namespace_name, nil, options)
150
- log.trace("raw metadata for #{namespace_name}: #{namespace_object}")
151
- metadata = parse_namespace_metadata(namespace_object)
152
- @stats.bump(:namespace_cache_api_updates)
153
- log.trace("parsed metadata for #{namespace_name}: #{metadata}")
154
- @namespace_cache[metadata['namespace_id']] = metadata
146
+ def fetch_namespace_metadata(namespace_name) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
147
+ log.trace("fetching namespace metadata: #{namespace_name}")
148
+ options = {
149
+ resource_version: '0' # Fetch from API server cache instead of etcd quorum read
150
+ }
151
+ namespace_object = @client.get_namespace(namespace_name, nil, options)
152
+ log.trace("raw metadata for #{namespace_name}: #{namespace_object}")
153
+ metadata = parse_namespace_metadata(namespace_object)
154
+ @stats.bump(:namespace_cache_api_updates)
155
+ log.trace("parsed metadata for #{namespace_name}: #{metadata}")
156
+ @namespace_cache[metadata['namespace_id']] = metadata
155
157
  rescue KubeException => e
156
158
  if e.error_code == 401
157
159
  # recreate client to refresh token
158
160
  log.info("Encountered '401 Unauthorized' exception, recreating client to refresh token")
159
- create_client()
161
+ create_client
160
162
  else
161
- log.error "Exception '#{e}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
163
+ log.error("Exception '#{e}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}") # rubocop:disable Layout/LineLength
162
164
  @stats.bump(:namespace_cache_api_nil_error)
163
165
  end
164
166
  {}
165
167
  rescue StandardError => e
166
168
  @stats.bump(:namespace_cache_api_nil_error)
167
- log.error "Exception '#{e}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
169
+ log.error("Exception '#{e}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}") # rubocop:disable Layout/LineLength
168
170
  {}
169
- end
171
+ end
170
172
 
171
- def initialize
172
- super
173
- @prev_time = Time.now
174
- @ssl_options = {}
175
- @auth_options = {}
176
- end
173
+ def initialize
174
+ super
175
+ @prev_time = Time.now
176
+ @ssl_options = {}
177
+ @auth_options = {}
178
+ end
177
179
 
178
- def configure(conf)
179
- super
180
+ def configure(conf) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
181
+ super
180
182
 
181
- require 'kubeclient'
182
- require 'lru_redux'
183
+ require 'kubeclient'
184
+ require 'lru_redux'
183
185
 
184
- @stats = KubernetesMetadata::Stats.new
185
- if @stats_interval <= 0
186
- @stats = KubernetesMetadata::NoOpStats.new
187
- self.define_singleton_method(:dump_stats) {}
188
- end
186
+ @stats = KubernetesMetadata::Stats.new
187
+ if @stats_interval <= 0
188
+ @stats = KubernetesMetadata::NoOpStats.new
189
+ define_singleton_method(:dump_stats) {} # rubocop:disable Lint/EmptyBlock
190
+ end
189
191
 
190
- if @cache_ttl < 0
191
- log.info 'Setting the cache TTL to :none because it was <= 0'
192
- @cache_ttl = :none
193
- end
192
+ if @cache_ttl < 0
193
+ log.info 'Setting the cache TTL to :none because it was <= 0'
194
+ @cache_ttl = :none
195
+ end
194
196
 
195
- # Caches pod/namespace UID tuples for a given container UID.
196
- @id_cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl, @ignore_nil)
197
+ # Caches pod/namespace UID tuples for a given container UID.
198
+ @id_cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl, @ignore_nil)
197
199
 
198
- # Use the container UID as the key to fetch a hash containing pod metadata
199
- @cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl, @ignore_nil)
200
+ # Use the container UID as the key to fetch a hash containing pod metadata
201
+ @cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl, @ignore_nil)
200
202
 
201
- # Use the namespace UID as the key to fetch a hash containing namespace metadata
202
- @namespace_cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl, @ignore_nil)
203
+ # Use the namespace UID as the key to fetch a hash containing namespace metadata
204
+ @namespace_cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl, @ignore_nil)
203
205
 
204
- @tag_to_kubernetes_name_regexp_compiled = Regexp.compile(@tag_to_kubernetes_name_regexp)
206
+ @tag_to_kubernetes_name_regexp_compiled = Regexp.compile(@tag_to_kubernetes_name_regexp)
205
207
 
206
- # Use Kubernetes default service account if we're in a pod.
207
- if @kubernetes_url.nil?
208
- log.debug 'Kubernetes URL is not set - inspecting environ'
208
+ # Use Kubernetes default service account if we're in a pod.
209
+ if @kubernetes_url.nil?
210
+ log.debug('Kubernetes URL is not set - inspecting environ')
211
+ env_host = ENV['KUBERNETES_SERVICE_HOST']
212
+ env_port = ENV['KUBERNETES_SERVICE_PORT']
209
213
 
210
- env_host = ENV['KUBERNETES_SERVICE_HOST']
211
- env_port = ENV['KUBERNETES_SERVICE_PORT']
212
- if present?(env_host) && present?(env_port)
213
- if env_host =~ Resolv::IPv6::Regex
214
- # Brackets are needed around IPv6 addresses
215
- env_host = "[#{env_host}]"
214
+ if present?(env_host) && present?(env_port)
215
+ if Resolv::IPv6::Regex.match?(env_host)
216
+ # Brackets are needed around IPv6 addresses
217
+ env_host = "[#{env_host}]"
218
+ end
219
+ @kubernetes_url = "https://#{env_host}:#{env_port}/api"
220
+ log.debug("Kubernetes URL is now '#{@kubernetes_url}'")
221
+ else
222
+ log.debug('No Kubernetes URL could be found in config or environ')
216
223
  end
217
- @kubernetes_url = "https://#{env_host}:#{env_port}/api"
218
- log.debug "Kubernetes URL is now '#{@kubernetes_url}'"
219
- else
220
- log.debug 'No Kubernetes URL could be found in config or environ'
221
224
  end
222
- end
223
225
 
224
- # Use SSL certificate and bearer token from Kubernetes service account.
225
- if Dir.exist?(@secret_dir)
226
- log.debug "Found directory with secrets: #{@secret_dir}"
227
- ca_cert = File.join(@secret_dir, K8_POD_CA_CERT)
228
- pod_token = File.join(@secret_dir, K8_POD_TOKEN)
226
+ # Use SSL certificate and bearer token from Kubernetes service account.
227
+ if Dir.exist?(@secret_dir)
228
+ log.debug("Found directory with secrets: #{@secret_dir}")
229
+ ca_cert = File.join(@secret_dir, K8_POD_CA_CERT)
230
+ pod_token = File.join(@secret_dir, K8_POD_TOKEN)
229
231
 
230
- if !present?(@ca_file) && File.exist?(ca_cert)
231
- log.debug "Found CA certificate: #{ca_cert}"
232
- @ca_file = ca_cert
233
- end
232
+ if !present?(@ca_file) && File.exist?(ca_cert)
233
+ log.debug("Found CA certificate: #{ca_cert}")
234
+ @ca_file = ca_cert
235
+ end
234
236
 
235
- if !present?(@bearer_token_file) && File.exist?(pod_token)
236
- log.debug "Found pod token: #{pod_token}"
237
- @bearer_token_file = pod_token
237
+ if !present?(@bearer_token_file) && File.exist?(pod_token)
238
+ log.debug("Found pod token: #{pod_token}")
239
+ @bearer_token_file = pod_token
240
+ end
238
241
  end
239
- end
240
242
 
241
- if present?(@kubernetes_url)
242
- @ssl_options = {
243
- client_cert: present?(@client_cert) ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil,
244
- client_key: present?(@client_key) ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
245
- ca_file: @ca_file,
246
- verify_ssl: @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
247
- }
243
+ if present?(@kubernetes_url)
244
+ @ssl_options = {
245
+ client_cert: present?(@client_cert) ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil,
246
+ client_key: present?(@client_key) ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
247
+ ca_file: @ca_file,
248
+ verify_ssl: @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
249
+ }
250
+
251
+ if @ssl_partial_chain
252
+ # taken from the ssl.rb OpenSSL::SSL::SSLContext code for DEFAULT_CERT_STORE
253
+ require 'openssl'
254
+
255
+ ssl_store = OpenSSL::X509::Store.new
256
+ ssl_store.set_default_paths
257
+ flagval = if defined? OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
258
+ OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
259
+ else
260
+ # this version of ruby does not define OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
261
+ 0x80000
262
+ end
263
+ ssl_store.flags = OpenSSL::X509::V_FLAG_CRL_CHECK_ALL | flagval
264
+ @ssl_options[:cert_store] = ssl_store
265
+ end
248
266
 
249
- if @ssl_partial_chain
250
- # taken from the ssl.rb OpenSSL::SSL::SSLContext code for DEFAULT_CERT_STORE
251
- require 'openssl'
252
- ssl_store = OpenSSL::X509::Store.new
253
- ssl_store.set_default_paths
254
- flagval = if defined? OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
255
- OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
256
- else
257
- # this version of ruby does not define OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
258
- 0x80000
259
- end
260
- ssl_store.flags = OpenSSL::X509::V_FLAG_CRL_CHECK_ALL | flagval
261
- @ssl_options[:cert_store] = ssl_store
262
- end
267
+ @auth_options[:bearer_token_file] = @bearer_token_file if present?(@bearer_token_file)
263
268
 
264
- if present?(@bearer_token_file)
265
- @auth_options[:bearer_token_file] = @bearer_token_file
266
- end
269
+ create_client
267
270
 
268
- create_client()
271
+ if @test_api_adapter
272
+ log.info "Extending client with test api adapter #{@test_api_adapter}"
273
+ @test_api_adapter = @test_api_adapter.gsub('::', '_')
274
+ .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
275
+ .gsub(/([a-z\d])([A-Z])/, '\1_\2')
276
+ .tr('-', '_')
277
+ .downcase
278
+ require_relative @test_api_adapter
279
+ @client.extend(eval(@test_api_adapter)) # rubocop:disable Security/Eval
280
+ end
269
281
 
270
- if @test_api_adapter
271
- log.info "Extending client with test api adaper #{@test_api_adapter}"
272
- require_relative @test_api_adapter.underscore
273
- @client.extend(eval(@test_api_adapter))
274
- end
282
+ begin
283
+ @client.api_valid?
284
+ rescue KubeException => e
285
+ raise Fluent::ConfigError, "Invalid Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}: #{e.message}"
286
+ end
275
287
 
276
- begin
277
- @client.api_valid?
278
- rescue KubeException => e
279
- raise Fluent::ConfigError, "Invalid Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}: #{e.message}"
280
- end
288
+ if @watch
289
+ if ENV['K8S_NODE_NAME'].nil? || ENV['K8S_NODE_NAME'].strip.empty?
290
+ log.warn("!! The environment variable 'K8S_NODE_NAME' is not set to the node name which can affect the API server and watch efficiency !!") # rubocop:disable Layout/LineLength
291
+ end
281
292
 
282
- if @watch
283
- if ENV['K8S_NODE_NAME'].nil? || ENV['K8S_NODE_NAME'].strip.empty?
284
- log.warn("!! The environment variable 'K8S_NODE_NAME' is not set to the node name which can affect the API server and watch efficiency !!")
285
- end
293
+ pod_thread = Thread.new(self, &:set_up_pod_thread)
294
+ pod_thread.abort_on_exception = true
286
295
 
287
- pod_thread = Thread.new(self, &:set_up_pod_thread)
288
- pod_thread.abort_on_exception = true
296
+ namespace_thread = Thread.new(self, &:set_up_namespace_thread)
297
+ namespace_thread.abort_on_exception = true
298
+ end
299
+ end
289
300
 
290
- namespace_thread = Thread.new(self, &:set_up_namespace_thread)
291
- namespace_thread.abort_on_exception = true
301
+ @annotations_regexps = []
302
+ @annotation_match.each do |regexp|
303
+ @annotations_regexps << Regexp.compile(regexp)
304
+ rescue RegexpError => e
305
+ log.error("Error: invalid regular expression in annotation_match: #{e}")
292
306
  end
293
307
  end
294
308
 
295
- @annotations_regexps = []
296
- @annotation_match.each do |regexp|
297
- @annotations_regexps << Regexp.compile(regexp)
298
- rescue RegexpError => e
299
- log.error "Error: invalid regular expression in annotation_match: #{e}"
309
+ def create_client # rubocop:disable Metrics/MethodLength
310
+ log.debug('Creating K8S client')
311
+ @client = nil
312
+ @client = Kubeclient::Client.new(
313
+ @kubernetes_url,
314
+ @apiVersion,
315
+ ssl_options: @ssl_options,
316
+ auth_options: @auth_options,
317
+ timeouts: {
318
+ open: @open_timeout,
319
+ read: @read_timeout
320
+ },
321
+ as: :parsed_symbolized
322
+ )
300
323
  end
301
- end
302
324
 
303
- def create_client()
304
- log.debug 'Creating K8S client'
305
- @client = nil
306
- @client = Kubeclient::Client.new(
307
- @kubernetes_url,
308
- @apiVersion,
309
- ssl_options: @ssl_options,
310
- auth_options: @auth_options,
311
- timeouts: {
312
- open: @open_timeout,
313
- read: @read_timeout
314
- },
315
- as: :parsed_symbolized
316
- )
317
- end
318
-
319
- def get_metadata_for_record(namespace_name, pod_name, container_name, cache_key, create_time, batch_miss_cache, docker_id)
320
- metadata = {
321
- 'docker' => { 'container_id' => "" },
322
- 'kubernetes' => {
323
- 'container_name' => container_name,
324
- 'namespace_name' => namespace_name,
325
- 'pod_name' => pod_name
325
+ def get_metadata_for_record(namespace_name, pod_name, container_name, cache_key, create_time, batch_miss_cache, # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/ParameterLists, Metrics/PerceivedComplexity
326
+ docker_id)
327
+ metadata = {
328
+ 'docker' => { 'container_id' => '' },
329
+ 'kubernetes' => {
330
+ 'container_name' => container_name,
331
+ 'namespace_name' => namespace_name,
332
+ 'pod_name' => pod_name
333
+ }
326
334
  }
327
- }
328
- metadata['docker']['container_id'] = docker_id unless docker_id.nil?
329
- container_cache_key = container_name
330
- if present?(@kubernetes_url)
331
- pod_metadata = get_pod_metadata(cache_key, namespace_name, pod_name, create_time, batch_miss_cache)
332
- if (pod_metadata.include? 'containers') && (pod_metadata['containers'].include? container_cache_key) && !@skip_container_metadata
333
- metadata['kubernetes']['container_image'] = pod_metadata['containers'][container_cache_key]['image']
334
- metadata['kubernetes']['container_image_id'] = pod_metadata['containers'][container_cache_key]['image_id'] unless pod_metadata['containers'][container_cache_key]['image_id'].empty?
335
- metadata['docker']['container_id'] = pod_metadata['containers'][container_cache_key]['containerID'] unless pod_metadata['containers'][container_cache_key]['containerID'].empty?
335
+ metadata['docker']['container_id'] = docker_id unless docker_id.nil?
336
+ container_cache_key = container_name
337
+ if present?(@kubernetes_url)
338
+ pod_metadata = get_pod_metadata(cache_key, namespace_name, pod_name, create_time, batch_miss_cache)
339
+ if (pod_metadata.include? 'containers') && (pod_metadata['containers'].include? container_cache_key) && !@skip_container_metadata # rubocop:disable Layout/LineLength
340
+ metadata['kubernetes']['container_image'] = pod_metadata['containers'][container_cache_key]['image']
341
+ unless pod_metadata['containers'][container_cache_key]['image_id'].empty?
342
+ metadata['kubernetes']['container_image_id'] =
343
+ pod_metadata['containers'][container_cache_key]['image_id']
344
+ end
345
+ unless pod_metadata['containers'][container_cache_key]['containerID'].empty?
346
+ metadata['docker']['container_id'] =
347
+ pod_metadata['containers'][container_cache_key]['containerID']
348
+ end
349
+ end
350
+ metadata['kubernetes'].merge!(pod_metadata) if pod_metadata
351
+ metadata['kubernetes'].delete('containers')
336
352
  end
337
-
338
- metadata['kubernetes'].merge!(pod_metadata) if pod_metadata
339
- metadata['kubernetes'].delete('containers')
340
- end
341
- metadata['kubernetes'].tap do |kube|
342
- kube.each_pair do |k,v|
343
- kube[k.dup] = v.dup
353
+ metadata['kubernetes'].tap do |kube|
354
+ kube.each_pair do |k, v|
355
+ kube[k.dup] = v.dup
356
+ end
344
357
  end
358
+ if metadata['docker'] && (metadata['docker']['container_id'].nil? || metadata['docker']['container_id'].empty?)
359
+ metadata.delete('docker')
360
+ end
361
+ metadata
345
362
  end
346
- metadata.delete('docker') if metadata['docker'] && (metadata['docker']['container_id'].nil? || metadata['docker']['container_id'].empty?)
347
- metadata
348
- end
349
363
 
350
- def filter(tag, time, record)
351
- tag_match_data = tag.match(@tag_to_kubernetes_name_regexp_compiled)
352
- batch_miss_cache = {}
353
- if tag_match_data
354
- cache_key = if tag_match_data.names.include?('pod_uuid') && !tag_match_data['pod_uuid'].nil?
355
- tag_match_data['pod_uuid']
356
- else
357
- tag_match_data['docker_id']
364
+ def filter(tag, time, record) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
365
+ tag_match_data = tag.match(@tag_to_kubernetes_name_regexp_compiled)
366
+ batch_miss_cache = {}
367
+ if tag_match_data
368
+ cache_key = if tag_match_data.names.include?('pod_uuid') && !tag_match_data['pod_uuid'].nil?
369
+ tag_match_data['pod_uuid']
370
+ else
371
+ tag_match_data['docker_id']
372
+ end
373
+ docker_id = tag_match_data.names.include?('docker_id') ? tag_match_data['docker_id'] : nil
374
+ metadata = get_metadata_for_record(
375
+ tag_match_data['namespace'],
376
+ tag_match_data['pod_name'],
377
+ tag_match_data['container_name'],
378
+ cache_key,
379
+ time,
380
+ batch_miss_cache,
381
+ docker_id
382
+ )
358
383
  end
359
- docker_id = tag_match_data.names.include?('docker_id') ? tag_match_data['docker_id'] : nil
360
- metadata = get_metadata_for_record(tag_match_data['namespace'], tag_match_data['pod_name'], tag_match_data['container_name'],
361
- cache_key, time, batch_miss_cache, docker_id)
362
- end
363
- if @lookup_from_k8s_field && record.key?('kubernetes') && record.key?('docker') &&
364
- record['kubernetes'].respond_to?(:has_key?) && record['docker'].respond_to?(:has_key?) &&
365
- record['kubernetes'].key?('namespace_name') &&
366
- record['kubernetes'].key?('pod_name') &&
367
- record['kubernetes'].key?('container_name') &&
368
- record['docker'].key?('container_id') &&
369
- (k_metadata = get_metadata_for_record(record['kubernetes']['namespace_name'], record['kubernetes']['pod_name'],
370
- record['kubernetes']['container_name'], record['docker']['container_id'],
371
- time, batch_miss_cache, record['docker']['container_id']))
372
- metadata = k_metadata
384
+ if @lookup_from_k8s_field && record.key?('kubernetes') && record.key?('docker') &&
385
+ record['kubernetes'].respond_to?(:has_key?) && record['docker'].respond_to?(:has_key?) &&
386
+ record['kubernetes'].key?('namespace_name') &&
387
+ record['kubernetes'].key?('pod_name') &&
388
+ record['kubernetes'].key?('container_name') &&
389
+ record['docker'].key?('container_id')
390
+ k_metadata = get_metadata_for_record(
391
+ record['kubernetes']['namespace_name'],
392
+ record['kubernetes']['pod_name'],
393
+ record['kubernetes']['container_name'],
394
+ record['docker']['container_id'],
395
+ time,
396
+ batch_miss_cache,
397
+ record['docker']['container_id']
398
+ )
399
+ metadata = k_metadata if k_metadata
400
+ end
401
+ dump_stats
402
+ metadata ? record.merge(metadata) : record
373
403
  end
374
- dump_stats
375
- metadata ? record.merge(metadata) : record
376
- end
377
404
 
378
- # copied from activesupport
379
- def present?(object)
380
- object.respond_to?(:empty?) ? !object.empty? : !!object
405
+ # copied from activesupport
406
+ def present?(object)
407
+ object.respond_to?(:empty?) ? !object.empty? : !!object
408
+ end
381
409
  end
382
410
  end
383
411
  end