fluent-plugin-kubernetes_metadata_filter 3.7.0 → 3.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2 -1
  3. data/Rakefile +3 -4
  4. data/lib/fluent/plugin/filter_kubernetes_metadata.rb +307 -279
  5. data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb +15 -11
  6. data/lib/fluent/plugin/kubernetes_metadata_common.rb +38 -40
  7. data/lib/fluent/plugin/kubernetes_metadata_stats.rb +4 -6
  8. data/lib/fluent/plugin/kubernetes_metadata_test_api_adapter.rb +39 -36
  9. data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +15 -12
  10. data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +18 -17
  11. metadata +2 -172
  12. data/.circleci/config.yml +0 -48
  13. data/.gitignore +0 -18
  14. data/.rubocop.yml +0 -57
  15. data/Gemfile +0 -9
  16. data/Gemfile.lock +0 -172
  17. data/fluent-plugin-kubernetes_metadata_filter.gemspec +0 -34
  18. data/lib/fluent/plugin/kubernetes_metadata_util.rb +0 -33
  19. data/test/cassettes/invalid_api_server_config.yml +0 -53
  20. data/test/cassettes/kubernetes_docker_metadata_annotations.yml +0 -205
  21. data/test/cassettes/kubernetes_docker_metadata_dotted_slashed_labels.yml +0 -197
  22. data/test/cassettes/kubernetes_get_api_v1.yml +0 -193
  23. data/test/cassettes/kubernetes_get_api_v1_using_token.yml +0 -195
  24. data/test/cassettes/kubernetes_get_namespace_default.yml +0 -72
  25. data/test/cassettes/kubernetes_get_namespace_default_using_token.yml +0 -71
  26. data/test/cassettes/kubernetes_get_pod.yml +0 -146
  27. data/test/cassettes/kubernetes_get_pod_container_init.yml +0 -145
  28. data/test/cassettes/kubernetes_get_pod_using_token.yml +0 -148
  29. data/test/cassettes/kubernetes_get_pod_with_ownerrefs.yml +0 -156
  30. data/test/cassettes/metadata_from_tag_and_journald_fields.yml +0 -153
  31. data/test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml +0 -285
  32. data/test/cassettes/valid_kubernetes_api_server.yml +0 -55
  33. data/test/cassettes/valid_kubernetes_api_server_using_token.yml +0 -57
  34. data/test/helper.rb +0 -82
  35. data/test/plugin/test.token +0 -1
  36. data/test/plugin/test_cache_stats.rb +0 -33
  37. data/test/plugin/test_cache_strategy.rb +0 -194
  38. data/test/plugin/test_filter_kubernetes_metadata.rb +0 -851
  39. data/test/plugin/test_watch_namespaces.rb +0 -245
  40. data/test/plugin/test_watch_pods.rb +0 -344
  41. data/test/plugin/watch_test.rb +0 -76
@@ -22,362 +22,390 @@
22
22
  require_relative 'kubernetes_metadata_cache_strategy'
23
23
  require_relative 'kubernetes_metadata_common'
24
24
  require_relative 'kubernetes_metadata_stats'
25
- require_relative 'kubernetes_metadata_util'
26
25
  require_relative 'kubernetes_metadata_watch_namespaces'
27
26
  require_relative 'kubernetes_metadata_watch_pods'
28
27
 
29
28
  require 'fluent/plugin/filter'
30
29
  require 'resolv'
31
30
 
32
- module Fluent::Plugin
33
- class KubernetesMetadataFilter < Fluent::Plugin::Filter
34
- K8_POD_CA_CERT = 'ca.crt'
35
- K8_POD_TOKEN = 'token'
36
-
37
- include KubernetesMetadata::CacheStrategy
38
- include KubernetesMetadata::Common
39
- include KubernetesMetadata::WatchNamespaces
40
- include KubernetesMetadata::WatchPods
41
-
42
- Fluent::Plugin.register_filter('kubernetes_metadata', self)
43
-
44
- config_param :kubernetes_url, :string, default: nil
45
- config_param :cache_size, :integer, default: 1000
46
- config_param :cache_ttl, :integer, default: 60 * 60
47
- config_param :ignore_nil, :integer, default: true
48
- config_param :watch, :bool, default: true
49
- config_param :apiVersion, :string, default: 'v1'
50
- config_param :client_cert, :string, default: nil
51
- config_param :client_key, :string, default: nil
52
- config_param :ca_file, :string, default: nil
53
- config_param :verify_ssl, :bool, default: true
54
- config_param :open_timeout, :integer, default: 3
55
- config_param :read_timeout, :integer, default: 10
56
-
57
- REGEX_VAR_LOG_PODS = '(var\.log\.pods)\.(?<namespace>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<pod_uuid>[a-z0-9-]*)\.(?<container_name>.+)\..*\.log$'
58
- REGEX_VAR_LOG_CONTAINERS = '(var\.log\.containers)\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
59
-
60
- #tag_to_kubernetes_name_regexp which must include named capture groups:
61
- # namespace - The namespace in which the pod is deployed
62
- # pod_name - The pod name
63
- # container_name - The name of the container
64
- # pod_uuid (/var/log/pods) | docker_id (/var/log/containers) - Unique identifier used in caching of either pod_uuid or the container hash
65
- config_param :tag_to_kubernetes_name_regexp, :string, default: "(#{REGEX_VAR_LOG_PODS}|#{REGEX_VAR_LOG_CONTAINERS})"
66
-
67
- config_param :bearer_token_file, :string, default: nil
68
- config_param :secret_dir, :string, default: '/var/run/secrets/kubernetes.io/serviceaccount'
69
-
70
- config_param :annotation_match, :array, default: []
71
- config_param :stats_interval, :integer, default: 30
72
- config_param :allow_orphans, :bool, default: true
73
- config_param :orphaned_namespace_name, :string, default: '.orphaned'
74
- config_param :orphaned_namespace_id, :string, default: 'orphaned'
75
- config_param :lookup_from_k8s_field, :bool, default: true
76
- # if `ca_file` is for an intermediate CA, or otherwise we do not have the root CA and want
77
- # to trust the intermediate CA certs we do have, set this to `true` - this corresponds to
78
- # the openssl s_client -partial_chain flag and X509_V_FLAG_PARTIAL_CHAIN
79
- config_param :ssl_partial_chain, :bool, default: false
80
- config_param :skip_labels, :bool, default: false
81
- config_param :skip_pod_labels, :bool, default: false
82
- config_param :skip_namespace_labels, :bool, default: false
83
- config_param :skip_container_metadata, :bool, default: false
84
- config_param :skip_master_url, :bool, default: false
85
- config_param :skip_namespace_metadata, :bool, default: false
86
- config_param :include_ownerrefs_metadata, :bool, default: false
87
-
88
- # A classname in the form of Test::APIAdapter which will try
89
- # to be resolved from a relative named file 'test_api_adapter'
90
- config_param :test_api_adapter, :string, default: nil
91
-
92
- # The time interval in seconds for retry backoffs when watch connections fail.
93
- config_param :watch_retry_interval, :integer, default: 1
94
- # The base number of exponential backoff for retries.
95
- config_param :watch_retry_exponential_backoff_base, :integer, default: 2
96
- # The maximum number of times to retry pod and namespace watches.
97
- config_param :watch_retry_max_times, :integer, default: 10
98
-
99
- def fetch_pod_metadata(namespace_name, pod_name)
100
- log.trace("fetching pod metadata: #{namespace_name}/#{pod_name}")
101
- options = {
102
- resource_version: '0' # Fetch from API server cache instead of etcd quorum read
103
- }
104
- pod_object = @client.get_pod(pod_name, namespace_name, options)
105
- log.trace("raw metadata for #{namespace_name}/#{pod_name}: #{pod_object}")
106
- metadata = parse_pod_metadata(pod_object)
107
- @stats.bump(:pod_cache_api_updates)
108
- log.trace("parsed metadata for #{namespace_name}/#{pod_name}: #{metadata}")
109
- @cache[metadata['pod_id']] = metadata
31
+ module Fluent
32
+ module Plugin
33
+ class KubernetesMetadataFilter < Fluent::Plugin::Filter # rubocop:disable Metrics/ClassLength
34
+ K8_POD_CA_CERT = 'ca.crt'
35
+ K8_POD_TOKEN = 'token'
36
+
37
+ include KubernetesMetadata::CacheStrategy
38
+ include KubernetesMetadata::Common
39
+ include KubernetesMetadata::WatchNamespaces
40
+ include KubernetesMetadata::WatchPods
41
+
42
+ Fluent::Plugin.register_filter('kubernetes_metadata', self)
43
+
44
+ config_param :kubernetes_url, :string, default: nil
45
+ config_param :cache_size, :integer, default: 1000
46
+ config_param :cache_ttl, :integer, default: 60 * 60
47
+ config_param :ignore_nil, :integer, default: true
48
+ config_param :watch, :bool, default: true
49
+ config_param :apiVersion, :string, default: 'v1'
50
+ config_param :client_cert, :string, default: nil
51
+ config_param :client_key, :string, default: nil
52
+ config_param :ca_file, :string, default: nil
53
+ config_param :verify_ssl, :bool, default: true
54
+ config_param :open_timeout, :integer, default: 3
55
+ config_param :read_timeout, :integer, default: 10
56
+
57
+ REGEX_VAR_LOG_PODS = '(var\.log\.pods)\.(?<namespace>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<pod_uuid>[a-z0-9-]*)\.(?<container_name>.+)\..*\.log$' # rubocop:disable Layout/LineLength
58
+ REGEX_VAR_LOG_CONTAINERS = '(var\.log\.containers)\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$' # rubocop:disable Layout/LineLength
59
+
60
+ # tag_to_kubernetes_name_regexp which must include named capture groups:
61
+ # namespace - The namespace in which the pod is deployed
62
+ # pod_name - The pod name
63
+ # container_name - The name of the container
64
+ # pod_uuid (/var/log/pods) | docker_id (/var/log/containers) - Unique identifier used in caching of either
65
+ # pod_uuid or the container hash
66
+ config_param :tag_to_kubernetes_name_regexp, :string,
67
+ default: "(#{REGEX_VAR_LOG_PODS}|#{REGEX_VAR_LOG_CONTAINERS})"
68
+
69
+ config_param :bearer_token_file, :string, default: nil
70
+ config_param :secret_dir, :string, default: '/var/run/secrets/kubernetes.io/serviceaccount'
71
+
72
+ config_param :annotation_match, :array, default: []
73
+ config_param :stats_interval, :integer, default: 30
74
+ config_param :allow_orphans, :bool, default: true
75
+ config_param :orphaned_namespace_name, :string, default: '.orphaned'
76
+ config_param :orphaned_namespace_id, :string, default: 'orphaned'
77
+ config_param :lookup_from_k8s_field, :bool, default: true
78
+ # if `ca_file` is for an intermediate CA, or otherwise we do not have the root CA and want
79
+ # to trust the intermediate CA certs we do have, set this to `true` - this corresponds to
80
+ # the openssl s_client -partial_chain flag and X509_V_FLAG_PARTIAL_CHAIN
81
+ config_param :ssl_partial_chain, :bool, default: false
82
+ config_param :skip_labels, :bool, default: false
83
+ config_param :skip_pod_labels, :bool, default: false
84
+ config_param :skip_namespace_labels, :bool, default: false
85
+ config_param :skip_container_metadata, :bool, default: false
86
+ config_param :skip_master_url, :bool, default: false
87
+ config_param :skip_namespace_metadata, :bool, default: false
88
+ config_param :include_ownerrefs_metadata, :bool, default: false
89
+
90
+ # A classname in the form of Test::APIAdapter which will try
91
+ # to be resolved from a relative named file 'test_api_adapter'
92
+ config_param :test_api_adapter, :string, default: nil
93
+
94
+ # The time interval in seconds for retry backoffs when watch connections fail.
95
+ config_param :watch_retry_interval, :integer, default: 1
96
+ # The base number of exponential backoff for retries.
97
+ config_param :watch_retry_exponential_backoff_base, :integer, default: 2
98
+ # The maximum number of times to retry pod and namespace watches.
99
+ config_param :watch_retry_max_times, :integer, default: 10
100
+
101
+ def fetch_pod_metadata(namespace_name, pod_name) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
102
+ log.trace("fetching pod metadata: #{namespace_name}/#{pod_name}")
103
+ options = {
104
+ resource_version: '0' # Fetch from API server cache instead of etcd quorum read
105
+ }
106
+ pod_object = @client.get_pod(pod_name, namespace_name, options)
107
+ log.trace("raw metadata for #{namespace_name}/#{pod_name}: #{pod_object}")
108
+ metadata = parse_pod_metadata(pod_object)
109
+ @stats.bump(:pod_cache_api_updates)
110
+ log.trace("parsed metadata for #{namespace_name}/#{pod_name}: #{metadata}")
111
+ @cache[metadata['pod_id']] = metadata
110
112
  rescue KubeException => e
111
113
  if e.error_code == 401
112
114
  # recreate client to refresh token
113
115
  log.info("Encountered '401 Unauthorized' exception, recreating client to refresh token")
114
- create_client()
116
+ create_client
115
117
  elsif e.error_code == 404
116
- log.debug "Encountered '404 Not Found' exception, pod not found"
118
+ log.debug("Encountered '404 Not Found' exception, pod not found")
117
119
  @stats.bump(:pod_cache_api_nil_error)
118
120
  else
119
- log.error "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
121
+ log.error("Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}") # rubocop:disable Layout/LineLength
120
122
  @stats.bump(:pod_cache_api_nil_error)
121
123
  end
122
124
  {}
123
125
  rescue StandardError => e
124
126
  @stats.bump(:pod_cache_api_nil_error)
125
- log.error "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
127
+ log.error("Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}") # rubocop:disable Layout/LineLength
126
128
  {}
127
129
  end
128
130
 
129
- def dump_stats
130
- @curr_time = Time.now
131
- return if @curr_time.to_i - @prev_time.to_i < @stats_interval
131
+ def dump_stats # rubocop:disable Metrics/AbcSize
132
+ @curr_time = Time.now
133
+ return if @curr_time.to_i - @prev_time.to_i < @stats_interval
134
+
135
+ @prev_time = @curr_time
136
+ @stats.set(:pod_cache_size, @cache.count)
137
+ @stats.set(:namespace_cache_size, @namespace_cache.count) if @namespace_cache
138
+ log.info(@stats)
139
+ return unless log.level == Fluent::Log::LEVEL_TRACE
132
140
 
133
- @prev_time = @curr_time
134
- @stats.set(:pod_cache_size, @cache.count)
135
- @stats.set(:namespace_cache_size, @namespace_cache.count) if @namespace_cache
136
- log.info(@stats)
137
- if log.level == Fluent::Log::LEVEL_TRACE
138
141
  log.trace(" id cache: #{@id_cache.to_a}")
139
142
  log.trace(" pod cache: #{@cache.to_a}")
140
143
  log.trace("namespace cache: #{@namespace_cache.to_a}")
141
144
  end
142
- end
143
145
 
144
- def fetch_namespace_metadata(namespace_name)
145
- log.trace("fetching namespace metadata: #{namespace_name}")
146
- options = {
147
- resource_version: '0' # Fetch from API server cache instead of etcd quorum read
148
- }
149
- namespace_object = @client.get_namespace(namespace_name, nil, options)
150
- log.trace("raw metadata for #{namespace_name}: #{namespace_object}")
151
- metadata = parse_namespace_metadata(namespace_object)
152
- @stats.bump(:namespace_cache_api_updates)
153
- log.trace("parsed metadata for #{namespace_name}: #{metadata}")
154
- @namespace_cache[metadata['namespace_id']] = metadata
146
+ def fetch_namespace_metadata(namespace_name) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
147
+ log.trace("fetching namespace metadata: #{namespace_name}")
148
+ options = {
149
+ resource_version: '0' # Fetch from API server cache instead of etcd quorum read
150
+ }
151
+ namespace_object = @client.get_namespace(namespace_name, nil, options)
152
+ log.trace("raw metadata for #{namespace_name}: #{namespace_object}")
153
+ metadata = parse_namespace_metadata(namespace_object)
154
+ @stats.bump(:namespace_cache_api_updates)
155
+ log.trace("parsed metadata for #{namespace_name}: #{metadata}")
156
+ @namespace_cache[metadata['namespace_id']] = metadata
155
157
  rescue KubeException => e
156
158
  if e.error_code == 401
157
159
  # recreate client to refresh token
158
160
  log.info("Encountered '401 Unauthorized' exception, recreating client to refresh token")
159
- create_client()
161
+ create_client
160
162
  else
161
- log.error "Exception '#{e}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
163
+ log.error("Exception '#{e}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}") # rubocop:disable Layout/LineLength
162
164
  @stats.bump(:namespace_cache_api_nil_error)
163
165
  end
164
166
  {}
165
167
  rescue StandardError => e
166
168
  @stats.bump(:namespace_cache_api_nil_error)
167
- log.error "Exception '#{e}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
169
+ log.error("Exception '#{e}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}") # rubocop:disable Layout/LineLength
168
170
  {}
169
- end
171
+ end
170
172
 
171
- def initialize
172
- super
173
- @prev_time = Time.now
174
- @ssl_options = {}
175
- @auth_options = {}
176
- end
173
+ def initialize
174
+ super
175
+ @prev_time = Time.now
176
+ @ssl_options = {}
177
+ @auth_options = {}
178
+ end
177
179
 
178
- def configure(conf)
179
- super
180
+ def configure(conf) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
181
+ super
180
182
 
181
- require 'kubeclient'
182
- require 'lru_redux'
183
+ require 'kubeclient'
184
+ require 'lru_redux'
183
185
 
184
- @stats = KubernetesMetadata::Stats.new
185
- if @stats_interval <= 0
186
- @stats = KubernetesMetadata::NoOpStats.new
187
- self.define_singleton_method(:dump_stats) {}
188
- end
186
+ @stats = KubernetesMetadata::Stats.new
187
+ if @stats_interval <= 0
188
+ @stats = KubernetesMetadata::NoOpStats.new
189
+ define_singleton_method(:dump_stats) {} # rubocop:disable Lint/EmptyBlock
190
+ end
189
191
 
190
- if @cache_ttl < 0
191
- log.info 'Setting the cache TTL to :none because it was <= 0'
192
- @cache_ttl = :none
193
- end
192
+ if @cache_ttl < 0
193
+ log.info 'Setting the cache TTL to :none because it was <= 0'
194
+ @cache_ttl = :none
195
+ end
194
196
 
195
- # Caches pod/namespace UID tuples for a given container UID.
196
- @id_cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl, @ignore_nil)
197
+ # Caches pod/namespace UID tuples for a given container UID.
198
+ @id_cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl, @ignore_nil)
197
199
 
198
- # Use the container UID as the key to fetch a hash containing pod metadata
199
- @cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl, @ignore_nil)
200
+ # Use the container UID as the key to fetch a hash containing pod metadata
201
+ @cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl, @ignore_nil)
200
202
 
201
- # Use the namespace UID as the key to fetch a hash containing namespace metadata
202
- @namespace_cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl, @ignore_nil)
203
+ # Use the namespace UID as the key to fetch a hash containing namespace metadata
204
+ @namespace_cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl, @ignore_nil)
203
205
 
204
- @tag_to_kubernetes_name_regexp_compiled = Regexp.compile(@tag_to_kubernetes_name_regexp)
206
+ @tag_to_kubernetes_name_regexp_compiled = Regexp.compile(@tag_to_kubernetes_name_regexp)
205
207
 
206
- # Use Kubernetes default service account if we're in a pod.
207
- if @kubernetes_url.nil?
208
- log.debug 'Kubernetes URL is not set - inspecting environ'
208
+ # Use Kubernetes default service account if we're in a pod.
209
+ if @kubernetes_url.nil?
210
+ log.debug('Kubernetes URL is not set - inspecting environ')
211
+ env_host = ENV['KUBERNETES_SERVICE_HOST']
212
+ env_port = ENV['KUBERNETES_SERVICE_PORT']
209
213
 
210
- env_host = ENV['KUBERNETES_SERVICE_HOST']
211
- env_port = ENV['KUBERNETES_SERVICE_PORT']
212
- if present?(env_host) && present?(env_port)
213
- if env_host =~ Resolv::IPv6::Regex
214
- # Brackets are needed around IPv6 addresses
215
- env_host = "[#{env_host}]"
214
+ if present?(env_host) && present?(env_port)
215
+ if Resolv::IPv6::Regex.match?(env_host)
216
+ # Brackets are needed around IPv6 addresses
217
+ env_host = "[#{env_host}]"
218
+ end
219
+ @kubernetes_url = "https://#{env_host}:#{env_port}/api"
220
+ log.debug("Kubernetes URL is now '#{@kubernetes_url}'")
221
+ else
222
+ log.debug('No Kubernetes URL could be found in config or environ')
216
223
  end
217
- @kubernetes_url = "https://#{env_host}:#{env_port}/api"
218
- log.debug "Kubernetes URL is now '#{@kubernetes_url}'"
219
- else
220
- log.debug 'No Kubernetes URL could be found in config or environ'
221
224
  end
222
- end
223
225
 
224
- # Use SSL certificate and bearer token from Kubernetes service account.
225
- if Dir.exist?(@secret_dir)
226
- log.debug "Found directory with secrets: #{@secret_dir}"
227
- ca_cert = File.join(@secret_dir, K8_POD_CA_CERT)
228
- pod_token = File.join(@secret_dir, K8_POD_TOKEN)
226
+ # Use SSL certificate and bearer token from Kubernetes service account.
227
+ if Dir.exist?(@secret_dir)
228
+ log.debug("Found directory with secrets: #{@secret_dir}")
229
+ ca_cert = File.join(@secret_dir, K8_POD_CA_CERT)
230
+ pod_token = File.join(@secret_dir, K8_POD_TOKEN)
229
231
 
230
- if !present?(@ca_file) && File.exist?(ca_cert)
231
- log.debug "Found CA certificate: #{ca_cert}"
232
- @ca_file = ca_cert
233
- end
232
+ if !present?(@ca_file) && File.exist?(ca_cert)
233
+ log.debug("Found CA certificate: #{ca_cert}")
234
+ @ca_file = ca_cert
235
+ end
234
236
 
235
- if !present?(@bearer_token_file) && File.exist?(pod_token)
236
- log.debug "Found pod token: #{pod_token}"
237
- @bearer_token_file = pod_token
237
+ if !present?(@bearer_token_file) && File.exist?(pod_token)
238
+ log.debug("Found pod token: #{pod_token}")
239
+ @bearer_token_file = pod_token
240
+ end
238
241
  end
239
- end
240
242
 
241
- if present?(@kubernetes_url)
242
- @ssl_options = {
243
- client_cert: present?(@client_cert) ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil,
244
- client_key: present?(@client_key) ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
245
- ca_file: @ca_file,
246
- verify_ssl: @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
247
- }
243
+ if present?(@kubernetes_url)
244
+ @ssl_options = {
245
+ client_cert: present?(@client_cert) ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil,
246
+ client_key: present?(@client_key) ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
247
+ ca_file: @ca_file,
248
+ verify_ssl: @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
249
+ }
250
+
251
+ if @ssl_partial_chain
252
+ # taken from the ssl.rb OpenSSL::SSL::SSLContext code for DEFAULT_CERT_STORE
253
+ require 'openssl'
254
+
255
+ ssl_store = OpenSSL::X509::Store.new
256
+ ssl_store.set_default_paths
257
+ flagval = if defined? OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
258
+ OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
259
+ else
260
+ # this version of ruby does not define OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
261
+ 0x80000
262
+ end
263
+ ssl_store.flags = OpenSSL::X509::V_FLAG_CRL_CHECK_ALL | flagval
264
+ @ssl_options[:cert_store] = ssl_store
265
+ end
248
266
 
249
- if @ssl_partial_chain
250
- # taken from the ssl.rb OpenSSL::SSL::SSLContext code for DEFAULT_CERT_STORE
251
- require 'openssl'
252
- ssl_store = OpenSSL::X509::Store.new
253
- ssl_store.set_default_paths
254
- flagval = if defined? OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
255
- OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
256
- else
257
- # this version of ruby does not define OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
258
- 0x80000
259
- end
260
- ssl_store.flags = OpenSSL::X509::V_FLAG_CRL_CHECK_ALL | flagval
261
- @ssl_options[:cert_store] = ssl_store
262
- end
267
+ @auth_options[:bearer_token_file] = @bearer_token_file if present?(@bearer_token_file)
263
268
 
264
- if present?(@bearer_token_file)
265
- @auth_options[:bearer_token_file] = @bearer_token_file
266
- end
269
+ create_client
267
270
 
268
- create_client()
271
+ if @test_api_adapter
272
+ log.info "Extending client with test api adapter #{@test_api_adapter}"
273
+ @test_api_adapter = @test_api_adapter.gsub('::', '_')
274
+ .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
275
+ .gsub(/([a-z\d])([A-Z])/, '\1_\2')
276
+ .tr('-', '_')
277
+ .downcase
278
+ require_relative @test_api_adapter
279
+ @client.extend(eval(@test_api_adapter)) # rubocop:disable Security/Eval
280
+ end
269
281
 
270
- if @test_api_adapter
271
- log.info "Extending client with test api adaper #{@test_api_adapter}"
272
- require_relative @test_api_adapter.underscore
273
- @client.extend(eval(@test_api_adapter))
274
- end
282
+ begin
283
+ @client.api_valid?
284
+ rescue KubeException => e
285
+ raise Fluent::ConfigError, "Invalid Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}: #{e.message}"
286
+ end
275
287
 
276
- begin
277
- @client.api_valid?
278
- rescue KubeException => e
279
- raise Fluent::ConfigError, "Invalid Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}: #{e.message}"
280
- end
288
+ if @watch
289
+ if ENV['K8S_NODE_NAME'].nil? || ENV['K8S_NODE_NAME'].strip.empty?
290
+ log.warn("!! The environment variable 'K8S_NODE_NAME' is not set to the node name which can affect the API server and watch efficiency !!") # rubocop:disable Layout/LineLength
291
+ end
281
292
 
282
- if @watch
283
- if ENV['K8S_NODE_NAME'].nil? || ENV['K8S_NODE_NAME'].strip.empty?
284
- log.warn("!! The environment variable 'K8S_NODE_NAME' is not set to the node name which can affect the API server and watch efficiency !!")
285
- end
293
+ pod_thread = Thread.new(self, &:set_up_pod_thread)
294
+ pod_thread.abort_on_exception = true
286
295
 
287
- pod_thread = Thread.new(self, &:set_up_pod_thread)
288
- pod_thread.abort_on_exception = true
296
+ namespace_thread = Thread.new(self, &:set_up_namespace_thread)
297
+ namespace_thread.abort_on_exception = true
298
+ end
299
+ end
289
300
 
290
- namespace_thread = Thread.new(self, &:set_up_namespace_thread)
291
- namespace_thread.abort_on_exception = true
301
+ @annotations_regexps = []
302
+ @annotation_match.each do |regexp|
303
+ @annotations_regexps << Regexp.compile(regexp)
304
+ rescue RegexpError => e
305
+ log.error("Error: invalid regular expression in annotation_match: #{e}")
292
306
  end
293
307
  end
294
308
 
295
- @annotations_regexps = []
296
- @annotation_match.each do |regexp|
297
- @annotations_regexps << Regexp.compile(regexp)
298
- rescue RegexpError => e
299
- log.error "Error: invalid regular expression in annotation_match: #{e}"
309
+ def create_client # rubocop:disable Metrics/MethodLength
310
+ log.debug('Creating K8S client')
311
+ @client = nil
312
+ @client = Kubeclient::Client.new(
313
+ @kubernetes_url,
314
+ @apiVersion,
315
+ ssl_options: @ssl_options,
316
+ auth_options: @auth_options,
317
+ timeouts: {
318
+ open: @open_timeout,
319
+ read: @read_timeout
320
+ },
321
+ as: :parsed_symbolized
322
+ )
300
323
  end
301
- end
302
324
 
303
- def create_client()
304
- log.debug 'Creating K8S client'
305
- @client = nil
306
- @client = Kubeclient::Client.new(
307
- @kubernetes_url,
308
- @apiVersion,
309
- ssl_options: @ssl_options,
310
- auth_options: @auth_options,
311
- timeouts: {
312
- open: @open_timeout,
313
- read: @read_timeout
314
- },
315
- as: :parsed_symbolized
316
- )
317
- end
318
-
319
- def get_metadata_for_record(namespace_name, pod_name, container_name, cache_key, create_time, batch_miss_cache, docker_id)
320
- metadata = {
321
- 'docker' => { 'container_id' => "" },
322
- 'kubernetes' => {
323
- 'container_name' => container_name,
324
- 'namespace_name' => namespace_name,
325
- 'pod_name' => pod_name
325
+ def get_metadata_for_record(namespace_name, pod_name, container_name, cache_key, create_time, batch_miss_cache, # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/ParameterLists, Metrics/PerceivedComplexity
326
+ docker_id)
327
+ metadata = {
328
+ 'docker' => { 'container_id' => '' },
329
+ 'kubernetes' => {
330
+ 'container_name' => container_name,
331
+ 'namespace_name' => namespace_name,
332
+ 'pod_name' => pod_name
333
+ }
326
334
  }
327
- }
328
- metadata['docker']['container_id'] = docker_id unless docker_id.nil?
329
- container_cache_key = container_name
330
- if present?(@kubernetes_url)
331
- pod_metadata = get_pod_metadata(cache_key, namespace_name, pod_name, create_time, batch_miss_cache)
332
- if (pod_metadata.include? 'containers') && (pod_metadata['containers'].include? container_cache_key) && !@skip_container_metadata
333
- metadata['kubernetes']['container_image'] = pod_metadata['containers'][container_cache_key]['image']
334
- metadata['kubernetes']['container_image_id'] = pod_metadata['containers'][container_cache_key]['image_id'] unless pod_metadata['containers'][container_cache_key]['image_id'].empty?
335
- metadata['docker']['container_id'] = pod_metadata['containers'][container_cache_key]['containerID'] unless pod_metadata['containers'][container_cache_key]['containerID'].empty?
335
+ metadata['docker']['container_id'] = docker_id unless docker_id.nil?
336
+ container_cache_key = container_name
337
+ if present?(@kubernetes_url)
338
+ pod_metadata = get_pod_metadata(cache_key, namespace_name, pod_name, create_time, batch_miss_cache)
339
+ if (pod_metadata.include? 'containers') && (pod_metadata['containers'].include? container_cache_key) && !@skip_container_metadata # rubocop:disable Layout/LineLength
340
+ metadata['kubernetes']['container_image'] = pod_metadata['containers'][container_cache_key]['image']
341
+ unless pod_metadata['containers'][container_cache_key]['image_id'].empty?
342
+ metadata['kubernetes']['container_image_id'] =
343
+ pod_metadata['containers'][container_cache_key]['image_id']
344
+ end
345
+ unless pod_metadata['containers'][container_cache_key]['containerID'].empty?
346
+ metadata['docker']['container_id'] =
347
+ pod_metadata['containers'][container_cache_key]['containerID']
348
+ end
349
+ end
350
+ metadata['kubernetes'].merge!(pod_metadata) if pod_metadata
351
+ metadata['kubernetes'].delete('containers')
336
352
  end
337
-
338
- metadata['kubernetes'].merge!(pod_metadata) if pod_metadata
339
- metadata['kubernetes'].delete('containers')
340
- end
341
- metadata['kubernetes'].tap do |kube|
342
- kube.each_pair do |k,v|
343
- kube[k.dup] = v.dup
353
+ metadata['kubernetes'].tap do |kube|
354
+ kube.each_pair do |k, v|
355
+ kube[k.dup] = v.dup
356
+ end
344
357
  end
358
+ if metadata['docker'] && (metadata['docker']['container_id'].nil? || metadata['docker']['container_id'].empty?)
359
+ metadata.delete('docker')
360
+ end
361
+ metadata
345
362
  end
346
- metadata.delete('docker') if metadata['docker'] && (metadata['docker']['container_id'].nil? || metadata['docker']['container_id'].empty?)
347
- metadata
348
- end
349
363
 
350
- def filter(tag, time, record)
351
- tag_match_data = tag.match(@tag_to_kubernetes_name_regexp_compiled)
352
- batch_miss_cache = {}
353
- if tag_match_data
354
- cache_key = if tag_match_data.names.include?('pod_uuid') && !tag_match_data['pod_uuid'].nil?
355
- tag_match_data['pod_uuid']
356
- else
357
- tag_match_data['docker_id']
364
+ def filter(tag, time, record) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
365
+ tag_match_data = tag.match(@tag_to_kubernetes_name_regexp_compiled)
366
+ batch_miss_cache = {}
367
+ if tag_match_data
368
+ cache_key = if tag_match_data.names.include?('pod_uuid') && !tag_match_data['pod_uuid'].nil?
369
+ tag_match_data['pod_uuid']
370
+ else
371
+ tag_match_data['docker_id']
372
+ end
373
+ docker_id = tag_match_data.names.include?('docker_id') ? tag_match_data['docker_id'] : nil
374
+ metadata = get_metadata_for_record(
375
+ tag_match_data['namespace'],
376
+ tag_match_data['pod_name'],
377
+ tag_match_data['container_name'],
378
+ cache_key,
379
+ time,
380
+ batch_miss_cache,
381
+ docker_id
382
+ )
358
383
  end
359
- docker_id = tag_match_data.names.include?('docker_id') ? tag_match_data['docker_id'] : nil
360
- metadata = get_metadata_for_record(tag_match_data['namespace'], tag_match_data['pod_name'], tag_match_data['container_name'],
361
- cache_key, time, batch_miss_cache, docker_id)
362
- end
363
- if @lookup_from_k8s_field && record.key?('kubernetes') && record.key?('docker') &&
364
- record['kubernetes'].respond_to?(:has_key?) && record['docker'].respond_to?(:has_key?) &&
365
- record['kubernetes'].key?('namespace_name') &&
366
- record['kubernetes'].key?('pod_name') &&
367
- record['kubernetes'].key?('container_name') &&
368
- record['docker'].key?('container_id') &&
369
- (k_metadata = get_metadata_for_record(record['kubernetes']['namespace_name'], record['kubernetes']['pod_name'],
370
- record['kubernetes']['container_name'], record['docker']['container_id'],
371
- time, batch_miss_cache, record['docker']['container_id']))
372
- metadata = k_metadata
384
+ if @lookup_from_k8s_field && record.key?('kubernetes') && record.key?('docker') &&
385
+ record['kubernetes'].respond_to?(:has_key?) && record['docker'].respond_to?(:has_key?) &&
386
+ record['kubernetes'].key?('namespace_name') &&
387
+ record['kubernetes'].key?('pod_name') &&
388
+ record['kubernetes'].key?('container_name') &&
389
+ record['docker'].key?('container_id')
390
+ k_metadata = get_metadata_for_record(
391
+ record['kubernetes']['namespace_name'],
392
+ record['kubernetes']['pod_name'],
393
+ record['kubernetes']['container_name'],
394
+ record['docker']['container_id'],
395
+ time,
396
+ batch_miss_cache,
397
+ record['docker']['container_id']
398
+ )
399
+ metadata = k_metadata if k_metadata
400
+ end
401
+ dump_stats
402
+ metadata ? record.merge(metadata) : record
373
403
  end
374
- dump_stats
375
- metadata ? record.merge(metadata) : record
376
- end
377
404
 
378
- # copied from activesupport
379
- def present?(object)
380
- object.respond_to?(:empty?) ? !object.empty? : !!object
405
+ # copied from activesupport
406
+ def present?(object)
407
+ object.respond_to?(:empty?) ? !object.empty? : !!object
408
+ end
381
409
  end
382
410
  end
383
411
  end