fluent-plugin-kubernetes_metadata_filter_fix 2.2.0.pre.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.circleci/config.yml +56 -0
- data/.gitignore +20 -0
- data/Gemfile +7 -0
- data/LICENSE.txt +201 -0
- data/README.md +210 -0
- data/Rakefile +37 -0
- data/fluent-plugin-kubernetes_metadata_filter.gemspec +36 -0
- data/lib/fluent/plugin/filter_kubernetes_metadata.rb +387 -0
- data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb +98 -0
- data/lib/fluent/plugin/kubernetes_metadata_common.rb +113 -0
- data/lib/fluent/plugin/kubernetes_metadata_stats.rb +46 -0
- data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +60 -0
- data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +63 -0
- data/test/cassettes/invalid_api_server_config.yml +53 -0
- data/test/cassettes/kubernetes_docker_metadata.yml +228 -0
- data/test/cassettes/kubernetes_docker_metadata_annotations.yml +239 -0
- data/test/cassettes/kubernetes_docker_metadata_dotted_labels.yml +231 -0
- data/test/cassettes/kubernetes_docker_metadata_using_bearer_token.yml +248 -0
- data/test/cassettes/metadata_from_tag_and_journald_fields.yml +408 -0
- data/test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml +540 -0
- data/test/cassettes/metadata_with_namespace_id.yml +276 -0
- data/test/cassettes/non_kubernetes_docker_metadata.yml +97 -0
- data/test/cassettes/valid_kubernetes_api_server.yml +55 -0
- data/test/helper.rb +64 -0
- data/test/plugin/test.token +1 -0
- data/test/plugin/test_cache_stats.rb +36 -0
- data/test/plugin/test_cache_strategy.rb +196 -0
- data/test/plugin/test_filter_kubernetes_metadata.rb +970 -0
- data/test/plugin/test_watch_namespaces.rb +91 -0
- data/test/plugin/test_watch_pods.rb +145 -0
- data/test/plugin/watch_test.rb +57 -0
- metadata +280 -0
data/Rakefile
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
require 'bundler/gem_tasks'
|
2
|
+
require 'rake/testtask'
|
3
|
+
require 'bump/tasks'
|
4
|
+
|
5
|
+
task :test => [:base_test]
|
6
|
+
|
7
|
+
task :default => [:test, :build]
|
8
|
+
|
9
|
+
desc 'Run test_unit based test'
|
10
|
+
Rake::TestTask.new(:base_test) do |t|
|
11
|
+
# To run test for only one file (or file path pattern)
|
12
|
+
# $ bundle exec rake base_test TEST=test/test_specified_path.rb
|
13
|
+
# $ bundle exec rake base_test TEST=test/test_*.rb
|
14
|
+
t.libs << 'test'
|
15
|
+
t.test_files = Dir['test/**/test_*.rb'].sort
|
16
|
+
#t.verbose = true
|
17
|
+
t.warning = false
|
18
|
+
end
|
19
|
+
|
20
|
+
desc 'Add copyright headers'
|
21
|
+
task :headers do
|
22
|
+
require 'rubygems'
|
23
|
+
require 'copyright_header'
|
24
|
+
|
25
|
+
args = {
|
26
|
+
:license => 'Apache-2.0',
|
27
|
+
:copyright_software => 'Fluentd Kubernetes Metadata Filter Plugin',
|
28
|
+
:copyright_software_description => 'Enrich Fluentd events with Kubernetes metadata',
|
29
|
+
:copyright_holders => ['Red Hat, Inc.'],
|
30
|
+
:copyright_years => ['2015-2017'],
|
31
|
+
:add_path => 'lib:test',
|
32
|
+
:output_dir => '.'
|
33
|
+
}
|
34
|
+
|
35
|
+
command_line = CopyrightHeader::CommandLine.new( args )
|
36
|
+
command_line.execute
|
37
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
|
5
|
+
Gem::Specification.new do |gem|
|
6
|
+
gem.name = "fluent-plugin-kubernetes_metadata_filter_fix"
|
7
|
+
gem.version = "2.2.0-1"
|
8
|
+
gem.authors = ["Jimmi Dyson"]
|
9
|
+
gem.email = ["jimmidyson@gmail.com"]
|
10
|
+
gem.description = %q{Filter plugin to add Kubernetes metadata}
|
11
|
+
gem.summary = %q{Fluentd filter plugin to add Kubernetes metadata}
|
12
|
+
gem.homepage = "https://github.com/vitkovskii/fluent-plugin-kubernetes_metadata_filter"
|
13
|
+
gem.license = "Apache-2.0"
|
14
|
+
|
15
|
+
gem.files = `git ls-files`.split($/)
|
16
|
+
gem.executables = gem.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
17
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
18
|
+
gem.require_paths = ["lib"]
|
19
|
+
|
20
|
+
gem.required_ruby_version = '>= 2.1.0'
|
21
|
+
|
22
|
+
gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 2']
|
23
|
+
gem.add_runtime_dependency "lru_redux"
|
24
|
+
gem.add_runtime_dependency "kubeclient", "~> 1.1.4"
|
25
|
+
|
26
|
+
gem.add_development_dependency "bundler", "~> 2.0.2"
|
27
|
+
gem.add_development_dependency "rake"
|
28
|
+
gem.add_development_dependency "minitest", "~> 4.0"
|
29
|
+
gem.add_development_dependency "test-unit", "~> 3.0.2"
|
30
|
+
gem.add_development_dependency "test-unit-rr", "~> 1.0.3"
|
31
|
+
gem.add_development_dependency "copyright-header"
|
32
|
+
gem.add_development_dependency "webmock"
|
33
|
+
gem.add_development_dependency "vcr"
|
34
|
+
gem.add_development_dependency "bump"
|
35
|
+
gem.add_development_dependency "yajl-ruby"
|
36
|
+
end
|
@@ -0,0 +1,387 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
|
3
|
+
# Kubernetes metadata
|
4
|
+
#
|
5
|
+
# Copyright 2017 Red Hat, Inc.
|
6
|
+
#
|
7
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
8
|
+
# you may not use this file except in compliance with the License.
|
9
|
+
# You may obtain a copy of the License at
|
10
|
+
#
|
11
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
12
|
+
#
|
13
|
+
# Unless required by applicable law or agreed to in writing, software
|
14
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
15
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
16
|
+
# See the License for the specific language governing permissions and
|
17
|
+
# limitations under the License.
|
18
|
+
#
|
19
|
+
|
20
|
+
require_relative 'kubernetes_metadata_cache_strategy'
|
21
|
+
require_relative 'kubernetes_metadata_common'
|
22
|
+
require_relative 'kubernetes_metadata_stats'
|
23
|
+
require_relative 'kubernetes_metadata_watch_namespaces'
|
24
|
+
require_relative 'kubernetes_metadata_watch_pods'
|
25
|
+
|
26
|
+
require 'fluent/plugin/filter'
|
27
|
+
|
28
|
+
module Fluent::Plugin
|
29
|
+
class KubernetesMetadataFilter < Fluent::Plugin::Filter
|
30
|
+
K8_POD_CA_CERT = 'ca.crt'
|
31
|
+
K8_POD_TOKEN = 'token'
|
32
|
+
|
33
|
+
include KubernetesMetadata::CacheStrategy
|
34
|
+
include KubernetesMetadata::Common
|
35
|
+
include KubernetesMetadata::WatchNamespaces
|
36
|
+
include KubernetesMetadata::WatchPods
|
37
|
+
|
38
|
+
Fluent::Plugin.register_filter('kubernetes_metadata', self)
|
39
|
+
|
40
|
+
config_param :kubernetes_url, :string, default: nil
|
41
|
+
config_param :cache_size, :integer, default: 1000
|
42
|
+
config_param :cache_ttl, :integer, default: 60 * 60
|
43
|
+
config_param :watch, :bool, default: true
|
44
|
+
config_param :apiVersion, :string, default: 'v1'
|
45
|
+
config_param :client_cert, :string, default: nil
|
46
|
+
config_param :client_key, :string, default: nil
|
47
|
+
config_param :ca_file, :string, default: nil
|
48
|
+
config_param :verify_ssl, :bool, default: true
|
49
|
+
config_param :tag_to_kubernetes_name_regexp,
|
50
|
+
:string,
|
51
|
+
:default => 'var\.log\.containers\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
|
52
|
+
config_param :bearer_token_file, :string, default: nil
|
53
|
+
config_param :secret_dir, :string, default: '/var/run/secrets/kubernetes.io/serviceaccount'
|
54
|
+
config_param :de_dot, :bool, default: true
|
55
|
+
config_param :de_dot_separator, :string, default: '_'
|
56
|
+
# if reading from the journal, the record will contain the following fields in the following
|
57
|
+
# format:
|
58
|
+
# CONTAINER_NAME=k8s_$containername.$containerhash_$podname_$namespacename_$poduuid_$rand32bitashex
|
59
|
+
# CONTAINER_FULL_ID=dockeridassha256hexvalue
|
60
|
+
config_param :use_journal, :bool, default: nil
|
61
|
+
# Field 2 is the container_hash, field 5 is the pod_id, and field 6 is the pod_randhex
|
62
|
+
# I would have included them as named groups, but you can't have named groups that are
|
63
|
+
# non-capturing :P
|
64
|
+
# parse format is defined here: https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/dockertools/docker.go#L317
|
65
|
+
config_param :container_name_to_kubernetes_regexp,
|
66
|
+
:string,
|
67
|
+
:default => '^(?<name_prefix>[^_]+)_(?<container_name>[^\._]+)(\.(?<container_hash>[^_]+))?_(?<pod_name>[^_]+)_(?<namespace>[^_]+)_[^_]+_[^_]+$'
|
68
|
+
|
69
|
+
config_param :annotation_match, :array, default: []
|
70
|
+
config_param :stats_interval, :integer, default: 30
|
71
|
+
config_param :allow_orphans, :bool, default: true
|
72
|
+
config_param :orphaned_namespace_name, :string, default: '.orphaned'
|
73
|
+
config_param :orphaned_namespace_id, :string, default: 'orphaned'
|
74
|
+
config_param :lookup_from_k8s_field, :bool, default: true
|
75
|
+
# if `ca_file` is for an intermediate CA, or otherwise we do not have the root CA and want
|
76
|
+
# to trust the intermediate CA certs we do have, set this to `true` - this corresponds to
|
77
|
+
# the openssl s_client -partial_chain flag and X509_V_FLAG_PARTIAL_CHAIN
|
78
|
+
config_param :ssl_partial_chain, :bool, default: false
|
79
|
+
config_param :skip_labels, :bool, default: false
|
80
|
+
config_param :skip_container_metadata, :bool, default: false
|
81
|
+
config_param :skip_master_url, :bool, default: false
|
82
|
+
config_param :skip_namespace_metadata, :bool, default: false
|
83
|
+
|
84
|
+
def fetch_pod_metadata(namespace_name, pod_name)
|
85
|
+
log.trace("fetching pod metadata: #{namespace_name}/#{pod_name}") if log.trace?
|
86
|
+
begin
|
87
|
+
metadata = @client.get_pod(pod_name, namespace_name)
|
88
|
+
unless metadata
|
89
|
+
log.trace("no metadata returned for: #{namespace_name}/#{pod_name}") if log.trace?
|
90
|
+
@stats.bump(:pod_cache_api_nil_not_found)
|
91
|
+
else
|
92
|
+
begin
|
93
|
+
log.trace("raw metadata for #{namespace_name}/#{pod_name}: #{metadata}") if log.trace?
|
94
|
+
metadata = parse_pod_metadata(metadata)
|
95
|
+
@stats.bump(:pod_cache_api_updates)
|
96
|
+
log.trace("parsed metadata for #{namespace_name}/#{pod_name}: #{metadata}") if log.trace?
|
97
|
+
@cache[metadata['pod_id']] = metadata
|
98
|
+
return metadata
|
99
|
+
rescue Exception=>e
|
100
|
+
log.debug(e)
|
101
|
+
@stats.bump(:pod_cache_api_nil_bad_resp_payload)
|
102
|
+
log.trace("returning empty metadata for #{namespace_name}/#{pod_name} due to error '#{e}'") if log.trace?
|
103
|
+
end
|
104
|
+
end
|
105
|
+
rescue Exception=>e
|
106
|
+
@stats.bump(:pod_cache_api_nil_error)
|
107
|
+
log.debug "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
|
108
|
+
end
|
109
|
+
{}
|
110
|
+
end
|
111
|
+
|
112
|
+
def dump_stats
|
113
|
+
@curr_time = Time.now
|
114
|
+
return if @curr_time.to_i - @prev_time.to_i < @stats_interval
|
115
|
+
@prev_time = @curr_time
|
116
|
+
@stats.set(:pod_cache_size, @cache.count)
|
117
|
+
@stats.set(:namespace_cache_size, @namespace_cache.count) if @namespace_cache
|
118
|
+
log.info(@stats)
|
119
|
+
if log.level == Fluent::Log::LEVEL_TRACE
|
120
|
+
log.trace(" id cache: #{@id_cache.to_a}")
|
121
|
+
log.trace(" pod cache: #{@cache.to_a}")
|
122
|
+
log.trace("namespace cache: #{@namespace_cache.to_a}")
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
def fetch_namespace_metadata(namespace_name)
|
127
|
+
log.trace("fetching namespace metadata: #{namespace_name}") if log.trace?
|
128
|
+
begin
|
129
|
+
metadata = @client.get_namespace(namespace_name)
|
130
|
+
unless metadata
|
131
|
+
log.trace("no metadata returned for: #{namespace_name}") if log.trace?
|
132
|
+
@stats.bump(:namespace_cache_api_nil_not_found)
|
133
|
+
else
|
134
|
+
begin
|
135
|
+
log.trace("raw metadata for #{namespace_name}: #{metadata}") if log.trace?
|
136
|
+
metadata = parse_namespace_metadata(metadata)
|
137
|
+
@stats.bump(:namespace_cache_api_updates)
|
138
|
+
log.trace("parsed metadata for #{namespace_name}: #{metadata}") if log.trace?
|
139
|
+
@namespace_cache[metadata['namespace_id']] = metadata
|
140
|
+
return metadata
|
141
|
+
rescue Exception => e
|
142
|
+
log.debug(e)
|
143
|
+
@stats.bump(:namespace_cache_api_nil_bad_resp_payload)
|
144
|
+
log.trace("returning empty metadata for #{namespace_name} due to error '#{e}'") if log.trace?
|
145
|
+
end
|
146
|
+
end
|
147
|
+
rescue Exception => kube_error
|
148
|
+
@stats.bump(:namespace_cache_api_nil_error)
|
149
|
+
log.debug "Exception '#{kube_error}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
|
150
|
+
end
|
151
|
+
{}
|
152
|
+
end
|
153
|
+
|
154
|
+
def initialize
|
155
|
+
super
|
156
|
+
@prev_time = Time.now
|
157
|
+
end
|
158
|
+
|
159
|
+
def configure(conf)
|
160
|
+
super
|
161
|
+
|
162
|
+
def log.trace?
|
163
|
+
level == Fluent::Log::LEVEL_TRACE
|
164
|
+
end
|
165
|
+
|
166
|
+
require 'kubeclient'
|
167
|
+
require 'active_support/core_ext/object/blank'
|
168
|
+
require 'lru_redux'
|
169
|
+
@stats = KubernetesMetadata::Stats.new
|
170
|
+
|
171
|
+
if @de_dot && (@de_dot_separator =~ /\./).present?
|
172
|
+
raise Fluent::ConfigError, "Invalid de_dot_separator: cannot be or contain '.'"
|
173
|
+
end
|
174
|
+
|
175
|
+
if @cache_ttl < 0
|
176
|
+
log.info "Setting the cache TTL to :none because it was <= 0"
|
177
|
+
@cache_ttl = :none
|
178
|
+
end
|
179
|
+
|
180
|
+
# Caches pod/namespace UID tuples for a given container UID.
|
181
|
+
@id_cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl)
|
182
|
+
|
183
|
+
# Use the container UID as the key to fetch a hash containing pod metadata
|
184
|
+
@cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl)
|
185
|
+
|
186
|
+
# Use the namespace UID as the key to fetch a hash containing namespace metadata
|
187
|
+
@namespace_cache = LruRedux::TTL::ThreadSafeCache.new(@cache_size, @cache_ttl)
|
188
|
+
|
189
|
+
@tag_to_kubernetes_name_regexp_compiled = Regexp.compile(@tag_to_kubernetes_name_regexp)
|
190
|
+
@container_name_to_kubernetes_regexp_compiled = Regexp.compile(@container_name_to_kubernetes_regexp)
|
191
|
+
|
192
|
+
# Use Kubernetes default service account if we're in a pod.
|
193
|
+
if @kubernetes_url.nil?
|
194
|
+
log.debug "Kubernetes URL is not set - inspecting environ"
|
195
|
+
|
196
|
+
env_host = ENV['KUBERNETES_SERVICE_HOST']
|
197
|
+
env_port = ENV['KUBERNETES_SERVICE_PORT']
|
198
|
+
if env_host.present? && env_port.present?
|
199
|
+
@kubernetes_url = "https://#{env_host}:#{env_port}/api"
|
200
|
+
log.debug "Kubernetes URL is now '#{@kubernetes_url}'"
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
# Use SSL certificate and bearer token from Kubernetes service account.
|
205
|
+
if Dir.exist?(@secret_dir)
|
206
|
+
log.debug "Found directory with secrets: #{@secret_dir}"
|
207
|
+
ca_cert = File.join(@secret_dir, K8_POD_CA_CERT)
|
208
|
+
pod_token = File.join(@secret_dir, K8_POD_TOKEN)
|
209
|
+
|
210
|
+
if !@ca_file.present? and File.exist?(ca_cert)
|
211
|
+
log.debug "Found CA certificate: #{ca_cert}"
|
212
|
+
@ca_file = ca_cert
|
213
|
+
end
|
214
|
+
|
215
|
+
if !@bearer_token_file.present? and File.exist?(pod_token)
|
216
|
+
log.debug "Found pod token: #{pod_token}"
|
217
|
+
@bearer_token_file = pod_token
|
218
|
+
end
|
219
|
+
end
|
220
|
+
|
221
|
+
if @kubernetes_url.present?
|
222
|
+
|
223
|
+
ssl_options = {
|
224
|
+
client_cert: @client_cert.present? ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil,
|
225
|
+
client_key: @client_key.present? ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
|
226
|
+
ca_file: @ca_file,
|
227
|
+
verify_ssl: @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
|
228
|
+
}
|
229
|
+
|
230
|
+
if @ssl_partial_chain
|
231
|
+
# taken from the ssl.rb OpenSSL::SSL::SSLContext code for DEFAULT_CERT_STORE
|
232
|
+
require 'openssl'
|
233
|
+
ssl_store = OpenSSL::X509::Store.new
|
234
|
+
ssl_store.set_default_paths
|
235
|
+
if defined? OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
|
236
|
+
flagval = OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
|
237
|
+
else
|
238
|
+
# this version of ruby does not define OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
|
239
|
+
flagval = 0x80000
|
240
|
+
end
|
241
|
+
ssl_store.flags = OpenSSL::X509::V_FLAG_CRL_CHECK_ALL | flagval
|
242
|
+
ssl_options[:cert_store] = ssl_store
|
243
|
+
end
|
244
|
+
|
245
|
+
auth_options = {}
|
246
|
+
|
247
|
+
if @bearer_token_file.present?
|
248
|
+
bearer_token = File.read(@bearer_token_file)
|
249
|
+
auth_options[:bearer_token] = bearer_token
|
250
|
+
end
|
251
|
+
|
252
|
+
log.debug "Creating K8S client"
|
253
|
+
@client = Kubeclient::Client.new @kubernetes_url, @apiVersion,
|
254
|
+
ssl_options: ssl_options,
|
255
|
+
auth_options: auth_options
|
256
|
+
|
257
|
+
begin
|
258
|
+
@client.api_valid?
|
259
|
+
rescue KubeException => kube_error
|
260
|
+
raise Fluent::ConfigError, "Invalid Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}: #{kube_error.message}"
|
261
|
+
end
|
262
|
+
|
263
|
+
if @watch
|
264
|
+
thread = Thread.new(self) { |this| this.start_pod_watch }
|
265
|
+
thread.abort_on_exception = true
|
266
|
+
namespace_thread = Thread.new(self) { |this| this.start_namespace_watch }
|
267
|
+
namespace_thread.abort_on_exception = true
|
268
|
+
end
|
269
|
+
end
|
270
|
+
@time_fields = []
|
271
|
+
@time_fields.push('_SOURCE_REALTIME_TIMESTAMP', '__REALTIME_TIMESTAMP') if @use_journal || @use_journal.nil?
|
272
|
+
@time_fields.push('time') unless @use_journal
|
273
|
+
@time_fields.push('@timestamp') if @lookup_from_k8s_field
|
274
|
+
|
275
|
+
@annotations_regexps = []
|
276
|
+
@annotation_match.each do |regexp|
|
277
|
+
begin
|
278
|
+
@annotations_regexps << Regexp.compile(regexp)
|
279
|
+
rescue RegexpError => e
|
280
|
+
log.error "Error: invalid regular expression in annotation_match: #{e}"
|
281
|
+
end
|
282
|
+
end
|
283
|
+
|
284
|
+
end
|
285
|
+
|
286
|
+
def get_metadata_for_record(namespace_name, pod_name, container_name, container_id, create_time, batch_miss_cache)
|
287
|
+
metadata = {
|
288
|
+
'docker' => {'container_id' => container_id},
|
289
|
+
'kubernetes' => {
|
290
|
+
'container_name' => container_name,
|
291
|
+
'namespace_name' => namespace_name,
|
292
|
+
'pod_name' => pod_name
|
293
|
+
}
|
294
|
+
}
|
295
|
+
if @kubernetes_url.present?
|
296
|
+
pod_metadata = get_pod_metadata(container_id, namespace_name, pod_name, create_time, batch_miss_cache)
|
297
|
+
|
298
|
+
if (pod_metadata.include? 'containers') && (pod_metadata['containers'].include? container_id) && !@skip_container_metadata
|
299
|
+
metadata['kubernetes']['container_image'] = pod_metadata['containers'][container_id]['image']
|
300
|
+
metadata['kubernetes']['container_image_id'] = pod_metadata['containers'][container_id]['image_id']
|
301
|
+
end
|
302
|
+
|
303
|
+
metadata['kubernetes'].merge!(pod_metadata) if pod_metadata
|
304
|
+
metadata['kubernetes'].delete('containers')
|
305
|
+
end
|
306
|
+
metadata
|
307
|
+
end
|
308
|
+
|
309
|
+
def create_time_from_record(record, internal_time)
|
310
|
+
time_key = @time_fields.detect{ |ii| record.has_key?(ii) }
|
311
|
+
time = record[time_key]
|
312
|
+
if time.nil? || time.chop.empty?
|
313
|
+
# `internal_time` is a Fluent::EventTime, it can't compare with Time.
|
314
|
+
return Time.at(internal_time.to_f)
|
315
|
+
end
|
316
|
+
if ['_SOURCE_REALTIME_TIMESTAMP', '__REALTIME_TIMESTAMP'].include?(time_key)
|
317
|
+
timei= time.to_i
|
318
|
+
return Time.at(timei / 1000000, timei % 1000000)
|
319
|
+
end
|
320
|
+
return Time.parse(time)
|
321
|
+
end
|
322
|
+
|
323
|
+
def filter_stream(tag, es)
|
324
|
+
return es if (es.respond_to?(:empty?) && es.empty?) || !es.is_a?(Fluent::EventStream)
|
325
|
+
new_es = Fluent::MultiEventStream.new
|
326
|
+
tag_match_data = tag.match(@tag_to_kubernetes_name_regexp_compiled) unless @use_journal
|
327
|
+
tag_metadata = nil
|
328
|
+
batch_miss_cache = {}
|
329
|
+
es.each do |time, record|
|
330
|
+
if tag_match_data && tag_metadata.nil?
|
331
|
+
tag_metadata = get_metadata_for_record(tag_match_data['namespace'], tag_match_data['pod_name'], tag_match_data['container_name'],
|
332
|
+
tag_match_data['docker_id'], create_time_from_record(record, time), batch_miss_cache)
|
333
|
+
end
|
334
|
+
metadata = Marshal.load(Marshal.dump(tag_metadata)) if tag_metadata
|
335
|
+
if (@use_journal || @use_journal.nil?) &&
|
336
|
+
(j_metadata = get_metadata_for_journal_record(record, time, batch_miss_cache))
|
337
|
+
metadata = j_metadata
|
338
|
+
end
|
339
|
+
if @lookup_from_k8s_field && record.has_key?('kubernetes') && record.has_key?('docker') &&
|
340
|
+
record['kubernetes'].respond_to?(:has_key?) && record['docker'].respond_to?(:has_key?) &&
|
341
|
+
record['kubernetes'].has_key?('namespace_name') &&
|
342
|
+
record['kubernetes'].has_key?('pod_name') &&
|
343
|
+
record['kubernetes'].has_key?('container_name') &&
|
344
|
+
record['docker'].has_key?('container_id') &&
|
345
|
+
(k_metadata = get_metadata_for_record(record['kubernetes']['namespace_name'], record['kubernetes']['pod_name'],
|
346
|
+
record['kubernetes']['container_name'], record['docker']['container_id'],
|
347
|
+
create_time_from_record(record, time), batch_miss_cache))
|
348
|
+
metadata = k_metadata
|
349
|
+
end
|
350
|
+
|
351
|
+
record = record.merge(metadata) if metadata
|
352
|
+
new_es.add(time, record)
|
353
|
+
end
|
354
|
+
dump_stats
|
355
|
+
new_es
|
356
|
+
end
|
357
|
+
|
358
|
+
def get_metadata_for_journal_record(record, time, batch_miss_cache)
|
359
|
+
metadata = nil
|
360
|
+
if record.has_key?('CONTAINER_NAME') && record.has_key?('CONTAINER_ID_FULL')
|
361
|
+
metadata = record['CONTAINER_NAME'].match(@container_name_to_kubernetes_regexp_compiled) do |match_data|
|
362
|
+
get_metadata_for_record(match_data['namespace'], match_data['pod_name'], match_data['container_name'],
|
363
|
+
record['CONTAINER_ID_FULL'], create_time_from_record(record, time), batch_miss_cache)
|
364
|
+
end
|
365
|
+
unless metadata
|
366
|
+
log.debug "Error: could not match CONTAINER_NAME from record #{record}"
|
367
|
+
@stats.bump(:container_name_match_failed)
|
368
|
+
end
|
369
|
+
elsif record.has_key?('CONTAINER_NAME') && record['CONTAINER_NAME'].start_with?('k8s_')
|
370
|
+
log.debug "Error: no container name and id in record #{record}"
|
371
|
+
@stats.bump(:container_name_id_missing)
|
372
|
+
end
|
373
|
+
metadata
|
374
|
+
end
|
375
|
+
|
376
|
+
def de_dot!(h)
|
377
|
+
h.keys.each do |ref|
|
378
|
+
if h[ref] && ref =~ /\./
|
379
|
+
v = h.delete(ref)
|
380
|
+
newref = ref.to_s.gsub('.', @de_dot_separator)
|
381
|
+
h[newref] = v
|
382
|
+
end
|
383
|
+
end
|
384
|
+
end
|
385
|
+
|
386
|
+
end
|
387
|
+
end
|