fluent-plugin-kubernetes_metadata_filter 2.6.0 → 2.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +1 -1
- data/.rubocop.yml +57 -0
- data/Gemfile +4 -2
- data/Gemfile.lock +11 -9
- data/README.md +4 -2
- data/Rakefile +15 -11
- data/fluent-plugin-kubernetes_metadata_filter.gemspec +23 -22
- data/lib/fluent/plugin/filter_kubernetes_metadata.rb +82 -72
- data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb +22 -18
- data/lib/fluent/plugin/kubernetes_metadata_common.rb +30 -29
- data/lib/fluent/plugin/kubernetes_metadata_stats.rb +6 -6
- data/lib/fluent/plugin/kubernetes_metadata_test_api_adapter.rb +68 -0
- data/lib/fluent/plugin/kubernetes_metadata_util.rb +53 -0
- data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +65 -65
- data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +69 -70
- data/test/helper.rb +5 -3
- data/test/plugin/test_cache_stats.rb +10 -10
- data/test/plugin/test_cache_strategy.rb +158 -157
- data/test/plugin/test_filter_kubernetes_metadata.rb +363 -344
- data/test/plugin/test_utils.rb +56 -0
- data/test/plugin/test_watch_namespaces.rb +191 -190
- data/test/plugin/test_watch_pods.rb +278 -267
- data/test/plugin/watch_test.rb +13 -7
- metadata +45 -41
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: aa36ffbcd940e1dbb15d59f6d1e331091aa44611fbbaf3e19f562dec80b8fa42
|
|
4
|
+
data.tar.gz: e9e3bc68ab9e2f5ed6e88ed42d609aa583d3a2fbe34cd1e10e826ad4a32b8f5b
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6fa67fa39f78c33377c5eab486d1cd4f24b38e5e81769a5a9e27c87c6002e5fecfe0f43da56302d798a31d69d83b14b9f56f684c0e2c729735669c0c9fa326db
|
|
7
|
+
data.tar.gz: 5c94737a18204e093672471b7f1b31221794a2aced8d5f65099aab1e97f1937635d4957b2f36406bf8ef43f4bd874e004a460a8b92ff7cc999299fa701d67f42
|
data/.circleci/config.yml
CHANGED
data/.rubocop.yml
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
AllCops:
|
|
2
|
+
TargetRubyVersion: 2.5 # keep in sync with .circleci/config.yml and gemspec
|
|
3
|
+
NewCops: enable
|
|
4
|
+
|
|
5
|
+
Style/EmptyMethod:
|
|
6
|
+
Enabled: false
|
|
7
|
+
|
|
8
|
+
Metrics:
|
|
9
|
+
Enabled: false
|
|
10
|
+
|
|
11
|
+
# not safe ... needs require 'English'
|
|
12
|
+
Style/SpecialGlobalVars:
|
|
13
|
+
Enabled: false
|
|
14
|
+
|
|
15
|
+
Layout/LineLength:
|
|
16
|
+
Max: 205 # TODO: lower
|
|
17
|
+
|
|
18
|
+
Style/Documentation:
|
|
19
|
+
Enabled: false
|
|
20
|
+
|
|
21
|
+
Naming/AccessorMethodName:
|
|
22
|
+
Enabled: false
|
|
23
|
+
|
|
24
|
+
Naming/MethodParameterName:
|
|
25
|
+
Enabled: false
|
|
26
|
+
|
|
27
|
+
Style/IfInsideElse:
|
|
28
|
+
Enabled: false
|
|
29
|
+
|
|
30
|
+
Style/GuardClause:
|
|
31
|
+
Enabled: false
|
|
32
|
+
|
|
33
|
+
Lint/NestedMethodDefinition:
|
|
34
|
+
Enabled: false
|
|
35
|
+
|
|
36
|
+
# TODO: fix
|
|
37
|
+
Style/StringConcatenation:
|
|
38
|
+
Enabled: false
|
|
39
|
+
|
|
40
|
+
Style/NumericPredicate:
|
|
41
|
+
EnforcedStyle: comparison
|
|
42
|
+
|
|
43
|
+
Style/IfUnlessModifier:
|
|
44
|
+
Enabled: false
|
|
45
|
+
|
|
46
|
+
Style/ClassAndModuleChildren:
|
|
47
|
+
Enabled: false
|
|
48
|
+
|
|
49
|
+
# TODO: enable ... somehow breaks tests
|
|
50
|
+
Style/HashEachMethods:
|
|
51
|
+
Enabled: false
|
|
52
|
+
|
|
53
|
+
Style/WordArray:
|
|
54
|
+
EnforcedStyle: brackets
|
|
55
|
+
|
|
56
|
+
Style/SymbolArray:
|
|
57
|
+
EnforcedStyle: brackets
|
data/Gemfile
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
source 'https://rubygems.org'
|
|
2
4
|
|
|
3
|
-
gem 'codeclimate-test-reporter', '<1.0.0', :
|
|
4
|
-
gem 'rubocop'
|
|
5
|
+
gem 'codeclimate-test-reporter', '<1.0.0', group: :test, require: nil
|
|
6
|
+
gem 'rubocop'
|
|
5
7
|
|
|
6
8
|
# Specify your gem's dependencies in fluent-plugin-add.gemspec
|
|
7
9
|
gemspec
|
data/Gemfile.lock
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
fluent-plugin-kubernetes_metadata_filter (2.
|
|
4
|
+
fluent-plugin-kubernetes_metadata_filter (2.7.0)
|
|
5
5
|
fluentd (>= 0.14.0, < 1.13)
|
|
6
6
|
kubeclient (< 5)
|
|
7
7
|
lru_redux
|
|
@@ -17,7 +17,7 @@ GEM
|
|
|
17
17
|
codeclimate-test-reporter (0.6.0)
|
|
18
18
|
simplecov (>= 0.7.1, < 1.0.0)
|
|
19
19
|
concurrent-ruby (1.1.8)
|
|
20
|
-
cool.io (1.7.
|
|
20
|
+
cool.io (1.7.1)
|
|
21
21
|
copyright-header (1.0.22)
|
|
22
22
|
github-linguist
|
|
23
23
|
crack (0.4.5)
|
|
@@ -26,11 +26,11 @@ GEM
|
|
|
26
26
|
domain_name (0.5.20190701)
|
|
27
27
|
unf (>= 0.0.5, < 1.0.0)
|
|
28
28
|
escape_utils (1.2.1)
|
|
29
|
-
ffi (1.
|
|
29
|
+
ffi (1.15.0)
|
|
30
30
|
ffi-compiler (1.0.1)
|
|
31
31
|
ffi (>= 1.0.0)
|
|
32
32
|
rake
|
|
33
|
-
fluentd (1.12.
|
|
33
|
+
fluentd (1.12.3)
|
|
34
34
|
bundler
|
|
35
35
|
cool.io (>= 1.4.5, < 2.0.0)
|
|
36
36
|
http_parser.rb (>= 0.5.1, < 0.7.0)
|
|
@@ -40,6 +40,7 @@ GEM
|
|
|
40
40
|
strptime (>= 0.2.2, < 1.0.0)
|
|
41
41
|
tzinfo (>= 1.0, < 3.0)
|
|
42
42
|
tzinfo-data (~> 1.0)
|
|
43
|
+
webrick (>= 1.4.2, < 1.8.0)
|
|
43
44
|
yajl-ruby (~> 1.0)
|
|
44
45
|
github-linguist (7.12.2)
|
|
45
46
|
charlock_holmes (~> 0.7.7)
|
|
@@ -69,10 +70,10 @@ GEM
|
|
|
69
70
|
lru_redux (1.1.0)
|
|
70
71
|
mime-types (3.3.1)
|
|
71
72
|
mime-types-data (~> 3.2015)
|
|
72
|
-
mime-types-data (3.
|
|
73
|
+
mime-types-data (3.2021.0225)
|
|
73
74
|
mini_mime (1.0.2)
|
|
74
75
|
minitest (4.7.5)
|
|
75
|
-
msgpack (1.
|
|
76
|
+
msgpack (1.4.2)
|
|
76
77
|
multi_json (1.15.0)
|
|
77
78
|
netrc (0.11.0)
|
|
78
79
|
parallel (1.20.1)
|
|
@@ -89,7 +90,7 @@ GEM
|
|
|
89
90
|
http-cookie (>= 1.0.2, < 2.0)
|
|
90
91
|
mime-types (>= 1.16, < 4.0)
|
|
91
92
|
netrc (~> 0.8)
|
|
92
|
-
rexml (3.2.
|
|
93
|
+
rexml (3.2.5)
|
|
93
94
|
rr (1.2.1)
|
|
94
95
|
rubocop (1.8.1)
|
|
95
96
|
parallel (~> 1.10)
|
|
@@ -104,7 +105,7 @@ GEM
|
|
|
104
105
|
parser (>= 2.7.1.5)
|
|
105
106
|
ruby-progressbar (1.11.0)
|
|
106
107
|
rugged (1.1.0)
|
|
107
|
-
serverengine (2.2.
|
|
108
|
+
serverengine (2.2.3)
|
|
108
109
|
sigdump (~> 0.2.2)
|
|
109
110
|
sigdump (0.2.4)
|
|
110
111
|
simplecov (0.21.2)
|
|
@@ -121,7 +122,7 @@ GEM
|
|
|
121
122
|
test-unit (>= 2.5.2)
|
|
122
123
|
tzinfo (2.0.4)
|
|
123
124
|
concurrent-ruby (~> 1.0)
|
|
124
|
-
tzinfo-data (1.
|
|
125
|
+
tzinfo-data (1.2021.1)
|
|
125
126
|
tzinfo (>= 1.0.0)
|
|
126
127
|
unf (0.1.4)
|
|
127
128
|
unf_ext
|
|
@@ -132,6 +133,7 @@ GEM
|
|
|
132
133
|
addressable (>= 2.3.6)
|
|
133
134
|
crack (>= 0.3.2)
|
|
134
135
|
hashdiff (>= 0.4.0, < 2.0.0)
|
|
136
|
+
webrick (1.7.0)
|
|
135
137
|
yajl-ruby (1.4.1)
|
|
136
138
|
|
|
137
139
|
PLATFORMS
|
data/README.md
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
[](https://circleci.com/gh/fabric8io/fluent-plugin-kubernetes_metadata_filter)
|
|
3
3
|
[](https://codeclimate.com/github/fabric8io/fluent-plugin-kubernetes_metadata_filter)
|
|
4
4
|
[](https://codeclimate.com/github/fabric8io/fluent-plugin-kubernetes_metadata_filter)
|
|
5
|
+
[](https://github.com/rubocop-hq/rubocop)
|
|
6
|
+
[](https://rubystyle.guide)
|
|
5
7
|
|
|
6
8
|
The Kubernetes metadata plugin filter enriches container log records with pod and namespace metadata.
|
|
7
9
|
|
|
@@ -39,7 +41,7 @@ Configuration options for fluent.conf are:
|
|
|
39
41
|
* `client_key` - path to a client key file to authenticate to the API server
|
|
40
42
|
* `bearer_token_file` - path to a file containing the bearer token to use for authentication
|
|
41
43
|
* `tag_to_kubernetes_name_regexp` - the regular expression used to extract kubernetes metadata (pod name, container name, namespace) from the current fluentd tag.
|
|
42
|
-
This must used named capture groups for `container_name`, `pod_name` & `namespace`
|
|
44
|
+
This must used named capture groups for `container_name`, `pod_name` & `namespace` default: See [code](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/blob/master/lib/fluent/plugin/filter_kubernetes_metadata.rb#L52)
|
|
43
45
|
* `cache_size` - size of the cache of Kubernetes metadata to reduce requests to the API server (default: `1000`)
|
|
44
46
|
* `cache_ttl` - TTL in seconds of each cached element. Set to negative value to disable TTL eviction (default: `3600` - 1 hour)
|
|
45
47
|
* `watch` - set up a watch on pods on the API server for updates to metadata (default: `true`)
|
|
@@ -47,7 +49,7 @@ This must used named capture groups for `container_name`, `pod_name` & `namespac
|
|
|
47
49
|
* `de_dot_separator` - separator to use if `de_dot` is enabled (default: `_`)
|
|
48
50
|
* *DEPRECATED* `use_journal` - If false, messages are expected to be formatted and tagged as if read by the fluentd in\_tail plugin with wildcard filename. If true, messages are expected to be formatted as if read from the systemd journal. The `MESSAGE` field has the full message. The `CONTAINER_NAME` field has the encoded k8s metadata (see below). The `CONTAINER_ID_FULL` field has the full container uuid. This requires docker to use the `--log-driver=journald` log driver. If unset (the default), the plugin will use the `CONTAINER_NAME` and `CONTAINER_ID_FULL` fields
|
|
49
51
|
if available, otherwise, will use the tag in the `tag_to_kubernetes_name_regexp` format.
|
|
50
|
-
* `container_name_to_kubernetes_regexp` - The regular expression used to extract the k8s metadata encoded in the journal `CONTAINER_NAME` field
|
|
52
|
+
* `container_name_to_kubernetes_regexp` - The regular expression used to extract the k8s metadata encoded in the journal `CONTAINER_NAME` field default: See [code](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/blob/master/lib/fluent/plugin/filter_kubernetes_metadata.rb#L68)
|
|
51
53
|
* This corresponds to the definition [in the source](https://github.com/kubernetes/kubernetes/blob/release-1.6/pkg/kubelet/dockertools/docker.go#L317)
|
|
52
54
|
* `annotation_match` - Array of regular expressions matching annotation field names. Matched annotations are added to a log record.
|
|
53
55
|
* `allow_orphans` - Modify the namespace and namespace id to the values of `orphaned_namespace_name` and `orphaned_namespace_id`
|
data/Rakefile
CHANGED
|
@@ -1,10 +1,15 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'bundler/setup'
|
|
1
4
|
require 'bundler/gem_tasks'
|
|
2
5
|
require 'rake/testtask'
|
|
3
6
|
require 'bump/tasks'
|
|
7
|
+
require 'rubocop/rake_task'
|
|
4
8
|
|
|
5
|
-
task :
|
|
9
|
+
task test: [:base_test]
|
|
10
|
+
task default: [:test, :build, :rubocop]
|
|
6
11
|
|
|
7
|
-
|
|
12
|
+
RuboCop::RakeTask.new
|
|
8
13
|
|
|
9
14
|
desc 'Run test_unit based test'
|
|
10
15
|
Rake::TestTask.new(:base_test) do |t|
|
|
@@ -13,7 +18,6 @@ Rake::TestTask.new(:base_test) do |t|
|
|
|
13
18
|
# $ bundle exec rake base_test TEST=test/test_*.rb
|
|
14
19
|
t.libs << 'test'
|
|
15
20
|
t.test_files = Dir['test/**/test_*.rb'].sort
|
|
16
|
-
#t.verbose = true
|
|
17
21
|
t.warning = false
|
|
18
22
|
end
|
|
19
23
|
|
|
@@ -23,15 +27,15 @@ task :headers do
|
|
|
23
27
|
require 'copyright_header'
|
|
24
28
|
|
|
25
29
|
args = {
|
|
26
|
-
:
|
|
27
|
-
:
|
|
28
|
-
:
|
|
29
|
-
:
|
|
30
|
-
:
|
|
31
|
-
:
|
|
32
|
-
:
|
|
30
|
+
license: 'Apache-2.0',
|
|
31
|
+
copyright_software: 'Fluentd Kubernetes Metadata Filter Plugin',
|
|
32
|
+
copyright_software_description: 'Enrich Fluentd events with Kubernetes metadata',
|
|
33
|
+
copyright_holders: ['Red Hat, Inc.'],
|
|
34
|
+
copyright_years: ['2015-2021'],
|
|
35
|
+
add_path: 'lib:test',
|
|
36
|
+
output_dir: '.'
|
|
33
37
|
}
|
|
34
38
|
|
|
35
|
-
command_line = CopyrightHeader::CommandLine.new(
|
|
39
|
+
command_line = CopyrightHeader::CommandLine.new(args)
|
|
36
40
|
command_line.execute
|
|
37
41
|
end
|
|
@@ -1,33 +1,34 @@
|
|
|
1
|
-
#
|
|
2
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
lib = File.expand_path('lib', __dir__)
|
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
5
|
|
|
5
6
|
Gem::Specification.new do |gem|
|
|
6
|
-
gem.name =
|
|
7
|
-
gem.version =
|
|
8
|
-
gem.authors = [
|
|
9
|
-
gem.email = [
|
|
10
|
-
gem.description =
|
|
11
|
-
gem.summary =
|
|
12
|
-
gem.homepage =
|
|
13
|
-
gem.license =
|
|
7
|
+
gem.name = 'fluent-plugin-kubernetes_metadata_filter'
|
|
8
|
+
gem.version = '2.7.0'
|
|
9
|
+
gem.authors = ['Jimmi Dyson']
|
|
10
|
+
gem.email = ['jimmidyson@gmail.com']
|
|
11
|
+
gem.description = 'Filter plugin to add Kubernetes metadata'
|
|
12
|
+
gem.summary = 'Fluentd filter plugin to add Kubernetes metadata'
|
|
13
|
+
gem.homepage = 'https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter'
|
|
14
|
+
gem.license = 'Apache-2.0'
|
|
14
15
|
|
|
15
16
|
gem.files = `git ls-files`.split($/)
|
|
16
17
|
|
|
17
18
|
gem.required_ruby_version = '>= 2.5.0'
|
|
18
19
|
|
|
19
20
|
gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 1.13']
|
|
20
|
-
gem.add_runtime_dependency
|
|
21
|
-
gem.add_runtime_dependency
|
|
21
|
+
gem.add_runtime_dependency 'kubeclient', '< 5'
|
|
22
|
+
gem.add_runtime_dependency 'lru_redux'
|
|
22
23
|
|
|
23
|
-
gem.add_development_dependency
|
|
24
|
-
gem.add_development_dependency
|
|
25
|
-
gem.add_development_dependency
|
|
26
|
-
gem.add_development_dependency
|
|
27
|
-
gem.add_development_dependency
|
|
28
|
-
gem.add_development_dependency
|
|
29
|
-
gem.add_development_dependency
|
|
30
|
-
gem.add_development_dependency
|
|
31
|
-
gem.add_development_dependency
|
|
32
|
-
gem.add_development_dependency
|
|
24
|
+
gem.add_development_dependency 'bump'
|
|
25
|
+
gem.add_development_dependency 'bundler', '~> 2.0'
|
|
26
|
+
gem.add_development_dependency 'copyright-header'
|
|
27
|
+
gem.add_development_dependency 'minitest', '~> 4.0'
|
|
28
|
+
gem.add_development_dependency 'rake'
|
|
29
|
+
gem.add_development_dependency 'test-unit', '~> 3.0.2'
|
|
30
|
+
gem.add_development_dependency 'test-unit-rr', '~> 1.0.3'
|
|
31
|
+
gem.add_development_dependency 'vcr'
|
|
32
|
+
gem.add_development_dependency 'webmock'
|
|
33
|
+
gem.add_development_dependency 'yajl-ruby'
|
|
33
34
|
end
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
#
|
|
2
4
|
# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
|
|
3
5
|
# Kubernetes metadata
|
|
@@ -20,6 +22,7 @@
|
|
|
20
22
|
require_relative 'kubernetes_metadata_cache_strategy'
|
|
21
23
|
require_relative 'kubernetes_metadata_common'
|
|
22
24
|
require_relative 'kubernetes_metadata_stats'
|
|
25
|
+
require_relative 'kubernetes_metadata_util'
|
|
23
26
|
require_relative 'kubernetes_metadata_watch_namespaces'
|
|
24
27
|
require_relative 'kubernetes_metadata_watch_pods'
|
|
25
28
|
|
|
@@ -33,6 +36,7 @@ module Fluent::Plugin
|
|
|
33
36
|
|
|
34
37
|
include KubernetesMetadata::CacheStrategy
|
|
35
38
|
include KubernetesMetadata::Common
|
|
39
|
+
include KubernetesMetadata::Util
|
|
36
40
|
include KubernetesMetadata::WatchNamespaces
|
|
37
41
|
include KubernetesMetadata::WatchPods
|
|
38
42
|
|
|
@@ -49,7 +53,7 @@ module Fluent::Plugin
|
|
|
49
53
|
config_param :verify_ssl, :bool, default: true
|
|
50
54
|
config_param :tag_to_kubernetes_name_regexp,
|
|
51
55
|
:string,
|
|
52
|
-
:
|
|
56
|
+
default: 'var\.log\.containers\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
|
|
53
57
|
config_param :bearer_token_file, :string, default: nil
|
|
54
58
|
config_param :secret_dir, :string, default: '/var/run/secrets/kubernetes.io/serviceaccount'
|
|
55
59
|
config_param :de_dot, :bool, default: true
|
|
@@ -65,7 +69,7 @@ module Fluent::Plugin
|
|
|
65
69
|
# parse format is defined here: https://github.com/kubernetes/kubernetes/blob/release-1.6/pkg/kubelet/dockertools/docker.go#L317
|
|
66
70
|
config_param :container_name_to_kubernetes_regexp,
|
|
67
71
|
:string,
|
|
68
|
-
:
|
|
72
|
+
default: '^(?<name_prefix>[^_]+)_(?<container_name>[^\._]+)(\.(?<container_hash>[^_]+))?_(?<pod_name>[^_]+)_(?<namespace>[^_]+)_[^_]+_[^_]+$'
|
|
69
73
|
|
|
70
74
|
config_param :annotation_match, :array, default: []
|
|
71
75
|
config_param :stats_interval, :integer, default: 30
|
|
@@ -81,6 +85,11 @@ module Fluent::Plugin
|
|
|
81
85
|
config_param :skip_container_metadata, :bool, default: false
|
|
82
86
|
config_param :skip_master_url, :bool, default: false
|
|
83
87
|
config_param :skip_namespace_metadata, :bool, default: false
|
|
88
|
+
|
|
89
|
+
# A classname in the form of Test::APIAdapter which will try
|
|
90
|
+
# to be resolved from a relative named file 'test_api_adapter'
|
|
91
|
+
config_param :test_api_adapter, :string, default: nil
|
|
92
|
+
|
|
84
93
|
# The time interval in seconds for retry backoffs when watch connections fail.
|
|
85
94
|
config_param :watch_retry_interval, :integer, default: 1
|
|
86
95
|
# The base number of exponential backoff for retries.
|
|
@@ -90,13 +99,16 @@ module Fluent::Plugin
|
|
|
90
99
|
|
|
91
100
|
def fetch_pod_metadata(namespace_name, pod_name)
|
|
92
101
|
log.trace("fetching pod metadata: #{namespace_name}/#{pod_name}") if log.trace?
|
|
93
|
-
|
|
102
|
+
options = {
|
|
103
|
+
resource_version: '0' # Fetch from API server cache instead of etcd quorum read
|
|
104
|
+
}
|
|
105
|
+
pod_object = @client.get_pod(pod_name, namespace_name, options)
|
|
94
106
|
log.trace("raw metadata for #{namespace_name}/#{pod_name}: #{pod_object}") if log.trace?
|
|
95
107
|
metadata = parse_pod_metadata(pod_object)
|
|
96
108
|
@stats.bump(:pod_cache_api_updates)
|
|
97
109
|
log.trace("parsed metadata for #{namespace_name}/#{pod_name}: #{metadata}") if log.trace?
|
|
98
110
|
@cache[metadata['pod_id']] = metadata
|
|
99
|
-
rescue => e
|
|
111
|
+
rescue StandardError => e
|
|
100
112
|
@stats.bump(:pod_cache_api_nil_error)
|
|
101
113
|
log.debug "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
|
|
102
114
|
{}
|
|
@@ -105,6 +117,7 @@ module Fluent::Plugin
|
|
|
105
117
|
def dump_stats
|
|
106
118
|
@curr_time = Time.now
|
|
107
119
|
return if @curr_time.to_i - @prev_time.to_i < @stats_interval
|
|
120
|
+
|
|
108
121
|
@prev_time = @curr_time
|
|
109
122
|
@stats.set(:pod_cache_size, @cache.count)
|
|
110
123
|
@stats.set(:namespace_cache_size, @namespace_cache.count) if @namespace_cache
|
|
@@ -118,15 +131,18 @@ module Fluent::Plugin
|
|
|
118
131
|
|
|
119
132
|
def fetch_namespace_metadata(namespace_name)
|
|
120
133
|
log.trace("fetching namespace metadata: #{namespace_name}") if log.trace?
|
|
121
|
-
|
|
134
|
+
options = {
|
|
135
|
+
resource_version: '0' # Fetch from API server cache instead of etcd quorum read
|
|
136
|
+
}
|
|
137
|
+
namespace_object = @client.get_namespace(namespace_name, nil, options)
|
|
122
138
|
log.trace("raw metadata for #{namespace_name}: #{namespace_object}") if log.trace?
|
|
123
139
|
metadata = parse_namespace_metadata(namespace_object)
|
|
124
140
|
@stats.bump(:namespace_cache_api_updates)
|
|
125
141
|
log.trace("parsed metadata for #{namespace_name}: #{metadata}") if log.trace?
|
|
126
142
|
@namespace_cache[metadata['namespace_id']] = metadata
|
|
127
|
-
rescue =>
|
|
143
|
+
rescue StandardError => e
|
|
128
144
|
@stats.bump(:namespace_cache_api_nil_error)
|
|
129
|
-
log.debug "Exception '#{
|
|
145
|
+
log.debug "Exception '#{e}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
|
|
130
146
|
{}
|
|
131
147
|
end
|
|
132
148
|
|
|
@@ -146,12 +162,12 @@ module Fluent::Plugin
|
|
|
146
162
|
require 'lru_redux'
|
|
147
163
|
@stats = KubernetesMetadata::Stats.new
|
|
148
164
|
|
|
149
|
-
if @de_dot && @de_dot_separator.include?(
|
|
165
|
+
if @de_dot && @de_dot_separator.include?('.')
|
|
150
166
|
raise Fluent::ConfigError, "Invalid de_dot_separator: cannot be or contain '.'"
|
|
151
167
|
end
|
|
152
168
|
|
|
153
169
|
if @cache_ttl < 0
|
|
154
|
-
log.info
|
|
170
|
+
log.info 'Setting the cache TTL to :none because it was <= 0'
|
|
155
171
|
@cache_ttl = :none
|
|
156
172
|
end
|
|
157
173
|
|
|
@@ -169,7 +185,7 @@ module Fluent::Plugin
|
|
|
169
185
|
|
|
170
186
|
# Use Kubernetes default service account if we're in a pod.
|
|
171
187
|
if @kubernetes_url.nil?
|
|
172
|
-
log.debug
|
|
188
|
+
log.debug 'Kubernetes URL is not set - inspecting environ'
|
|
173
189
|
|
|
174
190
|
env_host = ENV['KUBERNETES_SERVICE_HOST']
|
|
175
191
|
env_port = ENV['KUBERNETES_SERVICE_PORT']
|
|
@@ -181,7 +197,7 @@ module Fluent::Plugin
|
|
|
181
197
|
@kubernetes_url = "https://#{env_host}:#{env_port}/api"
|
|
182
198
|
log.debug "Kubernetes URL is now '#{@kubernetes_url}'"
|
|
183
199
|
else
|
|
184
|
-
log.debug
|
|
200
|
+
log.debug 'No Kubernetes URL could be found in config or environ'
|
|
185
201
|
end
|
|
186
202
|
end
|
|
187
203
|
|
|
@@ -191,12 +207,12 @@ module Fluent::Plugin
|
|
|
191
207
|
ca_cert = File.join(@secret_dir, K8_POD_CA_CERT)
|
|
192
208
|
pod_token = File.join(@secret_dir, K8_POD_TOKEN)
|
|
193
209
|
|
|
194
|
-
if !present?(@ca_file)
|
|
210
|
+
if !present?(@ca_file) && File.exist?(ca_cert)
|
|
195
211
|
log.debug "Found CA certificate: #{ca_cert}"
|
|
196
212
|
@ca_file = ca_cert
|
|
197
213
|
end
|
|
198
214
|
|
|
199
|
-
if !present?(@bearer_token_file)
|
|
215
|
+
if !present?(@bearer_token_file) && File.exist?(pod_token)
|
|
200
216
|
log.debug "Found pod token: #{pod_token}"
|
|
201
217
|
@bearer_token_file = pod_token
|
|
202
218
|
end
|
|
@@ -204,10 +220,10 @@ module Fluent::Plugin
|
|
|
204
220
|
|
|
205
221
|
if present?(@kubernetes_url)
|
|
206
222
|
ssl_options = {
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
223
|
+
client_cert: present?(@client_cert) ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil,
|
|
224
|
+
client_key: present?(@client_key) ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
|
|
225
|
+
ca_file: @ca_file,
|
|
226
|
+
verify_ssl: @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
|
|
211
227
|
}
|
|
212
228
|
|
|
213
229
|
if @ssl_partial_chain
|
|
@@ -215,12 +231,12 @@ module Fluent::Plugin
|
|
|
215
231
|
require 'openssl'
|
|
216
232
|
ssl_store = OpenSSL::X509::Store.new
|
|
217
233
|
ssl_store.set_default_paths
|
|
218
|
-
if defined? OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
234
|
+
flagval = if defined? OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
|
|
235
|
+
OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
|
|
236
|
+
else
|
|
237
|
+
# this version of ruby does not define OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
|
|
238
|
+
0x80000
|
|
239
|
+
end
|
|
224
240
|
ssl_store.flags = OpenSSL::X509::V_FLAG_CRL_CHECK_ALL | flagval
|
|
225
241
|
ssl_options[:cert_store] = ssl_store
|
|
226
242
|
end
|
|
@@ -232,7 +248,7 @@ module Fluent::Plugin
|
|
|
232
248
|
auth_options[:bearer_token] = bearer_token
|
|
233
249
|
end
|
|
234
250
|
|
|
235
|
-
log.debug
|
|
251
|
+
log.debug 'Creating K8S client'
|
|
236
252
|
@client = Kubeclient::Client.new(
|
|
237
253
|
@kubernetes_url,
|
|
238
254
|
@apiVersion,
|
|
@@ -241,17 +257,27 @@ module Fluent::Plugin
|
|
|
241
257
|
as: :parsed_symbolized
|
|
242
258
|
)
|
|
243
259
|
|
|
260
|
+
if @test_api_adapter
|
|
261
|
+
log.info "Extending client with test api adaper #{@test_api_adapter}"
|
|
262
|
+
require_relative @test_api_adapter.underscore
|
|
263
|
+
@client.extend(eval(@test_api_adapter))
|
|
264
|
+
end
|
|
265
|
+
|
|
244
266
|
begin
|
|
245
267
|
@client.api_valid?
|
|
246
|
-
rescue KubeException =>
|
|
247
|
-
raise Fluent::ConfigError, "Invalid Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}: #{
|
|
268
|
+
rescue KubeException => e
|
|
269
|
+
raise Fluent::ConfigError, "Invalid Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}: #{e.message}"
|
|
248
270
|
end
|
|
249
271
|
|
|
250
272
|
if @watch
|
|
251
|
-
|
|
273
|
+
if ENV['K8S_NODE_NAME'].nil? || ENV['K8S_NODE_NAME'].strip.empty?
|
|
274
|
+
log.warn("!! The environment variable 'K8S_NODE_NAME' is not set to the node name which can affect the API server and watch efficiency !!")
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
pod_thread = Thread.new(self, &:set_up_pod_thread)
|
|
252
278
|
pod_thread.abort_on_exception = true
|
|
253
279
|
|
|
254
|
-
namespace_thread = Thread.new(self
|
|
280
|
+
namespace_thread = Thread.new(self, &:set_up_namespace_thread)
|
|
255
281
|
namespace_thread.abort_on_exception = true
|
|
256
282
|
end
|
|
257
283
|
end
|
|
@@ -262,22 +288,19 @@ module Fluent::Plugin
|
|
|
262
288
|
|
|
263
289
|
@annotations_regexps = []
|
|
264
290
|
@annotation_match.each do |regexp|
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
log.error "Error: invalid regular expression in annotation_match: #{e}"
|
|
269
|
-
end
|
|
291
|
+
@annotations_regexps << Regexp.compile(regexp)
|
|
292
|
+
rescue RegexpError => e
|
|
293
|
+
log.error "Error: invalid regular expression in annotation_match: #{e}"
|
|
270
294
|
end
|
|
271
|
-
|
|
272
295
|
end
|
|
273
296
|
|
|
274
297
|
def get_metadata_for_record(namespace_name, pod_name, container_name, container_id, create_time, batch_miss_cache)
|
|
275
298
|
metadata = {
|
|
276
|
-
'docker' => {'container_id' => container_id},
|
|
299
|
+
'docker' => { 'container_id' => container_id },
|
|
277
300
|
'kubernetes' => {
|
|
278
|
-
'container_name'
|
|
279
|
-
'namespace_name'
|
|
280
|
-
'pod_name'
|
|
301
|
+
'container_name' => container_name,
|
|
302
|
+
'namespace_name' => namespace_name,
|
|
303
|
+
'pod_name' => pod_name
|
|
281
304
|
}
|
|
282
305
|
}
|
|
283
306
|
if present?(@kubernetes_url)
|
|
@@ -294,22 +317,9 @@ module Fluent::Plugin
|
|
|
294
317
|
metadata
|
|
295
318
|
end
|
|
296
319
|
|
|
297
|
-
def create_time_from_record(record, internal_time)
|
|
298
|
-
time_key = @time_fields.detect{ |ii| record.has_key?(ii) }
|
|
299
|
-
time = record[time_key]
|
|
300
|
-
if time.nil? || time.chop.empty?
|
|
301
|
-
# `internal_time` is a Fluent::EventTime, it can't compare with Time.
|
|
302
|
-
return Time.at(internal_time.to_f)
|
|
303
|
-
end
|
|
304
|
-
if ['_SOURCE_REALTIME_TIMESTAMP', '__REALTIME_TIMESTAMP'].include?(time_key)
|
|
305
|
-
timei= time.to_i
|
|
306
|
-
return Time.at(timei / 1000000, timei % 1000000)
|
|
307
|
-
end
|
|
308
|
-
return Time.parse(time)
|
|
309
|
-
end
|
|
310
|
-
|
|
311
320
|
def filter_stream(tag, es)
|
|
312
321
|
return es if (es.respond_to?(:empty?) && es.empty?) || !es.is_a?(Fluent::EventStream)
|
|
322
|
+
|
|
313
323
|
new_es = Fluent::MultiEventStream.new
|
|
314
324
|
tag_match_data = tag.match(@tag_to_kubernetes_name_regexp_compiled) unless @use_journal
|
|
315
325
|
tag_metadata = nil
|
|
@@ -317,23 +327,23 @@ module Fluent::Plugin
|
|
|
317
327
|
es.each do |time, record|
|
|
318
328
|
if tag_match_data && tag_metadata.nil?
|
|
319
329
|
tag_metadata = get_metadata_for_record(tag_match_data['namespace'], tag_match_data['pod_name'], tag_match_data['container_name'],
|
|
320
|
-
|
|
330
|
+
tag_match_data['docker_id'], create_time_from_record(record, time), batch_miss_cache)
|
|
321
331
|
end
|
|
322
332
|
metadata = Marshal.load(Marshal.dump(tag_metadata)) if tag_metadata
|
|
323
333
|
if (@use_journal || @use_journal.nil?) &&
|
|
324
|
-
|
|
334
|
+
(j_metadata = get_metadata_for_journal_record(record, time, batch_miss_cache))
|
|
325
335
|
metadata = j_metadata
|
|
326
336
|
end
|
|
327
|
-
if @lookup_from_k8s_field && record.
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
+
if @lookup_from_k8s_field && record.key?('kubernetes') && record.key?('docker') &&
|
|
338
|
+
record['kubernetes'].respond_to?(:has_key?) && record['docker'].respond_to?(:has_key?) &&
|
|
339
|
+
record['kubernetes'].key?('namespace_name') &&
|
|
340
|
+
record['kubernetes'].key?('pod_name') &&
|
|
341
|
+
record['kubernetes'].key?('container_name') &&
|
|
342
|
+
record['docker'].key?('container_id') &&
|
|
343
|
+
(k_metadata = get_metadata_for_record(record['kubernetes']['namespace_name'], record['kubernetes']['pod_name'],
|
|
344
|
+
record['kubernetes']['container_name'], record['docker']['container_id'],
|
|
345
|
+
create_time_from_record(record, time), batch_miss_cache))
|
|
346
|
+
metadata = k_metadata
|
|
337
347
|
end
|
|
338
348
|
|
|
339
349
|
record = record.merge(metadata) if metadata
|
|
@@ -345,16 +355,16 @@ module Fluent::Plugin
|
|
|
345
355
|
|
|
346
356
|
def get_metadata_for_journal_record(record, time, batch_miss_cache)
|
|
347
357
|
metadata = nil
|
|
348
|
-
if record.
|
|
358
|
+
if record.key?('CONTAINER_NAME') && record.key?('CONTAINER_ID_FULL')
|
|
349
359
|
metadata = record['CONTAINER_NAME'].match(@container_name_to_kubernetes_regexp_compiled) do |match_data|
|
|
350
360
|
get_metadata_for_record(match_data['namespace'], match_data['pod_name'], match_data['container_name'],
|
|
351
|
-
|
|
361
|
+
record['CONTAINER_ID_FULL'], create_time_from_record(record, time), batch_miss_cache)
|
|
352
362
|
end
|
|
353
363
|
unless metadata
|
|
354
364
|
log.debug "Error: could not match CONTAINER_NAME from record #{record}"
|
|
355
365
|
@stats.bump(:container_name_match_failed)
|
|
356
366
|
end
|
|
357
|
-
elsif record.
|
|
367
|
+
elsif record.key?('CONTAINER_NAME') && record['CONTAINER_NAME'].start_with?('k8s_')
|
|
358
368
|
log.debug "Error: no container name and id in record #{record}"
|
|
359
369
|
@stats.bump(:container_name_id_missing)
|
|
360
370
|
end
|
|
@@ -363,11 +373,11 @@ module Fluent::Plugin
|
|
|
363
373
|
|
|
364
374
|
def de_dot!(h)
|
|
365
375
|
h.keys.each do |ref|
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
376
|
+
next unless h[ref] && ref =~ /\./
|
|
377
|
+
|
|
378
|
+
v = h.delete(ref)
|
|
379
|
+
newref = ref.to_s.gsub('.', @de_dot_separator)
|
|
380
|
+
h[newref] = v
|
|
371
381
|
end
|
|
372
382
|
end
|
|
373
383
|
|