fluent-plugin-kubernetes_metadata_filter 2.6.0 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +2 -2
- data/.rubocop.yml +57 -0
- data/Gemfile +4 -2
- data/Gemfile.lock +19 -17
- data/README.md +4 -2
- data/Rakefile +15 -11
- data/fluent-plugin-kubernetes_metadata_filter.gemspec +24 -23
- data/lib/fluent/plugin/filter_kubernetes_metadata.rb +82 -72
- data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb +22 -18
- data/lib/fluent/plugin/kubernetes_metadata_common.rb +30 -29
- data/lib/fluent/plugin/kubernetes_metadata_stats.rb +6 -6
- data/lib/fluent/plugin/kubernetes_metadata_test_api_adapter.rb +68 -0
- data/lib/fluent/plugin/kubernetes_metadata_util.rb +53 -0
- data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +65 -65
- data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +69 -70
- data/test/helper.rb +5 -3
- data/test/plugin/test_cache_stats.rb +10 -10
- data/test/plugin/test_cache_strategy.rb +158 -157
- data/test/plugin/test_filter_kubernetes_metadata.rb +363 -344
- data/test/plugin/test_utils.rb +56 -0
- data/test/plugin/test_watch_namespaces.rb +191 -190
- data/test/plugin/test_watch_pods.rb +278 -267
- data/test/plugin/watch_test.rb +13 -7
- metadata +47 -43
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 452a507fb0cbb14f8eb17fe488d07a8ccd9b9f97bef38b592e56a5bb8acbe3bb
|
|
4
|
+
data.tar.gz: ff1346d50b7b31ebfd7105523eec621baea0db0a771c193d48094b3c8818e502
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 3f9257176000157861bf21980b91e0594494118d4d5cb10eee475127567c4803865e3c1404c02c1e035c015c264d0a3351660ed031566c5d399f774088205f21
|
|
7
|
+
data.tar.gz: 439308c8a534c7e0c5cdcf70f8ccd26eeb4ab455abacff1d411f86ae57e4da7b6b422c42f772d84508360a3fec60ed415c454633fecde854b4c288d24cd6d39b
|
data/.circleci/config.yml
CHANGED
|
@@ -12,12 +12,12 @@ missingdeps: &missingdeps
|
|
|
12
12
|
cat /etc/os-release
|
|
13
13
|
printf "deb http://deb.debian.org/debian buster main\ndeb http://security.debian.org buster/updates main\ndeb-src http://security.debian.org buster/updates main" > /tmp/sources.list
|
|
14
14
|
sudo cp /tmp/sources.list /etc/apt/sources.list
|
|
15
|
-
sudo apt-get update
|
|
15
|
+
sudo apt-get --allow-releaseinfo-change update
|
|
16
16
|
sudo apt-get install cmake libicu-dev libssl-dev
|
|
17
17
|
|
|
18
18
|
test: &test
|
|
19
19
|
name: Test bundle
|
|
20
|
-
command: bundle exec rake test
|
|
20
|
+
command: bundle exec rake test --trace
|
|
21
21
|
|
|
22
22
|
executors:
|
|
23
23
|
ruby-2-5:
|
data/.rubocop.yml
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
AllCops:
|
|
2
|
+
TargetRubyVersion: 2.5 # keep in sync with .circleci/config.yml and gemspec
|
|
3
|
+
NewCops: enable
|
|
4
|
+
|
|
5
|
+
Style/EmptyMethod:
|
|
6
|
+
Enabled: false
|
|
7
|
+
|
|
8
|
+
Metrics:
|
|
9
|
+
Enabled: false
|
|
10
|
+
|
|
11
|
+
# not safe ... needs require 'English'
|
|
12
|
+
Style/SpecialGlobalVars:
|
|
13
|
+
Enabled: false
|
|
14
|
+
|
|
15
|
+
Layout/LineLength:
|
|
16
|
+
Max: 205 # TODO: lower
|
|
17
|
+
|
|
18
|
+
Style/Documentation:
|
|
19
|
+
Enabled: false
|
|
20
|
+
|
|
21
|
+
Naming/AccessorMethodName:
|
|
22
|
+
Enabled: false
|
|
23
|
+
|
|
24
|
+
Naming/MethodParameterName:
|
|
25
|
+
Enabled: false
|
|
26
|
+
|
|
27
|
+
Style/IfInsideElse:
|
|
28
|
+
Enabled: false
|
|
29
|
+
|
|
30
|
+
Style/GuardClause:
|
|
31
|
+
Enabled: false
|
|
32
|
+
|
|
33
|
+
Lint/NestedMethodDefinition:
|
|
34
|
+
Enabled: false
|
|
35
|
+
|
|
36
|
+
# TODO: fix
|
|
37
|
+
Style/StringConcatenation:
|
|
38
|
+
Enabled: false
|
|
39
|
+
|
|
40
|
+
Style/NumericPredicate:
|
|
41
|
+
EnforcedStyle: comparison
|
|
42
|
+
|
|
43
|
+
Style/IfUnlessModifier:
|
|
44
|
+
Enabled: false
|
|
45
|
+
|
|
46
|
+
Style/ClassAndModuleChildren:
|
|
47
|
+
Enabled: false
|
|
48
|
+
|
|
49
|
+
# TODO: enable ... somehow breaks tests
|
|
50
|
+
Style/HashEachMethods:
|
|
51
|
+
Enabled: false
|
|
52
|
+
|
|
53
|
+
Style/WordArray:
|
|
54
|
+
EnforcedStyle: brackets
|
|
55
|
+
|
|
56
|
+
Style/SymbolArray:
|
|
57
|
+
EnforcedStyle: brackets
|
data/Gemfile
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
source 'https://rubygems.org'
|
|
2
4
|
|
|
3
|
-
gem 'codeclimate-test-reporter', '<1.0.0', :
|
|
4
|
-
gem 'rubocop'
|
|
5
|
+
gem 'codeclimate-test-reporter', '<1.0.0', group: :test, require: nil
|
|
6
|
+
gem 'rubocop'
|
|
5
7
|
|
|
6
8
|
# Specify your gem's dependencies in fluent-plugin-add.gemspec
|
|
7
9
|
gemspec
|
data/Gemfile.lock
CHANGED
|
@@ -1,23 +1,23 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
fluent-plugin-kubernetes_metadata_filter (2.
|
|
5
|
-
fluentd (>= 0.14.0, < 1.
|
|
4
|
+
fluent-plugin-kubernetes_metadata_filter (2.8.0)
|
|
5
|
+
fluentd (>= 0.14.0, < 1.15)
|
|
6
6
|
kubeclient (< 5)
|
|
7
7
|
lru_redux
|
|
8
8
|
|
|
9
9
|
GEM
|
|
10
10
|
remote: https://rubygems.org/
|
|
11
11
|
specs:
|
|
12
|
-
addressable (2.
|
|
12
|
+
addressable (2.8.0)
|
|
13
13
|
public_suffix (>= 2.0.2, < 5.0)
|
|
14
14
|
ast (2.4.1)
|
|
15
15
|
bump (0.10.0)
|
|
16
16
|
charlock_holmes (0.7.7)
|
|
17
17
|
codeclimate-test-reporter (0.6.0)
|
|
18
18
|
simplecov (>= 0.7.1, < 1.0.0)
|
|
19
|
-
concurrent-ruby (1.1.
|
|
20
|
-
cool.io (1.7.
|
|
19
|
+
concurrent-ruby (1.1.9)
|
|
20
|
+
cool.io (1.7.1)
|
|
21
21
|
copyright-header (1.0.22)
|
|
22
22
|
github-linguist
|
|
23
23
|
crack (0.4.5)
|
|
@@ -26,20 +26,21 @@ GEM
|
|
|
26
26
|
domain_name (0.5.20190701)
|
|
27
27
|
unf (>= 0.0.5, < 1.0.0)
|
|
28
28
|
escape_utils (1.2.1)
|
|
29
|
-
ffi (1.
|
|
29
|
+
ffi (1.15.3)
|
|
30
30
|
ffi-compiler (1.0.1)
|
|
31
31
|
ffi (>= 1.0.0)
|
|
32
32
|
rake
|
|
33
|
-
fluentd (1.
|
|
33
|
+
fluentd (1.14.0)
|
|
34
34
|
bundler
|
|
35
35
|
cool.io (>= 1.4.5, < 2.0.0)
|
|
36
|
-
http_parser.rb (>= 0.5.1, < 0.
|
|
36
|
+
http_parser.rb (>= 0.5.1, < 0.8.0)
|
|
37
37
|
msgpack (>= 1.3.1, < 2.0.0)
|
|
38
38
|
serverengine (>= 2.2.2, < 3.0.0)
|
|
39
39
|
sigdump (~> 0.2.2)
|
|
40
40
|
strptime (>= 0.2.2, < 1.0.0)
|
|
41
41
|
tzinfo (>= 1.0, < 3.0)
|
|
42
42
|
tzinfo-data (~> 1.0)
|
|
43
|
+
webrick (>= 1.4.2, < 1.8.0)
|
|
43
44
|
yajl-ruby (~> 1.0)
|
|
44
45
|
github-linguist (7.12.2)
|
|
45
46
|
charlock_holmes (~> 0.7.7)
|
|
@@ -53,15 +54,15 @@ GEM
|
|
|
53
54
|
http-form_data (~> 2.2)
|
|
54
55
|
http-parser (~> 1.2.0)
|
|
55
56
|
http-accept (1.7.0)
|
|
56
|
-
http-cookie (1.0.
|
|
57
|
+
http-cookie (1.0.4)
|
|
57
58
|
domain_name (~> 0.5)
|
|
58
59
|
http-form_data (2.3.0)
|
|
59
60
|
http-parser (1.2.3)
|
|
60
61
|
ffi-compiler (>= 1.0, < 2.0)
|
|
61
|
-
http_parser.rb (0.
|
|
62
|
+
http_parser.rb (0.7.0)
|
|
62
63
|
jsonpath (1.1.0)
|
|
63
64
|
multi_json
|
|
64
|
-
kubeclient (4.9.
|
|
65
|
+
kubeclient (4.9.2)
|
|
65
66
|
http (>= 3.0, < 5.0)
|
|
66
67
|
jsonpath (~> 1.0)
|
|
67
68
|
recursive-open-struct (~> 1.1, >= 1.1.1)
|
|
@@ -69,10 +70,10 @@ GEM
|
|
|
69
70
|
lru_redux (1.1.0)
|
|
70
71
|
mime-types (3.3.1)
|
|
71
72
|
mime-types-data (~> 3.2015)
|
|
72
|
-
mime-types-data (3.
|
|
73
|
+
mime-types-data (3.2021.0704)
|
|
73
74
|
mini_mime (1.0.2)
|
|
74
75
|
minitest (4.7.5)
|
|
75
|
-
msgpack (1.
|
|
76
|
+
msgpack (1.4.2)
|
|
76
77
|
multi_json (1.15.0)
|
|
77
78
|
netrc (0.11.0)
|
|
78
79
|
parallel (1.20.1)
|
|
@@ -89,7 +90,7 @@ GEM
|
|
|
89
90
|
http-cookie (>= 1.0.2, < 2.0)
|
|
90
91
|
mime-types (>= 1.16, < 4.0)
|
|
91
92
|
netrc (~> 0.8)
|
|
92
|
-
rexml (3.2.
|
|
93
|
+
rexml (3.2.5)
|
|
93
94
|
rr (1.2.1)
|
|
94
95
|
rubocop (1.8.1)
|
|
95
96
|
parallel (~> 1.10)
|
|
@@ -104,7 +105,7 @@ GEM
|
|
|
104
105
|
parser (>= 2.7.1.5)
|
|
105
106
|
ruby-progressbar (1.11.0)
|
|
106
107
|
rugged (1.1.0)
|
|
107
|
-
serverengine (2.2.
|
|
108
|
+
serverengine (2.2.4)
|
|
108
109
|
sigdump (~> 0.2.2)
|
|
109
110
|
sigdump (0.2.4)
|
|
110
111
|
simplecov (0.21.2)
|
|
@@ -121,7 +122,7 @@ GEM
|
|
|
121
122
|
test-unit (>= 2.5.2)
|
|
122
123
|
tzinfo (2.0.4)
|
|
123
124
|
concurrent-ruby (~> 1.0)
|
|
124
|
-
tzinfo-data (1.
|
|
125
|
+
tzinfo-data (1.2021.1)
|
|
125
126
|
tzinfo (>= 1.0.0)
|
|
126
127
|
unf (0.1.4)
|
|
127
128
|
unf_ext
|
|
@@ -132,6 +133,7 @@ GEM
|
|
|
132
133
|
addressable (>= 2.3.6)
|
|
133
134
|
crack (>= 0.3.2)
|
|
134
135
|
hashdiff (>= 0.4.0, < 2.0.0)
|
|
136
|
+
webrick (1.7.0)
|
|
135
137
|
yajl-ruby (1.4.1)
|
|
136
138
|
|
|
137
139
|
PLATFORMS
|
|
@@ -153,4 +155,4 @@ DEPENDENCIES
|
|
|
153
155
|
yajl-ruby
|
|
154
156
|
|
|
155
157
|
BUNDLED WITH
|
|
156
|
-
2.
|
|
158
|
+
2.2.19
|
data/README.md
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
[](https://circleci.com/gh/fabric8io/fluent-plugin-kubernetes_metadata_filter)
|
|
3
3
|
[](https://codeclimate.com/github/fabric8io/fluent-plugin-kubernetes_metadata_filter)
|
|
4
4
|
[](https://codeclimate.com/github/fabric8io/fluent-plugin-kubernetes_metadata_filter)
|
|
5
|
+
[](https://github.com/rubocop-hq/rubocop)
|
|
6
|
+
[](https://rubystyle.guide)
|
|
5
7
|
|
|
6
8
|
The Kubernetes metadata plugin filter enriches container log records with pod and namespace metadata.
|
|
7
9
|
|
|
@@ -39,7 +41,7 @@ Configuration options for fluent.conf are:
|
|
|
39
41
|
* `client_key` - path to a client key file to authenticate to the API server
|
|
40
42
|
* `bearer_token_file` - path to a file containing the bearer token to use for authentication
|
|
41
43
|
* `tag_to_kubernetes_name_regexp` - the regular expression used to extract kubernetes metadata (pod name, container name, namespace) from the current fluentd tag.
|
|
42
|
-
This must used named capture groups for `container_name`, `pod_name` & `namespace`
|
|
44
|
+
This must used named capture groups for `container_name`, `pod_name` & `namespace` default: See [code](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/blob/master/lib/fluent/plugin/filter_kubernetes_metadata.rb#L52)
|
|
43
45
|
* `cache_size` - size of the cache of Kubernetes metadata to reduce requests to the API server (default: `1000`)
|
|
44
46
|
* `cache_ttl` - TTL in seconds of each cached element. Set to negative value to disable TTL eviction (default: `3600` - 1 hour)
|
|
45
47
|
* `watch` - set up a watch on pods on the API server for updates to metadata (default: `true`)
|
|
@@ -47,7 +49,7 @@ This must used named capture groups for `container_name`, `pod_name` & `namespac
|
|
|
47
49
|
* `de_dot_separator` - separator to use if `de_dot` is enabled (default: `_`)
|
|
48
50
|
* *DEPRECATED* `use_journal` - If false, messages are expected to be formatted and tagged as if read by the fluentd in\_tail plugin with wildcard filename. If true, messages are expected to be formatted as if read from the systemd journal. The `MESSAGE` field has the full message. The `CONTAINER_NAME` field has the encoded k8s metadata (see below). The `CONTAINER_ID_FULL` field has the full container uuid. This requires docker to use the `--log-driver=journald` log driver. If unset (the default), the plugin will use the `CONTAINER_NAME` and `CONTAINER_ID_FULL` fields
|
|
49
51
|
if available, otherwise, will use the tag in the `tag_to_kubernetes_name_regexp` format.
|
|
50
|
-
* `container_name_to_kubernetes_regexp` - The regular expression used to extract the k8s metadata encoded in the journal `CONTAINER_NAME` field
|
|
52
|
+
* `container_name_to_kubernetes_regexp` - The regular expression used to extract the k8s metadata encoded in the journal `CONTAINER_NAME` field default: See [code](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/blob/master/lib/fluent/plugin/filter_kubernetes_metadata.rb#L68)
|
|
51
53
|
* This corresponds to the definition [in the source](https://github.com/kubernetes/kubernetes/blob/release-1.6/pkg/kubelet/dockertools/docker.go#L317)
|
|
52
54
|
* `annotation_match` - Array of regular expressions matching annotation field names. Matched annotations are added to a log record.
|
|
53
55
|
* `allow_orphans` - Modify the namespace and namespace id to the values of `orphaned_namespace_name` and `orphaned_namespace_id`
|
data/Rakefile
CHANGED
|
@@ -1,10 +1,15 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'bundler/setup'
|
|
1
4
|
require 'bundler/gem_tasks'
|
|
2
5
|
require 'rake/testtask'
|
|
3
6
|
require 'bump/tasks'
|
|
7
|
+
require 'rubocop/rake_task'
|
|
4
8
|
|
|
5
|
-
task :
|
|
9
|
+
task test: [:base_test]
|
|
10
|
+
task default: [:test, :build, :rubocop]
|
|
6
11
|
|
|
7
|
-
|
|
12
|
+
RuboCop::RakeTask.new
|
|
8
13
|
|
|
9
14
|
desc 'Run test_unit based test'
|
|
10
15
|
Rake::TestTask.new(:base_test) do |t|
|
|
@@ -13,7 +18,6 @@ Rake::TestTask.new(:base_test) do |t|
|
|
|
13
18
|
# $ bundle exec rake base_test TEST=test/test_*.rb
|
|
14
19
|
t.libs << 'test'
|
|
15
20
|
t.test_files = Dir['test/**/test_*.rb'].sort
|
|
16
|
-
#t.verbose = true
|
|
17
21
|
t.warning = false
|
|
18
22
|
end
|
|
19
23
|
|
|
@@ -23,15 +27,15 @@ task :headers do
|
|
|
23
27
|
require 'copyright_header'
|
|
24
28
|
|
|
25
29
|
args = {
|
|
26
|
-
:
|
|
27
|
-
:
|
|
28
|
-
:
|
|
29
|
-
:
|
|
30
|
-
:
|
|
31
|
-
:
|
|
32
|
-
:
|
|
30
|
+
license: 'Apache-2.0',
|
|
31
|
+
copyright_software: 'Fluentd Kubernetes Metadata Filter Plugin',
|
|
32
|
+
copyright_software_description: 'Enrich Fluentd events with Kubernetes metadata',
|
|
33
|
+
copyright_holders: ['Red Hat, Inc.'],
|
|
34
|
+
copyright_years: ['2015-2021'],
|
|
35
|
+
add_path: 'lib:test',
|
|
36
|
+
output_dir: '.'
|
|
33
37
|
}
|
|
34
38
|
|
|
35
|
-
command_line = CopyrightHeader::CommandLine.new(
|
|
39
|
+
command_line = CopyrightHeader::CommandLine.new(args)
|
|
36
40
|
command_line.execute
|
|
37
41
|
end
|
|
@@ -1,33 +1,34 @@
|
|
|
1
|
-
#
|
|
2
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
lib = File.expand_path('lib', __dir__)
|
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
5
|
|
|
5
6
|
Gem::Specification.new do |gem|
|
|
6
|
-
gem.name =
|
|
7
|
-
gem.version =
|
|
8
|
-
gem.authors = [
|
|
9
|
-
gem.email = [
|
|
10
|
-
gem.description =
|
|
11
|
-
gem.summary =
|
|
12
|
-
gem.homepage =
|
|
13
|
-
gem.license =
|
|
7
|
+
gem.name = 'fluent-plugin-kubernetes_metadata_filter'
|
|
8
|
+
gem.version = '2.8.0'
|
|
9
|
+
gem.authors = ['Jimmi Dyson']
|
|
10
|
+
gem.email = ['jimmidyson@gmail.com']
|
|
11
|
+
gem.description = 'Filter plugin to add Kubernetes metadata'
|
|
12
|
+
gem.summary = 'Fluentd filter plugin to add Kubernetes metadata'
|
|
13
|
+
gem.homepage = 'https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter'
|
|
14
|
+
gem.license = 'Apache-2.0'
|
|
14
15
|
|
|
15
16
|
gem.files = `git ls-files`.split($/)
|
|
16
17
|
|
|
17
18
|
gem.required_ruby_version = '>= 2.5.0'
|
|
18
19
|
|
|
19
|
-
gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 1.
|
|
20
|
-
gem.add_runtime_dependency
|
|
21
|
-
gem.add_runtime_dependency
|
|
20
|
+
gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 1.15']
|
|
21
|
+
gem.add_runtime_dependency 'kubeclient', '< 5'
|
|
22
|
+
gem.add_runtime_dependency 'lru_redux'
|
|
22
23
|
|
|
23
|
-
gem.add_development_dependency
|
|
24
|
-
gem.add_development_dependency
|
|
25
|
-
gem.add_development_dependency
|
|
26
|
-
gem.add_development_dependency
|
|
27
|
-
gem.add_development_dependency
|
|
28
|
-
gem.add_development_dependency
|
|
29
|
-
gem.add_development_dependency
|
|
30
|
-
gem.add_development_dependency
|
|
31
|
-
gem.add_development_dependency
|
|
32
|
-
gem.add_development_dependency
|
|
24
|
+
gem.add_development_dependency 'bump'
|
|
25
|
+
gem.add_development_dependency 'bundler', '~> 2.0'
|
|
26
|
+
gem.add_development_dependency 'copyright-header'
|
|
27
|
+
gem.add_development_dependency 'minitest', '~> 4.0'
|
|
28
|
+
gem.add_development_dependency 'rake'
|
|
29
|
+
gem.add_development_dependency 'test-unit', '~> 3.0.2'
|
|
30
|
+
gem.add_development_dependency 'test-unit-rr', '~> 1.0.3'
|
|
31
|
+
gem.add_development_dependency 'vcr'
|
|
32
|
+
gem.add_development_dependency 'webmock'
|
|
33
|
+
gem.add_development_dependency 'yajl-ruby'
|
|
33
34
|
end
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
#
|
|
2
4
|
# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
|
|
3
5
|
# Kubernetes metadata
|
|
@@ -20,6 +22,7 @@
|
|
|
20
22
|
require_relative 'kubernetes_metadata_cache_strategy'
|
|
21
23
|
require_relative 'kubernetes_metadata_common'
|
|
22
24
|
require_relative 'kubernetes_metadata_stats'
|
|
25
|
+
require_relative 'kubernetes_metadata_util'
|
|
23
26
|
require_relative 'kubernetes_metadata_watch_namespaces'
|
|
24
27
|
require_relative 'kubernetes_metadata_watch_pods'
|
|
25
28
|
|
|
@@ -33,6 +36,7 @@ module Fluent::Plugin
|
|
|
33
36
|
|
|
34
37
|
include KubernetesMetadata::CacheStrategy
|
|
35
38
|
include KubernetesMetadata::Common
|
|
39
|
+
include KubernetesMetadata::Util
|
|
36
40
|
include KubernetesMetadata::WatchNamespaces
|
|
37
41
|
include KubernetesMetadata::WatchPods
|
|
38
42
|
|
|
@@ -49,7 +53,7 @@ module Fluent::Plugin
|
|
|
49
53
|
config_param :verify_ssl, :bool, default: true
|
|
50
54
|
config_param :tag_to_kubernetes_name_regexp,
|
|
51
55
|
:string,
|
|
52
|
-
:
|
|
56
|
+
default: 'var\.log\.containers\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
|
|
53
57
|
config_param :bearer_token_file, :string, default: nil
|
|
54
58
|
config_param :secret_dir, :string, default: '/var/run/secrets/kubernetes.io/serviceaccount'
|
|
55
59
|
config_param :de_dot, :bool, default: true
|
|
@@ -65,7 +69,7 @@ module Fluent::Plugin
|
|
|
65
69
|
# parse format is defined here: https://github.com/kubernetes/kubernetes/blob/release-1.6/pkg/kubelet/dockertools/docker.go#L317
|
|
66
70
|
config_param :container_name_to_kubernetes_regexp,
|
|
67
71
|
:string,
|
|
68
|
-
:
|
|
72
|
+
default: '^(?<name_prefix>[^_]+)_(?<container_name>[^\._]+)(\.(?<container_hash>[^_]+))?_(?<pod_name>[^_]+)_(?<namespace>[^_]+)_[^_]+_[^_]+$'
|
|
69
73
|
|
|
70
74
|
config_param :annotation_match, :array, default: []
|
|
71
75
|
config_param :stats_interval, :integer, default: 30
|
|
@@ -81,6 +85,11 @@ module Fluent::Plugin
|
|
|
81
85
|
config_param :skip_container_metadata, :bool, default: false
|
|
82
86
|
config_param :skip_master_url, :bool, default: false
|
|
83
87
|
config_param :skip_namespace_metadata, :bool, default: false
|
|
88
|
+
|
|
89
|
+
# A classname in the form of Test::APIAdapter which will try
|
|
90
|
+
# to be resolved from a relative named file 'test_api_adapter'
|
|
91
|
+
config_param :test_api_adapter, :string, default: nil
|
|
92
|
+
|
|
84
93
|
# The time interval in seconds for retry backoffs when watch connections fail.
|
|
85
94
|
config_param :watch_retry_interval, :integer, default: 1
|
|
86
95
|
# The base number of exponential backoff for retries.
|
|
@@ -90,13 +99,16 @@ module Fluent::Plugin
|
|
|
90
99
|
|
|
91
100
|
def fetch_pod_metadata(namespace_name, pod_name)
|
|
92
101
|
log.trace("fetching pod metadata: #{namespace_name}/#{pod_name}") if log.trace?
|
|
93
|
-
|
|
102
|
+
options = {
|
|
103
|
+
resource_version: '0' # Fetch from API server cache instead of etcd quorum read
|
|
104
|
+
}
|
|
105
|
+
pod_object = @client.get_pod(pod_name, namespace_name, options)
|
|
94
106
|
log.trace("raw metadata for #{namespace_name}/#{pod_name}: #{pod_object}") if log.trace?
|
|
95
107
|
metadata = parse_pod_metadata(pod_object)
|
|
96
108
|
@stats.bump(:pod_cache_api_updates)
|
|
97
109
|
log.trace("parsed metadata for #{namespace_name}/#{pod_name}: #{metadata}") if log.trace?
|
|
98
110
|
@cache[metadata['pod_id']] = metadata
|
|
99
|
-
rescue => e
|
|
111
|
+
rescue StandardError => e
|
|
100
112
|
@stats.bump(:pod_cache_api_nil_error)
|
|
101
113
|
log.debug "Exception '#{e}' encountered fetching pod metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
|
|
102
114
|
{}
|
|
@@ -105,6 +117,7 @@ module Fluent::Plugin
|
|
|
105
117
|
def dump_stats
|
|
106
118
|
@curr_time = Time.now
|
|
107
119
|
return if @curr_time.to_i - @prev_time.to_i < @stats_interval
|
|
120
|
+
|
|
108
121
|
@prev_time = @curr_time
|
|
109
122
|
@stats.set(:pod_cache_size, @cache.count)
|
|
110
123
|
@stats.set(:namespace_cache_size, @namespace_cache.count) if @namespace_cache
|
|
@@ -118,15 +131,18 @@ module Fluent::Plugin
|
|
|
118
131
|
|
|
119
132
|
def fetch_namespace_metadata(namespace_name)
|
|
120
133
|
log.trace("fetching namespace metadata: #{namespace_name}") if log.trace?
|
|
121
|
-
|
|
134
|
+
options = {
|
|
135
|
+
resource_version: '0' # Fetch from API server cache instead of etcd quorum read
|
|
136
|
+
}
|
|
137
|
+
namespace_object = @client.get_namespace(namespace_name, nil, options)
|
|
122
138
|
log.trace("raw metadata for #{namespace_name}: #{namespace_object}") if log.trace?
|
|
123
139
|
metadata = parse_namespace_metadata(namespace_object)
|
|
124
140
|
@stats.bump(:namespace_cache_api_updates)
|
|
125
141
|
log.trace("parsed metadata for #{namespace_name}: #{metadata}") if log.trace?
|
|
126
142
|
@namespace_cache[metadata['namespace_id']] = metadata
|
|
127
|
-
rescue =>
|
|
143
|
+
rescue StandardError => e
|
|
128
144
|
@stats.bump(:namespace_cache_api_nil_error)
|
|
129
|
-
log.debug "Exception '#{
|
|
145
|
+
log.debug "Exception '#{e}' encountered fetching namespace metadata from Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}"
|
|
130
146
|
{}
|
|
131
147
|
end
|
|
132
148
|
|
|
@@ -146,12 +162,12 @@ module Fluent::Plugin
|
|
|
146
162
|
require 'lru_redux'
|
|
147
163
|
@stats = KubernetesMetadata::Stats.new
|
|
148
164
|
|
|
149
|
-
if @de_dot && @de_dot_separator.include?(
|
|
165
|
+
if @de_dot && @de_dot_separator.include?('.')
|
|
150
166
|
raise Fluent::ConfigError, "Invalid de_dot_separator: cannot be or contain '.'"
|
|
151
167
|
end
|
|
152
168
|
|
|
153
169
|
if @cache_ttl < 0
|
|
154
|
-
log.info
|
|
170
|
+
log.info 'Setting the cache TTL to :none because it was <= 0'
|
|
155
171
|
@cache_ttl = :none
|
|
156
172
|
end
|
|
157
173
|
|
|
@@ -169,7 +185,7 @@ module Fluent::Plugin
|
|
|
169
185
|
|
|
170
186
|
# Use Kubernetes default service account if we're in a pod.
|
|
171
187
|
if @kubernetes_url.nil?
|
|
172
|
-
log.debug
|
|
188
|
+
log.debug 'Kubernetes URL is not set - inspecting environ'
|
|
173
189
|
|
|
174
190
|
env_host = ENV['KUBERNETES_SERVICE_HOST']
|
|
175
191
|
env_port = ENV['KUBERNETES_SERVICE_PORT']
|
|
@@ -181,7 +197,7 @@ module Fluent::Plugin
|
|
|
181
197
|
@kubernetes_url = "https://#{env_host}:#{env_port}/api"
|
|
182
198
|
log.debug "Kubernetes URL is now '#{@kubernetes_url}'"
|
|
183
199
|
else
|
|
184
|
-
log.debug
|
|
200
|
+
log.debug 'No Kubernetes URL could be found in config or environ'
|
|
185
201
|
end
|
|
186
202
|
end
|
|
187
203
|
|
|
@@ -191,12 +207,12 @@ module Fluent::Plugin
|
|
|
191
207
|
ca_cert = File.join(@secret_dir, K8_POD_CA_CERT)
|
|
192
208
|
pod_token = File.join(@secret_dir, K8_POD_TOKEN)
|
|
193
209
|
|
|
194
|
-
if !present?(@ca_file)
|
|
210
|
+
if !present?(@ca_file) && File.exist?(ca_cert)
|
|
195
211
|
log.debug "Found CA certificate: #{ca_cert}"
|
|
196
212
|
@ca_file = ca_cert
|
|
197
213
|
end
|
|
198
214
|
|
|
199
|
-
if !present?(@bearer_token_file)
|
|
215
|
+
if !present?(@bearer_token_file) && File.exist?(pod_token)
|
|
200
216
|
log.debug "Found pod token: #{pod_token}"
|
|
201
217
|
@bearer_token_file = pod_token
|
|
202
218
|
end
|
|
@@ -204,10 +220,10 @@ module Fluent::Plugin
|
|
|
204
220
|
|
|
205
221
|
if present?(@kubernetes_url)
|
|
206
222
|
ssl_options = {
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
223
|
+
client_cert: present?(@client_cert) ? OpenSSL::X509::Certificate.new(File.read(@client_cert)) : nil,
|
|
224
|
+
client_key: present?(@client_key) ? OpenSSL::PKey::RSA.new(File.read(@client_key)) : nil,
|
|
225
|
+
ca_file: @ca_file,
|
|
226
|
+
verify_ssl: @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
|
|
211
227
|
}
|
|
212
228
|
|
|
213
229
|
if @ssl_partial_chain
|
|
@@ -215,12 +231,12 @@ module Fluent::Plugin
|
|
|
215
231
|
require 'openssl'
|
|
216
232
|
ssl_store = OpenSSL::X509::Store.new
|
|
217
233
|
ssl_store.set_default_paths
|
|
218
|
-
if defined? OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
234
|
+
flagval = if defined? OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
|
|
235
|
+
OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
|
|
236
|
+
else
|
|
237
|
+
# this version of ruby does not define OpenSSL::X509::V_FLAG_PARTIAL_CHAIN
|
|
238
|
+
0x80000
|
|
239
|
+
end
|
|
224
240
|
ssl_store.flags = OpenSSL::X509::V_FLAG_CRL_CHECK_ALL | flagval
|
|
225
241
|
ssl_options[:cert_store] = ssl_store
|
|
226
242
|
end
|
|
@@ -232,7 +248,7 @@ module Fluent::Plugin
|
|
|
232
248
|
auth_options[:bearer_token] = bearer_token
|
|
233
249
|
end
|
|
234
250
|
|
|
235
|
-
log.debug
|
|
251
|
+
log.debug 'Creating K8S client'
|
|
236
252
|
@client = Kubeclient::Client.new(
|
|
237
253
|
@kubernetes_url,
|
|
238
254
|
@apiVersion,
|
|
@@ -241,17 +257,27 @@ module Fluent::Plugin
|
|
|
241
257
|
as: :parsed_symbolized
|
|
242
258
|
)
|
|
243
259
|
|
|
260
|
+
if @test_api_adapter
|
|
261
|
+
log.info "Extending client with test api adaper #{@test_api_adapter}"
|
|
262
|
+
require_relative @test_api_adapter.underscore
|
|
263
|
+
@client.extend(eval(@test_api_adapter))
|
|
264
|
+
end
|
|
265
|
+
|
|
244
266
|
begin
|
|
245
267
|
@client.api_valid?
|
|
246
|
-
rescue KubeException =>
|
|
247
|
-
raise Fluent::ConfigError, "Invalid Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}: #{
|
|
268
|
+
rescue KubeException => e
|
|
269
|
+
raise Fluent::ConfigError, "Invalid Kubernetes API #{@apiVersion} endpoint #{@kubernetes_url}: #{e.message}"
|
|
248
270
|
end
|
|
249
271
|
|
|
250
272
|
if @watch
|
|
251
|
-
|
|
273
|
+
if ENV['K8S_NODE_NAME'].nil? || ENV['K8S_NODE_NAME'].strip.empty?
|
|
274
|
+
log.warn("!! The environment variable 'K8S_NODE_NAME' is not set to the node name which can affect the API server and watch efficiency !!")
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
pod_thread = Thread.new(self, &:set_up_pod_thread)
|
|
252
278
|
pod_thread.abort_on_exception = true
|
|
253
279
|
|
|
254
|
-
namespace_thread = Thread.new(self
|
|
280
|
+
namespace_thread = Thread.new(self, &:set_up_namespace_thread)
|
|
255
281
|
namespace_thread.abort_on_exception = true
|
|
256
282
|
end
|
|
257
283
|
end
|
|
@@ -262,22 +288,19 @@ module Fluent::Plugin
|
|
|
262
288
|
|
|
263
289
|
@annotations_regexps = []
|
|
264
290
|
@annotation_match.each do |regexp|
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
log.error "Error: invalid regular expression in annotation_match: #{e}"
|
|
269
|
-
end
|
|
291
|
+
@annotations_regexps << Regexp.compile(regexp)
|
|
292
|
+
rescue RegexpError => e
|
|
293
|
+
log.error "Error: invalid regular expression in annotation_match: #{e}"
|
|
270
294
|
end
|
|
271
|
-
|
|
272
295
|
end
|
|
273
296
|
|
|
274
297
|
def get_metadata_for_record(namespace_name, pod_name, container_name, container_id, create_time, batch_miss_cache)
|
|
275
298
|
metadata = {
|
|
276
|
-
'docker' => {'container_id' => container_id},
|
|
299
|
+
'docker' => { 'container_id' => container_id },
|
|
277
300
|
'kubernetes' => {
|
|
278
|
-
'container_name'
|
|
279
|
-
'namespace_name'
|
|
280
|
-
'pod_name'
|
|
301
|
+
'container_name' => container_name,
|
|
302
|
+
'namespace_name' => namespace_name,
|
|
303
|
+
'pod_name' => pod_name
|
|
281
304
|
}
|
|
282
305
|
}
|
|
283
306
|
if present?(@kubernetes_url)
|
|
@@ -294,22 +317,9 @@ module Fluent::Plugin
|
|
|
294
317
|
metadata
|
|
295
318
|
end
|
|
296
319
|
|
|
297
|
-
def create_time_from_record(record, internal_time)
|
|
298
|
-
time_key = @time_fields.detect{ |ii| record.has_key?(ii) }
|
|
299
|
-
time = record[time_key]
|
|
300
|
-
if time.nil? || time.chop.empty?
|
|
301
|
-
# `internal_time` is a Fluent::EventTime, it can't compare with Time.
|
|
302
|
-
return Time.at(internal_time.to_f)
|
|
303
|
-
end
|
|
304
|
-
if ['_SOURCE_REALTIME_TIMESTAMP', '__REALTIME_TIMESTAMP'].include?(time_key)
|
|
305
|
-
timei= time.to_i
|
|
306
|
-
return Time.at(timei / 1000000, timei % 1000000)
|
|
307
|
-
end
|
|
308
|
-
return Time.parse(time)
|
|
309
|
-
end
|
|
310
|
-
|
|
311
320
|
def filter_stream(tag, es)
|
|
312
321
|
return es if (es.respond_to?(:empty?) && es.empty?) || !es.is_a?(Fluent::EventStream)
|
|
322
|
+
|
|
313
323
|
new_es = Fluent::MultiEventStream.new
|
|
314
324
|
tag_match_data = tag.match(@tag_to_kubernetes_name_regexp_compiled) unless @use_journal
|
|
315
325
|
tag_metadata = nil
|
|
@@ -317,23 +327,23 @@ module Fluent::Plugin
|
|
|
317
327
|
es.each do |time, record|
|
|
318
328
|
if tag_match_data && tag_metadata.nil?
|
|
319
329
|
tag_metadata = get_metadata_for_record(tag_match_data['namespace'], tag_match_data['pod_name'], tag_match_data['container_name'],
|
|
320
|
-
|
|
330
|
+
tag_match_data['docker_id'], create_time_from_record(record, time), batch_miss_cache)
|
|
321
331
|
end
|
|
322
332
|
metadata = Marshal.load(Marshal.dump(tag_metadata)) if tag_metadata
|
|
323
333
|
if (@use_journal || @use_journal.nil?) &&
|
|
324
|
-
|
|
334
|
+
(j_metadata = get_metadata_for_journal_record(record, time, batch_miss_cache))
|
|
325
335
|
metadata = j_metadata
|
|
326
336
|
end
|
|
327
|
-
if @lookup_from_k8s_field && record.
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
+
if @lookup_from_k8s_field && record.key?('kubernetes') && record.key?('docker') &&
|
|
338
|
+
record['kubernetes'].respond_to?(:has_key?) && record['docker'].respond_to?(:has_key?) &&
|
|
339
|
+
record['kubernetes'].key?('namespace_name') &&
|
|
340
|
+
record['kubernetes'].key?('pod_name') &&
|
|
341
|
+
record['kubernetes'].key?('container_name') &&
|
|
342
|
+
record['docker'].key?('container_id') &&
|
|
343
|
+
(k_metadata = get_metadata_for_record(record['kubernetes']['namespace_name'], record['kubernetes']['pod_name'],
|
|
344
|
+
record['kubernetes']['container_name'], record['docker']['container_id'],
|
|
345
|
+
create_time_from_record(record, time), batch_miss_cache))
|
|
346
|
+
metadata = k_metadata
|
|
337
347
|
end
|
|
338
348
|
|
|
339
349
|
record = record.merge(metadata) if metadata
|
|
@@ -345,16 +355,16 @@ module Fluent::Plugin
|
|
|
345
355
|
|
|
346
356
|
def get_metadata_for_journal_record(record, time, batch_miss_cache)
|
|
347
357
|
metadata = nil
|
|
348
|
-
if record.
|
|
358
|
+
if record.key?('CONTAINER_NAME') && record.key?('CONTAINER_ID_FULL')
|
|
349
359
|
metadata = record['CONTAINER_NAME'].match(@container_name_to_kubernetes_regexp_compiled) do |match_data|
|
|
350
360
|
get_metadata_for_record(match_data['namespace'], match_data['pod_name'], match_data['container_name'],
|
|
351
|
-
|
|
361
|
+
record['CONTAINER_ID_FULL'], create_time_from_record(record, time), batch_miss_cache)
|
|
352
362
|
end
|
|
353
363
|
unless metadata
|
|
354
364
|
log.debug "Error: could not match CONTAINER_NAME from record #{record}"
|
|
355
365
|
@stats.bump(:container_name_match_failed)
|
|
356
366
|
end
|
|
357
|
-
elsif record.
|
|
367
|
+
elsif record.key?('CONTAINER_NAME') && record['CONTAINER_NAME'].start_with?('k8s_')
|
|
358
368
|
log.debug "Error: no container name and id in record #{record}"
|
|
359
369
|
@stats.bump(:container_name_id_missing)
|
|
360
370
|
end
|
|
@@ -363,11 +373,11 @@ module Fluent::Plugin
|
|
|
363
373
|
|
|
364
374
|
def de_dot!(h)
|
|
365
375
|
h.keys.each do |ref|
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
376
|
+
next unless h[ref] && ref =~ /\./
|
|
377
|
+
|
|
378
|
+
v = h.delete(ref)
|
|
379
|
+
newref = ref.to_s.gsub('.', @de_dot_separator)
|
|
380
|
+
h[newref] = v
|
|
371
381
|
end
|
|
372
382
|
end
|
|
373
383
|
|