fluent-plugin-kubernetes_metadata_filter 2.1.4 → 2.9.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +57 -0
- data/.gitignore +0 -1
- data/.rubocop.yml +57 -0
- data/Gemfile +4 -2
- data/Gemfile.lock +158 -0
- data/README.md +48 -28
- data/Rakefile +15 -11
- data/fluent-plugin-kubernetes_metadata_filter.gemspec +25 -28
- data/lib/fluent/plugin/filter_kubernetes_metadata.rb +185 -131
- data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb +27 -20
- data/lib/fluent/plugin/kubernetes_metadata_common.rb +59 -33
- data/lib/fluent/plugin/kubernetes_metadata_stats.rb +6 -6
- data/lib/fluent/plugin/kubernetes_metadata_test_api_adapter.rb +68 -0
- data/lib/fluent/plugin/kubernetes_metadata_util.rb +53 -0
- data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +121 -27
- data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +138 -29
- data/release_notes.md +42 -0
- data/test/cassettes/kubernetes_docker_metadata_annotations.yml +0 -34
- data/test/cassettes/{kubernetes_docker_metadata_dotted_labels.yml → kubernetes_docker_metadata_dotted_slashed_labels.yml} +0 -34
- data/test/cassettes/kubernetes_get_api_v1.yml +193 -0
- data/test/cassettes/kubernetes_get_api_v1_using_token.yml +195 -0
- data/test/cassettes/kubernetes_get_namespace_default.yml +69 -0
- data/test/cassettes/kubernetes_get_namespace_default_using_token.yml +71 -0
- data/test/cassettes/{kubernetes_docker_metadata.yml → kubernetes_get_pod.yml} +0 -82
- data/test/cassettes/{metadata_with_namespace_id.yml → kubernetes_get_pod_container_init.yml} +3 -134
- data/test/cassettes/{kubernetes_docker_metadata_using_bearer_token.yml → kubernetes_get_pod_using_token.yml} +5 -105
- data/test/cassettes/metadata_from_tag_and_journald_fields.yml +0 -255
- data/test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml +0 -255
- data/test/cassettes/{non_kubernetes_docker_metadata.yml → valid_kubernetes_api_server_using_token.yml} +4 -44
- data/test/helper.rb +20 -2
- data/test/plugin/test_cache_stats.rb +10 -13
- data/test/plugin/test_cache_strategy.rb +158 -160
- data/test/plugin/test_filter_kubernetes_metadata.rb +480 -320
- data/test/plugin/test_utils.rb +56 -0
- data/test/plugin/test_watch_namespaces.rb +209 -55
- data/test/plugin/test_watch_pods.rb +302 -103
- data/test/plugin/watch_test.rb +52 -33
- metadata +69 -72
- data/circle.yml +0 -17
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 98c4002df3754262c547e0b22acf366e62bca72f0142a20430f6a0f6c91dadea
|
|
4
|
+
data.tar.gz: 3f9e5d2d19b505c7cad15d113b69dea7ccb0943a3d4605d121fdf2234952c4a7
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 5474457d9a35d7951a8a380c0099deeb16af6d849a9f036c467ba47e0fcc47ce2ff7f77bbd75ad7d6487c9f172897e63406a91fd9f08dc6494eb32332e2b9373
|
|
7
|
+
data.tar.gz: 91a14c8edabcdd1eaa7b915c14bfd643b7829ae0cc8951ed8f9e77819483638207a5c6264094974971dcf4d7684c50c62a249cee4de36d23574b72cbe4e10589
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
version: 2.1
|
|
2
|
+
|
|
3
|
+
install: &install
|
|
4
|
+
name: Install bundle
|
|
5
|
+
command: |
|
|
6
|
+
gem install bundler
|
|
7
|
+
bundle install --path vendor/bundle
|
|
8
|
+
|
|
9
|
+
missingdeps: &missingdeps
|
|
10
|
+
name: Install missing dependecies
|
|
11
|
+
command: |
|
|
12
|
+
cat /etc/os-release
|
|
13
|
+
printf "deb http://deb.debian.org/debian buster main\ndeb http://security.debian.org buster/updates main\ndeb-src http://security.debian.org buster/updates main" > /tmp/sources.list
|
|
14
|
+
sudo cp /tmp/sources.list /etc/apt/sources.list
|
|
15
|
+
sudo apt-get --allow-releaseinfo-change update
|
|
16
|
+
sudo apt-get install cmake libicu-dev libssl-dev
|
|
17
|
+
|
|
18
|
+
test: &test
|
|
19
|
+
name: Test bundle
|
|
20
|
+
command: bundle exec rake test --trace
|
|
21
|
+
|
|
22
|
+
executors:
|
|
23
|
+
ruby-2-5:
|
|
24
|
+
docker:
|
|
25
|
+
- image: circleci/ruby:2.5.5
|
|
26
|
+
ruby-2-6:
|
|
27
|
+
docker:
|
|
28
|
+
- image: circleci/ruby:2.6.3
|
|
29
|
+
ruby-2-7:
|
|
30
|
+
docker:
|
|
31
|
+
- image: circleci/ruby:2.7.1
|
|
32
|
+
jobs:
|
|
33
|
+
"ruby-test":
|
|
34
|
+
parameters:
|
|
35
|
+
ruby-version:
|
|
36
|
+
type: executor
|
|
37
|
+
executor: << parameters.ruby-version >>
|
|
38
|
+
working_directory: ~/fluent-plugin-kubernetes_metadata_filter
|
|
39
|
+
steps:
|
|
40
|
+
- run: *missingdeps
|
|
41
|
+
- checkout
|
|
42
|
+
- run: *install
|
|
43
|
+
- run: *test
|
|
44
|
+
- store_test_results:
|
|
45
|
+
path: coverage
|
|
46
|
+
- store_artifacts:
|
|
47
|
+
path: coverage
|
|
48
|
+
|
|
49
|
+
workflows:
|
|
50
|
+
"test_multiple_ruby_versions":
|
|
51
|
+
jobs:
|
|
52
|
+
- ruby-test:
|
|
53
|
+
ruby-version: ruby-2-5
|
|
54
|
+
- ruby-test:
|
|
55
|
+
ruby-version: ruby-2-6
|
|
56
|
+
- ruby-test:
|
|
57
|
+
ruby-version: ruby-2-7
|
data/.gitignore
CHANGED
data/.rubocop.yml
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
AllCops:
|
|
2
|
+
TargetRubyVersion: 2.5 # keep in sync with .circleci/config.yml and gemspec
|
|
3
|
+
NewCops: enable
|
|
4
|
+
|
|
5
|
+
Style/EmptyMethod:
|
|
6
|
+
Enabled: false
|
|
7
|
+
|
|
8
|
+
Metrics:
|
|
9
|
+
Enabled: false
|
|
10
|
+
|
|
11
|
+
# not safe ... needs require 'English'
|
|
12
|
+
Style/SpecialGlobalVars:
|
|
13
|
+
Enabled: false
|
|
14
|
+
|
|
15
|
+
Layout/LineLength:
|
|
16
|
+
Max: 205 # TODO: lower
|
|
17
|
+
|
|
18
|
+
Style/Documentation:
|
|
19
|
+
Enabled: false
|
|
20
|
+
|
|
21
|
+
Naming/AccessorMethodName:
|
|
22
|
+
Enabled: false
|
|
23
|
+
|
|
24
|
+
Naming/MethodParameterName:
|
|
25
|
+
Enabled: false
|
|
26
|
+
|
|
27
|
+
Style/IfInsideElse:
|
|
28
|
+
Enabled: false
|
|
29
|
+
|
|
30
|
+
Style/GuardClause:
|
|
31
|
+
Enabled: false
|
|
32
|
+
|
|
33
|
+
Lint/NestedMethodDefinition:
|
|
34
|
+
Enabled: false
|
|
35
|
+
|
|
36
|
+
# TODO: fix
|
|
37
|
+
Style/StringConcatenation:
|
|
38
|
+
Enabled: false
|
|
39
|
+
|
|
40
|
+
Style/NumericPredicate:
|
|
41
|
+
EnforcedStyle: comparison
|
|
42
|
+
|
|
43
|
+
Style/IfUnlessModifier:
|
|
44
|
+
Enabled: false
|
|
45
|
+
|
|
46
|
+
Style/ClassAndModuleChildren:
|
|
47
|
+
Enabled: false
|
|
48
|
+
|
|
49
|
+
# TODO: enable ... somehow breaks tests
|
|
50
|
+
Style/HashEachMethods:
|
|
51
|
+
Enabled: false
|
|
52
|
+
|
|
53
|
+
Style/WordArray:
|
|
54
|
+
EnforcedStyle: brackets
|
|
55
|
+
|
|
56
|
+
Style/SymbolArray:
|
|
57
|
+
EnforcedStyle: brackets
|
data/Gemfile
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
source 'https://rubygems.org'
|
|
2
4
|
|
|
3
|
-
gem 'codeclimate-test-reporter', '<1.0.0', :
|
|
4
|
-
gem 'rubocop'
|
|
5
|
+
gem 'codeclimate-test-reporter', '<1.0.0', group: :test, require: nil
|
|
6
|
+
gem 'rubocop'
|
|
5
7
|
|
|
6
8
|
# Specify your gem's dependencies in fluent-plugin-add.gemspec
|
|
7
9
|
gemspec
|
data/Gemfile.lock
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
PATH
|
|
2
|
+
remote: .
|
|
3
|
+
specs:
|
|
4
|
+
fluent-plugin-kubernetes_metadata_filter (2.9.4)
|
|
5
|
+
fluentd (>= 0.14.0, < 1.15)
|
|
6
|
+
kubeclient (>= 4.0.0, < 5.0.0)
|
|
7
|
+
lru_redux
|
|
8
|
+
|
|
9
|
+
GEM
|
|
10
|
+
remote: https://rubygems.org/
|
|
11
|
+
specs:
|
|
12
|
+
addressable (2.8.0)
|
|
13
|
+
public_suffix (>= 2.0.2, < 5.0)
|
|
14
|
+
ast (2.4.2)
|
|
15
|
+
bump (0.10.0)
|
|
16
|
+
charlock_holmes (0.7.7)
|
|
17
|
+
codeclimate-test-reporter (0.6.0)
|
|
18
|
+
simplecov (>= 0.7.1, < 1.0.0)
|
|
19
|
+
concurrent-ruby (1.1.9)
|
|
20
|
+
cool.io (1.7.1)
|
|
21
|
+
copyright-header (1.0.22)
|
|
22
|
+
github-linguist
|
|
23
|
+
crack (0.4.5)
|
|
24
|
+
rexml
|
|
25
|
+
docile (1.4.0)
|
|
26
|
+
domain_name (0.5.20190701)
|
|
27
|
+
unf (>= 0.0.5, < 1.0.0)
|
|
28
|
+
escape_utils (1.2.1)
|
|
29
|
+
ffi (1.15.4)
|
|
30
|
+
ffi-compiler (1.0.1)
|
|
31
|
+
ffi (>= 1.0.0)
|
|
32
|
+
rake
|
|
33
|
+
fluentd (1.14.3)
|
|
34
|
+
bundler
|
|
35
|
+
cool.io (>= 1.4.5, < 2.0.0)
|
|
36
|
+
http_parser.rb (>= 0.5.1, < 0.9.0)
|
|
37
|
+
msgpack (>= 1.3.1, < 2.0.0)
|
|
38
|
+
serverengine (>= 2.2.2, < 3.0.0)
|
|
39
|
+
sigdump (~> 0.2.2)
|
|
40
|
+
strptime (>= 0.2.4, < 1.0.0)
|
|
41
|
+
tzinfo (>= 1.0, < 3.0)
|
|
42
|
+
tzinfo-data (~> 1.0)
|
|
43
|
+
webrick (>= 1.4.2, < 1.8.0)
|
|
44
|
+
yajl-ruby (~> 1.0)
|
|
45
|
+
github-linguist (7.17.0)
|
|
46
|
+
charlock_holmes (~> 0.7.7)
|
|
47
|
+
escape_utils (~> 1.2.0)
|
|
48
|
+
mini_mime (~> 1.0)
|
|
49
|
+
rugged (>= 0.25.1)
|
|
50
|
+
hashdiff (1.0.1)
|
|
51
|
+
http (4.4.1)
|
|
52
|
+
addressable (~> 2.3)
|
|
53
|
+
http-cookie (~> 1.0)
|
|
54
|
+
http-form_data (~> 2.2)
|
|
55
|
+
http-parser (~> 1.2.0)
|
|
56
|
+
http-accept (1.7.0)
|
|
57
|
+
http-cookie (1.0.4)
|
|
58
|
+
domain_name (~> 0.5)
|
|
59
|
+
http-form_data (2.3.0)
|
|
60
|
+
http-parser (1.2.3)
|
|
61
|
+
ffi-compiler (>= 1.0, < 2.0)
|
|
62
|
+
http_parser.rb (0.8.0)
|
|
63
|
+
jsonpath (1.1.0)
|
|
64
|
+
multi_json
|
|
65
|
+
kubeclient (4.9.2)
|
|
66
|
+
http (>= 3.0, < 5.0)
|
|
67
|
+
jsonpath (~> 1.0)
|
|
68
|
+
recursive-open-struct (~> 1.1, >= 1.1.1)
|
|
69
|
+
rest-client (~> 2.0)
|
|
70
|
+
lru_redux (1.1.0)
|
|
71
|
+
mime-types (3.4.1)
|
|
72
|
+
mime-types-data (~> 3.2015)
|
|
73
|
+
mime-types-data (3.2021.1115)
|
|
74
|
+
mini_mime (1.1.2)
|
|
75
|
+
minitest (4.7.5)
|
|
76
|
+
msgpack (1.4.2)
|
|
77
|
+
multi_json (1.15.0)
|
|
78
|
+
netrc (0.11.0)
|
|
79
|
+
parallel (1.21.0)
|
|
80
|
+
parser (3.0.2.0)
|
|
81
|
+
ast (~> 2.4.1)
|
|
82
|
+
power_assert (2.0.1)
|
|
83
|
+
public_suffix (4.0.6)
|
|
84
|
+
rainbow (3.0.0)
|
|
85
|
+
rake (13.0.6)
|
|
86
|
+
recursive-open-struct (1.1.3)
|
|
87
|
+
regexp_parser (2.1.1)
|
|
88
|
+
rest-client (2.1.0)
|
|
89
|
+
http-accept (>= 1.7.0, < 2.0)
|
|
90
|
+
http-cookie (>= 1.0.2, < 2.0)
|
|
91
|
+
mime-types (>= 1.16, < 4.0)
|
|
92
|
+
netrc (~> 0.8)
|
|
93
|
+
rexml (3.2.5)
|
|
94
|
+
rr (3.0.8)
|
|
95
|
+
rubocop (1.22.3)
|
|
96
|
+
parallel (~> 1.10)
|
|
97
|
+
parser (>= 3.0.0.0)
|
|
98
|
+
rainbow (>= 2.2.2, < 4.0)
|
|
99
|
+
regexp_parser (>= 1.8, < 3.0)
|
|
100
|
+
rexml
|
|
101
|
+
rubocop-ast (>= 1.12.0, < 2.0)
|
|
102
|
+
ruby-progressbar (~> 1.7)
|
|
103
|
+
unicode-display_width (>= 1.4.0, < 3.0)
|
|
104
|
+
rubocop-ast (1.12.0)
|
|
105
|
+
parser (>= 3.0.1.1)
|
|
106
|
+
ruby-progressbar (1.11.0)
|
|
107
|
+
rugged (1.2.0)
|
|
108
|
+
serverengine (2.2.4)
|
|
109
|
+
sigdump (~> 0.2.2)
|
|
110
|
+
sigdump (0.2.4)
|
|
111
|
+
simplecov (0.21.2)
|
|
112
|
+
docile (~> 1.1)
|
|
113
|
+
simplecov-html (~> 0.11)
|
|
114
|
+
simplecov_json_formatter (~> 0.1)
|
|
115
|
+
simplecov-html (0.12.3)
|
|
116
|
+
simplecov_json_formatter (0.1.3)
|
|
117
|
+
strptime (0.2.5)
|
|
118
|
+
test-unit (3.0.9)
|
|
119
|
+
power_assert
|
|
120
|
+
test-unit-rr (1.0.5)
|
|
121
|
+
rr (>= 1.1.1)
|
|
122
|
+
test-unit (>= 2.5.2)
|
|
123
|
+
tzinfo (2.0.4)
|
|
124
|
+
concurrent-ruby (~> 1.0)
|
|
125
|
+
tzinfo-data (1.2021.5)
|
|
126
|
+
tzinfo (>= 1.0.0)
|
|
127
|
+
unf (0.1.4)
|
|
128
|
+
unf_ext
|
|
129
|
+
unf_ext (0.0.8)
|
|
130
|
+
unicode-display_width (2.1.0)
|
|
131
|
+
vcr (6.0.0)
|
|
132
|
+
webmock (3.14.0)
|
|
133
|
+
addressable (>= 2.8.0)
|
|
134
|
+
crack (>= 0.3.2)
|
|
135
|
+
hashdiff (>= 0.4.0, < 2.0.0)
|
|
136
|
+
webrick (1.7.0)
|
|
137
|
+
yajl-ruby (1.4.1)
|
|
138
|
+
|
|
139
|
+
PLATFORMS
|
|
140
|
+
ruby
|
|
141
|
+
|
|
142
|
+
DEPENDENCIES
|
|
143
|
+
bump
|
|
144
|
+
bundler (~> 2.0)
|
|
145
|
+
codeclimate-test-reporter (< 1.0.0)
|
|
146
|
+
copyright-header
|
|
147
|
+
fluent-plugin-kubernetes_metadata_filter!
|
|
148
|
+
minitest (~> 4.0)
|
|
149
|
+
rake
|
|
150
|
+
rubocop
|
|
151
|
+
test-unit (~> 3.0.2)
|
|
152
|
+
test-unit-rr (~> 1.0.3)
|
|
153
|
+
vcr
|
|
154
|
+
webmock
|
|
155
|
+
yajl-ruby
|
|
156
|
+
|
|
157
|
+
BUNDLED WITH
|
|
158
|
+
2.3.4
|
data/README.md
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
[](https://circleci.com/gh/fabric8io/fluent-plugin-kubernetes_metadata_filter)
|
|
3
3
|
[](https://codeclimate.com/github/fabric8io/fluent-plugin-kubernetes_metadata_filter)
|
|
4
4
|
[](https://codeclimate.com/github/fabric8io/fluent-plugin-kubernetes_metadata_filter)
|
|
5
|
+
[](https://github.com/rubocop-hq/rubocop)
|
|
6
|
+
[](https://rubystyle.guide)
|
|
5
7
|
|
|
6
8
|
The Kubernetes metadata plugin filter enriches container log records with pod and namespace metadata.
|
|
7
9
|
|
|
@@ -15,6 +17,7 @@ that rely on the authenticity of the namespace for proper log isolation.
|
|
|
15
17
|
|
|
16
18
|
| fluent-plugin-kubernetes_metadata_filter | fluentd | ruby |
|
|
17
19
|
|-------------------|---------|------|
|
|
20
|
+
| >= 2.5.0 | >= v1.10.0 | >= 2.5 |
|
|
18
21
|
| >= 2.0.0 | >= v0.14.20 | >= 2.1 |
|
|
19
22
|
| < 2.0.0 | >= v0.12.0 | >= 1.9 |
|
|
20
23
|
|
|
@@ -38,64 +41,79 @@ Configuration options for fluent.conf are:
|
|
|
38
41
|
* `client_key` - path to a client key file to authenticate to the API server
|
|
39
42
|
* `bearer_token_file` - path to a file containing the bearer token to use for authentication
|
|
40
43
|
* `tag_to_kubernetes_name_regexp` - the regular expression used to extract kubernetes metadata (pod name, container name, namespace) from the current fluentd tag.
|
|
41
|
-
This must
|
|
44
|
+
This must use named capture groups for `container_name`, `pod_name`, `namespace`, and either `pod_uuid (/var/log/pods)` or `docker_id (/var/log/containers)`
|
|
42
45
|
* `cache_size` - size of the cache of Kubernetes metadata to reduce requests to the API server (default: `1000`)
|
|
43
46
|
* `cache_ttl` - TTL in seconds of each cached element. Set to negative value to disable TTL eviction (default: `3600` - 1 hour)
|
|
44
47
|
* `watch` - set up a watch on pods on the API server for updates to metadata (default: `true`)
|
|
45
|
-
*
|
|
46
|
-
*
|
|
48
|
+
* *DEPRECATED*`de_dot` - replace dots in labels and annotations with configured `de_dot_separator`, required for Datadog and ElasticSearch 2.x compatibility (default: `true`)
|
|
49
|
+
* *DEPRECATED*`de_dot_separator` - separator to use if `de_dot` is enabled (default: `_`)
|
|
50
|
+
* *DEPRECATED*`de_slash` - replace slashes in labels and annotations with configured `de_slash_separator`, required for Datadog compatibility (default: `false`)
|
|
51
|
+
* *DEPRECATED*`de_slash_separator` - separator to use if `de_slash` is enabled (default: `__`)
|
|
47
52
|
* *DEPRECATED* `use_journal` - If false, messages are expected to be formatted and tagged as if read by the fluentd in\_tail plugin with wildcard filename. If true, messages are expected to be formatted as if read from the systemd journal. The `MESSAGE` field has the full message. The `CONTAINER_NAME` field has the encoded k8s metadata (see below). The `CONTAINER_ID_FULL` field has the full container uuid. This requires docker to use the `--log-driver=journald` log driver. If unset (the default), the plugin will use the `CONTAINER_NAME` and `CONTAINER_ID_FULL` fields
|
|
48
53
|
if available, otherwise, will use the tag in the `tag_to_kubernetes_name_regexp` format.
|
|
49
|
-
* `container_name_to_kubernetes_regexp` - The regular expression used to extract the k8s metadata encoded in the journal `CONTAINER_NAME` field
|
|
50
|
-
* This corresponds to the definition [in the source](https://github.com/kubernetes/kubernetes/blob/
|
|
54
|
+
* `container_name_to_kubernetes_regexp` - The regular expression used to extract the k8s metadata encoded in the journal `CONTAINER_NAME` field default: See [code](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/blob/master/lib/fluent/plugin/filter_kubernetes_metadata.rb#L68)
|
|
55
|
+
* This corresponds to the definition [in the source](https://github.com/kubernetes/kubernetes/blob/release-1.6/pkg/kubelet/dockertools/docker.go#L317)
|
|
51
56
|
* `annotation_match` - Array of regular expressions matching annotation field names. Matched annotations are added to a log record.
|
|
52
57
|
* `allow_orphans` - Modify the namespace and namespace id to the values of `orphaned_namespace_name` and `orphaned_namespace_id`
|
|
53
58
|
when true (default: `true`)
|
|
54
59
|
* `orphaned_namespace_name` - The namespace to associate with records where the namespace can not be determined (default: `.orphaned`)
|
|
55
60
|
* `orphaned_namespace_id` - The namespace id to associate with records where the namespace can not be determined (default: `orphaned`)
|
|
56
61
|
* `lookup_from_k8s_field` - If the field `kubernetes` is present, lookup the metadata from the given subfields such as `kubernetes.namespace_name`, `kubernetes.pod_name`, etc. This allows you to avoid having to pass in metadata to lookup in an explicitly formatted tag name or in an explicitly formatted `CONTAINER_NAME` value. For example, set `kubernetes.namespace_name`, `kubernetes.pod_name`, `kubernetes.container_name`, and `docker.id` in the record, and the filter will fill in the rest. (default: `true`)
|
|
62
|
+
* `ssl_partial_chain` - if `ca_file` is for an intermediate CA, or otherwise we do not have the root CA and want
|
|
63
|
+
to trust the intermediate CA certs we do have, set this to `true` - this corresponds to
|
|
64
|
+
the `openssl s_client -partial_chain` flag and `X509_V_FLAG_PARTIAL_CHAIN` (default: `false`)
|
|
65
|
+
* `skip_labels` - Skip all label fields from the metadata.
|
|
66
|
+
* `skip_container_metadata` - Skip some of the container data of the metadata. The metadata will not contain the container_image and container_image_id fields.
|
|
67
|
+
* `skip_master_url` - Skip the master_url field from the metadata.
|
|
68
|
+
* `skip_namespace_metadata` - Skip the namespace_id field from the metadata. The fetch_namespace_metadata function will be skipped. The plugin will be faster and cpu consumption will be less.
|
|
69
|
+
* `watch_retry_interval` - The time interval in seconds for retry backoffs when watch connections fail. (default: `10`)
|
|
57
70
|
|
|
58
|
-
**NOTE:** As of the release 2.1.x of this plugin, it no longer supports parsing the source message into JSON and attaching it to the
|
|
59
|
-
payload. The following configuration options are removed:
|
|
60
71
|
|
|
61
|
-
|
|
62
|
-
* `preserve_json_log`
|
|
72
|
+
Reading from the JSON formatted log files with `in_tail` and wildcard filenames while respecting the CRI-o log format with the same config you need the fluent-plugin "multi-format-parser":
|
|
63
73
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
`docker.container_id`, `kubernetes.namespace_name`, `kubernetes.pod_name`, `kubernetes.container_name`,
|
|
68
|
-
then the plugin will use those values as the source to use to lookup the metadata
|
|
69
|
-
- If `use_journal true`, or `use_journal` is unset, and the fields `CONTAINER_NAME` and `CONTAINER_ID_FULL` are present in the record,
|
|
70
|
-
then the plugin will parse those values using `container_name_to_kubernetes_regexp` and use those as the source to lookup the metadata
|
|
71
|
-
- Otherwise, if the tag matches `tag_to_kubernetes_name_regexp`, the plugin will parse the tag and use those values to
|
|
72
|
-
lookup the metdata
|
|
74
|
+
```
|
|
75
|
+
fluent-gem install fluent-plugin-multi-format-parser
|
|
76
|
+
```
|
|
73
77
|
|
|
74
|
-
|
|
78
|
+
The config block could look like this:
|
|
75
79
|
```
|
|
76
80
|
<source>
|
|
77
|
-
type tail
|
|
81
|
+
@type tail
|
|
78
82
|
path /var/log/containers/*.log
|
|
79
83
|
pos_file fluentd-docker.pos
|
|
80
|
-
time_format %Y-%m-%dT%H:%M:%S
|
|
81
|
-
tag kubernetes.*
|
|
82
|
-
format json
|
|
83
84
|
read_from_head true
|
|
85
|
+
tag kubernetes.*
|
|
86
|
+
<parse>
|
|
87
|
+
@type multi_format
|
|
88
|
+
<pattern>
|
|
89
|
+
format json
|
|
90
|
+
time_key time
|
|
91
|
+
time_type string
|
|
92
|
+
time_format "%Y-%m-%dT%H:%M:%S.%NZ"
|
|
93
|
+
keep_time_key false
|
|
94
|
+
</pattern>
|
|
95
|
+
<pattern>
|
|
96
|
+
format regexp
|
|
97
|
+
expression /^(?<time>.+) (?<stream>stdout|stderr)( (?<logtag>.))? (?<log>.*)$/
|
|
98
|
+
time_format '%Y-%m-%dT%H:%M:%S.%N%:z'
|
|
99
|
+
keep_time_key false
|
|
100
|
+
</pattern>
|
|
101
|
+
</parse>
|
|
84
102
|
</source>
|
|
85
103
|
|
|
86
104
|
<filter kubernetes.var.log.containers.**.log>
|
|
87
|
-
type kubernetes_metadata
|
|
105
|
+
@type kubernetes_metadata
|
|
88
106
|
</filter>
|
|
89
107
|
|
|
90
108
|
<match **>
|
|
91
|
-
type stdout
|
|
109
|
+
@type stdout
|
|
92
110
|
</match>
|
|
93
111
|
```
|
|
94
112
|
|
|
95
113
|
Reading from the systemd journal (requires the fluentd `fluent-plugin-systemd` and `systemd-journal` plugins, and requires docker to use the `--log-driver=journald` log driver):
|
|
96
114
|
```
|
|
97
115
|
<source>
|
|
98
|
-
type systemd
|
|
116
|
+
@type systemd
|
|
99
117
|
path /run/log/journal
|
|
100
118
|
pos_file journal.pos
|
|
101
119
|
tag journal
|
|
@@ -111,15 +129,16 @@ Reading from the systemd journal (requires the fluentd `fluent-plugin-systemd` a
|
|
|
111
129
|
</match>
|
|
112
130
|
|
|
113
131
|
<filter kubernetes.**>
|
|
114
|
-
type kubernetes_metadata
|
|
132
|
+
@type kubernetes_metadata
|
|
115
133
|
use_journal true
|
|
116
134
|
</filter>
|
|
117
135
|
|
|
118
136
|
<match **>
|
|
119
|
-
type stdout
|
|
137
|
+
@type stdout
|
|
120
138
|
</match>
|
|
121
139
|
```
|
|
122
140
|
|
|
141
|
+
|
|
123
142
|
## Environment variables for Kubernetes
|
|
124
143
|
|
|
125
144
|
If the name of the Kubernetes node the plugin is running on is set as
|
|
@@ -162,6 +181,7 @@ Then output becomes as belows
|
|
|
162
181
|
"host": "jimmi-redhat.localnet",
|
|
163
182
|
"pod_name":"fabric8-console-controller-98rqc",
|
|
164
183
|
"pod_id": "c76927af-f563-11e4-b32d-54ee7527188d",
|
|
184
|
+
"pod_ip": "172.17.0.8",
|
|
165
185
|
"container_name": "fabric8-console-container",
|
|
166
186
|
"namespace_name": "default",
|
|
167
187
|
"namespace_id": "23437884-8e08-4d95-850b-e94378c9b2fd",
|
data/Rakefile
CHANGED
|
@@ -1,10 +1,15 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'bundler/setup'
|
|
1
4
|
require 'bundler/gem_tasks'
|
|
2
5
|
require 'rake/testtask'
|
|
3
6
|
require 'bump/tasks'
|
|
7
|
+
require 'rubocop/rake_task'
|
|
4
8
|
|
|
5
|
-
task :
|
|
9
|
+
task test: [:base_test]
|
|
10
|
+
task default: [:test, :build, :rubocop]
|
|
6
11
|
|
|
7
|
-
|
|
12
|
+
RuboCop::RakeTask.new
|
|
8
13
|
|
|
9
14
|
desc 'Run test_unit based test'
|
|
10
15
|
Rake::TestTask.new(:base_test) do |t|
|
|
@@ -13,7 +18,6 @@ Rake::TestTask.new(:base_test) do |t|
|
|
|
13
18
|
# $ bundle exec rake base_test TEST=test/test_*.rb
|
|
14
19
|
t.libs << 'test'
|
|
15
20
|
t.test_files = Dir['test/**/test_*.rb'].sort
|
|
16
|
-
#t.verbose = true
|
|
17
21
|
t.warning = false
|
|
18
22
|
end
|
|
19
23
|
|
|
@@ -23,15 +27,15 @@ task :headers do
|
|
|
23
27
|
require 'copyright_header'
|
|
24
28
|
|
|
25
29
|
args = {
|
|
26
|
-
:
|
|
27
|
-
:
|
|
28
|
-
:
|
|
29
|
-
:
|
|
30
|
-
:
|
|
31
|
-
:
|
|
32
|
-
:
|
|
30
|
+
license: 'Apache-2.0',
|
|
31
|
+
copyright_software: 'Fluentd Kubernetes Metadata Filter Plugin',
|
|
32
|
+
copyright_software_description: 'Enrich Fluentd events with Kubernetes metadata',
|
|
33
|
+
copyright_holders: ['Red Hat, Inc.'],
|
|
34
|
+
copyright_years: ['2015-2021'],
|
|
35
|
+
add_path: 'lib:test',
|
|
36
|
+
output_dir: '.'
|
|
33
37
|
}
|
|
34
38
|
|
|
35
|
-
command_line = CopyrightHeader::CommandLine.new(
|
|
39
|
+
command_line = CopyrightHeader::CommandLine.new(args)
|
|
36
40
|
command_line.execute
|
|
37
41
|
end
|
|
@@ -1,37 +1,34 @@
|
|
|
1
|
-
#
|
|
2
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
lib = File.expand_path('lib', __dir__)
|
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
5
|
|
|
5
6
|
Gem::Specification.new do |gem|
|
|
6
|
-
gem.name =
|
|
7
|
-
gem.version =
|
|
8
|
-
gem.authors = [
|
|
9
|
-
gem.email = [
|
|
10
|
-
gem.description =
|
|
11
|
-
gem.summary =
|
|
12
|
-
gem.homepage =
|
|
13
|
-
gem.license =
|
|
7
|
+
gem.name = 'fluent-plugin-kubernetes_metadata_filter'
|
|
8
|
+
gem.version = '2.9.4'
|
|
9
|
+
gem.authors = ['Jimmi Dyson']
|
|
10
|
+
gem.email = ['jimmidyson@gmail.com']
|
|
11
|
+
gem.description = 'Filter plugin to add Kubernetes metadata'
|
|
12
|
+
gem.summary = 'Fluentd filter plugin to add Kubernetes metadata'
|
|
13
|
+
gem.homepage = 'https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter'
|
|
14
|
+
gem.license = 'Apache-2.0'
|
|
14
15
|
|
|
15
16
|
gem.files = `git ls-files`.split($/)
|
|
16
|
-
gem.executables = gem.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
|
17
|
-
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
|
18
|
-
gem.require_paths = ["lib"]
|
|
19
|
-
gem.has_rdoc = false
|
|
20
17
|
|
|
21
|
-
gem.required_ruby_version = '>= 2.
|
|
18
|
+
gem.required_ruby_version = '>= 2.5.0'
|
|
22
19
|
|
|
23
|
-
gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '<
|
|
24
|
-
gem.add_runtime_dependency
|
|
25
|
-
gem.add_runtime_dependency
|
|
20
|
+
gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 1.15']
|
|
21
|
+
gem.add_runtime_dependency 'kubeclient', ['>= 4.0.0', '< 5.0.0']
|
|
22
|
+
gem.add_runtime_dependency 'lru_redux'
|
|
26
23
|
|
|
27
|
-
gem.add_development_dependency
|
|
28
|
-
gem.add_development_dependency
|
|
29
|
-
gem.add_development_dependency
|
|
30
|
-
gem.add_development_dependency
|
|
31
|
-
gem.add_development_dependency
|
|
32
|
-
gem.add_development_dependency
|
|
33
|
-
gem.add_development_dependency
|
|
34
|
-
gem.add_development_dependency
|
|
35
|
-
gem.add_development_dependency
|
|
36
|
-
gem.add_development_dependency
|
|
24
|
+
gem.add_development_dependency 'bump'
|
|
25
|
+
gem.add_development_dependency 'bundler', '~> 2.0'
|
|
26
|
+
gem.add_development_dependency 'copyright-header'
|
|
27
|
+
gem.add_development_dependency 'minitest', '~> 4.0'
|
|
28
|
+
gem.add_development_dependency 'rake'
|
|
29
|
+
gem.add_development_dependency 'test-unit', '~> 3.0.2'
|
|
30
|
+
gem.add_development_dependency 'test-unit-rr', '~> 1.0.3'
|
|
31
|
+
gem.add_development_dependency 'vcr'
|
|
32
|
+
gem.add_development_dependency 'webmock'
|
|
33
|
+
gem.add_development_dependency 'yajl-ruby'
|
|
37
34
|
end
|