fluent-plugin-kubernetes_metadata_filter 2.4.2 → 2.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +0 -1
- data/Gemfile.lock +156 -0
- data/README.md +3 -1
- data/fluent-plugin-kubernetes_metadata_filter.gemspec +2 -2
- data/lib/fluent/plugin/filter_kubernetes_metadata.rb +8 -13
- data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +15 -1
- data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +32 -1
- data/test/plugin/test_watch_namespaces.rb +39 -0
- data/test/plugin/test_watch_pods.rb +63 -0
- data/test/plugin/watch_test.rb +2 -0
- metadata +6 -5
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: fb4fedca66daf5e4073101607933eaf70d0ed6295cf8ac73207e70947b2bbe28
|
|
4
|
+
data.tar.gz: fb6a9d7a9e1eadfbb2d8e3c802611561d904ba4a7ed3948b5a305a140df848df
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 7f86f7472899a36b95316e5b37fb03ab0b893f6fe58b13cb9b5957ef5557ee6e7efd82bf74fe20525644002ac86d65c8b95c1f9e96e617355c0021c9b8e3689b
|
|
7
|
+
data.tar.gz: 505c4a63b0e87f068eaff8530bcb75fb0efa1eeb0abd5796d5bd41f9d3c49cc4fe3e59be2553203ed5b7b2901cff7f91046763bfefd40eb4049240d295dd7107
|
data/.gitignore
CHANGED
data/Gemfile.lock
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
PATH
|
|
2
|
+
remote: .
|
|
3
|
+
specs:
|
|
4
|
+
fluent-plugin-kubernetes_metadata_filter (2.5.1)
|
|
5
|
+
fluentd (>= 0.14.0, < 1.12)
|
|
6
|
+
kubeclient (< 5)
|
|
7
|
+
lru_redux
|
|
8
|
+
|
|
9
|
+
GEM
|
|
10
|
+
remote: https://rubygems.org/
|
|
11
|
+
specs:
|
|
12
|
+
addressable (2.7.0)
|
|
13
|
+
public_suffix (>= 2.0.2, < 5.0)
|
|
14
|
+
ast (2.4.1)
|
|
15
|
+
bump (0.9.0)
|
|
16
|
+
charlock_holmes (0.7.7)
|
|
17
|
+
codeclimate-test-reporter (0.6.0)
|
|
18
|
+
simplecov (>= 0.7.1, < 1.0.0)
|
|
19
|
+
concurrent-ruby (1.1.6)
|
|
20
|
+
cool.io (1.6.0)
|
|
21
|
+
copyright-header (1.0.22)
|
|
22
|
+
github-linguist
|
|
23
|
+
crack (0.4.3)
|
|
24
|
+
safe_yaml (~> 1.0.0)
|
|
25
|
+
docile (1.3.2)
|
|
26
|
+
domain_name (0.5.20190701)
|
|
27
|
+
unf (>= 0.0.5, < 1.0.0)
|
|
28
|
+
escape_utils (1.2.1)
|
|
29
|
+
ffi (1.13.1)
|
|
30
|
+
ffi-compiler (1.0.1)
|
|
31
|
+
ffi (>= 1.0.0)
|
|
32
|
+
rake
|
|
33
|
+
fluentd (1.11.1)
|
|
34
|
+
cool.io (>= 1.4.5, < 2.0.0)
|
|
35
|
+
http_parser.rb (>= 0.5.1, < 0.7.0)
|
|
36
|
+
msgpack (>= 1.3.1, < 2.0.0)
|
|
37
|
+
serverengine (>= 2.0.4, < 3.0.0)
|
|
38
|
+
sigdump (~> 0.2.2)
|
|
39
|
+
strptime (>= 0.2.2, < 1.0.0)
|
|
40
|
+
tzinfo (>= 1.0, < 3.0)
|
|
41
|
+
tzinfo-data (~> 1.0)
|
|
42
|
+
yajl-ruby (~> 1.0)
|
|
43
|
+
github-linguist (7.9.0)
|
|
44
|
+
charlock_holmes (~> 0.7.6)
|
|
45
|
+
escape_utils (~> 1.2.0)
|
|
46
|
+
mini_mime (~> 1.0)
|
|
47
|
+
rugged (>= 0.25.1)
|
|
48
|
+
hashdiff (1.0.1)
|
|
49
|
+
http (4.4.1)
|
|
50
|
+
addressable (~> 2.3)
|
|
51
|
+
http-cookie (~> 1.0)
|
|
52
|
+
http-form_data (~> 2.2)
|
|
53
|
+
http-parser (~> 1.2.0)
|
|
54
|
+
http-accept (1.7.0)
|
|
55
|
+
http-cookie (1.0.3)
|
|
56
|
+
domain_name (~> 0.5)
|
|
57
|
+
http-form_data (2.3.0)
|
|
58
|
+
http-parser (1.2.1)
|
|
59
|
+
ffi-compiler (>= 1.0, < 2.0)
|
|
60
|
+
http_parser.rb (0.6.0)
|
|
61
|
+
jsonpath (1.0.5)
|
|
62
|
+
multi_json
|
|
63
|
+
to_regexp (~> 0.2.1)
|
|
64
|
+
kubeclient (4.8.0)
|
|
65
|
+
http (>= 3.0, < 5.0)
|
|
66
|
+
jsonpath (~> 1.0)
|
|
67
|
+
recursive-open-struct (~> 1.1, >= 1.1.1)
|
|
68
|
+
rest-client (~> 2.0)
|
|
69
|
+
lru_redux (1.1.0)
|
|
70
|
+
mime-types (3.3.1)
|
|
71
|
+
mime-types-data (~> 3.2015)
|
|
72
|
+
mime-types-data (3.2020.0512)
|
|
73
|
+
mini_mime (1.0.2)
|
|
74
|
+
minitest (4.7.5)
|
|
75
|
+
msgpack (1.3.3)
|
|
76
|
+
multi_json (1.15.0)
|
|
77
|
+
netrc (0.11.0)
|
|
78
|
+
parallel (1.19.2)
|
|
79
|
+
parser (2.7.1.4)
|
|
80
|
+
ast (~> 2.4.1)
|
|
81
|
+
power_assert (1.2.0)
|
|
82
|
+
public_suffix (4.0.5)
|
|
83
|
+
rainbow (3.0.0)
|
|
84
|
+
rake (13.0.1)
|
|
85
|
+
recursive-open-struct (1.1.2)
|
|
86
|
+
regexp_parser (1.7.1)
|
|
87
|
+
rest-client (2.1.0)
|
|
88
|
+
http-accept (>= 1.7.0, < 2.0)
|
|
89
|
+
http-cookie (>= 1.0.2, < 2.0)
|
|
90
|
+
mime-types (>= 1.16, < 4.0)
|
|
91
|
+
netrc (~> 0.8)
|
|
92
|
+
rexml (3.2.4)
|
|
93
|
+
rr (1.2.1)
|
|
94
|
+
rubocop (0.86.0)
|
|
95
|
+
parallel (~> 1.10)
|
|
96
|
+
parser (>= 2.7.0.1)
|
|
97
|
+
rainbow (>= 2.2.2, < 4.0)
|
|
98
|
+
regexp_parser (>= 1.7)
|
|
99
|
+
rexml
|
|
100
|
+
rubocop-ast (>= 0.0.3, < 1.0)
|
|
101
|
+
ruby-progressbar (~> 1.7)
|
|
102
|
+
unicode-display_width (>= 1.4.0, < 2.0)
|
|
103
|
+
rubocop-ast (0.1.0)
|
|
104
|
+
parser (>= 2.7.0.1)
|
|
105
|
+
ruby-progressbar (1.10.1)
|
|
106
|
+
rugged (1.0.1)
|
|
107
|
+
safe_yaml (1.0.5)
|
|
108
|
+
serverengine (2.2.1)
|
|
109
|
+
sigdump (~> 0.2.2)
|
|
110
|
+
sigdump (0.2.4)
|
|
111
|
+
simplecov (0.18.5)
|
|
112
|
+
docile (~> 1.1)
|
|
113
|
+
simplecov-html (~> 0.11)
|
|
114
|
+
simplecov-html (0.12.2)
|
|
115
|
+
strptime (0.2.4)
|
|
116
|
+
test-unit (3.0.9)
|
|
117
|
+
power_assert
|
|
118
|
+
test-unit-rr (1.0.5)
|
|
119
|
+
rr (>= 1.1.1)
|
|
120
|
+
test-unit (>= 2.5.2)
|
|
121
|
+
to_regexp (0.2.1)
|
|
122
|
+
tzinfo (2.0.2)
|
|
123
|
+
concurrent-ruby (~> 1.0)
|
|
124
|
+
tzinfo-data (1.2020.1)
|
|
125
|
+
tzinfo (>= 1.0.0)
|
|
126
|
+
unf (0.1.4)
|
|
127
|
+
unf_ext
|
|
128
|
+
unf_ext (0.0.7.7)
|
|
129
|
+
unicode-display_width (1.7.0)
|
|
130
|
+
vcr (6.0.0)
|
|
131
|
+
webmock (3.8.3)
|
|
132
|
+
addressable (>= 2.3.6)
|
|
133
|
+
crack (>= 0.3.2)
|
|
134
|
+
hashdiff (>= 0.4.0, < 2.0.0)
|
|
135
|
+
yajl-ruby (1.4.1)
|
|
136
|
+
|
|
137
|
+
PLATFORMS
|
|
138
|
+
ruby
|
|
139
|
+
|
|
140
|
+
DEPENDENCIES
|
|
141
|
+
bump
|
|
142
|
+
bundler (~> 2.0)
|
|
143
|
+
codeclimate-test-reporter (< 1.0.0)
|
|
144
|
+
copyright-header
|
|
145
|
+
fluent-plugin-kubernetes_metadata_filter!
|
|
146
|
+
minitest (~> 4.0)
|
|
147
|
+
rake
|
|
148
|
+
rubocop
|
|
149
|
+
test-unit (~> 3.0.2)
|
|
150
|
+
test-unit-rr (~> 1.0.3)
|
|
151
|
+
vcr
|
|
152
|
+
webmock
|
|
153
|
+
yajl-ruby
|
|
154
|
+
|
|
155
|
+
BUNDLED WITH
|
|
156
|
+
2.1.4
|
data/README.md
CHANGED
|
@@ -15,6 +15,7 @@ that rely on the authenticity of the namespace for proper log isolation.
|
|
|
15
15
|
|
|
16
16
|
| fluent-plugin-kubernetes_metadata_filter | fluentd | ruby |
|
|
17
17
|
|-------------------|---------|------|
|
|
18
|
+
| >= 2.5.0 | >= v1.10.0 | >= 2.5 |
|
|
18
19
|
| >= 2.0.0 | >= v0.14.20 | >= 2.1 |
|
|
19
20
|
| < 2.0.0 | >= v0.12.0 | >= 1.9 |
|
|
20
21
|
|
|
@@ -47,7 +48,7 @@ This must used named capture groups for `container_name`, `pod_name` & `namespac
|
|
|
47
48
|
* *DEPRECATED* `use_journal` - If false, messages are expected to be formatted and tagged as if read by the fluentd in\_tail plugin with wildcard filename. If true, messages are expected to be formatted as if read from the systemd journal. The `MESSAGE` field has the full message. The `CONTAINER_NAME` field has the encoded k8s metadata (see below). The `CONTAINER_ID_FULL` field has the full container uuid. This requires docker to use the `--log-driver=journald` log driver. If unset (the default), the plugin will use the `CONTAINER_NAME` and `CONTAINER_ID_FULL` fields
|
|
48
49
|
if available, otherwise, will use the tag in the `tag_to_kubernetes_name_regexp` format.
|
|
49
50
|
* `container_name_to_kubernetes_regexp` - The regular expression used to extract the k8s metadata encoded in the journal `CONTAINER_NAME` field (default: `'^(?<name_prefix>[^_]+)_(?<container_name>[^\._]+)(\.(?<container_hash>[^_]+))?_(?<pod_name>[^_]+)_(?<namespace>[^_]+)_[^_]+_[^_]+$'`
|
|
50
|
-
* This corresponds to the definition [in the source](https://github.com/kubernetes/kubernetes/blob/
|
|
51
|
+
* This corresponds to the definition [in the source](https://github.com/kubernetes/kubernetes/blob/release-1.6/pkg/kubelet/dockertools/docker.go#L317)
|
|
51
52
|
* `annotation_match` - Array of regular expressions matching annotation field names. Matched annotations are added to a log record.
|
|
52
53
|
* `allow_orphans` - Modify the namespace and namespace id to the values of `orphaned_namespace_name` and `orphaned_namespace_id`
|
|
53
54
|
when true (default: `true`)
|
|
@@ -208,6 +209,7 @@ Then output becomes as belows
|
|
|
208
209
|
"host": "jimmi-redhat.localnet",
|
|
209
210
|
"pod_name":"fabric8-console-controller-98rqc",
|
|
210
211
|
"pod_id": "c76927af-f563-11e4-b32d-54ee7527188d",
|
|
212
|
+
"pod_ip": "172.17.0.8",
|
|
211
213
|
"container_name": "fabric8-console-container",
|
|
212
214
|
"namespace_name": "default",
|
|
213
215
|
"namespace_id": "23437884-8e08-4d95-850b-e94378c9b2fd",
|
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
|
4
4
|
|
|
5
5
|
Gem::Specification.new do |gem|
|
|
6
6
|
gem.name = "fluent-plugin-kubernetes_metadata_filter"
|
|
7
|
-
gem.version = "2.
|
|
7
|
+
gem.version = "2.5.1"
|
|
8
8
|
gem.authors = ["Jimmi Dyson"]
|
|
9
9
|
gem.email = ["jimmidyson@gmail.com"]
|
|
10
10
|
gem.description = %q{Filter plugin to add Kubernetes metadata}
|
|
@@ -19,7 +19,7 @@ Gem::Specification.new do |gem|
|
|
|
19
19
|
|
|
20
20
|
gem.required_ruby_version = '>= 2.1.0'
|
|
21
21
|
|
|
22
|
-
gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '<
|
|
22
|
+
gem.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 1.12']
|
|
23
23
|
gem.add_runtime_dependency "lru_redux"
|
|
24
24
|
gem.add_runtime_dependency "kubeclient", '< 5'
|
|
25
25
|
|
|
@@ -23,7 +23,6 @@ require_relative 'kubernetes_metadata_stats'
|
|
|
23
23
|
require_relative 'kubernetes_metadata_watch_namespaces'
|
|
24
24
|
require_relative 'kubernetes_metadata_watch_pods'
|
|
25
25
|
|
|
26
|
-
require 'fluent/plugin_helper/thread'
|
|
27
26
|
require 'fluent/plugin/filter'
|
|
28
27
|
require 'resolv'
|
|
29
28
|
|
|
@@ -39,8 +38,6 @@ module Fluent::Plugin
|
|
|
39
38
|
|
|
40
39
|
Fluent::Plugin.register_filter('kubernetes_metadata', self)
|
|
41
40
|
|
|
42
|
-
helpers :thread
|
|
43
|
-
|
|
44
41
|
config_param :kubernetes_url, :string, default: nil
|
|
45
42
|
config_param :cache_size, :integer, default: 1000
|
|
46
43
|
config_param :cache_ttl, :integer, default: 60 * 60
|
|
@@ -65,7 +62,7 @@ module Fluent::Plugin
|
|
|
65
62
|
# Field 2 is the container_hash, field 5 is the pod_id, and field 6 is the pod_randhex
|
|
66
63
|
# I would have included them as named groups, but you can't have named groups that are
|
|
67
64
|
# non-capturing :P
|
|
68
|
-
# parse format is defined here: https://github.com/kubernetes/kubernetes/blob/
|
|
65
|
+
# parse format is defined here: https://github.com/kubernetes/kubernetes/blob/release-1.6/pkg/kubelet/dockertools/docker.go#L317
|
|
69
66
|
config_param :container_name_to_kubernetes_regexp,
|
|
70
67
|
:string,
|
|
71
68
|
:default => '^(?<name_prefix>[^_]+)_(?<container_name>[^\._]+)(\.(?<container_hash>[^_]+))?_(?<pod_name>[^_]+)_(?<namespace>[^_]+)_[^_]+_[^_]+$'
|
|
@@ -85,11 +82,11 @@ module Fluent::Plugin
|
|
|
85
82
|
config_param :skip_master_url, :bool, default: false
|
|
86
83
|
config_param :skip_namespace_metadata, :bool, default: false
|
|
87
84
|
# The time interval in seconds for retry backoffs when watch connections fail.
|
|
88
|
-
config_param :watch_retry_interval, :
|
|
85
|
+
config_param :watch_retry_interval, :integer, default: 1
|
|
89
86
|
# The base number of exponential backoff for retries.
|
|
90
|
-
config_param :watch_retry_exponential_backoff_base, :
|
|
87
|
+
config_param :watch_retry_exponential_backoff_base, :integer, default: 2
|
|
91
88
|
# The maximum number of times to retry pod and namespace watches.
|
|
92
|
-
config_param :watch_retry_max_times, :
|
|
89
|
+
config_param :watch_retry_max_times, :integer, default: 10
|
|
93
90
|
|
|
94
91
|
def fetch_pod_metadata(namespace_name, pod_name)
|
|
95
92
|
log.trace("fetching pod metadata: #{namespace_name}/#{pod_name}") if log.trace?
|
|
@@ -211,6 +208,8 @@ module Fluent::Plugin
|
|
|
211
208
|
end
|
|
212
209
|
@kubernetes_url = "https://#{env_host}:#{env_port}/api"
|
|
213
210
|
log.debug "Kubernetes URL is now '#{@kubernetes_url}'"
|
|
211
|
+
else
|
|
212
|
+
log.debug "No Kubernetes URL could be found in config or environ"
|
|
214
213
|
end
|
|
215
214
|
end
|
|
216
215
|
|
|
@@ -274,14 +273,10 @@ module Fluent::Plugin
|
|
|
274
273
|
end
|
|
275
274
|
|
|
276
275
|
if @watch
|
|
277
|
-
pod_thread =
|
|
278
|
-
set_up_pod_thread
|
|
279
|
-
end
|
|
276
|
+
pod_thread = Thread.new(self) { |this| this.set_up_pod_thread }
|
|
280
277
|
pod_thread.abort_on_exception = true
|
|
281
278
|
|
|
282
|
-
namespace_thread =
|
|
283
|
-
set_up_namespace_thread
|
|
284
|
-
end
|
|
279
|
+
namespace_thread = Thread.new(self) { |this| this.set_up_namespace_thread }
|
|
285
280
|
namespace_thread.abort_on_exception = true
|
|
286
281
|
end
|
|
287
282
|
end
|
|
@@ -35,7 +35,7 @@ module KubernetesMetadata
|
|
|
35
35
|
# processing will be swallowed and retried. These failures /
|
|
36
36
|
# exceptions could be caused by Kubernetes API being temporarily
|
|
37
37
|
# down. We assume the configuration is correct at this point.
|
|
38
|
-
while
|
|
38
|
+
while true
|
|
39
39
|
begin
|
|
40
40
|
namespace_watcher ||= get_namespaces_and_start_watcher
|
|
41
41
|
process_namespace_watcher_notices(namespace_watcher)
|
|
@@ -96,11 +96,19 @@ module KubernetesMetadata
|
|
|
96
96
|
watcher
|
|
97
97
|
end
|
|
98
98
|
|
|
99
|
+
# Reset namespace watch retry count and backoff interval as there is a
|
|
100
|
+
# successful watch notice.
|
|
101
|
+
def reset_namespace_watch_retry_stats
|
|
102
|
+
Thread.current[:namespace_watch_retry_count] = 0
|
|
103
|
+
Thread.current[:namespace_watch_retry_backoff_interval] = @watch_retry_interval
|
|
104
|
+
end
|
|
105
|
+
|
|
99
106
|
# Process a watcher notice and potentially raise an exception.
|
|
100
107
|
def process_namespace_watcher_notices(watcher)
|
|
101
108
|
watcher.each do |notice|
|
|
102
109
|
case notice.type
|
|
103
110
|
when 'MODIFIED'
|
|
111
|
+
reset_namespace_watch_retry_stats
|
|
104
112
|
cache_key = notice.object['metadata']['uid']
|
|
105
113
|
cached = @namespace_cache[cache_key]
|
|
106
114
|
if cached
|
|
@@ -110,10 +118,16 @@ module KubernetesMetadata
|
|
|
110
118
|
@stats.bump(:namespace_cache_watch_misses)
|
|
111
119
|
end
|
|
112
120
|
when 'DELETED'
|
|
121
|
+
reset_namespace_watch_retry_stats
|
|
113
122
|
# ignore and let age out for cases where
|
|
114
123
|
# deleted but still processing logs
|
|
115
124
|
@stats.bump(:namespace_cache_watch_deletes_ignored)
|
|
125
|
+
when 'ERROR'
|
|
126
|
+
@stats.bump(:namespace_watch_error_type_notices)
|
|
127
|
+
message = notice['object']['message'] if notice['object'] && notice['object']['message']
|
|
128
|
+
raise "Error while watching namespaces: #{message}"
|
|
116
129
|
else
|
|
130
|
+
reset_namespace_watch_retry_stats
|
|
117
131
|
# Don't pay attention to creations, since the created namespace may not
|
|
118
132
|
# be used by any namespace on this node.
|
|
119
133
|
@stats.bump(:namespace_cache_watch_ignored)
|
|
@@ -19,6 +19,13 @@
|
|
|
19
19
|
require_relative 'kubernetes_metadata_common'
|
|
20
20
|
|
|
21
21
|
module KubernetesMetadata
|
|
22
|
+
|
|
23
|
+
class GoneError < StandardError
|
|
24
|
+
def initialize(msg="410 Gone")
|
|
25
|
+
super
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
22
29
|
module WatchPods
|
|
23
30
|
|
|
24
31
|
include ::KubernetesMetadata::Common
|
|
@@ -35,10 +42,15 @@ module KubernetesMetadata
|
|
|
35
42
|
# processing will be swallowed and retried. These failures /
|
|
36
43
|
# exceptions could be caused by Kubernetes API being temporarily
|
|
37
44
|
# down. We assume the configuration is correct at this point.
|
|
38
|
-
while
|
|
45
|
+
while true
|
|
39
46
|
begin
|
|
40
47
|
pod_watcher ||= get_pods_and_start_watcher
|
|
41
48
|
process_pod_watcher_notices(pod_watcher)
|
|
49
|
+
rescue GoneError
|
|
50
|
+
# Expected error. Quietly go back through the loop in order to
|
|
51
|
+
# start watching from the latest resource versions
|
|
52
|
+
log.debug("410 Gone encountered. Restarting watch to reset resource versions.")
|
|
53
|
+
pod_watcher = nil
|
|
42
54
|
rescue Exception => e
|
|
43
55
|
@stats.bump(:pod_watch_failures)
|
|
44
56
|
if Thread.current[:pod_watch_retry_count] < @watch_retry_max_times
|
|
@@ -99,11 +111,19 @@ module KubernetesMetadata
|
|
|
99
111
|
watcher
|
|
100
112
|
end
|
|
101
113
|
|
|
114
|
+
# Reset pod watch retry count and backoff interval as there is a
|
|
115
|
+
# successful watch notice.
|
|
116
|
+
def reset_pod_watch_retry_stats
|
|
117
|
+
Thread.current[:pod_watch_retry_count] = 0
|
|
118
|
+
Thread.current[:pod_watch_retry_backoff_interval] = @watch_retry_interval
|
|
119
|
+
end
|
|
120
|
+
|
|
102
121
|
# Process a watcher notice and potentially raise an exception.
|
|
103
122
|
def process_pod_watcher_notices(watcher)
|
|
104
123
|
watcher.each do |notice|
|
|
105
124
|
case notice.type
|
|
106
125
|
when 'MODIFIED'
|
|
126
|
+
reset_pod_watch_retry_stats
|
|
107
127
|
cache_key = notice.object['metadata']['uid']
|
|
108
128
|
cached = @cache[cache_key]
|
|
109
129
|
if cached
|
|
@@ -116,10 +136,21 @@ module KubernetesMetadata
|
|
|
116
136
|
@stats.bump(:pod_cache_watch_misses)
|
|
117
137
|
end
|
|
118
138
|
when 'DELETED'
|
|
139
|
+
reset_pod_watch_retry_stats
|
|
119
140
|
# ignore and let age out for cases where pods
|
|
120
141
|
# deleted but still processing logs
|
|
121
142
|
@stats.bump(:pod_cache_watch_delete_ignored)
|
|
143
|
+
when 'ERROR'
|
|
144
|
+
if notice.object && notice.object['code'] == 410
|
|
145
|
+
@stats.bump(:pod_watch_gone_notices)
|
|
146
|
+
raise GoneError
|
|
147
|
+
else
|
|
148
|
+
@stats.bump(:pod_watch_error_type_notices)
|
|
149
|
+
message = notice['object']['message'] if notice['object'] && notice['object']['message']
|
|
150
|
+
raise "Error while watching pods: #{message}"
|
|
151
|
+
end
|
|
122
152
|
else
|
|
153
|
+
reset_pod_watch_retry_stats
|
|
123
154
|
# Don't pay attention to creations, since the created pod may not
|
|
124
155
|
# end up on this node.
|
|
125
156
|
@stats.bump(:pod_cache_watch_ignored)
|
|
@@ -70,6 +70,12 @@ class WatchNamespacesTestTest < WatchTest
|
|
|
70
70
|
}
|
|
71
71
|
}
|
|
72
72
|
)
|
|
73
|
+
@error = OpenStruct.new(
|
|
74
|
+
type: 'ERROR',
|
|
75
|
+
object: {
|
|
76
|
+
'message' => 'some error message'
|
|
77
|
+
}
|
|
78
|
+
)
|
|
73
79
|
end
|
|
74
80
|
|
|
75
81
|
test 'namespace list caches namespaces' do
|
|
@@ -137,6 +143,39 @@ class WatchNamespacesTestTest < WatchTest
|
|
|
137
143
|
assert_equal(3, @stats[:namespace_watch_failures])
|
|
138
144
|
assert_equal(2, Thread.current[:namespace_watch_retry_count])
|
|
139
145
|
assert_equal(4, Thread.current[:namespace_watch_retry_backoff_interval])
|
|
146
|
+
assert_nil(@stats[:namespace_watch_error_type_notices])
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
test 'namespace watch retries when error is received' do
|
|
152
|
+
@client.stub :get_namespaces, @initial do
|
|
153
|
+
@client.stub :watch_namespaces, [@error] do
|
|
154
|
+
assert_raise Fluent::UnrecoverableError do
|
|
155
|
+
set_up_namespace_thread
|
|
156
|
+
end
|
|
157
|
+
assert_equal(3, @stats[:namespace_watch_failures])
|
|
158
|
+
assert_equal(2, Thread.current[:namespace_watch_retry_count])
|
|
159
|
+
assert_equal(4, Thread.current[:namespace_watch_retry_backoff_interval])
|
|
160
|
+
assert_equal(3, @stats[:namespace_watch_error_type_notices])
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
test 'namespace watch continues after retries succeed' do
|
|
166
|
+
@client.stub :get_namespaces, @initial do
|
|
167
|
+
@client.stub :watch_namespaces, [@modified, @error, @modified] do
|
|
168
|
+
# Force the infinite watch loop to exit after 3 seconds. Verifies that
|
|
169
|
+
# no unrecoverable error was thrown during this period of time.
|
|
170
|
+
assert_raise Timeout::Error.new('execution expired') do
|
|
171
|
+
Timeout.timeout(3) do
|
|
172
|
+
set_up_namespace_thread
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
assert_operator(@stats[:namespace_watch_failures], :>=, 3)
|
|
176
|
+
assert_operator(Thread.current[:namespace_watch_retry_count], :<=, 1)
|
|
177
|
+
assert_operator(Thread.current[:namespace_watch_retry_backoff_interval], :<=, 1)
|
|
178
|
+
assert_operator(@stats[:namespace_watch_error_type_notices], :>=, 3)
|
|
140
179
|
end
|
|
141
180
|
end
|
|
142
181
|
end
|
|
@@ -136,6 +136,26 @@ class DefaultPodWatchStrategyTest < WatchTest
|
|
|
136
136
|
}
|
|
137
137
|
}
|
|
138
138
|
)
|
|
139
|
+
@error = OpenStruct.new(
|
|
140
|
+
type: 'ERROR',
|
|
141
|
+
object: {
|
|
142
|
+
'message' => 'some error message'
|
|
143
|
+
}
|
|
144
|
+
)
|
|
145
|
+
@gone = OpenStruct.new(
|
|
146
|
+
type: 'ERROR',
|
|
147
|
+
object: {
|
|
148
|
+
'code' => 410,
|
|
149
|
+
'kind' => 'Status',
|
|
150
|
+
'message' => 'too old resource version: 123 (391079)',
|
|
151
|
+
'metadata' => {
|
|
152
|
+
'name' => 'gone',
|
|
153
|
+
'namespace' => 'gone',
|
|
154
|
+
'uid' => 'gone_uid'
|
|
155
|
+
},
|
|
156
|
+
'reason' => 'Gone'
|
|
157
|
+
}
|
|
158
|
+
)
|
|
139
159
|
end
|
|
140
160
|
|
|
141
161
|
test 'pod list caches pods' do
|
|
@@ -219,6 +239,49 @@ class DefaultPodWatchStrategyTest < WatchTest
|
|
|
219
239
|
assert_equal(3, @stats[:pod_watch_failures])
|
|
220
240
|
assert_equal(2, Thread.current[:pod_watch_retry_count])
|
|
221
241
|
assert_equal(4, Thread.current[:pod_watch_retry_backoff_interval])
|
|
242
|
+
assert_nil(@stats[:pod_watch_error_type_notices])
|
|
243
|
+
end
|
|
244
|
+
end
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
test 'pod watch raises a GoneError when a 410 Gone error is received' do
|
|
248
|
+
@cache['gone_uid'] = {}
|
|
249
|
+
@client.stub :watch_pods, [@gone] do
|
|
250
|
+
assert_raise KubernetesMetadata::GoneError do
|
|
251
|
+
process_pod_watcher_notices(start_pod_watch)
|
|
252
|
+
end
|
|
253
|
+
assert_equal(1, @stats[:pod_watch_gone_notices])
|
|
254
|
+
end
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
test 'pod watch retries when error is received' do
|
|
258
|
+
@client.stub :get_pods, @initial do
|
|
259
|
+
@client.stub :watch_pods, [@error] do
|
|
260
|
+
assert_raise Fluent::UnrecoverableError do
|
|
261
|
+
set_up_pod_thread
|
|
262
|
+
end
|
|
263
|
+
assert_equal(3, @stats[:pod_watch_failures])
|
|
264
|
+
assert_equal(2, Thread.current[:pod_watch_retry_count])
|
|
265
|
+
assert_equal(4, Thread.current[:pod_watch_retry_backoff_interval])
|
|
266
|
+
assert_equal(3, @stats[:pod_watch_error_type_notices])
|
|
267
|
+
end
|
|
268
|
+
end
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
test 'pod watch continues after retries succeed' do
|
|
272
|
+
@client.stub :get_pods, @initial do
|
|
273
|
+
@client.stub :watch_pods, [@modified, @error, @modified] do
|
|
274
|
+
# Force the infinite watch loop to exit after 3 seconds. Verifies that
|
|
275
|
+
# no unrecoverable error was thrown during this period of time.
|
|
276
|
+
assert_raise Timeout::Error.new('execution expired') do
|
|
277
|
+
Timeout.timeout(3) do
|
|
278
|
+
set_up_pod_thread
|
|
279
|
+
end
|
|
280
|
+
end
|
|
281
|
+
assert_operator(@stats[:pod_watch_failures], :>=, 3)
|
|
282
|
+
assert_operator(Thread.current[:pod_watch_retry_count], :<=, 1)
|
|
283
|
+
assert_operator(Thread.current[:pod_watch_retry_backoff_interval], :<=, 1)
|
|
284
|
+
assert_operator(@stats[:pod_watch_error_type_notices], :>=, 3)
|
|
222
285
|
end
|
|
223
286
|
end
|
|
224
287
|
end
|
data/test/plugin/watch_test.rb
CHANGED
|
@@ -33,6 +33,8 @@ class WatchTest < Test::Unit::TestCase
|
|
|
33
33
|
@watch_retry_exponential_backoff_base = 2
|
|
34
34
|
@cache = {}
|
|
35
35
|
@stats = KubernetesMetadata::Stats.new
|
|
36
|
+
Thread.current[:pod_watch_retry_count] = 0
|
|
37
|
+
Thread.current[:namespace_watch_retry_count] = 0
|
|
36
38
|
|
|
37
39
|
@client = OpenStruct.new
|
|
38
40
|
def @client.resourceVersion
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-kubernetes_metadata_filter
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.
|
|
4
|
+
version: 2.5.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Jimmi Dyson
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2020-
|
|
11
|
+
date: 2020-07-10 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: fluentd
|
|
@@ -19,7 +19,7 @@ dependencies:
|
|
|
19
19
|
version: 0.14.0
|
|
20
20
|
- - "<"
|
|
21
21
|
- !ruby/object:Gem::Version
|
|
22
|
-
version: '
|
|
22
|
+
version: '1.12'
|
|
23
23
|
type: :runtime
|
|
24
24
|
prerelease: false
|
|
25
25
|
version_requirements: !ruby/object:Gem::Requirement
|
|
@@ -29,7 +29,7 @@ dependencies:
|
|
|
29
29
|
version: 0.14.0
|
|
30
30
|
- - "<"
|
|
31
31
|
- !ruby/object:Gem::Version
|
|
32
|
-
version: '
|
|
32
|
+
version: '1.12'
|
|
33
33
|
- !ruby/object:Gem::Dependency
|
|
34
34
|
name: lru_redux
|
|
35
35
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -208,6 +208,7 @@ files:
|
|
|
208
208
|
- ".circleci/config.yml"
|
|
209
209
|
- ".gitignore"
|
|
210
210
|
- Gemfile
|
|
211
|
+
- Gemfile.lock
|
|
211
212
|
- LICENSE.txt
|
|
212
213
|
- README.md
|
|
213
214
|
- Rakefile
|
|
@@ -258,7 +259,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
258
259
|
- !ruby/object:Gem::Version
|
|
259
260
|
version: '0'
|
|
260
261
|
requirements: []
|
|
261
|
-
rubygems_version: 3.
|
|
262
|
+
rubygems_version: 3.1.2
|
|
262
263
|
signing_key:
|
|
263
264
|
specification_version: 4
|
|
264
265
|
summary: Fluentd filter plugin to add Kubernetes metadata
|