fluent-plugin-kubernetes_metadata_filter 2.5.0 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +10 -14
- data/.gitignore +0 -1
- data/.rubocop.yml +57 -0
- data/Gemfile +4 -2
- data/Gemfile.lock +76 -67
- data/README.md +9 -83
- data/Rakefile +15 -11
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/cpu.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/latency.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/loss.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/mem.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/readme.md +88 -0
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/results.html +127 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/cpu.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/latency.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/loss.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/mem.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/readme.md +97 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/results.html +136 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/cpu.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/latency.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/loss.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/mem.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/readme.md +97 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/results.html +136 -0
- data/fluent-plugin-kubernetes_metadata_filter.gemspec +25 -27
- data/lib/fluent/plugin/filter_kubernetes_metadata.rb +171 -192
- data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb +25 -23
- data/lib/fluent/plugin/kubernetes_metadata_common.rb +44 -69
- data/lib/fluent/plugin/kubernetes_metadata_stats.rb +21 -5
- data/lib/fluent/plugin/kubernetes_metadata_test_api_adapter.rb +68 -0
- data/lib/fluent/plugin/kubernetes_metadata_util.rb +33 -0
- data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +91 -42
- data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +108 -47
- data/release_notes.md +42 -0
- data/test/cassettes/kubernetes_get_pod_container_init.yml +145 -0
- data/test/helper.rb +20 -2
- data/test/plugin/test_cache_stats.rb +10 -13
- data/test/plugin/test_cache_strategy.rb +158 -160
- data/test/plugin/test_filter_kubernetes_metadata.rb +340 -616
- data/test/plugin/test_watch_namespaces.rb +188 -125
- data/test/plugin/test_watch_pods.rb +282 -202
- data/test/plugin/watch_test.rb +16 -15
- metadata +77 -67
- /data/test/cassettes/{kubernetes_docker_metadata_dotted_labels.yml → kubernetes_docker_metadata_dotted_slashed_labels.yml} +0 -0
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
#
|
2
4
|
# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
|
3
5
|
# Kubernetes metadata
|
@@ -18,6 +20,11 @@
|
|
18
20
|
#
|
19
21
|
module KubernetesMetadata
|
20
22
|
module Common
|
23
|
+
class GoneError < StandardError
|
24
|
+
def initialize(msg = '410 Gone')
|
25
|
+
super
|
26
|
+
end
|
27
|
+
end
|
21
28
|
|
22
29
|
def match_annotations(annotations)
|
23
30
|
result = {}
|
@@ -32,71 +39,66 @@ module KubernetesMetadata
|
|
32
39
|
end
|
33
40
|
|
34
41
|
def parse_namespace_metadata(namespace_object)
|
35
|
-
labels =
|
36
|
-
labels = syms_to_strs(namespace_object[
|
42
|
+
labels = ''
|
43
|
+
labels = syms_to_strs(namespace_object[:metadata][:labels].to_h) unless @skip_labels
|
44
|
+
|
45
|
+
annotations = match_annotations(syms_to_strs(namespace_object[:metadata][:annotations].to_h))
|
37
46
|
|
38
|
-
annotations = match_annotations(syms_to_strs(namespace_object['metadata']['annotations'].to_h))
|
39
|
-
if @de_dot
|
40
|
-
self.de_dot!(labels) unless @skip_labels
|
41
|
-
self.de_dot!(annotations)
|
42
|
-
end
|
43
47
|
kubernetes_metadata = {
|
44
|
-
'namespace_id' => namespace_object[
|
45
|
-
'creation_timestamp' => namespace_object[
|
48
|
+
'namespace_id' => namespace_object[:metadata][:uid],
|
49
|
+
'creation_timestamp' => namespace_object[:metadata][:creationTimestamp]
|
46
50
|
}
|
47
51
|
kubernetes_metadata['namespace_labels'] = labels unless labels.empty?
|
48
52
|
kubernetes_metadata['namespace_annotations'] = annotations unless annotations.empty?
|
49
|
-
|
53
|
+
kubernetes_metadata
|
50
54
|
end
|
51
55
|
|
52
56
|
def parse_pod_metadata(pod_object)
|
53
|
-
labels =
|
54
|
-
labels = syms_to_strs(pod_object[
|
57
|
+
labels = ''
|
58
|
+
labels = syms_to_strs(pod_object[:metadata][:labels].to_h) unless @skip_labels
|
55
59
|
|
56
|
-
annotations = match_annotations(syms_to_strs(pod_object[
|
57
|
-
if @de_dot
|
58
|
-
self.de_dot!(labels) unless @skip_labels
|
59
|
-
self.de_dot!(annotations)
|
60
|
-
end
|
60
|
+
annotations = match_annotations(syms_to_strs(pod_object[:metadata][:annotations].to_h))
|
61
61
|
|
62
|
-
# collect container
|
62
|
+
# collect container information
|
63
63
|
container_meta = {}
|
64
64
|
begin
|
65
|
-
pod_object[
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
65
|
+
pod_object[:status][:containerStatuses].each do |container_status|
|
66
|
+
container_id = (container_status[:containerID]||"").sub(%r{^[-_a-zA-Z0-9]+://}, '')
|
67
|
+
key = container_status[:name]
|
68
|
+
container_meta[key] = if @skip_container_metadata
|
69
|
+
{
|
70
|
+
'name' => container_status[:name]
|
71
|
+
}
|
72
|
+
else
|
73
|
+
{
|
74
|
+
'name' => container_status[:name],
|
75
|
+
'image' => container_status[:image],
|
76
|
+
'image_id' => container_status[:imageID],
|
77
|
+
:containerID => container_id
|
78
|
+
}
|
79
|
+
end
|
80
|
+
end if pod_object[:status] && pod_object[:status][:containerStatuses]
|
81
|
+
rescue StandardError=>e
|
82
|
+
log.warn("parsing container meta information failed for: #{pod_object[:metadata][:namespace]}/#{pod_object[:metadata][:name]}: #{e}")
|
82
83
|
end
|
83
84
|
|
84
85
|
kubernetes_metadata = {
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
86
|
+
'namespace_name' => pod_object[:metadata][:namespace],
|
87
|
+
'pod_id' => pod_object[:metadata][:uid],
|
88
|
+
'pod_name' => pod_object[:metadata][:name],
|
89
|
+
'pod_ip' => pod_object[:status][:podIP],
|
90
|
+
'containers' => syms_to_strs(container_meta),
|
91
|
+
'host' => pod_object[:spec][:nodeName]
|
90
92
|
}
|
91
93
|
kubernetes_metadata['annotations'] = annotations unless annotations.empty?
|
92
94
|
kubernetes_metadata['labels'] = labels unless labels.empty?
|
93
95
|
kubernetes_metadata['master_url'] = @kubernetes_url unless @skip_master_url
|
94
|
-
|
96
|
+
kubernetes_metadata
|
95
97
|
end
|
96
98
|
|
97
99
|
def syms_to_strs(hsh)
|
98
100
|
newhsh = {}
|
99
|
-
hsh.each_pair do |kk,vv|
|
101
|
+
hsh.each_pair do |kk, vv|
|
100
102
|
if vv.is_a?(Hash)
|
101
103
|
vv = syms_to_strs(vv)
|
102
104
|
end
|
@@ -108,32 +110,5 @@ module KubernetesMetadata
|
|
108
110
|
end
|
109
111
|
newhsh
|
110
112
|
end
|
111
|
-
|
112
|
-
end
|
113
|
-
end
|
114
|
-
|
115
|
-
# copied from activesupport
|
116
|
-
class Object
|
117
|
-
# An object is blank if it's false, empty, or a whitespace string.
|
118
|
-
# For example, +nil+, '', ' ', [], {}, and +false+ are all blank.
|
119
|
-
#
|
120
|
-
# This simplifies
|
121
|
-
#
|
122
|
-
# !address || address.empty?
|
123
|
-
#
|
124
|
-
# to
|
125
|
-
#
|
126
|
-
# address.blank?
|
127
|
-
#
|
128
|
-
# @return [true, false]
|
129
|
-
def blank?
|
130
|
-
respond_to?(:empty?) ? !!empty? : !self
|
131
|
-
end
|
132
|
-
|
133
|
-
# An object is present if it's not blank.
|
134
|
-
#
|
135
|
-
# @return [true, false]
|
136
|
-
def present?
|
137
|
-
!blank?
|
138
113
|
end
|
139
114
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
#
|
2
4
|
# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
|
3
5
|
# Kubernetes metadata
|
@@ -19,17 +21,16 @@
|
|
19
21
|
require 'lru_redux'
|
20
22
|
module KubernetesMetadata
|
21
23
|
class Stats
|
22
|
-
|
23
24
|
def initialize
|
24
25
|
@stats = ::LruRedux::TTL::ThreadSafeCache.new(1000, 3600)
|
25
26
|
end
|
26
27
|
|
27
28
|
def bump(key)
|
28
|
-
|
29
|
+
@stats[key] = @stats.getset(key) { 0 } + 1
|
29
30
|
end
|
30
31
|
|
31
32
|
def set(key, value)
|
32
|
-
|
33
|
+
@stats[key] = value
|
33
34
|
end
|
34
35
|
|
35
36
|
def [](key)
|
@@ -37,10 +38,25 @@ module KubernetesMetadata
|
|
37
38
|
end
|
38
39
|
|
39
40
|
def to_s
|
40
|
-
|
41
|
-
|
41
|
+
'stats - ' + [].tap do |a|
|
42
|
+
@stats.each { |k, v| a << "#{k}: #{v}" }
|
42
43
|
end.join(', ')
|
43
44
|
end
|
45
|
+
end
|
46
|
+
class NoOpStats
|
47
|
+
def initialize
|
48
|
+
end
|
49
|
+
|
50
|
+
def bump(key)
|
51
|
+
end
|
44
52
|
|
53
|
+
def set(key, value)
|
54
|
+
end
|
55
|
+
|
56
|
+
def [](key)
|
57
|
+
end
|
58
|
+
|
59
|
+
def to_s
|
60
|
+
end
|
45
61
|
end
|
46
62
|
end
|
@@ -0,0 +1,68 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
|
5
|
+
# Kubernetes metadata
|
6
|
+
#
|
7
|
+
# Copyright 2021 Red Hat, Inc.
|
8
|
+
#
|
9
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
10
|
+
# you may not use this file except in compliance with the License.
|
11
|
+
# You may obtain a copy of the License at
|
12
|
+
#
|
13
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
14
|
+
#
|
15
|
+
# Unless required by applicable law or agreed to in writing, software
|
16
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
17
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
18
|
+
# See the License for the specific language governing permissions and
|
19
|
+
# limitations under the License.
|
20
|
+
#
|
21
|
+
require 'kubeclient'
|
22
|
+
|
23
|
+
module KubernetesMetadata
|
24
|
+
module TestApiAdapter
|
25
|
+
|
26
|
+
def api_valid?
|
27
|
+
true
|
28
|
+
end
|
29
|
+
def get_namespace(namespace_name, unused, options)
|
30
|
+
return {
|
31
|
+
metadata: {
|
32
|
+
name: namespace_name,
|
33
|
+
uid: namespace_name + 'uuid',
|
34
|
+
labels: {
|
35
|
+
foo_ns: 'bar_ns'
|
36
|
+
}
|
37
|
+
}
|
38
|
+
}
|
39
|
+
end
|
40
|
+
|
41
|
+
def get_pod(pod_name, namespace_name, options)
|
42
|
+
return {
|
43
|
+
metadata: {
|
44
|
+
name: pod_name,
|
45
|
+
namespace: namespace_name,
|
46
|
+
uid: namespace_name + namespace_name + "uuid",
|
47
|
+
labels: {
|
48
|
+
foo: 'bar'
|
49
|
+
}
|
50
|
+
},
|
51
|
+
spec: {
|
52
|
+
nodeName: 'aNodeName',
|
53
|
+
containers: [{
|
54
|
+
name: 'foo',
|
55
|
+
image: 'bar'
|
56
|
+
}, {
|
57
|
+
name: 'bar',
|
58
|
+
image: 'foo'
|
59
|
+
}]
|
60
|
+
},
|
61
|
+
status: {
|
62
|
+
podIP: '172.17.0.8'
|
63
|
+
}
|
64
|
+
}
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
68
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
|
5
|
+
# Kubernetes metadata
|
6
|
+
#
|
7
|
+
# Copyright 2021 Red Hat, Inc.
|
8
|
+
#
|
9
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
10
|
+
# you may not use this file except in compliance with the License.
|
11
|
+
# You may obtain a copy of the License at
|
12
|
+
#
|
13
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
14
|
+
#
|
15
|
+
# Unless required by applicable law or agreed to in writing, software
|
16
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
17
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
18
|
+
# See the License for the specific language governing permissions and
|
19
|
+
# limitations under the License.
|
20
|
+
#
|
21
|
+
|
22
|
+
#https://stackoverflow.com/questions/5622435/how-do-i-convert-a-ruby-class-name-to-a-underscore-delimited-symbol
|
23
|
+
class String
|
24
|
+
def underscore
|
25
|
+
word = self.dup
|
26
|
+
word.gsub!(/::/, '_')
|
27
|
+
word.gsub!(/([A-Z]+)([A-Z][a-z])/,'\1_\2')
|
28
|
+
word.gsub!(/([a-z\d])([A-Z])/,'\1_\2')
|
29
|
+
word.tr!("-", "_")
|
30
|
+
word.downcase!
|
31
|
+
word
|
32
|
+
end
|
33
|
+
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
#
|
2
4
|
# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
|
3
5
|
# Kubernetes metadata
|
@@ -16,11 +18,11 @@
|
|
16
18
|
# See the License for the specific language governing permissions and
|
17
19
|
# limitations under the License.
|
18
20
|
#
|
21
|
+
# TODO: this is mostly copy-paste from kubernetes_metadata_watch_pods.rb unify them
|
19
22
|
require_relative 'kubernetes_metadata_common'
|
20
23
|
|
21
24
|
module KubernetesMetadata
|
22
25
|
module WatchNamespaces
|
23
|
-
|
24
26
|
include ::KubernetesMetadata::Common
|
25
27
|
|
26
28
|
def set_up_namespace_thread
|
@@ -35,20 +37,33 @@ module KubernetesMetadata
|
|
35
37
|
# processing will be swallowed and retried. These failures /
|
36
38
|
# exceptions could be caused by Kubernetes API being temporarily
|
37
39
|
# down. We assume the configuration is correct at this point.
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
40
|
+
loop do
|
41
|
+
namespace_watcher ||= get_namespaces_and_start_watcher
|
42
|
+
process_namespace_watcher_notices(namespace_watcher)
|
43
|
+
rescue GoneError => e
|
44
|
+
# Expected error. Quietly go back through the loop in order to
|
45
|
+
# start watching from the latest resource versions
|
46
|
+
@stats.bump(:namespace_watch_gone_errors)
|
47
|
+
log.info('410 Gone encountered. Restarting namespace watch to reset resource versions.', e)
|
48
|
+
namespace_watcher = nil
|
49
|
+
rescue KubeException => e
|
50
|
+
if e.error_code == 401
|
51
|
+
# recreate client to refresh token
|
52
|
+
log.info("Encountered '401 Unauthorized' exception in watch, recreating client to refresh token")
|
53
|
+
create_client()
|
54
|
+
namespace_watcher = nil
|
55
|
+
else
|
56
|
+
# treat all other errors the same as StandardError, log, swallow and reset
|
43
57
|
@stats.bump(:namespace_watch_failures)
|
44
58
|
if Thread.current[:namespace_watch_retry_count] < @watch_retry_max_times
|
45
59
|
# Instead of raising exceptions and crashing Fluentd, swallow
|
46
60
|
# the exception and reset the watcher.
|
47
61
|
log.info(
|
48
|
-
|
49
|
-
|
62
|
+
'Exception encountered parsing namespace watch event. ' \
|
63
|
+
'The connection might have been closed. Sleeping for ' \
|
50
64
|
"#{Thread.current[:namespace_watch_retry_backoff_interval]} " \
|
51
|
-
|
65
|
+
'seconds and resetting the namespace watcher.', e
|
66
|
+
)
|
52
67
|
sleep(Thread.current[:namespace_watch_retry_backoff_interval])
|
53
68
|
Thread.current[:namespace_watch_retry_count] += 1
|
54
69
|
Thread.current[:namespace_watch_retry_backoff_interval] *= @watch_retry_exponential_backoff_base
|
@@ -57,20 +72,45 @@ module KubernetesMetadata
|
|
57
72
|
# Since retries failed for many times, log as errors instead
|
58
73
|
# of info and raise exceptions and trigger Fluentd to restart.
|
59
74
|
message =
|
60
|
-
|
61
|
-
|
75
|
+
'Exception encountered parsing namespace watch event. The ' \
|
76
|
+
'connection might have been closed. Retried ' \
|
62
77
|
"#{@watch_retry_max_times} times yet still failing. Restarting."
|
63
78
|
log.error(message, e)
|
64
|
-
raise Fluent::UnrecoverableError
|
79
|
+
raise Fluent::UnrecoverableError, message
|
65
80
|
end
|
66
81
|
end
|
82
|
+
rescue StandardError => e
|
83
|
+
@stats.bump(:namespace_watch_failures)
|
84
|
+
if Thread.current[:namespace_watch_retry_count] < @watch_retry_max_times
|
85
|
+
# Instead of raising exceptions and crashing Fluentd, swallow
|
86
|
+
# the exception and reset the watcher.
|
87
|
+
log.info(
|
88
|
+
'Exception encountered parsing namespace watch event. ' \
|
89
|
+
'The connection might have been closed. Sleeping for ' \
|
90
|
+
"#{Thread.current[:namespace_watch_retry_backoff_interval]} " \
|
91
|
+
'seconds and resetting the namespace watcher.', e
|
92
|
+
)
|
93
|
+
sleep(Thread.current[:namespace_watch_retry_backoff_interval])
|
94
|
+
Thread.current[:namespace_watch_retry_count] += 1
|
95
|
+
Thread.current[:namespace_watch_retry_backoff_interval] *= @watch_retry_exponential_backoff_base
|
96
|
+
namespace_watcher = nil
|
97
|
+
else
|
98
|
+
# Since retries failed for many times, log as errors instead
|
99
|
+
# of info and raise exceptions and trigger Fluentd to restart.
|
100
|
+
message =
|
101
|
+
'Exception encountered parsing namespace watch event. The ' \
|
102
|
+
'connection might have been closed. Retried ' \
|
103
|
+
"#{@watch_retry_max_times} times yet still failing. Restarting."
|
104
|
+
log.error(message, e)
|
105
|
+
raise Fluent::UnrecoverableError, message
|
106
|
+
end
|
67
107
|
end
|
68
108
|
end
|
69
109
|
|
70
110
|
def start_namespace_watch
|
71
|
-
|
72
|
-
rescue
|
73
|
-
message =
|
111
|
+
get_namespaces_and_start_watcher
|
112
|
+
rescue StandardError => e
|
113
|
+
message = 'start_namespace_watch: Exception encountered setting up ' \
|
74
114
|
"namespace watch from Kubernetes API #{@apiVersion} endpoint " \
|
75
115
|
"#{@kubernetes_url}: #{e.message}"
|
76
116
|
message += " (#{e.response})" if e.respond_to?(:response)
|
@@ -83,16 +123,20 @@ module KubernetesMetadata
|
|
83
123
|
# starting from that resourceVersion.
|
84
124
|
def get_namespaces_and_start_watcher
|
85
125
|
options = {
|
86
|
-
resource_version: '0'
|
126
|
+
resource_version: '0' # Fetch from API server cache instead of etcd quorum read
|
87
127
|
}
|
88
128
|
namespaces = @client.get_namespaces(options)
|
89
|
-
namespaces.each do |namespace|
|
90
|
-
cache_key = namespace
|
129
|
+
namespaces[:items].each do |namespace|
|
130
|
+
cache_key = namespace[:metadata][:uid]
|
91
131
|
@namespace_cache[cache_key] = parse_namespace_metadata(namespace)
|
92
132
|
@stats.bump(:namespace_cache_host_updates)
|
93
133
|
end
|
94
|
-
|
134
|
+
|
135
|
+
# continue watching from most recent resourceVersion
|
136
|
+
options[:resource_version] = namespaces[:metadata][:resourceVersion]
|
137
|
+
|
95
138
|
watcher = @client.watch_namespaces(options)
|
139
|
+
reset_namespace_watch_retry_stats
|
96
140
|
watcher
|
97
141
|
end
|
98
142
|
|
@@ -106,31 +150,36 @@ module KubernetesMetadata
|
|
106
150
|
# Process a watcher notice and potentially raise an exception.
|
107
151
|
def process_namespace_watcher_notices(watcher)
|
108
152
|
watcher.each do |notice|
|
109
|
-
case notice
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
153
|
+
case notice[:type]
|
154
|
+
when 'MODIFIED'
|
155
|
+
reset_namespace_watch_retry_stats
|
156
|
+
cache_key = notice[:object][:metadata][:uid]
|
157
|
+
cached = @namespace_cache[cache_key]
|
158
|
+
if cached
|
159
|
+
@namespace_cache[cache_key] = parse_namespace_metadata(notice[:object])
|
160
|
+
@stats.bump(:namespace_cache_watch_updates)
|
161
|
+
else
|
162
|
+
@stats.bump(:namespace_cache_watch_misses)
|
163
|
+
end
|
164
|
+
when 'DELETED'
|
165
|
+
reset_namespace_watch_retry_stats
|
166
|
+
# ignore and let age out for cases where
|
167
|
+
# deleted but still processing logs
|
168
|
+
@stats.bump(:namespace_cache_watch_deletes_ignored)
|
169
|
+
when 'ERROR'
|
170
|
+
if notice[:object] && notice[:object][:code] == 410
|
171
|
+
@stats.bump(:namespace_watch_gone_notices)
|
172
|
+
raise GoneError
|
173
|
+
else
|
126
174
|
@stats.bump(:namespace_watch_error_type_notices)
|
127
|
-
message = notice[
|
175
|
+
message = notice[:object][:message] if notice[:object] && notice[:object][:message]
|
128
176
|
raise "Error while watching namespaces: #{message}"
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
177
|
+
end
|
178
|
+
else
|
179
|
+
reset_namespace_watch_retry_stats
|
180
|
+
# Don't pay attention to creations, since the created namespace may not
|
181
|
+
# be used by any namespace on this node.
|
182
|
+
@stats.bump(:namespace_cache_watch_ignored)
|
134
183
|
end
|
135
184
|
end
|
136
185
|
end
|