fluent-plugin-kubernetes_metadata_filter 2.5.0 → 3.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +10 -14
- data/.gitignore +0 -1
- data/.rubocop.yml +57 -0
- data/Gemfile +4 -2
- data/Gemfile.lock +76 -67
- data/README.md +9 -83
- data/Rakefile +15 -11
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/cpu.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/latency.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/loss.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/mem.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/readme.md +88 -0
- data/doc/benchmark/5m-1-2500lps-256b-baseline-01/results.html +127 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/cpu.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/latency.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/loss.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/mem.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/readme.md +97 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-01/results.html +136 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/cpu.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/latency.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/loss.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/mem.png +0 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/readme.md +97 -0
- data/doc/benchmark/5m-1-2500lps-256b-kube-01-marshal-02/results.html +136 -0
- data/fluent-plugin-kubernetes_metadata_filter.gemspec +25 -27
- data/lib/fluent/plugin/filter_kubernetes_metadata.rb +171 -192
- data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb +25 -23
- data/lib/fluent/plugin/kubernetes_metadata_common.rb +44 -69
- data/lib/fluent/plugin/kubernetes_metadata_stats.rb +21 -5
- data/lib/fluent/plugin/kubernetes_metadata_test_api_adapter.rb +68 -0
- data/lib/fluent/plugin/kubernetes_metadata_util.rb +33 -0
- data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +91 -42
- data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +108 -47
- data/release_notes.md +42 -0
- data/test/cassettes/kubernetes_get_pod_container_init.yml +145 -0
- data/test/helper.rb +20 -2
- data/test/plugin/test_cache_stats.rb +10 -13
- data/test/plugin/test_cache_strategy.rb +158 -160
- data/test/plugin/test_filter_kubernetes_metadata.rb +340 -616
- data/test/plugin/test_watch_namespaces.rb +188 -125
- data/test/plugin/test_watch_pods.rb +282 -202
- data/test/plugin/watch_test.rb +16 -15
- metadata +77 -67
- /data/test/cassettes/{kubernetes_docker_metadata_dotted_labels.yml → kubernetes_docker_metadata_dotted_slashed_labels.yml} +0 -0
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
#
|
2
4
|
# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
|
3
5
|
# Kubernetes metadata
|
@@ -18,6 +20,11 @@
|
|
18
20
|
#
|
19
21
|
module KubernetesMetadata
|
20
22
|
module Common
|
23
|
+
class GoneError < StandardError
|
24
|
+
def initialize(msg = '410 Gone')
|
25
|
+
super
|
26
|
+
end
|
27
|
+
end
|
21
28
|
|
22
29
|
def match_annotations(annotations)
|
23
30
|
result = {}
|
@@ -32,71 +39,66 @@ module KubernetesMetadata
|
|
32
39
|
end
|
33
40
|
|
34
41
|
def parse_namespace_metadata(namespace_object)
|
35
|
-
labels =
|
36
|
-
labels = syms_to_strs(namespace_object[
|
42
|
+
labels = ''
|
43
|
+
labels = syms_to_strs(namespace_object[:metadata][:labels].to_h) unless @skip_labels
|
44
|
+
|
45
|
+
annotations = match_annotations(syms_to_strs(namespace_object[:metadata][:annotations].to_h))
|
37
46
|
|
38
|
-
annotations = match_annotations(syms_to_strs(namespace_object['metadata']['annotations'].to_h))
|
39
|
-
if @de_dot
|
40
|
-
self.de_dot!(labels) unless @skip_labels
|
41
|
-
self.de_dot!(annotations)
|
42
|
-
end
|
43
47
|
kubernetes_metadata = {
|
44
|
-
'namespace_id' => namespace_object[
|
45
|
-
'creation_timestamp' => namespace_object[
|
48
|
+
'namespace_id' => namespace_object[:metadata][:uid],
|
49
|
+
'creation_timestamp' => namespace_object[:metadata][:creationTimestamp]
|
46
50
|
}
|
47
51
|
kubernetes_metadata['namespace_labels'] = labels unless labels.empty?
|
48
52
|
kubernetes_metadata['namespace_annotations'] = annotations unless annotations.empty?
|
49
|
-
|
53
|
+
kubernetes_metadata
|
50
54
|
end
|
51
55
|
|
52
56
|
def parse_pod_metadata(pod_object)
|
53
|
-
labels =
|
54
|
-
labels = syms_to_strs(pod_object[
|
57
|
+
labels = ''
|
58
|
+
labels = syms_to_strs(pod_object[:metadata][:labels].to_h) unless @skip_labels
|
55
59
|
|
56
|
-
annotations = match_annotations(syms_to_strs(pod_object[
|
57
|
-
if @de_dot
|
58
|
-
self.de_dot!(labels) unless @skip_labels
|
59
|
-
self.de_dot!(annotations)
|
60
|
-
end
|
60
|
+
annotations = match_annotations(syms_to_strs(pod_object[:metadata][:annotations].to_h))
|
61
61
|
|
62
|
-
# collect container
|
62
|
+
# collect container information
|
63
63
|
container_meta = {}
|
64
64
|
begin
|
65
|
-
pod_object[
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
65
|
+
pod_object[:status][:containerStatuses].each do |container_status|
|
66
|
+
container_id = (container_status[:containerID]||"").sub(%r{^[-_a-zA-Z0-9]+://}, '')
|
67
|
+
key = container_status[:name]
|
68
|
+
container_meta[key] = if @skip_container_metadata
|
69
|
+
{
|
70
|
+
'name' => container_status[:name]
|
71
|
+
}
|
72
|
+
else
|
73
|
+
{
|
74
|
+
'name' => container_status[:name],
|
75
|
+
'image' => container_status[:image],
|
76
|
+
'image_id' => container_status[:imageID],
|
77
|
+
:containerID => container_id
|
78
|
+
}
|
79
|
+
end
|
80
|
+
end if pod_object[:status] && pod_object[:status][:containerStatuses]
|
81
|
+
rescue StandardError=>e
|
82
|
+
log.warn("parsing container meta information failed for: #{pod_object[:metadata][:namespace]}/#{pod_object[:metadata][:name]}: #{e}")
|
82
83
|
end
|
83
84
|
|
84
85
|
kubernetes_metadata = {
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
86
|
+
'namespace_name' => pod_object[:metadata][:namespace],
|
87
|
+
'pod_id' => pod_object[:metadata][:uid],
|
88
|
+
'pod_name' => pod_object[:metadata][:name],
|
89
|
+
'pod_ip' => pod_object[:status][:podIP],
|
90
|
+
'containers' => syms_to_strs(container_meta),
|
91
|
+
'host' => pod_object[:spec][:nodeName]
|
90
92
|
}
|
91
93
|
kubernetes_metadata['annotations'] = annotations unless annotations.empty?
|
92
94
|
kubernetes_metadata['labels'] = labels unless labels.empty?
|
93
95
|
kubernetes_metadata['master_url'] = @kubernetes_url unless @skip_master_url
|
94
|
-
|
96
|
+
kubernetes_metadata
|
95
97
|
end
|
96
98
|
|
97
99
|
def syms_to_strs(hsh)
|
98
100
|
newhsh = {}
|
99
|
-
hsh.each_pair do |kk,vv|
|
101
|
+
hsh.each_pair do |kk, vv|
|
100
102
|
if vv.is_a?(Hash)
|
101
103
|
vv = syms_to_strs(vv)
|
102
104
|
end
|
@@ -108,32 +110,5 @@ module KubernetesMetadata
|
|
108
110
|
end
|
109
111
|
newhsh
|
110
112
|
end
|
111
|
-
|
112
|
-
end
|
113
|
-
end
|
114
|
-
|
115
|
-
# copied from activesupport
|
116
|
-
class Object
|
117
|
-
# An object is blank if it's false, empty, or a whitespace string.
|
118
|
-
# For example, +nil+, '', ' ', [], {}, and +false+ are all blank.
|
119
|
-
#
|
120
|
-
# This simplifies
|
121
|
-
#
|
122
|
-
# !address || address.empty?
|
123
|
-
#
|
124
|
-
# to
|
125
|
-
#
|
126
|
-
# address.blank?
|
127
|
-
#
|
128
|
-
# @return [true, false]
|
129
|
-
def blank?
|
130
|
-
respond_to?(:empty?) ? !!empty? : !self
|
131
|
-
end
|
132
|
-
|
133
|
-
# An object is present if it's not blank.
|
134
|
-
#
|
135
|
-
# @return [true, false]
|
136
|
-
def present?
|
137
|
-
!blank?
|
138
113
|
end
|
139
114
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
#
|
2
4
|
# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
|
3
5
|
# Kubernetes metadata
|
@@ -19,17 +21,16 @@
|
|
19
21
|
require 'lru_redux'
|
20
22
|
module KubernetesMetadata
|
21
23
|
class Stats
|
22
|
-
|
23
24
|
def initialize
|
24
25
|
@stats = ::LruRedux::TTL::ThreadSafeCache.new(1000, 3600)
|
25
26
|
end
|
26
27
|
|
27
28
|
def bump(key)
|
28
|
-
|
29
|
+
@stats[key] = @stats.getset(key) { 0 } + 1
|
29
30
|
end
|
30
31
|
|
31
32
|
def set(key, value)
|
32
|
-
|
33
|
+
@stats[key] = value
|
33
34
|
end
|
34
35
|
|
35
36
|
def [](key)
|
@@ -37,10 +38,25 @@ module KubernetesMetadata
|
|
37
38
|
end
|
38
39
|
|
39
40
|
def to_s
|
40
|
-
|
41
|
-
|
41
|
+
'stats - ' + [].tap do |a|
|
42
|
+
@stats.each { |k, v| a << "#{k}: #{v}" }
|
42
43
|
end.join(', ')
|
43
44
|
end
|
45
|
+
end
|
46
|
+
class NoOpStats
|
47
|
+
def initialize
|
48
|
+
end
|
49
|
+
|
50
|
+
def bump(key)
|
51
|
+
end
|
44
52
|
|
53
|
+
def set(key, value)
|
54
|
+
end
|
55
|
+
|
56
|
+
def [](key)
|
57
|
+
end
|
58
|
+
|
59
|
+
def to_s
|
60
|
+
end
|
45
61
|
end
|
46
62
|
end
|
@@ -0,0 +1,68 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
|
5
|
+
# Kubernetes metadata
|
6
|
+
#
|
7
|
+
# Copyright 2021 Red Hat, Inc.
|
8
|
+
#
|
9
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
10
|
+
# you may not use this file except in compliance with the License.
|
11
|
+
# You may obtain a copy of the License at
|
12
|
+
#
|
13
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
14
|
+
#
|
15
|
+
# Unless required by applicable law or agreed to in writing, software
|
16
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
17
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
18
|
+
# See the License for the specific language governing permissions and
|
19
|
+
# limitations under the License.
|
20
|
+
#
|
21
|
+
require 'kubeclient'
|
22
|
+
|
23
|
+
module KubernetesMetadata
|
24
|
+
module TestApiAdapter
|
25
|
+
|
26
|
+
def api_valid?
|
27
|
+
true
|
28
|
+
end
|
29
|
+
def get_namespace(namespace_name, unused, options)
|
30
|
+
return {
|
31
|
+
metadata: {
|
32
|
+
name: namespace_name,
|
33
|
+
uid: namespace_name + 'uuid',
|
34
|
+
labels: {
|
35
|
+
foo_ns: 'bar_ns'
|
36
|
+
}
|
37
|
+
}
|
38
|
+
}
|
39
|
+
end
|
40
|
+
|
41
|
+
def get_pod(pod_name, namespace_name, options)
|
42
|
+
return {
|
43
|
+
metadata: {
|
44
|
+
name: pod_name,
|
45
|
+
namespace: namespace_name,
|
46
|
+
uid: namespace_name + namespace_name + "uuid",
|
47
|
+
labels: {
|
48
|
+
foo: 'bar'
|
49
|
+
}
|
50
|
+
},
|
51
|
+
spec: {
|
52
|
+
nodeName: 'aNodeName',
|
53
|
+
containers: [{
|
54
|
+
name: 'foo',
|
55
|
+
image: 'bar'
|
56
|
+
}, {
|
57
|
+
name: 'bar',
|
58
|
+
image: 'foo'
|
59
|
+
}]
|
60
|
+
},
|
61
|
+
status: {
|
62
|
+
podIP: '172.17.0.8'
|
63
|
+
}
|
64
|
+
}
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
68
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
|
5
|
+
# Kubernetes metadata
|
6
|
+
#
|
7
|
+
# Copyright 2021 Red Hat, Inc.
|
8
|
+
#
|
9
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
10
|
+
# you may not use this file except in compliance with the License.
|
11
|
+
# You may obtain a copy of the License at
|
12
|
+
#
|
13
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
14
|
+
#
|
15
|
+
# Unless required by applicable law or agreed to in writing, software
|
16
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
17
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
18
|
+
# See the License for the specific language governing permissions and
|
19
|
+
# limitations under the License.
|
20
|
+
#
|
21
|
+
|
22
|
+
#https://stackoverflow.com/questions/5622435/how-do-i-convert-a-ruby-class-name-to-a-underscore-delimited-symbol
|
23
|
+
class String
|
24
|
+
def underscore
|
25
|
+
word = self.dup
|
26
|
+
word.gsub!(/::/, '_')
|
27
|
+
word.gsub!(/([A-Z]+)([A-Z][a-z])/,'\1_\2')
|
28
|
+
word.gsub!(/([a-z\d])([A-Z])/,'\1_\2')
|
29
|
+
word.tr!("-", "_")
|
30
|
+
word.downcase!
|
31
|
+
word
|
32
|
+
end
|
33
|
+
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
#
|
2
4
|
# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
|
3
5
|
# Kubernetes metadata
|
@@ -16,11 +18,11 @@
|
|
16
18
|
# See the License for the specific language governing permissions and
|
17
19
|
# limitations under the License.
|
18
20
|
#
|
21
|
+
# TODO: this is mostly copy-paste from kubernetes_metadata_watch_pods.rb unify them
|
19
22
|
require_relative 'kubernetes_metadata_common'
|
20
23
|
|
21
24
|
module KubernetesMetadata
|
22
25
|
module WatchNamespaces
|
23
|
-
|
24
26
|
include ::KubernetesMetadata::Common
|
25
27
|
|
26
28
|
def set_up_namespace_thread
|
@@ -35,20 +37,33 @@ module KubernetesMetadata
|
|
35
37
|
# processing will be swallowed and retried. These failures /
|
36
38
|
# exceptions could be caused by Kubernetes API being temporarily
|
37
39
|
# down. We assume the configuration is correct at this point.
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
40
|
+
loop do
|
41
|
+
namespace_watcher ||= get_namespaces_and_start_watcher
|
42
|
+
process_namespace_watcher_notices(namespace_watcher)
|
43
|
+
rescue GoneError => e
|
44
|
+
# Expected error. Quietly go back through the loop in order to
|
45
|
+
# start watching from the latest resource versions
|
46
|
+
@stats.bump(:namespace_watch_gone_errors)
|
47
|
+
log.info('410 Gone encountered. Restarting namespace watch to reset resource versions.', e)
|
48
|
+
namespace_watcher = nil
|
49
|
+
rescue KubeException => e
|
50
|
+
if e.error_code == 401
|
51
|
+
# recreate client to refresh token
|
52
|
+
log.info("Encountered '401 Unauthorized' exception in watch, recreating client to refresh token")
|
53
|
+
create_client()
|
54
|
+
namespace_watcher = nil
|
55
|
+
else
|
56
|
+
# treat all other errors the same as StandardError, log, swallow and reset
|
43
57
|
@stats.bump(:namespace_watch_failures)
|
44
58
|
if Thread.current[:namespace_watch_retry_count] < @watch_retry_max_times
|
45
59
|
# Instead of raising exceptions and crashing Fluentd, swallow
|
46
60
|
# the exception and reset the watcher.
|
47
61
|
log.info(
|
48
|
-
|
49
|
-
|
62
|
+
'Exception encountered parsing namespace watch event. ' \
|
63
|
+
'The connection might have been closed. Sleeping for ' \
|
50
64
|
"#{Thread.current[:namespace_watch_retry_backoff_interval]} " \
|
51
|
-
|
65
|
+
'seconds and resetting the namespace watcher.', e
|
66
|
+
)
|
52
67
|
sleep(Thread.current[:namespace_watch_retry_backoff_interval])
|
53
68
|
Thread.current[:namespace_watch_retry_count] += 1
|
54
69
|
Thread.current[:namespace_watch_retry_backoff_interval] *= @watch_retry_exponential_backoff_base
|
@@ -57,20 +72,45 @@ module KubernetesMetadata
|
|
57
72
|
# Since retries failed for many times, log as errors instead
|
58
73
|
# of info and raise exceptions and trigger Fluentd to restart.
|
59
74
|
message =
|
60
|
-
|
61
|
-
|
75
|
+
'Exception encountered parsing namespace watch event. The ' \
|
76
|
+
'connection might have been closed. Retried ' \
|
62
77
|
"#{@watch_retry_max_times} times yet still failing. Restarting."
|
63
78
|
log.error(message, e)
|
64
|
-
raise Fluent::UnrecoverableError
|
79
|
+
raise Fluent::UnrecoverableError, message
|
65
80
|
end
|
66
81
|
end
|
82
|
+
rescue StandardError => e
|
83
|
+
@stats.bump(:namespace_watch_failures)
|
84
|
+
if Thread.current[:namespace_watch_retry_count] < @watch_retry_max_times
|
85
|
+
# Instead of raising exceptions and crashing Fluentd, swallow
|
86
|
+
# the exception and reset the watcher.
|
87
|
+
log.info(
|
88
|
+
'Exception encountered parsing namespace watch event. ' \
|
89
|
+
'The connection might have been closed. Sleeping for ' \
|
90
|
+
"#{Thread.current[:namespace_watch_retry_backoff_interval]} " \
|
91
|
+
'seconds and resetting the namespace watcher.', e
|
92
|
+
)
|
93
|
+
sleep(Thread.current[:namespace_watch_retry_backoff_interval])
|
94
|
+
Thread.current[:namespace_watch_retry_count] += 1
|
95
|
+
Thread.current[:namespace_watch_retry_backoff_interval] *= @watch_retry_exponential_backoff_base
|
96
|
+
namespace_watcher = nil
|
97
|
+
else
|
98
|
+
# Since retries failed for many times, log as errors instead
|
99
|
+
# of info and raise exceptions and trigger Fluentd to restart.
|
100
|
+
message =
|
101
|
+
'Exception encountered parsing namespace watch event. The ' \
|
102
|
+
'connection might have been closed. Retried ' \
|
103
|
+
"#{@watch_retry_max_times} times yet still failing. Restarting."
|
104
|
+
log.error(message, e)
|
105
|
+
raise Fluent::UnrecoverableError, message
|
106
|
+
end
|
67
107
|
end
|
68
108
|
end
|
69
109
|
|
70
110
|
def start_namespace_watch
|
71
|
-
|
72
|
-
rescue
|
73
|
-
message =
|
111
|
+
get_namespaces_and_start_watcher
|
112
|
+
rescue StandardError => e
|
113
|
+
message = 'start_namespace_watch: Exception encountered setting up ' \
|
74
114
|
"namespace watch from Kubernetes API #{@apiVersion} endpoint " \
|
75
115
|
"#{@kubernetes_url}: #{e.message}"
|
76
116
|
message += " (#{e.response})" if e.respond_to?(:response)
|
@@ -83,16 +123,20 @@ module KubernetesMetadata
|
|
83
123
|
# starting from that resourceVersion.
|
84
124
|
def get_namespaces_and_start_watcher
|
85
125
|
options = {
|
86
|
-
resource_version: '0'
|
126
|
+
resource_version: '0' # Fetch from API server cache instead of etcd quorum read
|
87
127
|
}
|
88
128
|
namespaces = @client.get_namespaces(options)
|
89
|
-
namespaces.each do |namespace|
|
90
|
-
cache_key = namespace
|
129
|
+
namespaces[:items].each do |namespace|
|
130
|
+
cache_key = namespace[:metadata][:uid]
|
91
131
|
@namespace_cache[cache_key] = parse_namespace_metadata(namespace)
|
92
132
|
@stats.bump(:namespace_cache_host_updates)
|
93
133
|
end
|
94
|
-
|
134
|
+
|
135
|
+
# continue watching from most recent resourceVersion
|
136
|
+
options[:resource_version] = namespaces[:metadata][:resourceVersion]
|
137
|
+
|
95
138
|
watcher = @client.watch_namespaces(options)
|
139
|
+
reset_namespace_watch_retry_stats
|
96
140
|
watcher
|
97
141
|
end
|
98
142
|
|
@@ -106,31 +150,36 @@ module KubernetesMetadata
|
|
106
150
|
# Process a watcher notice and potentially raise an exception.
|
107
151
|
def process_namespace_watcher_notices(watcher)
|
108
152
|
watcher.each do |notice|
|
109
|
-
case notice
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
153
|
+
case notice[:type]
|
154
|
+
when 'MODIFIED'
|
155
|
+
reset_namespace_watch_retry_stats
|
156
|
+
cache_key = notice[:object][:metadata][:uid]
|
157
|
+
cached = @namespace_cache[cache_key]
|
158
|
+
if cached
|
159
|
+
@namespace_cache[cache_key] = parse_namespace_metadata(notice[:object])
|
160
|
+
@stats.bump(:namespace_cache_watch_updates)
|
161
|
+
else
|
162
|
+
@stats.bump(:namespace_cache_watch_misses)
|
163
|
+
end
|
164
|
+
when 'DELETED'
|
165
|
+
reset_namespace_watch_retry_stats
|
166
|
+
# ignore and let age out for cases where
|
167
|
+
# deleted but still processing logs
|
168
|
+
@stats.bump(:namespace_cache_watch_deletes_ignored)
|
169
|
+
when 'ERROR'
|
170
|
+
if notice[:object] && notice[:object][:code] == 410
|
171
|
+
@stats.bump(:namespace_watch_gone_notices)
|
172
|
+
raise GoneError
|
173
|
+
else
|
126
174
|
@stats.bump(:namespace_watch_error_type_notices)
|
127
|
-
message = notice[
|
175
|
+
message = notice[:object][:message] if notice[:object] && notice[:object][:message]
|
128
176
|
raise "Error while watching namespaces: #{message}"
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
177
|
+
end
|
178
|
+
else
|
179
|
+
reset_namespace_watch_retry_stats
|
180
|
+
# Don't pay attention to creations, since the created namespace may not
|
181
|
+
# be used by any namespace on this node.
|
182
|
+
@stats.bump(:namespace_cache_watch_ignored)
|
134
183
|
end
|
135
184
|
end
|
136
185
|
end
|