fluent-plugin-kubernetes_metadata_filter-rh 2.6.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.circleci/config.yml +57 -0
- data/.gitignore +19 -0
- data/.rubocop.yml +57 -0
- data/Gemfile +9 -0
- data/Gemfile.lock +156 -0
- data/LICENSE.txt +201 -0
- data/README.md +253 -0
- data/Rakefile +41 -0
- data/fluent-plugin-kubernetes_metadata_filter.gemspec +34 -0
- data/lib/fluent/plugin/filter_kubernetes_metadata.rb +378 -0
- data/lib/fluent/plugin/kubernetes_metadata_cache_strategy.rb +102 -0
- data/lib/fluent/plugin/kubernetes_metadata_common.rb +120 -0
- data/lib/fluent/plugin/kubernetes_metadata_stats.rb +46 -0
- data/lib/fluent/plugin/kubernetes_metadata_util.rb +40 -0
- data/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb +154 -0
- data/lib/fluent/plugin/kubernetes_metadata_watch_pods.rb +172 -0
- data/test/cassettes/invalid_api_server_config.yml +53 -0
- data/test/cassettes/kubernetes_docker_metadata_annotations.yml +205 -0
- data/test/cassettes/kubernetes_docker_metadata_dotted_labels.yml +197 -0
- data/test/cassettes/kubernetes_get_api_v1.yml +193 -0
- data/test/cassettes/kubernetes_get_api_v1_using_token.yml +195 -0
- data/test/cassettes/kubernetes_get_namespace_default.yml +69 -0
- data/test/cassettes/kubernetes_get_namespace_default_using_token.yml +71 -0
- data/test/cassettes/kubernetes_get_pod.yml +146 -0
- data/test/cassettes/kubernetes_get_pod_using_token.yml +148 -0
- data/test/cassettes/metadata_from_tag_and_journald_fields.yml +153 -0
- data/test/cassettes/metadata_from_tag_journald_and_kubernetes_fields.yml +285 -0
- data/test/cassettes/valid_kubernetes_api_server.yml +55 -0
- data/test/cassettes/valid_kubernetes_api_server_using_token.yml +57 -0
- data/test/helper.rb +82 -0
- data/test/plugin/test.token +1 -0
- data/test/plugin/test_cache_stats.rb +33 -0
- data/test/plugin/test_cache_strategy.rb +194 -0
- data/test/plugin/test_filter_kubernetes_metadata.rb +1012 -0
- data/test/plugin/test_utils.rb +56 -0
- data/test/plugin/test_watch_namespaces.rb +245 -0
- data/test/plugin/test_watch_pods.rb +344 -0
- data/test/plugin/watch_test.rb +74 -0
- metadata +269 -0
@@ -0,0 +1,102 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
|
5
|
+
# Kubernetes metadata
|
6
|
+
#
|
7
|
+
# Copyright 2017 Red Hat, Inc.
|
8
|
+
#
|
9
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
10
|
+
# you may not use this file except in compliance with the License.
|
11
|
+
# You may obtain a copy of the License at
|
12
|
+
#
|
13
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
14
|
+
#
|
15
|
+
# Unless required by applicable law or agreed to in writing, software
|
16
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
17
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
18
|
+
# See the License for the specific language governing permissions and
|
19
|
+
# limitations under the License.
|
20
|
+
#
|
21
|
+
module KubernetesMetadata
|
22
|
+
module CacheStrategy
|
23
|
+
def get_pod_metadata(key, namespace_name, pod_name, record_create_time, batch_miss_cache)
|
24
|
+
metadata = {}
|
25
|
+
ids = @id_cache[key]
|
26
|
+
if ids.nil?
|
27
|
+
# FAST PATH
|
28
|
+
# Cache hit, fetch metadata from the cache
|
29
|
+
@stats.bump(:id_cache_miss)
|
30
|
+
return batch_miss_cache["#{namespace_name}_#{pod_name}"] if batch_miss_cache.key?("#{namespace_name}_#{pod_name}")
|
31
|
+
|
32
|
+
pod_metadata = fetch_pod_metadata(namespace_name, pod_name)
|
33
|
+
if @skip_namespace_metadata
|
34
|
+
ids = { pod_id: pod_metadata['pod_id'] }
|
35
|
+
@id_cache[key] = ids
|
36
|
+
return pod_metadata
|
37
|
+
end
|
38
|
+
namespace_metadata = fetch_namespace_metadata(namespace_name)
|
39
|
+
ids = { pod_id: pod_metadata['pod_id'], namespace_id: namespace_metadata['namespace_id'] }
|
40
|
+
if !ids[:pod_id].nil? && !ids[:namespace_id].nil?
|
41
|
+
# pod found and namespace found
|
42
|
+
metadata = pod_metadata
|
43
|
+
metadata.merge!(namespace_metadata)
|
44
|
+
else
|
45
|
+
if ids[:pod_id].nil? && !ids[:namespace_id].nil?
|
46
|
+
# pod not found, but namespace found
|
47
|
+
@stats.bump(:id_cache_pod_not_found_namespace)
|
48
|
+
ns_time = Time.parse(namespace_metadata['creation_timestamp'])
|
49
|
+
if ns_time <= record_create_time
|
50
|
+
# namespace is older then record for pod
|
51
|
+
ids[:pod_id] = key
|
52
|
+
metadata = @cache.fetch(ids[:pod_id]) do
|
53
|
+
{ 'pod_id' => ids[:pod_id] }
|
54
|
+
end
|
55
|
+
end
|
56
|
+
metadata.merge!(namespace_metadata)
|
57
|
+
else
|
58
|
+
if !ids[:pod_id].nil? && ids[:namespace_id].nil?
|
59
|
+
# pod found, but namespace NOT found
|
60
|
+
# this should NEVER be possible since pod meta can
|
61
|
+
# only be retrieved with a namespace
|
62
|
+
@stats.bump(:id_cache_namespace_not_found_pod)
|
63
|
+
else
|
64
|
+
# nothing found
|
65
|
+
@stats.bump(:id_cache_orphaned_record)
|
66
|
+
end
|
67
|
+
if @allow_orphans
|
68
|
+
log.trace("orphaning message for: #{namespace_name}/#{pod_name} ") if log.trace?
|
69
|
+
metadata = {
|
70
|
+
'orphaned_namespace' => namespace_name,
|
71
|
+
'namespace_name' => @orphaned_namespace_name,
|
72
|
+
'namespace_id' => @orphaned_namespace_id
|
73
|
+
}
|
74
|
+
else
|
75
|
+
metadata = {}
|
76
|
+
end
|
77
|
+
batch_miss_cache["#{namespace_name}_#{pod_name}"] = metadata
|
78
|
+
end
|
79
|
+
end
|
80
|
+
@id_cache[key] = ids unless batch_miss_cache.key?("#{namespace_name}_#{pod_name}")
|
81
|
+
else
|
82
|
+
# SLOW PATH
|
83
|
+
metadata = @cache.fetch(ids[:pod_id]) do
|
84
|
+
@stats.bump(:pod_cache_miss)
|
85
|
+
m = fetch_pod_metadata(namespace_name, pod_name)
|
86
|
+
m.nil? || m.empty? ? { 'pod_id' => ids[:pod_id] } : m
|
87
|
+
end
|
88
|
+
metadata.merge!(@namespace_cache.fetch(ids[:namespace_id]) do
|
89
|
+
m = unless @skip_namespace_metadata
|
90
|
+
@stats.bump(:namespace_cache_miss)
|
91
|
+
fetch_namespace_metadata(namespace_name)
|
92
|
+
end
|
93
|
+
m.nil? || m.empty? ? { 'namespace_id' => ids[:namespace_id] } : m
|
94
|
+
end)
|
95
|
+
end
|
96
|
+
|
97
|
+
# remove namespace info that is only used for comparison
|
98
|
+
metadata.delete('creation_timestamp')
|
99
|
+
metadata.delete_if { |_k, v| v.nil? }
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
@@ -0,0 +1,120 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
|
5
|
+
# Kubernetes metadata
|
6
|
+
#
|
7
|
+
# Copyright 2017 Red Hat, Inc.
|
8
|
+
#
|
9
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
10
|
+
# you may not use this file except in compliance with the License.
|
11
|
+
# You may obtain a copy of the License at
|
12
|
+
#
|
13
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
14
|
+
#
|
15
|
+
# Unless required by applicable law or agreed to in writing, software
|
16
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
17
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
18
|
+
# See the License for the specific language governing permissions and
|
19
|
+
# limitations under the License.
|
20
|
+
#
|
21
|
+
module KubernetesMetadata
|
22
|
+
module Common
|
23
|
+
class GoneError < StandardError
|
24
|
+
def initialize(msg = '410 Gone')
|
25
|
+
super
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def match_annotations(annotations)
|
30
|
+
result = {}
|
31
|
+
@annotations_regexps.each do |regexp|
|
32
|
+
annotations.each do |key, value|
|
33
|
+
if ::Fluent::StringUtil.match_regexp(regexp, key.to_s)
|
34
|
+
result[key] = value
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
result
|
39
|
+
end
|
40
|
+
|
41
|
+
def parse_namespace_metadata(namespace_object)
|
42
|
+
labels = ''
|
43
|
+
labels = syms_to_strs(namespace_object[:metadata][:labels].to_h) unless @skip_labels
|
44
|
+
|
45
|
+
annotations = match_annotations(syms_to_strs(namespace_object[:metadata][:annotations].to_h))
|
46
|
+
if @de_dot
|
47
|
+
de_dot!(labels) unless @skip_labels
|
48
|
+
de_dot!(annotations)
|
49
|
+
end
|
50
|
+
kubernetes_metadata = {
|
51
|
+
'namespace_id' => namespace_object[:metadata][:uid],
|
52
|
+
'creation_timestamp' => namespace_object[:metadata][:creationTimestamp]
|
53
|
+
}
|
54
|
+
kubernetes_metadata['namespace_labels'] = labels unless labels.empty?
|
55
|
+
kubernetes_metadata['namespace_annotations'] = annotations unless annotations.empty?
|
56
|
+
kubernetes_metadata
|
57
|
+
end
|
58
|
+
|
59
|
+
def parse_pod_metadata(pod_object)
|
60
|
+
labels = ''
|
61
|
+
labels = syms_to_strs(pod_object[:metadata][:labels].to_h) unless @skip_labels
|
62
|
+
|
63
|
+
annotations = match_annotations(syms_to_strs(pod_object[:metadata][:annotations].to_h))
|
64
|
+
if @de_dot
|
65
|
+
de_dot!(labels) unless @skip_labels
|
66
|
+
de_dot!(annotations)
|
67
|
+
end
|
68
|
+
|
69
|
+
# collect container information
|
70
|
+
container_meta = {}
|
71
|
+
begin
|
72
|
+
pod_object[:status][:containerStatuses].each do |container_status|
|
73
|
+
# get plain container id (eg. docker://hash -> hash)
|
74
|
+
container_id = container_status[:containerID].sub(%r{^[-_a-zA-Z0-9]+://}, '')
|
75
|
+
container_meta[container_id] = if @skip_container_metadata
|
76
|
+
{
|
77
|
+
'name' => container_status[:name]
|
78
|
+
}
|
79
|
+
else
|
80
|
+
{
|
81
|
+
'name' => container_status[:name],
|
82
|
+
'image' => container_status[:image],
|
83
|
+
'image_id' => container_status[:imageID]
|
84
|
+
}
|
85
|
+
end
|
86
|
+
end
|
87
|
+
rescue StandardError
|
88
|
+
log.debug("parsing container meta information failed for: #{pod_object[:metadata][:namespace]}/#{pod_object[:metadata][:name]} ")
|
89
|
+
end
|
90
|
+
|
91
|
+
kubernetes_metadata = {
|
92
|
+
'namespace_name' => pod_object[:metadata][:namespace],
|
93
|
+
'pod_id' => pod_object[:metadata][:uid],
|
94
|
+
'pod_name' => pod_object[:metadata][:name],
|
95
|
+
'pod_ip' => pod_object[:status][:podIP],
|
96
|
+
'containers' => syms_to_strs(container_meta),
|
97
|
+
'host' => pod_object[:spec][:nodeName]
|
98
|
+
}
|
99
|
+
kubernetes_metadata['annotations'] = annotations unless annotations.empty?
|
100
|
+
kubernetes_metadata['labels'] = labels unless labels.empty?
|
101
|
+
kubernetes_metadata['master_url'] = @kubernetes_url unless @skip_master_url
|
102
|
+
kubernetes_metadata
|
103
|
+
end
|
104
|
+
|
105
|
+
def syms_to_strs(hsh)
|
106
|
+
newhsh = {}
|
107
|
+
hsh.each_pair do |kk, vv|
|
108
|
+
if vv.is_a?(Hash)
|
109
|
+
vv = syms_to_strs(vv)
|
110
|
+
end
|
111
|
+
if kk.is_a?(Symbol)
|
112
|
+
newhsh[kk.to_s] = vv
|
113
|
+
else
|
114
|
+
newhsh[kk] = vv
|
115
|
+
end
|
116
|
+
end
|
117
|
+
newhsh
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
|
5
|
+
# Kubernetes metadata
|
6
|
+
#
|
7
|
+
# Copyright 2017 Red Hat, Inc.
|
8
|
+
#
|
9
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
10
|
+
# you may not use this file except in compliance with the License.
|
11
|
+
# You may obtain a copy of the License at
|
12
|
+
#
|
13
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
14
|
+
#
|
15
|
+
# Unless required by applicable law or agreed to in writing, software
|
16
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
17
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
18
|
+
# See the License for the specific language governing permissions and
|
19
|
+
# limitations under the License.
|
20
|
+
#
|
21
|
+
require 'lru_redux'
|
22
|
+
module KubernetesMetadata
|
23
|
+
class Stats
|
24
|
+
def initialize
|
25
|
+
@stats = ::LruRedux::TTL::ThreadSafeCache.new(1000, 3600)
|
26
|
+
end
|
27
|
+
|
28
|
+
def bump(key)
|
29
|
+
@stats[key] = @stats.getset(key) { 0 } + 1
|
30
|
+
end
|
31
|
+
|
32
|
+
def set(key, value)
|
33
|
+
@stats[key] = value
|
34
|
+
end
|
35
|
+
|
36
|
+
def [](key)
|
37
|
+
@stats[key]
|
38
|
+
end
|
39
|
+
|
40
|
+
def to_s
|
41
|
+
'stats - ' + [].tap do |a|
|
42
|
+
@stats.each { |k, v| a << "#{k}: #{v}" }
|
43
|
+
end.join(', ')
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
|
5
|
+
# Kubernetes metadata
|
6
|
+
#
|
7
|
+
# Copyright 2021 Red Hat, Inc.
|
8
|
+
#
|
9
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
10
|
+
# you may not use this file except in compliance with the License.
|
11
|
+
# You may obtain a copy of the License at
|
12
|
+
#
|
13
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
14
|
+
#
|
15
|
+
# Unless required by applicable law or agreed to in writing, software
|
16
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
17
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
18
|
+
# See the License for the specific language governing permissions and
|
19
|
+
# limitations under the License.
|
20
|
+
#
|
21
|
+
module KubernetesMetadata
|
22
|
+
module Util
|
23
|
+
def create_time_from_record(record, internal_time)
|
24
|
+
time_key = @time_fields.detect { |ii| record.key?(ii) }
|
25
|
+
time = record[time_key]
|
26
|
+
if time.nil? || time.is_a?(String) && time.chop.empty?
|
27
|
+
# `internal_time` is a Fluent::EventTime, it can't compare with Time.
|
28
|
+
return Time.at(internal_time.to_f)
|
29
|
+
end
|
30
|
+
|
31
|
+
if ['_SOURCE_REALTIME_TIMESTAMP', '__REALTIME_TIMESTAMP'].include?(time_key)
|
32
|
+
timei = time.to_i
|
33
|
+
return Time.at(timei / 1_000_000, timei % 1_000_000)
|
34
|
+
end
|
35
|
+
return Time.at(time) if time.is_a?(Numeric)
|
36
|
+
|
37
|
+
Time.parse(time)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,154 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Fluentd Kubernetes Metadata Filter Plugin - Enrich Fluentd events with
|
5
|
+
# Kubernetes metadata
|
6
|
+
#
|
7
|
+
# Copyright 2017 Red Hat, Inc.
|
8
|
+
#
|
9
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
10
|
+
# you may not use this file except in compliance with the License.
|
11
|
+
# You may obtain a copy of the License at
|
12
|
+
#
|
13
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
14
|
+
#
|
15
|
+
# Unless required by applicable law or agreed to in writing, software
|
16
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
17
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
18
|
+
# See the License for the specific language governing permissions and
|
19
|
+
# limitations under the License.
|
20
|
+
#
|
21
|
+
# TODO: this is mostly copy-paste from kubernetes_metadata_watch_pods.rb unify them
|
22
|
+
require_relative 'kubernetes_metadata_common'
|
23
|
+
|
24
|
+
module KubernetesMetadata
|
25
|
+
module WatchNamespaces
|
26
|
+
include ::KubernetesMetadata::Common
|
27
|
+
|
28
|
+
def set_up_namespace_thread
|
29
|
+
# Any failures / exceptions in the initial setup should raise
|
30
|
+
# Fluent:ConfigError, so that users can inspect potential errors in
|
31
|
+
# the configuration.
|
32
|
+
namespace_watcher = start_namespace_watch
|
33
|
+
Thread.current[:namespace_watch_retry_backoff_interval] = @watch_retry_interval
|
34
|
+
Thread.current[:namespace_watch_retry_count] = 0
|
35
|
+
|
36
|
+
# Any failures / exceptions in the followup watcher notice
|
37
|
+
# processing will be swallowed and retried. These failures /
|
38
|
+
# exceptions could be caused by Kubernetes API being temporarily
|
39
|
+
# down. We assume the configuration is correct at this point.
|
40
|
+
loop do
|
41
|
+
namespace_watcher ||= get_namespaces_and_start_watcher
|
42
|
+
process_namespace_watcher_notices(namespace_watcher)
|
43
|
+
rescue GoneError => e
|
44
|
+
# Expected error. Quietly go back through the loop in order to
|
45
|
+
# start watching from the latest resource versions
|
46
|
+
@stats.bump(:namespace_watch_gone_errors)
|
47
|
+
log.info('410 Gone encountered. Restarting namespace watch to reset resource versions.', e)
|
48
|
+
namespace_watcher = nil
|
49
|
+
rescue StandardError => e
|
50
|
+
@stats.bump(:namespace_watch_failures)
|
51
|
+
if Thread.current[:namespace_watch_retry_count] < @watch_retry_max_times
|
52
|
+
# Instead of raising exceptions and crashing Fluentd, swallow
|
53
|
+
# the exception and reset the watcher.
|
54
|
+
log.info(
|
55
|
+
'Exception encountered parsing namespace watch event. ' \
|
56
|
+
'The connection might have been closed. Sleeping for ' \
|
57
|
+
"#{Thread.current[:namespace_watch_retry_backoff_interval]} " \
|
58
|
+
'seconds and resetting the namespace watcher.', e
|
59
|
+
)
|
60
|
+
sleep(Thread.current[:namespace_watch_retry_backoff_interval])
|
61
|
+
Thread.current[:namespace_watch_retry_count] += 1
|
62
|
+
Thread.current[:namespace_watch_retry_backoff_interval] *= @watch_retry_exponential_backoff_base
|
63
|
+
namespace_watcher = nil
|
64
|
+
else
|
65
|
+
# Since retries failed for many times, log as errors instead
|
66
|
+
# of info and raise exceptions and trigger Fluentd to restart.
|
67
|
+
message =
|
68
|
+
'Exception encountered parsing namespace watch event. The ' \
|
69
|
+
'connection might have been closed. Retried ' \
|
70
|
+
"#{@watch_retry_max_times} times yet still failing. Restarting."
|
71
|
+
log.error(message, e)
|
72
|
+
raise Fluent::UnrecoverableError, message
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def start_namespace_watch
|
78
|
+
get_namespaces_and_start_watcher
|
79
|
+
rescue StandardError => e
|
80
|
+
message = 'start_namespace_watch: Exception encountered setting up ' \
|
81
|
+
"namespace watch from Kubernetes API #{@apiVersion} endpoint " \
|
82
|
+
"#{@kubernetes_url}: #{e.message}"
|
83
|
+
message += " (#{e.response})" if e.respond_to?(:response)
|
84
|
+
log.debug(message)
|
85
|
+
|
86
|
+
raise Fluent::ConfigError, message
|
87
|
+
end
|
88
|
+
|
89
|
+
# List all namespaces, record the resourceVersion and return a watcher
|
90
|
+
# starting from that resourceVersion.
|
91
|
+
def get_namespaces_and_start_watcher
|
92
|
+
options = {
|
93
|
+
resource_version: '0' # Fetch from API server cache instead of etcd quorum read
|
94
|
+
}
|
95
|
+
namespaces = @client.get_namespaces(options)
|
96
|
+
namespaces[:items].each do |namespace|
|
97
|
+
cache_key = namespace[:metadata][:uid]
|
98
|
+
@namespace_cache[cache_key] = parse_namespace_metadata(namespace)
|
99
|
+
@stats.bump(:namespace_cache_host_updates)
|
100
|
+
end
|
101
|
+
|
102
|
+
# continue watching from most recent resourceVersion
|
103
|
+
options[:resource_version] = namespaces[:metadata][:resourceVersion]
|
104
|
+
|
105
|
+
watcher = @client.watch_namespaces(options)
|
106
|
+
reset_namespace_watch_retry_stats
|
107
|
+
watcher
|
108
|
+
end
|
109
|
+
|
110
|
+
# Reset namespace watch retry count and backoff interval as there is a
|
111
|
+
# successful watch notice.
|
112
|
+
def reset_namespace_watch_retry_stats
|
113
|
+
Thread.current[:namespace_watch_retry_count] = 0
|
114
|
+
Thread.current[:namespace_watch_retry_backoff_interval] = @watch_retry_interval
|
115
|
+
end
|
116
|
+
|
117
|
+
# Process a watcher notice and potentially raise an exception.
|
118
|
+
def process_namespace_watcher_notices(watcher)
|
119
|
+
watcher.each do |notice|
|
120
|
+
case notice[:type]
|
121
|
+
when 'MODIFIED'
|
122
|
+
reset_namespace_watch_retry_stats
|
123
|
+
cache_key = notice[:object][:metadata][:uid]
|
124
|
+
cached = @namespace_cache[cache_key]
|
125
|
+
if cached
|
126
|
+
@namespace_cache[cache_key] = parse_namespace_metadata(notice[:object])
|
127
|
+
@stats.bump(:namespace_cache_watch_updates)
|
128
|
+
else
|
129
|
+
@stats.bump(:namespace_cache_watch_misses)
|
130
|
+
end
|
131
|
+
when 'DELETED'
|
132
|
+
reset_namespace_watch_retry_stats
|
133
|
+
# ignore and let age out for cases where
|
134
|
+
# deleted but still processing logs
|
135
|
+
@stats.bump(:namespace_cache_watch_deletes_ignored)
|
136
|
+
when 'ERROR'
|
137
|
+
if notice[:object] && notice[:object][:code] == 410
|
138
|
+
@stats.bump(:namespace_watch_gone_notices)
|
139
|
+
raise GoneError
|
140
|
+
else
|
141
|
+
@stats.bump(:namespace_watch_error_type_notices)
|
142
|
+
message = notice[:object][:message] if notice[:object] && notice[:object][:message]
|
143
|
+
raise "Error while watching namespaces: #{message}"
|
144
|
+
end
|
145
|
+
else
|
146
|
+
reset_namespace_watch_retry_stats
|
147
|
+
# Don't pay attention to creations, since the created namespace may not
|
148
|
+
# be used by any namespace on this node.
|
149
|
+
@stats.bump(:namespace_cache_watch_ignored)
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
154
|
+
end
|