kicks_liveness 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +53 -1
- data/README.md +10 -5
- data/docs/DESIGN.md +124 -187
- data/docs/KUBERNETES.md +16 -8
- data/docs/LIMITATIONS.md +145 -61
- data/docs/SETUP.md +11 -0
- data/docs/VERIFYING.md +104 -7
- data/lib/kicks_liveness/attempts.rb +21 -16
- data/lib/kicks_liveness/configuration.rb +4 -9
- data/lib/kicks_liveness/heartbeat.rb +232 -109
- data/lib/kicks_liveness/hooks.rb +11 -29
- data/lib/kicks_liveness/monitor.rb +36 -46
- data/lib/kicks_liveness/registry.rb +3 -15
- data/lib/kicks_liveness/topology_report.rb +110 -0
- data/lib/kicks_liveness/version.rb +1 -1
- data/lib/kicks_liveness.rb +3 -1
- metadata +3 -2
|
@@ -14,11 +14,8 @@ module KicksLiveness
|
|
|
14
14
|
class << self
|
|
15
15
|
# Records a worker as subscribed.
|
|
16
16
|
#
|
|
17
|
-
#
|
|
18
|
-
#
|
|
19
|
-
# checked first. +equal?+ rather than +==+ on purpose: this is a registry
|
|
20
|
-
# of objects, and a worker class is free to define equality however it
|
|
21
|
-
# likes.
|
|
17
|
+
# Duplicate objects must not inflate the expected-count check. Use identity
|
|
18
|
+
# rather than == because worker classes can override equality.
|
|
22
19
|
#
|
|
23
20
|
# @param worker [Sneakers::Worker]
|
|
24
21
|
# @return [Array] the registry
|
|
@@ -36,16 +33,7 @@ module KicksLiveness
|
|
|
36
33
|
@mutex.synchronize { @workers.delete(worker) }
|
|
37
34
|
end
|
|
38
35
|
|
|
39
|
-
#
|
|
40
|
-
def size
|
|
41
|
-
@mutex.synchronize { @workers.size }
|
|
42
|
-
end
|
|
43
|
-
|
|
44
|
-
# Records that this process is shutting down on purpose.
|
|
45
|
-
#
|
|
46
|
-
# Without it a graceful shutdown is indistinguishable from a failure: the
|
|
47
|
-
# registry empties as each worker unsubscribes, so the monitor would see
|
|
48
|
-
# an incomplete set and report a genuine fault on every deploy.
|
|
36
|
+
# Mark deliberate shutdown before unsubscription changes the predicate.
|
|
49
37
|
#
|
|
50
38
|
# @return [true]
|
|
51
39
|
def stopping!
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
require 'fileutils'
|
|
2
|
+
|
|
3
|
+
module KicksLiveness
|
|
4
|
+
# Claims worker topology reports across fork respawns. Kept outside
|
|
5
|
+
# {Heartbeat} because the probe does not report these diagnostics.
|
|
6
|
+
#
|
|
7
|
+
# @see file:docs/LIMITATIONS.md#container-generations-need-linux-procfs-and-a-private-pid-namespace
|
|
8
|
+
# @api private
|
|
9
|
+
class TopologyReport
|
|
10
|
+
# @param dir [String] marks directory
|
|
11
|
+
# @param topology [ContainerGeneration::Result] what this worker proved
|
|
12
|
+
def initialize(dir, topology)
|
|
13
|
+
@dir = dir
|
|
14
|
+
@topology = topology
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# Claims the right to report this state, once.
|
|
18
|
+
#
|
|
19
|
+
# Unverifiable results use a generation key; unavailable and unsafe results
|
|
20
|
+
# have no trustworthy generation and use only their state. Verification
|
|
21
|
+
# clears the flag so a later degradation can be reported again.
|
|
22
|
+
#
|
|
23
|
+
# @return [Array(Symbol, String), nil] the level and the line to log, or nil
|
|
24
|
+
# when there is nothing new to say
|
|
25
|
+
def claim!
|
|
26
|
+
key = key_for(@topology)
|
|
27
|
+
|
|
28
|
+
unless key
|
|
29
|
+
forget! if @topology.verified?
|
|
30
|
+
return nil
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
return nil unless claim(key)
|
|
34
|
+
|
|
35
|
+
report_for(@topology)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private
|
|
39
|
+
|
|
40
|
+
def path
|
|
41
|
+
File.join(@dir, 'topology-report')
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def key_for(topology)
|
|
45
|
+
case topology.state
|
|
46
|
+
when :unsafe then 'unsafe'
|
|
47
|
+
when :unverifiable then "unverifiable #{topology.generation}"
|
|
48
|
+
when :unavailable then 'unavailable'
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Only unsafe changes the probe verdict and is reported at ERROR.
|
|
53
|
+
def report_for(topology)
|
|
54
|
+
case topology.state
|
|
55
|
+
when :unsafe
|
|
56
|
+
[:error, "#{Heartbeat::UNSUPPORTED_TOPOLOGY} (#{topology.detail}); " \
|
|
57
|
+
'liveness probes fail until the pod is recreated without those settings']
|
|
58
|
+
when :unverifiable
|
|
59
|
+
[:warn, "PID topology unverified (#{topology.detail}): the container generation still scopes the heartbeat, " \
|
|
60
|
+
'but shareProcessNamespace and hostPID cannot be ruled out']
|
|
61
|
+
else
|
|
62
|
+
[:warn, "no container generation (#{topology.detail}): falling back to freshness only, " \
|
|
63
|
+
'which cannot reject a heartbeat inherited from a previous container']
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# The runner starts all of its forks together, so the read and write have
|
|
68
|
+
# to be one inter-process operation: an atomic rename protects readers from
|
|
69
|
+
# a partial file, but does not stop every fork reading the old key and each
|
|
70
|
+
# claiming the same report. The state file is also the lock file, so no
|
|
71
|
+
# second path is added to the marks contract. Once it has existed it is
|
|
72
|
+
# cleared by truncation rather than unlink: every process must keep locking
|
|
73
|
+
# the same inode.
|
|
74
|
+
def claim(key)
|
|
75
|
+
synchronize(create: true) do |state|
|
|
76
|
+
state.rewind
|
|
77
|
+
next false if state.read.chomp == key
|
|
78
|
+
|
|
79
|
+
state.rewind
|
|
80
|
+
state.truncate(0)
|
|
81
|
+
state.write("#{key}\n")
|
|
82
|
+
state.flush
|
|
83
|
+
true
|
|
84
|
+
end
|
|
85
|
+
rescue StandardError
|
|
86
|
+
# A report that cannot be remembered must still be made. The cost is
|
|
87
|
+
# repetition on the next start, not silence about a lost guarantee.
|
|
88
|
+
true
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def forget!
|
|
92
|
+
return unless File.exist?(path)
|
|
93
|
+
|
|
94
|
+
synchronize(create: false) { |state| state.truncate(0) }
|
|
95
|
+
rescue StandardError
|
|
96
|
+
nil
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def synchronize(create:)
|
|
100
|
+
FileUtils.mkdir_p(@dir)
|
|
101
|
+
flags = File::RDWR
|
|
102
|
+
flags |= File::CREAT if create
|
|
103
|
+
File.open(path, flags, 0o644) do |state|
|
|
104
|
+
raise IOError, "cannot lock #{path}" unless state.flock(File::LOCK_EX)
|
|
105
|
+
|
|
106
|
+
yield state
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
data/lib/kicks_liveness.rb
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
require_relative 'kicks_liveness/version'
|
|
2
2
|
require_relative 'kicks_liveness/heartbeat'
|
|
3
3
|
require_relative 'kicks_liveness/attempts'
|
|
4
|
+
require_relative 'kicks_liveness/topology_report'
|
|
4
5
|
require_relative 'kicks_liveness/configuration'
|
|
5
6
|
require_relative 'kicks_liveness/registry'
|
|
6
7
|
require_relative 'kicks_liveness/monitor'
|
|
@@ -76,7 +77,8 @@ module KicksLiveness
|
|
|
76
77
|
# @param slot [Integer] supervisor slot of this fork
|
|
77
78
|
# @param processes [Integer] how many forks the probe must wait for
|
|
78
79
|
# @param consumers [Integer] how many workers must subscribe here
|
|
79
|
-
# @return [Thread, nil] nil when disabled by configuration
|
|
80
|
+
# @return [Thread, nil] nil when disabled by configuration, or when the PID
|
|
81
|
+
# topology is proven unsupported
|
|
80
82
|
# @api private
|
|
81
83
|
def start!(slot:, processes:, consumers:)
|
|
82
84
|
return unless config.enabled?
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: kicks_liveness
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- PoroshkinaVV
|
|
@@ -39,13 +39,14 @@ files:
|
|
|
39
39
|
- lib/kicks_liveness/probe.rb
|
|
40
40
|
- lib/kicks_liveness/railtie.rb
|
|
41
41
|
- lib/kicks_liveness/registry.rb
|
|
42
|
+
- lib/kicks_liveness/topology_report.rb
|
|
42
43
|
- lib/kicks_liveness/version.rb
|
|
43
44
|
homepage: https://github.com/PoroshkinaVV/kicks_liveness
|
|
44
45
|
licenses:
|
|
45
46
|
- MIT
|
|
46
47
|
metadata:
|
|
47
48
|
source_code_uri: https://github.com/PoroshkinaVV/kicks_liveness
|
|
48
|
-
documentation_uri: https://rubydoc.info/gems/kicks_liveness/0.1.
|
|
49
|
+
documentation_uri: https://rubydoc.info/gems/kicks_liveness/0.1.3
|
|
49
50
|
changelog_uri: https://github.com/PoroshkinaVV/kicks_liveness/blob/main/CHANGELOG.md
|
|
50
51
|
bug_tracker_uri: https://github.com/PoroshkinaVV/kicks_liveness/issues
|
|
51
52
|
rubygems_mfa_required: 'true'
|