kubernetes_leader_election 0.1.0 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- metadata +3 -6
- data/MIT-LICENSE +0 -20
- data/lib/kubernetes_leader_election/version.rb +0 -4
- data/lib/kubernetes_leader_election.rb +0 -123
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 41efa4758ac05050cf98224f65987ca7928b1bb8f6f83198fabfc3b272f1026a
|
4
|
+
data.tar.gz: 6d901c130466a461329dbf11caff97a7645f438e5c36e2a03d28ada0aa391147
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 383f9b7a4135872be64db97a0f41ead019cf990cd200823633a365285ec473092e8f7227791a2535695f37830c075e579231b140a19ad4d848cfa1598c87e247
|
7
|
+
data.tar.gz: c78cf533e76600728500baef08fb14628e67a4c5b26cd8789aba333d154cdd80457b71a2cfcf1f8fb64dd7b973679168200c39bdd2b2508005e6046ba874da75
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: kubernetes_leader_election
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Grosser
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-09-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: kubeclient
|
@@ -29,10 +29,7 @@ email: michael@grosser.it
|
|
29
29
|
executables: []
|
30
30
|
extensions: []
|
31
31
|
extra_rdoc_files: []
|
32
|
-
files:
|
33
|
-
- MIT-LICENSE
|
34
|
-
- lib/kubernetes_leader_election.rb
|
35
|
-
- lib/kubernetes_leader_election/version.rb
|
32
|
+
files: []
|
36
33
|
homepage: https://github.com/grosser/kubernetes_leader_election
|
37
34
|
licenses:
|
38
35
|
- MIT
|
data/MIT-LICENSE
DELETED
@@ -1,20 +0,0 @@
|
|
1
|
-
Copyright (C) 2013 Michael Grosser <michael@grosser.it>
|
2
|
-
|
3
|
-
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
-
a copy of this software and associated documentation files (the
|
5
|
-
"Software"), to deal in the Software without restriction, including
|
6
|
-
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
-
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
-
permit persons to whom the Software is furnished to do so, subject to
|
9
|
-
the following conditions:
|
10
|
-
|
11
|
-
The above copyright notice and this permission notice shall be
|
12
|
-
included in all copies or substantial portions of the Software.
|
13
|
-
|
14
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
-
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
-
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
-
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
-
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
-
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
-
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
@@ -1,123 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
require 'time'
|
3
|
-
require 'openssl'
|
4
|
-
require 'timeout'
|
5
|
-
require 'kubeclient'
|
6
|
-
|
7
|
-
class KubernetesLeaderElection
|
8
|
-
ALREADY_EXISTS_CODE = 409
|
9
|
-
FAILED_KUBERNETES_REQUEST =
|
10
|
-
[Timeout::Error, OpenSSL::SSL::SSLError, Kubeclient::HttpError, SystemCallError, HTTP::ConnectionError].freeze
|
11
|
-
|
12
|
-
def initialize(name, kubeclient, statsd:, logger:, interval: 30)
|
13
|
-
@name = name
|
14
|
-
@kubeclient = kubeclient
|
15
|
-
@statsd = statsd
|
16
|
-
@logger = logger
|
17
|
-
@interval = interval
|
18
|
-
end
|
19
|
-
|
20
|
-
# not using `call` since we never want to be restarted
|
21
|
-
def become_leader_for_life
|
22
|
-
@logger.info message: "trying to become leader ... if both pods show this, delete the #{@name} lease"
|
23
|
-
loop do
|
24
|
-
break if become_leader
|
25
|
-
sleep @interval
|
26
|
-
end
|
27
|
-
yield # signal we are leader, but keep reporting
|
28
|
-
loop do
|
29
|
-
@statsd.increment('leader_running') # we monitor this to make sure it's always exactly 1
|
30
|
-
sleep @interval
|
31
|
-
signal_alive
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
private
|
36
|
-
|
37
|
-
# show that we are alive or crash because we cannot reach the api (split-brain az)
|
38
|
-
def signal_alive
|
39
|
-
with_retries(*FAILED_KUBERNETES_REQUEST, times: 3) do
|
40
|
-
patch = { spec: { renewTime: microtime } }
|
41
|
-
reply = @kubeclient.patch_entity(
|
42
|
-
"leases", @name, patch, 'strategic-merge-patch', ENV.fetch("POD_NAMESPACE")
|
43
|
-
)
|
44
|
-
|
45
|
-
current_leader = reply.dig(:metadata, :ownerReferences, 0, :name)
|
46
|
-
raise "Lost leadership to #{current_leader}" if current_leader != ENV.fetch("POD_NAME")
|
47
|
-
end
|
48
|
-
end
|
49
|
-
|
50
|
-
# kubernetes needs exactly this format or it blows up
|
51
|
-
def microtime
|
52
|
-
Time.now.strftime('%FT%T.000000Z')
|
53
|
-
end
|
54
|
-
|
55
|
-
def alive?(lease)
|
56
|
-
Time.parse(lease.dig(:spec, :renewTime)) > Time.now - (2 * @interval)
|
57
|
-
end
|
58
|
-
|
59
|
-
# everyone tries to create the same leases, who succeeds is the owner,
|
60
|
-
# leases is auto-deleted by GC when owner is deleted
|
61
|
-
# same logic lives in kube-service-watcher & kube-stats
|
62
|
-
def become_leader
|
63
|
-
namespace = ENV.fetch("POD_NAMESPACE")
|
64
|
-
# retry request on regular api errors
|
65
|
-
reraise = ->(e) { e.is_a?(Kubeclient::HttpError) && e.error_code == ALREADY_EXISTS_CODE }
|
66
|
-
|
67
|
-
with_retries(*FAILED_KUBERNETES_REQUEST, reraise: reraise, times: 3) do
|
68
|
-
@kubeclient.create_entity(
|
69
|
-
"Lease",
|
70
|
-
"leases",
|
71
|
-
metadata: {
|
72
|
-
name: @name,
|
73
|
-
namespace: namespace,
|
74
|
-
ownerReferences: [{
|
75
|
-
apiVersion: "v1",
|
76
|
-
kind: "Pod",
|
77
|
-
name: ENV.fetch("POD_NAME"),
|
78
|
-
uid: ENV.fetch("POD_UID")
|
79
|
-
}]
|
80
|
-
},
|
81
|
-
spec: {
|
82
|
-
acquireTime: microtime,
|
83
|
-
holderIdentity: ENV.fetch("POD_NAME"), # shown in `kubectl get lease`
|
84
|
-
leaseDurationSeconds: @interval * 2,
|
85
|
-
leaseTransitions: 0, # will never change since we delete the lease
|
86
|
-
renewTime: microtime
|
87
|
-
}
|
88
|
-
)
|
89
|
-
end
|
90
|
-
@logger.info message: "became leader"
|
91
|
-
true # I'm the leader now
|
92
|
-
rescue Kubeclient::HttpError => e
|
93
|
-
raise e unless e.error_code == ALREADY_EXISTS_CODE # lease already exists
|
94
|
-
|
95
|
-
lease = with_retries(*FAILED_KUBERNETES_REQUEST, times: 3) do
|
96
|
-
@kubeclient.get_entity("leases", @name, namespace)
|
97
|
-
end
|
98
|
-
leader = lease.dig(:metadata, :ownerReferences, 0, :name)
|
99
|
-
if leader == ENV.fetch("POD_NAME")
|
100
|
-
@logger.info message: "still leader"
|
101
|
-
true # I restarted and am still the leader
|
102
|
-
elsif !alive?(lease)
|
103
|
-
@logger.info message: "deleting stale lease"
|
104
|
-
with_retries(*FAILED_KUBERNETES_REQUEST, times: 3) do
|
105
|
-
@kubeclient.delete_entity("leases", @name, namespace)
|
106
|
-
end
|
107
|
-
false
|
108
|
-
else
|
109
|
-
false # leader is still alive ... not logging to avoid repetitive noise
|
110
|
-
end
|
111
|
-
end
|
112
|
-
|
113
|
-
def with_retries(*errors, times:, reraise: nil, backoff: [0.1, 0.5, 1])
|
114
|
-
yield
|
115
|
-
rescue *errors => e
|
116
|
-
retries ||= -1
|
117
|
-
retries += 1
|
118
|
-
raise if retries >= times || reraise&.call(e)
|
119
|
-
@logger.warn message: "Retryable error", type: e.class.to_s, retries: times - retries
|
120
|
-
sleep backoff[retries] || backoff.last
|
121
|
-
retry
|
122
|
-
end
|
123
|
-
end
|