kubernetes_leader_election 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 41efa4758ac05050cf98224f65987ca7928b1bb8f6f83198fabfc3b272f1026a
-  data.tar.gz: 6d901c130466a461329dbf11caff97a7645f438e5c36e2a03d28ada0aa391147
+  metadata.gz: ebf5e530aef5aa2090bb6722def02ea09f0727b4445f1fac80391687b4b69c81
+  data.tar.gz: 69a5f3e755e6c401733ab118944a155a1bc706121588359912f01b937283af53
 SHA512:
-  metadata.gz: 383f9b7a4135872be64db97a0f41ead019cf990cd200823633a365285ec473092e8f7227791a2535695f37830c075e579231b140a19ad4d848cfa1598c87e247
-  data.tar.gz: c78cf533e76600728500baef08fb14628e67a4c5b26cd8789aba333d154cdd80457b71a2cfcf1f8fb64dd7b973679168200c39bdd2b2508005e6046ba874da75
+  metadata.gz: 86af0f3581d589358f6ccedeb9f04c4a39abb4dfc153c2b8e6ba69dd80b2153ffc8bf4183cd194e548e947eac9c12314517c8532ffb809f641f2e75dc1883002
+  data.tar.gz: f58367aff174456db85719244922b383c3972e796ecb5f6ed2f9295994c3ce323402650ffbd40c677ad851934b54220e129c9fed3216d7742d39e458586980a0

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (C) 2013 Michael Grosser <michael@grosser.it>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/kubernetes_leader_election/version.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+class KubernetesLeaderElection
+  VERSION = "0.1.2"
+end

data/lib/kubernetes_leader_election.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+require 'time'
+require 'openssl'
+require 'timeout'
+require 'kubeclient'
+
+class KubernetesLeaderElection
+  ALREADY_EXISTS_CODE = 409
+  FAILED_KUBERNETES_REQUEST =
+    [Timeout::Error, OpenSSL::SSL::SSLError, Kubeclient::HttpError, SystemCallError, HTTP::ConnectionError].freeze
+
+  def initialize(name, kubeclient, logger:, statsd: nil, interval: 30)
+    @name = name
+    @kubeclient = kubeclient
+    @statsd = statsd
+    @logger = logger
+    @interval = interval
+  end
+
+  # not using `call` since we never want to be restarted
+  def become_leader_for_life
+    @logger.info message: "trying to become leader ... if both pods show this, delete the #{@name} lease"
+    loop do
+      break if become_leader
+      sleep @interval
+    end
+    yield # signal we are leader, but keep reporting
+    loop do
+      @statsd&.increment('leader_running') # we monitor this to make sure it's always exactly 1
+      sleep @interval
+      signal_alive
+    end
+  end
+
+  private
+
+  # show that we are alive or crash because we cannot reach the api (split-brain az)
+  def signal_alive
+    with_retries(*FAILED_KUBERNETES_REQUEST, times: 3) do
+      patch = { spec: { renewTime: microtime } }
+      reply = @kubeclient.patch_entity(
+        "leases", @name, patch, 'strategic-merge-patch', ENV.fetch("POD_NAMESPACE")
+      )
+
+      current_leader = reply.dig(:metadata, :ownerReferences, 0, :name)
+      raise "Lost leadership to #{current_leader}" if current_leader != ENV.fetch("POD_NAME")
+    end
+  end
+
+  # kubernetes needs exactly this format or it blows up
+  def microtime
+    Time.now.strftime('%FT%T.000000Z')
+  end
+
+  def alive?(lease)
+    Time.parse(lease.dig(:spec, :renewTime)) > Time.now - (2 * @interval)
+  end
+
+  # everyone tries to create the same leases, who succeeds is the owner,
+  # leases is auto-deleted by GC when owner is deleted
+  # same logic lives in kube-service-watcher & kube-stats
+  def become_leader
+    namespace = ENV.fetch("POD_NAMESPACE")
+    # retry request on regular api errors
+    reraise = ->(e) { e.is_a?(Kubeclient::HttpError) && e.error_code == ALREADY_EXISTS_CODE }
+
+    with_retries(*FAILED_KUBERNETES_REQUEST, reraise: reraise, times: 3) do
+      @kubeclient.create_entity(
+        "Lease",
+        "leases",
+        metadata: {
+          name: @name,
+          namespace: namespace,
+          ownerReferences: [{
+            apiVersion: "v1",
+            kind: "Pod",
+            name: ENV.fetch("POD_NAME"),
+            uid: ENV.fetch("POD_UID")
+          }]
+        },
+        spec: {
+          acquireTime: microtime,
+          holderIdentity: ENV.fetch("POD_NAME"), # shown in `kubectl get lease`
+          leaseDurationSeconds: @interval * 2,
+          leaseTransitions: 0, # will never change since we delete the lease
+          renewTime: microtime
+        }
+      )
+    end
+    @logger.info message: "became leader"
+    true # I'm the leader now
+  rescue Kubeclient::HttpError => e
+    raise e unless e.error_code == ALREADY_EXISTS_CODE # lease already exists
+
+    lease = with_retries(*FAILED_KUBERNETES_REQUEST, times: 3) do
+      @kubeclient.get_entity("leases", @name, namespace)
+    rescue Kubeclient::ResourceNotFoundError
+      nil
+    end
+
+    if !lease
+      @logger.info message: "stale lease was deleted"
+      false
+    elsif lease.dig(:metadata, :ownerReferences, 0, :name) == ENV.fetch("POD_NAME")
+      @logger.info message: "still leader"
+      true # I restarted and am still the leader
+    elsif !alive?(lease)
+      # this is still a race-condition since we could be deleting the newly succeeded leader
+      # see https://github.com/kubernetes/kubernetes/issues/20572
+      @logger.info message: "deleting stale lease"
+      with_retries(*FAILED_KUBERNETES_REQUEST, times: 3) do
+        @kubeclient.delete_entity("leases", @name, namespace)
+      end
+      false # leader is dead, do not assume leadership here to avoid race condition
+    else
+      false # leader is still alive ... not logging to avoid repetitive noise
+    end
+  end
+
+  def with_retries(*errors, times:, reraise: nil, backoff: [0.1, 0.5, 1])
+    yield
+  rescue *errors => e
+    retries ||= -1
+    retries += 1
+    raise if retries >= times || reraise&.call(e)
+    @logger.warn message: "Retryable error", type: e.class.to_s, retries: times - retries
+    sleep backoff[retries] || backoff.last
+    retry
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kubernetes_leader_election
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Michael Grosser
@@ -29,7 +29,10 @@ email: michael@grosser.it
 executables: []
 extensions: []
 extra_rdoc_files: []
-files: []
+files:
+- MIT-LICENSE
+- lib/kubernetes_leader_election.rb
+- lib/kubernetes_leader_election/version.rb
 homepage: https://github.com/grosser/kubernetes_leader_election
 licenses:
 - MIT