kubernetes_leader_election 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 8b058cfd69bc5948abfc193c5486b6a8292c10d3364bfd0ebec98faf37b34fff
+  data.tar.gz: 3d800cc726a136c6664ff2aac8e2f1c2005159c6532929f3208590b9fa03fb8c
+SHA512:
+  metadata.gz: 6294c95ea8b4f283833fb398bf1119d42557341c4fbb2ce7612f2ed20906020bddf200bd0a6d8203fc3d0a7d57a068f1411f5979b0ee33c22f1fbb3ab91752a9
+  data.tar.gz: 4dbd4d8ac315543be78d73dcd156d926e7c0fd775cc637c5871a3b4e5a602b0ff81ad697c6a60343f66295669699696bb56fc799721801d2b5c578e0fbd954e3

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (C) 2013 Michael Grosser <michael@grosser.it>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/kubernetes_leader_election.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+require 'time'
+require 'openssl'
+require 'timeout'
+require 'kubeclient'
+
+class KubernetesLeaderElection
+  ALREADY_EXISTS_CODE = 409
+  FAILED_KUBERNETES_REQUEST =
+    [Timeout::Error, OpenSSL::SSL::SSLError, Kubeclient::HttpError, SystemCallError, HTTP::ConnectionError].freeze
+
+  def initialize(name, kubeclient, statsd:, logger:, interval: 30)
+    @name = name
+    @kubeclient = kubeclient
+    @statsd = statsd
+    @logger = logger
+    @interval = interval
+  end
+
+  # not using `call` since we never want to be restarted
+  def become_leader_for_life
+    @logger.info message: "trying to become leader ... if both pods show this, delete the #{@name} lease"
+    loop do
+      break if become_leader
+      sleep @interval
+    end
+    yield # signal we are leader, but keep reporting
+    loop do
+      @statsd.increment('leader_running') # we monitor this to make sure it's always exactly 1
+      sleep @interval
+      signal_alive
+    end
+  end
+
+  private
+
+  # show that we are alive or crash because we cannot reach the api (split-brain az)
+  def signal_alive
+    with_retries(*FAILED_KUBERNETES_REQUEST, times: 3) do
+      patch = { spec: { renewTime: microtime } }
+      reply = @kubeclient.patch_entity(
+        "leases", @name, patch, 'strategic-merge-patch', ENV.fetch("POD_NAMESPACE")
+      )
+
+      current_leader = reply.dig(:metadata, :ownerReferences, 0, :name)
+      raise "Lost leadership to #{current_leader}" if current_leader != ENV.fetch("POD_NAME")
+    end
+  end
+
+  # kubernetes needs exactly this format or it blows up
+  def microtime
+    Time.now.strftime('%FT%T.000000Z')
+  end
+
+  def alive?(lease)
+    Time.parse(lease.dig(:spec, :renewTime)) > Time.now - (2 * @interval)
+  end
+
+  # everyone tries to create the same leases, who succeeds is the owner,
+  # leases is auto-deleted by GC when owner is deleted
+  # same logic lives in kube-service-watcher & kube-stats
+  def become_leader
+    namespace = ENV.fetch("POD_NAMESPACE")
+    # retry request on regular api errors
+    reraise = ->(e) { e.is_a?(Kubeclient::HttpError) && e.error_code == ALREADY_EXISTS_CODE }
+
+    with_retries(*FAILED_KUBERNETES_REQUEST, reraise: reraise, times: 3) do
+      @kubeclient.create_entity(
+        "Lease",
+        "leases",
+        metadata: {
+          name: @name,
+          namespace: namespace,
+          ownerReferences: [{
+            apiVersion: "v1",
+            kind: "Pod",
+            name: ENV.fetch("POD_NAME"),
+            uid: ENV.fetch("POD_UID")
+          }]
+        },
+        spec: {
+          acquireTime: microtime,
+          holderIdentity: ENV.fetch("POD_NAME"), # shown in `kubectl get lease`
+          leaseDurationSeconds: @interval * 2,
+          leaseTransitions: 0, # will never change since we delete the lease
+          renewTime: microtime
+        }
+      )
+    end
+    @logger.info message: "became leader"
+    true # I'm the leader now
+  rescue Kubeclient::HttpError => e
+    raise e unless e.error_code == ALREADY_EXISTS_CODE # lease already exists
+
+    lease = with_retries(*FAILED_KUBERNETES_REQUEST, times: 3) do
+      @kubeclient.get_entity("leases", @name, namespace)
+    end
+    leader = lease.dig(:metadata, :ownerReferences, 0, :name)
+    if leader == ENV.fetch("POD_NAME")
+      @logger.info message: "still leader"
+      true # I restarted and am still the leader
+    elsif !alive?(lease)
+      @logger.info message: "deleting stale lease"
+      with_retries(*FAILED_KUBERNETES_REQUEST, times: 3) do
+        @kubeclient.delete_entity("leases", @name, namespace)
+      end
+      false
+    else
+      false # leader is still alive ... not logging to avoid repetitive noise
+    end
+  end
+
+  def with_retries(*errors, times:, reraise: nil, backoff: [0.1, 0.5, 1])
+    yield
+  rescue *errors => e
+    retries ||= -1
+    retries += 1
+    raise if retries >= times || reraise&.call(e)
+    @logger.warn message: "Retryable error", type: e.class.to_s, retries: times - retries
+    sleep backoff[retries] || backoff.last
+    retry
+  end
+end

data/lib/kubernetes_leader_election/version.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+class KubernetesLeaderElection
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,59 @@
+--- !ruby/object:Gem::Specification
+name: kubernetes_leader_election
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Michael Grosser
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2021-08-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: kubeclient
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email: michael@grosser.it
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- lib/kubernetes_leader_election.rb
+- lib/kubernetes_leader_election/version.rb
+homepage: https://github.com/grosser/kubernetes_leader_election
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.16
+signing_key: 
+specification_version: 4
+summary: Elect a kubernetes leader using leases for ruby
+test_files: []