sidekiq-power-fetch 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: afe6f74ebc8d0e8b63cbc035ca0fd7f0239a9148a6652a3f75a1a80ad4b24d55
+  data.tar.gz: c02dc4a2f5c8fd660841df3d082ce326440a981f73af3288ba7d61086cde9641
+SHA512:
+  metadata.gz: 4c9162f7bb42087096c35bebe4faa5273283a01a15a0f3393ea07f8e8386737cd64743d51cd2d235aebd9a7c9be28f4999461dddbef49178d8c0053491a73bfc
+  data.tar.gz: 93651ad8a7724d6121066a3df93468e5862a15e554afd3ac68dd204954f220f251b43134c732bc4d3e254aabef5317911089ab7373a9d18e290f0553cde0b0e9

data/lib/sidekiq/power_fetch/heartbeat.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  class PowerFetch
+    class Heartbeat
+      LIFESPAN = 60 # seconds
+
+      def self.start(config)
+        new(config)
+      end
+
+      def self.started?
+        @started
+      end
+
+      def self.started=(value)
+        @started = value
+      end
+
+      def initialize(config)
+        raise "#{self.class} already started" if self.class.started?
+
+        @config = config
+
+        # Must pulse on startup, else races: other workers think the current
+        # process is dead, but it just didn't heartbeat yet.
+        @config.on(:startup) { pulse }
+        @config.on(:heartbeat) { pulse }
+        self.class.started = true
+      end
+
+      def self.key(identity)
+        id = identity.tr(":", "-")
+        "sidekiq-power-fetch-heartbeat-#{id}"
+      end
+
+      def pulse
+        @config.redis do |conn|
+          conn.set(key, 1, ex: LIFESPAN)
+        end
+
+        @config.logger.debug("[PowerFetch] Heartbeat for #{PowerFetch.identity}")
+      end
+
+      private
+
+      def key
+        @key ||= self.class.key(PowerFetch.identity)
+      end
+    end
+  end
+end

data/lib/sidekiq/power_fetch/lock.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  class PowerFetch
+    class Lock
+      DEFAULT_INTERVAL = 120 # 2 minutes
+      DEFAULT_RECOVER = 3600 # 1 hour
+      KEY = "sidekiq-power-fetch-lock"
+
+      def initialize(capsule)
+        @capsule = capsule
+        @taken_at = 0
+
+        # Sidekiq processes should attempt recovery every 2 minutes.
+        @interval = @capsule.config[:power_fetch_lock] || DEFAULT_INTERVAL
+        # Perform recovery at most every 1 hour.
+        @recover = @capsule.config[:power_fetch_recover] || DEFAULT_RECOVER
+      end
+
+      def lock
+        return unless take?
+
+        @taken_at = time
+        # Return value is "OK" or nil.
+        @capsule.redis do |conn|
+          conn.set(KEY, 1, nx: true, ex: @recover)
+        end
+      end
+
+      private
+
+      def take?
+        time - @taken_at > @interval
+      end
+
+      def time
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/sidekiq/power_fetch/recover.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require_relative "lock"
+
+module Sidekiq
+  class PowerFetch
+    class Recover
+      # Defines the COUNT parameter that will be passed to Redis SCAN command
+      SCAN_COUNT = 1000
+
+      # How much time a job can be interrupted
+      RECOVERIES = 3
+
+      # Regexes for matching working queue keys
+      WORKING_QUEUE_REGEX = /\A#{WORKING_QUEUE_PREFIX}:(queue:.*):([^:]*:[0-9]*:[0-9a-f]*)\z/
+
+      def initialize(capsule)
+        @capsule = capsule
+        @lock = Lock.new(@capsule)
+        @recoveries = @capsule.config[:power_fetch_recoveries] || RECOVERIES
+      end
+
+      def lock
+        @lock.lock
+      end
+
+      # Detect "old" jobs and requeue them because the worker they were assigned
+      # to probably failed miserably.
+      def call
+        @capsule.logger.info("[PowerFetch] Recovering working queues")
+
+        @capsule.redis do |conn|
+          conn.scan(
+            match: "#{WORKING_QUEUE_PREFIX}:queue:*",
+            count: SCAN_COUNT
+          ) do |key|
+            # Identity format is "{hostname}:{pid}:{randomhex}
+            # Queue names may also have colons (namespaced).
+            # Expressing this in a single regex is unreadable
+            original_queue, identity = key.scan(WORKING_QUEUE_REGEX).flatten
+
+            next if original_queue.nil? || identity.nil?
+
+            if worker_dead?(identity, conn)
+              recover_working_queue!(original_queue, key)
+            end
+          end
+        end
+      end
+
+      # If you want this method to be run in a scope of multi connection
+      # you need to pass it
+      def requeue_job(queue, msg, conn)
+        with_connection(conn) do |conn|
+          conn.lpush(queue, Sidekiq.dump_json(msg))
+        end
+
+        @capsule.logger.info(
+          "[PowerFetch] Pushed job #{msg["jid"]} back to queue '#{queue}'"
+        )
+      end
+
+      private
+
+      def recover_working_queue!(original_queue, working_queue)
+        @capsule.redis do |conn|
+          while job = conn.rpop(working_queue)
+            preprocess_interrupted_job(job, original_queue)
+          end
+        end
+      end
+
+      def preprocess_interrupted_job(job, queue, conn = nil)
+        msg = Sidekiq.load_json(job)
+        msg["interrupted_count"] = msg["interrupted_count"].to_i + 1
+
+        if interruption_exhausted?(msg)
+          @capsule.logger.warn(
+            "[PowerFetch] Deleted job #{msg["class"]} jid #{msg["jid"]}, " \
+            "it was recovered too many times"
+          )
+        else
+          requeue_job(queue, msg, conn)
+        end
+      end
+
+      def interruption_exhausted?(msg)
+        return false if @recoveries < 0
+
+        msg["interrupted_count"].to_i >= @recoveries
+      end
+
+      # Yield block with an existing connection or creates another one
+      def with_connection(conn)
+        return yield(conn) if conn
+
+        @capsule.redis { |redis_conn| yield(redis_conn) }
+      end
+
+      def worker_dead?(identity, conn)
+        !conn.get(Heartbeat.key(identity))
+      end
+    end
+  end
+end

data/lib/sidekiq/power_fetch/unit_of_work.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  class PowerFetch
+    class UnitOfWork
+      attr_reader :queue
+      attr_reader :job
+
+      def initialize(queue, job)
+        @queue = queue
+        @job = job
+      end
+
+      def acknowledge
+        Sidekiq.redis do |conn|
+          conn.lrem(PowerFetch.working_queue_name(@queue), 1, @job)
+        end
+      end
+
+      def queue_name
+        @queue.sub(/.*queue:/, "")
+      end
+
+      def requeue
+        Sidekiq.redis do |conn|
+          conn.multi do |multi|
+            multi.lpush(@queue, @job)
+            multi.lrem(PowerFetch.working_queue_name(@queue), 1, @job)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/power_fetch.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require_relative "power_fetch/heartbeat"
+require_relative "power_fetch/unit_of_work"
+# "recover" is required at the bottom of the file, after
+# PowerFetch::WORKING_QUEUE_PREFIX constant is defined.
+
+module Sidekiq
+  class PowerFetch
+    WORKING_QUEUE_PREFIX = "working"
+
+    # We don't use Redis' blocking operations for fetch so
+    # we inject a regular sleep into the loop.
+    IDLE_TIMEOUT = 5 # seconds
+
+    def self.identity
+      @identity ||= begin
+        hostname = ENV["DYNO"] || Socket.gethostname
+        pid = ::Process.pid
+        process_nonce = SecureRandom.hex(6)
+
+        "#{hostname}:#{pid}:#{process_nonce}"
+      end
+    end
+
+    def self.working_queue_name(queue)
+      "#{WORKING_QUEUE_PREFIX}:#{queue}:#{identity}"
+    end
+
+    def initialize(capsule)
+      raise ArgumentError, "missing queue list" unless capsule.queues
+      @capsule = capsule
+      @strictly_ordered_queues = capsule.mode == :strict
+      @queues = @capsule.queues.map { |q| "queue:#{q}" }
+      @queues.uniq! if @strictly_ordered_queues
+
+      @recover = Recover.new(@capsule)
+      @capsule.logger.info("[PowerFetch] Activated!")
+    end
+
+    def retrieve_work
+      if @recover.lock
+        @recover.call
+      end
+
+      queues_list = @strictly_ordered_queues ? @queues : @queues.shuffle
+
+      queues_list.each do |queue|
+        work = @capsule.redis do |conn|
+          # Can't use 'blmove' here: empty blocked queue would then block
+          # other, potentially non-empty, queues.
+          conn.lmove(queue, self.class.working_queue_name(queue), :right, :left)
+        end
+
+        return UnitOfWork.new(queue, work) if work
+      end
+
+      # We didn't find a job in any of the configured queues. Let's sleep a bit
+      # to avoid uselessly burning too much CPU
+      sleep(IDLE_TIMEOUT)
+
+      nil
+    end
+
+    # Called by sidekiq on "hard shutdown": when shutdown is reached, and there
+    # are still busy threads. The threads are shutdown, but their jobs are
+    # requeued.
+    # https://github.com/sidekiq/sidekiq/blob/323a5cfaefdde20588f5ffdf0124691db83fd315/lib/sidekiq/manager.rb#L107
+    def bulk_requeue(inprogress)
+      return if inprogress.empty?
+
+      @capsule.redis do |conn|
+        inprogress.each do |unit_of_work|
+          conn.multi do |multi|
+            msg = Sidekiq.load_json(unit_of_work.job)
+            @recover.requeue_job(unit_of_work.queue, msg, multi)
+
+            multi.lrem(self.class.working_queue_name(unit_of_work.queue), 1, unit_of_work.job)
+          end
+        end
+      end
+    rescue => e
+      @capsule.logger.warn("[PowerFetch] Failed to requeue #{inprogress.size} jobs: #{e.message}")
+    end
+  end
+end
+
+require_relative "power_fetch/recover"

data/lib/sidekiq-power-fetch.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require "sidekiq"
+require "sidekiq/api"
+
+require_relative "sidekiq/power_fetch"
+
+Sidekiq.configure_server do |config|
+  config[:fetch_class] = Sidekiq::PowerFetch
+
+  # There's only one Heartbeat per process.
+  # Starting it as soon as possible.
+  Sidekiq::PowerFetch::Heartbeat.start(config)
+end

metadata

@@ -0,0 +1,106 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq-power-fetch
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- TEA
+- GitLab
+- Bruno Sutic
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-08-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '7.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '7.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.23'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.23'
+- !ruby/object:Gem::Dependency
+  name: standard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.30'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.30'
+description:
+email: code@brunosutic.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/sidekiq-power-fetch.rb
+- lib/sidekiq/power_fetch.rb
+- lib/sidekiq/power_fetch/heartbeat.rb
+- lib/sidekiq/power_fetch/lock.rb
+- lib/sidekiq/power_fetch/recover.rb
+- lib/sidekiq/power_fetch/unit_of_work.rb
+homepage: https://gitlab.com/bruno-/sidekiq-power-fetch
+licenses:
+- LGPL-3.0
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.14
+signing_key:
+specification_version: 4
+summary: Improved fetch for Sidekiq 7+ and Redis 6.2.0+
+test_files: []