sidekiq-routing 0.1.0

data/lib/sidekiq/routing/middleware/client.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Routing
+    module Middleware
+      # Handles inflow: new jobs at enqueue time (perform_async/perform_in,
+      # push_bulk). Blackhole aborts the push; park rewrites the queue so the job
+      # lands in the worker-less parking queue.
+      #
+      # No per-job log or metric here — during a flood that would emit millions
+      # of lines. Observability is the Web tab's live aggregates + operator-action
+      # logs only (see docs/routing-how-it-works.md).
+      class Client
+        def call(worker_class, job, queue, _redis_pool = nil)
+          return yield unless Routing.enabled?
+          return yield if job[NO_DIVERT_KEY] # process_parked / explicit bypass
+          return yield if queue.to_s == Routing.parked_queue # never re-divert parked jobs
+
+          route = Routing.route_for(job["wrapped"] || worker_class)
+          return yield unless route
+
+          case route["mode"]
+          when MODE_BLACKHOLE
+            false # abort push — never enters Redis
+          when MODE_PARK
+            job[ORIGINAL_QUEUE_KEY] ||= queue.to_s # preserve true origin across re-diverts
+            job["queue"] = Routing.parked_queue
+            yield # push, but to the parking queue
+          else
+            yield
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/routing/middleware/server.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Routing
+    module Middleware
+      # Handles the backlog: jobs already enqueued before the route was added,
+      # plus scheduled/retry-set jobs that re-enter their real queue (those do not
+      # pass through client middleware, so this is the only thing that catches
+      # them). Park re-pushes the job to the parking queue; blackhole acks and
+      # drops it (never the Dead set).
+      #
+      # Registered AFTER SidekiqUniqueJobs::Middleware::Server so that returning
+      # without yield still lets unique-jobs release the original's lock.
+      class Server
+        def call(worker, job, queue)
+          return yield unless Routing.enabled?
+          return yield if queue.to_s == Routing.parked_queue # loop guard: run parked jobs
+          return yield if job[NO_DIVERT_KEY]
+
+          route = Routing.route_for(job["wrapped"] || worker.class)
+          return yield unless route
+
+          case route["mode"]
+          when MODE_BLACKHOLE
+            nil # ack & drop; NOT the Dead set
+          when MODE_PARK
+            Mover.move(
+              job, Routing.parked_queue,
+              ORIGINAL_QUEUE_KEY => job[ORIGINAL_QUEUE_KEY] || queue.to_s)
+            nil # ack & remove original; the copy now lives in the parking queue
+          else
+            yield
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/routing/mover.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Routing
+    # Relocates an already-enqueued job payload to another queue by rewriting the
+    # "queue" field *inside* the payload and writing it straight to Redis.
+    #
+    # It deliberately bypasses the client middleware chain. Going through
+    # Sidekiq::Client.push would (a) risk re-diverting the job via our own client
+    # middleware and (b) make sidekiq-unique-jobs try to re-acquire a lock that
+    # the in-flight original still holds, which can fail the push and drop the job
+    # on a server-side park. Moving the raw payload sidesteps both.
+    #
+    # Used by the server middleware (park), the Sweeper, and the ParkedProcessor.
+    module Mover
+      class << self
+        # item:     the existing job hash (a Sidekiq::JobRecord#item or job hash)
+        # to_queue: destination queue name
+        # extra:    payload keys to merge in (e.g. the original-queue stamp)
+        def move(item, to_queue, extra = {})
+          target = to_queue.to_s
+          payload = item.merge("queue" => target).merge(extra)
+          json = Sidekiq.dump_json(payload)
+
+          Sidekiq.redis do |conn|
+            conn.sadd("queues", target) # keep the queue visible in the Web UI
+            conn.lpush("queue:#{target}", json)
+          end
+          true
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/routing/parked_processor.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Routing
+    # Moves parked jobs back to their original queue by rewriting the "queue"
+    # field inside the payload (read item -> set queue -> push -> delete). Because
+    # the payload's queue is now the original, a processed parked job that later
+    # fails retries to its original queue, NOT the parking queue.
+    #
+    # Stamps NO_DIVERT_KEY so the job is not bounced straight back to parked even
+    # if the route is still active (recommended order is still: unpark, then
+    # process_parked).
+    class ParkedProcessor
+      def call(klass: nil, limit: nil, batch_size: nil)
+        limit ||= Routing.configuration.batch_limit
+        fallback = Routing.configuration.process_parked_fallback_queue
+        moved = 0
+
+        Sidekiq::Queue.new(Routing.parked_queue).each do |job|
+          break if limit && moved >= limit
+          next if klass && job.display_class != klass
+
+          target = job.item[ORIGINAL_QUEUE_KEY]
+          unless target
+            target = fallback
+            Routing.logger.warn(
+              "[Routing] #{job.display_class} #{job.jid} had no original queue; processing parked job to #{fallback}"
+            )
+          end
+
+          payload = job.item.reject { |key, _| key == ORIGINAL_QUEUE_KEY }
+          Mover.move(payload, target, NO_DIVERT_KEY => true)
+          moved += 1 if job.delete
+        end
+
+        Routing.logger.warn("[Routing] processed #{moved} parked job(s) from #{Routing.parked_queue}")
+        moved
+      end
+    end
+  end
+end

data/lib/sidekiq/routing/store.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Sidekiq
+  module Routing
+    # Source of truth for manual route state: a single Redis hash in Sidekiq's Redis,
+    # field = job class name, value = JSON {mode, routed_at}.
+    # These reads are uncached (used by the operator API and Web tab); the
+    # middleware hot path uses Sidekiq::Routing.route_for instead.
+    module Store
+      HASH_KEY = "sidekiq:routing:routes"
+
+      class << self
+        def set(name, mode:)
+          value = JSON.dump(
+            "mode" => mode.to_s,
+            "routed_at" => Time.now.utc.iso8601
+          )
+          Sidekiq.redis { |conn| conn.hset(HASH_KEY, name, value) }
+        end
+
+        def delete(name)
+          Sidekiq.redis { |conn| conn.hdel(HASH_KEY, name) }
+        end
+
+        # -> {"mode"=>...} or nil
+        def fetch(name)
+          raw = Sidekiq.redis { |conn| conn.hget(HASH_KEY, name) }
+          raw && JSON.parse(raw)
+        end
+
+        # -> { "ClassName" => {"mode"=>...}, ... }
+        def all
+          Sidekiq.redis { |conn| conn.hgetall(HASH_KEY) }
+            .transform_values { |raw| JSON.parse(raw) }
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/routing/sweeper.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Routing
+    # Eagerly clears the already-enqueued backlog of a parked class out of its
+    # live queue(s) and into the parking queue. Explicit operator action — never
+    # automatic on park. For a 100Ks+ backlog, prefer drain-in-place over moving
+    # millions of jobs.
+    class Sweeper
+      def call(klass_name, queue: nil, limit: nil, batch_size: nil)
+        limit ||= Routing.configuration.batch_limit
+        requested_queue = queue.to_s.empty? ? nil : queue
+        target_queues = Array(requested_queue || default_queues_for(klass_name))
+        moved = 0
+
+        target_queues.each do |source|
+          next if source.to_s == Routing.parked_queue
+
+          Sidekiq::Queue.new(source).each do |job|
+            break if limit && moved >= limit
+            next unless job.display_class == klass_name
+
+            Mover.move(
+              job.item, Routing.parked_queue,
+              ORIGINAL_QUEUE_KEY => job.item[ORIGINAL_QUEUE_KEY] || source.to_s)
+            moved += 1 if job.delete
+          end
+        end
+
+        Routing.logger.warn(
+          "[Routing] swept #{moved} #{klass_name} job(s) into #{Routing.parked_queue}"
+        )
+        moved
+      end
+
+      private
+
+        # The class's configured queue if we can resolve it. We deliberately do NOT
+        # fall back to scanning every queue: during an incident that would hammer
+        # Redis and slow healthy queues. If the queue can't be resolved, the operator
+        # must say which queue(s) to sweep.
+        def default_queues_for(klass_name)
+          klass = safe_constantize(klass_name)
+          configured = klass.respond_to?(:get_sidekiq_options) ? klass.get_sidekiq_options["queue"] : nil
+          return [configured.to_s] if configured
+
+          raise ArgumentError,
+            "Cannot resolve a queue for #{klass_name}; pass queue: explicitly so the sweep " \
+            "does not scan every queue, e.g. Sidekiq::Routing.sweep(#{klass_name.inspect}, queue: \"within_1_minute\")."
+        end
+
+        # Plain-Ruby stand-in for ActiveSupport's String#safe_constantize:
+        # resolve a (possibly namespaced) class name, or nil if it isn't loaded.
+        def safe_constantize(name)
+          Object.const_get(name)
+        rescue NameError
+          nil
+        end
+    end
+  end
+end

data/lib/sidekiq/routing/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Routing
+    VERSION = "0.1.0"
+  end
+end

data/lib/sidekiq/routing/web/views/routing.erb

@@ -0,0 +1,74 @@
+<div class="row header">
+  <div class="col-sm-12">
+    <h3>Routing</h3>
+    <p class="text-muted">
+      Read-only view of runtime job-class routing.
+      <strong>Park</strong> diverts a class's jobs to the worker-less
+      <code><%= h @parked_queue %></code> queue (reversible);
+      <strong>Blackhole</strong> drops them entirely.
+      Actions (park, blackhole, unpark, sweep, process_parked) are performed from the
+      Rails console &mdash; see the routing runbook.
+    </p>
+  </div>
+</div>
+
+<h4>Active routes</h4>
+<% if @routes.empty? %>
+  <p class="text-muted">No active routes.</p>
+<% else %>
+  <table class="table table-striped table-bordered table-white">
+    <thead>
+      <tr>
+        <th>Job class</th>
+        <th>Mode</th>
+        <th>Since</th>
+      </tr>
+    </thead>
+    <tbody>
+      <% @routes.sort.each do |class_name, info| %>
+        <tr>
+          <td><code><%= h class_name %></code></td>
+          <td>
+            <span class="label <%= info["mode"] == "blackhole" ? "label-danger" : "label-warning" %>">
+              <%= h info["mode"] %>
+            </span>
+          </td>
+          <td><%= h info["routed_at"] %></td>
+        </tr>
+      <% end %>
+    </tbody>
+  </table>
+<% end %>
+
+<h4>Parking queue: <code><%= h @parked_queue %></code> (<%= @parked_size %> jobs)</h4>
+<p class="text-muted">
+  Recover by backlog size: <strong>process_parked</strong> moves parked jobs back to their original queue (modest volume);
+  for 100Ks+, scale a worker on <code><%= h @parked_queue %></code> to <strong>drain in place</strong> instead of moving them.
+  Run these from the Rails console &mdash; see the routing runbook.
+</p>
+
+<% if @parked_breakdown.any? %>
+  <p class="text-muted">Distribution over the first <%= @breakdown_sample %> parked jobs (sampled; the count above is the true total).</p>
+  <table class="table table-striped table-bordered table-white">
+    <thead>
+      <tr>
+        <th>Job class</th>
+        <th>Parked</th>
+        <th>By original queue</th>
+      </tr>
+    </thead>
+    <tbody>
+      <% @parked_breakdown.sort.each do |class_name, stats| %>
+        <tr>
+          <td><code><%= h class_name %></code></td>
+          <td><%= stats["count"] %></td>
+          <td>
+            <% stats["by_original_queue"].sort.each do |queue, count| %>
+              <%= h queue %>: <%= count %><br/>
+            <% end %>
+          </td>
+        </tr>
+      <% end %>
+    </tbody>
+  </table>
+<% end %>

data/lib/sidekiq/routing/web.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "sidekiq/routing/web_extension"
+
+# Registers the "Routing" tab. The registration API changed across Sidekiq
+# versions, so branch the way sidekiq-cron does (apps run 7.3.x; commons locks 8.x).
+if defined?(Sidekiq::Web)
+  if Gem::Version.new(Sidekiq::VERSION) >= Gem::Version.new("8.0.0")
+    Sidekiq::Web.configure do |config|
+      config.register(
+        Sidekiq::Routing::WebExtension,
+        name: "routing",
+        tab: "Routing",
+        index: "routing"
+      )
+    end
+  elsif Gem::Version.new(Sidekiq::VERSION) >= Gem::Version.new("7.3.0")
+    Sidekiq::Web.register(
+      Sidekiq::Routing::WebExtension,
+      name: "routing",
+      tab: "Routing",
+      index: "routing"
+    )
+  else
+    Sidekiq::Web.register Sidekiq::Routing::WebExtension
+    Sidekiq::Web.tabs["Routing"] = "routing"
+  end
+end

data/lib/sidekiq/routing/web_extension.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Routing
+    # Sidekiq Web "Routing" tab. Read-only: it displays routing state only
+    # (active routes + parking-queue depth/breakdown). Every mutating
+    # operation (park, blackhole, unpark, sweep, process_parked) is performed from the
+    # Rails console via the Sidekiq::Routing API — never from the dashboard — so
+    # destructive actions stay deliberate and don't ride on the shared
+    # Sidekiq Web credentials. Reads aggregates live from Redis; never emits
+    # per-job telemetry. Inherits the existing Sidekiq Web auth.
+    module WebExtension
+      VIEWS = File.expand_path("web/views", __dir__)
+
+      def self.registered(app)
+        app.get "/routing" do
+          @parked_queue = Sidekiq::Routing.parked_queue
+          @routes = Sidekiq::Routing.routes
+          @parked_size = Sidekiq::Routing.parked_size
+          @breakdown_sample = Sidekiq::Routing.configuration.breakdown_sample_size
+          @parked_breakdown = Sidekiq::Routing.parked_breakdown
+          erb(File.read(File.join(VIEWS, "routing.erb")))
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/routing.rb

@@ -0,0 +1,208 @@
+# frozen_string_literal: true
+
+require "sidekiq"
+require "json"
+
+# Runtime, per-job-class parking/blackhole mechanism for Sidekiq incident response.
+#
+# Two modes per job class (see lib/sidekiq/CONTEXT.md and
+# docs/routing-how-it-works.md):
+#   - park (default, reversible): divert jobs to a worker-less parking queue.
+#   - blackhole: drop jobs entirely (only for classes safe to lose).
+#
+# State lives in a single Redis hash; the hot path reads a whole-hash snapshot
+# refreshed at most once per cache_ttl_seconds, so per-job cost is an in-memory
+# lookup rather than a Redis round-trip.
+module Sidekiq
+  module Routing
+    PARKED_QUEUE_DEFAULT = "routing_parked"
+
+    # Keys stamped into the job payload.
+    ORIGINAL_QUEUE_KEY = "routing_original_queue"
+    NO_DIVERT_KEY = "routing_no_divert"
+
+    MODE_PARK = "park"
+    MODE_BLACKHOLE = "blackhole"
+    MODES = [MODE_PARK, MODE_BLACKHOLE].freeze
+
+    @snapshot = nil
+    @snapshot_at = nil
+    @snapshot_mutex = Mutex.new
+
+    class << self
+      def configuration
+        @_configuration ||= Configuration.new
+      end
+
+      def setup
+        yield configuration if block_given?
+      end
+
+      # Registers the manual routing client + server middleware on both client
+      # and server configurations. This mirrors Sidekiq::Lock.install! so host
+      # apps can add the gem first and opt in to routing per app.
+      def install!
+        require "sidekiq/routing/middleware/client"
+        require "sidekiq/routing/middleware/server"
+
+        prepend_routing = ->(chain) { chain.prepend Middleware::Client }
+
+        Sidekiq.configure_client do |config|
+          config.client_middleware(&prepend_routing)
+        end
+
+        Sidekiq.configure_server do |config|
+          config.client_middleware(&prepend_routing)
+          config.server_middleware { |chain| install_server_middleware(chain) }
+        end
+      end
+
+      def enabled?
+        configuration.enabled
+      end
+
+      def logger
+        configuration.logger
+      end
+
+      def parked_queue
+        configuration.parked_queue
+      end
+
+      # ---- operator API: managing manual routes ----
+
+      def park(klass)
+        write(klass, MODE_PARK)
+      end
+
+      def blackhole(klass)
+        write(klass, MODE_BLACKHOLE)
+      end
+
+      def unpark(klass)
+        name = class_name(klass)
+        Store.delete(name)
+        reset_cache!
+        logger.warn("[Routing] unparked #{name}")
+        name
+      end
+
+      def parked?(klass)
+        mode(klass) == MODE_PARK
+      end
+
+      def routed?(klass)
+        !Store.fetch(class_name(klass)).nil?
+      end
+
+      def mode(klass)
+        Store.fetch(class_name(klass))&.fetch("mode", nil)
+      end
+
+      # All active manual routes, read straight from Redis (uncached) so operators
+      # and the Web tab always see the truth.
+      def routes
+        Store.all
+      end
+
+      # ---- hot path: snapshot lookup, used by the middleware ----
+
+      # Returns the route hash ({"mode"=>...}) for a class, or nil. Reads from a
+      # process-local snapshot of all routes, refreshed at most once per
+      # cache_ttl_seconds.
+      def route_for(klass_or_name)
+        snapshot[class_name(klass_or_name)]
+      end
+
+      def reset_cache!
+        @snapshot_mutex.synchronize do
+          @snapshot = nil
+          @snapshot_at = nil
+        end
+      end
+
+      # ---- parking queue introspection ----
+
+      def parked_size
+        Sidekiq::Queue.new(parked_queue).size
+      end
+
+      # { "SomeJob" => { "count" => 12, "by_original_queue" => { "within_1_minute" => 12 } } }
+      #
+      # Scans at most `sample` jobs (default Configuration#breakdown_sample_size), not the
+      # whole queue: the parking queue can hold millions during a flood and this is called on
+      # every Web tab load. The result is a distribution over the sampled head, not exact
+      # totals — use parked_size (O(1) LLEN) for the true total. Pass sample: nil to scan all.
+      def parked_breakdown(sample: configuration.breakdown_sample_size)
+        result = Hash.new { |h, k| h[k] = { "count" => 0, "by_original_queue" => Hash.new(0) } }
+        scanned = 0
+        Sidekiq::Queue.new(parked_queue).each do |job|
+          break if sample && scanned >= sample
+
+          klass = job.display_class
+          original = job.item[ORIGINAL_QUEUE_KEY] || "unknown"
+          result[klass]["count"] += 1
+          result[klass]["by_original_queue"][original] += 1
+          scanned += 1
+        end
+        result
+      end
+
+      # ---- recovery (thin wrappers; logic in Sweeper/ParkedProcessor) ----
+
+      def sweep(klass, queue: nil, limit: nil, batch_size: nil)
+        Sweeper.new.call(class_name(klass), queue:, limit:, batch_size:)
+      end
+
+      def process_parked(klass: nil, limit: nil, batch_size: nil)
+        ParkedProcessor.new.call(klass: klass && class_name(klass), limit:, batch_size:)
+      end
+
+      # Resolve the effective job-class name. Prefers the ActiveJob "wrapped"
+      # class so a wrapped job is matched by its real class, not the JobWrapper
+      # (mirrors Sidekiq's own display_class). Accepts a Class, String, or nil.
+      def class_name(klass_or_name)
+        return klass_or_name if klass_or_name.is_a?(String)
+
+        klass_or_name&.name
+      end
+
+      private
+
+        def install_server_middleware(chain)
+          unique_jobs = defined?(SidekiqUniqueJobs::Middleware::Server) && SidekiqUniqueJobs::Middleware::Server
+          if unique_jobs && chain.exists?(unique_jobs)
+            chain.insert_after unique_jobs, Middleware::Server
+          else
+            chain.add Middleware::Server
+          end
+        end
+
+        def write(klass, mode)
+          name = class_name(klass)
+          Store.set(name, mode:)
+          reset_cache!
+          logger.warn("[Routing] #{name} routed to #{mode}")
+          name
+        end
+
+        def snapshot
+          ttl = configuration.cache_ttl_seconds.to_f
+          now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+          current = @snapshot
+          return current if current && ttl.positive? && (now - @snapshot_at) < ttl
+
+          @snapshot_mutex.synchronize do
+            current = @snapshot
+            fresh = current && ttl.positive? && (now - @snapshot_at) < ttl
+            unless fresh
+              @snapshot = Store.all.freeze
+              @snapshot_at = now
+              current = @snapshot
+            end
+          end
+          current
+        end
+    end
+  end
+end

data/lib/sidekiq-routing.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "sidekiq/routing/version"
+
+# Routing core: defines Sidekiq::Routing and the manual driver (operator
+# park / blackhole overrides). Loaded first so Sidekiq::Routing exists before
+# the Auto sub-namespace reopens it.
+#
+# The Web tab (sidekiq/routing/web) is intentionally NOT required here. Host
+# apps require it at the Sidekiq::Web mount site so worker processes never load
+# the web framework.
+require "sidekiq/routing"
+require "sidekiq/routing/configuration"
+require "sidekiq/routing/store"
+require "sidekiq/routing/mover"
+require "sidekiq/routing/sweeper"
+require "sidekiq/routing/parked_processor"
+require "sidekiq/routing/middleware/client"
+require "sidekiq/routing/middleware/server"
+
+# Routing::Auto: the automatic driver — latency-driven movement of jobs between
+# SLA tiers. Opt-in via SIDEKIQ_ROUTING_AUTO_REROUTE_ENABLED.
+require "sidekiq/routing/auto/configuration"
+require "sidekiq/routing/auto/job_duration_tracker"
+require "sidekiq/routing/auto/noisy_neighbor_detector"
+require "sidekiq/routing/auto/batch_rerouter"
+require "sidekiq/routing/auto/router"
+require "sidekiq/routing/auto/reroute_job"