cogworker 0.2.2 → 0.4.0

data/lib/cogworker/reliable_fetch.rb

@@ -0,0 +1,225 @@
+# frozen_string_literal: true
+
+require 'digest/sha1'
+
+module Cogworker
+  # The default fetch (`config.fetch = :reliable`): every job is moved
+  # atomically (`LMOVE`, Redis >= 6.2) from its queue onto this process's own
+  # `cogworker:inprogress:<identity>` list, and only removed from there
+  # (`acknowledge`) once it has finished — succeeded, or been routed to
+  # retry/dead. A job is therefore never only in a processor's memory: if the
+  # process dies mid-job (OOM, SIGKILL, a host going away), the job is still
+  # on that list, and `recover_orphans` (run by every live process's
+  # `Scheduled` poller) puts it back on its queue once the dead process has
+  # gone `config.orphan_threshold` (default 5 min) without a heartbeat. The trade-off is at-least-once delivery: a job
+  # that was partly done when its process died runs again from the start.
+  #
+  # `LMOVE` can only take from one list at a time, so there's no blocking
+  # wait across several weighted queues the way `BasicFetch`'s single
+  # `BRPOP` has: instead, one round trip (FETCH_SCRIPT) tries the queues in
+  # weighted-shuffled order and takes the first job it finds, and an empty
+  # round sleeps EMPTY_POLL_INTERVAL before the next. Weighting and
+  # `Queue#pause!` behave exactly as in BasicFetch.
+  class ReliableFetch < BasicFetch
+    # The first pause after an empty round; each further empty round in a
+    # row doubles it, up to `config.fetch_idle_max_interval` (default 1s),
+    # and finding a job resets it. That cap is the worst-case pickup delay
+    # on an idle process — the trade-off against how often idle processors
+    # poll Redis (at 0.25s flat, 4 calls/s per thread).
+    EMPTY_POLL_INTERVAL = 0.25
+    MIN_REDIS_VERSION = Gem::Version.new('6.2')
+
+    # KEYS[1..n-1] = queue keys, in the order to try; KEYS[n] = in-progress list.
+    FETCH_SCRIPT = <<~LUA
+      local dest = KEYS[#KEYS]
+      for i = 1, #KEYS - 1 do
+        local job = redis.call("LMOVE", KEYS[i], dest, "RIGHT", "LEFT")
+        if job then
+          return {KEYS[i], job}
+        end
+      end
+      return false
+    LUA
+
+    # The hot path runs this by hash (EVALSHA), not by shipping the whole
+    # script text on every poll; see #run_fetch_script.
+    FETCH_SCRIPT_SHA = Digest::SHA1.hexdigest(FETCH_SCRIPT)
+
+    # KEYS[1] = an in-progress list; ARGV[1] = queue key prefix; ARGV[2] =
+    # queue for a payload whose own `queue` can't be read (the Processor
+    # then buries it as unparseable); ARGV[3] = periodic running-lock key
+    # prefix, ARGV[4] = its queued TTL. Oldest first, each onto the end its
+    # queue is popped from, so recovered jobs run next, in their original
+    # order. Atomic per call: two processes recovering the same list can't
+    # both move one job. A job from an `until_executed` periodic entry
+    # re-takes its running lock on the way (same rule as
+    # Periodic::RunningLock::RETAKE_FUNCTION), so the ticker can't start
+    # the next slot alongside the recovered run.
+    REQUEUE_SCRIPT = Periodic::RunningLock::RETAKE_FUNCTION + <<~LUA
+      local moved = 0
+      while true do
+        local job = redis.call("LPOP", KEYS[1])
+        if not job then
+          return moved
+        end
+        local queue = ARGV[2]
+        local ok, decoded = pcall(cjson.decode, job)
+        if ok and type(decoded) == "table" then
+          if type(decoded["queue"]) == "string" then
+            queue = decoded["queue"]
+          end
+          local pjid, jid = decoded["periodic_pjid"], decoded["jid"]
+          if decoded["periodic_until_executed"] == true and type(pjid) == "string" and type(jid) == "string" then
+            retake_running_lock(ARGV[3] .. pjid, jid, ARGV[4])
+          end
+        end
+        redis.call("RPUSH", ARGV[1] .. queue, job)
+        moved = moved + 1
+      end
+    LUA
+
+    # KEYS[1] = in-progress list, KEYS[2] = the job's queue; ARGV[1] = job.
+    GIVE_BACK_SCRIPT = <<~LUA
+      if redis.call("LREM", KEYS[1], 1, ARGV[1]) == 1 then
+        redis.call("RPUSH", KEYS[2], ARGV[1])
+      end
+    LUA
+
+    class << self
+      def supported?
+        version = Cogworker.config.redis { |c| c.info('server')['redis_version'] }
+        Gem::Version.new(version) >= MIN_REDIS_VERSION
+      rescue StandardError
+        true # can't tell (Redis not reachable yet): assume a current server
+      end
+
+      # Puts every job still on `identity`'s in-progress list back on its
+      # queue; returns how many. Used for this process's own unfinished jobs
+      # on shutdown (`Manager#stop!`) and for dead processes' lists.
+      def requeue_in_progress(identity)
+        Cogworker.config.redis do |c|
+          c.eval(REQUEUE_SCRIPT, keys: [RedisKeys.in_progress(identity)],
+                                 argv: [RedisKeys::QUEUE_PREFIX, 'default', RedisKeys.periodic_running(''),
+                                        Periodic::RunningLock.queued_ttl])
+        end
+      end
+
+      # Requeues the in-progress list of every process that has gone
+      # `config.orphan_threshold` without a heartbeat — i.e. that died
+      # without a clean shutdown (see #dead?). A process always
+      # writes its heartbeat before its first fetch (`Heartbeat#start!`), so
+      # a live process's list is never mistaken for an orphan. Walks
+      # IN_PROGRESS_IDENTITIES (each reliable-fetch process registers itself
+      # there on every beat) rather than SCANning the keyspace; `scan: true`
+      # (once per process, at boot) also picks up lists from before that
+      # set existed.
+      def recover_orphans(scan: false)
+        own = Cogworker.identity
+        candidate_identities(scan: scan).sum do |identity|
+          next 0 if identity == own || !dead?(identity)
+
+          requeue_in_progress(identity).tap do |moved|
+            forget_if_empty(identity)
+            next unless moved.positive?
+
+            Cogworker.logger.warn { "requeued #{moved} job(s) left in progress by dead process #{identity}" }
+          end
+        end
+      end
+
+      # Its presence key gone (no beat for Heartbeat::TTL) is not enough on
+      # its own: a live process that just couldn't beat for a minute (a
+      # Redis outage or failover, a long GVL-holding call) would lose its
+      # in-progress jobs to a second, parallel run. It also has to have gone
+      # `orphan_threshold` since its last recorded beat — compared on
+      # Redis's clock on both sides, so host clock skew doesn't matter. No
+      # recorded beat at all (a list from before LAST_BEAT existed) falls
+      # back to the presence key alone.
+      def dead?(identity)
+        Cogworker.config.redis do |c|
+          next false if c.exists?(RedisKeys.process(identity))
+
+          last = c.hget(RedisKeys::LAST_BEAT, identity)
+          last.nil? || c.time.first - last.to_f > Cogworker.config.orphan_threshold
+        end
+      end
+
+      def candidate_identities(scan:)
+        Cogworker.config.redis do |c|
+          identities = c.smembers(RedisKeys::IN_PROGRESS_IDENTITIES)
+          next identities unless scan
+
+          scanned = c.scan_each(match: "#{RedisKeys::IN_PROGRESS_PREFIX}*").map do |key|
+            key.delete_prefix(RedisKeys::IN_PROGRESS_PREFIX)
+          end
+          identities | scanned
+        end
+      end
+
+      def forget_if_empty(identity)
+        Cogworker.config.redis do |c|
+          next unless c.llen(RedisKeys.in_progress(identity)).zero?
+
+          c.srem?(RedisKeys::IN_PROGRESS_IDENTITIES, identity)
+          c.hdel(RedisKeys::LAST_BEAT, identity)
+        end
+      end
+    end
+
+    def retrieve_work
+      keys = active_keys
+      if keys.empty?
+        sleep(TIMEOUT)
+        return nil
+      end
+
+      result = run_fetch_script(keys.shuffle.uniq + [RedisKeys.in_progress(Cogworker.identity)])
+      unless result
+        idle_sleep
+        return nil
+      end
+
+      @idle_interval = nil
+      queue_key, raw_job = result
+      UnitOfWork.new(queue_key.delete_prefix(RedisKeys::QUEUE_PREFIX), raw_job)
+    end
+
+    # Atomically moves a fetched-but-unstarted job off the in-progress list
+    # and back to the end of its queue that's popped next. A no-op if it's
+    # no longer in progress (already requeued by `requeue_in_progress`).
+    def give_back(work)
+      Cogworker.config.redis do |c|
+        c.eval(GIVE_BACK_SCRIPT, keys: [RedisKeys.in_progress(Cogworker.identity), RedisKeys.queue(work.queue)],
+                                 argv: [work.raw_job])
+      end
+    end
+
+    def acknowledge(work)
+      Cogworker.config.redis { |c| c.lrem(RedisKeys.in_progress(Cogworker.identity), 1, work.raw_job) }
+    end
+
+    private
+
+    def run_fetch_script(keys)
+      Cogworker.config.redis do |c|
+        c.evalsha(FETCH_SCRIPT_SHA, keys: keys)
+      rescue Redis::CommandError => e
+        raise unless e.message.start_with?('NOSCRIPT')
+
+        c.eval(FETCH_SCRIPT, keys: keys) # loads it into the script cache for next time
+      end
+    rescue Redis::CommandError => e
+      raise unless e.message.match?(/unknown (redis )?command/i)
+
+      raise UnsupportedError, e.message
+    end
+
+    # Jittered (75–100% of the interval) so a process's threads, all idle
+    # at once, don't keep polling in lockstep.
+    def idle_sleep
+      max = Cogworker.config.fetch_idle_max_interval
+      @idle_interval = @idle_interval ? [@idle_interval * 2, max].min : [EMPTY_POLL_INTERVAL, max].min
+      sleep(@idle_interval * (0.75 + (rand * 0.25)))
+    end
+  end
+end

data/lib/cogworker/scheduled.rb

@@ -11,6 +11,29 @@ module Cogworker
   class Scheduled
     SETS = [RedisKeys::SCHEDULE, RedisKeys::RETRY].freeze
     POLL_INTERVAL = 5
+    # How often this poller also looks for jobs left in progress by a dead
+    # process (ReliableFetch.recover_orphans) — the first time right away,
+    # at boot. Every live process does it, whatever its own fetch mode: in
+    # a mixed fleet, or after a runtime fall-back to `:basic`, the lists of
+    # crashed reliable processes still need someone to requeue them, and
+    # with nothing registered it's a single SMEMBERS. Only the one-off
+    # keyspace SCAN (for lists from before IN_PROGRESS_IDENTITIES existed)
+    # is limited to processes actually using ReliableFetch. The requeue is
+    # atomic, so any number of processes doing it at once is safe.
+    ORPHAN_CHECK_INTERVAL = 60
+
+    # KEYS[1] = the ZSET, KEYS[2] = cogworker:dead, KEYS[3] = stats:failed;
+    # ARGV[1] = the entry, ARGV[2] = its dead-set score, ARGV[3] = the
+    # wrapper to store. Same claim-and-move-atomically shape as
+    # JobUtil::CLAIM_AND_REQUEUE_SCRIPT.
+    BURY_SCRIPT = <<~LUA
+      if redis.call("ZREM", KEYS[1], ARGV[1]) == 0 then
+        return 0
+      end
+      redis.call("ZADD", KEYS[2], ARGV[2], ARGV[3])
+      redis.call("INCR", KEYS[3])
+      return 1
+    LUA
 
     def initialize(manager)
       @manager = manager
@@ -34,13 +57,28 @@ module Cogworker
 
     private
 
+    # A failed poll is logged and retried next interval rather than ending
+    # the thread — a single Redis blip used to stop every scheduled/retry
+    # job in this process from ever reaching its queue again.
     def run
       until @manager.stopping?
-        enqueue_due_jobs unless @manager.quiet?
+        begin
+          recover_orphans_if_due
+          enqueue_due_jobs unless @manager.quiet?
+        rescue StandardError => e
+          Cogworker.logger.error { "Scheduled poll failed: #{e.class}: #{e.message}" }
+        end
         sleep(POLL_INTERVAL)
       end
-    rescue StandardError => e
-      Cogworker.logger.error { "Scheduled poller died: #{e.class}: #{e.message}" }
+    end
+
+    def recover_orphans_if_due
+      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      return if @next_orphan_check && now < @next_orphan_check
+
+      scan = @next_orphan_check.nil? && @manager.fetch_class == ReliableFetch
+      @next_orphan_check = now + ORPHAN_CHECK_INTERVAL
+      ReliableFetch.recover_orphans(scan: scan)
     end
 
     def enqueue_due_jobs
@@ -51,17 +89,27 @@ module Cogworker
       end
     end
 
+    # An entry that can't be requeued (not JSON, no queue) is buried in
+    # `dead` instead: left in place it would sit first by score, failing
+    # every poll and starving everything behind it; raising after the
+    # `zrem` used to lose it silently (and cut the rest of this poll short).
     def graduate(set, raw)
       Cogworker.config.redis do |c|
-        won = c.zrem(set, raw)
-        next unless won
+        next unless JobUtil.claim_and_requeue(c, set, raw) == :invalid
 
-        job = JSON.parse(raw)
-        c.multi do |pipeline|
-          pipeline.sadd(RedisKeys::QUEUES, job['queue'])
-          pipeline.lpush(RedisKeys.queue(job['queue']), raw)
-        end
+        bury(c, set, raw)
       end
     end
+
+    def bury(conn, set, raw)
+      error = JSON::ParserError.new("not a job (JSON object with a queue) in #{set}")
+      job = JobUtil.unparseable_job(raw, queue: nil, error: error)
+      buried = conn.eval(BURY_SCRIPT, keys: [set, RedisKeys::DEAD, RedisKeys::STATS_FAILED],
+                                      argv: [raw, job['failed_at'], JSON.generate(job)])
+      return unless buried == 1
+
+      Throughput.record('failed') # counted like Processor#bury_unparseable
+      Cogworker.logger.error { "unparseable entry in #{set} moved to dead: #{raw.to_s[0, 200]}" }
+    end
   end
 end

data/lib/cogworker/stats.rb

@@ -30,7 +30,7 @@ module Cogworker
 
     # The raw `INFO` reply as a flat Hash (server/clients/memory/stats/...
     # sections all merged together, same as the `redis` gem always returns
-    # it) — the Web UI's Stats tab picks a handful of fields (version,
+    # it) — the Web UI's Overview tab picks a handful of fields (version,
     # uptime, connected clients, memory usage) back out of this itself,
     # rather than this class pre-selecting/renaming them, so a future
     # consumer isn't limited to whatever subset this method chose.

data/lib/cogworker/status.rb

@@ -29,8 +29,12 @@ module Cogworker
       config.server_middleware { |chain| chain.add(ServerMiddleware, expiration.to_i) }
     end
 
+    # A Symbol (`:queued`, `:working`, `:retrying`, `:complete`, `:failed`,
+    # or `:stopped`/`:interrupted` if something external wrote those), or
+    # `nil` for an unknown/expired jid. `get` below stays the raw stored
+    # Hash, string values and all.
     def status(jid)
-      Storage.read(jid)&.fetch('status', nil)
+      Storage.read(jid)&.fetch('status', nil)&.to_sym
     end
 
     def get(jid)

data/lib/cogworker/swarm.rb

@@ -20,14 +20,30 @@ module Cogworker
     # that never exits (a hung Redis call outliving even Manager#stop!'s
     # own 25s join, or anything else) must not be able to wedge the whole
     # restart cycle, and the supervisor's entire signal-handling loop with
-    # it, forever.
-    GRACEFUL_STOP_TIMEOUT = 20
+    # it, forever. Kept well above a child's own worst-case clean shutdown
+    # (Manager#stop!'s 25s drain + up to BasicFetch::TIMEOUT + 1 waiting out
+    # fetching threads + ~1s for Heartbeat#stop!) so a child is never killed
+    # mid-`requeue_unfinished` — for `:basic`, that would lose the jobs.
+    GRACEFUL_STOP_TIMEOUT = 35
+
+    # Respawn backoff for a child that exits unexpectedly: none for the
+    # first crash after a healthy run, then doubling from
+    # RESPAWN_BASE_DELAY up to RESPAWN_MAX_DELAY while the slot keeps dying
+    # within HEALTHY_UPTIME of starting. Without it, a child crashing during
+    # boot (bad deploy, Redis unreachable, a typo in the init file) made the
+    # supervisor fork/crash in a tight loop forever.
+    HEALTHY_UPTIME = 30
+    RESPAWN_BASE_DELAY = 1
+    RESPAWN_MAX_DELAY = 60
 
     def initialize(argv, count: ENV.fetch('COGWORKER_COUNT', 1).to_i, phased: ENV['PHASED_RESTART'] == 'true')
       @argv = argv
       @count = [count, 1].max
       @phased = phased
       @children = {} # pid => slot index
+      @started_at = {} # slot => monotonic time of its current child's fork
+      @crash_streak = Hash.new(0) # slot => consecutive quick crashes
+      @pending_respawns = {} # slot => monotonic time it may be forked again
       @signal_queue = ::Queue.new
       @stopping = false
     end
@@ -53,9 +69,11 @@ module Cogworker
         # indefinitely, so this OS process would never actually go away —
         # and `Swarm#initiate_restart`'s own `Process.waitpid(pid)` for this
         # exact child would then block forever right along with it.
+        Cogworker.flush_output!
         ::Process.exit!(true)
       end
       @children[pid] = slot
+      @started_at[slot] = monotonic_now
       Cogworker.logger.info { "swarm: started child pid=#{pid} slot=#{slot}" }
     end
 
@@ -70,13 +88,16 @@ module Cogworker
       until @stopping && @children.empty?
         drain_signals
         reap_children
+        respawn_due_children
+        kill_stragglers
         sleep 0.2
       end
     end
 
     def drain_signals
       until @signal_queue.empty?
-        case @signal_queue.pop
+        signal = @signal_queue.pop
+        case signal
         when :quiet then relay(Signals::QUIET)
         when :stop then initiate_stop
         when :restart
@@ -93,9 +114,25 @@ module Cogworker
 
     def initiate_stop
       @stopping = true
+      @stop_deadline = monotonic_now + GRACEFUL_STOP_TIMEOUT
+      @pending_respawns.clear
       relay(Signals::STOP)
     end
 
+    # Same bound as a phased restart's `wait_for_exit`: a child that hasn't
+    # exited GRACEFUL_STOP_TIMEOUT after the stop was relayed is
+    # force-killed (once), so one hung child can't keep the supervisor —
+    # and whatever is waiting on it — alive forever.
+    def kill_stragglers
+      return unless @stop_deadline && monotonic_now > @stop_deadline
+
+      @stop_deadline = nil
+      @children.each_key do |pid|
+        Cogworker.logger.warn { "swarm: child pid=#{pid} didn't stop within #{GRACEFUL_STOP_TIMEOUT}s, killing" }
+        safe_kill(pid, 'KILL')
+      end
+    end
+
     # Non-phased: signal every child at once; the normal reap/respawn path
     # below brings each one back as it exits, so there's a brief capacity
     # dip but no explicit special-casing needed. Phased: replace children
@@ -155,7 +192,31 @@ module Cogworker
       if @stopping
         Cogworker.logger.info { "swarm: child pid=#{pid} slot=#{slot} exited" }
       else
-        Cogworker.logger.warn { "swarm: child pid=#{pid} slot=#{slot} exited unexpectedly, respawning" }
+        schedule_respawn(pid, slot)
+      end
+    end
+
+    def schedule_respawn(pid, slot)
+      uptime = monotonic_now - @started_at.fetch(slot, monotonic_now)
+      @crash_streak[slot] = uptime < HEALTHY_UPTIME ? @crash_streak[slot] + 1 : 0
+      delay = respawn_delay(@crash_streak[slot])
+      Cogworker.logger.warn do
+        "swarm: child pid=#{pid} slot=#{slot} exited unexpectedly after #{uptime.round(1)}s, " \
+          "respawning in #{delay}s"
+      end
+      @pending_respawns[slot] = monotonic_now + delay
+    end
+
+    def respawn_delay(streak)
+      return 0 if streak <= 1
+
+      [RESPAWN_BASE_DELAY * (2**(streak - 2)), RESPAWN_MAX_DELAY].min
+    end
+
+    def respawn_due_children
+      now = monotonic_now
+      @pending_respawns.select { |_slot, at| at <= now }.each_key do |slot|
+        @pending_respawns.delete(slot)
         fork_child(slot)
       end
     end

data/lib/cogworker/testing/empty_queue_error.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Cogworker
+  module Testing
+    # Raised by `Testing.perform_one`/`SomeJob.perform_one` when fake mode
+    # has nothing recorded for that class.
+    class EmptyQueueError < StandardError; end
+  end
+end

data/lib/cogworker/testing.rb

@@ -14,8 +14,9 @@ module Cogworker
   #   but the normalized job hash is appended to `jobs_for(klass_name)`
   #   instead of touching Redis — nothing is scheduled or executed. Use
   #   `SomeJob.jobs` (added to `Job::ClassMethods`) to assert what got
-  #   pushed, `SomeJob.clear`/`Testing.clear_jobs!` to reset between
-  #   examples.
+  #   pushed, `SomeJob.perform_one`/`SomeJob.drain`/`Testing.drain_all` to
+  #   then actually run them, `SomeJob.clear`/`Testing.clear_jobs!` to reset
+  #   between examples.
   # - `:inline` — the job runs synchronously, right here in the calling
   #   thread, through the real server middleware chain
   #   (`Cogworker.config.server_chain`) — same as `Processor#execute`, minus
@@ -75,8 +76,31 @@ module Cogworker
         registry.clear
       end
 
+      # Fake mode's "now run them": `perform_one` takes the oldest recorded
+      # job for a class off its fake queue and runs it via `perform_inline`
+      # (real server middleware chain, errors propagate); `drain` repeats
+      # until that class's queue is empty, `drain_all` until every class's
+      # is — both also running whatever those jobs push in turn. Also
+      # reachable as `SomeJob.perform_one`/`SomeJob.drain`.
+      def perform_one(klass_name)
+        job = jobs_for(klass_name).shift
+        raise EmptyQueueError, "no #{klass_name} jobs recorded" unless job
+
+        perform_inline(job)
+      end
+
+      def drain(klass_name)
+        perform_one(klass_name) until jobs_for(klass_name).empty?
+      end
+
+      def drain_all
+        while (klass_name = registry.find { |_name, jobs| jobs.any? }&.first)
+          drain(klass_name)
+        end
+      end
+
       # Runs one job synchronously through the real server middleware chain.
-      # Only ever called from `Client.push` while `inline?`.
+      # Called from `Client.push` while `inline?`, and by `perform_one`.
       def perform_inline(job)
         klass = Object.const_get(job['class'])
         worker = klass.new

data/lib/cogworker/unique_jobs/client_middleware.rb

@@ -9,15 +9,14 @@ module Cogworker
     # continue down the chain to the real push. A job that loses the claim
     # (another copy is already enqueued/scheduled/running) never reaches
     # `dispatch` at all — it sets `job['unique_skipped']` instead, which
-    # `Client.push` reads to return `nil` rather than a jid, the same
-    # "nothing was actually pushed" signal `sidekiq-unique-jobs` gives.
+    # `Client.push` reads to return `nil` rather than a jid — the
+    # "nothing was actually pushed" signal to the caller.
     class ClientMiddleware
       def call(_worker_class, job, _queue, redis_pool = Cogworker.config.redis_pool)
         return yield unless UniqueJobs.until_executed?(job)
 
         acquired = redis_pool.with do |c|
-          c.set(RedisKeys.unique_lock(UniqueJobs.digest(job)), job['jid'],
-                nx: true, ex: Cogworker.config.unique_lock_ttl)
+          c.set(RedisKeys.unique_lock(UniqueJobs.digest(job)), job['jid'], nx: true, ex: lock_ttl(job))
         end
 
         if acquired
@@ -26,6 +25,16 @@ module Cogworker
           job['unique_skipped'] = true
         end
       end
+
+      private
+
+      # Counted from when the job is due, not from now: a `perform_in`/
+      # `perform_at` further out than `unique_lock_ttl` used to lose its
+      # lock before it even ran, letting a duplicate in.
+      def lock_ttl(job)
+        wait = job['at'] ? [job['at'].to_f - Time.now.to_f, 0].max : 0
+        (Cogworker.config.unique_lock_ttl + wait).ceil
+      end
     end
   end
 end

data/lib/cogworker/unique_jobs/release_middleware.rb

@@ -22,8 +22,11 @@ module Cogworker
 
       private
 
+      # Owner-checked: if this job's lock already expired and passed to a
+      # newer copy, this must not release *that* one's lock (which would let
+      # a third copy in while the newer one is still running).
       def release(job)
-        Cogworker.config.redis { |c| c.del(RedisKeys.unique_lock(UniqueJobs.digest(job))) }
+        OwnedKey.delete(RedisKeys.unique_lock(UniqueJobs.digest(job)), job['jid'])
       end
     end
   end

data/lib/cogworker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cogworker
-  VERSION = '0.2.2'
+  VERSION = '0.4.0'
 end

data/lib/cogworker/web/layout.rb

@@ -178,8 +178,8 @@ module Cogworker
       end
 
       # Cluster-wide "quiet every process" (`Routes::Workers`'s own `POST
-      # /workers/pause_all`) — the mock's "Pause intake" button. Unlike real
-      # Sidekiq OSS (where quiet is a one-way trip back to a fresh process),
+      # /workers/pause_all`) — the mock's "Pause intake" button. Quiet isn't
+      # a one-way trip that only a fresh process can undo:
       # `Manager#quiet` here is a plain in-memory flag, so `resume_intake_
       # button` below is a genuine, symmetric undo rather than a full
       # process restart.
@@ -237,7 +237,7 @@ module Cogworker
       # each one's visible text to the *browser's local* time, in
       # `Web.time_format`. Registered on `htmx:load`, which htmx fires once
       # for the initial page **and** again after every fragment swap — so
-      # newly-swapped-in timestamps (an auto-refreshed Busy/Scheduled/Dead
+      # newly-swapped-in timestamps (an auto-refreshed Workers/Jobs/Schedules
       # tab) get the same treatment without a full reload.
       #
       # `window.cogworkerFormatTime` is the same formatter exposed globally

data/lib/cogworker/web/optional_session.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require 'rack/session/cookie'
+
+module Cogworker
+  class Web
+    # The Web UI's own session cookie, applied only when nothing upstream
+    # already provides `env['rack.session']`. Mounted inside an app that has
+    # its own session (Rails, or any `Rack::Session::*` in front), an
+    # unconditional `Rack::Session::Cookie` here replaced that session with
+    # a copy for everything inside the Web UI — so an auth middleware added
+    # via `Web.use` read and wrote the copy, never the host's real session,
+    # and every response carried a second `cogworker.session` cookie.
+    class OptionalSession
+      def initialize(app, **options)
+        @app = app
+        @with_session = Rack::Session::Cookie.new(app, **options)
+      end
+
+      def call(env)
+        env['rack.session'] ? @app.call(env) : @with_session.call(env)
+      end
+    end
+  end
+end