wurk 1.7.6 → 1.8.1

data/lib/wurk/batch/callbacks.rb

@@ -1,45 +1,73 @@
 # frozen_string_literal: true
 
 require 'json'
+require_relative '../lua'
 
 module Wurk
   class Batch
+    # Raised by `Callbacks.enqueue_callbacks` once every callback for the event
+    # has been attempted and at least one could not be enqueued, so the caller
+    # leaves the event's dedup marker unwritten and the next fire retries it.
+    class CallbackEnqueueError < StandardError; end
+
     # Fires batch callbacks (`:success`, `:complete`, `:death`) by enqueuing
-    # them as ordinary jobs on the batch's `callback_queue`. Dedup is via
-    # b-<bid>-notify so the same callback can't be enqueued twice even
-    # if multiple workers race to ack the final job.
+    # them as ordinary jobs on the batch's `callback_queue`.
     #
     # Callback wrapper job: Wurk::Batch::CallbackJob — given a target spec
     # ("Klass" or "Klass#method") and options hash, it instantiates and
     # invokes on_<event> (or the named method) with the Status snapshot.
+    #
+    # A child batch's `:complete` and `:success` callback jobs are enqueued
+    # *into its parent batch* (their payload carries the parent's `bid`), so
+    # they are live jids of the parent: the parent cannot drain until they
+    # have run. That is what makes the spec §2.9 step workflow work —
+    # `step1_done` reopens the parent and adds `step2` before the parent can
+    # fire. A callback job that fails counts against the parent like any of
+    # its jobs (a death suppresses the parent's `:success`). `:death`
+    # callbacks stay outside: they fire while the child is still running, and
+    # a notification must not hold or poison the parent.
     module Callbacks
       module_function
 
-      # Called from the server middleware after BATCH_ACK_SUCCESS (and from
-      # DeathHandler when a death drains the last live jid). Fires `:complete`
-      # when live jids hit 0; fires `:success` when pending also hits 0 and
-      # there have been no deaths.
+      # In-process re-drive for the fire and ack paths. Nothing else re-drives
+      # a fire once the job that drained the batch has acked, so a transient
+      # Redis error there would otherwise strand the callbacks for good.
+      REDRIVE_ATTEMPTS = 3
+      REDRIVE_BACKOFF = 0.05
+
+      # Runs on the post-state of a batch script that moved a member out of the
+      # live jids or pending-children set (see the gate note heading
+      # lua/batch_ack_success.lua). Fires `:complete` once both sets are empty, and
+      # `:success` when pending is also 0 and nothing in the subtree is dead;
+      # then hands the drain to the parent.
       #
-      # Both fires are additionally gated on `b-<bid>-pkids` being empty —
-      # children whose own subtree hasn't finished yet (#209). Spec §2.4:
-      # child `:complete`/`:success` fire before the parent's, so when the
-      # parent's *own* last job acks while a child batch is still running,
-      # nothing fires here; the last child's propagate_to_parent re-invokes
-      # this and fires then. The SREM in pkids_drained? happens before that
-      # re-invocation, so at most one of the racing paths reaches a fire and
-      # the callback markers absorb the rest (see `fire_complete` for the one
-      # window that can still duplicate).
-      def maybe_fire(bid, pending:, live:)
-        return unless live.zero?
-        return unless kids_finished?(bid)
+      # Spec §2.4: child `:complete`/`:success` fire before the parent's, so a
+      # parent whose own jobs are done while a child batch is still running
+      # waits here (kids > 0); the child's `propagate_to_parent` re-enters with
+      # the parent's fresh state when it finishes.
+      def maybe_fire(bid, pending:, live:, kids:)
+        return unless live.zero? && kids.zero?
 
         fire_complete(bid)
         fire_success(bid) if pending.zero? && !subtree_dead?(bid)
         propagate_to_parent(bid)
       end
 
-      def kids_finished?(bid)
-        Wurk.redis { |conn| conn.call('SCARD', "b-#{bid}-pkids") }.to_i.zero?
+      # Yields the attempt number (0-based) and retries a StandardError up to
+      # REDRIVE_ATTEMPTS times in total, re-raising the last one. Every block
+      # handed to it is idempotent: the ack scripts are SREM/SADD-guarded and
+      # the fires are marker-guarded.
+      def retrying(attempts: REDRIVE_ATTEMPTS)
+        attempt = 0
+        begin
+          yield attempt
+        rescue StandardError
+          attempt += 1
+          raise if attempt >= attempts
+
+          sleep(REDRIVE_BACKOFF * attempt)
+          retry
+        end
       end
 
       # Fired from Wurk::Batch::DeathHandler whenever a death makes the died
@@ -58,13 +86,20 @@ module Wurk
       # dashboard and `subtree_dead?` all still see a dead batch. `:complete`
       # and `:success` have no such fallback — the callback is their whole
       # signal — and this claim additionally gates `cascade_death`, which
-      # would otherwise re-walk the ancestor chain on every re-invocation.
+      # would otherwise re-walk the ancestor chain on every re-invocation. An
+      # enqueue that *raises* is different from a crash: the claim is handed
+      # back so the caller's re-drive fires `:death` again.
       def fire_death(bid)
         record_event(bid, 'death_at')
         index_dead(bid)
         return unless dedup_set(bid, 'death')
 
-        enqueue_callbacks(bid, 'death')
+        begin
+          enqueue_callbacks(bid, 'death')
+        rescue CallbackEnqueueError
+          Wurk.redis { |conn| conn.call('DEL', "b-#{bid}-death") }
+          raise
+        end
         cascade_death(bid)
       end
 
@@ -95,22 +130,19 @@ module Wurk
       end
 
       # `:complete` and `:success` mark their dedup key *after* the enqueue,
-      # never before (F16). Nothing re-drives a fire once the acking job's
-      # BATCH_ACK_SUCCESS has SREM'd its jid — that job's retry gets
-      # `pending == -1` and returns before maybe_fire — so a claim-then-enqueue
-      # ordering turns a crash in between into callbacks that are never
-      # enqueued by anyone, ever. Enqueuing first makes the durable side effect
-      # happen before the marker that suppresses it.
+      # never before (F16): a claim-then-enqueue ordering turns a crash in
+      # between into callbacks nobody ever enqueues. Enqueuing first makes the
+      # durable side effect happen before the marker that suppresses it, and an
+      # enqueue that raised leaves the marker unwritten so the re-drive (the
+      # caller's `retrying`, or a reclaimed re-run of the draining job) fires
+      # the event again.
       #
       # The accepted direction is a duplicate over a lost callback: callback
       # jobs retry like any other job and must already be idempotent (spec
       # §2.4, §12 "Callback retries"), so firing one twice is a cost the app
       # is required to absorb, while losing one silently strands the batch.
-      #
-      # `dedup_marked?` still collapses every *sequential* re-invocation — a
-      # reclaimed child re-running propagate_to_parent, a second DeathHandler
-      # pass — so the duplicate window is only two genuinely concurrent acks
-      # interleaving between each other's check and mark.
+      # Concurrent duplicates are confined to re-drives: the fire gate admits
+      # exactly one caller per drain transition.
       #
       # `record_event` stays ahead of the enqueue: the callback job reads a
       # Status snapshot and must see `complete_at`/`success_at` already set.
@@ -140,11 +172,8 @@ module Wurk
       # full success. `created_at` shares the CLOCK_REALTIME epoch we record it
       # with. No-op without a dogstatsd client.
       #
-      # Strictly best-effort: this runs on the acking job's thread ahead of the
-      # enqueue, and that ack already removed the jid, so a raise here (e.g. a
-      # Redis hiccup on the HGET) would abort `fire_success` with nothing left
-      # to re-drive it — the success callbacks and linger would be stranded for
-      # good. Swallow and log instead.
+      # Strictly best-effort: a raise here would abort `fire_success` ahead of
+      # the enqueue for the sake of a metric. Swallow and log instead.
       def emit_duration_metric(bid)
         created = Wurk.redis { |conn| conn.call('HGET', "b-#{bid}", 'created_at') }
         return if created.nil? || created.to_s.empty?
@@ -163,14 +192,13 @@ module Wurk
         raw     = Wurk.redis { |conn| conn.call('HGET', "b-#{bid}", 'linger') }
         seconds = raw.nil? || raw.to_s.empty? ? Batch::POST_SUCCESS_EXPIRY_SECONDS : raw.to_i
         Wurk.redis do |conn|
-          Batch.keys_for(bid).each { |key| conn.call('EXPIRE', key, seconds) }
+          conn.pipelined { |pipe| Batch.keys_for(bid).each { |key| pipe.call('EXPIRE', key, seconds) } }
         end
       end
 
       # True once `b-<bid>-<event>` exists, i.e. an enqueue pass for `event`
       # has completed. The read-side half of the enqueue-then-mark ordering in
-      # `fire_complete`/`fire_success`; `fire_death` needs no equivalent
-      # because its `dedup_set` still doubles as the claim.
+      # `fire_complete`/`fire_success`.
       def dedup_marked?(bid, event)
         Wurk.redis { |conn| conn.call('EXISTS', "b-#{bid}-#{event}") }.to_i == 1
       end
@@ -190,17 +218,17 @@ module Wurk
         end
       end
 
-      # The HSETs resurrect the hash when a callback fires for a batch whose keys
-      # already expired (a child batch outliving its parent's 30d window), so the
-      # write is followed by an NX stamp — without it the resurrected hash would
-      # have no clock at all. NX leaves a live batch's expiry, and the shorter
-      # post-success `linger` window, untouched.
+      # The HSET resurrects the hash when a callback fires for a batch whose keys
+      # already expired, so the write is followed by an NX stamp — without it the
+      # resurrected hash would have no clock at all. NX leaves a live batch's
+      # expiry, and the shorter post-success `linger` window, untouched.
       def record_event(bid, field)
         now = ::Process.clock_gettime(::Process::CLOCK_REALTIME)
         Wurk.redis do |conn|
-          conn.call('HSET', "b-#{bid}", field, now.to_s)
-          conn.call('HSET', "b-#{bid}", field.to_s.sub('_at', ''), '1')
-          conn.call('EXPIRE', "b-#{bid}", Batch::DEFAULT_EXPIRY_SECONDS, 'NX')
+          conn.pipelined do |pipe|
+            pipe.call('HSET', "b-#{bid}", field, now.to_s, field.delete_suffix('_at'), '1')
+            pipe.call('EXPIRE', "b-#{bid}", Batch::DEFAULT_EXPIRY_SECONDS, 'NX')
+          end
         end
       end
 
@@ -233,13 +261,13 @@ module Wurk
       # last dead job is manually retried back to success, the descendant
       # clears its OWN death mark (#212, in BATCH_PUSH) — but every ancestor
       # was marked by the death *cascade*, not by a jid in its own died set,
-      # so nothing here ever cleared them and the ancestor's `:success`
-      # stayed suppressed forever. Re-evaluate this batch: drop its durable
-      # death mark and `dead-batches` membership once its own died set is
-      # empty AND no child still carries a death mark. The `b-<bid>-death`
-      # notify dedup key is deliberately left intact, so a later re-death
-      # re-marks the batch (fire_death restores the flag before its own
-      # dedup guard) without ever re-enqueuing `:death`.
+      # so nothing there ever clears them. Re-evaluated whenever a child
+      # drains (propagate_to_parent): drop the durable death mark and
+      # `dead-batches` membership once the batch's own died set is empty AND
+      # no child still carries a death mark. The `b-<bid>-death` notify dedup
+      # key is deliberately left intact, so a later re-death re-marks the batch
+      # (fire_death restores the flag before its own dedup guard) without ever
+      # re-enqueuing `:death`.
       def clear_death_on_recovery(bid)
         return unless death_fired?(bid)
         return if own_died_remaining?(bid)
@@ -261,26 +289,46 @@ module Wurk
       end
 
       # Per-callback rescue: one bad spec or a transient enqueue failure must
-      # not strand the batch with the remaining callbacks for this event
-      # un-enqueued. Log and move on so every other callback still fires.
+      # not keep the remaining callbacks for this event from being attempted.
+      # Each failure reaches the error handlers; once all were attempted, any
+      # failure raises so the caller does not write the event's marker.
       def enqueue_callbacks(bid, event)
-        callbacks, queue = callback_specs_for(bid)
+        failed = callback_items(bid, event).count { |item| !push_callback(bid, event, item) }
+        return if failed.zero?
+
+        raise CallbackEnqueueError, "batch #{bid}: #{failed} #{event} callback(s) failed to enqueue"
+      end
 
-        callbacks.each do |(cb_event, target, options)|
+      def callback_items(bid, event)
+        callbacks, queue, parent_bid = callback_specs_for(bid)
+        parent_bid = nil if event == 'death'
+        callbacks.filter_map do |(cb_event, target, options)|
           next unless cb_event == event
 
-          enqueue_callback_job(bid, target, event, options, queue)
-        rescue StandardError => e
-          Wurk.logger.warn("batch #{bid}: #{event} callback #{target.inspect} enqueue failed: #{e.class}: #{e.message}")
+          item = callback_item(bid, target, event, options, queue)
+          item['bid'] = parent_bid if parent_bid
+          item
         end
       end
 
+      # True when the callback job was enqueued.
+      def push_callback(bid, event, item)
+        Batch.with_thread_batch(nil, nil) { Wurk::Client.push(item) }
+        true
+      rescue StandardError => e
+        Wurk.configuration.handle_exception(
+          e, { context: "batch #{bid}: #{event} callback #{item['args'][1].inspect} enqueue failed", bid: bid }
+        )
+        false
+      end
+
       def callback_specs_for(bid)
-        raw = Wurk.redis { |conn| conn.call('HMGET', "b-#{bid}", 'callbacks', 'callback_queue') }
-        callbacks_json, queue = raw
+        callbacks_json, queue, parent_bid = Wurk.redis do |conn|
+          conn.call('HMGET', "b-#{bid}", 'callbacks', 'callback_queue', 'parent_bid')
+        end
         queue = 'default' if queue.nil? || queue.empty?
-        parsed = parse_callbacks(callbacks_json)
-        [parsed, queue]
+        parent_bid = nil if parent_bid.nil? || parent_bid.empty?
+        [parse_callbacks(callbacks_json), queue, parent_bid]
       end
 
       def parse_callbacks(raw)
@@ -291,46 +339,49 @@ module Wurk
         []
       end
 
-      def enqueue_callback_job(bid, target, event, options, queue)
-        Wurk::Client.push(
+      # Pushed (by push_callback) with the batch thread-locals cleared: a
+      # fire can run on a thread that is inside some other batch's `#jobs`
+      # block (a hold released at the end of a nested block), and the client
+      # middleware would otherwise stamp that batch's bid over the parent bid
+      # — or the buffer would swallow the push until that block exits.
+      def callback_item(bid, target, event, options, queue)
+        {
           'class' => 'Wurk::Batch::CallbackJob',
           'args' => [bid, target, event, options],
           'queue' => queue,
           'retry' => true
-        )
+        }
       end
 
-      # When a child batch finishes (its live jids hit 0 — by success or
-      # death), remove it from the parent's pkids set so the parent's own
-      # callbacks wait on the full subtree. When the parent's pkids hits 0,
-      # re-run the parent's maybe_fire: if its own counts are already at
-      # zero (the parent-acks-first race), this is what finally fires it.
+      # When a child batch drains (by success or death), drop it from the
+      # parent's pending children and re-evaluate the parent with the state
+      # BATCH_KID_DONE read atomically alongside the SREM. A re-run of the
+      # child (a reclaimed job, a re-drive) removes nothing but still reports
+      # the parent's state, so a parent fire lost to a crash is re-driven too.
+      #
+      # A recovered child may have lifted the last death from the parent's
+      # subtree — clear the parent's cascaded mark first, so the parent's
+      # `:success` can fire whenever it next drains (here, or when this
+      # child's callback jobs ack into it). Harmless on the death path: the
+      # dying child still carries its mark, so any_child_dead? keeps the
+      # parent dead.
       def propagate_to_parent(bid)
         parent_bid = parent_bid_for(bid)
         return if parent_bid.nil? || parent_bid.empty?
-        return unless pkids_drained?(parent_bid, bid)
 
-        # A recovered child may have lifted the last death from the parent's
-        # subtree — clear the parent's cascaded mark before its gate runs, so
-        # `:success` can fire. Harmless on the death path: the dying child
-        # still carries its mark, so any_child_dead? keeps the parent dead.
         clear_death_on_recovery(parent_bid)
-        maybe_fire(parent_bid, pending: pending_for(parent_bid), live: live_for(parent_bid))
+        pending, live, kids = Wurk.redis do |conn|
+          Wurk::Lua::Loader.eval_cached(
+            conn, :batch_kid_done,
+            keys: ["b-#{parent_bid}", "b-#{parent_bid}-jids", "b-#{parent_bid}-pkids"], argv: [bid]
+          )
+        end
+        maybe_fire(parent_bid, pending: pending.to_i, live: live.to_i, kids: kids.to_i)
       end
 
       def parent_bid_for(bid)
         Wurk.redis { |conn| conn.call('HGET', "b-#{bid}", 'parent_bid') }
       end
-
-      def pkids_drained?(parent_bid, child_bid)
-        Wurk.redis do |conn|
-          conn.call('SREM', "b-#{parent_bid}-pkids", child_bid)
-          conn.call('SCARD', "b-#{parent_bid}-pkids").to_i.zero?
-        end
-      end
-
-      def pending_for(bid) = Wurk.redis { |conn| conn.call('HGET', "b-#{bid}", 'pending') }.to_i
-      def live_for(bid)    = Wurk.redis { |conn| conn.call('SCARD', "b-#{bid}-jids") }.to_i
     end
   end
 end

data/lib/wurk/batch/death_handler.rb

@@ -10,32 +10,43 @@ module Wurk
     # carries a `bid`, we BATCH_ACK_COMPLETE → record the death → fire
     # `:death` callback exactly once per batch (first death only).
     #
+    # The retry layer runs this after the job is already in the morgue and
+    # acked off its private list, and swallows whatever it raises — nothing
+    # would ever deliver this death to the batch again. So the whole pass is
+    # re-driven in-process (`Callbacks.retrying`): the ack script is
+    # SREM/SADD-guarded and the fires are marker-guarded, so a replay is safe.
+    # A replay can't trust `first_death` (the lost attempt may already have
+    # moved the jid into the died set), so it falls back to the `:death`
+    # claim marker to decide whether `:death` still has to fire.
+    #
     # Spec: docs/target/sidekiq-pro.md §2.4 (`:death`).
     class DeathHandler
       def self.call(job, _exception)
         bid = job['bid']
         return unless bid
 
-        result = Wurk.redis do |conn|
+        Callbacks.retrying { |attempt| record(bid, job['jid'], replay: attempt.positive?) }
+      end
+
+      def self.record(bid, jid, replay:)
+        live, _died, first_death, kids, pending = Wurk.redis do |conn|
           Wurk::Lua::Loader.eval_cached(
             conn,
             :batch_ack_complete,
-            keys: ["b-#{bid}", "b-#{bid}-jids", "b-#{bid}-died", "b-#{bid}-failed"],
-            argv: [job['jid'], Batch::DEFAULT_EXPIRY_SECONDS]
+            keys: ["b-#{bid}", "b-#{bid}-jids", "b-#{bid}-died", "b-#{bid}-failed", "b-#{bid}-pkids"],
+            argv: [jid, Batch::DEFAULT_EXPIRY_SECONDS]
           )
-        end
-        live, _died, first_death = Array(result).map(&:to_i)
+        end.map(&:to_i)
 
         restamp_ttls(bid)
 
-        Wurk::Batch::Callbacks.fire_death(bid) if first_death == 1
-        return unless live.zero?
+        Callbacks.fire_death(bid) if first_death == 1 || (replay && !Callbacks.dedup_marked?(bid, 'death'))
 
         # Through the gated maybe_fire, not a direct fire_complete: this batch
         # may still have running child batches, and spec §2.4 ordering says
         # its `:complete` must wait for theirs (#209). `:success` stays
         # suppressed regardless — the death above set the durable death flag.
-        Wurk::Batch::Callbacks.maybe_fire(bid, pending: Wurk::Batch::Callbacks.pending_for(bid), live: 0)
+        Callbacks.maybe_fire(bid, pending: pending, live: live, kids: kids)
       end
 
       # BATCH_ACK_COMPLETE stamps the two keys it can itself resurrect; this

data/lib/wurk/batch/server_middleware.rb

@@ -1,17 +1,16 @@
 # frozen_string_literal: true
 
-require 'json'
 require_relative '../middleware'
 require_relative '../lua'
 require_relative '../job'
 require_relative '../job_retry'
+require_relative 'callbacks'
 
 module Wurk
   class Batch
     # Server middleware. Runs around `perform` for any job carrying a `bid`.
-    # On success → BATCH_ACK_SUCCESS → if pending hit zero and no deaths,
-    # enqueue `:success` callback jobs; if live jids hit zero, enqueue
-    # `:complete` callback jobs.
+    # On success → BATCH_ACK_SUCCESS → when that drains the batch, fire its
+    # callbacks (Callbacks.maybe_fire).
     #
     # On a job raising (and thus heading to retry), records a transient
     # failure → BATCH_ACK_FAILED → the jid joins `b-<bid>-failed` and
@@ -21,8 +20,18 @@ module Wurk
     # Limiter::Rescheduled re-enqueue — and cooperative IterableJob
     # interruption) are re-raised as neither success nor failure.
     #
+    # The success ack runs outside the job's rescue: once `perform` returned,
+    # nothing the batch bookkeeping raises may be mistaken for the job failing,
+    # or a job that already did its work is retried and runs twice. The ack
+    # itself is retried in-process and only raises (sending the job to retry,
+    # at-least-once) when Redis stays unreachable — a jid left in the live set
+    # would hold the batch open forever. The fire after it never raises: it is
+    # re-driven in-process and then reported to the error handlers.
+    #
     # Invalidated batches short-circuit: the job is skipped without
     # raising — counts as a "success" for batch purposes per spec §12.
+    # A callback job is never skipped: it rides in its *parent* batch (see
+    # Callbacks), and a cancelled parent must not swallow a child's callback.
     #
     # Death handling lives in Wurk::Batch::DeathHandler (registered as a
     # config death_handler) because death is signalled from the retry layer,
@@ -30,16 +39,15 @@ module Wurk
     class ServerMiddleware
       include Wurk::Middleware::ServerMiddleware
 
+      CALLBACK_JOB = 'Wurk::Batch::CallbackJob'
+
       def call(_worker, job, _queue, &)
         bid = job['bid']
         return yield unless bid
 
-        if invalidated?(bid)
-          ack_success(bid, job['jid'])
-          return
-        end
-
-        run_and_ack(bid, job['jid'], &)
+        jid = job['jid']
+        run(bid, jid, &) if job['class'] == CALLBACK_JOB || !invalidated?(bid)
+        ack_success(bid, jid)
       end
 
       private
@@ -50,9 +58,8 @@ module Wurk
       # failure: re-raise it untouched, acking neither. Any other exception
       # means the job failed and will retry (or eventually die): record it
       # before re-raising.
-      def run_and_ack(bid, jid)
+      def run(bid, jid)
         yield
-        ack_success(bid, jid)
       rescue Wurk::JobRetry::Handled, Wurk::Job::Interrupted
         raise
       rescue StandardError
@@ -64,19 +71,27 @@ module Wurk
         redis_pool.with { |conn| conn.call('HGET', "b-#{bid}", 'invalidated') } == '1'
       end
 
+      # A jid that was no longer live (`removed == 0`) is a re-run — most
+      # often a job reclaimed after a SIGKILL that landed between this ack and
+      # the fire. It still goes through the gate with the batch's current
+      # state: if the batch is drained, the dead run's fire may never have
+      # happened, and the callback markers absorb it if it did.
       def ack_success(bid, jid)
-        result = redis_pool.with do |conn|
-          Wurk::Lua::Loader.eval_cached(
-            conn,
-            :batch_ack_success,
-            keys: ["b-#{bid}", "b-#{bid}-jids", "b-#{bid}-failed"],
-            argv: [jid]
-          )
+        _removed, pending, live, kids = Callbacks.retrying do
+          redis_pool.with { |conn| Batch.ack_success(conn, bid, jid) }
         end
-        pending, live = Array(result).map(&:to_i)
-        return if pending.negative?
+        fire(bid, pending, live, kids)
+      end
 
-        Wurk::Batch::Callbacks.maybe_fire(bid, pending: pending, live: live)
+      def fire(bid, pending, live, kids)
+        Callbacks.retrying { Callbacks.maybe_fire(bid, pending: pending, live: live, kids: kids) }
+      rescue StandardError => e
+        error_config.handle_exception(e, { context: "batch #{bid}: firing callbacks after ack", bid: bid })
+      end
+
+      # `config` is a Configuration or a Capsule; only the former reports.
+      def error_config
+        config.respond_to?(:handle_exception) ? config : config.config
       end
 
       def ack_failed(bid, jid)
@@ -93,6 +108,4 @@ module Wurk
   end
 end
 
-require_relative 'callbacks'
-
 Wurk.configuration.server_middleware.add(Wurk::Batch::ServerMiddleware)

data/lib/wurk/batch/status.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require 'json'
+require_relative '../lua'
+require_relative 'callbacks'
 
 module Wurk
   class Batch
@@ -10,7 +12,7 @@ module Wurk
     #
     # `#data` returns the JSON-friendly hash served by the polling endpoint.
     # `#join` blocks the current thread until `complete?` — test/util only.
-    # `#delete` UNLINKs every key associated with the batch.
+    # `#delete` removes the batch and every reference to it.
     class Status
       JOIN_POLL_INTERVAL = 0.5
 
@@ -47,7 +49,7 @@ module Wurk
       def complete?
         return true if @data['complete'] == '1'
 
-        total.positive? && live_jids_count.zero?
+        complete_with?(live_jids_count)
       end
 
       def failed_jids
@@ -81,8 +83,10 @@ module Wurk
       end
 
       # JSON-serializable snapshot used by the polling middleware / web UI.
-      # Field names are wire-compat with Sidekiq Pro's BatchStatus.
+      # Field names are wire-compat with Sidekiq Pro's BatchStatus. The four
+      # set reads it needs ride one pipeline.
       def data
+        failed, dead, kids, live = set_snapshot
         {
           'bid' => @bid,
           'total' => total,
@@ -92,14 +96,14 @@ module Wurk
           'complete_at' => complete_at,
           'success_at' => success_at,
           'death_at' => death_at,
-          'complete' => complete?,
+          'complete' => complete_with?(live.to_i),
           'invalidated' => invalidated?,
           'description' => description,
           'parent_bid' => parent_bid,
           'tags' => tags,
-          'failed_jids' => failed_jids,
-          'dead_jids' => dead_jids,
-          'child_count' => child_count
+          'failed_jids' => failed,
+          'dead_jids' => dead,
+          'child_count' => kids.to_i
         }
       end
 
@@ -115,15 +119,22 @@ module Wurk
         end
       end
 
-      # Nukes every key for this batch. Dangerous if jobs are still in flight
-      # — they'll succeed/fail without a batch to ack against, callbacks won't
-      # fire, and counts get permanently inconsistent. Caller's problem.
+      # Nukes every key for this batch, its index entries and tag indexes,
+      # and its membership in the parent's `-kids`/`-pkids` — one atomic
+      # script (BATCH_DELETE). Dangerous if jobs are still in flight: they'll
+      # succeed/fail without a batch to ack against and this batch's callbacks
+      # won't fire. The *parent* is not left waiting on a child that no longer
+      # exists: when this was the last thing it waited on, its callbacks fire
+      # here.
       def delete
-        Wurk.redis do |conn|
-          conn.call('UNLINK', *Batch.keys_for(@bid))
-          conn.call('ZREM', 'batches', @bid)
-          conn.call('ZREM', 'dead-batches', @bid)
-        end
+        reload!
+        parent = parent_bid.to_s
+        own    = Batch.keys_for(@bid)
+        pending, live, kids = Wurk.redis do |conn|
+          Wurk::Lua::Loader.eval_cached(conn, :batch_delete, keys: delete_keys(own, parent),
+                                                             argv: [@bid, own.size, parent.empty? ? '0' : '1'])
+        end.map(&:to_i)
+        Callbacks.maybe_fire(parent, pending: pending, live: live, kids: kids) unless parent.empty?
         nil
       end
 
@@ -135,6 +146,29 @@ module Wurk
 
       private
 
+      # [failed jids, dead jids, child count, live count] in one round trip.
+      def set_snapshot
+        Wurk.redis do |conn|
+          conn.pipelined do |pipe|
+            pipe.call('SMEMBERS', "b-#{@bid}-failed")
+            pipe.call('SMEMBERS', "b-#{@bid}-died")
+            pipe.call('SCARD', "b-#{@bid}-kids")
+            pipe.call('SCARD', "b-#{@bid}-jids")
+          end
+        end
+      end
+
+      # KEYS layout BATCH_DELETE expects.
+      def delete_keys(own, parent)
+        keys = [*own, 'batches', 'dead-batches']
+        keys.push("b-#{parent}", "b-#{parent}-jids", "b-#{parent}-pkids", "b-#{parent}-kids") unless parent.empty?
+        keys.concat(tags.map { |t| "tags:#{t}" })
+      end
+
+      def complete_with?(live)
+        @data['complete'] == '1' || (total.positive? && live.zero?)
+      end
+
       def live_jids_count
         Wurk.redis { |conn| conn.call('SCARD', "b-#{@bid}-jids") }.to_i
       end