wurk 0.0.1

data/lib/wurk/batch/callback_job.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative '../worker'
+
+module Wurk
+  class Batch
+    # Worker that runs a single batch callback. Target may be a Class name
+    # ("MyCallback" → `MyCallback.new.on_<event>(status, options)`) or a
+    # "Klass#method" spec ("Foo#bar" → `Foo.new.bar(status, options)`).
+    # Failures retry like any ordinary job — callbacks MUST be idempotent.
+    #
+    # Spec: docs/target/sidekiq-pro.md §2.4.
+    class CallbackJob
+      include Wurk::Job
+
+      sidekiq_options retry: true
+
+      def perform(bid, target_spec, event, options)
+        klass, method = resolve(target_spec, event)
+        instance = klass.new
+        status   = Wurk::Batch::Status.new(bid)
+        instance.public_send(method, status, options || {})
+      end
+
+      private
+
+      def resolve(spec, event)
+        if spec.include?('#')
+          klass_name, method_name = spec.split('#', 2)
+          [Object.const_get(klass_name), method_name.to_sym]
+        else
+          [Object.const_get(spec), :"on_#{event}"]
+        end
+      end
+    end
+  end
+end

data/lib/wurk/batch/callbacks.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Wurk
+  class Batch
+    # Fires batch callbacks (`:success`, `:complete`, `:death`) by enqueuing
+    # them as ordinary jobs on the batch's `callback_queue`. Dedup is via
+    # b-<bid>-notify so the same callback can't be enqueued twice even
+    # if multiple workers race to ack the final job.
+    #
+    # Callback wrapper job: Wurk::Batch::CallbackJob — given a target spec
+    # ("Klass" or "Klass#method") and options hash, it instantiates and
+    # invokes on_<event> (or the named method) with the Status snapshot.
+    module Callbacks
+      module_function
+
+      # Called from the server middleware after BATCH_ACK_SUCCESS. Fires
+      # `:complete` when live jids hit 0; fires `:success` when pending
+      # also hits 0 and there have been no deaths.
+      def maybe_fire(bid, pending:, live:)
+        return unless live.zero?
+
+        fire_complete(bid)
+        fire_success(bid) if pending.zero? && !death_fired?(bid)
+        propagate_to_parent(bid)
+      end
+
+      # Fired from Wurk::Batch::DeathHandler on the FIRST permanent death
+      # in the batch only. Subsequent deaths bump the counter but do not
+      # re-enqueue the callback.
+      def fire_death(bid)
+        return unless dedup_set(bid, 'death')
+
+        record_event(bid, 'death_at')
+        Wurk.redis { |conn| conn.call('ZADD', 'dead-batches', Time.now.to_f.to_s, bid) }
+        enqueue_callbacks(bid, 'death')
+        cascade_death(bid)
+      end
+
+      # A child's death means the parent — and every ancestor — can never
+      # fully succeed, so `:death` propagates up the parent chain. The
+      # recursion bottoms out at the root (empty parent_bid); fire_death's own
+      # dedup_set guard makes each ancestor's `:death` fire exactly once even
+      # under racing children.
+      def cascade_death(bid)
+        parent_bid = parent_bid_for(bid)
+        return if parent_bid.nil? || parent_bid.empty?
+
+        fire_death(parent_bid)
+      end
+
+      def fire_complete(bid)
+        return unless dedup_set(bid, 'complete')
+
+        record_event(bid, 'complete_at')
+        enqueue_callbacks(bid, 'complete')
+      end
+
+      def fire_success(bid)
+        return unless dedup_set(bid, 'success')
+
+        record_event(bid, 'success_at')
+        enqueue_callbacks(bid, 'success')
+        apply_linger(bid)
+      end
+
+      # Post-success retention: a succeeded batch no longer coordinates any
+      # jobs, so its keys expire after the per-batch `linger` override (else
+      # 24h) instead of the 30d pending TTL. Mirrors Sidekiq Pro §2.8.
+      def apply_linger(bid)
+        raw     = Wurk.redis { |conn| conn.call('HGET', "b-#{bid}", 'linger') }
+        seconds = raw.nil? || raw.to_s.empty? ? Batch::POST_SUCCESS_EXPIRY_SECONDS : raw.to_i
+        Wurk.redis do |conn|
+          Batch.keys_for(bid).each { |key| conn.call('EXPIRE', key, seconds) }
+        end
+      end
+
+      # Atomically marks `bid` as having fired `event`. Returns true the
+      # first time, false thereafter — caller skips the enqueue when false.
+      # SET NX makes this safe under racing acks.
+      def dedup_set(bid, event)
+        Wurk.redis do |conn|
+          ok = conn.call('SET', "b-#{bid}-#{event}", '1', 'NX', 'EX', Batch::CALLBACK_NOTIFY_TTL)
+          ok == 'OK'
+        end
+      end
+
+      def record_event(bid, field)
+        now = ::Process.clock_gettime(::Process::CLOCK_REALTIME)
+        Wurk.redis do |conn|
+          conn.call('HSET', "b-#{bid}", field, now.to_s)
+          conn.call('HSET', "b-#{bid}", field.to_s.sub('_at', ''), '1')
+        end
+      end
+
+      # True once `:death` has fired for this batch — from one of its own
+      # jobs dying or from a descendant's death cascading up. Suppresses
+      # `:success`, which must never fire after any death in the subtree.
+      #
+      # Reads the durable `death` field on `b-<bid>` (written by `record_event`),
+      # not the `b-<bid>-death` dedup key — the dedup key has its own 30d TTL
+      # and can expire while an ancestor batch is still open, after which a
+      # late `maybe_fire` would wrongly emit `:success`.
+      def death_fired?(bid)
+        Wurk.redis { |conn| conn.call('HGET', "b-#{bid}", 'death') } == '1'
+      end
+
+      # Per-callback rescue: one bad spec or a transient enqueue failure must
+      # not strand the batch with the remaining callbacks for this event
+      # un-enqueued. Log and move on so every other callback still fires.
+      def enqueue_callbacks(bid, event)
+        callbacks, queue = callback_specs_for(bid)
+
+        callbacks.each do |(cb_event, target, options)|
+          next unless cb_event == event
+
+          enqueue_callback_job(bid, target, event, options, queue)
+        rescue StandardError => e
+          Wurk.logger.warn("batch #{bid}: #{event} callback #{target.inspect} enqueue failed: #{e.class}: #{e.message}")
+        end
+      end
+
+      def callback_specs_for(bid)
+        raw = Wurk.redis { |conn| conn.call('HMGET', "b-#{bid}", 'callbacks', 'callback_queue') }
+        callbacks_json, queue = raw
+        queue = 'default' if queue.nil? || queue.empty?
+        parsed = parse_callbacks(callbacks_json)
+        [parsed, queue]
+      end
+
+      def parse_callbacks(raw)
+        return [] if raw.nil? || raw.empty?
+
+        JSON.parse(raw)
+      rescue JSON::ParserError
+        []
+      end
+
+      def enqueue_callback_job(bid, target, event, options, queue)
+        Wurk::Client.push(
+          'class' => 'Wurk::Batch::CallbackJob',
+          'args' => [bid, target, event, options],
+          'queue' => queue,
+          'retry' => true
+        )
+      end
+
+      # When a child batch's `:success` fires, decrement the parent's pkids
+      # set so the parent's own `:success` waits on the full subtree. When
+      # parent's pkids hits 0 *and* its own pending is 0, parent's success
+      # fires too.
+      def propagate_to_parent(bid)
+        parent_bid = parent_bid_for(bid)
+        return if parent_bid.nil? || parent_bid.empty?
+        return unless pkids_drained?(parent_bid, bid)
+
+        maybe_fire(parent_bid, pending: pending_for(parent_bid), live: live_for(parent_bid))
+      end
+
+      def parent_bid_for(bid)
+        Wurk.redis { |conn| conn.call('HGET', "b-#{bid}", 'parent_bid') }
+      end
+
+      def pkids_drained?(parent_bid, child_bid)
+        Wurk.redis do |conn|
+          conn.call('SREM', "b-#{parent_bid}-pkids", child_bid)
+          conn.call('SCARD', "b-#{parent_bid}-pkids").to_i.zero?
+        end
+      end
+
+      def pending_for(bid) = Wurk.redis { |conn| conn.call('HGET', "b-#{bid}", 'pending') }.to_i
+      def live_for(bid)    = Wurk.redis { |conn| conn.call('SCARD', "b-#{bid}-jids") }.to_i
+    end
+  end
+end

data/lib/wurk/batch/client_middleware.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require_relative '../middleware'
+
+module Wurk
+  class Batch
+    # Client middleware. When a `Job.perform_async` runs inside a
+    # `batch.jobs { ... }` block, Thread.current[Batch::THREAD_KEY] holds
+    # the active batch — we stamp `bid` onto the payload so the worker
+    # can re-open the batch and Client#raw_push can route through
+    # BATCH_PUSH for atomic increment+LPUSH.
+    #
+    # Auto-registered at the head of the client chain when this file is
+    # required.
+    class ClientMiddleware
+      include Wurk::Middleware::ClientMiddleware
+
+      def call(_worker, job, _queue, _redis_pool)
+        batch = Thread.current[Batch::THREAD_KEY]
+        job['bid'] = batch.bid if batch
+        yield
+      end
+    end
+  end
+end
+
+Wurk.configuration.client_middleware.prepend(Wurk::Batch::ClientMiddleware)

data/lib/wurk/batch/death_handler.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative '../lua'
+require_relative 'callbacks'
+
+module Wurk
+  class Batch
+    # Registered as a config death_handler. Fires for every job that
+    # exhausts retries or carries `dead: false` and discards. If the job
+    # carries a `bid`, we BATCH_ACK_COMPLETE → record the death → fire
+    # `:death` callback exactly once per batch (first death only).
+    #
+    # Spec: docs/target/sidekiq-pro.md §2.4 (`:death`).
+    class DeathHandler
+      def self.call(job, _exception)
+        bid = job['bid']
+        return unless bid
+
+        result = Wurk.redis do |conn|
+          Wurk::Lua::Loader.eval_cached(
+            conn,
+            :batch_ack_complete,
+            keys: ["b-#{bid}", "b-#{bid}-jids", "b-#{bid}-died", "b-#{bid}-failed"],
+            argv: [job['jid']]
+          )
+        end
+        live, _died, first_death = Array(result).map(&:to_i)
+
+        Wurk::Batch::Callbacks.fire_death(bid) if first_death == 1
+        return unless live.zero?
+
+        Wurk::Batch::Callbacks.fire_complete(bid)
+        Wurk::Batch::Callbacks.propagate_to_parent(bid)
+      end
+    end
+  end
+end
+
+Wurk.configuration.death_handlers << Wurk::Batch::DeathHandler

data/lib/wurk/batch/empty.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require_relative '../worker'
+
+module Wurk
+  class Batch
+    # No-op marker job inserted into an empty `batch.jobs { }` block so
+    # `:complete` and `:success` callbacks still fire. Pro 7.1+ behaviour:
+    # without this marker, total=0 means no batch_push ever ran and the
+    # callback path can't tell "empty batch" from "never flushed batch".
+    #
+    # Spec: docs/target/sidekiq-pro.md §2.3 / §12.
+    class Empty
+      include Wurk::Job
+
+      sidekiq_options retry: false, queue: 'default'
+
+      def perform; end
+    end
+  end
+end

data/lib/wurk/batch/server_middleware.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'json'
+require_relative '../middleware'
+require_relative '../lua'
+
+module Wurk
+  class Batch
+    # Server middleware. Runs around `perform` for any job carrying a `bid`.
+    # On success → BATCH_ACK_SUCCESS → if pending hit zero and no deaths,
+    # enqueue `:success` callback jobs; if live jids hit zero, enqueue
+    # `:complete` callback jobs.
+    #
+    # Invalidated batches short-circuit: the job is skipped without
+    # raising — counts as a "success" for batch purposes per spec §12.
+    #
+    # Death handling lives in Wurk::Batch::DeathHandler (registered as a
+    # config death_handler) because death is signalled from the retry layer,
+    # not from this middleware's rescue path.
+    class ServerMiddleware
+      include Wurk::Middleware::ServerMiddleware
+
+      def call(_worker, job, _queue)
+        bid = job['bid']
+        return yield unless bid
+
+        if invalidated?(bid)
+          ack_success(bid, job['jid'])
+          return
+        end
+
+        yield
+        ack_success(bid, job['jid'])
+      end
+
+      private
+
+      def invalidated?(bid)
+        redis_pool.with { |conn| conn.call('HGET', "b-#{bid}", 'invalidated') } == '1'
+      end
+
+      def ack_success(bid, jid)
+        result = redis_pool.with do |conn|
+          Wurk::Lua::Loader.eval_cached(
+            conn,
+            :batch_ack_success,
+            keys: ["b-#{bid}", "b-#{bid}-jids"],
+            argv: [jid]
+          )
+        end
+        pending, live = Array(result).map(&:to_i)
+        return if pending.negative?
+
+        Wurk::Batch::Callbacks.maybe_fire(bid, pending: pending, live: live)
+      end
+    end
+  end
+end
+
+require_relative 'callbacks'
+
+Wurk.configuration.server_middleware.add(Wurk::Batch::ServerMiddleware)

data/lib/wurk/batch/status.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Wurk
+  class Batch
+    # Snapshot of a batch's state — read-only view backed by HGETALL of
+    # `b-<bid>` plus the supporting JIDs/failed/died sets. Mirrors the
+    # Sidekiq::Batch::Status surface from docs/target/sidekiq-pro.md §2.5.
+    #
+    # `#data` returns the JSON-friendly hash served by the polling endpoint.
+    # `#join` blocks the current thread until `complete?` — test/util only.
+    # `#delete` UNLINKs every key associated with the batch.
+    class Status
+      JOIN_POLL_INTERVAL = 0.5
+
+      attr_reader :bid
+
+      def initialize(bid)
+        raise ArgumentError, 'bid required' if bid.nil? || bid.to_s.empty?
+
+        @bid = bid.to_s
+        reload!
+      end
+
+      # False when no `b-<bid>` hash exists — a well-formed bid that was never
+      # created (or has expired). Lets callers 404 instead of serving an
+      # all-zero phantom batch.
+      def exists? = !@data.empty?
+
+      def total            = @data['total'].to_i
+      def pending          = @data['pending'].to_i
+      def failures         = @data['failures'].to_i
+      def created_at       = numeric_or_nil(@data['created_at'])
+      def complete_at      = numeric_or_nil(@data['complete_at'])
+      def success_at       = numeric_or_nil(@data['success_at'])
+      def death_at         = numeric_or_nil(@data['death_at'])
+      def description      = @data['description']
+      def parent_bid       = @data['parent_bid']
+      def callback_queue   = @data['callback_queue']
+      def invalidated?     = @data['invalidated'] == '1'
+
+      # `:complete` fires when the live jids set is empty (every job has
+      # either succeeded or died). Hash field is set by the server callback;
+      # falling back to a recompute keeps Status accurate even if the
+      # callback dispatch is in flight.
+      def complete?
+        return true if @data['complete'] == '1'
+
+        total.positive? && live_jids_count.zero?
+      end
+
+      def failed_jids
+        Wurk.redis { |conn| conn.call('SMEMBERS', "b-#{@bid}-failed") }
+      end
+
+      def dead_jids
+        Wurk.redis { |conn| conn.call('SMEMBERS', "b-#{@bid}-died") }
+      end
+
+      def child_count
+        Wurk.redis { |conn| conn.call('SCARD', "b-#{@bid}-kids") }.to_i
+      end
+
+      def tags
+        raw = @data['tags']
+        return [] if raw.nil? || raw.empty?
+
+        JSON.parse(raw)
+      rescue JSON::ParserError
+        []
+      end
+
+      # JSON-serializable snapshot used by the polling middleware / web UI.
+      # Field names are wire-compat with Sidekiq Pro's BatchStatus.
+      def data
+        {
+          'bid' => @bid,
+          'total' => total,
+          'pending' => pending,
+          'failures' => failures,
+          'created_at' => created_at,
+          'complete_at' => complete_at,
+          'success_at' => success_at,
+          'death_at' => death_at,
+          'complete' => complete?,
+          'invalidated' => invalidated?,
+          'description' => description,
+          'parent_bid' => parent_bid,
+          'tags' => tags,
+          'failed_jids' => failed_jids,
+          'dead_jids' => dead_jids,
+          'child_count' => child_count
+        }
+      end
+
+      # Blocks the current thread until `complete?` is true. Test/util only —
+      # polling Redis from a worker thread would defeat the whole point of
+      # asynchronous batches.
+      def join
+        loop do
+          reload!
+          break if complete?
+
+          sleep JOIN_POLL_INTERVAL
+        end
+      end
+
+      # Nukes every key for this batch. Dangerous if jobs are still in flight
+      # — they'll succeed/fail without a batch to ack against, callbacks won't
+      # fire, and counts get permanently inconsistent. Caller's problem.
+      def delete
+        Wurk.redis do |conn|
+          conn.call('UNLINK', *Batch.keys_for(@bid))
+          conn.call('ZREM', 'batches', @bid)
+          conn.call('ZREM', 'dead-batches', @bid)
+        end
+        nil
+      end
+
+      def reload!
+        raw = Wurk.redis { |conn| conn.call('HGETALL', "b-#{@bid}") }
+        @data = raw.is_a?(Hash) ? raw : raw.each_slice(2).to_h
+        self
+      end
+
+      private
+
+      def live_jids_count
+        Wurk.redis { |conn| conn.call('SCARD', "b-#{@bid}-jids") }.to_i
+      end
+
+      def numeric_or_nil(val)
+        return nil if val.nil? || val.to_s.empty?
+
+        val.to_f
+      end
+    end
+  end
+end