wurk 0.0.5 → 1.0.0

data/lib/wurk/redis_connection.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative 'redis_pool'
+
+module Wurk
+  # Sidekiq-compatible pool constructor. `Sidekiq::RedisConnection.create(...)`
+  # is the documented way to build a standalone Redis pool (spec §26) — it's the
+  # default pool for `Sidekiq::Deploy` (§23) and the constructor for per-shard
+  # pools in multi-shard web mounts (Pro §10.2). Returns a `Wurk::RedisPool`
+  # (connection_pool-backed, redis-client adapter), which is `.with`-compatible
+  # with everything that expects a Sidekiq pool.
+  #
+  # Accepts Sidekiq's option keys (`url`, `size`, `pool_timeout`, `name`),
+  # string- or symbol-keyed. Anything omitted falls back to RedisPool's defaults
+  # (URL = ENV["REDIS_URL"] or redis://localhost:6379/0).
+  module RedisConnection
+    # Housekeeping/standalone default; per-capsule pools size to concurrency.
+    DEFAULT_POOL_SIZE = 10
+
+    def self.create(options = {})
+      opts = options.transform_keys(&:to_sym)
+      RedisPool.new(
+        size: opts[:size] || DEFAULT_POOL_SIZE,
+        url: opts[:url] || RedisPool::DEFAULT_URL,
+        timeout: opts[:pool_timeout] || RedisPool::DEFAULT_TIMEOUT,
+        name: opts[:name] || RedisPool::DEFAULT_NAME
+      )
+    end
+  end
+end

data/lib/wurk/redis_pool.rb

@@ -2,6 +2,7 @@
 
 require 'redis-client'
 require 'connection_pool'
+require_relative 'redis_client_adapter'
 
 module Wurk
   # Per-process pool over redis-client + connection_pool. Never share a socket
@@ -56,8 +57,11 @@ module Wurk
 
     private
 
+    # Wrapped in the CompatClient decorator so `Sidekiq.redis { |c| c.smembers }`
+    # method-style commands work like Sidekiq 7+ (#204). Wurk's own code paths
+    # use #call, which the decorator forwards.
     def build_client
-      RedisClient.config(url: @url, timeout: @timeout).new_client
+      RedisClientAdapter::CompatClient.new(RedisClient.config(url: @url, timeout: @timeout).new_client)
     end
 
     def safe_close(conn)

data/lib/wurk/scheduled.rb

@@ -67,6 +67,48 @@ module Wurk
       end
     end
 
+    # Reliable variant of Enq (Pro §4). The default Enq pops then pushes —
+    # a crash between the ZPOPBYSCORE and the client push loses the job.
+    # ReliableEnq instead promotes every due job from each set onto its target
+    # queue in a single atomic Lua (ZRANGEBYSCORE → LPUSH queue:<q> → ZREM),
+    # so there is no window where a job exists in neither place. Swapped in by
+    # `config.reliable_scheduler!` via the `scheduled_enq` seam.
+    #
+    # Spec: docs/target/sidekiq-pro.md §4.
+    class ReliableEnq
+      include Component
+
+      def initialize(container)
+        @config = container
+        @done = false
+      end
+
+      def enqueue_jobs(sorted_sets = SETS)
+        @config.redis do |conn|
+          sorted_sets.each { |sset| promote(conn, sset) }
+        end
+      end
+
+      def terminate
+        @done = true
+      end
+
+      private
+
+      def promote(conn, sset)
+        Wurk::Lua::Loader.eval_cached(
+          conn,
+          :reliable_schedule_promote,
+          keys: [sset, Keys::QUEUES_SET],
+          argv: [real_time.to_s, Keys::QUEUE_PREFIX]
+        )
+      end
+
+      def real_time
+        ::Process.clock_gettime(::Process::CLOCK_REALTIME)
+      end
+    end
+
     # Single thread that wakes on a randomized interval, drains both ZSETs,
     # then sleeps again. Random spread prevents the cluster from dogpiling
     # Redis at the top of each cadence.

data/lib/wurk/sorted_entry.rb

@@ -42,35 +42,37 @@ module Wurk
     # ZINCRBY to shift the score; positive deltas reschedule into the future.
     # Sidekiq passes the absolute target time; we compute the delta here so
     # the call survives clock skew between caller and Redis.
-    def reschedule(at) # rubocop:disable Naming/MethodParameterName
+    def reschedule(at)
       Wurk.redis { |conn| conn.call('ZINCRBY', @parent.name, at.to_f - @score, value) }
     end
 
-    # Removes this entry, decrements `retry_count` by one (so the worker treats
-    # the next attempt as a re-do, not a fresh retry), and re-enqueues via the
-    # client. Wire-compat with Sidekiq's "Retry now" UI action.
+    # Removes this entry and re-enqueues it via the client with the payload
+    # untouched. Backs the scheduled/dead "add to queue" actions — Sidekiq's
+    # add_to_queue does not touch `retry_count`.
     def add_to_queue
       remove_job do |message|
-        message['retry_count'] = message['retry_count'].to_i - 1 if message['retry_count']
         Client.new.push(message)
       end
     end
 
-    # Same flow as add_to_queue but keeps `retry_count` intact. Used for the
-    # "retry" action from the retry set (count was already incremented when
-    # the job entered retry; don't double-bump).
+    # Same flow but decrements `retry_count` first: the count was already
+    # incremented when the job entered the retry set, and the next failure
+    # bumps it again — without the decrement a manual "Retry now" would
+    # consume an attempt. Wire-compat with Sidekiq's SortedEntry#retry.
     def retry
       remove_job do |message|
+        message['retry_count'] = message['retry_count'].to_i - 1 if message['retry_count']
         Client.new.push(message)
       end
     end
 
     # Removes this entry from its parent set and writes it to the dead set.
-    # `notify_failure: false` because the kill is user-initiated (UI action),
-    # not a retry-exhausted event — death_handlers don't fire.
+    # Death handlers fire with the synthesized "Job killed by API" exception
+    # (Sidekiq's default) so error trackers and the batch death path observe
+    # API/UI kills.
     def kill
       remove_job do |message|
-        DeadSet.new.kill(Wurk.dump_json(message), notify_failure: false)
+        DeadSet.new.kill(Wurk.dump_json(message))
       end
     end
 

data/lib/wurk/stats.rb

@@ -49,7 +49,11 @@ module Wurk
       end
     end
 
-    # @return [Hash{String=>Integer}] queue name → LLEN.
+    # @return [Hash{String=>Integer}] queue name → LLEN, largest queue first.
+    # `SMEMBERS` order is arbitrary; Sidekiq sorts the name/size pairs by size
+    # descending before building the hash (Stats#queues, spec §19.1) — gems
+    # and dashboards reading this rely on that order. Match it exactly with
+    # `sort_by { |_, size| -size }`.
     def queues
       Wurk.redis do |conn|
         names = conn.call('SMEMBERS', Keys::QUEUES_SET)
@@ -58,11 +62,14 @@ module Wurk
         sizes = conn.pipelined do |pipe|
           names.each { |q| pipe.call('LLEN', Keys.queue(q)) }
         end
-        names.zip(sizes.map(&:to_i)).to_h
+        names.zip(sizes.map(&:to_i)).sort_by { |_, size| -size }.to_h
       end
     end
 
-    # @return [Array<QueueSummary>] one per known queue.
+    # @return [Array<QueueSummary>] one per known queue, largest first.
+    # Same size-descending order as Sidekiq's detailed queue list
+    # (`queue_summaries.sort_by { |qd| -qd.size }`) — this feeds the
+    # dashboard's queue table (api_controller#queues).
     def queue_summaries
       Wurk.redis do |conn|
         names = conn.call('SMEMBERS', Keys::QUEUES_SET)
@@ -75,7 +82,7 @@ module Wurk
             pipe.call('LRANGE', Keys.queue(q), -1, -1)
           end
         end
-        build_summaries(names, results, paused_set)
+        build_summaries(names, results, paused_set).sort_by { |qd| -qd.size }
       end
     end
 

data/lib/wurk/swarm/child_boot.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative '../component'
 require_relative '../launcher'
 require_relative '../fetcher/reliable'
 
@@ -16,6 +17,8 @@ module Wurk
     # Kept separate from Wurk::Swarm so the parent supervisor stays
     # focused on PID supervision (SRP).
     class ChildBoot
+      include Component
+
       CHILD_SIGNALS = { 'TERM' => :term, 'INT' => :term, 'TSTP' => :tstp, 'USR2' => :usr2 }.freeze
 
       def initialize(config, slot, index)
@@ -29,11 +32,20 @@ module Wurk
         reset_inherited_signals
         reconnect_after_fork
         Wurk.server = true
+        # :fork runs in each child after our internal AR/Redis reconnect and
+        # before fetching, so apps can reopen sockets / restart threads /
+        # reconnect non-fork-safe libs (Ent §7.4). It never fires in the parent
+        # (which only forks + supervises). Like :startup, the child's forked
+        # copy of the bucket is cleared after dispatch, so siblings fire theirs.
+        fire_event(:fork)
         apply_slot_to_config
-        launcher = Wurk::Launcher.new(@config)
-        install_signal_handlers(launcher)
-        launcher.run
-        wait_loop(launcher)
+        # :startup must fire in each worker child before its managers spin up
+        # (Sidekiq contract, reraise: true). The parent supervisor never runs
+        # jobs, so the non-swarm CLI path fires it once per process — for the
+        # swarm, each child fires it here. Its own forked copy of the bucket is
+        # cleared after, so siblings still fire their own.
+        fire_event(:startup, reraise: true)
+        run_launcher
         exit 0
       rescue StandardError, ::Wurk::Shutdown => e
         @config.logger.error { "swarm child ##{@index} (#{::Process.pid}) crashed: #{e.class}: #{e.message}" }
@@ -42,6 +54,16 @@ module Wurk
 
       private
 
+      # Boot the launcher and block until shutdown. wait_loop joins the
+      # signal-dispatch thread, so the child can't fall through to `exit 0`
+      # mid-drain.
+      def run_launcher
+        launcher = Wurk::Launcher.new(@config)
+        install_signal_handlers(launcher)
+        launcher.run
+        wait_loop(launcher)
+      end
+
       # Parent installed traps for TERM/INT/TSTP/CONT/USR1 — the child
       # needs its own behavior, not the parent's.
       def reset_inherited_signals

data/lib/wurk/swarm.rb

@@ -36,7 +36,7 @@ module Wurk
 
     attr_reader :topology, :children
 
-    def initialize(topology:, config: Wurk.configuration, memory_limit: nil,
+    def initialize(topology:, config: Wurk.configuration, memory_limit: config.memory_limit_kb,
                    shutdown_timeout: DEFAULT_SHUTDOWN_TIMEOUT)
       @topology = topology
       @config = config

data/lib/wurk/transaction_aware_client.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+require_relative 'client'
+require_relative 'job_util'
+
+module Wurk
+  # Opt-in enqueue client that holds each `push` until the surrounding
+  # ActiveRecord transaction commits, so a job never references a row a
+  # rollback erased — the canonical "enqueue after commit" pattern.
+  #
+  # Enable globally with `Wurk.transactional_push!` (aliased
+  # `Sidekiq.transactional_push!`), which sets `default_job_options
+  # ["client_class"]`; the worker DSL then builds this client instead of the
+  # plain `Wurk::Client`.
+  #
+  # The jid is pre-allocated and returned synchronously so the caller can
+  # reference it inside the transaction; the actual Redis write runs in an
+  # after-commit hook. `push_bulk` is intentionally NOT deferred — it matches
+  # Sidekiq, whose batching/scheduling machinery can't ride the commit hook.
+  #
+  # Spec: docs/target/sidekiq-free.md §8.
+  class TransactionAwareClient
+    def initialize(pool: nil, config: nil)
+      @redis_client = Wurk::Client.new(pool: pool, config: config)
+    end
+
+    # True inside a `Batch#jobs` block. The batch counts jobs at push time, so
+    # deferring would desync its totals — push immediately instead.
+    def batching?
+      !Thread.current[Wurk::Batch::THREAD_KEY].nil?
+    end
+
+    # @return [String] the pre-allocated jid (returned before the deferred push runs).
+    def push(item)
+      item['jid'] ||= SecureRandom.hex(12)
+
+      if batching?
+        @redis_client.push(item)
+      else
+        register_after_commit { @redis_client.push(item) }
+      end
+
+      item['jid']
+    end
+
+    # Bulk enqueue is never transactional (Sidekiq parity) — straight to Redis.
+    def push_bulk(items)
+      @redis_client.push_bulk(items)
+    end
+
+    private
+
+    # Routes the push to the host's after-commit hook. AR 7.2+ exposes
+    # `ActiveRecord.after_all_transactions_commit`, which runs the block now
+    # when there's no open transaction and after commit when there is — so the
+    # "not in a transaction" and "ActiveRecord absent" cases both degrade to an
+    # immediate push, exactly the graceful no-op the spec calls for.
+    def register_after_commit(&)
+      if defined?(::ActiveRecord) && ::ActiveRecord.respond_to?(:after_all_transactions_commit)
+        ::ActiveRecord.after_all_transactions_commit(&)
+      elsif defined?(::AfterCommitEverywhere)
+        ::AfterCommitEverywhere.after_commit(&)
+      else
+        yield
+      end
+    end
+  end
+end

data/lib/wurk/unique.rb

@@ -3,6 +3,7 @@
 require 'json'
 require 'digest'
 require_relative 'middleware'
+require_relative 'lua'
 
 module Wurk
   # Sidekiq Enterprise unique jobs. Best-effort dedup at enqueue time keyed
@@ -16,6 +17,10 @@ module Wurk
   #     *before* invoking perform; a duplicate can be enqueued while the
   #     first is running.
   #
+  # A job that dies *automatically* (retries exhausted / discarded) releases
+  # its lock via a death handler; manual UI kills keep the lock until TTL
+  # expiry (Ent wiki, Ent-Unique-Jobs).
+  #
   # Wire-compat (§3.9): single-key Redis layout — `unique:<sha256>` STRING
   # holding the owning JID. Scheduled jobs extend the TTL by the delay so
   # the lock covers the entire wait+execution window (§3.4).
@@ -26,6 +31,31 @@ module Wurk
     DEFAULT_UNTIL = :success
     VALID_UNTIL = %i[success start].freeze
 
+    # Ent parity: a job that dies automatically releases its lock so a
+    # duplicate can enqueue immediately. Manual API/UI kills keep the lock
+    # until TTL expiry (Ent wiki, Ent-Unique-Jobs) — they reach death
+    # handlers too (Sidekiq fires them on API kills), so we recognize the
+    # synthesized kill exception and skip the release for it. Atomic
+    # CAS-DEL via the shared Lua script mirrors ServerMiddleware#release.
+    DEATH_HANDLER = lambda do |job, exception|
+      next unless Wurk::Unique.enabled?
+      next unless Wurk::Unique.coerce_ttl(job['unique_for'])
+      next if exception.instance_of?(::RuntimeError) && exception.message == DeadSet::API_KILL_MESSAGE
+
+      Wurk.redis { |conn| Wurk::Unique.release_if_owner(conn, Wurk::Unique.lock_key_for(job), job['jid']) }
+    end
+
+    # Atomic compare-and-delete of a unique lock key. Two-command
+    # GET-then-DEL is not a real CAS — the key can expire between the GET
+    # and DEL, letting a fresh enqueue grab it, and the bare DEL would
+    # then drop the new owner's lock. Routed through a single Lua script
+    # (`Wurk::Lua::RELEASE_IF_OWNER`) shared by `ServerMiddleware#release`
+    # (normal success/start release) and `DEATH_HANDLER` (automatic-death
+    # release) so the two paths cannot drift.
+    def self.release_if_owner(conn, key, jid)
+      Wurk::Lua::Loader.eval_cached(conn, :release_if_owner, keys: [key], argv: [jid])
+    end
+
     # `Sidekiq::Enterprise.unique!` flips this on. The middleware pair is
     # always loaded (so worker `sidekiq_options unique_for:` is a no-op
     # without `unique!`) — only when the flag is set does the client
@@ -91,6 +121,8 @@ module Wurk
           unless Wurk.configuration.client_middleware.exists?(ClientMiddleware)
         Wurk.configuration.server_middleware.add(ServerMiddleware) \
           unless Wurk.configuration.server_middleware.exists?(ServerMiddleware)
+        handlers = Wurk.configuration.death_handlers
+        handlers << DEATH_HANDLER unless handlers.include?(DEATH_HANDLER)
       end
     end
 
@@ -172,7 +204,17 @@ module Wurk
         pool.with do |conn|
           return yield if conn.call('SET', key, job['jid'], 'NX', 'EX', ttl) == 'OK'
 
-          log_duplicate(job, conn.call('GET', key))
+          holder = conn.call('GET', key)
+          # The job's own jid holding the lock is a re-push, not a duplicate:
+          # scheduled/retry promotion re-runs this chain while the
+          # enqueue-time lock is still live (§3.4/§3.7) — dropping here would
+          # silently lose the job.
+          return yield if holder == job['jid']
+          # nil holder: the lock expired between the failed SET NX and the
+          # GET — re-acquire instead of dropping.
+          return yield if holder.nil? && conn.call('SET', key, job['jid'], 'NX', 'EX', ttl) == 'OK'
+
+          log_duplicate(job, holder)
         end
         nil
       end
@@ -225,13 +267,13 @@ module Wurk
         VALID_UNTIL.include?(sym) ? sym : DEFAULT_UNTIL
       end
 
-      # CAS DEL: only drop the key if the owning JID still matches ours.
-      # Prevents a long-overdue retry from releasing a fresh lock held by
-      # a re-enqueued duplicate after the original TTL expired.
+      # Atomic CAS-DEL: only drop the key if the owning JID still matches
+      # ours. Prevents a long-overdue retry from releasing a fresh lock
+      # held by a re-enqueued duplicate after the original TTL expired.
+      # Shares the Lua script with `DEATH_HANDLER` so both release paths
+      # have identical semantics.
       def release(key, jid)
-        redis_pool.with do |conn|
-          conn.call('DEL', key) if conn.call('GET', key) == jid
-        end
+        redis_pool.with { |conn| Wurk::Unique.release_if_owner(conn, key, jid) }
       rescue StandardError => e
         Wurk.logger&.warn { "Wurk::Unique release failed: #{e.class}: #{e.message}" }
       end

data/lib/wurk/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Wurk
-  VERSION = "0.0.5"
+  VERSION = "1.0.0"
 end

data/lib/wurk/web/batch_status.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative '../batch/status'
+
+module Wurk
+  class Web
+    # Lightweight Rack middleware for batch-progress polling without mounting
+    # the full dashboard. Drop into a rack stack and app JS can drive progress
+    # bars off plain JSON:
+    #
+    #   # config.ru
+    #   use Sidekiq::Pro::BatchStatus
+    #   run Rails.application
+    #
+    # `GET /batch_status/<bid>.json` → `Wurk::Batch::Status#data` JSON
+    # (bid, total, pending, failures, complete, created_at, description, …).
+    # Unknown/expired bid → 404. Every other request passes straight through.
+    #
+    # Spec: docs/target/sidekiq-pro.md §10.3.
+    class BatchStatus
+      ROUTE = %r{\A/batch_status/(?<bid>[^/]+)\.json\z}
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        match = env['REQUEST_METHOD'] == 'GET' && ROUTE.match(env['PATH_INFO'])
+        return @app.call(env) unless match
+
+        status = Wurk::Batch::Status.new(match[:bid])
+        status.exists? ? json(200, status.data) : json(404, { 'error' => 'not_found' })
+      end
+
+      private
+
+      def json(code, payload)
+        [code, { 'content-type' => 'application/json' }, [Wurk.dump_json(payload)]]
+      end
+    end
+  end
+end