wurk 1.4.0 → 1.6.0

data/lib/wurk/cli.rb

@@ -24,6 +24,27 @@ module Wurk
     # format and Lua scripts rely on commands introduced in Redis 7.
     MIN_REDIS_VERSION = '7.0.0'
 
+    # Subcommands. Bare `wurk` runs the worker — the historical shape, and the
+    # only one the swarm binaries accept — so a subcommand is never anything
+    # but the first argument, and every argument after it is that command's.
+    COMMANDS = %w[api].freeze
+
+    # `wurk api` defaults. 0.0.0.0 because the point of the machine API is for
+    # another service to reach it; it is bearer-gated, and answers 404 to
+    # everything until a token exists. `--bind 127.0.0.1` narrows it.
+    DEFAULT_API_BIND = '0.0.0.0'
+    DEFAULT_API_PORT = 7433
+
+    # Serving the API with nothing registered would bind a port and 404 every
+    # request — the surface is off until a token exists, so say so at boot
+    # rather than leaving an operator to diagnose it over HTTP.
+    NO_API_TOKEN_MESSAGE = <<~MSG
+      No API token is registered, so every request would answer 404.
+      Register one where the app loaded by -r configures wurk:
+
+          Wurk.configuration.api_token(ENV.fetch('WURK_API_TOKEN'), scopes: %i[enqueue read])
+    MSG
+
     # Thread-backtrace dumper used by both TTIN and INFO. Same body — INFO is
     # the modern name, TTIN is kept for parity with older Sidekiq users.
     BACKTRACE_DUMPER = lambda do |cli|
@@ -68,8 +89,20 @@ module Wurk
       ['-C', '--config PATH',          :config_file, 'path to YAML config file']
     ].freeze
 
+    # Same table shape, offered only when `api` is the subcommand: none of it
+    # means anything to a worker process, and `wurk --help` stays the worker's.
+    API_OPTION_FLAGS = [
+      ['-b', '--bind ADDRESS', :api_bind,   "Address to bind (default #{DEFAULT_API_BIND})"],
+      ['-p', '--port PORT',    :api_port,   "Port to listen on (default #{DEFAULT_API_PORT})", :to_i],
+      ['-s', '--server NAME',  :api_server, 'Rack handler to serve with (default: the first installed)']
+    ].freeze
+
     attr_accessor :launcher, :environment, :config
 
+    # The subcommand `parse` pulled off argv, or nil for the worker runner.
+    # The binary picks the mode from it — `exe/wurk` runs the API for `api`.
+    attr_reader :command
+
     def self.instance
       @instance ||= new
     end
@@ -84,12 +117,14 @@ module Wurk
       @launcher = nil
       @environment = nil
       @parser = nil
+      @command = nil
     end
 
     # `parse` is split from `run` so tests can drive option parsing without
     # touching Redis or booting the host app.
     def parse(args = ARGV.dup)
       @config ||= Wurk.default_configuration
+      @command = extract_command!(args)
       setup_options(args)
       initialize_logger
       validate!
@@ -137,6 +172,8 @@ module Wurk
     # `SIDEKIQ_PRELOAD_APP` whole-app eager-load) and boots `Process.warmup`
     # before the fork so children share warmed pages (copy-on-write). Spec §7.
     def run_swarm(boot_app: true, warmup: true)
+      raise ArgumentError, "the swarm runner takes no subcommand; run `wurk #{@command}` instead" if @command
+
       # Server mode before the app loads — see #run. The flag rides through the
       # fork into every child (the config object is copied), so configure_server
       # blocks registered in the parent take effect in the workers.
@@ -163,6 +200,32 @@ module Wurk
       end
     end
 
+    # `wurk api` — serve the machine-facing HTTP API and nothing else: no
+    # fetcher, no heartbeat, no job ever runs here. This is mount mode 3, the
+    # standalone one: the same Rack app the engine nests and a host can mount
+    # itself, served without Rails.
+    #
+    # Server mode is entered for the reason #run does it — a host that
+    # registers its token inside a `configure_server` block would otherwise
+    # boot an API with no credential and 404 every request. Nothing starts as a
+    # side effect: no lifecycle event fires and no Launcher is ever built.
+    #
+    # `handler:` is a test seam; production resolves one from the installed
+    # gems.
+    def run_api(boot_app: true, handler: nil)
+      enter_server_mode
+      boot_application if boot_app
+      validate_redis!
+      # Load the app here, not on the first request: a broken load should fail
+      # the boot an operator is watching, not the first client to call.
+      require_relative 'api/app'
+      raise ArgumentError, NO_API_TOKEN_MESSAGE unless @config.api_enabled?
+
+      handler ||= api_handler(@config[:api_server])
+      logger.info { "Wurk API listening on http://#{api_bind}:#{api_port}#{Wurk::API::VERSION_PREFIX}" }
+      handler.run(Wurk::API, Host: api_bind, Port: api_port)
+    end
+
     def handle_signal(sig)
       logger.debug { "Got #{sig} signal" }
       handler = SIGNAL_HANDLERS[sig]
@@ -188,6 +251,29 @@ module Wurk
       ::Process.warmup
     end
 
+    def api_bind = @config[:api_bind] || DEFAULT_API_BIND
+    def api_port = @config[:api_port] || DEFAULT_API_PORT
+
+    # Rack 3 moved the server handlers out to the `rackup` gem; rack 2 still
+    # ships them itself. Neither is a wurk dependency — the web server is the
+    # host's choice here exactly as it is for the app this sits next to — so
+    # resolve one at run time and name the fix when there is none.
+    def api_handler(name)
+      registry = handler_registry
+      name ? registry.get(name) : registry.default
+    rescue ::LoadError, ::NameError, ::ArgumentError => e
+      raise ArgumentError, "wurk api could not start a web server (#{e.message}). Add one to your Gemfile " \
+                           '— puma, falcon, or rackup + webrick — or name an installed one with --server.'
+    end
+
+    def handler_registry
+      require 'rackup/handler'
+      ::Rackup::Handler
+    rescue ::LoadError
+      require 'rack/handler'
+      ::Rack::Handler
+    end
+
     def launch(self_read)
       @launcher = Wurk::Launcher.new(@config)
       begin
@@ -270,6 +356,14 @@ module Wurk
 
     # --- options + config-file --------------------------------------------
 
+    # Pulled off before OptionParser sees argv, so the banner and the flag
+    # table can depend on which command was named. A bare word OptionParser
+    # doesn't recognize would otherwise survive `parse!` and be silently
+    # ignored, which is how `wurk api` would look like it worked.
+    def extract_command!(args)
+      COMMANDS.include?(args.first) ? args.shift : nil
+    end
+
     def setup_options(args)
       opts = parse_options(args)
       set_environment(opts[:environment])
@@ -351,14 +445,15 @@ module Wurk
 
     def option_parser(opts)
       ::OptionParser.new do |o|
-        o.banner = 'wurk [options]'
-        define_value_flags(o, opts)
+        o.banner = @command ? "wurk #{@command} [options]" : 'wurk [options]'
+        define_value_flags(o, opts, OPTION_FLAGS)
+        define_value_flags(o, opts, API_OPTION_FLAGS) if @command == 'api'
         define_meta_flags(o, opts)
       end
     end
 
-    def define_value_flags(parser, opts)
-      OPTION_FLAGS.each do |short, long, key, desc, transform|
+    def define_value_flags(parser, opts, flags)
+      flags.each do |short, long, key, desc, transform|
         parser.on(short, long, desc) { |arg| assign_flag(opts, key, arg, transform) }
       end
     end

data/lib/wurk/client/buffered.rb

@@ -119,6 +119,13 @@ module Wurk
             @overflow_mode = nil
             @buffer_client_factory = nil
           end
+          # Stop before dropping: an unstopped drainer thread survives with
+          # its factory nil'd out from under it and ticks forever against
+          # nothing, leaking the thread and everything its closure retains.
+          install_mutex.synchronize do
+            @drainer&.stop
+            @drainer = nil
+          end
         end
 
         # Fork hook, called from the `Process._fork` prepend below and from
@@ -524,7 +531,7 @@ module Wurk
       # next push — this is a Wurk extension for issue #19's "Background
       # drain thread flushes on reconnect" so producer-stopped-mid-outage
       # buffers don't sit idle until next push.
-      def reliable_push_drainer(interval: Buffered::Drainer::DEFAULT_INTERVAL)
+      def reliable_push_drainer(interval: Buffered::Drainer::DEFAULT_INTERVAL) # rubocop:disable Naming/PredicateMethod
         Buffered.install!
         Buffered.start_drainer!(interval: interval)
         true

data/lib/wurk/client.rb

@@ -3,6 +3,7 @@
 require_relative 'iterable_job'
 require_relative 'job_util'
 require_relative 'lua'
+require_relative 'status'
 
 module Wurk
   # Enqueue interface. Pipelined LPUSH / ZADD writes against the canonical
@@ -17,7 +18,7 @@ module Wurk
   #   Wurk::Client.new.push_bulk("class" => "MyJob", "args" => [[1], [2], [3]])
   #
   # Spec: docs/target/sidekiq-free.md §7.
-  class Client
+  class Client # rubocop:disable Metrics/ClassLength
     include JobUtil
 
     # Sidekiq mirrors these exactly. Tests against the upstream parity suite
@@ -26,6 +27,20 @@ module Wurk
     SCHEDULED_BATCH_SIZE      = 100
     SPREAD_INTERVAL_FLOOR     = 5
 
+    # Batched (`bid`) payloads per EVALSHA pipeline. Ours, not Sidekiq's — it
+    # has no batches. Sized to DEFAULT_BATCH_SIZE so no existing caller's
+    # round-trip count moves: `push_bulk` already hands #raw_push at most that
+    # many payloads, so its batched pipeline stays exactly one round trip.
+    #
+    # The cap is for the one path that isn't pre-sliced: `autoflush = true`
+    # buffers a whole `Batch#jobs` block, so #flush_batched can be handed an
+    # unbounded payload set. Unsliced that is one pipeline holding every
+    # command and every reply in memory at once, and — Lua being atomic and
+    # single-threaded — one uninterrupted server-side sweep that blocks every
+    # other client for its duration. Same reasoning as the LIMIT on
+    # RELIABLE_SCHEDULE_PROMOTE.
+    BATCH_PIPELINE_SLICE      = 1_000
+
     # Thread-local slot holding the payloads of the current push whose Redis
     # write is confirmed applied. {Client::Buffered} subtracts them from the set
     # it re-buffers when a *later* phase of the same push loses the connection,
@@ -72,9 +87,10 @@ module Wurk
       validate_bulk_shape!(items, args)
       return [] if args.empty?
 
+      base      = bulk_base(items)
+      reject_bulk_collapse!(base)
       at_values = expand_at(items, args.size)
       batch_sz  = items['batch_size'] || items[:batch_size] || (at_values ? SCHEDULED_BATCH_SIZE : DEFAULT_BATCH_SIZE)
-      base      = bulk_base(items)
       flush_bulk(args, at_values, base, batch_sz)
     end
 
@@ -138,7 +154,21 @@ module Wurk
 
     private
 
+    # #push and #push_bulk verify at different points and both match Sidekiq
+    # exactly: push walks the payload the chain handed back (sidekiq
+    # client.rb:101 — normalize → middleware → verify → raw_push), bulk walks it
+    # inside the innermost block (sidekiq client.rb:165). Push used to do both,
+    # and since `strict_args_mode` defaults to :raise the second full recursive
+    # args walk was never skipped.
+    #
+    # Bulk keeps its walk inside the block on purpose: a client middleware that
+    # halts the job there short-circuits the walk, and hoisting it out would
+    # raise on args that middleware was about to drop.
     def invoke_chain(normed)
+      @chain.invoke(normed['class'], normed, normed['queue'], pool) { normed }
+    end
+
+    def invoke_chain_verified(normed)
       @chain.invoke(normed['class'], normed, normed['queue'], pool) do
         verify_json(normed)
         normed
@@ -170,6 +200,19 @@ module Wurk
       base
     end
 
+    # Up front, before the first slice is written — a policy discovered halfway
+    # through would leave the slices already flushed in Redis. Two lookups and no
+    # allocation on the untouched path.
+    #
+    # `class_defaults_for` is the same resolution #normalize_item does per item,
+    # so this sees a policy in exactly the cases the middleware would: a bulk
+    # pushed by class *name* never merges class options at all, and so never
+    # carries one. Spelled out in {Wurk::Collapse.reject_bulk!}.
+    def reject_bulk_collapse!(base)
+      key = Wurk::Collapse::OPTION
+      Wurk::Collapse.reject_bulk!(base['class'], base[key] || class_defaults_for(base['class'])[key])
+    end
+
     def flush_bulk(args, at_values, base, batch_size)
       jids = []
       args.each_slice(batch_size).with_index do |slice, slice_index|
@@ -191,7 +234,7 @@ module Wurk
         item = base.merge('args' => job_args)
         item['at'] = ats[idx] if ats
         normed = normalize_item(item)
-        invoke_chain(normed)
+        invoke_chain_verified(normed)
       end
     end
 
@@ -296,7 +339,7 @@ module Wurk
 
     def push_scheduled(conn, payloads)
       args = payloads.flat_map do |hash|
-        [hash['at'].to_s, Wurk.dump_json(hash.except('enqueued_at', 'at'))]
+        [hash['at'].to_s, JobUtil.scheduled_member(hash)]
       end
       conn.call('ZADD', 'schedule', *args)
     end
@@ -314,6 +357,27 @@ module Wurk
       push_batched_pipelined(conn, batched, now) unless batched.empty?
     end
 
+    # One pipeline per BATCH_PIPELINE_SLICE payloads, each marked delivered the
+    # moment its reply is in — same contract as push_plain_group, and for the
+    # same reason: a slice that Redis already accepted must stay out of the
+    # reliable_push ledger, or a later slice's failure would report it as
+    # undelivered.
+    def push_batched_pipelined(conn, batched, now)
+      batched.each_slice(BATCH_PIPELINE_SLICE) do |slice|
+        eval_batched_slice(conn) { |pipe, eval_method| push_batched(pipe, slice, now, eval_method: eval_method) }
+        mark_delivered(slice)
+      end
+    end
+
+    # Same slicing and NOSCRIPT recovery as push_batched_pipelined, for the
+    # scheduled batched path (BATCH_SCHEDULE instead of BATCH_PUSH).
+    def push_batched_scheduled_pipelined(conn, batched)
+      batched.each_slice(BATCH_PIPELINE_SLICE) do |slice|
+        eval_batched_slice(conn) { |pipe, eval_method| push_batched_scheduled(pipe, slice, eval_method: eval_method) }
+        mark_delivered(slice)
+      end
+    end
+
     # Outside of test boots and `SCRIPT FLUSH` the rescue branch is dead
     # code; the eager `script_load_all` after fork keeps the script cache
     # hot for the life of the connection. The retry uses EVAL (source-embedded)
@@ -321,24 +385,18 @@ module Wurk
     # NOSCRIPT a second time under heavy CI load (WorkerTest 3.4/7.2 flake).
     # `script_load_all` still primes the cache so the *next* pipeline returns
     # to the EVALSHA fast path.
-    def push_batched_pipelined(conn, batched, now)
-      conn.pipelined { |pipe| push_batched(pipe, batched, now) }
-    rescue RedisClient::CommandError => e
-      raise unless e.message.to_s.start_with?('NOSCRIPT')
-
-      Wurk::Lua::Loader.script_load_all(conn)
-      conn.pipelined { |pipe| push_batched(pipe, batched, now, eval_method: :eval_with_source) }
-    end
-
-    # Same NOSCRIPT-recovery shape as push_batched_pipelined, for the scheduled
-    # batched path (BATCH_SCHEDULE instead of BATCH_PUSH).
-    def push_batched_scheduled_pipelined(conn, batched)
-      conn.pipelined { |pipe| push_batched_scheduled(pipe, batched) }
+    #
+    # Replaying the slice is safe precisely because every command in it is the
+    # same script: a flushed cache NOSCRIPTs all of them and applies none.
+    # Recovery is per slice, so the slices already acknowledged above are never
+    # re-sent. Mirrors Fetcher::Reliable#requeue_pipelined.
+    def eval_batched_slice(conn)
+      conn.pipelined { |pipe| yield(pipe, :eval_cached) }
     rescue RedisClient::CommandError => e
       raise unless e.message.to_s.start_with?('NOSCRIPT')
 
       Wurk::Lua::Loader.script_load_all(conn)
-      conn.pipelined { |pipe| push_batched_scheduled(pipe, batched, eval_method: :eval_with_source) }
+      conn.pipelined { |pipe| yield(pipe, :eval_with_source) }
     end
 
     # One pipeline per queue, marked delivered the moment its reply is in.
@@ -355,27 +413,67 @@ module Wurk
     # Cost is a round trip per distinct queue. The single-queue push — every
     # `perform_async`, every same-class `push_bulk` — still writes exactly the
     # one SADD + LPUSH pipeline it did before.
+    #
+    # `uniform_queue` short-circuits the common case (one job, or many jobs
+    # all destined for the same queue) without paying for the `group_by`
+    # Hash + per-group Array allocations; only a genuinely mixed-queue batch
+    # falls through to grouping.
     def push_plain(conn, payloads, now)
-      payloads.group_by { |j| j['queue'] }.each do |queue, jobs|
-        serialized = jobs.map do |j|
-          j['enqueued_at'] = now
-          Wurk.dump_json(j)
-        end
-        conn.pipelined do |pipe|
-          pipe.call('SADD', 'queues', queue)
-          pipe.call('LPUSH', "queue:#{queue}", *serialized)
-        end
-        mark_delivered(jobs)
+      queue = uniform_queue(payloads)
+      return push_plain_group(conn, queue, payloads, now) if queue
+
+      payloads.group_by { |j| j['queue'] }.each { |q, jobs| push_plain_group(conn, q, jobs, now) }
+    end
+
+    def uniform_queue(payloads)
+      first = payloads[0]['queue']
+      return first if payloads.size == 1
+
+      first if payloads.all? { |j| j['queue'] == first }
+    end
+
+    def push_plain_group(conn, queue, jobs, now)
+      serialized = jobs.map do |j|
+        j['enqueued_at'] = now
+        Wurk.dump_json(j)
+      end
+      conn.pipelined do |pipe|
+        pipe.call('SADD', 'queues', queue)
+        pipe.call('LPUSH', "queue:#{queue}", *serialized)
+        track_enqueued(pipe, jobs, now)
+      end
+      mark_delivered(jobs)
+    end
+
+    # Ride the pipeline that is already open for the queue write, so a tracked
+    # push costs the same round trip an untracked one does. An app that never
+    # opts in pays one Hash lookup per payload here and appends nothing: the
+    # plain push stays exactly the two commands (SADD + LPUSH) it has always
+    # been, which is what `rake bench:command_count_tracked_off` asserts.
+    #
+    # The TTL is resolved once per group, and only when something is actually
+    # tracked — an untracked push never reads the config at all.
+    def track_enqueued(pipe, jobs, now)
+      ttl = nil
+      jobs.each do |job|
+        next unless job['track']
+
+        ttl ||= Wurk::Status.default_ttl(@config)
+        Wurk::Status.enqueued(pipe, job, now, ttl: ttl)
       end
     end
 
     # Batched jobs route through BATCH_PUSH: increments b-<bid> total+pending,
     # SADDs jid into the live set, registers the queue, LPUSHes the payload —
-    # all atomically. One Redis round-trip per job (no pipeline grouping)
-    # because the lua needs per-job KEYS bound. Acceptable cost: batch
-    # enqueue is not the hot path; correctness is. `eval_method` is the
+    # all atomically. The Lua binds per-job KEYS, so grouping N jobs into one
+    # EVALSHA isn't available; they ride one pipeline instead (the `conn` here
+    # is always the pipeline #push_batched_pipelined opened), so the cost is
+    # N commands and one round trip, not N round trips. `eval_method` is the
     # Wurk::Lua::Loader entry point (`:eval_cached` for the hot EVALSHA path,
-    # `:eval_with_source` for the EVAL-source retry).
+    # `:eval_with_source` for the EVAL-source retry) — neither reads the reply,
+    # which is what makes the pipelined form legal: `eval_cached`'s inline
+    # NOSCRIPT rescue can't fire against a buffered call, so recovery is the
+    # caller's finalize-time rescue.
     def push_batched(conn, payloads, now, eval_method: :eval_cached)
       payloads.each do |j|
         j['enqueued_at'] = now
@@ -388,15 +486,20 @@ module Wurk
           argv: [j['queue'], j['jid'], Wurk.dump_json(j), j['bid'], Wurk::Batch::DEFAULT_EXPIRY_SECONDS]
         )
       end
+      # Same pipeline, so a batched tracked push keeps its one round trip too.
+      # This slice is the one that gets replayed on NOSCRIPT (see
+      # #eval_batched_slice); HSET + EXPIRE are idempotent, so a replay writes
+      # the same row twice rather than double-counting anything.
+      track_enqueued(conn, payloads, now)
     end
 
     # Scheduled batched jobs route through BATCH_SCHEDULE: the SADD-guarded
     # total/pending increment registers the job in its batch at creation, and
     # the ZADD defers it onto `schedule`. Payload is stripped of `at`/
     # `enqueued_at` exactly like push_scheduled — `enqueued_at` is stamped fresh
-    # at promotion, never while the job sits scheduled (spec §7.1). One Redis
-    # round-trip per job because the Lua binds per-job KEYS; scheduled batch
-    # enqueue is not the hot path.
+    # at promotion, never while the job sits scheduled (spec §7.1). Per-job
+    # KEYS again, so one EVALSHA per job — pipelined by
+    # #push_batched_scheduled_pipelined into one round trip per slice.
     def push_batched_scheduled(conn, payloads, eval_method: :eval_cached)
       payloads.each do |j|
         Wurk::Lua::Loader.public_send(
@@ -436,7 +539,13 @@ module Wurk
     # evict them, and the drain that does land one counts it then — so booking
     # them here would inflate the counter on an outage and double-count every
     # payload that later replays.
+    #
+    # Resolve the client once for the whole batch and bail before touching a
+    # payload: unconfigured is the common case, and the tags below cost two
+    # Strings and an Array per job for `increment` to immediately drop.
     def emit_enqueued(payloads, buffered = nil)
+      return if Wurk::Metrics::Statsd.safe_client.nil?
+
       payloads = reject_by_identity(payloads, buffered) if buffered && !buffered.empty?
       payloads.each do |p|
         Wurk::Metrics::Statsd.increment(