wurk 1.5.0 → 1.7.0

data/lib/wurk/configuration.rb

@@ -8,6 +8,7 @@ require_relative 'pool_checkout'
 require_relative 'context'
 require_relative 'topology'
 require_relative 'redis_options'
+require_relative 'keys'
 
 module Wurk
   # Owns runtime knobs (concurrency, queues, timeouts, lifecycle events,
@@ -46,7 +47,58 @@ module Wurk
       backtrace_cleaner: ->(bt) { bt },
       logged_job_attributes: %w[bid tags],
       redis_idle_timeout: nil,
-      redis_error_handlers: []
+      redis_error_handlers: [],
+      # Wurk extras, appended after the mirrored keys so the Sidekiq prefix
+      # above stays byte-for-byte what a gem reading @options expects.
+      #
+      # Lifetime of a `status:<jid>` row (Wurk::Status), re-stamped on every
+      # write, and the separate lifetime a `complete` row gets once the job
+      # succeeded. Retention is off by default — nil means a finished job's
+      # row expires on the same clock as a running one. Set it to seconds to
+      # keep succeeded jobs around longer than they ran, or to 0 for Sidekiq's
+      # own behavior, where a job that succeeds leaves nothing behind.
+      status_ttl: Keys::STATUS_TTL,
+      status_retention: nil,
+      # Distributed tracing (Wurk::Telemetry). Seeded here, not written on
+      # demand, so the read side answers on a config that `freeze!` has already
+      # closed — and so a gem inspecting @options sees the same key set whether
+      # or not the host traces. Flipping it is only half the gate; the
+      # opentelemetry-api gem has to be present too (Wurk::Telemetry.enabled?).
+      telemetry: false,
+      # Bearer tokens for the machine-facing HTTP API (Wurk::API), as
+      # `token => [scopes]`. Empty means the API does not exist: the engine
+      # skips the mount and a directly mounted app answers 404, so a host that
+      # never registers a token carries no new HTTP surface at all. Seeded here
+      # rather than written on demand so the read side answers on a config that
+      # `freeze!` has already closed.
+      api_tokens: {},
+      # Boundary caps for the HTTP API's produce plane. A body larger than
+      # `api_max_body_bytes` is refused before it is parsed; a job whose args
+      # serialize larger than `api_max_args_bytes` before it is pushed — the
+      # second is the one that bounds what a bulk request lands in Redis, since
+      # one request becomes many payloads. `api_idempotency_ttl` is how long an
+      # `Idempotency-Key` is remembered: a replay window for a producer
+      # retrying a lost response, not an audit log.
+      api_max_body_bytes: 1_048_576,
+      api_max_args_bytes: 65_536,
+      api_idempotency_ttl: 3_600,
+      # Allow-list of classes the HTTP API may enqueue. nil (the default) means
+      # only an `admin` token may enqueue at all — see #api_enqueue_classes=.
+      api_enqueue_classes: nil,
+      # Three-state on purpose — see #api_read_only=. nil is "inherit", which
+      # only mount mode 1 has anything to inherit from.
+      api_read_only: nil,
+      # Per-token request ceiling. nil is off: the API adds no Redis work to a
+      # request until a host asks for a limit, and no number picked here would
+      # be right for both a cron relay and a fan-out producer.
+      api_rate_limit: nil,
+      api_rate_limit_interval: :minute,
+      # Cluster-wide ceiling per queue (Wurk::QueueSlot), as
+      # `{ 'critical' => 20 }`. Empty is the whole point of seeding it here:
+      # the fetch path reads this once at boot, so an app that caps nothing
+      # resolves to "no capped queues" before the first fetch and never asks
+      # again.
+      global_concurrency: {}
     }.freeze
 
     # :fork fires only inside swarm children, after fork + internal AR/Redis
@@ -100,6 +152,12 @@ module Wurk
 
     def initialize(options = {})
       @options = deep_dup_defaults.merge(options)
+      # Through the same door `global_concurrency=` uses, so a cap passed to
+      # `Configuration.new` is validated rather than trusted, and the default
+      # Hash arrives frozen: the fetch path resolves caps once at boot, so a
+      # Hash still mutable here is one a caller can add a queue to and have
+      # nothing read it. FrozenError is the honest answer to that.
+      @options[:global_concurrency] = normalize_global_concurrency(@options[:global_concurrency])
       @options[:error_handlers] << ERROR_HANDLER if @options[:error_handlers].empty?
       @capsules = {}
       @directory = {}
@@ -109,6 +167,7 @@ module Wurk
       @web_redis_pool = nil
       @logger = nil
       @thread_priority = DEFAULT_THREAD_PRIORITY
+      @telemetry_installed = false
       @frozen = false
     end
 
@@ -382,6 +441,206 @@ module Wurk
     def history_interval = @options.fetch(:history_interval, HISTORY_DEFAULT_INTERVAL)
     def history_collector = @options[:history_collector]
 
+    # --- Distributed tracing (OpenTelemetry) ------------------------------
+
+    # @return [Boolean] whether the host asked for tracing. Half the gate;
+    #   {Wurk::Telemetry.enabled?} is the whole one.
+    def telemetry? = @options[:telemetry] == true
+
+    # Opt into Wurk::Telemetry — a producer span carrying W3C trace context on
+    # the job hash at enqueue, a linked consumer span on execute:
+    #
+    #   Wurk.configure_server { |config| config.telemetry = true }
+    #
+    # Loading `wurk/telemetry` is deferred to this call, which is the only
+    # reason `require "wurk"` can leave `opentelemetry-api` untouched in an app
+    # that doesn't trace.
+    #
+    # Warns when the host opted in but the gem is missing: tracing then stays
+    # off, and a silent no-op is indistinguishable from a broken exporter.
+    def telemetry=(enabled)
+      guard_frozen!
+      @options[:telemetry] = enabled ? true : false
+      # Turning tracing back off still has to unregister, but must not *load*
+      # the module to do it — that require is the whole cost this flag gates,
+      # and a config that never opted in has nothing registered to undo. Tracked
+      # per instance rather than off `defined?(Wurk::Telemetry)`: whether some
+      # *other* config loaded the module says nothing about this one's chain.
+      return if !@options[:telemetry] && !@telemetry_installed
+
+      require_relative 'telemetry'
+      @telemetry_installed = true
+      Wurk::Telemetry.install!(self)
+      return if !@options[:telemetry] || Wurk::Telemetry.available?
+
+      logger.warn { 'config.telemetry = true but opentelemetry-api is not installed; tracing stays off' }
+    end
+
+    # --- Machine-facing HTTP API (Wurk::API) ------------------------------
+
+    # Registers a bearer token for the HTTP API and the scopes it grants —
+    # any of `:enqueue`, `:read`, `:admin` (which grants the other two):
+    #
+    #   # config/initializers/wurk.rb
+    #   Wurk.configuration.api_token ENV.fetch('WURK_API_TOKEN'), scopes: %i[enqueue read]
+    #
+    # Registered unconditionally, not inside `configure_server`/`configure_client`
+    # — the process that serves the API has to be the one holding the credential,
+    # and neither block runs in every such process. A Puma-cluster web process
+    # never enters server mode (Wurk::RailsBoot refuses to fork a swarm from
+    # one), so a token registered in `configure_server` would be absent from the
+    # exact process serving the engine-nested mount.
+    #
+    # Registering the first token is what brings the API into existence; with
+    # none the app is never mounted. Re-registering a token replaces its
+    # scopes. Loading `wurk/api/auth` is deferred to this call so a host that
+    # serves no HTTP API never pays for openssl/digest in every swarm child.
+    def api_token(token, scopes:)
+      guard_frozen!
+      require_relative 'api/auth'
+      value, granted = Wurk::API::Auth.credential!(token, scopes)
+      @options[:api_tokens][value] = granted
+      nil
+    end
+
+    # @return [Hash{String => Array<Symbol>}] registered tokens and their scopes
+    def api_tokens = @options[:api_tokens]
+
+    # @return [Boolean] whether the HTTP API exists at all. The engine's mount
+    #   is conditional on this.
+    def api_enabled? = !@options[:api_tokens].empty?
+
+    # @return [Integer] largest request body the produce plane will read.
+    def api_max_body_bytes = @options[:api_max_body_bytes]
+
+    def api_max_body_bytes=(bytes)
+      @options[:api_max_body_bytes] = api_limit!(:api_max_body_bytes, bytes)
+    end
+
+    # @return [Integer] largest `args` any single job may serialize to.
+    def api_max_args_bytes = @options[:api_max_args_bytes]
+
+    def api_max_args_bytes=(bytes)
+      @options[:api_max_args_bytes] = api_limit!(:api_max_args_bytes, bytes)
+    end
+
+    # @return [Integer] seconds an `Idempotency-Key` stays replayable.
+    def api_idempotency_ttl = @options[:api_idempotency_ttl]
+
+    def api_idempotency_ttl=(seconds)
+      @options[:api_idempotency_ttl] = api_limit!(:api_idempotency_ttl, seconds)
+    end
+
+    # @return [Array<String>, :any, nil] classes the HTTP API may enqueue.
+    def api_enqueue_classes = @options[:api_enqueue_classes]
+
+    # The allow-list `JobUtil::TRANSIENT_ATTRIBUTES` deliberately is not — that
+    # is a strip-list, right for a producer already running your code:
+    #
+    #   config.api_enqueue_classes = %w[Billing::Charge ReportJob]
+    #
+    # Unset, only an `admin` token may enqueue at all. `class` is a code
+    # selector — a producer token that can name any constant in the host is
+    # choosing which of its jobs to run — so an `enqueue`-scoped credential has
+    # to be told exactly what it is for. Once a list exists it binds every
+    # token, `admin` included: a host that writes one means it.
+    #
+    # `:any` turns the check off, which is what an enqueue-scoped relay that
+    # legitimately pushes arbitrary classes needs — the alternative is handing
+    # it `admin` and the destructive routes that come with it.
+    def api_enqueue_classes=(classes)
+      guard_frozen!
+      require_relative 'api/validation'
+      @options[:api_enqueue_classes] = Wurk::API::Validation.enqueueable_classes!(classes)
+    end
+
+    # @return [Boolean, nil] true/false when the host said so, nil to inherit.
+    def api_read_only = @options[:api_read_only]
+
+    # Freezes the API's writes — enqueue included, not only the destructive
+    # `admin` routes. Same rule as the dashboard's `Wurk::Web.config.read_only`
+    # and deliberately the same sentence: a read-only deployment answers reads.
+    # Carving enqueue out would mean a viewer-only deploy could still be made
+    # to run arbitrary work, and would leave an operator holding two different
+    # definitions of the same word.
+    #
+    # Three states, because the two planes are not always one deployment:
+    #
+    #   nil (default)  inherit. Mounted inside the engine, the API is part of
+    #                  the dashboard's deployment and `WURK_WEB_READ_ONLY=1`
+    #                  freezes both. Mounted on its own path or run standalone
+    #                  there is nothing to inherit from and this means off.
+    #   true           frozen wherever it is mounted. `WURK_API_READ_ONLY=1`
+    #                  sets it, which is the only door mode 3 has.
+    #   false          live even inside a read-only dashboard — the host that
+    #                  wants a viewer-only UI and a working producer on the
+    #                  same mount, said out loud.
+    def api_read_only=(value)
+      guard_frozen!
+      @options[:api_read_only] = value.nil? ? nil : truthy_setting?(value)
+    end
+
+    # @return [Boolean] the resolved verdict for a mount with nothing to
+    #   inherit; App layers mode 1's inherited flag over it.
+    def api_read_only?
+      value = @options[:api_read_only]
+      value.nil? ? ENV['WURK_API_READ_ONLY'] == '1' : value
+    end
+
+    # @return [Integer, nil] requests one token may make per interval.
+    def api_rate_limit = @options[:api_rate_limit]
+
+    # Requests per `api_rate_limit_interval`, per token, across the fleet —
+    # enforced by a `Wurk::Limiter` sliding window rather than a second
+    # limiter, so an API ceiling shows up on the Limiters page beside every
+    # other one and resets on the same Redis clock. nil turns it off.
+    def api_rate_limit=(count)
+      guard_frozen!
+      @options[:api_rate_limit] = count.nil? ? nil : api_limit!(:api_rate_limit, count)
+    end
+
+    # @return [Symbol, Integer] the window the limit is counted over.
+    def api_rate_limit_interval = @options[:api_rate_limit_interval]
+
+    # `:second :minute :hour :day` or a raw Integer of seconds — whatever
+    # `Limiter.window` accepts, validated here so a typo raises in the
+    # initializer that wrote it rather than on the first throttled request.
+    def api_rate_limit_interval=(interval)
+      guard_frozen!
+      Wurk::Limiter.interval_seconds(interval, allow_integer: true)
+      @options[:api_rate_limit_interval] = interval
+    end
+
+    # --- Global per-queue concurrency (Wurk::QueueSlot) -------------------
+
+    # @return [Hash{String => Integer}] queue name → cluster-wide ceiling.
+    #   Always frozen — the default included, normalized in #initialize; see
+    #   #global_concurrency=.
+    def global_concurrency = @options[:global_concurrency]
+
+    # Caps how many jobs from a queue may run at once across the whole
+    # cluster, whatever the worker count:
+    #
+    #   Wurk.configure_server { |config| config.global_concurrency = { critical: 20 } }
+    #
+    # Not the same thing as `concurrency`, which is threads in one process, nor
+    # as a {Wurk::Limiter}, which is keyed per lock and gates inside the job
+    # (by which point the job is already claimed off the queue). This is a
+    # ceiling on the queue itself, enforced where the work is picked up.
+    #
+    # Names are normalized to Strings and the result is frozen, because the
+    # fetch path resolves it once at boot: a cap added by mutating this Hash
+    # afterwards would be read by nothing, so it raises rather than doing
+    # nothing. Assign a new Hash to change one.
+    def global_concurrency=(caps)
+      guard_frozen!
+      @options[:global_concurrency] = normalize_global_concurrency(caps)
+    end
+
+    # @return [Boolean] whether any queue is capped. The one question the
+    #   fetcher asks at boot to decide whether it has a gate to run at all.
+    def global_concurrency? = !@options[:global_concurrency].empty?
+
     # --- Web dashboard ----------------------------------------------------
 
     # Web UI configuration: the authorization hook and read-only mode. Returns
@@ -409,7 +668,7 @@ module Wurk
       guard_frozen!
       p = Integer(port)
       rw = Integer(ready_window)
-      raise ArgumentError, 'port must be between 0 and 65535' unless (0..65535).cover?(p)
+      raise ArgumentError, 'port must be between 0 and 65535' unless (0..65_535).cover?(p)
       raise ArgumentError, 'ready_window must be > 0' unless rw.positive?
 
       b = bind.to_s
@@ -557,7 +816,7 @@ module Wurk
     # native name, SIDEKIQ_COUNT the drop-in alias. Unparseable input falls back
     # to the CPU count; the result is floored at 1 so the swarm always forks.
     def default_child_count
-      raw = ENV['WURK_COUNT'] || ENV['SIDEKIQ_COUNT']
+      raw = ENV['WURK_COUNT'] || ENV.fetch('SIDEKIQ_COUNT', nil)
       return Etc.nprocessors if raw.nil? || raw.strip.empty?
 
       value = Float(raw)
@@ -570,7 +829,7 @@ module Wurk
     # SIDEKIQ_MAXMEM_MB is the drop-in name; WURK_MAXMEM_MB the native alias.
     # Unparseable / empty input disables recycling rather than raising at boot.
     def env_memory_limit_mb
-      raw = ENV['WURK_MAXMEM_MB'] || ENV['SIDEKIQ_MAXMEM_MB']
+      raw = ENV['WURK_MAXMEM_MB'] || ENV.fetch('SIDEKIQ_MAXMEM_MB', nil)
       return nil if raw.nil? || raw.strip.empty?
 
       Integer(raw)
@@ -582,6 +841,46 @@ module Wurk
       raise FrozenError, 'Wurk::Configuration is frozen' if @frozen
     end
 
+    # `config.api_read_only = ENV['SOMETHING']` is the shape a host reaches for,
+    # and `!!'false'` is true. Borrow the dashboard's list of strings that mean
+    # off rather than growing a second one that could disagree with it about the
+    # same env var.
+    def truthy_setting?(value)
+      return !!value unless value.is_a?(String)
+
+      require_relative 'web/config'
+      !Wurk::Web::Config::FALSEY_STRINGS.include?(value.strip.downcase)
+    end
+
+    # Boundary caps are refused where they are written, not where they are
+    # read: a cap that silently coerced to 0 is a cap that isn't one, and the
+    # request path is the wrong place to discover that.
+    def api_limit!(name, value)
+      guard_frozen!
+      size = Integer(value, exception: false)
+      raise ArgumentError, "#{name} must be a positive Integer" unless size&.positive?
+
+      size
+    end
+
+    # A cap is a promise about how much of a queue may run; a typo in one is a
+    # queue that silently runs unbounded (name) or never runs (count), and
+    # neither shows up until production is either overloaded or stalled. Both
+    # are refused in the initializer that wrote them.
+    def normalize_global_concurrency(caps)
+      raise ArgumentError, "global_concurrency must be a Hash, got #{caps.class}" unless caps.is_a?(Hash)
+
+      caps.each_with_object({}) do |(queue, cap), resolved|
+        name = queue.to_s
+        raise ArgumentError, 'global_concurrency queue names cannot be empty' if name.empty?
+
+        limit = Integer(cap, exception: false)
+        raise ArgumentError, "global_concurrency[#{name.inspect}] must be a positive Integer" unless limit&.positive?
+
+        resolved[name] = limit
+      end.freeze
+    end
+
     def deep_dup_defaults
       DEFAULTS.each_with_object({}) do |(k, v), h|
         h[k] = case v

data/lib/wurk/debounce.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+require_relative 'job_util'
+require_relative 'keys'
+require_relative 'lua'
+require_relative 'pool_checkout'
+require_relative 'unique'
+
+module Wurk
+  # Debounce — collapse a burst of enqueues into one job carrying the last
+  # payload, fired after a period of quiet. A Wurk extra, not a Sidekiq
+  # surface, and deliberately not a second flavour of {Wurk::Unique}: unique
+  # *rejects* the duplicate and keeps the first, debounce *replaces* the
+  # pending job and keeps the last.
+  #
+  # The classic use is a re-derivation triggered by something a user does
+  # repeatedly — reindex a record, warm a cache, recompute a projection. Fifty
+  # edits in a minute should produce one rebuild at the end, not fifty.
+  #
+  # Redis schema — one new key, plus an ordinary entry in the ZSET that already
+  # exists:
+  #
+  #   debounce:<digest>   HASH   member (the pending job JSON), first (epoch)
+  #   schedule            ZSET   the collapsed job, exactly as `perform_in`
+  #                              would have written it
+  #
+  # Nothing reading `schedule` can tell a debounced job from any other
+  # scheduled one — {Wurk::ScheduledSet}, the dashboard and a stock Sidekiq
+  # poller all handle it unchanged. There is no parallel delayed structure.
+  #
+  # Identity is {Wurk::Unique}'s digest of `[class, queue, args]`, so a class
+  # that already narrows its key with `sidekiq_unique_context` narrows its
+  # debounce key the same way — that hook is the supported "collapse on a
+  # subset of the args" escape hatch, and there is no second one.
+  module Debounce
+    # How long `debounce:<digest>` outlives the entry it points at.
+    #
+    # The key's only job is to remember which member to pull back out of
+    # `schedule` when the burst is extended, so it must not expire while that
+    # member is still pending — a burst that lost its key ZADDs a second entry
+    # next to the first and stops collapsing. The window that matters is
+    # therefore "fire time → poller promotes it", which the scheduler keeps at
+    # roughly `average_scheduled_poll_interval` across the cluster however many
+    # processes are running. Five minutes is far past that; overshooting costs
+    # one ~100-byte key per idle debounce identity, and the script treats an
+    # outlived key as a finished burst anyway.
+    GRACE = 300
+
+    # What one debounced enqueue did to the pending entry.
+    #
+    # `replaced` is the only part a caller cannot work out for itself: from
+    # outside the script, extending a live burst and opening a new one look
+    # identical. `at` is the epoch second the job will run — capped, so it is
+    # not necessarily `now + wait`. `opened_at` is the burst's own start, the
+    # clock `max_wait` is measured against.
+    Outcome = Struct.new(:replaced, :at, :opened_at) do
+      # True when this push displaced a still-pending sibling.
+      def replaced? = replaced
+    end
+
+    class << self
+      # @param job [Hash] a normalized job payload
+      # @return [String] `debounce:<digest>` for that job's identity
+      def key_for(job) = Keys.debounce(Unique.digest_for(job))
+
+      # Collapse `job` into the pending entry for its key, or open a new burst.
+      #
+      # @param job [Hash] normalized payload; stored minus `at`/`enqueued_at`,
+      #   the same bytes {Wurk::Client#push_scheduled} would have written
+      # @param wait [Numeric] seconds of quiet before the job fires; every
+      #   further enqueue of the same key pushes this out again
+      # @param max_wait [Numeric, nil] hard cap measured from the *first*
+      #   enqueue of the burst. nil leaves the job able to starve — a key
+      #   re-enqueued faster than `wait` never fires — so callers that cannot
+      #   rule that out should always set it.
+      # @param pool [#with, nil] defaults to this process's pool
+      # @return [Outcome]
+      def schedule(job, wait:, max_wait: nil, pool: nil)
+        seconds = positive_seconds!(wait, 'wait')
+        cap = max_wait.nil? ? 0.0 : positive_seconds!(max_wait, 'max_wait')
+        reject_shorter_cap!(seconds, cap)
+
+        argv = [seconds, cap, JobUtil.scheduled_member(job), GRACE]
+        # Replay-safe: a re-run finds the member it just wrote, ZREMs it and
+        # ZADDs it back, so a pool retry after a lost reply converges on one
+        # entry rather than a duplicate — the whole point of doing this in a
+        # script rather than a pipeline.
+        #
+        # The replay is not byte-identical: it recomputes `now`, so the fire
+        # time moves out by the retry's own latency (and `first` is carried
+        # forward, so `max_wait` still caps where it lands). That is the claim
+        # worth making anyway. A lost reply means the write probably applied,
+        # and refusing the replay only converts a bounded shift into an
+        # exception out of `perform_async` for a job that is already scheduled
+        # — whose caller then re-enqueues and shifts the fire time regardless.
+        raw = with_pool(pool, idempotent: true) do |conn|
+          Lua::Loader.eval_cached(conn, :debounce, keys: [key_for(job), Keys::SCHEDULE], argv: argv)
+        end
+        Outcome.new(raw[0].to_i == 1, raw[1].to_f, raw[2].to_f)
+      end
+
+      private
+
+      def positive_seconds!(value, name)
+        raise ArgumentError, "debounce #{name} must be a positive number of seconds: #{value.inspect}" \
+          unless JobUtil.positive_seconds?(value)
+
+        value.to_f
+      end
+
+      # A cap below the quiet period fires every burst at `first + max_wait`
+      # regardless of traffic, which is a throttle wearing a debounce's name.
+      # Nobody writes that on purpose, so it fails where it was declared.
+      def reject_shorter_cap!(wait, cap)
+        return if cap.zero? || cap >= wait
+
+        raise ArgumentError, "debounce max_wait (#{cap}) must be at least wait (#{wait})"
+      end
+
+      def with_pool(pool, idempotent: false, &)
+        pool ? PoolCheckout.with(pool, idempotent, &) : Wurk.redis(idempotent:, &)
+      end
+    end
+  end
+end

data/lib/wurk/encryption.rb

@@ -45,10 +45,15 @@ module Wurk
   #     remain plaintext.
   #   * Incompatible with Wurk::Unique (each ciphertext differs → digest
   #     defeats the lock). Documented invariant.
+  #   * A worker that also sets `track: true` gets its full Wurk::Status
+  #     lifecycle but no stored result — `perform`'s return value is dropped and
+  #     the row is flagged `result_withheld`, rather than landing in a plaintext
+  #     HASH beside the arguments this envelope exists to protect. See
+  #     Wurk::Middleware::Status#withhold_result?.
   #   * Web UI redacts the last arg when `encrypt: true` is set on the job.
   #
   # Spec: docs/target/sidekiq-ent.md §4.
-  module Encryption # rubocop:disable Metrics/ModuleLength
+  module Encryption
     CIPHER_NAME = 'aes-256-gcm'
     KEY_BYTES = 32
     IV_BYTES = 12 # GCM standard: 96-bit IV.