wurk 1.4.0 → 1.6.0

data/lib/wurk/api/roll_up.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require_relative 'serializers'
+
+module Wurk
+  module API
+    # `GET /swarm`: many heartbeats folded into one answer to "how is the
+    # swarm actually working". Serializers shapes one process; this shapes the
+    # cluster, and it is the only place the two differ in kind.
+    #
+    # A pure function of the {Wurk::Process} list it is handed — no Redis of
+    # its own — so the roll-up and the `/processes` listing beside it can only
+    # ever describe the same fleet.
+    #
+    # What it deliberately does not report: the swarm parent's rolling-restart
+    # state. `Wurk::Swarm::Restart` is a state machine in the parent's memory —
+    # which slot is being replaced, whether its replacement has beaten yet —
+    # and no part of it is ever written to Redis. The API usually runs in a
+    # different process entirely (a Rails web worker, or standalone `wurk
+    # api`), so the only way to serve it would be new heartbeat traffic, which
+    # is exactly what this plane may not add. The observable signature is here
+    # instead: `versions` holds more than one entry while a deploy is
+    # mid-flight, `processes.quiet` counts the children already told to drain,
+    # and `hosts` shows the per-host child count the replacements land in.
+    module RollUp
+      module_function
+
+      # @param processes [Array<Wurk::Process>] live heartbeats, already
+      #   filtered of identities whose `info` has expired (ProcessSet#each).
+      # @param leader_identity [String] the `dear-leader` value, read once.
+      # @param now [Float] epoch seconds, taken once for the whole answer.
+      def cluster(processes, leader_identity:, now:)
+        ages = processes.map { |process| Serializers.beat_age_seconds(process['beat'], now) }
+        {
+          processes: process_counts(processes, ages),
+          **totals(processes),
+          queues: processes.flat_map(&:queues).compact.uniq.sort,
+          versions: processes.filter_map(&:version).uniq.sort,
+          leader: leader(processes, leader_identity),
+          hosts: hosts(processes),
+          slots: slots(processes),
+          beat: beat(ages)
+        }
+      end
+
+      def totals(processes)
+        concurrency = sum(processes, 'concurrency')
+        busy = sum(processes, 'busy')
+        {
+          concurrency: concurrency, busy: busy,
+          utilization: utilization(busy, concurrency), rss_kb: sum(processes, 'rss')
+        }
+      end
+
+      def sum(processes, field) = processes.sum { |process| process[field].to_i }
+
+      def process_counts(processes, ages)
+        {
+          total: processes.size,
+          quiet: processes.count(&:stopping?),
+          stale: ages.count { |age| Serializers.stale?(age) }
+        }
+      end
+
+      # Busy threads over total threads, as a fraction. Zero rather than a
+      # division by zero when nothing is running: an empty fleet is 0% busy,
+      # and a null here would only push the special case onto every client.
+      def utilization(busy, concurrency)
+        return 0.0 if concurrency.zero?
+
+        (busy.to_f / concurrency).round(4)
+      end
+
+      # `live` distinguishes a leader that is still beating from a lock left
+      # behind by a process that died holding it — the key outlives the
+      # heartbeat by up to its own TTL, and during that gap the cluster has a
+      # recorded leader and no leader.
+      def leader(processes, identity)
+        identity = identity.to_s
+        return { identity: nil, live: false } if identity.empty?
+
+        { identity: identity, live: processes.any? { |process| process.identity == identity } }
+      end
+
+      # Per-host roll-up, which is what "how many children is this box
+      # running" actually asks. Hardware facts ride here rather than on every
+      # process row: they describe the box, and repeating them per child would
+      # invite a client to average them.
+      def hosts(processes)
+        grouped = processes.group_by { |process| process['hostname'] }
+        grouped.map { |hostname, group| host(hostname, group) }.sort_by { |host| host[:hostname].to_s }
+      end
+
+      def host(hostname, group)
+        facts = group.first
+        {
+          hostname: hostname, processes: group.size,
+          concurrency: sum(group, 'concurrency'), busy: sum(group, 'busy'), rss_kb: sum(group, 'rss'),
+          cpu_model: facts['cpu_model'], cores: facts['cores'], memory_total_kb: facts['memory_total_kb']
+        }
+      end
+
+      # The topology as the heartbeats describe it, in {Wurk::Topology::Slot}'s
+      # own vocabulary: how many children of each (queues, concurrency) kind
+      # are live. Observed rather than read off `config.topology`, because the
+      # declaration lives in the swarm parent's configuration and the process
+      # answering this request may not share it — a standalone `wurk api` would
+      # otherwise report a slot table derived from its own CPU count, which
+      # describes nothing that is running.
+      def slots(processes)
+        grouped = processes.group_by { |process| [process.queues.to_a.sort, process['concurrency'].to_i] }
+        rows = grouped.map do |(queues, concurrency), group|
+          { count: group.size, queues: queues, concurrency: concurrency }
+        end
+        rows.sort_by { |slot| [-slot[:count], slot[:queues]] }
+      end
+
+      # The oldest beat is the one that says whether anything has gone quiet on
+      # the wire; the threshold ships with it so a client reads the verdict and
+      # the rule behind it in the same document.
+      def beat(ages)
+        { oldest_age_seconds: ages.max, stale_after_seconds: Serializers::STALE_AFTER_SECONDS }
+      end
+    end
+  end
+end

data/lib/wurk/api/router.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require 'rack'
+require_relative 'auth'
+
+module Wurk
+  module API
+    # Path table for the HTTP API. Deliberately not ActionDispatch: the API has
+    # to route in standalone mode, where Rails is never loaded (CLAUDE.md —
+    # "standalone mode must run without loading the engine").
+    #
+    # Patterns are literal segments plus `:name` captures ('/jobs/:jid').
+    # A capture never spans '/', so a queue named "a/b" has to arrive
+    # percent-encoded — the same constraint `config/routes.rb` puts on the
+    # dashboard's `:name`.
+    #
+    # Every route names the Auth scope its caller must hold. The router only
+    # records it — App#dispatch enforces it — but the keyword is required, so
+    # adding an endpoint without deciding who may call it is impossible rather
+    # than merely discouraged. A new route can never default open.
+    class Router
+      Route = Struct.new(:verb, :segments, :scope, :handler, keyword_init: true)
+      # `handler` nil means nothing matched the path; `allowed` then carries the
+      # verbs registered for a path that *did* match, so the caller can answer
+      # 405 with an Allow header instead of a misleading 404.
+      Match = Struct.new(:handler, :params, :scope, :allowed, keyword_init: true)
+
+      NO_PARAMS = {}.freeze
+
+      def initialize
+        @routes = []
+      end
+
+      def get(pattern, scope:, &handler) = add('GET', pattern, scope, handler)
+      def post(pattern, scope:, &handler) = add('POST', pattern, scope, handler)
+      def delete(pattern, scope:, &handler) = add('DELETE', pattern, scope, handler)
+
+      # HEAD is served by the GET route (RFC 9110 §9.3.2); the caller drops the
+      # body.
+      def match(verb, path)
+        verb = 'GET' if verb == 'HEAD'
+        segments = split(path).map { |seg| ::Rack::Utils.unescape_path(seg) }
+        allowed = []
+        @routes.each do |route|
+          params = bind(route.segments, segments)
+          next unless params
+          if route.verb == verb
+            return Match.new(handler: route.handler, params: params, scope: route.scope, allowed: nil)
+          end
+
+          allowed << route.verb
+        end
+        Match.new(handler: nil, params: NO_PARAMS, scope: nil, allowed: allowed.uniq)
+      end
+
+      private
+
+      # A misspelled scope would 403 the route forever; reject it at draw time,
+      # which is boot, rather than letting it surface as a runtime denial.
+      def add(verb, pattern, scope, handler)
+        unless Auth::ROUTE_SCOPES.include?(scope)
+          raise ArgumentError, "unknown route scope #{scope.inspect}; valid scopes are #{Auth::ROUTE_SCOPES.inspect}"
+        end
+
+        @routes << Route.new(verb: verb, segments: split(pattern), scope: scope, handler: handler)
+        self
+      end
+
+      def split(path) = path.to_s.split('/').reject(&:empty?)
+
+      # nil (no match) is a different answer from {} (matched, no captures) —
+      # the caller branches on it, so don't collapse them.
+      def bind(pattern, segments)
+        return nil unless pattern.size == segments.size
+
+        params = {}
+        pattern.each_with_index do |seg, index|
+          if seg.start_with?(':')
+            params[seg[1..].to_sym] = segments[index]
+          elsif seg != segments[index]
+            return nil
+          end
+        end
+        params
+      end
+    end
+  end
+end

data/lib/wurk/api/serializers.rb

@@ -0,0 +1,232 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Wurk
+  module API
+    # Wire shapes for the observe plane.
+    #
+    # Every value below is read off a canonical inspector — Stats, Queue,
+    # JobRecord, SortedEntry — so the API and the dashboard cannot report
+    # different numbers for the same Redis. Only the shape differs, and it
+    # differs on purpose: the dashboard's serializers
+    # (app/controllers/wurk/api/serializers.rb) answer to the SPA and change
+    # whenever it does, and they reach into Wurk::Web for host-registered
+    # extension rows — an engine dependency the API cannot take, because it
+    # also runs standalone. Under /v1 these field names are a contract.
+    module Serializers
+      # Three missed beats at {Wurk::Heartbeat::BEAT_PAUSE} cadence, which is
+      # also the window Wurk::Health calls a heartbeat stale, and half the
+      # heartbeat key's TTL — so a process that stopped beating is reported
+      # stale for ~30s before Redis reaps the row out from under this API.
+      STALE_AFTER_SECONDS = ::Wurk::Heartbeat::BEAT_PAUSE * 3
+
+      module_function
+
+      # Seconds since this process last beat, measured against a `now` the
+      # caller took once for the whole listing so rows never disagree by a
+      # round trip. Floored at zero: `beat` is stamped by the beating process's
+      # clock and this is read on another host's, so a skew of a few
+      # milliseconds must not surface as a negative age.
+      def beat_age_seconds(beat, now)
+        [now - beat.to_f, 0.0].max.round(3)
+      end
+
+      def stale?(age) = age > STALE_AFTER_SECONDS
+
+      # One live process. `beat_age_seconds` and `stale` are derived here
+      # rather than left to the client: a client comparing `beat` against its
+      # own clock is comparing two clocks, which is the one comparison this
+      # roll-up exists to save it from.
+      #
+      # `leader` is passed in because the two callers learn it differently — a
+      # listing compares against one memoized `dear-leader` read, a single
+      # process asks itself — and neither should pay the other's round trips.
+      def process_row(process, leader:, now:)
+        declared(process, now).merge(measured(process, now)).merge(leader: leader)
+      end
+
+      # The half a process wrote about itself in the `info` JSON: who it is and
+      # what it was configured to do. Split along the same seam the heartbeat
+      # itself uses (Heartbeat#info_hash versus #beat_hash_args), so a field
+      # that moves between the two halves upstream moves between these.
+      def declared(process, now)
+        {
+          identity: process.identity, hostname: process['hostname'], pid: process['pid'],
+          tag: process.tag, version: process.version, embedded: process.embedded?,
+          concurrency: process['concurrency'], queues: process.queues,
+          weights: process.weights, labels: process.labels,
+          started_at: process['started_at'], uptime_seconds: uptime_seconds(process['started_at'], now)
+        }
+      end
+
+      # The half its last beat measured, plus the two values derived from it.
+      def measured(process, now)
+        age = beat_age_seconds(process['beat'], now)
+        {
+          busy: process['busy'], rss_kb: process['rss'], rtt_us: process['rtt_us'],
+          beat: process['beat'], beat_age_seconds: age, stale: stale?(age),
+          quiet: process.stopping?
+        }
+      end
+
+      # One in-flight job. `class`/`args` are the display view for the reason
+      # {#job_record} gives, and `elapsed_seconds` is derived for the reason
+      # `beat_age_seconds` is — the client's clock is not the swarm's.
+      def work_row(process_id, thread_id, work, now:)
+        record = work.job
+        run_at = work.run_at.to_f
+        {
+          process_id: process_id, thread_id: thread_id, queue: work.queue,
+          jid: record.jid, class: record.display_class, args: record.display_args,
+          run_at: run_at, elapsed_seconds: [now - run_at, 0.0].max.round(3)
+        }
+      end
+
+      # `available`, not the `available?` {Wurk::Limiter::Base#build_status}
+      # keys it with: a trailing `?` is Ruby's convention for a predicate, not
+      # JSON's for a Boolean field. `status` is null when the limiter's
+      # metadata expired between the listing that named it and this read.
+      def limiter_row(name, meta, status)
+        {
+          name: name,
+          type: meta['type'].to_s,
+          fingerprint: meta['fingerprint'].to_s,
+          options: parse_options(meta['options']),
+          status: status && {
+            used: status[:used], limit: status[:limit],
+            reset_at: status[:reset_at], available: status[:available?]
+          }
+        }
+      end
+
+      # One registered cron loop. `next_fire_at` is evaluated against a `now`
+      # the caller took once, so every row in a listing answers "next after the
+      # same instant" rather than drifting a row at a time.
+      def cron_loop(loop_obj, now:)
+        {
+          lid: loop_obj.lid, schedule: loop_obj.schedule, class: loop_obj.klass,
+          queue: loop_obj.queue, args: loop_obj.args, tz: loop_obj.tz_name,
+          paused: loop_obj.paused?,
+          last_fired_at: loop_obj.last_fired_at,
+          next_fire_at: loop_obj.next_fire_at(now)
+        }
+      end
+
+      def parse_options(raw)
+        return {} if raw.nil? || raw.to_s.empty?
+
+        ::JSON.parse(raw)
+      rescue ::JSON::ParserError
+        {}
+      end
+
+      # nil rather than 0 when the heartbeat carries no `started_at`: a process
+      # of unknown age is not one that just booted.
+      def uptime_seconds(started_at, now)
+        return nil if started_at.nil?
+
+        [now - started_at.to_f, 0.0].max.round(3)
+      end
+
+      # `default_queue_latency`, not the dashboard's `latency`: sitting beside
+      # a `queues` array, a bare `latency` reads as the whole fleet's rather
+      # than the `default` queue's, which is what Stats measures.
+      def stats(snapshot)
+        {
+          processed: snapshot.processed,
+          failed: snapshot.failed,
+          expired: snapshot.expired,
+          enqueued: snapshot.enqueued,
+          busy: snapshot.workers_size,
+          scheduled: snapshot.scheduled_size,
+          retries: snapshot.retry_size,
+          dead: snapshot.dead_size,
+          processes: snapshot.processes_size,
+          default_queue_latency: snapshot.default_queue_latency,
+          queues: snapshot.queue_summaries.map { |summary| queue_summary(summary) }
+        }
+      end
+
+      def queue_summary(summary)
+        { name: summary.name, size: summary.size, latency: summary.latency, paused: summary.paused? }
+      end
+
+      # The same four gauges read off a Wurk::Queue instead of a pipelined
+      # Stats::QueueSummary — three round trips rather than a share of one,
+      # which is what asking about a single queue costs. Same field names on
+      # purpose: a client that read a queue out of the listing and then fetched
+      # it should not have to reshape anything.
+      def queue_gauges(queue)
+        { name: queue.name, size: queue.size, latency: queue.latency, paused: queue.paused? }
+      end
+
+      # `class` and `args` are the *display* view, the same one the dashboard
+      # renders: an ActiveJob wrapper is unwrapped to the job the host actually
+      # wrote, and an `encrypt: true` job's ciphertext argument is masked.
+      # Serving `record.args` here would publish the raw encrypted envelope of
+      # every such job over HTTP.
+      def job_record(record)
+        {
+          jid: record.jid,
+          class: record.display_class,
+          args: record.display_args,
+          queue: record.queue,
+          enqueued_at: record.enqueued_at&.to_f,
+          created_at: record.created_at&.to_f
+        }
+      end
+
+      # One flow and its graph. `succeeded` is emitted alongside `pending`
+      # because a client rendering progress would otherwise have to know that
+      # the two are complements of `total` — a relation this contract should
+      # not require anyone to rediscover.
+      #
+      # `nodes` is empty for an abandoned flow: the kill switch releases the
+      # node records, and inventing rows for keys that are gone would report a
+      # graph nothing can still act on.
+      def flow(status)
+        {
+          fid: status.fid, state: status.state, terminal: status.terminal?,
+          total: status.total, pending: status.pending, succeeded: status.succeeded_count,
+          depth: status.depth, width: status.width,
+          created_at: status.created_at, finished_at: status.finished_at,
+          failed_at: status.failed_at, abandoned_at: status.abandoned_at,
+          dead_nodes: status.dead_indexes,
+          nodes: status.nodes.map { |node| flow_node(node) }
+        }
+      end
+
+      # `depends_on` and `dependents` are node indexes, which is what a node is
+      # addressed by: `name` is optional, and two nodes of the same class are
+      # otherwise indistinguishable. `error` is set only on a `broken` node —
+      # one whose piped input never arrived, so no job ran to leave a failure
+      # anywhere else.
+      def flow_node(node)
+        {
+          index: node.index, name: node.name, class: node.klass, queue: node.queue,
+          jid: node.jid, bid: node.bid, state: node.state,
+          depends_on: node.dependencies, dependents: node.dependents,
+          remaining: node.remaining, piped: node.piped?, error: node.error
+        }
+      end
+
+      # `at` is the member's ZSET score in epoch seconds — when it is due to
+      # retry, due to run, or when it died, depending on which set it came out
+      # of. Emitted once: the score and the timestamp are the same number, and
+      # a contract that ships both invites clients to disagree about which is
+      # authoritative.
+      def sorted_entry(entry)
+        job_record(entry).merge(
+          at: entry.at.to_f,
+          retry_count: entry['retry_count'],
+          error_class: entry['error_class'],
+          error_message: entry['error_message'],
+          failed_at: entry.failed_at&.to_f,
+          retried_at: entry.retried_at&.to_f,
+          error_backtrace: entry.error_backtrace
+        )
+      end
+    end
+  end
+end