pgbus 0.17.0 → 0.17.2

data/lib/pgbus/web/event_state.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Web
+    # The Events page's words for one pending event (issue #494): JobState's
+    # state and reason, re-keyed into event vocabulary with the handler named
+    # in every reason, plus the one thing only events have: a pattern no
+    # running consumer subscribes to.
+    #
+    # Rows come from DataSource#event_rows (JobList SQL scoped to the handler
+    # queues) and carry :handler_class and :pattern. Pure, like JobState.
+    module EventState
+      STATES = %w[ready scheduled running retrying].freeze
+
+      # Waiting reasons a missing consumer explains better.
+      UNCOVERED = %w[waiting waiting_priority waiting_unknown lease_expired no_workers not_drained].freeze
+
+      # Handler queues have no priority sub-tables and are always drained.
+      RENAMED = {
+        "running" => "handling", "no_workers" => "no_consumers",
+        "waiting_priority" => "waiting", "not_drained" => "no_consumer_for_queue"
+      }.freeze
+
+      PROCESSED_TONES = { "completed" => :green, "handling" => :indigo, "abandoned" => :yellow }.freeze
+
+      # jobs: the JobState::Context. covered_queues: Set of physical handler
+      # queues some healthy consumer's topics overlap, or nil when unknown.
+      # claim_window: EventBus::Handler.claim_ownership_window, in seconds.
+      Context = Data.define(:jobs, :covered_queues, :claim_window)
+      Result = Data.define(:state, :reason_key, :reason_args, :next_run_at, :badge_tone, :handler_class)
+      ProcessedResult = Data.define(:state, :reason_key, :reason_args, :badge_tone)
+
+      module_function
+
+      def present(row, context, ahead: nil)
+        job = JobState.present(row, context.jobs, ahead: ahead)
+        key = event_reason_key(job.reason_key, row, context)
+        handler = row[:handler_class] || row[:queue_name]
+        args = key == "no_consumer_for_queue" ? { pattern: row[:pattern] } : job.reason_args
+        Result.new(state: job.state, reason_key: key, reason_args: args.merge(handler: handler),
+                   next_run_at: job.next_run_at, badge_tone: job.badge_tone, handler_class: row[:handler_class])
+      end
+
+      def event_reason_key(key, row, context)
+        covered = context.covered_queues
+        return "no_consumer_for_queue" if UNCOVERED.include?(key) && covered && !covered.include?(row[:queue_name])
+
+        RENAMED.fetch(key, key)
+      end
+
+      # The audit state of one pgbus_processed_events row. processed_at is the
+      # claim's last heartbeat (ClaimBeat refreshes it while the handler runs),
+      # so a pending claim older than the ownership window went silent.
+      def processed(row, now:, claim_window:)
+        processed_at = JobState.time(row["processed_at"])
+        return processed_result("completed", "completed_legacy", processed_at) unless row.key?("completed_at")
+
+        completed_at = JobState.time(row["completed_at"])
+        return processed_result("completed", "completed", completed_at) if completed_at
+        return processed_result("handling", "handling", processed_at) if processed_at && now - processed_at < claim_window
+
+        processed_result("abandoned", "abandoned", processed_at)
+      end
+
+      def processed_result(state, key, ago)
+        ProcessedResult.new(state: state, reason_key: key, reason_args: { ago: ago },
+                            badge_tone: PROCESSED_TONES.fetch(state))
+      end
+    end
+  end
+end

data/lib/pgbus/web/health_app.rb

@@ -106,10 +106,14 @@ module Pgbus
       end
 
       # Reuse an injected DataSource (tests, an app that wants one shared
-      # instance); otherwise build a fresh one each check so per-instance
-      # memoization can never serve stale queue/process metrics.
+      # instance), dropping its per-instance memos first; otherwise build a
+      # fresh one each check. Either way no check sees the last one's queue,
+      # process, stream or table metrics.
       def data_source_for_check
-        @data_source || DataSource.new
+        return DataSource.new unless @data_source
+
+        @data_source.reset_cache! if @data_source.respond_to?(:reset_cache!)
+        @data_source
       end
 
       def not_found

data/lib/pgbus/web/health_verdict.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Web
+    # Turns queue_health_stats into what the home page's Queue Health cards
+    # say (issue #492): each number's level, and the thresholds behind it, so
+    # the words can state them instead of colouring a percentage unexplained.
+    #
+    # Pure like QueueSummary: no I/O. Each Entry carries a translation key
+    # under pgbus.dashboard.queue_health.words and raw args (seconds, ratios,
+    # counts) the helper formats.
+    module HealthVerdict
+      Entry = Data.define(:level, :key, :args)
+      Result = Data.define(:dead_tuples, :bloat, :vacuum, :transaction)
+
+      DEAD_TUPLES_WARN = 10_000
+      BLOAT_WARN = 0.10
+      BLOAT_CRITICAL = 0.20
+      VACUUM_WARN_SEC = 3600
+      VACUUM_CRITICAL_SEC = 86_400
+      TXN_WARN_SEC = 30
+      TXN_CRITICAL_SEC = 60
+      # Where Pgbus::MCP::HealthAnalyzer turns the verdict DEGRADED (its MVCC
+      # check); the card's own thresholds are earlier warnings.
+      VERDICT_TXN_SEC = 300
+
+      module_function
+
+      def present(health)
+        Result.new(dead_tuples: dead_tuples(health), bloat: bloat(health), vacuum: vacuum(health),
+                   transaction: transaction(health))
+      end
+
+      def dead_tuples(health)
+        count = health[:total_dead_tuples].to_i
+        entry("dead_tuples", count > DEAD_TUPLES_WARN ? :warn : :ok, count: count, limit: DEAD_TUPLES_WARN)
+      end
+
+      def bloat(health)
+        worst = Array(health[:tables]).max_by { |t| t[:bloat_ratio].to_f }
+        return entry("bloat", :ok, :none) unless worst
+
+        ratio = health[:worst_bloat_ratio].to_f
+        entry("bloat", level(ratio, BLOAT_WARN, BLOAT_CRITICAL), ratio: ratio, table: worst[:table],
+                                                                 warn: BLOAT_WARN, critical: BLOAT_CRITICAL)
+      end
+
+      def vacuum(health)
+        age = health[:oldest_vacuum_ago_sec]
+        return entry("vacuum", :unknown) if age.nil?
+
+        entry("vacuum", level(age, VACUUM_WARN_SEC, VACUUM_CRITICAL_SEC), age: age, warn: VACUUM_WARN_SEC,
+                                                                          critical: VACUUM_CRITICAL_SEC)
+      end
+
+      def transaction(health)
+        age = health[:oldest_transaction_age_sec]
+        args = { warn: TXN_WARN_SEC, critical: TXN_CRITICAL_SEC, verdict: VERDICT_TXN_SEC }
+        return entry("transaction", :ok, :none, **args) if age.nil?
+
+        entry("transaction", level(age, TXN_WARN_SEC, TXN_CRITICAL_SEC), age: age, **args)
+      end
+
+      def level(value, warn, critical)
+        return :critical if value > critical
+        return :warn if value > warn
+
+        :ok
+      end
+
+      # The key is the metric and the level, or a named case (none) that has
+      # its own sentence.
+      def entry(metric, level, variant = level, **args)
+        Entry.new(level: level, key: "#{metric}.#{variant}", args: args)
+      end
+    end
+  end
+end

data/lib/pgbus/web/insights_charts.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Web
+    # What each Insights chart says in words, and the rows of its data table
+    # (issue #492): the accessible alternative to the SVG, rendered on the
+    # server. Pure: it takes the DataSource's chart series and does no I/O.
+    #
+    # Series are minute buckets; a 30-day window has up to 43,200 of them, so
+    # the tables roll consecutive minutes up into at most MAX_TABLE_ROWS rows
+    # (the chart itself still plots every minute).
+    module InsightsCharts
+      MAX_TABLE_ROWS = 60
+      STATUS_ORDER = %w[success failed dead_lettered].freeze
+
+      Throughput = Data.define(:total, :peak, :peak_at, :step_minutes, :rows) do
+        def empty? = rows.empty?
+      end
+
+      Status = Data.define(:total, :rows) do
+        def empty? = total.zero?
+
+        # 0 for a status with no row: grouped counts omit statuses with none.
+        def [](status)
+          row = rows.find { |r| r[:status] == status.to_s }
+          row ? row[:count] : 0
+        end
+      end
+
+      Latency = Data.define(:step_minutes, :rows) do
+        def empty? = rows.empty?
+      end
+
+      module_function
+
+      def throughput(points, minutes:)
+        series = points.filter_map { |p| (time = TimeFormat.coerce(p[:time])) && [time, p[:count].to_i] }
+        busiest = series.max_by(&:last)
+        step = step_minutes(minutes)
+        rows = bucket(series, step).map { |from, group| { from: from, count: group.sum(&:last) } }
+        Throughput.new(total: series.sum(&:last), peak: busiest&.last, peak_at: busiest&.first, step_minutes: step,
+                       rows: rows)
+      end
+
+      def status(counts)
+        rows = counts.to_h { |status, count| [status.to_s, count.to_i] }
+                     .sort_by { |status, _| [STATUS_ORDER.index(status) || STATUS_ORDER.size, status] }
+        total = rows.sum(&:last)
+        Status.new(total: total, rows: rows.map do |status, count|
+          { status: status, count: count, share: total.zero? ? 0.0 : count.to_f / total }
+        end)
+      end
+
+      # Each row: the mean of its minutes' averages and the highest minute p95
+      # (percentiles do not average).
+      def latency(points, minutes:)
+        series = points.filter_map { |p| (time = TimeFormat.coerce(p[:time])) && [time, p[:avg_ms].to_f, p[:p95_ms].to_f] }
+        step = step_minutes(minutes)
+        rows = bucket(series, step).map do |from, group|
+          { from: from, avg_ms: (group.sum { |g| g[1] } / group.size).round, p95_ms: group.map(&:last).max.round }
+        end
+        Latency.new(step_minutes: step, rows: rows)
+      end
+
+      def step_minutes(minutes)
+        [(minutes.to_f / MAX_TABLE_ROWS).ceil, 1].max
+      end
+
+      # [[bucket start, [entries]], ...] in time order; entries start with their
+      # Time. Buckets are anchored at the first entry, and the last one absorbs
+      # an entry exactly a window's length later, so a table never exceeds
+      # MAX_TABLE_ROWS rows.
+      def bucket(series, step)
+        return [] if series.empty?
+
+        seconds = step * 60
+        first = series.map(&:first).min.to_i
+        series.group_by { |entry| [(entry.first.to_i - first) / seconds, MAX_TABLE_ROWS - 1].min }
+              .sort_by(&:first)
+              .map { |index, group| [Time.at(first + (index * seconds)).utc, group] }
+      end
+    end
+  end
+end

data/lib/pgbus/web/job_state.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Pgbus
+  module Web
+    # Turns one Jobs-list row into the words an operator reads: why the job is
+    # sitting there and when it is expected to run (issue #489).
+    #
+    # The row's `state` is derived in SQL (DataSource::JobList::STATE_CASE) and
+    # taken as given; this class never re-derives it. It only adds the reason
+    # and the ETA, from facts the row and the request context already carry, so
+    # it does no I/O and is fully unit-testable.
+    #
+    # reason_args carry raw values: Integers, Strings, and Times under :time
+    # (a future moment) and :ago (a past one). The view helper formats them.
+    module JobState
+      STATES = %w[ready scheduled running retrying blocked].freeze
+      AHEAD_CAP = 10_000
+      PRIORITY_TABLE = /_p\d+\z/
+
+      BADGE_TONES = {
+        "ready" => :blue, "scheduled" => :gray, "running" => :indigo,
+        "retrying" => :yellow, "blocked" => :purple
+      }.freeze
+
+      # paused: Set of logical queue names. drained: Set of physical queue
+      # names, or nil when a wildcard capsule drains every queue.
+      # handler_queues: physical EventBus handler queues, which consumers
+      # drain; every other queue needs a live worker.
+      Context = Data.define(:now, :max_retries, :paused, :drained, :workers_alive, :handler_queues,
+                            :consumers_alive)
+      Result = Data.define(:state, :reason_key, :reason_args, :next_run_at, :badge_tone)
+
+      module_function
+
+      def present(row, context, ahead: nil)
+        state = row[:state].to_s
+        key, args, next_run_at = reason_for(state, row, context, ahead)
+        Result.new(state: state, reason_key: key, reason_args: args, next_run_at: next_run_at,
+                   badge_tone: BADGE_TONES.fetch(state, :gray))
+      end
+
+      def reason_for(state, row, context, ahead)
+        case state
+        when "scheduled"
+          vt = time(row[:vt])
+          ["scheduled", { time: vt }, vt]
+        when "running" then ["running", { ago: time(row[:last_read_at]), time: time(row[:vt]) }, nil]
+        when "retrying" then retrying_reason(row, context)
+        when "blocked" then blocked_reason(row)
+        else ready_reason(row, context, ahead)
+        end
+      end
+
+      def ready_reason(row, context, ahead)
+        overlay = ready_overlay(row, context)
+        return [overlay, {}, nil] if overlay
+        return ["lease_expired", { attempt: row[:read_ct].to_i + 1 }, nil] if row[:read_ct].to_i.positive?
+        return ["waiting_unknown", {}, nil] if ahead.nil?
+
+        key = row[:queue_name].to_s.match?(PRIORITY_TABLE) ? "waiting_priority" : "waiting"
+        [key, { count: ahead_count(ahead) }, nil]
+      end
+
+      # First match wins: a paused queue explains everything below it, a queue
+      # no capsule drains explains why live workers do not help, and only then
+      # does the absence of any healthy worker matter.
+      def ready_overlay(row, context)
+        return "paused" if context.paused.include?(row[:logical_queue])
+        return "not_drained" if context.drained && !context.drained.include?(row[:queue_name])
+
+        alive = context.handler_queues.include?(row[:queue_name]) ? context.consumers_alive : context.workers_alive
+        "no_workers" unless alive
+      end
+
+      def retrying_reason(row, context)
+        error = row[:error_class]
+        return ["orphaned", { error: error }, nil] if row[:source].to_s == "failed"
+
+        attempt = row[:read_ct].to_i
+        args = { attempt: attempt, max: context.max_retries, error: error }
+        vt = time(row[:vt])
+        return ["retrying_dlq", args, vt] if attempt >= context.max_retries
+        return ["retrying_due", args, vt] if vt.nil? || vt <= context.now
+
+        ["retrying", args.merge(time: vt), vt]
+      end
+
+      def blocked_reason(row)
+        key = row[:concurrency_key]
+        ago = time(row[:enqueued_at])
+        return ["blocked_no_slots", { key: key, ago: ago }, nil] if row[:slots_max].nil?
+
+        ["blocked", { key: key, held: row[:slots_held].to_i, limit: row[:slots_max].to_i, ago: ago }, nil]
+      end
+
+      def ahead_count(ahead)
+        ahead.to_i >= AHEAD_CAP ? "10k+" : ahead.to_i
+      end
+
+      def time(value)
+        case value
+        when nil, "" then nil
+        when String then Time.parse(value)
+        else value
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/web/process_activity.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Web
+    # What a process is doing and how close it is to a recycle (issue #493),
+    # as lists of [key, args] parts the view joins with " · ". A part shows up
+    # only when its fact was reported, so a process from before a key existed
+    # just says less. Pure, like ProcessState.
+    #
+    # args carry raw values: Integers and Floats, seconds under :interval,
+    # :uptime and :limit (lifetime), Strings for queue and topic lists.
+    module ProcessActivity
+      KIND_ORDER = %w[worker consumer dispatcher scheduler outbox_poller].freeze
+      RECYCLE_LIMITS = [%w[jobs max_jobs jobs_processed done], %w[memory max_memory_mb memory_mb used]].freeze
+
+      module_function
+
+      def activity(kind:, metadata:, children:, thresholds:)
+        case kind
+        when "worker" then worker(metadata)
+        when "consumer" then consumer(metadata)
+        when "dispatcher" then [["maintenance", { interval: thresholds.dispatch_interval }]]
+        when "scheduler"
+          [["schedule", { count: metadata["tasks"].to_i, interval: thresholds.recurring_schedule_interval }]]
+        when "supervisor" then supervisor(children)
+        when "outbox_poller" then [["outbox", {}]]
+        else []
+        end
+      end
+
+      def worker(metadata)
+        queues = list(metadata["queues"])
+        # A "*" anywhere puts the worker in wildcard mode (Worker#@wildcard).
+        drains = if queues.include?("*") then ["drains_all", {}]
+                 elsif queues.any? then ["drains", { queues: queues.join(", ") }]
+                 end
+        [drains, *pool(metadata), buffered(metadata), *rates(metadata["rates"])].compact
+      end
+
+      def rates(rates)
+        return [] unless rates.is_a?(Hash)
+
+        { "rate" => rates["processed"], "failed_rate" => rates["failed"] }
+          .filter_map { |key, rate| [key, { rate: rate.to_f }] if rate.to_f.positive? }
+      end
+
+      def consumer(metadata)
+        topics = list(metadata["topics"])
+        handled = metadata["jobs_processed"]
+        [(["consumes", { topics: topics.join(", ") }] if topics.any?), *pool(metadata), buffered(metadata),
+         (["handled", { number: handled.to_i }] if handled)].compact
+      end
+
+      # "3/5 threads busy" from the pool's capacity/busy, or "5 threads" from
+      # the registration's thread count when the pool was not reported.
+      def pool(metadata)
+        unit = metadata["execution_mode"].to_s == "async" ? "slots" : "threads"
+        capacity = metadata["capacity"]
+        return [["busy_#{unit}", { busy: metadata["busy"].to_i, capacity: capacity.to_i }]] if capacity
+
+        metadata["threads"] ? [[unit, { count: metadata["threads"].to_i }]] : []
+      end
+
+      def buffered(metadata)
+        count = metadata["buffered"].to_i
+        ["buffered", { number: count }] if count.positive?
+      end
+
+      def supervisor(children)
+        return [["supervises_none", {}]] if children.empty?
+
+        sorted = children.sort_by { |kind, _| [KIND_ORDER.index(kind) || KIND_ORDER.size, kind] }
+        [["supervises", { children: sorted }]]
+      end
+
+      # Each configured limit next to how far the process has come; nil when
+      # the process reported no limit.
+      def recycle(metadata, uptime_seconds)
+        parts = RECYCLE_LIMITS.filter_map do |key, limit_key, value_key, arg|
+          limit = metadata[limit_key]
+          [key, { arg.to_sym => metadata[value_key]&.to_i, limit: limit.to_i }] if limit
+        end
+        lifetime = metadata["max_lifetime"]
+        parts << ["lifetime", { uptime: uptime_seconds, limit: lifetime.to_i }] if lifetime
+        parts.empty? ? nil : parts
+      end
+
+      def list(value)
+        case value
+        when Array then value.map(&:to_s)
+        when String then value.split(",").map(&:strip).reject(&:empty?)
+        else []
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/web/process_state.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+require "json"
+require "time"
+
+module Pgbus
+  module Web
+    # The Processes page's words for one pgbus_processes row (issue #493):
+    # why its status is what it is and what happens next.
+    #
+    # Owns the status derivation too (DataSource#processes calls derive), so
+    # the badge and the sentence "loop last ticked 45s ago (limit 90s)" come
+    # from the same now and the same limits. present takes the row's status as
+    # given, like JobState. Pure: no I/O, no config reads.
+    #
+    # reason_args/next_args carry raw values: Times under :ago, seconds under
+    # :limit, :age and :within. The view helper formats them.
+    module ProcessState
+      # Kinds that stamp a loop beacon (loop_tick_at) on every iteration.
+      LOOP_KINDS = %w[worker consumer dispatcher scheduler].freeze
+      # Kinds the supervisor's watchdog SIGKILLs when their beacon stalls.
+      WATCHED_KINDS = %w[worker consumer].freeze
+
+      BADGE_TONES = { "healthy" => :green, "stalled" => :yellow, "stale" => :red }.freeze
+
+      # Everything derive needs, captured once per request so every row is
+      # judged against the same now.
+      Thresholds = Data.define(:now, :alive_threshold, :stall_threshold, :dispatch_interval,
+                               :recurring_schedule_interval, :polling_interval, :notify_fallback_poll,
+                               :notify_wakeup, :reap_interval)
+
+      # supervisors: { [hostname, pid] => status } for every supervisor row.
+      # children: { [hostname, supervisor_pid] => { kind => count } }.
+      # live_dispatcher: some dispatcher row is healthy (the reaper runs in it).
+      Context = Data.define(:thresholds, :supervisors, :children, :live_dispatcher)
+
+      Result = Data.define(:status, :badge_tone, :reason_key, :reason_args, :next_key, :next_args,
+                           :activity, :recycle, :started_at, :uptime_seconds, :supervisor_pid)
+
+      module_function
+
+      def derive(kind:, metadata:, heartbeat_at:, thresholds:)
+        return :stale if stale?(heartbeat_at, thresholds)
+        return :healthy unless metadata.is_a?(Hash)
+
+        limit = stall_limit_for(kind, thresholds)
+        tick = metadata["loop_tick_at"]
+        return :healthy unless limit && tick
+
+        thresholds.now.to_f - tick.to_f > limit ? :stalled : :healthy
+      end
+
+      def stale?(heartbeat_at, thresholds)
+        heartbeat = JobState.time(heartbeat_at)
+        heartbeat ? thresholds.now - heartbeat > thresholds.alive_threshold : false
+      end
+
+      # Kind-aware stall limit. Workers run a tight loop, so the base
+      # stall_threshold applies directly. The dispatcher and scheduler sleep
+      # between iterations, so their beacon is naturally up to one sleep
+      # interval old even when healthy. Consumers idle on an empty-read wait
+      # that reaches notify_fallback_poll when a live listener drives wake-up,
+      # or polling_interval otherwise. Kinds with no loop beacon: nil.
+      def stall_limit_for(kind, thresholds)
+        base = thresholds.stall_threshold
+        return nil unless base&.positive?
+
+        case kind.to_s
+        when "worker" then base
+        when "dispatcher" then base + thresholds.dispatch_interval
+        when "scheduler" then base + thresholds.recurring_schedule_interval
+        when "consumer" then base + consumer_slack(thresholds)
+        end
+      end
+
+      def consumer_slack(thresholds)
+        return thresholds.polling_interval unless thresholds.notify_wakeup
+
+        [thresholds.polling_interval, thresholds.notify_fallback_poll].max
+      end
+
+      # The cross-row facts, from the already-derived rows; shared by
+      # DataSource and the test stub.
+      def context_for(rows, thresholds)
+        supervisors = rows.select { |r| r[:kind].to_s == "supervisor" }
+                          .to_h { |r| [[r[:hostname], r[:pid].to_i], status_of(r)] }
+        children = Hash.new { |h, k| h[k] = Hash.new(0) }
+        rows.each do |r|
+          parent = parse_metadata(r[:metadata])["supervisor_pid"]
+          children[[r[:hostname], parent.to_i]][r[:kind].to_s] += 1 if parent && r[:kind].to_s != "supervisor"
+        end
+        live = rows.any? { |r| r[:kind].to_s == "dispatcher" && status_of(r) == :healthy }
+        Context.new(thresholds: thresholds, supervisors: supervisors,
+                    children: children.transform_values(&:to_h), live_dispatcher: live)
+      end
+
+      # Host by host: each supervisor followed by its children, then the
+      # standalone rows, each group in the order given.
+      def order(rows, context)
+        rows.group_by { |r| r[:hostname].to_s }.sort_by(&:first).flat_map do |_host, host_rows|
+          supervisors, others = host_rows.partition { |r| r[:kind].to_s == "supervisor" }
+          children = others.group_by { |r| supervisor_pid(r, parse_metadata(r[:metadata]), context) }
+          supervisors.flat_map { |s| [s, *children.delete(s[:pid].to_i)] } + children.values.flatten
+        end
+      end
+
+      def present(row, context)
+        thresholds = context.thresholds
+        metadata = parse_metadata(row[:metadata])
+        status = status_of(row).to_s
+        reason_key, reason_args = reason_for(status, row, metadata, thresholds)
+        next_key, next_args = next_for(status, row, metadata, context)
+        started_at = JobState.time(metadata["started_at"] && Time.at(metadata["started_at"].to_f))
+        uptime = started_at && (thresholds.now - started_at).round
+        Result.new(
+          status: status, badge_tone: BADGE_TONES.fetch(status, :red),
+          reason_key: reason_key, reason_args: reason_args, next_key: next_key, next_args: next_args,
+          activity: ProcessActivity.activity(kind: row[:kind].to_s, metadata: metadata,
+                                             children: context.children.fetch([row[:hostname], row[:pid].to_i], {}),
+                                             thresholds: thresholds),
+          recycle: ProcessActivity.recycle(metadata, uptime),
+          started_at: started_at, uptime_seconds: uptime, supervisor_pid: supervisor_pid(row, metadata, context)
+        )
+      end
+
+      def reason_for(status, row, metadata, thresholds)
+        heartbeat = JobState.time(row[:last_heartbeat_at])
+        alive = thresholds.alive_threshold
+        limit = stall_limit_for(row[:kind], thresholds)
+        tick = metadata["loop_tick_at"] && Time.at(metadata["loop_tick_at"].to_f)
+        case status
+        when "stale" then ["stale", { age: heartbeat && (thresholds.now - heartbeat).round, limit: alive }]
+        when "stalled" then ["stalled", { ago: tick, limit: limit }]
+        else healthy_reason(row[:kind].to_s, heartbeat, tick, limit, alive)
+        end
+      end
+
+      def healthy_reason(kind, heartbeat, tick, limit, alive)
+        return ["healthy_loop", { ago: tick, limit: limit }] if limit && tick
+
+        key = LOOP_KINDS.include?(kind) && limit.nil? ? "healthy_detection_off" : "healthy_heartbeat"
+        [key, { ago: heartbeat, limit: alive }]
+      end
+
+      def next_for(status, row, metadata, context)
+        case status
+        when "stale"
+          context.live_dispatcher ? ["stale_reaped", { within: context.thresholds.reap_interval }] : ["stale_unreaped", {}]
+        when "stalled" then [stalled_next(row, metadata, context), {}]
+        else [nil, {}]
+        end
+      end
+
+      def stalled_next(row, metadata, context)
+        kind = row[:kind].to_s
+        return { "dispatcher" => "stalled_maintenance", "scheduler" => "stalled_schedule" }[kind] unless WATCHED_KINDS.include?(kind)
+
+        parent = supervisor_pid(row, metadata, context)
+        parent && context.supervisors[[row[:hostname], parent]] == :healthy ? "stalled_supervised" : "stalled_standalone"
+      end
+
+      # The supervisor's pid, only when that supervisor's row is on the same
+      # host: a standalone process's parent is a shell, foreman or systemd.
+      def supervisor_pid(row, metadata, context)
+        parent = metadata["supervisor_pid"]&.to_i
+        parent if parent && context.supervisors.key?([row[:hostname], parent])
+      end
+
+      def status_of(row)
+        status = row[:status]
+        return status.to_sym if status
+
+        row[:healthy] ? :healthy : :stale
+      end
+
+      def parse_metadata(metadata)
+        case metadata
+        when Hash then metadata
+        when String then JSON.parse(metadata).then { |h| h.is_a?(Hash) ? h : {} }
+        else {}
+        end
+      rescue JSON::ParserError
+        {}
+      end
+    end
+  end
+end