@humwick/runner 0.0.0-stage → 0.130.0

package/node_modules/@humwick/offload/src/subprocess-runner.ts

@@ -0,0 +1,334 @@
+import PQueue from "p-queue";
+import { beginRun, finishLog, recordResult, startLog, type RunContext } from "./envelope.js";
+import { OffloadError, StrikeCounter, normalizeError, validateTimeout, type OffloadDefaults, type OffloadEnvelope } from "./types.js";
+
+export interface SubprocessSpec {
+  command: string[];
+  cwd?: string;
+  env?: Record<string, string | undefined>;
+  /**
+   * Exit codes that are a normal ANSWER rather than a failure. Defaults to `[0]`; pass
+   * `"any"` when every code is data (a land gate whose failing codes are not enumerable).
+   *
+   * Probe-shaped callers use a non-zero exit as data — `tmux has-session` exits 1 when the
+   * session is absent, `git rev-parse` exits 128 outside a repo, `which` exits 1. Without
+   * this they reject AND take an escalation strike for a healthy result, which inverts the
+   * signal: the sink pages on a healthy fleet, then latches (it only re-arms on a success,
+   * which never comes in that steady state) so the real outage is silent.
+   *
+   * `"any"` includes SIGNAL DEATHS — a child killed by SIGKILL (137) or SIGSEGV (139)
+   * returns as a success and RE-ARMS the strike counter. That is intended for a land gate,
+   * where an OOM-killed gate is simply a failed gate, but it means an infrastructure death
+   * reads as a healthy answer and will never escalate. Prefer an explicit code list unless
+   * the failing codes genuinely cannot be enumerated. Timeout and abort still throw.
+   */
+  okExitCodes?: readonly number[] | "any";
+}
+
+export interface SubprocessResult {
+  exitCode: number;
+  stdout: string;
+  stderr: string;
+  /** True when either stream hit `maxOutputBytes` and the captured text was cut short. */
+  truncated: boolean;
+}
+
+export interface SubprocessRunnerOptions extends OffloadDefaults {
+  concurrency?: number;
+  /**
+   * How long a child may keep running after SIGTERM before it is SIGKILLed. This is the
+   * slack on the deadline: a run cannot outlive `timeoutMs + killGraceMs` (plus a short
+   * reap window) even if it traps or ignores SIGTERM. Defaults to 2000ms.
+   */
+  killGraceMs?: number;
+  /** Captured bytes per stream before output is truncated. Defaults to 8 MiB. */
+  maxOutputBytes?: number;
+}
+
+export interface SubprocessRunner {
+  run(taskName: string, process: SubprocessSpec, env?: OffloadEnvelope): Promise<SubprocessResult>;
+  readonly stats: { active: number; queued: number };
+  /** A subprocess owns the heavy work, so maintenance callers may mark their await window off-thread. */
+  readonly offThread: true;
+  /** Resolves when everything accepted so far has finished. Does not cancel anything. */
+  drain(): Promise<void>;
+  /** SIGKILLs and awaits every live child, settles every outstanding caller, and refuses new work. */
+  destroy(): Promise<void>;
+}
+
+type Child = ReturnType<typeof Bun.spawn>;
+
+const DEFAULT_KILL_GRACE_MS = 2_000;
+const DEFAULT_MAX_OUTPUT_BYTES = 8 * 1024 * 1024;
+/** How long to wait for a SIGKILLed child to be reaped before giving its slot back anyway. */
+const REAP_GRACE_MS = 1_000;
+
+/**
+ * `detached` children are reparented rather than reaped when the host dies (#1809 P1-5), so
+ * every live group is killed from a single process-level exit hook. One module-scoped hook
+ * keeps runner instances from piling up `exit` listeners.
+ */
+const liveChildren = new Set<Child>();
+let exitHookInstalled = false;
+
+function killGroup(child: Child, signal: NodeJS.Signals): void {
+  try {
+    if (process.platform !== "win32") process.kill(-child.pid, signal);
+    else child.kill();
+  } catch {
+    try { child.kill(signal); } catch { /* the child is already gone */ }
+  }
+}
+
+function trackChild(child: Child): void {
+  liveChildren.add(child);
+  if (exitHookInstalled) return;
+  exitHookInstalled = true;
+  process.on("exit", () => {
+    for (const live of liveChildren) killGroup(live, "SIGKILL");
+  });
+}
+
+/** Reads a stream up to `limit` bytes, then keeps draining so the child never blocks on a full pipe. */
+async function readCapped(stream: ReadableStream<Uint8Array> | null, limit: number): Promise<{ text: string; truncated: boolean }> {
+  if (!stream) return { text: "", truncated: false };
+  const reader = stream.getReader();
+  const chunks: Uint8Array[] = [];
+  let size = 0;
+  let truncated = false;
+  try {
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      if (!value) continue;
+      const remaining = limit - size;
+      if (remaining <= 0) {
+        truncated = true;
+        continue;
+      }
+      if (value.byteLength > remaining) {
+        chunks.push(value.subarray(0, remaining));
+        size = limit;
+        truncated = true;
+        continue;
+      }
+      chunks.push(value);
+      size += value.byteLength;
+    }
+  } finally {
+    reader.releaseLock();
+  }
+  const buffer = new Uint8Array(size);
+  let offset = 0;
+  for (const chunk of chunks) {
+    buffer.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return { text: new TextDecoder().decode(buffer), truncated };
+}
+
+type Outcome = { ok: true; value: SubprocessResult } | { ok: false; error: OffloadError };
+
+type LiveRun = {
+  taskName: string;
+  teardown: AbortController;
+  settled: Promise<void>;
+  /** Settles a caller whose child has not been spawned yet; a no-op once it has. */
+  cancelIfQueued(error: OffloadError): void;
+};
+
+/**
+ * Bounded wrapper around the repo's standard `Bun.spawn(... { stdout: "pipe", stderr: "pipe" })`
+ * pattern. On POSIX, each run gets its own process group so timeout/abort also kills
+ * descendants (for example shell, git, or install subprocess trees); Windows falls back
+ * to Bun's direct-child kill because POSIX process groups do not exist there.
+ */
+export function createSubprocessRunner(options: SubprocessRunnerOptions = {}): SubprocessRunner {
+  const queue = new PQueue({ concurrency: options.concurrency ?? 1 });
+  const counter = new StrikeCounter(options.escalationThreshold, options.escalationWindowAttempts);
+  const killGraceMs = options.killGraceMs ?? DEFAULT_KILL_GRACE_MS;
+  const maxOutputBytes = options.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
+  if (!Number.isFinite(killGraceMs) || killGraceMs < 0) throw new RangeError("killGraceMs must be a non-negative finite number");
+  if (!Number.isSafeInteger(maxOutputBytes) || maxOutputBytes < 1) throw new RangeError("maxOutputBytes must be a positive integer");
+  const live = new Set<LiveRun>();
+  const children = new Set<Child>();
+  let destroyed = false;
+
+  const abortedError = (taskName: string, cause?: unknown): OffloadError =>
+    new OffloadError({ kind: "aborted", taskName, message: `${taskName} was aborted`, cause });
+  const destroyedError = (taskName: string): OffloadError =>
+    new OffloadError({ kind: "aborted", taskName, message: `${taskName} was cancelled: subprocess runner destroyed` });
+
+  return {
+    offThread: true,
+    get stats() { return { active: queue.pending, queued: queue.size }; },
+    async run(taskName: string, spec: SubprocessSpec, env?: OffloadEnvelope): Promise<SubprocessResult> {
+      const labels = env?.labels ?? {};
+      const logger = env?.logger ?? options.logger;
+      const sink = env?.onEscalate ?? options.onEscalate;
+      const reportRejection = async (error: OffloadError): Promise<never> => {
+        // Rejecting before the run reaches startLog is still a terminal outcome; without this
+        // the structured stream shows nothing at all for it.
+        finishLog(logger, "subprocess", taskName, labels, performance.now(), error);
+        await recordResult({ counter, taskName, backend: "subprocess", labels, error, sink, logger });
+        throw error;
+      };
+      if (!spec.command.length) return reportRejection(new OffloadError({ kind: "rejected", taskName, message: "Subprocess command must not be empty" }));
+      if (destroyed) return reportRejection(destroyedError(taskName));
+      if (env?.signal?.aborted) return reportRejection(abortedError(taskName, env.signal.reason));
+      try {
+        validateTimeout(env?.timeoutMs ?? options.defaultTimeoutMs);
+      } catch (error) {
+        return reportRejection(normalizeError(taskName, error, "rejected"));
+      }
+
+      let publish!: (outcome: Outcome) => void;
+      let published = false;
+      const outcome = new Promise<Outcome>((resolve) => {
+        publish = (value) => { if (!published) { published = true; resolve(value); } };
+      });
+      let started = false;
+      let cancelled = false;
+      const entry: LiveRun = {
+        taskName,
+        teardown: new AbortController(),
+        settled: outcome.then(() => undefined),
+        cancelIfQueued: (error) => {
+          if (started || cancelled) return;
+          cancelled = true;
+          // The single funnel for both abort-while-queued and destroy()-cancels-queued, so
+          // the one line each terminal path owes the structured stream is emitted here. The
+          // callback bails on `cancelled` without logging, which keeps it to exactly one.
+          finishLog(logger, "subprocess", taskName, labels, performance.now(), error);
+          publish({ ok: false, error });
+        },
+      };
+      live.add(entry);
+
+      // p-queue's own `signal` option used to provide prompt abort for queued runs, but it
+      // also released the slot while the child kept running (#1809 P1-7).
+      const signal = env?.signal;
+      const onQueuedAbort = signal ? () => entry.cancelIfQueued(abortedError(taskName, signal.reason)) : undefined;
+      if (signal && onQueuedAbort) signal.addEventListener("abort", onQueuedAbort, { once: true });
+
+      const task = queue.add(async () => {
+        if (signal && onQueuedAbort) signal.removeEventListener("abort", onQueuedAbort);
+        let context: RunContext | undefined;
+        let child: Child | undefined;
+        let onAbort: (() => void) | undefined;
+        let killTimer: ReturnType<typeof setTimeout> | undefined;
+        let abandonTimer: ReturnType<typeof setTimeout> | undefined;
+        let reaped = false;
+        let abandon!: () => void;
+        const abandoned = new Promise<"abandoned">((resolve) => { abandon = () => resolve("abandoned"); });
+        try {
+          if (cancelled) return;
+          if (destroyed || entry.teardown.signal.aborted) {
+            const error = destroyedError(taskName);
+            finishLog(logger, "subprocess", taskName, labels, performance.now(), error);
+            publish({ ok: false, error });
+            return;
+          }
+          started = true;
+          context = beginRun(taskName, env, options, entry.teardown.signal);
+          const run = context;
+          const startedAt = startLog(logger, "subprocess", taskName, labels);
+          // SIGTERM alone bounds nothing: a child that traps or ignores it runs to completion
+          // while holding the slot (#1809 P0-3). Escalate to a group SIGKILL, then stop
+          // waiting altogether so the deadline is real even for an unkillable child.
+          onAbort = () => {
+            if (!child || reaped) return;
+            killGroup(child, "SIGTERM");
+            killTimer = setTimeout(() => {
+              // `reaped` keeps a late timer from signalling a process group whose pid the
+              // kernel may already have handed to somebody else.
+              if (!child || reaped) return;
+              killGroup(child, "SIGKILL");
+              abandonTimer = setTimeout(abandon, REAP_GRACE_MS);
+            }, killGraceMs);
+          };
+          run.signal.addEventListener("abort", onAbort, { once: true });
+          try {
+            if (run.signal.aborted) throw run.abortError();
+            child = Bun.spawn(spec.command, {
+              cwd: spec.cwd,
+              env: spec.env ? { ...process.env, ...spec.env } : process.env,
+              stdin: "ignore", stdout: "pipe", stderr: "pipe",
+              detached: process.platform !== "win32",
+            });
+            trackChild(child);
+            children.add(child);
+            const exiting = child;
+            void exiting.exited.then(() => {
+              reaped = true;
+              liveChildren.delete(exiting);
+              children.delete(exiting);
+            });
+            const completed = Promise.all([
+              readCapped(child.stdout as ReadableStream<Uint8Array> | null, maxOutputBytes),
+              readCapped(child.stderr as ReadableStream<Uint8Array> | null, maxOutputBytes),
+              child.exited,
+            ]);
+            const settledRun = await Promise.race([completed, abandoned]);
+            if (settledRun === "abandoned") {
+              throw run.signal.aborted
+                ? run.abortError()
+                : new OffloadError({ kind: "timeout", taskName, message: `${taskName} did not exit after SIGKILL` });
+            }
+            const [stdout, stderr, exitCode] = settledRun;
+            if (run.signal.aborted) throw run.abortError();
+            const okExitCodes = spec.okExitCodes ?? [0];
+            if (okExitCodes !== "any" && !okExitCodes.includes(exitCode)) throw new OffloadError({
+              kind: "rejected", taskName, code: String(exitCode), message: `${taskName} exited ${exitCode}: ${stderr.text.trim() || stdout.text.trim()}`,
+            });
+            const result: SubprocessResult = { exitCode, stdout: stdout.text, stderr: stderr.text, truncated: stdout.truncated || stderr.truncated };
+            finishLog(logger, "subprocess", taskName, labels, startedAt);
+            await recordResult({ counter, taskName, backend: "subprocess", labels, sink, logger });
+            publish({ ok: true, value: result });
+          } catch (error) {
+            const normalized = run.signal.aborted ? run.abortError() : normalizeError(taskName, error);
+            finishLog(logger, "subprocess", taskName, labels, startedAt, normalized);
+            await recordResult({ counter, taskName, backend: "subprocess", labels, error: normalized, sink, logger });
+            publish({ ok: false, error: normalized });
+          } finally {
+            if (onAbort) run.signal.removeEventListener("abort", onAbort);
+            run.cleanup();
+          }
+        } catch (error) {
+          const normalized = normalizeError(taskName, error, "rejected");
+          finishLog(logger, "subprocess", taskName, labels, performance.now(), normalized);
+          await recordResult({ counter, taskName, backend: "subprocess", labels, error: normalized, sink, logger });
+          publish({ ok: false, error: normalized });
+        } finally {
+          context?.cleanup();
+          // Keep the slot until the child is actually reaped, so the concurrency cap and
+          // `stats` stay true while a kill is in flight (#1809 P1-7). `abandoned` bounds
+          // this: the SIGKILL escalation above always ends the wait.
+          if (child) await Promise.race([child.exited, abandoned]);
+          if (killTimer) clearTimeout(killTimer);
+          if (abandonTimer) clearTimeout(abandonTimer);
+          live.delete(entry);
+        }
+      });
+      void task.catch(() => undefined);
+
+      const result = await outcome;
+      if (!result.ok) throw result.error;
+      return result.value;
+    },
+    drain: () => queue.onIdle(),
+    async destroy(): Promise<void> {
+      destroyed = true;
+      const outstanding = [...live];
+      for (const entry of outstanding) {
+        entry.cancelIfQueued(destroyedError(entry.taskName));
+        entry.teardown.abort(new DOMException("Subprocess runner destroyed", "AbortError"));
+      }
+      for (const child of children) killGroup(child, "SIGKILL");
+      await Promise.all([
+        ...outstanding.map((entry) => entry.settled),
+        ...[...children].map((child) => child.exited),
+      ]);
+    },
+  };
+}

package/node_modules/@humwick/offload/src/types.ts

@@ -0,0 +1,235 @@
+import { errMessage } from "@humwick/sdk";
+
+export type OffloadErrorKind = "timeout" | "aborted" | "worker-error" | "worker-exit" | "rejected";
+
+/** A stable, backend-independent failure surface for callers and escalation sinks. */
+export class OffloadError extends Error {
+  readonly kind: OffloadErrorKind;
+  readonly code?: string;
+  readonly taskName: string;
+
+  constructor(options: {
+    kind: OffloadErrorKind;
+    taskName: string;
+    code?: string;
+    message?: string;
+    cause?: unknown;
+  }) {
+    super(options.message ?? `${options.taskName} ${options.kind}`, { cause: options.cause });
+    this.name = "OffloadError";
+    this.kind = options.kind;
+    this.code = options.code;
+    this.taskName = options.taskName;
+  }
+}
+
+export type OffloadLogFields = Record<string, unknown>;
+
+/** Deliberately structural: server, runner and orchestrator can pass their existing logger adapters. */
+export interface OffloadLogger {
+  debug?(message: string, fields?: OffloadLogFields): void;
+  info?(message: string, fields?: OffloadLogFields): void;
+  warn?(message: string, fields?: OffloadLogFields): void;
+  error?(message: string, fields?: OffloadLogFields): void;
+}
+
+export interface EscalationEvent {
+  taskName: string;
+  backend: "worker" | "async" | "subprocess";
+  error: OffloadError;
+  strikes: number;
+  labels: Record<string, string>;
+}
+
+export type EscalationSink = (event: EscalationEvent) => void | Promise<void>;
+
+/** The common per-run contract. No backend-specific options leak into this envelope. */
+export interface OffloadEnvelope {
+  timeoutMs?: number;
+  signal?: AbortSignal;
+  logger?: OffloadLogger;
+  labels?: Record<string, string>;
+  onEscalate?: EscalationSink;
+  /**
+   * Opt in to automatic re-dispatch after an ambiguous worker death (WorkerPool only; the async
+   * and subprocess runners never re-run). Default is AT-MOST-ONCE: if the worker dies with this
+   * job in flight, the job is rejected (`worker-exit`) and never re-run — because a re-run would
+   * repeat any side effect the job already performed (an HTTP POST, a mutation). Set this ONLY
+   * when the job is idempotent/retry-safe (pure compute), in which case a worker death is
+   * absorbed by re-dispatching to a fresh worker up to a bounded number of times (#1822 HIGH-1).
+   * Command/handler offload (side-effecting) must leave this unset.
+   */
+  retrySafe?: boolean;
+}
+
+export interface OffloadDefaults {
+  defaultTimeoutMs?: number;
+  logger?: OffloadLogger;
+  onEscalate?: EscalationSink;
+  /** Consecutive failed runs for a task before its injected sink fires. Defaults to 3. */
+  escalationThreshold?: number;
+  /**
+   * Size, in failure-attempts, of the sliding window the strike counter escalates over: a task
+   * escalates when it accumulates `escalationThreshold` failures within this many consecutive
+   * failure-attempts (counted across ALL tasks). A task with no failure inside the window is
+   * forgotten, which is ALSO what bounds memory — the counter holds at most this many failure
+   * events, so at most this many distinct names, at once (#1822 items 1 and 4). Adoption keys
+   * runs by dynamic names (workspace/agent ids); the window is what keeps that from growing
+   * without bound. Defaults to 4096.
+   */
+  escalationWindowAttempts?: number;
+}
+
+/**
+ * Sliding-window failure counter driving edge-triggered escalation (#1822 F1/F4, round-3
+ * redesign). A task escalates when it accumulates `threshold` failures within the last
+ * `window` failure-attempts (counted across all tasks); a failure older than the window no
+ * longer counts. This is attempt-based — no wall-clock, no timers — so it is deterministic and
+ * needs no time injection to test.
+ *
+ * WHY windowed rather than a size-capped consecutive counter: any hard *count* cap on a
+ * consecutive-strike map either scans the map per failure (O(cap); an earlier attempt blocked
+ * the event loop ~11s at 40k / ~31s at 100k distinct failures) or silently drops a task that is
+ * genuinely accumulating (a missed page — strictly worse than the leak it replaced). The window
+ * gives all four invariants at once: no lost escalation, no page storm, amortized O(1) per
+ * failure, and a hard memory bound.
+ *
+ * IMPLEMENTATION — the standard sliding-window multiset over a ring of failure events:
+ *  - `#ring[(clock-1) % window]` holds the failure event `window` attempts ago. Each new failure
+ *    overwrites exactly that slot, and the event it evicts is exactly the one aging out — so a
+ *    failure is pushed once and expired once: amortized O(1), never a scan.
+ *  - `#live[name]` is the name's failure count still inside the window = its escalation score.
+ *  - Memory is hard-bounded: the ring holds ≤ `window` events, hence ≤ `window` distinct names.
+ *  - `succeed()` bumps a per-name `#epoch`; events carry the epoch they were recorded under, so a
+ *    success invalidates the name's in-window failures (score resets) in O(1) without scanning
+ *    the ring — the stale events simply don't count as they age out.
+ *  - Latch (`trip`): a per-name flag so an escalation fires ONCE and does not re-fire while the
+ *    task stays continuously failing; it clears when the task succeeds or every failure ages out.
+ *
+ * SEMANTIC NOTE (deliberate change from the first #1822 attempt): escalation is now "threshold
+ * failures WITHIN the window", not "threshold consecutive failures ever". Failures spread wider
+ * apart than the window decay and never trip — which is the actual flap-detection semantic.
+ */
+export class StrikeCounter {
+  #ring: Array<{ name: string; epoch: number } | undefined>;
+  /** Failures still inside the window under the name's CURRENT epoch = its escalation score. */
+  #live = new Map<string, number>();
+  /** Total ring occurrences per name across all epochs — for O(1) cleanup and the memory bound. */
+  #present = new Map<string, number>();
+  /** Per-name epoch, bumped by succeed() so pre-success failures stop counting. */
+  #epoch = new Map<string, number>();
+  /** Names that have escalated and not yet re-armed (edge-trigger latch). */
+  #latched = new Set<string>();
+  #clock = 0;
+  readonly threshold: number;
+  readonly window: number;
+
+  constructor(threshold = 3, escalationWindowAttempts = 4096) {
+    if (!Number.isSafeInteger(threshold) || threshold < 1) throw new RangeError("escalationThreshold must be a positive integer");
+    if (!Number.isSafeInteger(escalationWindowAttempts) || escalationWindowAttempts < 1) throw new RangeError("escalationWindowAttempts must be a positive integer");
+    // A window smaller than the threshold can never hold `threshold` of one name's failures at
+    // once (the ring evicts the oldest before the count gets there), so it would score 1,2,…,
+    // threshold-1 forever and SILENTLY never escalate. Reject it rather than mislead (#1822 LOW-4).
+    if (escalationWindowAttempts < threshold) throw new RangeError("escalationWindowAttempts must be >= escalationThreshold");
+    this.threshold = threshold;
+    this.window = escalationWindowAttempts;
+    this.#ring = new Array(escalationWindowAttempts).fill(undefined);
+  }
+
+  /** Distinct task names with a failure still inside the window. Bounded by `window`. */
+  get size(): number {
+    return this.#present.size;
+  }
+
+  // Removes the event in `slot` (the one aging out of the window) and updates the name's counts.
+  #expire(slot: number): void {
+    const event = this.#ring[slot];
+    if (event === undefined) return;
+    this.#ring[slot] = undefined;
+    const { name, epoch } = event;
+    // Only decrement the score if the event is still under the name's live epoch; an event from
+    // before a succeed() carries a stale epoch and was never counted toward the current score.
+    if (epoch === (this.#epoch.get(name) ?? 0)) {
+      const live = (this.#live.get(name) ?? 0) - 1;
+      if (live <= 0) { this.#live.delete(name); this.#latched.delete(name); }
+      else this.#live.set(name, live);
+    }
+    const present = (this.#present.get(name) ?? 0) - 1;
+    if (present <= 0) {
+      // Every trace of the name has aged out of the window: forget it entirely (this is the
+      // natural memory bound, and it re-arms a latch that went quiet).
+      this.#present.delete(name);
+      this.#epoch.delete(name);
+      this.#live.delete(name);
+      this.#latched.delete(name);
+    } else {
+      this.#present.set(name, present);
+    }
+  }
+
+  fail(taskName: string): number {
+    this.#clock += 1;
+    const slot = (this.#clock - 1) % this.window;
+    // The event this slot holds is exactly `window` attempts old — evict it before reusing it.
+    this.#expire(slot);
+    const epoch = this.#epoch.get(taskName) ?? 0;
+    this.#ring[slot] = { name: taskName, epoch };
+    this.#present.set(taskName, (this.#present.get(taskName) ?? 0) + 1);
+    const score = (this.#live.get(taskName) ?? 0) + 1;
+    this.#live.set(taskName, score);
+    // A latched name must not re-page while it stays continuously live.
+    if (this.#latched.has(taskName)) return 0;
+    return score;
+  }
+
+  /** Latch an escalation so it fires once until the task re-arms (succeeds or ages out). */
+  trip(taskName: string): void {
+    // Latch ONLY a name that has a failure in the window. Otherwise a caller (or a bug) tripping
+    // arbitrary names would grow #latched without any ring presence to age it out — breaking the
+    // "memory bounded BY the window" invariant, since `size` counts only #present (#1822 MED-3).
+    // In practice trip() is always called right after fail() returned >= threshold, so the name
+    // is present; this guard just makes the bound hold unconditionally.
+    if (this.#present.has(taskName)) this.#latched.add(taskName);
+  }
+
+  succeed(taskName: string): void {
+    // Nothing tracked → nothing to reset (and never create an epoch entry for a name that only
+    // ever succeeds, which would leak one map entry per distinct successful task name).
+    if (!this.#present.has(taskName)) return;
+    // Bump the epoch: every in-window failure recorded under the old epoch stops counting, so
+    // the score resets to 0 in O(1). The stale ring events are reclaimed as they age out.
+    this.#epoch.set(taskName, (this.#epoch.get(taskName) ?? 0) + 1);
+    this.#live.delete(taskName);
+    this.#latched.delete(taskName);
+  }
+}
+
+export function normalizeError(taskName: string, error: unknown, fallback: OffloadErrorKind = "rejected"): OffloadError {
+  if (error instanceof OffloadError) return error;
+  const detail = error as { code?: unknown; message?: unknown } | undefined;
+  return new OffloadError({
+    kind: fallback,
+    taskName,
+    code: typeof detail?.code === "string" ? detail.code : undefined,
+    message: errMessage(error),
+    cause: error,
+  });
+}
+
+export function log(logger: OffloadLogger | undefined, level: keyof OffloadLogger, message: string, fields: OffloadLogFields): void {
+  const method = logger?.[level];
+  if (typeof method !== "function") return;
+  try {
+    method.call(logger, message, fields);
+  } catch {
+    // A caller-supplied transport that throws must never break the run it is describing:
+    // logging happens on the settle path, so an escaping throw wedges the backend (#1809 P0-1).
+    // There is no second transport to report this to, so it is deliberately swallowed.
+  }
+}
+
+export function validateTimeout(timeoutMs: number | undefined): number | undefined {
+  if (timeoutMs === undefined) return undefined;
+  if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) throw new RangeError("timeoutMs must be a positive finite number");
+  return timeoutMs;
+}