@humwick/server 0.0.0-stage → 0.130.1

package/src/services/agent-hang-watchdog.ts

@@ -0,0 +1,395 @@
+// #2035 (leaf L4 of #2031) — the agent-hang-watchdog DECISION CORE.
+//
+// THE HOLE THIS FILLS. An agent that launches a never-finishing `run_in_background` job ends its
+// provider turn without a final report and is then invisible to every existing watchdog:
+//   - `runStuckBusyWatchdog` → `deriveAgentBusyHealth` requires `!hasLiveWork`, but a
+//     `background-script` record sets `hasLiveWork = true`, so it can never fire for a
+//     background-script-only busy;
+//   - `auditProviderTurnWatchdog` needs a live provider-turn timeline event → never fires either;
+//   - after 10 min #1856's `isStaleBackgroundScriptOnlyBusy` force-clears the record and the card
+//     goes idle — healing busy→idle WITHOUT healing the missing report, so the last surface that
+//     hinted at trouble goes green.
+// Result: an idle agent that owes a report, watched by nothing — the observed silent team stall.
+//
+// THIS MODULE is PURE (no I/O, no wiring — that is #2036). Given a stall snapshot it returns a typed
+// decision: fire a self-check rung, escalate to the coordinator, or do nothing. It is a sibling of
+// `runner/src/background-script-reconcile.ts` in shape: primitives in, verdict out, unit-testable
+// without standing up a server.
+//
+// FOUR TRAPS IT IS BUILT AROUND (each is a real regression this must not reintroduce):
+//   1. The stall signal is the TEAM-STALL condition — an unmet OBLIGATION plus no progress — never
+//      "watch the background script directly". The script may run forever legitimately; what is
+//      wrong is the missing report / unanswered reply / stuck Task.
+//   2. Rungs key on the STALL EPISODE, not on `lastProgressAt`. A self-check that WORKS advances
+//      `lastProgressAt` (the agent woke, looked, went back to waiting); keying the ladder on last
+//      progress would drop to R1 every time and nag politely forever, never escalating. The episode
+//      ends — and only then does the rung reset — when the OBLIGATION resolves. This is the same
+//      defect family as #2010: an event that is the watchdog's OWN footprint must never be read as
+//      evidence the watched thing is healthy.
+//   3. It must not reintroduce #1980's false-cancel of slow-but-alive turns. Positive busy evidence
+//      (a live pid, recent genuine progress, an `active` phase, a dead transport that is the
+//      offline reaper's problem) SUPPRESSES the rung. Suppression fires ONLY on AFFIRMATIVE
+//      evidence, never on its absence (the #1069/#1980/#2010 discipline); a derivation error fails
+//      OPEN to notifying, exactly as `auditProviderTurnWatchdog` does.
+//   4. INVARIANT H1 — this module may NEVER clear a provider-turn claim. Its entire authority is to
+//      RETURN a decision. Only the runner's positive-evidence path (`BusyReconciler` →
+//      `clearProviderTurn`) may end a turn. An import-graph test pins that this module cannot reach
+//      any turn-clear API; that is what keeps #1980's false-cancel from returning.
+
+import { STUCK_BUSY_TRANSPORT_STALE_MS } from "../config";
+import type { AgentBusyHealth } from "../agent-busy-health";
+import type { AgentKind, AgentStatus } from "../types";
+
+/** R1 fires at this episode age. 10 min matches `STALE_BACKGROUND_SCRIPT_BUSY_MS`, so R1 lands
+ *  exactly when #1856 would otherwise green the card. Also the "recent genuine progress" grace
+ *  window: progress newer than this defers the next rung (see suppression). */
+export const HANG_WATCHDOG_BASE_RUNG_MS = 10 * 60_000;
+/** A follow-up turn within this window of a delivered rung is an ACK: the agent woke and looked, so
+ *  it is alive. An ack resets the unacked streak and buys the (doubled) interval to the next rung —
+ *  it NEVER rewinds the ladder (trap #2). */
+export const HANG_WATCHDOG_ACK_WINDOW_MS = 5 * 60_000;
+/** Two CONSECUTIVE delivered-but-unacked rungs ⇒ the agent is unresponsive ⇒ escalate. */
+export const HANG_WATCHDOG_ESCALATE_AFTER_UNACKED = 2;
+
+export interface AgentHangWatchdogConfig {
+  /** Base rung interval; the ladder doubles from here (see {@link rungDueAtElapsedMs}). */
+  baseRungMs: number;
+  /** Grace window for a follow-up turn to count as an ack. */
+  ackWindowMs: number;
+  /** Consecutive delivered-but-unacked rungs that trip escalation. */
+  escalateAfterUnacked: number;
+  /** A transport quieter than this is a dead socket — the offline reaper's problem, not a hang. */
+  transportStaleMs: number;
+}
+
+export const DEFAULT_HANG_WATCHDOG_CONFIG: AgentHangWatchdogConfig = {
+  baseRungMs: HANG_WATCHDOG_BASE_RUNG_MS,
+  ackWindowMs: HANG_WATCHDOG_ACK_WINDOW_MS,
+  escalateAfterUnacked: HANG_WATCHDOG_ESCALATE_AFTER_UNACKED,
+  transportStaleMs: STUCK_BUSY_TRANSPORT_STALE_MS,
+};
+
+/** The unmet obligation that defines a stall episode. Its `key` is the episode's identity: as long
+ *  as the SAME obligation stays unmet the episode continues (and rungs climb monotonically); when it
+ *  resolves the caller passes `null` and the episode ends. `openedAt` — when the obligation FIRST
+ *  went unmet — is the FLOOR of the episode anchor; the anchor itself is stall onset, the later of
+ *  `openedAt` and the last genuine progress, read once at open (see `decideAgentHangWatchdog`). */
+export type AgentHangObligationKind = "reply" | "task" | "work-final" | "background-script";
+export interface AgentHangObligation {
+  kind: AgentHangObligationKind;
+  /** Stable identity, e.g. `reply:1234`, `task:2148`, `work-final:<group>`, `background-script`. */
+  key: string;
+  /** Epoch-ms the obligation first went unmet — the episode's anchor. */
+  openedAt: number;
+  /** Human-readable subject of the stalled work, surfaced on the decision. */
+  subject?: string;
+}
+
+/** The just-enough agent facts the decision needs. The caller (server-side, #2036) reads these off
+ *  the `AgentCard`; kept as a narrow struct so the module stays pure and trivially testable. */
+export interface AgentHangAgentSnapshot {
+  id: string;
+  kind: AgentKind;
+  ready: boolean;
+  status: AgentStatus;
+  /** Derived from the card's lifecycle markers (`shutdown`/`killing`). A teardown is not a hang. */
+  shuttingDown?: boolean;
+}
+
+/** Per-agent episode/rung ledger. The caller persists the `episode` returned on the last decision
+ *  and passes it back as `prior` on the next tick. After DELIVERING a self-check the caller stamps
+ *  `lastRungDeliveredAt` with the delivery-ack time before persisting — an emitted-but-undelivered
+ *  rung leaves it `undefined`, which is how "delivered (delivery-acked), not merely sent" is
+ *  enforced: an undelivered rung never counts toward escalation. */
+export interface AgentHangRungState {
+  episodeId: string;
+  obligationKey: string;
+  episodeStartedAt: number;
+  /** Highest rung fired so far (0 = none). */
+  rung: number;
+  lastRungEmittedAt?: number;
+  /** Delivery-ack time of the last emitted rung; `undefined` until the caller confirms delivery. */
+  lastRungDeliveredAt?: number;
+  /** Consecutive delivered-but-unacked rungs accumulated within this episode. */
+  consecutiveUnackedDelivered: number;
+}
+
+export interface AgentHangWatchdogInput {
+  now: number;
+  agent: AgentHangAgentSnapshot;
+  /** `deriveAgentBusyHealth` output; `undefined` when the agent is not busy (the idle-obligation
+   *  path). NOT re-derived here — the caller derives it (and sets {@link healthError} on a throw). */
+  health?: AgentBusyHealth;
+  /** True when `deriveAgentBusyHealth` THREW for this agent. Fails OPEN: suppression is skipped so a
+   *  genuine stall can never be swallowed by an exception (mirrors `auditProviderTurnWatchdog`). */
+  healthError?: boolean;
+  /** The agent is parked on a provider hold (rate-limit / awaiting input). A hold is NOT a hang —
+   *  #286/#1610 own it. Also implied by `health.busyPhase === "blocked"`. */
+  providerBlocked?: boolean;
+  /** Workflow step turns are watched for missing progress even while marked active. */
+  watchActiveTurn?: boolean;
+  /** The open obligation, or `null` when nothing is owed (which ends any episode). */
+  obligation: AgentHangObligation | null;
+  /** Latest GENUINE forward-progress epoch-ms — a real turn/token, never the watchdog's own
+   *  footprint. Drives both the "recent progress" suppressor and ack detection. */
+  lastProgressAt: number;
+  /** A tracked pid on a work record is alive — positive proof the work is genuinely running. */
+  pidAlive?: boolean;
+  /** Transport-liveness age; defaults to `health.transportStaleForMs`. A dead socket suppresses. */
+  transportStaleForMs?: number;
+  /** The ledger returned on the previous decision for this agent, or `null`/absent on the first. */
+  prior?: AgentHangRungState | null;
+  config?: Partial<AgentHangWatchdogConfig>;
+}
+
+export type AgentHangAction = "none" | "self-check" | "escalate";
+
+export interface AgentHangDecision {
+  action: AgentHangAction;
+  /** The stall episode this decision pertains to; `null` only when no obligation is owed. */
+  episodeId: string | null;
+  /** The rung this decision fires (self-check/escalate), or the current highest rung for `none`. */
+  rung: number;
+  /** Machine-readable reason, for tests and observability. */
+  reason: string;
+  /** Age of the episode (`now - episodeStartedAt`); 0 when there is no episode. */
+  elapsedMs: number;
+  subjectWork?: string;
+  /** The ledger to persist for the next tick; `null` when the episode is closed. */
+  episode: AgentHangRungState | null;
+}
+
+/** #2036 review nit N5 — a non-finite / zero / negative interval is config corruption, never intent.
+ *  `baseRungMs <= 0` in particular is not merely wrong but UNSAFE: every rung would be due at
+ *  elapsed 0, so `highestDueRung`'s `while (elapsed >= due(k+1))` would never terminate. Clamp to
+ *  the declared minimum instead of trusting the caller. `src/config.ts` already rejects a bad env
+ *  override before it gets here; this is the second, in-module line of defence for programmatic
+ *  callers and tests. */
+function clampAtLeast(value: number | undefined, minimum: number, fallback: number): number {
+  if (value === undefined || !Number.isFinite(value)) return fallback;
+  return Math.max(minimum, value);
+}
+
+function resolveConfig(overrides?: Partial<AgentHangWatchdogConfig>): AgentHangWatchdogConfig {
+  if (!overrides) return DEFAULT_HANG_WATCHDOG_CONFIG;
+  const merged = { ...DEFAULT_HANG_WATCHDOG_CONFIG, ...overrides };
+  return {
+    baseRungMs: clampAtLeast(merged.baseRungMs, 1, DEFAULT_HANG_WATCHDOG_CONFIG.baseRungMs),
+    ackWindowMs: clampAtLeast(merged.ackWindowMs, 0, DEFAULT_HANG_WATCHDOG_CONFIG.ackWindowMs),
+    escalateAfterUnacked: clampAtLeast(merged.escalateAfterUnacked, 1, DEFAULT_HANG_WATCHDOG_CONFIG.escalateAfterUnacked),
+    transportStaleMs: clampAtLeast(merged.transportStaleMs, 1, DEFAULT_HANG_WATCHDOG_CONFIG.transportStaleMs),
+  };
+}
+
+/** Deterministic (no RNG — the module is replayable): the episode id is a function of the agent, the
+ *  obligation identity, and the anchor timestamp. */
+function makeEpisodeId(agentId: string, obligationKey: string, episodeStartedAt: number): string {
+  return `hang:${agentId}:${obligationKey}:${episodeStartedAt}`;
+}
+
+/**
+ * Scheduled episode age at which rung `k` (1-based) becomes due. Gaps DOUBLE from `baseMs`:
+ *   R1 = base·(2¹−1) = base           (10 min)
+ *   R2 = base·(2²−1) = 3·base         (30 min)
+ *   R3 = base·(2³−1) = 7·base         (70 min)
+ *   Rk = base·(2ᵏ−1)                  … indefinitely, per "keep laddering with doubled intervals".
+ * A rung acked (agent woke and looked) is thereby a doubled interval from the previous one — the
+ * "buys the doubled interval" grace — WITHOUT the ladder ever rewinding (it is a fixed function of
+ * the episode age alone).
+ */
+export function rungDueAtElapsedMs(k: number, baseMs: number = HANG_WATCHDOG_BASE_RUNG_MS): number {
+  if (k <= 0) return 0;
+  return baseMs * (2 ** k - 1);
+}
+
+/** The scheduled GAP between rung `k-1` and rung `k` — `base·2^(k-1)`, the difference of two
+ *  consecutive {@link rungDueAtElapsedMs}. Doubles as the rung's TTL and its "I'll re-check in …". */
+export function rungGapMs(k: number, baseMs: number = HANG_WATCHDOG_BASE_RUNG_MS): number {
+  if (k <= 0) return 0;
+  return Math.max(1, baseMs * 2 ** (k - 1));
+}
+
+/** Highest rung whose scheduled due-time the episode age has reached (0 = none due yet). */
+function highestDueRung(elapsedMs: number, baseMs: number): number {
+  let k = 0;
+  while (elapsedMs >= rungDueAtElapsedMs(k + 1, baseMs)) k++;
+  return k;
+}
+
+/**
+ * The one entry point. Pure and deterministic: same inputs → same decision.
+ *
+ * EPISODE LIFECYCLE (the anti-#2010 crux):
+ *   - obligation `null`                → episode CLOSES (the only reset). Progress never closes it.
+ *   - same obligation `key` as `prior` → episode CONTINUES. Its anchor tracks stall onset until the
+ *                                        first rung fires and is frozen from then on, so the age
+ *                                        keeps growing and rungs climb even as the agent "works".
+ *   - a different `key`                → a NEW episode opens at STALL ONSET, i.e.
+ *                                        `max(obligation.openedAt, lastProgressAt)` (see below).
+ */
+export function decideAgentHangWatchdog(input: AgentHangWatchdogInput): AgentHangDecision {
+  const cfg = resolveConfig(input.config);
+  const { now, agent, health, obligation } = input;
+  const prior = input.prior ?? null;
+
+  // Obligation resolved / none owed — the SOLE episode-reset path. Report sent, reply settled, Task
+  // off `in_progress`, Stop hook posted idle(background-script): whatever it was, there is nothing to
+  // watch, so the episode ends and the rung ledger is dropped.
+  if (!obligation) {
+    return { action: "none", episodeId: null, rung: 0, reason: "no-obligation", elapsedMs: 0, episode: null };
+  }
+
+  // Episode IDENTITY keys on the obligation, NEVER on lastProgressAt — that is what makes the ladder
+  // monotonic (trap #2). The episode ANCHOR is a separate question, settled ONCE at open:
+  //
+  //   #2036 review GAP 3 — anchoring on `obligation.openedAt` alone made rung numbers meaningless
+  //   for long-lived obligations. A Task claimed six hours ago whose worker has been productive the
+  //   whole time would open its FIRST-EVER episode at rung 3 and prose about "twice the gap of the
+  //   previous warning" that never happened. The stall did not start when the obligation opened; it
+  //   started when the agent last did real work. So the anchor is the LATER of the two — stall
+  //   onset — while the obligation still decides identity and lifetime.
+  //
+  //   Freezing that anchor at OPEN is not enough, and this is the subtle half (#2036 round-2
+  //   OPEN-1). An episode opens on the FIRST tick the agent is even a candidate — including ticks
+  //   the suppressors then answer `none`, which still carry the ledger forward. So a branch worker
+  //   whose Task obligation predates its stall (i.e. every branch worker) opens its episode the
+  //   moment it claims the Task, and every productive hour afterwards accumulates into `elapsedMs`.
+  //   Five productive hours later its FIRST-EVER rung fires as R5, promising to "re-check in 5h
+  //   20m", with escalation nearly two days out: the R1/R2/escalate ladder disarmed for the common
+  //   shape.
+  //
+  //   So the anchor RE-ANCHORS while no rung has fired, and freezes the moment one does. Before any
+  //   rung there is nothing to rewind — no message was sent, no idempotency key exists, and the
+  //   suppressors are the only thing that has happened. Once rung >= 1 the anchor is immovable, so a
+  //   self-check's OWN ack can never rewind the ladder it is climbing, which is the property trap #2
+  //   protects. The episode ID is minted once and preserved across a re-anchor: it keys idempotency,
+  //   and churning it at rung 0 would buy nothing but noise.
+  const continuing = prior !== null && prior.obligationKey === obligation.key;
+  const shouldReanchor = !continuing || prior!.rung === 0;
+  const episodeStartedAt = shouldReanchor
+    ? Math.min(now, Math.max(obligation.openedAt, input.lastProgressAt))
+    : prior!.episodeStartedAt;
+  const episodeId = continuing ? prior!.episodeId : makeEpisodeId(agent.id, obligation.key, episodeStartedAt);
+  const baseRung = continuing ? prior!.rung : 0;
+  const baseUnacked = continuing ? prior!.consecutiveUnackedDelivered : 0;
+  const lastDeliveredAt = continuing ? prior!.lastRungDeliveredAt : undefined;
+  const lastEmittedAt = continuing ? prior!.lastRungEmittedAt : undefined;
+  const elapsedMs = Math.max(0, now - episodeStartedAt);
+  const subjectWork = obligation.subject ?? health?.activeWorkLabel ?? health?.activeWorkId;
+
+  // The ledger we carry forward unchanged when we take no rung action (suppression / not-yet-due).
+  const carried: AgentHangRungState = {
+    episodeId,
+    obligationKey: obligation.key,
+    episodeStartedAt,
+    rung: baseRung,
+    ...(lastEmittedAt !== undefined ? { lastRungEmittedAt: lastEmittedAt } : {}),
+    ...(lastDeliveredAt !== undefined ? { lastRungDeliveredAt: lastDeliveredAt } : {}),
+    consecutiveUnackedDelivered: baseUnacked,
+  };
+  const none = (reason: string): AgentHangDecision => ({
+    action: "none",
+    episodeId,
+    rung: baseRung,
+    reason,
+    elapsedMs,
+    ...(subjectWork ? { subjectWork } : {}),
+    episode: carried,
+  });
+
+  // --- Candidate gate. A non-candidate is NOT an episode reset (a live agent may flip in and out of
+  //     `active`); the episode is preserved so the ladder never rewinds on a transient state. ---
+  if (agent.kind !== "provider") return none("not-a-provider");
+  if (!agent.ready) return none("not-ready");
+  if (agent.shuttingDown) return none("shutting-down");
+  if (input.providerBlocked || health?.busyPhase === "blocked") return none("provider-blocked");
+
+  // Two stall shapes: (i) background-script-only busy (`passive`), or (ii) idle with an open
+  // obligation. A busy-`active` agent is doing turn work now — #1980's domain, never ours.
+  const passiveBusy = health?.busyPhase === "passive";
+  const idleWithObligation = agent.status === "idle";
+  if (!passiveBusy && !idleWithObligation && !(input.watchActiveTurn && agent.status === "busy")) return none("not-a-stall-candidate");
+
+  // --- Positive-evidence suppression. Fires ONLY on affirmative proof of life; a derivation error
+  //     fails OPEN (skips suppression) so a stall is never swallowed by an exception. ---
+  if (!input.healthError) {
+    if (input.pidAlive) return none("suppressed:pid-alive");
+    if (health?.busyPhase === "active" && !input.watchActiveTurn) return none("suppressed:active");
+    const transportStaleForMs = input.transportStaleForMs ?? health?.transportStaleForMs;
+    if (transportStaleForMs !== undefined && transportStaleForMs >= cfg.transportStaleMs) {
+      return none("suppressed:transport-stale");
+    }
+    // Recent genuine progress ⇒ actively working (#1980's long-tool-run) OR the agent just acked a
+    // rung. Either way defer — this DELAYS the next rung; it never rewinds the episode.
+    if (now - input.lastProgressAt < cfg.baseRungMs) return none("suppressed:recent-progress");
+  }
+
+  // --- Ladder. Two regimes, and the split is what keeps the ladder armed for the WHOLE episode.
+  //
+  //   BEFORE the first rung (rung 0): due-times key on the EPISODE age, whose anchor is still
+  //   tracking stall onset. R1 lands `base` after the agent went quiet.
+  //
+  //   AFTER a rung has fired: the anchor is frozen (anti-#2010), so keying later rungs on episode
+  //   age would make the ladder progressively meaningless as the episode outlives its first stall.
+  //   #2036 round-3 OPEN-1: one early idle spell fires R1 and pins the anchor for the rest of a
+  //   Task's life; the worker then works productively for hours, genuinely wedges, and the SECOND
+  //   stall's first rung is R5 with escalation a day out — never reached in practice. So the next
+  //   due-time is measured from the LAST RUNG instead: `lastRungEmittedAt + base·2^rung`.
+  //
+  //   For an uninterrupted episode the two are IDENTICAL — base·(2^k − 1) = base·(2^(k-1) − 1) +
+  //   base·2^(k-1) — so the doubling schedule is unchanged. They diverge only after a long
+  //   suppressed stretch, which is exactly where the old form was wrong: the ladder self-corrects,
+  //   resuming at the next rung with its proper gap rather than teleporting up the ladder.
+  //
+  //   This moves the DUE-TIME calculation only. The anchor still never rewinds, so a self-check's
+  //   own ack still cannot reset the ladder (trap #2 / #2010).
+  let dueRung: number;
+  if (baseRung >= 1 && lastEmittedAt !== undefined) {
+    if (now < lastEmittedAt + rungGapMs(baseRung + 1, cfg.baseRungMs)) return none("no-rung-due");
+    dueRung = baseRung + 1;
+  } else {
+    dueRung = highestDueRung(elapsedMs, cfg.baseRungMs);
+    if (dueRung <= baseRung) return none("no-rung-due");
+  }
+
+  // A new rung is due. Account the PRECEDING rung's ack status. The ack (a follow-up turn after the
+  // rung was delivered) only resets the unacked streak; it NEVER rewinds `rung` or `episodeStartedAt`.
+  const precedingDelivered = lastDeliveredAt !== undefined;
+  const precedingAcked = precedingDelivered && input.lastProgressAt > lastDeliveredAt!;
+  const precedingUnacked =
+    precedingDelivered && !precedingAcked && now - lastDeliveredAt! >= cfg.ackWindowMs;
+  const streak = precedingAcked ? 0 : baseUnacked + (precedingUnacked ? 1 : 0);
+
+  const nextLedger: AgentHangRungState = {
+    episodeId,
+    obligationKey: obligation.key,
+    episodeStartedAt,
+    rung: dueRung,
+    lastRungEmittedAt: now,
+    // The freshly-fired rung is not yet delivered; the caller stamps delivery before persisting.
+    consecutiveUnackedDelivered: streak,
+  };
+
+  if (streak >= cfg.escalateAfterUnacked) {
+    // Two consecutive delivered-but-unacked rungs ⇒ agent unresponsive ⇒ hand to the coordinator.
+    return {
+      action: "escalate",
+      episodeId,
+      rung: dueRung,
+      reason: "unacked-escalation",
+      elapsedMs,
+      ...(subjectWork ? { subjectWork } : {}),
+      episode: nextLedger,
+    };
+  }
+
+  return {
+    action: "self-check",
+    episodeId,
+    rung: dueRung,
+    reason: precedingAcked ? "rung-after-ack" : "rung-due",
+    elapsedMs,
+    ...(subjectWork ? { subjectWork } : {}),
+    episode: nextLedger,
+  };
+}

package/src/services/agent-keep.ts

@@ -0,0 +1,72 @@
+// #3428 — the one keep operation behind relay_agent_keep, `humwick agent keep`, the HTTP route
+// and the dashboard agent card. Any authenticated agent:write caller may keep or release any agent
+// at any time: a keep only withholds automatic disposal and idle digests, never explicit shutdown.
+import type { AgentCard, AgentKeep } from "@humwick/sdk";
+import { ValidationError, createActivityEventAsync, getAgent, setAgentKeepAsync } from "../db";
+import { emitActivityEvent, emitAgentStatus } from "../sse";
+import { optionalString } from "../validation";
+import type { AuthContext } from "./auth-context";
+
+export class AgentKeepTargetError extends Error {}
+
+export interface SetAgentKeepInput {
+  agentId: string;
+  keep: boolean;
+  /** Epoch ms. Omitted means kept until cleared. */
+  until?: number;
+  reason?: string;
+  requestedVia: "http" | "mcp";
+  auditMetadata?: Record<string, unknown>;
+}
+
+/** Transport-neutral argument parsing shared by MCP and HTTP. `until` takes epoch ms or an ISO time. */
+export function parseAgentKeepArgs(args: Record<string, unknown>): Pick<SetAgentKeepInput, "keep" | "until" | "reason"> {
+  if (typeof args.keep !== "boolean") throw new ValidationError("keep must be true or false");
+  let until: number | undefined;
+  if (typeof args.until === "string") {
+    until = Date.parse(args.until);
+    if (!Number.isFinite(until)) throw new ValidationError("until must be epoch ms or an ISO time");
+  } else if (args.until !== undefined && args.until !== null) {
+    if (typeof args.until !== "number") throw new ValidationError("until must be epoch ms or an ISO time");
+    until = args.until;
+  }
+  return { keep: args.keep, ...(until !== undefined ? { until } : {}), ...(args.reason != null ? { reason: optionalString(args.reason, "reason", 500) } : {}) };
+}
+
+/** The keep that is in force now; an expired keep restores eligibility without a write. */
+export function activeAgentKeep(agent: Pick<AgentCard, "keep"> | null | undefined, now: number): AgentKeep | undefined {
+  const keep = agent?.keep;
+  return keep && (keep.until === undefined || keep.until > now) ? keep : undefined;
+}
+
+export async function setAgentKeep(input: SetAgentKeepInput, ctx: AuthContext, now = Date.now()): Promise<{ agentId: string; keep: AgentKeep | null }> {
+  const agent = getAgent(input.agentId);
+  if (!agent) throw new AgentKeepTargetError(`agent not found: ${input.agentId}`);
+  if (input.until !== undefined && (!Number.isSafeInteger(input.until) || input.until <= now)) {
+    throw new ValidationError("until must be a future epoch-ms time");
+  }
+  if (!input.keep && (input.until !== undefined || input.reason !== undefined)) {
+    throw new ValidationError("until and reason apply only when keep is true");
+  }
+  const keep: AgentKeep | null = input.keep ? {
+    ...(input.until !== undefined ? { until: input.until } : {}),
+    ...(input.reason ? { reason: input.reason } : {}),
+    setBy: ctx.callerAgentId ?? ctx.actor.id,
+    setAt: now,
+  } : null;
+  const event = await createActivityEventAsync({
+    clientId: `agent-keep-${agent.id}-${now}`,
+    kind: "state",
+    title: keep ? "Agent kept" : "Agent keep cleared",
+    body: keep?.reason ?? "",
+    meta: agent.id,
+    icon: keep ? "ti-pin" : "ti-pinned-off",
+    view: "agents",
+    agentId: agent.id,
+    metadata: { action: "keep", keep, previous: agent.keep ?? null, actor: ctx.actor.id, actorKind: ctx.actor.kind, requestedVia: input.requestedVia, ...input.auditMetadata, source: "server" },
+  });
+  if (!(await setAgentKeepAsync(agent.id, keep))) throw new AgentKeepTargetError(`agent not found: ${input.agentId}`);
+  emitActivityEvent(event);
+  await emitAgentStatus(agent.id);
+  return { agentId: agent.id, keep };
+}

package/src/services/agent-message-fanout.ts

@@ -0,0 +1,91 @@
+// The shared server-side seam for agent-bound message delivery.
+import { isRecord } from "@humwick/sdk";
+import { isFanoutTarget } from "../channel-target";
+import { isDisallowedChannelContent } from "../db/channel-egress-allowlist";
+import { isChannelEgressTarget } from "../db/delivery";
+import type { Message } from "../types";
+
+export type DeliverySurface = { surface: "turn" } | { surface: "history" };
+
+/** The transport-specific write a site performs once delivery is decided `turn`. It is a pure sink:
+ *  it emits the already-built agent-bound frame (or collects the polled message) and returns nothing. */
+export type EmitFrameFn = () => void;
+
+// #3008: exclude history BEFORE a turn poll's LIMIT, or unread history permanently fills the page.
+// Keep this SQL projection beside the authoritative decision; display polls do not use it.
+export function historyOnlyMessageSql(alias: string): string {
+  return `(${alias}.kind = 'session'
+    OR json_type(${alias}.payload, '$.wake') IS 'false'
+    OR json_extract(${alias}.payload, '$.kind') IS 'agent.self_resume_continuation'
+    OR json_type(${alias}.payload, '$.channelReportSuppressed') IS 'true'
+    OR coalesce(json_extract(${alias}.payload, '$.reportSuppression') NOT IN (0, ''), 0)
+    OR json_type(${alias}.payload, '$.workflowHeld') IS 'object'
+    OR json_type(${alias}.payload, '$.coordinatorDigestHeld') IS 'true')`;
+}
+
+/**
+ * The `history` rules: recipient-independent, so the message writer settles these rows at insert (#3013).
+ */
+export function isHistoryOnlyMessage(message: Pick<Message, "kind" | "payload">): boolean {
+  // #2029 (#2026-B) — the FIRST real policy on the delivery seam. Enforced ONCE, server-side, so it holds for
+  // every recipient regardless of which runner checkout they pin (stale runners included). Both arms
+  // resolve to `history`: still persisted, still on the thread, still on the dashboard SSE stream
+  // (connections with `agentId: null` bypass this seam), but NEVER fanned to the recipient's turn.
+  //
+  //   - `kind === "session"` — session-capture forwards, injected via prompt.inject, never a turn frame.
+  //     Migrated here from the runner's client-side skip (was runner-core.enqueueMessage) so it lives in
+  //     ONE place instead of each client implementation.
+  //   - `payload.wake === false` — the #2007 suppression ("submitted for landing" and any future
+  //     wake:false notice). Previously enforced only by the runner (runner-core.ts:752), which could not
+  //     hold on an already-running agent; now it cannot be missed by any client, stale or not.
+  //
+  // A replyExpected:false notification is still transport delivery. Routing it to history
+  // withholds it from the runner's turn poll, so no adapter can acknowledge the row and its
+  // delivery_status remains pending. Empty-turn work classification belongs to the work gate;
+  // the runner keeps notification-only batches from arming its input-stalled watch.
+  if (message.kind === "session") return true;
+  if (message.payload?.wake === false) return true;
+  // #2550 — the durable self-resume continuation recovery artifact is never a turn frame. Its payload
+  // reaches the provider through the COMMAND plane (coupled clear-inject, native /compact instructions,
+  // or the #1023 injectContext backstop); the message row exists only as the relay_get_message recovery
+  // artifact for the fresh context. Before this rule the row stayed an ordinary wakable poll candidate,
+  // so the runner's ~20s liveness bootstrap re-poll flushed the byte-identical envelope again as normal
+  // queued mail at the first idle boundary — the deferred duplicate. `history` keeps it persisted and
+  // dashboard/thread-visible on every surface (live, batch, replay, poll) while withholding it from the
+  // recipient's turn; it is decided here, server-side, so stale runners cannot miss it either.
+  if (message.payload?.kind === "agent.self_resume_continuation") return true;
+  // #2335 (P1 privacy, supersedes #2089/T2219) — anything addressed to a channel bridge that is not
+  // deliberate channel content is stamped `channelReportSuppressed` at send time by the channel-egress
+  // ALLOWLIST (isDisallowedChannelContent, db/channel-egress-allowlist.ts) — auto-captured agent narration above all.
+  // Route it to `history` exactly like the session mirror above — persisted + dashboard-visible, never
+  // fanned to the channels-host bridge, which would forward it to a real human's DM. #2089 stamped only
+  // the narrower case (a capture report duplicating the agent's own explicit reply, the #2181-unmasked
+  // Telegram double); that case is strictly implied here, so the double stays fixed while the lane #2089
+  // deliberately left open — narration standing in AS the reply — is closed. A channel reply must be an
+  // EXPLICIT relay_reply / relay_egress_to_channel, which carries no capture marker and still delivers.
+  // Stamped once and persisted, so live/batch/replay/poll decide identically.
+  if (message.payload?.channelReportSuppressed === true) return true;
+  // #2831 — a capture made while its sender waited on its own relay_run; decided once at capture time.
+  if (message.payload?.reportSuppression) return true;
+  // #3045 — stored on a workflow run instead of reaching its coordinator (db/workflow-contact.ts).
+  if (isRecord(message.payload?.workflowHeld)) return true;
+  if (message.payload?.coordinatorDigestHeld === true) return true;
+  return false;
+}
+
+export function decideDelivery(recipientAgentId: string, message: Message): DeliverySurface {
+  // #2029 (#2026-B) — the history rules live in isHistoryOnlyMessage, enforced ONCE, server-side.
+  if (isHistoryOnlyMessage(message)) return { surface: "history" };
+  return { surface: "turn" };
+}
+
+/**
+ * The ONE agent-bound emission helper. Every server site that can put a message in front of an agent
+ * calls this and nothing else: it asks {@link decideDelivery} for the surface and, only on `turn`, runs
+ * the site's `emit` sink. A non-`turn` decision emits no agent-bound frame. This is the census guard's
+ * anchor — {@link decideDelivery} is called from nowhere else, and every emission is behind this call.
+ */
+export function fanOutToAgent(recipientAgentId: string, message: Message, emit: EmitFrameFn): void {
+  if ((isFanoutTarget(message.to) || message.to.startsWith("policy:")) && isDisallowedChannelContent(message.payload, message.kind, message.meta) && isChannelEgressTarget(recipientAgentId)) return;
+  if (decideDelivery(recipientAgentId, message).surface === "turn") emit();
+}

package/src/services/agent-outstanding-work.ts

@@ -0,0 +1,66 @@
+// #2503 — distinguish "stuck against pending work" from "idle after delivery with no pending
+// inbound work".
+//
+// The runner's empty-turn ladder (`consecutiveEmptyTurns`, #2389) escalates whenever input it
+// armed sits unanswered. After a builder has DELIVERED its Task (needs-review / complete /
+// blocked) there is legitimately nothing for it to do until review reports back — the messages
+// that reach it then (lifecycle/task notifications, land receipts) are fire-and-forget FYIs, not
+// work — so an empty provider turn in that state is EXPECTED behavior, and escalating it to the
+// spawn parent as "steer it or re-spawn" trains the coordinator to ignore the signal it exists
+// for. This module is the server-side gate the escalation is routed through: it reads the
+// distinguishing state from authoritative sources (the task_events delivery ledger and the
+// messages tables, see src/db/agent-outstanding-work.ts), never from guesses about runner
+// internals.
+//
+// Deliberately NARROW (do not over-suppress): suppression requires BOTH
+//   (a) the agent's most recent lifecycle action was a delivery it made itself — or (#2545) its
+//       workspace is parked for review, the same hand-back state that Path-B/taskless lands leave
+//       without a deliverable task event, AND
+//   (b) nobody has given it work since — no reply-expected message arrived after that delivery /
+//       review park, nothing addressed to it is still undelivered, it holds no read-but-unanswered
+//       reply obligation, and it holds no Task claim it has failed to advance.
+// Any outstanding work ⇒ escalate, exactly as before this gate.
+//
+// FAIL OPEN: every read error falls through to "escalate". A suppression bug that silently
+// swallows a genuinely wedged agent is strictly worse than the spurious wake this gate removes
+// (same posture as #2010 in turn-reconcile-audit.ts).
+
+import { listPendingReplyObligations } from "../db";
+import {
+  hasUnadvancedTask,
+  hasUndeliveredDirectedWork,
+  hasWorkMessageAfter,
+  latestDeliverableEventByAgent,
+  latestReviewParkedWorkspaceByAgent,
+} from "../db";
+
+/**
+ * #2503 — true when this agent's most recent lifecycle action was a delivery AND nothing
+ * outstanding remains: no directed work message arrived after the delivery, nothing addressed to
+ * it is still undelivered, it holds no read-but-unanswered reply obligation, and it holds no
+ * claimed Task it is failing to advance. Only such an agent may have its empty-turn escalation
+ * suppressed; anything else escalates exactly as before.
+ *
+ * #2545 — a workspace parked for review (`review_requested` / landState `delivery:needs-review`)
+ * is the same "handed the work back, awaiting an external verdict" state, and is accepted as the
+ * idle anchor even when no `deliverable` task event exists (Path-B/taskless lands). The anchor is
+ * whichever of {latest delivery, review park} happened LAST, so a fresh re-delivery or a fresh
+ * park each re-arm the gate from that point. Anything outstanding ⇒ escalate, exactly as before.
+ */
+export function agentDeliveredAndIdle(agentId: string): boolean {
+  try {
+    const delivery = latestDeliverableEventByAgent(agentId);
+    // No delivery on record and no review park ⇒ nothing distinguished ⇒ the #2389 escalation
+    // stands (the original spawn-prompt strand must keep waking the parent).
+    const park = latestReviewParkedWorkspaceByAgent(agentId);
+    if (!delivery && !park) return false;
+    const anchoredAt = Math.max(delivery?.createdAt ?? 0, park?.parkedAt ?? 0);
+    if (hasWorkMessageAfter(agentId, anchoredAt)) return false;
+    if (hasUndeliveredDirectedWork(agentId)) return false;
+    if (listPendingReplyObligations(agentId, 1).length > 0) return false;
+    if (hasUnadvancedTask(agentId)) return false;
+    return true;
+  } catch {
+    return false;
+  }
+}

package/src/services/agent-reporting.ts

@@ -0,0 +1,27 @@
+import { isReportVerbosity, type ReportVerbosity } from "@humwick/sdk";
+import { createActivityEventAsync, getAgent, mergeAgentMetaAsync, ValidationError } from "../db";
+import { emitActivityEvent, emitAgentStatus } from "../sse";
+import { authorizeShutdown, ShutdownTargetError } from "./shutdown-agent";
+import type { AuthContext } from "./auth-context";
+
+export async function setAgentReportVerbosity(input: {
+  agentId: string;
+  reportVerbosity: unknown;
+  requestedVia: "http" | "mcp";
+  reason?: string;
+}, ctx: AuthContext): Promise<{ action: "reportVerbosity"; reportVerbosity: ReportVerbosity }> {
+  authorizeShutdown(ctx, { agentId: input.agentId, requestedVia: input.requestedVia });
+  const agent = getAgent(input.agentId);
+  if (!agent) throw new ShutdownTargetError("agent not found");
+  if (!isReportVerbosity(input.reportVerbosity)) throw new ValidationError("reportVerbosity must be off, quiet, normal or verbose");
+  const reportVerbosity = input.reportVerbosity;
+  const changed = await mergeAgentMetaAsync(agent.id, { reportVerbosity }, { instanceId: agent.instanceId, epoch: agent.epoch });
+  if (!changed) throw new ValidationError("agent instance changed while updating reportVerbosity");
+  const activity = await createActivityEventAsync({
+    kind: "state", title: "Agent report mode changed", body: reportVerbosity, agentId: agent.id,
+    metadata: { action: "reportVerbosity", reportVerbosity, actor: ctx.actor.id, actorKind: ctx.actor.kind, requestedVia: input.requestedVia, ...(input.reason ? { reason: input.reason } : {}) },
+  });
+  emitActivityEvent(activity);
+  await emitAgentStatus(agent.id);
+  return { action: "reportVerbosity", reportVerbosity };
+}