@humwick/server 0.130.1 → 0.130.3

package/src/db/agents-async.ts

@@ -11,8 +11,16 @@
 
 import { agentSessionGuardResult } from "./agent-predicates.ts";
 import {
+  agentLifecycleWriteArgs,
+  bufferOrBypassHeartbeat,
   getAgent,
+  prepareAgentLifecycleUpdate,
   runBeforeAgentStatusTransitionHandlers,
+  type AgentLifecycleUpdate,
+  type AgentLifecycleUpdateResult,
+  type HeartbeatInput,
+  type OrphanedTokenPage,
+  type OrphanedTokenRevocation,
   validateAgentSession,
   type UpsertAgentTxResult,
   type UpsertAgentOptions,
@@ -123,7 +131,7 @@ export async function setStatusAsync(id: string, status: AgentCard["status"], gu
   const before = await getAgentAsync(id);
   if (!agentSessionGuardResult(before, guard).ok) return false;
   if (!before) return false;
-  if (before.status !== status) runBeforeAgentStatusTransitionHandlers(before, status);
+  if (before.status !== status) await runBeforeAgentStatusTransitionHandlers(before, status);
   if (status === "offline" || status === "stale") evictPendingPresence(id);
   const now = Date.now();
   // #2111 — buffered into the short lifecycle window and submitted with its peers as ONE batch job.
@@ -209,6 +217,32 @@ export async function recordAgentExitDiagnosticsAsync(
   return getAgentAsync(id);
 }
 
+/** `applyAgentLifecycleUpdate` with the write on the writer thread (#1870 S7). The guard, the `before` read
+ *  and the transition handlers stay on the main thread ahead of submission; the op CAS-fences on
+ *  `before.epoch`, so a takeover that lands in the hop turns this into a no-op instead of a clobber. */
+export async function applyAgentLifecycleUpdateAsync(
+  id: string,
+  update: AgentLifecycleUpdate,
+  guard?: AgentSessionGuard,
+  now: number = Date.now(),
+): Promise<AgentLifecycleUpdateResult> {
+  // Allocated first, per the ordinal-timing rule at the top of setStatusAsync.
+  const seq = nextLifecycleSeq();
+  const before = prepareAgentLifecycleUpdate(id, update, guard, false);
+  if (!before) return { ok: false, changed: false, metaWriteBytes: 0, before: null, after: null };
+  if (update.status !== undefined && update.status !== before.status) await runBeforeAgentStatusTransitionHandlers(before, update.status);
+  const result = (await getGateway().write("agents.applyLifecycleUpdate", agentLifecycleWriteArgs(before, update, seq, now))) as unknown as Omit<AgentLifecycleUpdateResult, "ok" | "before">;
+  return { ok: true, before, ...result };
+}
+
+/** `recordHeartbeat` for the transports: a steady-state beat is buffered at once; a presence-critical one
+ *  writes on the writer thread (#1870 S7). */
+export async function recordHeartbeatAsync(id: string, opts: HeartbeatInput = {}): Promise<boolean> {
+  const beat = bufferOrBypassHeartbeat(id, opts);
+  if (typeof beat === "boolean") return beat;
+  return (await applyAgentLifecycleUpdateAsync(id, beat, opts.guard, opts.now ?? Date.now())).changed;
+}
+
 export async function heartbeatAsync(
   id: string,
   guard?: AgentSessionGuard,
@@ -263,7 +297,7 @@ export async function reapStaleAgentsAsync(ttlMs: number = STALE_TTL_MS, lagGrac
  */
 export async function pruneOfflineAgentsAsync(
   maxOfflineMs: number = DAY_MS,
-  options: { onFinality?: (result: WorkFinalFlushResult) => void } = {},
+  options: { onFinality?: (result: WorkFinalFlushResult) => void; onArchivePurgeFailure?: () => void } = {},
 ): Promise<string[]> {
   const cutoff = Date.now() - maxOfflineMs;
   const candidates = (await getGateway().read("agents.offlinePruneCandidates", {
@@ -271,7 +305,7 @@ export async function pruneOfflineAgentsAsync(
   })) as unknown as AgentCard[];
   const candidateIds = candidates.map((candidate) => candidate.id);
   if (candidateIds.length === 0) return [];
-  await purgeArchivesBeforeDelete(candidateIds, cutoff);
+  if (!await purgeArchivesBeforeDelete(candidateIds, cutoff)) options.onArchivePurgeFailure?.();
 
   const finalityQueues = takeDeferredLineageReports(candidateIds);
   const refreshFinalityCandidates = candidates
@@ -301,11 +335,13 @@ export async function pruneOfflineAgentsAsync(
 }
 
 /** #3012 — best effort: whatever the off-loop purge leaves, the delete's own FK cascade still removes. */
-async function purgeArchivesBeforeDelete(agentIds: string[], offlineBefore?: number): Promise<void> {
+async function purgeArchivesBeforeDelete(agentIds: string[], offlineBefore?: number): Promise<boolean> {
   try {
     await purgeContinuationArchivesAsync({ agentIds, offlineBefore });
+    return true;
   } catch (error) {
     console.warn("[agents] continuation archive purge before delete failed; the cascade will remove them:", error);
+    return false;
   }
 }
 
@@ -319,14 +355,28 @@ export async function deleteAgentAsync(id: string): Promise<{ ok: boolean; error
   return deleted ? { ok: true } : { ok: false, error: "agent not found" };
 }
 
+// #3776 — the sweep is one short writer op per page of agents (each agent's lineage revoke stays atomic), so
+// other writes interleave between pages. It stops when aborted or past its budget; the next run starts over.
+const ORPHAN_SWEEP_PAGE_SIZE = 8;
+const ORPHAN_SWEEP_BUDGET_MS = 30_000;
+
 export async function revokeOrphanedRuntimeTokensAsync(
   graceMs: number = STALE_TTL_MS,
   lagGraceMs = 0,
-): Promise<Array<{ agentId: string; jtis: string[] }>> {
-  return (await getGateway().write("agents.revokeOrphanedTokens", { graceMs, lagGraceMs })) as unknown as Array<{
-    agentId: string;
-    jtis: string[];
-  }>;
+  options: { signal?: AbortSignal; budgetMs?: number; pageSize?: number } = {},
+): Promise<{ revoked: OrphanedTokenRevocation[]; complete: boolean }> {
+  const { signal, budgetMs = ORPHAN_SWEEP_BUDGET_MS, pageSize = ORPHAN_SWEEP_PAGE_SIZE } = options;
+  const deadline = Date.now() + budgetMs;
+  const revoked: OrphanedTokenRevocation[] = [];
+  let afterId: string | undefined;
+  do {
+    if (signal?.aborted) return { revoked, complete: false };
+    const page = (await getGateway().write("agents.revokeOrphanedTokens", { graceMs, lagGraceMs, afterId, limit: pageSize })) as unknown as OrphanedTokenPage;
+    revoked.push(...page.revoked);
+    if (page.nextAfterId === null) return { revoked, complete: true };
+    afterId = page.nextAfterId;
+  } while (Date.now() < deadline);
+  return { revoked, complete: false };
 }
 
 export async function revokeRuntimeTokensForAgentAsync(

package/src/db/agents-write-ops.ts

@@ -19,6 +19,8 @@ import { applyIdleSelfCare } from "./idle-self-care";
 // write (none of these are) would need `idempotent: false` plus a caller-supplied key.
 
 import {
+  applyAgentLifecycleUpdateInternal,
+  type AgentLifecycleWriteArgs,
   clearAgentMetaKeys,
   deleteAgentInternal,
   heartbeat,
@@ -41,6 +43,7 @@ import {
 import { pruneOfflineAgentsWithFinality, type AgentsPruneOfflineOpArgs } from "./agents-prune-write-op.ts";
 import { setAgentKeepState } from "./agent-keep.ts";
 import type { AgentCard, AgentKeep, AgentSessionGuard, ContextState, ManagedSessionExitDiagnostics, ProviderCapabilities, QuotaState, RegisterAgentInput } from "../types";
+import type { OrphanedTokenPage } from "./agents.ts";
 import type { LineageRevokeReason } from "./token-standing.ts";
 import type { LifecycleWriteOutcome } from "./lifecycle-seq.ts";
 import type { WriterOpContext, WriterOpSpec } from "./write-ops.ts";
@@ -69,7 +72,7 @@ export interface AgentsHeartbeatOpArgs {
 export interface AgentsRecordContextSnapshotOpArgs { agentId: string; context: ContextState }
 export interface AgentsReapStaleOpArgs { ttlMs: number; lagGraceMs: number; protectedIds?: string[]; seq: number }
 export interface AgentsDeleteOpArgs { id: string }
-export interface AgentsRevokeOrphanedTokensOpArgs { graceMs: number; lagGraceMs: number }
+export interface AgentsRevokeOrphanedTokensOpArgs { graceMs: number; lagGraceMs: number; afterId?: string; limit?: number }
 export interface AgentsRevokeTokensForAgentOpArgs { agentId: string; reason?: LineageRevokeReason }
 
 export const AGENTS_WRITE_OPS: Readonly<Record<string, WriterOpSpec>> = Object.freeze({
@@ -107,6 +110,12 @@ export const AGENTS_WRITE_OPS: Readonly<Record<string, WriterOpSpec>> = Object.f
   // coalescing window buffered them) and return a tri-state: `superseded` means a later-submitted
   // lifecycle write already landed, which is a SUCCESS for the caller and must not be mistaken for a
   // missing agent. See lifecycle-seq.ts.
+  "agents.applyLifecycleUpdate": {
+    // #1870 S7: bus status frame and heartbeat bypass; a replay of the same `seq` is a no-op on the fenced columns.
+    offLoop: true,
+    idempotent: true,
+    fn: (args: AgentLifecycleWriteArgs) => applyAgentLifecycleUpdateInternal(args),
+  },
   "agents.setStatus": {
     offLoop: true,
     idempotent: true,
@@ -176,8 +185,8 @@ export const AGENTS_WRITE_OPS: Readonly<Record<string, WriterOpSpec>> = Object.f
   "agents.revokeOrphanedTokens": {
     offLoop: true,
     idempotent: true,
-    fn: (args: AgentsRevokeOrphanedTokensOpArgs, ctx: WriterOpContext): Array<{ agentId: string; jtis: string[] }> =>
-      revokeOrphanedRuntimeTokens(args.graceMs, args.lagGraceMs, ctx.now),
+    fn: (args: AgentsRevokeOrphanedTokensOpArgs, ctx: WriterOpContext): OrphanedTokenPage =>
+      revokeOrphanedRuntimeTokens(args.graceMs, args.lagGraceMs, ctx.now, { afterId: args.afterId, limit: args.limit }),
   },
   "agents.revokeTokensForAgent": {
     offLoop: true,

package/src/db/agents.ts

@@ -1,3 +1,4 @@
+import { trackRelayEvent } from "../relay-event-drain";
 import { mainThreadRegistry } from "./main-thread-state";
 import { spawnTreeParentIds } from "./spawn-tree";
 import { recordSpawnProcessExit } from "./spawn-process-evidence";
@@ -8,6 +9,7 @@ import { STALE_TTL_MS, DAY_MS } from "../config";
 import { withoutAuthzMetaKeys } from "../authz-meta-keys";
 import {
   jsonBytes,
+  serializeProfileProjection,
   splitAgentMetaForStorage,
   splitAgentMetaPatchForStorage,
   topLevelJsonPath,
@@ -48,10 +50,10 @@ import type {
   QuotaState,
 } from "../types";
 
-type BeforeAgentStatusTransition = (agent: AgentCard, nextStatus: AgentCard["status"]) => void;
+type BeforeAgentStatusTransition = (agent: AgentCard, nextStatus: AgentCard["status"]) => unknown;
 const beforeAgentStatusTransitionHandlers = mainThreadRegistry("agents.ts:beforeAgentStatusTransitionHandlers", new Set<BeforeAgentStatusTransition>());
 
-/** Register synchronous work that must finish before a status transition is persisted. */
+/** Register work that must finish before a status transition is persisted. */
 export function onBeforeAgentStatusTransition(handler: BeforeAgentStatusTransition): () => void {
   beforeAgentStatusTransitionHandlers.add(handler);
   return () => beforeAgentStatusTransitionHandlers.delete(handler);
@@ -62,8 +64,9 @@ export function onBeforeAgentStatusTransition(handler: BeforeAgentStatusTransiti
  *  a writer-worker's own instance), so this must only ever be invoked main-thread-side: from
  *  `setStatus` directly, or from `setStatusAsync`'s pre-write orchestration. Never from the
  *  `agents.setStatus` op body. */
-export function runBeforeAgentStatusTransitionHandlers(before: AgentCard, next: AgentCard["status"]): void {
-  for (const handler of beforeAgentStatusTransitionHandlers) handler(before, next);
+export function runBeforeAgentStatusTransitionHandlers(before: AgentCard, next: AgentCard["status"]): Promise<void> {
+  const pending = [...beforeAgentStatusTransitionHandlers].map(handler => handler(before, next));
+  return trackRelayEvent(Promise.all(pending).then(() => {}), "agent.before-status");
 }
 
 const CONTEXT_SNAPSHOT_DEBOUNCE_MS = 60_000;
@@ -161,7 +164,7 @@ export function upsertAgentInternal(
   const contextState = mergeRelayContextMarkers(input.id, input.context);
   const storedMeta = splitAgentMetaForStorage(withCanonicalSpawnRequestId(input.meta));
   const metaJson = JSON.stringify(storedMeta.meta);
-  const profileProjectionJson = storedMeta.profileProjection == null ? null : JSON.stringify(storedMeta.profileProjection);
+  const profileProjectionJson = storedMeta.profileProjectionProvided ? serializeProfileProjection(input.id, storedMeta.profileProjection) : null;
   warnIfAgentMetaOversized(input.id, metaJson, "stored meta");
   const stmt = getDb().query(`
     INSERT INTO agents (id, name, kind, label, tags, system_tags, machine, rig, capabilities, ready, status, instance_id, epoch, provider_capabilities, context_state, quota_state, profile_projection, meta, lifecycle, spawned_by, team_id, status_seq, ready_seq, last_seen, created_at)
@@ -479,28 +482,51 @@ export function applyAgentLifecycleUpdate(
   guard?: AgentSessionGuard,
   now: number = Date.now(),
 ): AgentLifecycleUpdateResult {
-  const miss: AgentLifecycleUpdateResult = { ok: false, changed: false, metaWriteBytes: 0, before: null, after: null };
-  const before = getPersistedAgentRuntimeCard(id);
-  if (!before) return miss;
-  if (!agentSessionGuardResult(before, guard).ok) return miss;
+  const seq = nextLifecycleSeq();
+  const before = prepareAgentLifecycleUpdate(id, update, guard);
+  if (!before) return { ok: false, changed: false, metaWriteBytes: 0, before: null, after: null };
+  return { ok: true, before, ...inWriteTransaction(() => applyAgentLifecycleUpdateInternal(agentLifecycleWriteArgs(before, update, seq, now))) };
+}
 
+/** The writer-op args for one lifecycle delta (#1870 S7). `seq` is taken at the caller's entry (lifecycle-seq.ts
+ *  rule 1); `expectedEpoch` fences the write to the instance `before` was read from across the writer hop. */
+export interface AgentLifecycleWriteArgs {
+  id: string;
+  update: AgentLifecycleUpdate;
+  seq: number;
+  now: number;
+  expectedEpoch: number;
+  before: { instanceId?: string; spawnRequestId: string; terminalProviderExit?: unknown };
+}
+
+export function agentLifecycleWriteArgs(before: AgentCard, update: AgentLifecycleUpdate, seq: number, now: number): AgentLifecycleWriteArgs {
+  return { id: before.id, update, seq, now, expectedEpoch: before.epoch, before: { instanceId: before.instanceId, spawnRequestId: String(before.meta?.spawnRequestId ?? ""), terminalProviderExit: before.meta?.terminalProviderExit } };
+}
+
+/** The main-thread half of a lifecycle delta: guard, `before` snapshot, transition handlers and presence
+ *  eviction (main-thread state), all before the write is submitted. Null when the guard rejects. */
+export function prepareAgentLifecycleUpdate(id: string, update: AgentLifecycleUpdate, guard?: AgentSessionGuard, runHandlers = true): AgentCard | null {
+  const before = getPersistedAgentRuntimeCard(id);
+  if (!before || !agentSessionGuardResult(before, guard).ok) return null;
   const nextStatus = update.status;
-  // Pre-transition handlers are main-thread-only state (see runBeforeAgentStatusTransitionHandlers)
-  // and must run BEFORE the write lands, exactly as `setStatus` does.
-  if (nextStatus !== undefined && nextStatus !== before.status) runBeforeAgentStatusTransitionHandlers(before, nextStatus);
+  // Pre-transition handlers must run BEFORE the write lands, exactly as `setStatus` does.
+  if (runHandlers && nextStatus !== undefined && nextStatus !== before.status) runBeforeAgentStatusTransitionHandlers(before, nextStatus);
   // #1468 — a terminal transition evicts any buffered beat so a coalesced advance cannot later
   // resurrect a gone agent at flush.
   if (nextStatus === "offline" || nextStatus === "stale") evictPendingPresence(id);
+  return before;
+}
 
-  // #2111 review BLOCKER 1 — raise each written column's ordinal so a lifecycle write the coalescing
-  // window deferred cannot commit afterwards and undo it. This path only ever WINS its own races (it
-  // commits synchronously, and any submission carrying a higher ordinal has not been allocated yet),
-  // but it is fenced anyway so every writer of these columns obeys one rule.
-  const seq = nextLifecycleSeq();
+/** Op body (`agents.applyLifecycleUpdate`): the fenced UPDATE, its effects, and the offline post-commit, in the
+ *  caller's transaction. #2111 review BLOCKER 1 — each written column's ordinal is raised, so a lifecycle write
+ *  the coalescing window deferred cannot commit afterwards and undo it. */
+export function applyAgentLifecycleUpdateInternal(args: AgentLifecycleWriteArgs): Omit<AgentLifecycleUpdateResult, "ok" | "before"> {
+  const { id, update, seq, now, before } = args;
+  const nextStatus = update.status;
   // Named params throughout: the fenced-column fragments repeat `$seq`, which positional `?` could
   // only express by counting placeholders across a dynamically-built SET list.
   const sets: string[] = ["last_seen = $now"];
-  const params: Record<string, unknown> = { $now: now, $seq: seq, $id: id };
+  const params: Record<string, unknown> = { $now: now, $seq: seq, $id: id, $epoch: args.expectedEpoch };
   let metaWriteBytes = 0;
   if (nextStatus !== undefined) {
     sets.push(fencedColumnSql("status", "$status", "$seq"));
@@ -543,7 +569,7 @@ export function applyAgentLifecycleUpdate(
     }
     if (patch.profileProjectionProvided) {
       sets.push("profile_projection = $profileProjection");
-      params.$profileProjection = patch.profileProjection == null ? null : JSON.stringify(patch.profileProjection);
+      params.$profileProjection = serializeProfileProjection(id, patch.profileProjection);
     }
   }
   const runtime = update.runtime;
@@ -558,16 +584,10 @@ export function applyAgentLifecycleUpdate(
     params.$qs = runtime?.quota ? JSON.stringify(runtime.quota) : null;
   }
 
-  // BEGIN IMMEDIATE (#1428/#1431) so busy_timeout governs contention, joining an ambient writer-op
-  // transaction rather than nesting when this body is reached through the gateway.
-  const changed = inWriteTransaction(() => {
-    const applied = getDb().query(`UPDATE agents SET ${sets.join(", ")} WHERE id = $id`).run(params as never).changes > 0;
-    const terminalLaunch = terminalProviderLaunchId(update.meta?.terminalProviderExit ?? before.meta?.terminalProviderExit);
-    if (applied && nextStatus === "offline" && terminalLaunch) revokeVaultTokens(id, now, "provider_terminal_exit", before.instanceId, String(before.meta?.spawnRequestId ?? ""), terminalLaunch);
-    if (applied && contextState) recordContextSnapshot(id, contextState, now);
-    return applied;
-  });
-
+  const changed = getDb().query(`UPDATE agents SET ${sets.join(", ")} WHERE id = $id AND epoch = $epoch`).run(params as never).changes > 0;
+  const terminalLaunch = terminalProviderLaunchId(update.meta?.terminalProviderExit ?? before.terminalProviderExit);
+  if (changed && nextStatus === "offline" && terminalLaunch) revokeVaultTokens(id, now, "provider_terminal_exit", before.instanceId, before.spawnRequestId, terminalLaunch);
+  if (changed && contextState) recordContextSnapshot(id, contextState, now);
   // NOTE (from setStatus): `offline` is a LIVENESS observation, not a confirmed death — it re-elects
   // stewards and closes pairs but must NOT release a steward's workspace claim (#1789 part 2, F2).
   // Gated on the requested status, not on a status CHANGE, matching setStatus's own condition.
@@ -575,7 +595,7 @@ export function applyAgentLifecycleUpdate(
     closeOpenPairsForAgent(id, now);
     electWorkspaceStewards();
   }
-  return { ok: true, changed, metaWriteBytes, before, after: getPersistedAgentRuntimeCard(id) };
+  return { changed, metaWriteBytes, after: getPersistedAgentRuntimeCard(id) };
 }
 
 export function mergeAgentMeta(id: string, meta: Record<string, unknown>, guard?: AgentSessionGuard, now: number = Date.now()): boolean {
@@ -598,7 +618,7 @@ export function mergeAgentMeta(id: string, meta: Record<string, unknown>, guard?
   warnIfAgentMetaOversized(id, metaJson, "stored meta");
   const result = getDb()
     .query("UPDATE agents SET meta = ?, profile_projection = ?, last_seen = ? WHERE id = ?")
-    .run(metaJson, stored.profileProjection == null ? null : JSON.stringify(stored.profileProjection), now, id);
+    .run(metaJson, serializeProfileProjection(id, stored.profileProjection), now, id);
   return result.changes > 0;
 }
 
@@ -614,7 +634,7 @@ export function markAgentReadyNotified(id: string, at: number = Date.now()): boo
   const merged = { ...(agent.meta ?? {}), readyNotifiedAt: at };
   const stored = splitAgentMetaForStorage(merged);
   return getDb().query("UPDATE agents SET meta = ?, profile_projection = ? WHERE id = ?")
-    .run(JSON.stringify(stored.meta), stored.profileProjection == null ? null : JSON.stringify(stored.profileProjection), id).changes > 0;
+    .run(JSON.stringify(stored.meta), serializeProfileProjection(id, stored.profileProjection), id).changes > 0;
 }
 
 /** Delete specific keys from an agent's meta blob. Returns true if anything changed. Used to clear
@@ -633,7 +653,7 @@ export function clearAgentMetaKeys(id: string, keys: string[]): boolean {
   if (!changed) return false;
   const stored = splitAgentMetaForStorage(meta);
   return getDb().query("UPDATE agents SET meta = ?, profile_projection = ? WHERE id = ?")
-    .run(JSON.stringify(stored.meta), stored.profileProjection == null ? null : JSON.stringify(stored.profileProjection), id).changes > 0;
+    .run(JSON.stringify(stored.meta), serializeProfileProjection(id, stored.profileProjection), id).changes > 0;
 }
 
 /**
@@ -724,7 +744,7 @@ export function recordAgentExitDiagnosticsInternal(
     .run({
       $seq: lifecycleSeq,
       $meta: JSON.stringify(stored.meta),
-      $profileProjection: stored.profileProjection == null ? null : JSON.stringify(stored.profileProjection),
+      $profileProjection: serializeProfileProjection(id, stored.profileProjection),
       $id: id,
     } as never);
   return result.changes > 0 ? agent.epoch : null;
@@ -828,21 +848,27 @@ export function heartbeat(
 // withBufferedPresence / bufferedLiveAgentIds) live in the leaf module ./presence-buffer.ts so every
 // liveness reader can fold in the buffer without an import cycle (agents.ts imports channels.ts).
 
+/** Heartbeat: buffers the steady-state liveness advance (flushed by `flushPresence`), writes presence-critical
+ * transitions at once. False if the session guard rejects or the agent is gone. Sync twin for tests; the
+ * transports use `recordHeartbeatAsync`, which writes the bypass on the writer thread (#1870 S7). */
+export function recordHeartbeat(id: string, opts: HeartbeatInput = {}): boolean {
+  const beat = bufferOrBypassHeartbeat(id, opts);
+  return typeof beat === "boolean" ? beat : applyAgentLifecycleUpdate(id, beat, opts.guard, opts.now ?? Date.now()).changed;
+}
+
+export interface HeartbeatInput {
+  guard?: AgentSessionGuard;
+  /** Semantic status reported by the beat (bus path); undefined for the runtime-less API path. */
+  status?: AgentCard["status"];
+  runtime?: { providerCapabilities?: ProviderCapabilities; context?: ContextState; quota?: QuotaState };
+  now?: number;
+}
+
 /**
- * Hot-path heartbeat entry point for the bus/API transports. Coalesces the steady-state liveness
- * advance into the in-memory buffer (flushed by `flushPresence`), but writes synchronously for
- * presence-critical transitions. Returns false if the session guard rejects or the agent is gone.
+ * The shared decision of `recordHeartbeat` and `recordHeartbeatAsync`: buffer a steady-state beat (returns
+ * the result) or hand back the lifecycle delta a presence-critical beat must write now.
  */
-export function recordHeartbeat(
-  id: string,
-  opts: {
-    guard?: AgentSessionGuard;
-    /** Semantic status reported by the beat (bus path); undefined for the runtime-less API path. */
-    status?: AgentCard["status"];
-    runtime?: { providerCapabilities?: ProviderCapabilities; context?: ContextState; quota?: QuotaState };
-    now?: number;
-  } = {},
-): boolean {
+export function bufferOrBypassHeartbeat(id: string, opts: HeartbeatInput): boolean | AgentLifecycleUpdate {
   if (!validateAgentSession(id, opts.guard).ok) return false;
   const current = getAgent(id);
   if (!current) return false;
@@ -861,9 +887,9 @@ export function recordHeartbeat(
     opts.runtime.context.lifecycleState !== current.context?.lifecycleState;
   const prev = pendingPresence.get(id);
   if (resurrection || statusChanged || lifecycleTransition) {
-    // Merge any buffered-but-unflushed runtime cols into this synchronous write so the bypass never
-    // DISCARDS a coalesced advance (a beat that carried context but was buffered, then a status
-    // change arrives carrying none). Per column: this beat wins, else the pending value.
+    // Merge any buffered-but-unflushed runtime cols into this write so the bypass never DISCARDS a
+    // coalesced advance (a beat that carried context but was buffered, then a status change arrives
+    // carrying none). Per column: this beat wins, else the pending value.
     const runtime = prev
       ? {
           providerCapabilities: opts.runtime?.providerCapabilities ?? prev.providerCapabilities,
@@ -871,16 +897,10 @@ export function recordHeartbeat(
           quota: opts.runtime?.quota ?? prev.quota,
         }
       : opts.runtime;
-    pendingPresence.delete(id); // this synchronous write supersedes any buffered advance
-    // #2111 — was `heartbeat()` (one UPDATE) followed by `setStatus()` (a second UPDATE of the SAME
-    // row inside its own BEGIN IMMEDIATE). One consolidated update now carries both; the explicit
-    // status overrides the resurrection CASE exactly as the second write used to.
-    return applyAgentLifecycleUpdate(
-      id,
-      { status: statusChanged ? opts.status : undefined, resurrectOnBeat: true, runtime },
-      opts.guard,
-      now,
-    ).changed;
+    pendingPresence.delete(id); // this write supersedes any buffered advance
+    // #2111 — one consolidated update carries both the beat and the explicit status, which overrides
+    // the resurrection CASE.
+    return { status: statusChanged ? opts.status : undefined, resurrectOnBeat: true, runtime };
   }
 
   // Steady state: buffer. Merge context advisory markers now (matches heartbeat()'s per-beat merge),
@@ -1192,6 +1212,10 @@ export function deleteAgent(id: string): { ok: boolean; error?: string } {
   return deleted ? { ok: true } : { ok: false, error: "agent not found" };
 }
 
+export type OrphanedTokenRevocation = { agentId: string; jtis: string[] };
+/** One page of the orphan sweep: agents after `afterId` in id order, up to `limit` (#3776: one short write each). */
+export type OrphanedTokenPage = { revoked: OrphanedTokenRevocation[]; nextAfterId: string | null };
+
 // #1057(1) — close the orphan-token window #1051 left open. A dead runner with no replacement
 // used to keep its token valid until the 24h offline prune (~720× the pre-#1051 ~2min horizon).
 // This sweep revokes ("stale", re-mintable — #484) the runtime tokens of agents whose process the
@@ -1204,12 +1228,18 @@ export function deleteAgent(id: string): { ok: boolean; error?: string } {
 // `lagGraceMs` mirrors the stale reaper's #1048/#1055 discipline: while the relay's own event
 // loop was stalled, a live claimant's heartbeats may not have been processed yet, so the window
 // widens by the observed lag rather than misreading the backlog as post-mortem silence.
-export function revokeOrphanedRuntimeTokens(graceMs: number = STALE_TTL_MS, lagGraceMs = 0, now = Date.now()): Array<{ agentId: string; jtis: string[] }> {
+export function revokeOrphanedRuntimeTokens(
+  graceMs: number = STALE_TTL_MS,
+  lagGraceMs = 0,
+  now = Date.now(),
+  page: { afterId?: string; limit?: number } = {},
+): OrphanedTokenPage {
   const cutoff = now - graceMs - Math.max(0, lagGraceMs);
+  const limit = page.limit ?? -1;
   const rows = getDb()
-    .query(`SELECT id, meta, last_seen FROM agents WHERE status = 'offline' AND meta LIKE '%lastExitReceivedAt%' AND NOT ${reservedAgentIdSql()}`)
-    .all() as Array<{ id: string; meta: string | null; last_seen: number }>;
-  const revoked: Array<{ agentId: string; jtis: string[] }> = [];
+    .query(`SELECT id, meta, last_seen FROM agents WHERE status = 'offline' AND meta LIKE '%lastExitReceivedAt%' AND NOT ${reservedAgentIdSql()} AND id > ? ORDER BY id LIMIT ?`)
+    .all(page.afterId ?? "", limit) as Array<{ id: string; meta: string | null; last_seen: number }>;
+  const revoked: OrphanedTokenRevocation[] = [];
   for (const row of rows) {
     const meta = parseJson<Record<string, unknown>>(row.meta ?? "{}", {});
     const receivedAt = typeof meta.lastExitReceivedAt === "number" ? meta.lastExitReceivedAt : undefined;
@@ -1218,7 +1248,7 @@ export function revokeOrphanedRuntimeTokens(graceMs: number = STALE_TTL_MS, lagG
     const jtis = revokeProviderRuntimeTokensByJti(runtimeTokenJtisFromMeta(meta), "stale", now);
     if (jtis.length > 0) revoked.push({ agentId: row.id, jtis });
   }
-  return revoked;
+  return { revoked, nextAfterId: limit > 0 && rows.length === limit ? rows[rows.length - 1]!.id : null };
 }
 
 // #484: `reason` is stamped on each revocation so a stale-on-restart revocation

package/src/db/auth-identities.ts

@@ -93,9 +93,9 @@ export function createAuthIdentity(input: {
   providerSubject: string;
   secretHash?: string | null;
   metadata?: Record<string, unknown>;
-}): AuthIdentity {
+}, submittedAt: number = Date.now()): AuthIdentity {
   const id = randomUUID();
-  const now = Date.now();
+  const now = submittedAt;
   getDb()
     .query(
       `INSERT INTO auth_identities (id, user_id, provider, provider_subject, secret_hash, metadata, created_at)

package/src/db/automation-read-ops.ts

@@ -0,0 +1,11 @@
+// #3799 — automation scans (run listing, reconcile/precheck/dispatch ticks) on the reader pool.
+// A cold scan of automation_runs once blocked the hub loop for seconds on swap-bound I/O.
+import { listAutomationRuns, listDispatchableQueuedRuns, listDueAutomations, listPendingPrecheckRuns, listReconcilableRuns } from "../automations/storage.ts";
+
+export const AUTOMATION_READ_OPS = {
+  "automations.listRuns": (args: { automationId?: string; status?: string; limit?: number }) => listAutomationRuns(args),
+  "automations.pendingPrecheckRuns": () => listPendingPrecheckRuns(),
+  "automations.reconcilableRuns": () => listReconcilableRuns(),
+  "automations.dispatchableQueuedRuns": () => listDispatchableQueuedRuns(),
+  "automations.dueAutomations": (args: { now: number }) => listDueAutomations(args.now),
+};

package/src/db/automation-write-ops.ts

@@ -0,0 +1,46 @@
+import { ValidationError } from "@humwick/sdk";
+import { insertAutomation, updateAutomationNormalized, deleteAutomation } from "../automations/persistence";
+import { nextScheduledAt } from "../automations/cron";
+import { getAutomation, getAutomationRun, hasActiveRun, insertRun, enqueueAutomationRun, updateRun, cancelActiveRuns, rescheduleAutomation } from "../automations/storage";
+import type { Automation } from "../types";
+import type { WriterOpSpec } from "./write-ops";
+
+function admitRun(input: { automationId: string; now: number; manual?: boolean; queuedRunId?: string }) {
+  const automation = getAutomation(input.automationId);
+  if (!automation) return null;
+  const active = hasActiveRun(automation.id);
+  if (input.queuedRunId) {
+    const run = getAutomationRun(input.queuedRunId);
+    if (!run || run.automationId !== automation.id || run.status !== "scheduled" || active) return null;
+    updateRun(run.id, { status: "dispatching" }, input.now);
+    return { automation, run: getAutomationRun(run.id)! };
+  }
+  if (input.manual) {
+    if (active) throw new ValidationError("automation already has an active run");
+  } else {
+    if (!automation.enabled || automation.nextRunAt === undefined || automation.nextRunAt > input.now) return null;
+    if (active && automation.concurrencyPolicy === "replace") cancelActiveRuns(automation.id, input.now, "Replaced by newer scheduled run");
+    else if (active && automation.concurrencyPolicy === "queue") enqueueAutomationRun(automation, automation.nextRunAt, input.now);
+    const next = automation.enabled ? nextScheduledAt(automation.schedule, automation.timezone, input.now) : undefined;
+    rescheduleAutomation(automation, next, input.now);
+    if (active && automation.concurrencyPolicy !== "replace") return null;
+  }
+  const run = insertRun(automation, input.manual ? input.now : automation.nextRunAt ?? input.now, input.now);
+  return { automation: getAutomation(automation.id)!, run };
+}
+
+export const AUTOMATION_WRITE_OPS: Readonly<Record<string, WriterOpSpec>> = Object.freeze({
+  "automations.create": { offLoop: true, idempotent: false, fn: (a: { normalized: Parameters<typeof insertAutomation>[0]; now: number }) => insertAutomation(a.normalized, a.now) },
+  "automations.update": { offLoop: true, idempotent: false, fn: (a: { id: string; normalized: Parameters<typeof updateAutomationNormalized>[1]; expected: Automation; now: number }) => {
+    const current = getAutomation(a.id);
+    if (!current || JSON.stringify(current) !== JSON.stringify(a.expected)) throw new ValidationError("automation changed or was deleted during catalog validation");
+    return updateAutomationNormalized(a.id, a.normalized, a.now);
+  } },
+  "automations.delete": { offLoop: true, idempotent: true, fn: (a: { id: string }) => deleteAutomation(a.id) },
+  "automations.admitRun": { offLoop: true, idempotent: false, fn: admitRun },
+  "automations.insertRun": { offLoop: true, idempotent: false, fn: (a: Parameters<typeof insertRun>) => insertRun(...a) },
+  "automations.enqueueRun": { offLoop: true, idempotent: false, fn: (a: Parameters<typeof enqueueAutomationRun>) => enqueueAutomationRun(...a) },
+  "automations.updateRun": { offLoop: true, idempotent: false, fn: (a: Parameters<typeof updateRun>) => updateRun(...a) ?? null },
+  "automations.cancelRuns": { offLoop: true, idempotent: false, fn: (a: Parameters<typeof cancelActiveRuns>) => cancelActiveRuns(...a) ?? null },
+  "automations.reschedule": { offLoop: true, idempotent: false, fn: (a: Parameters<typeof rescheduleAutomation>) => rescheduleAutomation(...a) ?? null },
+});

package/src/db/blackboard.ts

@@ -161,11 +161,11 @@ export function searchEntries(
   return rows.map(rowToEntry);
 }
 
-export function retractEntry(id: string): BlackboardEntry | null {
+export function retractEntry(id: string, submittedAt: number = Date.now()): BlackboardEntry | null {
   const existing = getEntry(id);
   if (!existing) return null;
   assertStatus("retracted");
   getDb().query("UPDATE blackboard_entries SET status = 'retracted', updated_at = ? WHERE id = ?")
-    .run(Date.now(), id);
+    .run(submittedAt, id);
   return getEntry(id);
 }

package/src/db/browser-session-write.ts

@@ -0,0 +1,32 @@
+import { getAgent } from "./agents";
+import { createCommandInternal } from "../commands-db";
+import { countActiveBrowserSessions, getBrowserSessionRow, insertBrowserSession, transitionBrowserSession } from "./browser-sessions";
+import { getOrchestrator } from "./orchestrators";
+import type { BrowserLimits } from "@humwick/sdk";
+
+export interface StartBrowserWrite {
+  id: string; commandId: string; agentId: string; instanceId?: string; epoch?: number;
+  orchestratorId: string; limits: BrowserLimits;
+}
+export function startBrowserBody(input: StartBrowserWrite, now: number) {
+  const agent = getAgent(input.agentId);
+  if (!agent || agent.status === "offline" || agent.instanceId !== input.instanceId || agent.epoch !== input.epoch) return { conflict: "agent" as const };
+  const host = getOrchestrator(input.orchestratorId);
+  if (!host || host.status !== "online") return { conflict: "host" as const };
+  if (countActiveBrowserSessions({ agentId: input.agentId }) >= input.limits.perAgent) return { conflict: "agentLimit" as const };
+  if (countActiveBrowserSessions({ orchestratorId: host.id }) >= input.limits.perHost) return { conflict: "hostLimit" as const };
+  const command = createCommandInternal({ id: input.commandId, input: { type: "browser.start", source: input.agentId, target: host.agentId, correlationId: input.id,
+    params: { sessionId: input.id, agentId: input.agentId, orchestratorId: host.id, idleTimeoutMs: input.limits.idleTimeoutMinutes * 60_000, hostSessionLimit: input.limits.perHost } } }, now);
+  const row = insertBrowserSession({ id: input.id, agentId: input.agentId, orchestratorId: host.id, startCommandId: command.id, now });
+  return { row, command };
+}
+
+export function stopBrowserBody(input: { id: string; reason: string; commandId: string }, now: number) {
+  const row = getBrowserSessionRow(input.id);
+  if (!row || (row.status !== "starting" && row.status !== "running")) return { row };
+  const host = getOrchestrator(row.orchestratorId);
+  if (!host) return { row: transitionBrowserSession(row.id, ["starting", "running"], { status: "stopped", stoppedAt: now, error: "orchestrator no longer registered" }, now) };
+  const command = createCommandInternal({ id: input.commandId, input: { type: "browser.stop", source: "system", target: host.agentId, correlationId: row.id,
+    params: { sessionId: row.id, agentId: row.agentId, orchestratorId: host.id, reason: input.reason } } }, now);
+  return { row: transitionBrowserSession(row.id, ["starting", "running"], { status: "stopping", stopCommandId: command.id }, now), command };
+}

package/src/db/bus-outbox-write-ops.ts

@@ -31,8 +31,10 @@ export const BUS_OUTBOX_WRITE_OPS: Readonly<Record<string, WriterOpSpec>> = Obje
   "bus.append": {
     offLoop: true,
     // Appending assigns a new sequence row; retries reconcile by idempotency key rather than
-    // inserting a second durable event after a phantom acknowledgement.
+    // inserting a second durable event after a phantom acknowledgement. A plain emit never retries,
+    // so it goes keyless and writes no ledger row (#3614).
     idempotent: false,
+    atMostOnce: true,
     fn: (args: BusAppendOpArgs, ctx: WriterOpContext) => ({
       ...appendRelayEventInternal(args.type, args.source, args.data, args.subject, ctx.now)!, replayed: false,
     }),