@zgeoff/atc 2.11.0 → 2.12.0

package/src/daemon/daemon.ts

@@ -2,13 +2,18 @@ import { unlinkSync, writeFileSync } from 'node:fs';
 import { hostname } from 'node:os';
 import { dirname, join } from 'node:path';
 import type { AdapterEvent, AgentAdapter } from '../agents/agent-adapter';
+import { DaemonError } from '../protocol/daemon-error';
 import { MAX_CHUNK, PROTOCOL_V } from '../protocol/protocol';
 import type { EventMsg } from '../protocol/protocol';
 import type { HooksConfig } from '../shared/collect-hooks';
+import type { TargetConfigError } from '../shared/collect-targets';
 import { findDaemonRecord } from '../shared/find-daemon-record';
 import type { MessageID } from '../shared/message-id';
+import { isRecord } from '../shared/report';
 import type { SessionID } from '../shared/session-id';
+import { toMessageID } from '../shared/to-message-id';
 import { truncateToBytes } from '../shared/truncate-to-bytes';
+import type { IdempotencyRecord } from '../store/idempotency-record';
 import type { MessageOwner } from '../store/message-owner';
 import type { MessageRecord, MessageStatus } from '../store/message-record';
 import { StateStore } from '../store/state-store';
@@ -17,22 +22,37 @@ import type { TrailEntry } from '../store/trail-entry';
 import { ANSWER_BYTE_CAP } from './answer-byte-cap';
 import { AttachRegistry } from './attach-registry';
 import { buildAgentList } from './build-agent-list';
+import { buildConfigRevision } from './build-config-revision';
+import { buildExecutionTargets } from './build-execution-targets';
+import type { ExecutionTarget } from './build-execution-targets';
 import { buildFleetEvents } from './build-fleet-events';
 import { buildMessageTrailEntry } from './build-message-trail-entry';
 import { buildReportTrailEntry } from './build-report-trail-entry';
 import { buildSessionEvent } from './build-session-event';
 import { buildSessionMessageEvent } from './build-session-message-event';
 import { buildSessionReportEvent } from './build-session-report-event';
+import { buildTargetAccess } from './build-target-access';
+import { buildTargetForbiddenError } from './build-target-forbidden-error';
+import { buildTargetList } from './build-target-list';
 import { claimDaemonLock } from './claim-daemon-lock';
 import { DaemonConnection } from './daemon-connection';
-import type { DaemonContext, OutputClient, TapClient } from './daemon-connection';
+import type {
+  DaemonContext,
+  MessageRefusal,
+  OutputClient,
+  SpawnParams,
+  TapClient,
+} from './daemon-connection';
+import { EffectRemainsError } from './effect-remains-error';
 import { EventSignal } from './event-signal';
 import { startHookServer } from './hooks';
 import type { HookEvent } from './hooks';
+import { IdempotencyLedger } from './idempotency-ledger';
 import { loadTranscriptPage } from './load-transcript-page';
 import { makeHookRunner } from './make-hook-runner';
 import type { HookScope } from './make-hook-runner';
 import { mintMessageID } from './mint-message-id';
+import { mintSessionID } from './mint-session-id';
 import { parseReport } from './parse-report';
 import { PermissionRegistry } from './permission-registry';
 import { restoreFleet } from './restore-fleet';
@@ -44,6 +64,7 @@ import type { Session, SessionDescriptor, SessionState } from './sessions';
 import { startEventsServer } from './start-events-server';
 import { startHeadlessTurn } from './start-headless-turn';
 import { TapRegistry } from './tap-registry';
+import type { TargetAccess } from './target-access';
 import { writeDaemonRecord } from './write-daemon-record';
 
 export interface DaemonOptions {
@@ -59,6 +80,21 @@ export interface DaemonOptions {
   // grok adapter is unsupported, not a Claude spawn.
   readonly adapters?: readonly AgentAdapter[];
 
+  // Where sessions run, in config order; one `local` target on the local
+  // pseudo-terminal provider when unset. A spawn without a target runs on
+  // the default one: `local` when unset and the targets hold it, and none
+  // otherwise, which refuses such a spawn.
+  readonly targets?: readonly ExecutionTarget[];
+  readonly defaultTarget?: string | null;
+
+  // The target config problems the daemon started with. A target they cover
+  // refuses every session, and `agents.list` returns them.
+  readonly targetErrors?: readonly TargetConfigError[];
+
+  // The targets each principal may use; unset or null when the config has
+  // no principals, which leaves every principal the implicit local target.
+  readonly principals?: ReadonlyMap<string, readonly string[]> | null;
+
   // SQLite path for daemon state; a fleet.json at legacyFleetPath seeds the
   // fleet table once so upgrading keeps the restorable fleet.
   readonly dbPath: string;
@@ -109,6 +145,14 @@ export interface DaemonHandle {
 // it is refused.
 const TAP_GRACE_MS = 15_000;
 
+// The principal every request on the local socket acts as.
+const LOCAL_PRINCIPAL = 'local';
+
+// A completed idempotency key is kept this long after it completed, and the
+// sweep that drops older ones runs this often.
+const IDEMPOTENCY_TTL_MS = 24 * 60 * 60 * 1000;
+const IDEMPOTENCY_SWEEP_MS = 60 * 60 * 1000;
+
 // How long startup waits for a daemon that is shutting down to release the
 // state lock before refusing to start.
 const LOCK_WAIT_MS = 2000;
@@ -148,7 +192,42 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
 
   const store = await StateStore.open(opts.dbPath, opts.legacyFleetPath);
 
-  const mgr = new SessionManager(opts.adapter, store, opts.statusPath, opts.adapters ?? []);
+  // Before any request is served: a key left in progress by a daemon that
+  // stopped mid-effect is settled first, so no retry can race it.
+  await store.reconcileIdempotencyKeys(Date.now());
+
+  await tryRemoveExpiredIdempotencyKeys(store);
+
+  const idempotencySweep = setInterval(() => {
+    void tryRemoveExpiredIdempotencyKeys(store);
+  }, IDEMPOTENCY_SWEEP_MS);
+
+  idempotencySweep.unref();
+
+  const targets =
+    opts.targets ?? buildExecutionTargets([{ id: 'local', provider: 'local-pty', options: {} }]);
+
+  const targetErrors = opts.targetErrors ?? [];
+  const principals = opts.principals ?? null;
+
+  const targetsByID = new Map(targets.map((target) => [target.id, target]));
+
+  const defaultTarget =
+    opts.defaultTarget === undefined
+      ? (targets.find((target) => target.id === 'local')?.id ?? null)
+      : opts.defaultTarget;
+
+  const configRevision = buildConfigRevision(targets, defaultTarget, targetErrors);
+
+  const mgr = new SessionManager(
+    opts.adapter,
+    store,
+    opts.statusPath,
+    opts.adapters ?? [],
+    targets,
+    targetErrors,
+  );
+
   const clients = new Set<DaemonConnection>();
 
   const runHooks = makeHookRunner(opts.hooks ?? {});
@@ -207,11 +286,37 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
   };
 
   // A live session's scope also matches the rows it wrote under an earlier
-  // atc id, through the agent session id a restore carries on.
-  const buildEventScope = (sessionID: SessionID): EventScope => ({
-    atcID: sessionID,
-    agentSessionID: mgr.sessions.find((x) => x.id === sessionID)?.agentSessionID,
-  });
+  // atc id, through the agent session id a restore carries on. Under an
+  // access, the scope holds only the sessions on targets the access holds,
+  // so a session outside it matches nothing, as a session never seen does.
+  const buildEventScope = (
+    sessionID: SessionID | null,
+    access: TargetAccess | null,
+  ): EventScope | null => {
+    if (access === null) {
+      if (sessionID === null) {
+        return null;
+      }
+
+      const agentSessionID = mgr.sessions.find((x) => x.id === sessionID)?.agentSessionID;
+
+      return {
+        atcIDs: [sessionID],
+        agentSessionIDs: agentSessionID === undefined ? [] : [agentSessionID],
+      };
+    }
+
+    const visible = mgr.sessions.filter(
+      (x) => access.canUse(x) && (sessionID === null || x.id === sessionID),
+    );
+
+    return {
+      atcIDs: visible.map((x) => x.id),
+      agentSessionIDs: visible.flatMap((x) =>
+        x.agentSessionID === undefined ? [] : [x.agentSessionID],
+      ),
+    };
+  };
 
   // A message belongs to the live session holding its atc id or its agent
   // session id, else to the atc id it was sent to.
@@ -421,7 +526,12 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
 
     const s = mgr.sessions.find((x) => x.id === sessionID);
 
-    s?.pty?.resize(dims.cols, dims.rows);
+    // A host that cannot resize keeps its terminal at the size it started
+    // with; the screen model still follows the clients.
+    if (s !== undefined && (mgr.findProvider(s)?.capabilities.resize ?? false)) {
+      s.pty?.resize(dims.cols, dims.rows);
+    }
+
     runtime?.screen?.updateDims(dims.cols, dims.rows);
 
     if (runtime !== undefined) {
@@ -652,8 +762,199 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
     }
   }, opts.reporterSocketPath);
 
+  // A spawn that throws once its process has started takes the session back
+  // before it throws, so a failed start leaves nothing running and a keyed
+  // retry spawns once. When taking it back fails too, the session may still
+  // stand, and the throw says so.
+  const startSpawn = async (p: SpawnParams, id: SessionID): Promise<SessionDescriptor> => {
+    try {
+      return startSpawnedSession(p, id);
+    } catch (error) {
+      try {
+        await mgr.removeFailedSpawn(id);
+      } catch (cleanupError) {
+        throw new EffectRemainsError(
+          `spawn of session ${id} failed and taking it back failed too`,
+          {
+            cause: cleanupError,
+          },
+        );
+      }
+
+      throw error;
+    }
+  };
+
+  const startSpawnedSession = (p: SpawnParams, id: SessionID): SessionDescriptor => {
+    const s = mgr.spawn(
+      p.cwd,
+      p.name,
+      p.prompt,
+      p.cols,
+      p.rows,
+      p.resume,
+      p.namedBy,
+      p.agent,
+      p.parent,
+      p.overrides,
+      id,
+      p.target,
+    );
+
+    const runtime = runtimes.get(s.id);
+
+    if (runtime !== undefined) {
+      runtime.pendingLastUsed = true;
+      runtime.dims = { cols: p.cols, rows: p.rows };
+
+      runtime.screen = new ScreenModel(p.cols, p.rows);
+    }
+
+    void store.recordSpawnDir(p.cwd);
+
+    return getDescriptor(mgr, s.id);
+  };
+
+  // A retried spawn answers with the session as it stands now when it is
+  // listed, else with the descriptor the first spawn answered. A key that
+  // start-up reconciliation completed has no stored answer, and its session
+  // lists only once the fleet is restored.
+  const loadSpawnReplay = (record: IdempotencyRecord): Readonly<Record<string, unknown>> => {
+    const listed = mgr.collectDescriptors().find((d) => d.id === record.effectRef);
+
+    if (listed !== undefined) {
+      return { session: listed };
+    }
+
+    const stored: unknown = record.result === null ? null : JSON.parse(record.result);
+
+    if (isRecord(stored)) {
+      return stored;
+    }
+
+    throw new DaemonError(
+      'no_such_session',
+      `the spawn under idempotency key '${record.key}' created session '${record.effectRef}', which is not listed; restore the fleet to list it`,
+      { effectRef: record.effectRef },
+    );
+  };
+
+  const ledger = new IdempotencyLedger(store, LOCAL_PRINCIPAL);
+
+  // Throws the refusal a fresh spawn to the target gets when the access
+  // does not reach the target, and the identity, a held spawn key recorded
+  // for its session. A key that records none is refused: nothing it holds
+  // shows where its session ran.
+  const requireReplayInReach = (record: IdempotencyRecord, access: TargetAccess): void => {
+    const bound = record.effectTarget;
+
+    if (bound === null) {
+      throw buildTargetForbiddenError(findReplayTarget(record));
+    }
+
+    if (!access.canUse(bound)) {
+      throw buildTargetForbiddenError(bound.target);
+    }
+  };
+
+  // Why the session refuses a message right now, or null when it takes one.
+  const findMessageRefusal = (sessionID: SessionID): MessageRefusal | null => {
+    const s = mgr.sessions.find((x) => x.id === sessionID);
+
+    if (s === undefined) {
+      return 'missing';
+    }
+
+    if (!(s.pty !== null || (s.kind === 'headless' && s.state !== 'exited'))) {
+      return 'dead';
+    }
+
+    if (mgr.findAdapter(s.agent)?.takesMessages !== true) {
+      return 'unsupported';
+    }
+
+    const runtime = runtimes.get(sessionID);
+
+    if (
+      runtime !== undefined &&
+      runtime.startedAt !== null &&
+      !runtime.tapAttached &&
+      Date.now() - runtime.startedAt >= (opts.tapGraceMs ?? TAP_GRACE_MS)
+    ) {
+      return 'no_tap';
+    }
+
+    return null;
+  };
+
+  // Writes the message under the id given, notes it in the trail, and hands
+  // it to the session's tap when one is attached. Only the write itself can
+  // reject: every later step swallows its own failure, so a rejection means
+  // no message was written.
+  const writeAcceptedMessage = async (
+    sessionID: SessionID,
+    from: string,
+    text: string,
+    id: MessageID,
+  ): Promise<MessageRecord> => {
+    const s = mgr.sessions.find((x) => x.id === sessionID);
+    const previousWrite = lastMessageWrite;
+    const written = Promise.withResolvers<void>();
+
+    lastMessageWrite = written.promise;
+
+    await previousWrite;
+
+    const record: MessageRecord = {
+      id,
+      atcID: sessionID,
+      ...(s?.agentSessionID === undefined ? {} : { agentSessionID: s.agentSessionID }),
+      from,
+      text,
+      status: 'accepted',
+      sentAt: Date.now(),
+    };
+
+    try {
+      await store.writeMessage(record);
+
+      await recordMessageStatus(sessionID, record);
+    } finally {
+      written.resolve();
+    }
+
+    await drainInbox(sessionID);
+
+    return record;
+  };
+
+  // A retried send answers with the message's current status, so a caller
+  // that lost the first answer learns where its message stands now.
+  const loadMessageReplay = async (
+    record: IdempotencyRecord,
+  ): Promise<Readonly<Record<string, unknown>>> => {
+    const current = await store.findMessageByID(toMessageID(record.effectRef));
+
+    if (current !== null) {
+      return { message: current.id, status: current.status };
+    }
+
+    const stored: unknown = record.result === null ? null : JSON.parse(record.result);
+
+    if (isRecord(stored)) {
+      return stored;
+    }
+
+    throw new DaemonError(
+      'internal',
+      `the message under idempotency key '${record.key}' has no stored row`,
+      { effectRef: record.effectRef },
+    );
+  };
+
   const ctx: DaemonContext = {
     build: opts.build,
+    daemonID: store.daemonID,
     collectSessions: () => mgr.collectDescriptors(),
     collectSpawnDirs: () => store.collectSpawnDirs(),
     collectAgents: () => ({
@@ -664,35 +965,63 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
         build: opts.build,
       },
       agents: buildAgentList(mgr.collectAdapters(), (bin) => Bun.which(bin) !== null),
+      targets: buildTargetList(targets, defaultTarget),
+      spawnDefaults: { agent: 'claude', target: defaultTarget },
+      configRevision,
+      targetErrors,
     }),
     collectFleet: () => store.loadFleet(),
     loadLastUsedAgent: () => store.loadLastUsedAgent(),
     findAdapter: (kind) => mgr.findAdapter(kind),
-    spawnSession: (p) => {
-      const s = mgr.spawn(
-        p.cwd,
-        p.name,
-        p.prompt,
-        p.cols,
-        p.rows,
-        p.resume,
-        p.namedBy,
-        p.agent,
-        p.parent,
-      );
-
-      const runtime = runtimes.get(s.id);
+    buildTargetAccess: (principal) => buildTargetAccess(principals, targetsByID, principal),
+    findSessionGrant: (id) => {
+      const s = mgr.sessions.find((x) => x.id === id);
 
-      if (runtime !== undefined) {
-        runtime.pendingLastUsed = true;
-        runtime.dims = { cols: p.cols, rows: p.rows };
+      return s === undefined ? null : { target: s.target, targetIdentity: s.targetIdentity };
+    },
+    findTargetIdentity: (target) => targets.find((x) => x.id === target)?.identity ?? null,
+    collectChildIDs: (id) => mgr.collectChildren(id).map((child) => child.id),
+    findPermissionSession: (request) => registry.findSessionID(request),
+    resolveSpawnTarget: (requested) => {
+      const target = requested ?? defaultTarget;
+      const refusal = mgr.findExecutionRefusal({ target, targetIdentity: null }, 'spawn');
+
+      if (refusal !== null) {
+        throw refusal;
+      }
 
-        runtime.screen = new ScreenModel(p.cols, p.rows);
+      if (target === null) {
+        throw new Error('a spawn without a target passed its checks');
       }
 
-      void store.recordSpawnDir(p.cwd);
+      return target;
+    },
+    spawnSession: (plan, keyed, access) => {
+      if (keyed === null) {
+        return startSpawn(plan(), mintSessionID()).then((session) => ({ session }));
+      }
 
-      return getDescriptor(mgr, s.id);
+      const effectRef = mintSessionID();
+
+      return ledger.run({
+        operation: 'session.spawn',
+        keyed,
+        effectRef,
+        start: async () => ({ session: await startSpawn(plan(), effectRef) }),
+        settle: () => mgr.writeFleet(),
+        replay: (record) => {
+          if (access !== null) {
+            requireReplayInReach(record, access);
+          }
+
+          return loadSpawnReplay(record);
+        },
+        findEffectTarget: () => {
+          const s = mgr.sessions.find((x) => x.id === effectRef);
+
+          return s === undefined ? null : { target: s.target, targetIdentity: s.targetIdentity };
+        },
+      });
     },
     updateSession: (id, name, pinned) => mgr.updateSession(id, name, pinned),
     quitDaemon: () => {
@@ -707,10 +1036,18 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
       }, 80);
     },
     killSession: async (id) => {
-      if (!mgr.sessions.some((s) => s.id === id)) {
+      const s = mgr.sessions.find((x) => x.id === id);
+
+      if (s === undefined) {
         return false;
       }
 
+      for (const live of [s, ...mgr.collectChildren(id)]) {
+        if (live.pty !== null) {
+          mgr.requireExecution(live, 'kill');
+        }
+      }
+
       for (const child of mgr.collectChildren(id)) {
         runtimes.get(child.id)?.stopHeadlessRun();
       }
@@ -738,6 +1075,9 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
         return 'no_transcript';
       }
 
+      mgr.requireExecution(s, 'kill');
+      mgr.requireExecution(s, 'headless');
+
       const yanked = mgr.yankHeadless(id);
 
       if (yanked === null) {
@@ -776,9 +1116,7 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
       }
 
       if (runtime !== undefined) {
-        runtime.dims = { cols, rows };
-        runtime.startedAt = null;
-        runtime.tapAttached = false;
+        runtime.resetBoot({ cols, rows });
 
         runtime.screen ??= new ScreenModel(cols, rows);
       }
@@ -821,6 +1159,7 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
         return 'dead';
       }
 
+      mgr.requireExecution(s, 'attach');
       attachments.attach(sessionID, client, dims);
       mgr.attach(sessionID);
 
@@ -861,7 +1200,15 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
         return 'missing';
       }
 
+      // A headless turn runs only through the session's own target, never
+      // anywhere else, and a dead session takes no input.
       if (s.kind === 'headless') {
+        mgr.requireExecution(s, 'headless');
+
+        if (s.state === 'exited') {
+          return 'dead';
+        }
+
         if ((runtimes.get(sessionID)?.headlessRun ?? null) !== null) {
           return 'busy';
         }
@@ -881,6 +1228,7 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
         return 'dead';
       }
 
+      mgr.requireExecution(s, 'input');
       s.pty.write(data);
 
       return 'ok';
@@ -950,9 +1298,9 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
 
       return { path, page };
     },
-    readEvents: async (afterID, limit, waitMs, sessionID) => {
+    readEvents: async (afterID, limit, waitMs, sessionID, access) => {
       const deadline = Date.now() + waitMs;
-      const scope = sessionID === null ? null : buildEventScope(sessionID);
+      const scope = buildEventScope(sessionID, access);
 
       // A wake for an event the read leaves out (a heartbeat, or another
       // session's event under a session filter) loops back to wait out the
@@ -979,60 +1327,51 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
         await eventSignal.waitForNext(generation, remaining);
       }
     },
-    writeSessionMessage: async (sessionID, from, text) => {
-      const s = mgr.sessions.find((x) => x.id === sessionID);
+    writeSessionMessage: async (sessionID, from, text, keyed) => {
+      if (keyed === null) {
+        const refusal = findMessageRefusal(sessionID);
 
-      if (s === undefined) {
-        return 'missing';
-      }
+        if (refusal !== null) {
+          return refusal;
+        }
 
-      if (!(s.pty !== null || (s.kind === 'headless' && s.state !== 'exited'))) {
-        return 'dead';
-      }
+        const record = await writeAcceptedMessage(sessionID, from, text, mintMessageID());
 
-      if (mgr.findAdapter(s.agent)?.takesMessages !== true) {
-        return 'unsupported';
+        return { message: record.id, status: record.status };
       }
 
-      const runtime = runtimes.get(sessionID);
+      const effectRef = mintMessageID();
 
-      if (
-        runtime !== undefined &&
-        runtime.startedAt !== null &&
-        !runtime.tapAttached &&
-        Date.now() - runtime.startedAt >= (opts.tapGraceMs ?? TAP_GRACE_MS)
-      ) {
-        return 'no_tap';
-      }
+      try {
+        return await ledger.run({
+          operation: 'session.message',
+          keyed,
+          effectRef,
 
-      const previousWrite = lastMessageWrite;
-      const written = Promise.withResolvers<void>();
+          // A refusal throws, so the claim drops and a retry runs fresh.
+          start: async () => {
+            const refusal = findMessageRefusal(sessionID);
 
-      lastMessageWrite = written.promise;
+            if (refusal !== null) {
+              throw new MessageRefusedError(refusal);
+            }
 
-      await previousWrite;
+            const record = await writeAcceptedMessage(sessionID, from, text, effectRef);
 
-      const record: MessageRecord = {
-        id: mintMessageID(),
-        atcID: sessionID,
-        ...(s.agentSessionID === undefined ? {} : { agentSessionID: s.agentSessionID }),
-        from,
-        text,
-        status: 'accepted',
-        sentAt: Date.now(),
-      };
+            return { message: record.id, status: record.status };
+          },
 
-      try {
-        await store.writeMessage(record);
+          // The message row is the effect, and the start already wrote it.
+          settle: () => Promise.resolve(),
+          replay: (record) => loadMessageReplay(record),
+        });
+      } catch (error) {
+        if (error instanceof MessageRefusedError) {
+          return error.refusal;
+        }
 
-        await recordMessageStatus(sessionID, record);
-      } finally {
-        written.resolve();
+        throw error;
       }
-
-      await drainInbox(sessionID);
-
-      return record;
     },
     attachTap: (client, sessionID) => {
       const s = mgr.sessions.find((x) => x.id === sessionID);
@@ -1149,6 +1488,8 @@ export async function startDaemon(opts: DaemonOptions): Promise<DaemonHandle> {
       client.dispose();
     }
 
+    clearInterval(idempotencySweep);
+
     server.stop(true);
     eventsServer?.stop();
     reporter.stop(true);
@@ -1231,3 +1572,39 @@ function getDescriptor(mgr: SessionManager, id: SessionID): SessionDescriptor {
 
   return d;
 }
+
+// A sweep that fails leaves the keys for the next one.
+async function tryRemoveExpiredIdempotencyKeys(store: StateStore): Promise<boolean> {
+  try {
+    await store.removeExpiredIdempotencyKeys(Date.now() - IDEMPOTENCY_TTL_MS);
+  } catch {
+    return false;
+  }
+
+  return true;
+}
+
+// Carries a message refusal out of a keyed send's start, so the claim drops
+// and the refusal still answers the request.
+class MessageRefusedError extends Error {
+  readonly refusal: MessageRefusal;
+
+  constructor(refusal: MessageRefusal) {
+    super(`message refused: ${refusal}`);
+
+    this.refusal = refusal;
+    this.name = 'MessageRefusedError';
+  }
+}
+
+// The target a held spawn key's stored answer holds its session on, for a
+// refusal of its replay; the answer holds no other target to name.
+function findReplayTarget(record: IdempotencyRecord): string {
+  const stored: unknown = record.result === null ? null : JSON.parse(record.result);
+  const session = isRecord(stored) ? stored['session'] : null;
+  const locator = isRecord(session) ? session['locator'] : null;
+
+  return isRecord(locator) && typeof locator['targetID'] === 'string'
+    ? locator['targetID']
+    : 'unknown';
+}