deepline 0.3.43 → 0.3.45

package/dist/bundling-sources/shared_libs/play-runtime/tool-call/receipt-cohort.ts

@@ -0,0 +1,904 @@
+import type { ToolExecuteResult } from '../tool-result';
+import { createRuntimeReceiptHeartbeatSupervisor } from '../receipt-heartbeat-supervisor';
+import {
+  ToolCallReceiptLeaseLostError,
+  type ToolCallReceiptLease,
+  type ToolCallReceiptRecoverySource,
+  type ToolResultReceipts,
+} from './contract';
+
+/**
+ * The durable state of one map-member Tool Result Receipt. This is deliberately
+ * smaller than RuntimeStepReceipt: row persistence and provider transport are
+ * not receipt-cohort concerns.
+ */
+export type ToolCallReceiptRecord = {
+  resultReceiptKey: string;
+  status: 'queued' | 'pending' | 'running' | 'completed' | 'failed' | 'skipped';
+  result?: ToolExecuteResult;
+  error?: unknown;
+  lease?: ToolCallReceiptLease;
+  leaseExpiresAt?: string | null;
+  claimState?: 'claimed' | 'existing';
+};
+
+export type ToolCallReceiptCohortMember = {
+  /** A caller-local identity; it is never persisted. */
+  memberId: string;
+  resultReceiptKey: string;
+  force?: boolean;
+  forceFailedRefresh?: boolean;
+  /** Number of poll reads before a foreign/incomplete receipt is reclaimed. */
+  waitMaxAttempts?: number;
+};
+
+export type ToolCallReceiptCohortStore = {
+  read(input: {
+    resultReceiptKeys: string[];
+  }): Promise<ToolCallReceiptRecord[]>;
+  /**
+   * The Adapter may partition this vector by refresh policy to reach a legacy
+   * gateway that accepts one policy per RPC. Results are key-aligned by value,
+   * not array position, so duplicates are forbidden at this seam.
+   */
+  claim(input: {
+    ownerRunId: string;
+    ownerAttempt: number;
+    receipts: Array<{
+      resultReceiptKey: string;
+      force: boolean;
+      forceFailedRefresh: boolean;
+      reclaimRunning: boolean;
+    }>;
+  }): Promise<ToolCallReceiptRecord[]>;
+  heartbeat(input: {
+    ownerRunId: string;
+    ownerAttempt: number;
+    receipts: Array<{ resultReceiptKey: string; lease: ToolCallReceiptLease }>;
+  }): Promise<ToolCallReceiptRecord[]>;
+  complete(input: {
+    ownerRunId: string;
+    ownerAttempt: number;
+    receipts: Array<{
+      resultReceiptKey: string;
+      lease: ToolCallReceiptLease;
+      result: ToolExecuteResult;
+    }>;
+  }): Promise<ToolCallReceiptRecord[]>;
+  fail(input: {
+    ownerRunId: string;
+    ownerAttempt: number;
+    receipts: Array<{
+      resultReceiptKey: string;
+      lease: ToolCallReceiptLease;
+      error: unknown;
+    }>;
+  }): Promise<ToolCallReceiptRecord[]>;
+};
+
+export type ToolCallReceiptCohortMemberState =
+  | {
+      kind: 'completed';
+      result: ToolExecuteResult;
+      source: 'cache' | 'in_flight' | 'owner';
+    }
+  | { kind: 'failed'; error: unknown }
+  | {
+      kind: 'owned';
+      resultReceiptKey: string;
+      lease: ToolCallReceiptLease;
+      leaseExpiresAt?: string | null;
+      receipts: ToolResultReceipts;
+    }
+  | {
+      kind: 'following';
+      resultReceiptKey: string;
+      ownerMemberId: string;
+      result: Promise<ToolExecuteResult>;
+    }
+  | {
+      /** A foreign receipt is being hydrated/reclaimed by the cohort. */
+      kind: 'recovering';
+      resultReceiptKey: string;
+      result: Promise<ToolExecuteResult>;
+    };
+
+export type ToolCallReceiptCohort = {
+  member(memberId: string): ToolCallReceiptCohortMemberState;
+  /** A native batch uses this after splitting one physical provider response. */
+  complete(
+    input: Array<{ memberId: string; result: ToolExecuteResult }>,
+  ): Promise<Map<string, ToolExecuteResult>>;
+  fail(input: Array<{ memberId: string; error: unknown }>): Promise<void>;
+  /** One RPC for every distinct owned receipt, including queued map tail work. */
+  heartbeat(): Promise<'active' | 'terminal'>;
+  /**
+   * Starts the cohort's ownership supervisor. Per-member receipt views only
+   * assert this shared fence; they never fragment it into one heartbeat RPC per
+   * provider call.
+   */
+  startHeartbeat(input: {
+    intervalMs: number;
+    onLeaseLost(error: ToolCallReceiptLeaseLostError): void;
+    onTransientFailure?(error: unknown): void;
+  }): { stop(): void; assertOwned(): void };
+  /** Wait for foreign-receipt hydration/reclaim work started by eager acquire. */
+  settle(): Promise<void>;
+};
+
+export type ToolCallReceiptCohortJob = {
+  acquire(input: {
+    owner: { runId: string; attempt: number };
+    members: ToolCallReceiptCohortMember[];
+    /**
+     * Return after the atomic initial claim so independently owned map rows
+     * can start provider work while foreign rows hydrate. The default keeps
+     * the standalone Module's all-settled acquisition contract.
+     */
+    eager?: boolean;
+  }): Promise<ToolCallReceiptCohort>;
+};
+
+export class ToolCallReceiptCohortWaitTimeoutError extends Error {
+  constructor(resultReceiptKey: string) {
+    super(`Timed out waiting for Tool Result Receipt ${resultReceiptKey}.`);
+    this.name = 'ToolCallReceiptCohortWaitTimeoutError';
+  }
+}
+
+/**
+ * Owns the receipt protocol for one mapped Tool Call cohort.
+ *
+ * Its leverage is deliberately map-specific: scalar ToolCall needs one receipt;
+ * a map needs deduplicated bulk ownership, bounded foreign-owner recovery,
+ * one heartbeat for undispatched work, and a terminal fanout after a native
+ * batch splits. Keeping that protocol out of PlayContext makes ToolCall's
+ * scalar Interface remain small without discarding the real map behaviour.
+ */
+export function createToolCallReceiptCohortJob(input: {
+  store: ToolCallReceiptCohortStore;
+  sleep?: (milliseconds: number) => Promise<void>;
+  waitDelayMs?: number;
+}): ToolCallReceiptCohortJob {
+  const sleep = input.sleep ?? defaultSleep;
+  const waitDelayMs = positive(input.waitDelayMs ?? 250, 'wait delay');
+
+  return {
+    async acquire({ owner, members, eager = false }) {
+      const ownerRunId = required(owner.runId, 'owner run id');
+      if (!Number.isInteger(owner.attempt) || owner.attempt < 0) {
+        throw new Error(
+          'Tool Call Receipt Cohort owner attempt must be a non-negative integer.',
+        );
+      }
+      if (members.length === 0) {
+        throw new Error(
+          'Tool Call Receipt Cohort requires at least one member.',
+        );
+      }
+
+      const groups = groupMembers(members);
+      const states = new Map<string, ToolCallReceiptCohortMemberState>();
+      const resultDeferred = new Map<string, Deferred<ToolExecuteResult>>();
+      const owned = new Map<string, OwnedGroup>();
+      let leaseLost: ToolCallReceiptLeaseLostError | null = null;
+      const queuedCompletions: Array<{
+        memberId: string;
+        result: ToolExecuteResult;
+        resolve(result: ToolExecuteResult): void;
+        reject(error: unknown): void;
+      }> = [];
+      const queuedFailures: Array<{
+        memberId: string;
+        error: unknown;
+        resolve(): void;
+        reject(error: unknown): void;
+      }> = [];
+      let completionFlushScheduled = false;
+      let failureFlushScheduled = false;
+
+      const resolveGroup = (
+        group: Group,
+        result: ToolExecuteResult,
+        source: 'cache' | 'in_flight' | 'owner',
+      ) => {
+        const deferred = deferredFor(group, resultDeferred);
+        deferred.resolve(result);
+        for (const [index, member] of group.members.entries()) {
+          states.set(member.memberId, {
+            kind: 'completed',
+            result,
+            // The physical owner receives its live dispatch result. Duplicate
+            // map cells attached to that receipt are followers even though
+            // the completion fanout is local to this process.
+            source: source === 'owner' && index > 0 ? 'in_flight' : source,
+          });
+        }
+      };
+      const rejectGroup = (group: Group, error: unknown) => {
+        const deferred = deferredFor(group, resultDeferred);
+        deferred.reject(error);
+        for (const member of group.members) {
+          states.set(member.memberId, { kind: 'failed', error });
+        }
+      };
+
+      const ownGroup = (group: Group, record: ToolCallReceiptRecord) => {
+        const lease = record.lease;
+        if (!lease || lease.ownerRunId !== ownerRunId) {
+          throw new ToolCallReceiptLeaseLostError(group.resultReceiptKey);
+        }
+        const ownerMember = group.members[0]!;
+        const groupOwned: OwnedGroup = { group, lease };
+        owned.set(group.resultReceiptKey, groupOwned);
+        states.set(ownerMember.memberId, {
+          kind: 'owned',
+          resultReceiptKey: group.resultReceiptKey,
+          lease,
+          ...(record.leaseExpiresAt
+            ? { leaseExpiresAt: record.leaseExpiresAt }
+            : {}),
+          receipts: receiptView(ownerMember.memberId, groupOwned),
+        });
+        const deferred = deferredFor(group, resultDeferred);
+        for (const follower of group.members.slice(1)) {
+          states.set(follower.memberId, {
+            kind: 'following',
+            resultReceiptKey: group.resultReceiptKey,
+            ownerMemberId: ownerMember.memberId,
+            result: deferred.promise,
+          });
+        }
+      };
+
+      const applyClaim = (
+        group: Group,
+        record: ToolCallReceiptRecord | undefined,
+        source: 'cache' | 'in_flight',
+      ) => {
+        if (!record) return false;
+        if (isCompleted(record)) {
+          resolveGroup(group, record.result, source);
+          return true;
+        }
+        if (record.status === 'failed') {
+          rejectGroup(
+            group,
+            record.error ??
+              new Error(
+                `Tool Result Receipt ${group.resultReceiptKey} failed.`,
+              ),
+          );
+          return true;
+        }
+        if (isOwned(record, ownerRunId)) {
+          ownGroup(group, record);
+          return true;
+        }
+        return false;
+      };
+
+      const claimGroups = async (targets: Group[], reclaimRunning: boolean) => {
+        if (targets.length === 0)
+          return new Map<string, ToolCallReceiptRecord>();
+        const response = await input.store.claim({
+          ownerRunId,
+          ownerAttempt: owner.attempt,
+          receipts: targets.map((group) => ({
+            resultReceiptKey: group.resultReceiptKey,
+            force: group.force,
+            forceFailedRefresh: group.forceFailedRefresh,
+            reclaimRunning,
+          })),
+        });
+        return byKey(response);
+      };
+
+      // A forced refresh must not overlap an active foreign lease. Other
+      // requests go straight to the atomic bulk claim, matching the existing
+      // gateway protocol.
+      const forceGroups = [...groups.values()].filter((group) => group.force);
+      const forceExisting = forceGroups.length
+        ? byKey(
+            await input.store.read({
+              resultReceiptKeys: forceGroups.map(
+                (group) => group.resultReceiptKey,
+              ),
+            }),
+          )
+        : new Map<string, ToolCallReceiptRecord>();
+      const waitGroups: Group[] = [];
+      const initialClaimGroups: Group[] = [];
+      for (const group of groups.values()) {
+        const existing = forceExisting.get(group.resultReceiptKey);
+        if (group.force && existing && isLiveLeased(existing)) {
+          waitGroups.push(group);
+        } else {
+          initialClaimGroups.push(group);
+        }
+      }
+      const claims = await claimGroups(initialClaimGroups, false);
+      for (const group of initialClaimGroups) {
+        if (!applyClaim(group, claims.get(group.resultReceiptKey), 'cache')) {
+          waitGroups.push(group);
+        }
+      }
+
+      const waitAttempts = new Map(
+        waitGroups.map((group) => [
+          group.resultReceiptKey,
+          Math.max(
+            1,
+            ...group.members.map((member) => member.waitMaxAttempts ?? 240),
+          ),
+        ]),
+      );
+      for (const group of waitGroups) {
+        states.set(group.members[0]!.memberId, {
+          kind: 'recovering',
+          resultReceiptKey: group.resultReceiptKey,
+          result: deferredFor(group, resultDeferred).promise,
+        });
+        for (const member of group.members.slice(1)) {
+          states.set(member.memberId, {
+            kind: 'recovering',
+            resultReceiptKey: group.resultReceiptKey,
+            result: deferredFor(group, resultDeferred).promise,
+          });
+        }
+      }
+
+      const settleWaitGroups = async (): Promise<void> => {
+        const pending = new Map(
+          waitGroups.map((group) => [group.resultReceiptKey, group]),
+        );
+        try {
+          const maxAttempts = Math.max(0, ...waitAttempts.values());
+          for (
+            let attempt = 0;
+            attempt < maxAttempts && pending.size;
+            attempt += 1
+          ) {
+            if (attempt > 0) await sleep(waitDelayMs);
+            const targets = [...pending.values()].filter(
+              (group) =>
+                attempt < (waitAttempts.get(group.resultReceiptKey) ?? 0),
+            );
+            if (targets.length === 0) continue;
+            // The context Adapter chunks this one vector sequentially. This is
+            // the bounded hydration lane: never turn an existing map into N
+            // per-row reads just because each member has its own recovery.
+            const latest = byKey(
+              await input.store.read({
+                resultReceiptKeys: targets.map(
+                  (group) => group.resultReceiptKey,
+                ),
+              }),
+            );
+            for (const group of targets) {
+              const record = latest.get(group.resultReceiptKey);
+              // A forced caller cannot reuse the foreign terminal result, but
+              // it also must not spend its whole recovery window polling a
+              // receipt that is no longer live. Claim a fresh forced lease as
+              // soon as the observed owner has finished (or the row vanished).
+              // Waiting until the final attempt strands the map resolver for
+              // the entire bounded-recovery budget after a perfectly healthy
+              // first call completes.
+              if (group.force) {
+                if (!record || !isLiveLeased(record)) {
+                  const claimed = await claimGroups([group], true);
+                  if (
+                    applyClaim(
+                      group,
+                      claimed.get(group.resultReceiptKey),
+                      'cache',
+                    )
+                  ) {
+                    pending.delete(group.resultReceiptKey);
+                  }
+                }
+                continue;
+              }
+              if (!record) continue;
+              if (isCompleted(record)) {
+                resolveGroup(group, record.result, 'in_flight');
+                pending.delete(group.resultReceiptKey);
+              } else if (record.status === 'failed') {
+                rejectGroup(
+                  group,
+                  record.error ??
+                    new Error(
+                      `Tool Result Receipt ${group.resultReceiptKey} failed.`,
+                    ),
+                );
+                pending.delete(group.resultReceiptKey);
+              }
+            }
+          }
+          if (pending.size === 0) return;
+          const unresolved = [...pending.values()];
+          const reclaimed = await claimGroups(unresolved, true);
+          for (const group of unresolved) {
+            if (
+              applyClaim(group, reclaimed.get(group.resultReceiptKey), 'cache')
+            ) {
+              pending.delete(group.resultReceiptKey);
+              continue;
+            }
+            const error = new ToolCallReceiptCohortWaitTimeoutError(
+              group.resultReceiptKey,
+            );
+            rejectGroup(group, error);
+            throw error;
+          }
+        } catch (error) {
+          for (const group of pending.values()) rejectGroup(group, error);
+          throw error;
+        }
+      };
+      const settlement = settleWaitGroups();
+      // An eager caller deliberately awaits this after independently owned
+      // work settles. Keep a rejection observed meanwhile so a prompt adapter
+      // read failure cannot surface as an unhandled rejection.
+      void settlement.catch(() => undefined);
+      if (!eager) await settlement;
+
+      const assertOwned = () => {
+        if (leaseLost) throw leaseLost;
+        for (const group of owned.values()) {
+          const state = states.get(group.group.members[0]!.memberId);
+          if (
+            !state ||
+            state.kind !== 'owned' ||
+            state.lease.leaseId !== group.lease.leaseId
+          ) {
+            throw new ToolCallReceiptLeaseLostError(
+              group.group.resultReceiptKey,
+            );
+          }
+        }
+      };
+
+      const heartbeat = async (): Promise<'active' | 'terminal'> => {
+        if (owned.size === 0) return 'terminal';
+        const current = [...owned.values()];
+        const response = byKey(
+          await input.store.heartbeat({
+            ownerRunId,
+            ownerAttempt: owner.attempt,
+            receipts: current.map(({ group, lease }) => ({
+              resultReceiptKey: group.resultReceiptKey,
+              lease,
+            })),
+          }),
+        );
+        let active = false;
+        for (const groupOwned of current) {
+          const record = response.get(groupOwned.group.resultReceiptKey);
+          if (record && isCompleted(record)) {
+            owned.delete(groupOwned.group.resultReceiptKey);
+            resolveGroup(groupOwned.group, record.result, 'in_flight');
+            continue;
+          }
+          if (record?.status === 'failed') {
+            owned.delete(groupOwned.group.resultReceiptKey);
+            rejectGroup(
+              groupOwned.group,
+              record.error ??
+                new Error(
+                  `Tool Result Receipt ${groupOwned.group.resultReceiptKey} failed.`,
+                ),
+            );
+            continue;
+          }
+          if (!record || !isSameLease(record, groupOwned.lease)) {
+            leaseLost ??= new ToolCallReceiptLeaseLostError(
+              groupOwned.group.resultReceiptKey,
+            );
+            throw leaseLost;
+          }
+          active = true;
+        }
+        return active ? 'active' : 'terminal';
+      };
+
+      const complete = async (
+        entries: Array<{ memberId: string; result: ToolExecuteResult }>,
+      ) => {
+        const requested = ownedEntries(entries, states, owned);
+        if (requested.length === 0) return new Map<string, ToolExecuteResult>();
+        const response = byKey(
+          await input.store.complete({
+            ownerRunId,
+            ownerAttempt: owner.attempt,
+            receipts: requested.map(({ group, lease, result }) => ({
+              resultReceiptKey: group.resultReceiptKey,
+              lease,
+              result,
+            })),
+          }),
+        );
+        const results = new Map<string, ToolExecuteResult>();
+        for (const entry of requested) {
+          const persisted = response.get(entry.group.resultReceiptKey);
+          // Completion gateways are allowed to return a compact terminal
+          // acknowledgement after durably storing the result. That response
+          // is represented as a `no_result` wrapper by the adapter; it is not
+          // the provider result and must not replace this invocation's live
+          // value. A real no-result provider response is identical to the
+          // submitted entry, so retaining the entry is correct there too.
+          const result =
+            isCompleted(persisted) && persisted.result.status !== 'no_result'
+              ? persisted.result
+              : entry.result;
+          owned.delete(entry.group.resultReceiptKey);
+          resolveGroup(entry.group, result, 'owner');
+          for (const member of entry.group.members)
+            results.set(member.memberId, result);
+        }
+        return results;
+      };
+
+      const fail = async (
+        entries: Array<{ memberId: string; error: unknown }>,
+      ) => {
+        const requested = ownedEntries(entries, states, owned);
+        if (requested.length === 0) return;
+        await input.store.fail({
+          ownerRunId,
+          ownerAttempt: owner.attempt,
+          receipts: requested.map(({ group, lease, error }) => ({
+            resultReceiptKey: group.resultReceiptKey,
+            lease,
+            error,
+          })),
+        });
+        for (const entry of requested) {
+          owned.delete(entry.group.resultReceiptKey);
+          rejectGroup(entry.group, entry.error);
+        }
+      };
+
+      const flushCompletions = async (): Promise<void> => {
+        completionFlushScheduled = false;
+        const entries = queuedCompletions.splice(0);
+        if (entries.length === 0) return;
+        try {
+          const results = await complete(
+            entries.map(({ memberId, result }) => ({ memberId, result })),
+          );
+          for (const entry of entries) {
+            entry.resolve(results.get(entry.memberId) ?? entry.result);
+          }
+        } catch (error) {
+          for (const entry of entries) entry.reject(error);
+        }
+      };
+      const flushFailures = async (): Promise<void> => {
+        failureFlushScheduled = false;
+        const entries = queuedFailures.splice(0);
+        if (entries.length === 0) return;
+        try {
+          await fail(
+            entries.map(({ memberId, error }) => ({ memberId, error })),
+          );
+          for (const entry of entries) entry.resolve();
+        } catch (error) {
+          for (const entry of entries) entry.reject(error);
+        }
+      };
+
+      function receiptView(
+        memberId: string,
+        groupOwned: OwnedGroup,
+      ): ToolResultReceipts {
+        const { group, lease } = groupOwned;
+        return {
+          async claim(candidate) {
+            if (
+              candidate.resultReceiptKey !== group.resultReceiptKey ||
+              candidate.ownerRunId !== ownerRunId
+            ) {
+              throw new ToolCallReceiptLeaseLostError(
+                candidate.resultReceiptKey,
+              );
+            }
+            return { kind: 'owned', lease };
+          },
+          async recover(candidate) {
+            if (
+              candidate.resultReceiptKey !== group.resultReceiptKey ||
+              candidate.ownerRunId !== ownerRunId
+            ) {
+              throw new ToolCallReceiptLeaseLostError(
+                candidate.resultReceiptKey,
+              );
+            }
+            return await deferredFor(group, resultDeferred).promise;
+          },
+          async heartbeat(candidate) {
+            if (
+              candidate.resultReceiptKey !== group.resultReceiptKey ||
+              !sameLease(candidate.lease, lease)
+            )
+              return 'lost';
+            try {
+              assertOwned();
+              // Preserve the per-provider-admission fence from the old map
+              // path. The cohort-wide heartbeat keeps queued work alive;
+              // this one-member check closes the exact dispatch window.
+              const record = byKey(
+                await input.store.heartbeat({
+                  ownerRunId,
+                  ownerAttempt: owner.attempt,
+                  receipts: [
+                    { resultReceiptKey: group.resultReceiptKey, lease },
+                  ],
+                }),
+              ).get(group.resultReceiptKey);
+              if (record && isCompleted(record)) {
+                owned.delete(group.resultReceiptKey);
+                resolveGroup(group, record.result, 'in_flight');
+                return 'completed';
+              }
+              if (!record || !isSameLease(record, lease)) return 'lost';
+              return 'active';
+            } catch (error) {
+              if (error instanceof ToolCallReceiptLeaseLostError) return 'lost';
+              throw error;
+            }
+          },
+          async complete(candidate) {
+            if (
+              candidate.resultReceiptKey !== group.resultReceiptKey ||
+              !sameLease(candidate.lease, lease)
+            ) {
+              throw new ToolCallReceiptLeaseLostError(
+                candidate.resultReceiptKey,
+              );
+            }
+            const result = await new Promise<ToolExecuteResult>(
+              (resolve, reject) => {
+                queuedCompletions.push({
+                  memberId,
+                  result: candidate.result,
+                  resolve,
+                  reject,
+                });
+                if (!completionFlushScheduled) {
+                  completionFlushScheduled = true;
+                  setTimeout(() => void flushCompletions(), 0);
+                }
+              },
+            );
+            return { kind: 'stored', result };
+          },
+          async fail(candidate) {
+            if (
+              candidate.resultReceiptKey !== group.resultReceiptKey ||
+              !sameLease(candidate.lease, lease)
+            ) {
+              throw new ToolCallReceiptLeaseLostError(
+                candidate.resultReceiptKey,
+              );
+            }
+            await new Promise<void>((resolve, reject) => {
+              queuedFailures.push({
+                memberId,
+                error: candidate.error,
+                resolve,
+                reject,
+              });
+              if (!failureFlushScheduled) {
+                failureFlushScheduled = true;
+                setTimeout(() => void flushFailures(), 0);
+              }
+            });
+          },
+        };
+      }
+
+      return {
+        member(memberId) {
+          const state = states.get(memberId);
+          if (!state)
+            throw new Error(
+              `Unknown Tool Call Receipt Cohort member ${memberId}.`,
+            );
+          return state;
+        },
+        complete,
+        fail,
+        heartbeat,
+        startHeartbeat({ intervalMs, onLeaseLost, onTransientFailure }) {
+          const supervisor = createRuntimeReceiptHeartbeatSupervisor({
+            intervalMs: positive(intervalMs, 'heartbeat interval'),
+            heartbeat,
+            isLeaseLost: (error) =>
+              error instanceof ToolCallReceiptLeaseLostError,
+            onLeaseLost: (error) => {
+              leaseLost ??= error as ToolCallReceiptLeaseLostError;
+              onLeaseLost(leaseLost);
+            },
+            onTransientFailure,
+          });
+          supervisor.start();
+          return { stop: () => supervisor.stop(), assertOwned };
+        },
+        async settle() {
+          await settlement;
+        },
+      };
+    },
+  };
+}
+
+type Group = {
+  resultReceiptKey: string;
+  members: ToolCallReceiptCohortMember[];
+  force: boolean;
+  forceFailedRefresh: boolean;
+};
+
+type OwnedGroup = { group: Group; lease: ToolCallReceiptLease };
+
+type Deferred<T> = {
+  promise: Promise<T>;
+  resolve(value: T): void;
+  reject(error: unknown): void;
+};
+
+function groupMembers(
+  members: ToolCallReceiptCohortMember[],
+): Map<string, Group> {
+  const groups = new Map<string, Group>();
+  const memberIds = new Set<string>();
+  for (const member of members) {
+    const memberId = required(member.memberId, 'member id');
+    if (memberIds.has(memberId))
+      throw new Error(`Duplicate Tool Call Receipt Cohort member ${memberId}.`);
+    memberIds.add(memberId);
+    const resultReceiptKey = required(
+      member.resultReceiptKey,
+      'result receipt key',
+    );
+    const group = groups.get(resultReceiptKey) ?? {
+      resultReceiptKey,
+      members: [],
+      force: false,
+      forceFailedRefresh: false,
+    };
+    group.members.push({ ...member, memberId, resultReceiptKey });
+    group.force ||= member.force === true;
+    group.forceFailedRefresh ||= member.forceFailedRefresh === true;
+    groups.set(resultReceiptKey, group);
+  }
+  for (const group of groups.values()) {
+    // Full force dominates failed-only refresh for every duplicate member of
+    // the same content-addressed receipt, just as the current map grouping.
+    if (group.force) group.forceFailedRefresh = false;
+  }
+  return groups;
+}
+
+function deferredFor(
+  group: Group,
+  deferred: Map<string, Deferred<ToolExecuteResult>>,
+): Deferred<ToolExecuteResult> {
+  let value = deferred.get(group.resultReceiptKey);
+  if (value) return value;
+  let resolve!: (result: ToolExecuteResult) => void;
+  let reject!: (error: unknown) => void;
+  const promise = new Promise<ToolExecuteResult>(
+    (resolvePromise, rejectPromise) => {
+      resolve = resolvePromise;
+      reject = rejectPromise;
+    },
+  );
+  // A receipt can fail before a same-run follower attaches. Do not create an
+  // unhandled rejection while preserving the rejection for real followers.
+  void promise.catch(() => undefined);
+  value = { promise, resolve, reject };
+  deferred.set(group.resultReceiptKey, value);
+  return value;
+}
+
+function ownedEntries<T extends { memberId: string }>(
+  entries: T[],
+  states: Map<string, ToolCallReceiptCohortMemberState>,
+  owned: Map<string, OwnedGroup>,
+): Array<T & OwnedGroup> {
+  const byKey = new Map<string, T & OwnedGroup>();
+  for (const entry of entries) {
+    const state = states.get(entry.memberId);
+    if (!state || state.kind !== 'owned') {
+      throw new Error(
+        `Tool Call Receipt Cohort member ${entry.memberId} does not own a receipt.`,
+      );
+    }
+    const groupOwned = owned.get(state.resultReceiptKey);
+    if (!groupOwned || !sameLease(groupOwned.lease, state.lease)) {
+      throw new ToolCallReceiptLeaseLostError(state.resultReceiptKey);
+    }
+    const existing = byKey.get(groupOwned.group.resultReceiptKey);
+    if (existing && existing.memberId !== entry.memberId) {
+      throw new Error(
+        `Tool Call Receipt Cohort received two terminal outcomes for ${groupOwned.group.resultReceiptKey}.`,
+      );
+    }
+    byKey.set(groupOwned.group.resultReceiptKey, { ...entry, ...groupOwned });
+  }
+  return [...byKey.values()];
+}
+
+function byKey(
+  records: ToolCallReceiptRecord[],
+): Map<string, ToolCallReceiptRecord> {
+  const result = new Map<string, ToolCallReceiptRecord>();
+  for (const record of records) result.set(record.resultReceiptKey, record);
+  return result;
+}
+
+function isCompleted(
+  record: ToolCallReceiptRecord | undefined,
+): record is ToolCallReceiptRecord & { result: ToolExecuteResult } {
+  if (!record) return false;
+  return (
+    (record.status === 'completed' || record.status === 'skipped') &&
+    record.result !== undefined
+  );
+}
+
+function isOwned(record: ToolCallReceiptRecord, ownerRunId: string): boolean {
+  return (
+    record.claimState !== 'existing' &&
+    Boolean(record.lease) &&
+    record.lease?.ownerRunId === ownerRunId &&
+    (record.status === 'queued' ||
+      record.status === 'pending' ||
+      record.status === 'running')
+  );
+}
+
+function isLiveLeased(record: ToolCallReceiptRecord): boolean {
+  if (
+    record.status !== 'queued' &&
+    record.status !== 'pending' &&
+    record.status !== 'running'
+  )
+    return false;
+  if (!record.lease) return false;
+  const expiresAt = Date.parse(record.leaseExpiresAt ?? '');
+  return !Number.isFinite(expiresAt) || expiresAt > Date.now();
+}
+
+function isSameLease(
+  record: ToolCallReceiptRecord,
+  lease: ToolCallReceiptLease,
+): boolean {
+  return Boolean(record.lease) && sameLease(record.lease!, lease);
+}
+
+function sameLease(
+  left: ToolCallReceiptLease,
+  right: ToolCallReceiptLease,
+): boolean {
+  return left.leaseId === right.leaseId && left.ownerRunId === right.ownerRunId;
+}
+
+function required(value: string, name: string): string {
+  const normalized = value?.trim();
+  if (!normalized)
+    throw new Error(`Tool Call Receipt Cohort requires a ${name}.`);
+  return normalized;
+}
+
+function positive(value: number, name: string): number {
+  if (!Number.isFinite(value) || value <= 0)
+    throw new Error(`Tool Call Receipt Cohort ${name} must be positive.`);
+  return Math.floor(value);
+}
+
+async function defaultSleep(milliseconds: number): Promise<void> {
+  await new Promise<void>((resolve) => setTimeout(resolve, milliseconds));
+}