deepline 0.3.283 → 0.3.285

package/dist/bundling-sources/shared_libs/play-runtime/tool-call/receipt-cohort.ts

@@ -37,7 +37,7 @@ export type ToolCallReceiptCohortStore = {
    *
    * Deliberately returns nothing. A completed-result insert is immutable, so a
    * caller that already executed keeps its own result whether or not it won
-   * the insert (see `flushPublications`). Materializing the stored rows here
+   * the insert. Materializing the stored rows here
    * only bought a second deserialize-and-wrap of every result, inside the
    * delivery interval a row is blocked on.
    */
@@ -46,6 +46,8 @@ export type ToolCallReceiptCohortStore = {
     ownerAttempt: number;
     receipts: Array<{
       resultReceiptKey: string;
+      /** Successful invocations represented by this coalesced cache write. */
+      memberIds?: readonly string[];
       lease: ToolCallReceiptLease;
       result: ToolExecuteResult;
     }>;
@@ -149,250 +151,210 @@ export function createToolCallReceiptCohortJob(input: {
           'Tool Call Receipt Cohort requires at least one member.',
         );
       }
-
-      const groups = groupMembers(members);
-      // A forced member deliberately discards any completed cache value and
-      // executes a fresh provider operation. Reading it first is not merely
-      // redundant: a large persisted result is fully hydrated and wrapped
-      // before the force branch below ignores it. Keep completed-result cache
-      // reads scoped to groups that can actually consume their values. This
-      // also leaves a mixed group correct: `groupMembers` marks it forced when
-      // any duplicate member explicitly requires fresh execution.
-      const cacheReadKeys = [...groups.values()]
-        .filter((group) => !group.force)
-        .map((group) => group.resultReceiptKey);
-      const records = byKey(
-        cacheReadKeys.length > 0
+      const normalized = new Map<string, ToolCallReceiptCohortMember>();
+      for (const member of members) {
+        const memberId = required(member.memberId, 'member id');
+        if (normalized.has(memberId)) {
+          throw new Error(
+            `Duplicate Tool Call Receipt Cohort member ${memberId}.`,
+          );
+        }
+        normalized.set(memberId, {
+          ...member,
+          memberId,
+          resultReceiptKey: required(
+            member.resultReceiptKey,
+            'result receipt key',
+          ),
+        });
+      }
+      const cacheReadKeys = [
+        ...new Set(
+          [...normalized.values()]
+            .filter((member) => !member.force)
+            .map((member) => member.resultReceiptKey),
+        ),
+      ];
+      const cached = new Map(
+        (cacheReadKeys.length > 0
           ? await input.store.read({ resultReceiptKeys: cacheReadKeys })
-          : [],
+          : []
+        ).map((record) => [record.resultReceiptKey, record]),
       );
       const states = new Map<string, ToolCallReceiptCohortMemberState>();
-      const deferred = new Map<string, Deferred<ToolExecuteResult>>();
-      const owned = new Map<string, OwnedGroup>();
-      const pendingPublications: PendingPublication[] = [];
-      let publicationFlushScheduled = false;
+      const owned = new Map<string, OwnedMember>();
+      const pending: Publication[] = [];
+      let flushScheduled = false;
 
-      const resolveGroup = (group: Group, result: ToolExecuteResult) => {
-        deferredFor(group, deferred).resolve(result);
-        for (const member of group.members) {
+      for (const member of normalized.values()) {
+        const record = !member.force && cached.get(member.resultReceiptKey);
+        if (record) {
           states.set(member.memberId, {
             kind: 'completed',
-            result,
-            source: 'owner',
+            result: record.result,
+            source: 'cache',
           });
-        }
-      };
-      const rejectGroup = (group: Group, error: unknown) => {
-        deferredFor(group, deferred).reject(error);
-      };
-
-      for (const group of groups.values()) {
-        const cached = records.get(group.resultReceiptKey);
-        if (!group.force && cached) {
-          for (const member of group.members) {
-            states.set(member.memberId, {
-              kind: 'completed',
-              result: cached.result,
-              source: 'cache',
-            });
-          }
           continue;
         }
         const lease = {
           leaseId: COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID,
           ownerRunId,
         };
-        const groupOwned = { group, lease };
-        owned.set(group.resultReceiptKey, groupOwned);
-        const ownerMember = group.members[0]!;
-        states.set(ownerMember.memberId, {
+        const ownedMember = { member, lease };
+        owned.set(member.memberId, ownedMember);
+        states.set(member.memberId, {
           kind: 'owned',
-          resultReceiptKey: group.resultReceiptKey,
+          resultReceiptKey: member.resultReceiptKey,
           lease,
-          receipts: receiptView(ownerMember.memberId, groupOwned),
+          receipts: receiptView(ownedMember),
         });
-        for (const follower of group.members.slice(1)) {
-          states.set(follower.memberId, {
-            kind: 'following',
-            resultReceiptKey: group.resultReceiptKey,
-            ownerMemberId: ownerMember.memberId,
-            result: deferredFor(group, deferred).promise,
-          });
-        }
       }
 
-      const flushPublications = async (): Promise<void> => {
-        publicationFlushScheduled = false;
-        const pending = pendingPublications.splice(0);
-        if (pending.length === 0) return;
-        // Which receipts each publication is waiting on, resolved before the
-        // write so a publication can be settled the moment ITS receipts are
-        // durable rather than when the whole wave is. A lease lost between the
-        // queue and this flush is a pre-write fault of the wave, so it still
-        // rejects every publication in it.
-        const claims: PublicationClaim[] = [];
-        const requestedByKey = new Map<string, RequestedReceipt>();
-        try {
-          for (const publication of pending) {
-            const entries = ownedEntries(publication.entries, states, owned);
-            for (const entry of entries) {
-              const key = entry.group.resultReceiptKey;
-              if (!requestedByKey.has(key)) requestedByKey.set(key, entry);
-            }
-            claims.push({
-              publication,
-              resultReceiptKeys: entries.map(
-                (entry) => entry.group.resultReceiptKey,
-              ),
-              settled: false,
-            });
-          }
-        } catch (error) {
-          for (const publication of pending) publication.reject(error);
-          return;
-        }
-        const requested = [...requestedByKey.values()];
-        const settledByKey = new Map<string, ReceiptSettlement>();
-        const results = new Map<string, ToolExecuteResult>();
+      const assertOwned = (memberId: string): OwnedMember => {
+        const current = owned.get(memberId);
+        if (!current) throw new ToolCallReceiptLeaseLostError(memberId);
+        return current;
+      };
 
-        const settleReadyClaims = () => {
-          for (const claim of claims) {
-            if (claim.settled) continue;
-            const settlements = claim.resultReceiptKeys.map((key) =>
-              settledByKey.get(key),
-            );
-            if (settlements.some((settlement) => settlement === undefined)) {
-              continue;
-            }
-            claim.settled = true;
-            const failed = settlements.find(
-              (settlement) => settlement?.failed === true,
-            );
-            if (failed) {
-              // Only this publication: a receipt that failed to persist says
-              // nothing about the other receipts of the same wave.
-              claim.publication.reject(failed.error);
-              continue;
-            }
-            claim.publication.resolve(
-              new Map(
-                claim.publication.entries.flatMap((entry) => {
-                  const result = results.get(entry.memberId);
-                  return result ? [[entry.memberId, result] as const] : [];
-                }),
-              ),
-            );
+      const flush = async (): Promise<void> => {
+        flushScheduled = false;
+        const publications = pending.splice(0);
+        const writes = new Map<string, PublicationEntry>();
+        const entriesByKey = new Map<string, PublicationEntry[]>();
+        const publicationsByKey = new Map<string, Publication[]>();
+        for (const publication of publications) {
+          for (const entry of publication.entries) {
+            const key = entry.member.resultReceiptKey;
+            // Coalesce cache writes only. Every member already executed and
+            // retains its own result, even when this insert loses a race.
+            if (!writes.has(key)) writes.set(key, entry);
+            const matching = entriesByKey.get(key) ?? [];
+            matching.push(entry);
+            entriesByKey.set(key, matching);
           }
-        };
-
-        const settleReceipt = (
-          resultReceiptKey: string,
-          settlement: ReceiptSettlement,
-        ) => {
-          if (settledByKey.has(resultReceiptKey)) return;
-          const entry = requestedByKey.get(resultReceiptKey);
-          if (!entry) return;
-          settledByKey.set(resultReceiptKey, settlement);
-          // Per receipt, not per wave: the interval a caller opens here is
-          // "this row's write settled -> this row resumed", and stamping it
-          // when the last receipt of the wave landed charged an earlier row
-          // for a later row's flight. Still stamped on failure too, so the
-          // interval it opens is closed on both outcomes.
+          for (const key of publication.remaining) {
+            const matching = publicationsByKey.get(key) ?? [];
+            matching.push(publication);
+            publicationsByKey.set(key, matching);
+          }
+        }
+        const settled = new Set<string>();
+        const settle = (settlement: ToolCallReceiptCohortSettlement) => {
+          if (
+            !writes.has(settlement.resultReceiptKey) ||
+            settled.has(settlement.resultReceiptKey)
+          )
+            return;
+          settled.add(settlement.resultReceiptKey);
+          const matching = entriesByKey.get(settlement.resultReceiptKey) ?? [];
           input.timings?.onCompleteSettled?.({
-            memberIds: entry.group.members.map((member) => member.memberId),
+            memberIds: matching.map((entry) => entry.member.memberId),
             atMs: Date.now(),
           });
-          if (!settlement.failed) {
-            // A completed-result insert is immutable: another independent
-            // cold miss may have published first. That does not retroactively
-            // turn this caller into a cache follower. It already executed, so
-            // it returns its own result, and the stored row matters only to a
-            // later cache read.
-            owned.delete(resultReceiptKey);
-            resolveGroup(entry.group, entry.result);
-            for (const member of entry.group.members) {
-              results.set(member.memberId, entry.result);
+          if (settlement.status === 'stored') {
+            for (const entry of matching) {
+              owned.delete(entry.member.memberId);
+              states.set(entry.member.memberId, {
+                kind: 'completed',
+                result: entry.result,
+                source: 'owner',
+              });
             }
           }
-          settleReadyClaims();
+          for (const publication of publicationsByKey.get(
+            settlement.resultReceiptKey,
+          ) ?? []) {
+            publication.remaining.delete(settlement.resultReceiptKey);
+            if (settlement.status === 'failed')
+              publication.failure = settlement;
+            if (publication.remaining.size > 0) continue;
+            if (publication.failure)
+              publication.reject(publication.failure.error);
+            else
+              publication.resolve(
+                new Map(
+                  publication.entries.map((entry) => [
+                    entry.member.memberId,
+                    entry.result,
+                  ]),
+                ),
+              );
+          }
         };
-
         try {
           await input.store.complete({
             ownerRunId,
             ownerAttempt: owner.attempt,
-            receipts: requested.map(({ group, lease, result }) => ({
-              resultReceiptKey: group.resultReceiptKey,
-              lease,
-              result,
+            receipts: [...writes.values()].map((entry) => ({
+              resultReceiptKey: entry.member.resultReceiptKey,
+              memberIds: (
+                entriesByKey.get(entry.member.resultReceiptKey) ?? []
+              ).map((member) => member.member.memberId),
+              lease: entry.lease,
+              result: entry.result,
             })),
-            onReceiptSettled: (settlement) =>
-              settleReceipt(
-                settlement.resultReceiptKey,
-                settlement.status === 'failed'
-                  ? { failed: true, error: settlement.error }
-                  : { failed: false, error: undefined },
-              ),
+            onReceiptSettled: settle,
           });
-          for (const entry of requested) {
-            settleReceipt(entry.group.resultReceiptKey, {
-              failed: false,
-              error: undefined,
-            });
+          for (const resultReceiptKey of writes.keys()) {
+            settle({ resultReceiptKey, status: 'stored' });
           }
         } catch (error) {
-          for (const entry of requested) {
-            settleReceipt(entry.group.resultReceiptKey, {
-              failed: true,
-              error,
-            });
+          for (const resultReceiptKey of writes.keys()) {
+            settle({ resultReceiptKey, status: 'failed', error });
           }
         }
       };
 
-      const publish = async (
+      const publish = (
         entries: Array<{ memberId: string; result: ToolExecuteResult }>,
       ): Promise<Map<string, ToolExecuteResult>> => {
-        // Validate ownership before queuing, then coalesce writes that settle
-        // in this event-loop turn. This batches cache persistence only; it
-        // never waits for or coordinates another run's provider call.
-        if (ownedEntries(entries, states, owned).length === 0) {
-          return new Map();
-        }
-        return await new Promise<Map<string, ToolExecuteResult>>(
+        if (entries.length === 0) return Promise.resolve(new Map());
+        const publication = new Promise<Map<string, ToolExecuteResult>>(
           (resolve, reject) => {
-            pendingPublications.push({ entries, resolve, reject });
-            if (publicationFlushScheduled) return;
-            publicationFlushScheduled = true;
-            setImmediate(() => {
-              void flushPublications();
+            const publications = entries.map((entry) => ({
+              ...assertOwned(entry.memberId),
+              result: entry.result,
+            }));
+            pending.push({
+              entries: publications,
+              resolve,
+              reject,
+              remaining: new Set(
+                publications.map((entry) => entry.member.resultReceiptKey),
+              ),
             });
+            if (!flushScheduled) {
+              flushScheduled = true;
+              setImmediate(() => {
+                void flush();
+              });
+            }
           },
         );
+        // Siblings can settle while a caller is still awaiting another one.
+        // Observe rejection now, but return the original rejecting promise.
+        void publication.catch(() => undefined);
+        return publication;
       };
 
-      function receiptView(
-        memberId: string,
-        groupOwned: OwnedGroup,
-      ): ToolResultReceipts {
-        const { group, lease } = groupOwned;
+      function receiptView(current: OwnedMember): ToolResultReceipts {
+        const { member, lease } = current;
         const assert = (candidate: {
           resultReceiptKey: string;
-          lease: ToolCallReceiptLease;
+          lease?: ToolCallReceiptLease;
         }) => {
           if (
-            candidate.resultReceiptKey !== group.resultReceiptKey ||
-            !sameLease(candidate.lease, lease)
+            candidate.resultReceiptKey !== member.resultReceiptKey ||
+            (candidate.lease && !sameLease(candidate.lease, lease))
           ) {
             throw new ToolCallReceiptLeaseLostError(candidate.resultReceiptKey);
           }
+          assertOwned(member.memberId);
         };
         return {
           async claim(candidate) {
-            if (
-              candidate.resultReceiptKey !== group.resultReceiptKey ||
-              candidate.ownerRunId !== ownerRunId
-            ) {
+            assert(candidate);
+            if (candidate.ownerRunId !== ownerRunId) {
               throw new ToolCallReceiptLeaseLostError(
                 candidate.resultReceiptKey,
               );
@@ -400,34 +362,31 @@ export function createToolCallReceiptCohortJob(input: {
             return { kind: 'owned', lease };
           },
           async recover(candidate) {
-            if (
-              candidate.resultReceiptKey !== group.resultReceiptKey ||
-              candidate.ownerRunId !== ownerRunId
-            ) {
-              throw new ToolCallReceiptLeaseLostError(
-                candidate.resultReceiptKey,
-              );
-            }
-            return await deferredFor(group, deferred).promise;
+            throw new Error(
+              `An independently executing Tool Call cannot follow ${candidate.resultReceiptKey}.`,
+            );
           },
           async heartbeat(candidate) {
             assert(candidate);
-            return owned.has(group.resultReceiptKey) ? 'active' : 'completed';
+            return 'active';
           },
-          async complete(candidate) {
-            assert(candidate);
-            const results = await publish([
-              { memberId, result: candidate.result },
-            ]);
-            return {
-              kind: 'stored',
-              result: results.get(memberId) ?? candidate.result,
-            };
+          complete(candidate) {
+            const completion = (async () => {
+              assert(candidate);
+              const result = candidate.result;
+              await publish([{ memberId: member.memberId, result }]);
+              return {
+                kind: 'stored' as const,
+                result,
+              };
+            })();
+            // The member result wraps the queued promise; guard this promise too.
+            void completion.catch(() => undefined);
+            return completion;
           },
           async fail(candidate) {
             assert(candidate);
-            owned.delete(group.resultReceiptKey);
-            rejectGroup(group, candidate.error);
+            owned.delete(member.memberId);
           },
         };
       }
@@ -435,26 +394,17 @@ export function createToolCallReceiptCohortJob(input: {
       return {
         member(memberId) {
           const state = states.get(memberId);
-          if (!state) {
+          if (!state)
             throw new Error(
               `Unknown Tool Call Receipt Cohort member ${memberId}.`,
             );
-          }
           return state;
         },
         complete: publish,
         async fail(entries) {
           for (const entry of entries) {
-            const state = states.get(entry.memberId);
-            if (!state || state.kind !== 'owned') {
-              throw new ToolCallReceiptLeaseLostError(entry.memberId);
-            }
-            const groupOwned = owned.get(state.resultReceiptKey);
-            if (!groupOwned) {
-              throw new ToolCallReceiptLeaseLostError(state.resultReceiptKey);
-            }
-            owned.delete(state.resultReceiptKey);
-            rejectGroup(groupOwned.group, entry.error);
+            assertOwned(entry.memberId);
+            owned.delete(entry.memberId);
           }
         },
         async heartbeat() {
@@ -469,113 +419,19 @@ export function createToolCallReceiptCohortJob(input: {
   };
 }
 
-type Group = {
-  resultReceiptKey: string;
-  members: ToolCallReceiptCohortMember[];
-  force: boolean;
+type OwnedMember = {
+  member: ToolCallReceiptCohortMember;
+  lease: ToolCallReceiptLease;
 };
-
-type OwnedGroup = { group: Group; lease: ToolCallReceiptLease };
-
-type Deferred<T> = {
-  promise: Promise<T>;
-  resolve(value: T): void;
-  reject(error: unknown): void;
-};
-
-type PendingPublication = {
-  entries: Array<{ memberId: string; result: ToolExecuteResult }>;
+type PublicationEntry = OwnedMember & { result: ToolExecuteResult };
+type Publication = {
+  entries: PublicationEntry[];
   resolve(value: Map<string, ToolExecuteResult>): void;
   reject(error: unknown): void;
+  remaining: Set<string>;
+  failure?: Extract<ToolCallReceiptCohortSettlement, { status: 'failed' }>;
 };
 
-type RequestedReceipt = {
-  memberId: string;
-  result: ToolExecuteResult;
-} & OwnedGroup;
-
-/** One queued publication and the receipts it is waiting to become durable. */
-type PublicationClaim = {
-  publication: PendingPublication;
-  resultReceiptKeys: string[];
-  settled: boolean;
-};
-
-type ReceiptSettlement = { failed: boolean; error: unknown };
-
-function groupMembers(
-  members: ToolCallReceiptCohortMember[],
-): Map<string, Group> {
-  const groups = new Map<string, Group>();
-  const memberIds = new Set<string>();
-  for (const member of members) {
-    const memberId = required(member.memberId, 'member id');
-    if (memberIds.has(memberId)) {
-      throw new Error(`Duplicate Tool Call Receipt Cohort member ${memberId}.`);
-    }
-    memberIds.add(memberId);
-    const resultReceiptKey = required(
-      member.resultReceiptKey,
-      'result receipt key',
-    );
-    const group = groups.get(resultReceiptKey) ?? {
-      resultReceiptKey,
-      members: [],
-      force: false,
-    };
-    group.members.push({ ...member, memberId, resultReceiptKey });
-    group.force ||= member.force === true;
-    groups.set(resultReceiptKey, group);
-  }
-  return groups;
-}
-
-function deferredFor(
-  group: Group,
-  deferred: Map<string, Deferred<ToolExecuteResult>>,
-): Deferred<ToolExecuteResult> {
-  let value = deferred.get(group.resultReceiptKey);
-  if (value) return value;
-  let resolve!: (result: ToolExecuteResult) => void;
-  let reject!: (error: unknown) => void;
-  const promise = new Promise<ToolExecuteResult>(
-    (resolvePromise, rejectPromise) => {
-      resolve = resolvePromise;
-      reject = rejectPromise;
-    },
-  );
-  void promise.catch(() => undefined);
-  value = { promise, resolve, reject };
-  deferred.set(group.resultReceiptKey, value);
-  return value;
-}
-
-function ownedEntries<T extends { memberId: string }>(
-  entries: T[],
-  states: Map<string, ToolCallReceiptCohortMemberState>,
-  owned: Map<string, OwnedGroup>,
-): Array<T & OwnedGroup> {
-  const byKey = new Map<string, T & OwnedGroup>();
-  for (const entry of entries) {
-    const state = states.get(entry.memberId);
-    if (!state || state.kind !== 'owned') {
-      throw new ToolCallReceiptLeaseLostError(entry.memberId);
-    }
-    const groupOwned = owned.get(state.resultReceiptKey);
-    if (!groupOwned || !sameLease(groupOwned.lease, state.lease)) {
-      throw new ToolCallReceiptLeaseLostError(state.resultReceiptKey);
-    }
-    byKey.set(groupOwned.group.resultReceiptKey, { ...entry, ...groupOwned });
-  }
-  return [...byKey.values()];
-}
-
-function byKey(
-  records: ToolCallReceiptRecord[],
-): Map<string, ToolCallReceiptRecord> {
-  return new Map(records.map((record) => [record.resultReceiptKey, record]));
-}
-
 function sameLease(
   left: ToolCallReceiptLease,
   right: ToolCallReceiptLease,
@@ -585,8 +441,7 @@ function sameLease(
 
 function required(value: string, name: string): string {
   const normalized = value?.trim();
-  if (!normalized) {
+  if (!normalized)
     throw new Error(`Tool Call Receipt Cohort requires a ${name}.`);
-  }
   return normalized;
 }