deepline 0.1.210 → 0.1.211

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -18,6 +18,7 @@ import {
   isPlayDataset,
   iteratePlayDatasetInputPages,
   materializePlayDatasetInput,
+  resolveMaterializeLimitCap,
 } from '@shared_libs/plays/dataset';
 import type { PlayDataset, PlayDatasetInput } from '@shared_libs/plays/dataset';
 import {
@@ -261,6 +262,7 @@ const rowContext = new AsyncLocalStorage<{
   rowKey?: string;
   mapScope?: MapExecutionScope;
 }>();
+const toolExecutionOverrides = new AsyncLocalStorage<{ force: boolean }>();
 const PROGRESS_HEARTBEAT_INTERVAL_MS = 1_000;
 const PURE_JS_HEARTBEAT_ROW_INTERVAL = 250;
 /**
@@ -819,6 +821,111 @@ function stableDigest(value: string): string {
   return sha256Hex(value);
 }
 
+// Receipt keys contain canonicalized tool inputs. Runtime diagnostics must never
+// emit them directly; a short digest is enough to correlate gateway, runner,
+// and waiter observations for one failing run.
+function runtimeReceiptKeyDigest(key: string): string {
+  return stableDigest(key).slice(0, 12);
+}
+
+function runtimeReceiptReadSummary(input: {
+  requested: readonly string[];
+  receipts: readonly (RuntimeStepReceipt | null | undefined)[];
+  resolved: ReadonlyMap<string, RuntimeStepReceipt>;
+}): string {
+  const statuses = input.receipts.reduce<Record<string, number>>(
+    (counts, receipt) => {
+      const status = receipt?.status ?? 'missing';
+      counts[status] = (counts[status] ?? 0) + 1;
+      return counts;
+    },
+    {},
+  );
+  const positionalMismatches = input.receipts.reduce(
+    (count, receipt, index) =>
+      receipt && receipt.key.trim() !== (input.requested[index] ?? '').trim()
+        ? count + 1
+        : count,
+    0,
+  );
+  const missingRequested = input.requested.filter(
+    (key) => !input.resolved.has(key),
+  );
+  return (
+    `requested=${input.requested.length} returned=${input.receipts.length} ` +
+    `resolved=${input.resolved.size} positional_mismatches=${positionalMismatches} ` +
+    `request_digests=${input.requested.map(runtimeReceiptKeyDigest).join(',')} ` +
+    `returned_digests=${input.receipts
+      .map((receipt) =>
+        receipt ? runtimeReceiptKeyDigest(receipt.key) : 'null',
+      )
+      .join(',')} ` +
+    `missing_digests=${missingRequested
+      .map(runtimeReceiptKeyDigest)
+      .join(',')} ` +
+    `statuses=${Object.entries(statuses)
+      .map(([status, count]) => `${status}:${count}`)
+      .join(',')}`
+  );
+}
+
+const runtimeReceiptReadTraceEnabled =
+  process.env.DEEPLINE_RUNTIME_RECEIPT_TRACE === '1';
+
+function runtimeReceiptReadTrace(input: {
+  keys: readonly string[];
+  receipts: Array<RuntimeStepReceipt | null>;
+  byKey: ReadonlyMap<string, RuntimeStepReceipt>;
+}): string | null {
+  if (!runtimeReceiptReadTraceEnabled) return null;
+  const digest = (value: string) => stableDigest(value).slice(0, 16);
+  const statusCounts = input.receipts.reduce<Record<string, number>>(
+    (counts, receipt) => {
+      if (receipt) {
+        counts[receipt.status] = (counts[receipt.status] ?? 0) + 1;
+      }
+      return counts;
+    },
+    {},
+  );
+  const expiredInFlightCount = input.receipts.filter((receipt) => {
+    if (
+      receipt?.status !== 'queued' &&
+      receipt?.status !== 'pending' &&
+      receipt?.status !== 'running'
+    ) {
+      return false;
+    }
+    const expiresAt = receipt.leaseExpiresAt
+      ? Date.parse(receipt.leaseExpiresAt)
+      : Number.NaN;
+    return Number.isFinite(expiresAt) && expiresAt <= Date.now();
+  }).length;
+  return (
+    '[runtime-receipt-context.read-trace] ' +
+    JSON.stringify({
+      requestedCount: input.keys.length,
+      returnedCount: input.receipts.filter(Boolean).length,
+      mapSize: input.byKey.size,
+      statusCounts,
+      expiredInFlightCount,
+      requestedOrderDigest: digest(input.keys.join('\u0000')),
+      returnedOrderDigest: digest(
+        input.receipts
+          .map((receipt) =>
+            receipt ? `${digest(receipt.key)}:${receipt.status}` : '-',
+          )
+          .join('\u0000'),
+      ),
+      mapOrderDigest: digest(
+        [...input.byKey.entries()]
+          .map(([key, receipt]) => `${digest(key)}:${receipt.status}`)
+          .join('\u0000'),
+      ),
+    })
+  );
+}
+
 type DurableCtxOperation = 'step' | 'tool' | 'fetch';
 
 function durableCtxKey(input: {
@@ -1176,12 +1283,15 @@ export class PlayContextImpl {
         );
       }
       assertNoSecretTaint(request.input, 'ctx.tools.execute input');
+      const force =
+        request.force === true ||
+        toolExecutionOverrides.getStore()?.force === true;
       return this.executeTool(
         request.id.trim(),
         request.tool,
         request.input,
         request.description ||
-          request.force === true ||
+          force ||
           request.staleAfterSeconds !== undefined ||
           request.timeoutMs !== undefined ||
           request.receiptWaitMs !== undefined
@@ -1189,7 +1299,7 @@ export class PlayContextImpl {
               ...(request.description
                 ? { description: request.description }
                 : {}),
-              ...(request.force === true ? { force: true } : {}),
+              ...(force ? { force: true } : {}),
               ...(request.staleAfterSeconds !== undefined
                 ? { staleAfterSeconds: request.staleAfterSeconds }
                 : {}),
@@ -1204,6 +1314,10 @@ export class PlayContextImpl {
       ) as Promise<TOutput>;
     },
   };
+
+  async __deeplineRunWithForcedTools<T>(run: () => Promise<T>): Promise<T> {
+    return await toolExecutionOverrides.run({ force: true }, run);
+  }
   readonly customerDb = {
     query: async <TRow extends object = Record<string, unknown>>(
       statement:
@@ -2161,6 +2275,21 @@ export class PlayContextImpl {
       );
       if (normalized) byKey.set(normalized.key, normalized);
     }
+    if (runtimeReceiptReadTraceEnabled) {
+      this.log(
+        `[runtime-receipt-normalize] ${runtimeReceiptReadSummary({
+          requested: uniqueKeys,
+          receipts,
+          resolved: byKey,
+        })}`,
+      );
+      const trace = runtimeReceiptReadTrace({
+        keys: uniqueKeys,
+        receipts,
+        byKey,
+      });
+      if (trace) this.log(trace);
+    }
     return byKey;
   }
 
@@ -2207,6 +2336,16 @@ export class PlayContextImpl {
       );
       if (normalized) byKey.set(normalized.key, normalized);
     }
+    if (runtimeReceiptReadTraceEnabled) {
+      this.log(
+        `[runtime-receipt-claim-normalize] reclaim_running=${reclaimRunning} force_refresh=${forceRefresh} force_failed_refresh=${forceFailedRefresh} ` +
+          runtimeReceiptReadSummary({
+            requested: uniqueKeys,
+            receipts,
+            resolved: byKey,
+          }),
+      );
+    }
     return byKey;
   }
 
@@ -2667,6 +2806,7 @@ export class PlayContextImpl {
       receiptKeys: keys,
       store: this.durableReceiptExecutionStore(),
       maxAttempts,
+      log: (message) => this.log(message),
     });
   }
 
@@ -3755,6 +3895,13 @@ export class PlayContextImpl {
       let duplicateReuseCount = 0;
       let executedCount = 0;
       const previewRows: Record<string, unknown>[] = [];
+      // A bounded dataset may be materialized by authored code immediately
+      // after run(). Keep the rows produced by this exact awaited execution so
+      // that read-after-write does not depend on a separate sheet read becoming
+      // visible. Datasets above the public materialize cap remain fully
+      // streaming and sheet-backed.
+      const materializedResultRowLimit = resolveMaterializeLimitCap();
+      const immediateMaterializedRows: Record<string, unknown>[] = [];
 
       const persistMapRows = async (rows: PersistableMapRow[]) => {
         if (!this.#options.onMapRowsCompleted || rows.length === 0) {
@@ -4038,8 +4185,11 @@ export class PlayContextImpl {
         this.activeMapCellMeta = null;
 
         for (const row of mapResult.completedRows) {
-          if (previewRows.length >= 5) break;
-          previewRows.push(this.toMaterializedOutputRow(row.data));
+          const materializedRow = this.toMaterializedOutputRow(row.data);
+          if (previewRows.length < 5) previewRows.push(materializedRow);
+          if (immediateMaterializedRows.length < materializedResultRowLimit) {
+            immediateMaterializedRows.push(materializedRow);
+          }
         }
         executedCount += rowsToExecute.length;
         successfulCount += mapResult.completedRows.length;
@@ -4100,6 +4250,42 @@ export class PlayContextImpl {
         return result.rows.map((row) => this.toMaterializedOutputRow(row));
       };
       const materializeRuntimeBackedMapRows = async (limit?: number) => {
+        if (limit !== undefined && limit <= 0) return [];
+        const availableResultRows = Math.min(
+          successfulCount,
+          materializedResultRowLimit,
+        );
+        if (
+          immediateMaterializedRows.length === availableResultRows &&
+          (limit !== undefined || successfulCount <= materializedResultRowLimit)
+        ) {
+          return limit === undefined
+            ? immediateMaterializedRows.slice()
+            : immediateMaterializedRows.slice(0, limit);
+        }
+        const pageSize = 1000;
+        const materialized: Record<string, unknown>[] = [];
+        let offset = 0;
+        while (true) {
+          const remaining =
+            limit === undefined
+              ? pageSize
+              : Math.max(0, limit - materialized.length);
+          if (remaining === 0) break;
+          const rows = await readRuntimeBackedMapRows({
+            limit: Math.min(pageSize, remaining),
+            offset,
+          });
+          if (rows.length === 0) break;
+          materialized.push(...rows);
+          if (limit !== undefined && materialized.length >= limit) {
+            return materialized.slice(0, limit);
+          }
+          offset += rows.length;
+        }
+        return materialized;
+      };
+      const materializeFullPersistedMapRows = async (limit?: number) => {
         if (limit !== undefined && limit <= 0) return [];
         const pageSize = 1000;
         const materialized: Record<string, unknown>[] = [];
@@ -4156,14 +4342,22 @@ export class PlayContextImpl {
           peek: async (limit) =>
             limit <= 0
               ? []
-              : await readRuntimeBackedMapRows({
-                  limit,
-                  offset: 0,
-                }),
+              : immediateMaterializedRows.length >=
+                  Math.min(limit, successfulCount)
+                ? immediateMaterializedRows.slice(0, limit)
+                : await readRuntimeBackedMapRows({
+                    limit,
+                    offset: 0,
+                  }),
           materialize: materializeRuntimeBackedMapRows,
+          materializeFullPersistedDataset: materializeFullPersistedMapRows,
           iterate: () =>
             ({
               async *[Symbol.asyncIterator]() {
+                if (immediateMaterializedRows.length === successfulCount) {
+                  for (const row of immediateMaterializedRows) yield row;
+                  return;
+                }
                 const pageSize = 1000;
                 let offset = 0;
                 while (true) {
@@ -4598,39 +4792,33 @@ export class PlayContextImpl {
       });
     }
 
-    const readRuntimeBackedMapRows =
+    // This map's terminal rows have already crossed the persistence barrier
+    // above. Keep that verified snapshot as the authored return value instead
+    // of immediately re-reading the Runtime Sheet projection. A second read
+    // can observe a lagging/dynamically incomplete physical projection even
+    // though the terminal write succeeded (notably for inline-child object
+    // outputs on Absurd), making `materialize()` disagree with durable truth.
+    // Neon remains the backing for later API/export reads of this handle.
+    const terminalRows = results.map((row) =>
+      this.toMaterializedOutputRow(row),
+    );
+    const materializeTerminalRows = async (limit?: number) =>
+      limit === undefined
+        ? terminalRows.slice()
+        : terminalRows.slice(0, Math.max(0, limit));
+    const runtimeSheetBacked =
       this.#options.runtimeSheetBackedMapDatasets === true &&
-      this.#options.baseUrl &&
-      this.#options.executorToken &&
-      this.#options.playName &&
-      this.#options.runId
-        ? async (input: { limit: number; offset: number }) => {
-            const result = await readRuntimeSheetDatasetRows(
-              {
-                baseUrl: this.#options.baseUrl!,
-                executorToken: this.#options.executorToken!,
-                dbSessionStrategy: this.#options.dbSessionStrategy,
-                playName: this.#options.playName!,
-                userEmail: this.#options.userEmail,
-                runId: this.#options.runId!,
-              },
-              {
-                tableNamespace: resolvedTableNamespace,
-                runId: this.#options.runId!,
-                limit: input.limit,
-                offset: input.offset,
-              },
-            );
-            return result.rows.map((row) => this.toMaterializedOutputRow(row));
-          }
-        : null;
-    const materializeRuntimeBackedMapRows = async (limit?: number) => {
-      if (!readRuntimeBackedMapRows) {
-        return limit === undefined
-          ? results.map((row) => this.toMaterializedOutputRow(row))
-          : results
-              .slice(0, Math.max(0, limit))
-              .map((row) => this.toMaterializedOutputRow(row));
+      Boolean(
+        this.#options.baseUrl &&
+        this.#options.executorToken &&
+        this.#options.playName &&
+        this.#options.runId,
+      );
+    const materializeFullPersistedMapRows = async (limit?: number) => {
+      if (!runtimeSheetBacked) {
+        throw new Error(
+          'The full persisted dataset is unavailable because this run result has no Runtime Sheet backing.',
+        );
       }
       if (limit !== undefined && limit <= 0) return [];
       const pageSize = 1000;
@@ -4642,10 +4830,25 @@ export class PlayContextImpl {
             ? pageSize
             : Math.max(0, limit - materialized.length);
         if (remaining === 0) break;
-        const rows = await readRuntimeBackedMapRows({
-          limit: Math.min(pageSize, remaining),
-          offset,
-        });
+        const result = await readRuntimeSheetDatasetRows(
+          {
+            baseUrl: this.#options.baseUrl!,
+            executorToken: this.#options.executorToken!,
+            dbSessionStrategy: this.#options.dbSessionStrategy,
+            playName: this.#options.playName!,
+            userEmail: this.#options.userEmail,
+            runId: this.#options.runId!,
+          },
+          {
+            tableNamespace: resolvedTableNamespace,
+            runId: this.#options.runId!,
+            limit: Math.min(pageSize, remaining),
+            offset,
+          },
+        );
+        const rows = result.rows.map((row) =>
+          this.toMaterializedOutputRow(row),
+        );
         if (rows.length === 0) break;
         materialized.push(...rows);
         if (limit !== undefined && materialized.length >= limit) {
@@ -4663,7 +4866,7 @@ export class PlayContextImpl {
         resolvedTableNamespace,
       ),
       count: results.length,
-      backing: readRuntimeBackedMapRows
+      backing: runtimeSheetBacked
         ? {
             storage: 'neon_sheet',
             sheet: {
@@ -4689,37 +4892,17 @@ export class PlayContextImpl {
       },
       resolvers: {
         count: async () => results.length,
-        peek: async (limit) =>
-          readRuntimeBackedMapRows
-            ? limit <= 0
-              ? []
-              : await readRuntimeBackedMapRows({
-                  limit,
-                  offset: 0,
-                })
-            : results
-                .slice(0, Math.max(0, limit))
-                .map((row) => this.toMaterializedOutputRow(row)),
-        materialize: materializeRuntimeBackedMapRows,
+        peek: async (limit) => terminalRows.slice(0, Math.max(0, limit)),
+        materialize: materializeTerminalRows,
+        ...(runtimeSheetBacked
+          ? {
+              materializeFullPersistedDataset: materializeFullPersistedMapRows,
+            }
+          : {}),
         iterate: () =>
           ({
             async *[Symbol.asyncIterator]() {
-              if (readRuntimeBackedMapRows) {
-                const pageSize = 1000;
-                let offset = 0;
-                while (true) {
-                  const rows = await readRuntimeBackedMapRows({
-                    limit: pageSize,
-                    offset,
-                  });
-                  if (rows.length === 0) return;
-                  for (const row of rows) {
-                    yield row;
-                  }
-                  offset += rows.length;
-                }
-              }
-              for (const row of results) {
+              for (const row of terminalRows) {
                 yield row;
               }
             },
@@ -7431,18 +7614,32 @@ export class PlayContextImpl {
           const forcedExisting = forcedReceiptKeys.length
             ? await this.getRuntimeStepReceipts(forcedReceiptKeys)
             : new Map<string, RuntimeStepReceipt>();
-          const forcedRunningReceiptKeys = new Set(
+          // A forced refresh must never duplicate a provider call owned by a
+          // live lease. It can, however, immediately claim pending/unleased
+          // work and any expired lease. Waiting first for those receipts turns
+          // an already-recoverable interruption into a five-minute stall.
+          const forcedLiveReceiptKeys = new Set(
             [...forcedExisting]
-              .filter(
-                ([, receipt]) =>
-                  receipt.status === 'queued' ||
-                  receipt.status === 'pending' ||
-                  receipt.status === 'running',
-              )
+              .filter(([, receipt]) => {
+                if (
+                  receipt.status !== 'queued' &&
+                  receipt.status !== 'pending' &&
+                  receipt.status !== 'running'
+                ) {
+                  return false;
+                }
+                if (!receipt.leaseId?.trim()) return false;
+                const expiresAt = receipt.leaseExpiresAt
+                  ? Date.parse(receipt.leaseExpiresAt)
+                  : Number.NaN;
+                // Keep an unparseable leased receipt conservative. The
+                // fenced claim remains the recovery path after the wait.
+                return !Number.isFinite(expiresAt) || expiresAt > Date.now();
+              })
               .map(([receiptKey]) => receiptKey),
           );
           const claimableForcedReceiptKeys = forcedReceiptKeys.filter(
-            (receiptKey) => !forcedRunningReceiptKeys.has(receiptKey),
+            (receiptKey) => !forcedLiveReceiptKeys.has(receiptKey),
           );
           const claims =
             normalReceiptKeys.length > 0
@@ -7706,16 +7903,39 @@ export class PlayContextImpl {
           const processExistingRunningReceiptWaits =
             async (): Promise<void> => {
               if (durableExistingRunningWaits.length === 0) return;
+              const waitingReceiptKeys = durableExistingRunningWaits.map(
+                (wait) => wait.receiptKey,
+              );
               const waitMaxAttempts = Math.max(
                 ...durableExistingRunningWaits.map(
                   (wait) => wait.waitMaxAttempts ?? 0,
                 ),
               );
+              if (runtimeReceiptReadTraceEnabled) {
+                this.log(
+                  `[runtime-receipt-wait] phase=start requested=${waitingReceiptKeys.length} ` +
+                    `request_digests=${waitingReceiptKeys
+                      .map(runtimeReceiptKeyDigest)
+                      .join(',')} max_attempts=${waitMaxAttempts || 'default'}`,
+                );
+              }
               const waited = await this.waitForCompletedRuntimeToolReceipts(
-                durableExistingRunningWaits.map((wait) => wait.receiptKey),
+                waitingReceiptKeys,
                 waitMaxAttempts > 0 ? waitMaxAttempts : undefined,
               );
-              await Promise.allSettled(
+              if (runtimeReceiptReadTraceEnabled) {
+                this.log(
+                  `[runtime-receipt-wait] phase=settled completed=${waited.completed.size} ` +
+                    `failed=${waited.failed.size} timed_out=${waited.timedOut.size} ` +
+                    `completed_digests=${[...waited.completed.keys()]
+                      .map(runtimeReceiptKeyDigest)
+                      .join(',')} ` +
+                    `timed_out_digests=${[...waited.timedOut]
+                      .map(runtimeReceiptKeyDigest)
+                      .join(',')}`,
+                );
+              }
+              const deliverySettlements = await Promise.allSettled(
                 durableExistingRunningWaits.map(async (wait) => {
                   const failed = waited.failed.get(wait.receiptKey);
                   if (failed) {
@@ -7737,6 +7957,28 @@ export class PlayContextImpl {
                   );
                 }),
               );
+              if (runtimeReceiptReadTraceEnabled) {
+                const rejected = deliverySettlements.flatMap(
+                  (settlement, index) => {
+                    if (settlement.status !== 'rejected') return [];
+                    const receiptKey =
+                      durableExistingRunningWaits[index]?.receiptKey;
+                    return receiptKey
+                      ? [
+                          `${runtimeReceiptKeyDigest(receiptKey)}:${
+                            settlement.reason instanceof Error
+                              ? settlement.reason.name
+                              : 'non_error'
+                          }`,
+                        ]
+                      : ['unknown'];
+                  },
+                );
+                this.log(
+                  `[runtime-receipt-wait] phase=delivery settled=${deliverySettlements.length} ` +
+                    `rejected=${rejected.length} rejected_digests=${rejected.join(',')}`,
+                );
+              }
             };
 
           for (const [receiptKey, group] of requestsByReceiptKey) {
@@ -8222,11 +8464,17 @@ export class PlayContextImpl {
     // The Governor's tool slot is the single seam for tool-call budget + the
     // global tool-concurrency backstop + per-(org, provider) pacing. It blocks
     // until all three are satisfied and is held (across retries) until release.
+    const admissionStartedAt = Date.now();
     const toolSlot = await this.resourceGovernor.acquireTool({
       orgId: this.#options.orgId ?? null,
       providerResourceKey: `tool:${toolId}`,
       toolId,
     });
+    if (runtimeReceiptReadTraceEnabled) {
+      this.log(
+        `[perf] tool call id=${toolId} phase=governor_admission elapsed_ms=${Date.now() - admissionStartedAt}`,
+      );
+    }
     try {
       return await withActiveSpan(
         'plays.tool.execute',
@@ -8286,7 +8534,13 @@ export class PlayContextImpl {
               : null;
             try {
               const providerCallStartedAt = Date.now();
+              const ownershipStartedAt = Date.now();
               await options?.beforeProviderCall?.();
+              if (runtimeReceiptReadTraceEnabled) {
+                this.log(
+                  `[perf] tool call id=${toolId} phase=before_provider_ownership elapsed_ms=${Date.now() - ownershipStartedAt}`,
+                );
+              }
               const heartbeatIntervalMs = hasReceiptHeartbeat
                 ? runtimeLeaseHeartbeatIntervalFromExpiry({
                     leaseExpiresAt: options?.receiptLeaseExpiresAt,
@@ -8316,6 +8570,7 @@ export class PlayContextImpl {
                   : null;
               receiptHeartbeatSupervisor?.start();
               try {
+                const integrationFetchStartedAt = Date.now();
                 response = await fetch(url, {
                   method: 'POST',
                   signal: abortController?.signal,
@@ -8380,6 +8635,11 @@ export class PlayContextImpl {
                       : {}),
                   }),
                 });
+                if (runtimeReceiptReadTraceEnabled) {
+                  this.log(
+                    `[perf] tool call id=${toolId} phase=integration_fetch_headers elapsed_ms=${Date.now() - integrationFetchStartedAt} status=${response.status}`,
+                  );
+                }
                 this.resourceGovernor.observe({
                   toolId,
                   providerResourceKey: `tool:${toolId}`,