deepline 0.3.461 → 0.3.462

package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts

@@ -72,6 +72,7 @@ import {
 } from './runtime-heartbeat';
 import {
   formatRunnerGatewayTransportSummary,
+  isRunnerGatewayNoResponseError,
   type RunnerGatewayTransportEvent,
 } from './runtime-gateway-transport';
 import { startConfiguredPlayRunnerRuntimeDeadline } from './runtime-deadline';
@@ -118,7 +119,6 @@ import {
   formatDatasetRegistrationLog,
 } from '../dataset-lifecycle-log';
 import {
-  assertCustomerPlayReturnWithinLimit,
   RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_BYTES,
   RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_ITEMS,
   RUNTIME_RECEIPT_COMPLETION_MAX_IN_FLIGHT_BATCHES,
@@ -126,9 +126,15 @@ import {
   RUNTIME_RECEIPT_CONTROL_BUFFER_MAX_ITEMS,
   RUNTIME_RECEIPT_CONTROL_MAX_IN_FLIGHT_BATCHES,
   RUNNER_POST_TERMINAL_DIAGNOSTIC_MAX_BYTES,
+  truncateCustomerPlayReturnWithinLimit,
 } from '../output-size-limits';
 import {
   resolveRuntimeReferencedPlay,
+  getRuntimeArrayInputManifest,
+  beginRuntimeArrayInput,
+  putRuntimeArrayInputPage,
+  sealRuntimeArrayInput,
+  readRuntimeArrayInputPage,
   createCompleteRuntimeMapRowsOperation,
   finishRuntimeSheetRunMembership,
   createRuntimeSheetDatasetPageReader,
@@ -137,6 +143,7 @@ import {
   readRuntimeSheetDatasetRows,
   releaseRuntimeWorkReceipts,
   startRuntimeSheetDataset,
+  startRuntimeSheetDatasetPages,
   supersededRuntimeMapRows,
   type RuntimeMapRowsStaleRow,
   type RuntimeMapRowsWriteResult,
@@ -240,6 +247,7 @@ import {
 } from '../app-runtime-api';
 import { RUNTIME_CAPACITY_POLICY } from '../runtime-capacity-policy';
 import { RUNTIME_RELIABILITY_POLICY } from '../runtime-reliability-policy';
+import { RUNNER_GATEWAY_CONNECTION_RETRY_BUDGET_MS } from '../runtime-constants';
 
 const playRuntimeRequire = createRequire(`${process.cwd()}/package.json`);
 const moduleRuntime = playRuntimeRequire('module') as {
@@ -576,11 +584,11 @@ export async function fetchRunnerRuntimeControl(input: {
 }): Promise<Response> {
   const fetchImpl = input.fetchImpl ?? fetch;
   const sleepImpl = input.sleepImpl ?? sleepRunnerRuntimeControlRetry;
-  for (
-    let attempt = 1;
-    attempt <= RUNNER_RUNTIME_CONTROL_MAX_ATTEMPTS;
-    attempt += 1
-  ) {
+  const startedAt = Date.now();
+  // Error responses spend the ordinary attempts. A request that gets no
+  // response at all keeps retrying through a sandbox network cut.
+  let respondedAttempts = 0;
+  for (let attempt = 1; ; attempt += 1) {
     const timeoutMs =
       input.timeoutMs ?? RUNNER_RUNTIME_CONTROL_REQUEST_TIMEOUT_MS;
     const timeoutSignal = AbortSignal.timeout(timeoutMs);
@@ -613,14 +621,15 @@ export async function fetchRunnerRuntimeControl(input: {
             `gateway and the launch carries vercelProtectionBypassToken.`,
         );
       }
+      respondedAttempts += 1;
       if (
         input.retrySafe &&
-        attempt < RUNNER_RUNTIME_CONTROL_MAX_ATTEMPTS &&
+        respondedAttempts < RUNNER_RUNTIME_CONTROL_MAX_ATTEMPTS &&
         isRetryableRuntimeControlStatus(response.status)
       ) {
         await response.text().catch(() => '');
         await sleepImpl(
-          RUNNER_RUNTIME_CONTROL_RETRY_DELAYS_MS[attempt - 1] ?? 250,
+          RUNNER_RUNTIME_CONTROL_RETRY_DELAYS_MS[respondedAttempts - 1] ?? 250,
         );
         continue;
       }
@@ -646,17 +655,29 @@ export async function fetchRunnerRuntimeControl(input: {
       const transportMessage = timeoutSignal.aborted
         ? `timed out after ${timeoutMs}ms`
         : message;
-      if (!input.retrySafe || attempt >= RUNNER_RUNTIME_CONTROL_MAX_ATTEMPTS) {
-        throw new Error(
-          `${input.description} transport failed: ${transportMessage}`,
+      const failure = new Error(
+        `${input.description} transport failed: ${transportMessage}`,
+      );
+      if (!input.retrySafe || input.init.signal?.aborted) throw failure;
+      const delayMs =
+        RUNNER_RUNTIME_CONTROL_RETRY_DELAYS_MS[attempt - 1] ??
+        Math.min(
+          8_000,
+          1_000 * (attempt - RUNNER_RUNTIME_CONTROL_RETRY_DELAYS_MS.length),
         );
+      const outageRetryAllowed =
+        (timeoutSignal.aborted || isRunnerGatewayNoResponseError(error)) &&
+        Date.now() - startedAt + delayMs <=
+          RUNNER_GATEWAY_CONNECTION_RETRY_BUDGET_MS;
+      if (
+        attempt >= RUNNER_RUNTIME_CONTROL_MAX_ATTEMPTS &&
+        !outageRetryAllowed
+      ) {
+        throw failure;
       }
-      await sleepImpl(
-        RUNNER_RUNTIME_CONTROL_RETRY_DELAYS_MS[attempt - 1] ?? 250,
-      );
+      await sleepImpl(delayMs);
     }
   }
-  throw new Error(`${input.description} transport failed.`);
 }
 
 function createRuntimeToolMetadataResolver(config: PlayRunnerExecutionConfig) {
@@ -888,6 +909,7 @@ function createRateStateBackendForConfig(
     vercelProtectionBypassToken:
       config.context.vercelProtectionBypassToken ?? null,
     integrationMode: config.context.integrationMode,
+    connectionRetryBudgetMs: RUNNER_GATEWAY_CONNECTION_RETRY_BUDGET_MS,
   };
   return {
     requireSharedRateState: true,
@@ -2211,6 +2233,7 @@ async function run(
     executorToken: config.context.executorToken ?? '',
     vercelProtectionBypassToken: config.context.vercelProtectionBypassToken,
     runtimeTestFaultHeader: config.context.runtimeTestFaultHeader ?? null,
+    connectionRetryBudgetMs: RUNNER_GATEWAY_CONNECTION_RETRY_BUDGET_MS,
   };
   const receiptGatewayResponseObserver = createReceiptGatewayResponseObserver();
   const receiptGatewayBaseUrl = config.context.receiptGatewayBaseUrl?.trim();
@@ -2360,6 +2383,24 @@ async function run(
     toolResponseContract: config.context.toolResponseContract,
     toolResponseReceiptRevision: config.context.toolResponseReceiptRevision,
   });
+  const arrayInputRuntimeContext = (scope: {
+    playName: string;
+    runId: string;
+  }) => ({
+    baseUrl: receiptGatewayBaseUrl || config.context.baseUrl,
+    executorToken: config.context.executorToken,
+    // Input snapshots must cross the gateway's current-attempt fence, even
+    // when ordinary row persistence can use a preloaded tenant session.
+    dbSessionStrategy: 'gateway_only' as const,
+    preloadedDbSessions: runtimePreloadedDbSessions,
+    postgresSessionUnwrapKey: runtimePostgresSessionUnwrapKey,
+    abortSignal: persistenceSignal,
+    playName: scope.playName,
+    runId: scope.runId,
+    orgId: config.context.orgId,
+    userEmail: config.context.userEmail,
+    vercelProtectionBypassToken: config.context.vercelProtectionBypassToken,
+  });
   const ctx = createPlayContext({
     artifactRuntimeContract,
     rowAdmissionSignal: options?.rowAdmissionSignal,
@@ -2435,6 +2476,52 @@ async function run(
     convexUrl: config.context.convexUrl,
     runtimeSchedulerSchema: config.context.runtimeSchedulerSchema ?? null,
     runtimeSheetBackedMapDatasets: true,
+    runtimeArrayInputStagingEnabled: true,
+    runtimeArrayInputStore: {
+      getManifest: (scope) =>
+        retryRuntimeSheetPersistence({
+          description: `runtime array input getManifest table=${scope.tableNamespace}`,
+          log: (line) => emitWorkerLog(line, options),
+          signal: persistenceSignal,
+          run: () =>
+            getRuntimeArrayInputManifest(
+              arrayInputRuntimeContext(scope),
+              scope,
+            ),
+        }),
+      begin: (input) =>
+        retryRuntimeSheetPersistence({
+          description: `runtime array input begin table=${input.tableNamespace}`,
+          log: (line) => emitWorkerLog(line, options),
+          signal: persistenceSignal,
+          run: () =>
+            beginRuntimeArrayInput(arrayInputRuntimeContext(input), input),
+        }),
+      putPage: (input) =>
+        retryRuntimeSheetPersistence({
+          description: `runtime array input putPage table=${input.tableNamespace}`,
+          log: (line) => emitWorkerLog(line, options),
+          signal: persistenceSignal,
+          run: () =>
+            putRuntimeArrayInputPage(arrayInputRuntimeContext(input), input),
+        }),
+      seal: (input) =>
+        retryRuntimeSheetPersistence({
+          description: `runtime array input seal table=${input.tableNamespace}`,
+          log: (line) => emitWorkerLog(line, options),
+          signal: persistenceSignal,
+          run: () =>
+            sealRuntimeArrayInput(arrayInputRuntimeContext(input), input),
+        }),
+      readPage: (input) =>
+        retryRuntimeSheetPersistence({
+          description: `runtime array input readPage table=${input.tableNamespace}`,
+          log: (line) => emitWorkerLog(line, options),
+          signal: persistenceSignal,
+          run: () =>
+            readRuntimeArrayInputPage(arrayInputRuntimeContext(input), input),
+        }),
+    },
     runtimeReceiptClaimsEstablishExecutionFence:
       receiptStoreAdapter?.claimsEstablishExecutionFence === true,
     workflowId: config.context.workflowId,
@@ -2644,7 +2731,7 @@ async function run(
         options,
       );
     },
-    onMapStart: async (items, tableNamespace, mapContext) => {
+    onMapStartPages: async function* (items, tableNamespace, mapContext) {
       assertPlayRunnerRuntimeSheetPersistenceContext(config.context);
       const { baseUrl, orgId } = config.context;
       const startedAt = Date.now();
@@ -2681,153 +2768,161 @@ async function run(
       // after a scheduler wake recreates this Map. Retain the local version as
       // an additional consistency check across pages of this runner.
       const existingAttempt = runtimeSheetAttempts.get(attemptKey);
-      const mapStartResult = await retryRuntimeSheetPersistence({
-        description: `runtime map start table=${tableNamespace}`,
-        log: (line) => emitWorkerLog(line, options),
-        signal: persistenceSignal,
-        run: () =>
-          startRuntimeSheetDataset(
-            {
-              executorToken,
-              baseUrl,
-              vercelProtectionBypassToken:
-                config.context.vercelProtectionBypassToken,
-              orgId,
-              playName,
-              runId,
-              userEmail,
-              dbSessionStrategy: config.context.dbSessionStrategy,
-              preloadedDbSessions: canUseRootPreloadedDbSessions
-                ? runtimePreloadedDbSessions
-                : undefined,
-              postgresSessionUnwrapKey: canUseRootPreloadedDbSessions
-                ? runtimePostgresSessionUnwrapKey
-                : executorToken,
-              abortSignal: persistenceSignal,
-            },
-            {
-              playName,
-              tableNamespace,
-              playInput: runtimeInputObject,
-              sheetContract: augmentSheetContractWithDatasetFields({
-                contract: sheetContract,
-                rows: items,
-                columnTypes: mapContext.columnTypes,
-              }),
-              rows: items,
-              stableWriteIntent: true,
-              pendingRerunAdmission: true,
-              includePreviousCells: mapContext.includePreviousCells,
-              previousCellsBudgetBytes: mapContext.previousCellsBudgetBytes,
-              inputOffset: mapContext.inputOffset,
-              mode: mapContext.mode,
-              runId,
-              attemptId: deterministicRuntimeSheetAttemptId({
-                playName,
-                runId,
-                tableNamespace,
-                attemptSeq: runnerRunAttempt,
-              }),
-              attemptOwnerRunId: runId,
-              attemptSeq: runnerRunAttempt,
-              ...runtimeSheetStartWriteVersion(existingAttempt),
-              ...(playRunnerContextRuntimeSheetForce(config.context)
-                ? { force: true }
-                : {}),
-            },
-          ),
-      });
-      if (mapStartResult.membershipCapture === true)
-        membershipCapturingMaps.add(attemptKey);
-      const datasetId = createRuntimeDatasetId(playName, tableNamespace);
-      const datasetPath = `datasets.${tableNamespace}`;
-      const settlement = runtimeMapDatasetSettlements.get(datasetId) ?? {
-        datasetId,
-        path: datasetPath,
-        tableNamespace,
-        runId,
-        persistedRows: 0,
-        succeededRows: 0,
-        failedRows: 0,
-      };
-      // `completedRows` is the sheet admission result, not a terminal write
-      // for this run. Normal maps still execute every admitted row (the tool
-      // receipt may be reused inside that execution), and their terminal
-      // callback below is the authoritative settlement. Counting these rows
-      // here as well would report N rows at map start and another N rows at
-      // terminal persistence, advertising 2N rows for an N-row Dataset Handle.
-      // `net_new` maps exclude these rows before execution, so they must not
-      // be counted here either.
-      runtimeMapDatasetSettlements.set(datasetId, settlement);
-      emitRunnerEvent(
+      const admissionPages = startRuntimeSheetDatasetPages(
         {
-          type: 'execution_event',
-          event: {
-            type: RUN_EVENT_KIND.datasetLifecycle,
-            datasetId,
-            path: datasetPath,
-            tableNamespace,
-            phase: 'registered',
-            persistedRows: 0,
-            succeededRows: 0,
-            failedRows: 0,
-            complete: false,
-            // ADR 0019 dataset-grain row birth. These rows entered from
-            // play-body code, so the only witnessed fact is how many arrived at
-            // this table — never a per-row source mapping, which the runtime
-            // does not see here. `rowCountIn` counts rows admitted through this
-            // boundary so far, so a paged dataset converges upward instead of
-            // reporting only its last page.
-            bornFrom: {
-              table: tableNamespace,
-              rowCountIn: (mapContext.inputOffset ?? 0) + items.length,
-            },
-            at: Date.now(),
-          },
+          executorToken,
+          baseUrl,
+          vercelProtectionBypassToken:
+            config.context.vercelProtectionBypassToken,
+          orgId,
+          playName,
+          runId,
+          userEmail,
+          dbSessionStrategy: config.context.dbSessionStrategy,
+          preloadedDbSessions: canUseRootPreloadedDbSessions
+            ? runtimePreloadedDbSessions
+            : undefined,
+          postgresSessionUnwrapKey: canUseRootPreloadedDbSessions
+            ? runtimePostgresSessionUnwrapKey
+            : executorToken,
+          abortSignal: persistenceSignal,
         },
-        options,
-      );
-      emitRunnerEvent(
         {
-          type: 'log',
-          at: new Date().toISOString(),
-          source: 'play',
-          line: formatDatasetRegistrationLog({
-            dataset: { path: datasetPath, tableNamespace },
-            rows: items.length,
+          playName,
+          tableNamespace,
+          sheetContract: augmentSheetContractWithDatasetFields({
+            contract: sheetContract,
+            rows: items,
+            columnTypes: mapContext.columnTypes,
           }),
+          rows: items,
+          stableWriteIntent: true,
+          pendingRerunAdmission: true,
+          includePreviousCells: mapContext.includePreviousCells,
+          previousCellsBudgetBytes: mapContext.previousCellsBudgetBytes,
+          inputOffset: mapContext.inputOffset,
+          mode: mapContext.mode,
+          runId,
+          attemptId: deterministicRuntimeSheetAttemptId({
+            playName,
+            runId,
+            tableNamespace,
+            attemptSeq: runnerRunAttempt,
+          }),
+          attemptOwnerRunId: runId,
+          attemptSeq: runnerRunAttempt,
+          ...runtimeSheetStartWriteVersion(existingAttempt),
+          ...(playRunnerContextRuntimeSheetForce(config.context)
+            ? { force: true }
+            : {}),
+        },
+        {
+          runPage: (run) =>
+            retryRuntimeSheetPersistence({
+              description: `runtime map start table=${tableNamespace}`,
+              log: (line) => emitWorkerLog(line, options),
+              signal: persistenceSignal,
+              run,
+            }),
         },
-        options,
       );
-      if (
-        mapStartResult.attemptId &&
-        mapStartResult.attemptOwnerRunId &&
-        mapStartResult.attemptExpiresAt
-      ) {
-        runtimeSheetAttempts.set(
-          attemptKey,
-          runtimeSheetAttemptForMapStart({
+      let registered = false;
+      for await (const mapStartResult of admissionPages) {
+        if (mapStartResult.membershipCapture === true)
+          membershipCapturingMaps.add(attemptKey);
+        if (!registered) {
+          registered = true;
+          const datasetId = createRuntimeDatasetId(playName, tableNamespace);
+          const datasetPath = `datasets.${tableNamespace}`;
+          const settlement = runtimeMapDatasetSettlements.get(datasetId) ?? {
+            datasetId,
+            path: datasetPath,
             tableNamespace,
             runId,
-            existing: existingAttempt,
-            mapStartResult: {
-              attemptId: mapStartResult.attemptId,
-              attemptOwnerRunId: mapStartResult.attemptOwnerRunId,
-              attemptExpiresAt: mapStartResult.attemptExpiresAt,
-              attemptSeq: mapStartResult.attemptSeq,
-              writeVersion: mapStartResult.writeVersion,
+            persistedRows: 0,
+            succeededRows: 0,
+            failedRows: 0,
+          };
+          // `completedRows` is the sheet admission result, not a terminal write
+          // for this run. Normal maps still execute every admitted row (the tool
+          // receipt may be reused inside that execution), and their terminal
+          // callback below is the authoritative settlement. Counting these rows
+          // here as well would report N rows at map start and another N rows at
+          // terminal persistence, advertising 2N rows for an N-row Dataset Handle.
+          // `net_new` maps exclude these rows before execution, so they must not
+          // be counted here either.
+          runtimeMapDatasetSettlements.set(datasetId, settlement);
+          emitRunnerEvent(
+            {
+              type: 'execution_event',
+              event: {
+                type: RUN_EVENT_KIND.datasetLifecycle,
+                datasetId,
+                path: datasetPath,
+                tableNamespace,
+                phase: 'registered',
+                persistedRows: 0,
+                succeededRows: 0,
+                failedRows: 0,
+                complete: false,
+                // ADR 0019 dataset-grain row birth. These rows entered from
+                // play-body code, so the only witnessed fact is how many arrived at
+                // this table — never a per-row source mapping, which the runtime
+                // does not see here. `rowCountIn` counts rows admitted through this
+                // boundary so far, so a paged dataset converges upward instead of
+                // reporting only its last page.
+                bornFrom: {
+                  table: tableNamespace,
+                  rowCountIn: (mapContext.inputOffset ?? 0) + items.length,
+                },
+                at: Date.now(),
+              },
             },
-            runnerRunAttempt,
-          }),
-        );
+            options,
+          );
+          emitRunnerEvent(
+            {
+              type: 'log',
+              at: new Date().toISOString(),
+              source: 'play',
+              line: formatDatasetRegistrationLog({
+                dataset: { path: datasetPath, tableNamespace },
+                rows: items.length,
+              }),
+            },
+            options,
+          );
+        }
+        if (
+          mapStartResult.attemptId &&
+          mapStartResult.attemptOwnerRunId &&
+          mapStartResult.attemptExpiresAt
+        ) {
+          runtimeSheetAttempts.set(
+            attemptKey,
+            runtimeSheetAttemptForMapStart({
+              tableNamespace,
+              runId,
+              existing: runtimeSheetAttempts.get(attemptKey),
+              mapStartResult: {
+                attemptId: mapStartResult.attemptId,
+                attemptOwnerRunId: mapStartResult.attemptOwnerRunId,
+                attemptExpiresAt: mapStartResult.attemptExpiresAt,
+                attemptSeq: mapStartResult.attemptSeq,
+                writeVersion: mapStartResult.writeVersion,
+              },
+              runnerRunAttempt,
+            }),
+          );
+        }
+        inserted += mapStartResult.inserted;
+        skipped += mapStartResult.skipped;
+        yield mapStartResult;
       }
       emitWorkerLog(
         `[perf] runtime map setup table=${tableNamespace} rows=${items.length} elapsed_ms=${Date.now() - startedAt}`,
         options,
       );
-      inserted += mapStartResult.inserted;
-      skipped += mapStartResult.skipped;
-      return mapStartResult;
     },
     onMapRowsCheckpoint: async (input) => {
       if (input.updates.length === 0) return;
@@ -2878,6 +2973,8 @@ async function run(
               vercelProtectionBypassToken:
                 config.context.vercelProtectionBypassToken,
               signal: persistenceSignal,
+              connectionRetryBudgetMs:
+                RUNNER_GATEWAY_CONNECTION_RETRY_BUDGET_MS,
             },
             {
               playName,
@@ -3800,10 +3897,27 @@ export function projectPlayResultForTransport(
     output[key] = entry;
   }
 
-  assertCustomerPlayReturnWithinLimit({
+  const boundedOutput = truncateCustomerPlayReturnWithinLimit({
     value: output,
   });
-  return { output, warnings: [] };
+  return {
+    output: boundedOutput.value,
+    warnings: boundedOutput.truncated
+      ? [
+          {
+            code: 'RUN_OUTPUT_TRUNCATED',
+            reason: 'byte_limit',
+            message:
+              'The Play return was truncated to fit the output limit. Return a Dataset Handle to keep a large result recoverable.',
+            originalBytesAtLeast: boundedOutput.originalBytesAtLeast,
+            retainedBytes: boundedOutput.retainedBytes,
+            limitBytes: boundedOutput.limitBytes,
+            retryable: false,
+            recoverableFrom: 'none',
+          },
+        ]
+      : [],
+  };
 }
 
 function serializePlayResult<T>(value: T): T {

package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-gateway-transport.ts

@@ -58,6 +58,23 @@ export function runnerGatewayTransportErrorCode(error: unknown): string {
   return SAFE_ERROR_NAMES.has(name) ? name : 'unknown';
 }
 
+/**
+ * True when a request got no HTTP response at all: DNS, connect, reset, or a
+ * socket that stalled. Node's fetch reports these as `fetch failed` or
+ * `terminated` with an errno or undici code on the cause.
+ */
+export function isRunnerGatewayNoResponseError(error: unknown): boolean {
+  if (
+    /^(?:E[A-Z0-9_]+|UND_ERR_[A-Z0-9_]+)$/.test(
+      runnerGatewayTransportErrorCode(error),
+    )
+  ) {
+    return true;
+  }
+  const message = error instanceof Error ? error.message : '';
+  return /^(?:fetch failed|terminated)$/i.test(message);
+}
+
 export function updateRunnerGatewayTransportSummary(
   previous: RunnerGatewayTransportSummary | null,
   event: RunnerGatewayTransportEvent,