deepline 0.3.461 → 0.3.463

package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts

@@ -365,6 +365,7 @@ type RuntimeApiRequest =
       action: 'governor_budget_charge';
     } & PlayRunnerBudgetChargeInput)
   | { action: 'get_billing_entitlements' }
+  | ({ action: 'write_page' } & PlayPageWriteRequest)
   | { action: 'get_runtime_traffic_policy' }
   | RuntimeReceiptAction;
 
@@ -383,6 +384,13 @@ export type WorkerRuntimeApiContext = {
   requestTimeoutMs?: number | null;
   /** A higher-level writer owns retries and preserves the exact request body. */
   retryPolicy?: 'default' | 'none';
+  /**
+   * Opt-in for a sandbox runner, whose network can be cut for minutes while
+   * its process keeps running. When a retryable action's ordinary attempts
+   * all get no response, keep sending it until this long after the first
+   * attempt. An error response never extends the retries.
+   */
+  connectionRetryBudgetMs?: number;
   /**
    * Observe the gateway request identity of a SUCCESSFUL response. Failures
    * already carry it on the thrown error; a success carried it nowhere, so a
@@ -570,6 +578,11 @@ export function parseRuntimeGatewayServerTiming(
 }
 
 const APP_RUNTIME_API_RETRY_DELAYS_MS = [100, 250, 500, 1_000] as const;
+// Vercel Function ingress rejects request bodies above 4.5 MB. Keep normal
+// map update requests well below it and leave room for a single large row.
+const APP_RUNTIME_ROW_UPDATE_BATCH_TARGET_BYTES = 3 * 1024 * 1024;
+const APP_RUNTIME_ROW_UPDATE_REQUEST_MAX_BYTES = 4_450_000;
+const APP_RUNTIME_JSON_ENCODER = new TextEncoder();
 const APP_RUNTIME_API_RECEIPT_RETRY_DELAYS_MS = [
   250, 500, 1_000, 2_000, 4_000, 8_000, 12_000,
 ] as const;
@@ -1277,6 +1290,8 @@ function emitAppRuntimeRetrySummary(input: {
 async function retryAppRuntimeBodyTimeoutOrThrow(input: {
   action: RuntimeApiRequest['action'];
   attempt: number;
+  /** Position on the error-response ladder; decides the retry and its delay. */
+  ladderAttempt: number;
   transportAttemptId: string;
   maxAttempts: number;
   error: unknown;
@@ -1285,6 +1300,7 @@ async function retryAppRuntimeBodyTimeoutOrThrow(input: {
   httpStatus?: number;
   retryAfterMs?: number | null;
   boundaryLabel: string;
+  signal?: AbortSignal;
 }): Promise<void> {
   if (!isRequestTimeoutAbort(input.error)) {
     recordAppRuntimeRetryFailure({
@@ -1307,14 +1323,14 @@ async function retryAppRuntimeBodyTimeoutOrThrow(input: {
     throw input.error;
   }
   if (
-    input.attempt < input.maxAttempts &&
+    input.ladderAttempt < input.maxAttempts &&
     isRetryableAppRuntimeFetchError({
       action: input.action,
       error: input.error,
     })
   ) {
     const retryDelayMs = appRuntimeRetrySleepMs(
-      appRuntimeRetryDelayMs(input.action, input.attempt),
+      appRuntimeRetryDelayMs(input.action, input.ladderAttempt),
       input.retryAfterMs,
     );
     recordAppRuntimeRetryFailure({
@@ -1326,7 +1342,7 @@ async function retryAppRuntimeBodyTimeoutOrThrow(input: {
       retryDelayMs,
       httpStatus: input.httpStatus,
     });
-    await sleep(retryDelayMs);
+    await sleepUnlessAborted(retryDelayMs, input.signal);
     return;
   }
   recordAppRuntimeRetryFailure({
@@ -1512,14 +1528,111 @@ export function isAppRuntimeApiCapacityError(
   );
 }
 
+const CAPACITY_SHED_DEFAULT_RETRY_AFTER_MS = 1_000;
+
+/**
+ * How long a working store asked the caller to wait, or null when `error` is
+ * not a load shed. A shed proves only that this one delivery did not run, so a
+ * caller re-sending the work needs its own idempotency.
+ */
+export function capacityShedRetryAfterMs(error: unknown): number | null {
+  const shed =
+    isAppRuntimeApiCapacityError(error) ||
+    (error instanceof AppRuntimeApiResponseError &&
+      error.retryable &&
+      error.status === 429);
+  if (!shed) return null;
+  const retryAfterMs = (error as AppRuntimeApiResponseError).retryAfterMs;
+  return retryAfterMs && retryAfterMs > 0
+    ? retryAfterMs
+    : CAPACITY_SHED_DEFAULT_RETRY_AFTER_MS;
+}
+
 function resolveAppRuntimeApiUrl(context: WorkerRuntimeApiContext): string {
   const baseUrl = context.baseUrl.trim().replace(/\/$/, '');
   return `${baseUrl}${PLAY_RUNTIME_API_COMPAT_PATH}`;
 }
 
+const APP_RUNTIME_CONNECTION_RETRY_MAX_DELAY_MS = 8_000;
+
 async function postAppRuntimeApi<TResponse>(
   context: WorkerRuntimeApiContext,
   body: RuntimeApiRequest,
+): Promise<TResponse> {
+  const budgetMs =
+    context.retryPolicy === 'none'
+      ? undefined
+      : context.connectionRetryBudgetMs;
+  if (!budgetMs || !isRetryableAppRuntimeAction(body.action)) {
+    return await postAppRuntimeApiAttempts<TResponse>(context, body, null);
+  }
+  const startedAt = Date.now();
+  // Shared by every round below, so error responses spend one attempt ladder
+  // for the whole operation however the cut splits the rounds.
+  const ladder: AppRuntimeResponseLadder = {
+    respondedAttempts: 0,
+    lastAttemptResponded: false,
+  };
+  for (let outageRetry = 1; ; outageRetry += 1) {
+    try {
+      return await postAppRuntimeApiAttempts<TResponse>(context, body, ladder);
+    } catch (error) {
+      if (
+        !(error instanceof AppRuntimeApiTransportError) ||
+        // A status line arrived, so the error-response ladder already decided.
+        ladder.lastAttemptResponded ||
+        !isRetryableAppRuntimeFetchError({
+          action: body.action,
+          error: error.cause,
+        })
+      ) {
+        throw error;
+      }
+      const delayMs = applyRetryJitter(
+        Math.min(
+          APP_RUNTIME_CONNECTION_RETRY_MAX_DELAY_MS,
+          1_000 * outageRetry,
+        ),
+      );
+      if (Date.now() - startedAt + delayMs > budgetMs) throw error;
+      await sleepUnlessAborted(delayMs, context.signal);
+    }
+  }
+}
+
+function sleepUnlessAborted(ms: number, signal?: AbortSignal): Promise<void> {
+  return new Promise((resolve, reject) => {
+    if (signal?.aborted) {
+      reject(signal.reason);
+      return;
+    }
+    const onAbort = () => {
+      clearTimeout(timer);
+      reject(signal?.reason);
+    };
+    const timer = setTimeout(() => {
+      signal?.removeEventListener('abort', onAbort);
+      resolve();
+    }, ms);
+    signal?.addEventListener('abort', onAbort, { once: true });
+  });
+}
+
+/**
+ * Attempts that got an HTTP response, counted across one operation. Present
+ * only under a runner's connection retry budget: attempts that got no response
+ * then never spend the ladder that error responses use.
+ */
+type AppRuntimeResponseLadder = {
+  respondedAttempts: number;
+  /** The latest attempt got a status line, even if its body then failed. */
+  lastAttemptResponded: boolean;
+};
+
+async function postAppRuntimeApiAttempts<TResponse>(
+  context: WorkerRuntimeApiContext,
+  body: RuntimeApiRequest,
+  ladder: AppRuntimeResponseLadder | null,
 ): Promise<TResponse> {
   const baseUrl = context.baseUrl.trim().replace(/\/$/, '');
   const token = context.executorToken.trim();
@@ -1575,7 +1688,11 @@ async function postAppRuntimeApi<TResponse>(
   // immediate plain-JSON resend even when the caller disabled normal retries.
   // It does not consume an application retry attempt or sleep.
   let attempt = 1;
-  while (attempt <= maxAttempts) {
+  // With a ladder, attempts that got no response do not spend it, so `attempt`
+  // can pass `maxAttempts` while error responses still have retries left. The
+  // loop still ends: a response retries only while the ladder has room, and a
+  // failure with no response retries here only while `attempt < maxAttempts`.
+  while (ladder !== null || attempt <= maxAttempts) {
     // Re-evaluated per attempt rather than hoisted: the capability can be
     // learned from THIS request's own first attempt (or from a sibling request
     // that landed in between), so a retry compresses as soon as the origin has
@@ -1594,6 +1711,7 @@ async function postAppRuntimeApi<TResponse>(
     // would show up as network cost that no round trip explains.
     const attemptStartedAt = Date.now();
     const transportAttemptId = randomUUID();
+    if (ladder) ladder.lastAttemptResponded = false;
     if (context.onRequestAttempt) {
       try {
         context.onRequestAttempt({ action: body.action, transportAttemptId });
@@ -1687,7 +1805,7 @@ async function postAppRuntimeApi<TResponse>(
           attemptStartedAt,
           retryDelayMs,
         });
-        await sleep(retryDelayMs);
+        await sleepUnlessAborted(retryDelayMs, context.signal);
         attempt += 1;
         continue;
       }
@@ -1715,6 +1833,13 @@ async function postAppRuntimeApi<TResponse>(
         boundaryLabel,
       });
     }
+    if (ladder) {
+      ladder.respondedAttempts += 1;
+      ladder.lastAttemptResponded = true;
+    }
+    // The position on the error-response ladder: every attempt, or under a
+    // connection retry budget only the attempts that got a response.
+    const ladderAttempt = ladder ? ladder.respondedAttempts : attempt;
     // Learn the capability from EVERY response, including 4xx/5xx: an error
     // response still proves the gateway that produced it can decode a
     // compressed body, and a run whose first request fails must not be stuck
@@ -1784,6 +1909,7 @@ async function postAppRuntimeApi<TResponse>(
         await retryAppRuntimeBodyTimeoutOrThrow({
           action: body.action,
           attempt,
+          ladderAttempt,
           transportAttemptId,
           maxAttempts,
           error,
@@ -1791,6 +1917,7 @@ async function postAppRuntimeApi<TResponse>(
           attemptStartedAt,
           httpStatus: response.status,
           boundaryLabel,
+          signal: context.signal,
         });
         attempt += 1;
         continue;
@@ -1829,6 +1956,7 @@ async function postAppRuntimeApi<TResponse>(
       await retryAppRuntimeBodyTimeoutOrThrow({
         action: body.action,
         attempt,
+        ladderAttempt,
         transportAttemptId,
         maxAttempts,
         error,
@@ -1837,6 +1965,7 @@ async function postAppRuntimeApi<TResponse>(
         httpStatus: response.status,
         retryAfterMs: appRuntimeRetryAfterMs(response, ''),
         boundaryLabel,
+        signal: context.signal,
       });
       attempt += 1;
       continue;
@@ -1855,6 +1984,8 @@ async function postAppRuntimeApi<TResponse>(
       revokeRuntimeApiCompressionCapability(baseUrl);
       forceUncompressedNextAttempt = true;
       usedLegacyDecoderFallback = true;
+      // Nothing was dispatched, so the resend spends no ladder step either.
+      if (ladder) ladder.respondedAttempts -= 1;
       continue;
     }
     const responseCode = appRuntimeErrorCode(response, responseText);
@@ -1873,7 +2004,7 @@ async function postAppRuntimeApi<TResponse>(
       });
     }
     if (
-      attempt < maxAttempts &&
+      ladderAttempt < maxAttempts &&
       isRetryableAppRuntimeResponse({
         action: body.action,
         status: response.status,
@@ -1882,7 +2013,7 @@ async function postAppRuntimeApi<TResponse>(
       })
     ) {
       const retryDelayMs = appRuntimeRetrySleepMs(
-        appRuntimeRetryDelayMs(body.action, attempt),
+        appRuntimeRetryDelayMs(body.action, ladderAttempt),
         appRuntimeRetryAfterMs(response, responseText),
       );
       recordAppRuntimeRetryFailure({
@@ -2214,6 +2345,57 @@ export async function getBillingEntitlementsViaAppRuntime(
   }
 }
 
+export type PlayPageWriteRequest = {
+  name: string;
+  public: boolean;
+  contentType: string;
+  description?: string;
+  /** Exactly one of these carries the content. */
+  contentText?: string;
+  contentBase64?: string;
+};
+
+export type PlayPageWriteResult = {
+  name: string;
+  url: string;
+  public: boolean;
+  created: boolean;
+  warnings: string[];
+};
+
+export async function writePageViaAppRuntime(
+  context: WorkerRuntimeApiContext,
+  input: PlayPageWriteRequest,
+): Promise<PlayPageWriteResult> {
+  try {
+    return await postAppRuntimeApi<PlayPageWriteResult>(context, {
+      action: 'write_page',
+      ...input,
+    });
+  } catch (error) {
+    // An older receiver does not know write_page, and a token minted while
+    // page.write is withheld from the defaults lacks it. write_page needs only
+    // page.write, so either exact 403 means page writes are not available yet.
+    if (
+      error instanceof AppRuntimeApiResponseError &&
+      error.status === 403 &&
+      error.code === null &&
+      (error.detail === 'Unsupported runtime action capability scope.' ||
+        error.detail ===
+          'Workflow executor token does not permit this operation.')
+    ) {
+      throw new AppRuntimeApiResponseError({
+        action: 'write_page',
+        status: 503,
+        code: 'PAGES_UNAVAILABLE',
+        retryable: false,
+        detail: 'Page writes are not available to this run yet.',
+      });
+    }
+    throw error;
+  }
+}
+
 export async function getRuntimeTrafficPolicyViaAppRuntime(
   context: WorkerRuntimeApiContext,
 ): Promise<RuntimeTrafficPolicy> {
@@ -2814,10 +2996,67 @@ export async function applyRowUpdatesViaAppRuntime(
     ? Omit<T & { action: 'apply_row_updates' }, 'action'>
     : never,
 ): Promise<void> {
-  await postAppRuntimeApi<{ ok: true }>(context, {
-    action: 'apply_row_updates',
-    ...input,
-  });
+  const requestForUpdates = (updates: typeof input.updates) =>
+    ({
+      action: 'apply_row_updates',
+      ...input,
+      updates,
+    }) as Extract<RuntimeApiRequest, { action: 'apply_row_updates' }>;
+  if (input.updates.length === 0) {
+    await postAppRuntimeApi<{ ok: true }>(
+      context,
+      requestForUpdates(input.updates),
+    );
+    return;
+  }
+  const measureJsonBytes = (value: unknown) => {
+    const serialized = JSON.stringify(
+      prepareRetainedPlayPipelinesForTransport(value),
+    );
+    if (serialized === undefined) {
+      throw new Error(
+        'Runtime row update request is not serializable as JSON.',
+      );
+    }
+    return APP_RUNTIME_JSON_ENCODER.encode(serialized).byteLength;
+  };
+  const emptyRequestBytes = measureJsonBytes(requestForUpdates([]));
+  let batch: typeof input.updates = [];
+  let batchBytes = emptyRequestBytes;
+
+  const flush = async () => {
+    if (batch.length === 0) return;
+    // Map checkpoint updates carry the same row keys and write-version fence
+    // in every page. Replaying a successful page after a later page fails is
+    // therefore an idempotent merge of the same cell patches.
+    await postAppRuntimeApi<{ ok: true }>(context, requestForUpdates(batch));
+    batch = [];
+    batchBytes = emptyRequestBytes;
+  };
+
+  for (const update of input.updates) {
+    const updateBytes = measureJsonBytes(update);
+    const separatorBytes = batch.length > 0 ? 1 : 0;
+    let nextBytes = batchBytes + separatorBytes + updateBytes;
+    if (
+      batch.length > 0 &&
+      nextBytes > APP_RUNTIME_ROW_UPDATE_BATCH_TARGET_BYTES
+    ) {
+      await flush();
+      nextBytes = batchBytes + updateBytes;
+    }
+    if (nextBytes > APP_RUNTIME_ROW_UPDATE_REQUEST_MAX_BYTES) {
+      throw new Error(
+        `A single runtime row update needs ${nextBytes} JSON bytes, above the ${APP_RUNTIME_ROW_UPDATE_REQUEST_MAX_BYTES}-byte app request budget.`,
+      );
+    }
+    batch.push(update);
+    batchBytes = nextBytes;
+    if (batchBytes > APP_RUNTIME_ROW_UPDATE_BATCH_TARGET_BYTES) {
+      await flush();
+    }
+  }
+  await flush();
 }
 
 export async function createDbSessionViaAppRuntime(