deepline 0.3.231 → 0.3.233

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -92,7 +92,9 @@ import {
 } from './run-execution-scope';
 import { DEFAULT_RUNTIME_EXECUTION_CAPABILITIES } from './execution-capabilities';
 import { vercelProtectionBypassHeader } from './vercel-protection';
-import { RUNTIME_RELIABILITY_POLICY } from './runtime-reliability-policy';
+import { positiveTimeoutMs } from './runtime-reliability-policy';
+import { sendPlayHttpRequest } from './play-http-request';
+export { CtxFetchTimeoutError } from './play-http-request';
 export {
   RuntimeSheetRowsBlockedError,
   type RuntimeSheetBlockedRowDetail,
@@ -205,10 +207,6 @@ import {
   TOOL_EXECUTE_PROVIDER_THROTTLE_HEADER,
   classifyToolExecuteHttpFailure,
   createToolExecuteHttpFailureAttemptTracker,
-  TOOL_EXECUTE_BARE_RATE_LIMIT_MAX_ATTEMPTS,
-  TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS,
-  TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS,
-  TOOL_EXECUTE_RATE_LIMIT_RETRY_DELAY_MAX_MS,
   parseToolExecuteRetryAfterMs,
   parseToolExecuteRetryAtMs,
   parseToolExecuteAuthScopeChangedError,
@@ -590,19 +588,10 @@ const DEEPLINEAGENT_TOOL_RUNTIME_TIMEOUT_MS = 15 * 60 * 1000;
 // a bounded, repairable transport failure that row-isolation settles per row
 // instead of hanging the whole run.
 const DEFAULT_TOOL_RUNTIME_TIMEOUT_MS = 15 * 60 * 1000 + 30_000;
-const FETCH_TRANSPORT_MAX_ATTEMPTS =
-  RUNTIME_RELIABILITY_POLICY.egress.fetchMaxAttempts;
-const FETCH_TRANSPORT_RETRY_DELAY_MS = 100;
 // Diagnostic logs are retained in the context until terminalization. Keep a
 // representative, deduplicated set so row-heavy continued failures cannot turn
 // customer-safe observability into unbounded runner memory or log traffic.
 const MAX_CTX_FETCH_HTTP_FAILURE_DIAGNOSTICS = 16;
-const CTX_FETCH_HEADERS_TIMEOUT_MS =
-  RUNTIME_RELIABILITY_POLICY.egress.fetchHeadersTimeoutMs;
-const CTX_FETCH_BODY_TIMEOUT_MS =
-  RUNTIME_RELIABILITY_POLICY.egress.fetchBodyTimeoutMs;
-const CTX_FETCH_TOTAL_TIMEOUT_MS =
-  RUNTIME_RELIABILITY_POLICY.egress.fetchTotalTimeoutMs;
 // cloudflared returns this branded HTML when its connection to the local app
 // resets. It is a transport failure before the app can reach a provider, not a
 // provider 502; require both the branded page and our development tunnel host.
@@ -647,185 +636,6 @@ class ToolExecuteResponseBodyTimeoutError extends Error {
   }
 }
 
-export class CtxFetchTimeoutError extends Error {
-  readonly code = 'CTX_FETCH_TIMEOUT';
-
-  constructor(
-    readonly phase: 'headers' | 'body' | 'total',
-    readonly timeoutMs: number,
-  ) {
-    super(
-      `[CTX_FETCH_TIMEOUT phase=${phase} timeout_ms=${timeoutMs}] ` +
-        `ctx.fetch exceeded its ${phase} deadline after ${timeoutMs}ms.`,
-    );
-    this.name = 'CtxFetchTimeoutError';
-  }
-}
-
-type CtxFetchDeadline = {
-  bodyMs: number;
-  headersMs: number;
-  startedAt: number;
-  totalMs: number;
-};
-
-function positiveTimeoutMs(
-  value: number | undefined,
-  fallback: number,
-): number {
-  return value === undefined || !Number.isFinite(value) || value <= 0
-    ? fallback
-    : Math.max(1, Math.floor(value));
-}
-
-function remainingCtxFetchMs(deadline: CtxFetchDeadline): number {
-  return Math.max(0, deadline.totalMs - (Date.now() - deadline.startedAt));
-}
-
-async function runCtxFetchPhase<T>(input: {
-  callerSignal?: AbortSignal | null;
-  controller: AbortController;
-  deadline: CtxFetchDeadline;
-  phase: 'headers' | 'body' | 'total';
-  run: () => Promise<T>;
-}): Promise<T> {
-  const remainingMs = remainingCtxFetchMs(input.deadline);
-  if (remainingMs <= 0) {
-    throw new CtxFetchTimeoutError('total', input.deadline.totalMs);
-  }
-  if (input.callerSignal?.aborted) {
-    throw (
-      input.callerSignal.reason ?? new DOMException('Aborted', 'AbortError')
-    );
-  }
-  const phaseLimitMs =
-    input.phase === 'headers'
-      ? input.deadline.headersMs
-      : input.phase === 'body'
-        ? input.deadline.bodyMs
-        : remainingMs;
-  const timeoutMs = Math.min(phaseLimitMs, remainingMs);
-  const timeoutPhase = timeoutMs >= remainingMs ? 'total' : input.phase;
-  let timeout: ReturnType<typeof setTimeout> | null = null;
-  let onCallerAbort: (() => void) | null = null;
-  const phaseTimeout = new Promise<never>((_resolve, reject) => {
-    timeout = setTimeout(() => {
-      const error = new CtxFetchTimeoutError(
-        timeoutPhase,
-        timeoutPhase === 'total' ? input.deadline.totalMs : phaseLimitMs,
-      );
-      reject(error);
-      input.controller.abort(error);
-    }, timeoutMs);
-  });
-  const callerAbort = input.callerSignal
-    ? new Promise<never>((_resolve, reject) => {
-        onCallerAbort = () => {
-          const reason =
-            input.callerSignal?.reason ??
-            new DOMException('Aborted', 'AbortError');
-          reject(reason);
-          input.controller.abort(reason);
-        };
-        input.callerSignal!.addEventListener('abort', onCallerAbort, {
-          once: true,
-        });
-      })
-    : null;
-  try {
-    return await Promise.race([
-      input.run(),
-      phaseTimeout,
-      ...(callerAbort ? [callerAbort] : []),
-    ]);
-  } finally {
-    if (timeout) clearTimeout(timeout);
-    if (onCallerAbort) {
-      input.callerSignal?.removeEventListener('abort', onCallerAbort);
-    }
-  }
-}
-
-async function sleepWithinCtxFetchDeadline(input: {
-  callerSignal?: AbortSignal | null;
-  deadline: CtxFetchDeadline;
-  delayMs: number;
-}): Promise<void> {
-  const controller = new AbortController();
-  let timer: ReturnType<typeof setTimeout> | undefined;
-  try {
-    await runCtxFetchPhase({
-      callerSignal: input.callerSignal,
-      controller,
-      deadline: input.deadline,
-      phase: 'total',
-      run: () =>
-        new Promise<void>((resolve) => {
-          timer = setTimeout(resolve, input.delayMs);
-        }),
-    });
-  } finally {
-    if (timer !== undefined) clearTimeout(timer);
-  }
-}
-
-function isCtxFetchBackpressure(response: Response, bodyText: string): boolean {
-  if (response.status === 429) return true;
-  if (response.status !== 503) return false;
-  // This Salesforce endpoint reports a rejected burst as 503. Match the final
-  // destination and exact response contract; arbitrary 503s remain failures.
-  try {
-    const destination = new URL(response.url);
-    const body: unknown = JSON.parse(bodyText);
-    return (
-      destination.protocol === 'https:' &&
-      destination.hostname.endsWith('.salesforce-sites.com') &&
-      body !== null &&
-      typeof body === 'object' &&
-      !Array.isArray(body) &&
-      (body as Record<string, unknown>).status === 'burst_circuit_open'
-    );
-  } catch {
-    return false;
-  }
-}
-
-async function readCtxFetchBody(input: {
-  callerSignal?: AbortSignal | null;
-  controller: AbortController;
-  deadline: CtxFetchDeadline;
-  response: Response;
-}): Promise<string> {
-  if (!input.response.body) return '';
-  const reader = input.response.body.getReader();
-  const decoder = new TextDecoder();
-  try {
-    return await runCtxFetchPhase({
-      callerSignal: input.callerSignal,
-      controller: input.controller,
-      deadline: input.deadline,
-      phase: 'body',
-      run: async () => {
-        let body = '';
-        for (;;) {
-          const chunk = await reader.read();
-          if (chunk.done) return body + decoder.decode();
-          body += decoder.decode(chunk.value, { stream: true });
-        }
-      },
-    });
-  } catch (error) {
-    // Cancellation is socket hygiene, not part of the timeout contract. A
-    // hostile/custom stream may never resolve its cancel hook; awaiting it
-    // here would recreate the exact forever-pending ctx.fetch this deadline
-    // is meant to eliminate.
-    void reader.cancel(error).catch(() => {});
-    throw error;
-  } finally {
-    reader.releaseLock();
-  }
-}
-
 /** What a map is parked on when nothing in it is settling. */
 export type RuntimeMapReceiptWaitParking = {
   /** Receipt-wait groups parked right now. */
@@ -1336,9 +1146,6 @@ export function chunkArrayForReceiptRequest<T>(items: readonly T[]): T[][] {
   return chunks;
 }
 
-type SafeFetchModule = typeof import('../security/safe-fetch');
-let safeFetchModule: Promise<SafeFetchModule> | null = null;
-
 export async function waitForNodeRuntimeMapRowsVisible(input: {
   mapName: string;
   tableNamespace: string;
@@ -1546,11 +1353,6 @@ function isUnmarkedExecutionGateway502(input: {
   }
 }
 
-function loadSafeFetch(): Promise<SafeFetchModule> {
-  safeFetchModule ??= import('../security/safe-fetch');
-  return safeFetchModule;
-}
-
 function waitForSecretResolutionRetry(ms: number): Promise<void> {
   if (ms <= 0) return Promise.resolve();
   return new Promise<void>((resolve) => setTimeout(resolve, ms));
@@ -1578,10 +1380,6 @@ const NO_SECRET_RESOLUTION_RETRY: SecretResolutionRetryDecision = {
   retryAfterExceedsCap: false,
 };
 
-function isUnsafeOutboundUrlError(error: unknown): boolean {
-  return error instanceof Error && error.name === 'UnsafeOutboundUrlError';
-}
-
 function publicToolResponseEnvelope(value: unknown): {
   status: string;
   raw: unknown;
@@ -2635,7 +2433,9 @@ const WAITING_ROW = Symbol('deepline.waiting_row');
 const FAILED_ROW = Symbol('deepline.failed_row');
 const COMPLETED_ROW = Symbol('deepline.completed_row');
 type MapRowExecutionResult =
-  typeof COMPLETED_ROW | typeof WAITING_ROW | typeof FAILED_ROW;
+  | typeof COMPLETED_ROW
+  | typeof WAITING_ROW
+  | typeof FAILED_ROW;
 
 function normalizeFetchHeaders(
   headers: RequestInit['headers'],
@@ -4433,7 +4233,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     artifactTableNamespace: string;
     mapNodeId?: string | null;
     explicitKey?:
-      ((row: Record<string, unknown>, index: number) => string) | null;
+      | ((row: Record<string, unknown>, index: number) => string)
+      | null;
   }): MapExecutionScope {
     const mapInvocationId = `${input.logicalNamespace}:${this.mapInvocationIndex}`;
     this.mapInvocationIndex += 1;
@@ -4683,10 +4484,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
       // run actually owns. The map/cohort publisher keeps the claim-free
       // batched write, where the pointer genuinely shares the canonical
       // result's single bounded request.
-      if (!(
-        this.#options.claimRuntimeStepReceipt ||
-        this.#options.claimRuntimeStepReceipts
-      )) {
+      if (
+        !(
+          this.#options.claimRuntimeStepReceipt ||
+          this.#options.claimRuntimeStepReceipts
+        )
+      ) {
         return;
       }
       const existing = await this.getRuntimeStepReceipts(
@@ -8470,7 +8273,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     _key: string,
     _items: PlayDatasetInput<T>,
     _input?:
-      MapFieldDefinition<T, Record<string, unknown>> | RuntimeStepProgram,
+      | MapFieldDefinition<T, Record<string, unknown>>
+      | RuntimeStepProgram,
     _options?: RuntimeDatasetOptions<T>,
   ): never {
     void _key;
@@ -8594,7 +8398,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
 
     const userKeyOption = options?.key;
     let explicitKeyResolver:
-      ((row: Record<string, unknown>, index: number) => string) | null = null;
+      | ((row: Record<string, unknown>, index: number) => string)
+      | null = null;
     if (userKeyOption !== undefined) {
       explicitKeyResolver = (row, index) => {
         const inputRow = stripFieldOutputs(row);
@@ -15080,206 +14885,17 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
             rowStore?.mapStallObserver?.fetchStarted(
               rowStore.mapStallResolverToken,
             );
-          const fetchDeadline: CtxFetchDeadline = {
-            startedAt: Date.now(),
-            headersMs: positiveTimeoutMs(
-              this.#options.ctxFetchTimeouts?.headersMs,
-              CTX_FETCH_HEADERS_TIMEOUT_MS,
-            ),
-            bodyMs: positiveTimeoutMs(
-              this.#options.ctxFetchTimeouts?.bodyMs,
-              CTX_FETCH_BODY_TIMEOUT_MS,
-            ),
-            totalMs: positiveTimeoutMs(
-              this.#options.ctxFetchTimeouts?.totalMs,
-              CTX_FETCH_TOTAL_TIMEOUT_MS,
-            ),
-          };
-          let response: Response | null = null;
-          let bodyText: string | null = null;
           try {
-            // Only replay a body whose bytes are stable. In particular, a
-            // consumed ReadableStream cannot be retried even with a write key.
-            const replayableBody =
-              init.body == null || typeof init.body === 'string';
-            const canRetryTransport =
-              replayableBody && ['GET', 'HEAD', 'OPTIONS'].includes(method);
-            const { safePublicFetch } = await loadSafeFetch();
-            let transportAttempt = 1;
-            let rateLimitAttempt = 0;
-            while (true) {
-              const attemptController = new AbortController();
-              let permit: Awaited<
-                ReturnType<RuntimeResourceGovernor['acquireProviderPermit']>
-              > | null = null;
-              let dispatched = false;
-              const releasePhysicalPermit = async () => {
-                const current = permit;
-                permit = null;
-                if (!current) return;
-                if (dispatched) current.release();
-                else await current.cancel();
-                dispatched = false;
-              };
-              let transportRetry = false;
-              try {
-                const attemptInit = {
-                  ...fetchInit,
-                  signal: init.signal
-                    ? AbortSignal.any([init.signal, attemptController.signal])
-                    : attemptController.signal,
-                };
-                response = await runCtxFetchPhase({
-                  callerSignal: init.signal,
-                  controller: attemptController,
-                  deadline: fetchDeadline,
-                  phase: 'total',
-                  run: () =>
-                    safePublicFetch(url, attemptInit, {
-                      // safePublicFetch validates each URL before invoking this
-                      // wrapper. Redirect hops are physical dispatches too.
-                      fetchImpl: async (physicalUrl, physicalInit) => {
-                        await releasePhysicalPermit();
-                        const signal = attemptInit.signal;
-                        permit = await runCtxFetchPhase({
-                          callerSignal: init.signal,
-                          controller: attemptController,
-                          deadline: fetchDeadline,
-                          phase: 'total',
-                          run: async () => {
-                            for (;;) {
-                              const acquired =
-                                await this.resourceGovernor.acquireProviderPermit(
-                                  { toolId: CTX_FETCH_EGRESS_TOOL_ID, signal },
-                                );
-                              try {
-                                signal.throwIfAborted();
-                                const valid =
-                                  !acquired.validate ||
-                                  (await acquired.validate());
-                                signal.throwIfAborted();
-                                if (valid) return acquired;
-                              } catch (error) {
-                                await acquired.cancel();
-                                throw error;
-                              }
-                              await acquired.cancel();
-                            }
-                          },
-                        });
-                        return await runCtxFetchPhase({
-                          callerSignal: init.signal,
-                          controller: attemptController,
-                          deadline: fetchDeadline,
-                          phase: 'headers',
-                          run: () => {
-                            dispatched = true;
-                            return (
-                              this.#options.fetchImpl ?? globalThis.fetch
-                            )(physicalUrl, physicalInit);
-                          },
-                        });
-                      },
-                      sensitiveHeaders: Object.keys(secretHeaderMarkers),
-                      stripHeadersOnCrossOriginRedirect: true,
-                      headersOnCrossOriginRedirect: {
-                        'user-agent': 'Deepline Play Runtime',
-                      },
-                    }),
-                });
-                bodyText = await readCtxFetchBody({
-                  callerSignal: init.signal,
-                  controller: attemptController,
-                  deadline: fetchDeadline,
-                  response,
-                });
-              } catch (error) {
-                void response?.body?.cancel(error).catch(() => undefined);
-                response = null;
-                bodyText = null;
-                if (error instanceof CtxFetchTimeoutError) throw error;
-                if (init.signal?.aborted) {
-                  throw init.signal.reason ?? error;
-                }
-                if (isUnsafeOutboundUrlError(error)) {
-                  throw error;
-                }
-                const message =
-                  error instanceof Error ? error.message : String(error);
-                if (
-                  canRetryTransport &&
-                  transportAttempt < FETCH_TRANSPORT_MAX_ATTEMPTS
-                ) {
-                  this.runtimeLog(
-                    `ctx.fetch(${method} ${url}) transport failed on attempt ${transportAttempt}/${FETCH_TRANSPORT_MAX_ATTEMPTS}; retrying: ${message}`,
-                  );
-                  transportRetry = true;
-                } else {
-                  throw new Error(
-                    `ctx.fetch(${method} ${url}) failed on attempt ${transportAttempt}/${FETCH_TRANSPORT_MAX_ATTEMPTS}: ${message}`,
-                  );
-                }
-              } finally {
-                await releasePhysicalPermit();
-              }
-              if (transportRetry) {
-                await sleepWithinCtxFetchDeadline({
-                  callerSignal: init.signal,
-                  deadline: fetchDeadline,
-                  delayMs: FETCH_TRANSPORT_RETRY_DELAY_MS * transportAttempt,
-                });
-                transportAttempt += 1;
-                continue;
-              }
-              if (
-                !response ||
-                bodyText === null ||
-                !replayableBody ||
-                !isCtxFetchBackpressure(response, bodyText)
-              )
-                break;
-              rateLimitAttempt += 1;
-              const retryAfter = response.headers.get('retry-after');
-              const attemptCap = retryAfter?.trim()
-                ? TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS
-                : TOOL_EXECUTE_BARE_RATE_LIMIT_MAX_ATTEMPTS;
-              if (rateLimitAttempt >= attemptCap) break;
-              const delayMs = Math.max(
-                parseToolExecuteRetryAfterMs(retryAfter),
-                Math.min(
-                  TOOL_EXECUTE_RATE_LIMIT_RETRY_DELAY_MAX_MS,
-                  TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS * rateLimitAttempt,
-                ),
-              );
-              this.runtimeLog(
-                `[runtime.ctx_fetch_backpressure] ${JSON.stringify({
-                  key: normalizedKey,
-                  method,
-                  origin: new URL(response.url || url).origin,
-                  http_status: response.status,
-                  attempt: rateLimitAttempt,
-                  max_attempts: attemptCap,
-                  retry_delay_ms: delayMs,
-                })}`,
-              );
-              // Destination feedback must not cool the shared generic_http
-              // bucket used by unrelated websites. Pace this logical request.
-              await sleepWithinCtxFetchDeadline({
-                callerSignal: init.signal,
-                deadline: fetchDeadline,
-                delayMs,
-              });
-            }
-            if (!response) {
-              throw new Error(
-                `ctx.fetch(${method} ${url}) failed before receiving a response.`,
-              );
-            }
-            if (bodyText === null) {
-              throw new Error(
-                `ctx.fetch(${method} ${url}) failed while reading the response body.`,
-              );
-            }
+            const { response, bodyText } = await sendPlayHttpRequest({
+              key: normalizedKey,
+              url,
+              init: fetchInit,
+              governor: this.resourceGovernor,
+              sensitiveHeaders: Object.keys(secretHeaderMarkers),
+              timeouts: this.#options.ctxFetchTimeouts,
+              fetchImpl: this.#options.fetchImpl,
+              log: (message) => this.runtimeLog(message),
+            });
             const rawJson = parseJsonOrNull(bodyText);
             const redactedBodyText = this.secretRedactor.redactString(bodyText);
             const output: PlayFetchResponse = {
@@ -16685,8 +16301,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
               },
             })),
             buildPhysicalCall: async ({ members }) => {
-              const owned = members.map((member) =>
-                batchRequestByCallId.get(member.memberId)!,
+              const owned = members.map(
+                (member) => batchRequestByCallId.get(member.memberId)!,
               );
               const compiled = compileOwnedPhysicalBatch(owned);
               const providerActionVersion =
@@ -19012,7 +18628,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
     // running for.
     const activeRowScope = rowContext.getStore();
     const memoryLedgerRowScopes: readonly (
-      MemoryLedgerRowScope | null | undefined
+      | MemoryLedgerRowScope
+      | null
+      | undefined
     )[] = options?.memoryLedgerRowScopes ?? [
       { rowKey: activeRowScope?.rowKey, fieldName: activeRowScope?.fieldName },
     ];

package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-machines.ts

@@ -366,6 +366,7 @@ export function buildBootMachineConfig(input: {
   componentInputHash?: string;
   region?: string;
   rollbackHoldUntilMs?: number;
+  draining?: boolean;
 }): Record<string, unknown> {
   return {
     region: input.region ?? DEFAULT_ABSURD_FLY_REGION,
@@ -389,6 +390,7 @@ export function buildBootMachineConfig(input: {
       },
       env: {
         [RUNTIME_DEPLOY_VERSION_ENV]: input.releaseId,
+        ...(input.draining ? { [DRAIN_EXIT_ENV]: '1' } : {}),
       },
       // Override the image CMD so rollback and recovery of an older image still
       // honor today's fleet-wide slot ceiling. Every historical Absurd worker