deepline 0.3.231 → 0.3.233

package/dist/bundling-sources/shared_libs/play-runtime/play-http-request.ts

@@ -0,0 +1,419 @@
+import type { SafeFetchImplementation } from '../security/safe-fetch';
+import type { RuntimeResourceGovernor } from './resource-governor';
+import { CTX_FETCH_EGRESS_TOOL_ID } from './builtin-pacing';
+import {
+  positiveTimeoutMs,
+  RUNTIME_RELIABILITY_POLICY,
+} from './runtime-reliability-policy';
+import {
+  TOOL_EXECUTE_BARE_RATE_LIMIT_MAX_ATTEMPTS,
+  TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS,
+  TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS,
+  TOOL_EXECUTE_RATE_LIMIT_RETRY_DELAY_MAX_MS,
+  parseToolExecuteRetryAfterMs,
+} from './tool-execute-retry-policy';
+
+const FETCH_TRANSPORT_MAX_ATTEMPTS =
+  RUNTIME_RELIABILITY_POLICY.egress.fetchMaxAttempts;
+const FETCH_TRANSPORT_RETRY_DELAY_MS = 100;
+const CTX_FETCH_HEADERS_TIMEOUT_MS =
+  RUNTIME_RELIABILITY_POLICY.egress.fetchHeadersTimeoutMs;
+const CTX_FETCH_BODY_TIMEOUT_MS =
+  RUNTIME_RELIABILITY_POLICY.egress.fetchBodyTimeoutMs;
+const CTX_FETCH_TOTAL_TIMEOUT_MS =
+  RUNTIME_RELIABILITY_POLICY.egress.fetchTotalTimeoutMs;
+
+type SafeFetchModule = typeof import('../security/safe-fetch');
+let safeFetchModule: Promise<SafeFetchModule> | null = null;
+function loadSafeFetch(): Promise<SafeFetchModule> {
+  safeFetchModule ??= import('../security/safe-fetch');
+  return safeFetchModule;
+}
+
+function isUnsafeOutboundUrlError(error: unknown): boolean {
+  return error instanceof Error && error.name === 'UnsafeOutboundUrlError';
+}
+
+export class CtxFetchTimeoutError extends Error {
+  readonly code = 'CTX_FETCH_TIMEOUT';
+
+  constructor(
+    readonly phase: 'headers' | 'body' | 'total',
+    readonly timeoutMs: number,
+  ) {
+    super(
+      `[CTX_FETCH_TIMEOUT phase=${phase} timeout_ms=${timeoutMs}] ` +
+        `ctx.fetch exceeded its ${phase} deadline after ${timeoutMs}ms.`,
+    );
+    this.name = 'CtxFetchTimeoutError';
+  }
+}
+
+type CtxFetchDeadline = {
+  bodyMs: number;
+  headersMs: number;
+  startedAt: number;
+  totalMs: number;
+};
+
+function remainingCtxFetchMs(deadline: CtxFetchDeadline): number {
+  return Math.max(0, deadline.totalMs - (Date.now() - deadline.startedAt));
+}
+
+async function runCtxFetchPhase<T>(input: {
+  callerSignal?: AbortSignal | null;
+  controller: AbortController;
+  deadline: CtxFetchDeadline;
+  phase: 'headers' | 'body' | 'total';
+  run: () => Promise<T>;
+}): Promise<T> {
+  const remainingMs = remainingCtxFetchMs(input.deadline);
+  if (remainingMs <= 0) {
+    throw new CtxFetchTimeoutError('total', input.deadline.totalMs);
+  }
+  if (input.callerSignal?.aborted) {
+    throw (
+      input.callerSignal.reason ?? new DOMException('Aborted', 'AbortError')
+    );
+  }
+  const phaseLimitMs =
+    input.phase === 'headers'
+      ? input.deadline.headersMs
+      : input.phase === 'body'
+        ? input.deadline.bodyMs
+        : remainingMs;
+  const timeoutMs = Math.min(phaseLimitMs, remainingMs);
+  const timeoutPhase = timeoutMs >= remainingMs ? 'total' : input.phase;
+  let timeout: ReturnType<typeof setTimeout> | null = null;
+  let onCallerAbort: (() => void) | null = null;
+  const phaseTimeout = new Promise<never>((_resolve, reject) => {
+    timeout = setTimeout(() => {
+      const error = new CtxFetchTimeoutError(
+        timeoutPhase,
+        timeoutPhase === 'total' ? input.deadline.totalMs : phaseLimitMs,
+      );
+      reject(error);
+      input.controller.abort(error);
+    }, timeoutMs);
+  });
+  const callerAbort = input.callerSignal
+    ? new Promise<never>((_resolve, reject) => {
+        onCallerAbort = () => {
+          const reason =
+            input.callerSignal?.reason ??
+            new DOMException('Aborted', 'AbortError');
+          reject(reason);
+          input.controller.abort(reason);
+        };
+        input.callerSignal!.addEventListener('abort', onCallerAbort, {
+          once: true,
+        });
+      })
+    : null;
+  try {
+    return await Promise.race([
+      input.run(),
+      phaseTimeout,
+      ...(callerAbort ? [callerAbort] : []),
+    ]);
+  } finally {
+    if (timeout) clearTimeout(timeout);
+    if (onCallerAbort) {
+      input.callerSignal?.removeEventListener('abort', onCallerAbort);
+    }
+  }
+}
+
+async function sleepWithinCtxFetchDeadline(input: {
+  callerSignal?: AbortSignal | null;
+  deadline: CtxFetchDeadline;
+  delayMs: number;
+}): Promise<void> {
+  const controller = new AbortController();
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  try {
+    await runCtxFetchPhase({
+      callerSignal: input.callerSignal,
+      controller,
+      deadline: input.deadline,
+      phase: 'total',
+      run: () =>
+        new Promise<void>((resolve) => {
+          timer = setTimeout(resolve, input.delayMs);
+        }),
+    });
+  } finally {
+    if (timer !== undefined) clearTimeout(timer);
+  }
+}
+
+function isCtxFetchBackpressure(response: Response, bodyText: string): boolean {
+  if (response.status === 429) return true;
+  if (response.status !== 503) return false;
+  // This Salesforce endpoint reports a rejected burst as 503. Match the final
+  // destination and exact response contract; arbitrary 503s remain failures.
+  try {
+    const destination = new URL(response.url);
+    const body: unknown = JSON.parse(bodyText);
+    return (
+      destination.protocol === 'https:' &&
+      destination.hostname.endsWith('.salesforce-sites.com') &&
+      body !== null &&
+      typeof body === 'object' &&
+      !Array.isArray(body) &&
+      (body as Record<string, unknown>).status === 'burst_circuit_open'
+    );
+  } catch {
+    return false;
+  }
+}
+
+async function readCtxFetchBody(input: {
+  callerSignal?: AbortSignal | null;
+  controller: AbortController;
+  deadline: CtxFetchDeadline;
+  response: Response;
+}): Promise<string> {
+  if (!input.response.body) return '';
+  const reader = input.response.body.getReader();
+  const decoder = new TextDecoder();
+  try {
+    return await runCtxFetchPhase({
+      callerSignal: input.callerSignal,
+      controller: input.controller,
+      deadline: input.deadline,
+      phase: 'body',
+      run: async () => {
+        let body = '';
+        for (;;) {
+          const chunk = await reader.read();
+          if (chunk.done) return body + decoder.decode();
+          body += decoder.decode(chunk.value, { stream: true });
+        }
+      },
+    });
+  } catch (error) {
+    // Cancellation is socket hygiene, not part of the timeout contract. A
+    // hostile/custom stream may never resolve its cancel hook; awaiting it
+    // here would recreate the exact forever-pending ctx.fetch this deadline
+    // is meant to eliminate.
+    void reader.cancel(error).catch(() => {});
+    throw error;
+  } finally {
+    reader.releaseLock();
+  }
+}
+
+/** Sends one logical Play HTTP request within its permit and I/O budgets.
+ * Durable receipt policy and secret redaction belong to the caller.
+ */
+export async function sendPlayHttpRequest(input: {
+  key: string;
+  url: string;
+  init: RequestInit;
+  governor: Pick<RuntimeResourceGovernor, 'acquireProviderPermit'>;
+  sensitiveHeaders: string[];
+  timeouts?: { headersMs?: number; bodyMs?: number; totalMs?: number };
+  fetchImpl?: SafeFetchImplementation;
+  log: (message: string) => void;
+}): Promise<{ response: Response; bodyText: string }> {
+  const { url, init } = input;
+  const method = String(init.method ?? 'GET').toUpperCase();
+  const fetchDeadline: CtxFetchDeadline = {
+    startedAt: Date.now(),
+    headersMs: positiveTimeoutMs(
+      input.timeouts?.headersMs,
+      CTX_FETCH_HEADERS_TIMEOUT_MS,
+    ),
+    bodyMs: positiveTimeoutMs(
+      input.timeouts?.bodyMs,
+      CTX_FETCH_BODY_TIMEOUT_MS,
+    ),
+    totalMs: positiveTimeoutMs(
+      input.timeouts?.totalMs,
+      CTX_FETCH_TOTAL_TIMEOUT_MS,
+    ),
+  };
+  let response: Response | null = null;
+  let bodyText: string | null = null;
+  // Only replay a body whose bytes are stable. In particular, a
+  // consumed ReadableStream cannot be retried even with a write key.
+  const replayableBody = init.body == null || typeof init.body === 'string';
+  const canRetryTransport =
+    replayableBody && ['GET', 'HEAD', 'OPTIONS'].includes(method);
+  const { safePublicFetch } = await loadSafeFetch();
+  let transportAttempt = 1;
+  let rateLimitAttempt = 0;
+  while (true) {
+    const attemptController = new AbortController();
+    let permit: Awaited<
+      ReturnType<RuntimeResourceGovernor['acquireProviderPermit']>
+    > | null = null;
+    let dispatched = false;
+    const releasePhysicalPermit = async () => {
+      const current = permit;
+      permit = null;
+      if (!current) return;
+      if (dispatched) current.release();
+      else await current.cancel();
+      dispatched = false;
+    };
+    let transportRetry = false;
+    try {
+      const attemptInit = {
+        ...init,
+        signal: init.signal
+          ? AbortSignal.any([init.signal, attemptController.signal])
+          : attemptController.signal,
+      };
+      response = await runCtxFetchPhase({
+        callerSignal: init.signal,
+        controller: attemptController,
+        deadline: fetchDeadline,
+        phase: 'total',
+        run: () =>
+          safePublicFetch(url, attemptInit, {
+            // safePublicFetch validates each URL before invoking this
+            // wrapper. Redirect hops are physical dispatches too.
+            fetchImpl: async (physicalUrl, physicalInit) => {
+              await releasePhysicalPermit();
+              const signal = attemptInit.signal;
+              permit = await runCtxFetchPhase({
+                callerSignal: init.signal,
+                controller: attemptController,
+                deadline: fetchDeadline,
+                phase: 'total',
+                run: async () => {
+                  for (;;) {
+                    const acquired = await input.governor.acquireProviderPermit(
+                      { toolId: CTX_FETCH_EGRESS_TOOL_ID, signal },
+                    );
+                    try {
+                      signal.throwIfAborted();
+                      const valid =
+                        !acquired.validate || (await acquired.validate());
+                      signal.throwIfAborted();
+                      if (valid) return acquired;
+                    } catch (error) {
+                      await acquired.cancel();
+                      throw error;
+                    }
+                    await acquired.cancel();
+                  }
+                },
+              });
+              return await runCtxFetchPhase({
+                callerSignal: init.signal,
+                controller: attemptController,
+                deadline: fetchDeadline,
+                phase: 'headers',
+                run: () => {
+                  dispatched = true;
+                  return (input.fetchImpl ?? globalThis.fetch)(
+                    physicalUrl,
+                    physicalInit,
+                  );
+                },
+              });
+            },
+            sensitiveHeaders: input.sensitiveHeaders,
+            stripHeadersOnCrossOriginRedirect: true,
+            headersOnCrossOriginRedirect: {
+              'user-agent': 'Deepline Play Runtime',
+            },
+          }),
+      });
+      bodyText = await readCtxFetchBody({
+        callerSignal: init.signal,
+        controller: attemptController,
+        deadline: fetchDeadline,
+        response,
+      });
+    } catch (error) {
+      void response?.body?.cancel(error).catch(() => undefined);
+      response = null;
+      bodyText = null;
+      if (error instanceof CtxFetchTimeoutError) throw error;
+      if (init.signal?.aborted) {
+        throw init.signal.reason ?? error;
+      }
+      if (isUnsafeOutboundUrlError(error)) {
+        throw error;
+      }
+      const message = error instanceof Error ? error.message : String(error);
+      if (
+        canRetryTransport &&
+        transportAttempt < FETCH_TRANSPORT_MAX_ATTEMPTS
+      ) {
+        input.log(
+          `ctx.fetch(${method} ${url}) transport failed on attempt ${transportAttempt}/${FETCH_TRANSPORT_MAX_ATTEMPTS}; retrying: ${message}`,
+        );
+        transportRetry = true;
+      } else {
+        throw new Error(
+          `ctx.fetch(${method} ${url}) failed on attempt ${transportAttempt}/${FETCH_TRANSPORT_MAX_ATTEMPTS}: ${message}`,
+        );
+      }
+    } finally {
+      await releasePhysicalPermit();
+    }
+    if (transportRetry) {
+      await sleepWithinCtxFetchDeadline({
+        callerSignal: init.signal,
+        deadline: fetchDeadline,
+        delayMs: FETCH_TRANSPORT_RETRY_DELAY_MS * transportAttempt,
+      });
+      transportAttempt += 1;
+      continue;
+    }
+    if (
+      !response ||
+      bodyText === null ||
+      !replayableBody ||
+      !isCtxFetchBackpressure(response, bodyText)
+    )
+      break;
+    rateLimitAttempt += 1;
+    const retryAfter = response.headers.get('retry-after');
+    const attemptCap = retryAfter?.trim()
+      ? TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS
+      : TOOL_EXECUTE_BARE_RATE_LIMIT_MAX_ATTEMPTS;
+    if (rateLimitAttempt >= attemptCap) break;
+    const delayMs = Math.max(
+      parseToolExecuteRetryAfterMs(retryAfter),
+      Math.min(
+        TOOL_EXECUTE_RATE_LIMIT_RETRY_DELAY_MAX_MS,
+        TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS * rateLimitAttempt,
+      ),
+    );
+    input.log(
+      `[runtime.ctx_fetch_backpressure] ${JSON.stringify({
+        key: input.key,
+        method,
+        origin: new URL(response.url || url).origin,
+        http_status: response.status,
+        attempt: rateLimitAttempt,
+        max_attempts: attemptCap,
+        retry_delay_ms: delayMs,
+      })}`,
+    );
+    // Destination feedback must not cool the shared generic_http
+    // bucket used by unrelated websites. Pace this logical request.
+    await sleepWithinCtxFetchDeadline({
+      callerSignal: init.signal,
+      deadline: fetchDeadline,
+      delayMs,
+    });
+  }
+  if (!response) {
+    throw new Error(
+      `ctx.fetch(${method} ${url}) failed before receiving a response.`,
+    );
+  }
+  if (bodyText === null) {
+    throw new Error(
+      `ctx.fetch(${method} ${url}) failed while reading the response body.`,
+    );
+  }
+  return { response, bodyText };
+}

package/dist/bundling-sources/shared_libs/play-runtime/runtime-reliability-policy.ts

@@ -389,3 +389,13 @@ export function resolveRuntimeReliabilityPolicy(
 }
 
 export const RUNTIME_RELIABILITY_POLICY = resolveRuntimeReliabilityPolicy();
+
+/** Resolve optional runtime intervals and I/O deadlines with the same fallback policy. */
+export function positiveTimeoutMs(
+  value: number | undefined,
+  fallback: number,
+): number {
+  return value === undefined || !Number.isFinite(value) || value <= 0
+    ? fallback
+    : Math.max(1, Math.floor(value));
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts

@@ -7091,6 +7091,7 @@ export type AbsurdWorkerWakeHook = ((
   wake: () => void,
 ) => (() => void) | void) & {
   isListening: () => boolean;
+  close: () => void;
 };
 
 export function resolveAbsurdIdlePollFallbackMs(
@@ -7117,7 +7118,10 @@ export function createAbsurdWorkerWakeHook(input: {
 }): AbsurdWorkerWakeHook | undefined {
   if (input.draining || input.useDirectNotify === false) return undefined;
   let listening = false;
+  let closed = false;
+  const subscriptions = new Set<() => void>();
   const hook: AbsurdWorkerWakeHook = (wake) => {
+    if (closed) return;
     const acquireNotifyHub =
       input.acquireNotifyHub ??
       ((notifyInput: {
@@ -7143,13 +7147,24 @@ export function createAbsurdWorkerWakeHook(input: {
       listening = state === 'listening';
       if (listening || wasListening) wake();
     });
-    return () => {
+    let released = false;
+    const release = () => {
+      if (released) return;
+      released = true;
       unsubscribeNotify();
       unsubscribeState();
       lease.release();
+      subscriptions.delete(release);
     };
+    subscriptions.add(release);
+    return release;
   };
   hook.isListening = () => listening;
+  hook.close = () => {
+    closed = true;
+    listening = false;
+    for (const release of subscriptions) release();
+  };
   return hook;
 }
 
@@ -7530,6 +7545,10 @@ export async function startRuntimeWorkerHost(
 
   return {
     close: async () => {
+      // Accepted handlers and maintenance can take minutes to settle. Stop
+      // owning a direct LISTEN session before waiting for that drain, so a
+      // replacement can elect its listener without exhausting primary slots.
+      onWake?.close();
       drainStopped = true;
       billingStopped = true;
       cleanupStopped = true;