deepline 0.3.461 → 0.3.463

package/dist/bundling-sources/shared_libs/play-runtime/runtime-constants.ts

@@ -17,6 +17,22 @@ export const PLAY_RUNNER_TERMINAL_GRACE_SECONDS = 90;
  */
 export const PLAY_RUNNER_STARTUP_GRACE_SECONDS = 4 * 60;
 
+/**
+ * Longest sandbox network cut a running Play must survive. Daytona cut running
+ * sandboxes off for 564 to 621 seconds between 2026-10-27 and 2026-10-30 while
+ * their processes kept running.
+ */
+export const SANDBOX_NETWORK_CUT_MAX_MS = 11 * 60_000;
+
+/**
+ * How long the runner keeps retrying a gateway request that gets no HTTP
+ * response at all: the longest cut, plus a 30-second request timeout and its
+ * backoff on each side of it. A request that gets an error response keeps its
+ * own, shorter budget.
+ */
+export const RUNNER_GATEWAY_CONNECTION_RETRY_BUDGET_MS =
+  SANDBOX_NETWORK_CUT_MAX_MS + 60_000;
+
 /**
  * Runner timeout includes setup, cleanup, and billing headroom after the
  * user-code runtime cap.

package/dist/bundling-sources/shared_libs/play-runtime/runtime-environment.ts

@@ -86,6 +86,44 @@ export function normalizePlayRuntimeSelection(
   };
 }
 
+/**
+ * The app's own runtime, on a named sandbox placement policy. Internal-only:
+ * the `play_run.sandbox_placement` capability in
+ * `@deepline/auth/internal-capabilities` decides who may send it.
+ */
+export type PlayNativeRuntimeSelection = {
+  environment: 'native';
+  sandboxPlacementPolicy: string;
+};
+
+/** Every `runtime` value a Play run accepts. */
+export type PlayRunRuntimeSelection =
+  | PlayRuntimeSelection
+  | PlayNativeRuntimeSelection;
+
+export function normalizePlayRunRuntimeSelection(
+  value: unknown,
+): PlayRunRuntimeSelection | null {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
+  const record = value as Record<string, unknown>;
+  if (record.environment !== 'native') {
+    return normalizePlayRuntimeSelection(value);
+  }
+  if (
+    Object.keys(record).some(
+      (key) => key !== 'environment' && key !== 'sandboxPlacementPolicy',
+    )
+  ) {
+    return null;
+  }
+  const sandboxPlacementPolicy = normalizePlayRuntimeSandboxPlacementPolicy(
+    record.sandboxPlacementPolicy,
+  );
+  return sandboxPlacementPolicy
+    ? { environment: 'native', sandboxPlacementPolicy }
+    : null;
+}
+
 export function normalizePlayRuntimeEnvironment(
   value: unknown,
 ): PlayRuntimeEnvironment | null {

package/dist/bundling-sources/shared_libs/play-runtime/runtime-flow.ts

@@ -252,6 +252,12 @@ export function createRuntimeDispatchFlow<
             : null;
           if (!providerPermit) {
             releaseExecutionSlot();
+            // A backend without a local due permit must re-enter provider
+            // admission. First let FIFO waiters return their reserved slots:
+            // immediately resolved provider admissions can otherwise circulate
+            // that reservation forever without allowing timers or aborts to run.
+            // Neither a provider permit nor an execution slot is held here.
+            await new Promise<void>((resolve) => setTimeout(resolve, 0));
             continue;
           }
         }

package/dist/bundling-sources/shared_libs/play-runtime/runtime-incident-drills.ts

@@ -19,6 +19,7 @@ export const RUNTIME_TEST_FAULT_IDS = [
   'receipt_claim_response_timeout',
   'modal_sandbox_create_resource_exhausted',
   'runtime_sheet_page_tail_hold_ms',
+  'runner_heartbeat_silence_ms',
 ] as const;
 
 export type RuntimeTestFaultName = (typeof RUNTIME_TEST_FAULT_IDS)[number];
@@ -40,7 +41,8 @@ export type RuntimeIncidentSafetyFact =
   | 'bounded_provider_execution'
   | 'retry_or_explicit_recovery'
   | 'no_user_code_before_recovery'
-  | 'all_rows_settle_once';
+  | 'all_rows_settle_once'
+  | 'no_loss_verdict_without_provider_proof';
 
 /**
  * The automated evidence that proves this drill.  A focused regression owns a
@@ -353,6 +355,29 @@ export const RUNTIME_INCIDENT_DRILL_CATALOG: readonly RuntimeIncidentDrill[] = [
       target: 'page-tail-sheet-write-hold-e2e',
     },
   },
+  {
+    id: 'runner_heartbeat_silence_ms',
+    description:
+      'Stop a live detached runner heartbeating for a bounded time while its process keeps running.',
+    topologies: LOCAL_AND_PREVIEW_TOPOLOGIES,
+    control: {
+      kind: 'runtime_test_fault',
+      // The 120-second lease plus 80 seconds of confirmation puts the first
+      // provider read at about 200 seconds. The rest absorbs late wakes, so
+      // that read lands inside the silence and not after it.
+      defaultValue: 300_000,
+      valueMeaning: 'milliseconds',
+    },
+    expected: {
+      outcome:
+        'The worker asks the provider once the lease and confirmation pass, the sandbox is not proven gone, and the same attempt finishes after heartbeats return.',
+      safetyFacts: ['same_logical_run', 'no_loss_verdict_without_provider_proof'],
+    },
+    verification: {
+      kind: 'preview_black_box',
+      target: 'silent-runner-liveness-e2e',
+    },
+  },
 ];
 
 export type RuntimeTestFaultDrill = Omit<

package/dist/bundling-sources/shared_libs/play-runtime/runtime-operation-contract.ts

@@ -24,6 +24,31 @@ export const RUNTIME_SHEET_ADMISSION_OPERATION_CONTRACT: DeployedOperationContra
     },
   });
 
+/** Named, run-scoped array source snapshots. Older receivers retain legacy array admission. */
+export const RUNTIME_ARRAY_INPUT_CAPABILITIES = {
+  runtime_array_input_get_manifest: 'receipt.read',
+  runtime_array_input_begin: 'receipt.write',
+  runtime_array_input_put_page: 'receipt.write',
+  runtime_array_input_seal: 'receipt.write',
+  runtime_array_input_read_page: 'receipt.read',
+} as const satisfies Readonly<Record<string, RuntimeExecutionCapability>>;
+
+export const RUNTIME_ARRAY_INPUT_OPERATION_CONTRACT: DeployedOperationContract =
+  deployed.operation({
+    id: 'runtime-array-input-actions',
+    actions: RUNTIME_ARRAY_INPUT_CAPABILITIES,
+    olderReceiver: {
+      runtime_array_input_get_manifest: {
+        kind: 'feature-unavailable',
+        code: 'RUNTIME_ARRAY_INPUT_UNAVAILABLE',
+      },
+      runtime_array_input_begin: {
+        kind: 'feature-unavailable',
+        code: 'RUNTIME_ARRAY_INPUT_UNAVAILABLE',
+      },
+    },
+  });
+
 /**
  * One declarative owner for the runtime action → authority relationship. The
  * route still owns authentication and execution; callers can add an action in
@@ -73,6 +98,7 @@ export const RUNTIME_OPERATION_CAPABILITIES = {
   resolve_secret: 'secret.resolve',
   // Provider prices and availability use the existing discovery authority.
   get_billing_entitlements: 'tool.inspect',
+  write_page: 'page.write',
 } as const satisfies Readonly<Record<string, RuntimeExecutionCapability>>;
 
 export const RUNTIME_OPERATION_CONTRACT: DeployedOperationContract =
@@ -84,6 +110,10 @@ export const RUNTIME_OPERATION_CONTRACT: DeployedOperationContract =
         kind: 'feature-unavailable',
         code: 'BILLING_ENTITLEMENTS_UNAVAILABLE',
       },
+      write_page: {
+        kind: 'feature-unavailable',
+        code: 'PAGES_UNAVAILABLE',
+      },
       project_run_events: {
         kind: 'equivalent-fallback',
         action: 'append_run_events',

package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-store-adapter.ts

@@ -35,6 +35,7 @@ import {
   RuntimeReceiptWriterResultCountError,
   writeRuntimeReceiptWithBackpressure,
 } from './runtime-receipt-writer';
+import { RUNNER_GATEWAY_CONNECTION_RETRY_BUDGET_MS } from './runtime-constants';
 
 type ReceiptOutput =
   | RuntimeStepReceipt
@@ -122,8 +123,12 @@ export type RuntimeReceiptStoreHandlers = Required<
 // Four ambiguous 30-second receipt requests previously consumed the writer's
 // entire two-minute budget. Receipt operations are idempotent and buffered, so
 // keep retry ownership here and allow independent transports enough attempts
-// to recover from a short gateway/network impairment.
+// to recover from a short gateway impairment. This budget covers retryable
+// error responses; requests that get no response at all retry through a whole
+// sandbox network cut instead.
 export const RUNTIME_RECEIPT_STORE_RETRY_BUDGET_MS = 5 * 60_000;
+export const RUNTIME_RECEIPT_STORE_CONNECTION_RETRY_BUDGET_MS =
+  RUNNER_GATEWAY_CONNECTION_RETRY_BUDGET_MS;
 
 /**
  * The runner's single Work Receipt Adapter.
@@ -221,6 +226,8 @@ export class RuntimeReceiptStoreAdapter {
       maxBufferedItems: options.maxBufferedItems,
       maxBufferedBytes: options.maxBufferedBytes,
       maxRetryElapsedMs: RUNTIME_RECEIPT_STORE_RETRY_BUDGET_MS,
+      maxConnectionRetryElapsedMs:
+        RUNTIME_RECEIPT_STORE_CONNECTION_RETRY_BUDGET_MS,
       classifyRetryableError: classifyReceiptRetry,
       onRetryEvent: (event) =>
         emitReceiptWriterRetryTelemetry({
@@ -534,12 +541,16 @@ function receiptCommandOrderingKeys(
   }
 }
 
-function classifyReceiptRetry(error: unknown): { retryAfterMs: number } | null {
+function classifyReceiptRetry(
+  error: unknown,
+): { retryAfterMs: number; connectionFailure?: boolean } | null {
   if (isAppRuntimeApiCapacityError(error)) {
     return { retryAfterMs: error.retryAfterMs };
   }
   if (error instanceof AppRuntimeApiTransportError) {
-    return { retryAfterMs: 250 };
+    // The one-attempt transport throws this only when no complete HTTP
+    // response arrived: DNS, connect, reset, or its request timeout.
+    return { retryAfterMs: 250, connectionFailure: true };
   }
   if (error instanceof AppRuntimeApiResponseError && error.retryable) {
     return { retryAfterMs: 250 };

package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-writer.ts

@@ -1,5 +1,10 @@
 type RetryableReceiptError = {
   retryAfterMs: number;
+  /**
+   * The request got no HTTP response at all. Time spent this way counts only
+   * against `maxConnectionRetryElapsedMs`, never against `maxRetryElapsedMs`.
+   */
+  connectionFailure?: boolean;
 };
 
 const MAX_RETRY_JITTER_RATIO = 0.25;
@@ -155,6 +160,7 @@ export class RuntimeReceiptWriter<Input, Output> {
     Required<RuntimeReceiptWriterLaneBufferLimit>
   >;
   readonly #maxRetryElapsedMs: number;
+  readonly #maxConnectionRetryElapsedMs: number;
   readonly #onRetryEvent:
     | ((event: RuntimeReceiptWriterRetryEvent<Input>) => void)
     | null;
@@ -219,7 +225,13 @@ export class RuntimeReceiptWriter<Input, Output> {
     maxBufferedByLane?: Readonly<
       Record<string, RuntimeReceiptWriterLaneBufferLimit>
     >;
+    /** Retry budget for failures that returned an HTTP response. */
     maxRetryElapsedMs?: number;
+    /**
+     * Retry budget for a batch whose latest failure got no HTTP response,
+     * measured from its first attempt. Defaults to `maxRetryElapsedMs`.
+     */
+    maxConnectionRetryElapsedMs?: number;
     onRetryEvent?: (event: RuntimeReceiptWriterRetryEvent<Input>) => void;
     /**
      * Called with the writer's buffered byte total every time it changes. The
@@ -299,6 +311,10 @@ export class RuntimeReceiptWriter<Input, Output> {
       options.maxRetryElapsedMs,
       2 * 60_000,
     );
+    this.#maxConnectionRetryElapsedMs = normalizePositiveInteger(
+      options.maxConnectionRetryElapsedMs,
+      this.#maxRetryElapsedMs,
+    );
     this.#onRetryEvent = options.onRetryEvent ?? null;
     this.#onBufferedBytesChanged = options.onBufferedBytesChanged ?? null;
     this.#onStatsChanged = options.onStatsChanged ?? null;
@@ -793,9 +809,14 @@ export class RuntimeReceiptWriter<Input, Output> {
   ): Promise<readonly Output[]> {
     const inputs = batch.map((pending) => pending.input);
     const startedAt = Date.now();
+    // Attempts that got no response, with the waits after them. They do not
+    // spend the error-response budget, so a 503 just after a long network cut
+    // still gets the full ordinary budget.
+    let connectionFailureMs = 0;
     let attempt = 0;
     while (true) {
       attempt += 1;
+      const attemptStartedAt = Date.now();
       if (this.#closeController.signal.aborted) {
         throw this.#closeReason;
       }
@@ -821,20 +842,23 @@ export class RuntimeReceiptWriter<Input, Output> {
         const retry = this.#classifyRetryableError(error);
         if (!retry) throw error;
         const elapsedMs = Date.now() - startedAt;
-        if (elapsedMs >= this.#maxRetryElapsedMs) {
+        const [budgetElapsedMs, budgetMs] = retry.connectionFailure
+          ? [elapsedMs, this.#maxConnectionRetryElapsedMs]
+          : [elapsedMs - connectionFailureMs, this.#maxRetryElapsedMs];
+        if (budgetElapsedMs >= budgetMs) {
           throw new RuntimeReceiptWriterRetryDeadlineError(
             attempt,
             elapsedMs,
-            this.#maxRetryElapsedMs,
+            budgetMs,
             { cause: error },
           );
         }
         const retryAfterMs = jitteredRetryDelayMs(retry.retryAfterMs);
-        if (elapsedMs + retryAfterMs > this.#maxRetryElapsedMs) {
+        if (budgetElapsedMs + retryAfterMs > budgetMs) {
           throw new RuntimeReceiptWriterRetryDeadlineError(
             attempt,
             elapsedMs,
-            this.#maxRetryElapsedMs,
+            budgetMs,
             { cause: error },
           );
         }
@@ -848,6 +872,9 @@ export class RuntimeReceiptWriter<Input, Output> {
           error,
         });
         await this.#waitForRetry(retryAfterMs, batch);
+        if (retry.connectionFailure) {
+          connectionFailureMs += Date.now() - attemptStartedAt;
+        }
       }
     }
   }

package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-row-transition.ts

@@ -24,6 +24,53 @@ export type RuntimePreparedFailedRow = RuntimePreparedCompletedRow & {
   error: string;
 };
 
+// Logical parameter framing for index/null tags and array-element overhead.
+// This is a batching estimate, not an exact wire or executor-memory bound.
+const RUNTIME_PREPARED_ROW_FRAMING_BYTES = 64;
+const preparedRowEncoder = new TextEncoder();
+
+/** Chunk already-serialized SQL inputs without touching authored object graphs. */
+export function chunkPreparedRuntimeSheetRows<
+  T extends RuntimePreparedCompletedRow & { error?: string },
+>(rows: readonly T[], input: { maxRows: number; maxBytes: number }): T[][] {
+  if (
+    !Number.isSafeInteger(input.maxRows) ||
+    input.maxRows <= 0 ||
+    !Number.isSafeInteger(input.maxBytes) ||
+    input.maxBytes <= 0
+  ) {
+    throw new Error(
+      'Prepared Runtime Sheet chunk limits must be positive safe integers.',
+    );
+  }
+  const chunks: T[][] = [];
+  let chunk: T[] = [];
+  let chunkBytes = 0;
+  for (const row of rows) {
+    const bytes =
+      RUNTIME_PREPARED_ROW_FRAMING_BYTES +
+      preparedRowEncoder.encode(row.key).byteLength +
+      preparedRowEncoder.encode(row.data_patch_json).byteLength +
+      preparedRowEncoder.encode(row.cell_meta_patch_json).byteLength +
+      (row.error === undefined
+        ? 0
+        : preparedRowEncoder.encode(row.error).byteLength);
+    if (
+      chunk.length > 0 &&
+      (chunk.length >= input.maxRows || bytes > input.maxBytes - chunkBytes)
+    ) {
+      chunks.push(chunk);
+      chunk = [];
+      chunkBytes = 0;
+    }
+    // A valid row larger than the target budget remains one intact singleton.
+    chunk.push(row);
+    chunkBytes += bytes;
+  }
+  if (chunk.length > 0) chunks.push(chunk);
+  return chunks;
+}
+
 export function normalizeRuntimeMapInputIndex(value: unknown): number | null {
   if (typeof value !== 'number' || !Number.isFinite(value)) {
     return null;

package/dist/bundling-sources/shared_libs/play-runtime/sandbox-compute-usage.ts

@@ -90,19 +90,32 @@ export function resolveRuntimeSandboxComputeItems(input: {
  * A provider create may finish after the original run has terminalized. Those
  * resources cannot be appended to the run's already-finalized billing
  * session, so the scheduler settles each through a stable adjustment session.
+ *
+ * A recovered create is recorded from its reservation, without the ceiling a
+ * live create carries, and cleanup may find it weeks later. It still bills no
+ * more than one run could: `maxBillingDurationSeconds` fills any missing one.
  */
 export function resolveLateRuntimeSandboxComputeItems(input: {
   resources: readonly unknown[];
   endedAt: number;
+  maxBillingDurationSeconds: number;
 }): ComputeBillingItem[] {
+  const lateResources: Record<string, unknown>[] = [];
+  for (const resource of input.resources) {
+    if (!resource || typeof resource !== 'object' || Array.isArray(resource)) {
+      continue;
+    }
+    const value = resource as Record<string, unknown>;
+    if (value.lateAcquired !== true) continue;
+    lateResources.push({
+      ...value,
+      maxBillingDurationSeconds:
+        finiteNonNegative(value.maxBillingDurationSeconds) ??
+        input.maxBillingDurationSeconds,
+    });
+  }
   return resolveRuntimeSandboxComputeItems({
-    resources: input.resources.filter(
-      (resource) =>
-        Boolean(resource) &&
-        typeof resource === 'object' &&
-        !Array.isArray(resource) &&
-        (resource as Record<string, unknown>).lateAcquired === true,
-    ),
+    resources: lateResources,
     endedAt: input.endedAt,
   });
 }

package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/contract.ts

@@ -1,3 +1,5 @@
+import type { PlayRunnerDaytonaCleanupIntent } from '../runner-backends/types';
+
 /**
  * Sandbox Runs is the portable lifecycle boundary around a physical execution
  * resource.  It is deliberately narrower than the scheduler attempt: it does
@@ -245,12 +247,17 @@ export type SandboxExecutionAdmission = {
         providerOperationToken?: string | null;
         /** Resume the existing durable provider operation before any new create. */
         resumeExistingProviderOperation?: boolean;
+        /** Existing operation has durable delete intent; cleanup owns it. */
+        resumePendingDelete?: boolean;
         /** Scheduler-clock age of the operation being resumed. */
         existingProviderOperationAgeMs?: number;
       }
     | { kind: 'pre_code_deferred'; reason: SandboxPreCodeDeferralReason }
   >;
   releaseCapacity(input: { leaseId: string }): Promise<void>;
+  recordDaytonaCleanupIntent?(
+    intent: PlayRunnerDaytonaCleanupIntent,
+  ): Promise<void>;
   claimProviderCircuit(input: {
     provider: Exclude<SandboxProvider, 'local'>;
   }): Promise<{ allowed: boolean; probeToken: string | null }>;
@@ -312,12 +319,17 @@ export type SandboxExecutionLifecycle = {
         providerOperationToken?: string | null;
         /** Resume the existing durable provider operation before any new create. */
         resumeExistingProviderOperation?: boolean;
+        /** Existing operation has durable delete intent; cleanup owns it. */
+        resumePendingDelete?: boolean;
         /** Scheduler-clock age of the operation being resumed. */
         existingProviderOperationAgeMs?: number;
       }
     | { kind: 'pre_code_deferred'; reason: SandboxPreCodeDeferralReason }
   >;
   releaseCapacity(input: { leaseId: string }): Promise<void>;
+  recordDaytonaCleanupIntent?(
+    intent: PlayRunnerDaytonaCleanupIntent,
+  ): Promise<void>;
   claimProviderCircuit(input: {
     provider: Exclude<SandboxProvider, 'local'>;
   }): Promise<{ allowed: boolean; probeToken: string | null }>;

package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/index.ts

@@ -249,6 +249,15 @@ function createExecutionLifecycle<Launch, Checkpoint, Suspension, RunnerResult>(
         await requireAdmission().reserveCapacity(value),
       releaseCapacity: async (value) =>
         await requireAdmission().releaseCapacity(value),
+      recordDaytonaCleanupIntent: async (intent) => {
+        const recorder = requireAdmission().recordDaytonaCleanupIntent;
+        if (!recorder) {
+          throw new Error(
+            'Sandbox Runs admission has no Daytona cleanup-intent owner.',
+          );
+        }
+        await recorder(intent);
+      },
       claimProviderCircuit: async (value) =>
         await requireAdmission().claimProviderCircuit(value),
       markProviderHealthy: async (value) =>

package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/runner-backend-adapter.ts

@@ -188,6 +188,9 @@ function createRunnerCallbacks(input: {
         ...(outcome.resumeExistingProviderOperation === true
           ? { resumeExistingProviderOperation: true }
           : {}),
+        ...(outcome.resumePendingDelete === true
+          ? { resumePendingDelete: true }
+          : {}),
         ...(outcome.existingProviderOperationAgeMs === undefined
           ? {}
           : {
@@ -199,6 +202,13 @@ function createRunnerCallbacks(input: {
     releaseSandboxCapacity: async (leaseId) => {
       await input.lifecycle.releaseCapacity({ leaseId });
     },
+    ...(input.lifecycle.recordDaytonaCleanupIntent
+      ? {
+          recordDaytonaCleanupIntent: async (intent) => {
+            await input.lifecycle.recordDaytonaCleanupIntent?.(intent);
+          },
+        }
+      : {}),
     shouldAttemptSandboxProvider: async (provider) =>
       await input.lifecycle.claimProviderCircuit({ provider }),
     markSandboxProviderHealthy: async (provider, probeToken) => {

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts

@@ -158,6 +158,10 @@ export type PlaySchedulerSubmitInput = RetainedPipelineCarrier & {
    * When omitted, legacy schedulers fall back to baseUrl.
    */
   runtimeApiBaseUrl?: string | null;
+  /** The pinned runtime API origin can retire durable array input on terminal delivery. */
+  arrayInputTerminalRetirementSupported?: boolean;
+  /** Immutable app build that owns terminal array-input retirement. */
+  arrayInputTerminalProjectionOrigin?: string | null;
   /**
    * Stable signing origin for executor JWTs. This is intentionally distinct
    * from runtimeApiBaseUrl: a staged Vercel deployment must receive the