deepline 0.3.87 → 0.3.89

package/dist/bundling-sources/shared_libs/play-runtime/coordinator-headers.ts

@@ -1,20 +1,7 @@
 /**
  * HTTP headers attached to every server-to-coordinator request.
- *
- * The coordinator is a Cloudflare Worker (`deepline-play-coordinator-*`) that
- * we publish through Cloudflare's Versions + Deployments primitives. Whenever
- * a deployment is split across two versions (e.g. 10% canary), Cloudflare
- * uses the `Cloudflare-Workers-Version-Key` header to pick a stable version
- * per request. Sending the runId as the key keeps every request for one play
- * run pinned to whichever version started the run, so a partial rollout
- * cannot strand a Workflow's `/submit` and `/result` calls on different
- * code versions.
- *
- * The header is harmless when there is only one active version (the default).
  */
-export const COORDINATOR_VERSION_KEY_HEADER = 'Cloudflare-Workers-Version-Key';
-export const COORDINATOR_VERSION_OVERRIDES_HEADER =
-  'Cloudflare-Workers-Version-Overrides';
+
 /**
  * Shared secret the coordinator uses to authenticate dispatcher (Vercel app)
  * traffic. The coordinator also sanity-checks `x-deepline-run-scope` matches
@@ -88,8 +75,6 @@ export function coordinatorRequestHeaders(input: {
   runId: string;
   contentType?: string | null;
   internalToken?: string | null;
-  runtimeDeployVersion?: string | null;
-  coordinatorWorkerName?: string | null;
   /**
    * When set, the coordinator validates this matches the runId in the URL.
    * Pass the runId on `/cancel` / `/signal` calls so a leaked dispatcher
@@ -102,22 +87,6 @@ export function coordinatorRequestHeaders(input: {
     headers['content-type'] = input.contentType;
   }
   const trimmed = input.runId.trim();
-  if (trimmed) {
-    headers[COORDINATOR_VERSION_KEY_HEADER] = trimmed;
-  }
-  const runtimeDeployVersion = input.runtimeDeployVersion?.trim();
-  const coordinatorWorkerName = input.coordinatorWorkerName?.trim();
-  if (
-    runtimeDeployVersion &&
-    coordinatorWorkerName &&
-    /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
-      runtimeDeployVersion,
-    ) &&
-    /^[A-Za-z0-9_-]+$/.test(coordinatorWorkerName)
-  ) {
-    headers[COORDINATOR_VERSION_OVERRIDES_HEADER] =
-      `${coordinatorWorkerName}="${runtimeDeployVersion}"`;
-  }
   const internalToken =
     input.internalToken?.trim() || resolveInternalCoordinatorToken();
   if (internalToken) {

package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts

@@ -10,8 +10,10 @@ import type { PlayRunContractSnapshot } from '../plays/contracts';
 import type { PlayStructuredDefinition } from '../plays/definition';
 import type { PlayStaticPipeline } from '../plays/static-pipeline';
 import type {
+  AsyncPlayRunRef,
   DurableCallStaleAfterSeconds,
   PlayAuthoringCallOptions,
+  PlayAuthoringAsyncCallOptions,
   PlayAuthoringCsvOptions,
   PlayAuthoringFetchOptions,
   PlayAuthoringRuntimeStepOptions,
@@ -331,6 +333,19 @@ export type PlayCallOptions = Omit<
   timeoutMs?: number;
 };
 
+export type PlayAsyncCallOptions = PlayAuthoringAsyncCallOptions;
+
+export type AsyncPlayLaunchRequest = Readonly<{
+  key: string;
+  invocationKey: string;
+  playRef: string;
+  input: Record<string, unknown>;
+  description: string;
+  schedulerSchema: string | null;
+  /** Governor ancestry at the authored call site, before entering the child. */
+  ancestryPlayIds: readonly string[];
+}>;
+
 export type RuntimeStepOptions = PlayAuthoringRuntimeStepOptions;
 
 export type FetchOptions = PlayAuthoringFetchOptions;
@@ -345,6 +360,16 @@ export interface ResolvedPlayExecution {
   definition?: PlayStructuredDefinition | null;
   staticPipeline?: PlayStaticPipeline | null;
   contractSnapshot?: PlayRunContractSnapshot | null;
+  launchMetadata?: {
+    revisionId?: string | null;
+    artifactStorageKey: string;
+    artifactHash: string;
+    graphHash: string;
+    definitionScope: 'org' | 'system';
+    authoringContract?:
+      | import('../plays/authoring-contract').AdmittedPlayAuthoringContract
+      | null;
+  } | null;
 }
 
 export interface BatchRequest {
@@ -548,6 +573,7 @@ export type IntegrationEventWaitRuntimeContext = {
   workflowId?: string;
   orgId?: string;
   executorToken?: string;
+  slackHitlDefaultTimeoutMs?: number;
 };
 
 export type IntegrationEventWaitBoundary = {
@@ -560,6 +586,14 @@ export type IntegrationEventWaitBoundary = {
     channel: string;
     ts: string;
   };
+  /**
+   * Optional control-plane timeout behavior. Missing preserves the legacy
+   * "return the timed_out result" replay behavior for older app deployments.
+   */
+  timeoutPolicy?: {
+    suspendAfterTimeoutMs?: number;
+    finalizeOnTimeout?: boolean;
+  };
 };
 
 export type IntegrationEventWaitHandler = {
@@ -584,6 +618,11 @@ export type IntegrationEventWaitHandler = {
     context: IntegrationEventWaitRuntimeContext;
     boundary: IntegrationEventWaitBoundary;
   }) => Promise<void>;
+  /**
+   * When set, timeout finalization is side-effect-only: the Play stays parked
+   * instead of consuming a `timed_out` result.
+   */
+  suspendAfterTimeoutMs?: number;
 };
 
 export interface ContextOptions {
@@ -769,6 +808,8 @@ export interface ContextOptions {
   }) => Promise<void>;
   playId?: string;
   runId?: string;
+  /** Shared governance/billing root. Differs from runId for async children. */
+  rootRunId?: string;
   /** Physical executor run that owns leases for an inline child context. */
   runtimeReceiptOwnerRunId?: string;
   /** Logical invocation scope for receipts created inside an inline child. */
@@ -800,6 +841,10 @@ export interface ContextOptions {
    */
   runtimeSheetBackedMapDatasets?: boolean;
   resolvePlay?: (playRef: string) => Promise<ResolvedPlayExecution | null>;
+  /** Admit an independently scheduled scalar/HITL Play Run. */
+  launchPlayAsync?: (
+    request: AsyncPlayLaunchRequest,
+  ) => Promise<AsyncPlayRunRef>;
   getToolQueueHints?: (toolId: string) => Promise<readonly PlayQueueHint[]>;
   getToolProvider?: (toolId: string) => Promise<string | null>;
   getToolOperation?: (toolId: string) => Promise<string | null>;
@@ -1147,6 +1192,12 @@ export type PlayStep =
       nestedSteps: PlayStep[];
       description?: string;
     }
+  | {
+      type: 'play_launch';
+      playId: string;
+      runId: string;
+      description?: string;
+    }
   | {
       type: 'run_javascript';
       alias: string;
@@ -1187,6 +1238,12 @@ export type PlayDatasetSubstep =
       nestedSteps: PlayStep[];
       description?: string;
     }
+  | {
+      type: 'play_launch';
+      playId: string;
+      runId: string;
+      description?: string;
+    }
   | {
       type: 'run_javascript';
       alias: string;

package/dist/bundling-sources/shared_libs/play-runtime/default-batch-strategies.ts

@@ -10,6 +10,15 @@ type TestRateLimitPayload = {
   lead_id?: string;
   row_number?: number;
   simulated_delay_ms?: number;
+  emit_security_gateway?: boolean;
+  /**
+   * Synthetic-provider-only switch. The large-payload gauntlet uses the batch
+   * response as a transport fixture and does not inspect per-item execution
+   * counters, so it can avoid creating thousands of unrelated test-state rows.
+   */
+  track_item_execution?: boolean;
+  simulated_cooldown_ms?: number;
+  cooldown_failures_before_success?: number;
 };
 
 type TestRateLimitBatchPayload = {
@@ -83,7 +92,17 @@ const testRateLimitBatchStrategy: BatchOperationStrategy<
   maxBatchSize: 200,
   bucketKeyPayloadFields: ['stage'],
   canBatchWith(left, right) {
-    return String(left.stage || '') === String(right.stage || '');
+    // One upstream native batch has one 429/Retry-After outcome. Cooldown
+    // fixtures therefore only share a physical request when they requested
+    // the exact same policy; a normal member must never inherit another
+    // member's synthetic provider failure.
+    return (
+      String(left.stage || '') === String(right.stage || '') &&
+      Number(left.simulated_cooldown_ms ?? 0) ===
+        Number(right.simulated_cooldown_ms ?? 0) &&
+      Number(left.cooldown_failures_before_success ?? 0) ===
+        Number(right.cooldown_failures_before_success ?? 0)
+    );
   },
   toBucketKey(payload) {
     return `test_batch_rate_limit:${String(payload.stage || '')}`;

package/dist/bundling-sources/shared_libs/play-runtime/execution-capabilities.ts

@@ -2,6 +2,7 @@ import {
   validateFixtureBehavior,
   type FixtureBehavior,
 } from './fixture-behavior';
+import { normalizeRuntimeSchedulerSchema } from './runtime-scheduler-topology';
 
 export const RUNTIME_EXECUTION_CAPABILITIES = [
   'artifact.read',
@@ -46,6 +47,8 @@ export type RuntimeAuthorityDescriptor = {
   synthetic?: boolean | null;
   actorUserId?: string | null;
   actorEmail?: string | null;
+  /** Durable scheduler topology, carried into each newly minted attempt token. */
+  schedulerSchema?: string | null;
   /**
    * Whether THIS run may capture the docflow provenance trace.
    *
@@ -142,6 +145,9 @@ export function requireRuntimeAuthorityDescriptor(
     synthetic: descriptor.synthetic === true,
     actorUserId: descriptor.actorUserId ?? null,
     actorEmail: descriptor.actorEmail ?? null,
+    schedulerSchema: normalizeRuntimeSchedulerSchema(
+      descriptor.schedulerSchema,
+    ),
     // `=== true`, not `?? false`: the normalizer is what a durable launch is
     // read back through, so anything that is not an explicit yes — missing,
     // null, a string that survived a JSON round trip — denies.

package/dist/bundling-sources/shared_libs/play-runtime/execution-protocol.ts

@@ -11,7 +11,13 @@
 export const PLAY_RUNTIME_PROTOCOL = {
   /** Launches written before this field existed. */
   legacy: 0,
-  /** Current scheduler/runner protocol. */
+  /**
+   * Current scheduler/runner protocol. This number STAYS AT 1: the additive
+   * compatibility rule (`docs/runtime-compatibility-contract.md`) covers new
+   * fields and new named actions without a bump. A bump is a fence release —
+   * two PRs, expand worker support first, then enable — and needs a written
+   * reason the additive rule cannot serve.
+   */
   current: 1,
 } as const;
 

package/dist/bundling-sources/shared_libs/play-runtime/lane-retirement-gate.ts

@@ -0,0 +1,189 @@
+/**
+ * The Absurd release-lane retirement gate, as dependency-free pure logic.
+ *
+ * This lives in its own module because three very different processes must
+ * reach exactly the same verdict and none of them may drag the others'
+ * dependencies in:
+ *
+ *   - the Next.js app (`src/lib/plays/runtime-lanes.ts`, the admin and internal
+ *     lane endpoints), which also talks to Convex;
+ *   - `scripts/fly-absurd-lane.ts` retire, run by the release controller; and
+ *   - `apps/absurd-release-scaler/index.ts`, which owns retirement of every lane
+ *     the release controller handed over, and whose Fly image does not contain
+ *     `packages/convex/`.
+ *
+ * The gate clears two ways:
+ *
+ *   1. **Empty.** No non-terminal task on the lane's queue and no non-terminal
+ *      run row referencing its release id.
+ *   2. **Adopted parked (lane handoff).** The lane has been marked not claiming,
+ *      nothing on it is claimable or running, and every remaining non-terminal
+ *      run is parked in a durable wait the newest lane adopts.
+ *
+ * Anything else is blocked, and `blockedBy` names which of the two halves is
+ * responsible so an operator is not left guessing at a bare depth count.
+ */
+
+/**
+ * The subset of a lane's depth the gate reads.
+ *
+ * Every field is optional because the gate is also evaluated against a lane
+ * view that crossed the wire from a control plane which may be older than its
+ * caller: the release-aware scaler is deployed from the same release as the
+ * app, but a scaler image can outlive the app deployment it polls. A fact that
+ * did not arrive is never guessed — see `adoption-facts-unavailable`.
+ */
+export type LaneRetirementGateFacts = {
+  /** Non-terminal tasks on `absurd.t_<queue>`. */
+  queuedTasks?: number;
+  claimableTasks?: number;
+  runningTasks?: number;
+  /** Non-terminal `runs` rows whose launch envelope pins this release id. */
+  nonTerminalRuns?: number;
+  /** Subset of `nonTerminalRuns` parked in a wait the newest lane adopts. */
+  adoptableParkedRuns?: number;
+  /** Whether this lane still claims work off its own queue. */
+  claiming?: boolean;
+  /** `queuedTasks === 0 && nonTerminalRuns === 0`. */
+  empty?: boolean;
+  /** `claimableTasks === 0 && runningTasks === 0 && nonTerminalRuns === adoptableParkedRuns`. */
+  emptyExceptAdoptableParked?: boolean;
+};
+
+export type LaneRetirementGateBasis = 'empty' | 'adopted-parked' | 'blocked';
+
+/**
+ * Why a blocked lane is blocked. This is the operator-facing half of the gate:
+ * each value maps to exactly one remediation.
+ */
+export type LaneRetirementBlockedReason =
+  /** Not blocked. */
+  | 'none'
+  /** Something on this lane is claimable or running right now. Wait. */
+  | 'executable-work'
+  /**
+   * The lane holds nothing but adoptable parked waits, and the ONLY thing
+   * keeping it alive is that its claiming switch was never flipped. The lane
+   * cannot flip it itself, so without an external write this lane is immortal.
+   */
+  | 'still-claiming'
+  /**
+   * Non-terminal runs this lane can no longer execute and no other lane can
+   * adopt: a `detached_runner` park whose sandbox is gone, or a run row with no
+   * engine task at all. Nothing will ever wake these; they need an explicit,
+   * evidence-gated terminal transition.
+   */
+  | 'unadoptable-non-terminal-runs'
+  /**
+   * The caller's lane view predates the adoption fields. The gate refuses to
+   * infer adoption from absent evidence and falls back to the strict `empty`
+   * half only.
+   */
+  | 'adoption-facts-unavailable';
+
+export type LaneRetirementGateResult = {
+  ok: boolean;
+  basis: LaneRetirementGateBasis;
+  blockedBy: LaneRetirementBlockedReason;
+};
+
+/**
+ * Evaluate the retirement gate.
+ *
+ * Deliberately conservative in one direction only: it never clears a lane on a
+ * fact it did not receive. It is safe for the caller to retire on `ok`, because
+ * retirement deletes Machines and flips the registry row; it does not drop the
+ * queue tables an adopting lane still claims from.
+ */
+export function laneRetirementGate(
+  depth: LaneRetirementGateFacts,
+): LaneRetirementGateResult {
+  if (depth.empty === true) {
+    return { ok: true, basis: 'empty', blockedBy: 'none' };
+  }
+  if (
+    typeof depth.claiming !== 'boolean' ||
+    typeof depth.emptyExceptAdoptableParked !== 'boolean' ||
+    typeof depth.adoptableParkedRuns !== 'number'
+  ) {
+    return {
+      ok: false,
+      basis: 'blocked',
+      blockedBy: 'adoption-facts-unavailable',
+    };
+  }
+  if (!depth.claiming && depth.emptyExceptAdoptableParked) {
+    return { ok: true, basis: 'adopted-parked', blockedBy: 'none' };
+  }
+  const executable =
+    (depth.claimableTasks ?? 0) > 0 || (depth.runningTasks ?? 0) > 0;
+  if (executable) {
+    return { ok: false, basis: 'blocked', blockedBy: 'executable-work' };
+  }
+  if (depth.claiming && depth.emptyExceptAdoptableParked) {
+    return { ok: false, basis: 'blocked', blockedBy: 'still-claiming' };
+  }
+  return {
+    ok: false,
+    basis: 'blocked',
+    blockedBy: 'unadoptable-non-terminal-runs',
+  };
+}
+
+/**
+ * The one operator-actionable sentence for a blocked lane.
+ *
+ * A depth count alone ("11 queued") has repeatedly sent operators looking for
+ * live work that is not there. Each reason below has exactly one next step.
+ */
+export function laneRetirementRemediation(
+  result: LaneRetirementGateResult,
+): string {
+  switch (result.blockedBy) {
+    case 'none':
+      return result.basis === 'adopted-parked'
+        ? 'Retirable: nothing here is claimable or running and every remaining run is a parked wait the newest lane adopts.'
+        : 'Retirable: this lane holds no non-terminal task and no non-terminal run.';
+    case 'executable-work':
+      return 'Wait. Work on this lane is claimable or running right now; the scaler keeps a version-matched machine for it.';
+    case 'still-claiming':
+      return 'Demote the lane. It holds only adoptable parked waits; the newest lane cannot adopt them until this lane stops claiming (lane handoff). The release-aware scaler converges this, or run `bun scripts/fly-absurd-lane.ts stop-claiming --release <id>`.';
+    case 'unadoptable-non-terminal-runs':
+      return 'Inspect the runs. This lane holds non-terminal runs it can no longer execute and no lane can adopt. Run `deepline-admin runtime stranded scan --release <id> --environment production --json`, then remediate the reviewed cohort with `runtime stranded plan` / `runtime stranded resolve --dry-run`.';
+    case 'adoption-facts-unavailable':
+      return 'The lane view did not carry the lane handoff adoption fields, so only the strict empty gate was evaluated. Re-read the lane against a control plane from this release.';
+  }
+}
+
+/**
+ * Should this lane's claiming switch be converged to `false`?
+ *
+ * The switch is written exactly once, by the release controller, as a
+ * `reclamation` effect whose failure is explicitly tolerated — a failed
+ * `stop-prior-lane-claiming` is logged and the release still SUCCEEDS. Absence
+ * of the control row reads as "claiming", so a lost write leaves a demoted lane
+ * claiming forever. With no machines the lane cannot make progress, and with
+ * `claiming` true the newest lane may not adopt it: a permanently unclaimable
+ * steady state.
+ *
+ * This predicate is intentionally narrow. It fires only for a lane whose gate
+ * is blocked by the lost write and nothing else, so it can never demote a lane
+ * that is still doing something, and it cannot fight `abort_retire` (which
+ * re-arms claiming for a lane that has real work, and therefore is not blocked
+ * by `still-claiming`).
+ */
+export function shouldConvergeLaneToNotClaiming(lane: {
+  active?: boolean;
+  status?: string;
+  queueDepth?: LaneRetirementGateFacts;
+}): boolean {
+  if (lane.active !== false) return false;
+  if (lane.status !== 'draining' && lane.status !== 'retiring') return false;
+  const depth = lane.queueDepth;
+  if (!depth) return false;
+  if ((depth.claimableTasks ?? 0) > 0 || (depth.runningTasks ?? 0) > 0) {
+    return false;
+  }
+  if ((depth.adoptableParkedRuns ?? 0) <= 0) return false;
+  return laneRetirementGate(depth).blockedBy === 'still-claiming';
+}

package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-semantics.ts

@@ -34,6 +34,9 @@ export type PlayRuntimeSemantics = Readonly<{
      */
     receiptIdentityEdition: PlayAuthoringContractEdition | null;
   }>;
+  slackHitl: Readonly<{
+    defaultTimeoutMs: number;
+  }>;
 }>;
 
 function definePlayRuntimeSemantics(
@@ -43,6 +46,7 @@ function definePlayRuntimeSemantics(
     ...semantics,
     secrets: Object.freeze({ ...semantics.secrets }),
     fetch: Object.freeze({ ...semantics.fetch }),
+    slackHitl: Object.freeze({ ...semantics.slackHitl }),
   });
 }
 
@@ -66,6 +70,14 @@ const PLAINTEXT_SECRET_SEMANTICS = {
   secrets: { value: 'plaintext_promise' },
 } as const;
 
+const LEGACY_SLACK_HITL_SEMANTICS = {
+  defaultTimeoutMs: 24 * 60 * 60 * 1000,
+} as const;
+
+const WEEK_LONG_SLACK_HITL_SEMANTICS = {
+  defaultTimeoutMs: 7 * 24 * 60 * 60 * 1000,
+} as const;
+
 /**
  * Exhaustive compatibility registry for worker-visible Play behavior.
  *
@@ -77,22 +89,27 @@ const PLAY_RUNTIME_SEMANTICS_BY_EDITION = {
   1: definePlayRuntimeSemantics({
     authoringContractEdition: 1,
     ...LEGACY_SEMANTICS,
+    slackHitl: LEGACY_SLACK_HITL_SEMANTICS,
   }),
   2: definePlayRuntimeSemantics({
     authoringContractEdition: 2,
     ...STRICT_SEMANTICS,
+    slackHitl: LEGACY_SLACK_HITL_SEMANTICS,
   }),
   3: definePlayRuntimeSemantics({
     authoringContractEdition: 3,
     ...STRICT_SEMANTICS,
+    slackHitl: LEGACY_SLACK_HITL_SEMANTICS,
   }),
   4: definePlayRuntimeSemantics({
     authoringContractEdition: 4,
     ...PLAINTEXT_SECRET_SEMANTICS,
+    slackHitl: LEGACY_SLACK_HITL_SEMANTICS,
   }),
   5: definePlayRuntimeSemantics({
     authoringContractEdition: 5,
     ...PLAINTEXT_SECRET_SEMANTICS,
+    slackHitl: LEGACY_SLACK_HITL_SEMANTICS,
     fetch: {
       httpFailure: 'throw_typed_error',
       persistHttpFailure: false,
@@ -102,6 +119,7 @@ const PLAY_RUNTIME_SEMANTICS_BY_EDITION = {
   6: definePlayRuntimeSemantics({
     authoringContractEdition: 6,
     ...PLAINTEXT_SECRET_SEMANTICS,
+    slackHitl: LEGACY_SLACK_HITL_SEMANTICS,
     fetch: {
       httpFailure: 'throw_typed_error',
       persistHttpFailure: false,
@@ -110,6 +128,16 @@ const PLAY_RUNTIME_SEMANTICS_BY_EDITION = {
       receiptIdentityEdition: 6,
     },
   }),
+  7: definePlayRuntimeSemantics({
+    authoringContractEdition: 7,
+    ...PLAINTEXT_SECRET_SEMANTICS,
+    slackHitl: WEEK_LONG_SLACK_HITL_SEMANTICS,
+    fetch: {
+      httpFailure: 'throw_typed_error',
+      persistHttpFailure: false,
+      receiptIdentityEdition: 6,
+    },
+  }),
 } satisfies Record<PlayAuthoringContractEdition, PlayRuntimeSemantics>;
 
 export function resolvePlayRuntimeSemantics(

package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts

@@ -152,6 +152,8 @@ export interface PlayRunnerContextConfig {
   workflowId?: string;
   playId?: string;
   runId?: string;
+  /** Shared governance/billing root. Differs from runId for async children. */
+  rootRunId?: string;
   runAttempt?: number | null;
   playName?: string;
   userEmail?: string;

package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts

@@ -170,6 +170,29 @@ export class ProviderExhaustedError extends Error {
   }
 }
 
+/**
+ * `ProviderExhaustedError` is created inside the runtime, but a tool execution
+ * can cross the execution gateway before a step program sees it. The gateway
+ * deliberately serializes that error as the public, provider-owned
+ * `ToolExecutionError` shape. Treat those two representations as the same
+ * typed exhaustion outcome; do not fall back based on a message substring.
+ */
+export function isProviderExhaustedFailure(
+  error: unknown,
+): error is
+  | ProviderExhaustedError
+  | (ToolExecutionError & {
+      readonly origin: 'provider';
+      readonly code: typeof PROVIDER_EXHAUSTED_CODE;
+    }) {
+  return (
+    error instanceof ProviderExhaustedError ||
+    (error instanceof ToolExecutionError &&
+      error.origin === 'provider' &&
+      error.code === PROVIDER_EXHAUSTED_CODE)
+  );
+}
+
 /**
  * Derive the human provider name for a PROVIDER_EXHAUSTED error from a pacing
  * bucketId. Bucket ids are `${orgId}:${provider}`; the provider itself is

package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts

@@ -130,7 +130,7 @@ export function isActivePlayRunStatus(status: unknown): boolean {
  * The minimal persisted Play Run document shape required to project the live
  * snapshot. This is a structural subset of the Convex `playRuns` doc and of
  * the run-observer projection returned by
- * `convex/runObservers.getPlayRunSnapshotForObserver`.
+ * `packages/convex/runObservers.getPlayRunSnapshotForObserver`.
  */
 export type LedgerBackedRunLike = {
   workflowId: string;

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts

@@ -11,8 +11,15 @@ import {
 } from '../../../plays/sandbox-runtime-limits';
 import type { PlayRunnerRuntimeLifecycleEvent } from '../types';
 import { DAYTONA_PLAY_RUNNER_LABEL_SOURCE } from './daytona-labels';
-
-const DAYTONA_CREATE_TIMEOUT_SECONDS = 10;
+import type { DaytonaSandboxStartAdmission } from './daytona-start-admission';
+
+// Match Daytona's supported default create window. Ten seconds is shorter
+// than normal provider startup under burst load: the SDK abandons an otherwise
+// healthy create, and the resulting provider-side transition cannot safely be
+// deleted or hedged with a second sandbox. The runner deadline still bounds
+// the full lifecycle, while ambiguous creates retain the fail-closed cleanup
+// path below.
+const DAYTONA_CREATE_TIMEOUT_SECONDS = 60;
 const DAYTONA_CREATE_RETRY_DELAYS_MS = [0, 500, 1_500] as const;
 const DAYTONA_TIMED_OUT_CREATE_RECONCILE_DELAYS_MS = [
   0, 1_000, 3_000, 6_000, 12_000, 24_000, 36_000,
@@ -105,7 +112,7 @@ export function resolveDaytonaSandboxAcquisitionUnavailableReason(
 }
 
 function isDaytonaSandboxStartTimeout(error: string): boolean {
-  return /Failed to create and start sandbox within 10 seconds\. Operation timed out\./.test(
+  return /Failed to create and start sandbox within \d+ seconds\. Operation timed out\./.test(
     error,
   );
 }
@@ -575,6 +582,7 @@ async function acquireOneShotDaytonaSandbox(input: {
   releaseSandboxCapacity?: (leaseId: string) => Promise<void>;
   nextProviderAttempt: () => number;
   startedAt: number;
+  startAdmission?: DaytonaSandboxStartAdmission;
 }): Promise<AcquiredDaytonaSandbox> {
   const limits = validatePlaySandboxRuntimeLimits(
     input.context.sandboxRuntimeLimits ?? {
@@ -582,7 +590,11 @@ async function acquireOneShotDaytonaSandbox(input: {
     },
   );
   input.emitStage('create:start');
-  const result = await createRetriedOneShotDaytonaSandbox(input);
+  const result = input.startAdmission
+    ? await input.startAdmission.run(() =>
+        createRetriedOneShotDaytonaSandbox(input),
+      )
+    : await createRetriedOneShotDaytonaSandbox(input);
   const granted = {
     cpu: result.sandbox.cpu,
     memoryGiB: result.sandbox.memory,
@@ -683,6 +695,7 @@ export function createOneShotDaytonaSandboxLifecycle(input: {
     providerOperationToken: string | null;
   }>;
   releaseSandboxCapacity?: (leaseId: string) => Promise<void>;
+  startAdmission?: DaytonaSandboxStartAdmission;
   startedAt?: number;
 }): OneShotDaytonaSandboxLifecycle {
   const orgId = validateDaytonaExecutionContext(input.context);
@@ -700,6 +713,7 @@ export function createOneShotDaytonaSandboxLifecycle(input: {
       observeCreateCall: input.observeCreateCall,
       reserveSandboxCapacity: input.reserveSandboxCapacity,
       releaseSandboxCapacity: input.releaseSandboxCapacity,
+      startAdmission: input.startAdmission,
       nextProviderAttempt: () => {
         providerAttempt += 1;
         return providerAttempt;