deepline 0.3.283 → 0.3.285

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts

@@ -606,6 +606,8 @@ export type PostgresSchedulerRunStatusSnapshot = {
   runId: string;
   /** Owning org from the scheduler runs row; '' on pre-org legacy rows. */
   orgId: string;
+  /** Included only for callers that explicitly request admission identity. */
+  definitionScope?: 'org' | 'system';
   status: string;
   playName: string;
   graphHash: string | null;
@@ -630,6 +632,7 @@ type RunStatusRow = RunRow & {
   graph_hash: string | null;
   runtime_profile: string | null;
   launch_json: unknown;
+  definition_scope?: string | null;
   suspension_json: unknown | null;
   updated_at: Date | null;
 };
@@ -5649,6 +5652,7 @@ export class PostgresSchedulerCronLaunchConflictError extends PostgresSchedulerC
  */
 export type PostgresSchedulerCronSourcePin = {
   orgId: string;
+  definitionScope?: 'org' | 'system';
   userId: string | null;
   playName: string;
   revisionId: string | null;
@@ -5675,6 +5679,9 @@ export function postgresSchedulerCronSourcePinFromLaunch(
       : null;
   return {
     orgId: launch.orgId,
+    ...(launch.definitionScope === 'org' || launch.definitionScope === 'system'
+      ? { definitionScope: launch.definitionScope }
+      : {}),
     userId: launch.userId ?? null,
     playName: launch.playName,
     revisionId: launch.revisionId ?? null,
@@ -5785,7 +5792,23 @@ export async function seedPostgresSchedulerCronSchedule(
                  schedule.cron = $4::text
                  AND schedule.timezone IS NOT DISTINCT FROM $5::text
                  AND schedule.current_due_at = to_timestamp($3::double precision / 1000.0)
-                 AND schedule.source_pin_json = $6::jsonb
+                 AND (
+                   schedule.source_pin_json = $6::jsonb
+                   OR (
+                     schedule.source_pin_json - 'definitionScope' =
+                       $6::jsonb - 'definitionScope'
+                     AND (
+                       (
+                         NOT (schedule.source_pin_json ? 'definitionScope')
+                         AND $6::jsonb ->> 'definitionScope' = 'org'
+                       )
+                       OR (
+                         NOT ($6::jsonb ? 'definitionScope')
+                         AND schedule.source_pin_json ->> 'definitionScope' = 'org'
+                       )
+                     )
+                   )
+                 )
                ) AS matches_input
         FROM schedule_lock
         JOIN ${tableName(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE, options)} AS schedule
@@ -7083,16 +7106,21 @@ export async function assignPostgresSchedulerRunAttempt(
         SELECT
           runs.run_id,
           runs.started_at,
-          max(attempts.attempt) AS active_attempt
+          max(attempts.attempt) FILTER (
+            WHERE attempts.status IN ('claimed', 'running', 'waiting')
+          ) AS active_attempt
         FROM locked_run AS runs
         LEFT JOIN ${tableName('attempts', options)} AS attempts
           ON attempts.run_id = runs.run_id
-         AND attempts.status IN ('claimed', 'running', 'waiting')
         WHERE runs.run_id = $1
           AND runs.status IN ('queued', 'running', 'waiting')
         GROUP BY runs.run_id, runs.started_at
-        HAVING max(attempts.attempt) IS NULL
-            OR max(attempts.attempt) < $2
+        HAVING max(attempts.attempt) FILTER (
+                 WHERE attempts.status IN ('claimed', 'running', 'waiting')
+               ) IS NULL
+            OR max(attempts.attempt) FILTER (
+                 WHERE attempts.status IN ('claimed', 'running', 'waiting')
+               ) < $2
       ),
       assigned AS (
         INSERT INTO ${tableName('attempts', options)} (
@@ -7124,6 +7152,45 @@ export async function assignPostgresSchedulerRunAttempt(
           AND attempt < $2
           AND status IN ('claimed', 'running', 'waiting')
       ),
+      -- A run can only have one live, unbound Daytona create in flight. Reuse
+      -- its durable provider identity across attempt retries and move its
+      -- fence to the new owner; the provider idempotency key makes replay safe.
+      adopted_daytona_operations AS (
+        UPDATE ${tableName('runtime_sandbox_operations', options)} AS operation
+        SET attempt = $2,
+            updated_at = clock_timestamp()
+        FROM target_run
+        JOIN assigned USING (run_id)
+        WHERE operation.run_id = target_run.run_id
+          AND operation.attempt < $2
+          AND operation.provider = 'daytona'
+          AND operation.state = 'reserved'
+          AND EXISTS (
+            SELECT 1
+            FROM ${tableName('runtime_sandbox_capacity_leases', options)} AS lease
+            WHERE lease.lease_id = operation.operation_key
+              AND lease.released_at IS NULL
+              AND lease.sandbox_id IS NULL
+              AND lease.expires_at > clock_timestamp()
+          )
+          AND 1 = (
+            SELECT count(*)
+            FROM ${tableName('runtime_sandbox_operations', options)} AS pending
+            WHERE pending.run_id = target_run.run_id
+              AND pending.attempt < $2
+              AND pending.provider = 'daytona'
+              AND pending.state = 'reserved'
+              AND EXISTS (
+                SELECT 1
+                FROM ${tableName('runtime_sandbox_capacity_leases', options)} AS pending_lease
+                WHERE pending_lease.lease_id = pending.operation_key
+                  AND pending_lease.released_at IS NULL
+                  AND pending_lease.sandbox_id IS NULL
+                  AND pending_lease.expires_at > clock_timestamp()
+              )
+          )
+        RETURNING operation.operation_key
+      ),
       first_start AS (
         SELECT assigned.run_id
         FROM assigned
@@ -10096,7 +10163,11 @@ export async function markPostgresSchedulerRuntimeResourceDetached(
 
 export async function readPostgresSchedulerRuntimeResourcesForRun(
   client: PostgresSchedulerQueryClient,
-  input: { runId: string; attempt?: number | null },
+  input: {
+    runId: string;
+    attempt?: number | null;
+    throughAttempt?: number;
+  },
   options?: Pick<PostgresSchedulerOptions, 'schema'>,
 ): Promise<unknown[]> {
   const resources = await client.query<{ resource_json: unknown }>(
@@ -10105,10 +10176,11 @@ export async function readPostgresSchedulerRuntimeResourcesForRun(
       FROM ${tableName('attempts', options)}
       WHERE run_id = $1
         AND ($2::integer IS NULL OR attempt = $2)
+        AND ($3::integer IS NULL OR attempt <= $3)
         AND resource_json IS NOT NULL
       ORDER BY attempt DESC
     `,
-    [input.runId, input.attempt ?? null],
+    [input.runId, input.attempt ?? null, input.throughAttempt ?? null],
   );
   return resources.rows.flatMap((row) =>
     Array.isArray(row.resource_json) ? row.resource_json : [row.resource_json],
@@ -10137,6 +10209,7 @@ export async function settlePostgresSchedulerRuntimeResources(
       | 'runtime_limit'
       | 'cancelled'
       | 'lease_expired';
+    throughAttempt?: number;
   },
   options?: Pick<PostgresSchedulerOptions, 'schema'>,
 ): Promise<number> {
@@ -10169,6 +10242,7 @@ export async function settlePostgresSchedulerRuntimeResources(
           ),
           updated_at = now()
       WHERE run_id = $1
+        AND ($4::integer IS NULL OR attempt <= $4)
         AND resource_json IS NOT NULL
         AND jsonb_typeof(resource_json) = 'array'
         AND EXISTS (
@@ -10182,7 +10256,12 @@ export async function settlePostgresSchedulerRuntimeResources(
         )
       RETURNING run_id
     `,
-    [input.runId, input.billingEndedAt, input.terminalReason],
+    [
+      input.runId,
+      input.billingEndedAt,
+      input.terminalReason,
+      input.throughAttempt ?? null,
+    ],
   );
   return stamped.rows.length;
 }
@@ -11605,7 +11684,7 @@ export async function enqueuePostgresSchedulerSandboxCleanupForAttempt(
  * admits the current runner before cancellation terminalizes. */
 export async function enqueuePostgresSchedulerSandboxCleanupForRun(
   client: PostgresSchedulerQueryClient,
-  input: { runId: string },
+  input: { runId: string; throughAttempt?: number },
   options?: Pick<PostgresSchedulerOptions, 'schema'>,
 ): Promise<void> {
   await client.query(
@@ -11639,6 +11718,7 @@ export async function enqueuePostgresSchedulerSandboxCleanupForRun(
         END
       ) AS resource
       WHERE attempt.run_id = $1
+        AND ($2::integer IS NULL OR attempt.attempt <= $2)
         AND attempt.resource_json IS NOT NULL
         AND resource->>'kind' IN ('daytona_sandbox', 'modal_sandbox')
         AND NULLIF(resource->>'sandboxId', '') IS NOT NULL
@@ -11648,12 +11728,13 @@ export async function enqueuePostgresSchedulerSandboxCleanupForRun(
              operation.attempt, operation.environment, operation.routing_domain
       FROM ${tableName('runtime_sandbox_operations', options)} AS operation
       WHERE operation.run_id = $1
+        AND ($2::integer IS NULL OR operation.attempt <= $2)
         AND operation.state IN ('live', 'deleting')
         AND operation.sandbox_id IS NOT NULL
         AND NOT ${protectedDetachedSandboxSql({ runId: 'operation.run_id', provider: 'operation.provider', sandboxId: 'operation.sandbox_id', routingDomain: 'operation.routing_domain' }, options)}
       ON CONFLICT (provider, sandbox_id, routing_domain_key) DO NOTHING
     `,
-    [input.runId],
+    [input.runId, input.throughAttempt ?? null],
   );
 }
 
@@ -11734,6 +11815,7 @@ export async function claimPostgresSchedulerSandboxCleanupJobs(
   input: {
     leaseOwner: string;
     runId?: string;
+    throughAttempt?: number;
     limit?: number;
     leaseSeconds?: number;
     environment?: string | null;
@@ -11828,6 +11910,7 @@ export async function claimPostgresSchedulerSandboxCleanupJobs(
           AND ($4::text IS NULL OR cleanup.environment = $4)
           AND ($5::text[] IS NULL OR cleanup.provider = ANY($5::text[]))
           AND ($8::text IS NULL OR cleanup.run_id = $8)
+          AND ($9::integer IS NULL OR cleanup.attempt <= $9)
           AND NOT ${protectedDetachedSandboxSql({ runId: 'cleanup.run_id', provider: 'cleanup.provider', sandboxId: 'cleanup.sandbox_id', routingDomain: 'cleanup.routing_domain' }, options)}
         ORDER BY
           CASE WHEN cleanup.diagnostic_json IS NOT NULL THEN 0 ELSE 1 END ASC,
@@ -11914,6 +11997,7 @@ export async function claimPostgresSchedulerSandboxCleanupJobs(
       RuntimeSandboxOperationState.Deleting,
       RuntimeSandboxOperationState.Live,
       input.runId ?? null,
+      input.throughAttempt ?? null,
     ],
   );
   return claimed.rows.map((row) => ({
@@ -16914,8 +16998,12 @@ export async function readPostgresSchedulerRunStatus(
   runId: string,
   options?: Pick<PostgresSchedulerOptions, 'schema'> & {
     includeDetails?: boolean;
+    includeAdmissionScope?: boolean;
   },
 ): Promise<PostgresSchedulerRunStatusSnapshot | null> {
+  const admissionScopeSelection = options?.includeAdmissionScope
+    ? ", launch_json ->> 'definitionScope' AS definition_scope"
+    : '';
   if (options?.includeDetails === false) {
     // An admission projection gap needs lifecycle truth only. Launch, result,
     // suspension and error JSON may contain row/call-sized detail: never even
@@ -16933,11 +17021,12 @@ export async function readPostgresSchedulerRunStatus(
         | 'updated_at'
         | 'started_at'
         | 'finished_at'
+        | 'definition_scope'
       >
     >(
       `
       SELECT run_id, org_id, status, play_name, graph_hash, runtime_profile,
-        created_at, updated_at, started_at, finished_at
+        created_at, updated_at, started_at, finished_at${admissionScopeSelection}
       FROM ${tableName('runs', options)}
       WHERE run_id = $1
     `,
@@ -16948,6 +17037,10 @@ export async function readPostgresSchedulerRunStatus(
     return {
       runId: row.run_id,
       orgId: row.org_id ?? '',
+      ...(options.includeAdmissionScope &&
+      (row.definition_scope === 'org' || row.definition_scope === 'system')
+        ? { definitionScope: row.definition_scope }
+        : {}),
       status: row.status,
       playName: row.play_name,
       graphHash: row.graph_hash,
@@ -16973,7 +17066,7 @@ export async function readPostgresSchedulerRunStatus(
     `
       SELECT run_id, org_id, status, play_name, graph_hash, runtime_profile,
         launch_json, suspension_json, terminal_result_json, terminal_error_json,
-        runtime_timing_json, created_at, updated_at, started_at, finished_at
+        runtime_timing_json, created_at, updated_at, started_at, finished_at${admissionScopeSelection}
       FROM ${tableName('runs', options)}
       WHERE run_id = $1
     `,
@@ -16990,6 +17083,10 @@ export async function readPostgresSchedulerRunStatus(
   return {
     runId: row.run_id,
     orgId: row.org_id ?? '',
+    ...(options?.includeAdmissionScope &&
+    (row.definition_scope === 'org' || row.definition_scope === 'system')
+      ? { definitionScope: row.definition_scope }
+      : {}),
     status: row.status,
     playName: row.play_name,
     graphHash: row.graph_hash,
@@ -18810,9 +18907,14 @@ type RunLedgerOutboxDrainScope = {
   leaseOwner: string;
   targetEventId: string;
   nextClaimed: PostgresSchedulerOutboxEvent[];
+};
+
+type RunLedgerOutboxAuthorityScope = {
   authority: {
+    runId: string;
+    leaseOwner: string;
     runtimeApi: WorkerRuntimeApiContext;
-    expiresAtMs: number;
+    expiresAtMs: number | null;
   } | null;
 };
 
@@ -18826,6 +18928,7 @@ async function dispatchRunLedgerOutboxThroughEventWithScope(
   },
   options?: Pick<PostgresSchedulerOptions, 'schema'>,
   scope?: RunLedgerOutboxDrainScope,
+  authorityScope: RunLedgerOutboxAuthorityScope = { authority: null },
 ): Promise<PostgresSchedulerOutboxDrainResult> {
   const runId = input.runId?.trim();
   if (!runId) {
@@ -18841,6 +18944,11 @@ async function dispatchRunLedgerOutboxThroughEventWithScope(
     input.maxEvents === undefined
       ? null
       : Math.max(1, Math.min(32, input.maxEvents));
+  const leaseOwner =
+    input.leaseOwner?.trim() ||
+    scope?.leaseOwner ||
+    `run-ledger-outbox-dispatch:${randomUUID()}`;
+  const scopedInput = { ...input, leaseOwner };
   const batchSize = RUN_LEDGER_PROJECTION_PREFIX_MAX_FACTS;
   const eventIds: string[] = [];
   const failedEventIds: string[] = [];
@@ -18855,13 +18963,14 @@ async function dispatchRunLedgerOutboxThroughEventWithScope(
     const result = await dispatchRunLedgerOutboxBatchWithScope(
       withClient,
       {
-        ...input,
+        ...scopedInput,
         waitForRunLock: true,
         continueThroughTerminal: false,
         limit: remaining === null ? batchSize : Math.min(batchSize, remaining),
       },
       options,
       scope,
+      authorityScope,
     );
     attempted += result.attempted;
     projected += result.projected;
@@ -18940,7 +19049,13 @@ export async function dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope(
   input: RunLedgerOutboxDispatchInput & { waitForRunLock?: boolean },
   options?: Pick<PostgresSchedulerOptions, 'schema'>,
 ): Promise<PostgresSchedulerOutboxProjectionResult> {
-  return dispatchRunLedgerOutboxBatchWithScope(withClient, input, options);
+  return dispatchRunLedgerOutboxBatchWithScope(
+    withClient,
+    input,
+    options,
+    undefined,
+    { authority: null },
+  );
 }
 
 async function dispatchRunLedgerOutboxBatchWithScope(
@@ -18950,6 +19065,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
   input: RunLedgerOutboxDispatchInput & { waitForRunLock?: boolean },
   options?: Pick<PostgresSchedulerOptions, 'schema'>,
   scope?: RunLedgerOutboxDrainScope,
+  authorityScope: RunLedgerOutboxAuthorityScope = { authority: null },
 ): Promise<PostgresSchedulerOutboxProjectionResult> {
   const dispatchStartedAtMs = Date.now();
   const leaseOwner =
@@ -19126,23 +19242,45 @@ async function dispatchRunLedgerOutboxBatchWithScope(
     ) {
       throw new Error('Run-ledger continuation is outside its dispatch scope.');
     }
-    if (scope?.authority && scope.authority.expiresAtMs <= Date.now()) {
-      throw new Error('Run-ledger continuation authority expired.');
-    }
+    // Minted executor authority is reused only inside this finite run/owner
+    // drain and only while it can cover the HTTP request plus its SQL renewal.
+    const reuseMarginMs =
+      Math.max(
+        1,
+        Math.floor(
+          input.runtimeApi?.requestTimeoutMs ??
+            RUNTIME_CAPACITY_POLICY.receiptGateway.requestTimeoutMs,
+        ),
+      ) + RUNTIME_RELIABILITY_POLICY.schedulerDatabase.queryTimeoutMs;
+    const cachedAuthority = authorityScope.authority;
     const gatewayRuntimeApi =
       tokenAuthority === 'gateway'
-        ? (scope?.authority?.runtimeApi ??
-          (await loadPostgresSchedulerRuntimeApiForRun(
-            withClient,
-            first.runId,
-            first.leaseOwner
-              ? { eventId: first.eventId, leaseOwner: first.leaseOwner }
-              : null,
-            tokenAuthority,
-            options,
-            input.runtimeApi?.signal,
-          )))
+        ? cachedAuthority?.runId === first.runId &&
+          cachedAuthority.leaseOwner === leaseOwner &&
+          cachedAuthority.expiresAtMs !== null &&
+          cachedAuthority.expiresAtMs > Date.now() + reuseMarginMs
+          ? cachedAuthority.runtimeApi
+          : await loadPostgresSchedulerRuntimeApiForRun(
+              withClient,
+              first.runId,
+              first.leaseOwner
+                ? { eventId: first.eventId, leaseOwner: first.leaseOwner }
+                : null,
+              tokenAuthority,
+              options,
+              input.runtimeApi?.signal,
+            )
         : null;
+    if (
+      gatewayRuntimeApi &&
+      cachedAuthority?.runId === first.runId &&
+      cachedAuthority.leaseOwner === leaseOwner &&
+      cachedAuthority.runtimeApi.baseUrl !== gatewayRuntimeApi.baseUrl
+    ) {
+      throw new Error(
+        'Run Ledger dispatch origin changed within an immutable launch.',
+      );
+    }
     const resolvedRuntimeApi =
       gatewayRuntimeApi ??
       input.runtimeApi ??
@@ -19186,12 +19324,48 @@ async function dispatchRunLedgerOutboxBatchWithScope(
     const tokenExpiresAtMs = runtimeApiTokenExpiresAtMs(
       runtimeApi.executorToken,
     );
-    if (scope && gatewayRuntimeApi && tokenExpiresAtMs !== null) {
-      scope.authority = { runtimeApi, expiresAtMs: tokenExpiresAtMs };
+    if (tokenAuthority === 'gateway' && gatewayRuntimeApi) {
+      authorityScope.authority = {
+        runId: first.runId,
+        leaseOwner,
+        runtimeApi,
+        expiresAtMs: tokenExpiresAtMs,
+      };
     }
     authorityReadyAtMs = Date.now();
-    const renewDelivery = () =>
-      withClient((client) =>
+    const renewDelivery = async () => {
+      const currentAuthority = authorityScope.authority;
+      if (
+        tokenAuthority === 'gateway' &&
+        currentAuthority?.runId === first.runId &&
+        currentAuthority.leaseOwner === leaseOwner &&
+        currentAuthority.expiresAtMs !== null &&
+        currentAuthority.expiresAtMs <= Date.now() + reuseMarginMs
+      ) {
+        const refreshed = await loadPostgresSchedulerRuntimeApiForRun(
+          withClient,
+          first.runId,
+          { eventId: first.eventId, leaseOwner },
+          tokenAuthority,
+          options,
+          runtimeApi.signal,
+        );
+        if (refreshed.baseUrl !== runtimeApi.baseUrl) {
+          throw new Error(
+            'Run Ledger dispatch origin changed within an immutable launch.',
+          );
+        }
+        runtimeApi.executorToken = refreshed.executorToken;
+        runtimeApi.vercelProtectionBypassToken =
+          refreshed.vercelProtectionBypassToken;
+        authorityScope.authority = {
+          runId: first.runId,
+          leaseOwner,
+          runtimeApi,
+          expiresAtMs: runtimeApiTokenExpiresAtMs(refreshed.executorToken),
+        };
+      }
+      await withClient((client) =>
         renewPostgresRunLedgerDelivery(
           client,
           {
@@ -19202,6 +19376,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
           options,
         ),
       );
+    };
     // A pending start is always claimed alone because later events require its
     // snapshot. It cannot share the append endpoint, so retain that mutation
     // as its own exact delivery.
@@ -19299,6 +19474,7 @@ async function dispatchRunLedgerOutboxBatchWithScope(
       withClient,
       { input, delivered, scope: acknowledgement.continuation },
       options,
+      authorityScope,
     );
   } catch (error) {
     await releasePendingContinuation();
@@ -19404,6 +19580,7 @@ async function continueRunLedgerOutboxThroughTerminal(
     scope: RunLedgerOutboxDrainScope;
   },
   options?: Pick<PostgresSchedulerOptions, 'schema'>,
+  authorityScope: RunLedgerOutboxAuthorityScope = { authority: null },
 ): Promise<PostgresSchedulerOutboxProjectionResult> {
   const drained = await dispatchRunLedgerOutboxThroughEventWithScope(
     withClient,
@@ -19415,6 +19592,7 @@ async function continueRunLedgerOutboxThroughTerminal(
     },
     options,
     input.scope,
+    authorityScope,
   );
   return {
     attempted: input.delivered.attempted + drained.attempted,
@@ -19430,7 +19608,10 @@ async function continueRunLedgerOutboxThroughTerminal(
 function runtimeApiTokenExpiresAtMs(token: string): number | null {
   try {
     const expiresAt = decodeJwt(token).exp;
-    return typeof expiresAt === 'number' && Number.isFinite(expiresAt)
+    return typeof expiresAt === 'number' &&
+      Number.isSafeInteger(expiresAt) &&
+      expiresAt > 0 &&
+      Number.isSafeInteger(expiresAt * 1_000)
       ? expiresAt * 1_000
       : null;
   } catch {
@@ -19514,18 +19695,11 @@ async function acknowledgeRunLedgerPrefixWithContinuation(
             options,
           );
           if (nextClaimed.length > 0) {
-            const expiresAtMs = runtimeApiTokenExpiresAtMs(
-              input.runtimeApi.executorToken,
-            );
             continuation = input.scope ?? {
               runId,
               leaseOwner: input.leaseOwner,
               targetEventId,
               nextClaimed: [],
-              authority:
-                input.input.tokenAuthority === 'gateway' && expiresAtMs !== null
-                  ? { runtimeApi: input.runtimeApi, expiresAtMs }
-                  : null,
             };
             continuation.nextClaimed = nextClaimed;
             input.continuation.current = continuation;
@@ -20127,6 +20301,9 @@ function parsePostgresSchedulerCronSourcePin(
   return {
     ...pin,
     orgId: pin.orgId,
+    ...(pin.definitionScope === 'org' || pin.definitionScope === 'system'
+      ? { definitionScope: pin.definitionScope }
+      : {}),
     userId: typeof pin.userId === 'string' ? pin.userId : null,
     playName: pin.playName,
     revisionId: typeof pin.revisionId === 'string' ? pin.revisionId : null,

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-sandbox-capacity.ts

@@ -539,12 +539,19 @@ export async function reserveRuntimeSandboxCapacityLease(
     provider: RuntimeSandboxProviderId;
     operation_attempt: number | null;
     operation_environment: 'preview' | 'production' | null;
+    operation_provider: RuntimeSandboxProviderId | null;
     provider_operation_token: string | null;
+    operation_state: RuntimeSandboxOperationState | null;
+    released_at: Date | null;
+    sandbox_id: string | null;
   }>(
     `SELECT lease.org_id, lease.run_id, lease.provider,
             operation.attempt AS operation_attempt,
             operation.environment AS operation_environment,
-            operation.provider_operation_token
+            operation.provider AS operation_provider,
+            operation.provider_operation_token,
+            operation.state AS operation_state,
+            lease.released_at, lease.sandbox_id
        FROM ${leases} AS lease
        LEFT JOIN ${operations} AS operation
          ON operation.operation_key = lease.lease_id
@@ -559,12 +566,23 @@ export async function reserveRuntimeSandboxCapacityLease(
     row.provider !== input.provider ||
     (input.attempt !== undefined &&
       (row.operation_attempt !== input.attempt ||
-        row.operation_environment !== (input.environment ?? 'production')))
+        row.operation_environment !== (input.environment ?? 'production') ||
+        row.operation_provider !== input.provider))
   ) {
     throw new Error(
       'operationKey reservation did not persist the requested sandbox operation.',
     );
   }
+  if (
+    input.attempt !== undefined &&
+    (row.operation_state !== RuntimeSandboxOperationState.Reserved ||
+      row.released_at !== null ||
+      row.sandbox_id !== null)
+  ) {
+    throw new RuntimeSandboxOperationNotRetryableError(
+      row.operation_state ?? RuntimeSandboxOperationState.Abandoned,
+    );
+  }
   const resolvedProviderOperationToken =
     row.provider_operation_token === null && input.attempt !== undefined
       ? await backfillReservedProviderOperationToken(

package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts

@@ -124,13 +124,24 @@ function parseRuntimeTestFaultHeader(
     const count =
       rawCount === undefined || rawCount.trim() === ''
         ? 1
-        : Number.parseInt(rawCount.trim(), 10);
+        : name === 'execution_upstream_response_loss'
+          ? Number(rawCount.trim())
+          : Number.parseInt(rawCount.trim(), 10);
     if (!Number.isInteger(count) || count < 1) {
       return {
         error: `Runtime test fault "${name}" count must be a positive integer.`,
       };
     }
-    faults.set(name, (faults.get(name) ?? 0) + count);
+    const total = (faults.get(name) ?? 0) + count;
+    // This seam deliberately interrupts only the first buffered response of
+    // each invocation. Repeated losses would test retry exhaustion instead of
+    // the durable-result recovery contract.
+    if (name === 'execution_upstream_response_loss' && total !== 1) {
+      return {
+        error: `Runtime test fault "${name}" count must be 1.`,
+      };
+    }
+    faults.set(name, total);
   }
   return faults;
 }