deepline 0.3.17 → 0.3.18

package/dist/bundling-sources/shared_libs/play-runtime/runtime-reliability-policy.ts

@@ -0,0 +1,391 @@
+/**
+ * Runtime availability policy shared by the scheduler, Fly workers, receipt
+ * gateway, and sandbox runner. These are platform controls, not Play options:
+ * a Play author must never be able to make an unsafe side effect retry longer
+ * or more often.
+ *
+ * Defaults are deliberately compiled into the runtime so a missing deploy
+ * variable is safe. A changed value is accepted only when an incident operator
+ * also sets `DEEPLINE_RUNTIME_RELIABILITY_SEV_OVERRIDE=enabled`. This makes an
+ * emergency tuning change explicit, auditable in deployment configuration, and
+ * impossible to smuggle through an ordinary per-run request.
+ */
+
+export const RUNTIME_RELIABILITY_SEV_OVERRIDE_ENV =
+  'DEEPLINE_RUNTIME_RELIABILITY_SEV_OVERRIDE';
+
+export const RUNTIME_RELIABILITY_ENV = {
+  engineMaxAttempts: 'DEEPLINE_RUNTIME_ENGINE_MAX_ATTEMPTS',
+  engineRetryBaseSeconds: 'DEEPLINE_RUNTIME_ENGINE_RETRY_BASE_SECONDS',
+  engineRetryMaxSeconds: 'DEEPLINE_RUNTIME_ENGINE_RETRY_MAX_SECONDS',
+  schedulerSubmitCapacityMaxAttempts:
+    'DEEPLINE_RUNTIME_SCHEDULER_SUBMIT_CAPACITY_MAX_ATTEMPTS',
+  claimLeaseSeconds: 'DEEPLINE_RUNTIME_CLAIM_LEASE_SECONDS',
+  schedulerDbAcquireTimeoutMs:
+    'DEEPLINE_RUNTIME_SCHEDULER_DB_ACQUIRE_TIMEOUT_MS',
+  schedulerDbQueryTimeoutMs: 'DEEPLINE_RUNTIME_SCHEDULER_DB_QUERY_TIMEOUT_MS',
+  schedulerDbRetryAttempts: 'DEEPLINE_RUNTIME_SCHEDULER_DB_RETRY_ATTEMPTS',
+  schedulerDbRetryBaseDelayMs:
+    'DEEPLINE_RUNTIME_SCHEDULER_DB_RETRY_BASE_DELAY_MS',
+  schedulerDbRetryMaxDelayMs:
+    'DEEPLINE_RUNTIME_SCHEDULER_DB_RETRY_MAX_DELAY_MS',
+  schedulerDbCircuitOpenMs: 'DEEPLINE_RUNTIME_SCHEDULER_DB_CIRCUIT_OPEN_MS',
+  recoveryMaxDefers: 'DEEPLINE_RUNTIME_RECOVERY_MAX_DEFERS',
+  recoveryHorizonMs: 'DEEPLINE_RUNTIME_RECOVERY_HORIZON_MS',
+  recoveryDeferBaseSeconds: 'DEEPLINE_RUNTIME_RECOVERY_DEFER_BASE_SECONDS',
+  recoveryDeferMaxSeconds: 'DEEPLINE_RUNTIME_RECOVERY_DEFER_MAX_SECONDS',
+  recoveryCircuitOpenMs: 'DEEPLINE_RUNTIME_RECOVERY_CIRCUIT_OPEN_MS',
+  recoveryCircuitProbeLeaseMs:
+    'DEEPLINE_RUNTIME_RECOVERY_CIRCUIT_PROBE_LEASE_MS',
+  sandboxRunnerReadyTimeoutMs:
+    'DEEPLINE_RUNTIME_SANDBOX_RUNNER_READY_TIMEOUT_MS',
+  modalSandboxCreateTimeoutMs:
+    'DEEPLINE_RUNTIME_MODAL_SANDBOX_CREATE_TIMEOUT_MS',
+  daytonaCommandRecoveryTimeoutMs:
+    'DEEPLINE_RUNTIME_DAYTONA_COMMAND_RECOVERY_TIMEOUT_MS',
+  daytonaCommandRecoveryPollMs:
+    'DEEPLINE_RUNTIME_DAYTONA_COMMAND_RECOVERY_POLL_MS',
+  runnerControlRequestTimeoutMs:
+    'DEEPLINE_RUNTIME_RUNNER_CONTROL_REQUEST_TIMEOUT_MS',
+  runnerControlMaxAttempts: 'DEEPLINE_RUNTIME_RUNNER_CONTROL_MAX_ATTEMPTS',
+  terminalPushRequestTimeoutMs:
+    'DEEPLINE_RUNTIME_TERMINAL_PUSH_REQUEST_TIMEOUT_MS',
+  terminalPushMaxAttempts: 'DEEPLINE_RUNTIME_TERMINAL_PUSH_MAX_ATTEMPTS',
+  egressFetchMaxAttempts: 'DEEPLINE_RUNTIME_EGRESS_FETCH_MAX_ATTEMPTS',
+  egressFetchHeadersTimeoutMs:
+    'DEEPLINE_RUNTIME_EGRESS_FETCH_HEADERS_TIMEOUT_MS',
+  egressFetchBodyTimeoutMs: 'DEEPLINE_RUNTIME_EGRESS_FETCH_BODY_TIMEOUT_MS',
+  egressFetchTotalTimeoutMs: 'DEEPLINE_RUNTIME_EGRESS_FETCH_TOTAL_TIMEOUT_MS',
+} as const;
+
+export type RuntimeReliabilityPolicy = Readonly<{
+  sevOverrideEnabled: boolean;
+  engine: Readonly<{
+    maxAttempts: number;
+    retryBaseSeconds: number;
+    retryMaxSeconds: number;
+  }>;
+  schedulerSubmit: Readonly<{ capacityMaxAttempts: number }>;
+  worker: Readonly<{ claimLeaseSeconds: number }>;
+  schedulerDatabase: Readonly<{
+    acquireTimeoutMs: number;
+    queryTimeoutMs: number;
+    retryAttempts: number;
+    retryBaseDelayMs: number;
+    retryMaxDelayMs: number;
+    circuitOpenMs: number;
+  }>;
+  recovery: Readonly<{
+    maxDefers: number;
+    horizonMs: number;
+    deferBaseSeconds: number;
+    deferMaxSeconds: number;
+    circuitOpenMs: number;
+    circuitProbeLeaseMs: number;
+  }>;
+  sandbox: Readonly<{
+    modalSandboxCreateTimeoutMs: number;
+    runnerReadyTimeoutMs: number;
+    daytonaCommandRecoveryTimeoutMs: number;
+    daytonaCommandRecoveryPollMs: number;
+  }>;
+  gateway: Readonly<{
+    runnerControlRequestTimeoutMs: number;
+    runnerControlMaxAttempts: number;
+    terminalPushRequestTimeoutMs: number;
+    terminalPushMaxAttempts: number;
+  }>;
+  egress: Readonly<{
+    fetchMaxAttempts: number;
+    fetchHeadersTimeoutMs: number;
+    fetchBodyTimeoutMs: number;
+    fetchTotalTimeoutMs: number;
+  }>;
+}>;
+
+type Env = Readonly<Record<string, string | undefined>>;
+
+type NumberSetting = Readonly<{
+  name: string;
+  defaultValue: number;
+  min: number;
+  max: number;
+}>;
+
+function readProcessEnvironment(): Env {
+  return typeof process === 'undefined' ? {} : (process.env ?? {});
+}
+
+function enabled(raw: string | undefined): boolean {
+  return raw?.trim().toLowerCase() === 'enabled';
+}
+
+function resolveSetting(
+  env: Env,
+  sevOverrideEnabled: boolean,
+  setting: NumberSetting,
+): number {
+  const raw = env[setting.name]?.trim();
+  if (!raw) return setting.defaultValue;
+  if (!/^\d+$/.test(raw)) {
+    throw new Error(
+      `${setting.name} must be an integer between ${setting.min} and ${setting.max}.`,
+    );
+  }
+  const value = Number(raw);
+  if (
+    !Number.isSafeInteger(value) ||
+    value < setting.min ||
+    value > setting.max
+  ) {
+    throw new Error(
+      `${setting.name} must be an integer between ${setting.min} and ${setting.max}.`,
+    );
+  }
+  if (value !== setting.defaultValue && !sevOverrideEnabled) {
+    throw new Error(
+      `${setting.name} differs from its safe platform default. Set ${RUNTIME_RELIABILITY_SEV_OVERRIDE_ENV}=enabled only for an active SEV.`,
+    );
+  }
+  return value;
+}
+
+function setting(
+  name: string,
+  defaultValue: number,
+  min: number,
+  max: number,
+): NumberSetting {
+  return { name, defaultValue, min, max };
+}
+
+/** Pure resolver so tests can prove the deployment guard without mutating env. */
+export function resolveRuntimeReliabilityPolicy(
+  env: Env = readProcessEnvironment(),
+): RuntimeReliabilityPolicy {
+  const sevOverrideEnabled = enabled(env[RUNTIME_RELIABILITY_SEV_OVERRIDE_ENV]);
+  const read = (name: string, defaultValue: number, min: number, max: number) =>
+    resolveSetting(
+      env,
+      sevOverrideEnabled,
+      setting(name, defaultValue, min, max),
+    );
+
+  const policy: RuntimeReliabilityPolicy = {
+    sevOverrideEnabled,
+    engine: {
+      maxAttempts: read(RUNTIME_RELIABILITY_ENV.engineMaxAttempts, 5, 1, 8),
+      retryBaseSeconds: read(
+        RUNTIME_RELIABILITY_ENV.engineRetryBaseSeconds,
+        5,
+        1,
+        60,
+      ),
+      retryMaxSeconds: read(
+        RUNTIME_RELIABILITY_ENV.engineRetryMaxSeconds,
+        60,
+        5,
+        300,
+      ),
+    },
+    schedulerSubmit: {
+      capacityMaxAttempts: read(
+        RUNTIME_RELIABILITY_ENV.schedulerSubmitCapacityMaxAttempts,
+        8,
+        1,
+        12,
+      ),
+    },
+    worker: {
+      claimLeaseSeconds: read(
+        RUNTIME_RELIABILITY_ENV.claimLeaseSeconds,
+        120,
+        30,
+        600,
+      ),
+    },
+    schedulerDatabase: {
+      acquireTimeoutMs: read(
+        RUNTIME_RELIABILITY_ENV.schedulerDbAcquireTimeoutMs,
+        5_000,
+        1_000,
+        30_000,
+      ),
+      queryTimeoutMs: read(
+        RUNTIME_RELIABILITY_ENV.schedulerDbQueryTimeoutMs,
+        30_000,
+        5_000,
+        120_000,
+      ),
+      retryAttempts: read(
+        RUNTIME_RELIABILITY_ENV.schedulerDbRetryAttempts,
+        3,
+        1,
+        6,
+      ),
+      retryBaseDelayMs: read(
+        RUNTIME_RELIABILITY_ENV.schedulerDbRetryBaseDelayMs,
+        50,
+        10,
+        5_000,
+      ),
+      retryMaxDelayMs: read(
+        RUNTIME_RELIABILITY_ENV.schedulerDbRetryMaxDelayMs,
+        1_000,
+        50,
+        30_000,
+      ),
+      circuitOpenMs: read(
+        RUNTIME_RELIABILITY_ENV.schedulerDbCircuitOpenMs,
+        2_000,
+        500,
+        60_000,
+      ),
+    },
+    recovery: {
+      maxDefers: read(RUNTIME_RELIABILITY_ENV.recoveryMaxDefers, 8, 1, 12),
+      horizonMs: read(
+        RUNTIME_RELIABILITY_ENV.recoveryHorizonMs,
+        30 * 60_000,
+        60_000,
+        2 * 60 * 60_000,
+      ),
+      deferBaseSeconds: read(
+        RUNTIME_RELIABILITY_ENV.recoveryDeferBaseSeconds,
+        15,
+        1,
+        300,
+      ),
+      deferMaxSeconds: read(
+        RUNTIME_RELIABILITY_ENV.recoveryDeferMaxSeconds,
+        300,
+        15,
+        900,
+      ),
+      circuitOpenMs: read(
+        RUNTIME_RELIABILITY_ENV.recoveryCircuitOpenMs,
+        60_000,
+        5_000,
+        5 * 60_000,
+      ),
+      circuitProbeLeaseMs: read(
+        RUNTIME_RELIABILITY_ENV.recoveryCircuitProbeLeaseMs,
+        60_000,
+        5_000,
+        5 * 60_000,
+      ),
+    },
+    sandbox: {
+      modalSandboxCreateTimeoutMs: read(
+        RUNTIME_RELIABILITY_ENV.modalSandboxCreateTimeoutMs,
+        60_000,
+        5_000,
+        5 * 60_000,
+      ),
+      runnerReadyTimeoutMs: read(
+        RUNTIME_RELIABILITY_ENV.sandboxRunnerReadyTimeoutMs,
+        30_000,
+        5_000,
+        5 * 60_000,
+      ),
+      daytonaCommandRecoveryTimeoutMs: read(
+        RUNTIME_RELIABILITY_ENV.daytonaCommandRecoveryTimeoutMs,
+        60_000,
+        5_000,
+        5 * 60_000,
+      ),
+      daytonaCommandRecoveryPollMs: read(
+        RUNTIME_RELIABILITY_ENV.daytonaCommandRecoveryPollMs,
+        2_000,
+        100,
+        30_000,
+      ),
+    },
+    gateway: {
+      runnerControlRequestTimeoutMs: read(
+        RUNTIME_RELIABILITY_ENV.runnerControlRequestTimeoutMs,
+        20_000,
+        1_000,
+        120_000,
+      ),
+      runnerControlMaxAttempts: read(
+        RUNTIME_RELIABILITY_ENV.runnerControlMaxAttempts,
+        3,
+        1,
+        6,
+      ),
+      terminalPushRequestTimeoutMs: read(
+        RUNTIME_RELIABILITY_ENV.terminalPushRequestTimeoutMs,
+        15_000,
+        1_000,
+        120_000,
+      ),
+      terminalPushMaxAttempts: read(
+        RUNTIME_RELIABILITY_ENV.terminalPushMaxAttempts,
+        5,
+        1,
+        8,
+      ),
+    },
+    egress: {
+      fetchMaxAttempts: read(
+        RUNTIME_RELIABILITY_ENV.egressFetchMaxAttempts,
+        3,
+        1,
+        5,
+      ),
+      fetchHeadersTimeoutMs: read(
+        RUNTIME_RELIABILITY_ENV.egressFetchHeadersTimeoutMs,
+        30_000,
+        1_000,
+        5 * 60_000,
+      ),
+      fetchBodyTimeoutMs: read(
+        RUNTIME_RELIABILITY_ENV.egressFetchBodyTimeoutMs,
+        30_000,
+        1_000,
+        5 * 60_000,
+      ),
+      fetchTotalTimeoutMs: read(
+        RUNTIME_RELIABILITY_ENV.egressFetchTotalTimeoutMs,
+        120_000,
+        5_000,
+        15 * 60_000,
+      ),
+    },
+  };
+
+  if (policy.engine.retryMaxSeconds < policy.engine.retryBaseSeconds) {
+    throw new Error('Engine retry maximum must be at least the retry base.');
+  }
+  if (
+    policy.schedulerDatabase.retryMaxDelayMs <
+    policy.schedulerDatabase.retryBaseDelayMs
+  ) {
+    throw new Error(
+      'Scheduler DB retry maximum must be at least the retry base.',
+    );
+  }
+  if (policy.recovery.deferMaxSeconds < policy.recovery.deferBaseSeconds) {
+    throw new Error('Recovery defer maximum must be at least the defer base.');
+  }
+  if (
+    policy.recovery.circuitProbeLeaseMs <
+    policy.sandbox.modalSandboxCreateTimeoutMs
+  ) {
+    throw new Error(
+      'Recovery circuit probe lease must cover the Modal sandbox create deadline.',
+    );
+  }
+  if (
+    policy.egress.fetchTotalTimeoutMs < policy.egress.fetchHeadersTimeoutMs ||
+    policy.egress.fetchTotalTimeoutMs < policy.egress.fetchBodyTimeoutMs
+  ) {
+    throw new Error(
+      'Egress total timeout must cover both headers and body deadlines.',
+    );
+  }
+  return policy;
+}
+
+export const RUNTIME_RELIABILITY_POLICY = resolveRuntimeReliabilityPolicy();

package/dist/bundling-sources/shared_libs/play-runtime/runtime-traffic-policy.ts

@@ -0,0 +1,125 @@
+/**
+ * Operator-owned limits for the shared Play Runtime traffic path.
+ *
+ * The compiled defaults are intentionally permissive and safe.  A live
+ * override is a short-lived incident control, stored in Convex and read by the
+ * worker before it can create a sandbox.  It can only reduce new work; it can
+ * never make retries, side effects, or provider limits less safe.
+ */
+
+export const RUNTIME_TRAFFIC_POLICY_SCOPE = 'play_runtime' as const;
+
+export const RUNTIME_TRAFFIC_POLICY_LIMITS = {
+  /** An operator must renew a mitigation rather than accidentally leave it on. */
+  maxOverrideDurationMs: 2 * 60 * 60_000,
+  /** A global start throttle may only shed load, never increase it. */
+  maxSandboxStartsPerMinute: 600,
+} as const;
+
+export type RuntimeTrafficAdmissionMode = 'normal' | 'hold_new_sandbox_starts';
+
+export type RuntimeTrafficPolicy = Readonly<{
+  scope: typeof RUNTIME_TRAFFIC_POLICY_SCOPE;
+  version: number;
+  admissionMode: RuntimeTrafficAdmissionMode;
+  /** Null means no extra global pacing beyond the compiled runtime policy. */
+  sandboxStartsPerMinute: number | null;
+  expiresAt: number | null;
+  reason: string | null;
+  updatedAt: number | null;
+}>;
+
+export const DEFAULT_RUNTIME_TRAFFIC_POLICY: RuntimeTrafficPolicy = {
+  scope: RUNTIME_TRAFFIC_POLICY_SCOPE,
+  version: 0,
+  admissionMode: 'normal',
+  sandboxStartsPerMinute: null,
+  expiresAt: null,
+  reason: null,
+  updatedAt: null,
+};
+
+/**
+ * The worker reads this value over an app-runtime boundary.  Treat a malformed
+ * successful response as an unavailable control plane, rather than allowing
+ * JavaScript's `undefined` coercions to accidentally turn it into a throttle.
+ */
+export function isRuntimeTrafficPolicy(
+  value: unknown,
+): value is RuntimeTrafficPolicy {
+  if (!value || typeof value !== 'object') return false;
+  const candidate = value as Record<string, unknown>;
+  const sandboxStartsPerMinute = candidate.sandboxStartsPerMinute;
+  return (
+    candidate.scope === RUNTIME_TRAFFIC_POLICY_SCOPE &&
+    typeof candidate.version === 'number' &&
+    Number.isSafeInteger(candidate.version) &&
+    candidate.version >= 0 &&
+    (candidate.admissionMode === 'normal' ||
+      candidate.admissionMode === 'hold_new_sandbox_starts') &&
+    (sandboxStartsPerMinute === null ||
+      (typeof sandboxStartsPerMinute === 'number' &&
+        Number.isSafeInteger(sandboxStartsPerMinute) &&
+        sandboxStartsPerMinute >= 1 &&
+        sandboxStartsPerMinute <=
+          RUNTIME_TRAFFIC_POLICY_LIMITS.maxSandboxStartsPerMinute)) &&
+    (candidate.expiresAt === null ||
+      (typeof candidate.expiresAt === 'number' &&
+        Number.isSafeInteger(candidate.expiresAt))) &&
+    (candidate.reason === null || typeof candidate.reason === 'string') &&
+    (candidate.updatedAt === null ||
+      (typeof candidate.updatedAt === 'number' &&
+        Number.isSafeInteger(candidate.updatedAt)))
+  );
+}
+
+export function runtimeTrafficPolicyIsActive(
+  policy: Pick<RuntimeTrafficPolicy, 'expiresAt'>,
+  nowMs = Date.now(),
+): boolean {
+  return policy.expiresAt === null || policy.expiresAt > nowMs;
+}
+
+/**
+ * Expired rows remain as audit evidence but must behave exactly like no
+ * override.  Keeping this pure makes the app, worker, and CLI agree without
+ * relying on a best-effort expiry job.
+ */
+export function effectiveRuntimeTrafficPolicy(
+  policy: RuntimeTrafficPolicy | null | undefined,
+  nowMs = Date.now(),
+): RuntimeTrafficPolicy {
+  if (!policy || !runtimeTrafficPolicyIsActive(policy, nowMs)) {
+    return DEFAULT_RUNTIME_TRAFFIC_POLICY;
+  }
+  return policy;
+}
+
+export function validateRuntimeTrafficOverride(input: {
+  admissionMode: RuntimeTrafficAdmissionMode;
+  sandboxStartsPerMinute: number | null;
+  expiresAt: number;
+  nowMs?: number;
+}): void {
+  const nowMs = input.nowMs ?? Date.now();
+  if (!Number.isSafeInteger(input.expiresAt) || input.expiresAt <= nowMs) {
+    throw new Error('Runtime traffic override expiry must be in the future.');
+  }
+  if (
+    input.expiresAt - nowMs >
+    RUNTIME_TRAFFIC_POLICY_LIMITS.maxOverrideDurationMs
+  ) {
+    throw new Error('Runtime traffic overrides may last at most two hours.');
+  }
+  if (
+    input.sandboxStartsPerMinute !== null &&
+    (!Number.isSafeInteger(input.sandboxStartsPerMinute) ||
+      input.sandboxStartsPerMinute < 1 ||
+      input.sandboxStartsPerMinute >
+        RUNTIME_TRAFFIC_POLICY_LIMITS.maxSandboxStartsPerMinute)
+  ) {
+    throw new Error(
+      `sandboxStartsPerMinute must be a whole number from 1 to ${RUNTIME_TRAFFIC_POLICY_LIMITS.maxSandboxStartsPerMinute}.`,
+    );
+  }
+}

package/dist/bundling-sources/shared_libs/play-runtime/sandbox-compute-usage.ts

@@ -86,6 +86,27 @@ export function resolveRuntimeSandboxComputeItems(input: {
   return items;
 }
 
+/**
+ * A provider create may finish after the original run has terminalized. Those
+ * resources cannot be appended to the run's already-finalized billing
+ * session, so the scheduler settles each through a stable adjustment session.
+ */
+export function resolveLateRuntimeSandboxComputeItems(input: {
+  resources: readonly unknown[];
+  endedAt: number;
+}): ComputeBillingItem[] {
+  return resolveRuntimeSandboxComputeItems({
+    resources: input.resources.filter(
+      (resource) =>
+        Boolean(resource) &&
+        typeof resource === 'object' &&
+        !Array.isArray(resource) &&
+        (resource as Record<string, unknown>).lateAcquired === true,
+    ),
+    endedAt: input.endedAt,
+  });
+}
+
 /** Project executor resource facts into one idempotent item per real sandbox. */
 export function resolveDaytonaRuntimeComputeItems(input: {
   resources: readonly unknown[];

package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts

@@ -1,37 +1,12 @@
 import { getRuntimeEnv } from '../runtime-env';
 import type { PlayRunnerEvent } from './protocol';
+import {
+  runtimeTestFaultDrills,
+  type RuntimeTestFaultName,
+} from './runtime-incident-drills';
 
 export const PLAY_RUNTIME_TEST_FAULT_HEADER = 'x-deepline-test-fault';
 
-export type RuntimeTestFaultName =
-  | 'receipt_complete_write_fail'
-  | 'receipt_fail_write_fail'
-  | 'worker_receipt_complete_write_fail'
-  | 'invocation_response_delivery_abort'
-  /**
-   * Holds an already checked-out receipt-gateway scheduler client for the
-   * supplied bounded millisecond value. Preview/CI only: this creates a
-   * deterministic real-Pg-pool pressure window without changing an action's
-   * SQL or outcome.
-   */
-  | 'receipt_gateway_hold_ms'
-  /**
-   * Synthetic-only bulk-claim fault. The runner forwards this token on the
-   * first qualifying physical claim request only. The gateway holds the real
-   * checked-out scheduler client for the supplied milliseconds, allowing the
-   * claim-operation deadline and connection destruction path to be exercised
-   * end to end without making the fault Machine-local.
-   */
-  | 'receipt_claim_query_hold_once_ms'
-  /**
-   * Synthetic-only response-boundary fault. The runner fully receives a
-   * successful bulk receipt-claim response, then reports the configured first
-   * N physical requests as request timeouts. This proves ambiguous-delivery
-   * replay without depending on which gateway Machine accepted the request.
-   */
-  | 'receipt_claim_response_timeout'
-  | 'runtime_sheet_page_tail_hold_ms';
-
 export type RuntimeTestFaultRegistry = {
   consume(name: RuntimeTestFaultName): boolean;
   remaining(): Record<string, number>;
@@ -86,16 +61,9 @@ type ValidatedRuntimeTestPolicyOverrides =
   | { ok: true; overrides: RuntimeTestPolicyOverrides | null }
   | { ok: false; status: 400 | 403; error: string };
 
-const SUPPORTED_RUNTIME_TEST_FAULTS = new Set<RuntimeTestFaultName>([
-  'receipt_complete_write_fail',
-  'receipt_fail_write_fail',
-  'worker_receipt_complete_write_fail',
-  'invocation_response_delivery_abort',
-  'receipt_gateway_hold_ms',
-  'receipt_claim_query_hold_once_ms',
-  'receipt_claim_response_timeout',
-  'runtime_sheet_page_tail_hold_ms',
-]);
+const SUPPORTED_RUNTIME_TEST_FAULTS = new Set<RuntimeTestFaultName>(
+  runtimeTestFaultDrills().map(({ id }) => id),
+);
 
 const RUNTIME_TEST_POLICY_MS_FIELDS = new Set([
   'receiptLeaseTtlMs',
@@ -181,6 +149,35 @@ export function recognizedRuntimeTestFaultCount(
   return parsed.get(name) ?? 0;
 }
 
+/**
+ * A provider-create fault cannot use process-local "already consumed" state:
+ * a deferred Play Run normally resumes on another worker. Instead, the
+ * persisted scheduler attempt is its consumption fence. `:1` means inject on
+ * generation one only; `:N` intentionally selects generation N for diagnosis.
+ *
+ * The header itself is admitted only through `parseRuntimeTestFaultCounts` at
+ * run admission. This second production guard ensures a persisted historical
+ * test header can never synthesize a provider failure in a live runtime.
+ */
+export function shouldInjectModalSandboxCreateResourceExhausted(input: {
+  runtimeTestFaultHeader: string | null | undefined;
+  runAttempt: number | null | undefined;
+}): boolean {
+  if (getRuntimeEnv() === 'prod') return false;
+  const requestedAttempt = recognizedRuntimeTestFaultCount(
+    input.runtimeTestFaultHeader,
+    'modal_sandbox_create_resource_exhausted',
+  );
+  if (requestedAttempt < 1) return false;
+  const runAttempt = input.runAttempt;
+  return (
+    typeof runAttempt === 'number' &&
+    Number.isSafeInteger(runAttempt) &&
+    runAttempt >= 1 &&
+    runAttempt === requestedAttempt
+  );
+}
+
 export function runtimeSheetPageTailRowsContainTarget(
   rows: readonly unknown[],
 ): boolean {