deepline 0.3.209 → 0.3.211

package/dist/bundling-sources/sdk/src/release.ts

@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: '0.3.209',
+  version: '0.3.211',
   updateSummary:
     'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
   packageCapabilities: {

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -122,6 +122,7 @@ import {
 import {
   AllAdmittedRowsFailedError,
   isProviderExhaustedFailure,
+  ProviderAdmissionWaitTimeoutError,
   ProviderExhaustedError,
 } from './run-failure';
 import { PROVIDER_EXHAUSTED_MAX_WAIT_MS } from './governor/app-runtime-rate-state-backend';
@@ -18682,6 +18683,30 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
           networkScope: null,
         },
       );
+    const providerAdmissionTimeoutError = (
+      error: ProviderAdmissionWaitTimeoutError,
+    ) =>
+      createToolHttpError(
+        toolErrorSchemaVersion,
+        error.message,
+        null,
+        503,
+        'repairable',
+        {
+          toolId,
+          provider: error.provider,
+          operation: toolId,
+          code: error.code,
+          origin: 'deepline',
+          category: 'rate_limit',
+          retryable: true,
+          statusCode: 503,
+          requestId: null,
+          retryAfterMs: null,
+          networkKind: null,
+          networkScope: null,
+        },
+      );
     const observeProviderWait = (retryAtMs: number) => {
       const startedAtMsByScope =
         (this.checkpoint.providerNoProgressStartedAtMsByScope ??= {});
@@ -19934,6 +19959,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
               if (heartbeatFailure) {
                 throw heartbeatFailure;
               }
+              if (error instanceof ProviderAdmissionWaitTimeoutError) {
+                throw providerAdmissionTimeoutError(error);
+              }
               if (error instanceof ProviderExhaustedError) {
                 const retryAtMs = Date.parse(error.retryAt);
                 observeProviderWait(

package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts

@@ -20,6 +20,7 @@ import {
   type RateStateBackendKind,
 } from './rate-state-backend';
 import {
+  ProviderAdmissionWaitTimeoutError,
   ProviderExhaustedError,
   providerNameFromBucketId,
 } from '../run-failure';
@@ -36,12 +37,18 @@ import type { PlayRunnerRateStateValidateResult } from '../protocol';
  */
 export const PROVIDER_EXHAUSTED_MAX_WAIT_MS = 300_000;
 /**
- * Total time one acquirer may spend cycling through window-exhausted refills.
- * Each refill's sleep is bounded, but nothing bounded how many of them a row
- * could take; with several runs sharing one org's provider window a late
- * starter parked here for twenty minutes with its core idle.
+ * Max total duration of a cached rate-state hold before it is yielded or its
+ * stale local no-progress warning is cleared. This is distinct from the shorter
+ * local permit-admission budget below.
  */
 export const RATE_ACQUIRE_MAX_WAIT_MS = 300_000;
+/**
+ * Total time one acquirer may spend waiting for a locally paced permit. This
+ * covers both empty-block refills and future-scheduled tickets. A spent budget
+ * is a repairable Deepline admission error, not another retry loop or a
+ * provider-owned Retry-After hold.
+ */
+export const PROVIDER_ADMISSION_WAIT_BUDGET_MS = 120_000;
 const PROVIDER_NO_PROGRESS_WARNING_MS = 180_000;
 
 /**
@@ -91,6 +98,31 @@ export const APP_RUNTIME_RATE_STATE_LEASE_BLOCK_SIZE = 8;
 const LEASE_BLOCK_MIN_TTL_MS = 250;
 const LEASE_BLOCK_MAX_TTL_MS = 5_000;
 const MAX_ACQUIRE_SLEEP_MS = 5_000;
+
+function abortReason(signal: AbortSignal): Error {
+  return signal.reason instanceof Error
+    ? signal.reason
+    : new Error('Rate-state acquire aborted.');
+}
+
+function sleepWithAbort(ms: number, signal?: AbortSignal): Promise<void> {
+  if (signal?.aborted) return Promise.reject(abortReason(signal));
+  return new Promise((resolve, reject) => {
+    const cleanup = () => signal?.removeEventListener('abort', onAbort);
+    const timer = setTimeout(() => {
+      cleanup();
+      resolve();
+    }, ms);
+    const onAbort = () => {
+      clearTimeout(timer);
+      cleanup();
+      reject(abortReason(signal!));
+    };
+    signal?.addEventListener('abort', onAbort, { once: true });
+    if (signal?.aborted) onAbort();
+  });
+}
+
 /**
  * Horizon used to size the next lease block from the rate actually in force.
  * Kept short so a block stays inside its own TTL and a stalled runner can never
@@ -156,7 +188,7 @@ interface PendingRelease {
 interface Options {
   schedulerSchema?: string | null;
   now?: () => number;
-  sleep?: (ms: number) => Promise<void>;
+  sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
   onStoreFailure?: (info: { bucketId: string; error: string }) => void;
   /** Override the transport (tests inject a counting/faulting port). */
   port?: AppRuntimeRatePort;
@@ -177,7 +209,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
   private readonly port: AppRuntimeRatePort;
   private readonly schedulerSchema: string | null;
   private readonly now: () => number;
-  private readonly sleep: (ms: number) => Promise<void>;
+  private readonly sleep: (ms: number, signal?: AbortSignal) => Promise<void>;
   private readonly jitter: () => number;
   private readonly onStoreFailure: (info: {
     bucketId: string;
@@ -238,9 +270,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     this.port = options.port ?? contextRatePort(context);
     this.schedulerSchema = options.schedulerSchema?.trim() || null;
     this.now = options.now ?? (() => Date.now());
-    this.sleep =
-      options.sleep ??
-      ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms)));
+    this.sleep = options.sleep ?? sleepWithAbort;
     this.jitter = options.jitter ?? Math.random;
     this.onStoreFailure =
       options.onStoreFailure ??
@@ -412,20 +442,47 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
       const localPermit = this.drawFromBlock(bucketId, laneKey);
       if (localPermit) {
         const waitMs = localPermit.scheduledAtMs - this.now();
-        if (waitMs > 0) await this.sleep(waitMs);
+        const elapsedMs = Math.max(0, this.now() - acquireStartedAt);
+        const remainingWaitMs = Math.max(
+          0,
+          PROVIDER_ADMISSION_WAIT_BUDGET_MS - elapsedMs,
+        );
+        if (
+          elapsedMs >= PROVIDER_ADMISSION_WAIT_BUDGET_MS ||
+          waitMs > remainingWaitMs
+        ) {
+          await this.cancelUndispatchedPermit(localPermit);
+          throw new ProviderAdmissionWaitTimeoutError({
+            provider: providerNameFromBucketId(bucketId),
+            waitBudgetMs: PROVIDER_ADMISSION_WAIT_BUDGET_MS,
+          });
+        }
+        try {
+          if (waitMs > 0) await this.sleep(waitMs, signal);
+        } catch (error) {
+          await this.cancelUndispatchedPermit(localPermit);
+          throw error;
+        }
         if (signal?.aborted) {
-          localPermit.permit.release();
-          throw signal.reason instanceof Error
-            ? signal.reason
-            : new Error('Rate-state acquire aborted.');
+          await this.cancelUndispatchedPermit(localPermit);
+          throw abortReason(signal);
+        }
+        if (
+          this.now() - acquireStartedAt >= PROVIDER_ADMISSION_WAIT_BUDGET_MS
+        ) {
+          await this.cancelUndispatchedPermit(localPermit);
+          throw new ProviderAdmissionWaitTimeoutError({
+            provider: providerNameFromBucketId(bucketId),
+            waitBudgetMs: PROVIDER_ADMISSION_WAIT_BUDGET_MS,
+          });
         }
         return localPermit.permit;
       }
       const waitedMs = this.now() - acquireStartedAt;
-      if (waitedMs >= RATE_ACQUIRE_MAX_WAIT_MS) {
-        throw new ProviderExhaustedError({
+      if (waitedMs >= PROVIDER_ADMISSION_WAIT_BUDGET_MS) {
+        throw new ProviderAdmissionWaitTimeoutError({
           provider: providerNameFromBucketId(bucketId),
-          retryAtMs: this.now() + MAX_ACQUIRE_SLEEP_MS,
+          waitBudgetMs: PROVIDER_ADMISSION_WAIT_BUDGET_MS,
         });
       }
       // Count this acquirer as waiting on the refill so the single-flight refill
@@ -1389,7 +1446,11 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
   private drawFromBlock(
     bucketId: string,
     laneKey: string,
-  ): { permit: PacingPermit; scheduledAtMs: number } | null {
+  ): {
+    permit: PacingPermit;
+    scheduledAtMs: number;
+    reservationId: string | null;
+  } | null {
     const block = this.liveBlock(bucketId, laneKey);
     if (!block) return null;
     // An emptied block stays until it expires: the next refill merges into it,
@@ -1399,9 +1460,24 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
     return {
       permit: this.permitFor(bucketId, laneKey, block, scheduled),
       scheduledAtMs: scheduled[0],
+      reservationId: scheduled[2],
     };
   }
 
+  private async cancelUndispatchedPermit(input: {
+    permit: PacingPermit;
+    reservationId: string | null;
+  }): Promise<void> {
+    if (input.reservationId) {
+      await input.permit.cancel();
+      return;
+    }
+    // Older gateways may grant a permit without a refundable reservation id.
+    // Still release any concurrency lease; losing one rate token is safer than
+    // manufacturing an immediate permit during a rolling deploy.
+    input.permit.release();
+  }
+
   private drainBlocksForBucket(
     bucketId: string,
     beforeRetryAtMs?: number,
@@ -1546,7 +1622,7 @@ function withAdaptiveMaxRps(rules: readonly PacingRule[]): PacingRule[] {
     // This one shared default rule is synthesized for unknown providers whose
     // credential bucket spans operations. Keep the generic policy unchanged
     // for local/coordinator backends, but let Postgres start a new provider
-    // scope with one probe/sec before learning its sustainable rate.
+    // scope at 10 RPS before learning its sustainable rate.
     const seed =
       rule.ruleId === 'default:provider-scope'
         ? {

package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts

@@ -511,9 +511,9 @@ export function createPlayExecutionGovernor(
       if (!declaredPacing) {
         // bucketId is the authenticated quota scope. Unknown tools sharing a
         // credential must not each mint an independent copy of the default rate.
-        // Start its one shared lane at one probe/sec; the DB-owned AIMD quickly
-        // ramps on sustained success and reacts to Retry-After if the provider
-        // tells us the quota is lower.
+        // Start its one shared lane at 10 RPS, matching the generic fallback.
+        // The DB-owned AIMD ramps on sustained success and reacts to
+        // Retry-After if the provider tells us the quota is lower.
         pacing.rules = pacing.rules.map((rule) => ({
           ...rule,
           ruleId: 'default:provider-scope',

package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts

@@ -189,7 +189,7 @@ export const SHARED_EXECUTION_POLICY: ResolvedExecutionPolicy = {
   },
   pacing: {
     // Keep the generic fallback rate. The shared Postgres backend separately
-    // cold-starts unknown provider scopes at one probe/sec.
+    // cold-starts unknown provider scopes at 10 RPS.
     // Declared providers (rate-limit-definitions.ts) keep their own rates.
     defaultProviderRequestsPerSecond: 10,
     // This is the scheduler's resident-call envelope, not an RPS target.

package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts

@@ -15,7 +15,7 @@
 export const DEFAULT_ADVISORY_PROVIDER_MAX_RPS = 100;
 
 /** Cold-start rate for an unknown provider bucket shared across a fleet key. */
-export const DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS = 1;
+export const DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS = 10;
 
 /** AIMD state cooldown cap; provider Retry-After holds remain uncapped. */
 export const PROVIDER_AIMD_COOLDOWN_CAP_MS = 45 * 60 * 1_000;

package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts

@@ -240,6 +240,27 @@ export function providerExhaustedMessage(input: {
   );
 }
 
+/**
+ * The local rate governor could not offer a provider permit inside its bounded
+ * admission window. No provider request was dispatched, so this is a Deepline
+ * pacing failure rather than a provider Retry-After hold.
+ */
+export class ProviderAdmissionWaitTimeoutError extends Error {
+  readonly code = 'PROVIDER_ADMISSION_TIMEOUT';
+  readonly provider: string;
+  readonly waitBudgetMs: number;
+
+  constructor(input: { provider: string; waitBudgetMs: number }) {
+    const minutes = Math.max(1, Math.ceil(input.waitBudgetMs / 60_000));
+    super(
+      `PROVIDER_ADMISSION_TIMEOUT: Deepline could not grant a provider permit for ${input.provider} within its ${minutes}-minute wait budget. No provider request was sent.`,
+    );
+    this.name = 'ProviderAdmissionWaitTimeoutError';
+    this.provider = input.provider;
+    this.waitBudgetMs = input.waitBudgetMs;
+  }
+}
+
 /**
  * Control signal from the DB-authoritative pacer when a provider hold exceeds
  * the in-process wait threshold. The next provider call is not dispatched.

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts

@@ -42,6 +42,7 @@ import type {
 import {
   buildDaytonaRuntimeWatchdogSource,
   PLAY_RUNNER_UV_THREADPOOL_SIZE,
+  RUNNER_MEMORY_CGROUP_ROOT,
   RUNTIME_SANDBOX_MEMORY_ABORT_REASON,
 } from './daytona-runtime-watchdog';
 
@@ -72,6 +73,9 @@ export type StagedDaytonaPayload = {
   command: string;
   outputPath: string;
   exitCodePath: string;
+  oomKillBaselinePath: string;
+  runnerPidPath: string;
+  runtimeStartedPath: string;
   runtimeCompletedPath: string;
   terminationDiagnosticPath: string;
   progressEventPath: string;
@@ -656,6 +660,7 @@ export async function stageRunnerPayload(input: {
   const crashPusherLogPath = `${workDir}/deepline-play-crash-terminal-${randomUUID()}.log`;
   const exitCodePath = `${workDir}/deepline-play-exit-${randomUUID()}.txt`;
   const oomKillBaselinePath = `${workDir}/deepline-play-oom-kill-${randomUUID()}.txt`;
+  const runnerPidPath = `${workDir}/deepline-play-runner-pid-${randomUUID()}.txt`;
   const runtimeStartedPath = `${workDir}/deepline-play-runtime-started-${randomUUID()}.json`;
   const runtimeCompletedPath = `${workDir}/deepline-play-runtime-completed-${randomUUID()}.json`;
   const runtimeLimitMarkerPath = `${workDir}/deepline-play-runtime-limit-${randomUUID()}.txt`;
@@ -691,6 +696,8 @@ export async function stageRunnerPayload(input: {
     shellQuote(terminationDiagnosticPath),
     shellQuote(memoryLedgerPath),
     shellQuote(memoryPressurePath),
+    shellQuote(RUNNER_MEMORY_CGROUP_ROOT),
+    shellQuote(runnerPidPath),
   ].join(' ');
   const runnerCommand = `${nodeMaterializePayloadCommand({
     envelopePath,
@@ -708,13 +715,16 @@ export async function stageRunnerPayload(input: {
   // Its own retry diagnostics are kept in a separate log: appending them to
   // `outputPath` after a near-gateway-sized result could hide that result from
   // a later bounded-tail recovery read.
-  const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(crashPusherLogPath)} ${shellQuote(exitCodePath)} ${shellQuote(oomKillBaselinePath)} ${shellQuote(runtimeStartedPath)} ${shellQuote(runtimeCompletedPath)} ${shellQuote(runtimeLimitMarkerPath)} ${shellQuote(terminationDiagnosticPath)} ${shellQuote(memoryLedgerPath)} ${shellQuote(`${memoryLedgerPath}.tmp`)} ${shellQuote(memoryPressurePath)} ${shellQuote(progressEventPath)} ${shellQuote(`${progressEventPath}.*`)} ${shellQuote(startupDiagnosticPath)}; ( awk '$1 == "oom_kill" { print $2 }' /sys/fs/cgroup/memory.events 2>/dev/null || true ) > ${shellQuote(oomKillBaselinePath)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; ${runtimeReliabilityEnv}node ${shellQuote(crashPusherPath)} ${shellQuote(configPath)} "$code" ${shellQuote(outputPath)} ${shellQuote(runtimeLimitMarkerPath)} ${shellQuote(oomKillBaselinePath)} /sys/fs/cgroup/memory.events ${shellQuote(terminationDiagnosticPath)} > ${shellQuote(crashPusherLogPath)} 2>&1 || true; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
+  const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(crashPusherLogPath)} ${shellQuote(exitCodePath)} ${shellQuote(oomKillBaselinePath)} ${shellQuote(runnerPidPath)} ${shellQuote(runtimeStartedPath)} ${shellQuote(runtimeCompletedPath)} ${shellQuote(runtimeLimitMarkerPath)} ${shellQuote(terminationDiagnosticPath)} ${shellQuote(memoryLedgerPath)} ${shellQuote(`${memoryLedgerPath}.tmp`)} ${shellQuote(memoryPressurePath)} ${shellQuote(progressEventPath)} ${shellQuote(`${progressEventPath}.*`)} ${shellQuote(startupDiagnosticPath)}; ( awk '$1 == "oom_kill" { print $2 }' /sys/fs/cgroup/memory.events 2>/dev/null || true ) > ${shellQuote(oomKillBaselinePath)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; ${runtimeReliabilityEnv}node ${shellQuote(crashPusherPath)} ${shellQuote(configPath)} "$code" ${shellQuote(outputPath)} ${shellQuote(runtimeLimitMarkerPath)} ${shellQuote(oomKillBaselinePath)} /sys/fs/cgroup/memory.events ${shellQuote(terminationDiagnosticPath)} > ${shellQuote(crashPusherLogPath)} 2>&1 || true; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
 
   return {
     workDir: input.workDir,
     command,
     outputPath,
     exitCodePath,
+    oomKillBaselinePath,
+    runnerPidPath,
+    runtimeStartedPath,
     runtimeCompletedPath,
     terminationDiagnosticPath,
     progressEventPath,

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts

@@ -126,7 +126,7 @@ export const RUNTIME_SANDBOX_MEMORY_ABORT_REASON =
  *   1 runnerPath, 2 configPath, 3 runtimeStartedPath, 4 runtimeCompletedPath,
  *   5 limitMs, 6 startupLimitMs, 7 terminalGraceMs, 8 runtimeLimitMarkerPath,
  *   9 terminationDiagnosticPath, 10 memoryLedgerPath, 11 memoryPressurePath,
- *   12 cgroupRoot.
+ *   12 cgroupRoot, 13 runnerPidPath.
  */
 export function buildDaytonaRuntimeWatchdogSource(): string {
   return `
@@ -144,6 +144,7 @@ const terminationDiagnosticPath = process.argv[9] || '';
 const memoryLedgerPath = process.argv[10] || '';
 const memoryPressurePath = process.argv[11] || '';
 const cgroupRoot = process.argv[12] || ${JSON.stringify(RUNNER_MEMORY_CGROUP_ROOT)};
+const runnerPidPath = process.argv[13] || '';
 const runtimeLimitSeconds = Math.max(1, Math.ceil(limitMs / 1_000));
 const child = spawn(
   process.execPath,
@@ -161,6 +162,21 @@ const child = spawn(
   stdio: 'inherit',
   },
 );
+if (runnerPidPath && Number.isSafeInteger(child.pid)) {
+  try {
+    const rawStat = fs.readFileSync('/proc/' + child.pid + '/stat', 'utf8');
+    const close = rawStat.lastIndexOf(')');
+    const fields = close >= 0 ? rawStat.slice(close + 1).trim().split(/\\s+/) : [];
+    const startTimeTicks = Number.parseInt(fields[19], 10);
+    if (Number.isSafeInteger(startTimeTicks) && startTimeTicks >= 0) {
+      fs.writeFileSync(
+        runnerPidPath,
+        JSON.stringify({ pid: child.pid, startTimeTicks }),
+        { flag: 'wx' },
+      );
+    }
+  } catch {}
+}
 let timedOut = false;
 let startupTimedOut = false;
 let terminalFlushTimedOut = false;

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts

@@ -46,6 +46,7 @@ import {
 } from './daytona-session-execution';
 import { RUNTIME_RELIABILITY_POLICY } from '../../runtime-reliability-policy';
 import { daytonaSandboxStartAdmission } from './daytona-start-admission';
+import { RUNNER_MEMORY_CGROUP_ROOT } from './daytona-runtime-watchdog';
 
 const DAYTONA_COMMAND_RECOVERY_TIMEOUT_MS =
   RUNTIME_RELIABILITY_POLICY.sandbox.daytonaCommandRecoveryTimeoutMs;
@@ -94,13 +95,37 @@ export type DaytonaCrashDiagnostic = {
   exitCodeFile: number | null;
   sessionExitCode: number | null;
   outputBytes: number | null;
+  outputFileAvailable?: boolean | null;
+  exitCodeFileAvailable?: boolean | null;
+  terminationFileAvailable?: boolean | null;
+  runtimeStarted?: boolean | null;
+  runtimeCompleted?: boolean | null;
   tailBytes: number;
   tailTruncated: boolean;
   signals: string[];
   unclassifiedLineCount: number;
   termination: DaytonaTerminationDiagnostic | null;
+  resourceSnapshot?: DaytonaCrashResourceSnapshot | null;
   collectionFailure: DaytonaLookupFailure | null;
 };
+export type DaytonaCrashResourceSnapshot = {
+  memorySource: 'cgroup_v1' | 'cgroup_v1_nested' | 'cgroup_v2' | null;
+  memoryUsageBytes: number | null;
+  memoryLimitBytes: number | null;
+  memoryUsageRatio: number | null;
+  memoryOomCount: number | null;
+  memoryOomKillCount: number | null;
+  memoryOomKillDelta: number | null;
+  memoryOomGroupKillCount: number | null;
+  memoryHighCount: number | null;
+  memoryMaxCount: number | null;
+  cgroupCpuUsageUsecDelta: number | null;
+  cgroupCpuSampleWindowMs: number | null;
+  runnerPresent: boolean | null;
+  runnerState: string | null;
+  runnerRssPages: number | null;
+  runnerCpuTicksDelta: number | null;
+};
 export type DaytonaTerminationDiagnostic = {
   schemaVersion: 1;
   reason:
@@ -185,6 +210,56 @@ function parseDaytonaTerminationDiagnostic(
   }
   return diagnostic;
 }
+
+export function parseDaytonaCrashResourceSnapshot(
+  value: unknown,
+): DaytonaCrashResourceSnapshot | null {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
+  const candidate = value as Record<string, unknown>;
+  const nonNegativeNumber = (key: string): number | null => {
+    const value = candidate[key];
+    return typeof value === 'number' &&
+      Number.isSafeInteger(value) &&
+      value >= 0
+      ? value
+      : null;
+  };
+  const ratio = candidate.memoryUsageRatio;
+  const source = candidate.memorySource;
+  return {
+    memorySource:
+      source === 'cgroup_v1' ||
+      source === 'cgroup_v1_nested' ||
+      source === 'cgroup_v2'
+        ? source
+        : null,
+    memoryUsageBytes: nonNegativeNumber('memoryUsageBytes'),
+    memoryLimitBytes: nonNegativeNumber('memoryLimitBytes'),
+    memoryUsageRatio:
+      typeof ratio === 'number' && Number.isFinite(ratio) && ratio >= 0
+        ? ratio
+        : null,
+    memoryOomCount: nonNegativeNumber('memoryOomCount'),
+    memoryOomKillCount: nonNegativeNumber('memoryOomKillCount'),
+    memoryOomKillDelta: nonNegativeNumber('memoryOomKillDelta'),
+    memoryOomGroupKillCount: nonNegativeNumber('memoryOomGroupKillCount'),
+    memoryHighCount: nonNegativeNumber('memoryHighCount'),
+    memoryMaxCount: nonNegativeNumber('memoryMaxCount'),
+    cgroupCpuUsageUsecDelta: nonNegativeNumber('cgroupCpuUsageUsecDelta'),
+    cgroupCpuSampleWindowMs: nonNegativeNumber('cgroupCpuSampleWindowMs'),
+    runnerPresent:
+      typeof candidate.runnerPresent === 'boolean'
+        ? candidate.runnerPresent
+        : null,
+    runnerState:
+      typeof candidate.runnerState === 'string' &&
+      /^[A-Za-z]$/.test(candidate.runnerState)
+        ? candidate.runnerState
+        : null,
+    runnerRssPages: nonNegativeNumber('runnerRssPages'),
+    runnerCpuTicksDelta: nonNegativeNumber('runnerCpuTicksDelta'),
+  };
+}
 type DaytonaUploadAttemptTiming = {
   attempt: number;
   sandboxId: string;
@@ -368,6 +443,189 @@ export function extractDaytonaCrashDiagnosticSignals(outputTail: string): {
   return { signals, unclassifiedLineCount };
 }
 
+export function buildDaytonaCrashDiagnosticSnapshotSource(): string {
+  return String.raw`
+const fs = require('node:fs');
+(async () => {
+const output = process.argv[1];
+const exit = process.argv[2];
+const max = Number(process.argv[3]);
+const terminationPath = process.argv[4] || '';
+const startedPath = process.argv[5] || '';
+const completedPath = process.argv[6] || '';
+const pidPath = process.argv[7] || '';
+const oomBaselinePath = process.argv[8] || '';
+const cgroupRoot = process.argv[9] || '/sys/fs/cgroup';
+const read = (path) => {
+  try { return fs.readFileSync(path, 'utf8'); } catch { return null; }
+};
+const exists = (path) => {
+  try { return fs.statSync(path).isFile(); } catch { return false; }
+};
+let outputStat = null;
+try { outputStat = fs.statSync(output); } catch {}
+let tail = Buffer.alloc(0);
+if (outputStat) {
+  try {
+    const start = Math.max(0, outputStat.size - max);
+    const fd = fs.openSync(output, 'r');
+    tail = Buffer.alloc(outputStat.size - start);
+    fs.readSync(fd, tail, 0, tail.length, start);
+    fs.closeSync(fd);
+  } catch {}
+}
+const exitRaw = read(exit);
+let exitCode = null;
+if (exitRaw !== null) {
+  const value = Number.parseInt(exitRaw.trim(), 10);
+  if (Number.isFinite(value)) exitCode = value;
+}
+let termination = null;
+try {
+  const value = JSON.parse(read(terminationPath) || 'null');
+  if (value && typeof value === 'object') termination = value;
+} catch {}
+const memoryPaths = [
+  ['cgroup_v2', cgroupRoot + '/memory.current', cgroupRoot + '/memory.max'],
+  ['cgroup_v1', cgroupRoot + '/memory.usage_in_bytes', cgroupRoot + '/memory.limit_in_bytes'],
+  ['cgroup_v1_nested', cgroupRoot + '/memory/memory.usage_in_bytes', cgroupRoot + '/memory/memory.limit_in_bytes'],
+];
+const readNumber = (path) => {
+  const raw = read(path);
+  if (raw === null || !raw.trim() || raw.trim() === 'max') return null;
+  const value = Number.parseInt(raw.trim(), 10);
+  return Number.isSafeInteger(value) && value >= 0 ? value : null;
+};
+const readMemory = () => {
+  for (const [source, usagePath, limitPath] of memoryPaths) {
+    const usage = readNumber(usagePath);
+    const limit = readNumber(limitPath);
+    if (usage !== null && limit !== null && limit > 0 && limit < 2 ** 50) {
+      return {
+        memorySource: source,
+        memoryUsageBytes: usage,
+        memoryLimitBytes: limit,
+        memoryUsageRatio: usage / limit,
+      };
+    }
+  }
+  return {
+    memorySource: null,
+    memoryUsageBytes: null,
+    memoryLimitBytes: null,
+    memoryUsageRatio: null,
+  };
+};
+const readEvents = () => {
+  const result = {};
+  for (const line of (read(cgroupRoot + '/memory.events') || '').split(/\r?\n/)) {
+    const match = line.match(/^(oom|oom_kill|oom_group_kill|high|max)\s+(\d+)$/);
+    if (match) result[match[1]] = Number.parseInt(match[2], 10);
+  }
+  return result;
+};
+const readCpu = () => {
+  const line = (read(cgroupRoot + '/cpu.stat') || '')
+    .split(/\r?\n/)
+    .find((entry) => /^usage_usec\s+\d+$/.test(entry));
+  return line ? Number.parseInt(line.split(/\s+/)[1], 10) : null;
+};
+const readRunner = () => {
+  let identity = null;
+  try { identity = JSON.parse(read(pidPath) || 'null'); } catch {}
+  const pid = identity && identity.pid;
+  const expectedStartTimeTicks = identity && identity.startTimeTicks;
+  if (
+    !Number.isSafeInteger(pid) ||
+    pid < 1 ||
+    !Number.isSafeInteger(expectedStartTimeTicks) ||
+    expectedStartTimeTicks < 0
+  ) return null;
+  const rawStat = read('/proc/' + pid + '/stat');
+  if (rawStat === null) return { present: false };
+  const close = rawStat.lastIndexOf(')');
+  if (close < 0) return { present: true };
+  const fields = rawStat.slice(close + 1).trim().split(/\s+/);
+  const startTimeTicks = Number.parseInt(fields[19], 10);
+  if (startTimeTicks !== expectedStartTimeTicks) return null;
+  const userTicks = Number.parseInt(fields[11], 10);
+  const systemTicks = Number.parseInt(fields[12], 10);
+  const rssRaw = (read('/proc/' + pid + '/statm') || '').trim().split(/\s+/)[1];
+  const rssPages = Number.parseInt(rssRaw || '', 10);
+  return {
+    present: true,
+    state: fields[0] || null,
+    cpuTicks:
+      Number.isFinite(userTicks) && Number.isFinite(systemTicks)
+        ? userTicks + systemTicks
+        : null,
+    rssPages: Number.isSafeInteger(rssPages) && rssPages >= 0 ? rssPages : null,
+  };
+};
+const beforeCpu = readCpu();
+const beforeRunner = readRunner();
+const sampleStarted = Date.now();
+await new Promise((resolve) => setTimeout(resolve, 200));
+const afterCpu = readCpu();
+const afterRunner = readRunner();
+const sampleWindowMs = Date.now() - sampleStarted;
+const events = readEvents();
+const baseline = readNumber(oomBaselinePath);
+const resourceSnapshot = {
+  ...readMemory(),
+  memoryOomCount: Number.isSafeInteger(events.oom) ? events.oom : null,
+  memoryOomKillCount: Number.isSafeInteger(events.oom_kill) ? events.oom_kill : null,
+  memoryOomKillDelta:
+    Number.isSafeInteger(events.oom_kill) &&
+    baseline !== null &&
+    events.oom_kill >= baseline
+      ? events.oom_kill - baseline
+      : null,
+  memoryOomGroupKillCount:
+    Number.isSafeInteger(events.oom_group_kill) ? events.oom_group_kill : null,
+  memoryHighCount: Number.isSafeInteger(events.high) ? events.high : null,
+  memoryMaxCount: Number.isSafeInteger(events.max) ? events.max : null,
+  cgroupCpuUsageUsecDelta:
+    Number.isSafeInteger(beforeCpu) &&
+    Number.isSafeInteger(afterCpu) &&
+    afterCpu >= beforeCpu
+      ? afterCpu - beforeCpu
+      : null,
+  cgroupCpuSampleWindowMs: sampleWindowMs,
+  runnerPresent: afterRunner ? afterRunner.present : null,
+  runnerState:
+    afterRunner && typeof afterRunner.state === 'string'
+      ? afterRunner.state
+      : null,
+  runnerRssPages:
+    afterRunner && Number.isSafeInteger(afterRunner.rssPages)
+      ? afterRunner.rssPages
+      : null,
+  runnerCpuTicksDelta:
+    beforeRunner &&
+    afterRunner &&
+    Number.isSafeInteger(beforeRunner.cpuTicks) &&
+    Number.isSafeInteger(afterRunner.cpuTicks) &&
+    afterRunner.cpuTicks >= beforeRunner.cpuTicks
+      ? afterRunner.cpuTicks - beforeRunner.cpuTicks
+      : null,
+};
+process.stdout.write(JSON.stringify({
+  outputBytes: outputStat ? outputStat.size : null,
+  outputFileAvailable: Boolean(outputStat),
+  exitCodeFileAvailable: exitRaw !== null,
+  tail: tail.toString('base64'),
+  exitCode,
+  terminationFileAvailable: exists(terminationPath),
+  termination,
+  runtimeStarted: exists(startedPath),
+  runtimeCompleted: exists(completedPath),
+  resourceSnapshot,
+}));
+})().catch(() => { process.exitCode = 1; });
+`;
+}
+
 /**
  * Read bounded crash evidence while the failed sandbox still exists. This is
  * operator telemetry only: it never becomes a run result or customer log.
@@ -378,6 +636,10 @@ export async function readDetachedDaytonaCrashDiagnostic(input: {
   cmdId: string;
   outputPath: string;
   exitCodePath: string;
+  oomKillBaselinePath?: string;
+  runnerPidPath?: string;
+  runtimeStartedPath?: string;
+  runtimeCompletedPath?: string;
   terminationDiagnosticPath?: string;
   expectedOrganizationId?: string | null;
 }): Promise<DaytonaCrashDiagnostic> {
@@ -389,11 +651,17 @@ export async function readDetachedDaytonaCrashDiagnostic(input: {
     exitCodeFile: null,
     sessionExitCode: null,
     outputBytes: null,
+    outputFileAvailable: null,
+    exitCodeFileAvailable: null,
+    terminationFileAvailable: null,
+    runtimeStarted: null,
+    runtimeCompleted: null,
     tailBytes: 0,
     tailTruncated: false,
     signals: [],
     unclassifiedLineCount: 0,
     termination: null,
+    resourceSnapshot: null,
     collectionFailure,
   });
   try {
@@ -419,10 +687,10 @@ export async function readDetachedDaytonaCrashDiagnostic(input: {
       };
     }
 
-    const script = `const fs=require('node:fs');const output=process.argv[1];const exit=process.argv[2];const max=Number(process.argv[3]);const terminationPath=process.argv[4]||'';const stat=fs.statSync(output);const start=Math.max(0,stat.size-max);const fd=fs.openSync(output,'r');const tail=Buffer.alloc(stat.size-start);fs.readSync(fd,tail,0,tail.length,start);fs.closeSync(fd);let exitCode=null;try{const value=Number.parseInt(fs.readFileSync(exit,'utf8').trim(),10);if(Number.isFinite(value))exitCode=value}catch{}let termination=null;try{const value=JSON.parse(fs.readFileSync(terminationPath,'utf8'));if(value&&typeof value==='object')termination=value}catch{}process.stdout.write(JSON.stringify({outputBytes:stat.size,tail:tail.toString('base64'),exitCode,termination}));`;
+    const script = buildDaytonaCrashDiagnosticSnapshotSource();
     const [snapshotResult, sessionResult] = await Promise.allSettled([
       sandbox.process.executeCommand(
-        `node -e ${shellQuoteDaytonaDiagnostic(script)} ${shellQuoteDaytonaDiagnostic(input.outputPath)} ${shellQuoteDaytonaDiagnostic(input.exitCodePath)} ${DAYTONA_CRASH_DIAGNOSTIC_TAIL_MAX_BYTES} ${shellQuoteDaytonaDiagnostic(input.terminationDiagnosticPath ?? '')}`,
+        `node -e ${shellQuoteDaytonaDiagnostic(script)} ${shellQuoteDaytonaDiagnostic(input.outputPath)} ${shellQuoteDaytonaDiagnostic(input.exitCodePath)} ${DAYTONA_CRASH_DIAGNOSTIC_TAIL_MAX_BYTES} ${shellQuoteDaytonaDiagnostic(input.terminationDiagnosticPath ?? '')} ${shellQuoteDaytonaDiagnostic(input.runtimeStartedPath ?? '')} ${shellQuoteDaytonaDiagnostic(input.runtimeCompletedPath ?? '')} ${shellQuoteDaytonaDiagnostic(input.runnerPidPath ?? '')} ${shellQuoteDaytonaDiagnostic(input.oomKillBaselinePath ?? '')} ${shellQuoteDaytonaDiagnostic(RUNNER_MEMORY_CGROUP_ROOT)}`,
         undefined,
         {},
         8,
@@ -434,9 +702,15 @@ export async function readDetachedDaytonaCrashDiagnostic(input: {
     }
     const snapshot = JSON.parse(snapshotResult.value.result) as {
       outputBytes?: unknown;
+      outputFileAvailable?: unknown;
+      exitCodeFileAvailable?: unknown;
+      terminationFileAvailable?: unknown;
+      runtimeStarted?: unknown;
+      runtimeCompleted?: unknown;
       tail?: unknown;
       exitCode?: unknown;
       termination?: unknown;
+      resourceSnapshot?: unknown;
     };
     const tailBuffer =
       typeof snapshot.tail === 'string'
@@ -456,6 +730,9 @@ export async function readDetachedDaytonaCrashDiagnostic(input: {
         ? sessionResult.value.exitCode
         : null;
     const termination = parseDaytonaTerminationDiagnostic(snapshot.termination);
+    const resourceSnapshot = parseDaytonaCrashResourceSnapshot(
+      snapshot.resourceSnapshot,
+    );
     return {
       schemaVersion: 1,
       collectionStatus: 'collected',
@@ -466,6 +743,26 @@ export async function readDetachedDaytonaCrashDiagnostic(input: {
           : null,
       sessionExitCode,
       outputBytes,
+      outputFileAvailable:
+        typeof snapshot.outputFileAvailable === 'boolean'
+          ? snapshot.outputFileAvailable
+          : null,
+      exitCodeFileAvailable:
+        typeof snapshot.exitCodeFileAvailable === 'boolean'
+          ? snapshot.exitCodeFileAvailable
+          : null,
+      terminationFileAvailable:
+        typeof snapshot.terminationFileAvailable === 'boolean'
+          ? snapshot.terminationFileAvailable
+          : null,
+      runtimeStarted:
+        typeof snapshot.runtimeStarted === 'boolean'
+          ? snapshot.runtimeStarted
+          : null,
+      runtimeCompleted:
+        typeof snapshot.runtimeCompleted === 'boolean'
+          ? snapshot.runtimeCompleted
+          : null,
       tailBytes: tailBuffer.byteLength,
       tailTruncated:
         outputBytes !== null &&
@@ -473,6 +770,7 @@ export async function readDetachedDaytonaCrashDiagnostic(input: {
       signals,
       unclassifiedLineCount,
       termination,
+      resourceSnapshot,
       collectionFailure:
         sessionResult.status === 'rejected'
           ? describeDaytonaLookupFailure(sessionResult.reason)
@@ -1418,6 +1716,9 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
               cmdId: start.cmdId,
               outputPath: stagedPayload.outputPath,
               exitCodePath: stagedPayload.exitCodePath,
+              oomKillBaselinePath: stagedPayload.oomKillBaselinePath,
+              runnerPidPath: stagedPayload.runnerPidPath,
+              runtimeStartedPath: stagedPayload.runtimeStartedPath,
               runtimeCompletedPath: stagedPayload.runtimeCompletedPath,
               terminationDiagnosticPath:
                 stagedPayload.terminationDiagnosticPath,

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts

@@ -834,6 +834,10 @@ function buildDefaultDeps(absurd: Absurd): AbsurdRuntimeWorkerDeps {
         cmdId: request.cmdId,
         outputPath: request.outputPath,
         exitCodePath: request.exitCodePath,
+        oomKillBaselinePath: request.oomKillBaselinePath,
+        runnerPidPath: request.runnerPidPath,
+        runtimeStartedPath: request.runtimeStartedPath,
+        runtimeCompletedPath: request.runtimeCompletedPath,
         terminationDiagnosticPath: request.terminationDiagnosticPath,
         expectedOrganizationId: routingDomain,
       }),
@@ -4057,6 +4061,18 @@ function sandboxCrashDiagnosticRequest(input: {
     cmdId: suspension.cmdId,
     outputPath: suspension.outputPath,
     exitCodePath: suspension.exitCodePath,
+    ...(suspension.oomKillBaselinePath
+      ? { oomKillBaselinePath: suspension.oomKillBaselinePath }
+      : {}),
+    ...(suspension.runnerPidPath
+      ? { runnerPidPath: suspension.runnerPidPath }
+      : {}),
+    ...(suspension.runtimeStartedPath
+      ? { runtimeStartedPath: suspension.runtimeStartedPath }
+      : {}),
+    ...(suspension.runtimeCompletedPath
+      ? { runtimeCompletedPath: suspension.runtimeCompletedPath }
+      : {}),
     ...(suspension.terminationDiagnosticPath
       ? { terminationDiagnosticPath: suspension.terminationDiagnosticPath }
       : {}),
@@ -4371,7 +4387,7 @@ async function resolveDetachedRunnerOutcome(input: {
             runnerAttempt: suspension.runnerAttempt,
             sandboxId: suspension.sandboxId,
             exitCode: inspection.exitCode,
-            diagnosis: inspection.diagnosis?.stage,
+            diagnosis: inspection.diagnosis?.stage ?? 'unknown',
             resultStatus: inspection.result.status,
           }),
         );
@@ -4385,7 +4401,7 @@ async function resolveDetachedRunnerOutcome(input: {
             runnerAttempt: suspension.runnerAttempt,
             sandboxId: suspension.sandboxId,
             exitCode: inspection.exitCode,
-            diagnosis: inspection.diagnosis?.stage,
+            diagnosis: inspection.diagnosis?.stage ?? 'unknown',
             resultStatus: inspection.result?.status ?? null,
           }),
         );
@@ -6172,6 +6188,20 @@ function parseSandboxCrashDiagnosticRequest(
   ) {
     return null;
   }
+  for (const key of [
+    'oomKillBaselinePath',
+    'runnerPidPath',
+    'runtimeStartedPath',
+    'runtimeCompletedPath',
+    'terminationDiagnosticPath',
+  ] as const) {
+    if (
+      candidate[key] !== undefined &&
+      (typeof candidate[key] !== 'string' || !(candidate[key] as string).trim())
+    ) {
+      return null;
+    }
+  }
   return candidate as PostgresSchedulerSandboxCrashDiagnosticRequest;
 }
 
@@ -6184,11 +6214,17 @@ function unavailableSandboxCrashDiagnostic(
     exitCodeFile: null,
     sessionExitCode: null,
     outputBytes: null,
+    outputFileAvailable: null,
+    exitCodeFileAvailable: null,
+    terminationFileAvailable: null,
+    runtimeStarted: null,
+    runtimeCompleted: null,
     tailBytes: 0,
     tailTruncated: false,
     signals: [],
     unclassifiedLineCount: 0,
     termination: null,
+    resourceSnapshot: null,
     collectionFailure: {
       errorName: 'CrashDiagnosticUnavailable',
       errorCode: null,

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-scope.ts

@@ -14,18 +14,22 @@ const AIMD_ADDITIVE_STEP_FRACTION = 0.25;
 export function acquireRateScopeSql(
   table: string,
   successesParameter: string,
+  minimumRateSql = 'NULL::float8',
 ): string {
+  const refillRate = `CASE WHEN ${minimumRateSql} IS NULL
+    THEN h.refill_per_sec
+    ELSE GREATEST(h.refill_per_sec, ${minimumRateSql}) END`;
   return `provider_hold_rows AS (
     SELECT * FROM ${table} WHERE bucket_id = $1 FOR UPDATE
   ), provider_hold AS (
     SELECT h.*,
       LEAST(
-        GREATEST(CASE WHEN h.seen_backpressure THEN 1 ELSE h.capacity END, h.refill_per_sec),
-        h.tokens + GREATEST(0, nv.now_ms - h.refilled_at_ms) * h.refill_per_sec / 1000.0
+        GREATEST(CASE WHEN h.seen_backpressure THEN 1 ELSE h.capacity END, ${refillRate}),
+        h.tokens + GREATEST(0, nv.now_ms - h.refilled_at_ms) * ${refillRate} / 1000.0
       ) AS tokens_after_refill,
       ${successesParameter}::int = 0
         AND nv.now_ms - GREATEST(h.refilled_at_ms, h.next_dispatch_at_ms, h.retry_at_ms)
-          > GREATEST(1000, 2000.0 / h.refill_per_sec) AS observation_was_idle,
+          > GREATEST(1000, 2000.0 / ${refillRate}) AS observation_was_idle,
       CASE WHEN ${successesParameter}::int > 0
           AND h.rate_observation_started_at_ms IS NOT NULL
           AND nv.now_ms - h.last_ramp_at_ms >= CASE WHEN h.seen_backpressure
@@ -33,9 +37,9 @@ export function acquireRateScopeSql(
             ELSE h.rate_recovery_interval_ms END
           AND h.retry_at_ms <= nv.now_ms
         THEN LEAST(h.max_rps, CASE WHEN h.seen_backpressure
-          THEN h.refill_per_sec * (1 + ${AIMD_ADDITIVE_STEP_FRACTION})
-          ELSE h.refill_per_sec * 2 END)
-        ELSE h.refill_per_sec END AS next_refill_per_sec,
+          THEN ${refillRate} * (1 + ${AIMD_ADDITIVE_STEP_FRACTION})
+          ELSE ${refillRate} * 2 END)
+        ELSE ${refillRate} END AS next_refill_per_sec,
       CASE WHEN ${successesParameter}::int > 0
           AND h.rate_observation_started_at_ms IS NOT NULL
           AND nv.now_ms - h.last_ramp_at_ms >= CASE WHEN h.seen_backpressure

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts

@@ -390,9 +390,9 @@ function normalizeRules(
       return {
         ruleId: rule.ruleId,
         // Rates are fractional; only dispatched requests must be whole.
-        // Keep the shared, unhinted credential scope conservative even when
-        // an older runner submits the generic 10-RPS policy seed. The shared
-        // row's AIMD controller learns upward from successful observations.
+        // Normalize every runner to the canonical 10-RPS seed for an unknown
+        // shared credential scope. The shared row's AIMD state can still be
+        // lower after backpressure and learns upward from successful calls.
         requestsPerWindow: isUnknownProviderScope
           ? DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS
           : rule.requestsPerWindow,
@@ -446,7 +446,13 @@ function seedForRule(rule: AcquirePacingRule): RateBucketSeed {
   // Preserve its fractional refill rate, with dispatch times resolved in ms.
   const capacity = Math.max(1, rule.requestsPerWindow);
   const base = (rule.requestsPerWindow / rule.windowMs) * 1000;
-  const minRps = Math.min(base, MIN_ADAPTIVE_RPS_FLOOR);
+  // Undeclared providers use the shared fallback rate as their operating
+  // floor. Explicit provider Retry-After deadlines still block dispatch; a
+  // prior learned AIMD rate must not leave later runs paced below the fallback.
+  const minRps =
+    rule.ruleId === 'default:provider-scope'
+      ? base
+      : Math.min(base, MIN_ADAPTIVE_RPS_FLOOR);
   const maxRps =
     rule.adaptiveMaxRps != null && rule.adaptiveMaxRps > 0
       ? Math.max(rule.adaptiveMaxRps, base)
@@ -461,6 +467,12 @@ function seedForRule(rule: AcquirePacingRule): RateBucketSeed {
   };
 }
 
+function minimumRateSqlForRules(rules: readonly AcquirePacingRule[]): string {
+  return rules.some((rule) => rule.ruleId === 'default:provider-scope')
+    ? `${DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS}::float8`
+    : 'NULL::float8';
+}
+
 function positiveInt(value: number, label: string): number {
   if (!Number.isFinite(value) || value <= 0) {
     throw new Error(`${label} must be a positive finite number.`);
@@ -743,11 +755,10 @@ async function seedBucketRows(
         ON CONFLICT (bucket_id) DO UPDATE SET
           retry_at_ms = GREATEST(hold.retry_at_ms, EXCLUDED.retry_at_ms),
           refill_per_sec = CASE
-            -- Older runners seeded this shared credential row at the generic
-            -- default (10 RPS). If that old value never moved from its seed,
-            -- adopt the conservative shared-scope seed on the first acquire
-            -- after upgrade. Preserve any rate with actual ramp or throttle
-            -- evidence, and never erase a provider Retry-After hold.
+            -- If this row never moved from its seed, adopt the canonical
+            -- shared-scope seed on the first acquire after upgrade. Preserve
+            -- any rate with actual ramp or throttle evidence, and never erase
+            -- a provider Retry-After hold.
             WHEN EXISTS (
               SELECT 1 FROM seed_config
               WHERE rule_id = 'default:provider-scope'
@@ -1295,7 +1306,11 @@ async function acquirePureRate(
           adaptive_max_configured boolean
         )
       ),
-      ${acquireRateScopeSql(scopeHoldTableName(input), '$5')},
+      ${acquireRateScopeSql(
+        scopeHoldTableName(input),
+        '$5',
+        minimumRateSqlForRules(rules),
+      )},
       reclaim_reservations AS (
         DELETE FROM ${reservationTableName(input)}
         WHERE bucket_id = $1
@@ -1735,7 +1750,11 @@ async function acquireWithConcurrency(
           FLOOR(EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::bigint
         ) AS now_ms
       ),
-      ${acquireRateScopeSql(scopeHoldTableName(input), '$10')},
+      ${acquireRateScopeSql(
+        scopeHoldTableName(input),
+        '$10',
+        minimumRateSqlForRules(rules),
+      )},
       reclaim_reservations AS (
         DELETE FROM ${reservationTableName(input)}
         WHERE bucket_id = $1
@@ -3015,6 +3034,11 @@ async function penalizeRateStateStatements(
     return emptyRateStatePenaltyTelemetry();
   }
   const rules = normalizeRules((input.rules ?? []) as AcquirePacingRule[]);
+  const minimumRateRps = rules.some(
+    (rule) => rule.ruleId === 'default:provider-scope',
+  )
+    ? DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS
+    : MIN_ADAPTIVE_RPS_FLOOR;
   await applyConcurrencyObservations(
     client,
     input,
@@ -3091,7 +3115,7 @@ async function penalizeRateStateStatements(
         FROM held h CROSS JOIN now_value nv
       ), throttle_ceiling AS (
         SELECT e.*,
-          GREATEST(${MIN_ADAPTIVE_RPS_FLOOR},
+          GREATEST(${minimumRateRps},
             CASE WHEN e.seen_backpressure AND e.rate_successes = 0 THEN
               CASE WHEN $5::bigint IS NOT NULL
                 -- A fresh provider deadline is not a throughput measurement.
@@ -3106,7 +3130,7 @@ async function penalizeRateStateStatements(
         FROM evidence e
       ), learned AS (
         SELECT e.*,
-          CASE WHEN e.new_episode THEN GREATEST(${MIN_ADAPTIVE_RPS_FLOOR}, LEAST(
+          CASE WHEN e.new_episode THEN GREATEST(${minimumRateRps}, LEAST(
             e.next_ceiling_rps,
             GREATEST(1, e.rate_successes) * 1000.0 / e.cycle_ms
           )) ELSE e.refill_before END AS refill_after

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts

@@ -392,6 +392,10 @@ export type PostgresSchedulerSandboxCrashDiagnosticRequest = {
   cmdId: string;
   outputPath: string;
   exitCodePath: string;
+  oomKillBaselinePath?: string;
+  runnerPidPath?: string;
+  runtimeStartedPath?: string;
+  runtimeCompletedPath?: string;
   terminationDiagnosticPath?: string;
 };
 

package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts

@@ -68,6 +68,10 @@ export type PlayExecutionSuspension =
        * timeout-wake salvage verification. */
       outputPath: string;
       exitCodePath: string;
+      /** Bounded crash evidence paths for sandbox state at liveness loss. */
+      oomKillBaselinePath?: string;
+      runnerPidPath?: string;
+      runtimeStartedPath?: string;
       /** Exact customer-code completion fence inside the Daytona sandbox. */
       runtimeCompletedPath?: string;
       /** Bounded watchdog-owned cause marker for crash-only diagnostics. */

package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts

@@ -8,7 +8,12 @@ import {
   type ToolExecutionErrorSchemaVersion,
 } from '../plays/tool-execution-error';
 
-export const TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS = 2;
+// Give an explicitly replay-safe transient 5xx one additional bounded retry.
+// Preview gateway load runs can overlap brief Convex billing-plane 503s; this
+// adds a second recovery chance without replaying operations that did not opt
+// into the transient HTTP safety contract.
+export const TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS = 3;
+const TOOL_EXECUTE_CUSTOMER_DB_STORAGE_UNAVAILABLE_MAX_ATTEMPTS = 2;
 // A provider may need to finish the original request before it can replay the
 // result for this key. With no Retry-After signal, these two polls give the
 // original request ten seconds to finish without repeatedly hammering it.
@@ -344,7 +349,7 @@ function decideToolExecuteHttpRetry(input: {
   if (input.customerDbStorageUnavailable) {
     return {
       retryable: true,
-      attemptCap: TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS,
+      attemptCap: TOOL_EXECUTE_CUSTOMER_DB_STORAGE_UNAVAILABLE_MAX_ATTEMPTS,
       reason: 'customer_db_storage_unavailable',
     };
   }

package/dist/cli/index.js

@@ -3051,7 +3051,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.209",
+  version: "0.3.211",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1

package/dist/cli/index.mjs

@@ -3046,7 +3046,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.209",
+  version: "0.3.211",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1

package/dist/index.js

@@ -864,7 +864,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.209",
+  version: "0.3.211",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1

package/dist/index.mjs

@@ -768,7 +768,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.209",
+  version: "0.3.211",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1

package/dist/release.d.mts

@@ -149,7 +149,7 @@ type SdkRelease = {
     supportPolicy: SdkSupportPolicy;
 };
 declare const SDK_RELEASE: {
-    readonly version: "0.3.209";
+    readonly version: "0.3.211";
     readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
     readonly packageCapabilities: {
         readonly updatePreferences: 1;

package/dist/release.d.ts

@@ -149,7 +149,7 @@ type SdkRelease = {
     supportPolicy: SdkSupportPolicy;
 };
 declare const SDK_RELEASE: {
-    readonly version: "0.3.209";
+    readonly version: "0.3.211";
     readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
     readonly packageCapabilities: {
         readonly updatePreferences: 1;

package/dist/release.js

@@ -74,7 +74,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.209",
+  version: "0.3.211",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1

package/dist/release.mjs

@@ -48,7 +48,7 @@ var SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: "0.3.209",
+  version: "0.3.211",
   updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
   packageCapabilities: {
     updatePreferences: 1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepline",
-  "version": "0.3.209",
+  "version": "0.3.211",
   "description": "GTM data CLI and TypeScript SDK for coding agents",
   "license": "MIT",
   "homepage": "https://code.deepline.com",