deepline 0.3.476 → 0.3.478

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts

@@ -29,6 +29,18 @@ export type DetachedDaytonaStartupDiagnostic = {
     errorCode?: string;
     transportAttemptId?: string;
   } | null;
+  /** Fixed summaries only: raw runner output can contain customer secrets. */
+  launcherFailure?: {
+    reason:
+      | 'node_not_found'
+      | 'node_loader_failed'
+      | 'permission_denied'
+      | 'argument_list_too_long'
+      | 'exec_format_error'
+      | 'unknown_exit_127'
+      | 'output_unavailable';
+    nodeCheckExitCode: number;
+  };
   errors: string[];
 };
 
@@ -124,6 +136,7 @@ function summarizeProcesses(value: unknown): {
  * Process names are reduced to a Node count before logging; runner logs,
  * output, and arbitrary process names are deliberately excluded because no
  * heuristic redactor can guarantee removal of arbitrary customer secrets.
+ * Exit-127 shell failures are classified inside the sandbox into fixed enums.
  */
 export async function inspectDetachedDaytonaStartup(input: {
   sandbox: DaytonaSandbox;
@@ -131,6 +144,7 @@ export async function inspectDetachedDaytonaStartup(input: {
   cmdId: string;
   exitCodePath: string;
   startupDiagnosticPath?: string;
+  outputPath?: string;
 }): Promise<DetachedDaytonaStartupDiagnostic> {
   const errors: string[] = [];
   const diagnostic: DetachedDaytonaStartupDiagnostic = {
@@ -189,6 +203,70 @@ export async function inspectDetachedDaytonaStartup(input: {
       .catch(() => errors.push('process_snapshot_failed')),
   ]);
 
+  if (
+    input.outputPath &&
+    diagnostic.exitCodeFile === 127 &&
+    !diagnostic.runnerStartup
+  ) {
+    // The failed shell is retired immediately after this inspection. Classify
+    // its bounded output inside the sandbox, returning only known enum values
+    // and an exit code. Never ship arbitrary output through a heuristic redactor.
+    const quotedPath = "'" + input.outputPath.replace(/'/g, "'\\''") + "'";
+    try {
+      const result = await input.sandbox.process.executeCommand(
+        `
+        if [ -r ${quotedPath} ]; then
+          startup_tail="$(tail -c 4096 -- ${quotedPath} 2>/dev/null)"
+          case "$startup_tail" in
+            *"error while loading shared libraries"*) startup_reason=node_loader_failed ;;
+            *"not found: node"*|*node*"not found"*|*node*"No such file"*) startup_reason=node_not_found ;;
+            *"Permission denied"*) startup_reason=permission_denied ;;
+            *"Argument list too long"*) startup_reason=argument_list_too_long ;;
+            *"Exec format error"*) startup_reason=exec_format_error ;;
+            *) startup_reason=unknown_exit_127 ;;
+          esac
+        else
+          startup_reason=output_unavailable
+        fi
+        node --version >/dev/null 2>&1
+        startup_node_code=$?
+        printf '{"reason":"%s","nodeCheckExitCode":%s}\\n' "$startup_reason" "$startup_node_code"
+      `,
+        undefined,
+        {},
+        5,
+      );
+      const value = JSON.parse(result.result) as Record<string, unknown>;
+      const reasons = [
+        'node_not_found',
+        'node_loader_failed',
+        'permission_denied',
+        'argument_list_too_long',
+        'exec_format_error',
+        'unknown_exit_127',
+        'output_unavailable',
+      ];
+      if (
+        typeof value.reason !== 'string' ||
+        !reasons.includes(value.reason) ||
+        typeof value.nodeCheckExitCode !== 'number' ||
+        !Number.isInteger(value.nodeCheckExitCode) ||
+        value.nodeCheckExitCode < 0 ||
+        value.nodeCheckExitCode > 255
+      ) {
+        errors.push('launcher_diagnostic_invalid');
+      } else {
+        diagnostic.launcherFailure = {
+          reason: value.reason as NonNullable<
+            DetachedDaytonaStartupDiagnostic['launcherFailure']
+          >['reason'],
+          nodeCheckExitCode: value.nodeCheckExitCode,
+        };
+      }
+    } catch {
+      errors.push('launcher_diagnostic_unavailable');
+    }
+  }
   return diagnostic;
 }
 
@@ -225,6 +303,7 @@ export type DetachedDaytonaRunnerSupervisor = {
     command: string;
     exitCodePath: string;
     startupDiagnosticPath?: string;
+    outputPath?: string;
   }): Promise<Extract<DetachedDaytonaRunnerState, { phase: 'ready' }>>;
 };
 
@@ -394,7 +473,7 @@ export function prepareDetachedDaytonaRunner(input: {
   let started = false;
 
   return {
-    async start({ command, exitCodePath, startupDiagnosticPath }) {
+    async start({ command, exitCodePath, startupDiagnosticPath, outputPath }) {
       if (started) {
         throw new Error(
           `Detached Daytona runner session ${input.sessionId} was already started.`,
@@ -435,6 +514,7 @@ export function prepareDetachedDaytonaRunner(input: {
           cmdId: state.cmdId,
           exitCodePath,
           startupDiagnosticPath,
+          outputPath,
           readiness: input.readiness,
           baselineHeartbeatAt: state.baselineHeartbeatAt,
           runControl: input.runControl,
@@ -461,6 +541,7 @@ export async function confirmDetachedDaytonaRunnerReady(input: {
   cmdId: string;
   exitCodePath: string;
   startupDiagnosticPath?: string;
+  outputPath?: string;
   readiness: DetachedRunnerReadinessPort;
   baselineHeartbeatAt: string | null;
   runControl?: <T>(
@@ -517,6 +598,7 @@ export async function confirmDetachedDaytonaRunnerReady(input: {
       cmdId: input.cmdId,
       exitCodePath: input.exitCodePath,
       startupDiagnosticPath: input.startupDiagnosticPath,
+      outputPath: input.outputPath,
     });
   const diagnostic = await (input.runControl
     ? input.runControl(inspect)

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts

@@ -67,6 +67,7 @@ import {
   recoveryForRunnerStartupNotLive,
 } from '../../play-run-recovery-policy';
 import {
+  DaytonaNodeUnavailableError,
   stageDaytonaRunnerPayload,
   withRunnerSandboxProvider,
 } from './daytona-payload-transport';
@@ -1506,6 +1507,7 @@ const DAYTONA_UNANSWERED_REQUEST_CODES: ReadonlySet<string> = new Set([
  * loud too, because its input may be too large for any sandbox.
  */
 function isFailedDaytonaRunnerStart(error: unknown): boolean {
+  if (error instanceof DaytonaNodeUnavailableError) return true;
   if (error instanceof DaytonaControlTimeoutError) return true;
   if (!isDaytonaProviderError(error)) return false;
   const status = daytonaErrorStatusForLog(error);
@@ -1536,6 +1538,9 @@ function startFailureForLog(error: unknown) {
     errorName: error instanceof Error ? error.name : null,
     providerStatus: daytonaErrorStatusForLog(error),
     providerCode: daytonaErrorCodeForLog(error),
+    ...(error instanceof DaytonaNodeUnavailableError
+      ? { nodeReadiness: error.diagnostic }
+      : {}),
     ...(diagnostic
       ? {
           // Retain the already-sanitized evidence before sandbox retirement.
@@ -1563,6 +1568,7 @@ function startFailureForLog(error: unknown) {
                   transportAttemptId: startup.transportAttemptId,
                 }
               : null,
+            launcherFailure: diagnostic.launcherFailure,
             inspectionErrorCount: diagnostic.errors.length,
           },
         }
@@ -2092,6 +2098,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
               command: stagedPayload.command,
               exitCodePath: stagedPayload.exitCodePath,
               startupDiagnosticPath: stagedPayload.startupDiagnosticPath,
+              outputPath: stagedPayload.outputPath,
             });
           } catch (error) {
             if (

package/dist/bundling-sources/shared_libs/play-runtime/sandbox-compute-usage.ts

@@ -88,6 +88,9 @@ export function resolveRuntimeSandboxComputeItems(input: {
         sandboxId,
         startedAt: resource.startedAt,
         endedAt,
+        maxBillingDurationSeconds: resource.maxBillingDurationSeconds,
+        observedWallTimeSeconds:
+          Math.max(0, observedEndedAt - resource.startedAt) / 1_000,
       }),
     );
   }
@@ -201,6 +204,9 @@ export function resolveDaytonaRuntimeComputeItems(input: {
         sandboxId,
         startedAt: resource.startedAt,
         endedAt,
+        maxBillingDurationSeconds: resource.maxBillingDurationSeconds,
+        observedWallTimeSeconds:
+          Math.max(0, observedEndedAt - resource.startedAt) / 1_000,
       }),
     );
   }

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts

@@ -38,6 +38,26 @@ const RATE_SCOPE_HOLD_TABLE = 'rate_scope_holds';
 const PHYSICAL_DISPATCH_RULE_ID = 'runtime:physical-dispatch';
 
 const MIN_CONCURRENCY_WAIT_MS = 10;
+/**
+ * Longest re-ask when every concurrency slot is taken. A full scope re-asks
+ * about when its next slot should free: the average age of its live leases
+ * divided by their count. A fast provider stays at the 10 ms floor; one whose
+ * calls hold a slot for minutes no longer re-locks its scope row every 10 ms.
+ * On 2026-11-06 five such runs re-asked about 160 times a second for hours.
+ */
+const MAX_CONCURRENCY_WAIT_MS = 1_000;
+/**
+ * A healthy scope lock is held for milliseconds. An acquire that cannot get it
+ * within this budget stops waiting on the server instead of leaving a backend
+ * queued behind a stuck holder after its caller gave up. On 2026-11-06 one
+ * stuck holder queued such abandoned waits for 16 minutes until they exhausted
+ * the shared scheduler pool.
+ */
+const RATE_STATE_LOCK_TIMEOUT_MS = 3_000;
+/** Ends a bounded acquire statement before the gateway's 30 s caller budget. */
+const RATE_STATE_STATEMENT_TIMEOUT_MS = 15_000;
+/** Re-ask after a lock timeout, as for any other busy scope. */
+const RATE_STATE_LOCK_BUSY_WAIT_MS = 1_000;
 /**
  * Longest a requester held back by the fair-share cap sleeps before it re-asks.
  * The natural wait is the rest of the current fair-share window (a sibling's
@@ -308,13 +328,13 @@ async function queryRateStateWithProbeBaseline<
   parameters: unknown[],
 ) {
   try {
-    return await client.query<Row>(sql(true), parameters);
+    return await queryBoundedRateState<Row>(client, sql(true), parameters);
   } catch (error) {
     if (!isMissingProbeBaselineColumnError(error)) throw error;
     // PostgreSQL rejects the statement before any grant or reservation writes.
     // Retry only that statement; previously accepted success feedback is not
     // applied twice. No DDL, migration preflight, or global schema floor.
-    return await client.query<Row>(sql(false), parameters);
+    return await queryBoundedRateState<Row>(client, sql(false), parameters);
   }
 }
 
@@ -771,6 +791,32 @@ async function seedBucketRows(
   },
   seeds: RateBucketSeed[],
   nowMs: number | null,
+): Promise<void> {
+  // Cold path only, so its own bounded transaction costs two round trips once
+  // per bucket per process. Its insert waits on the same scope row as a grant.
+  await client.query('BEGIN');
+  try {
+    await boundRateStateTransaction(client);
+    await seedBucketRowsStatement(client, input, seeds, nowMs);
+    await client.query('COMMIT');
+  } catch (error) {
+    try {
+      await client.query('ROLLBACK');
+    } catch {
+      // Preserve the original seed/storage error.
+    }
+    throw rateStateLockBusyOr(error);
+  }
+}
+
+async function seedBucketRowsStatement(
+  client: PostgresSchedulerQueryClient,
+  input: { bucketId: string } & {
+    schema?: string | null;
+    schedulerSchema?: string | null;
+  },
+  seeds: RateBucketSeed[],
+  nowMs: number | null,
 ): Promise<void> {
   await client.query(
     `
@@ -1797,6 +1843,7 @@ async function acquireWithConcurrency(
   while (true) {
     await client.query('BEGIN ISOLATION LEVEL READ COMMITTED');
     try {
+      await boundRateStateTransaction(client);
       await client.query(
         `SELECT bucket_id FROM ${scopeHoldTableName(input)}
        WHERE bucket_id = $1 FOR UPDATE`,
@@ -1824,11 +1871,73 @@ async function acquireWithConcurrency(
         probeBaselineSupported = false;
         continue;
       }
-      throw error;
+      throw rateStateLockBusyOr(error);
     }
   }
 }
 
+/**
+ * Bound the current rate-state transaction. `SET LOCAL` keeps the limits inside
+ * this transaction: the scheduler pool runs behind PgBouncer transaction
+ * pooling, where a session setting would leak to another caller.
+ */
+async function boundRateStateTransaction(
+  client: PostgresSchedulerQueryClient,
+): Promise<void> {
+  await client.query(
+    `SELECT set_config('lock_timeout', $1, true),
+            set_config('statement_timeout', $2, true)`,
+    [`${RATE_STATE_LOCK_TIMEOUT_MS}ms`, `${RATE_STATE_STATEMENT_TIMEOUT_MS}ms`],
+  );
+}
+
+/** Bound one formerly-autocommit statement without leaking session settings. */
+async function queryBoundedRateState<
+  Row extends Record<string, unknown> = Record<string, unknown>,
+>(client: PostgresSchedulerQueryClient, sql: string, parameters?: unknown[]) {
+  await client.query('BEGIN');
+  try {
+    await boundRateStateTransaction(client);
+    const result = await client.query<Row>(sql, parameters);
+    await client.query('COMMIT');
+    return result;
+  } catch (error) {
+    try {
+      await client.query('ROLLBACK');
+    } catch {
+      // Preserve the original statement error.
+    }
+    throw rateStateLockBusyOr(error);
+  }
+}
+
+/** A scope row another transaction still holds after the lock budget. */
+class RateStateLockBusyError extends Error {
+  constructor(cause: unknown) {
+    super('Rate-state scope lock was not available within its budget.', {
+      cause,
+    });
+    this.name = 'RateStateLockBusyError';
+  }
+}
+
+/**
+ * A bounded rate-state transaction that ran out of its lock or statement
+ * budget (57014 is statement_timeout).
+ */
+function isRateStateBusy(error: unknown): boolean {
+  return (
+    error instanceof RateStateLockBusyError ||
+    (error as { code?: unknown } | null)?.code === '57014'
+  );
+}
+
+function rateStateLockBusyOr(error: unknown): unknown {
+  return (error as { code?: unknown } | null)?.code === '55P03'
+    ? new RateStateLockBusyError(error)
+    : error;
+}
+
 async function grantWithConcurrency(
   client: PostgresSchedulerQueryClient,
   input: RateStateAcquireInput,
@@ -2315,7 +2424,29 @@ async function grantWithConcurrency(
               (SELECT cool_until_ms FROM cooling),
               (SELECT retry_at_ms FROM cooling)
             ) - (SELECT now_ms FROM refreshed LIMIT 1)${fair?.waitBranch ?? ''}
-          WHEN (SELECT free FROM free_slots) <= 0 THEN $7::bigint
+          WHEN (SELECT free FROM free_slots) <= 0 THEN (
+            -- Re-ask about when the next slot should free. Leases are never
+            -- renewed, so a lease's age is its TTL minus its remaining life.
+            SELECT LEAST(
+              ${MAX_CONCURRENCY_WAIT_MS}::bigint,
+              GREATEST(
+                $7::bigint,
+                CEIL(
+                  COALESCE(
+                    AVG(
+                      (SELECT now_ms FROM now_value)
+                        - (cl.expires_at_ms - $8::bigint)
+                    ),
+                    0
+                  ) / GREATEST((SELECT active_count FROM concurrency_context), 1)
+                )::bigint
+              )
+            )
+            FROM ${concurrencyLeaseTableName(input)} cl
+            WHERE cl.bucket_id = $1
+              AND cl.rule_id = ANY($5::text[])
+              AND cl.expires_at_ms > (SELECT now_ms FROM now_value)
+          )
           ELSE LEAST((SELECT pending_recovery_wait_ms FROM provider_hold), (
             SELECT CEIL(
               MAX(
@@ -2417,9 +2548,28 @@ export async function acquirePostgresRateState(
   client: PostgresSchedulerQueryClient,
   input: RateStateAcquireInput,
 ): Promise<PlayRunnerRateStateAcquireResult> {
-  return await withRateStateAvailabilityGuard(input, () =>
-    acquireRateStateStatements(client, input),
-  );
+  return await withRateStateAvailabilityGuard(input, async () => {
+    try {
+      const result = await acquireRateStateStatements(client, input);
+      return {
+        ...result,
+        acknowledgedObservationBatch: concurrencyObservationBatchFrom(
+          input.concurrencyObservationBatch,
+        ),
+      };
+    } catch (error) {
+      // A busy scope is normal backpressure. A rolled-back feedback fold must
+      // keep its exact batch on the runner; a committed fold is safe to replay
+      // after an uncertain grant because the observer watermark deduplicates it.
+      if (!(error instanceof RateStateLockBusyError)) throw error;
+      return {
+        granted: 0,
+        waitMs: RATE_STATE_LOCK_BUSY_WAIT_MS,
+        leaseIds: [],
+        acknowledgedObservationBatch: null,
+      };
+    }
+  });
 }
 
 /**
@@ -2434,10 +2584,11 @@ export async function validatePostgresRateStatePermit(
   },
 ): Promise<PlayRunnerRateStateValidateResult> {
   return await withRateStateAvailabilityGuard(input, async () => {
-    const result = await client.query<{
+    const result = await queryBoundedRateState<{
       server_now_ms: string | number;
       retry_at_ms: string | number;
     }>(
+      client,
       `
         WITH now_value AS (
           SELECT COALESCE(
@@ -2495,13 +2646,14 @@ export async function claimPostgresProviderDispatch(
   }
 > {
   return withRateStateAvailabilityGuard(input, async () => {
-    const result = await client.query<{
+    const result = await queryBoundedRateState<{
       server_now_ms: string | number;
       retry_at_ms: string | number;
       allowed: boolean;
       cadence_wait: boolean;
       stale_permit: boolean;
     }>(
+      client,
       `
         WITH now_value AS (
           SELECT COALESCE($2::bigint,
@@ -2758,11 +2910,12 @@ async function concurrencyThrottleFrom(
   if (concurrencyRuleIds.length === 0) {
     return { attribution: 'not_observed', observations: [], batch: null };
   }
-  const attributedLease = await client.query<{
+  const attributedLease = await queryBoundedRateState<{
     in_flight_at_start: number | null;
     target_requests_per_second: number | null;
     minimum_rtt_ms: number | null;
   }>(
+    client,
     `
       SELECT
         in_flight_at_start,
@@ -2850,13 +3003,17 @@ async function applyConcurrencyObservations(
   const concurrencyRules = rules.filter(
     (rule) => rule.maxConcurrency != null && rule.maxConcurrency > 0,
   );
-  const countSuccesses = async () => {
+  const countSuccesses = async (bounded = false) => {
     const successes = Math.max(
       0,
       Math.min(1_000_000, Math.trunc(input.observedSuccesses ?? 0)),
     );
     if (successes <= 0) return;
-    await client.query(
+    const query = bounded
+      ? (sql: string, params: unknown[]) =>
+          queryBoundedRateState(client, sql, params)
+      : (sql: string, params: unknown[]) => client.query(sql, params);
+    await query(
       `
       UPDATE ${scopeHoldTableName(input)}
       SET rate_successes = rate_successes + $2::bigint,
@@ -2881,7 +3038,7 @@ async function applyConcurrencyObservations(
   if (batch === null) {
     // Legacy writers have no observation identity. Preserve their success
     // input; new writers always supply the replay-safe batch below.
-    await countSuccesses();
+    await countSuccesses(true);
     return {
       disposition: 'unbatched',
       ruleCount: 0,
@@ -2895,6 +3052,7 @@ async function applyConcurrencyObservations(
 
   await client.query('BEGIN');
   try {
+    await boundRateStateTransaction(client);
     const accepted = await client.query<{ last_sequence: number }>(
       `
         WITH now_value AS (
@@ -3103,7 +3261,7 @@ async function applyConcurrencyObservations(
     } catch {
       // Preserve the original controller/storage error.
     }
-    throw error;
+    throw rateStateLockBusyOr(error);
   }
 }
 
@@ -3130,8 +3288,8 @@ async function acquireRateStateStatements(
       : acquirePureRate(client, grantInput, rules, seeds, requested);
 
   // Cold path only: seed missing rows / refresh changed configuration. A warm
-  // pure-rate acquire is exactly one statement. Concurrency-bearing acquires
-  // take the scope lock before the lease-count statement gets its snapshot.
+  // pure-rate acquire has one grant statement in a bounded transaction.
+  // Concurrency-bearing acquires lock the scope before the lease snapshot.
   let seeded = false;
   if (!warmRateBuckets.has(warmKey)) {
     await seedBucketRows(client, input, seeds, nowMs);
@@ -3179,7 +3337,16 @@ async function acquireRateStateStatements(
     // so a row widened out of band is repaired here instead of waiting for a
     // process restart. Repairing after the grant keeps the acquire itself to a
     // single granting statement, so no permit is ever issued twice.
-    await seedBucketRows(client, input, seeds, nowMs);
+    try {
+      await seedBucketRows(client, input, seeds, nowMs);
+    } catch (error) {
+      // The grant above already committed. A busy scope must not turn its
+      // leases into a zero answer the runner never releases. Leave the bucket
+      // cold so the next acquire repairs before it grants.
+      if (!isRateStateBusy(error)) throw error;
+      warmRateBuckets.delete(warmKey);
+      return acquireResultOf(outcome);
+    }
   }
   markRateBucketWarm(warmKey);
   return acquireResultOf(outcome);
@@ -3199,6 +3366,27 @@ async function releaseRateStateStatements(
   input: PlayRunnerRateStateReleaseInput & { nowMs?: number },
 ): Promise<void> {
   const allRules = normalizeRules(input.rules as AcquirePacingRule[]);
+  if (
+    input.observedSuccesses !== undefined ||
+    input.concurrencyObservations !== undefined ||
+    input.concurrencyObservationBatch !== undefined
+  ) {
+    const batch = concurrencyObservationBatchFrom(
+      input.concurrencyObservationBatch,
+    );
+    if (!batch) {
+      throw new Error(
+        'Rate-state success feedback requires a replay-safe observation batch.',
+      );
+    }
+    await applyConcurrencyObservations(
+      client,
+      input,
+      allRules,
+      concurrencyObservationsFrom(input.concurrencyObservations),
+      batch,
+    );
+  }
   if (allRules.length === 0) return;
   const refundReservationIds = [
     ...new Set(
@@ -3216,7 +3404,8 @@ async function releaseRateStateStatements(
     // no scope owner. Create its uninitialized hold before taking the normal
     // hold -> reservation -> rule lock path; seedBucketRows will initialize its
     // learner on the next admission and preserve this inherited deadline.
-    await client.query(
+    await queryBoundedRateState(
+      client,
       `
         INSERT INTO ${scopeHoldTableName(input)} (bucket_id, retry_at_ms)
         SELECT bucket_id, MAX(claimed_retry_at_ms)
@@ -3232,7 +3421,8 @@ async function releaseRateStateStatements(
     // a bucket, and an HTTP retry sees no reservation left to refund. The shared
     // dispatch cursor deliberately stays monotonic, so cancellation can never
     // compress provider starts into a burst.
-    await client.query(
+    await queryBoundedRateState(
+      client,
       `
         WITH now_value AS (
           SELECT COALESCE(
@@ -3318,7 +3508,8 @@ async function releaseRateStateStatements(
   ];
   if (leaseIds.length === 0) return;
   // DELETE by holder token is naturally idempotent, including HTTP retries.
-  await client.query(
+  await queryBoundedRateState(
+    client,
     `
       DELETE FROM ${concurrencyLeaseTableName(input)}
       WHERE bucket_id = $1
@@ -3376,11 +3567,12 @@ async function penalizeRateStateStatements(
   //    min_rps.
   //  - seen_backpressure flips the ramp from slow-start (x2) to additive.
   //  - tokens drained: no more grants until refill re-accrues past the cooldown.
-  await client.query(
+  await queryBoundedRateState(
+    client,
     `INSERT INTO ${scopeHoldTableName(input)} (bucket_id) VALUES ($1) ON CONFLICT DO NOTHING`,
     [input.bucketId],
   );
-  const rateStateResult = await client.query<{
+  const rateStateResult = await queryBoundedRateState<{
     retry_at_ms: string | number | null;
     cool_until_ms: string | number | null;
     rate_rule_count: string | number;
@@ -3392,6 +3584,7 @@ async function penalizeRateStateStatements(
     observation_cycle_max_ms: string | number | null;
     confirmed_successes_max: string | number | null;
   }>(
+    client,
     `
       WITH now_value AS (
         SELECT COALESCE(

package/dist/bundling-sources/shared_libs/play-runtime/worker-api-types.ts

@@ -98,6 +98,32 @@ function sandboxSizeMetadata(cpu: number, memoryGiB: number) {
   return entry ? { sandboxSize: entry[0] } : {};
 }
 
+/**
+ * Lets settlement tell a run that used its declared runtime from a duration no
+ * runtime limit explains. Billed time stops at the ceiling, so the unclamped
+ * duration travels beside it: without it an overrun looks like a run that
+ * stayed inside its limit. Items without a ceiling keep their historical shape.
+ */
+function runtimeLimitMetadata(
+  maxBillingDurationSeconds: number | null | undefined,
+  observedWallTimeSeconds: number | undefined,
+) {
+  if (
+    typeof maxBillingDurationSeconds !== 'number' ||
+    !Number.isFinite(maxBillingDurationSeconds) ||
+    maxBillingDurationSeconds <= 0
+  ) {
+    return {};
+  }
+  return {
+    maxBillingDurationSeconds,
+    ...(typeof observedWallTimeSeconds === 'number' &&
+    Number.isFinite(observedWallTimeSeconds)
+      ? { observedWallTimeSeconds: Math.max(0, observedWallTimeSeconds) }
+      : {}),
+  };
+}
+
 export function resolveDaytonaSandboxComputeItem(input: {
   itemId: string;
   source?: string;
@@ -108,6 +134,10 @@ export function resolveDaytonaSandboxComputeItem(input: {
   sandboxId?: string | null;
   startedAt?: number;
   endedAt?: number;
+  /** The resource's own billing ceiling: its timeout plus terminal grace. */
+  maxBillingDurationSeconds?: number | null;
+  /** Wall time before the ceiling clamped it; recorded only with a ceiling. */
+  observedWallTimeSeconds?: number;
 }): ComputeBillingItem {
   const billableSeconds = Math.max(1, Math.ceil(input.wallTimeSeconds));
   const cpu = Math.max(0, input.cpu);
@@ -138,6 +168,10 @@ export function resolveDaytonaSandboxComputeItem(input: {
       billableDiskGiB,
       startedAt: input.startedAt,
       endedAt: input.endedAt,
+      ...runtimeLimitMetadata(
+        input.maxBillingDurationSeconds,
+        input.observedWallTimeSeconds,
+      ),
     },
   };
 }
@@ -152,6 +186,10 @@ export function resolveModalSandboxComputeItem(input: {
   sandboxId?: string | null;
   startedAt?: number;
   endedAt?: number;
+  /** The resource's own billing ceiling: its timeout plus terminal grace. */
+  maxBillingDurationSeconds?: number | null;
+  /** Wall time before the ceiling clamped it; recorded only with a ceiling. */
+  observedWallTimeSeconds?: number;
 }): ComputeBillingItem {
   const billableSeconds = Math.max(1, Math.ceil(input.wallTimeSeconds));
   const physicalCpuCores = Math.max(0, input.physicalCpuCores);
@@ -186,6 +224,10 @@ export function resolveModalSandboxComputeItem(input: {
           }),
       startedAt: input.startedAt,
       endedAt: input.endedAt,
+      ...runtimeLimitMetadata(
+        input.maxBillingDurationSeconds,
+        input.observedWallTimeSeconds,
+      ),
     },
   };
 }