@opengeni/worker-bundle 2.0.3 → 2.1.0-canary.36239117573001

package/src/activities/knowledge-indexing.ts

@@ -20,6 +20,95 @@ import {
 import type { DocumentServices } from "@opengeni/documents";
 import type { ControlActivityServices } from "./types";
 
+/** The configured monthly indexed-chunk limit, not a provider failure. */
+export class KnowledgeIndexUsageLimitError extends Error {
+  constructor() {
+    super("monthly document indexing limit reached");
+    this.name = "KnowledgeIndexUsageLimitError";
+  }
+}
+
+export type KnowledgeIndexFailureStage = "embedding" | "processing";
+
+/**
+ * Content-free classification for a deferred Knowledge index batch. Only
+ * protocol constants, an HTTP status, and a PostgreSQL SQLSTATE are retained;
+ * provider messages, bodies, SQL, and identifiers never leave the process.
+ * Outside the embedding call, only a PostgreSQL error in the cause chain is
+ * attributed to the database; any other failure stays a worker failure.
+ */
+export function knowledgeIndexFailureDiagnostic(
+  stage: KnowledgeIndexFailureStage,
+  error: unknown,
+): {
+  errorClass: "KnowledgeIndexOperationError";
+  errorCode:
+    | "knowledge_index_usage_limit_reached"
+    | "knowledge_index_embedding_failed"
+    | "knowledge_index_persistence_failed"
+    | "knowledge_index_failed";
+  origin: "worker" | "db";
+  status?: number;
+  sqlState?: string;
+} {
+  if (error instanceof KnowledgeIndexUsageLimitError) {
+    return {
+      errorClass: "KnowledgeIndexOperationError",
+      errorCode: "knowledge_index_usage_limit_reached",
+      origin: "worker",
+    };
+  }
+  if (stage === "embedding") {
+    const status = ownValue(error, "status");
+    return {
+      errorClass: "KnowledgeIndexOperationError",
+      errorCode: "knowledge_index_embedding_failed",
+      origin: "worker",
+      ...(typeof status === "number" && Number.isInteger(status) && status >= 100 && status <= 599
+        ? { status }
+        : {}),
+    };
+  }
+  const postgres = postgresErrorState(error);
+  if (!postgres) {
+    return {
+      errorClass: "KnowledgeIndexOperationError",
+      errorCode: "knowledge_index_failed",
+      origin: "worker",
+    };
+  }
+  return {
+    errorClass: "KnowledgeIndexOperationError",
+    errorCode: "knowledge_index_persistence_failed",
+    origin: "db",
+    ...(postgres.sqlState ? { sqlState: postgres.sqlState } : {}),
+  };
+}
+
+function ownValue(value: unknown, key: string): unknown {
+  try {
+    if (!value || typeof value !== "object") return undefined;
+    const descriptor = Object.getOwnPropertyDescriptor(value, key);
+    return descriptor && "value" in descriptor ? descriptor.value : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+/** The nearest PostgreSQL error in a short cause chain, with its SQLSTATE
+ * when that is a well-formed five-character code. */
+function postgresErrorState(error: unknown): { sqlState?: string } | undefined {
+  let current = error;
+  for (let depth = 0; depth < 4 && current && typeof current === "object"; depth += 1) {
+    if (ownValue(current, "name") === "PostgresError") {
+      const code = ownValue(current, "code");
+      return typeof code === "string" && /^[0-9A-Z]{5}$/.test(code) ? { sqlState: code } : {};
+    }
+    current = ownValue(current, "cause");
+  }
+  return undefined;
+}
+
 export function createKnowledgeIndexingActivities(
   services: () => Promise<ControlActivityServices>,
   resolveDocumentServices?: () => Promise<DocumentServices>,
@@ -52,6 +141,7 @@ export function createKnowledgeIndexingActivities(
         limit: 2,
       });
       for (const claim of claims) {
+        let stage: KnowledgeIndexFailureStage = "processing";
         try {
           const source = await readKnowledgeIndexSource(db, claim);
           if (!source) {
@@ -126,8 +216,7 @@ export function createKnowledgeIndexingActivities(
                     eventType: "document.indexed",
                     since: new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1)),
                   });
-                  if (used + chunks.length > limit)
-                    throw new Error("monthly document indexing limit reached");
+                  if (used + chunks.length > limit) throw new KnowledgeIndexUsageLimitError();
                 }
               }
               const inputs = chunks.map((chunk) => chunk.embeddingInput);
@@ -135,9 +224,11 @@ export function createKnowledgeIndexingActivities(
                 (sum, input) => sum + Buffer.byteLength(input, "utf8"),
                 0,
               );
+              stage = "embedding";
               const vectors = await embedder.embedMany(inputs);
               if (vectors.length !== chunks.length)
                 throw new Error("Incomplete Knowledge embeddings");
+              stage = "processing";
               // A reviewer may have rejected this revision during the provider
               // call. The DB guard holds its publication row through settlement.
               if (paid) {
@@ -239,16 +330,23 @@ export function createKnowledgeIndexingActivities(
               else result.unavailable++;
             }
           });
-        } catch {
+        } catch (error) {
           // Provider failures retain the last completed projection. The durable
           // queue owns retry/backoff; do not retry an entire activity implicitly.
-          await deferKnowledgeIndexJob(db, claim).catch(() => undefined);
-          result.deferred++;
-          observability.warn("Knowledge indexing batch deferred", {
-            accountId: claim.accountId,
-            entryId: claim.entryId,
-            revisionId: claim.revisionId,
+          // The stored reason stays the SQL lifecycle's fixed code; the log
+          // carries the content-free class/code of the actual cause.
+          observability.warn(
+            "Knowledge indexing batch deferred",
+            knowledgeIndexFailureDiagnostic(stage, error),
+          );
+          await deferKnowledgeIndexJob(db, claim).catch((deferError: unknown) => {
+            const deferDiagnostic = knowledgeIndexFailureDiagnostic("processing", deferError);
+            observability.warn("Knowledge indexing batch deferral failed", {
+              ...deferDiagnostic,
+              errorCode: "knowledge_index_defer_failed",
+            });
           });
+          result.deferred++;
         }
       }
       return result;

package/src/activities/scheduled-tasks.ts

@@ -20,6 +20,7 @@ import {
   openGeniSlackBotMetadata,
   requireOpenGeniSlackBotConnection,
   resolveWorkspaceCatalogSettings,
+  resolveScheduledTaskDefaultModel,
   resolveSessionToolPolicy,
   workspaceCustomModelReference,
   lockActiveCustomModelForAdmission,
@@ -438,8 +439,23 @@ export function createScheduledTaskActivities(services: () => Promise<ControlAct
           )
         : null;
       const settings = await settingsForTask(task, targetSessionExecutionBase?.model);
-      const model = task.agentConfig.model ?? settings.openaiModel;
-      const reasoningEffort = task.agentConfig.reasoningEffort ?? settings.openaiReasoningEffort;
+      // A task that names no model resolves the new-chat default at each fresh
+      // occurrence (saved workspace default, then a usable connected
+      // subscription under the task's frozen SuperGrok authority, then the
+      // credits default while the organization holds a credit balance, then the
+      // deployment default). A frozen SuperGrok pool that is gone only means
+      // SuperGrok is not ready. The accepted execution freezes the result, so
+      // retries and recovery never resolve again. Existing-session runs keep
+      // that session's model.
+      const resolvedDefault =
+        task.agentConfig.model || targetSessionExecutionBase
+          ? null
+          : await resolveScheduledTaskDefaultModel(db, settings, task);
+      const model = task.agentConfig.model ?? resolvedDefault?.model ?? settings.openaiModel;
+      const reasoningEffort =
+        task.agentConfig.reasoningEffort ??
+        resolvedDefault?.reasoningEffort ??
+        settings.openaiReasoningEffort;
       let sandboxBackend = task.agentConfig.sandboxBackend ?? settings.sandboxBackend;
       let sandboxOs: "linux" | "macos" | "windows" = "linux";
       const taskTools = withFirstPartyTools(settings, task.agentConfig.tools);

package/src/activity-services.ts

@@ -1,11 +1,20 @@
-import { dbSearchPath, getSettings, resolveNatsControlPlaneAuth } from "@opengeni/config";
-import { configureChildLifecycleNotices, createDb } from "@opengeni/db";
+import {
+  codeSearchDeploymentPolicy,
+  dbSearchPath,
+  getSettings,
+  resolveNatsControlPlaneAuth,
+} from "@opengeni/config";
+import {
+  configureChildLifecycleNotices,
+  configureCodeSearchDeploymentPolicy,
+  createDb,
+} from "@opengeni/db";
 import { createNatsEventBus } from "@opengeni/events";
 import { createObservability } from "@opengeni/observability";
 import { resolveCatalogSettings } from "@opengeni/core";
 import { createObjectStorage } from "@opengeni/storage";
 import type { ActivityDependencies, SharedActivityServices } from "./activities/types";
-import { observabilityEventLogger } from "./observability-metrics";
+import { observabilityEventBusOptions } from "./observability-metrics";
 import { buildPersonalGitHubGitCredentials } from "./personal-github-git-credentials";
 import { createStandaloneConnectionCredentialsPort } from "./pr-review-credentials";
 
@@ -40,6 +49,9 @@ export function createSharedActivityServices(
       // Child lifecycle notice producers are process-global in @opengeni/db;
       // install the boot-validated rollout flag once for this worker.
       configureChildLifecycleNotices({ enabled: settings.childLifecycleNoticesEnabled });
+      // Child, scheduled and automation sessions created here freeze their
+      // code_search decision from the same deployment policy as the API.
+      configureCodeSearchDeploymentPolicy(codeSearchDeploymentPolicy(settings));
       return {
         settings: resolvedSettings,
         catalogSourceSettings: settings,
@@ -51,7 +63,7 @@ export function createSharedActivityServices(
             controlPlaneAuth
               ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password }
               : undefined,
-            { logger: observabilityEventLogger(observability) },
+            observabilityEventBusOptions(observability),
           )),
         objectStorage: dependencies.objectStorage ?? createObjectStorage(settings),
         observability,

package/src/editable-artifact-hint-broker.ts

@@ -161,6 +161,9 @@ export async function connectEditableArtifactHintBroker(
     name,
     reconnect: true,
     maxReconnectAttempts: -1,
+    // Same policy as the event bus: a broker or auth-callout restart must not
+    // permanently close this connection after two identical auth errors.
+    ignoreAuthErrorAbort: true,
     reconnectTimeWait: 2_000,
     reconnectJitter: 1_000,
     reconnectJitterTLS: 1_000,

package/src/index.ts

@@ -61,7 +61,7 @@ import {
   initializeContextCompactionMetrics,
   initializeWorkerOutcomeMetrics,
   normalizeTurnTaskQueueStats,
-  observabilityEventLogger,
+  observabilityEventBusOptions,
   startContextCompactionPendingMonitor,
   startSessionRecoveryMonitor,
   startTurnCapacityMonitor,
@@ -1173,7 +1173,7 @@ export async function startWorker() {
           controlPlaneAuth
             ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password }
             : undefined,
-          { logger: observabilityEventLogger(observability) },
+          observabilityEventBusOptions(observability),
         ),
       { ...retryOptions, onRetry },
     );

package/src/observability-metrics.ts

@@ -2,7 +2,11 @@ import { createHash } from "node:crypto";
 import { errorCodeToJSON } from "@opengeni/agent-proto";
 import { SandboxBackend, type SessionEventType } from "@opengeni/contracts";
 import type { SessionEventAppendPhaseObservation } from "@opengeni/db";
-import type { EventLogger } from "@opengeni/events";
+import {
+  natsSubscriptionTerminationCounter,
+  type EventBusOptions,
+  type EventLogger,
+} from "@opengeni/events";
 import type { Attributes, AttributeValue, Observability } from "@opengeni/observability";
 import type { CompanyBrainContributionReceipt } from "./model-context-contributions";
 import {
@@ -80,6 +84,16 @@ const CONTEXT_COMPACTIONS_METRIC = {
   help: "Total completed context compactions, by trigger.",
 } as const;
 
+/** Logger plus the closed-label subscription-termination counter for NATS connections. */
+export function observabilityEventBusOptions(
+  observability: Observability,
+): Pick<EventBusOptions, "logger" | "onSubscriptionTerminated"> {
+  return {
+    logger: observabilityEventLogger(observability),
+    onSubscriptionTerminated: natsSubscriptionTerminationCounter(observability),
+  };
+}
+
 export function observabilityEventLogger(observability: Observability): EventLogger {
   return {
     debug: (message, attributes) => observability.debug(message, eventAttributes(attributes)),
@@ -156,6 +170,13 @@ export function runtimeMetricsHooksForObservability(
         labels: { backend, stage },
       });
     },
+    onSandboxReadinessReplacement: ({ backend, outcome }) => {
+      observability.incrementCounter({
+        name: "opengeni_sandbox_readiness_replacements_total",
+        help: "Fresh sandbox command-readiness replacement decisions by backend and outcome.",
+        labels: { backend, outcome },
+      });
+    },
     onSandboxProviderApiThrottle: ({ backend, operation }) => {
       observability.incrementCounter({
         name: "opengeni_sandbox_provider_api_throttles_total",
@@ -2264,3 +2285,51 @@ export function modelCallAccountContext(input: {
     accountChangedFromPrevCall,
   };
 }
+
+export type CodeSearchCallOutcome =
+  | "completed"
+  | "jev_unavailable"
+  | "jev_rejected"
+  | "workspace_unavailable"
+  | "invalid_arguments"
+  | "breaker_open"
+  | "cancelled"
+  | "failed";
+
+/** One `code_search` tool call: outcome, wall time and the Jev work it used. */
+export function recordCodeSearchCall(
+  observability: Observability,
+  input: {
+    outcome: CodeSearchCallOutcome;
+    durationSeconds: number;
+    jevRequests: number;
+    jevCostUsd: number;
+  },
+): void {
+  observability.incrementCounter({
+    name: "opengeni_code_search_calls_total",
+    help: "Jev-backed code_search tool calls by outcome.",
+    labels: { outcome: input.outcome },
+  });
+  observability.observeHistogram({
+    name: "opengeni_code_search_duration_seconds",
+    help: "Wall time of one code_search tool call.",
+    buckets: [0.5, 1, 2, 4, 8, 15, 30, 60],
+    labels: { outcome: input.outcome },
+    value: Math.max(0, input.durationSeconds),
+  });
+  if (input.jevRequests > 0) {
+    observability.incrementCounter({
+      name: "opengeni_code_search_jev_requests_total",
+      help: "Jev requests made by code_search.",
+      amount: input.jevRequests,
+    });
+  }
+  if (input.jevCostUsd > 0) {
+    observability.incrementCounter({
+      name: "opengeni_code_search_jev_cost_micro_usd_total",
+      help: "Estimated Jev list-price cost of code_search, in micro-USD.",
+      amount: Math.round(input.jevCostUsd * 1_000_000),
+    });
+  }
+}

package/src/personal-github-git-credentials.ts

@@ -295,6 +295,8 @@ export function personalGitBrokerOrigin(settings: Settings): string | null {
     (url.hostname === "host.docker.internal" ||
       url.hostname === cachedDockerBridgeGateway ||
       /^[a-z][a-z0-9-]{0,62}$/iu.test(url.hostname));
+  // The `local` and `test` environments accept any origin, which covers the
+  // local launcher's Linux Docker sandbox route on the Compose network gateway.
   const privateExecutionOrigin =
     (url.protocol === "http:" &&
       (url.hostname === "127.0.0.1" || url.hostname === "localhost" || dockerPrivateOrigin)) ||

package/src/sandbox-resume.ts

@@ -81,6 +81,7 @@ import {
   withoutSandboxProviderIdentity,
   type EstablishedSandboxSession,
   type RuntimeMetricsHooks,
+  type SandboxReadinessReplacementOutcome,
   type WorkspaceArchiveDescriptor,
 } from "@opengeni/runtime";
 import {
@@ -150,6 +151,24 @@ export type SandboxResumeServices = {
   /** Test seam for the bounded command-readiness proof performed before an
    * attached provider box is handed to the agent. */
   verifyAttachedSandboxReadiness?: (established: EstablishedSandboxSession) => Promise<void>;
+  /** Test seam for the bounded command-readiness proof the elected spawner runs
+   * on its freshly created box before publishing the lease warm. Production
+   * uses {@link waitForSandboxExecReadiness} with the shared 60 s budget. */
+  verifySpawnedSandboxReadiness?: (
+    established: EstablishedSandboxSession,
+    identity: { sandboxGroupId: string },
+  ) => Promise<void>;
+  /** Test seam for the jittered pause before the single fresh-box readiness
+   * replacement. Production uses {@link freshSandboxReadinessReplacementDelayMs}.
+   * It may be async so a test can observe the rolled-back lease in between. */
+  freshSandboxReadinessReplacementDelayMs?: () => number | Promise<number>;
+  /**
+   * The turn attempt's fresh-box readiness replacement budget. The lazy
+   * provisioner may call resumeBoxForTurn again after a typed lease
+   * supersession, so the turn creates the budget once and shares it with every
+   * call. Absent, the call gets its own single replacement.
+   */
+  freshSandboxReadinessReplacementBudget?: FreshSandboxReadinessReplacementBudget;
   /** Called only by the observer that wins the exact warm->cold loss CAS. */
   onSandboxLost?: (input: {
     sandboxGroupId: string;
@@ -318,6 +337,80 @@ export class SandboxSiblingWarmingTimeoutError extends SandboxWarmingTimeoutErro
   }
 }
 
+/**
+ * Out-of-band proof attached to one exact {@link SandboxExecReadinessTimeoutError}
+ * thrown by the elected spawner: the box that missed its readiness budget was
+ * freshly created by this caller (not an attached, resumed, or provider-continuity
+ * box), it was never published warm, the provider terminated it, and the holder
+ * was released. Only such an error may be replaced by re-entering admission.
+ */
+const disposedUnreadyFreshSandboxErrors = new WeakSet<object>();
+
+/** True only for a spawner readiness timeout whose unready fresh box is proven gone. */
+export function isReplaceableFreshSandboxReadinessTimeout(error: unknown): boolean {
+  return (
+    error instanceof SandboxExecReadinessTimeoutError &&
+    disposedUnreadyFreshSandboxErrors.has(error)
+  );
+}
+
+/**
+ * A burst of simultaneous turn starts can leave many freshly created provider
+ * boxes command-unready at once. The single replacement re-enters admission
+ * after a jittered pause so the replacement creates do not re-synchronize into
+ * the same burst. Bounded to [2 s, 10 s).
+ */
+export const FRESH_SANDBOX_READINESS_REPLACEMENT_BASE_DELAY_MS = 2_000;
+export const FRESH_SANDBOX_READINESS_REPLACEMENT_JITTER_MS = 8_000;
+
+export function freshSandboxReadinessReplacementDelayMs(
+  random: () => number = Math.random,
+): number {
+  const sample = random();
+  const unit = Number.isFinite(sample) ? Math.min(Math.max(sample, 0), 1 - Number.EPSILON) : 0;
+  return (
+    FRESH_SANDBOX_READINESS_REPLACEMENT_BASE_DELAY_MS +
+    Math.floor(unit * FRESH_SANDBOX_READINESS_REPLACEMENT_JITTER_MS)
+  );
+}
+
+/** Replacements allowed per turn attempt, across every resumeBoxForTurn call. */
+export const FRESH_SANDBOX_READINESS_REPLACEMENTS_PER_TURN_ATTEMPT = 1;
+
+export type FreshSandboxReadinessReplacementBudget = { remaining: number };
+
+export function createFreshSandboxReadinessReplacementBudget(): FreshSandboxReadinessReplacementBudget {
+  return { remaining: FRESH_SANDBOX_READINESS_REPLACEMENTS_PER_TURN_ATTEMPT };
+}
+
+function recordFreshSandboxReadinessReplacement(
+  metrics: RuntimeMetricsHooks | undefined,
+  backend: string,
+  outcome: SandboxReadinessReplacementOutcome,
+): void {
+  try {
+    metrics?.onSandboxReadinessReplacement?.({ backend, outcome });
+  } catch {
+    // Metrics emission must never affect sandbox recovery or error propagation.
+  }
+}
+
+async function sleepUnlessCancelled(ms: number, signal: AbortSignal | undefined): Promise<boolean> {
+  if (signal?.aborted) return false;
+  if (ms <= 0) return true;
+  return await new Promise<boolean>((resolve) => {
+    const onAbort = () => {
+      clearTimeout(timer);
+      resolve(false);
+    };
+    const timer = setTimeout(() => {
+      signal?.removeEventListener("abort", onAbort);
+      resolve(true);
+    }, ms);
+    signal?.addEventListener("abort", onAbort, { once: true });
+  });
+}
+
 /** The exact attached caller that won warm->cold after proving the provider
  * instance gone. It is a lease supersession (the same logical turn recovers),
  * plus the lost id needed for one durable observability event. */
@@ -1057,12 +1150,103 @@ async function persistWarmWorkspaceSnapshot(
  * holderId is the globally unique durable turn-attempt id. It must not be a
  * Temporal activity id, because activity ids are only workflow-local and
  * collide when sibling sessions share one sandbox group.
+ *
+ * A freshly created box that misses its bounded command-readiness budget is
+ * replaced at most once per turn attempt (the budget is shared by every call
+ * the turn's provisioner makes). The first establish has already terminated that exact
+ * unpublished box, rolled its exact warming epoch back to cold (advancing the
+ * epoch), and released its holder, so the replacement is an ordinary new
+ * admission: it re-runs the cold->warming CAS, persists its own provider
+ * instance before readiness, and may instead attach to a sibling that won the
+ * CAS first. Nothing model- or tool-visible ran on the discarded box, so no
+ * side effect is replayed. A second readiness timeout fails the turn exactly
+ * as before; attached/resumed boxes and unconfirmed terminations never
+ * replace.
  */
 export async function resumeBoxForTurn(
   services: SandboxResumeServices,
   ids: ResumeBoxIds,
   kind: "turn",
   holderId: TurnSandboxLeaseHolderId,
+): Promise<ResumedTurnSandbox> {
+  try {
+    return await resumeBoxForTurnOnce(services, ids, kind, holderId);
+  } catch (error) {
+    if (!isReplaceableFreshSandboxReadinessTimeout(error)) throw error;
+    const readiness = error as SandboxExecReadinessTimeoutError;
+    const warn = (message: string, fields: Parameters<Observability["warn"]>[1]) => {
+      if (services.observability) services.observability.warn(message, fields);
+      else console.warn(message, fields);
+    };
+    const identity = {
+      workspaceId: ids.workspaceId,
+      sessionId: ids.sessionId,
+      sandboxGroupId: ids.sandboxGroupId,
+      backend: readiness.backend,
+      instanceId: readiness.instanceId,
+      readinessTimeoutMs: readiness.timeoutMs,
+    };
+    const budget =
+      services.freshSandboxReadinessReplacementBudget ??
+      createFreshSandboxReadinessReplacementBudget();
+    if (budget.remaining <= 0) {
+      recordFreshSandboxReadinessReplacement(
+        services.sandboxMetrics,
+        readiness.backend,
+        "budget_spent",
+      );
+      warn(
+        "sandbox command-readiness timed out on a fresh box; this turn attempt already used its replacement",
+        identity,
+      );
+      throw error;
+    }
+    budget.remaining -= 1;
+    const delayMs = await (
+      services.freshSandboxReadinessReplacementDelayMs ?? freshSandboxReadinessReplacementDelayMs
+    )();
+    warn("sandbox command-readiness timed out on a fresh box; replacing it once after jitter", {
+      ...identity,
+      replacementDelayMs: delayMs,
+    });
+    if (!(await sleepUnlessCancelled(delayMs, services.cancellationSignal))) {
+      // Cancellation owns the turn boundary; surface the original typed
+      // readiness failure rather than starting a replacement.
+      recordFreshSandboxReadinessReplacement(
+        services.sandboxMetrics,
+        readiness.backend,
+        "cancelled",
+      );
+      throw error;
+    }
+    try {
+      const replaced = await resumeBoxForTurnOnce(services, ids, kind, holderId);
+      recordFreshSandboxReadinessReplacement(
+        services.sandboxMetrics,
+        readiness.backend,
+        "replaced",
+      );
+      return replaced;
+    } catch (replacementError) {
+      recordFreshSandboxReadinessReplacement(
+        services.sandboxMetrics,
+        readiness.backend,
+        services.cancellationSignal?.aborted
+          ? "cancelled"
+          : replacementError instanceof SandboxExecReadinessTimeoutError
+            ? "failed_again"
+            : "replacement_failed",
+      );
+      throw replacementError;
+    }
+  }
+}
+
+async function resumeBoxForTurnOnce(
+  services: SandboxResumeServices,
+  ids: ResumeBoxIds,
+  kind: "turn",
+  holderId: TurnSandboxLeaseHolderId,
 ): Promise<ResumedTurnSandbox> {
   const { db, settings } = services;
   const os = ids.os ?? "linux";
@@ -1656,11 +1840,17 @@ export async function resumeBoxForTurn(
       // A sandbox handle is not sufficient evidence that an asynchronous
       // provider's command router is live. Do not publish a warm lease until
       // one bounded no-op exec works.
-      // On timeout the catch below terminates the box and rolls warming -> cold,
-      // so the next turn cold-creates instead of hanging forever on first use.
-      await waitForSandboxExecReadiness(established, MODAL_EXEC_READINESS_TIMEOUT_MS, {
-        sandboxGroupId: ids.sandboxGroupId,
-      });
+      // On timeout the catch below terminates the box and rolls warming -> cold;
+      // resumeBoxForTurn may then replace a proven-terminated fresh box once.
+      if (services.verifySpawnedSandboxReadiness) {
+        await services.verifySpawnedSandboxReadiness(established, {
+          sandboxGroupId: ids.sandboxGroupId,
+        });
+      } else {
+        await waitForSandboxExecReadiness(established, MODAL_EXEC_READINESS_TIMEOUT_MS, {
+          sandboxGroupId: ids.sandboxGroupId,
+        });
+      }
       await maybeRenewProviderExpiration(true);
       throwIfReleasedOrCancelled();
       // Fold the LIVE box into a re-resumable envelope and persist it as the
@@ -1796,6 +1986,16 @@ export async function resumeBoxForTurn(
       }
       await release();
       recordSandboxWarmingTimeout(services.sandboxMetrics, error);
+      if (
+        terminated &&
+        createdEstablished &&
+        error instanceof SandboxExecReadinessTimeoutError &&
+        error.instanceId === createdEstablished.instanceId &&
+        (createdEstablished.origin === "created" || createdEstablished.origin === "restored") &&
+        !createdEstablished.providerContinuity
+      ) {
+        disposedUnreadyFreshSandboxErrors.add(error);
+      }
       throw sandboxProvisionStageError(rematerialization ? "archive_recovery" : "create", error);
     }
   }