@opengeni/worker-bundle 1.0.0-canary.0 → 1.0.0

package/dist/observability-metrics.d.ts

@@ -1,10 +1,10 @@
-import { type SessionEventType } from "@opengeni/contracts";
-import type { SessionEventAppendPhaseObservation } from "@opengeni/db";
-import type { EventLogger } from "@opengeni/events";
+import { type SessionEventType, type SkillReadKind, type SkillUseSource } from "@opengeni/contracts";
+import { type ActiveUserWindow, type CreditGrantTotals, type SessionEventAppendPhaseObservation } from "@opengeni/db";
+import { type EventBusOptions, type EventLogger } from "@opengeni/events";
 import type { Observability } from "@opengeni/observability";
 import type { CompanyBrainContributionReceipt } from "./model-context-contributions.js";
 import { type OpenSandboxKubernetesInventory } from "./opensandbox-kubernetes-inventory.js";
-import { type RuntimeMetricsHooks, type SelfhostedOpObservation, type SelfhostedOpObserver } from "@opengeni/runtime";
+import { type RuntimeMetricsHooks, type SelfhostedOpObservation, type SelfhostedOpObserver, type ToolPreparationPhaseMeasurement } from "@opengeni/runtime";
 export type TurnOutcome = "completed" | "failed" | "cancelled" | "recovering";
 export type WorkerDeathRecoveryOutcome = "recovering" | "exhausted";
 export type CreditMicrosKind = "usage" | "grant" | "topup" | "refund";
@@ -39,6 +39,8 @@ export type TemporalTurnTaskQueueStats = {
 } | null;
 declare const CONTEXT_COMPACTION_TRIGGERS: readonly ["auto", "operator", "proactive", "overflow"];
 export type ContextCompactionTrigger = (typeof CONTEXT_COMPACTION_TRIGGERS)[number];
+/** Logger plus the closed-label subscription-termination counter for NATS connections. */
+export declare function observabilityEventBusOptions(observability: Observability): Pick<EventBusOptions, "logger" | "onSubscriptionTerminated">;
 export declare function observabilityEventLogger(observability: Observability): EventLogger;
 export declare function runtimeMetricsHooksForObservability(observability: Observability): RuntimeMetricsHooks;
 /**
@@ -174,17 +176,79 @@ export declare function startContextCompactionPendingMonitor(input: {
 };
 export declare function recordSandboxLeaseGauges(observability: Observability, counts: Partial<Record<SandboxLeaseLiveness, number>>): void;
 export declare function recordOpenSandboxKubernetesInventoryGauges(observability: Observability, inventory: OpenSandboxKubernetesInventory): void;
-export declare const SANDBOX_INVENTORY_PROJECTION_DOMAINS: readonly ["leases", "checkpoint_artifacts", "rotation_backlog", "retained_processes", "expired_drains", "opensandbox_kubernetes"];
+export declare const SANDBOX_INVENTORY_PROJECTION_DOMAINS: readonly ["leases", "checkpoint_artifacts", "recovery_observations", "rotation_backlog", "retained_processes", "expired_drains", "opensandbox_kubernetes"];
 export type SandboxInventoryProjectionDomain = (typeof SANDBOX_INVENTORY_PROJECTION_DOMAINS)[number];
 export declare function recordSandboxInventoryProjectionSuccess(observability: Observability, domain: SandboxInventoryProjectionDomain, timestampSeconds?: number): void;
 export declare function recordSandboxInventoryProjectionFailure(observability: Observability, domain: SandboxInventoryProjectionDomain): void;
 export declare function recordCreditBalanceGauges(observability: Observability, balances: CreditBalanceGauge[]): void;
+/**
+ * Distinct managed people with authenticated browser activity in each fixed
+ * window (migration 0565 presence). Every control worker publishes the same
+ * global value, so dashboards must aggregate with `max()` across pods, never
+ * `sum()`. API keys, services and embedded hosts are never counted.
+ */
+export declare function recordActiveUserGauges(observability: Observability, counts: Record<ActiveUserWindow, number>): void;
+/**
+ * Positive credit grants observed by the ledger trigger since migration 0565,
+ * by closed class. These are cumulative database totals published as gauges by
+ * every control worker: aggregate with `max()` across pods, then `increase()`.
+ * Counting in the database covers the grants no application process writes:
+ * the verified-signup trial grant (a setup trigger) and operator grants.
+ */
+export declare function recordCreditGrantGauges(observability: Observability, totals: CreditGrantTotals): void;
+/**
+ * The deployment-level runtime switch for the one-time verified signup trial
+ * credit (migration 0521): 1 while it allows grants, 0 when an operator has
+ * disabled it or no revision exists. A grant also needs the API's
+ * OPENGENI_VERIFIED_SIGNUP_TRIAL_CREDITS_ENABLED master opt-in, which
+ * {@link recordVerifiedSignupTrialDeploymentFlagGauge} reports separately.
+ */
+export declare function recordVerifiedSignupTrialSwitchGauge(observability: Observability, grantsEnabled: boolean): void;
+/**
+ * The deployment-level runtime switch for new managed account sign-ups
+ * (migration 0585): 1 while it allows new accounts (also when no revision
+ * exists), 0 when an operator has paused them. The API additionally honours
+ * its OPENGENI_MANAGED_AUTH_NEW_SIGNUPS_ENABLED ceiling, which this gauge does
+ * not see; `GET /v1/config/client` reports the combined decision.
+ */
+export declare function recordManagedAuthNewSignupsSwitchGauge(observability: Observability, signupsEnabled: boolean): void;
+/**
+ * The OPENGENI_VERIFIED_SIGNUP_TRIAL_CREDITS_ENABLED master opt-in as this
+ * worker's configuration sees it. The API reads the same shared setting; new
+ * grants happen only while this gauge and the runtime switch gauge are both 1.
+ */
+export declare function recordVerifiedSignupTrialDeploymentFlagGauge(observability: Observability, enabled: boolean): void;
 export declare function recordSandboxOrphansTerminated(observability: Observability, count: number): void;
 export type SandboxCheckpointArtifactMetricState = "candidate" | "current" | "previous" | "delete_pending" | "deleting" | "delete_failed" | "deleted";
 export declare function recordSandboxCheckpointArtifactGauges(observability: Observability, counts: Record<SandboxCheckpointArtifactMetricState, number>): void;
 export type SandboxCheckpointArtifactOutcome = "legacy_adopted" | "claimed" | "deleted" | "delete_failed" | "tombstone_pruned";
 export declare function recordSandboxCheckpointArtifactOutcome(observability: Observability, outcome: SandboxCheckpointArtifactOutcome, count?: number): void;
 export declare function recordSandboxDeadlineRotationsRequested(observability: Observability, count: number): void;
+/** Fixed outcomes of legacy retained-command containment: per-candidate
+ * enrollment inspection, then the enrolled drain's exact cold commit. */
+export declare const SANDBOX_COMMAND_CONTAINMENT_OUTCOMES: readonly ["idle_enrolled", "deadline_enrolled", "resumed_enrolled", "not_eligible", "inspection_failed", "contained", "provider_missing"];
+export type SandboxCommandContainmentOutcome = (typeof SANDBOX_COMMAND_CONTAINMENT_OUTCOMES)[number];
+export declare function recordSandboxCommandContainment(observability: Observability, outcome: SandboxCommandContainmentOutcome): void;
+/** Only call after the exact draining->cold commit reports wentCold. The
+ * backend is validated against the closed contract so provider IDs and other
+ * per-sandbox values can never become metric labels. */
+export declare function recordSandboxProviderMissingBeforeCapture(observability: Observability, backend: string): void;
+export declare function recordSandboxRecoveryObservationGauges(observability: Observability, observations: {
+    providerLosses: number;
+    fallbackSelections: number;
+    freshWorkspaceSelections?: number;
+}): void;
+/** Fixed outcomes for a committed automatic continuity decision after
+ * definitive managed-provider loss: a singleton checkpoint (`selected`), a
+ * shared-group checkpoint (`selected_shared`), or a new empty workspace
+ * (`fresh_workspace`). Call only after the durable authorization committed. */
+export declare const SANDBOX_AUTOMATIC_RECOVERY_OUTCOMES: readonly ["selected", "selected_shared", "fresh_workspace"];
+export type SandboxAutomaticRecoveryOutcome = (typeof SANDBOX_AUTOMATIC_RECOVERY_OUTCOMES)[number];
+export declare function sandboxAutomaticRecoveryOutcome(input: {
+    lane: "checkpoint" | "fresh_workspace";
+    groupSessionCount: number;
+}): SandboxAutomaticRecoveryOutcome;
+export declare function recordSandboxAutomaticRecoverySelected(observability: Observability, backend: string, outcome: SandboxAutomaticRecoveryOutcome): void;
 export declare function recordSandboxRotationBacklogGauges(observability: Observability, backlog: {
     requested: number;
     overdue: number;
@@ -205,7 +269,7 @@ export declare function recordExpiredDrainingSandboxLeaseGauges(observability: O
     ageBucket: string;
     count: number;
 }>): void;
-export declare const RETAINED_PROCESS_RECONCILIATION_OUTCOMES: readonly ["claim_failed", "proof_exited", "proof_lost", "proof_checkpoint_failed", "settled_exited", "settled_lost", "settlement_failed", "identity_mismatch", "resume_state_missing", "backend_unsupported", "provider_running", "provider_unknown", "provider_timeout", "provider_error", "provider_binding_missing", "provider_binding_mismatch", "provider_binding_adopted", "process_observation_unavailable", "quarantined_process_observation_unavailable", "quarantined_binding_missing", "quarantined_binding_mismatch", "defer_failed"];
+export declare const RETAINED_PROCESS_RECONCILIATION_OUTCOMES: readonly ["claim_failed", "background_recovered", "proof_exited", "proof_lost", "proof_checkpoint_failed", "settled_exited", "settled_lost", "settlement_failed", "identity_mismatch", "resume_state_missing", "backend_unsupported", "provider_running", "provider_unknown", "provider_timeout", "provider_error", "provider_binding_missing", "provider_binding_mismatch", "provider_binding_adopted", "process_observation_unavailable", "quarantined_process_observation_unavailable", "quarantined_binding_missing", "quarantined_binding_mismatch", "defer_failed"];
 export type RetainedProcessReconciliationOutcome = (typeof RETAINED_PROCESS_RECONCILIATION_OUTCOMES)[number];
 export declare function recordRetainedProcessReconciliation(observability: Observability, outcome: RetainedProcessReconciliationOutcome): void;
 export declare function recordCreditMicros(observability: Observability | undefined, kind: CreditMicrosKind, amountMicros: number): void;
@@ -220,7 +284,7 @@ export declare function recordModelRequestPhase(observability: Observability, in
     outcome?: "completed" | "failed" | "timed_out";
     durationSeconds: number;
 }): void;
-export type TurnStartupPhase = "claim_and_policy" | "turn_start_settlement" | "credential_selection" | "runtime_preparation" | "sandbox_establish" | "file_resolution" | "tool_context_preparation" | "tool_preparation" | "tool_server_construction" | "tool_required_connect" | "tool_optional_connect" | "tool_attempt_catalog_build" | "tool_attempt_catalog_persist" | "tool_workspace_gateway_catalog_build" | "post_tool_preparation" | "agent_construction" | "post_agent_preparation" | "file_materialization" | "history_preparation" | "history_system_update_load" | "history_current_attachment_resolution" | "history_durable_history_load" | "history_sandbox_envelope_load" | "history_canonical_projection" | "history_provider_projection" | "history_attachment_ref_projection" | "history_screenshot_materialization" | "history_model_attachment_projection" | "history_runtime_input_assembly" | "history_artifact_candidate_scan" | "history_generated_image_materialization" | "history_position_load" | "owned_sandbox_setup" | "provider_dispatch" | "model_request_preparation" | "model_sdk_serialization" | "model_prepare_sandbox_agent_preparation" | "model_prepare_sandbox_agent_manifest_inventory" | "model_prepare_sandbox_session_manifest_inventory" | "model_prepare_sandbox_manifest_apply" | "model_prepare_sandbox_entry_materialization" | "model_prepare_sandbox_running_check" | "model_prepare_sandbox_start" | "model_prepare_sandbox_client_create" | "model_prepare_sandbox_client_resume" | "model_prepare_sandbox_client_delete" | "model_prepare_sandbox_client_state_serialize" | "model_prepare_sandbox_client_reuse_check" | "model_prepare_sandbox_workspace_mutation_admission" | "model_prepare_sandbox_workspace_mutation_provider" | "model_prepare_sandbox_workspace_mutation_settlement" | "model_prepare_sandbox_first_routed_resolution_other" | "model_prepare_sandbox_first_routed_mutation_admission" | "model_prepare_sandbox_first_routed_provider_operation" | "model_prepare_sandbox_first_routed_mutation_settlement" | "model_prepare_sandbox_first_routed_other" | "model_prepare_sandbox_snapshot_wait" | "model_prepare_runner_before_first_sandbox_operation" | "model_prepare_sdk_after_first_sandbox_operation" | "model_prepare_runner_before_mcp_tools" | "model_prepare_mcp_tools_snapshot" | "model_prepare_mcp_tools_before_input_filter" | "model_prepare_mcp_tools_before_repository_skill_discovery" | "model_prepare_repository_skill_discovery" | "model_prepare_repository_skill_discovery_before_input_filter" | "model_prepare_input_filter_base" | "model_prepare_input_filter_genesis" | "model_prepare_input_filter_host" | "model_prepare_input_filter_tool_output" | "model_prepare_input_filter_modality" | "model_prepare_input_filter_context" | "model_prepare_responses_input_conversion" | "model_prepare_responses_request_build" | "model_credential_resolution" | "model_wire_normalization" | "model_request_audit" | "stream_bootstrap";
+export type TurnStartupPhase = "services_initialization" | "claim_catalog_read" | "claim_atomic" | "claim_session_read" | "claim_capability_settings" | "learning_policy_freeze" | "attachment_authority_projection" | "claim_and_policy" | "turn_start_settlement" | "credential_selection" | "runtime_preparation" | "session_mcp_settings" | "sandbox_establish" | "file_resolution" | "tool_context_preparation" | "tool_preparation" | "tool_server_construction" | "tool_required_connect" | "tool_optional_connect" | "tool_attempt_catalog_build" | "tool_attempt_catalog_persist" | "tool_workspace_gateway_catalog_build" | "post_tool_preparation" | "agent_construction" | "post_agent_preparation" | "file_materialization" | "history_preparation" | "history_system_update_load" | "history_current_attachment_resolution" | "history_durable_history_load" | "history_sandbox_envelope_load" | "history_canonical_projection" | "history_provider_projection" | "history_attachment_ref_projection" | "history_screenshot_materialization" | "history_model_attachment_projection" | "history_runtime_input_assembly" | "history_artifact_candidate_scan" | "history_generated_image_materialization" | "history_position_load" | "owned_sandbox_setup" | "runtime_stream_initialization" | "model_request_preparation" | "model_sdk_serialization" | "model_prepare_sandbox_agent_preparation" | "model_prepare_sandbox_agent_manifest_inventory" | "model_prepare_sandbox_session_manifest_inventory" | "model_prepare_sandbox_manifest_apply" | "model_prepare_sandbox_entry_materialization" | "model_prepare_sandbox_running_check" | "model_prepare_sandbox_start" | "model_prepare_sandbox_client_create" | "model_prepare_sandbox_client_resume" | "model_prepare_sandbox_client_delete" | "model_prepare_sandbox_client_state_serialize" | "model_prepare_sandbox_client_reuse_check" | "model_prepare_sandbox_workspace_mutation_admission" | "model_prepare_sandbox_workspace_mutation_provider" | "model_prepare_sandbox_workspace_mutation_settlement" | "model_prepare_sandbox_first_routed_resolution_other" | "model_prepare_sandbox_first_routed_mutation_admission" | "model_prepare_sandbox_first_routed_provider_operation" | "model_prepare_sandbox_first_routed_mutation_settlement" | "model_prepare_sandbox_first_routed_other" | "model_prepare_sandbox_snapshot_wait" | "model_prepare_runner_before_first_sandbox_operation" | "model_prepare_sdk_after_first_sandbox_operation" | "model_prepare_runner_before_mcp_tools" | "model_prepare_mcp_tools_snapshot" | "model_prepare_mcp_tools_before_input_filter" | "model_prepare_mcp_tools_before_repository_skill_discovery" | "model_prepare_repository_skill_discovery" | "model_prepare_repository_skill_discovery_before_input_filter" | "model_prepare_input_filter_base" | "model_prepare_input_filter_genesis" | "model_prepare_input_filter_host" | "model_prepare_input_filter_tool_output" | "model_prepare_input_filter_modality" | "model_prepare_input_filter_context" | "model_prepare_responses_input_conversion" | "model_prepare_responses_request_build" | "model_credential_resolution" | "model_wire_normalization" | "model_request_audit" | "stream_bootstrap";
 export type TurnStartupOutcome = "completed" | "failed";
 export type TurnStartupMilestone = "queue" | "provider_dispatch" | "first_byte";
 export type TurnStartupCache = "hit" | "miss" | "disabled" | "none";
@@ -233,7 +297,8 @@ export type SandboxLogicalProvisionOutcome = "completed" | "expected_transition"
 export type SandboxProvisionAttemptOutcome = "completed" | "retrying" | "failed";
 export declare function turnStartupCountBucket(count: number | null): TurnStartupCountBucket;
 /**
- * Measure the critical path from a durable turn start to provider dispatch.
+ * Measure startup operations, including nested and parallel work. These are
+ * not additive critical-path intervals; use milestones for elapsed latency.
  * Every label is a closed or configuration-derived enum; high-cardinality turn,
  * session, credential, connection, file, and model identifiers are forbidden.
  */
@@ -245,6 +310,16 @@ export declare function recordTurnStartupPhase(observability: Observability, inp
     durationSeconds: number;
     count?: number | null;
     cache?: TurnStartupCache;
+    /** Trace-only lookup for the completed claim while its execution root is still open. */
+    executionCorrelationId?: string;
+}): void;
+/** Observe an existing dependency only. Child intervals can overlap and are
+ * not additive; observer failure must never change authority or error identity. */
+export declare function measureTurnStartupPhase<T>(observability: Observability, input: Omit<Parameters<typeof recordTurnStartupPhase>[1], "durationSeconds" | "outcome">, work: () => Promise<T>): Promise<T>;
+/** Background MCP preparation must not inflate startup phase distributions. */
+export declare function recordToolPreparationPhase(observability: Observability, input: ToolPreparationPhaseMeasurement & {
+    provider: string;
+    backend: string;
 }): void;
 /**
  * Explain why a turn did or did not keep provider sandbox work off the
@@ -414,4 +489,39 @@ export declare function modelCallAccountContext(input: {
     servingAccountHash: string;
     accountChangedFromPrevCall: boolean;
 };
+export type CodeSearchCallOutcome = "completed" | "jev_unavailable" | "jev_rejected" | "workspace_unavailable" | "invalid_arguments" | "breaker_open" | "cancelled" | "failed";
+/** One `code_search` tool call: outcome, wall time and the Jev work it used. */
+export declare function recordCodeSearchCall(observability: Observability, input: {
+    outcome: CodeSearchCallOutcome;
+    durationSeconds: number;
+    jevRequests: number;
+    jevCostUsd: number;
+}): void;
+/**
+ * One skill_read call. Labels are closed sets: `skill` is a built-in id or
+ * `custom` for every other Skill, so tenant Skill ids, names, and requested
+ * identifiers never become labels. `source` is `unknown` when the read was
+ * refused before a Skill resolved.
+ */
+export declare function recordSkillRead(observability: Observability, read: {
+    caller: "model" | "codemode";
+    kind: SkillReadKind;
+    source: SkillUseSource | null;
+    /** The resolved Skill id, or the requested identifier when none resolved. */
+    skill: string;
+}): void;
+/**
+ * One skill_checkout call. Labels are closed sets and never carry Skill ids,
+ * names, or paths. Phases that did not run are not observed.
+ */
+export declare function recordSkillCheckout(observability: Observability, checkout: {
+    outcome: "written" | "unchanged" | "refused" | "failed";
+    selection: "all" | "paths";
+    written: number;
+    unchanged: number;
+    resolveSeconds: number | null;
+    sandboxSeconds: number | null;
+    writeSeconds: number | null;
+    totalSeconds: number;
+}): void;
 export {};

package/dist/retained-process-diagnostics.d.ts

@@ -0,0 +1,4 @@
+import { type SandboxRetainedProcess } from "@opengeni/db";
+import type { Observability } from "@opengeni/observability";
+/** Protected cause locations and schema facts; public logs receive only the diagnostic key. */
+export declare function warnRetainedProcessProofFailure(observability: Pick<Observability, "recordFailureDiagnostic" | "warn">, error: unknown, process: Pick<SandboxRetainedProcess, "id" | "sessionId" | "ownerAttemptId" | "ownerTurnId" | "reconcileAttempts">): void;

package/dist/retained-process-retry.d.ts

@@ -0,0 +1,7 @@
+import type { LeaseSnapshot, SandboxRetainedProcess } from "@opengeni/db";
+/** Observation backoff must leave the exact provider's rotation window usable.
+ * This changes only the next probe time, never cancellation or settlement proof. */
+export declare function retainedProcessDeadlineRetryMs(process: Pick<SandboxRetainedProcess, "leaseId" | "leaseEpoch" | "providerBackend" | "providerInstanceId" | "routeTargetId">, lease: Pick<LeaseSnapshot, "id" | "leaseEpoch" | "backend" | "instanceId" | "providerDeadlineAt"> | null, settings: {
+    sandboxRotationLeadMs: number;
+    sandboxLeaseReaperPeriodMs: number;
+}, retryAfterMs: number, now?: number): number;

package/dist/sandbox-resume.d.ts

@@ -1,8 +1,10 @@
+import type { EventBus } from "@opengeni/events";
 import { type Settings } from "@opengeni/config";
 import { readLease, reinstateTurnLeaseHolder, SandboxLeaseSupersededError, type Database, type LeaseHolderKind } from "@opengeni/db";
 import { establishSandboxSessionFromEnvelope, type EstablishedSandboxSession, type RuntimeMetricsHooks } from "@opengeni/runtime";
 import { type ObjectStorage } from "@opengeni/storage";
-import type { Observability } from "@opengeni/observability";
+import { type Observability } from "@opengeni/observability";
+export { safeSnapshotError } from "./sandbox-snapshot-diagnostics.js";
 export type ResumeHolderKind = LeaseHolderKind;
 /**
  * The durable identity of a turn holding a shared sandbox lease.
@@ -22,9 +24,6 @@ export type SandboxResumeServices = {
     db: Database;
     settings: Settings;
     objectStorage?: ObjectStorage | null;
-    /** Exact settings before a verified rig provider image overlaid the logical
-     * pack/deployment image. Fresh-create NotFound fallback must preserve an
-     * ID-only logical base instead of selecting the provider default. */
     logicalFallbackSettings?: Settings;
     sandboxMetrics?: RuntimeMetricsHooks;
     /** Structured logger for lifecycle decisions (holder release reasons). Falls
@@ -45,6 +44,25 @@ export type SandboxResumeServices = {
     /** Test seam for the bounded command-readiness proof performed before an
      * attached provider box is handed to the agent. */
     verifyAttachedSandboxReadiness?: (established: EstablishedSandboxSession) => Promise<void>;
+    /** Test seam for the bounded command-readiness proof the elected spawner runs
+     * on its freshly created box before publishing the lease warm. Production
+     * uses {@link waitForSandboxExecReadiness} with the shared 60 s budget. */
+    verifySpawnedSandboxReadiness?: (established: EstablishedSandboxSession, identity: {
+        sandboxGroupId: string;
+    }) => Promise<void>;
+    /** Test seam for the jittered pause before the single fresh-box readiness
+     * replacement. Production uses {@link freshSandboxReadinessReplacementDelayMs}.
+     * It may be async so a test can observe the rolled-back lease in between. */
+    freshSandboxReadinessReplacementDelayMs?: () => number | Promise<number>;
+    /**
+     * The turn attempt's fresh-box readiness replacement budget. The lazy
+     * provisioner may call resumeBoxForTurn again after a typed lease
+     * supersession, so the turn creates the budget once and shares it with every
+     * call. Absent, the call gets its own single replacement.
+     */
+    freshSandboxReadinessReplacementBudget?: FreshSandboxReadinessReplacementBudget;
+    /** Live fanout for background commands a provider loss settled. */
+    bus?: EventBus | null;
     /** Called only by the observer that wins the exact warm->cold loss CAS. */
     onSandboxLost?: (input: {
         sandboxGroupId: string;
@@ -83,6 +101,9 @@ export type ResumeBoxIds = {
      * never conflicts).
      */
     image?: string;
+    /** Deployment/workspace image pins apply only to new creates; an existing
+     * group keeps its image. Omission preserves explicit image matching (B3). */
+    imagePolicy?: "require_match" | "new_creates_only";
     /**
      * RIG IS SHARED STATE (M3): the frozen rig version this run rides. Threaded to
      * acquireLease, which stamps it on the cold-create and conflicts on a live box
@@ -147,6 +168,23 @@ export declare class SandboxSiblingWarmingTimeoutError extends SandboxWarmingTim
         instanceId?: string | null;
     });
 }
+/** True only for a spawner readiness timeout whose unready fresh box is proven gone. */
+export declare function isReplaceableFreshSandboxReadinessTimeout(error: unknown): boolean;
+/**
+ * A burst of simultaneous turn starts can leave many freshly created provider
+ * boxes command-unready at once. The single replacement re-enters admission
+ * after a jittered pause so the replacement creates do not re-synchronize into
+ * the same burst. Bounded to [2 s, 10 s).
+ */
+export declare const FRESH_SANDBOX_READINESS_REPLACEMENT_BASE_DELAY_MS = 2000;
+export declare const FRESH_SANDBOX_READINESS_REPLACEMENT_JITTER_MS = 8000;
+export declare function freshSandboxReadinessReplacementDelayMs(random?: () => number): number;
+/** Replacements allowed per turn attempt, across every resumeBoxForTurn call. */
+export declare const FRESH_SANDBOX_READINESS_REPLACEMENTS_PER_TURN_ATTEMPT = 1;
+export type FreshSandboxReadinessReplacementBudget = {
+    remaining: number;
+};
+export declare function createFreshSandboxReadinessReplacementBudget(): FreshSandboxReadinessReplacementBudget;
 /** The exact attached caller that won warm->cold after proving the provider
  * instance gone. It is a lease supersession (the same logical turn recovers),
  * plus the lost id needed for one durable observability event. */
@@ -162,15 +200,7 @@ export declare class SandboxLeaseInstanceLostError extends SandboxLeaseSupersede
  */
 export declare function waitForSandboxExecReadiness(established: EstablishedSandboxSession, timeoutMs?: number, identity?: {
     sandboxGroupId?: string | null;
-}): Promise<void>;
-export declare function safeSnapshotError(error: unknown): {
-    errorClass: "SnapshotOperationError";
-    errorCode: "snapshot_operation_failed";
-    status?: number;
-    origin: "sandbox-resume";
-    causeName?: string;
-    integrityCode?: string;
-};
+}, signal?: AbortSignal): Promise<void>;
 export declare function waitForWarmSnapshot(snapshot: Promise<unknown>, timeoutMs: number, signal?: AbortSignal): Promise<boolean>;
 export type SandboxDeadlineRotationCheckpointResult = {
     /** The exact established epoch still carries a requested rotation and a
@@ -245,5 +275,17 @@ export declare function maybePersistWarmWorkspaceSnapshot(services: SandboxResum
  * holderId is the globally unique durable turn-attempt id. It must not be a
  * Temporal activity id, because activity ids are only workflow-local and
  * collide when sibling sessions share one sandbox group.
+ *
+ * A freshly created box that misses its bounded command-readiness budget is
+ * replaced at most once per turn attempt (the budget is shared by every call
+ * the turn's provisioner makes). The first establish has already terminated that exact
+ * unpublished box, rolled its exact warming epoch back to cold (advancing the
+ * epoch), and released its holder, so the replacement is an ordinary new
+ * admission: it re-runs the cold->warming CAS, persists its own provider
+ * instance before readiness, and may instead attach to a sibling that won the
+ * CAS first. Nothing model- or tool-visible ran on the discarded box, so no
+ * side effect is replayed. A second readiness timeout fails the turn exactly
+ * as before; attached/resumed boxes and unconfirmed terminations never
+ * replace.
  */
 export declare function resumeBoxForTurn(services: SandboxResumeServices, ids: ResumeBoxIds, kind: "turn", holderId: TurnSandboxLeaseHolderId): Promise<ResumedTurnSandbox>;

package/dist/sandbox-routing.d.ts

@@ -2,7 +2,7 @@ import { type Settings } from "@opengeni/config";
 import { type Database, type EnrollmentRecord } from "@opengeni/db";
 import type { OpStreamOutputFrame } from "@opengeni/runtime/sandbox";
 import type { EventBus } from "@opengeni/events";
-import { type EstablishedSandboxSession, type RoutingSandboxFirstOperationObserver, type RoutingSandboxOperationObserver, type SelfhostedOpObserver, type SelfhostedConnectionBinding, type SelfhostedRelayConfig, type OpStreamJournal } from "@opengeni/runtime";
+import { type EstablishedSandboxSession, type RoutingSandboxFirstOperationObserver, type RoutingSandboxOperationObserver, type RoutingSandboxCaptureWaitObserver, type SelfhostedOpObserver, type SelfhostedConnectionBinding, type SelfhostedRelayConfig, type OpStreamJournal } from "@opengeni/runtime";
 export type RoutingWiringServices = {
     db: Database;
     settings: Settings;
@@ -15,6 +15,7 @@ export type RoutingWiringServices = {
     onOp?: SelfhostedOpObserver;
     /** Every physical routed provider call, across cloud and selfhosted homes. */
     onSandboxOperation?: RoutingSandboxOperationObserver;
+    onSandboxCaptureWait?: RoutingSandboxCaptureWaitObserver;
     /** The op-stream durable-resume journal (the Temporal adaptation from
      *  op-journal.ts): attach generation + settled-frontier persistence. Absent ⇒
      *  the runtime defaults (generation "1", no persistence) — tests / non-turn
@@ -27,11 +28,12 @@ export type RoutingWiringServices = {
         instanceId: string;
         leaseEpoch: number;
     }) => Promise<void>;
-    /** Called when a route repair replaces the worker's original home handle. */
+    /** Prepare and publish the worker's home handle. Rejection leaves the route
+     * unpublished; replacement preparation must settle before the first tool op. */
     onHomeSandboxRebound?: (input: {
         established: EstablishedSandboxSession;
         leaseEpoch: number;
-    }) => void;
+    }) => void | Promise<void>;
     /** Cancel bounded capture waits with the owning turn activity. */
     waitSignal?: AbortSignal;
 };

package/dist/sandbox-snapshot-diagnostics.d.ts

@@ -0,0 +1,20 @@
+import { type Observability } from "@opengeni/observability";
+type SafeSnapshotError = {
+    errorClass: "SnapshotOperationError";
+    errorCode: "snapshot_operation_failed";
+    status?: number;
+    origin: "sandbox-resume" | "worker-lifecycle";
+    causeName?: string;
+    integrityCode?: string;
+    providerErrorName?: string;
+    providerGrpcCode?: number;
+    providerHttpStatus?: number;
+    providerRetryable?: boolean;
+};
+export declare function safeSnapshotError(error: unknown, origin?: SafeSnapshotError["origin"]): SafeSnapshotError;
+export declare function warnDrainSnapshotFailure(observability: Pick<Observability, "warn">, message: string, error: unknown, lease: {
+    workspaceId?: string;
+    sandboxGroupId: string;
+    leaseEpoch: number;
+}): void;
+export {};

package/dist/session-event-fanout.d.ts

@@ -0,0 +1,6 @@
+import type { SessionEvent } from "@opengeni/contracts";
+import type { EventBus } from "@opengeni/events";
+/** Best-effort live fanout of events that are already durable, grouped by
+ * session. Durable catch-up remains the source of truth, so failures are only
+ * reported. */
+export declare function publishDurableSessionEvents(bus: EventBus | null | undefined, workspaceId: string, events: readonly SessionEvent[] | undefined, onError?: (error: unknown) => void): Promise<void>;

package/dist/warm-capture-holder.d.ts

@@ -0,0 +1,3 @@
+import type { LeaseSnapshot } from "@opengeni/db";
+/** Physical capture retention is not permission to heartbeat or execute a closed turn. */
+export declare function retainsHolderForWarmCapture(lease: Pick<LeaseSnapshot, "liveness" | "leaseEpoch" | "archiveCapture"> | null, expectedEpoch: number, now?: number): boolean;

package/dist/worker-service-lifecycle.d.ts

@@ -22,3 +22,15 @@ export declare function createWorkerServiceLifecycle(input: {
     closeOwnedResources: () => Promise<void>;
     onReady?: () => void;
 }): WorkerServiceLifecycle;
+/** A cleanup stall first drains peers through WORKER_SHUTDOWN. The standalone
+ * process owns the final exit, because Temporal cannot kill JavaScript writers.
+ * Embedded hosts may supply their own termination policy at this same edge. */
+export declare function createWorkerCleanupContainment(input: {
+    drain: () => boolean;
+    terminate?: () => void;
+    observability: Observability;
+    timeoutMs?: number;
+}): {
+    request(): void;
+    finished(): void;
+};