@opengeni/worker-bundle 1.0.1 → 2.0.0

package/dist/observability-metrics.d.ts

@@ -4,7 +4,7 @@ import type { EventLogger } from "@opengeni/events";
 import type { Observability } from "@opengeni/observability";
 import type { CompanyBrainContributionReceipt } from "./model-context-contributions.js";
 import { type OpenSandboxKubernetesInventory } from "./opensandbox-kubernetes-inventory.js";
-import { type RuntimeMetricsHooks, type SelfhostedOpObservation, type SelfhostedOpObserver } from "@opengeni/runtime";
+import { type RuntimeMetricsHooks, type SelfhostedOpObservation, type SelfhostedOpObserver, type ToolPreparationPhaseMeasurement } from "@opengeni/runtime";
 export type TurnOutcome = "completed" | "failed" | "cancelled" | "recovering";
 export type WorkerDeathRecoveryOutcome = "recovering" | "exhausted";
 export type CreditMicrosKind = "usage" | "grant" | "topup" | "refund";
@@ -220,7 +220,7 @@ export declare function recordModelRequestPhase(observability: Observability, in
     outcome?: "completed" | "failed" | "timed_out";
     durationSeconds: number;
 }): void;
-export type TurnStartupPhase = "claim_and_policy" | "turn_start_settlement" | "credential_selection" | "runtime_preparation" | "sandbox_establish" | "file_resolution" | "tool_context_preparation" | "tool_preparation" | "tool_server_construction" | "tool_required_connect" | "tool_optional_connect" | "tool_attempt_catalog_build" | "tool_attempt_catalog_persist" | "tool_workspace_gateway_catalog_build" | "post_tool_preparation" | "agent_construction" | "post_agent_preparation" | "file_materialization" | "history_preparation" | "history_system_update_load" | "history_current_attachment_resolution" | "history_durable_history_load" | "history_sandbox_envelope_load" | "history_canonical_projection" | "history_provider_projection" | "history_attachment_ref_projection" | "history_screenshot_materialization" | "history_model_attachment_projection" | "history_runtime_input_assembly" | "history_artifact_candidate_scan" | "history_generated_image_materialization" | "history_position_load" | "owned_sandbox_setup" | "provider_dispatch" | "model_request_preparation" | "model_sdk_serialization" | "model_prepare_sandbox_agent_preparation" | "model_prepare_sandbox_agent_manifest_inventory" | "model_prepare_sandbox_session_manifest_inventory" | "model_prepare_sandbox_manifest_apply" | "model_prepare_sandbox_entry_materialization" | "model_prepare_sandbox_running_check" | "model_prepare_sandbox_start" | "model_prepare_sandbox_client_create" | "model_prepare_sandbox_client_resume" | "model_prepare_sandbox_client_delete" | "model_prepare_sandbox_client_state_serialize" | "model_prepare_sandbox_client_reuse_check" | "model_prepare_sandbox_workspace_mutation_admission" | "model_prepare_sandbox_workspace_mutation_provider" | "model_prepare_sandbox_workspace_mutation_settlement" | "model_prepare_sandbox_first_routed_resolution_other" | "model_prepare_sandbox_first_routed_mutation_admission" | "model_prepare_sandbox_first_routed_provider_operation" | "model_prepare_sandbox_first_routed_mutation_settlement" | "model_prepare_sandbox_first_routed_other" | "model_prepare_sandbox_snapshot_wait" | "model_prepare_runner_before_first_sandbox_operation" | "model_prepare_sdk_after_first_sandbox_operation" | "model_prepare_runner_before_mcp_tools" | "model_prepare_mcp_tools_snapshot" | "model_prepare_mcp_tools_before_input_filter" | "model_prepare_mcp_tools_before_repository_skill_discovery" | "model_prepare_repository_skill_discovery" | "model_prepare_repository_skill_discovery_before_input_filter" | "model_prepare_input_filter_base" | "model_prepare_input_filter_genesis" | "model_prepare_input_filter_host" | "model_prepare_input_filter_tool_output" | "model_prepare_input_filter_modality" | "model_prepare_input_filter_context" | "model_prepare_responses_input_conversion" | "model_prepare_responses_request_build" | "model_credential_resolution" | "model_wire_normalization" | "model_request_audit" | "stream_bootstrap";
+export type TurnStartupPhase = "claim_and_policy" | "turn_start_settlement" | "credential_selection" | "runtime_preparation" | "sandbox_establish" | "file_resolution" | "tool_context_preparation" | "tool_preparation" | "tool_server_construction" | "tool_required_connect" | "tool_optional_connect" | "tool_attempt_catalog_build" | "tool_attempt_catalog_persist" | "tool_workspace_gateway_catalog_build" | "post_tool_preparation" | "agent_construction" | "post_agent_preparation" | "file_materialization" | "history_preparation" | "history_system_update_load" | "history_current_attachment_resolution" | "history_durable_history_load" | "history_sandbox_envelope_load" | "history_canonical_projection" | "history_provider_projection" | "history_attachment_ref_projection" | "history_screenshot_materialization" | "history_model_attachment_projection" | "history_runtime_input_assembly" | "history_artifact_candidate_scan" | "history_generated_image_materialization" | "history_position_load" | "owned_sandbox_setup" | "runtime_stream_initialization" | "model_request_preparation" | "model_sdk_serialization" | "model_prepare_sandbox_agent_preparation" | "model_prepare_sandbox_agent_manifest_inventory" | "model_prepare_sandbox_session_manifest_inventory" | "model_prepare_sandbox_manifest_apply" | "model_prepare_sandbox_entry_materialization" | "model_prepare_sandbox_running_check" | "model_prepare_sandbox_start" | "model_prepare_sandbox_client_create" | "model_prepare_sandbox_client_resume" | "model_prepare_sandbox_client_delete" | "model_prepare_sandbox_client_state_serialize" | "model_prepare_sandbox_client_reuse_check" | "model_prepare_sandbox_workspace_mutation_admission" | "model_prepare_sandbox_workspace_mutation_provider" | "model_prepare_sandbox_workspace_mutation_settlement" | "model_prepare_sandbox_first_routed_resolution_other" | "model_prepare_sandbox_first_routed_mutation_admission" | "model_prepare_sandbox_first_routed_provider_operation" | "model_prepare_sandbox_first_routed_mutation_settlement" | "model_prepare_sandbox_first_routed_other" | "model_prepare_sandbox_snapshot_wait" | "model_prepare_runner_before_first_sandbox_operation" | "model_prepare_sdk_after_first_sandbox_operation" | "model_prepare_runner_before_mcp_tools" | "model_prepare_mcp_tools_snapshot" | "model_prepare_mcp_tools_before_input_filter" | "model_prepare_mcp_tools_before_repository_skill_discovery" | "model_prepare_repository_skill_discovery" | "model_prepare_repository_skill_discovery_before_input_filter" | "model_prepare_input_filter_base" | "model_prepare_input_filter_genesis" | "model_prepare_input_filter_host" | "model_prepare_input_filter_tool_output" | "model_prepare_input_filter_modality" | "model_prepare_input_filter_context" | "model_prepare_responses_input_conversion" | "model_prepare_responses_request_build" | "model_credential_resolution" | "model_wire_normalization" | "model_request_audit" | "stream_bootstrap";
 export type TurnStartupOutcome = "completed" | "failed";
 export type TurnStartupMilestone = "queue" | "provider_dispatch" | "first_byte";
 export type TurnStartupCache = "hit" | "miss" | "disabled" | "none";
@@ -233,7 +233,8 @@ export type SandboxLogicalProvisionOutcome = "completed" | "expected_transition"
 export type SandboxProvisionAttemptOutcome = "completed" | "retrying" | "failed";
 export declare function turnStartupCountBucket(count: number | null): TurnStartupCountBucket;
 /**
- * Measure the critical path from a durable turn start to provider dispatch.
+ * Measure startup operations, including nested and parallel work. These are
+ * not additive critical-path intervals; use milestones for elapsed latency.
  * Every label is a closed or configuration-derived enum; high-cardinality turn,
  * session, credential, connection, file, and model identifiers are forbidden.
  */
@@ -245,6 +246,13 @@ export declare function recordTurnStartupPhase(observability: Observability, inp
     durationSeconds: number;
     count?: number | null;
     cache?: TurnStartupCache;
+    /** Trace-only lookup for the completed claim while its execution root is still open. */
+    executionCorrelationId?: string;
+}): void;
+/** Background MCP preparation must not inflate startup phase distributions. */
+export declare function recordToolPreparationPhase(observability: Observability, input: ToolPreparationPhaseMeasurement & {
+    provider: string;
+    backend: string;
 }): void;
 /**
  * Explain why a turn did or did not keep provider sandbox work off the

package/dist/retained-process-diagnostics.d.ts

@@ -0,0 +1,4 @@
+import { type SandboxRetainedProcess } from "@opengeni/db";
+import type { Observability } from "@opengeni/observability";
+/** Protected cause locations and schema facts; public logs receive only the diagnostic key. */
+export declare function warnRetainedProcessProofFailure(observability: Pick<Observability, "recordFailureDiagnostic" | "warn">, error: unknown, process: Pick<SandboxRetainedProcess, "id" | "sessionId" | "ownerAttemptId" | "ownerTurnId" | "reconcileAttempts">): void;

package/dist/sandbox-resume.d.ts

@@ -2,7 +2,8 @@ import { type Settings } from "@opengeni/config";
 import { readLease, reinstateTurnLeaseHolder, SandboxLeaseSupersededError, type Database, type LeaseHolderKind } from "@opengeni/db";
 import { establishSandboxSessionFromEnvelope, type EstablishedSandboxSession, type RuntimeMetricsHooks } from "@opengeni/runtime";
 import { type ObjectStorage } from "@opengeni/storage";
-import type { Observability } from "@opengeni/observability";
+import { type Observability } from "@opengeni/observability";
+export { safeSnapshotError } from "./sandbox-snapshot-diagnostics.js";
 export type ResumeHolderKind = LeaseHolderKind;
 /**
  * The durable identity of a turn holding a shared sandbox lease.
@@ -22,9 +23,6 @@ export type SandboxResumeServices = {
     db: Database;
     settings: Settings;
     objectStorage?: ObjectStorage | null;
-    /** Exact settings before a verified rig provider image overlaid the logical
-     * pack/deployment image. Fresh-create NotFound fallback must preserve an
-     * ID-only logical base instead of selecting the provider default. */
     logicalFallbackSettings?: Settings;
     sandboxMetrics?: RuntimeMetricsHooks;
     /** Structured logger for lifecycle decisions (holder release reasons). Falls
@@ -163,14 +161,6 @@ export declare class SandboxLeaseInstanceLostError extends SandboxLeaseSupersede
 export declare function waitForSandboxExecReadiness(established: EstablishedSandboxSession, timeoutMs?: number, identity?: {
     sandboxGroupId?: string | null;
 }): Promise<void>;
-export declare function safeSnapshotError(error: unknown): {
-    errorClass: "SnapshotOperationError";
-    errorCode: "snapshot_operation_failed";
-    status?: number;
-    origin: "sandbox-resume";
-    causeName?: string;
-    integrityCode?: string;
-};
 export declare function waitForWarmSnapshot(snapshot: Promise<unknown>, timeoutMs: number, signal?: AbortSignal): Promise<boolean>;
 export type SandboxDeadlineRotationCheckpointResult = {
     /** The exact established epoch still carries a requested rotation and a

package/dist/sandbox-routing.d.ts

@@ -2,7 +2,7 @@ import { type Settings } from "@opengeni/config";
 import { type Database, type EnrollmentRecord } from "@opengeni/db";
 import type { OpStreamOutputFrame } from "@opengeni/runtime/sandbox";
 import type { EventBus } from "@opengeni/events";
-import { type EstablishedSandboxSession, type RoutingSandboxFirstOperationObserver, type RoutingSandboxOperationObserver, type SelfhostedOpObserver, type SelfhostedConnectionBinding, type SelfhostedRelayConfig, type OpStreamJournal } from "@opengeni/runtime";
+import { type EstablishedSandboxSession, type RoutingSandboxFirstOperationObserver, type RoutingSandboxOperationObserver, type RoutingSandboxCaptureWaitObserver, type SelfhostedOpObserver, type SelfhostedConnectionBinding, type SelfhostedRelayConfig, type OpStreamJournal } from "@opengeni/runtime";
 export type RoutingWiringServices = {
     db: Database;
     settings: Settings;
@@ -15,6 +15,7 @@ export type RoutingWiringServices = {
     onOp?: SelfhostedOpObserver;
     /** Every physical routed provider call, across cloud and selfhosted homes. */
     onSandboxOperation?: RoutingSandboxOperationObserver;
+    onSandboxCaptureWait?: RoutingSandboxCaptureWaitObserver;
     /** The op-stream durable-resume journal (the Temporal adaptation from
      *  op-journal.ts): attach generation + settled-frontier persistence. Absent ⇒
      *  the runtime defaults (generation "1", no persistence) — tests / non-turn
@@ -27,11 +28,12 @@ export type RoutingWiringServices = {
         instanceId: string;
         leaseEpoch: number;
     }) => Promise<void>;
-    /** Called when a route repair replaces the worker's original home handle. */
+    /** Prepare and publish the worker's home handle. Rejection leaves the route
+     * unpublished; replacement preparation must settle before the first tool op. */
     onHomeSandboxRebound?: (input: {
         established: EstablishedSandboxSession;
         leaseEpoch: number;
-    }) => void;
+    }) => void | Promise<void>;
     /** Cancel bounded capture waits with the owning turn activity. */
     waitSignal?: AbortSignal;
 };

package/dist/sandbox-snapshot-diagnostics.d.ts

@@ -0,0 +1,20 @@
+import { type Observability } from "@opengeni/observability";
+type SafeSnapshotError = {
+    errorClass: "SnapshotOperationError";
+    errorCode: "snapshot_operation_failed";
+    status?: number;
+    origin: "sandbox-resume" | "worker-lifecycle";
+    causeName?: string;
+    integrityCode?: string;
+    providerErrorName?: string;
+    providerGrpcCode?: number;
+    providerHttpStatus?: number;
+    providerRetryable?: boolean;
+};
+export declare function safeSnapshotError(error: unknown, origin?: SafeSnapshotError["origin"]): SafeSnapshotError;
+export declare function warnDrainSnapshotFailure(observability: Pick<Observability, "warn">, message: string, error: unknown, lease: {
+    workspaceId?: string;
+    sandboxGroupId: string;
+    leaseEpoch: number;
+}): void;
+export {};

package/dist/warm-capture-holder.d.ts

@@ -0,0 +1,3 @@
+import type { LeaseSnapshot } from "@opengeni/db";
+/** Physical capture retention is not permission to heartbeat or execute a closed turn. */
+export declare function retainsHolderForWarmCapture(lease: Pick<LeaseSnapshot, "liveness" | "leaseEpoch" | "archiveCapture"> | null, expectedEpoch: number, now?: number): boolean;