@opengeni/worker-bundle 1.4.0 → 1.4.2

package/src/observability-metrics.ts

@@ -1071,6 +1071,8 @@ export const SANDBOX_INVENTORY_PROJECTION_DOMAINS = [
   "retained_processes",
   "expired_drains",
   "opensandbox_kubernetes",
+  "modal_provider",
+  "interaction_idle",
 ] as const;
 
 export type SandboxInventoryProjectionDomain =
@@ -1228,6 +1230,49 @@ export function recordVerifiedSignupTrialDeploymentFlagGauge(
   });
 }
 
+/**
+ * Provider-side Modal inventory reconciled against live leases by the orphan
+ * sweep: `running` = every running box in the app, `unleased` = running boxes
+ * no live lease protects (orphans, counted before termination), and
+ * `lease_missing_instance` = live (warming/warm/draining) leases whose exact
+ * provider instance is no longer running (zombies).
+ */
+export function recordModalSandboxInventoryGauges(
+  observability: Observability,
+  inventory: { running: number; unleased: number; liveLeaseInstancesMissing: number },
+): void {
+  const states = {
+    running: inventory.running,
+    unleased: inventory.unleased,
+    lease_missing_instance: inventory.liveLeaseInstancesMissing,
+  };
+  for (const [state, value] of Object.entries(states)) {
+    observability.setGauge({
+      name: "opengeni_modal_sandbox_inventory",
+      help: "Running Modal sandboxes reconciled against live sandbox leases by the orphan sweep.",
+      labels: { state },
+      value: Math.max(0, value),
+    });
+  }
+}
+
+/** Warm leases held only by Browser/Computer interaction holders, by time since
+ * the newest interaction activity. Such boxes never drain on their own until the
+ * interaction session ends or the provider deadline. */
+export function recordInteractionOnlyLeaseGauges(
+  observability: Observability,
+  counts: Record<string, number>,
+): void {
+  for (const [idleBucket, value] of Object.entries(counts)) {
+    observability.setGauge({
+      name: "opengeni_sandbox_leases_interaction_only",
+      help: "Warm sandbox leases held only by Browser/Computer sessions, by time since last interaction activity.",
+      labels: { idle_bucket: idleBucket },
+      value: Math.max(0, value),
+    });
+  }
+}
+
 export function recordSandboxOrphansTerminated(observability: Observability, count: number): void {
   if (count <= 0) {
     return;
@@ -1629,7 +1674,6 @@ export type TurnStartupPhase =
   | "post_tool_preparation"
   | "agent_construction"
   | "post_agent_preparation"
-  | "programmatic_operation_recovery"
   | "file_materialization"
   | "history_preparation"
   | "history_system_update_load"
@@ -2104,34 +2148,61 @@ function contentDeltaClass(type: SessionEventType): StreamDeltaClass | null {
   return null;
 }
 
-// TTFT and inter-delta live on a human-perceptible scale (tens of ms to a few
-// seconds), so they get their own SHORT buckets — the default duration buckets
-// (which run to 3600s) would collapse every real streaming value into one bucket.
-const STREAM_TTFT_BUCKETS = [0.02, 0.05, 0.1, 0.2, 0.35, 0.5, 0.75, 1, 1.5, 2, 3, 5, 10];
+// TTFT lives on a human-perceptible scale, but reasoning models legitimately
+// think for tens of seconds before their first streamed token. The finite
+// buckets therefore run well past 10s: a top bucket at 10s made every slower
+// first token land in +Inf, so p50/p99 saturated at exactly "10" and hid how
+// slow the tail really was.
+const STREAM_TTFT_BUCKETS = [
+  0.02, 0.05, 0.1, 0.2, 0.35, 0.5, 0.75, 1, 1.5, 2, 3, 5, 10, 15, 20, 30, 45, 60, 90, 120, 180, 300,
+];
 const STREAM_INTER_DELTA_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.35, 0.5, 1, 2, 5];
+// OpenGeni's own per-request work before the provider sees bytes: admission,
+// durable history/audit checkpoints, request build. Normally milliseconds to a
+// second; tens of seconds means our database or consumer is the bottleneck.
+const MODEL_REQUEST_PRE_DISPATCH_BUCKETS = [
+  0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60, 120,
+];
+
+/** First streamed content of a provider response. `any` is the first reasoning
+ *  or answer delta; `text` is the first answer-text delta. They overlap — never
+ *  sum across `content`. */
+export type ProviderFirstContent = "any" | "text";
 
 /**
- * Per-turn stream-timing tracker fed every normalized runtime event in push order.
- * It emits two model-responsiveness SLIs from the worker's seat on the stream:
+ * Per-stream timing tracker fed every normalized runtime event in push order,
+ * plus two producer-side hooks. It separates OUR latency from the PROVIDER's:
  *
- *   - `opengeni_stream_ttft_seconds{provider}` — time from a model (re)start to its
- *     first streamed content delta. The anchor starts at construction (≈ runStream
- *     start, so the first observation is "how long until text appears") and re-arms
- *     on every non-content event (a tool call, a completed message, a usage frame),
- *     so a post-tool response measures the model's restart latency, NOT our own
- *     tool-execution time.
+ *   - `opengeni_stream_ttft_seconds{provider}` — user-perceived (re)start latency:
+ *     time from a stream start / structural boundary (tool call, tool result,
+ *     completed message, usage frame) to the first streamed content delta. It
+ *     mixes our between-call work with provider time; keep it for the absolute
+ *     dashboard view, not for provider alerting.
+ *   - `opengeni_model_request_pre_dispatch_seconds{provider}` — OUR per-request
+ *     latency: SDK model entry (the first admission check) to the literal
+ *     provider dispatch. Covers admission, the durable history checkpoint,
+ *     credit revalidation, the durable request audit and request build.
+ *   - `opengeni_model_provider_ttft_seconds{provider,content}` — the PROVIDER's
+ *     latency: literal dispatch to the first streamed `any` (reasoning or text)
+ *     delta and to the first answer `text` delta. Includes network, provider
+ *     queueing, prompt prefill and reasoning before the first streamed token.
  *   - `opengeni_stream_inter_delta_gap_seconds{provider,class}` — gap between
- *     consecutive content deltas of the SAME class. The run resets on any
- *     non-content event so a gap never spans a tool call or a model boundary — it
- *     measures only the choppiness of a live token stream.
+ *     consecutive content deltas of the SAME class, reset at every structural
+ *     event so a gap never spans a tool call or a model boundary.
  *
- * Purely observational and clock-injectable; it never touches the events it sees.
+ * Dispatch timing deliberately survives structural events: the consumer may
+ * still be persisting the previous response's tool results when the producer
+ * dispatches the next request, but the next request's first delta always
+ * follows its own dispatch. Purely observational and clock-injectable.
  */
 export class StreamTimingMetrics {
   private readonly now: () => number;
   private ttftAnchor: number;
   private ttftArmed = true;
   private readonly lastDeltaAt = new Map<StreamDeltaClass, number>();
+  private modelEntryAt: number | null = null;
+  private dispatchedAt: number | null = null;
+  private readonly providerFirstRecorded = new Set<ProviderFirstContent>();
 
   constructor(
     private readonly observability: Observability,
@@ -2141,6 +2212,31 @@ export class StreamTimingMetrics {
     this.ttftAnchor = this.now();
   }
 
+  /** Producer side: the SDK entered a model request (first admission check).
+   *  Admission can be re-entered for the same request; keep the earliest. */
+  onModelRequestEntry(): void {
+    if (this.modelEntryAt === null) this.modelEntryAt = this.now();
+  }
+
+  /** Producer side: request bytes are about to leave this process. */
+  onProviderDispatch(): void {
+    const at = this.now();
+    if (this.modelEntryAt !== null) {
+      this.observability.observeHistogram({
+        name: "opengeni_model_request_pre_dispatch_seconds",
+        help: "Seconds of OpenGeni work from SDK model-request entry to literal provider dispatch.",
+        buckets: MODEL_REQUEST_PRE_DISPATCH_BUCKETS,
+        labels: { provider: this.options.provider },
+        value: Math.max(0, (at - this.modelEntryAt) / 1000),
+      });
+      this.modelEntryAt = null;
+    }
+    // A transport retry re-dispatches the same request; the first delta then
+    // belongs to the latest dispatch.
+    this.dispatchedAt = at;
+    this.providerFirstRecorded.clear();
+  }
+
   onEvent(type: SessionEventType): void {
     const deltaClass = contentDeltaClass(type);
     if (deltaClass === null) {
@@ -2156,13 +2252,17 @@ export class StreamTimingMetrics {
     if (this.ttftArmed) {
       this.observability.observeHistogram({
         name: "opengeni_stream_ttft_seconds",
-        help: "Seconds from a model (re)start to its first streamed content delta.",
+        help: "Seconds from a stream start or structural boundary to the next streamed content delta (user-perceived; includes OpenGeni between-call work).",
         buckets: STREAM_TTFT_BUCKETS,
         labels: { provider: this.options.provider },
         value: Math.max(0, (at - this.ttftAnchor) / 1000),
       });
       this.ttftArmed = false;
     }
+    if (this.dispatchedAt !== null) {
+      this.observeProviderFirst("any", at);
+      if (deltaClass === "message") this.observeProviderFirst("text", at);
+    }
     const last = this.lastDeltaAt.get(deltaClass);
     if (last !== undefined) {
       this.observability.observeHistogram({
@@ -2175,6 +2275,18 @@ export class StreamTimingMetrics {
     }
     this.lastDeltaAt.set(deltaClass, at);
   }
+
+  private observeProviderFirst(content: ProviderFirstContent, at: number): void {
+    if (this.dispatchedAt === null || this.providerFirstRecorded.has(content)) return;
+    this.providerFirstRecorded.add(content);
+    this.observability.observeHistogram({
+      name: "opengeni_model_provider_ttft_seconds",
+      help: "Seconds from literal provider dispatch to the first streamed content delta (any = reasoning or text, text = answer text).",
+      buckets: STREAM_TTFT_BUCKETS,
+      labels: { provider: this.options.provider, content },
+      value: Math.max(0, (at - this.dispatchedAt) / 1000),
+    });
+  }
 }
 
 // Batch shapes: sizes are small integers; durations are the append+publish round

package/src/sandbox-resume.ts

@@ -166,6 +166,9 @@ export type SandboxResumeServices = {
    * replacement. Production uses {@link freshSandboxReadinessReplacementDelayMs}.
    * It may be async so a test can observe the rolled-back lease in between. */
   freshSandboxReadinessReplacementDelayMs?: () => number | Promise<number>;
+  /** Test seam: runs immediately after the elected spawner published its box
+   * warm, before the final cancellation check. */
+  onSpawnedSandboxPublished?: () => void | Promise<void>;
   /**
    * The turn attempt's fresh-box readiness replacement budget. The lazy
    * provisioner may call resumeBoxForTurn again after a typed lease
@@ -438,10 +441,6 @@ export class SandboxLeaseInstanceLostError extends SandboxLeaseSupersededError {
 // user-facing and separate from the lease TTL heartbeat/reaper horizon.
 const WARMING_POLL_INTERVAL_MS = 250;
 
-async function sleep(ms: number): Promise<void> {
-  await new Promise<void>((resolve) => setTimeout(resolve, ms));
-}
-
 /**
  * A remote provider may return a sandbox handle before its command router
  * accepts the first exec. The upstream session's yieldTimeMs starts only after
@@ -1576,6 +1575,12 @@ async function resumeBoxForTurnOnce(
   if (acquired.role === "spawner") {
     const expectedEpoch = acquired.lease.leaseEpoch;
     let createdEstablished: EstablishedSandboxSession | null = null;
+    // Set once commitWarmingToWarm publishes this box. From then on the box is
+    // the group's shared, attachable workspace: a later cancellation (Pause,
+    // Steer, worker shutdown) only drops this attempt's holder. Terminating it
+    // here would hand the next attempt a warm lease naming a dead box, whose
+    // exact-id resume then records a lost workspace that never lost anything.
+    let published = false;
     let providerCreateOperationId: string | undefined;
     let providerCreateBindingKey: string | undefined;
     let rematerialization: {
@@ -2021,13 +2026,21 @@ async function resumeBoxForTurnOnce(
         await release();
         throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
       }
+      published = true;
       holderLeaseHeartbeat = {
         expectedEpoch: committed.lease.leaseEpoch,
         leaseTtlMs,
       };
+      await services.onSpawnedSandboxPublished?.();
       throwIfReleasedOrCancelled();
       return { established, leaseEpoch: committed.lease.leaseEpoch, release };
     } catch (error) {
+      if (published) {
+        // The published warm box stays for the replacement attempt to resume
+        // by exact provider id; ordinary idle drain owns its capture/teardown.
+        await release();
+        throw error;
+      }
       if (error instanceof SandboxLeaseSupersededError) {
         await terminateEstablishedSandbox(createdEstablished);
         await release();
@@ -2203,7 +2216,13 @@ async function waitForWarm(
   const deadline = Date.now() + settings.sandboxWarmingTimeoutMs;
   let instanceId: string | null = null;
   while (Date.now() < deadline) {
-    await sleep(WARMING_POLL_INTERVAL_MS);
+    // A cancelled waiter owns no box and must not keep polling for up to the
+    // warming budget: its activity finalizer joins this exact promise.
+    if (!(await sleepUnlessCancelled(WARMING_POLL_INTERVAL_MS, services.cancellationSignal))) {
+      throw services.cancellationSignal?.reason instanceof Error
+        ? services.cancellationSignal.reason
+        : new Error("Sandbox warming wait was cancelled with its owning turn attempt");
+    }
     const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
     if (!lease) {
       // Lease vanished (cold-reaped). Re-dispatch from scratch.

package/src/sandbox-routing.ts

@@ -40,6 +40,7 @@ import {
   readActiveSandbox,
   resolvePersonalMachineConnectionForAttempt,
   SandboxRetainedProcessPromotionFencedError,
+  SandboxRetainedProcessTerminalError,
   SandboxWorkspaceMutationOutputRejectedError,
   type Database,
   type EnrollmentRecord,
@@ -700,6 +701,17 @@ function afterRetainedProcessMutation(
           outcome,
         })
       ) {
+        if (error.retainedProcessTerminal) {
+          // Another authority (the reaper's exact-proof reconciliation) settled
+          // this retained process terminal between admission and settlement.
+          // The admission is physically settled and its output stays rejected;
+          // surface the durable terminal truth so routing reports completion
+          // instead of failing the turn. Nothing is replayed.
+          throw new SandboxRetainedProcessTerminalError(
+            error.retainedProcessTerminal.state,
+            error.retainedProcessTerminal.exitCode,
+          );
+        }
         throw new RoutingMutationOutputRejectedError(op, error.code, { cause: error });
       }
       throw error;