@opengeni/worker-bundle 1.4.0 → 1.4.4

package/dist/activities/agent-turn/errors.d.ts

@@ -246,6 +246,12 @@ export declare function collectErrorStrings(value: unknown, seen?: WeakSet<objec
  */
 export declare const STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE = "unable to connect. is the computer able to access the url?";
 export declare function isExactStatuslessUpstreamConnectivityMessage(message: string): boolean;
+/**
+ * A fetch-layer timeout: the runtime's DOMException `TimeoutError` (message
+ * "The operation timed out."), possibly wrapped by SDK/runner `cause` chains.
+ * Explicit cancellation (`AbortError`) is deliberately not matched.
+ */
+export declare function isTransportTimeoutError(error: unknown): boolean;
 export declare function isTransientProviderError(error: unknown): boolean;
 /**
  * Recognize an exhausted API-key provider quota (a daily or monthly allowance,

package/dist/activities/agent-turn/sandbox-provision.d.ts

@@ -48,6 +48,28 @@ export declare class SandboxDeadlineRotationError extends Error {
 export declare function turnOperationCancellationFailure(error: unknown): CancelledFailure | null;
 export declare function throwIfTurnOperationCancelled(signal: AbortSignal | undefined): void;
 export declare function waitForTurnOperation<T>(operation: Promise<T>, signal: AbortSignal | undefined, disposeLateResult: ((result: T) => Promise<void> | void) | undefined): Promise<T>;
+/**
+ * Upper bound for joining a cancelled attempt's in-flight provider establish.
+ * Worker shutdown cancels activities 5s after SIGTERM and force-stops 100s
+ * after it; this leaves room for the remaining finalization steps.
+ */
+export declare const PHYSICAL_SANDBOX_RESUME_DRAIN_TIMEOUT_MS = 60000;
+/** Register the PHYSICAL lease acquire/provider establish promise, not the
+ * cancellable wrapper awaiting it. */
+export declare function trackPhysicalSandboxResume<T>(inFlight: Set<Promise<unknown>>, operation: Promise<T>): Promise<T>;
+/**
+ * Join every lease acquire/provider establish this attempt started, even after
+ * the turn-facing wrapper rejected on cancellation. An establish has no
+ * portable abort: after cancellation it still finishes its own exact cleanup
+ * (terminate an unpublished box and roll warming back to cold, or keep a
+ * published box and drop its holder). If the activity returned first, a
+ * shutting-down worker exited mid-cleanup and stranded a warming lease naming a
+ * terminated box; the replacement attempt then waited out the warming TTL and
+ * inherited a lost-workspace verdict. This wait is deliberately independent of
+ * the finalizer cancellation signal (cancellation is what started it) and is
+ * bounded so an uncooperative provider call cannot pin the activity.
+ */
+export declare function drainPhysicalSandboxResumes(inFlight: Set<Promise<unknown>>, timeoutMs?: number): Promise<"none" | "settled" | "timed_out">;
 export declare function sleep(ms: number): Promise<void>;
 export declare function sandboxProvisionErrorChain(error: unknown): unknown[];
 export declare function firstProvisionError<T>(chain: readonly unknown[], constructor: abstract new (...args: never[]) => T): T | null;

package/dist/activities/agent-turn/turn-context.d.ts

@@ -78,6 +78,9 @@ export type SandboxRuntimeState = {
     }>;
     turnSandboxProvisioner: TurnSandboxProvisioner<ResumedTurnSandbox> | null;
     resumeManagedGroupBox: (() => Promise<ResumedTurnSandbox>) | null;
+    /** Physical resumeBoxForTurn promises still running for this attempt; the
+     * finalizer joins them so a cancelled establish completes its own cleanup. */
+    inFlightSandboxResumes: Set<Promise<unknown>>;
     prefetchedManagedBox: Promise<ResumedTurnSandbox> | null;
     prefetchedManagedBoxResult: ResumedTurnSandbox | null;
     setupBoxSession: unknown;
@@ -127,6 +130,9 @@ export type EventingState = {
     firstModelRequestPreparationRecorded: boolean;
     firstModelRequestCheckpointAt: number | null;
     initialModelWireDispatch: InitialModelWireDispatchClock;
+    /** Diagnostic only: the current stream's timing hook for a native transport's
+     *  literal provider dispatch. Never joins, fences or fails the request. */
+    providerDispatchObserver: (() => void) | null;
     companyBrainContextContributions: readonly ModelContextContributionSummary[] | null;
     /** Skill ids in this turn's frozen, model-visible Skill index; telemetry only. */
     modelVisibleSkillIds: ReadonlySet<string> | null;

package/dist/activities/agent-turn/web-search.d.ts

@@ -18,6 +18,9 @@ export declare function turnWebSearchPlan(resolvedModel: {
     hostedWebSearch: boolean;
     providerTools: WebSearchProviderToolName[];
 };
+/** Closed outcome set for `opengeni_web_search_calls_total`. */
+export type WebSearchCallOutcome = "ok" | "provider_error" | "provider_retryable" | "billing_refused" | "error";
+type WebSearchObservability = Pick<Observability, "warn" | "incrementCounter" | "observeHistogram">;
 /**
  * The `web_search` and `web_fetch` attempt tools for one turn. The caller
  * decides which names the turn gets ({@link webSearchToolPlan}); this only
@@ -28,6 +31,7 @@ export declare function webSearchToolDefinitions(input: {
     tools: readonly WebSearchProviderToolName[];
     scope: WebSearchCallScope;
     billing: WebSearchBilling;
-    observability: Pick<Observability, "warn">;
+    observability: WebSearchObservability;
     fetch?: typeof fetch;
 }): AttemptToolDefinition[];
+export {};

package/dist/activities/codemode-dispatcher.d.ts

@@ -43,7 +43,15 @@ export declare class CodemodeAttemptDispatcher {
         beginWait(): (accepted: boolean) => void;
     } | undefined);
     start(): void;
+    /**
+     * Join-on-use recovery before the first model request. The journal read needs
+     * no tool environment, so a turn with nothing unfinished returns at once and
+     * never waits on lazy MCP preparation. Only a stored waiting/queued/running
+     * operation awaits the prepared dispatcher to adopt and execute it.
+     */
+    static resumeApproved(db: Database, scope: CodemodeDispatcherScope, callerSubjectId: string, decisionOperationId: string | undefined, prepared: () => Promise<CodemodeAttemptDispatcher | null>): Promise<CodemodeOperation[]>;
     resumeApproved(callerSubjectId: string, decisionOperationId?: string): Promise<CodemodeOperation[]>;
+    private resumeOperations;
     close(reason?: string): Promise<void>;
     private execute;
     private executeClaimed;

package/dist/activities/programmatic-approvals.d.ts

@@ -8,20 +8,3 @@ export declare function programmaticApproval(operation: CodemodeOperation): {
     requestId: string | undefined;
 };
 export declare function programmaticContinuationNote(operations: readonly CodemodeOperation[]): string | undefined;
-/**
- * Recover committed programmatic approval waits/results before the first model
- * request without turning lazy MCP preparation into a first-request barrier.
- *
- * Only eager servers may gate the first request. Resuming a stored operation
- * needs the prepared attempt tool environment, so this waits for full tool
- * preparation only when there is something to resume: an approval decision
- * trigger, or a cheap durable read showing unfinished operations for this
- * logical turn (which covers every recovery that has stored work). Every other
- * turn sends its first request without waiting on any lazy server.
- */
-export declare function recoverProgrammaticOperationsBeforeFirstRequest(input: {
-    approvalDecisionId: string | undefined;
-    hasUnfinishedOperations: () => Promise<boolean>;
-    toolPreparationReady: Promise<void> | null;
-    resumeApproved: (decisionOperationId: string | undefined) => Promise<CodemodeOperation[]> | undefined;
-}): Promise<CodemodeOperation[]>;

package/dist/activities/sandbox-lease.d.ts

@@ -209,6 +209,15 @@ export declare function probeRetainedProcessAtProvider(settings: ActivityService
 export declare function classifyRetainedProcessPollResult(result: unknown, providerSessionId: number, source?: object): RetainedProcessProbeResult;
 export declare function reconcileModalProviderCreates(db: ActivityServices["db"], settings: ActivityServices["settings"], observability: ActivityServices["observability"], discover?: typeof findModalProviderCreateReceipt): Promise<void>;
 export declare function modalOrphanTerminationStillEligible(latest: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>, candidate: ModalOrphanSweepTermination): boolean;
+/** Live leases still WARM on an instance the complete listing did not show.
+ * A draining lease legitimately loses its box mid-drain (capture, terminate,
+ * then cold), and warming has not published a box yet, so only a warm lease
+ * whose exact box is gone is a zombie. */
+export declare function countWarmModalLeasesMissingInstance(latest: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>, missingInstanceIds: string[]): number;
+/** Whether no live lease owns the candidate box by exact instance or by its
+ * attribution tags. Unlike termination eligibility this ignores the pending
+ * create postponement: a postponed orphan is still running without a lease. */
+export declare function modalOrphanCandidateStillUnowned(latest: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>, candidate: ModalOrphanSweepTermination): boolean;
 export type SandboxDrainReason = "idle" | "provider_deadline" | "operator" | "teardown_claim" | "rotation_requested";
 /** The bounded drain reason carried on `sandbox.box.terminated`: the lease's
  * rotation reason when a rotation was requested, otherwise the ordinary idle

package/dist/{activities-control-D5KVEPHI.js → activities-control-VRFS65FT.js}

@@ -34,7 +34,7 @@ import {
   warnDrainSnapshotFailure,
   withFirstPartyTools,
   workflowIdForSession
-} from "./chunk-BMTWULF7.js";
+} from "./chunk-BICGFGO7.js";
 import {
   inspectOpenSandboxKubernetesInventory,
   recordActiveUserGauges,
@@ -42,7 +42,9 @@ import {
   recordCreditGrantGauges,
   recordCreditMicros,
   recordExpiredDrainingSandboxLeaseGauges,
+  recordInteractionOnlyLeaseGauges,
   recordManagedAuthNewSignupsSwitchGauge,
+  recordModalSandboxInventoryGauges,
   recordOpenSandboxKubernetesInventoryGauges,
   recordRetainedProcessInventoryGauges,
   recordRetainedProcessReconciliation,
@@ -62,7 +64,7 @@ import {
   recordVerifiedSignupTrialSwitchGauge,
   recordWorkerDeathRecoveryMetrics,
   runtimeMetricsHooksForObservability
-} from "./chunk-FF4HPY7L.js";
+} from "./chunk-HVNNCWEU.js";
 import "./chunk-PZ5AY32C.js";
 
 // src/activities/knowledge-indexing.ts
@@ -2790,6 +2792,7 @@ import {
   recordRecoveredModalProviderCreate,
   markWarmBillingStopCutoff,
   listMeterableWarmLeases,
+  countInteractionOnlyWarmLeasesByIdle,
   listSandboxViewerForceDrainWorkspaceIds,
   markSandboxCheckpointArtifactDeletePending as markSandboxCheckpointArtifactDeletePending2,
   persistDrainSnapshot,
@@ -4599,6 +4602,17 @@ async function refreshQueueLeaseAndCreditGauges(db, settings, observability, ins
         );
       }
     ),
+    refreshSandboxInventoryGauge(
+      observability,
+      "interaction_idle",
+      "interaction-only-lease",
+      async () => {
+        recordInteractionOnlyLeaseGauges(
+          observability,
+          await countInteractionOnlyWarmLeasesByIdle(db, await listMeterableWarmLeases(db))
+        );
+      }
+    ),
     refreshSandboxInventoryGauge(observability, "expired_drains", "expired-draining", async () => {
       recordExpiredDrainingSandboxLeaseGauges(
         observability,
@@ -4751,6 +4765,14 @@ function modalOrphanTerminationStillEligible(latest, candidate) {
   if (latest.some((lease) => lease.liveness === "warming" && lease.instanceId === null)) {
     return false;
   }
+  return modalOrphanCandidateStillUnowned(latest, candidate);
+}
+function countWarmModalLeasesMissingInstance(latest, missingInstanceIds) {
+  return missingInstanceIds.filter(
+    (instanceId) => latest.some((lease) => lease.liveness === "warm" && lease.instanceId === instanceId)
+  ).length;
+}
+function modalOrphanCandidateStillUnowned(latest, candidate) {
   if (latest.some((lease) => lease.instanceId === candidate.sandboxId)) {
     return false;
   }
@@ -4786,6 +4808,27 @@ async function sweepModalOrphansForConfiguredBackend(settings, db, observability
       return modalOrphanTerminationStillEligible(latest, candidate);
     }
   });
+  if (result.inventory.complete) {
+    try {
+      const latest = await listLiveModalSandboxLeaseAttributions(db);
+      recordModalSandboxInventoryGauges(observability, {
+        running: result.inventory.running,
+        unleased: result.inventory.unterminated.filter(
+          (candidate) => modalOrphanCandidateStillUnowned(latest, candidate)
+        ).length,
+        liveLeaseInstancesMissing: countWarmModalLeasesMissingInstance(
+          latest,
+          result.inventory.missingLiveLeaseInstanceIds
+        )
+      });
+      recordSandboxInventoryProjectionSuccess(observability, "modal_provider");
+    } catch (error) {
+      recordSandboxInventoryProjectionFailure(observability, "modal_provider");
+      observability.warn("sandbox reaper: Modal inventory ownership re-read failed", {
+        error: error instanceof Error ? error.message : String(error)
+      });
+    }
+  }
   for (const terminated of result.terminated) {
     observability.warn("sandbox reaper: terminated Modal orphan sandbox", {
       sandboxId: terminated.sandboxId,
@@ -5174,15 +5217,18 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
     }
   }
   const providerMissing = providerMissingBeforeCapture || (typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture);
-  const { wentCold, backgroundCommandEvents } = await confirmDrainCold(db, {
-    accountId,
-    workspaceId: row.workspaceId,
-    sandboxGroupId: row.sandboxGroupId,
-    expectedEpoch: row.leaseEpoch,
-    ...captureClaim ? { expectedCaptureId: captureClaim.id } : {},
-    providerMissingBeforeCapture: providerMissing,
-    idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs
-  });
+  const { wentCold, unpublishedProviderLost, backgroundCommandEvents } = await confirmDrainCold(
+    db,
+    {
+      accountId,
+      workspaceId: row.workspaceId,
+      sandboxGroupId: row.sandboxGroupId,
+      expectedEpoch: row.leaseEpoch,
+      ...captureClaim ? { expectedCaptureId: captureClaim.id } : {},
+      providerMissingBeforeCapture: providerMissing,
+      idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs
+    }
+  );
   await publishDurableSessionEvents(bus, row.workspaceId, backgroundCommandEvents, (error) => {
     observability.warn("sandbox reaper: contained command event fanout failed", {
       sandboxGroupId: row.sandboxGroupId,
@@ -5190,7 +5236,7 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
     });
   });
   if (wentCold) {
-    if (providerMissing) {
+    if (providerMissing && !unpublishedProviderLost) {
       recordSandboxProviderMissingBeforeCapture(observability, backend);
     }
     if (lease.unobservableCommandDrainIds?.length) {
@@ -8813,4 +8859,4 @@ export {
   createControlActivities,
   createControlActivitiesFromServices
 };
-//# sourceMappingURL=activities-control-D5KVEPHI.js.map
+//# sourceMappingURL=activities-control-VRFS65FT.js.map