@opengeni/worker-bundle 1.2.1-canary.37222965366001 → 1.4.0

package/src/activities/agent-turn/errors.ts

@@ -27,8 +27,12 @@ import {
   isModalTaskExecStartPreDispatchUnavailableError,
   isModalCommandStartOutcomeUnknownError,
   isProviderCommandObservationUnavailableError,
+  ProviderCommandInputOutcomeUnknownError,
+  ProviderCommandStartOutcomeUnknownError,
   isRoutingMutationOutcomeUnknownError,
+  isRoutingMutationOutputRejectedError,
   RoutingWorkspaceRootChangedError,
+  RoutingBackendRecoveryRequiredError,
   ResponsesStreamingTerminalError,
   SandboxMaterializationVerificationError,
   materializationVerificationDiagnostic,
@@ -380,7 +384,7 @@ export function preClaimAdmissionFailure(error: unknown): ApplicationFailure {
  * objects (including array containers), not queue positions/duplicate refs.
  * The separate link ceiling bounds huge duplicate arrays without truncating
  * them into permission. Cycles are harmless; overflow/unreadable edges hold. */
-function databaseRecoveryCauseGraph(error: unknown): Map<object, Set<object>> | null {
+function structuredRecoveryCauseGraph(error: unknown): Map<object, Set<object>> | null {
   const graph = new Map<object, Set<object>>();
   const queue: object[] = [];
   let links = 0;
@@ -414,7 +418,7 @@ function retryableDatabaseFailureCode(
   requireDatabaseProvenance = false,
 ): PostClaimDatabaseRecoveryDetail["code"] | null {
   try {
-    const graph = databaseRecoveryCauseGraph(error);
+    const graph = structuredRecoveryCauseGraph(error);
     if (!graph) return null;
     const transports = new Set<object>();
     const boundaries = new Set<object>();
@@ -429,7 +433,8 @@ function retryableDatabaseFailureCode(
     // Ask the canonical transport predicate about ONLY this node's facts. Its
     // recursive search must not pair a DB sibling with an unrelated provider.
     for (const node of graph.keys()) {
-      if (isRoutingMutationOutcomeUnknownError(node)) return null;
+      if (isRoutingMutationOutcomeUnknownError(node) || isRoutingMutationOutputRejectedError(node))
+        return null;
       const record = node as Record<string, unknown>;
       if (
         requireDatabaseProvenance
@@ -742,13 +747,56 @@ export function modelPreparationFailureEventPayload(error: unknown, durationMs:
  * only those structural error links with a strict bound; never classify from
  * message text, which could originate in model or tool content.
  */
+/**
+ * Only the home resolver's positive pre-dispatch signal permits a fresh attempt.
+ * The same error class can also escape AFTER a provider mutation, so neither its
+ * retryable property nor message alone establishes replay safety. Inspect the
+ * complete bounded SDK cause graph and let uncertain peer outcomes veto this
+ * narrow recovery; unreadable or overflowing graphs fail closed.
+ */
+function isPreDispatchHomeBackendRecoveryRequired(error: unknown): boolean {
+  const graph = structuredRecoveryCauseGraph(error);
+  if (!graph) return false;
+  let found = false;
+  try {
+    for (const node of graph.keys()) {
+      if (
+        isRoutingMutationOutcomeUnknownError(node) ||
+        isRoutingMutationOutputRejectedError(node) ||
+        isRawProviderCommandOutcomeUnknown(node) ||
+        isModalCommandStartOutcomeUnknownError(node) ||
+        isProviderCommandObservationUnavailableError(node)
+      )
+        return false;
+      if (node instanceof RoutingBackendRecoveryRequiredError) {
+        if (
+          node.op !== "resolve_home_backend" ||
+          !Number.isSafeInteger(node.leaseEpoch) ||
+          node.leaseEpoch < 0 ||
+          !node.retryable ||
+          (node.recovery !== "pending" && node.recovery !== "superseded")
+        )
+          return false;
+        found = true;
+      }
+    }
+  } catch {
+    return false;
+  }
+  return found;
+}
+
 export function sandboxRouteTransitionCode(
   error: unknown,
 ):
   | "home_unavailable_this_turn"
   | "workspace_root_changed_this_turn"
   | "native_capabilities_changed_this_attempt"
+  | "home_backend_recovery_pending"
   | null {
+  if (isPreDispatchHomeBackendRecoveryRequired(error)) {
+    return "home_backend_recovery_pending";
+  }
   const pending: unknown[] = [error];
   const seen = new WeakSet<object>();
   let inspected = 0;
@@ -1273,10 +1321,41 @@ function anthropicRequestDiagnostic(error: unknown): AnthropicRequestError | und
     : undefined;
 }
 
+function isRawProviderCommandOutcomeUnknown(error: unknown): boolean {
+  try {
+    return (
+      error instanceof ProviderCommandInputOutcomeUnknownError ||
+      error instanceof ProviderCommandStartOutcomeUnknownError
+    );
+  } catch {
+    return false;
+  }
+}
+
 export function agentRunFailurePayload(
   error: unknown,
   options: { isCodexTurn?: boolean } = {},
 ): ReturnType<typeof baseAgentRunFailurePayload> {
+  const graph = structuredRecoveryCauseGraph(error);
+  const nodes = graph ? [...graph.keys()] : [];
+  const outputRejected = nodes.find(isRoutingMutationOutputRejectedError);
+  if (outputRejected) {
+    // A known receipt applies only to its own mutation. Uncertain peers retain
+    // their established terminal classification, regardless of SDK graph order.
+    const unknown = nodes.find(
+      (node) =>
+        isRoutingMutationOutcomeUnknownError(node) ||
+        isModalCommandStartOutcomeUnknownError(node) ||
+        isProviderCommandObservationUnavailableError(node) ||
+        isRawProviderCommandOutcomeUnknown(node),
+    );
+    if (unknown) return { ...baseAgentRunFailurePayload(unknown, options), retryable: false };
+    return {
+      error: outputRejected.message,
+      code: outputRejected.code,
+      retryable: false,
+    };
+  }
   const failure = baseAgentRunFailurePayload(error, options);
   const diagnostic = materializationVerificationDiagnostic(error);
   const anthropic = anthropicRequestDiagnostic(error);

package/src/activities/agent-turn/governance-model.ts

@@ -59,6 +59,7 @@ import {
   lazyToolTransportForTurn,
   openAiHostedImageProviderBindingForTurn,
   modelAttachmentInputPolicyForTurn,
+  resolveAcceptedTurnModel,
 } from "./tool-policy";
 
 import type { ClaimTurnOk } from "./claim";
@@ -361,10 +362,11 @@ export async function prepareGovernanceAndModel(
   // a chat-only Fireworks model. Resolving against the default-model settings
   // keeps gating consistent with the router. Cost accounting covers registry
   // models via configuredModelPricing.
-  const resolvedModel = runtime.resolveTurnModel(
-    capabilitySettings,
-    turnExecutionPolicy.productModelId,
-  );
+  //
+  // The accepted policy is then projected back onto the resolved shape: a
+  // turn frozen before hosted web search was enabled keeps its frozen tool set
+  // on every attempt, so recovery never adds a tool mid-turn.
+  const resolvedModel = resolveAcceptedTurnModel(runtime, capabilitySettings, turnExecutionPolicy);
   const providerApi = resolvedModel?.provider.api ?? "responses";
   const nativeImageProviderBinding =
     providerApi === "responses"
@@ -372,10 +374,8 @@ export async function prepareGovernanceAndModel(
       : null;
   const lazyToolTransport = lazyToolTransportForTurn(resolvedModel);
   const modelInputPolicy = modelAttachmentInputPolicyForTurn(resolvedModel);
-  // Use the proven wire capability, not the catalogue modality alone. Chat
-  // providers may advertise vision, but OpenGeni intentionally has no typed
-  // image transport for that wire yet; exposing view_image there would turn
-  // pixels into a multi-megabyte text/base64 function result.
+  // The shared input policy combines model modality with the supported wire
+  // transport, including typed image projection for vision-capable Chat models.
   const supportsImageInput = modelInputPolicy.supportsImageInput;
   media.modelCanReceiveRetainedSessionImages = supportsImageInput;
   const attachmentProjector = createModelHistoryAttachmentProjector(

package/src/activities/agent-turn/sandbox-runtime.ts

@@ -5,6 +5,7 @@ import {
   readLease,
   accrueWarmSeconds,
   SandboxWorkspaceMutationFencedError,
+  SandboxWorkspaceMutationOutputRejectedError,
   retainWorkspaceMutationProcess,
   retainedProviderCommandPersistence,
   SandboxRetainedProcessPromotionFencedError,
@@ -12,6 +13,7 @@ import {
 import { createProviderCommandRetainer } from "@opengeni/db/retained-provider-commands";
 import {
   RoutingMutationOutcomeUnknownError,
+  RoutingMutationOutputRejectedError,
   ProviderCommandStartOutcomeUnknownError,
   ProviderCommandObservationUnavailableError,
   isModalCommandStartOutcomeUnknownError,
@@ -622,6 +624,20 @@ export function createSandboxTurnRuntime(deps: SandboxTurnRuntimeDeps) {
     try {
       await settleMutation("resolved");
     } catch (settlementError) {
+      if (
+        settlementError instanceof SandboxWorkspaceMutationOutputRejectedError &&
+        settlementError.matchesPhysicalSettlement({
+          accountId: identity.accountId,
+          workspaceId: identity.workspaceId,
+          admission,
+          operation,
+          outcome: "resolved",
+        })
+      ) {
+        throw new RoutingMutationOutputRejectedError(operation, settlementError.code, {
+          cause: settlementError,
+        });
+      }
       throw new RoutingMutationOutcomeUnknownError(
         operation,
         `Platform workspace mutation "${operation}" returned from the provider but lost its durable settlement fence; its outcome is unknown and it was not replayed`,

package/src/activities/agent-turn/stream-attempt.ts

@@ -7,6 +7,7 @@ import {
   getHumanInputResumeForEvent,
   getSessionHumanInputRequest,
   getWorkspace,
+  listTurnCodemodeApprovals,
   recordUsageEvent,
   registerPendingSessionToolCall,
   recordPendingSessionToolCallResult,
@@ -77,6 +78,11 @@ import {
   generatedImageFromSdkEvent,
   isCompletedGeneratedImageSdkEvent,
 } from "../generated-images";
+import {
+  programmaticApproval,
+  programmaticContinuationNote,
+  recoverProgrammaticOperationsBeforeFirstRequest,
+} from "../programmatic-approvals";
 import { ToolResultSpill } from "./tool-result-spill";
 import { ownedTurnSandboxForAgent } from "./turn-sandbox-access";
 import { createTurnCredentialLeases } from "./credential-leases";
@@ -449,6 +455,8 @@ export async function runTurnStreamAttempt(
     await parallelSessionTitle?.cancel();
   };
   let runInput: Awaited<ReturnType<typeof turnInput>>["input"] | null = null;
+  let codemodeContinuationNote: string | undefined;
+  let programmaticApprovalAcknowledged = false;
   const prepareRunAttemptInput = async () => {
     const historyPreparationStartedAt = performance.now();
     let historyPreparationOutcome: "completed" | "failed" = "completed";
@@ -461,6 +469,8 @@ export async function runTurnStreamAttempt(
           subjectId: fileAuthoritySubjectId,
         },
         recovering: turn.executionGeneration > 1,
+        programmaticApprovalAcknowledged,
+        ...(codemodeContinuationNote ? { codemodeContinuationNote } : {}),
         ...(unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {}),
         ...(runCredentialsNote ? { runCredentialsNote } : {}),
         ...(mcpAvailabilityNote ? { mcpAvailabilityNote } : {}),
@@ -1656,10 +1666,21 @@ export async function runTurnStreamAttempt(
         });
       }
     }
-    if (eventing.stream.interruptions.length > 0) {
+    const programmaticPending = (
+      await listTurnCodemodeApprovals(db, {
+        accountId: input.accountId,
+        workspaceId: input.workspaceId,
+        sessionId: input.sessionId,
+        turnId: activeTurnId,
+      })
+    )
+      .filter((operation) => operation.state === "waiting_for_approval")
+      .map(programmaticApproval);
+    if (eventing.stream.interruptions.length > 0 || programmaticPending.length > 0) {
       await eventing.preparedTools?.inputWaitYield?.sealForSettlement(runtimeCancellationSignal);
       await historySink.reconcileConversationTruth({ requireDurable: true });
-      const approvals = runtime.serializeApprovals(eventing.stream.interruptions);
+      const sdkApprovals = runtime.serializeApprovals(eventing.stream.interruptions);
+      const approvals = [...sdkApprovals, ...programmaticPending];
       const humanInputInterruptions =
         runtime.serializeHumanInputRequests?.(eventing.stream.interruptions) ?? [];
       const interactionInterventionInterruptions =
@@ -1739,28 +1760,34 @@ export async function runTurnStreamAttempt(
       const interruptionCallIds = interruptionCallIdsFromPause({
         humanInputRequests,
         interactionInterventionRequests,
-        pendingApprovals,
+        pendingApprovals: [
+          ...sdkApprovals,
+          ...interactionInterventionInterruptions.map((interruption) => interruption.approval),
+        ],
       });
       assertOpenSuffixResumable(suffixMembers, interruptionCallIds);
       const suffixByCallId = new Map(suffixMembers.map((member) => [member.callId, member]));
-      const attached = await attachOpenSuffixToPendingToolCalls(db, {
-        accountId: input.accountId,
-        workspaceId: input.workspaceId,
-        sessionId: input.sessionId,
-        turnId: activeTurnId,
-        executionGeneration: attempt.executionGeneration,
-        attemptId: input.attemptId,
-        members: interruptionCallIds.map((callId) => {
-          const member = suffixByCallId.get(callId)!;
-          return {
-            callId,
-            interruptionKind: interruptionKindForCallItem(
-              member.callItem as Record<string, unknown>,
-            ),
-            reasoningItems: member.reasoningItems as Array<Record<string, unknown>>,
-          };
-        }),
-      });
+      const attached =
+        interruptionCallIds.length === 0
+          ? { accepted: true }
+          : await attachOpenSuffixToPendingToolCalls(db, {
+              accountId: input.accountId,
+              workspaceId: input.workspaceId,
+              sessionId: input.sessionId,
+              turnId: activeTurnId,
+              executionGeneration: attempt.executionGeneration,
+              attemptId: input.attemptId,
+              members: interruptionCallIds.map((callId) => {
+                const member = suffixByCallId.get(callId)!;
+                return {
+                  callId,
+                  interruptionKind: interruptionKindForCallItem(
+                    member.callItem as Record<string, unknown>,
+                  ),
+                  reasoningItems: member.reasoningItems as Array<Record<string, unknown>>,
+                };
+              }),
+            });
       if (!attached.accepted) {
         return claimedResult({ status: "cancelled" });
       }
@@ -1935,7 +1962,59 @@ export async function runTurnStreamAttempt(
     return claimedResult({ status: "idle" });
   };
 
+  // Recover committed waits/results before any model dispatch, including crash replacement.
+  // A new attempt uses its own signed caller; origin provenance never changes.
+  // Lazy MCP preparation is never awaited unless stored work must resume.
+  // Measured as its own startup phase so a first-request barrier cannot hide.
+  const programmaticRecoveryStartedAt = performance.now();
+  let programmaticRecoveryOutcome: "completed" | "failed" = "failed";
+  const codemodeOperations = await recoverProgrammaticOperationsBeforeFirstRequest({
+    approvalDecisionId:
+      trigger.type === "user.approvalDecision"
+        ? (trigger.payload as { approvalId: string }).approvalId
+        : undefined,
+    hasUnfinishedOperations: async () =>
+      (
+        await listTurnCodemodeApprovals(db, {
+          accountId: input.accountId,
+          workspaceId: input.workspaceId,
+          sessionId: input.sessionId,
+          turnId: activeTurnId,
+        })
+      ).length > 0,
+    toolPreparationReady: eventing.toolPreparationReady,
+    resumeApproved: (decisionOperationId) =>
+      eventing.codemodeDispatcher?.resumeApproved(
+        `sandbox:${input.attemptId}`,
+        decisionOperationId,
+      ),
+  })
+    .then((operations) => {
+      programmaticRecoveryOutcome = "completed";
+      return operations;
+    })
+    .finally(() => {
+      recordTurnStartupPhase(observability, {
+        phase: "programmatic_operation_recovery",
+        provider: turnExecutionPolicy.providerId,
+        backend: activeSandboxBackend ?? groupBoxBackend,
+        outcome: programmaticRecoveryOutcome,
+        durationSeconds: (performance.now() - programmaticRecoveryStartedAt) / 1_000,
+      });
+    });
+  codemodeContinuationNote = programmaticContinuationNote(codemodeOperations);
+  programmaticApprovalAcknowledged =
+    trigger.type === "user.approvalDecision" &&
+    codemodeOperations.some(
+      (operation) =>
+        operation.operationId === (trigger.payload as { approvalId?: unknown }).approvalId,
+    );
+  const pendingProgrammaticApprovals = codemodeOperations
+    .filter((operation) => operation.state === "waiting_for_approval")
+    .map(programmaticApproval);
   const openSuffixResume = await settleOpenSuffixResumeIfNeeded({
+    additionalApprovals: pendingProgrammaticApprovals,
+    programmaticApprovalAcknowledged,
     db,
     agent,
     accountId: input.accountId,

package/src/activities/agent-turn/subscription-lease.ts

@@ -1,6 +1,8 @@
 export type LeaseRenewReason = "timer" | "runtime_event" | "model_usage";
 export type LeaseLossReason = "deadline" | "not_found";
 
+const HEARTBEAT_INTERVAL_MS = 60_000;
+
 type LeaseIdentity = { turnId: string; holderId: string; generation: number };
 export type SubscriptionLeaseDeps = {
   ttlMs: number;
@@ -24,11 +26,13 @@ export class SubscriptionTurnLease {
   private heartbeatInFlight = false;
   private heartbeatTimer: ReturnType<typeof setInterval> | undefined;
   private readonly now: () => number;
+  private readonly renewalIntervalMs: number;
 
   constructor(private readonly leaseDeps: SubscriptionLeaseDeps) {
     if (!Number.isFinite(leaseDeps.ttlMs) || leaseDeps.ttlMs <= 0)
       throw new Error("Subscription lease TTL must be positive");
     this.now = leaseDeps.now ?? (() => performance.now());
+    this.renewalIntervalMs = Math.min(HEARTBEAT_INTERVAL_MS, leaseDeps.ttlMs / 5);
   }
 
   private expired = (deadline: number | null): boolean =>
@@ -62,6 +66,11 @@ export class SubscriptionTurnLease {
       return;
     }
     if (this.heartbeatInFlight) return;
+    // Acquisition and successful renewal confirm ownership from request start.
+    // Reuse that proof between heartbeats; a failed renewal leaves it unchanged
+    // so the next checkpoint can retry immediately. Expiry is checked first.
+    const renewAt = this.confirmedUntilMs! - this.leaseDeps.ttlMs + this.renewalIntervalMs;
+    if (this.now() < renewAt) return;
     this.heartbeatInFlight = true;
     const priorDeadline = this.confirmedUntilMs;
     const startedAt = this.now();
@@ -103,7 +112,7 @@ export class SubscriptionTurnLease {
 
   startHeartbeat = (): void => {
     if (!this.leaseDeps.getTurnId() || this.heartbeatTimer) return;
-    this.heartbeatTimer = setInterval(() => void this.renew("timer"), 60_000);
+    this.heartbeatTimer = setInterval(() => void this.renew("timer"), HEARTBEAT_INTERVAL_MS);
     this.heartbeatTimer.unref?.();
   };
 

package/src/activities/agent-turn/tool-environment.ts

@@ -67,6 +67,7 @@ import {
   scheduledTurnMcpServerIds,
   hasPermission,
   requireExplicitPermissionDelegation,
+  createWebSearchBilling,
 } from "@opengeni/core";
 import { loadWorkspaceEnvironmentForRunWithCredentials } from "../environment";
 import { withFirstPartyTools } from "../goals";
@@ -116,6 +117,7 @@ import { resolveTurnSandboxAccess } from "./turn-sandbox-access";
 import { createListModelsAttemptToolDefinition } from "./list-models";
 import { createRefreshCredentialsAttemptToolDefinition } from "./refresh-credentials";
 import { codeSearchToolDefinitions, codeSearchWorkspaceFromChannel } from "./code-search";
+import { turnWebSearchPlan, webSearchToolDefinitions } from "./web-search";
 import { createWorkspaceSkillTools } from "./skill-tools";
 import { loadConfiguredBundledSkills } from "./skill-selection";
 import { guardSkillFilesystem } from "./skill-transfer";
@@ -886,6 +888,22 @@ export async function prepareTurnToolRuntime(deps: PrepareTurnToolRuntimeDeps) {
   const attemptToolFamilies = resolveAgentToolFamilies(session.agent, {
     hasSkills: skillCatalog.length > 0,
   });
+  // Provider web search: only where the turn has no hosted search (or the
+  // operator chose `replace`), only when a provider is configured, and only
+  // when the session's agent configuration allows web search (filtered below).
+  const webSearchTools = webSearchToolDefinitions({
+    settings: runSettings,
+    tools: turnWebSearchPlan(resolvedModel, runSettings).providerTools,
+    scope: {
+      accountId: input.accountId,
+      workspaceId: input.workspaceId,
+      sessionId: input.sessionId,
+      turnId: turn.id,
+      attemptId: input.attemptId,
+    },
+    billing: createWebSearchBilling({ db, settings: runSettings }),
+    observability,
+  });
   const attemptToolDefinitions = [
     ...(operationReadStore
       ? [
@@ -976,6 +994,7 @@ export async function prepareTurnToolRuntime(deps: PrepareTurnToolRuntimeDeps) {
       ? [googleDrivePublicationTool]
       : []),
     ...codeSearchTools,
+    ...webSearchTools,
   ].filter((tool) => attemptToolFamilies.allowsFunctionTool(tool.modelName));
   recordTurnStartupPhase(observability, {
     phase: "tool_context_preparation",
@@ -1234,6 +1253,7 @@ export async function prepareTurnToolRuntime(deps: PrepareTurnToolRuntimeDeps) {
       {},
       (name) => mcpToolDisplayMetadata(tools.mcpServers, name),
       (entry) => toolFamilyForCatalogIdentity(entry, runSettings.mcpServers),
+      tools.inputWaitYield,
     );
     eventing.codemodeDispatcher.start();
   };

package/src/activities/agent-turn/tool-policy.ts

@@ -2,8 +2,11 @@ import {
   desktopCapableBackend,
   type ComputerToolMode,
   type LazyToolTransport,
+  type OpenGeniRuntime,
 } from "@opengeni/runtime";
+import type { TurnExecutionPolicyV1 } from "@opengeni/contracts";
 import {
+  configuredModelForAcceptedTurnExecutionPolicy,
   isDirectOpenAiApiBaseUrl,
   type ModelProviderApi,
   type ResolvedModelProvider,
@@ -208,6 +211,28 @@ export function shouldDeferNonEagerToolPreparation(args: {
   );
 }
 
+/**
+ * Resolve the provider routing/gating shape for an accepted turn. The current
+ * definition is projected back onto the verified frozen policy: a turn
+ * accepted before hosted web search was enabled keeps its frozen tool set on
+ * every recovery attempt (stable tool prefix, exact accepted definition); the
+ * next accepted logical turn resolves the newly enabled tool.
+ */
+export function resolveAcceptedTurnModel(
+  runtime: Pick<OpenGeniRuntime, "resolveTurnModel">,
+  settings: Settings,
+  policy: TurnExecutionPolicyV1,
+): ReturnType<OpenGeniRuntime["resolveTurnModel"]> {
+  const current = runtime.resolveTurnModel(settings, policy.productModelId);
+  if (!current) return null;
+  const configured = configuredModelForAcceptedTurnExecutionPolicy(
+    current.configured,
+    current.provider,
+    policy,
+  );
+  return configured === current.configured ? current : { ...current, configured };
+}
+
 /**
  * Native web search is a runtime capability, not part of the session's MCP
  * allow-list. Attach it whenever the resolved provider advertises runnable
@@ -294,12 +319,8 @@ export function modelAttachmentInputPolicyForTurn(
     };
   } | null,
 ): ModelAttachmentInputPolicy {
-  const typedTransport =
-    resolvedModel === null ||
-    resolvedModel.provider.api === "responses" ||
-    resolvedModel.provider.api === "anthropic-messages";
   return {
-    supportsImageInput: typedTransport && modelSupportsImageInputForTurn(resolvedModel),
+    supportsImageInput: modelSupportsImageInputForTurn(resolvedModel),
     inputFileMediaTypes: [],
   };
 }