@opengeni/worker-bundle 0.28.4-canary.0 → 0.28.4-canary.1

package/dist/index.js

@@ -782,7 +782,7 @@ function temporalWorkflowExecutionNotFound(error) {
   return isGrpcServiceError(error) && error.code === 5;
 }
 async function createDefaultWorkerActivities(role, dependencies = {}) {
-  return role === "control" ? (await import("./activities-control-J5QNRWVH.js")).createControlActivities(dependencies) : (await import("./activities-turn-ZJBY3W5D.js")).createTurnActivities(dependencies);
+  return role === "control" ? (await import("./activities-control-MKGEOBXT.js")).createControlActivities(dependencies) : (await import("./activities-turn-5MZZ6HQA.js")).createTurnActivities(dependencies);
 }
 function resolveOpenGeniWorkflowDefinition(moduleUrl = import.meta.url) {
   const modulePath = fileURLToPath(moduleUrl).replaceAll("\\", "/");

package/dist/sandbox-routing.d.ts

@@ -2,7 +2,7 @@ import { type Settings } from "@opengeni/config";
 import { type Database, type EnrollmentRecord } from "@opengeni/db";
 import type { OpStreamOutputFrame } from "@opengeni/runtime/sandbox";
 import type { EventBus } from "@opengeni/events";
-import { type EstablishedSandboxSession, type RoutingSandboxFirstOperationObserver, type RoutingSandboxOperationObserver, type SelfhostedOpObserver, type SelfhostedRelayConfig, type OpStreamJournal } from "@opengeni/runtime";
+import { type EstablishedSandboxSession, type RoutingSandboxFirstOperationObserver, type RoutingSandboxOperationObserver, type SelfhostedOpObserver, type SelfhostedConnectionBinding, type SelfhostedRelayConfig, type OpStreamJournal } from "@opengeni/runtime";
 export type RoutingWiringServices = {
     db: Database;
     settings: Settings;
@@ -179,6 +179,8 @@ export type SelfhostedTurnSessionArgs = {
     operationResourcePolicy: EnrollmentRecord["operationPolicy"];
     operationResourcePolicySupported: boolean;
     operationCpuQuotaSupported: boolean;
+    /** Feature advertisement only; refreshed by exact live admission for each operation. */
+    transactionalFsWriteSupported?: boolean;
     /** The active pointer's epoch — the control-op fence echoed to the agent. */
     epoch: number;
     /** The run's declared sandbox environment (the SAME object fed to buildAgent +
@@ -197,6 +199,9 @@ export type SelfhostedTurnSessionArgs = {
         sessionId: string;
     };
 };
+/** Project one enrollment read into one exact runtime route. Capability and
+ * process identity must be taken from the same snapshot. */
+export declare function connectionBindingFor(services: RoutingWiringServices, enrollment: EnrollmentRecord | null): SelfhostedConnectionBinding | null;
 export declare function establishSelfhostedTurnSession(services: RoutingWiringServices, args: SelfhostedTurnSessionArgs): Promise<EstablishedSandboxSession>;
 /** Whether the routing proxy should wrap the turn box: the hot-swap feature is
  *  gated by the selfhosted flag (the active pointer + swap tools are only

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opengeni/worker-bundle",
-  "version": "0.28.4-canary.0",
+  "version": "0.28.4-canary.1",
   "description": "OpenGeni worker entry and reusable embedded lifecycle, shipped with a release-coherent pre-bundled Temporal workflow artifact.",
   "license": "Apache-2.0",
   "repository": {
@@ -54,23 +54,23 @@
   },
   "dependencies": {
     "@llamaindex/liteparse": "2.14.2",
-    "@opengeni/agent-proto": "^0.5.1-canary.26",
-    "@opengeni/capabilities": "^0.3.3-canary.8",
-    "@opengeni/codemode": "^0.5.6-canary.0",
-    "@opengeni/codex": "^0.2.22-canary.8",
-    "@opengeni/config": "^1.1.2-canary.0",
-    "@opengeni/contracts": "^3.0.2-canary.0",
-    "@opengeni/core": "^2.9.4-canary.0",
-    "@opengeni/db": "^4.3.3-canary.0",
-    "@opengeni/documents": "^0.8.27-canary.0",
-    "@opengeni/events": "^0.4.25-canary.0",
-    "@opengeni/github": "^0.7.10-canary.0",
-    "@opengeni/network": "^0.3.1-canary.8",
-    "@opengeni/observability": "^0.8.26-canary.0",
-    "@opengeni/runtime": "^2.5.3-canary.0",
-    "@opengeni/storage": "^0.2.127-canary.0",
-    "@opengeni/tool-gateway": "^0.1.7-canary.0",
-    "@opengeni/xai-subscription": "^0.1.4-canary.8",
+    "@opengeni/agent-proto": "^0.5.1-canary.27",
+    "@opengeni/capabilities": "^0.3.3-canary.9",
+    "@opengeni/codemode": "^0.5.6-canary.1",
+    "@opengeni/codex": "^0.2.22-canary.9",
+    "@opengeni/config": "^1.1.2-canary.1",
+    "@opengeni/contracts": "^3.0.2-canary.1",
+    "@opengeni/core": "^2.9.4-canary.1",
+    "@opengeni/db": "^4.3.3-canary.1",
+    "@opengeni/documents": "^0.8.27-canary.1",
+    "@opengeni/events": "^0.4.25-canary.1",
+    "@opengeni/github": "^0.7.10-canary.1",
+    "@opengeni/network": "^0.3.1-canary.9",
+    "@opengeni/observability": "^0.8.26-canary.1",
+    "@opengeni/runtime": "^2.5.3-canary.1",
+    "@opengeni/storage": "^0.2.127-canary.1",
+    "@opengeni/tool-gateway": "^0.1.7-canary.1",
+    "@opengeni/xai-subscription": "^0.1.4-canary.9",
     "@temporalio/activity": "^1.17.0",
     "@temporalio/client": "^1.17.0",
     "@temporalio/worker": "^1.17.0",

package/src/activities/agent-turn/codex-capacity.ts

@@ -5,7 +5,7 @@ import {
   CodexCredentialLeaseAttemptFencedError,
   CodexCredentialFailoverExhaustedError,
   CODEX_CREDENTIAL_LEASE_TTL_MS,
-  recordSessionActiveCodexCredential,
+  recordSessionCodexSelectionForTurnAttempt,
   setSessionCodexPinInTransaction,
   settleCodexCredentialFailover,
   withSessionCodexCapacityMutation,
@@ -288,19 +288,40 @@ export async function selectCodexTurnCapacity(
         leases.codex.confirmedUntilMs !== null;
       if (leases.codex.held) leases.codex.startHeartbeat();
 
+      const eligibleCount = leased.accounts.filter((account) =>
+        isCodexCredentialEligible(account, new Date()),
+      ).length;
+      const selectionReceipt = providerTurn.effectiveCodexCredentialId
+        ? await recordSessionCodexSelectionForTurnAttempt(db, {
+            workspaceId: input.workspaceId,
+            sessionId: input.sessionId,
+            turnId,
+            attemptId: input.attemptId,
+            executionGeneration: attempt.executionGeneration,
+            credentialId: providerTurn.effectiveCodexCredentialId,
+            strategy: leased.rotationStrategy,
+            reusedLease: leased.reused,
+            pinnedCredentialId: sessionPin,
+            pinSource: sessionPinSource,
+            eligibleCount,
+            connectedCount: leased.accounts.length,
+          })
+        : null;
+      if (selectionReceipt)
+        await publishDurableSessionEvents(
+          bus,
+          input.workspaceId,
+          input.sessionId,
+          selectionReceipt.events,
+        );
+      const selectionDiagnostics = selectionReceipt?.diagnostics ?? null;
       const actualOutcome = providerTurn.effectiveCodexCredentialId
         ? "selected"
         : rotationDecision.kind === "allCapped" || rotationDecision.kind === "allocatorDisabled"
           ? "waiting"
           : "none";
-      const actualReason = providerTurn.effectiveCodexCredentialId
-        ? leased.reused
-          ? "lease_reused"
-          : sessionPin === providerTurn.effectiveCodexCredentialId
-            ? "pin"
-            : rotationDecision.kind === "active" && rotationDecision.moved
-              ? "rotation"
-              : "active"
+      const actualReason = selectionDiagnostics
+        ? selectionDiagnostics.reason
         : rotationDecision.kind === "allCapped"
           ? "all_capped"
           : rotationDecision.kind === "allocatorDisabled"
@@ -362,9 +383,6 @@ export async function selectCodexTurnCapacity(
         });
       }
 
-      const eligibleCount = leased.accounts.filter((account) =>
-        isCodexCredentialEligible(account, new Date()),
-      ).length;
       const poolDepth = eligibleCount === 0 ? "zero" : eligibleCount === 1 ? "one" : "many";
       observability.incrementCounter({
         name: "opengeni_codex_pool_observations_total",
@@ -690,34 +708,7 @@ export async function selectCodexTurnCapacity(
         return { exit: claimedResult({ status: "cancelled" }) };
       }
       if (providerTurn.effectiveCodexCredentialId) {
-        const priorAccountId = lockedSessionCodexState.lastCredentialId;
-        if (priorAccountId !== providerTurn.effectiveCodexCredentialId) {
-          await recordSessionActiveCodexCredential(
-            db,
-            input.workspaceId,
-            input.sessionId,
-            providerTurn.effectiveCodexCredentialId,
-          );
-          const rotated = rotationDecision.kind === "active" && rotationDecision.moved;
-          await eventing.publish([
-            {
-              type: "codex.account.switched",
-              payload: {
-                fromAccountId: priorAccountId,
-                toAccountId: providerTurn.effectiveCodexCredentialId,
-                reason: rotated ? "rotation" : "manual",
-              },
-            },
-          ]);
-        }
-
-        const selectionReason = leased.reused
-          ? "lease_reused"
-          : sessionPin === providerTurn.effectiveCodexCredentialId
-            ? "pin"
-            : rotationDecision.kind === "active" && rotationDecision.moved
-              ? "rotation"
-              : "active";
+        const selectionReason = selectionDiagnostics!.reason;
         observability.incrementCounter({
           name: "opengeni_codex_credential_selections_total",
           help: "Codex credential selections by strategy and reason.",
@@ -727,19 +718,6 @@ export async function selectCodexTurnCapacity(
             reason: selectionReason,
           },
         });
-        await eventing.publish([
-          {
-            type: "codex.credential.selected",
-            payload: {
-              credentialId: providerTurn.effectiveCodexCredentialId,
-              strategy: leased.rotationStrategy,
-              reason: selectionReason,
-              eligibleCount,
-              connectedCount: leased.accounts.length,
-              reused: leased.reused,
-            },
-          },
-        ]);
       }
     } catch (error) {
       credentialSelectionOutcome = "failed";

package/src/activities/agent-turn/codex-selection-diagnostics.ts

@@ -0,0 +1 @@
+export { codexSelectionDiagnostics } from "@opengeni/db";

package/src/activities/agent-turn/sandbox-establish.ts

@@ -475,6 +475,8 @@ export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Prom
               liveMachineEnrollment?.agentCapabilities.operationResourcePolicy === true,
             operationCpuQuotaSupported:
               liveMachineEnrollment?.agentCapabilities.operationCpuQuota === true,
+            transactionalFsWriteSupported:
+              liveMachineEnrollment?.agentCapabilities.transactionalFsWrite === true,
             ...(activeSandboxRecord!.scope === "user" && fileAuthoritySubjectId
               ? {
                   personalMachineAttempt: {

package/src/activities/agent-turn/skill-search.ts

@@ -1,6 +1,5 @@
 import type { AttemptToolDefinition } from "@opengeni/codemode";
 import { PublicSkillSearchError, type PublicSkillSearchClient } from "@opengeni/core";
-import { listSkillLibraryEntries } from "@opengeni/runtime/skill-library";
 
 export type WorkspaceSkillSearchEntry = Readonly<{
   id: string;
@@ -23,7 +22,7 @@ export function createSkillSearchAttemptToolDefinition(input: {
     codemodePath: ["opengeni", "skill_search"],
     title: "Search Skills",
     description:
-      "Find installed workspace Skills or available curated and public Skills. Returns identifiers and install sources, not file contents. Search never installs or starts a sandbox. Scope defaults to all; use installed to avoid external search.",
+      "Find installed workspace Skills or available public Skills. Returns identifiers and install sources, not file contents. Search never installs or starts a sandbox. Scope defaults to all; use installed to avoid external search.",
     inputSchema: {
       type: "object",
       properties: {
@@ -60,19 +59,6 @@ export function createSkillSearchAttemptToolDefinition(input: {
           source: entry.source ?? "workspace",
           installed: !entry.source || entry.source === "workspace",
         }));
-      const libraryHits =
-        scope === "installed"
-          ? []
-          : listSkillLibraryEntries()
-              .filter(matches)
-              .slice(0, limit)
-              .map((entry) => ({
-                id: `library:${entry.id}`,
-                name: entry.name,
-                description: entry.description,
-                source: "library" as const,
-                libraryId: entry.id,
-              }));
       let publicResult: Awaited<ReturnType<PublicSkillSearchClient["search"]>> | null = null;
       let publicError: {
         source: "skills_sh";
@@ -93,7 +79,7 @@ export function createSkillSearchAttemptToolDefinition(input: {
       }
       const output = {
         workspace: workspaceHits,
-        library: libraryHits,
+        library: [], // Retained response field for older clients.
         public: publicResult?.items.map((entry) => ({ ...entry })) ?? [],
         // A provider outage is not an empty search. Keep local results useful
         // and expose partial failure explicitly, without leaking network details.

package/src/activities/codex-fleet-shadow.ts

@@ -18,6 +18,9 @@ export type CodexFleetActualDecisionV1 = {
   candidateKey: string | null;
   reason:
     | "lease_reused"
+    | "affinity_reused"
+    | "assigned"
+    | "switched"
     | "pin"
     | "rotation"
     | "active"

package/src/activities/sandbox-lease.ts

@@ -26,6 +26,7 @@ import {
   appendSessionEventToSandboxGroup,
   bindRetainedProcessProviderIdentity,
   claimWorkspaceArchiveCapture,
+  releaseWorkspaceArchiveCapture,
   claimSandboxCheckpointArtifactsForGc,
   claimTerminalRetainedProcesses,
   countActiveRetainedProcessesByOwnerState,
@@ -249,6 +250,7 @@ export type TerminateBoxFn = (
   captureDisposition?: DrainCaptureDisposition,
   capturePolicy?: ProviderWorkspaceCapturePolicy | null,
   diskBackedArchives?: boolean,
+  releaseFailedCapture?: () => Promise<void>,
 ) => Promise<boolean | ProviderTerminationOutcome>;
 
 export type SweepModalOrphansFn = (
@@ -350,12 +352,13 @@ export class SandboxProviderCaptureTimeoutError extends Error {
   }
 }
 
-async function awaitProviderCaptureWithLatePublication<T>(input: {
+export async function awaitProviderCaptureWithLatePublication<T>(input: {
   capture: Promise<T>;
   timeoutMs: number;
   timeoutError: SandboxProviderCaptureTimeoutError;
   publishLate: (value: T) => Promise<void>;
   observeLateFailure: (error: unknown) => void;
+  observeLatePublicationFailure: (error: unknown) => void;
 }): Promise<T> {
   let timeout: ReturnType<typeof setTimeout> | undefined;
   const timeoutError = input.timeoutError;
@@ -373,7 +376,9 @@ async function awaitProviderCaptureWithLatePublication<T>(input: {
     // durable archive publication: never provider teardown or a cold commit from
     // an activity Temporal has already rejected. A replaced claim fences this
     // callback; Modal candidates then flow to artifact-ledger GC.
-    void input.capture.then(input.publishLate).catch(input.observeLateFailure);
+    void input.capture
+      .then(input.publishLate, input.observeLateFailure)
+      .catch(input.observeLatePublicationFailure);
     throw error;
   } finally {
     if (timeout) clearTimeout(timeout);
@@ -482,6 +487,7 @@ export function createSandboxLeaseActivities(
       captureDisposition,
       capturePolicy,
       diskBackedArchives,
+      releaseFailedCapture,
     ) =>
       await terminateProviderBox(
         settings,
@@ -494,6 +500,7 @@ export function createSandboxLeaseActivities(
         captureDisposition,
         capturePolicy,
         diskBackedArchives,
+        releaseFailedCapture,
       ));
   const sweepModalOrphans: SweepModalOrphansFn =
     options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
@@ -2803,6 +2810,17 @@ async function terminateDrainableBox(
         captureDisposition,
         capturePolicy,
         Boolean(objectStorage),
+        async () => {
+          if (!captureClaim || !lease.instanceId) return;
+          await releaseWorkspaceArchiveCapture(db, {
+            accountId,
+            workspaceId: row.workspaceId,
+            sandboxGroupId: row.sandboxGroupId,
+            captureId: captureClaim.id,
+            expectedEpoch: row.leaseEpoch,
+            expectedInstanceId: lease.instanceId,
+          });
+        },
       );
   const terminated = typeof termination === "boolean" ? termination : termination.terminated;
   if (!terminated) {
@@ -2936,10 +2954,21 @@ export async function terminateProviderBox(
   captureDisposition: DrainCaptureDisposition = "capture_required",
   claimedCapturePolicy?: ProviderWorkspaceCapturePolicy | null,
   diskBackedArchives = false,
+  releaseFailedCapture?: () => Promise<void>,
 ): Promise<ProviderTerminationOutcome> {
   const durableBackendId = (lease.resumeBackendId ?? lease.backend) as string;
   const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
   const logIdentity = providerDrainLogIdentity(lease, backend);
+  const releaseAfterCaptureFailure = async (): Promise<void> => {
+    try {
+      await releaseFailedCapture?.();
+    } catch (error) {
+      observability.warn("sandbox reaper: failed capture claim release failed", {
+        ...logIdentity,
+        error: error instanceof Error ? error.message : String(error),
+      });
+    }
+  };
   // 'none' / no backend -> nothing to terminate.
   if (!backend || backend === "none") {
     return { terminated: true, providerMissingBeforeCapture: false };
@@ -3187,7 +3216,17 @@ export async function terminateProviderBox(
             await disposeWorkspaceArchive(archive);
           }
         },
+        observeLatePublicationFailure: (error) => {
+          observability.warn("sandbox reaper: late capture publication cleanup failed", {
+            ...logIdentity,
+            error: error instanceof Error ? error.message : String(error),
+          });
+        },
         observeLateFailure: (error) => {
+          // Capture has now rejected and this timed-out path can never reach
+          // provider teardown. Release only our unpublished claim so a waiting
+          // turn can re-arm the intact live box; stale callbacks are DB-fenced.
+          void releaseAfterCaptureFailure();
           observability.warn("sandbox reaper: timed-out provider capture later failed", {
             ...logIdentity,
             error: error instanceof Error ? error.message : String(error),
@@ -3209,6 +3248,12 @@ export async function terminateProviderBox(
         error: error instanceof Error ? error.message : String(error),
       },
     );
+    // An unresolved timeout is not proof of capture failure: retain the fence
+    // until its provider promise settles. A settled failure cannot enter the
+    // teardown path below, so its unpublished claim may safely be released.
+    if (!(error instanceof SandboxProviderCaptureTimeoutError)) {
+      await releaseAfterCaptureFailure();
+    }
     // NEVER terminate a box whose snapshot we could not capture.
     throw error;
   }

package/src/sandbox-routing.ts

@@ -1383,6 +1383,8 @@ export type SelfhostedTurnSessionArgs = {
   operationResourcePolicy: EnrollmentRecord["operationPolicy"];
   operationResourcePolicySupported: boolean;
   operationCpuQuotaSupported: boolean;
+  /** Feature advertisement only; refreshed by exact live admission for each operation. */
+  transactionalFsWriteSupported?: boolean;
   /** The active pointer's epoch — the control-op fence echoed to the agent. */
   epoch: number;
   /** The run's declared sandbox environment (the SAME object fed to buildAgent +
@@ -1442,7 +1444,7 @@ function opStreamDepsFor(
 
 /** Project one enrollment read into one exact runtime route. Capability and
  * process identity must be taken from the same snapshot. */
-function connectionBindingFor(
+export function connectionBindingFor(
   services: RoutingWiringServices,
   enrollment: EnrollmentRecord | null,
 ): SelfhostedConnectionBinding | null {
@@ -1456,6 +1458,7 @@ function connectionBindingFor(
     operationResourcePolicy: enrollment.operationPolicy,
     operationResourcePolicySupported: enrollment.agentCapabilities.operationResourcePolicy === true,
     operationCpuQuotaSupported: enrollment.agentCapabilities.operationCpuQuota === true,
+    transactionalFsWriteSupported: enrollment.agentCapabilities.transactionalFsWrite === true,
   };
 }
 
@@ -1526,6 +1529,7 @@ export async function establishSelfhostedTurnSession(
     operationResourcePolicy: args.operationResourcePolicy,
     operationResourcePolicySupported: args.operationResourcePolicySupported,
     operationCpuQuotaSupported: args.operationCpuQuotaSupported,
+    transactionalFsWriteSupported: args.transactionalFsWriteSupported === true,
     resolveOperationAdmission: async () => {
       const enrollment = args.personalMachineAttempt
         ? await resolvePersonalMachineConnectionForAttempt(db, {