@opengeni/worker-bundle 0.27.0-canary.0 → 0.27.2-canary.0

package/dist/{activities-control-VMA3XMHE.js → activities-control-WZRFEIJH.js}

@@ -33,7 +33,7 @@ import {
   scheduledUserMessagePayload,
   withFirstPartyTools,
   workflowIdForSession
-} from "./chunk-7LS42PHP.js";
+} from "./chunk-C54VDT6I.js";
 import {
   inspectOpenSandboxKubernetesInventory,
   recordCreditBalanceGauges,
@@ -53,7 +53,7 @@ import {
   recordTurnsQueuedGauge,
   recordWorkerDeathRecoveryMetrics,
   runtimeMetricsHooksForObservability
-} from "./chunk-D4ZEFOYH.js";
+} from "./chunk-BARTLVBK.js";
 
 // src/activities/browser-state-artifact-reaper.ts
 import {
@@ -159,6 +159,7 @@ import {
   claimAutomationRun,
   settleAutomationRun
 } from "@opengeni/db";
+import { SessionSkills } from "@opengeni/contracts";
 
 // src/activities/agent-run-admission.ts
 import { configuredStaticUsageLimits } from "@opengeni/config";
@@ -355,7 +356,8 @@ function createAutomationActivities(services, options = {}) {
           workspaceId: input.workspaceId,
           initialMessage: accepted.initialMessage,
           resources: template.resources,
-          skills: template.skills,
+          skills: SessionSkills.parse(template.skills),
+          bundledSkillIds: template.bundledSkillIds,
           tools: template.tools,
           toolPolicy: { mode: "explicit", inheritedFromSessionId: null },
           model,
@@ -5188,6 +5190,7 @@ function createRigVerificationActivities(services) {
 
 // src/activities/sandbox-lease.ts
 import { createHash as createHash3, randomUUID as randomUUID2 } from "crypto";
+import { retainedProviderCommandPersistence } from "@opengeni/db/retained-provider-commands";
 import { Context } from "@temporalio/activity";
 import { OpLostReason, OpState } from "@opengeni/agent-proto";
 import {
@@ -5250,6 +5253,7 @@ import {
   inspectModalSandboxLifecycle,
   isExecSessionLostBanner,
   isProviderSandboxNotFoundError,
+  isProviderSandboxGoneDuringRoutedOperation,
   NatsControlRpc,
   NatsOpStreamTransport,
   OpStreamExecClient,
@@ -5378,8 +5382,14 @@ function createSandboxLeaseActivities(services, options = {}) {
     try {
       const { db, settings, observability } = await services();
       const timing = sandboxDrainTiming(settings);
+      const onUnobservableCommandDrainError = (error) => {
+        observability.warn("sandbox reaper: unobservable command drain inspection failed", {
+          error: error instanceof Error ? error.message : String(error)
+        });
+      };
       if (!settings.sandboxOwnershipEnabled) {
         const drainableInventory2 = await reapStaleLeaseHoldersGlobal(db, {
+          onUnobservableCommandDrainError,
           viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
           turnHolderTtlMs: settings.sandboxLeaseTtlMs,
           interactionHolderTtlMs: settings.sandboxInteractionHolderTtlMs,
@@ -5406,6 +5416,7 @@ function createSandboxLeaseActivities(services, options = {}) {
       });
       recordSandboxDeadlineRotationsRequested(observability, rotationsRequested);
       const drainableInventory = await reapStaleLeaseHoldersGlobal(db, {
+        onUnobservableCommandDrainError,
         viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
         // Dead-worker turn holders: a live holder is touched every 10s from the
         // moment it is registered (resumeBoxForTurn's holder-liveness loop covers
@@ -6160,8 +6171,8 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
               lease,
               process,
               claim.ownerState === "background_stopping" ? "cancel" : "observe",
-              async (result, chunkId) => {
-                if (typeof result !== "string" || isExecSessionLostBanner(result, process.providerSessionId))
+              async (result, chunkId, stream, streamFidelity) => {
+                if (typeof result !== "string" || !stream && isExecSessionLostBanner(result, process.providerSessionId))
                   return;
                 const events = await appendSessionCommandOutput(db, {
                   accountId: process.accountId,
@@ -6169,13 +6180,19 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
                   sessionId: process.sessionId,
                   commandId: process.id,
                   chunkId,
-                  stream: "stdout",
-                  streamFidelity: "merged",
-                  chunk: stripExecBanner(result)
+                  stream: stream ?? "stdout",
+                  streamFidelity: streamFidelity ?? (stream ? "separate" : "merged"),
+                  chunk: stream ? result : stripExecBanner(result)
                 });
                 if (events.length && bus)
                   await bus.publish(process.workspaceId, process.sessionId, events).catch(() => void 0);
-              }
+              },
+              retainedProviderCommandPersistence(db, {
+                accountId: process.accountId,
+                workspaceId: process.workspaceId,
+                sessionId: process.sessionId,
+                processId: process.id
+              })
             );
           } catch (error) {
             observability.warn("sandbox reaper: retained-process provider probe failed", {
@@ -6314,11 +6331,11 @@ async function deferRetainedProcessClaim(db, settings, observability, process, e
   }
 }
 function retainedProcessReconciliationDeferral(settings, process, outcome) {
-  const bindingQuarantine = process.reconcileAttempts >= RETAINED_PROCESS_BINDING_QUARANTINE_AFTER_ATTEMPTS && (outcome === "provider_binding_missing" || outcome === "provider_binding_mismatch");
+  const bindingQuarantine = process.reconcileAttempts >= RETAINED_PROCESS_BINDING_QUARANTINE_AFTER_ATTEMPTS && (outcome === "provider_binding_missing" || outcome === "provider_binding_mismatch" || outcome === "process_observation_unavailable");
   if (bindingQuarantine) {
     return {
       durableOutcome: `quarantined_${outcome}`,
-      metricOutcome: outcome === "provider_binding_missing" ? "quarantined_binding_missing" : "quarantined_binding_mismatch",
+      metricOutcome: outcome === "provider_binding_missing" ? "quarantined_binding_missing" : outcome === "provider_binding_mismatch" ? "quarantined_binding_mismatch" : "quarantined_process_observation_unavailable",
       retryAfterMs: RETAINED_PROCESS_BINDING_QUARANTINE_RETRY_MS,
       quarantined: true
     };
@@ -6363,7 +6380,7 @@ async function captureRetainedProbeOutput(processId, result, capture) {
   await capture(pending.result, pending.chunkId);
   pendingRetainedProbeOutput.delete(processId);
 }
-async function probeRetainedProcessAtProvider(settings, lease, process, mode = "observe", captureOutput) {
+async function probeRetainedProcessAtProvider(settings, lease, process, mode = "observe", captureOutput, providerPersistence) {
   if (lease.id !== process.leaseId || lease.sandboxGroupId !== process.sandboxGroupId || lease.leaseEpoch !== process.leaseEpoch || lease.backend !== process.providerBackend || lease.instanceId !== process.providerInstanceId || lease.resumeBackendId !== process.providerBackend || process.routeTargetId !== null) {
     return { status: "deferred", reason: "identity_mismatch" };
   }
@@ -6477,7 +6494,27 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
     if (liveBinding.key !== process.providerBindingKey || configuredModalBinding?.key !== process.providerBindingKey) {
       return { status: "deferred", reason: "provider_binding_mismatch" };
     }
+    const command = await providerPersistence?.load();
+    if (!command || !providerPersistence || !session.bindProviderCommand)
+      return { status: "deferred", reason: "process_observation_unavailable" };
+    session.bindProviderCommand(process.providerSessionId, command, providerPersistence);
   }
+  const capturePage = async (value) => {
+    if (!captureOutput) {
+      if (session.getProviderCommandOutput?.(value))
+        throw new Error("Provider command output requires durable capture before settlement");
+      return;
+    }
+    const page = session.getProviderCommandOutput?.(value);
+    if (page) {
+      for (const chunk of page.chunks)
+        if (chunk.text)
+          await captureOutput(chunk.text, chunk.chunkId, chunk.stream, page.streamFidelity);
+    } else {
+      await captureRetainedProbeOutput(process.id, value, captureOutput);
+    }
+    if (typeof value === "string") await session.acknowledgeCommandOutput?.(value);
+  };
   let result;
   try {
     result = await withRetainedProcessProbeTimeout(
@@ -6492,7 +6529,7 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
     if (error === RETAINED_PROCESS_PROBE_TIMEOUT) {
       return { status: "deferred", reason: "provider_timeout" };
     }
-    if (isProviderSandboxNotFoundError(client.backendId, error)) {
+    if (isProviderSandboxGoneDuringRoutedOperation(client.backendId, error)) {
       if (process.providerBackend === "modal" && !process.providerBindingKey) {
         return { status: "deferred", reason: "provider_binding_missing" };
       }
@@ -6507,8 +6544,8 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
     }
     return { status: "deferred", reason: "provider_error" };
   }
-  if (captureOutput) await captureRetainedProbeOutput(process.id, result, captureOutput);
-  const observation = classifyRetainedProcessPollResult(result, process.providerSessionId);
+  await capturePage(result);
+  const observation = classifyRetainedProcessPollResult(result, process.providerSessionId, session);
   if (observation.status === "deferred" && observation.reason === "provider_running" && lease.rotationRequestedAt !== null) {
     try {
       const interrupted = await withRetainedProcessProbeTimeout(
@@ -6519,10 +6556,10 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
           maxOutputTokens: 2e3
         })
       );
-      if (captureOutput) await captureRetainedProbeOutput(process.id, interrupted, captureOutput);
-      return classifyRetainedProcessPollResult(interrupted, process.providerSessionId);
+      await capturePage(interrupted);
+      return classifyRetainedProcessPollResult(interrupted, process.providerSessionId, session);
     } catch (error) {
-      if (isProviderSandboxNotFoundError(client.backendId, error)) {
+      if (isProviderSandboxGoneDuringRoutedOperation(client.backendId, error)) {
         if (process.providerBackend === "modal" && !process.providerBindingKey) {
           return { status: "deferred", reason: "provider_binding_missing" };
         }
@@ -6543,11 +6580,11 @@ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "
   }
   return observation;
 }
-function classifyRetainedProcessPollResult(result, providerSessionId) {
+function classifyRetainedProcessPollResult(result, providerSessionId, source) {
   if (typeof result !== "string") {
     return { status: "deferred", reason: "provider_unknown" };
   }
-  if (isExecSessionLostBanner(result, providerSessionId)) {
+  if (isExecSessionLostBanner(result, providerSessionId, source)) {
     return {
       status: "proved",
       proof: {
@@ -6848,7 +6885,7 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
   if (!lease) {
     return false;
   }
-  if (lease.liveness !== "draining" || lease.refcount !== 0 || lease.leaseEpoch !== row.leaseEpoch) {
+  if (lease.liveness !== "draining" || lease.refcount !== 0 && !lease.unobservableCommandDrainIds?.length || lease.leaseEpoch !== row.leaseEpoch) {
     return false;
   }
   const durableBackendId = lease.resumeBackendId ?? lease.backend;
@@ -8294,6 +8331,7 @@ function createScheduledTaskActivities(services) {
                 workspaceId: task.workspaceId,
                 initialMessage: task.agentConfig.prompt,
                 resources: task.agentConfig.resources,
+                bundledSkillIds: task.agentConfig.bundledSkillIds,
                 tools: taskTools,
                 firstPartyMcpTools,
                 firstPartyMcpPermissions,
@@ -9148,6 +9186,7 @@ async function recoverBoundScheduledTaskDispatch(input) {
       workspaceId: task.workspaceId,
       initialMessage: task.agentConfig.prompt,
       resources: task.agentConfig.resources,
+      bundledSkillIds: task.agentConfig.bundledSkillIds,
       tools: input.acceptedExecution.resolvedTools,
       firstPartyMcpTools: input.acceptedExecution.resolvedFirstPartyMcpTools,
       firstPartyMcpPermissions: input.acceptedExecution.resolvedFirstPartyMcpPermissions,
@@ -10176,7 +10215,7 @@ function createWorkflowWakeActivities(services) {
       const timers = await listDueWorkspacePauseTimers(service.db, 100);
       for (const timer of timers) {
         try {
-          const result = await withWorkspaceRls(
+          const result2 = await withWorkspaceRls(
             service.db,
             timer.workspace_id,
             (scoped) => scoped.transaction(
@@ -10186,11 +10225,11 @@ function createWorkflowWakeActivities(services) {
               })
             )
           );
-          if (result?.workspaceControlEventId) {
+          if (result2?.workspaceControlEventId) {
             const event = await getWorkspaceControlEvent(
               service.db,
               timer.workspace_id,
-              result.workspaceControlEventId
+              result2.workspaceControlEventId
             );
             if (event)
               await publishDurableWorkspaceControlEvent(service.bus, timer.workspace_id, event);
@@ -10203,6 +10242,10 @@ function createWorkflowWakeActivities(services) {
         }
       }
       let claimed = 0;
+      let signaled = 0;
+      let pendingAdmission = 0;
+      let unconfirmed = 0;
+      const pendingAdmissionBlockers = {};
       let delivered = 0;
       let failed = 0;
       let exhaustedBatchLimit = false;
@@ -10215,7 +10258,14 @@ function createWorkflowWakeActivities(services) {
         const limit = Math.min(BATCH_SIZE, remaining);
         const batch = await reconcilePendingSessionWorkflowWakes(service, limit);
         claimed += batch.claimed;
+        signaled += batch.signaled;
         delivered += batch.delivered;
+        pendingAdmission += batch.pendingAdmission;
+        unconfirmed += batch.unconfirmed;
+        for (const [blocker, count] of Object.entries(batch.pendingAdmissionBlockers)) {
+          const key = blocker;
+          pendingAdmissionBlockers[key] = (pendingAdmissionBlockers[key] ?? 0) + count;
+        }
         failed += batch.failed;
         if (batch.claimed < limit) break;
       }
@@ -10230,7 +10280,34 @@ function createWorkflowWakeActivities(services) {
           failed: titleFanout.failed
         });
       }
-      return { claimed, delivered, failed, exhaustedBatchLimit };
+      const result = {
+        claimed,
+        signaled,
+        delivered,
+        pendingAdmission,
+        unconfirmed,
+        failed,
+        pendingAdmissionBlockers,
+        exhaustedBatchLimit
+      };
+      if (claimed > 0) {
+        service.observability.info("session workflow wake delivery reconciled", {
+          claimed,
+          signaled,
+          delivered,
+          pendingAdmission,
+          unconfirmed,
+          failed,
+          exhaustedBatchLimit,
+          ...Object.fromEntries(
+            Object.entries(pendingAdmissionBlockers).map(([blocker, count]) => [
+              `pendingAdmissionBlockers.${blocker}`,
+              count
+            ])
+          )
+        });
+      }
+      return result;
     }
   };
 }
@@ -10273,4 +10350,4 @@ export {
   createControlActivities,
   createControlActivitiesFromServices
 };
-//# sourceMappingURL=activities-control-VMA3XMHE.js.map
+//# sourceMappingURL=activities-control-WZRFEIJH.js.map