@opengeni/worker-bundle 2.0.1 → 2.0.3

package/dist/{activities-turn-YVVKCGAK.js → activities-turn-EZEDZTEZ.js}

@@ -86,7 +86,7 @@ import {
   withFirstPartyTools,
   wrapLazyTurnBoxWithRouting,
   wrapTurnBoxWithRouting
-} from "./chunk-M3NVMF2V.js";
+} from "./chunk-JMO5ZUW3.js";
 import {
   StreamTimingMetrics,
   makeMachineOpObserver,
@@ -1445,11 +1445,13 @@ import {
   COMPUTER_SCREENSHOT_RETENTION_MS,
   COMPUTER_SCREENSHOT_WORKSPACE_QUOTA_BYTES,
   RetainedArtifactMetadataSchema,
-  retainedScreenshotReferenceFromFile
+  retainedScreenshotReferenceFromFile,
+  retainedSessionScreenshotKindFromObjectKey
 } from "@opengeni/contracts";
 import {
   RetainedScreenshotQuotaExceededError,
   getRetainedScreenshotArtifact,
+  getRetainedScreenshotArtifactForToolCall,
   isDatabasePersistenceFailure,
   isSessionEventPersistenceError,
   prepareRetainedScreenshotArtifact,
@@ -1554,13 +1556,16 @@ function toolOutputContainsInlineImage(output) {
     return output.some((entry) => {
       if (!entry || typeof entry !== "object" || Array.isArray(entry)) return false;
       const record7 = entry;
-      return record7.type === "input_image" && sdkImageSourceContainsInlineImage(record7.image ?? record7.image_url ?? record7.imageUrl) || record7.type === "image" && decodeInlineMcpImage(record7.data, record7.mimeType) !== null;
+      return record7.type === "input_image" && sdkImageSourceContainsInlineImage(record7.image ?? record7.image_url ?? record7.imageUrl) || record7.type === "image" && isDeclaredInlineMcpImage(record7);
     });
   }
   if (!output || typeof output !== "object") return false;
   const record6 = output;
   if (Array.isArray(record6.content)) return toolOutputContainsInlineImage(record6.content);
-  return record6.type === "image" && sdkImageSourceContainsInlineImage(record6.image);
+  return record6.type === "image" && (sdkImageSourceContainsInlineImage(record6.image) || isDeclaredInlineMcpImage(record6));
+}
+function isDeclaredInlineMcpImage(record6) {
+  return typeof record6.data === "string" && typeof record6.mimeType === "string" && record6.mimeType.toLowerCase().startsWith("image/");
 }
 function decodeInlineMcpImage(data, mimeType) {
   if (typeof data !== "string" || typeof mimeType !== "string" || !["image/png", "image/jpeg", "image/webp"].includes(mimeType)) {
@@ -1798,7 +1803,9 @@ function validateRetainableSessionImage(input, limits = {}) {
   };
 }
 function retainedScreenshotIdentity(input) {
-  const settlementKey = createHash3("sha256").update("opengeni:computer-screenshot:v1\0").update(input.sessionId).update("\0").update(input.turnId).update("\0").update(input.attemptId).update("\0").update(input.toolCallId).update("\0").update(input.toolOutputId).digest("hex");
+  const settlementKey = createHash3("sha256").update(
+    input.kind === "browser_screenshot" ? "opengeni:browser-screenshot:v1\0" : "opengeni:computer-screenshot:v1\0"
+  ).update(input.sessionId).update("\0").update(input.turnId).update("\0").update(input.attemptId).update("\0").update(input.toolCallId).update("\0").update(input.toolOutputId).digest("hex");
   const uuidBytes = Uint8Array.from(Buffer.from(settlementKey.slice(0, 32), "hex"));
   uuidBytes[6] = uuidBytes[6] & 15 | 128;
   uuidBytes[8] = uuidBytes[8] & 63 | 128;
@@ -1812,13 +1819,14 @@ function unavailableRetainedSessionImage(input) {
   const identity = retainedScreenshotIdentity(input);
   return unavailable(identity.artifactId, input.reason);
 }
-async function retainComputerScreenshot(input) {
+async function retainSessionScreenshot(input) {
   const identity = retainedScreenshotIdentity({
     sessionId: input.sessionId,
     turnId: input.turnId,
     attemptId: input.attemptId,
     toolCallId: input.output.callId,
-    toolOutputId: input.output.toolOutputId
+    toolOutputId: input.output.toolOutputId,
+    kind: input.kind
   });
   let screenshot;
   try {
@@ -1837,7 +1845,8 @@ async function retainComputerScreenshot(input) {
   const retentionExpiresAt = new Date(
     now.getTime() + (input.retentionMs ?? COMPUTER_SCREENSHOT_RETENTION_MS)
   );
-  const objectKey = `workspaces/${input.workspaceId}/files/${identity.artifactId}/retained/session-image.${screenshot.extension}`;
+  const objectName = input.kind === "browser_screenshot" ? "browser-screenshot" : "session-image";
+  const objectKey = `workspaces/${input.workspaceId}/files/${identity.artifactId}/retained/${objectName}.${screenshot.extension}`;
   let prepared;
   try {
     prepared = (await prepareRetainedScreenshotArtifact(input.db, {
@@ -1924,6 +1933,7 @@ async function retainComputerScreenshot(input) {
     return unavailable(identity.artifactId, "pending");
   }
 }
+var retainComputerScreenshot = retainSessionScreenshot;
 function compactRetainedScreenshotHistory(history, receiptsByCallId) {
   if (receiptsByCallId.size === 0) return history;
   return history.map((item) => {
@@ -2005,6 +2015,20 @@ async function materializeRetainedScreenshotHistory(input) {
 }
 async function materializeRetainedScreenshotHistoryWithCache(input, cache) {
   const now = input.now ?? /* @__PURE__ */ new Date();
+  const receiptForMarker = async (marker, callId) => {
+    if (!isRetainedImageMarker(marker)) return null;
+    const receipt = retainedReceipt(marker.artifact);
+    if (receipt) return receipt;
+    if (!callId) throw new Error("Retained screenshot receipt has no tool call identity");
+    const artifact = await getRetainedScreenshotArtifactForToolCall(
+      input.db,
+      input.workspaceId,
+      input.sessionId,
+      callId
+    );
+    if (!artifact) throw new Error("Retained screenshot receipt cannot be recovered");
+    return reference(artifact);
+  };
   const dataUrlForReceipt = async (receipt) => {
     let dataUrl = cache.get(receipt.artifactId);
     if (!dataUrl) {
@@ -2045,15 +2069,17 @@ async function materializeRetainedScreenshotHistoryWithCache(input, cache) {
     }
     return dataUrl;
   };
-  const materializeEntry = async (entry) => {
-    const receipt = retainedReceiptFromImageContent(entry);
+  const materializeEntry = async (entry, callId) => {
+    const image = entry && typeof entry === "object" && !Array.isArray(entry) ? entry.image : null;
+    const receipt = await receiptForMarker(image, callId);
     if (!receipt) return entry;
     const dataUrl = await dataUrlForReceipt(receipt);
     return { ...entry, image: dataUrl };
   };
   const materialized = [];
   for (const item of input.history) {
-    const directReceipt = retainedReceiptFromDirectOutput(item.output);
+    const callId = historyCallId(item);
+    const directReceipt = await receiptForMarker(item.output, callId);
     if (directReceipt) {
       materialized.push({
         ...item,
@@ -2070,7 +2096,7 @@ async function materializeRetainedScreenshotHistoryWithCache(input, cache) {
     let changed = false;
     const output = [];
     for (const entry of outputEntries) {
-      const next = await materializeEntry(entry);
+      const next = await materializeEntry(entry, callId);
       changed ||= next !== entry;
       output.push(next);
     }
@@ -2124,12 +2150,15 @@ function reference(artifact) {
   if (!artifact.sessionId) {
     throw new Error(`Ready retained screenshot is detached: ${artifact.artifactId}`);
   }
+  const kind = retainedSessionScreenshotKindFromObjectKey(artifact.file.objectKey);
+  if (!kind) throw new Error(`Ready retained screenshot has unknown key: ${artifact.artifactId}`);
   const value = retainedScreenshotReferenceFromFile({
     ...artifact.file,
     sessionId: artifact.sessionId,
     width: artifact.width,
     height: artifact.height,
-    expiresAt: artifact.retentionExpiresAt.toISOString()
+    expiresAt: artifact.retentionExpiresAt.toISOString(),
+    kind
   });
   if (!value)
     throw new Error(`Ready retained screenshot receipt is invalid: ${artifact.artifactId}`);
@@ -2166,6 +2195,11 @@ function retainedReceiptFromImageContent(entry) {
   if (marker.type !== RETAINED_IMAGE_MARKER) return null;
   return retainedReceipt(marker.artifact);
 }
+function isRetainedImageMarker(value) {
+  return Boolean(
+    value && typeof value === "object" && !Array.isArray(value) && value.type === RETAINED_IMAGE_MARKER
+  );
+}
 function retainedReceiptFromDirectOutput(output) {
   if (!output || typeof output !== "object" || Array.isArray(output)) return null;
   const marker = output;
@@ -2175,12 +2209,12 @@ function retainedReceiptFromDirectOutput(output) {
 function retainedReceipt(value) {
   const parsed = RetainedArtifactMetadataSchema.safeParse(value);
   if (!parsed.success) return null;
-  return parsed.data.available && parsed.data.kind !== "computer_screenshot" ? null : parsed.data;
+  return parsed.data.available && parsed.data.kind !== "computer_screenshot" && parsed.data.kind !== "browser_screenshot" ? null : parsed.data;
 }
 function isInlineImageContent(entry) {
   if (!entry || typeof entry !== "object" || Array.isArray(entry)) return false;
   const record6 = entry;
-  return record6.type === "input_image" && typeof record6.image === "string" && record6.image.startsWith("data:image/") || record6.type === "image" && decodeInlineMcpImage(record6.data, record6.mimeType) !== null;
+  return record6.type === "input_image" && typeof record6.image === "string" && record6.image.startsWith("data:image/") || record6.type === "image" && isDeclaredInlineMcpImage(record6);
 }
 function historyCallId(item) {
   const value = item.callId ?? item.call_id;
@@ -2882,6 +2916,7 @@ var TurnMediaArtifacts = class {
   sandboxFileDownloadBackend;
   nativeImageGenerationRetention = null;
   retainedSessionImageCallIds = /* @__PURE__ */ new Set();
+  retainedSessionImageKindsByCallId = /* @__PURE__ */ new Map();
   rememberGeneratedImageCreatedThisTurn = (receipt) => {
     this.generatedImageReceiptsCreatedThisTurn.set(receipt.artifact.artifactId, receipt);
   };
@@ -3217,7 +3252,10 @@ function pendingToolCallFromSdkEvent(event) {
   };
 }
 function toolCallProducesRetainableSessionImage(name) {
-  return name === "computer_screenshot" || name === "view_image" || name === "interaction__computer_observe";
+  return name === "computer_screenshot" || name === "view_image" || name === "interaction__computer_observe" || retainableBrowserScreenshotToolCall(name);
+}
+function retainableBrowserScreenshotToolCall(name) {
+  return name === "browser_screenshot" || name === "interaction__browser_screenshot" || name === "browser_observe" || name === "interaction__browser_observe" || name === "browser_act" || name === "interaction__browser_act";
 }
 function completedToolCallFromSdkEvent(event) {
   if (!event || typeof event !== "object") return null;
@@ -5110,7 +5148,8 @@ import {
 } from "@opengeni/db";
 import { publishDurableSessionEvents as publishDurableSessionEvents2 } from "@opengeni/events";
 import { maxTurnsExceededRunState } from "@opengeni/runtime";
-import { CancelledFailure } from "@temporalio/activity";
+import { ApplicationFailure, CancelledFailure } from "@temporalio/activity";
+import { TurnExecutionPolicyDefinitionMismatchError } from "@opengeni/config";
 import {
   classifyCodexEncryptedArtifactRejection,
   classifyCodexUsageLimitError,
@@ -6853,10 +6892,11 @@ async function settleTurnFailure(deps) {
       return claimedResult({ status: "recovering" });
     }
   }
-  let failure = agentRunFailurePayload(error51, {
+  const earlyDefinitionMismatch = error51 instanceof TurnExecutionPolicyDefinitionMismatchError && !attempt.modelRequestStarted && !eventing.turnStartedPublished && !!attempt.turnId && !!attempt.triggerEventId && attempt.executionGeneration > 0;
+  let failure = earlyDefinitionMismatch ? { error: error51.message, code: error51.code, retryable: true } : agentRunFailurePayload(error51, {
     isCodexTurn: billingState.isCodexTurn
   });
-  if (failure.retryable && eventing.publish && attempt.turnId && eventing.turnStartedPublished) {
+  if (attempt.turnId && (earlyDefinitionMismatch || failure.retryable && eventing.publish && eventing.turnStartedPublished)) {
     const nextProviderRecoveryCount = attempt.providerRecoveryCount + 1;
     const recoveryResult = providerRecoveryResult({
       failureCode: failure.code,
@@ -6865,8 +6905,10 @@ async function settleTurnFailure(deps) {
     });
     try {
       if (recoveryResult.status === "recovering") {
-        await flushRuntimeBatcher();
-        await historySink.reconcileConversationTruth({ requireDurable: true });
+        if (!earlyDefinitionMismatch) {
+          await flushRuntimeBatcher();
+          await historySink.reconcileConversationTruth({ requireDurable: true });
+        }
         const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
           sessionId: input.sessionId,
           turnId: attempt.turnId,
@@ -6894,6 +6936,16 @@ async function settleTurnFailure(deps) {
         return claimedResult(recoveryResult);
       }
       failure = providerRecoveryExhaustedFailure(failure, recoveryResult);
+      if (earlyDefinitionMismatch) {
+        control.activityStatus = "failed";
+        control.turnMetricOutcome = "failed";
+        control.activityError = error51;
+        throw ApplicationFailure.create({
+          message: `${error51.message}. Automatic same-turn configuration recovery exhausted after ${recoveryResult.providerRecoveryCount} retries.`,
+          type: "TurnExecutionPolicyDefinitionMismatchError",
+          nonRetryable: true
+        });
+      }
     } catch (recoveryError) {
       const escaped = recoveryResult.status === "recovering" ? escapedMcpTimeoutRecoveryFailure({
         failureCode: failure.code,
@@ -8695,6 +8747,12 @@ async function runTurnStreamAttempt(deps) {
           currentToolBatchCallIds.add(pendingToolCall.callId);
           if (toolCallProducesRetainableSessionImage(pendingToolCall.callName)) {
             media.retainedSessionImageCallIds.add(pendingToolCall.callId);
+            if (retainableBrowserScreenshotToolCall(pendingToolCall.callName)) {
+              media.retainedSessionImageKindsByCallId.set(
+                pendingToolCall.callId,
+                "browser_screenshot"
+              );
+            }
           }
         }
         const completedToolCall = completedToolCallFromSdkEvent(durableSdkEvent);
@@ -8708,7 +8766,8 @@ async function runTurnStreamAttempt(deps) {
               attemptId: input.attemptId,
               toolCallId: completedToolCall.callId,
               toolOutputId: completedToolCall.callId,
-              reason: "unsupported"
+              reason: "unsupported",
+              kind: media.retainedSessionImageKindsByCallId.get(completedToolCall.callId)
             });
             media.retainedScreenshotReceiptsByCallId.set(
               completedToolCall.callId,
@@ -8722,13 +8781,14 @@ async function runTurnStreamAttempt(deps) {
               attemptId: input.attemptId,
               toolCallId: typedScreenshot.callId,
               toolOutputId: typedScreenshot.toolOutputId,
-              reason: "pending"
+              reason: "pending",
+              kind: media.retainedSessionImageKindsByCallId.get(completedToolCall.callId)
             });
             media.retainedScreenshotReceiptsByCallId.set(
               completedToolCall.callId,
               retainedScreenshotMetadata
             );
-            retainedScreenshotMetadata = await retainComputerScreenshot({
+            retainedScreenshotMetadata = await retainSessionScreenshot({
               db,
               objectStorage,
               accountId: input.accountId,
@@ -8736,7 +8796,8 @@ async function runTurnStreamAttempt(deps) {
               sessionId: input.sessionId,
               turnId: activeTurnId,
               attemptId: input.attemptId,
-              output: typedScreenshot
+              output: typedScreenshot,
+              kind: media.retainedSessionImageKindsByCallId.get(completedToolCall.callId)
             });
             media.retainedScreenshotReceiptsByCallId.set(
               completedToolCall.callId,
@@ -8855,6 +8916,7 @@ async function runTurnStreamAttempt(deps) {
           }
           for (const callId of stableToolCallIdsToClear) {
             media.retainedSessionImageCallIds.delete(callId);
+            media.retainedSessionImageKindsByCallId.delete(callId);
           }
         }
       }
@@ -12004,7 +12066,7 @@ async function runPostAgentCompaction(deps) {
 import {
   advanceWorkspaceGeneration,
   verifyWorkspaceMutationSettlement,
-  heartbeatLeaseHolder,
+  heartbeatLeaseHolderStatus,
   readLease as readLease2,
   accrueWarmSeconds,
   SandboxWorkspaceMutationFencedError
@@ -12312,21 +12374,29 @@ function createSandboxTurnRuntime(deps) {
     const heartbeatEpoch = sandbox.leaseEpoch;
     const heartbeatHolderId = sandboxState.sandboxHolderId;
     const heartbeatGroupId = sandboxState.sandboxGroupId;
-    const warmRate = sandboxWarmRateMicrosPerSecond(
+    const warmRate = settings.sandboxWarmBillingMode === "usage_only" ? 0 : sandboxWarmRateMicrosPerSecond(
       settings,
       warmBackend ?? sandbox.established.backendId
     );
     sandboxState.leaseHeartbeatTimer = setInterval(() => {
-      void heartbeatLeaseHolder(db, {
+      void heartbeatLeaseHolderStatus(db, {
         accountId: input.accountId,
         workspaceId: input.workspaceId,
         sandboxGroupId: heartbeatGroupId,
         kind: "turn",
         holderId: heartbeatHolderId,
         leaseTtlMs: settings.sandboxLeaseTtlMs,
-        expectedEpoch: heartbeatEpoch
-      }).then(async (alive) => {
-        if (alive) return;
+        expectedEpoch: heartbeatEpoch,
+        billingMode: settings.sandboxWarmBillingMode
+      }).then(async (status) => {
+        if (status.fence === "funding") {
+          stopLeaseHeartbeat();
+          sandboxRotationController.abort(
+            new Error("Insufficient OpenGeni credits to extend paid sandbox compute")
+          );
+          return;
+        }
+        if (status.leaseExtended) return;
         const rotation = await beginRotationPreemption(sandbox, heartbeatEpoch, heartbeatGroupId);
         if (rotation === "not_rotating") {
           stopLeaseHeartbeat();
@@ -12338,8 +12408,13 @@ function createSandboxTurnRuntime(deps) {
         sandboxGroupId: heartbeatGroupId,
         expectedEpoch: heartbeatEpoch,
         warmRateMicrosPerSecond: warmRate,
+        billingMode: settings.sandboxWarmBillingMode,
         subjectId: input.sessionId
-      }).then((result) => recordCreditMicros(observability, "usage", result.costMicros)).catch(() => void 0);
+      }).then((result) => {
+        if (settings.sandboxWarmBillingMode === "credits") {
+          recordCreditMicros(observability, "usage", result.costMicros);
+        }
+      }).catch(() => void 0);
       const snapshotSession = sandboxState.setupBoxSession;
       const snapshotTurnId = attempt.turnId;
       if (snapshotSession && snapshotTurnId && shouldStartPeriodicWorkspaceSnapshot({
@@ -37533,4 +37608,4 @@ export {
   createTurnActivities,
   createTurnActivitiesFromServices
 };
-//# sourceMappingURL=activities-turn-YVVKCGAK.js.map
+//# sourceMappingURL=activities-turn-EZEDZTEZ.js.map