@opengeni/worker-bundle 3.0.1-canary.38872461489001 → 3.1.2

package/dist/index.js

@@ -1434,7 +1434,7 @@ async function createDefaultWorkerActivities(role, dependencies = {}) {
   if (dependencies.objectStorage == null) {
     prepareObjectStorageProvider(dependencies.settings ?? getSettings());
   }
-  return role === "control" ? (await import("./activities-control-3PJDM3QQ.js")).createControlActivities(dependencies) : (await import("./activities-turn-2ILW6PRK.js")).createTurnActivities(dependencies);
+  return role === "control" ? (await import("./activities-control-ROJJNNTJ.js")).createControlActivities(dependencies) : (await import("./activities-turn-4SWYWKI5.js")).createTurnActivities(dependencies);
 }
 function resolveOpenGeniWorkflowDefinition(moduleUrl = import.meta.url) {
   const modulePath = fileURLToPath(moduleUrl).replaceAll("\\", "/");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opengeni/worker-bundle",
-  "version": "3.0.1-canary.38872461489001",
+  "version": "3.1.2",
   "description": "Opengeni worker entry and reusable embedded lifecycle, shipped with a release-coherent pre-bundled Temporal workflow artifact.",
   "license": "Apache-2.0",
   "repository": {
@@ -55,29 +55,29 @@
   "dependencies": {
     "@1password/sdk": "0.5.0",
     "@llamaindex/liteparse": "2.14.2",
-    "@opengeni/agent-proto": "3.0.1-canary.38872461489001",
-    "@opengeni/capabilities": "3.0.1-canary.38872461489001",
-    "@opengeni/codemode": "3.0.1-canary.38872461489001",
-    "@opengeni/codex": "3.0.1-canary.38872461489001",
-    "@opengeni/config": "3.0.1-canary.38872461489001",
-    "@opengeni/contracts": "3.0.1-canary.38872461489001",
-    "@opengeni/core": "3.0.1-canary.38872461489001",
-    "@opengeni/db": "3.0.1-canary.38872461489001",
-    "@opengeni/documents": "3.0.1-canary.38872461489001",
-    "@opengeni/events": "3.0.1-canary.38872461489001",
-    "@opengeni/github": "3.0.1-canary.38872461489001",
-    "@opengeni/interaction": "3.0.1-canary.38872461489001",
-    "@opengeni/jev": "3.0.1-canary.38872461489001",
-    "@opengeni/network": "3.0.1-canary.38872461489001",
-    "@opengeni/observability": "3.0.1-canary.38872461489001",
-    "@opengeni/runtime": "3.0.1-canary.38872461489001",
-    "@opengeni/sdk": "3.0.1-canary.38872461489001",
-    "@opengeni/storage": "3.0.1-canary.38872461489001",
-    "@opengeni/subscriptions": "3.0.1-canary.38872461489001",
-    "@opengeni/tool-gateway": "3.0.1-canary.38872461489001",
-    "@opengeni/vaults": "3.0.1-canary.38872461489001",
-    "@opengeni/vaults-onepassword": "3.0.1-canary.38872461489001",
-    "@opengeni/xai-subscription": "3.0.1-canary.38872461489001",
+    "@opengeni/agent-proto": "3.1.2",
+    "@opengeni/capabilities": "3.1.2",
+    "@opengeni/codemode": "3.1.2",
+    "@opengeni/codex": "3.1.2",
+    "@opengeni/config": "3.1.2",
+    "@opengeni/contracts": "3.1.2",
+    "@opengeni/core": "3.1.2",
+    "@opengeni/db": "3.1.2",
+    "@opengeni/documents": "3.1.2",
+    "@opengeni/events": "3.1.2",
+    "@opengeni/github": "3.1.2",
+    "@opengeni/interaction": "3.1.2",
+    "@opengeni/jev": "3.1.2",
+    "@opengeni/network": "3.1.2",
+    "@opengeni/observability": "3.1.2",
+    "@opengeni/runtime": "3.1.2",
+    "@opengeni/sdk": "3.1.2",
+    "@opengeni/storage": "3.1.2",
+    "@opengeni/subscriptions": "3.1.2",
+    "@opengeni/tool-gateway": "3.1.2",
+    "@opengeni/vaults": "3.1.2",
+    "@opengeni/vaults-onepassword": "3.1.2",
+    "@opengeni/xai-subscription": "3.1.2",
     "@temporalio/activity": "^1.17.0",
     "@temporalio/client": "^1.17.0",
     "@temporalio/worker": "^1.17.0",

package/src/activities/agent-turn/errors.ts

@@ -57,7 +57,10 @@ import {
 import { ApplicationFailure, CancelledFailure } from "@temporalio/activity";
 import type { ProviderErrorOutcome } from "@opengeni/subscriptions";
 import { CODEX_USAGE_EXHAUSTED_PCT } from "./codex-usage-policy";
-import { RetainedAttachmentTransportLimitError } from "../run-input";
+import {
+  RetainedAttachmentTransportLimitError,
+  RetainedAttachmentUnavailableError,
+} from "../run-input";
 import type { CodexAccountStatus } from "@opengeni/db";
 import {
   CODEX_USAGE_LIMIT_ERROR_TYPE,
@@ -144,6 +147,21 @@ export function providerRecoveryJitterMs(delayMs: number, sample: number): numbe
   return Math.floor(Math.min(5_000, delayMs * 0.2) * bounded);
 }
 export const POST_COMPACTION_CONTINUATION_EMPTY_CODE = "post_compaction_continuation_empty";
+export const RETAINED_ATTACHMENT_UNAVAILABLE_CODE = "retained_attachment_unavailable";
+
+function findRetainedAttachmentUnavailableError(
+  error: unknown,
+): RetainedAttachmentUnavailableError | null {
+  let current = error;
+  const seen = new Set<unknown>();
+  for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
+    if (current instanceof RetainedAttachmentUnavailableError) return current;
+    if (seen.has(current)) break;
+    seen.add(current);
+    current = (current as { cause?: unknown }).cause;
+  }
+  return null;
+}
 
 export class PostCompactionContinuationEmptyError extends Error {
   readonly code = POST_COMPACTION_CONTINUATION_EMPTY_CODE;
@@ -221,6 +239,7 @@ export function providerRecoveryResult(input: {
           input.failureCode === "provider_unavailable" ||
           input.failureCode === "upstream_connectivity_unavailable" ||
           input.failureCode === "sandbox_command_start_unavailable" ||
+          input.failureCode === RETAINED_ATTACHMENT_UNAVAILABLE_CODE ||
           input.failureCode === "mcp_transport_timeout" ||
           input.failureCode === "mcp_transport_unavailable" ||
           input.failureCode === "turn_execution_policy_definition_mismatch" ||
@@ -1809,6 +1828,16 @@ function baseAgentRunFailurePayload(
   if (error instanceof RetainedAttachmentTransportLimitError) {
     return { error: error.message, code: "retained_attachment_transport_limit", retryable: false };
   }
+  if (findRetainedAttachmentUnavailableError(error)) {
+    // Storage may lag after a replica outage. The same turn retries with the
+    // connectivity backoff and never continues without the image's pixels.
+    return {
+      error:
+        "An image attached earlier in this conversation can't be read from storage right now. The same turn will retry after a short delay.",
+      code: RETAINED_ATTACHMENT_UNAVAILABLE_CODE,
+      retryable: true,
+    };
+  }
   if (error instanceof MandatoryHistoryPersistenceError) {
     const databaseFailure =
       isSessionEventPersistenceError(error.cause) ||

package/src/activities/agent-turn/provider-recovery-copy.ts

@@ -56,6 +56,8 @@ export function providerRecoverySubject(input: ProviderRecoveryCopyInput): strin
       return "A required MCP server is unreachable";
     case "sandbox_command_start_unavailable":
       return "The sandbox isn't ready yet";
+    case "retained_attachment_unavailable":
+      return "An image from earlier in this conversation can't be read from storage";
     case "turn_execution_policy_definition_mismatch":
       return "Opengeni is applying a configuration update";
     default:

package/src/activities/run-input.ts

@@ -2,6 +2,7 @@ import {
   MODEL_ATTACHMENT_REFS_FIELD,
   MODEL_ATTACHMENT_CATALOG_MARKER,
   MODEL_ATTACHMENT_CATALOG_RENDERED_FIELD,
+  MODEL_ATTACHMENT_IMAGE_MAX_DIMENSION_FIELD,
   FileResourceRef,
   resourceMountPath,
   readTurnExecutionPolicyV1,
@@ -21,6 +22,7 @@ import {
   type Database,
 } from "@opengeni/db";
 import {
+  createModelImageSizer,
   projectHistoryForProvider,
   projectRejectedProviderArtifactsFromSerializedRunState,
   projectRejectedReasoningArtifact,
@@ -219,6 +221,25 @@ export class RetainedAttachmentTransportLimitError extends Error {
   }
 }
 
+/**
+ * A retained image the conversation already sent could not be read from
+ * object storage, for example while a storage replica catches up after an
+ * outage. Its pixels are part of the accepted prompt, so the turn waits and
+ * retries rather than continuing without them. Integrity mismatches are not
+ * this error: retrying cannot repair different bytes.
+ */
+export class RetainedAttachmentUnavailableError extends Error {
+  readonly fileIds: readonly string[];
+
+  constructor(fileIds: readonly string[]) {
+    super("An image attached earlier in this conversation can't be read from storage right now.");
+    this.name = "RetainedAttachmentUnavailableError";
+    this.fileIds = [...fileIds];
+  }
+}
+
+export type ModelAttachmentReadFailure = "unavailable" | "integrity_mismatch";
+
 export type ModelAttachmentContent = {
   kind: "image" | "file";
   fileId: string;
@@ -286,7 +307,11 @@ function modelAttachmentDescriptor(
 export async function modelAttachmentContentForFiles(
   files: FileAsset[],
   readFileBytes: (file: FileAsset) => Promise<Uint8Array>,
-  options: { retainAll?: boolean } = {},
+  options: {
+    retainAll?: boolean;
+    /** Why a selected file produced no content; omitted files are not reported. */
+    onReadFailure?: (fileId: string, failure: ModelAttachmentReadFailure) => void;
+  } = {},
 ): Promise<ModelAttachmentContent[]> {
   const selected: Array<{
     file: FileAsset;
@@ -327,12 +352,14 @@ export async function modelAttachmentContentForFiles(
             expectedSizeBytes: file.sizeBytes,
             actualSizeBytes: bytes.byteLength,
           });
+          options.onReadFailure?.(file.id, "integrity_mismatch");
           continue;
         }
         if (createHash("sha256").update(bytes).digest("hex") !== checksum) {
           console.error("model attachment checksum did not match finalized metadata", {
             fileId: file.id,
           });
+          options.onReadFailure?.(file.id, "integrity_mismatch");
           continue;
         }
         attachments[index] = {
@@ -350,6 +377,7 @@ export async function modelAttachmentContentForFiles(
           fileId: file.id,
           errorType: safeErrorType(error),
         });
+        options.onReadFailure?.(file.id, "unavailable");
       }
     }
   };
@@ -392,10 +420,64 @@ function unavailableAttachmentReceiptText(ref: FileResourceRef): string {
   return requesterUnavailableReceiptText(`Attachment: fileId=${ref.fileId}`, "file");
 }
 
+/** The image bound stamped on a message when it was accepted, if any. */
+function attachmentImageBound(item: Record<string, unknown>): number | undefined {
+  const raw = item[MODEL_ATTACHMENT_IMAGE_MAX_DIMENSION_FIELD];
+  return typeof raw === "number" && Number.isSafeInteger(raw) && raw >= 64 && raw <= 8192
+    ? raw
+    : undefined;
+}
+
+// Process-wide, so a long conversation does not re-encode the same photos on
+// every turn. The projection depends only on the image bytes and the bound.
+const boundedImageSizers = new Map<number, ReturnType<typeof createModelImageSizer>>();
+
+type BoundedAttachmentImage = { image: string; note: string };
+
+/**
+ * The model-facing rendition of one image under the bound its message was
+ * stamped with. Smaller images keep their original bytes. The file in the
+ * sandbox and in storage stays at full resolution.
+ */
+async function boundedAttachmentImage(
+  attachment: ModelAttachmentContent,
+  bound: number,
+): Promise<BoundedAttachmentImage> {
+  const original = { image: attachment.dataUrl, note: "" };
+  const comma = attachment.dataUrl.indexOf(",");
+  if (comma < 0) return original;
+  const data = attachment.dataUrl.slice(comma + 1);
+  let sizer = boundedImageSizers.get(bound);
+  if (!sizer) {
+    sizer = createModelImageSizer(bound, 10 * 1024 * 1024, { jpegQuality: 85 });
+    boundedImageSizers.set(bound, sizer);
+  }
+  try {
+    const sized = await sizer(data, attachment.contentType);
+    if (sized.data === data && sized.mediaType === attachment.contentType) return original;
+    return {
+      image: `data:${sized.mediaType};base64,${sized.data}`,
+      note:
+        ` Shown to you downscaled to ${sized.width}x${sized.height} from ` +
+        `${sized.originalWidth}x${sized.originalHeight}; the file itself is full resolution.`,
+    };
+  } catch (error) {
+    // Decoding is deterministic: an image that cannot be resized here keeps
+    // the original projection on every turn rather than being dropped.
+    console.error("model attachment image could not be resized; sending original", {
+      fileId: attachment.fileId,
+      errorType: safeErrorType(error),
+    });
+    return original;
+  }
+}
+
 /**
  * Build one turn-scoped durable-attachment projector. Resolve active refs under
  * current file authority and memoize metadata/bytes for retries and compaction.
  * A turn boundary never changes an authorized attachment's representation.
+ * Images of a message stamped with an image bound are shown downscaled to fit
+ * it; unstamped (older) messages keep their full-resolution projection.
  */
 export function createModelHistoryAttachmentProjector(
   policy: ModelAttachmentInputPolicy,
@@ -407,6 +489,7 @@ export function createModelHistoryAttachmentProjector(
   const fileById = new Map<string, FileAsset>();
   const resolvedFileIds = new Set<string>();
   const retainedByteSizes = new Map<string, number>();
+  const boundedById = new Map<string, BoundedAttachmentImage>();
 
   return async (items, options = {}) => {
     for (const file of options.inlineFiles ?? []) {
@@ -475,25 +558,44 @@ export function createModelHistoryAttachmentProjector(
           return Boolean(descriptor && descriptor.kind === "image" && policy.supportsImageInput);
         });
       for (const file of readable) attemptedContentIds.add(file.id);
+      const readFailures = new Map<string, ModelAttachmentReadFailure>();
       const content = await modelAttachmentContentForFiles(readable, readFileBytes, {
         retainAll: true,
+        onReadFailure: (fileId, failure) => readFailures.set(fileId, failure),
       });
       const loadedIds = new Set(content.map((entry) => entry.fileId));
-      for (const file of readable) {
-        if (
+      const missing = readable.filter(
+        (file) =>
           file.status === "ready" &&
           file.sizeBytes <= MAX_INLINE_MODEL_ATTACHMENT_BYTES &&
           /^[a-f0-9]{64}$/i.test(file.sha256 ?? "") &&
-          !loadedIds.has(file.id)
-        ) {
-          // Do not let a retry silently proceed with pixels removed.
-          for (const candidate of readable) attemptedContentIds.delete(candidate.id);
-          throw new Error(`Retained attachment bytes unavailable or invalid: ${file.id}`);
+          !loadedIds.has(file.id),
+      );
+      if (missing.length > 0) {
+        // Do not let a retry silently proceed with pixels removed.
+        for (const candidate of readable) attemptedContentIds.delete(candidate.id);
+        const corrupt = missing.find((file) => readFailures.get(file.id) === "integrity_mismatch");
+        if (corrupt) {
+          throw new Error(`Retained attachment bytes unavailable or invalid: ${corrupt.id}`);
         }
+        throw new RetainedAttachmentUnavailableError(missing.map((file) => file.id));
       }
       for (const attachment of content) contentById.set(attachment.fileId, attachment);
     }
 
+    for (const [index, refs] of refsByIndex) {
+      const item = items[index]!;
+      if (item[MODEL_ATTACHMENT_CATALOG_MARKER] === true) continue;
+      const bound = attachmentImageBound(item);
+      if (bound === undefined) continue;
+      for (const ref of refs) {
+        const attachment = fileById.has(ref.fileId) ? contentById.get(ref.fileId) : undefined;
+        const key = `${bound}:${ref.fileId}`;
+        if (!attachment || attachment.kind !== "image" || boundedById.has(key)) continue;
+        boundedById.set(key, await boundedAttachmentImage(attachment, bound));
+      }
+    }
+
     const projected = [...items];
     for (const [index, refs] of refsByIndex) {
       const original = items[index]!;
@@ -519,14 +621,20 @@ export function createModelHistoryAttachmentProjector(
           resolvedFileIds.has(ref.fileId) &&
           currentFile === undefined;
         const attachment = currentFile ? contentById.get(ref.fileId) : undefined;
-        const receipt = {
-          type: "input_text",
-          text: excluded ? unavailableAttachmentReceiptText(ref) : attachmentReceiptText(ref),
-        };
         if (!attachment || attachment.kind !== "image") {
-          return [receipt];
+          return [
+            {
+              type: "input_text",
+              text: excluded ? unavailableAttachmentReceiptText(ref) : attachmentReceiptText(ref),
+            },
+          ];
         }
-        return [receipt, { type: "input_image", image: attachment.dataUrl }];
+        const bound = attachmentImageBound(original);
+        const bounded = bound === undefined ? undefined : boundedById.get(`${bound}:${ref.fileId}`);
+        return [
+          { type: "input_text", text: attachmentReceiptText(ref) + (bounded?.note ?? "") },
+          { type: "input_image", image: bounded?.image ?? attachment.dataUrl },
+        ];
       });
       const clone: Record<string, unknown> = {
         ...original,