@opengeni/runtime 3.0.2-canary.38894290271001 → 3.1.2

package/dist/{chunk-EQQ3YOKR.js → chunk-7OUG7WBV.js}

@@ -76,6 +76,7 @@ import {
 import {
   MODEL_ATTACHMENT_CATALOG_MARKER,
   MODEL_ATTACHMENT_CATALOG_RENDERED_FIELD,
+  MODEL_ATTACHMENT_IMAGE_MAX_DIMENSION_FIELD,
   MODEL_ATTACHMENT_REFS_FIELD,
   MODEL_TIMELINE_ANNOTATIONS_FIELD
 } from "@opengeni/contracts";
@@ -239,12 +240,13 @@ function sanitizeHistoryItemsForModel(items, toolOutputTruncationTokens = DEFAUL
 var INTERNAL_RESUME_MESSAGE_MARKER = "opengeni_internal_resume";
 function stripInternalModelMetadata(item) {
   const withoutResume = stripInternalResumeMarker(item);
-  if (!(MODEL_ATTACHMENT_CATALOG_MARKER in withoutResume) && !(MODEL_ATTACHMENT_CATALOG_RENDERED_FIELD in withoutResume) && !(MODEL_ATTACHMENT_REFS_FIELD in withoutResume) && !(MODEL_TIMELINE_ANNOTATIONS_FIELD in withoutResume)) {
+  if (!(MODEL_ATTACHMENT_CATALOG_MARKER in withoutResume) && !(MODEL_ATTACHMENT_CATALOG_RENDERED_FIELD in withoutResume) && !(MODEL_ATTACHMENT_IMAGE_MAX_DIMENSION_FIELD in withoutResume) && !(MODEL_ATTACHMENT_REFS_FIELD in withoutResume) && !(MODEL_TIMELINE_ANNOTATIONS_FIELD in withoutResume)) {
     return withoutResume;
   }
   const clone = { ...withoutResume };
   delete clone[MODEL_ATTACHMENT_CATALOG_MARKER];
   delete clone[MODEL_ATTACHMENT_CATALOG_RENDERED_FIELD];
+  delete clone[MODEL_ATTACHMENT_IMAGE_MAX_DIMENSION_FIELD];
   delete clone[MODEL_ATTACHMENT_REFS_FIELD];
   delete clone[MODEL_TIMELINE_ANNOTATIONS_FIELD];
   return clone;
@@ -490,6 +492,7 @@ import {
   FileResourceRef,
   MODEL_ATTACHMENT_CATALOG_MARKER as MODEL_ATTACHMENT_CATALOG_MARKER2,
   MODEL_ATTACHMENT_CATALOG_RENDERED_FIELD as MODEL_ATTACHMENT_CATALOG_RENDERED_FIELD2,
+  MODEL_ATTACHMENT_IMAGE_MAX_DIMENSION_FIELD as MODEL_ATTACHMENT_IMAGE_MAX_DIMENSION_FIELD2,
   MODEL_ATTACHMENT_REFS_FIELD as MODEL_ATTACHMENT_REFS_FIELD2,
   collapseRepeatedSessionGoalContexts,
   expandUnchangedSessionGoalContexts,
@@ -1056,13 +1059,9 @@ function isCanonicalArrayIndex(property, length) {
   return Number.isSafeInteger(index) && index >= 0 && index < length && String(index) === property;
 }
 var IMAGE_LOW_DETAIL_TOKENS = 85;
-var IMAGE_HIGH_DETAIL_BASE_TOKENS = 85;
-var IMAGE_HIGH_DETAIL_TILE_TOKENS = 170;
-var IMAGE_HIGH_DETAIL_TILE_SIZE = 512;
-var IMAGE_HIGH_DETAIL_MAX_SIDE = 2048;
-var IMAGE_HIGH_DETAIL_TARGET_SHORT_SIDE = 768;
+var IMAGE_PATCH_SIZE = 28;
 var UNKNOWN_IMAGE_TOKENS = 4096;
-var MAX_NATIVE_IMAGE_TOKENS = 8192;
+var MAX_NATIVE_IMAGE_TOKENS = 36e3;
 function finitePositiveInteger(value) {
   return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.floor(value) : null;
 }
@@ -1168,25 +1167,10 @@ function dimensionsFromImageSource(source) {
   const rawDataDimensions = rawData instanceof Uint8Array ? dimensionsFromImageBytes(rawData) : typeof rawData === "string" && typeof mediaType === "string" && mediaType.toLowerCase().startsWith("image/") ? dimensionsFromImageBytes(base64BytesPrefix(rawData) ?? new Uint8Array()) : null;
   return dimensionsFromRecord(record3) ?? rawDataDimensions ?? dimensionsFromImageSource(record3.url) ?? dimensionsFromImageSource(rawData);
 }
-function highDetailImageTokens(width, height) {
-  let scaledWidth = width;
-  let scaledHeight = height;
-  const longest = Math.max(scaledWidth, scaledHeight);
-  if (longest > IMAGE_HIGH_DETAIL_MAX_SIDE) {
-    const scale = IMAGE_HIGH_DETAIL_MAX_SIDE / longest;
-    scaledWidth *= scale;
-    scaledHeight *= scale;
-  }
-  const shortest = Math.min(scaledWidth, scaledHeight);
-  if (shortest > IMAGE_HIGH_DETAIL_TARGET_SHORT_SIDE) {
-    const scale = IMAGE_HIGH_DETAIL_TARGET_SHORT_SIDE / shortest;
-    scaledWidth *= scale;
-    scaledHeight *= scale;
-  }
-  const tiles = Math.ceil(scaledWidth / IMAGE_HIGH_DETAIL_TILE_SIZE) * Math.ceil(scaledHeight / IMAGE_HIGH_DETAIL_TILE_SIZE);
+function patchImageTokens(width, height) {
   return Math.min(
     MAX_NATIVE_IMAGE_TOKENS,
-    IMAGE_HIGH_DETAIL_BASE_TOKENS + IMAGE_HIGH_DETAIL_TILE_TOKENS * Math.max(1, tiles)
+    Math.max(1, Math.ceil(width / IMAGE_PATCH_SIZE) * Math.ceil(height / IMAGE_PATCH_SIZE))
   );
 }
 function estimateNativeImageTokens(input) {
@@ -1209,7 +1193,7 @@ function estimateNativeImageTokens(input) {
     };
   }
   return {
-    tokens: highDetailImageTokens(width, height),
+    tokens: patchImageTokens(width, height),
     width,
     height,
     detail,
@@ -1792,13 +1776,37 @@ function prepareRetainedInputs(items, isCandidate) {
 function isPortableRetentionCandidate(item) {
   return isPortableInputMessage(item) && !isCompactionSummary(item) && !isAttachmentCatalog(item);
 }
+function hasAttachmentRefs(item) {
+  const raw = item[MODEL_ATTACHMENT_REFS_FIELD2];
+  return Array.isArray(raw) && raw.length > 0;
+}
+function withOnlyNewestAttachmentsInline(items) {
+  let newest = -1;
+  for (let index = items.length - 1; index >= 0; index -= 1) {
+    const item = items[index];
+    if (itemRole(item) === "user" && !isAttachmentCatalog(item) && hasAttachmentRefs(item)) {
+      newest = index;
+      break;
+    }
+  }
+  return items.map((item, index) => {
+    if (index === newest || isAttachmentCatalog(item) || !hasAttachmentRefs(item)) return item;
+    const stripped = { ...item };
+    delete stripped[MODEL_ATTACHMENT_REFS_FIELD2];
+    delete stripped[MODEL_ATTACHMENT_IMAGE_MAX_DIMENSION_FIELD2];
+    return stripped;
+  });
+}
 function buildCompactionReplacementHistory(items, summaryBody, retainedItemTokens = (item) => estimateTokens([item]), retainedMessageBudgetTokens = COMPACT_USER_MESSAGE_MAX_TOKENS, attachmentUploadTimes = /* @__PURE__ */ new Map()) {
   const retainedReversed = [];
   let remaining = Math.max(
     0,
     Math.min(COMPACT_USER_MESSAGE_MAX_TOKENS, retainedMessageBudgetTokens)
   );
-  const prepared = prepareRetainedInputs(items, isPortableRetentionCandidate);
+  const prepared = prepareRetainedInputs(
+    withOnlyNewestAttachmentsInline(items),
+    isPortableRetentionCandidate
+  );
   for (let index = prepared.charged.length - 1; index >= 0 && remaining > 0; index -= 1) {
     const item = prepared.charged[index];
     if (!isPortableRetentionCandidate(item) || prepared.skip(index)) {
@@ -1853,7 +1861,7 @@ function buildRemoteV2ReplacementHistory(items, compactionItem, retainedItemToke
   const retainedReversed = [];
   let remaining = REMOTE_V2_RETAINED_MESSAGE_TOKEN_BUDGET;
   const prepared = prepareRetainedInputs(
-    items,
+    withOnlyNewestAttachmentsInline(items),
     (item) => readSkillCatalogContext(item) === null && isRetainedRemoteV2Message(item)
   );
   for (let index = prepared.charged.length - 1; index >= 0 && remaining > 0; index -= 1) {
@@ -8940,6 +8948,100 @@ function captureClaudeRequestToken(input, init) {
   return headers.get("authorization")?.match(/^Bearer (.+)$/i)?.[1] ?? null;
 }
 
+// src/model-image-sizing.ts
+import { createHash as createHash3 } from "crypto";
+import sharp from "sharp";
+function createModelImageSizer(maxDimension, maxEncodedBytes = 10 * 1024 * 1024, options = {}) {
+  if (!Number.isSafeInteger(maxDimension) || maxDimension < 1) {
+    throw new RangeError("Model image dimension must be a positive integer.");
+  }
+  if (!Number.isSafeInteger(maxEncodedBytes) || maxEncodedBytes < 1) {
+    throw new RangeError("Model image byte limit must be a positive integer.");
+  }
+  const jpegQuality = options.jpegQuality;
+  if (jpegQuality !== void 0 && (!Number.isSafeInteger(jpegQuality) || jpegQuality < 1 || jpegQuality > 100)) {
+    throw new RangeError("Model image JPEG quality must be an integer from 1 to 100.");
+  }
+  const cache = /* @__PURE__ */ new Map();
+  let cacheBytes = 0;
+  const maxCacheBytes = 32 * 1024 * 1024;
+  return async (data, mediaType) => {
+    const key = createHash3("sha256").update(mediaType).update(data).digest("hex");
+    const prior = cache.get(key);
+    if (prior) return prior;
+    try {
+      if (data.length > Math.ceil(64 * 1024 * 1024 / 3) * 4) throw new Error();
+      const bytes = Buffer.from(data, "base64");
+      const signature = bytes.subarray(0, 12);
+      const matches = mediaType === "image/png" && signature.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])) || mediaType === "image/jpeg" && signature[0] === 255 && signature[1] === 216 && signature[2] === 255 || mediaType === "image/webp" && signature.toString("ascii", 0, 4) === "RIFF" && signature.toString("ascii", 8, 12) === "WEBP" || mediaType === "image/gif" && ["GIF87a", "GIF89a"].includes(signature.toString("ascii", 0, 6));
+      if (!matches) throw new Error();
+      const decoder = sharp(bytes, { limitInputPixels: 64e6, failOn: "error" });
+      const metadata = await decoder.metadata();
+      if (!metadata.width || !metadata.height) throw new Error();
+      let result = {
+        data,
+        mediaType,
+        originalWidth: metadata.width,
+        originalHeight: metadata.height,
+        width: metadata.width,
+        height: metadata.height
+      };
+      if (metadata.width > maxDimension || metadata.height > maxDimension || data.length > maxEncodedBytes) {
+        const oriented = (metadata.orientation ?? 1) >= 5;
+        const pipeline = decoder.autoOrient().resize({
+          width: maxDimension,
+          height: maxDimension,
+          fit: "inside",
+          withoutEnlargement: true
+        });
+        const encodedBudget = Math.min(maxEncodedBytes, data.length);
+        const asJpeg = jpegQuality !== void 0 && mediaType === "image/jpeg";
+        let resized = asJpeg ? await pipeline.clone().jpeg({ quality: jpegQuality }).toBuffer({ resolveWithObject: true }) : await pipeline.clone().png().toBuffer({ resolveWithObject: true });
+        let outputType = asJpeg ? "image/jpeg" : "image/png";
+        for (const quality of [90, 65, 45]) {
+          if (Math.ceil(resized.data.length / 3) * 4 <= encodedBudget) break;
+          resized = await pipeline.clone().webp({ quality }).toBuffer({ resolveWithObject: true });
+          outputType = "image/webp";
+        }
+        let dimension = Math.min(maxDimension, Math.max(metadata.width, metadata.height));
+        while (Math.ceil(resized.data.length / 3) * 4 > encodedBudget && dimension > 1) {
+          dimension = Math.max(1, Math.floor(dimension * 0.75));
+          resized = await decoder.clone().autoOrient().resize({
+            width: dimension,
+            height: dimension,
+            fit: "inside",
+            withoutEnlargement: true
+          }).webp({ quality: 75 }).toBuffer({ resolveWithObject: true });
+          outputType = "image/webp";
+        }
+        if (Math.ceil(resized.data.length / 3) * 4 > encodedBudget) throw new Error();
+        result = {
+          ...result,
+          data: resized.data.toString("base64"),
+          mediaType: outputType,
+          originalWidth: oriented ? metadata.height : metadata.width,
+          originalHeight: oriented ? metadata.width : metadata.height,
+          width: resized.info.width,
+          height: resized.info.height
+        };
+      }
+      const size = result.data.length;
+      if (size <= maxCacheBytes) {
+        while (cache.size && (cacheBytes + size > maxCacheBytes || cache.size >= 256)) {
+          const oldest = cache.keys().next().value;
+          cacheBytes -= cache.get(oldest).data.length;
+          cache.delete(oldest);
+        }
+        cache.set(key, result);
+        cacheBytes += size;
+      }
+      return result;
+    } catch {
+      throw new Error("Model image could not be decoded within the image sizing limit.");
+    }
+  };
+}
+
 // src/provider-history-adapter.ts
 import { collapseRepeatedSessionGoalContexts as collapseRepeatedSessionGoalContexts2 } from "@opengeni/contracts";
 
@@ -9360,7 +9462,7 @@ function splitStableSystemPromptPrefix(instructions) {
 import { createHash as createHash5, randomUUID as randomUUID2 } from "crypto";
 
 // src/claude-code-identity.ts
-import { createHash as createHash3, randomUUID } from "crypto";
+import { createHash as createHash4, randomUUID } from "crypto";
 var CLAUDE_CODE_HEADERS = {
   "user-agent": "claude-cli/2.1.285 (external, sdk-ts, agent-sdk/0.3.276)",
   "x-app": "cli",
@@ -9383,7 +9485,7 @@ function claudeCodeFingerprint(input) {
   const first = typeof content === "string" ? content : Array.isArray(content) ? content.find((block) => block.type === "input_text" || String(block.type) === "text") : void 0;
   const prompt = typeof first === "string" ? first : first && "text" in first ? String(first.text) : "";
   const sample = [4, 7, 20].map((index) => prompt[index] || "0").join("");
-  return createHash3("sha256").update(`59cf53e54c78${sample}2.1.285`).digest("hex").slice(0, 3);
+  return createHash4("sha256").update(`59cf53e54c78${sample}2.1.285`).digest("hex").slice(0, 3);
 }
 function applyClaudeCodeIdentity(body, headers, url, request, account, identity) {
   url.searchParams.set("beta", "true");
@@ -9423,97 +9525,6 @@ function applyClaudeCodeIdentity(body, headers, url, request, account, identity)
 
 // src/anthropic-messages.ts
 import { claudeNativeModelProfile } from "@opengeni/config";
-
-// src/model-image-sizing.ts
-import { createHash as createHash4 } from "crypto";
-import sharp from "sharp";
-function createModelImageSizer(maxDimension, maxEncodedBytes = 10 * 1024 * 1024) {
-  if (!Number.isSafeInteger(maxDimension) || maxDimension < 1) {
-    throw new RangeError("Model image dimension must be a positive integer.");
-  }
-  if (!Number.isSafeInteger(maxEncodedBytes) || maxEncodedBytes < 1) {
-    throw new RangeError("Model image byte limit must be a positive integer.");
-  }
-  const cache = /* @__PURE__ */ new Map();
-  let cacheBytes = 0;
-  const maxCacheBytes = 32 * 1024 * 1024;
-  return async (data, mediaType) => {
-    const key = createHash4("sha256").update(mediaType).update(data).digest("hex");
-    const prior = cache.get(key);
-    if (prior) return prior;
-    try {
-      if (data.length > Math.ceil(64 * 1024 * 1024 / 3) * 4) throw new Error();
-      const bytes = Buffer.from(data, "base64");
-      const signature = bytes.subarray(0, 12);
-      const matches = mediaType === "image/png" && signature.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])) || mediaType === "image/jpeg" && signature[0] === 255 && signature[1] === 216 && signature[2] === 255 || mediaType === "image/webp" && signature.toString("ascii", 0, 4) === "RIFF" && signature.toString("ascii", 8, 12) === "WEBP" || mediaType === "image/gif" && ["GIF87a", "GIF89a"].includes(signature.toString("ascii", 0, 6));
-      if (!matches) throw new Error();
-      const decoder = sharp(bytes, { limitInputPixels: 64e6, failOn: "error" });
-      const metadata = await decoder.metadata();
-      if (!metadata.width || !metadata.height) throw new Error();
-      let result = {
-        data,
-        mediaType,
-        originalWidth: metadata.width,
-        originalHeight: metadata.height,
-        width: metadata.width,
-        height: metadata.height
-      };
-      if (metadata.width > maxDimension || metadata.height > maxDimension || data.length > maxEncodedBytes) {
-        const oriented = (metadata.orientation ?? 1) >= 5;
-        const pipeline = decoder.autoOrient().resize({
-          width: maxDimension,
-          height: maxDimension,
-          fit: "inside",
-          withoutEnlargement: true
-        });
-        const encodedBudget = Math.min(maxEncodedBytes, data.length);
-        let resized = await pipeline.clone().png().toBuffer({ resolveWithObject: true });
-        let outputType = "image/png";
-        for (const quality of [90, 65, 45]) {
-          if (Math.ceil(resized.data.length / 3) * 4 <= encodedBudget) break;
-          resized = await pipeline.clone().webp({ quality }).toBuffer({ resolveWithObject: true });
-          outputType = "image/webp";
-        }
-        let dimension = Math.min(maxDimension, Math.max(metadata.width, metadata.height));
-        while (Math.ceil(resized.data.length / 3) * 4 > encodedBudget && dimension > 1) {
-          dimension = Math.max(1, Math.floor(dimension * 0.75));
-          resized = await decoder.clone().autoOrient().resize({
-            width: dimension,
-            height: dimension,
-            fit: "inside",
-            withoutEnlargement: true
-          }).webp({ quality: 75 }).toBuffer({ resolveWithObject: true });
-          outputType = "image/webp";
-        }
-        if (Math.ceil(resized.data.length / 3) * 4 > encodedBudget) throw new Error();
-        result = {
-          ...result,
-          data: resized.data.toString("base64"),
-          mediaType: outputType,
-          originalWidth: oriented ? metadata.height : metadata.width,
-          originalHeight: oriented ? metadata.width : metadata.height,
-          width: resized.info.width,
-          height: resized.info.height
-        };
-      }
-      const size = result.data.length;
-      if (size <= maxCacheBytes) {
-        while (cache.size && (cacheBytes + size > maxCacheBytes || cache.size >= 256)) {
-          const oldest = cache.keys().next().value;
-          cacheBytes -= cache.get(oldest).data.length;
-          cache.delete(oldest);
-        }
-        cache.set(key, result);
-        cacheBytes += size;
-      }
-      return result;
-    } catch {
-      throw new Error("Model image could not be decoded within the image sizing limit.");
-    }
-  };
-}
-
-// src/anthropic-messages.ts
 var ANTHROPIC_PAUSE_TURN_MAX_REQUESTS = 8;
 var AnthropicProtocolError = class extends Error {
   code = "anthropic_protocol_error";
@@ -13090,6 +13101,30 @@ var HARNESS_CONTROL_MODEL_TOOL_NAMES = new Set(
 function isHarnessControlMcpToolName(registryId, modelName) {
   return registryId === HARNESS_CONTROL_MCP_SERVER_ID && HARNESS_CONTROL_MODEL_TOOL_NAMES.has(modelName);
 }
+var INTERACTION_MCP_SERVER_ID = "interaction";
+var INTERACTION_TOOL_FAMILIES = [
+  ["browser_open", "browser_observe", "browser_act", "browser_screenshot", "browser_tabs"],
+  ["computer_open", "computer_observe", "computer_act"]
+];
+function interactionToolName(name) {
+  const prefix = `${INTERACTION_MCP_SERVER_ID}${MCP_TOOL_NAME_SEPARATOR}`;
+  return name.startsWith(prefix) ? name.slice(prefix.length) : null;
+}
+function familyCompanions(disclosed, pool) {
+  const disclosedShort = new Set(
+    disclosed.map((tool2) => interactionToolName(tool2.name)).filter((name) => name !== null)
+  );
+  const wanted = INTERACTION_TOOL_FAMILIES.filter(
+    (family) => family.some((member) => disclosedShort.has(member))
+  ).flat();
+  const companions = [];
+  for (const member of wanted) {
+    if (disclosedShort.has(member)) continue;
+    const tool2 = pool.find((candidate) => interactionToolName(candidate.name) === member);
+    if (tool2 && !companions.includes(tool2)) companions.push(tool2);
+  }
+  return companions;
+}
 function isHarnessControlMcpFunctionTool(tool2, mcpServerIds = NO_MCP_SERVER_IDS, modelServerIds) {
   const serverId = mcpServerIdForTool(tool2, mcpServerIds, modelServerIds);
   return serverId !== null && isHarnessControlMcpToolName(serverId, tool2.name);
@@ -13257,6 +13292,14 @@ function searchToolPool(availableTools, rawArguments) {
     bounded2.push(tool2);
     if (bounded2.length >= limit) break;
   }
+  for (const tool2 of familyCompanions(bounded2, searchable)) {
+    if (bounded2.includes(tool2)) continue;
+    const toolBytes = mcpSerializedSizeBytes(tool2);
+    if (toolBytes > MCP_MAX_TOOL_DEFINITION_BYTES) continue;
+    if (disclosedBytes + toolBytes > MCP_MAX_TOOL_SEARCH_DISCLOSURE_BYTES) continue;
+    disclosedBytes += toolBytes;
+    bounded2.push(tool2);
+  }
   return bounded2;
 }
 
@@ -16084,6 +16127,7 @@ export {
   OpenAIResponsesModel,
   OpenAIChatCompletionsModel,
   withClaudeUsageObserver,
+  createModelImageSizer,
   ProviderHistoryIncompatibleError,
   projectHistoryForProvider,
   recordStableSystemPromptPrefix,
@@ -16174,4 +16218,4 @@ export {
   normalizeModelCallUsage,
   modelCallUsageTelemetry
 };
-//# sourceMappingURL=chunk-EQQ3YOKR.js.map
+//# sourceMappingURL=chunk-7OUG7WBV.js.map