eve-hub 0.14.6 → 0.14.8

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # eve-hub
 
+## 0.14.8
+
+### Patch Changes
+
+- `packKit` (the tarball the hub's tests and hosted workspaces' staging use) compiles the kit inside `packages/agent-kit/dist`, where a publish compiles it, instead of a temp dir. The compiler now resolves eve and the kit's other dependencies there too, so a declaration only a publish rejected (TS2883 in 0.14.5) fails the tests before merge.
+
+## 0.14.7
+
+### Patch Changes
+
+- Bots see animated GIF motion as sampled frames.
+
 ## 0.14.6
 
 ### Patch Changes

package/animated-images.mjs

@@ -0,0 +1,185 @@
+// A model sees only the first frame of an animated image: Anthropic, OpenAI
+// and the rest read a GIF as one still. So every model call a bot makes swaps
+// each inline animated image in its prompt (a message attachment, a read_file
+// or view_image result, an image read_thread returns) for a header and up to
+// MAX_FRAMES labeled stills sampled evenly across it, in time order.
+//
+// The swap happens on the Gateway request itself (installAnimatedFrames wraps
+// global fetch, as model-timeout.mjs does), not by wrapping the model: a model
+// object changes the id eve records for every step, while the request body is
+// the AI SDK prompt verbatim. A body with no animated image goes through
+// byte-identical.
+//
+// A decoder turns one media type into frames; video (mp4/mov) plugs in as a
+// second entry in DECODERS.
+import { createHash } from "node:crypto";
+import { crc32, deflateSync } from "node:zlib";
+import omggif from "omggif";
+
+export const MAX_FRAMES = 8;
+
+const INSTALLED = Symbol.for("hub.animatedFrames");
+const GATEWAY_HOST = "ai-gateway.vercel.sh";
+
+function pngChunk(type, data) {
+  const body = Buffer.concat([Buffer.from(type, "latin1"), data]);
+  const out = Buffer.alloc(body.length + 8);
+  out.writeUInt32BE(data.length, 0);
+  body.copy(out, 4);
+  out.writeUInt32BE(crc32(body), body.length + 4);
+  return out;
+}
+
+export function encodePng(width, height, rgba) {
+  const ihdr = Buffer.alloc(13);
+  ihdr.writeUInt32BE(width, 0);
+  ihdr.writeUInt32BE(height, 4);
+  ihdr.set([8, 6, 0, 0, 0], 8);
+  const stride = width * 4;
+  const raw = Buffer.alloc((stride + 1) * height);
+  for (let y = 0; y < height; y++) Buffer.from(rgba.buffer, rgba.byteOffset + y * stride, stride).copy(raw, y * (stride + 1) + 1);
+  return Buffer.concat([Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), pngChunk("IHDR", ihdr), pngChunk("IDAT", deflateSync(raw)), pngChunk("IEND", Buffer.alloc(0))]);
+}
+
+// `count` indices spread evenly over `total` frames, first and last included.
+export function sampleIndices(total, count = MAX_FRAMES) {
+  if (total <= count) return Array.from({ length: total }, (_, i) => i);
+  return Array.from({ length: count }, (_, i) => Math.round((i * (total - 1)) / (count - 1)));
+}
+
+// Browsers play a delay of 0 or 1 hundredths as 10.
+const delayMs = (centis) => (centis <= 1 ? 10 : centis) * 10;
+
+// Anthropic rejects a request with more than 20 images once any is over
+// 2000 px on a side, and the decode below runs inside fetch, synchronously:
+// a GIF bigger than either bound goes to the provider as it is.
+export const MAX_SIDE = 2000;
+export const MAX_DECODED_PIXELS = 32_000_000;
+
+// Up to `max` composited frames of an animated GIF, or null for a still or
+// one over the bounds.
+function sampleGif(bytes, max) {
+  const gif = new omggif.GifReader(bytes);
+  const total = gif.numFrames();
+  const { width, height } = gif;
+  if (total < 2 || width > MAX_SIDE || height > MAX_SIDE || width * height * total > MAX_DECODED_PIXELS) return null;
+  const wanted = new Set(sampleIndices(total, max));
+  const canvas = new Uint8Array(width * height * 4);
+  const frames = [];
+  let timeMs = 0;
+  for (let i = 0; i < total; i++) {
+    const info = gif.frameInfo(i);
+    const previous = info.disposal === 3 ? canvas.slice() : null;
+    gif.decodeAndBlitFrameRGBA(i, canvas);
+    if (wanted.has(i)) frames.push({ index: i, timeMs, png: encodePng(width, height, canvas) });
+    timeMs += delayMs(info.delay);
+    if (info.disposal === 2) {
+      for (let y = info.y; y < info.y + info.height; y++) canvas.fill(0, (y * width + info.x) * 4, (y * width + info.x + info.width) * 4);
+    } else if (previous) canvas.set(previous);
+  }
+  return { frameCount: total, durationMs: timeMs, frames };
+}
+
+// media type -> (bytes, max frames) => { frameCount, durationMs, frames: [{ index, timeMs, png }] } | null
+export const DECODERS = { "image/gif": sampleGif };
+
+const seconds = (ms) => `${(ms / 1000).toFixed(2)}s`;
+
+// An image stays in a session's history, so every later model call carries it
+// again; decoding a 1 MB GIF takes about 400 ms.
+const recent = new Map();
+const RECENT_MAX = 16;
+
+function expandImage(mediaType, base64) {
+  if (!DECODERS[mediaType]) return null;
+  const key = createHash("sha256").update(mediaType).update("\0").update(base64).digest("hex");
+  if (recent.has(key)) return recent.get(key);
+  const parts = decodeImage(mediaType, base64);
+  recent.set(key, parts);
+  if (recent.size > RECENT_MAX) recent.delete(recent.keys().next().value);
+  return parts;
+}
+
+// The parts that replace one animated image: a header, then a label and a
+// PNG per sampled frame. null when the image is a still or can't be decoded;
+// then the provider gets the original and reads it as it always has.
+function decodeImage(mediaType, base64) {
+  let sampled;
+  try {
+    sampled = DECODERS[mediaType](Buffer.from(base64, "base64"), MAX_FRAMES);
+  } catch {
+    return null;
+  }
+  if (!sampled) return null;
+  const { frameCount, durationMs, frames } = sampled;
+  const header = `Animated image (${mediaType}), ${frameCount} frames over ${seconds(durationMs)}. The ${frames.length} images below are frames sampled from this one animation, in time order; motion between the sampled frames is not shown.`;
+  return [
+    { type: "text", text: header },
+    ...frames.flatMap(({ index, timeMs, png }) => [
+      { type: "text", text: `Frame ${index + 1} of ${frameCount}, at ${seconds(timeMs)}` },
+      { type: "file", mediaType: "image/png", data: { type: "data", data: png.toString("base64") } },
+    ]),
+  ];
+}
+
+// `prompt` (AI SDK language model v4 messages) with each animated image in
+// a user message or a tool result expanded, and how many were.
+export function expandAnimatedImages(prompt) {
+  let expanded = 0;
+  const expandParts = (parts) =>
+    parts.flatMap((part) => {
+      const data = part?.type === "file" && part.data?.type === "data" && typeof part.data.data === "string" ? part.data.data : null;
+      const replacement = data === null ? null : expandImage(part.mediaType, data);
+      if (!replacement) return [part];
+      expanded++;
+      return replacement;
+    });
+  const out = prompt.map((message) => {
+    if (!Array.isArray(message?.content)) return message;
+    if (message.role === "user") return { ...message, content: expandParts(message.content) };
+    if (message.role !== "tool") return message;
+    return {
+      ...message,
+      content: message.content.map((part) =>
+        part?.type === "tool-result" && part.output?.type === "content" && Array.isArray(part.output.value)
+          ? { ...part, output: { ...part.output, value: expandParts(part.output.value) } }
+          : part,
+      ),
+    };
+  });
+  return { prompt: out, expanded };
+}
+
+// A Gateway language-model request body with its animated images expanded,
+// or the same string when it has none.
+export function expandRequestBody(body) {
+  if (!Object.keys(DECODERS).some((type) => body.includes(`"${type}"`))) return body;
+  const parsed = JSON.parse(body);
+  if (!Array.isArray(parsed.prompt)) return body;
+  const { prompt, expanded } = expandAnimatedImages(parsed.prompt);
+  return expanded === 0 ? body : JSON.stringify({ ...parsed, prompt });
+}
+
+function isLanguageModelCall(input) {
+  try {
+    const url = new URL(typeof input === "string" || input instanceof URL ? input : input.url);
+    return url.hostname === GATEWAY_HOST && url.pathname.endsWith("/language-model");
+  } catch {
+    return false;
+  }
+}
+
+export function withAnimatedFrames(baseFetch) {
+  return function fetchWithAnimatedFrames(input, init) {
+    if (typeof init?.body !== "string" || !isLanguageModelCall(input)) return baseFetch(input, init);
+    const body = expandRequestBody(init.body);
+    return baseFetch(input, body === init.body ? init : { ...init, body });
+  };
+}
+
+export function installAnimatedFrames() {
+  if (globalThis.fetch[INSTALLED]) return;
+  const wrapped = withAnimatedFrames(globalThis.fetch.bind(globalThis));
+  wrapped[INSTALLED] = true;
+  globalThis.fetch = wrapped;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "eve-hub",
-  "version": "0.14.6",
+  "version": "0.14.8",
   "license": "Apache-2.0",
   "description": "Bot-side code shared by every hub agent: registry tools, the roster hook, the eve channel, the host sandbox and browser, and model helpers.",
   "type": "module",
@@ -32,15 +32,16 @@
     "./vercel": "./vercel.mjs"
   },
   "peerDependencies": {
-    "eve": "*",
-    "zod": "4.5.4",
     "@vercel/connect": "^2.4.1",
-    "@vercel/oidc": "4.0.0"
+    "@vercel/oidc": "4.0.0",
+    "eve": "*",
+    "zod": "4.5.4"
   },
   "dependencies": {
     "ai": "^7.0.137",
     "jose": "^6.2.3",
     "jsonc-parser": "3.3.1",
+    "omggif": "1.0.10",
     "smol-toml": "^1.9.1",
     "tree-sitter-bash": "0.25.1",
     "web-tree-sitter": "0.27.1"

package/roster-model.mjs

@@ -18,8 +18,10 @@ import { CONTEXT_WINDOW, selectModel } from "./bot-model.mjs";
 import { DEFAULT_SUBAGENT_CAP } from "./subagent-cap.mjs";
 import { sessionLimits } from "./session-limits.mjs";
 import { installModelCallTimeout } from "./model-timeout.mjs";
+import { installAnimatedFrames } from "./animated-images.mjs";
 import { findWorkspaceRoot, loadWorkspace, parseWorkspace } from "./workspace.mjs";
 const real = process.env.ROSTER_MODELS !== "mock";
+installAnimatedFrames();
 installModelCallTimeout();
 
 export function createRosterModel({ agents, subagentCap }) {