eve-hub 0.14.5 → 0.14.7

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # eve-hub
 
+## 0.14.7
+
+### Patch Changes
+
+- Bots see animated GIF motion as sampled frames.
+
+## 0.14.6
+
+### Patch Changes
+
+- New `fetch_upload` tool: a bot copies a file another bot shared as a `hub://upload/<id>` link into its sandbox (for example a patch to `git apply`), with the hub credential its kit tools already send. A shell can't fetch these links: the hosted hub's Deployment Protection sends it to a login page.
+
 ## 0.14.5
 
 ### Patch Changes

package/animated-images.mjs

@@ -0,0 +1,185 @@
+// A model sees only the first frame of an animated image: Anthropic, OpenAI
+// and the rest read a GIF as one still. So every model call a bot makes swaps
+// each inline animated image in its prompt (a message attachment, a read_file
+// or view_image result, an image read_thread returns) for a header and up to
+// MAX_FRAMES labeled stills sampled evenly across it, in time order.
+//
+// The swap happens on the Gateway request itself (installAnimatedFrames wraps
+// global fetch, as model-timeout.mjs does), not by wrapping the model: a model
+// object changes the id eve records for every step, while the request body is
+// the AI SDK prompt verbatim. A body with no animated image goes through
+// byte-identical.
+//
+// A decoder turns one media type into frames; video (mp4/mov) plugs in as a
+// second entry in DECODERS.
+import { createHash } from "node:crypto";
+import { crc32, deflateSync } from "node:zlib";
+import omggif from "omggif";
+
+export const MAX_FRAMES = 8;
+
+const INSTALLED = Symbol.for("hub.animatedFrames");
+const GATEWAY_HOST = "ai-gateway.vercel.sh";
+
+function pngChunk(type, data) {
+  const body = Buffer.concat([Buffer.from(type, "latin1"), data]);
+  const out = Buffer.alloc(body.length + 8);
+  out.writeUInt32BE(data.length, 0);
+  body.copy(out, 4);
+  out.writeUInt32BE(crc32(body), body.length + 4);
+  return out;
+}
+
+export function encodePng(width, height, rgba) {
+  const ihdr = Buffer.alloc(13);
+  ihdr.writeUInt32BE(width, 0);
+  ihdr.writeUInt32BE(height, 4);
+  ihdr.set([8, 6, 0, 0, 0], 8);
+  const stride = width * 4;
+  const raw = Buffer.alloc((stride + 1) * height);
+  for (let y = 0; y < height; y++) Buffer.from(rgba.buffer, rgba.byteOffset + y * stride, stride).copy(raw, y * (stride + 1) + 1);
+  return Buffer.concat([Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), pngChunk("IHDR", ihdr), pngChunk("IDAT", deflateSync(raw)), pngChunk("IEND", Buffer.alloc(0))]);
+}
+
+// `count` indices spread evenly over `total` frames, first and last included.
+export function sampleIndices(total, count = MAX_FRAMES) {
+  if (total <= count) return Array.from({ length: total }, (_, i) => i);
+  return Array.from({ length: count }, (_, i) => Math.round((i * (total - 1)) / (count - 1)));
+}
+
+// Browsers play a delay of 0 or 1 hundredths as 10.
+const delayMs = (centis) => (centis <= 1 ? 10 : centis) * 10;
+
+// Anthropic rejects a request with more than 20 images once any is over
+// 2000 px on a side, and the decode below runs inside fetch, synchronously:
+// a GIF bigger than either bound goes to the provider as it is.
+export const MAX_SIDE = 2000;
+export const MAX_DECODED_PIXELS = 32_000_000;
+
+// Up to `max` composited frames of an animated GIF, or null for a still or
+// one over the bounds.
+function sampleGif(bytes, max) {
+  const gif = new omggif.GifReader(bytes);
+  const total = gif.numFrames();
+  const { width, height } = gif;
+  if (total < 2 || width > MAX_SIDE || height > MAX_SIDE || width * height * total > MAX_DECODED_PIXELS) return null;
+  const wanted = new Set(sampleIndices(total, max));
+  const canvas = new Uint8Array(width * height * 4);
+  const frames = [];
+  let timeMs = 0;
+  for (let i = 0; i < total; i++) {
+    const info = gif.frameInfo(i);
+    const previous = info.disposal === 3 ? canvas.slice() : null;
+    gif.decodeAndBlitFrameRGBA(i, canvas);
+    if (wanted.has(i)) frames.push({ index: i, timeMs, png: encodePng(width, height, canvas) });
+    timeMs += delayMs(info.delay);
+    if (info.disposal === 2) {
+      for (let y = info.y; y < info.y + info.height; y++) canvas.fill(0, (y * width + info.x) * 4, (y * width + info.x + info.width) * 4);
+    } else if (previous) canvas.set(previous);
+  }
+  return { frameCount: total, durationMs: timeMs, frames };
+}
+
+// media type -> (bytes, max frames) => { frameCount, durationMs, frames: [{ index, timeMs, png }] } | null
+export const DECODERS = { "image/gif": sampleGif };
+
+const seconds = (ms) => `${(ms / 1000).toFixed(2)}s`;
+
+// An image stays in a session's history, so every later model call carries it
+// again; decoding a 1 MB GIF takes about 400 ms.
+const recent = new Map();
+const RECENT_MAX = 16;
+
+function expandImage(mediaType, base64) {
+  if (!DECODERS[mediaType]) return null;
+  const key = createHash("sha256").update(mediaType).update("\0").update(base64).digest("hex");
+  if (recent.has(key)) return recent.get(key);
+  const parts = decodeImage(mediaType, base64);
+  recent.set(key, parts);
+  if (recent.size > RECENT_MAX) recent.delete(recent.keys().next().value);
+  return parts;
+}
+
+// The parts that replace one animated image: a header, then a label and a
+// PNG per sampled frame. null when the image is a still or can't be decoded;
+// then the provider gets the original and reads it as it always has.
+function decodeImage(mediaType, base64) {
+  let sampled;
+  try {
+    sampled = DECODERS[mediaType](Buffer.from(base64, "base64"), MAX_FRAMES);
+  } catch {
+    return null;
+  }
+  if (!sampled) return null;
+  const { frameCount, durationMs, frames } = sampled;
+  const header = `Animated image (${mediaType}), ${frameCount} frames over ${seconds(durationMs)}. The ${frames.length} images below are frames sampled from this one animation, in time order; motion between the sampled frames is not shown.`;
+  return [
+    { type: "text", text: header },
+    ...frames.flatMap(({ index, timeMs, png }) => [
+      { type: "text", text: `Frame ${index + 1} of ${frameCount}, at ${seconds(timeMs)}` },
+      { type: "file", mediaType: "image/png", data: { type: "data", data: png.toString("base64") } },
+    ]),
+  ];
+}
+
+// `prompt` (AI SDK language model v4 messages) with each animated image in
+// a user message or a tool result expanded, and how many were.
+export function expandAnimatedImages(prompt) {
+  let expanded = 0;
+  const expandParts = (parts) =>
+    parts.flatMap((part) => {
+      const data = part?.type === "file" && part.data?.type === "data" && typeof part.data.data === "string" ? part.data.data : null;
+      const replacement = data === null ? null : expandImage(part.mediaType, data);
+      if (!replacement) return [part];
+      expanded++;
+      return replacement;
+    });
+  const out = prompt.map((message) => {
+    if (!Array.isArray(message?.content)) return message;
+    if (message.role === "user") return { ...message, content: expandParts(message.content) };
+    if (message.role !== "tool") return message;
+    return {
+      ...message,
+      content: message.content.map((part) =>
+        part?.type === "tool-result" && part.output?.type === "content" && Array.isArray(part.output.value)
+          ? { ...part, output: { ...part.output, value: expandParts(part.output.value) } }
+          : part,
+      ),
+    };
+  });
+  return { prompt: out, expanded };
+}
+
+// A Gateway language-model request body with its animated images expanded,
+// or the same string when it has none.
+export function expandRequestBody(body) {
+  if (!Object.keys(DECODERS).some((type) => body.includes(`"${type}"`))) return body;
+  const parsed = JSON.parse(body);
+  if (!Array.isArray(parsed.prompt)) return body;
+  const { prompt, expanded } = expandAnimatedImages(parsed.prompt);
+  return expanded === 0 ? body : JSON.stringify({ ...parsed, prompt });
+}
+
+function isLanguageModelCall(input) {
+  try {
+    const url = new URL(typeof input === "string" || input instanceof URL ? input : input.url);
+    return url.hostname === GATEWAY_HOST && url.pathname.endsWith("/language-model");
+  } catch {
+    return false;
+  }
+}
+
+export function withAnimatedFrames(baseFetch) {
+  return function fetchWithAnimatedFrames(input, init) {
+    if (typeof init?.body !== "string" || !isLanguageModelCall(input)) return baseFetch(input, init);
+    const body = expandRequestBody(init.body);
+    return baseFetch(input, body === init.body ? init : { ...init, body });
+  };
+}
+
+export function installAnimatedFrames() {
+  if (globalThis.fetch[INSTALLED]) return;
+  const wrapped = withAnimatedFrames(globalThis.fetch.bind(globalThis));
+  wrapped[INSTALLED] = true;
+  globalThis.fetch = wrapped;
+}

package/kit-modules.json

@@ -23,6 +23,7 @@
   "tools/create_thread",
   "tools/dash_update",
   "tools/disabled",
+  "tools/fetch_upload",
   "tools/list_agents",
   "tools/list_threads",
   "tools/move_path",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "eve-hub",
-  "version": "0.14.5",
+  "version": "0.14.7",
   "license": "Apache-2.0",
   "description": "Bot-side code shared by every hub agent: registry tools, the roster hook, the eve channel, the host sandbox and browser, and model helpers.",
   "type": "module",
@@ -32,15 +32,16 @@
     "./vercel": "./vercel.mjs"
   },
   "peerDependencies": {
-    "eve": "*",
-    "zod": "4.5.4",
     "@vercel/connect": "^2.4.1",
-    "@vercel/oidc": "4.0.0"
+    "@vercel/oidc": "4.0.0",
+    "eve": "*",
+    "zod": "4.5.4"
   },
   "dependencies": {
     "ai": "^7.0.137",
     "jose": "^6.2.3",
     "jsonc-parser": "3.3.1",
+    "omggif": "1.0.10",
     "smol-toml": "^1.9.1",
     "tree-sitter-bash": "0.25.1",
     "web-tree-sitter": "0.27.1"

package/roster-model.mjs

@@ -18,8 +18,10 @@ import { CONTEXT_WINDOW, selectModel } from "./bot-model.mjs";
 import { DEFAULT_SUBAGENT_CAP } from "./subagent-cap.mjs";
 import { sessionLimits } from "./session-limits.mjs";
 import { installModelCallTimeout } from "./model-timeout.mjs";
+import { installAnimatedFrames } from "./animated-images.mjs";
 import { findWorkspaceRoot, loadWorkspace, parseWorkspace } from "./workspace.mjs";
 const real = process.env.ROSTER_MODELS !== "mock";
+installAnimatedFrames();
 installModelCallTimeout();
 
 export function createRosterModel({ agents, subagentCap }) {

package/stubs.mjs

@@ -10,6 +10,7 @@ export const KIT_FILES = {
   "tools/react.ts": ["React to a message with an emoji."],
   "tools/show.ts": ["Move the human's screen to a thread, DM, or message, or open a page or file beside the thread."],
   "tools/browser.ts": ["Its own headless browser, watched live in the Computer view."],
+  "tools/fetch_upload.ts": ["Copies a file another bot shared (hub://upload/<id>) into the sandbox."],
   "tools/view_image.ts": ["Shows the bot an image (a sandbox file or a browser screenshot) so it can see it."],
   "tools/update_thread.ts": ["Update a thread this bot is in: title, track, domain and summary."],
   "tools/list_threads.ts": ["Every workspace thread, from any session."],
@@ -57,7 +58,7 @@ export const DEPLOYED_KIT_FILES = new Set([
   "schedules/dash.ts",
   "skills/onboarding-profile.ts",
   "tools/onboarding_scripts.ts",
-  ...["react", "show", "update_thread", "list_threads", "read_thread", "post_message"].map((t) => `tools/${t}.ts`),
+  ...["react", "show", "fetch_upload", "update_thread", "list_threads", "read_thread", "post_message"].map((t) => `tools/${t}.ts`),
   "memory/team.ts",
   "skills/forms.ts",
   ...["create_thread", "set_summary", "move_path", "archive_thread", "unarchive_thread", "list_agents", "create_agent", "remove_agent", "restart_agent", "tag_threads", "dash_update", "apply_tags"].map((t) => `tools/${t}.ts`),

package/tools/fetch_upload.d.ts

@@ -0,0 +1,18 @@
+declare const _default: import("eve/tools").ToolDefinition<{
+    uri: string;
+    path?: string | undefined;
+}, {
+    path: string;
+    size: number;
+    mediaType: string | null;
+}> & {
+    execute(input: {
+        uri: string;
+        path?: string | undefined;
+    }, ctx: import("eve/tools").ToolContext): Promise<{
+        path: string;
+        size: number;
+        mediaType: string | null;
+    }>;
+};
+export default _default;

package/tools/fetch_upload.js

@@ -0,0 +1,40 @@
+// The `fetch_upload` tool: copies a file another bot shared (`hub://upload/<id>`, what
+// `show(path)` returns on a hosted hub) into this bot's sandbox, e.g. a patch to `git apply`.
+// A shell can't fetch it: the hosted hub's Deployment Protection and its auth take only the bot
+// credential that registryFetch sends (registry-fetch.mjs).
+import { defineTool } from "eve/tools";
+import { z } from "zod";
+import { registryFetch } from "../registry-fetch.mjs";
+const UPLOAD = /^hub:\/\/upload\/([0-9a-f]{64}\.[a-z0-9]{1,10})$/;
+export default defineTool({
+    description: "Fetch a file another bot shared as a hub://upload/<id> link (e.g. a patch, a log, a brief) into your sandbox, then read or apply it there. Your shell can't fetch these links; this tool sends your hub credential. Returns the path it wrote, the size in bytes and the media type.",
+    inputSchema: z.object({
+        uri: z.string().describe("The hub://upload/<id> link."),
+        path: z.string().optional().describe("Where to write it in your sandbox. Default /tmp/uploads/<id>."),
+    }),
+    async execute({ uri, path }, ctx) {
+        const id = UPLOAD.exec(uri.trim())?.[1];
+        if (!id)
+            throw new Error(`not a hub://upload/<id> link: ${uri}`);
+        // A redirect is Deployment Protection sending the request to a login page: an error, not the file.
+        const res = await registryFetch(`/api/uploads/${id}`, { redirect: "manual", signal: AbortSignal.timeout(30_000) });
+        if (!res.ok) {
+            const text = await res.text();
+            // Deployment Protection answers with a whole HTML page: name the redirect, or keep the body short.
+            const error = (() => { try {
+                return JSON.parse(text).error;
+            }
+            catch {
+                return undefined;
+            } })();
+            const location = res.headers.get("location");
+            const why = error ?? (location ? `redirected to ${location}` : text.slice(0, 200));
+            throw new Error(`fetch_upload failed: ${res.status} ${why}`.trim());
+        }
+        const bytes = new Uint8Array(await res.arrayBuffer());
+        const sandbox = await ctx.getSandbox();
+        const target = path ?? `/tmp/uploads/${id}`;
+        await sandbox.writeBinaryFile({ path: target, content: bytes });
+        return { path: sandbox.resolvePath(target), size: bytes.length, mediaType: res.headers.get("content-type") };
+    },
+});