@bivy/bivy 0.4.0-staging.54 → 0.4.0-staging.55

package/dist/harness/tool-image-attachments.js

@@ -0,0 +1,83 @@
+// SPDX-License-Identifier: FSL-1.1-ALv2
+// Copyright (c) 2026 Petter André Sjulstad
+//
+// Config gate + noise guard for PASSIVELY surfacing tool-produced images (e.g. a
+// screenshot MCP tool's output riding home on a tool_result) into the chat as
+// attachments — see issue #292. Today a runtime adapter that notices an image
+// inside a tool_result (e.g. src/runtime/claude-code.ts's toolResultText, which
+// deliberately keeps only the text parts) can route the image through here to
+// decide whether it's allowed at all, and if so, whether this turn's budget has
+// room for it.
+//
+// Kept independent of both src/runtime and src/session/server so nothing new
+// crosses that layering: a runtime adapter only needs the pure functions below;
+// src/server.ts wires the config setter to settings.json/env — mirroring
+// src/harness/sandbox.ts's live-config pattern — and owns the actual
+// store+persist+broadcast (see handlePassiveToolImage in src/server.ts), the
+// same way it already does for an explicit `bivy attach`.
+// ---- Opt-in gate ---------------------------------------------------------
+//
+// Off by default: an image silently appearing in the chat with no explicit
+// attach call is new, surprising behavior, and a badly-behaved tool could turn
+// it into transcript noise (the per-turn budget below bounds that too, but the
+// gate is the first line of defense — no existing installation should see any
+// change unless it opts in).
+let configuredEnabled;
+/** Set from settings.json at boot and whenever node settings change (mirrors
+ *  setConfiguredSandboxTier in src/harness/sandbox.ts). */
+export function setConfiguredAutoAttachToolImages(value) {
+    configuredEnabled = value === true;
+}
+/** Whether a tool_result image should be captured as a passive attachment.
+ *  Precedence: `BIVY_AUTO_ATTACH_TOOL_IMAGES` env (plain truthiness — any
+ *  non-empty value enables, matching BIVY_MCP_PROXY's convention) > the node's
+ *  persisted `settings.json` setting > off. */
+export function autoAttachToolImagesEnabled() {
+    if (process.env.BIVY_AUTO_ATTACH_TOOL_IMAGES)
+        return true;
+    return configuredEnabled === true;
+}
+// ---- Per-turn noise guard -------------------------------------------------
+//
+// A chatty tool (one that screenshots after every step, say) must never be
+// able to flood the transcript just because the feature is on. Both caps are
+// deliberately small relative to the explicit-attach ceiling
+// (MAX_AGENT_ATTACHMENT_BYTES in src/session/attach-to-chat.ts) — passive
+// images are unreviewed by a human before they land in the chat.
+/** Max images passively surfaced from a single turn's tool results. */
+export const MAX_PASSIVE_IMAGES_PER_TURN = 4;
+/** Max total decoded bytes passively surfaced from a single turn. */
+export const MAX_PASSIVE_IMAGE_BYTES_PER_TURN = 12 * 1024 * 1024;
+/**
+ * Tracks how much of one turn's passive-image budget has been spent so a
+ * runtime can decide, per image, whether to surface it or drop it. Reset at
+ * the start of every turn (a fresh instance is the simplest reset).
+ */
+export class PassiveImageBudget {
+    count = 0;
+    bytes = 0;
+    droppedCount = 0;
+    droppedBytes = 0;
+    /** Reserves the budget and returns true if `byteLength` fits under both caps;
+     *  otherwise records the drop (for droppedSummary) and returns false. */
+    admit(byteLength) {
+        if (this.count >= MAX_PASSIVE_IMAGES_PER_TURN || this.bytes + byteLength > MAX_PASSIVE_IMAGE_BYTES_PER_TURN) {
+            this.droppedCount += 1;
+            this.droppedBytes += byteLength;
+            return false;
+        }
+        this.count += 1;
+        this.bytes += byteLength;
+        return true;
+    }
+    /** True once anything has been dropped this turn (for logging at the call site). */
+    get hasDropped() {
+        return this.droppedCount > 0;
+    }
+    /** Human-readable summary of what this turn dropped, or "" if nothing was. */
+    droppedSummary() {
+        if (!this.droppedCount)
+            return "";
+        return `dropped ${this.droppedCount} tool-produced image(s) totaling ${this.droppedBytes} bytes (per-turn cap: ${MAX_PASSIVE_IMAGES_PER_TURN} images / ${MAX_PASSIVE_IMAGE_BYTES_PER_TURN} bytes)`;
+    }
+}

package/dist/runtime/claude-code.js

@@ -23,6 +23,7 @@ import { depCacheEnv } from "../harness/dep-cache.js";
 import os from "node:os";
 import path from "node:path";
 import { sandboxTier, claudePermissionModeFor } from "../harness/sandbox.js";
+import { autoAttachToolImagesEnabled, PassiveImageBudget } from "../harness/tool-image-attachments.js";
 import { anthropicCredentialPreflight, describeAnthropicError, isAnthropicAuthError } from "./anthropic-preflight.js";
 import { toModelInfo as sharedToModelInfo } from "./normalize.js";
 import { hasLiveProcessForCwd } from "./native-process-scan.js";
@@ -239,6 +240,28 @@ function toolResultText(block) {
         .map((part) => part.text)
         .join("");
 }
+/** Sibling of toolResultText that keeps what that one discards: the `image`
+ *  content parts of a tool_result (e.g. a Playwright/screenshot MCP tool's
+ *  output), so they can be passively surfaced as chat attachments (issue #292)
+ *  instead of silently vanishing. Only base64-sourced images are collected — a
+ *  `url`-sourced image block (rare for a local tool) is skipped rather than
+ *  fetched, since this passive path must never make its own network call. */
+function toolResultImages(block) {
+    const content = block?.content;
+    if (!Array.isArray(content))
+        return [];
+    const out = [];
+    for (const part of content) {
+        if (part?.type !== "image")
+            continue;
+        const source = part.source;
+        if (!source || source.type !== "base64" || typeof source.data !== "string" || !source.data)
+            continue;
+        const mimeType = typeof source.media_type === "string" && source.media_type ? source.media_type : "image/png";
+        out.push({ mimeType, data: source.data });
+    }
+    return out;
+}
 /** Sums per-model token usage (SDK's ModelUsage) into a single totals object. */
 export function sumModelUsage(modelUsage) {
     let input = 0, output = 0, cacheRead = 0, cacheWrite = 0;
@@ -579,6 +602,13 @@ class ClaudeSession {
      *  "interrupted" notice. One-shot; cleared when consumed, on the next result,
      *  and on a real abort() (a user Stop must never be silenced by a stale flag). */
     suppressNextInterrupt = false;
+    /** tool_use id → tool name, learned as "assistant" turns emit tool_use blocks.
+     *  Used only to label a passively-surfaced tool_image (see #292); never
+     *  cleared mid-session since a tool_use_id is unique for the session's life. */
+    toolNamesByUseId = new Map();
+    /** This turn's passive-image noise guard (see PassiveImageBudget); replaced
+     *  with a fresh budget at the start of every prompt(). */
+    passiveImageBudget = new PassiveImageBudget();
     /** The agent's own slash commands for this session, learned from the SDK's
      *  system/init message (slash_commands + skills). Empty until the first turn's
      *  init arrives; getCommands() exposes them and a `runtime.commands` event lets
@@ -846,6 +876,31 @@ class ClaudeSession {
         this.startedMessage = true;
         this.emit({ type: "message_start", message: { role: "assistant", content: "" } });
     }
+    /**
+     * Passively surface any images riding home on a tool_result (issue #292) —
+     * e.g. a Playwright/screenshot MCP tool's output, which toolResultText above
+     * deliberately drops. Gated by autoAttachToolImagesEnabled() at the call site
+     * and bounded here by this turn's PassiveImageBudget so a chatty tool can't
+     * flood the transcript; a drop is logged (with the responsible tool's name)
+     * rather than silently discarded. Emits one `tool_image` RuntimeEvent per
+     * admitted image; src/server.ts's session listener does the actual
+     * store+persist+broadcast, the same way an explicit `bivy attach` does.
+     */
+    emitPassiveToolImages(block) {
+        const images = toolResultImages(block);
+        if (!images.length)
+            return;
+        const toolUseId = String(block.tool_use_id ?? "");
+        const toolName = this.toolNamesByUseId.get(toolUseId) ?? "tool";
+        for (const image of images) {
+            const byteLength = Buffer.byteLength(image.data, "base64");
+            if (!this.passiveImageBudget.admit(byteLength)) {
+                console.warn(`[claude-code] dropped a passively-surfaced tool image from "${toolName}" (tool_use_id=${toolUseId}, ~${byteLength} bytes): ${this.passiveImageBudget.droppedSummary()}`);
+                continue;
+            }
+            this.emit({ type: "tool_image", toolUseId, toolName, mimeType: image.mimeType, data: image.data });
+        }
+    }
     handle(message) {
         switch (message?.type) {
             case "stream_event": {
@@ -880,6 +935,8 @@ class ClaudeSession {
                 for (const block of content) {
                     if (block?.type === "tool_use") {
                         this.emit({ type: "tool_call", toolName: block.name, input: block.input, toolUseId: block.id });
+                        if (typeof block.id === "string" && typeof block.name === "string")
+                            this.toolNamesByUseId.set(block.id, block.name);
                     }
                 }
                 const text = extractText(message.message);
@@ -921,6 +978,8 @@ class ClaudeSession {
                     for (const block of toolResults) {
                         this.runningTools.delete(String(block.tool_use_id));
                         this.emit({ type: "tool_result", toolUseId: block.tool_use_id, result: toolResultText(block), isError: Boolean(block.is_error) });
+                        if (autoAttachToolImagesEnabled())
+                            this.emitPassiveToolImages(block);
                     }
                     this.emit({ type: "user", raw: message });
                     break;
@@ -1020,6 +1079,9 @@ class ClaudeSession {
         const hasImages = Boolean(options?.images?.length);
         if (!prompt && !hasImages)
             return;
+        // Fresh per-turn noise-guard budget (see PassiveImageBudget) — a prior
+        // turn's usage must never carry over and eat into this one's allowance.
+        this.passiveImageBudget = new PassiveImageBudget();
         // Credential preflight (first turn only): if no Anthropic credential will
         // reach the SDK, surface an actionable message instead of letting it spawn
         // and fail its first request with an opaque `401 Unauthorized`.

package/dist/server.js

@@ -54,6 +54,7 @@ import { initSharedDepCache, sharedDepCacheRoot } from "./harness/dep-cache.js";
 import { evictToCap, dirSizeBytes } from "./harness/cache-evict.js";
 import { checkDiskAdmission } from "./harness/disk-admission.js";
 import { sandboxTier, setConfiguredSandboxTier, normalizeSandboxTier } from "./harness/sandbox.js";
+import { setConfiguredAutoAttachToolImages } from "./harness/tool-image-attachments.js";
 import { injectMcpProxyForSession } from "./harness/mcp-inject.js";
 import { parseRepo, inferGitHubRepoFromWorkspace, isSharedCloneRoot, resolveGitHubToken, cloneOrUpdateRepo, resolveDefaultBaseRef, resolveBranchBaseRef, fetchOrigin } from "./repo-workspace.js";
 import { configureGitAuth, writeGitCredentialEndpoint } from "./git-auth.js";
@@ -71,7 +72,7 @@ import { normalizeMessages } from "./session/transcript-normal.js";
 import { buildNativeImportSeedPrompt } from "./session/native-import.js";
 import { EventLog } from "./session/event-log.js";
 import { AttachmentStore, isValidAttachmentHash } from "./session/attachment-store.js";
-import { planAttachment, isAttachPlanError } from "./session/attach-to-chat.js";
+import { planAttachment, isAttachPlanError, MAX_AGENT_ATTACHMENT_BYTES } from "./session/attach-to-chat.js";
 import { ReplicationService } from "./session/replication-service.js";
 import { createSessionNewDedupe } from "./session/session-new-dedupe.js";
 import { evaluateForkPrereqs, blockingForkPrereqs, missingForkPrereqs } from "./session/fork-prereqs.js";
@@ -1034,30 +1035,25 @@ function materializeAttachments(record, files) {
     return { note: notes.join("\n"), refs };
 }
 /**
- * Surface an AGENT-produced file into the chat as an attachment (image or file)
- * — the reverse of the composer paperclip. Confines to the session workspace,
- * stores the bytes in the content-addressed AttachmentStore, persists a durable
- * outbound reference anchored at the current transcript position (so a reload or
- * another device shows it), and emits the live `attachment` event so attached
- * devices render the chip/thumbnail immediately. Shared by the HTTP endpoint and
- * the `bivy attach` CLI. Returns the stored ref, or a human-readable error.
+ * Store attachment bytes, persist a durable outbound reference anchored at the
+ * current transcript position (so a reload or another device shows it), and
+ * emit the live `attachment` event so attached devices render the chip/
+ * thumbnail immediately. The common tail of both `attachToChat` (an explicit
+ * `bivy attach`) and `handlePassiveToolImage` (an image a tool produced,
+ * surfaced with no explicit attach call — see issue #292); the only difference
+ * between the two callers is how the bytes were obtained. Records the stored
+ * hash onto `record.seenAttachmentHashes` so a later passive image with
+ * identical bytes de-dupes against this one for free.
  */
-function attachToChat(record, opts) {
-    const plan = planAttachment({
-        workspaceDir: harnessDirFor(record),
-        filePath: opts.filePath,
-        mimeType: opts.mimeType,
-        name: opts.name,
-    });
-    if (isAttachPlanError(plan))
-        return { error: plan.error };
+function recordAttachment(record, bytes, opts) {
     let ref;
     try {
-        ref = attachmentStore.put(plan.bytes, { name: plan.name, mimeType: plan.mimeType, kind: plan.kind });
+        ref = attachmentStore.put(bytes, { name: opts.name, mimeType: opts.mimeType, kind: opts.kind });
     }
     catch (error) {
         return { error: `Could not store the attachment: ${error instanceof Error ? error.message : String(error)}` };
     }
+    (record.seenAttachmentHashes ??= new Set()).add(ref.hash);
     const entryId = `att-${randomBytes(8).toString("hex")}`;
     const caption = opts.caption ? String(opts.caption).slice(0, 2000) : undefined;
     // Anchor at the current base length so history replay interleaves the
@@ -1067,6 +1063,72 @@ function attachToChat(record, opts) {
     broadcast({ type: "session.event", sessionId: record.id, event: { type: "attachment", id: entryId, ref, caption } });
     return { ref };
 }
+/**
+ * Surface an AGENT-produced file into the chat as an attachment (image or file)
+ * — the reverse of the composer paperclip. Confines to the session workspace,
+ * then hands off to recordAttachment for the store+persist+broadcast. Shared by
+ * the HTTP endpoint and the `bivy attach` CLI. Returns the stored ref, or a
+ * human-readable error.
+ */
+function attachToChat(record, opts) {
+    const plan = planAttachment({
+        workspaceDir: harnessDirFor(record),
+        filePath: opts.filePath,
+        mimeType: opts.mimeType,
+        name: opts.name,
+    });
+    if (isAttachPlanError(plan))
+        return { error: plan.error };
+    return recordAttachment(record, plan.bytes, { name: plan.name, mimeType: plan.mimeType, kind: plan.kind, caption: opts.caption });
+}
+/** Extension guess for a passively-surfaced tool image, from its mime type. */
+function extFromImageMime(mimeType) {
+    if (mimeType === "image/jpeg")
+        return "jpg";
+    if (mimeType === "image/gif")
+        return "gif";
+    if (mimeType === "image/webp")
+        return "webp";
+    if (mimeType === "image/svg+xml")
+        return "svg";
+    return "png";
+}
+/**
+ * Handle a `tool_image` RuntimeEvent — a runtime adapter (see
+ * src/runtime/claude-code.ts's emitPassiveToolImages) noticed an image inside a
+ * tool_result and, gated on autoAttachToolImagesEnabled() and bounded by its own
+ * per-turn budget, forwarded the raw bytes here. Stores it exactly like an
+ * explicit `bivy attach` (see recordAttachment), except de-duplicated against
+ * anything already surfaced in this session — explicit or passive — by content
+ * hash, so identical bytes (a tool that returns the same screenshot twice, or a
+ * tool result that duplicates bytes the agent already attached) never produce a
+ * second chip. Best-effort: a malformed or oversized payload is dropped with a
+ * warning rather than erroring the turn.
+ */
+function handlePassiveToolImage(record, event) {
+    const dataB64 = typeof event.data === "string" ? event.data : "";
+    if (!dataB64)
+        return;
+    let bytes;
+    try {
+        bytes = Buffer.from(dataB64, "base64");
+    }
+    catch {
+        return;
+    }
+    if (!bytes.length || bytes.length > MAX_AGENT_ATTACHMENT_BYTES)
+        return;
+    const hash = createHash("sha256").update(bytes).digest("hex");
+    if (record.seenAttachmentHashes?.has(hash))
+        return;
+    const mimeType = typeof event.mimeType === "string" && event.mimeType ? event.mimeType : "image/png";
+    const toolName = typeof event.toolName === "string" && event.toolName.trim() ? event.toolName.trim() : "tool";
+    const name = sanitizeAttachmentFilename(`${toolName}-${hash.slice(0, 8)}.${extFromImageMime(mimeType)}`);
+    const result = recordAttachment(record, bytes, { name, mimeType, kind: "image", caption: `From ${toolName}` });
+    if ("error" in result) {
+        console.warn("[attachments] failed to store a passively-surfaced tool image:", result.error);
+    }
+}
 function approvalModeFrom(value) {
     return value === "never" || value === "risky" || value === "always" || value === "autonomous" ? value : undefined;
 }
@@ -1234,6 +1296,7 @@ function nodeSettingsSnapshot() {
             return typeof v === "string" && v.trim() ? v.trim() : undefined;
         })(),
         sessionResumeMode: nodeSessionResumeMode(),
+        autoAttachToolImages: readSettings().autoAttachToolImages === true,
     };
 }
 async function applyNodeSettings(patch) {
@@ -1290,14 +1353,21 @@ async function applyNodeSettings(patch) {
     if ("sessionResumeMode" in patch) {
         settings.sessionResumeMode = patch.sessionResumeMode === "manual" ? "manual" : "auto";
     }
+    if ("autoAttachToolImages" in patch) {
+        settings.autoAttachToolImages = patch.autoAttachToolImages === true;
+        setConfiguredAutoAttachToolImages(settings.autoAttachToolImages);
+    }
     writeSettings(settings);
     const snapshot = nodeSettingsSnapshot();
     broadcast({ type: "node.settings", settings: snapshot });
     return snapshot;
 }
 // Apply persisted node settings at boot: seed the effective sandbox tier and the
-// default runtime from settings.json (env still wins for the sandbox).
+// default runtime from settings.json (env still wins for the sandbox), plus the
+// passive tool-image-attachment gate (issue #292; BIVY_AUTO_ATTACH_TOOL_IMAGES
+// still wins — see src/harness/tool-image-attachments.ts).
 setConfiguredSandboxTier(readSettings().defaultSandbox);
+setConfiguredAutoAttachToolImages(readSettings().autoAttachToolImages);
 {
     const savedAgent = readSettings().defaultAgent;
     if (typeof savedAgent === "string" && savedAgent.trim()) {
@@ -6599,6 +6669,15 @@ function attachSessionListeners(record) {
         // text the user is watching stream in.
         if (event.type !== "message_update")
             sessionEvents.flush(record.id);
+        if (event.type === "tool_image") {
+            // A runtime adapter (e.g. Claude Code) noticed an image inside a
+            // tool_result and forwarded the raw bytes — store/persist/broadcast it as
+            // a chat attachment (see handlePassiveToolImage) instead of the generic
+            // session.event wrap below, which would otherwise ship the raw base64
+            // payload to every client.
+            handlePassiveToolImage(record, event);
+            return;
+        }
         const currentSessionFile = record.session.sessionFile;
         if (currentSessionFile && currentSessionFile !== record.sessionFile) {
             record.sessionFile = currentSessionFile;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bivy/bivy",
-  "version": "0.4.0-staging.54",
+  "version": "0.4.0-staging.55",
   "type": "module",
   "license": "FSL-1.1-ALv2",
   "description": "Run coding agents on machines you own. Source-available, self-hostable agent workspace.",