@bivy/bivy 0.20.9-staging.9 → 0.21.0

package/dist/session/event-log.js

@@ -17,10 +17,12 @@
 //     text). The base is a SNAPSHOT that is overwritten each turn and SHRINKS on
 //     compaction, so it cannot be appended naively. Instead each snapshot is stored
 //     as a bounded DELTA against the previous one: a `reset:false` record carrying
-//     only the new tail when the snapshot merely extends the last (the common case),
-//     or a `reset:true` record carrying the full snapshot when it shrinks or a
-//     prefix message changed (compaction / in-place mutation). `baseReplay` folds
-//     these back — reset replaces, non-reset extends — reproducing the last snapshot
+//     only the new tail when the snapshot merely extends the last (the common case)
+//     or changes after a shared prefix (`keep` = how much of the prefix survives;
+//     a retried turn replacing its trailing error), or a `reset:true` record
+//     carrying the full snapshot when its first message changed (compaction).
+//     `baseReplay` folds these back — reset replaces, non-reset truncates to `keep`
+//     and extends — reproducing the last snapshot
 //     exactly, so the fold is behaviour-preserving while keeping the file bounded
 //     (~O(total messages)) instead of O(turns × transcript size).
 //
@@ -234,7 +236,46 @@ export function mergeBases(logged, runtime) {
         if (add(m))
             sequence.push({ fresh: m }); // not in the logged base
     }
-    return placeFresh(merged, sequence);
+    return placeLoggedOnly(placeFresh(merged, sequence), sequence);
+}
+function timeOf(m) {
+    const t = m.timestamp;
+    return typeof t === "number" && Number.isFinite(t) ? t : undefined;
+}
+/**
+ * Put runs of messages only the log has back where they happened. A reload can
+ * drop messages the log stored: Claude's reloaded transcript has no subagent
+ * sidechain. Such a run kept its spot after its last shared message, so every
+ * later turn went in ahead of it and the old subagent work sat below the newest
+ * reply. A run that is older than the shared message before it moves to just
+ * before the first shared message that came later.
+ */
+function placeLoggedOnly(list, sequence) {
+    const shared = new Set(sequence.map((item) => ("anchor" in item ? item.anchor : item.fresh)));
+    const out = [];
+    let previous; // last shared message already in `out`
+    for (let i = 0; i < list.length;) {
+        if (shared.has(list[i])) {
+            previous = list[i];
+            out.push(previous);
+            i++;
+            continue;
+        }
+        let end = i;
+        while (end < list.length && !shared.has(list[end]))
+            end++;
+        const run = list.slice(i, end);
+        i = end;
+        const t = timeOf(run[0]);
+        const before = previous && timeOf(previous);
+        if (t === undefined || before === undefined || before <= t) {
+            out.push(...run);
+            continue;
+        }
+        const at = out.findIndex((m) => shared.has(m) && (timeOf(m) ?? -Infinity) > t);
+        out.splice(at < 0 ? out.length : at, 0, ...run);
+    }
+    return out;
 }
 /**
  * Put messages only the runtime had where they happened, not at the end. A
@@ -249,10 +290,6 @@ function placeFresh(merged, sequence) {
     const freshSet = new Set(sequence.flatMap((item) => ("fresh" in item ? [item.fresh] : [])));
     if (!freshSet.size)
         return merged;
-    const timeOf = (m) => {
-        const t = m.timestamp;
-        return typeof t === "number" && Number.isFinite(t) ? t : undefined;
-    };
     const placed = merged.filter((m) => !freshSet.has(m));
     let previous; // last anchor seen, or last fresh placed after it
     for (let i = 0; i < sequence.length; i++) {
@@ -291,7 +328,9 @@ function isBase(value) {
     if (!value || typeof value !== "object")
         return false;
     const record = value;
-    return record.bivyKind === "base" && Array.isArray(record.messages) && typeof record.reset === "boolean";
+    const keep = value.keep;
+    return record.bivyKind === "base" && Array.isArray(record.messages) && typeof record.reset === "boolean"
+        && (keep === undefined || (Number.isInteger(keep) && keep >= 0));
 }
 function isForkDisplay(value) {
     if (!value || typeof value !== "object")
@@ -544,34 +583,70 @@ export function baseReplay(entries) {
         if (entry.reset)
             base = [...entry.messages];
         else
-            base = base.concat(entry.messages);
+            base = (entry.keep === undefined ? base : base.slice(0, entry.keep)).concat(entry.messages);
     }
     return base;
 }
-function parseLogDetailed(body, keep = () => true) {
-    const records = [];
+function parseLogDetailed(lines, keep = () => true) {
+    let records = [];
     let malformedLines = 0;
-    for (const line of body.split("\n")) {
+    let superseded = 0;
+    for (const line of lines) {
         const trimmed = line.trim();
         if (!trimmed || !keep(trimmed))
             continue;
         try {
             const value = JSON.parse(trimmed);
-            if (isRecord(value))
-                records.push(value);
-            else
+            if (!isRecord(value)) {
                 malformedLines += 1;
+                continue;
+            }
+            // A full reset replaces every earlier base record (baseReplay discards
+            // them), so don't hold old snapshots and their images in memory.
+            if (isBase(value) && value.reset) {
+                const before = records.length;
+                records = records.filter((record) => record.bivyKind !== "base");
+                superseded += before - records.length;
+            }
+            records.push(value);
         }
         catch {
             malformedLines += 1;
         }
     }
-    return { records, malformedLines };
+    return { records, malformedLines, superseded };
+}
+/** A log this large that carries replaced base snapshots is rewritten without
+ *  them when it's next read. */
+const COMPACT_LOG_BYTES = 32 * 1024 * 1024;
+/** A file's lines, read in chunks: a log can outgrow the longest string V8
+ *  allows (~512 MB), which made the whole session unreadable. */
+function* fileLines(file) {
+    const fd = fs.openSync(file, "r");
+    try {
+        const chunk = Buffer.allocUnsafe(4 * 1024 * 1024);
+        let partial = [];
+        for (let n = fs.readSync(fd, chunk); n > 0; n = fs.readSync(fd, chunk)) {
+            let start = 0;
+            for (let nl = chunk.indexOf(10, start); nl !== -1 && nl < n; nl = chunk.indexOf(10, start)) {
+                yield Buffer.concat([...partial, chunk.subarray(start, nl)]).toString("utf8");
+                partial = [];
+                start = nl + 1;
+            }
+            if (start < n)
+                partial.push(Buffer.from(chunk.subarray(start, n)));
+        }
+        if (partial.length)
+            yield Buffer.concat(partial).toString("utf8");
+    }
+    finally {
+        fs.closeSync(fd);
+    }
 }
 /** Parse a JSONL log body into valid records. Callers that need corruption
  * diagnostics use EventLog.load, which reports malformed lines via onIssue. */
 export function parseLog(body) {
-    return parseLogDetailed(body).records;
+    return parseLogDetailed(body.split("\n")).records;
 }
 /**
  * Append-only, coalesced, throttled JSONL log store — the write+read companion to
@@ -655,11 +730,14 @@ export class EventLog {
     /** Read + parse a session's log file (lines passing `keep`), reporting problems. */
     readLog(id, keep) {
         try {
-            const body = fs.readFileSync(this.pathFor(id), "utf8");
-            const parsed = parseLogDetailed(body, keep);
+            const file = this.pathFor(id);
+            const bytes = fs.statSync(file).size;
+            const parsed = parseLogDetailed(fileLines(file), keep);
             if (parsed.malformedLines > 0)
                 this.report(id, "parse", new Error(`${parsed.malformedLines} malformed record(s); valid history was recovered`));
-            return { records: parsed.records, bytes: body.length };
+            if (!keep && parsed.superseded > 0 && bytes > COMPACT_LOG_BYTES)
+                return { records: parsed.records, bytes: this.compact(id, parsed.records) ?? bytes };
+            return { records: parsed.records, bytes };
         }
         catch (error) {
             if (error?.code !== "ENOENT")
@@ -667,6 +745,31 @@ export class EventLog {
             return { records: [], bytes: 0 };
         }
     }
+    /** Rewrite a log without the base snapshots a later reset replaced (logs
+     *  written before truncate-and-extend records repeated the whole transcript on
+     *  every retry). Atomic; returns the new size, or undefined if it failed. */
+    compact(id, records) {
+        const file = this.pathFor(id);
+        const tmp = `${file}.compact-${process.pid}`;
+        try {
+            const fd = fs.openSync(tmp, "w");
+            let bytes = 0;
+            try {
+                for (const record of records)
+                    bytes += fs.writeSync(fd, `${JSON.stringify(record)}\n`);
+            }
+            finally {
+                fs.closeSync(fd);
+            }
+            fs.renameSync(tmp, file);
+            return bytes;
+        }
+        catch (error) {
+            fs.rmSync(tmp, { force: true });
+            this.report(id, "rewrite", error);
+            return undefined;
+        }
+    }
     load(id) {
         const cached = this.disk.get(id);
         if (cached) {
@@ -762,15 +865,19 @@ export class EventLog {
         // Identical snapshot → nothing to record.
         if (nextKeys.length === prevKeys.length && prevKeys.every((k, i) => k === nextKeys[i]))
             return;
-        // A prefix-EXTEND (strictly longer, every prior message unchanged) appends only
-        // the new tail; anything else — a shrink (compaction), a changed prefix message
-        // (in-place mutation), or the first snapshot (empty prev, nothing to extend) —
+        // Record only what follows the prefix both snapshots share: a plain extend, or
+        // a truncate-and-extend (`keep`) when a later message changed — a retried turn
+        // replacing its trailing error rewrote the whole transcript, images and all,
+        // on every retry. A changed first message (compaction) or the first snapshot
         // appends a full reset. Rebuild the recorded messages from the serialized form:
         // a free deep copy that also matches exactly what will land on disk.
-        const extend = prevKeys.length > 0 && nextKeys.length > prevKeys.length && prevKeys.every((k, i) => k === nextKeys[i]);
-        const record = extend
-            ? { bivyKind: "base", reset: false, createdAt: Date.now(), messages: nextKeys.slice(prevKeys.length).map((s) => JSON.parse(s)) }
-            : { bivyKind: "base", reset: true, createdAt: Date.now(), messages: nextKeys.map((s) => JSON.parse(s)) };
+        let shared = 0;
+        while (shared < prevKeys.length && shared < nextKeys.length && prevKeys[shared] === nextKeys[shared])
+            shared++;
+        const tail = nextKeys.slice(shared).map((s) => JSON.parse(s));
+        const record = shared === 0
+            ? { bivyKind: "base", reset: true, createdAt: Date.now(), messages: tail }
+            : { bivyKind: "base", reset: false, createdAt: Date.now(), messages: tail, ...(shared < prevKeys.length ? { keep: shared } : {}) };
         this.baseKeys.set(id, nextKeys);
         this.enqueue(id, this.syntheticKey(id), record);
     }

package/dist/session/inline-image-fetch.js

@@ -10,27 +10,21 @@
 // otherwise get the *viewer's* browser/IP to hit an arbitrary URL by embedding
 // it in a reply).
 //
-// This is the PURE-ish, testable half — URL extraction, host/SSRF validation,
-// and the guarded fetch itself — all dependency-injectable so tests never hit
-// the real network or DNS. The server half (src/server.ts) owns the AttachmentStore
+// This is the PURE-ish, testable half — host/SSRF validation and the guarded
+// fetch itself — all dependency-injectable so tests never hit the real network
+// or DNS. Which references a message makes at all is the renderer's and this
+// module's shared concern, so it lives in ./message-components.js (the same file
+// @bivy/core compiles). The server half (src/server.ts) owns the AttachmentStore
 // write, the durable event-log ref, and the live broadcast; see resolveInlineImages.
 import dns from "node:dns/promises";
 import { hostnameIsLocal } from "../auth.js";
 import { sanitizeAttachmentName, sniffMime } from "./attach-to-chat.js";
-/**
- * The image-markdown pattern, `![alt](https://…)` — MUST match the image regex
- * in `inline()` in packages/core/src/markdown.ts exactly. Not a shared import: the
- * node (src/) intentionally does not depend on @bivy/core (a browser/client
- * package — see packages/core's own description). Kept in lock-step by comment
- * instead, the same convention EPHEMERAL_ALLOWED_HOSTS uses in
- * src/ephemeral-exec.ts for its cross-copy host allowlist. If you change one,
- * change the other.
- */
-const INLINE_IMAGE_MD_RE = /!\[[^\]]*\]\((https:\/\/[^)\s]+)\)/g;
-/** Bound how many distinct remote images a single message can trigger a fetch
- *  for — a pathological/malicious message can't fan out into an unbounded
- *  number of outbound requests. */
-export const MAX_INLINE_IMAGES_PER_MESSAGE = 6;
+import { extractImageReferences, extractMessageReferences, MAX_COMPONENTS_PER_MESSAGE } from "./message-components.js";
+/** Bound how many distinct images a single message can trigger a resolve for —
+ *  a pathological/malicious message can't fan out into an unbounded number of
+ *  outbound requests or file reads. Re-exported from the shared grammar, which
+ *  the renderer applies the same cap from. */
+export const MAX_INLINE_IMAGES_PER_MESSAGE = MAX_COMPONENTS_PER_MESSAGE;
 /** Ceiling for a single fetched inline image. Smaller than
  *  MAX_AGENT_ATTACHMENT_BYTES (attach-to-chat.ts) because these bytes come from
  *  an arbitrary, untrusted remote origin rather than the local workspace —
@@ -40,23 +34,20 @@ export const MAX_INLINE_IMAGE_BYTES = 8 * 1024 * 1024;
 const FETCH_TIMEOUT_MS = 10_000;
 /** Hard cap on redirect hops, mirroring execEphemeralRequest's guard. */
 const MAX_REDIRECTS = 5;
-/** Extract the distinct `https://` URLs a message's raw markdown references via
- *  `![alt](url)`, in first-seen order, capped at MAX_INLINE_IMAGES_PER_MESSAGE. */
+/** The distinct `https://` URLs a message's raw markdown references via
+ *  `![alt](url)`, in first-seen order, capped at MAX_INLINE_IMAGES_PER_MESSAGE.
+ *  Workspace-relative references are a different resolver's job (they are read,
+ *  not fetched) — see resolveWorkspaceImageRefs in transcript-persistence.ts. */
 export function extractInlineImageUrls(text) {
-    if (!text)
-        return [];
-    const out = [];
-    const seen = new Set();
-    for (const match of text.matchAll(INLINE_IMAGE_MD_RE)) {
-        const url = match[1];
-        if (!url || seen.has(url))
-            continue;
-        seen.add(url);
-        out.push(url);
-        if (out.length >= MAX_INLINE_IMAGES_PER_MESSAGE)
-            break;
-    }
-    return out;
+    return extractImageReferences(text).filter((i) => i.origin === "remote").map((i) => i.ref);
+}
+/** The distinct workspace references a message makes — from image syntax or a
+ *  `::view{src=…}` directive — in first-seen order, under the same per-message
+ *  cap. `syntax` tells the resolver whether non-image bytes are acceptable. */
+export function extractWorkspaceRefs(text) {
+    return extractMessageReferences(text)
+        .filter((r) => r.origin === "workspace")
+        .map(({ ref, syntax }) => ({ ref, syntax }));
 }
 /** Best-effort plain text for an assistant RuntimeMessage's `content`, which is
  *  either a plain string or an array of typed blocks (`{type:"text", text}`