@earendil-works/pi-agent-core 0.85.1 → 0.86.0

package/dist/harness/pico3/jsonl.js

@@ -0,0 +1,325 @@
+import { closeSync, existsSync, fsyncSync, ftruncateSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, statSync, unlinkSync, writeSync, } from "node:fs";
+import { join } from "node:path";
+import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
+import { isBase } from "@earendil-works/chord/delta";
+import { MemoryStorage } from "./memory.js";
+/**
+ * Files in a session directory:
+ *   main.jsonl                   conversations, entries, inputs, rewindable/session doc ops, each task's
+ *                                CREATE and TERMINAL records, and one marker per commit. Append-only, never rewritten.
+ *   sticky-<conversation>.jsonl  sticky doc ops; rewritten from its last base by `truncate` (on the Session line)
+ *   task-<id>.jsonl              a live task's intermediate patches (running, checkpoints); unlinked after its
+ *                                terminal record is published in main
+ *
+ * Publication (§10): one commit `Seq`; sidecar records are appended first, then exactly one main
+ * record, last, listing the sidecar refs it expects — the publication point. Replay applies a sidecar
+ * record only when main has the marker for that seq naming that file; unconfirmed sidecar tails are
+ * ignored. Every record carries the committed-ID high-water. Bytes after the last newline of any file
+ * are a torn write and are truncated before the file is opened for append. There is no compaction.
+ *
+ * With `fsync: false`, recovery covers process termination while the OS and filesystem remain alive.
+ * It does not promise that an acknowledged commit survives power loss, kernel/host failure,
+ * storage-cache loss, or a filesystem that loses/reorders completed writes. An acknowledged tail may
+ * roll back, or a surviving main marker may reference a missing sidecar and make open fail rather
+ * than expose partial state. A lost pre-effect checkpoint can cause an external effect to be attempted
+ * again; external idempotency remains required.
+ *
+ * With `fsync: true`, each sidecar is flushed before the main publication marker is flushed. This
+ * strengthens file-data durability and ordering across machine failure. It is not a complete database
+ * guarantee: newly created files, renames, and unlinks are not followed by a parent-directory fsync,
+ * and storage hardware/filesystems may provide weaker guarantees.
+ *
+ * One process owns a directory at a time; a second process is unsupported.
+ */
+export class JsonlStorage extends MemoryStorage {
+    mainFd;
+    sidecars = new Map(); // sticky docs, by conversation
+    taskSidecars = new Map();
+    dir;
+    fsync;
+    closedOnce = false;
+    constructor(dir, fsync) {
+        super();
+        this.dir = dir;
+        this.fsync = fsync;
+        this.mainFd = openSync(join(dir, "main.jsonl"), "a");
+    }
+    static async open(dir, opts = {}) {
+        mkdirSync(dir, { recursive: true });
+        for (const f of readdirSync(dir))
+            if (f.endsWith(".jsonl"))
+                truncateTornTail(join(dir, f));
+        const storage = new JsonlStorage(dir, opts.fsync ?? true);
+        try {
+            await storage.replay();
+            return storage;
+        }
+        catch (error) {
+            await storage.close();
+            throw error;
+        }
+    }
+    /** For tests and tooling: sizes of every file in the directory. */
+    sizes() {
+        const out = {};
+        for (const f of readdirSync(this.dir))
+            out[f] = statSync(join(this.dir, f)).size;
+        return out;
+    }
+    async replay() {
+        const read = (file) => {
+            if (!existsSync(file))
+                return [];
+            const bytes = readFileSync(file);
+            const out = [];
+            let start = 0;
+            let last = 0;
+            for (let end = bytes.indexOf(0x0a, start); end >= 0; end = bytes.indexOf(0x0a, start)) {
+                const line = bytes.subarray(start, end).toString("utf8");
+                start = end + 1;
+                if (line === "")
+                    continue;
+                let record;
+                try {
+                    record = JSON.parse(line);
+                }
+                catch (error) {
+                    throw new Error(`${file}: malformed record: ${String(error)}`);
+                }
+                if (typeof record.seq !== "number" || typeof record.maxId !== "number" || !Array.isArray(record.writes))
+                    throw new Error(`${file}: record lacks seq/maxId/writes`);
+                if (record.seq <= last)
+                    throw new Error(`${file}: sequence not increasing at ${record.seq}`);
+                last = record.seq;
+                out.push({ record, end: start });
+            }
+            return out;
+        };
+        let maxId = 0;
+        // Main is the commit log. Confirmed = (seq → refs) for every marker.
+        const main = read(join(this.dir, "main.jsonl")).map(({ record }) => record);
+        const confirmed = new Map();
+        for (const record of main)
+            confirmed.set(record.seq, new Set(record.refs ?? []));
+        // Merge every file's records by seq so batches apply in commit order.
+        const sidecarFiles = readdirSync(this.dir).filter((file) => file.startsWith("sticky-") || file.startsWith("task-"));
+        const bySeq = new Map();
+        const found = new Map();
+        const retainedStickyBase = new Map();
+        const retiredTasks = new Map();
+        for (const record of main) {
+            for (const write of record.writes) {
+                if (write.type === "task.patch" && write.patch.status === "terminal")
+                    retiredTasks.set(write.patch.id, record.seq);
+            }
+            bySeq.set(record.seq, [...record.writes]);
+            maxId = Math.max(maxId, record.maxId);
+        }
+        for (const file of sidecarFiles) {
+            const path = join(this.dir, file);
+            const records = read(path);
+            const first = records[0]?.record;
+            if (file.startsWith("sticky-") &&
+                first !== undefined &&
+                isBase(first.writes.flatMap((write) => (write.type === "doc" ? write.ops : [])))) {
+                retainedStickyBase.set(file, first.seq);
+            }
+            let confirmedEnd = 0;
+            let sawUnconfirmed = false;
+            for (const { record, end } of records) {
+                if (!confirmed.get(record.seq)?.has(file)) {
+                    sawUnconfirmed = true;
+                    continue;
+                }
+                if (sawUnconfirmed)
+                    throw new Error(`${path}: confirmed record follows an unconfirmed tail at ${record.seq}`);
+                confirmedEnd = end;
+                const files = found.get(record.seq) ?? new Set();
+                files.add(file);
+                found.set(record.seq, files);
+                bySeq.get(record.seq).push(...record.writes);
+                maxId = Math.max(maxId, record.maxId);
+            }
+            if (sawUnconfirmed)
+                truncateTo(path, confirmedEnd);
+        }
+        for (const record of main) {
+            for (const file of record.refs ?? []) {
+                if (found.get(record.seq)?.has(file))
+                    continue;
+                const retainedFrom = retainedStickyBase.get(file);
+                if (retainedFrom !== undefined && record.seq < retainedFrom)
+                    continue;
+                if (file.startsWith("task-")) {
+                    const id = Number(file.slice(5, -6));
+                    const retiredAt = retiredTasks.get(id);
+                    if (retiredAt !== undefined && record.seq < retiredAt)
+                        continue;
+                }
+                throw new Error(`${join(this.dir, file)}: missing record for published sequence ${record.seq}`);
+            }
+        }
+        for (const seq of [...bySeq.keys()].sort((a, b) => a - b)) {
+            this.seq = seq - 1;
+            await super.commit(bySeq.get(seq), BACKGROUND_CONTEXT);
+        }
+        this.setNextId(maxId + 1);
+        // Task sidecars belong to live tasks only; a leftover for a terminal (or unknown) task cannot resurrect anything.
+        for (const f of sidecarFiles) {
+            if (!f.startsWith("task-"))
+                continue;
+            const id = Number(f.slice(5, -6));
+            const t = await super.task(id);
+            if (t === undefined || t.status === "terminal")
+                unlinkSync(join(this.dir, f));
+        }
+    }
+    async commit(writes, ctx) {
+        if (this.closedOnce)
+            throw new Error("storage closed");
+        const seq = this.seq + 1;
+        const apply = this.stage(writes, seq); // validates the whole batch first; throws before any file is touched
+        const maxId = this.nextIdAfter(writes);
+        const main = [];
+        const sticky = new Map();
+        const tasks = new Map();
+        const retired = [];
+        for (const w of writes) {
+            if (w.type === "doc" && w.ref.doc === "sticky")
+                (sticky.get(w.ref.conversationId) ?? sticky.set(w.ref.conversationId, []).get(w.ref.conversationId)).push(w);
+            else if (w.type === "task.patch" && w.patch.status !== "terminal")
+                (tasks.get(w.patch.id) ?? tasks.set(w.patch.id, []).get(w.patch.id)).push(w);
+            else {
+                main.push(w);
+                if (w.type === "task.patch" && w.patch.status === "terminal")
+                    retired.push(w.patch.id);
+            }
+        }
+        const refs = [];
+        for (const [id, ws] of tasks) {
+            this.append(this.taskSidecar(id), { seq, maxId, writes: ws });
+            refs.push(`task-${id}.jsonl`);
+        }
+        for (const [c, ws] of sticky) {
+            this.append(this.sidecar(c), { seq, maxId, writes: ws });
+            refs.push(`sticky-${c}.jsonl`);
+        }
+        this.append(this.mainFd, { seq, maxId, writes: main, ...(refs.length ? { refs } : {}) }); // the publication point
+        apply();
+        this.seq = seq;
+        for (const id of retired)
+            this.retireTaskSidecar(id); // after the terminal record is durably published
+        void ctx;
+        return seq;
+    }
+    nextIdAfter(writes) {
+        let max = this.nextIdValue - 1;
+        for (const w of writes) {
+            const id = w.type === "conversation"
+                ? w.conversation.id
+                : w.type === "entry"
+                    ? w.entry.id
+                    : w.type === "task"
+                        ? w.task.id
+                        : w.type === "input"
+                            ? w.input.id
+                            : 0;
+            max = Math.max(max, id);
+        }
+        return max;
+    }
+    append(fd, record) {
+        writeSync(fd, `${JSON.stringify(record)}\n`);
+        if (this.fsync)
+            fsyncSync(fd);
+    }
+    sidecar(conversationId) {
+        let fd = this.sidecars.get(conversationId);
+        if (fd === undefined) {
+            fd = openSync(join(this.dir, `sticky-${conversationId}.jsonl`), "a");
+            this.sidecars.set(conversationId, fd);
+        }
+        return fd;
+    }
+    taskSidecar(id) {
+        let fd = this.taskSidecars.get(id);
+        if (fd === undefined) {
+            fd = openSync(join(this.dir, `task-${id}.jsonl`), "a");
+            this.taskSidecars.set(id, fd);
+        }
+        return fd;
+    }
+    retireTaskSidecar(id) {
+        const fd = this.taskSidecars.get(id);
+        if (fd !== undefined)
+            closeSync(fd);
+        this.taskSidecars.delete(id);
+        try {
+            unlinkSync(join(this.dir, `task-${id}.jsonl`));
+        }
+        catch { }
+    }
+    /** Rewrite the sticky sidecar from its last base: temp file, optional fsync, rename. Called on the Session line. */
+    async truncate(ref, _ctx) {
+        await super.truncate(ref);
+        if (ref.doc !== "sticky")
+            return;
+        const file = join(this.dir, `sticky-${ref.conversationId}.jsonl`);
+        if (!existsSync(file))
+            return;
+        const lines = readFileSync(file, "utf8").split("\n").filter(Boolean);
+        let start = 0;
+        for (let i = lines.length - 1; i >= 0; i--)
+            if (isBase(JSON.parse(lines[i]).writes.flatMap((w) => (w.type === "doc" ? w.ops : [])))) {
+                start = i;
+                break;
+            }
+        if (start === 0)
+            return;
+        const tmp = `${file}.tmp`;
+        const fd = openSync(tmp, "w");
+        writeSync(fd, lines
+            .slice(start)
+            .map((l) => `${l}\n`)
+            .join(""));
+        if (this.fsync)
+            fsyncSync(fd);
+        closeSync(fd);
+        const old = this.sidecars.get(ref.conversationId);
+        if (old !== undefined)
+            closeSync(old);
+        renameSync(tmp, file);
+        this.sidecars.set(ref.conversationId, openSync(file, "a"));
+    }
+    async close(_ctx) {
+        if (this.closedOnce)
+            return;
+        this.closedOnce = true;
+        await super.close();
+        closeSync(this.mainFd);
+        for (const fd of this.sidecars.values())
+            closeSync(fd);
+        for (const fd of this.taskSidecars.values())
+            closeSync(fd);
+    }
+}
+function truncateTo(file, length) {
+    const fd = openSync(file, "r+");
+    try {
+        ftruncateSync(fd, length);
+    }
+    finally {
+        closeSync(fd);
+    }
+}
+/** Bytes after the last newline are a torn write: cut them so a later append cannot extend a corrupt line. */
+function truncateTornTail(file) {
+    const size = statSync(file).size;
+    if (size === 0)
+        return;
+    const buf = readFileSync(file);
+    const end = buf.lastIndexOf(0x0a);
+    if (end + 1 !== buf.length)
+        truncateTo(file, end + 1);
+}
+//# sourceMappingURL=jsonl.js.map

package/dist/harness/pico3/jsonl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"jsonl.js","sourceRoot":"","sources":["../../../src/harness/pico3/jsonl.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,SAAS,EACT,UAAU,EACV,SAAS,EACT,aAAa,EACb,SAAS,EACT,QAAQ,EACR,WAAW,EACX,YAAY,EACZ,UAAU,EACV,QAAQ,EACR,UAAU,EACV,SAAS,GACT,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,MAAM,EAAW,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAW5C;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAM,OAAO,YAAa,SAAQ,aAAa;IACtC,MAAM,CAAS;IACN,QAAQ,GAAG,IAAI,GAAG,EAAc,CAAC,CAAC,+BAA+B;IACjE,YAAY,GAAG,IAAI,GAAG,EAAc,CAAC;IACrC,GAAG,CAAS;IACpB,KAAK,CAAU;IAChB,UAAU,GAAG,KAAK,CAAC;IAE3B,YAAoB,GAAW,EAAE,KAAc,EAAE;QAChD,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,EAAE,GAAG,CAAC,CAAC;IAAA,CACrD;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAW,EAAE,IAAI,GAAwB,EAAE,EAAyB;QACrF,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACpC,KAAK,MAAM,CAAC,IAAI,WAAW,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAAE,gBAAgB,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3F,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC;QAC1D,IAAI,CAAC;YACJ,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC;YACvB,OAAO,OAAO,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;YACtB,MAAM,KAAK,CAAC;QACb,CAAC;IAAA,CACD;IAED,mEAAmE;IACnE,KAAK,GAA+B;QACnC,MAAM,GAAG,GAA+B,EAAE,CAAC;QAC3C,KAAK,MAAM,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,GAAG,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACjF,OAAO,GAAG,CAAC;IAAA,CACX;IAEO,KAAK,CAAC,MAAM,GAAG;QACtB,MAAM,IAAI,GAAG,CAAC,IAAY,EAAsC,EAAE,CAAC;YAClE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;gBAAE,OAAO,EAAE,CAAC;YACjC,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;YACjC,MAAM,GAAG,GAAuC,EAAE,CAAC;YACnD,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,IAAI,IAAI,GAAG,CAAC,CAAC;YACb,KAAK,IAAI,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,CAAC;gBACvF,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACzD,KAAK,GAAG,GAAG,GAAG,CAAC,CAAC;gBAChB,IAAI,IAAI,KAAK,EAAE;oBAAE,SAAS;gBAC1B,IAAI,MAAe,CAAC;gBACpB,IAAI,CAAC;oBACJ,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC;gBACtC,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,uBAAuB,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAChE,CAAC;gBACD,IAAI,OAAO,MAAM,CAAC,GAAG,KAAK,QAAQ,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;oBACtG,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,iCAAiC,CAAC,CAAC;gBAC3D,IAAI,MAAM,CAAC,GAAG,IAAI,IAAI;oBAAE,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,gCAAgC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;gBAC7F,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC;gBAClB,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;YAClC,CAAC;YACD,OAAO,GAAG,CAAC;QAAA,CACX,CAAC;QACF,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,uEAAqE;QACrE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QAC5E,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoB,CAAC;QAC9C,KAAK,MAAM,MAAM,IAAI,IAAI;YAAE,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC;QACjF,sEAAsE;QACtE,MAAM,YAAY,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,CAChD,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAChE,CAAC;QACF,MAAM,KAAK,GAAG,IAAI,GAAG,EAAgB,CAAC;QACtC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAC;QAC1C,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAe,CAAC;QAClD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAW,CAAC;QACxC,KAAK,MAAM,MAAM,IAAI,IAAI,EAAE,CAAC;YAC3B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;gBACnC,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,UAAU;oBACnE,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;YAC/C,CAAC;YACD,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;YAC1C,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QACvC,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YAClC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3B,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC;YACjC,IACC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;gBAC1B,KAAK,KAAK,SAAS;gBACnB,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAC/E,CAAC;gBACF,kBAAkB,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;YACzC,CAAC;YACD,IAAI,YAAY,GAAG,CAAC,CAAC;YACrB,IAAI,cAAc,GAAG,KAAK,CAAC;YAC3B,KAAK,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,OAAO,EAAE,CAAC;gBACvC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC3C,cAAc,GAAG,IAAI,CAAC;oBACtB,SAAS;gBACV,CAAC;gBACD,IAAI,cAAc;oBACjB,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,qDAAqD,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;gBAC3F,YAAY,GAAG,GAAG,CAAC;gBACnB,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,EAAU,CAAC;gBACzD,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAChB,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;gBAC7B,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC9C,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;YACvC,CAAC;YACD,IAAI,cAAc;gBAAE,UAAU,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QACpD,CAAC;QACD,KAAK,MAAM,MAAM,IAAI,IAAI,EAAE,CAAC;YAC3B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,IAAI,EAAE,EAAE,CAAC;gBACtC,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC;oBAAE,SAAS;gBAC/C,MAAM,YAAY,GAAG,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAClD,IAAI,YAAY,KAAK,SAAS,IAAI,MAAM,CAAC,GAAG,GAAG,YAAY;oBAAE,SAAS;gBACtE,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC9B,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;oBACrC,MAAM,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBACvC,IAAI,SAAS,KAAK,SAAS,IAAI,MAAM,CAAC,GAAG,GAAG,SAAS;wBAAE,SAAS;gBACjE,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,2CAA2C,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;YACjG,CAAC;QACF,CAAC;QACD,KAAK,MAAM,GAAG,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YAC3D,IAAI,CAAC,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC;YACnB,MAAM,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAE,EAAE,kBAAkB,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QAC1B,kHAAkH;QAClH,KAAK,MAAM,CAAC,IAAI,YAAY,EAAE,CAAC;YAC9B,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC;gBAAE,SAAS;YACrC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;YAClC,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC/B,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,KAAK,UAAU;gBAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QAC/E,CAAC;IAAA,CACD;IAEQ,KAAK,CAAC,MAAM,CAAC,MAAwB,EAAE,GAAY,EAAgB;QAC3E,IAAI,IAAI,CAAC,UAAU;YAAE,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC;QACvD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,qEAAqE;QAC5G,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACvC,MAAM,IAAI,GAAY,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,IAAI,GAAG,EAAe,CAAC;QACtC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAe,CAAC;QACrC,MAAM,OAAO,GAAS,EAAE,CAAC;QACzB,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACxB,IAAI,CAAC,CAAC,IAAI,KAAK,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,KAAK,QAAQ;gBAC7C,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,cAAc,CAAE,CAAC,CAAC,IAAI,CACzG,CAAC,CACD,CAAC;iBACE,IAAI,CAAC,CAAC,IAAI,KAAK,YAAY,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM,KAAK,UAAU;gBAChE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;iBAC1E,CAAC;gBACL,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACb,IAAI,CAAC,CAAC,IAAI,KAAK,YAAY,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM,KAAK,UAAU;oBAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACxF,CAAC;QACF,CAAC;QACD,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,KAAK,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;YAC9B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;YAC9D,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAC/B,CAAC;QACD,KAAK,MAAM,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,EAAE,CAAC;YAC9B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;YACzD,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QAChC,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,wBAAwB;QAClH,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,KAAK,MAAM,EAAE,IAAI,OAAO;YAAE,IAAI,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC,CAAC,iDAAiD;QACvG,KAAK,GAAG,CAAC;QACT,OAAO,GAAG,CAAC;IAAA,CACX;IACO,WAAW,CAAC,MAAwB,EAAM;QACjD,IAAI,GAAG,GAAG,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACxB,MAAM,EAAE,GACP,CAAC,CAAC,IAAI,KAAK,cAAc;gBACxB,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,EAAE;gBACnB,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO;oBACnB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE;oBACZ,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM;wBAClB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE;wBACX,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO;4BACnB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE;4BACZ,CAAC,CAAC,CAAC,CAAC;YACT,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACzB,CAAC;QACD,OAAO,GAAG,CAAC;IAAA,CACX;IAEO,MAAM,CAAC,EAAU,EAAE,MAAe,EAAE;QAC3C,SAAS,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,IAAI,CAAC,KAAK;YAAE,SAAS,CAAC,EAAE,CAAC,CAAC;IAAA,CAC9B;IAEO,OAAO,CAAC,cAAkB,EAAU;QAC3C,IAAI,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QAC3C,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;YACtB,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,cAAc,QAAQ,CAAC,EAAE,GAAG,CAAC,CAAC;YACrE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QACvC,CAAC;QACD,OAAO,EAAE,CAAC;IAAA,CACV;IACO,WAAW,CAAC,EAAM,EAAU;QACnC,IAAI,EAAE,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACnC,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;YACtB,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,GAAG,CAAC,CAAC;YACvD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QAC/B,CAAC;QACD,OAAO,EAAE,CAAC;IAAA,CACV;IACO,iBAAiB,CAAC,EAAM,EAAE;QACjC,MAAM,EAAE,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACrC,IAAI,EAAE,KAAK,SAAS;YAAE,SAAS,CAAC,EAAE,CAAC,CAAC;QACpC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC7B,IAAI,CAAC;YACJ,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;QAChD,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;IAAA,CACV;IAED,oHAAoH;IAC3G,KAAK,CAAC,QAAQ,CAAC,GAAW,EAAE,IAAc,EAAiB;QACnE,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAC1B,IAAI,GAAG,CAAC,GAAG,KAAK,QAAQ;YAAE,OAAO;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,GAAG,CAAC,cAAc,QAAQ,CAAC,CAAC;QAClE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,OAAO;QAC9B,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACrE,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE;YACzC,IACC,MAAM,CAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAE,CAAa,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAS,CAAC,EACxG,CAAC;gBACF,KAAK,GAAG,CAAC,CAAC;gBACV,MAAM;YACP,CAAC;QACF,IAAI,KAAK,KAAK,CAAC;YAAE,OAAO;QACxB,MAAM,GAAG,GAAG,GAAG,IAAI,MAAM,CAAC;QAC1B,MAAM,EAAE,GAAG,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAC9B,SAAS,CACR,EAAE,EACF,KAAK;aACH,KAAK,CAAC,KAAK,CAAC;aACZ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC;aACpB,IAAI,CAAC,EAAE,CAAC,CACV,CAAC;QACF,IAAI,IAAI,CAAC,KAAK;YAAE,SAAS,CAAC,EAAE,CAAC,CAAC;QAC9B,SAAS,CAAC,EAAE,CAAC,CAAC;QACd,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QAClD,IAAI,GAAG,KAAK,SAAS;YAAE,SAAS,CAAC,GAAG,CAAC,CAAC;QACtC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QACtB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,cAAc,EAAE,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;IAAA,CAC3D;IAEQ,KAAK,CAAC,KAAK,CAAC,IAAc,EAAE;QACpC,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO;QAC5B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACpB,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;YAAE,SAAS,CAAC,EAAE,CAAC,CAAC;QACvD,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE;YAAE,SAAS,CAAC,EAAE,CAAC,CAAC;IAAA,CAC3D;CACD;AAED,SAAS,UAAU,CAAC,IAAY,EAAE,MAAc,EAAQ;IACvD,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAChC,IAAI,CAAC;QACJ,aAAa,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC;YAAS,CAAC;QACV,SAAS,CAAC,EAAE,CAAC,CAAC;IACf,CAAC;AAAA,CACD;AAED,8GAA8G;AAC9G,SAAS,gBAAgB,CAAC,IAAY,EAAE;IACvC,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC;IACjC,IAAI,IAAI,KAAK,CAAC;QAAE,OAAO;IACvB,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,GAAG,GAAG,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAClC,IAAI,GAAG,GAAG,CAAC,KAAK,GAAG,CAAC,MAAM;QAAE,UAAU,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;AAAA,CACtD","sourcesContent":["import {\n\tcloseSync,\n\texistsSync,\n\tfsyncSync,\n\tftruncateSync,\n\tmkdirSync,\n\topenSync,\n\treaddirSync,\n\treadFileSync,\n\trenameSync,\n\tstatSync,\n\tunlinkSync,\n\twriteSync,\n} from \"node:fs\";\nimport { join } from \"node:path\";\nimport type { Context } from \"@earendil-works/chord\";\nimport { BACKGROUND_CONTEXT } from \"@earendil-works/chord/context\";\nimport { isBase, type Op } from \"@earendil-works/chord/delta\";\nimport { MemoryStorage } from \"./memory.ts\";\nimport type { DocRef, Id, Seq, Storage, Write } from \"./types.ts\";\n\n/** One line of any file. `refs` (main only) lists the sidecar files this commit also wrote. */\ninterface Record_ {\n\tseq: Seq;\n\tmaxId: Id;\n\twrites: Write[];\n\trefs?: string[];\n}\n\n/**\n * Files in a session directory:\n *   main.jsonl                   conversations, entries, inputs, rewindable/session doc ops, each task's\n *                                CREATE and TERMINAL records, and one marker per commit. Append-only, never rewritten.\n *   sticky-<conversation>.jsonl  sticky doc ops; rewritten from its last base by `truncate` (on the Session line)\n *   task-<id>.jsonl              a live task's intermediate patches (running, checkpoints); unlinked after its\n *                                terminal record is published in main\n *\n * Publication (§10): one commit `Seq`; sidecar records are appended first, then exactly one main\n * record, last, listing the sidecar refs it expects — the publication point. Replay applies a sidecar\n * record only when main has the marker for that seq naming that file; unconfirmed sidecar tails are\n * ignored. Every record carries the committed-ID high-water. Bytes after the last newline of any file\n * are a torn write and are truncated before the file is opened for append. There is no compaction.\n *\n * With `fsync: false`, recovery covers process termination while the OS and filesystem remain alive.\n * It does not promise that an acknowledged commit survives power loss, kernel/host failure,\n * storage-cache loss, or a filesystem that loses/reorders completed writes. An acknowledged tail may\n * roll back, or a surviving main marker may reference a missing sidecar and make open fail rather\n * than expose partial state. A lost pre-effect checkpoint can cause an external effect to be attempted\n * again; external idempotency remains required.\n *\n * With `fsync: true`, each sidecar is flushed before the main publication marker is flushed. This\n * strengthens file-data durability and ordering across machine failure. It is not a complete database\n * guarantee: newly created files, renames, and unlinks are not followed by a parent-directory fsync,\n * and storage hardware/filesystems may provide weaker guarantees.\n *\n * One process owns a directory at a time; a second process is unsupported.\n */\nexport class JsonlStorage extends MemoryStorage implements Storage {\n\tprivate mainFd: number;\n\tprivate readonly sidecars = new Map<Id, number>(); // sticky docs, by conversation\n\tprivate readonly taskSidecars = new Map<Id, number>();\n\tprivate readonly dir: string;\n\treadonly fsync: boolean;\n\tprivate closedOnce = false;\n\n\tprivate constructor(dir: string, fsync: boolean) {\n\t\tsuper();\n\t\tthis.dir = dir;\n\t\tthis.fsync = fsync;\n\t\tthis.mainFd = openSync(join(dir, \"main.jsonl\"), \"a\");\n\t}\n\n\tstatic async open(dir: string, opts: { fsync?: boolean } = {}): Promise<JsonlStorage> {\n\t\tmkdirSync(dir, { recursive: true });\n\t\tfor (const f of readdirSync(dir)) if (f.endsWith(\".jsonl\")) truncateTornTail(join(dir, f));\n\t\tconst storage = new JsonlStorage(dir, opts.fsync ?? true);\n\t\ttry {\n\t\t\tawait storage.replay();\n\t\t\treturn storage;\n\t\t} catch (error) {\n\t\t\tawait storage.close();\n\t\t\tthrow error;\n\t\t}\n\t}\n\n\t/** For tests and tooling: sizes of every file in the directory. */\n\tsizes(): { [file: string]: number } {\n\t\tconst out: { [file: string]: number } = {};\n\t\tfor (const f of readdirSync(this.dir)) out[f] = statSync(join(this.dir, f)).size;\n\t\treturn out;\n\t}\n\n\tprivate async replay() {\n\t\tconst read = (file: string): { record: Record_; end: number }[] => {\n\t\t\tif (!existsSync(file)) return [];\n\t\t\tconst bytes = readFileSync(file);\n\t\t\tconst out: { record: Record_; end: number }[] = [];\n\t\t\tlet start = 0;\n\t\t\tlet last = 0;\n\t\t\tfor (let end = bytes.indexOf(0x0a, start); end >= 0; end = bytes.indexOf(0x0a, start)) {\n\t\t\t\tconst line = bytes.subarray(start, end).toString(\"utf8\");\n\t\t\t\tstart = end + 1;\n\t\t\t\tif (line === \"\") continue;\n\t\t\t\tlet record: Record_;\n\t\t\t\ttry {\n\t\t\t\t\trecord = JSON.parse(line) as Record_;\n\t\t\t\t} catch (error) {\n\t\t\t\t\tthrow new Error(`${file}: malformed record: ${String(error)}`);\n\t\t\t\t}\n\t\t\t\tif (typeof record.seq !== \"number\" || typeof record.maxId !== \"number\" || !Array.isArray(record.writes))\n\t\t\t\t\tthrow new Error(`${file}: record lacks seq/maxId/writes`);\n\t\t\t\tif (record.seq <= last) throw new Error(`${file}: sequence not increasing at ${record.seq}`);\n\t\t\t\tlast = record.seq;\n\t\t\t\tout.push({ record, end: start });\n\t\t\t}\n\t\t\treturn out;\n\t\t};\n\t\tlet maxId = 0;\n\t\t// Main is the commit log. Confirmed = (seq → refs) for every marker.\n\t\tconst main = read(join(this.dir, \"main.jsonl\")).map(({ record }) => record);\n\t\tconst confirmed = new Map<Seq, Set<string>>();\n\t\tfor (const record of main) confirmed.set(record.seq, new Set(record.refs ?? []));\n\t\t// Merge every file's records by seq so batches apply in commit order.\n\t\tconst sidecarFiles = readdirSync(this.dir).filter(\n\t\t\t(file) => file.startsWith(\"sticky-\") || file.startsWith(\"task-\"),\n\t\t);\n\t\tconst bySeq = new Map<Seq, Write[]>();\n\t\tconst found = new Map<Seq, Set<string>>();\n\t\tconst retainedStickyBase = new Map<string, Seq>();\n\t\tconst retiredTasks = new Map<Id, Seq>();\n\t\tfor (const record of main) {\n\t\t\tfor (const write of record.writes) {\n\t\t\t\tif (write.type === \"task.patch\" && write.patch.status === \"terminal\")\n\t\t\t\t\tretiredTasks.set(write.patch.id, record.seq);\n\t\t\t}\n\t\t\tbySeq.set(record.seq, [...record.writes]);\n\t\t\tmaxId = Math.max(maxId, record.maxId);\n\t\t}\n\t\tfor (const file of sidecarFiles) {\n\t\t\tconst path = join(this.dir, file);\n\t\t\tconst records = read(path);\n\t\t\tconst first = records[0]?.record;\n\t\t\tif (\n\t\t\t\tfile.startsWith(\"sticky-\") &&\n\t\t\t\tfirst !== undefined &&\n\t\t\t\tisBase(first.writes.flatMap((write) => (write.type === \"doc\" ? write.ops : [])))\n\t\t\t) {\n\t\t\t\tretainedStickyBase.set(file, first.seq);\n\t\t\t}\n\t\t\tlet confirmedEnd = 0;\n\t\t\tlet sawUnconfirmed = false;\n\t\t\tfor (const { record, end } of records) {\n\t\t\t\tif (!confirmed.get(record.seq)?.has(file)) {\n\t\t\t\t\tsawUnconfirmed = true;\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\tif (sawUnconfirmed)\n\t\t\t\t\tthrow new Error(`${path}: confirmed record follows an unconfirmed tail at ${record.seq}`);\n\t\t\t\tconfirmedEnd = end;\n\t\t\t\tconst files = found.get(record.seq) ?? new Set<string>();\n\t\t\t\tfiles.add(file);\n\t\t\t\tfound.set(record.seq, files);\n\t\t\t\tbySeq.get(record.seq)!.push(...record.writes);\n\t\t\t\tmaxId = Math.max(maxId, record.maxId);\n\t\t\t}\n\t\t\tif (sawUnconfirmed) truncateTo(path, confirmedEnd);\n\t\t}\n\t\tfor (const record of main) {\n\t\t\tfor (const file of record.refs ?? []) {\n\t\t\t\tif (found.get(record.seq)?.has(file)) continue;\n\t\t\t\tconst retainedFrom = retainedStickyBase.get(file);\n\t\t\t\tif (retainedFrom !== undefined && record.seq < retainedFrom) continue;\n\t\t\t\tif (file.startsWith(\"task-\")) {\n\t\t\t\t\tconst id = Number(file.slice(5, -6));\n\t\t\t\t\tconst retiredAt = retiredTasks.get(id);\n\t\t\t\t\tif (retiredAt !== undefined && record.seq < retiredAt) continue;\n\t\t\t\t}\n\t\t\t\tthrow new Error(`${join(this.dir, file)}: missing record for published sequence ${record.seq}`);\n\t\t\t}\n\t\t}\n\t\tfor (const seq of [...bySeq.keys()].sort((a, b) => a - b)) {\n\t\t\tthis.seq = seq - 1;\n\t\t\tawait super.commit(bySeq.get(seq)!, BACKGROUND_CONTEXT);\n\t\t}\n\t\tthis.setNextId(maxId + 1);\n\t\t// Task sidecars belong to live tasks only; a leftover for a terminal (or unknown) task cannot resurrect anything.\n\t\tfor (const f of sidecarFiles) {\n\t\t\tif (!f.startsWith(\"task-\")) continue;\n\t\t\tconst id = Number(f.slice(5, -6));\n\t\t\tconst t = await super.task(id);\n\t\t\tif (t === undefined || t.status === \"terminal\") unlinkSync(join(this.dir, f));\n\t\t}\n\t}\n\n\toverride async commit(writes: readonly Write[], ctx: Context): Promise<Seq> {\n\t\tif (this.closedOnce) throw new Error(\"storage closed\");\n\t\tconst seq = this.seq + 1;\n\t\tconst apply = this.stage(writes, seq); // validates the whole batch first; throws before any file is touched\n\t\tconst maxId = this.nextIdAfter(writes);\n\t\tconst main: Write[] = [];\n\t\tconst sticky = new Map<Id, Write[]>();\n\t\tconst tasks = new Map<Id, Write[]>();\n\t\tconst retired: Id[] = [];\n\t\tfor (const w of writes) {\n\t\t\tif (w.type === \"doc\" && w.ref.doc === \"sticky\")\n\t\t\t\t(sticky.get(w.ref.conversationId) ?? sticky.set(w.ref.conversationId, []).get(w.ref.conversationId)!).push(\n\t\t\t\t\tw,\n\t\t\t\t);\n\t\t\telse if (w.type === \"task.patch\" && w.patch.status !== \"terminal\")\n\t\t\t\t(tasks.get(w.patch.id) ?? tasks.set(w.patch.id, []).get(w.patch.id)!).push(w);\n\t\t\telse {\n\t\t\t\tmain.push(w);\n\t\t\t\tif (w.type === \"task.patch\" && w.patch.status === \"terminal\") retired.push(w.patch.id);\n\t\t\t}\n\t\t}\n\t\tconst refs: string[] = [];\n\t\tfor (const [id, ws] of tasks) {\n\t\t\tthis.append(this.taskSidecar(id), { seq, maxId, writes: ws });\n\t\t\trefs.push(`task-${id}.jsonl`);\n\t\t}\n\t\tfor (const [c, ws] of sticky) {\n\t\t\tthis.append(this.sidecar(c), { seq, maxId, writes: ws });\n\t\t\trefs.push(`sticky-${c}.jsonl`);\n\t\t}\n\t\tthis.append(this.mainFd, { seq, maxId, writes: main, ...(refs.length ? { refs } : {}) }); // the publication point\n\t\tapply();\n\t\tthis.seq = seq;\n\t\tfor (const id of retired) this.retireTaskSidecar(id); // after the terminal record is durably published\n\t\tvoid ctx;\n\t\treturn seq;\n\t}\n\tprivate nextIdAfter(writes: readonly Write[]): Id {\n\t\tlet max = this.nextIdValue - 1;\n\t\tfor (const w of writes) {\n\t\t\tconst id =\n\t\t\t\tw.type === \"conversation\"\n\t\t\t\t\t? w.conversation.id\n\t\t\t\t\t: w.type === \"entry\"\n\t\t\t\t\t\t? w.entry.id\n\t\t\t\t\t\t: w.type === \"task\"\n\t\t\t\t\t\t\t? w.task.id\n\t\t\t\t\t\t\t: w.type === \"input\"\n\t\t\t\t\t\t\t\t? w.input.id\n\t\t\t\t\t\t\t\t: 0;\n\t\t\tmax = Math.max(max, id);\n\t\t}\n\t\treturn max;\n\t}\n\n\tprivate append(fd: number, record: Record_) {\n\t\twriteSync(fd, `${JSON.stringify(record)}\\n`);\n\t\tif (this.fsync) fsyncSync(fd);\n\t}\n\n\tprivate sidecar(conversationId: Id): number {\n\t\tlet fd = this.sidecars.get(conversationId);\n\t\tif (fd === undefined) {\n\t\t\tfd = openSync(join(this.dir, `sticky-${conversationId}.jsonl`), \"a\");\n\t\t\tthis.sidecars.set(conversationId, fd);\n\t\t}\n\t\treturn fd;\n\t}\n\tprivate taskSidecar(id: Id): number {\n\t\tlet fd = this.taskSidecars.get(id);\n\t\tif (fd === undefined) {\n\t\t\tfd = openSync(join(this.dir, `task-${id}.jsonl`), \"a\");\n\t\t\tthis.taskSidecars.set(id, fd);\n\t\t}\n\t\treturn fd;\n\t}\n\tprivate retireTaskSidecar(id: Id) {\n\t\tconst fd = this.taskSidecars.get(id);\n\t\tif (fd !== undefined) closeSync(fd);\n\t\tthis.taskSidecars.delete(id);\n\t\ttry {\n\t\t\tunlinkSync(join(this.dir, `task-${id}.jsonl`));\n\t\t} catch {}\n\t}\n\n\t/** Rewrite the sticky sidecar from its last base: temp file, optional fsync, rename. Called on the Session line. */\n\toverride async truncate(ref: DocRef, _ctx?: Context): Promise<void> {\n\t\tawait super.truncate(ref);\n\t\tif (ref.doc !== \"sticky\") return;\n\t\tconst file = join(this.dir, `sticky-${ref.conversationId}.jsonl`);\n\t\tif (!existsSync(file)) return;\n\t\tconst lines = readFileSync(file, \"utf8\").split(\"\\n\").filter(Boolean);\n\t\tlet start = 0;\n\t\tfor (let i = lines.length - 1; i >= 0; i--)\n\t\t\tif (\n\t\t\t\tisBase((JSON.parse(lines[i]!) as Record_).writes.flatMap((w) => (w.type === \"doc\" ? w.ops : [])) as Op[])\n\t\t\t) {\n\t\t\t\tstart = i;\n\t\t\t\tbreak;\n\t\t\t}\n\t\tif (start === 0) return;\n\t\tconst tmp = `${file}.tmp`;\n\t\tconst fd = openSync(tmp, \"w\");\n\t\twriteSync(\n\t\t\tfd,\n\t\t\tlines\n\t\t\t\t.slice(start)\n\t\t\t\t.map((l) => `${l}\\n`)\n\t\t\t\t.join(\"\"),\n\t\t);\n\t\tif (this.fsync) fsyncSync(fd);\n\t\tcloseSync(fd);\n\t\tconst old = this.sidecars.get(ref.conversationId);\n\t\tif (old !== undefined) closeSync(old);\n\t\trenameSync(tmp, file);\n\t\tthis.sidecars.set(ref.conversationId, openSync(file, \"a\"));\n\t}\n\n\toverride async close(_ctx?: Context) {\n\t\tif (this.closedOnce) return;\n\t\tthis.closedOnce = true;\n\t\tawait super.close();\n\t\tcloseSync(this.mainFd);\n\t\tfor (const fd of this.sidecars.values()) closeSync(fd);\n\t\tfor (const fd of this.taskSidecars.values()) closeSync(fd);\n\t}\n}\n\nfunction truncateTo(file: string, length: number): void {\n\tconst fd = openSync(file, \"r+\");\n\ttry {\n\t\tftruncateSync(fd, length);\n\t} finally {\n\t\tcloseSync(fd);\n\t}\n}\n\n/** Bytes after the last newline are a torn write: cut them so a later append cannot extend a corrupt line. */\nfunction truncateTornTail(file: string) {\n\tconst size = statSync(file).size;\n\tif (size === 0) return;\n\tconst buf = readFileSync(file);\n\tconst end = buf.lastIndexOf(0x0a);\n\tif (end + 1 !== buf.length) truncateTo(file, end + 1);\n}\n"]}

package/dist/harness/pico3/kinds/collapse.d.ts

@@ -0,0 +1,49 @@
+import type { Context } from "@earendil-works/chord";
+import { type CoreKind, type Entry, type HookInfo, type HookResult, type Id, type ModelRef, type RetryPolicy, type ThinkingLevel } from "../types.ts";
+export type CollapseInput = {
+    reason: "threshold" | "manual" | "overflow";
+    through: Id;
+    instructions?: string;
+};
+type Base = {
+    expectedHead: Id | null;
+    instructions?: string;
+    model: ModelRef;
+    thinkingLevel: ThinkingLevel;
+    retry: RetryPolicy;
+    attempt: number;
+};
+export type CollapseCheckpoint = ({
+    phase: "summarizing";
+} & Base) | ({
+    phase: "retrying";
+    untilMs: number;
+    lastError: string;
+} & Base) | ({
+    phase: "prepared";
+    summary: string;
+} & Base);
+export type CollapseFailure = {
+    reason: "stale" | "declined" | "provider" | "retries_exhausted" | "no_model";
+    detail: string;
+};
+export interface CollapseHooks {
+    beforeCollapse(reason: "threshold" | "manual" | "overflow", through: Id, entries: Entry[], info: HookInfo, ctx: Context): HookResult<{
+        decline: true;
+    } | {
+        instructions?: string;
+        summary?: string;
+    }>;
+}
+export declare const collapseConfig: {
+    readonly rewindable: {
+        readonly threshold: number;
+        readonly keepRecent: number;
+    };
+};
+export declare const collapse: CoreKind<CollapseInput, CollapseCheckpoint, {
+    summary: Id;
+}, CollapseFailure, null, CollapseHooks, typeof collapseConfig>;
+export declare function chooseThrough(entries: Entry[], keepRecent: number): Id | undefined;
+export {};
+//# sourceMappingURL=collapse.d.ts.map

package/dist/harness/pico3/kinds/collapse.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"collapse.d.ts","sourceRoot":"","sources":["../../../../src/harness/pico3/kinds/collapse.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAIrD,OAAO,EAEN,KAAK,QAAQ,EAEb,KAAK,KAAK,EACV,KAAK,QAAQ,EACb,KAAK,UAAU,EACf,KAAK,EAAE,EACP,KAAK,QAAQ,EAEb,KAAK,WAAW,EAKhB,KAAK,aAAa,EAElB,MAAM,aAAa,CAAC;AAQrB,MAAM,MAAM,aAAa,GAAG;IAAE,MAAM,EAAE,WAAW,GAAG,QAAQ,GAAG,UAAU,CAAC;IAAC,OAAO,EAAE,EAAE,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAChH,KAAK,IAAI,GAAG;IACX,YAAY,EAAE,EAAE,GAAG,IAAI,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,QAAQ,CAAC;IAChB,aAAa,EAAE,aAAa,CAAC;IAC7B,KAAK,EAAE,WAAW,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;CAChB,CAAC;AACF,MAAM,MAAM,kBAAkB,GAC3B,CAAC;IAAE,KAAK,EAAE,aAAa,CAAA;CAAE,GAAG,IAAI,CAAC,GACjC,CAAC;IAAE,KAAK,EAAE,UAAU,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,GAClE,CAAC;IAAE,KAAK,EAAE,UAAU,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,CAAC;AACnD,MAAM,MAAM,eAAe,GAAG;IAC7B,MAAM,EAAE,OAAO,GAAG,UAAU,GAAG,UAAU,GAAG,mBAAmB,GAAG,UAAU,CAAC;IAC7E,MAAM,EAAE,MAAM,CAAC;CACf,CAAC;AACF,MAAM,WAAW,aAAa;IAC7B,cAAc,CACb,MAAM,EAAE,WAAW,GAAG,QAAQ,GAAG,UAAU,EAC3C,OAAO,EAAE,EAAE,EACX,OAAO,EAAE,KAAK,EAAE,EAChB,IAAI,EAAE,QAAQ,EACd,GAAG,EAAE,OAAO,GACV,UAAU,CAAC;QAAE,OAAO,EAAE,IAAI,CAAA;KAAE,GAAG;QAAE,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC/E;AACD,eAAO,MAAM,cAAc;;;;;CAAoF,CAAC;AAwBhH,eAAO,MAAM,QAAQ,EAAE,QAAQ,CAC9B,aAAa,EACb,kBAAkB,EAClB;IAAE,OAAO,EAAE,EAAE,CAAA;CAAE,EACf,eAAe,EACf,IAAI,EACJ,aAAa,EACb,OAAO,cAAc,CA4ErB,CAAC;AAwFF,wBAAgB,aAAa,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,EAAE,GAAG,SAAS,CAuBlF","sourcesContent":["import type { Context } from \"@earendil-works/chord\";\nimport type { AssistantMessage } from \"@earendil-works/pi-ai\";\nimport { estimateContextTokens } from \"@earendil-works/pi-ai/utils/estimate\";\nimport { effectiveTools } from \"../system.ts\";\nimport {\n\ttype Completion,\n\ttype CoreKind,\n\ttype CoreTx,\n\ttype Entry,\n\ttype HookInfo,\n\ttype HookResult,\n\ttype Id,\n\ttype ModelRef,\n\ttype RequestMessage,\n\ttype RetryPolicy,\n\ttype Runtime,\n\ttype Step,\n\ttype SystemMessage,\n\ttype Task,\n\ttype ThinkingLevel,\n\ttoStored,\n} from \"../types.ts\";\nimport { retryDecision } from \"./generation.ts\";\n\nconst estimate = (messages: RequestMessage[]) =>\n\testimateContextTokens(\n\t\tmessages.filter((message): message is Exclude<RequestMessage, SystemMessage> => message.role !== \"system\"),\n\t).tokens;\n\nexport type CollapseInput = { reason: \"threshold\" | \"manual\" | \"overflow\"; through: Id; instructions?: string };\ntype Base = {\n\texpectedHead: Id | null;\n\tinstructions?: string;\n\tmodel: ModelRef;\n\tthinkingLevel: ThinkingLevel;\n\tretry: RetryPolicy;\n\tattempt: number;\n};\nexport type CollapseCheckpoint =\n\t| ({ phase: \"summarizing\" } & Base)\n\t| ({ phase: \"retrying\"; untilMs: number; lastError: string } & Base)\n\t| ({ phase: \"prepared\"; summary: string } & Base);\nexport type CollapseFailure = {\n\treason: \"stale\" | \"declined\" | \"provider\" | \"retries_exhausted\" | \"no_model\";\n\tdetail: string;\n};\nexport interface CollapseHooks {\n\tbeforeCollapse(\n\t\treason: \"threshold\" | \"manual\" | \"overflow\",\n\t\tthrough: Id,\n\t\tentries: Entry[],\n\t\tinfo: HookInfo,\n\t\tctx: Context,\n\t): HookResult<{ decline: true } | { instructions?: string; summary?: string }>;\n}\nexport const collapseConfig = { rewindable: { threshold: 0 as number, keepRecent: 20_000 as number } } as const;\n\ntype CollapseTask = Task<CollapseInput, CollapseCheckpoint>;\ntype CollapseRuntime = Runtime<CollapseHooks, CoreTx>;\ntype CollapseStep = Step<CollapseCheckpoint, { summary: Id }, CollapseFailure, CoreTx>;\nconst failed = (reason: CollapseFailure[\"reason\"], detail: string): Completion<{ summary: Id }, CollapseFailure> => ({\n\tstatus: \"failed\",\n\tfailure: { reason, detail },\n});\nconst baseOf = (checkpoint: CollapseCheckpoint): Base => {\n\tconst { phase: _phase, ...rest } = checkpoint;\n\tconst {\n\t\tuntilMs: _untilMs,\n\t\tlastError: _lastError,\n\t\tsummary: _summary,\n\t\t...base\n\t} = rest as Base & {\n\t\tuntilMs?: number;\n\t\tlastError?: string;\n\t\tsummary?: string;\n\t};\n\treturn base;\n};\n\nexport const collapse: CoreKind<\n\tCollapseInput,\n\tCollapseCheckpoint,\n\t{ summary: Id },\n\tCollapseFailure,\n\tnull,\n\tCollapseHooks,\n\ttypeof collapseConfig\n> = {\n\tname: \"pi.collapse\",\n\tconfig: collapseConfig,\n\tinflight: [\"summarizing\"],\n\n\tasync initial(task, runtime, ctx) {\n\t\tconst state = await runtime.rewindable(task.conversationId, ctx);\n\t\tif (state.model === undefined) return { done: () => failed(\"no_model\", \"no model configured\") };\n\t\tconst sticky = await runtime.sticky(task.conversationId, ctx);\n\t\tconst head = await runtime.newestEntry(task.conversationId, { withHead: true }, ctx);\n\t\tconst { entries } = await runtime.context(task.conversationId, task.input.through, ctx);\n\t\tlet decision: { decline: true } | { instructions?: string; summary?: string } = {};\n\t\tawait runtime.hooks.each(\n\t\t\tctx,\n\t\t\t(hook, api) => hook.beforeCollapse?.(task.input.reason, task.input.through, entries, api, ctx),\n\t\t\t(value) => {\n\t\t\t\tif (value === undefined) return;\n\t\t\t\tdecision = value;\n\t\t\t\treturn true;\n\t\t\t},\n\t\t);\n\t\tif (\"decline\" in decision) return { done: () => failed(\"declined\", \"declined by beforeCollapse\") };\n\t\tconst choice: { instructions?: string; summary?: string } = decision;\n\t\tconst base: Base = {\n\t\t\texpectedHead: head?.id ?? null,\n\t\t\t...((choice.instructions ?? task.input.instructions)\n\t\t\t\t? { instructions: choice.instructions ?? task.input.instructions }\n\t\t\t\t: {}),\n\t\t\tmodel: state.model,\n\t\t\tthinkingLevel: state.thinkingLevel,\n\t\t\tretry: sticky.retry,\n\t\t\tattempt: 1,\n\t\t};\n\t\tif (choice.summary !== undefined) {\n\t\t\treturn {\n\t\t\t\tnext: async (tx, current) =>\n\t\t\t\t\t(await headMoved(tx, current.conversationId, base))\n\t\t\t\t\t\t? failed(\"stale\", \"head moved during beforeCollapse\")\n\t\t\t\t\t\t: { phase: \"prepared\", summary: choice.summary!, ...base },\n\t\t\t};\n\t\t}\n\t\treturn summarizeNow(task, base, runtime, ctx, \"check head\");\n\t},\n\n\tphases: {\n\t\tasync summarizing(task, runtime) {\n\t\t\treturn afterFailure(baseOf(task.checkpoint), null, \"interrupted\", runtime);\n\t\t},\n\t\tasync retrying(task, runtime, ctx) {\n\t\t\tawait runtime.sleep(task.checkpoint.untilMs, ctx);\n\t\t\treturn summarizeNow(task, { ...baseOf(task.checkpoint), attempt: task.checkpoint.attempt + 1 }, runtime, ctx);\n\t\t},\n\t\tasync prepared(task, runtime) {\n\t\t\tconst { summary } = task.checkpoint;\n\t\t\tconst base = baseOf(task.checkpoint);\n\t\t\treturn {\n\t\t\t\tdone: async (tx, current) => {\n\t\t\t\t\tconst { head, entries } = await tx.context(current.conversationId);\n\t\t\t\t\tif ((head?.id ?? null) !== base.expectedHead) return failed(\"stale\", \"head moved during collapse\");\n\t\t\t\t\tconst retained = entries.find((entry) => entry.id > task.input.through);\n\t\t\t\t\tconst id = tx.appendEntry(current.conversationId, {\n\t\t\t\t\t\tkind: \"pi.summary\",\n\t\t\t\t\t\tdata: { through: task.input.through },\n\t\t\t\t\t\tmodel: [{ role: \"user\", content: summary, timestamp: runtime.now() }],\n\t\t\t\t\t\thead: retained?.id ?? \"self\",\n\t\t\t\t\t});\n\t\t\t\t\treturn { status: \"completed\", result: { summary: id } };\n\t\t\t\t},\n\t\t\t};\n\t\t},\n\t},\n\n\tasync abort() {\n\t\treturn async () => null;\n\t},\n};\n\nconst headMoved = async (tx: Pick<CoreTx, \"newestEntry\">, conversationId: Id, base: Base) =>\n\t((await tx.newestEntry(conversationId, { withHead: true }))?.id ?? null) !== base.expectedHead;\n\nasync function summarizeNow(\n\ttask: CollapseTask,\n\tbase: Base,\n\truntime: CollapseRuntime,\n\tctx: Context,\n\tcheck?: \"check head\",\n): Promise<CollapseStep> {\n\tconst stale = await runtime.commit(async (tx, current) => {\n\t\tif (check && (await headMoved(tx, current.conversationId, base))) return true;\n\t\ttx.checkpoint({ phase: \"summarizing\", ...base });\n\t\treturn false;\n\t}, ctx);\n\tif (stale) return { done: () => failed(\"stale\", \"head moved during beforeCollapse\") };\n\tconst model = runtime.models.resolve(base.model);\n\tif (model === undefined) return { done: () => failed(\"no_model\", \"model unavailable\") };\n\tconst { messages } = await runtime.context(task.conversationId, task.input.through, ctx);\n\tconst tools = effectiveTools(messages);\n\tconst request: RequestMessage[] = [\n\t\t...messages,\n\t\t...(tools.length > 0\n\t\t\t? [toStored({ role: \"system\", content: \"\", toolsRemoved: tools, timestamp: runtime.now() } as SystemMessage)]\n\t\t\t: []),\n\t\t{\n\t\t\trole: \"user\",\n\t\t\tcontent: `${base.instructions ?? \"Summarize the conversation so far for continuation.\"}\\n\\nRespond with the summary only.`,\n\t\t\ttimestamp: runtime.now(),\n\t\t},\n\t];\n\tlet message: AssistantMessage | undefined;\n\ttry {\n\t\tfor await (const event of runtime.models.stream(\n\t\t\tmodel,\n\t\t\t{ messages: request, thinkingLevel: base.thinkingLevel },\n\t\t\tctx,\n\t\t)) {\n\t\t\tif (event.type === \"done\") {\n\t\t\t\tmessage = event.message;\n\t\t\t\tbreak;\n\t\t\t}\n\t\t\tif (event.type === \"error\") {\n\t\t\t\tmessage = event.error;\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\t} catch (error) {\n\t\tif (ctx.abortSignal?.aborted) throw error;\n\t\treturn afterFailure(base, null, String(error), runtime);\n\t}\n\tif (message === undefined) return afterFailure(base, null, \"no message\", runtime);\n\tif (message.content.some((content) => content.type === \"toolCall\")) {\n\t\treturn afterFailure(base, { ...message, stopReason: \"error\" }, \"summarizer returned tool calls\", runtime);\n\t}\n\tif (message.stopReason === \"error\")\n\t\treturn afterFailure(base, message, message.errorMessage ?? \"provider error\", runtime);\n\tconst summary = message.content\n\t\t.filter((content): content is { type: \"text\"; text: string } => content.type === \"text\")\n\t\t.map((content) => content.text)\n\t\t.join(\"\");\n\treturn { next: { phase: \"prepared\", summary, ...base } };\n}\n\nfunction afterFailure(\n\tbase: Base,\n\tmessage: AssistantMessage | null,\n\tdetail: string,\n\truntime: CollapseRuntime,\n): CollapseStep {\n\tconst decision = retryDecision(base, message, runtime.now());\n\tif (decision.kind === \"fail\") return { done: () => failed(decision.reason, detail) };\n\treturn {\n\t\tnext: (tx, current) => {\n\t\t\ttx.emit({\n\t\t\t\ttype: \"compaction.retrying\",\n\t\t\t\ttaskId: current.id,\n\t\t\t\tattempt: base.attempt,\n\t\t\t\tretryAt: decision.untilMs,\n\t\t\t\terror: detail,\n\t\t\t});\n\t\t\treturn { phase: \"retrying\", ...base, untilMs: decision.untilMs, lastError: detail };\n\t\t},\n\t};\n}\n\nexport function chooseThrough(entries: Entry[], keepRecent: number): Id | undefined {\n\tconst exchanges: { last: Id; tokens: number }[] = [];\n\tlet open: { last: Id; tokens: number } | undefined;\n\tfor (const entry of entries) {\n\t\tconst role = entry.model?.[0]?.role;\n\t\tconst tokens = estimate((entry.model ?? []) as RequestMessage[]);\n\t\tif (role === \"assistant\") {\n\t\t\topen = { last: entry.id, tokens };\n\t\t\texchanges.push(open);\n\t\t} else if (role === \"toolResult\" && open) {\n\t\t\topen.last = entry.id;\n\t\t\topen.tokens += tokens;\n\t\t} else {\n\t\t\topen = undefined;\n\t\t\texchanges.push({ last: entry.id, tokens });\n\t\t}\n\t}\n\tlet retained = 0;\n\tlet index = exchanges.length - 1;\n\twhile (index >= 0 && retained + exchanges[index]!.tokens <= keepRecent) retained += exchanges[index--]!.tokens;\n\tif (index < 0) return undefined;\n\tif (index === exchanges.length - 1) return exchanges[index - 1]?.last;\n\treturn exchanges[index]!.last;\n}\n"]}

package/dist/harness/pico3/kinds/collapse.js

@@ -0,0 +1,191 @@
+import { estimateContextTokens } from "@earendil-works/pi-ai/utils/estimate";
+import { effectiveTools } from "../system.js";
+import { toStored, } from "../types.js";
+import { retryDecision } from "./generation.js";
+const estimate = (messages) => estimateContextTokens(messages.filter((message) => message.role !== "system")).tokens;
+export const collapseConfig = { rewindable: { threshold: 0, keepRecent: 20_000 } };
+const failed = (reason, detail) => ({
+    status: "failed",
+    failure: { reason, detail },
+});
+const baseOf = (checkpoint) => {
+    const { phase: _phase, ...rest } = checkpoint;
+    const { untilMs: _untilMs, lastError: _lastError, summary: _summary, ...base } = rest;
+    return base;
+};
+export const collapse = {
+    name: "pi.collapse",
+    config: collapseConfig,
+    inflight: ["summarizing"],
+    async initial(task, runtime, ctx) {
+        const state = await runtime.rewindable(task.conversationId, ctx);
+        if (state.model === undefined)
+            return { done: () => failed("no_model", "no model configured") };
+        const sticky = await runtime.sticky(task.conversationId, ctx);
+        const head = await runtime.newestEntry(task.conversationId, { withHead: true }, ctx);
+        const { entries } = await runtime.context(task.conversationId, task.input.through, ctx);
+        let decision = {};
+        await runtime.hooks.each(ctx, (hook, api) => hook.beforeCollapse?.(task.input.reason, task.input.through, entries, api, ctx), (value) => {
+            if (value === undefined)
+                return;
+            decision = value;
+            return true;
+        });
+        if ("decline" in decision)
+            return { done: () => failed("declined", "declined by beforeCollapse") };
+        const choice = decision;
+        const base = {
+            expectedHead: head?.id ?? null,
+            ...((choice.instructions ?? task.input.instructions)
+                ? { instructions: choice.instructions ?? task.input.instructions }
+                : {}),
+            model: state.model,
+            thinkingLevel: state.thinkingLevel,
+            retry: sticky.retry,
+            attempt: 1,
+        };
+        if (choice.summary !== undefined) {
+            return {
+                next: async (tx, current) => (await headMoved(tx, current.conversationId, base))
+                    ? failed("stale", "head moved during beforeCollapse")
+                    : { phase: "prepared", summary: choice.summary, ...base },
+            };
+        }
+        return summarizeNow(task, base, runtime, ctx, "check head");
+    },
+    phases: {
+        async summarizing(task, runtime) {
+            return afterFailure(baseOf(task.checkpoint), null, "interrupted", runtime);
+        },
+        async retrying(task, runtime, ctx) {
+            await runtime.sleep(task.checkpoint.untilMs, ctx);
+            return summarizeNow(task, { ...baseOf(task.checkpoint), attempt: task.checkpoint.attempt + 1 }, runtime, ctx);
+        },
+        async prepared(task, runtime) {
+            const { summary } = task.checkpoint;
+            const base = baseOf(task.checkpoint);
+            return {
+                done: async (tx, current) => {
+                    const { head, entries } = await tx.context(current.conversationId);
+                    if ((head?.id ?? null) !== base.expectedHead)
+                        return failed("stale", "head moved during collapse");
+                    const retained = entries.find((entry) => entry.id > task.input.through);
+                    const id = tx.appendEntry(current.conversationId, {
+                        kind: "pi.summary",
+                        data: { through: task.input.through },
+                        model: [{ role: "user", content: summary, timestamp: runtime.now() }],
+                        head: retained?.id ?? "self",
+                    });
+                    return { status: "completed", result: { summary: id } };
+                },
+            };
+        },
+    },
+    async abort() {
+        return async () => null;
+    },
+};
+const headMoved = async (tx, conversationId, base) => ((await tx.newestEntry(conversationId, { withHead: true }))?.id ?? null) !== base.expectedHead;
+async function summarizeNow(task, base, runtime, ctx, check) {
+    const stale = await runtime.commit(async (tx, current) => {
+        if (check && (await headMoved(tx, current.conversationId, base)))
+            return true;
+        tx.checkpoint({ phase: "summarizing", ...base });
+        return false;
+    }, ctx);
+    if (stale)
+        return { done: () => failed("stale", "head moved during beforeCollapse") };
+    const model = runtime.models.resolve(base.model);
+    if (model === undefined)
+        return { done: () => failed("no_model", "model unavailable") };
+    const { messages } = await runtime.context(task.conversationId, task.input.through, ctx);
+    const tools = effectiveTools(messages);
+    const request = [
+        ...messages,
+        ...(tools.length > 0
+            ? [toStored({ role: "system", content: "", toolsRemoved: tools, timestamp: runtime.now() })]
+            : []),
+        {
+            role: "user",
+            content: `${base.instructions ?? "Summarize the conversation so far for continuation."}\n\nRespond with the summary only.`,
+            timestamp: runtime.now(),
+        },
+    ];
+    let message;
+    try {
+        for await (const event of runtime.models.stream(model, { messages: request, thinkingLevel: base.thinkingLevel }, ctx)) {
+            if (event.type === "done") {
+                message = event.message;
+                break;
+            }
+            if (event.type === "error") {
+                message = event.error;
+                break;
+            }
+        }
+    }
+    catch (error) {
+        if (ctx.abortSignal?.aborted)
+            throw error;
+        return afterFailure(base, null, String(error), runtime);
+    }
+    if (message === undefined)
+        return afterFailure(base, null, "no message", runtime);
+    if (message.content.some((content) => content.type === "toolCall")) {
+        return afterFailure(base, { ...message, stopReason: "error" }, "summarizer returned tool calls", runtime);
+    }
+    if (message.stopReason === "error")
+        return afterFailure(base, message, message.errorMessage ?? "provider error", runtime);
+    const summary = message.content
+        .filter((content) => content.type === "text")
+        .map((content) => content.text)
+        .join("");
+    return { next: { phase: "prepared", summary, ...base } };
+}
+function afterFailure(base, message, detail, runtime) {
+    const decision = retryDecision(base, message, runtime.now());
+    if (decision.kind === "fail")
+        return { done: () => failed(decision.reason, detail) };
+    return {
+        next: (tx, current) => {
+            tx.emit({
+                type: "compaction.retrying",
+                taskId: current.id,
+                attempt: base.attempt,
+                retryAt: decision.untilMs,
+                error: detail,
+            });
+            return { phase: "retrying", ...base, untilMs: decision.untilMs, lastError: detail };
+        },
+    };
+}
+export function chooseThrough(entries, keepRecent) {
+    const exchanges = [];
+    let open;
+    for (const entry of entries) {
+        const role = entry.model?.[0]?.role;
+        const tokens = estimate((entry.model ?? []));
+        if (role === "assistant") {
+            open = { last: entry.id, tokens };
+            exchanges.push(open);
+        }
+        else if (role === "toolResult" && open) {
+            open.last = entry.id;
+            open.tokens += tokens;
+        }
+        else {
+            open = undefined;
+            exchanges.push({ last: entry.id, tokens });
+        }
+    }
+    let retained = 0;
+    let index = exchanges.length - 1;
+    while (index >= 0 && retained + exchanges[index].tokens <= keepRecent)
+        retained += exchanges[index--].tokens;
+    if (index < 0)
+        return undefined;
+    if (index === exchanges.length - 1)
+        return exchanges[index - 1]?.last;
+    return exchanges[index].last;
+}
+//# sourceMappingURL=collapse.js.map