@adecore/agents 0.41.1 → 0.43.0

package/dist/usage/usage-store.d.ts

@@ -0,0 +1,26 @@
+import { type FileUsage } from './history-file.ts';
+import { type UsageRecord } from './record.ts';
+import { type ScanState } from './scan-state.ts';
+export declare class UsageStore {
+    readonly usage: Map<string, FileUsage>;
+    readonly state: Map<string, ScanState>;
+    private readonly dir;
+    private pending;
+    private stateDirty;
+    private gen;
+    private base;
+    private bytes;
+    private rewriteNext;
+    constructor(dir: string);
+    load(): Promise<void>;
+    records(): UsageRecord[];
+    replace(path: string, usage: FileUsage, state: ScanState): void;
+    append(path: string, added: FileUsage, state: ScanState): void;
+    forget(path: string): void;
+    commit(): Promise<void>;
+    private setState;
+    private appendPending;
+    private rewrite;
+    private saveState;
+    private migrate;
+}

package/dist/usage/usage-store.js

@@ -0,0 +1,197 @@
+import { randomUUID } from 'node:crypto';
+import { isDeepStrictEqual } from 'node:util';
+import { appendFile, mkdir, readFile, rename, truncate } from 'node:fs/promises';
+import { join } from 'node:path';
+import { errorText } from '../error-text.js';
+import { isNotFound, removeFile, writeAtomic } from '../fs.js';
+import { applyEntry, decodeHistory, encodeEntry, encodeHeader } from './history-file.js';
+import { decodeIndex } from './index-file.js';
+import { foldByKey } from './record.js';
+import { decodeState, encodeState } from './scan-state.js';
+const HISTORY_FILE = 'history.jsonl';
+const STATE_FILE = 'scan-state.json';
+const LEGACY_INDEX_FILE = 'index.json';
+// Below this, appended lines are not worth a rewrite of the whole history.
+const COMPACT_FLOOR_BYTES = 1 << 20;
+async function readOptional(path) {
+    try {
+        return await readFile(path);
+    }
+    catch (e) {
+        if (isNotFound(e)) {
+            return null;
+        }
+        throw e;
+    }
+}
+/*
+ * The usage of every transcript ever read, kept apart from where the scanner left each one. The state
+ * is saved after the lines it accounts for, against the history's size, so after a crash in between it
+ * does not match: it is dropped and every transcript is read from the start again, which replaces what
+ * a path counted instead of adding to it.
+ */
+export class UsageStore {
+    usage = new Map();
+    state = new Map();
+    dir;
+    pending = [];
+    stateDirty = false;
+    gen = null;
+    base = 0;
+    bytes = 0;
+    // Set when the history on disk may not end where this store thinks it does.
+    rewriteNext = false;
+    constructor(dir) {
+        this.dir = dir;
+    }
+    async load() {
+        this.usage.clear();
+        this.state.clear();
+        const history = await readOptional(join(this.dir, HISTORY_FILE));
+        if (history === null) {
+            await this.migrate();
+            return;
+        }
+        const decoded = decodeHistory(history);
+        for (const [path, usage] of decoded.usage) {
+            this.usage.set(path, usage);
+        }
+        if (decoded.complete < history.length) {
+            await truncate(join(this.dir, HISTORY_FILE), decoded.complete);
+        }
+        this.gen = decoded.header?.gen ?? null;
+        this.base = decoded.header?.base ?? 0;
+        this.bytes = decoded.complete;
+        this.rewriteNext = this.gen === null;
+        const saved = await readOptional(join(this.dir, STATE_FILE));
+        const state = saved === null ? null : decodeState(saved.toString('utf8'));
+        if (state !== null && state.history.gen === this.gen && state.history.bytes === this.bytes) {
+            for (const [path, file] of state.files) {
+                this.state.set(path, file);
+            }
+        }
+        // A migration that stopped after writing the history; the index has nothing the history lacks.
+        await removeFile(join(this.dir, LEGACY_INDEX_FILE));
+    }
+    records() {
+        const all = [];
+        for (const file of this.usage.values()) {
+            all.push(...file.records, ...file.tail);
+        }
+        return foldByKey(all);
+    }
+    /* A transcript read from the start: what it counts now replaces what it counted. */
+    replace(path, usage, state) {
+        this.setState(path, state);
+        if (isDeepStrictEqual(this.usage.get(path), usage)) {
+            return;
+        }
+        this.usage.set(path, usage);
+        this.pending.push({ path, op: 'set', usage });
+    }
+    /* A transcript read from where it was left: `added` counts on top of what it had. */
+    append(path, added, state) {
+        this.setState(path, state);
+        const known = this.usage.get(path);
+        if (added.records.length === 0 && known?.provider === added.provider && isDeepStrictEqual(known.tail, added.tail)) {
+            return;
+        }
+        const entry = { path, op: 'add', usage: { ...added, records: foldByKey(added.records) } };
+        applyEntry(this.usage, entry);
+        this.pending.push(entry);
+    }
+    /* Where the scanner left a transcript that is gone; what it counted stays. */
+    forget(path) {
+        if (this.state.delete(path)) {
+            this.stateDirty = true;
+        }
+    }
+    async commit() {
+        if (this.pending.length === 0 && !this.stateDirty) {
+            return;
+        }
+        await mkdir(this.dir, { recursive: true, mode: 0o700 });
+        try {
+            if (this.rewriteNext || (this.pending.length > 0 && this.gen === null)) {
+                await this.rewrite();
+            }
+            else if (this.pending.length > 0) {
+                await this.appendPending();
+            }
+            await this.saveState();
+        }
+        catch (e) {
+            // What failed to land is in memory, and a rewrite puts all of it on disk.
+            this.pending = [];
+            this.rewriteNext = true;
+            throw e;
+        }
+    }
+    setState(path, state) {
+        this.state.set(path, state);
+        this.stateDirty = true;
+    }
+    async appendPending() {
+        const text = this.pending.map(encodeEntry).join('');
+        this.pending = [];
+        await appendFile(join(this.dir, HISTORY_FILE), text, { mode: 0o600 });
+        this.bytes += Buffer.byteLength(text);
+        if (this.bytes > 2 * this.base && this.bytes - this.base > COMPACT_FLOOR_BYTES) {
+            await this.rewrite();
+        }
+    }
+    /* The history as one `set` line per path, which drops every line a later one superseded. */
+    async rewrite() {
+        const lines = [...this.usage].map(([path, usage]) => encodeEntry({ path, op: 'set', usage }));
+        const gen = randomUUID();
+        const body = lines.join('');
+        const base = Buffer.byteLength(body);
+        const header = encodeHeader({ gen, base });
+        this.pending = [];
+        await writeAtomic(join(this.dir, HISTORY_FILE), header + body);
+        this.gen = gen;
+        this.base = base;
+        this.bytes = Buffer.byteLength(header + body);
+        this.rewriteNext = false;
+    }
+    async saveState() {
+        if (this.gen === null) {
+            return;
+        }
+        await writeAtomic(join(this.dir, STATE_FILE), encodeState(this.state, { gen: this.gen, bytes: this.bytes }));
+        this.stateDirty = false;
+    }
+    /*
+     * Moves an index of the single-file format into history and scan state. The history goes first:
+     * once it is there, a load treats the index as moved, so an interrupted migration finishes on the next load.
+     */
+    async migrate() {
+        const legacy = join(this.dir, LEGACY_INDEX_FILE);
+        const text = await readOptional(legacy);
+        if (text === null) {
+            return;
+        }
+        const index = decodeIndex(text.toString('utf8'));
+        if (index === null) {
+            console.warn('The usage index would not read; it is kept beside a fresh history.');
+            await rename(legacy, `${legacy}.unreadable`);
+            return;
+        }
+        for (const [path, file] of index) {
+            const { records, tail, provider, ...state } = file;
+            this.usage.set(path, { provider, records, tail });
+            this.state.set(path, { provider, ...state });
+        }
+        try {
+            await this.rewrite();
+            await this.saveState();
+            await removeFile(legacy);
+        }
+        catch (e) {
+            // The index stays and is moved again on the next load; this run counts from memory.
+            this.rewriteNext = true;
+            this.stateDirty = true;
+            console.warn('The usage index could not be moved to the history yet:', errorText(e));
+        }
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adecore/agents",
-  "version": "0.41.1",
+  "version": "0.43.0",
   "description": "Node-compatible agent chat hosts, providers, accounts, usage and durable task coordination.",
   "type": "module",
   "exports": {
@@ -93,8 +93,8 @@
     "test": "bun --conditions=source test"
   },
   "dependencies": {
-    "@adecore/agent-contracts": "0.41.1",
-    "@adecore/intelligent-ui": "0.41.1",
+    "@adecore/agent-contracts": "0.43.0",
+    "@adecore/intelligent-ui": "0.43.0",
     "zod": "^4.6.7"
   },
   "devDependencies": {

package/src/chat/chat-core.ts

@@ -1148,9 +1148,31 @@ export class ChatCore {
         }
     }
 
-    /* What a turn changed against the tree it started from; null when it has no checkpoint. */
-    turnDiff(chatId: string, turnId: string): Promise<ChatCheckpointDiff | null> {
-        return this.require(chatId).turnDiff(turnId);
+    /* What a turn changed against the tree it started from, whole also where the thread keeps only its first diffs; null when it has no checkpoint. */
+    async turnDiff(chatId: string, turnId: string): Promise<ChatCheckpointDiff | null> {
+        const session = this.require(chatId);
+        const turn = session.thread.get(turnId);
+
+        if (turn?.kind === 'turn' && turn.checkpointDiffRef && this.store) {
+            return await this.store.readDiff(chatId, turn.checkpointDiffRef.id);
+        }
+
+        return await session.turnDiff(turnId);
+    }
+
+    /* The whole of what a tool call answered, also where the thread keeps only its beginning. */
+    async toolOutput(chatId: string, itemId: string): Promise<string | null> {
+        const tool = this.require(chatId).thread.get(itemId);
+
+        if (tool?.kind !== 'tool') {
+            throw new ChatError('item-not-found', `No tool call ${itemId} in chat ${chatId}`);
+        }
+
+        if (!tool.outputRef || !this.store) {
+            return tool.output;
+        }
+
+        return await this.store.readPayload(chatId, tool.outputRef.id);
     }
 
     approve(chatId: string, requestId: string, decision: 'allow' | 'allow-always' | 'deny', message?: string): void {
@@ -1575,6 +1597,8 @@ export class ChatCore {
             // A chat killed while the write was out has no log left to fold.
             if (this.logs.get(chatId) === log && folds) {
                 log.compact(at.seq);
+                // With the log folded, the record alone says which payloads the thread still needs.
+                await this.store.prunePayloads(chatId);
             }
         });
 

package/src/chat/chat-store.ts

@@ -1,13 +1,14 @@
 import { mkdirSync } from 'node:fs';
 import { mkdir, open, readdir, readFile, rm, stat } from 'node:fs/promises';
 import { join } from 'node:path';
-import { ChatInfoSchema, ChatItemSchema, type ChatInfo, type ChatItem } from '@adecore/agent-contracts';
+import { ChatInfoSchema, ChatItemSchema, type ChatCheckpointDiff, type ChatInfo, type ChatItem } from '@adecore/agent-contracts';
 import { z } from 'zod';
 import { isNotFound, writeAtomic, writeAtomicSync } from '../fs.ts';
 import { recordFileName } from '../record-directory.ts';
 import { migrateInlineAttachments, type AttachmentStore } from './attachment-store.ts';
 import { isBookmarkFileName } from './bookmark-store.ts';
 import { parseLog, type ChatLogLine } from './chat-log.ts';
+import { compactItems, PayloadStore, type PayloadSources } from './payload-store.ts';
 import { ChatThread } from './thread.ts';
 import { isVisualFileName } from './visual-store.ts';
 
@@ -103,17 +104,23 @@ export interface ChatStoreOptions {
 
 /*
  * One JSON file per chat under `<home>/chats`, written while a turn runs and on shutdown, with
- * the log of what happened since beside it (`ChatLog` appends to it; reading plays it back).
+ * the log of what happened since beside it (`ChatLog` appends to it; reading plays it back). An
+ * output or diff past the inline limit goes to the chat's payload folder, and the record keeps
+ * its beginning (`PayloadStore`).
  */
 export class ChatStore {
     readonly dir: string;
     private readonly attachments: AttachmentStore | null;
     private readonly isSidecar: (name: string) => boolean;
+    private readonly payloads: PayloadStore;
+    // The payloads the record this process wrote last refers to, which is what a prune keeps.
+    private readonly referenced = new Map<string, Set<string>>();
 
     constructor(home: string, options: ChatStoreOptions = {}) {
         this.dir = join(home, 'chats');
         this.attachments = options.attachments ?? null;
         this.isSidecar = options.isSidecar ?? (() => false);
+        this.payloads = new PayloadStore(this.dir);
     }
 
     logPath(chatId: string): string {
@@ -169,9 +176,13 @@ export class ChatStore {
         extras: ChatRecordExtras = {},
         uiAccess: Readonly<Record<string, unknown>> = {}
     ): Promise<number> {
-        const body = recordBody(info, items, at, preambles, extras, uiAccess);
+        const compacted = compactItems(items);
         await mkdir(this.dir, { recursive: true, mode: 0o700 });
+        // Before the record, so a record on disk never names a payload that is not.
+        await this.payloads.ensure(chatId, compacted.payloads);
+        const body = recordBody(info, compacted.items, at, preambles, extras, uiAccess);
         await writeAtomic(join(this.dir, recordFileName(chatId)), body);
+        this.referenced.set(chatId, new Set(compacted.payloads.keys()));
 
         return body.length;
     }
@@ -189,8 +200,32 @@ export class ChatStore {
         extras: ChatRecordExtras = {},
         uiAccess: Readonly<Record<string, unknown>> = {}
     ): void {
+        const compacted = compactItems(items);
         mkdirSync(this.dir, { recursive: true, mode: 0o700 });
-        writeAtomicSync(join(this.dir, recordFileName(chatId)), recordBody(info, items, at, preambles, extras, uiAccess));
+        this.payloads.ensureSync(chatId, compacted.payloads);
+        writeAtomicSync(join(this.dir, recordFileName(chatId)), recordBody(info, compacted.items, at, preambles, extras, uiAccess));
+        this.referenced.set(chatId, new Set(compacted.payloads.keys()));
+    }
+
+    /* The whole of a tool output or a diff kept apart from the record, by the id its item's ref carries. */
+    readPayload(chatId: string, id: string): Promise<string> {
+        return this.payloads.read(chatId, id);
+    }
+
+    readDiff(chatId: string, id: string): Promise<ChatCheckpointDiff> {
+        return this.payloads.readDiff(chatId, id);
+    }
+
+    /*
+     * Removes the payloads the record written last no longer refers to, such as everything a clear
+     * emptied. Nothing before this process wrote the record, since only that write says what it holds.
+     */
+    async prunePayloads(chatId: string): Promise<void> {
+        const keep = this.referenced.get(chatId);
+
+        if (keep) {
+            await this.payloads.prune(chatId, keep);
+        }
     }
 
     /* Null for a chat with nothing on disk to make it of, and for one whose record does not parse (see `unreadable`). */
@@ -203,7 +238,9 @@ export class ChatStore {
         }
 
         if (snapshot === null) {
-            return fromLogAlone(lines);
+            const rebuilt = fromLogAlone(lines);
+
+            return rebuilt && { ...rebuilt, items: await this.compact(chatId, rebuilt.items) };
         }
 
         const seq = snapshot.seq ?? 0;
@@ -223,8 +260,12 @@ export class ChatStore {
             }
         }
 
+        const { info, items } = thread.snapshot();
+
         return {
-            ...thread.snapshot(),
+            info,
+            // A large output the log still holds whole goes apart now, so an open thread never holds one.
+            items: await this.compact(chatId, items),
             seq,
             resetSeq,
             preambles: snapshot.preambles ?? [],
@@ -284,16 +325,50 @@ export class ChatStore {
 
         const parsed = RecordSchema.safeParse(liftUiAccess(record));
 
-        return parsed.success ? parsed.data : UNREADABLE;
+        if (!parsed.success) {
+            return UNREADABLE;
+        }
+
+        const compacted = compactItems(parsed.data.items);
+
+        if (compacted.items.every((item, index) => item === parsed.data.items[index])) {
+            return parsed.data;
+        }
+
+        // A record from before payloads were kept apart moves its large ones out once. The payloads land
+        // first and the record is replaced whole, so a move cut short leaves the old record as it was.
+        const migrated = { ...parsed.data, items: compacted.items };
+        await this.payloads.ensure(chatId, onlyWritten(compacted.payloads));
+        await writeAtomic(join(this.dir, recordFileName(chatId)), JSON.stringify(migrated));
+
+        return migrated;
+    }
+
+    private async compact(chatId: string, items: ChatItem[]): Promise<ChatItem[]> {
+        const compacted = compactItems(items);
+        await this.payloads.ensure(chatId, onlyWritten(compacted.payloads));
+
+        return compacted.items;
     }
 
     async delete(chatId: string): Promise<void> {
-        await Promise.all([rm(join(this.dir, recordFileName(chatId)), { force: true }), rm(this.logPath(chatId), { force: true })]);
+        this.referenced.delete(chatId);
+
+        await Promise.all([
+            rm(join(this.dir, recordFileName(chatId)), { force: true }),
+            rm(this.logPath(chatId), { force: true }),
+            this.payloads.remove(chatId)
+        ]);
     }
 }
 
 const UNREADABLE = Symbol('unreadable');
 
+/* The payloads this process holds the text of; a read leaves the ones a record already refers to for the next write to check. */
+function onlyWritten(payloads: PayloadSources): PayloadSources {
+    return new Map([...payloads].filter(([, source]) => source !== null));
+}
+
 const RECORD_HEAD = Buffer.from('{"info":{');
 const HEAD_CHUNK_BYTES = 16384;
 const QUOTE = 0x22;

package/src/chat/errors.ts

@@ -23,6 +23,8 @@ export type ChatErrorCode =
     | 'transcript-format'
     | 'fork-failed'
     | 'turn-not-found'
+    // An output or diff stored apart from the thread that is no longer on disk.
+    | 'payload-missing'
     // What a bookmark is refused with.
     | 'item-not-found'
     | 'bookmark-not-found'