@adecore/agents 0.41.1 → 0.43.0

package/src/chat/payload-store.ts

@@ -0,0 +1,459 @@
+import { createHash } from 'node:crypto';
+import { constants, copyFileSync, mkdirSync, readdirSync, statSync } from 'node:fs';
+import { copyFile, mkdir, readdir, readFile, rm, stat } from 'node:fs/promises';
+import { join } from 'node:path';
+import { promisify } from 'node:util';
+import { gunzip, gzip, gzipSync } from 'node:zlib';
+import { CHAT_PAYLOAD_INLINE_BYTES, ChatCheckpointDiffSchema, type ChatCheckpointDiff, type ChatItem, type ChatPayloadRef } from '@adecore/agent-contracts';
+import { isNotFound, writeAtomic, writeAtomicSync } from '../fs.ts';
+import { ChatError } from './errors.ts';
+
+/* How much of a value stored apart the item keeps: what a timeline shows before anyone asks for the rest. */
+export const PAYLOAD_PREVIEW_BYTES = 8 * 1024;
+
+const FOLDER_SUFFIX = '.payloads';
+const ID_PATTERN = /^[0-9a-f]{64}$/;
+const CONTINUATION_MASK = 0xc0;
+const CONTINUATION = 0x80;
+
+const gzipAsync = promisify(gzip);
+const gunzipAsync = promisify(gunzip);
+
+/* What a thread needs on disk beside its record: each payload by id, with its text when this process still holds it. */
+export type PayloadSources = Map<string, (() => string) | null>;
+
+export interface CompactedItems {
+    items: ChatItem[];
+    payloads: PayloadSources;
+}
+
+interface Compacted {
+    item: ChatItem;
+    payloads: [string, (() => string) | null][];
+}
+
+// Items never change in place, so an item compacted once answers the same for every write of every chat after it.
+const compactedItems = new WeakMap<ChatItem, Compacted>();
+
+/* The folder of a chat's payloads beside its record, which `ChatStore.list` passes over since it is no JSON. */
+export function payloadFolderName(chatId: string): string {
+    return `${encodeURIComponent(chatId)}${FOLDER_SUFFIX}`;
+}
+
+function payloadId(text: string): string {
+    return createHash('sha256').update(text).digest('hex');
+}
+
+function byteLength(text: string): number {
+    return Buffer.byteLength(text, 'utf8');
+}
+
+/* Whether a text is past the inline limit, without encoding the short ones: a UTF-16 unit is at most three bytes. */
+function pastInline(text: string): boolean {
+    return text.length * 3 > CHAT_PAYLOAD_INLINE_BYTES && byteLength(text) > CHAT_PAYLOAD_INLINE_BYTES;
+}
+
+/*
+ * The beginning of a text in at most `maxBytes` of UTF-8, never cut inside a character, and cut after
+ * the last line break when that keeps at least half of it, so a preview ends on a whole line.
+ */
+export function previewOf(text: string, maxBytes: number): string {
+    let units = Math.min(text.length, maxBytes);
+
+    // Half a surrogate pair would encode as a replacement character.
+    if (units < text.length && isHighSurrogate(text.charCodeAt(units - 1))) {
+        units -= 1;
+    }
+
+    const head = Buffer.from(text.slice(0, units), 'utf8');
+    let end = Math.min(head.length, maxBytes);
+
+    while (end > 0 && end < head.length && (head[end]! & CONTINUATION_MASK) === CONTINUATION) {
+        end -= 1;
+    }
+
+    const prefix = head.subarray(0, end).toString('utf8');
+
+    if (prefix.length === text.length) {
+        return text;
+    }
+
+    const lineEnd = prefix.lastIndexOf('\n');
+
+    return lineEnd >= prefix.length / 2 ? prefix.slice(0, lineEnd + 1) : prefix;
+}
+
+function isHighSurrogate(code: number): boolean {
+    return code >= 0xd800 && code <= 0xdbff;
+}
+
+/* Every file of a diff, with the diffs that fit the preview in order and the rest held back as empty. */
+function diffPreview(diff: ChatCheckpointDiff): ChatCheckpointDiff {
+    let budget = PAYLOAD_PREVIEW_BYTES;
+
+    const files = diff.files.map((file) => {
+        const size = byteLength(file.diff);
+
+        if (size <= budget) {
+            budget -= size;
+
+            return file;
+        }
+
+        budget = 0;
+
+        return { ...file, diff: '' };
+    });
+
+    return { ...diff, files };
+}
+
+function compactOne(item: ChatItem): Compacted {
+    if (item.kind === 'tool') {
+        if (item.outputRef) {
+            return { item, payloads: [[item.outputRef.id, null]] };
+        }
+
+        const output = item.output;
+
+        if (output === null || !pastInline(output)) {
+            return { item, payloads: [] };
+        }
+
+        const ref: ChatPayloadRef = { id: payloadId(output), bytes: byteLength(output) };
+
+        return { item: { ...item, output: previewOf(output, PAYLOAD_PREVIEW_BYTES), outputRef: ref }, payloads: [[ref.id, () => output]] };
+    }
+
+    if (item.kind === 'turn' && item.checkpointDiff) {
+        if (item.checkpointDiffRef) {
+            return { item, payloads: [[item.checkpointDiffRef.id, null]] };
+        }
+
+        const json = JSON.stringify(item.checkpointDiff);
+
+        if (!pastInline(json)) {
+            return { item, payloads: [] };
+        }
+
+        const ref: ChatPayloadRef = { id: payloadId(json), bytes: byteLength(json) };
+        const diff = item.checkpointDiff;
+
+        return { item: { ...item, checkpointDiff: diffPreview(diff), checkpointDiffRef: ref }, payloads: [[ref.id, () => JSON.stringify(diff)]] };
+    }
+
+    return { item, payloads: [] };
+}
+
+/*
+ * A thread with every output and diff past the inline limit cut to its beginning, and what has to be
+ * on disk for the rest. A payload is named by the hash of its text, so the same text written twice, or
+ * copied in from another chat, is one file.
+ */
+export function compactItems(items: readonly ChatItem[]): CompactedItems {
+    const payloads: PayloadSources = new Map();
+
+    const compacted = items.map((item) => {
+        let known = compactedItems.get(item);
+
+        if (known === undefined) {
+            known = compactOne(item);
+            compactedItems.set(item, known);
+        }
+
+        for (const [id, source] of known.payloads) {
+            if (source !== null || !payloads.has(id)) {
+                payloads.set(id, source);
+            }
+        }
+
+        return known.item;
+    });
+
+    return { items: compacted, payloads };
+}
+
+export function payloadRefsOf(items: readonly ChatItem[]): Set<string> {
+    const ids = new Set<string>();
+
+    for (const item of items) {
+        if (item.kind === 'tool' && item.outputRef) {
+            ids.add(item.outputRef.id);
+        } else if (item.kind === 'turn' && item.checkpointDiffRef) {
+            ids.add(item.checkpointDiffRef.id);
+        }
+    }
+
+    return ids;
+}
+
+/*
+ * The outputs and diffs of every chat that are too large to keep in its record, one gzip file per
+ * payload under `<chats>/<chat id>.payloads/<sha-256>.gz`. Each chat has its own folder, so removing
+ * a chat never reaches into a fork that copied from it.
+ */
+export class PayloadStore {
+    private readonly dir: string;
+    // What this process knows is on disk per chat, so a write checks a payload once and not every time.
+    private readonly present = new Map<string, Set<string>>();
+    // What was written while a prune ran, which that prune must not take away again.
+    private readonly writtenDuringPrune = new Map<string, Set<string>>();
+
+    constructor(dir: string) {
+        this.dir = dir;
+    }
+
+    folder(chatId: string): string {
+        return join(this.dir, payloadFolderName(chatId));
+    }
+
+    /* Puts every payload on disk that is not there yet: from its text, or copied from another chat that has it. */
+    async ensure(chatId: string, payloads: PayloadSources): Promise<void> {
+        const present = this.presentIn(chatId);
+        const folder = this.folder(chatId);
+        let others: string[] | null = null;
+
+        for (const [id, source] of payloads) {
+            if (present.has(id)) {
+                continue;
+            }
+
+            const path = join(folder, `${id}.gz`);
+
+            if (await existsAsync(path)) {
+                this.mark(chatId, id);
+
+                continue;
+            }
+
+            await mkdir(folder, { recursive: true, mode: 0o700 });
+
+            if (source !== null) {
+                await writeAtomic(path, await gzipAsync(source()));
+                this.mark(chatId, id);
+
+                continue;
+            }
+
+            others ??= await this.otherFolders(chatId);
+
+            if (await copyFromAny(others, id, path)) {
+                this.mark(chatId, id);
+            }
+        }
+    }
+
+    /* `ensure` without a turn of the event loop, for a write on the way out. */
+    ensureSync(chatId: string, payloads: PayloadSources): void {
+        const present = this.presentIn(chatId);
+        const folder = this.folder(chatId);
+        let others: string[] | null = null;
+
+        for (const [id, source] of payloads) {
+            if (present.has(id)) {
+                continue;
+            }
+
+            const path = join(folder, `${id}.gz`);
+
+            if (existsSync(path)) {
+                this.mark(chatId, id);
+
+                continue;
+            }
+
+            mkdirSync(folder, { recursive: true, mode: 0o700 });
+
+            if (source !== null) {
+                writeAtomicSync(path, gzipSync(source()));
+                this.mark(chatId, id);
+
+                continue;
+            }
+
+            others ??= this.otherFoldersSync(chatId);
+
+            if (copyFromAnySync(others, id, path)) {
+                this.mark(chatId, id);
+            }
+        }
+    }
+
+    async read(chatId: string, id: string): Promise<string> {
+        if (!ID_PATTERN.test(id)) {
+            throw new ChatError('payload-missing', `No stored payload ${id} in chat ${chatId}`);
+        }
+
+        let packed: Buffer;
+
+        try {
+            packed = await readFile(join(this.folder(chatId), `${id}.gz`));
+        } catch (e) {
+            if (isNotFound(e)) {
+                throw new ChatError('payload-missing', `The stored payload ${id} of chat ${chatId} is gone`);
+            }
+
+            throw e;
+        }
+
+        return (await gunzipAsync(packed)).toString('utf8');
+    }
+
+    async readDiff(chatId: string, id: string): Promise<ChatCheckpointDiff> {
+        const parsed = ChatCheckpointDiffSchema.safeParse(JSON.parse(await this.read(chatId, id)));
+
+        if (!parsed.success) {
+            throw new ChatError('payload-missing', `The stored diff ${id} of chat ${chatId} does not read`);
+        }
+
+        return parsed.data;
+    }
+
+    /* Removes the payloads of a chat its thread no longer refers to, such as everything before a clear. */
+    async prune(chatId: string, keep: ReadonlySet<string>): Promise<void> {
+        const folder = this.folder(chatId);
+        let names: string[];
+
+        try {
+            names = await readdir(folder);
+        } catch (e) {
+            if (isNotFound(e)) {
+                return;
+            }
+
+            throw e;
+        }
+
+        const written = new Set<string>();
+        this.writtenDuringPrune.set(chatId, written);
+
+        try {
+            for (const name of names) {
+                const id = name.replace(/\.gz$/, '');
+
+                if (keep.has(id) || written.has(id)) {
+                    continue;
+                }
+
+                this.present.get(chatId)?.delete(id);
+                await rm(join(folder, name), { force: true });
+            }
+        } finally {
+            this.writtenDuringPrune.delete(chatId);
+        }
+    }
+
+    async remove(chatId: string): Promise<void> {
+        this.present.delete(chatId);
+        await rm(this.folder(chatId), { recursive: true, force: true });
+    }
+
+    private presentIn(chatId: string): Set<string> {
+        let present = this.present.get(chatId);
+
+        if (!present) {
+            present = new Set();
+            this.present.set(chatId, present);
+        }
+
+        return present;
+    }
+
+    private mark(chatId: string, id: string): void {
+        this.presentIn(chatId).add(id);
+        this.writtenDuringPrune.get(chatId)?.add(id);
+    }
+
+    private async otherFolders(chatId: string): Promise<string[]> {
+        try {
+            return foldersOf(this.dir, await readdir(this.dir), chatId);
+        } catch (e) {
+            if (isNotFound(e)) {
+                return [];
+            }
+
+            throw e;
+        }
+    }
+
+    private otherFoldersSync(chatId: string): string[] {
+        try {
+            return foldersOf(this.dir, readdirSync(this.dir), chatId);
+        } catch (e) {
+            if (isNotFound(e)) {
+                return [];
+            }
+
+            throw e;
+        }
+    }
+}
+
+function foldersOf(dir: string, names: readonly string[], chatId: string): string[] {
+    const own = payloadFolderName(chatId);
+
+    return names.filter((name) => name.endsWith(FOLDER_SUFFIX) && name !== own).map((name) => join(dir, name));
+}
+
+// A clone where the file system has one, so a fork of a large chat takes no space until either side changes.
+const COPY_MODE = constants.COPYFILE_FICLONE;
+
+async function copyFromAny(folders: readonly string[], id: string, target: string): Promise<boolean> {
+    for (const folder of folders) {
+        try {
+            await copyFile(join(folder, `${id}.gz`), target, COPY_MODE);
+
+            return true;
+        } catch (e) {
+            if (!isNotFound(e)) {
+                throw e;
+            }
+        }
+    }
+
+    return false;
+}
+
+function copyFromAnySync(folders: readonly string[], id: string, target: string): boolean {
+    for (const folder of folders) {
+        try {
+            copyFileSync(join(folder, `${id}.gz`), target, COPY_MODE);
+
+            return true;
+        } catch (e) {
+            if (!isNotFound(e)) {
+                throw e;
+            }
+        }
+    }
+
+    return false;
+}
+
+async function existsAsync(path: string): Promise<boolean> {
+    try {
+        await stat(path);
+
+        return true;
+    } catch (e) {
+        if (isNotFound(e)) {
+            return false;
+        }
+
+        throw e;
+    }
+}
+
+function existsSync(path: string): boolean {
+    try {
+        statSync(path);
+
+        return true;
+    } catch (e) {
+        if (isNotFound(e)) {
+            return false;
+        }
+
+        throw e;
+    }
+}

package/src/chat/projector.ts

@@ -293,7 +293,8 @@ export class ThreadProjector {
                 if (target?.kind === 'tool' && target.name === 'Workflow' && event.state === 'done' && event.output?.startsWith(WORKFLOW_LAUNCHED)) {
                     // The call only launched the workflow; the row runs on past the turn until the workflow's task ends.
                     this.backgroundCalls.add(target.id);
-                    events.push(this.thread.upsert({ ...target, output: event.output }));
+                    const { outputRef: _earlier, ...launched } = target;
+                    events.push(this.thread.upsert({ ...launched, output: event.output }));
 
                     break;
                 }
@@ -667,6 +668,10 @@ export class ThreadProjector {
             item.progress = previous.progress;
         }
 
+        if (previous?.outputRef) {
+            item.outputRef = previous.outputRef;
+        }
+
         const changes = event.changes ?? previous?.changes;
 
         if (changes) {
@@ -697,6 +702,11 @@ export class ThreadProjector {
         const settled: ChatToolItem = { ...tool, output: event.output ?? tool.output, state: event.state };
         delete settled.progress;
 
+        // A stored whole of an earlier output would answer for this one.
+        if (event.output != null) {
+            delete settled.outputRef;
+        }
+
         if (event.changes) {
             settled.changes = event.changes;
         }

package/src/host/handlers.ts

@@ -103,6 +103,7 @@ export function chatHandlers(chats: ChatCore, providers: ProviderRegistry): { [T
             return {};
         },
         'chat.turnDiff': async (payload) => ({ diff: await chats.turnDiff(payload.chatId, payload.turnId) }),
+        'chat.toolOutput': async (payload) => ({ output: await chats.toolOutput(payload.chatId, payload.itemId) }),
         'chat.fork': unsupported('fork chats'),
         'chat.forkInfo': unsupported('fork chats'),
         'chat.summarize': unsupported('fork chats'),

package/src/host/wiring.ts

@@ -25,7 +25,7 @@ import { cliEnvironment, type CliEnvironmentPolicy } from './environment.ts';
 import { accountHandlers, chatHandlers, usageHandlers, type AgentHandlers } from './handlers.ts';
 
 export interface AgentWiringOptions<Core extends ChatCore = ChatCore> {
-    // Where the chats, their attachments, bookmarks and visuals, the accounts and the usage index are kept.
+    // Where the chats, their attachments, bookmarks and visuals, the accounts and the usage history are kept.
     dataDir: string;
     // The environment the CLIs start in; absent, this process's own through `cliEnvironment`.
     env?: Record<string, string | undefined>;

package/src/usage/history-file.ts

@@ -0,0 +1,140 @@
+import { UsageProviderSchema, type UsageProvider } from '@adecore/agent-contracts';
+import { foldByKey, type UsageRecord } from './record.ts';
+import { CURRENT_ROW, decodeRows, RowWriter, type RecordRow } from './rows.ts';
+
+const HISTORY_VERSION = 1;
+const NEWLINE = 0x0a;
+
+/* What the transcript at one path has counted, kept after the transcript itself is gone. */
+export interface FileUsage {
+    provider: UsageProvider;
+    records: UsageRecord[];
+    /* What the line still being written at the end of the transcript says; replaced on every read of it. */
+    tail: UsageRecord[];
+}
+
+/*
+ * One line of the history. `set` replaces what a path counted, since its transcript was read from the
+ * start; `add` folds its records into what the path had, since they came from an append.
+ */
+export interface HistoryEntry {
+    path: string;
+    op: 'set' | 'add';
+    usage: FileUsage;
+}
+
+/*
+ * The first line. `gen` is new on every rewrite of the file, so scan state saved against another
+ * rewrite is never trusted; `base` is the size of the lines it was rewritten with, which says when to
+ * rewrite it again.
+ */
+export interface HistoryHeader {
+    gen: string;
+    base: number;
+}
+
+/* Each line names its own models, sessions and folders, so it reads back without the lines before it. */
+interface EntryLine {
+    p: string;
+    k: UsageProvider;
+    op: 'set' | 'add';
+    m: string[];
+    s: string[];
+    f: string[];
+    a: string[];
+    r: RecordRow[];
+    t: RecordRow[];
+}
+
+export function encodeHeader(header: HistoryHeader): string {
+    return `${JSON.stringify({ v: HISTORY_VERSION, gen: header.gen, base: header.base })}\n`;
+}
+
+export function encodeEntry(entry: HistoryEntry): string {
+    const writer = new RowWriter();
+    const rows = entry.usage.records.map((record) => writer.row(record));
+    const tail = entry.usage.tail.map((record) => writer.row(record));
+    const { models, sessions, folders, accounts } = writer.tables;
+    const line: EntryLine = { p: entry.path, k: entry.usage.provider, op: entry.op, m: models, s: sessions, f: folders, a: accounts, r: rows, t: tail };
+
+    return `${JSON.stringify(line)}\n`;
+}
+
+function parseLine(text: string): unknown {
+    try {
+        return JSON.parse(text);
+    } catch {
+        return null;
+    }
+}
+
+function decodeHeader(text: string): HistoryHeader | null {
+    const header = parseLine(text) as Partial<HistoryHeader & { v: number }> | null;
+
+    if (header?.v !== HISTORY_VERSION || typeof header.gen !== 'string' || typeof header.base !== 'number') {
+        return null;
+    }
+
+    return { gen: header.gen, base: header.base };
+}
+
+function decodeEntry(text: string): HistoryEntry | null {
+    const line = parseLine(text) as Partial<EntryLine> | null;
+
+    if (typeof line?.p !== 'string' || (line.op !== 'set' && line.op !== 'add') || !UsageProviderSchema.safeParse(line.k).success) {
+        return null;
+    }
+
+    const provider = line.k as UsageProvider;
+    const tables = { models: line.m, sessions: line.s, folders: line.f, accounts: line.a };
+    const records = decodeRows(line.r, tables, provider, CURRENT_ROW);
+    const tail = decodeRows(line.t, tables, provider, CURRENT_ROW);
+
+    if (records === null || tail === null) {
+        return null;
+    }
+
+    return { path: line.p, op: line.op, usage: { provider, records, tail } };
+}
+
+/* Folds one line into what every path counted. */
+export function applyEntry(usage: Map<string, FileUsage>, entry: HistoryEntry): void {
+    const known = usage.get(entry.path);
+
+    if (entry.op === 'set' || known === undefined || known.provider !== entry.usage.provider) {
+        usage.set(entry.path, entry.usage);
+
+        return;
+    }
+
+    usage.set(entry.path, { provider: known.provider, records: foldByKey([...known.records, ...entry.usage.records]), tail: entry.usage.tail });
+}
+
+export interface DecodedHistory {
+    header: HistoryHeader | null;
+    usage: Map<string, FileUsage>;
+    /* The bytes up to the last newline; past it is a line an append left half-written. */
+    complete: number;
+}
+
+/*
+ * Every line stands alone, so a line that does not read back costs only that line. Only the end can
+ * be half-written, since lines are only ever appended.
+ */
+export function decodeHistory(bytes: Uint8Array): DecodedHistory {
+    const buffer = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    const complete = buffer.lastIndexOf(NEWLINE) + 1;
+    const lines = buffer.toString('utf8', 0, complete).split('\n');
+    const usage = new Map<string, FileUsage>();
+    const header = complete === 0 ? null : decodeHeader(lines[0]!);
+
+    for (let i = header === null ? 0 : 1; i < lines.length; i++) {
+        const entry = lines[i] === '' ? null : decodeEntry(lines[i]!);
+
+        if (entry !== null) {
+            applyEntry(usage, entry);
+        }
+    }
+
+    return { header, usage, complete };
+}