@adecore/agents 0.0.1 → 0.17.0-beta.2

package/src/usage/scanner.ts

@@ -0,0 +1,265 @@
+import { mkdir, open, readFile, readdir, stat } from 'node:fs/promises';
+import { join } from 'node:path';
+import type { UsageRoot } from '@adecore/agent-contracts';
+import { isNotFound, writeAtomic } from '../fs.ts';
+import { decodeIndex, encodeIndex, type IndexedFile, type UsageIndex } from './index-file.ts';
+import { foldByKey, type UsageRecord } from './record.ts';
+import { claudeMightCarryUsage, parseClaudeLine } from './readers/claude.ts';
+import { cloneCodexState, codexMightCarryUsage, createCodexState, parseCodexLine, type CodexParserState } from './readers/codex.ts';
+import { usageRoots, type UsageRootPath } from './roots.ts';
+import { errorText } from '../error-text.ts';
+
+/* What a file holds from `from` to its end, as it is at the read. */
+async function readFrom(path: string, from: number): Promise<string> {
+    const handle = await open(path, 'r');
+    try {
+        const { size } = await handle.stat();
+        const buffer = Buffer.alloc(Math.max(0, size - from));
+        let filled = 0;
+        while (filled < buffer.length) {
+            const { bytesRead } = await handle.read(buffer, filled, buffer.length - filled, from + filled);
+            if (bytesRead === 0) {
+                break;
+            }
+            filled += bytesRead;
+        }
+        return buffer.subarray(0, filled).toString('utf8');
+    } finally {
+        await handle.close();
+    }
+}
+
+export interface ScanReport {
+    at: number;
+    files: number;
+    changedFiles: number;
+    durationMs: number;
+    roots: UsageRoot[];
+}
+
+interface ParsedFile {
+    records: UsageRecord[];
+    tail: UsageRecord[];
+    offset: number;
+    codex: CodexParserState | null;
+}
+
+async function listJsonl(dir: string, out: string[] = []): Promise<string[]> {
+    let entries;
+    try {
+        entries = await readdir(dir, { withFileTypes: true });
+    } catch {
+        // A directory that cannot be read is one the scan walks past; the root itself reports status.
+        return out;
+    }
+    for (const entry of entries) {
+        const path = join(dir, entry.name);
+        if (entry.isDirectory()) {
+            await listJsonl(path, out);
+        } else if (entry.isFile() && entry.name.endsWith('.jsonl')) {
+            out.push(path);
+        }
+    }
+    return out;
+}
+
+/*
+ * Reads what a file added since the last scan. Only whole lines count towards the offset; the piece
+ * after the last newline is a line still being written, so it is parsed for what it is worth and
+ * read again next time. The Codex state is snapshotted before that tail for the same reason.
+ */
+function parseChunk(text: string, provider: 'claude' | 'codex', from: number, state: CodexParserState | null): ParsedFile {
+    let records: UsageRecord[] = [];
+    const end = text.lastIndexOf('\n');
+    const whole = end === -1 ? '' : text.slice(0, end);
+    const tail = text.slice(end + 1);
+    const read = (chunk: string, codex: CodexParserState | null): void => {
+        for (const raw of chunk.split('\n')) {
+            const line = raw.endsWith('\r') ? raw.slice(0, -1) : raw;
+            if (line === '') {
+                continue;
+            }
+            if (provider === 'claude') {
+                if (claudeMightCarryUsage(line)) {
+                    const record = parseClaudeLine(line);
+                    if (record !== null) {
+                        records.push(record);
+                    }
+                }
+                continue;
+            }
+            if (codex !== null && codexMightCarryUsage(line)) {
+                const record = parseCodexLine(line, codex);
+                if (record !== null) {
+                    records.push(record);
+                }
+            }
+        }
+    };
+    const codex = provider === 'codex' ? (state ?? createCodexState()) : null;
+    read(whole, codex);
+    const committed = codex === null ? null : cloneCodexState(codex);
+    const counted = records;
+    records = [];
+    read(tail, codex);
+    return { records: counted, tail: records, offset: from + (end === -1 ? 0 : Buffer.byteLength(whole, 'utf8') + 1), codex: committed };
+}
+
+export interface UsageScannerOptions {
+    /* The account of every session the host knows ran, by `<provider>\0<sessionId>`, the default one under its kind. */
+    sessionAccounts?: () => ReadonlyMap<string, string>;
+}
+
+type AccountOf = (record: UsageRecord) => string | undefined;
+
+/*
+ * The account a record of a root belongs to, as a record keeps it (absent for the default account):
+ * the one account that writes there, else the one the host knows ran the session, else the
+ * default account of the CLI, which is the only one a session nobody started through the host ran under.
+ */
+export function accountResolver(root: UsageRootPath, sessionAccounts: () => ReadonlyMap<string, string>): AccountOf {
+    const accounts = root.accounts ?? [root.provider];
+    const stored = (id: string): string | undefined => (id === root.provider ? undefined : id);
+    if (accounts.length === 1) {
+        const only = stored(accounts[0]!);
+        return () => only;
+    }
+    const fallback = accounts.includes(root.provider) ? root.provider : accounts[0]!;
+    return (record) => {
+        const ran = sessionAccounts().get(`${root.provider}\0${record.sessionId}`);
+        return stored(ran !== undefined && accounts.includes(ran) ? ran : fallback);
+    };
+}
+
+function withAccount(records: UsageRecord[], accountOf: AccountOf): UsageRecord[] {
+    return records.map((record) => {
+        const account = accountOf(record);
+        return account === undefined ? record : { ...record, account };
+    });
+}
+
+/*
+ * Read appended transcript bytes incrementally, but restart after rewrites or truncation. Preserve
+ * totals for deleted transcripts because their usage still happened.
+ */
+export class UsageScanner {
+    private readonly home: string;
+    private readonly roots: () => UsageRootPath[];
+    private readonly sessionAccounts: () => ReadonlyMap<string, string>;
+    private index: UsageIndex = new Map();
+    private loaded = false;
+    private dirty = false;
+
+    /* `roots` as a function is asked again on every scan, so an account added since is walked too. */
+    constructor(home: string, roots: UsageRootPath[] | (() => UsageRootPath[]) = usageRoots(), options: UsageScannerOptions = {}) {
+        this.home = home;
+        this.roots = typeof roots === 'function' ? roots : () => roots;
+        this.sessionAccounts = options.sessionAccounts ?? (() => new Map());
+    }
+
+    private get file(): string {
+        return join(this.home, 'usage', 'index.json');
+    }
+
+    async load(): Promise<void> {
+        if (this.loaded) {
+            return;
+        }
+        this.loaded = true;
+        try {
+            this.index = decodeIndex(await readFile(this.file, 'utf8'));
+        } catch (e) {
+            if (!isNotFound(e)) {
+                console.warn('The usage index would not read; starting from a cold scan:', errorText(e));
+            }
+        }
+    }
+
+    async scan(): Promise<ScanReport> {
+        const started = Date.now();
+        await this.load();
+        const roots: UsageRoot[] = [];
+        let files = 0;
+        let changedFiles = 0;
+        let sessions: ReadonlyMap<string, string> | null = null;
+        const sessionAccounts = (): ReadonlyMap<string, string> => {
+            sessions ??= this.sessionAccounts();
+            return sessions;
+        };
+
+        for (const root of this.roots()) {
+            try {
+                const info = await stat(root.path);
+                if (!info.isDirectory()) {
+                    roots.push({ provider: root.provider, path: root.path, status: 'missing', message: null });
+                    continue;
+                }
+            } catch (e) {
+                const missing = isNotFound(e);
+                roots.push({
+                    provider: root.provider,
+                    path: root.path,
+                    status: missing ? 'missing' : 'failed',
+                    message: missing ? null : errorText(e)
+                });
+                continue;
+            }
+            let failure: string | null = null;
+            const accountOf = accountResolver(root, sessionAccounts);
+            for (const path of await listJsonl(root.path)) {
+                files += 1;
+                try {
+                    if (await this.readFileInto(path, root.provider, accountOf)) {
+                        changedFiles += 1;
+                    }
+                } catch (e) {
+                    // One unreadable transcript is not the root failing; the count simply misses it.
+                    failure ??= errorText(e);
+                }
+            }
+            roots.push({ provider: root.provider, path: root.path, status: failure === null ? 'ok' : 'failed', message: failure });
+        }
+
+        if (this.dirty) {
+            this.dirty = false;
+            await mkdir(join(this.home, 'usage'), { recursive: true, mode: 0o700 });
+            await writeAtomic(this.file, encodeIndex(this.index));
+        }
+        return { at: Date.now(), files, changedFiles, durationMs: Date.now() - started, roots };
+    }
+
+    /* Every record the index holds, with the copies one message left in several transcripts folded. */
+    records(): UsageRecord[] {
+        const all: UsageRecord[] = [];
+        for (const file of this.index.values()) {
+            all.push(...file.records, ...file.tail);
+        }
+        return foldByKey(all);
+    }
+
+    private async readFileInto(path: string, provider: 'claude' | 'codex', accountOf: AccountOf): Promise<boolean> {
+        const info = await stat(path);
+        const known = this.index.get(path);
+        if (known !== undefined && known.provider === provider && known.size === info.size && known.mtimeMs === info.mtimeMs) {
+            return false;
+        }
+        const grown = known !== undefined && known.provider === provider && info.size > known.size;
+        const from = grown ? known.offset : 0;
+        const text = await readFrom(path, from);
+        const parsed = parseChunk(text, provider, from, grown ? known.codex : null);
+        const before = grown ? known.records : [];
+        const entry: IndexedFile = {
+            provider,
+            size: info.size,
+            mtimeMs: info.mtimeMs,
+            offset: parsed.offset,
+            // Folding here keeps the streaming copies of one message out of the index entirely.
+            records: foldByKey([...before, ...withAccount(parsed.records, accountOf)]),
+            tail: withAccount(parsed.tail, accountOf),
+            codex: parsed.codex
+        };
+        this.index.set(path, entry);
+        this.dirty = true;
+        return true;
+    }
+}

package/src/usage/usage-service.ts

@@ -0,0 +1,161 @@
+import type { UsageAccount, UsageSummaryPayload, UsageSummaryResult } from '@adecore/agent-contracts';
+import type { AgentEvent, AgentSink } from '../events.ts';
+import { accountOfRecord, aggregate } from './aggregate.ts';
+import { ExchangeRates } from './exchange.ts';
+import { PriceBook } from './pricing.ts';
+import type { KnownProject } from './projects.ts';
+import type { UsageRecord } from './record.ts';
+import { UsageScanner, type ScanReport, type UsageScannerOptions } from './scanner.ts';
+import type { UsageRootPath } from './roots.ts';
+import { errorText } from '../error-text.ts';
+import { ClientSinks } from '../client-sinks.ts';
+
+/* A scan this fresh answers the question the page is asking, so nothing is opened for it. */
+const SCAN_TTL_MS = 60_000;
+
+export interface UsageServiceOptions {
+    home: string;
+    /* Off with `--no-price-fetch`: the bundled table then prices everything and no rate is asked for. */
+    allowPriceFetch?: boolean;
+    /* The projects the host knows, so a folder can wear the name it has in the app. */
+    knownProjects(): Promise<KnownProject[]>;
+    /* Where the transcripts are, asked again on every scan when a function. */
+    roots?: UsageRootPath[] | (() => UsageRootPath[]);
+    sessionAccounts?: UsageScannerOptions['sessionAccounts'];
+    /* The accounts of the machine, for the page to filter by; absent names none. */
+    accounts?: () => UsageAccount[];
+}
+
+/* The accounts of the machine, and after them any a record names that the machine no longer has, under its id. */
+function withRecordAccounts(accounts: UsageAccount[], records: readonly UsageRecord[]): UsageAccount[] {
+    const listed = new Set(accounts.map((account) => account.id));
+    const gone: UsageAccount[] = [];
+    for (const record of records) {
+        const id = accountOfRecord(record);
+        if (!listed.has(id)) {
+            listed.add(id);
+            gone.push({ id, kind: record.provider, label: id });
+        }
+    }
+    return [...accounts, ...gone];
+}
+
+const EMPTY_SCAN: ScanReport = { at: 0, files: 0, changedFiles: 0, durationMs: 0, roots: [] };
+
+/*
+ * What the usage page asks the host. It owns one scanner and one price table for every client:
+ * the transcripts are machine-wide, so a second person looking costs a second aggregation and not a
+ * second scan. The scan runs when a request finds the last one stale and, while any client has the
+ * page open, once a minute, which is what `usage.changed` announces.
+ */
+export class UsageService {
+    private readonly scanner: UsageScanner;
+    private readonly prices: PriceBook;
+    private readonly rates: ExchangeRates;
+    private readonly knownProjects: UsageServiceOptions['knownProjects'];
+    private readonly accounts: (() => UsageAccount[]) | null;
+    private readonly sinks = new ClientSinks<AgentEvent>((clientId) => this.unfollow(clientId));
+    private readonly followers = new Set<string>();
+    private report: ScanReport = EMPTY_SCAN;
+    private failed = false;
+    private inFlight: Promise<void> | null = null;
+    private timer: ReturnType<typeof setInterval> | null = null;
+
+    constructor(options: UsageServiceOptions) {
+        this.scanner = new UsageScanner(options.home, options.roots, options.sessionAccounts ? { sessionAccounts: options.sessionAccounts } : {});
+        this.accounts = options.accounts ?? null;
+        this.prices = new PriceBook(options.home, options.allowPriceFetch ?? true);
+        this.rates = new ExchangeRates(options.home, options.allowPriceFetch ?? true);
+        this.knownProjects = options.knownProjects;
+    }
+
+    subscribe(clientId: string, sink: AgentSink): () => void {
+        return this.sinks.subscribe(clientId, sink);
+    }
+
+    /* The page is open here: keep the numbers moving until it closes or the socket does. */
+    follow(clientId: string): void {
+        this.followers.add(clientId);
+        this.timer ??= setInterval(() => {
+            void this.rescan();
+        }, SCAN_TTL_MS);
+    }
+
+    unfollow(clientId: string): void {
+        this.followers.delete(clientId);
+        if (this.followers.size === 0 && this.timer !== null) {
+            clearInterval(this.timer);
+            this.timer = null;
+        }
+    }
+
+    async summary(payload: UsageSummaryPayload): Promise<UsageSummaryResult> {
+        if (Date.now() - this.report.at > SCAN_TTL_MS) {
+            await this.rescan();
+        }
+        const records = this.scanner.records();
+        const { buckets, models, projects, sessions } = await aggregate(records, payload, this.prices, await this.knownProjects());
+        return {
+            from: payload.from,
+            to: payload.to,
+            resolution: payload.resolution,
+            timeZone: payload.timeZone,
+            buckets,
+            models,
+            projects,
+            sessions,
+            scan: {
+                at: this.report.at,
+                files: this.report.files,
+                changedFiles: this.report.changedFiles,
+                durationMs: this.report.durationMs,
+                running: this.inFlight !== null,
+                failed: this.failed
+            },
+            pricing: this.prices.pricing,
+            rate: this.rates.current,
+            roots: this.report.roots,
+            ...(this.accounts === null ? {} : { accounts: withRecordAccounts(this.accounts(), records) })
+        };
+    }
+
+    stop(): void {
+        if (this.timer !== null) {
+            clearInterval(this.timer);
+            this.timer = null;
+        }
+    }
+
+    /* One pass at a time: a request that arrives while a pass runs waits for that one's answer. */
+    private rescan(): Promise<void> {
+        this.inFlight ??= this.runScan().finally(() => {
+            this.inFlight = null;
+        });
+        return this.inFlight;
+    }
+
+    private async runScan(): Promise<void> {
+        // The table and the rate are fetched beside the walk, so a slow answer never holds up the scan.
+        const [report] = await Promise.all([
+            this.scanner.scan().catch((e: unknown) => {
+                console.error('The usage scan failed:', errorText(e));
+                return null;
+            }),
+            this.prices.ensure(),
+            this.rates.ensure()
+        ]);
+        this.failed = report === null;
+        if (report === null) {
+            // The last good numbers stay on screen; only the moment of the scan is not moved on.
+            return;
+        }
+        this.report = report;
+        if (report.changedFiles > 0) {
+            this.emit({ event: 'usage.changed', payload: { scannedAt: report.at } });
+        }
+    }
+
+    private emit(event: AgentEvent): void {
+        this.sinks.emit(event);
+    }
+}

package/src/watch-seam.ts

@@ -0,0 +1,111 @@
+import { watch } from 'node:fs';
+
+/* The part of an `fs.watch` handle the watchers use. */
+export interface DirectoryWatcher {
+    on(event: 'error', listener: (error: Error) => void): unknown;
+    close(): void;
+}
+
+export type WatchDirectory = (path: string, options: { recursive: boolean }, listener: (event: string, filename: string | null) => void) => DirectoryWatcher;
+
+/* Runs `callback` after `ms` and answers with the function that cancels it. */
+export type Schedule = (callback: () => unknown, ms: number) => () => void;
+
+/* What every watcher reaches the file system and the clock through, so a test can hand it fakes that it drives by hand. */
+export interface WatchSeams {
+    watch: WatchDirectory;
+    schedule: Schedule;
+}
+
+/*
+ * Recursive watching is one call to the platform on macOS and Windows; elsewhere it costs a
+ * descriptor per directory, which a folder tree has too many of to be worth it.
+ */
+export function supportsRecursive(platform: NodeJS.Platform): boolean {
+    return platform === 'darwin' || platform === 'win32';
+}
+
+export const SYSTEM_WATCH: WatchSeams = {
+    watch: (path, options, listener) => watch(path, options, listener),
+    schedule: (callback, ms) => {
+        const timer = setTimeout(callback, ms);
+        return () => clearTimeout(timer);
+    }
+};
+
+/* One run after a burst of changes: every nudge puts it off again, and a stop drops the one waiting. */
+export interface Settled {
+    nudge(): void;
+    stop(): void;
+}
+
+/*
+ * A save, a formatter and a build all touch a folder in a burst, and one answer per burst is enough.
+ * How long a burst is differs per watcher: reading one file again is cheap, a status run over a
+ * whole repository is not.
+ */
+export function settled(seams: WatchSeams, ms: number, run: () => unknown): Settled {
+    let cancel: (() => void) | null = null;
+    return {
+        nudge: () => {
+            cancel?.();
+            cancel = seams.schedule(() => {
+                cancel = null;
+                return run();
+            }, ms);
+        },
+        stop: () => {
+            cancel?.();
+            cancel = null;
+        }
+    };
+}
+
+/*
+ * What each client watches, keyed on the path it asked for. A watch is per client, like a session
+ * attach: two windows on the same folder each get their own, and a socket that goes takes only its
+ * own with it.
+ */
+export class PerClientWatches<TState> {
+    private readonly byClient = new Map<string, Map<string, TState>>();
+    private readonly stop: (state: TState) => void;
+
+    constructor(stop: (state: TState) => void) {
+        this.stop = stop;
+    }
+
+    get(clientId: string, key: string): TState | undefined {
+        return this.byClient.get(clientId)?.get(key);
+    }
+
+    put(clientId: string, key: string, state: TState): void {
+        const watches = this.byClient.get(clientId) ?? new Map<string, TState>();
+        watches.set(key, state);
+        this.byClient.set(clientId, watches);
+    }
+
+    /* Everything this client watches, as a copy, so a caller may close one while it walks them. */
+    all(clientId: string): Array<[string, TState]> {
+        return [...(this.byClient.get(clientId) ?? [])];
+    }
+
+    remove(clientId: string, key: string): void {
+        const watches = this.byClient.get(clientId);
+        const state = watches?.get(key);
+        if (!watches || state === undefined) {
+            return;
+        }
+        this.stop(state);
+        watches.delete(key);
+        if (watches.size === 0) {
+            this.byClient.delete(clientId);
+        }
+    }
+
+    detachAll(clientId: string): void {
+        for (const state of this.byClient.get(clientId)?.values() ?? []) {
+            this.stop(state);
+        }
+        this.byClient.delete(clientId);
+    }
+}

package/src/watch-test-helpers.ts

@@ -0,0 +1,102 @@
+import type { DirectoryWatcher, WatchDirectory, WatchSeams } from './watch-seam.ts';
+
+export class FakeDirectoryWatcher implements DirectoryWatcher {
+    readonly path: string;
+    readonly recursive: boolean;
+    closed = false;
+    private readonly listener: (event: string, filename: string | null) => void;
+
+    constructor(path: string, recursive: boolean, listener: (event: string, filename: string | null) => void) {
+        this.path = path;
+        this.recursive = recursive;
+        this.listener = listener;
+    }
+
+    /* What the platform would report for a write to `filename`, relative to the watched directory. */
+    emit(filename: string | null, event: 'change' | 'rename' = 'change'): void {
+        if (this.closed) {
+            return;
+        }
+        this.listener(event, filename);
+    }
+
+    on(): this {
+        return this;
+    }
+
+    close(): void {
+        this.closed = true;
+    }
+}
+
+/*
+ * A watch function and a settle timer that only move when a test says so: `emit` on a watcher stands
+ * in for the platform, and `settle` runs every timer that is due and waits for the work it started.
+ */
+export class FakeWatch implements WatchSeams {
+    readonly watchers: FakeDirectoryWatcher[] = [];
+    private readonly due = new Map<number, () => unknown>();
+    private nextId = 1;
+
+    readonly watch: WatchDirectory = (path, options, listener) => {
+        const watcher = new FakeDirectoryWatcher(path, options.recursive, listener);
+        this.watchers.push(watcher);
+        return watcher;
+    };
+
+    readonly schedule = (callback: () => unknown, _ms: number): (() => void) => {
+        const id = this.nextId++;
+        this.due.set(id, callback);
+        return () => {
+            this.due.delete(id);
+        };
+    };
+
+    /* How many timers are waiting to run. */
+    get pending(): number {
+        return this.due.size;
+    }
+
+    openOn(path: string): FakeDirectoryWatcher[] {
+        return this.watchers.filter((watcher) => watcher.path === path && !watcher.closed);
+    }
+
+    /* The one open watcher on `path`, which a test that emits on it expects to exist. */
+    on(path: string): FakeDirectoryWatcher {
+        const [watcher, ...rest] = this.openOn(path);
+        if (!watcher || rest.length > 0) {
+            throw new Error(`Expected one open watcher on ${path}, found ${rest.length + (watcher ? 1 : 0)}`);
+        }
+        return watcher;
+    }
+
+    async settle(): Promise<void> {
+        const callbacks = [...this.due.values()];
+        this.due.clear();
+        await Promise.all(callbacks.map((callback) => callback()));
+    }
+}
+
+/* A sink's events, with a promise for the moment one of them passes a check, so a test awaits the event itself rather than a deadline. */
+export class EventLog<T> {
+    readonly events: T[] = [];
+    private waiters: Array<{ check: (events: T[]) => boolean; resolve: () => void }> = [];
+
+    readonly push = (event: T): void => {
+        this.events.push(event);
+        const ready = this.waiters.filter((waiter) => waiter.check(this.events));
+        this.waiters = this.waiters.filter((waiter) => !ready.includes(waiter));
+        for (const waiter of ready) {
+            waiter.resolve();
+        }
+    };
+
+    until(check: (events: T[]) => boolean): Promise<void> {
+        if (check(this.events)) {
+            return Promise.resolve();
+        }
+        return new Promise((resolve) => {
+            this.waiters.push({ check, resolve });
+        });
+    }
+}