@adecore/agents 0.41.0 → 0.42.0

package/src/usage/index-file.ts

@@ -1,8 +1,12 @@
 import { UsageProviderSchema, type UsageProvider } from '@adecore/agent-contracts';
 import type { UsageRecord } from './record.ts';
 import type { CodexParserState } from './readers/codex.ts';
+import { CURRENT_ROW, decodeRows, RowWriter, type RecordRow } from './rows.ts';
 
-/* Old records survive deleted transcripts; the scanner reparses existing Codex files with its current reader. */
+/*
+ * The single index usage was kept in before history and scan state went apart. The store reads it
+ * once to migrate it; nothing writes it any more but the tests that need one to migrate.
+ */
 const INDEX_VERSION = 3;
 /* Before accounts: every record was the default account's, so it reads as that without a scan. */
 const ACCOUNTLESS_VERSION = 1;
@@ -11,15 +15,10 @@ export interface IndexedFile {
     provider: UsageProvider;
     size: number;
     mtimeMs: number;
-    /* Just past the last newline that was read, so a file that grew is read from there. */
     offset: number;
     records: UsageRecord[];
-    /* What the line still being written at the end of the file says; read again on the next scan,
-       so it is kept apart from the records the offset has already accounted for. */
     tail: UsageRecord[];
-    /* What the reader has to remember to carry on halfway through a Codex file. */
     codex: CodexParserState | null;
-    /* A versioned hash of all bytes before the offset; absent in indexes that cannot verify an append. */
     head: string | null;
     parserVersion?: number;
     compressed?: boolean;
@@ -27,9 +26,6 @@ export interface IndexedFile {
 
 export type UsageIndex = Map<string, IndexedFile>;
 
-/* Interned names keep the index compact; optional cost and tier fields preserve per-call pricing. */
-type RecordRow = [number, number, number, number, number, number, number, number, number, number, number, string | 0, number?, number?, string?];
-
 interface FileRow {
     p: UsageProvider;
     s: number;
@@ -52,60 +48,10 @@ interface IndexFile {
     files: Record<string, FileRow>;
 }
 
-class Interner {
-    readonly values: string[] = [];
-    private readonly at = new Map<string, number>();
-
-    index(value: string): number {
-        const seen = this.at.get(value);
-
-        if (seen !== undefined) {
-            return seen;
-        }
-
-        const next = this.values.length;
-        this.values.push(value);
-        this.at.set(value, next);
-
-        return next;
-    }
-}
-
 export function encodeIndex(index: UsageIndex): string {
-    const models = new Interner();
-    const sessions = new Interner();
-    const folders = new Interner();
-    const accounts = new Interner();
+    const writer = new RowWriter();
     const files: Record<string, FileRow> = {};
 
-    const row = (record: UsageRecord): RecordRow => {
-        const counted: RecordRow = [
-            record.timestampMs,
-            models.index(record.model),
-            sessions.index(record.sessionId),
-            folders.index(record.cwd),
-            record.totals.calls,
-            record.totals.input,
-            record.totals.cacheRead,
-            record.totals.cacheWrite,
-            record.totals.cacheWrite1h,
-            record.totals.output,
-            record.totals.reasoning,
-            record.dedupeKey ?? 0,
-            record.account === undefined ? 0 : accounts.index(record.account) + 1
-        ];
-
-        if (record.costUsd !== undefined || record.serviceTier !== undefined) {
-            counted[13] = record.costUsd;
-        }
-
-        if (record.serviceTier !== undefined) {
-            counted[14] = record.serviceTier;
-        }
-
-        return counted;
-    };
-
     for (const [path, file] of index) {
         files[path] = {
             p: file.provider,
@@ -113,50 +59,30 @@ export function encodeIndex(index: UsageIndex): string {
             m: file.mtimeMs,
             o: file.offset,
             c: file.codex,
-            r: file.records.map(row),
-            t: file.tail.map(row),
+            r: file.records.map((record) => writer.row(record)),
+            t: file.tail.map((record) => writer.row(record)),
             ...(file.head === null ? {} : { h: file.head }),
             ...(file.parserVersion === undefined ? {} : { v: file.parserVersion }),
             ...(file.compressed === undefined ? {} : { z: file.compressed })
         };
     }
 
-    const document: IndexFile = {
-        version: INDEX_VERSION,
-        models: models.values,
-        sessions: sessions.values,
-        folders: folders.values,
-        accounts: accounts.values,
-        files
-    };
+    const document: IndexFile = { version: INDEX_VERSION, ...writer.tables, files };
 
     return JSON.stringify(document);
 }
 
-function isRow(row: unknown, version: number): row is RecordRow {
-    return (
-        Array.isArray(row) &&
-        (version === ACCOUNTLESS_VERSION ? row.length === 12 : row.length >= 13 && row.length <= (version === INDEX_VERSION ? 15 : 14)) &&
-        row.slice(0, 11).every((value) => typeof value === 'number') &&
-        (typeof row[11] === 'string' || row[11] === 0) &&
-        (version === ACCOUNTLESS_VERSION || typeof row[12] === 'number') &&
-        (row.length < 14 || row[13] === null || row[13] === undefined || typeof row[13] === 'number') &&
-        (row.length < 15 || typeof row[14] === 'string')
-    );
-}
-
 /*
- * A row that does not read back costs a cold parse of the file it belonged to, never a wrong total,
- * so anything unexpected drops that file's entry rather than being repaired.
+ * Null for text that is no index of a version this reads. A file whose rows do not read back is left
+ * out, so only that file's usage is missing from what migrates, never a wrong total.
  */
-export function decodeIndex(text: string): UsageIndex {
-    const index: UsageIndex = new Map();
+export function decodeIndex(text: string): UsageIndex | null {
     let document: IndexFile;
 
     try {
         document = JSON.parse(text) as IndexFile;
     } catch {
-        return index;
+        return null;
     }
 
     const version = document?.version;
@@ -167,12 +93,12 @@ export function decodeIndex(text: string): UsageIndex {
         !Array.isArray(document.sessions) ||
         !Array.isArray(document.folders)
     ) {
-        return index;
+        return null;
     }
 
-    const accounts = Array.isArray(document.accounts) ? document.accounts : [];
-
-    const name = (table: string[], at: number): string | null => (typeof table[at] === 'string' ? table[at] : null);
+    const index: UsageIndex = new Map();
+    const tables = { ...document, accounts: Array.isArray(document.accounts) ? document.accounts : [] };
+    const shape = { accountless: version === ACCOUNTLESS_VERSION, maxLength: version === INDEX_VERSION ? CURRENT_ROW.maxLength : 14 };
 
     for (const [path, file] of Object.entries(document.files ?? {})) {
         // Every provider the schema knows, so a new one is never dropped from the index it was saved in.
@@ -180,63 +106,29 @@ export function decodeIndex(text: string): UsageIndex {
             continue;
         }
 
-        if (typeof file.s !== 'number' || typeof file.m !== 'number' || typeof file.o !== 'number' || file.o < 0 || !Array.isArray(file.r)) {
+        if (typeof file.s !== 'number' || typeof file.m !== 'number' || typeof file.o !== 'number' || file.o < 0) {
             continue;
         }
 
-        let broken = false;
-
-        const decode = (rows: RecordRow[]): UsageRecord[] => {
-            const records: UsageRecord[] = [];
-
-            for (const row of rows) {
-                const valid = isRow(row, version);
-                const model = valid ? name(document.models, row[1]) : null;
-                const sessionId = valid ? name(document.sessions, row[2]) : null;
-                const cwd = valid ? name(document.folders, row[3]) : null;
-                const accountAt = valid ? (row[12] ?? 0) : 0;
-                const account = accountAt === 0 ? undefined : name(accounts, accountAt - 1);
-
-                if (!valid || model === null || sessionId === null || cwd === null || account === null) {
-                    broken = true;
-
-                    return records;
-                }
-
-                records.push({
-                    provider: file.p,
-                    timestampMs: row[0],
-                    model,
-                    sessionId,
-                    cwd,
-                    ...(account === undefined ? {} : { account }),
-                    totals: { calls: row[4], input: row[5], cacheRead: row[6], cacheWrite: row[7], cacheWrite1h: row[8], output: row[9], reasoning: row[10] },
-                    ...(typeof row[13] === 'number' ? { costUsd: row[13] } : {}),
-                    ...(typeof row[14] === 'string' ? { serviceTier: row[14] } : {}),
-                    dedupeKey: row[11] === 0 ? null : row[11]
-                });
-            }
-
-            return records;
-        };
+        const records = decodeRows(file.r, tables, file.p, shape);
+        const tail = decodeRows(Array.isArray(file.t) ? file.t : [], tables, file.p, shape);
 
-        const records = decode(file.r);
-        const tail = decode(Array.isArray(file.t) ? file.t : []);
-
-        if (!broken) {
-            index.set(path, {
-                provider: file.p,
-                size: file.s,
-                mtimeMs: file.m,
-                offset: file.o,
-                records,
-                tail,
-                codex: file.c ?? null,
-                head: typeof file.h === 'string' ? file.h : null,
-                ...(typeof file.v === 'number' ? { parserVersion: file.v } : {}),
-                ...(typeof file.z === 'boolean' ? { compressed: file.z } : {})
-            });
+        if (records === null || tail === null) {
+            continue;
         }
+
+        index.set(path, {
+            provider: file.p,
+            size: file.s,
+            mtimeMs: file.m,
+            offset: file.o,
+            records,
+            tail,
+            codex: file.c ?? null,
+            head: typeof file.h === 'string' ? file.h : null,
+            ...(typeof file.v === 'number' ? { parserVersion: file.v } : {}),
+            ...(typeof file.z === 'boolean' ? { compressed: file.z } : {})
+        });
     }
 
     return index;

package/src/usage/rows.ts

@@ -0,0 +1,136 @@
+import type { UsageProvider } from '@adecore/agent-contracts';
+import type { UsageRecord } from './record.ts';
+
+/* Interned names keep the files compact; optional cost and tier fields preserve per-call pricing. */
+export type RecordRow = [number, number, number, number, number, number, number, number, number, number, number, string | 0, number?, number?, string?];
+
+/* The names the rows of one document point into. */
+export interface RowTables {
+    models: string[];
+    sessions: string[];
+    folders: string[];
+    accounts: string[];
+}
+
+class Interner {
+    readonly values: string[] = [];
+    private readonly at = new Map<string, number>();
+
+    index(value: string): number {
+        const seen = this.at.get(value);
+
+        if (seen !== undefined) {
+            return seen;
+        }
+
+        const next = this.values.length;
+        this.values.push(value);
+        this.at.set(value, next);
+
+        return next;
+    }
+}
+
+/* Turns records into rows, collecting the names they share into one set of tables. */
+export class RowWriter {
+    private readonly models = new Interner();
+    private readonly sessions = new Interner();
+    private readonly folders = new Interner();
+    private readonly accounts = new Interner();
+
+    get tables(): RowTables {
+        return { models: this.models.values, sessions: this.sessions.values, folders: this.folders.values, accounts: this.accounts.values };
+    }
+
+    row(record: UsageRecord): RecordRow {
+        const counted: RecordRow = [
+            record.timestampMs,
+            this.models.index(record.model),
+            this.sessions.index(record.sessionId),
+            this.folders.index(record.cwd),
+            record.totals.calls,
+            record.totals.input,
+            record.totals.cacheRead,
+            record.totals.cacheWrite,
+            record.totals.cacheWrite1h,
+            record.totals.output,
+            record.totals.reasoning,
+            record.dedupeKey ?? 0,
+            record.account === undefined ? 0 : this.accounts.index(record.account) + 1
+        ];
+
+        if (record.costUsd !== undefined || record.serviceTier !== undefined) {
+            counted[13] = record.costUsd;
+        }
+
+        if (record.serviceTier !== undefined) {
+            counted[14] = record.serviceTier;
+        }
+
+        return counted;
+    }
+}
+
+/* Rows written before accounts carry twelve fields, and before per-call pricing at most fourteen. */
+export interface RowShape {
+    accountless: boolean;
+    maxLength: number;
+}
+
+export const CURRENT_ROW: RowShape = { accountless: false, maxLength: 15 };
+
+function isRow(row: unknown, shape: RowShape): row is RecordRow {
+    return (
+        Array.isArray(row) &&
+        (shape.accountless ? row.length === 12 : row.length >= 13 && row.length <= shape.maxLength) &&
+        row.slice(0, 11).every((value) => typeof value === 'number') &&
+        (typeof row[11] === 'string' || row[11] === 0) &&
+        (shape.accountless || typeof row[12] === 'number') &&
+        (row.length < 14 || row[13] === null || row[13] === undefined || typeof row[13] === 'number') &&
+        (row.length < 15 || typeof row[14] === 'string')
+    );
+}
+
+function nameAt(table: unknown, at: number): string | null {
+    return Array.isArray(table) && typeof table[at] === 'string' ? table[at] : null;
+}
+
+/* The records of `rows`, or null when any row does not read back against its tables. */
+export function decodeRows(rows: unknown, tables: Partial<Record<keyof RowTables, unknown>>, provider: UsageProvider, shape: RowShape): UsageRecord[] | null {
+    if (!Array.isArray(rows)) {
+        return null;
+    }
+
+    const records: UsageRecord[] = [];
+
+    for (const row of rows) {
+        if (!isRow(row, shape)) {
+            return null;
+        }
+
+        const model = nameAt(tables.models, row[1]);
+        const sessionId = nameAt(tables.sessions, row[2]);
+        const cwd = nameAt(tables.folders, row[3]);
+        const accountAt = row[12] ?? 0;
+        const account = accountAt === 0 ? undefined : nameAt(tables.accounts, accountAt - 1);
+
+        if (model === null || sessionId === null || cwd === null || account === null) {
+            return null;
+        }
+
+        records.push({
+            provider,
+            timestampMs: row[0],
+            model,
+            sessionId,
+            cwd,
+            ...(account === undefined ? {} : { account }),
+            totals: { calls: row[4], input: row[5], cacheRead: row[6], cacheWrite: row[7], cacheWrite1h: row[8], output: row[9], reasoning: row[10] },
+            ...(typeof row[13] === 'number' ? { costUsd: row[13] } : {}),
+            ...(typeof row[14] === 'string' ? { serviceTier: row[14] } : {}),
+            dedupeKey: row[11] === 0 ? null : row[11]
+        });
+    }
+
+    return records;
+}

package/src/usage/scan-state.ts

@@ -0,0 +1,105 @@
+import { UsageProviderSchema, type UsageProvider } from '@adecore/agent-contracts';
+import type { CodexParserState } from './readers/codex.ts';
+
+const STATE_VERSION = 1;
+
+/* Where the scanner left a transcript. All of it can be rebuilt by reading the transcript again. */
+export interface ScanState {
+    provider: UsageProvider;
+    size: number;
+    mtimeMs: number;
+    /* Just past the last newline that was read, so a file that grew is read from there. */
+    offset: number;
+    /* What the reader has to remember to carry on halfway through a Codex file. */
+    codex: CodexParserState | null;
+    /* A versioned hash of all bytes before the offset; absent where an append cannot be verified. */
+    head: string | null;
+    parserVersion?: number;
+    compressed?: boolean;
+}
+
+/* The history this state was saved against: the rewrite it belongs to and how far into it. */
+export interface HistoryMark {
+    gen: string;
+    bytes: number;
+}
+
+interface StateRow {
+    p: UsageProvider;
+    s: number;
+    m: number;
+    o: number;
+    c: CodexParserState | null;
+    h?: string;
+    v?: number;
+    z?: boolean;
+}
+
+interface StateFile {
+    version: number;
+    history: HistoryMark;
+    files: Record<string, StateRow>;
+}
+
+export function encodeState(state: ReadonlyMap<string, ScanState>, history: HistoryMark): string {
+    const files: Record<string, StateRow> = {};
+
+    for (const [path, file] of state) {
+        files[path] = {
+            p: file.provider,
+            s: file.size,
+            m: file.mtimeMs,
+            o: file.offset,
+            c: file.codex,
+            ...(file.head === null ? {} : { h: file.head }),
+            ...(file.parserVersion === undefined ? {} : { v: file.parserVersion }),
+            ...(file.compressed === undefined ? {} : { z: file.compressed })
+        };
+    }
+
+    const document: StateFile = { version: STATE_VERSION, history, files };
+
+    return JSON.stringify(document);
+}
+
+/* Null for text that is no state of this version; an entry that does not read only costs a read of its transcript. */
+export function decodeState(text: string): { history: HistoryMark; files: Map<string, ScanState> } | null {
+    let document: Partial<StateFile> | null;
+
+    try {
+        document = JSON.parse(text) as Partial<StateFile> | null;
+    } catch {
+        return null;
+    }
+
+    const history = document?.history;
+
+    if (document?.version !== STATE_VERSION || typeof history?.gen !== 'string' || typeof history.bytes !== 'number') {
+        return null;
+    }
+
+    const files = new Map<string, ScanState>();
+
+    for (const [path, file] of Object.entries(document.files ?? {})) {
+        if (!UsageProviderSchema.safeParse(file.p).success) {
+            continue;
+        }
+
+        if (typeof file.s !== 'number' || typeof file.m !== 'number' || typeof file.o !== 'number' || file.o < 0) {
+            continue;
+        }
+
+        files.set(path, {
+            provider: file.p,
+            size: file.s,
+            mtimeMs: file.m,
+            offset: file.o,
+            codex: file.c ?? null,
+            head: typeof file.h === 'string' ? file.h : null,
+            ...(typeof file.v === 'number' ? { parserVersion: file.v } : {}),
+            ...(typeof file.z === 'boolean' ? { compressed: file.z } : {})
+        });
+    }
+
+    return { history: { gen: history.gen, bytes: history.bytes }, files };
+}