@krtclcdy/tersio 2.24.1 → 2.25.1

package/extensions/shared/usage-store.js

@@ -1,9 +1,8 @@
-// extensions/shared/usage-store.ts — tersio-owned usage.db. A cache of the
-// live parse, never a fork; missing sqlite3 → sync false / read null.
+// extensions/shared/usage-store.ts — tersio-owned usage.db. A cache of the live parse, never a fork; missing sqlite3 → sync false / read null.
 import { execFileSync } from 'node:child_process';
 import fs from 'node:fs';
 import path from 'node:path';
-import { RECENT_LIMIT, canonicalModelId, classifySessionLine, classifyOpencodeMessage, codexSessionsDir, costOf, durOf, hostOfSessionFile, ingestSessionRow, opencodeSessionsDir, newSessionAccum, sessionsDirs, walkJsonl, } from './usage-ledger.js';
+import { RECENT_LIMIT, SQLITE_READ_BUFFER, canonicalModelId, classifySessionLine, classifyOpencodeMessage, costOf, durOf, hostOfSessionFile, ingestSessionRow, messageRowId, OPENCODE_DB_OVERLAP_MS, opencodeDbPath, opencodeSessionsDir, newSessionAccum, readOpencodeDbRows, sessionsDirs, walkJsonl, } from './usage-ledger.js';
 import { tersioDataPath } from '../lib/utils.js';
 import { MODEL_ALIASES } from './pricing.js';
 import { tersioSettingsFile } from './plugin-settings.js';
@@ -26,13 +25,15 @@ function nullStr(v) {
     return v === undefined ? 'NULL' : esc(v);
 }
 function run(db, sql) {
-    execFileSync('sqlite3', [db, sql], { stdio: ['ignore', 'ignore', 'ignore'], timeout: 30000 });
+    // -bail stops the script before COMMIT, so a failed INSERT never leaves DELETEs behind.
+    execFileSync('sqlite3', ['-bail', db, sql], { stdio: ['ignore', 'ignore', 'ignore'], timeout: 30000 });
 }
 function query(db, sql) {
     const out = execFileSync('sqlite3', ['-separator', '\t', db, sql], {
         encoding: 'utf8',
         timeout: 30000,
         stdio: ['ignore', 'pipe', 'ignore'],
+        maxBuffer: SQLITE_READ_BUFFER,
     });
     return out
         .split('\n')
@@ -40,38 +41,146 @@ function query(db, sql) {
         .map((line) => line.split('\t'));
 }
 // Bump on a parse change: unchanged transcripts are never re-read.
-const PARSER_VERSION = '6';
+const PARSER_VERSION = '11';
+// A re-parse that keeps less than half the rows means transcripts vanished mid-migration; the backup is restored instead of publishing the loss.
+const REPARSE_GUARD_RATIO = 0.5;
+let guardReport = null;
+/** Why the last sync refused to publish a re-parse, if it did. */
+export function reparseGuardReport() {
+    return guardReport;
+}
+const SCHEMA_SQL = `CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v TEXT NOT NULL);` +
+    `CREATE TABLE IF NOT EXISTS files (id INTEGER PRIMARY KEY AUTOINCREMENT, path TEXT UNIQUE NOT NULL, mtime REAL NOT NULL, size INTEGER NOT NULL);` +
+    `CREATE TABLE IF NOT EXISTS messages (file_id INTEGER NOT NULL REFERENCES files(id), t REAL, model TEXT NOT NULL,` +
+    ` i INTEGER NOT NULL, o INTEGER NOT NULL, d REAL, cr INTEGER NOT NULL, cw INTEGER NOT NULL,` +
+    ` usd REAL, st TEXT NOT NULL, code REAL, note TEXT, tools TEXT NOT NULL DEFAULT '[]', h TEXT, id TEXT);` +
+    `CREATE INDEX IF NOT EXISTS idx_messages_file ON messages(file_id);`;
+function storedParserVersion(db) {
+    try {
+        const stored = query(db, `SELECT v FROM meta WHERE k='parser_version';`);
+        return stored.length ? stored[0][0] : null;
+    }
+    catch {
+        return null;
+    }
+}
+function countMessages(db) {
+    try {
+        const rows = query(db, `SELECT COUNT(*) FROM messages;`);
+        return Number(rows[0]?.[0]) || 0;
+    }
+    catch {
+        return 0;
+    }
+}
+// Watermark for the sqlite scan: the newest opencode row already stored.
+function newestOpencodeMs(db) {
+    try {
+        const rows = query(db, `SELECT MAX(t) FROM messages WHERE h='opencode';`);
+        const v = Number(rows[0]?.[0]);
+        return Number.isFinite(v) && v > 0 ? v : null;
+    }
+    catch {
+        return null;
+    }
+}
+// Same pre-migration state as the newest backup: another copy churns the 3-slot rotation for nothing.
+function newestBackupMatches(stored) {
+    if (stored === null)
+        return false;
+    try {
+        const newest = listUsageBackups()[0];
+        if (!newest)
+            return false;
+        const out = query(newest.file, `SELECT v FROM meta WHERE k='parser_version';`);
+        return out.length > 0 && out[0][0] === stored;
+    }
+    catch {
+        return false;
+    }
+}
+// v9 shape (path-keyed rows) migrates in place with zero re-parse, so history survives even when transcripts are gone.
+function canMigrateInPlace(db) {
+    try {
+        const names = query(db, `PRAGMA table_info(messages);`).map((r) => r[1]);
+        return names.includes('file') && names.includes('h') && !names.includes('file_id');
+    }
+    catch {
+        return false;
+    }
+}
+function migrateInPlace(db, preCount) {
+    const cols = `t, model, i, o, d, cr, cw, usd, st, code, note, tools, h`;
+    try {
+        run(db, `BEGIN;` +
+            `CREATE TABLE files_new (id INTEGER PRIMARY KEY AUTOINCREMENT, path TEXT UNIQUE NOT NULL, mtime REAL NOT NULL, size INTEGER NOT NULL);` +
+            `INSERT INTO files_new (path, mtime, size) SELECT path, mtime, size FROM files;` +
+            `CREATE TABLE messages_new (file_id INTEGER NOT NULL REFERENCES files_new(id), t REAL, model TEXT NOT NULL,` +
+            ` i INTEGER NOT NULL, o INTEGER NOT NULL, d REAL, cr INTEGER NOT NULL, cw INTEGER NOT NULL,` +
+            ` usd REAL, st TEXT NOT NULL, code REAL, note TEXT, tools TEXT NOT NULL DEFAULT '[]', h TEXT, id TEXT);` +
+            `INSERT INTO messages_new (file_id, ${cols}) SELECT (SELECT id FROM files_new WHERE files_new.path = messages.file), ${cols} FROM messages;` +
+            `DROP TABLE messages;DROP TABLE files;` +
+            `ALTER TABLE files_new RENAME TO files;ALTER TABLE messages_new RENAME TO messages;` +
+            `CREATE INDEX idx_messages_file ON messages(file_id);` +
+            `COMMIT;`);
+    }
+    catch {
+        return false;
+    }
+    try {
+        const names = query(db, `PRAGMA table_info(messages);`).map((r) => r[1]);
+        return names.includes('file_id') && !names.includes('file') && countMessages(db) === preCount;
+    }
+    catch {
+        return false;
+    }
+}
+// v10 shape gains the message id with a bare ALTER: no rows move, so no re-parse and no guard trip.
+function canAlterInPlace(db) {
+    try {
+        const names = query(db, `PRAGMA table_info(messages);`).map((r) => r[1]);
+        return names.includes('file_id') && !names.includes('id');
+    }
+    catch {
+        return false;
+    }
+}
+function alterInPlace(db) {
+    try {
+        run(db, `ALTER TABLE messages ADD COLUMN id TEXT;`);
+    }
+    catch {
+        return false;
+    }
+    return !canAlterInPlace(db);
+}
 function ensureSchema(db) {
     fs.mkdirSync(path.dirname(db), { recursive: true });
-    // Fold in rows stored under a pre-alias spelling; the mtime ledger will
-    // not re-read those transcripts.
+    // Fold in rows stored under a pre-alias spelling; the mtime ledger will not re-read those transcripts.
     const rekeys = MODEL_ALIASES.map((a) => `UPDATE messages SET model='${a.id}' WHERE model='${a.feed}';`).join('');
-    run(db, `CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v TEXT NOT NULL);` +
-        `CREATE TABLE IF NOT EXISTS files (path TEXT PRIMARY KEY, mtime REAL NOT NULL, size INTEGER NOT NULL);` +
-        `CREATE TABLE IF NOT EXISTS messages (file TEXT NOT NULL, t REAL, model TEXT NOT NULL,` +
-        ` i INTEGER NOT NULL, o INTEGER NOT NULL, d REAL, cr INTEGER NOT NULL, cw INTEGER NOT NULL,` +
-        ` usd REAL, st TEXT NOT NULL, code REAL, note TEXT, tools TEXT NOT NULL DEFAULT '[]', h TEXT);` +
-        `CREATE INDEX IF NOT EXISTS idx_messages_file ON messages(file);` + rekeys);
-    // An older db has no h column; CREATE TABLE IF NOT EXISTS will not add one.
-    try {
-        run(db, `ALTER TABLE messages ADD COLUMN h TEXT;`);
+    const stored = storedParserVersion(db);
+    if (stored === PARSER_VERSION) {
+        run(db, SCHEMA_SQL + rekeys);
+        return;
     }
-    catch { /* already present */ }
-    // Host is derivable from the path, so backfill without re-reading transcripts.
+    // A version bump rebuilds both tables, so no ALTER patchwork survives a migration.
     try {
-        for (const [file] of query(db, `SELECT DISTINCT file FROM messages WHERE h IS NULL;`)) {
-            run(db, `UPDATE messages SET h='${esc(hostOfSessionFile(file))}' WHERE file=${esc(file)};`);
+        if (countMessages(db) > 0 && !newestBackupMatches(stored))
+            backupUsageDb(db);
+        if (stored !== null && canMigrateInPlace(db) && migrateInPlace(db, countMessages(db))) {
+            run(db, `INSERT OR REPLACE INTO meta (k,v) VALUES ('parser_version','${PARSER_VERSION}');`);
+            try {
+                run(db, `VACUUM;`);
+            }
+            catch { /* best-effort */ }
+            return;
         }
-    }
-    catch { /* fresh or unreadable db */ }
-    // Drop the cache when the parser has moved on, so every transcript is read
-    // again and the new columns actually fill.
-    try {
-        const stored = query(db, `SELECT v FROM meta WHERE k='parser_version';`);
-        if (stored.length && stored[0][0] === PARSER_VERSION)
+        if (stored !== null && canAlterInPlace(db) && alterInPlace(db)) {
+            run(db, `INSERT OR REPLACE INTO meta (k,v) VALUES ('parser_version','${PARSER_VERSION}');`);
             return;
-        backupUsageDb(db);
-        run(db, `DELETE FROM messages;DELETE FROM files;INSERT OR REPLACE INTO meta (k,v) VALUES ('parser_version','${PARSER_VERSION}');`);
+        }
+        run(db, `DROP TABLE IF EXISTS messages;DROP TABLE IF EXISTS files;` + SCHEMA_SQL +
+            `INSERT OR REPLACE INTO meta (k,v) VALUES ('parser_version','${PARSER_VERSION}');`);
     }
     catch { /* fresh db, nothing to invalidate */ }
 }
@@ -95,7 +204,7 @@ export function listUsageBackups() {
         return [];
     }
 }
-function backupUsageDb(db) {
+export function backupUsageDb(db) {
     try {
         const dir = backupUsageDir();
         fs.mkdirSync(dir, { recursive: true });
@@ -178,7 +287,26 @@ function readFiles(db) {
     catch { /* fresh db */ }
     return known;
 }
-function parseFile(text, host) {
+// OpenCode rows carry no status, code, note, or tool list; only tokens and the msg id.
+function opencodeStoredRow(oc, host, id) {
+    return {
+        t: oc.ms ?? null,
+        model: canonicalModelId(oc.model),
+        i: intOf(oc.usage.input),
+        o: intOf(oc.usage.output),
+        d: durOf(oc.durMs),
+        cr: intOf(oc.usage.cacheRead),
+        cw: intOf(oc.usage.cacheWrite),
+        usd: costOf(oc.usage),
+        st: 'completed',
+        code: undefined,
+        note: undefined,
+        tools: [],
+        host,
+        id,
+    };
+}
+function parseFile(text, host, file) {
     const rows = [];
     // Guarded: a JSONL transcript must fall through, not abort the sync.
     let oc = null;
@@ -187,21 +315,7 @@ function parseFile(text, host) {
     }
     catch { /* JSONL */ }
     if (oc) {
-        rows.push({
-            t: oc.ms ?? null,
-            model: canonicalModelId(oc.model),
-            i: intOf(oc.usage.input),
-            o: intOf(oc.usage.output),
-            d: durOf(oc.durMs),
-            cr: intOf(oc.usage.cacheRead),
-            cw: intOf(oc.usage.cacheWrite),
-            usd: costOf(oc.usage),
-            st: 'completed',
-            code: undefined,
-            note: undefined,
-            tools: [],
-            host,
-        });
+        rows.push(opencodeStoredRow(oc, host, file ? path.basename(file).replace(/\.[^.]+$/, '') : undefined));
         return rows;
     }
     let codexProvider = null;
@@ -211,6 +325,7 @@ function parseFile(text, host) {
             continue;
         try {
             const row = JSON.parse(line);
+            const rid = messageRowId(row);
             const parsed = classifySessionLine(row);
             if (parsed.kind === 'codex_provider') {
                 if (parsed.provider)
@@ -242,6 +357,7 @@ function parseFile(text, host) {
                     note: undefined,
                     tools: [],
                     host,
+                    id: rid,
                 });
                 continue;
             }
@@ -265,6 +381,7 @@ function parseFile(text, host) {
                 note,
                 tools: parsed.tools ?? [],
                 host,
+                id: rid,
             });
         }
         catch { /* skip corrupt lines */ }
@@ -272,17 +389,25 @@ function parseFile(text, host) {
     return rows;
 }
 function insertSql(file, r) {
-    return `INSERT INTO messages (file, t, model, i, o, d, cr, cw, usd, st, code, note, tools, h) VALUES (` +
-        `${esc(file)},${r.t === null ? 'NULL' : String(r.t)},${esc(r.model)},${r.i},${r.o},` +
+    return `INSERT INTO messages (file_id, t, model, i, o, d, cr, cw, usd, st, code, note, tools, h, id) VALUES (` +
+        `(SELECT id FROM files WHERE path=${esc(file)}),${r.t === null ? 'NULL' : String(r.t)},${esc(r.model)},${r.i},${r.o},` +
         `${nullNum(r.d)},${r.cr},${r.cw},${nullNum(r.usd)},${esc(r.st)},` +
-        `${nullNum(r.code)},${nullStr(r.note)},${esc(JSON.stringify(r.tools))},${nullStr(r.host)});`;
+        `${nullNum(r.code)},${nullStr(r.note)},${esc(JSON.stringify(r.tools))},${nullStr(r.host)},${nullStr(r.id)});`;
 }
-// Unchanged transcripts are skipped via mtime+size; rows for deleted ones are
-// kept so rotation never erases history. False when sqlite3 is unavailable.
+// Unchanged transcripts are skipped via mtime+size; rows for deleted ones are kept so rotation never erases history. False when sqlite3 is unavailable.
 export function syncUsageDb() {
     let db;
+    guardReport = null;
+    let preWipe = -1;
+    let migrated = false;
     try {
         db = usageDbPath();
+        const stored = storedParserVersion(db);
+        migrated = stored !== null && stored !== PARSER_VERSION;
+        // The operator deleted sessions on purpose and wants the mirror to match: bypass the guard.
+        const forced = process.env.TERSIO_FORCE_REPARSE === '1';
+        if (migrated && !forced)
+            preWipe = countMessages(db);
         ensureSchema(db);
         maybeScheduledBackup(db);
     }
@@ -294,9 +419,6 @@ export function syncUsageDb() {
     try {
         for (const dir of sessionsDirs())
             walkJsonl(dir, files, 2000);
-        if (process.env.TERSIO_SESSIONS_DIR === undefined || process.env.TERSIO_CODEX_DIR !== undefined) {
-            walkJsonl(codexSessionsDir(), files, 2000);
-        }
         // OpenCode stores one JSON document per message, not JSONL.
         if (process.env.TERSIO_SESSIONS_DIR === undefined || process.env.TERSIO_OPENCODE_DIR !== undefined) {
             walkJsonl(opencodeSessionsDir(), ocFiles, 5000, '.json');
@@ -306,6 +428,15 @@ export function syncUsageDb() {
         return false;
     }
     files.push(...ocFiles);
+    // v2 hosts keep messages in sqlite; rows past the watermark are new. Overlap re-reads the tail idempotently.
+    const ocDbRows = [];
+    const ocDbPath = opencodeDbPath();
+    if (process.env.TERSIO_SESSIONS_DIR === undefined || process.env.TERSIO_OPENCODE_DB !== undefined) {
+        const newest = newestOpencodeMs(db);
+        if (newest !== null || fs.existsSync(ocDbPath)) {
+            ocDbRows.push(...readOpencodeDbRows(ocDbPath, newest === null ? 0 : newest - OPENCODE_DB_OVERLAP_MS));
+        }
+    }
     const known = readFiles(db);
     const current = {};
     const changed = [];
@@ -323,7 +454,7 @@ export function syncUsageDb() {
         if (!prev || prev.mtime !== entry.mtime || prev.size !== entry.size)
             changed.push(file);
     }
-    if (!changed.length)
+    if (!changed.length && !ocDbRows.length)
         return true;
     const chunks = [];
     let chunkBytes = 0;
@@ -343,23 +474,36 @@ export function syncUsageDb() {
     const push = (sql) => {
         chunks.push(sql);
         chunkBytes += sql.length;
-        if (chunkBytes < 400_000)
+        // One argv string caps at 128KB on Linux (E2BIG); stay well under it.
+        if (chunkBytes < 100_000)
             return true;
         return flush();
     };
     for (const file of changed) {
-        if (!push(`DELETE FROM messages WHERE file=${esc(file)};DELETE FROM files WHERE path=${esc(file)};`))
+        const entry = current[file];
+        if (!push(`INSERT INTO files (path, mtime, size) VALUES (${esc(file)},${entry.mtime},${entry.size})` +
+            ` ON CONFLICT(path) DO UPDATE SET mtime=excluded.mtime,size=excluded.size;` +
+            `DELETE FROM messages WHERE file_id=(SELECT id FROM files WHERE path=${esc(file)});`))
             return false;
     }
     try {
         for (const file of changed) {
-            const entry = current[file];
             const text = fs.readFileSync(file, 'utf8');
-            for (const r of parseFile(text, hostOfSessionFile(file))) {
+            for (const r of parseFile(text, hostOfSessionFile(file), file)) {
                 if (!push(insertSql(file, r)))
                     return false;
             }
-            if (!push(`INSERT OR REPLACE INTO files (path, mtime, size) VALUES (${esc(file)},${entry.mtime},${entry.size});`))
+        }
+        for (const row of ocDbRows) {
+            const parsed = classifyOpencodeMessage(row.row);
+            if (!parsed)
+                continue;
+            // One synthetic path per row id, so the delete-and-reinsert that follows is idempotent.
+            const key = `${ocDbPath}#${row.id}`;
+            if (!push(`INSERT INTO files (path, mtime, size) VALUES (${esc(key)},${row.created},0)` +
+                ` ON CONFLICT(path) DO UPDATE SET mtime=excluded.mtime;` +
+                `DELETE FROM messages WHERE file_id=(SELECT id FROM files WHERE path=${esc(key)});` +
+                insertSql(key, opencodeStoredRow(parsed, 'opencode', row.id))))
                 return false;
         }
     }
@@ -368,7 +512,26 @@ export function syncUsageDb() {
     }
     if (!push(`INSERT OR REPLACE INTO meta (k, v) VALUES ('last_sync',${esc(String(Date.now()))});`))
         return false;
-    return flush();
+    if (!flush())
+        return false;
+    if (preWipe > 0) {
+        const kept = countMessages(db);
+        if (kept < preWipe * REPARSE_GUARD_RATIO) {
+            const newest = listUsageBackups()[0];
+            if (newest && restoreUsageBackup(path.basename(newest.file))) {
+                guardReport = `re-parse kept ${kept} of ${preWipe} rows; restored ${path.basename(newest.file)} (set TERSIO_FORCE_REPARSE=1 to accept the loss)`;
+                return false;
+            }
+        }
+    }
+    // Reclaim the migration's DELETE storm so the file stays small.
+    if (migrated) {
+        try {
+            run(db, `VACUUM;`);
+        }
+        catch { /* best-effort */ }
+    }
+    return true;
 }
 export function readUsageDb() {
     const db = usageDbPath();
@@ -381,13 +544,13 @@ export function readUsageDb() {
     }
     let rows;
     try {
-        rows = query(db, `SELECT t, model, i, o, d, cr, cw, usd, st, code, note, tools, h FROM messages;`);
+        rows = query(db, `SELECT t, model, i, o, d, cr, cw, usd, st, code, note, tools, h, id FROM messages;`);
     }
     catch {
         return null;
     }
     const accum = newSessionAccum();
-    for (const [t, model, i, o, d, cr, cw, usd, st, code, note, tools, h] of rows) {
+    for (const [t, model, i, o, d, cr, cw, usd, st, code, note, tools, h, id] of rows) {
         const ts = t === '' ? undefined : Number(t);
         let toolNames = [];
         try {
@@ -411,7 +574,7 @@ export function readUsageDb() {
             st: st === 'error' || st === 'aborted' ? st : 'completed',
             code: code === '' ? undefined : Number(code),
             note: note === '' ? undefined : note,
-        }, toolNames, h === '' ? undefined : h);
+        }, toolNames, h === '' ? undefined : h, id === '' || id === undefined ? undefined : id);
     }
     accum.recent.sort((a, b) => b.t - a.t);
     let syncedAt = 0;
@@ -430,6 +593,7 @@ export function readUsageDb() {
             byDayModel: accum.byDayModel,
             byTool: accum.byTool,
             byModelMessages: accum.byModelMessages,
+            byHost: accum.byHost,
             costMeasured: accum.costMeasured,
             recent: accum.recent.slice(0, RECENT_LIMIT),
         },