@krtclcdy/tersio 2.24.0 → 2.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -24
- package/cli/banner.js +7 -5
- package/cli/common.js +14 -13
- package/cli/currency.js +2 -7
- package/cli/dashboard.js +220 -13
- package/cli/doctor-fix.js +99 -29
- package/cli/doctor.js +32 -35
- package/cli/hosts.js +23 -26
- package/cli/install.js +228 -115
- package/cli/interactive.js +65 -21
- package/cli/manifest.js +8 -4
- package/cli/profile.js +49 -16
- package/cli/reset.js +3 -7
- package/cli/rtk-wiring.js +29 -32
- package/cli/settings.js +40 -16
- package/cli/uninstall.js +79 -29
- package/cli/update.js +27 -20
- package/cli/usage.js +54 -19
- package/dashboard/dist/index.html +18 -18
- package/extensions/ai-addons-updater/index.js +75 -20
- package/extensions/ai-addons-updater/index.ts +67 -23
- package/extensions/caveman-session/index.js +33 -75
- package/extensions/caveman-session/index.ts +35 -77
- package/extensions/combo-toggle/index.js +40 -62
- package/extensions/combo-toggle/index.ts +38 -72
- package/extensions/lib/utils.js +64 -15
- package/extensions/lib/utils.ts +62 -15
- package/extensions/opencode/index.js +104 -0
- package/extensions/opencode/server.js +190 -0
- package/extensions/opencode/server.ts +200 -0
- package/extensions/rtk-session/index.js +38 -57
- package/extensions/rtk-session/index.ts +36 -55
- package/extensions/shared/carbon.js +4 -10
- package/extensions/shared/carbon.ts +5 -10
- package/extensions/shared/host.js +95 -56
- package/extensions/shared/host.ts +106 -60
- package/extensions/shared/plugin-settings.js +6 -18
- package/extensions/shared/plugin-settings.ts +6 -18
- package/extensions/shared/pricing.js +79 -27
- package/extensions/shared/pricing.ts +80 -28
- package/extensions/shared/rtk-gain.js +4 -11
- package/extensions/shared/rtk-gain.ts +4 -11
- package/extensions/shared/session-state.js +28 -39
- package/extensions/shared/session-state.ts +29 -38
- package/extensions/shared/status.js +38 -0
- package/extensions/shared/status.ts +45 -0
- package/extensions/shared/types.js +1 -2
- package/extensions/shared/types.ts +2 -2
- package/extensions/shared/usage-ledger.js +312 -57
- package/extensions/shared/usage-ledger.ts +321 -61
- package/extensions/shared/usage-store.js +354 -38
- package/extensions/shared/usage-store.ts +360 -43
- package/extensions/tersio-commands/index.js +5 -14
- package/extensions/tersio-commands/index.ts +5 -14
- package/package.json +12 -7
- package/tersio.js +4 -6
- package/cli/agent-hosts.js +0 -42
- package/cli/agents.js +0 -266
- package/cli/layer.js +0 -24
- package/cli/omp-layer.js +0 -51
- package/cli/pi-layer.js +0 -37
- package/cli/pi-wiring.js +0 -92
- package/extensions/shared/omp-prompt.js +0 -12
|
@@ -1,15 +1,11 @@
|
|
|
1
|
-
// extensions/shared/usage-store.ts — tersio-owned usage.db.
|
|
2
|
-
// Persists the rows importSessionTokens derives live, so reports survive
|
|
3
|
-
// session-file rotation. Sync is incremental (a files ledger skips unchanged
|
|
4
|
-
// transcripts) and every read applies the reset watermark, so the store is a
|
|
5
|
-
// cache, never a fork. Missing sqlite3 → sync false / read null. Never touches
|
|
6
|
-
// RTK's history.db or host transcripts.
|
|
1
|
+
// extensions/shared/usage-store.ts — tersio-owned usage.db. A cache of the live parse, never a fork; missing sqlite3 → sync false / read null.
|
|
7
2
|
import { execFileSync } from 'node:child_process';
|
|
8
3
|
import fs from 'node:fs';
|
|
9
4
|
import path from 'node:path';
|
|
10
|
-
import { RECENT_LIMIT, canonicalModelId, classifySessionLine,
|
|
5
|
+
import { RECENT_LIMIT, SQLITE_READ_BUFFER, canonicalModelId, classifySessionLine, classifyOpencodeMessage, costOf, durOf, hostOfSessionFile, ingestSessionRow, messageRowId, OPENCODE_DB_OVERLAP_MS, opencodeDbPath, opencodeSessionsDir, newSessionAccum, readOpencodeDbRows, sessionsDirs, walkJsonl, } from './usage-ledger.js';
|
|
11
6
|
import { tersioDataPath } from '../lib/utils.js';
|
|
12
7
|
import { MODEL_ALIASES } from './pricing.js';
|
|
8
|
+
import { tersioSettingsFile } from './plugin-settings.js';
|
|
13
9
|
export function usageDbPath() {
|
|
14
10
|
const override = process.env.TERSIO_USAGE_DB;
|
|
15
11
|
if (override)
|
|
@@ -29,31 +25,257 @@ function nullStr(v) {
|
|
|
29
25
|
return v === undefined ? 'NULL' : esc(v);
|
|
30
26
|
}
|
|
31
27
|
function run(db, sql) {
|
|
32
|
-
|
|
28
|
+
// -bail stops the script before COMMIT, so a failed INSERT never leaves DELETEs behind.
|
|
29
|
+
execFileSync('sqlite3', ['-bail', db, sql], { stdio: ['ignore', 'ignore', 'ignore'], timeout: 30000 });
|
|
33
30
|
}
|
|
34
31
|
function query(db, sql) {
|
|
35
32
|
const out = execFileSync('sqlite3', ['-separator', '\t', db, sql], {
|
|
36
33
|
encoding: 'utf8',
|
|
37
34
|
timeout: 30000,
|
|
38
35
|
stdio: ['ignore', 'pipe', 'ignore'],
|
|
36
|
+
maxBuffer: SQLITE_READ_BUFFER,
|
|
39
37
|
});
|
|
40
38
|
return out
|
|
41
39
|
.split('\n')
|
|
42
40
|
.filter((line) => line.trim())
|
|
43
41
|
.map((line) => line.split('\t'));
|
|
44
42
|
}
|
|
43
|
+
// Bump on a parse change: unchanged transcripts are never re-read.
|
|
44
|
+
const PARSER_VERSION = '11';
|
|
45
|
+
// A re-parse that keeps less than half the rows means transcripts vanished mid-migration; the backup is restored instead of publishing the loss.
|
|
46
|
+
const REPARSE_GUARD_RATIO = 0.5;
|
|
47
|
+
let guardReport = null;
|
|
48
|
+
/** Why the last sync refused to publish a re-parse, if it did. */
|
|
49
|
+
export function reparseGuardReport() {
|
|
50
|
+
return guardReport;
|
|
51
|
+
}
|
|
52
|
+
const SCHEMA_SQL = `CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v TEXT NOT NULL);` +
|
|
53
|
+
`CREATE TABLE IF NOT EXISTS files (id INTEGER PRIMARY KEY AUTOINCREMENT, path TEXT UNIQUE NOT NULL, mtime REAL NOT NULL, size INTEGER NOT NULL);` +
|
|
54
|
+
`CREATE TABLE IF NOT EXISTS messages (file_id INTEGER NOT NULL REFERENCES files(id), t REAL, model TEXT NOT NULL,` +
|
|
55
|
+
` i INTEGER NOT NULL, o INTEGER NOT NULL, d REAL, cr INTEGER NOT NULL, cw INTEGER NOT NULL,` +
|
|
56
|
+
` usd REAL, st TEXT NOT NULL, code REAL, note TEXT, tools TEXT NOT NULL DEFAULT '[]', h TEXT, id TEXT);` +
|
|
57
|
+
`CREATE INDEX IF NOT EXISTS idx_messages_file ON messages(file_id);`;
|
|
58
|
+
function storedParserVersion(db) {
|
|
59
|
+
try {
|
|
60
|
+
const stored = query(db, `SELECT v FROM meta WHERE k='parser_version';`);
|
|
61
|
+
return stored.length ? stored[0][0] : null;
|
|
62
|
+
}
|
|
63
|
+
catch {
|
|
64
|
+
return null;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
function countMessages(db) {
|
|
68
|
+
try {
|
|
69
|
+
const rows = query(db, `SELECT COUNT(*) FROM messages;`);
|
|
70
|
+
return Number(rows[0]?.[0]) || 0;
|
|
71
|
+
}
|
|
72
|
+
catch {
|
|
73
|
+
return 0;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
// Watermark for the sqlite scan: the newest opencode row already stored.
|
|
77
|
+
function newestOpencodeMs(db) {
|
|
78
|
+
try {
|
|
79
|
+
const rows = query(db, `SELECT MAX(t) FROM messages WHERE h='opencode';`);
|
|
80
|
+
const v = Number(rows[0]?.[0]);
|
|
81
|
+
return Number.isFinite(v) && v > 0 ? v : null;
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
return null;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
// Same pre-migration state as the newest backup: another copy churns the 3-slot rotation for nothing.
|
|
88
|
+
function newestBackupMatches(stored) {
|
|
89
|
+
if (stored === null)
|
|
90
|
+
return false;
|
|
91
|
+
try {
|
|
92
|
+
const newest = listUsageBackups()[0];
|
|
93
|
+
if (!newest)
|
|
94
|
+
return false;
|
|
95
|
+
const out = query(newest.file, `SELECT v FROM meta WHERE k='parser_version';`);
|
|
96
|
+
return out.length > 0 && out[0][0] === stored;
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
return false;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
// v9 shape (path-keyed rows) migrates in place with zero re-parse, so history survives even when transcripts are gone.
|
|
103
|
+
function canMigrateInPlace(db) {
|
|
104
|
+
try {
|
|
105
|
+
const names = query(db, `PRAGMA table_info(messages);`).map((r) => r[1]);
|
|
106
|
+
return names.includes('file') && names.includes('h') && !names.includes('file_id');
|
|
107
|
+
}
|
|
108
|
+
catch {
|
|
109
|
+
return false;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
function migrateInPlace(db, preCount) {
|
|
113
|
+
const cols = `t, model, i, o, d, cr, cw, usd, st, code, note, tools, h`;
|
|
114
|
+
try {
|
|
115
|
+
run(db, `BEGIN;` +
|
|
116
|
+
`CREATE TABLE files_new (id INTEGER PRIMARY KEY AUTOINCREMENT, path TEXT UNIQUE NOT NULL, mtime REAL NOT NULL, size INTEGER NOT NULL);` +
|
|
117
|
+
`INSERT INTO files_new (path, mtime, size) SELECT path, mtime, size FROM files;` +
|
|
118
|
+
`CREATE TABLE messages_new (file_id INTEGER NOT NULL REFERENCES files_new(id), t REAL, model TEXT NOT NULL,` +
|
|
119
|
+
` i INTEGER NOT NULL, o INTEGER NOT NULL, d REAL, cr INTEGER NOT NULL, cw INTEGER NOT NULL,` +
|
|
120
|
+
` usd REAL, st TEXT NOT NULL, code REAL, note TEXT, tools TEXT NOT NULL DEFAULT '[]', h TEXT, id TEXT);` +
|
|
121
|
+
`INSERT INTO messages_new (file_id, ${cols}) SELECT (SELECT id FROM files_new WHERE files_new.path = messages.file), ${cols} FROM messages;` +
|
|
122
|
+
`DROP TABLE messages;DROP TABLE files;` +
|
|
123
|
+
`ALTER TABLE files_new RENAME TO files;ALTER TABLE messages_new RENAME TO messages;` +
|
|
124
|
+
`CREATE INDEX idx_messages_file ON messages(file_id);` +
|
|
125
|
+
`COMMIT;`);
|
|
126
|
+
}
|
|
127
|
+
catch {
|
|
128
|
+
return false;
|
|
129
|
+
}
|
|
130
|
+
try {
|
|
131
|
+
const names = query(db, `PRAGMA table_info(messages);`).map((r) => r[1]);
|
|
132
|
+
return names.includes('file_id') && !names.includes('file') && countMessages(db) === preCount;
|
|
133
|
+
}
|
|
134
|
+
catch {
|
|
135
|
+
return false;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
// v10 shape gains the message id with a bare ALTER: no rows move, so no re-parse and no guard trip.
|
|
139
|
+
function canAlterInPlace(db) {
|
|
140
|
+
try {
|
|
141
|
+
const names = query(db, `PRAGMA table_info(messages);`).map((r) => r[1]);
|
|
142
|
+
return names.includes('file_id') && !names.includes('id');
|
|
143
|
+
}
|
|
144
|
+
catch {
|
|
145
|
+
return false;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
function alterInPlace(db) {
|
|
149
|
+
try {
|
|
150
|
+
run(db, `ALTER TABLE messages ADD COLUMN id TEXT;`);
|
|
151
|
+
}
|
|
152
|
+
catch {
|
|
153
|
+
return false;
|
|
154
|
+
}
|
|
155
|
+
return !canAlterInPlace(db);
|
|
156
|
+
}
|
|
45
157
|
function ensureSchema(db) {
|
|
46
158
|
fs.mkdirSync(path.dirname(db), { recursive: true });
|
|
47
|
-
//
|
|
48
|
-
// with, and the mtime ledger will not re-read those transcripts. Fold them in
|
|
49
|
-
// once, so the merge shows up without a reset.
|
|
159
|
+
// Fold in rows stored under a pre-alias spelling; the mtime ledger will not re-read those transcripts.
|
|
50
160
|
const rekeys = MODEL_ALIASES.map((a) => `UPDATE messages SET model='${a.id}' WHERE model='${a.feed}';`).join('');
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
161
|
+
const stored = storedParserVersion(db);
|
|
162
|
+
if (stored === PARSER_VERSION) {
|
|
163
|
+
run(db, SCHEMA_SQL + rekeys);
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
// A version bump rebuilds both tables, so no ALTER patchwork survives a migration.
|
|
167
|
+
try {
|
|
168
|
+
if (countMessages(db) > 0 && !newestBackupMatches(stored))
|
|
169
|
+
backupUsageDb(db);
|
|
170
|
+
if (stored !== null && canMigrateInPlace(db) && migrateInPlace(db, countMessages(db))) {
|
|
171
|
+
run(db, `INSERT OR REPLACE INTO meta (k,v) VALUES ('parser_version','${PARSER_VERSION}');`);
|
|
172
|
+
try {
|
|
173
|
+
run(db, `VACUUM;`);
|
|
174
|
+
}
|
|
175
|
+
catch { /* best-effort */ }
|
|
176
|
+
return;
|
|
177
|
+
}
|
|
178
|
+
if (stored !== null && canAlterInPlace(db) && alterInPlace(db)) {
|
|
179
|
+
run(db, `INSERT OR REPLACE INTO meta (k,v) VALUES ('parser_version','${PARSER_VERSION}');`);
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
run(db, `DROP TABLE IF EXISTS messages;DROP TABLE IF EXISTS files;` + SCHEMA_SQL +
|
|
183
|
+
`INSERT OR REPLACE INTO meta (k,v) VALUES ('parser_version','${PARSER_VERSION}');`);
|
|
184
|
+
}
|
|
185
|
+
catch { /* fresh db, nothing to invalidate */ }
|
|
186
|
+
}
|
|
187
|
+
// The one destructive thing we do; keep the last few so a bad bump is recoverable.
|
|
188
|
+
const BACKUP_KEEP = 3;
|
|
189
|
+
export function backupUsageDir() {
|
|
190
|
+
return path.join(path.dirname(usageDbPath()), 'backups');
|
|
191
|
+
}
|
|
192
|
+
export function listUsageBackups() {
|
|
193
|
+
try {
|
|
194
|
+
return fs.readdirSync(backupUsageDir())
|
|
195
|
+
.filter((n) => /^usage-\d{14}\.db$/.test(n))
|
|
196
|
+
.map((n) => {
|
|
197
|
+
const full = path.join(backupUsageDir(), n);
|
|
198
|
+
const st = fs.statSync(full);
|
|
199
|
+
return { file: full, mtime: st.mtimeMs, size: st.size };
|
|
200
|
+
})
|
|
201
|
+
.sort((a, b) => b.mtime - a.mtime);
|
|
202
|
+
}
|
|
203
|
+
catch {
|
|
204
|
+
return [];
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
export function backupUsageDb(db) {
|
|
208
|
+
try {
|
|
209
|
+
const dir = backupUsageDir();
|
|
210
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
211
|
+
const stamp = new Date().toISOString().replace(/[-:T]/g, '').slice(0, 14);
|
|
212
|
+
fs.copyFileSync(db, path.join(dir, `usage-${stamp}.db`));
|
|
213
|
+
for (const old of listUsageBackups().slice(BACKUP_KEEP)) {
|
|
214
|
+
fs.rmSync(old.file, { force: true });
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
catch { /* a missing backup must not block the sync */ }
|
|
218
|
+
}
|
|
219
|
+
const SCHEDULE_MS = { daily: 86_400_000, weekly: 604_800_000, monthly: 2_592_000_000 };
|
|
220
|
+
/** Read the schedule straight from settings; this module must not import cli/. */
|
|
221
|
+
export function readBackupSchedule() {
|
|
222
|
+
try {
|
|
223
|
+
const raw = JSON.parse(fs.readFileSync(tersioSettingsFile(), 'utf8'));
|
|
224
|
+
return typeof raw?.backupSchedule === 'string' ? raw.backupSchedule : 'monthly';
|
|
225
|
+
}
|
|
226
|
+
catch {
|
|
227
|
+
return 'monthly';
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
/** Snapshot on the schedule, skipping when the newest one is still fresh. */
|
|
231
|
+
export function maybeScheduledBackup(db) {
|
|
232
|
+
const every = SCHEDULE_MS[readBackupSchedule()];
|
|
233
|
+
if (!every)
|
|
234
|
+
return false;
|
|
235
|
+
try {
|
|
236
|
+
const newest = listUsageBackups()[0];
|
|
237
|
+
if (newest && Date.now() - newest.mtime < every)
|
|
238
|
+
return false;
|
|
239
|
+
backupUsageDb(db);
|
|
240
|
+
return true;
|
|
241
|
+
}
|
|
242
|
+
catch {
|
|
243
|
+
return false;
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
// basename only: a crafted `../` must not escape the backup dir.
|
|
247
|
+
function resolveBackupFile(file) {
|
|
248
|
+
const dir = path.resolve(backupUsageDir());
|
|
249
|
+
const target = path.join(dir, path.basename(String(file || '')));
|
|
250
|
+
if (!target.startsWith(dir + path.sep))
|
|
251
|
+
return null;
|
|
252
|
+
return fs.existsSync(target) ? target : null;
|
|
253
|
+
}
|
|
254
|
+
/** Put a backup back in place. The next sync then re-parses over it. */
|
|
255
|
+
export function restoreUsageBackup(file) {
|
|
256
|
+
try {
|
|
257
|
+
const src = resolveBackupFile(file);
|
|
258
|
+
if (!src)
|
|
259
|
+
return false;
|
|
260
|
+
fs.copyFileSync(src, usageDbPath());
|
|
261
|
+
return true;
|
|
262
|
+
}
|
|
263
|
+
catch {
|
|
264
|
+
return false;
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
/** Delete one snapshot. Only ever touches a file inside the backup dir. */
|
|
268
|
+
export function deleteUsageBackup(file) {
|
|
269
|
+
try {
|
|
270
|
+
const target = resolveBackupFile(file);
|
|
271
|
+
if (!target)
|
|
272
|
+
return false;
|
|
273
|
+
fs.rmSync(target, { force: true });
|
|
274
|
+
return true;
|
|
275
|
+
}
|
|
276
|
+
catch {
|
|
277
|
+
return false;
|
|
278
|
+
}
|
|
57
279
|
}
|
|
58
280
|
function readFiles(db) {
|
|
59
281
|
const known = {};
|
|
@@ -65,20 +287,56 @@ function readFiles(db) {
|
|
|
65
287
|
catch { /* fresh db */ }
|
|
66
288
|
return known;
|
|
67
289
|
}
|
|
68
|
-
|
|
290
|
+
// OpenCode rows carry no status, code, note, or tool list; only tokens and the msg id.
|
|
291
|
+
function opencodeStoredRow(oc, host, id) {
|
|
292
|
+
return {
|
|
293
|
+
t: oc.ms ?? null,
|
|
294
|
+
model: canonicalModelId(oc.model),
|
|
295
|
+
i: intOf(oc.usage.input),
|
|
296
|
+
o: intOf(oc.usage.output),
|
|
297
|
+
d: durOf(oc.durMs),
|
|
298
|
+
cr: intOf(oc.usage.cacheRead),
|
|
299
|
+
cw: intOf(oc.usage.cacheWrite),
|
|
300
|
+
usd: costOf(oc.usage),
|
|
301
|
+
st: 'completed',
|
|
302
|
+
code: undefined,
|
|
303
|
+
note: undefined,
|
|
304
|
+
tools: [],
|
|
305
|
+
host,
|
|
306
|
+
id,
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
function parseFile(text, host, file) {
|
|
69
310
|
const rows = [];
|
|
311
|
+
// Guarded: a JSONL transcript must fall through, not abort the sync.
|
|
312
|
+
let oc = null;
|
|
313
|
+
try {
|
|
314
|
+
oc = classifyOpencodeMessage(JSON.parse(text));
|
|
315
|
+
}
|
|
316
|
+
catch { /* JSONL */ }
|
|
317
|
+
if (oc) {
|
|
318
|
+
rows.push(opencodeStoredRow(oc, host, file ? path.basename(file).replace(/\.[^.]+$/, '') : undefined));
|
|
319
|
+
return rows;
|
|
320
|
+
}
|
|
70
321
|
let codexProvider = null;
|
|
322
|
+
let codexModel = null;
|
|
71
323
|
for (const line of text.split('\n')) {
|
|
72
324
|
if (!line.trim())
|
|
73
325
|
continue;
|
|
74
326
|
try {
|
|
75
327
|
const row = JSON.parse(line);
|
|
328
|
+
const rid = messageRowId(row);
|
|
76
329
|
const parsed = classifySessionLine(row);
|
|
77
330
|
if (parsed.kind === 'codex_provider') {
|
|
78
331
|
if (parsed.provider)
|
|
79
332
|
codexProvider = parsed.provider;
|
|
80
333
|
continue;
|
|
81
334
|
}
|
|
335
|
+
if (parsed.kind === 'codex_model') {
|
|
336
|
+
if (parsed.model)
|
|
337
|
+
codexModel = parsed.model;
|
|
338
|
+
continue;
|
|
339
|
+
}
|
|
82
340
|
const ts = row.timestamp;
|
|
83
341
|
const ms = ts === undefined ? NaN : typeof ts === 'number' ? ts : Date.parse(ts);
|
|
84
342
|
const t = Number.isFinite(ms) ? ms : null;
|
|
@@ -87,7 +345,7 @@ function parseFile(text) {
|
|
|
87
345
|
const usage = parsed.usage ?? {};
|
|
88
346
|
rows.push({
|
|
89
347
|
t,
|
|
90
|
-
model: canonicalModelId(provider ? `codex/${provider}` : 'codex'),
|
|
348
|
+
model: canonicalModelId(provider ? (codexModel ? `codex/${provider}/${codexModel}` : `codex/${provider}`) : 'codex'),
|
|
91
349
|
i: intOf(usage.input),
|
|
92
350
|
o: intOf(usage.output),
|
|
93
351
|
d: undefined,
|
|
@@ -98,6 +356,8 @@ function parseFile(text) {
|
|
|
98
356
|
code: undefined,
|
|
99
357
|
note: undefined,
|
|
100
358
|
tools: [],
|
|
359
|
+
host,
|
|
360
|
+
id: rid,
|
|
101
361
|
});
|
|
102
362
|
continue;
|
|
103
363
|
}
|
|
@@ -120,6 +380,8 @@ function parseFile(text) {
|
|
|
120
380
|
code: run.code,
|
|
121
381
|
note,
|
|
122
382
|
tools: parsed.tools ?? [],
|
|
383
|
+
host,
|
|
384
|
+
id: rid,
|
|
123
385
|
});
|
|
124
386
|
}
|
|
125
387
|
catch { /* skip corrupt lines */ }
|
|
@@ -127,33 +389,54 @@ function parseFile(text) {
|
|
|
127
389
|
return rows;
|
|
128
390
|
}
|
|
129
391
|
function insertSql(file, r) {
|
|
130
|
-
return `INSERT INTO messages (
|
|
131
|
-
|
|
392
|
+
return `INSERT INTO messages (file_id, t, model, i, o, d, cr, cw, usd, st, code, note, tools, h, id) VALUES (` +
|
|
393
|
+
`(SELECT id FROM files WHERE path=${esc(file)}),${r.t === null ? 'NULL' : String(r.t)},${esc(r.model)},${r.i},${r.o},` +
|
|
132
394
|
`${nullNum(r.d)},${r.cr},${r.cw},${nullNum(r.usd)},${esc(r.st)},` +
|
|
133
|
-
`${nullNum(r.code)},${nullStr(r.note)},${esc(JSON.stringify(r.tools))});`;
|
|
395
|
+
`${nullNum(r.code)},${nullStr(r.note)},${esc(JSON.stringify(r.tools))},${nullStr(r.host)},${nullStr(r.id)});`;
|
|
134
396
|
}
|
|
135
|
-
//
|
|
136
|
-
// ones re-inserted. Rows for deleted transcripts are kept, so rotation never
|
|
137
|
-
// erases history. False when sqlite3 or disk is unavailable.
|
|
397
|
+
// Unchanged transcripts are skipped via mtime+size; rows for deleted ones are kept so rotation never erases history. False when sqlite3 is unavailable.
|
|
138
398
|
export function syncUsageDb() {
|
|
139
399
|
let db;
|
|
400
|
+
guardReport = null;
|
|
401
|
+
let preWipe = -1;
|
|
402
|
+
let migrated = false;
|
|
140
403
|
try {
|
|
141
404
|
db = usageDbPath();
|
|
405
|
+
const stored = storedParserVersion(db);
|
|
406
|
+
migrated = stored !== null && stored !== PARSER_VERSION;
|
|
407
|
+
// The operator deleted sessions on purpose and wants the mirror to match: bypass the guard.
|
|
408
|
+
const forced = process.env.TERSIO_FORCE_REPARSE === '1';
|
|
409
|
+
if (migrated && !forced)
|
|
410
|
+
preWipe = countMessages(db);
|
|
142
411
|
ensureSchema(db);
|
|
412
|
+
maybeScheduledBackup(db);
|
|
143
413
|
}
|
|
144
414
|
catch {
|
|
145
415
|
return false;
|
|
146
416
|
}
|
|
147
417
|
const files = [];
|
|
418
|
+
const ocFiles = [];
|
|
148
419
|
try {
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
420
|
+
for (const dir of sessionsDirs())
|
|
421
|
+
walkJsonl(dir, files, 2000);
|
|
422
|
+
// OpenCode stores one JSON document per message, not JSONL.
|
|
423
|
+
if (process.env.TERSIO_SESSIONS_DIR === undefined || process.env.TERSIO_OPENCODE_DIR !== undefined) {
|
|
424
|
+
walkJsonl(opencodeSessionsDir(), ocFiles, 5000, '.json');
|
|
152
425
|
}
|
|
153
426
|
}
|
|
154
427
|
catch {
|
|
155
428
|
return false;
|
|
156
429
|
}
|
|
430
|
+
files.push(...ocFiles);
|
|
431
|
+
// v2 hosts keep messages in sqlite; rows past the watermark are new. Overlap re-reads the tail idempotently.
|
|
432
|
+
const ocDbRows = [];
|
|
433
|
+
const ocDbPath = opencodeDbPath();
|
|
434
|
+
if (process.env.TERSIO_SESSIONS_DIR === undefined || process.env.TERSIO_OPENCODE_DB !== undefined) {
|
|
435
|
+
const newest = newestOpencodeMs(db);
|
|
436
|
+
if (newest !== null || fs.existsSync(ocDbPath)) {
|
|
437
|
+
ocDbRows.push(...readOpencodeDbRows(ocDbPath, newest === null ? 0 : newest - OPENCODE_DB_OVERLAP_MS));
|
|
438
|
+
}
|
|
439
|
+
}
|
|
157
440
|
const known = readFiles(db);
|
|
158
441
|
const current = {};
|
|
159
442
|
const changed = [];
|
|
@@ -171,7 +454,7 @@ export function syncUsageDb() {
|
|
|
171
454
|
if (!prev || prev.mtime !== entry.mtime || prev.size !== entry.size)
|
|
172
455
|
changed.push(file);
|
|
173
456
|
}
|
|
174
|
-
if (!changed.length)
|
|
457
|
+
if (!changed.length && !ocDbRows.length)
|
|
175
458
|
return true;
|
|
176
459
|
const chunks = [];
|
|
177
460
|
let chunkBytes = 0;
|
|
@@ -191,23 +474,36 @@ export function syncUsageDb() {
|
|
|
191
474
|
const push = (sql) => {
|
|
192
475
|
chunks.push(sql);
|
|
193
476
|
chunkBytes += sql.length;
|
|
194
|
-
|
|
477
|
+
// One argv string caps at 128KB on Linux (E2BIG); stay well under it.
|
|
478
|
+
if (chunkBytes < 100_000)
|
|
195
479
|
return true;
|
|
196
480
|
return flush();
|
|
197
481
|
};
|
|
198
482
|
for (const file of changed) {
|
|
199
|
-
|
|
483
|
+
const entry = current[file];
|
|
484
|
+
if (!push(`INSERT INTO files (path, mtime, size) VALUES (${esc(file)},${entry.mtime},${entry.size})` +
|
|
485
|
+
` ON CONFLICT(path) DO UPDATE SET mtime=excluded.mtime,size=excluded.size;` +
|
|
486
|
+
`DELETE FROM messages WHERE file_id=(SELECT id FROM files WHERE path=${esc(file)});`))
|
|
200
487
|
return false;
|
|
201
488
|
}
|
|
202
489
|
try {
|
|
203
490
|
for (const file of changed) {
|
|
204
|
-
const entry = current[file];
|
|
205
491
|
const text = fs.readFileSync(file, 'utf8');
|
|
206
|
-
for (const r of parseFile(text)) {
|
|
492
|
+
for (const r of parseFile(text, hostOfSessionFile(file), file)) {
|
|
207
493
|
if (!push(insertSql(file, r)))
|
|
208
494
|
return false;
|
|
209
495
|
}
|
|
210
|
-
|
|
496
|
+
}
|
|
497
|
+
for (const row of ocDbRows) {
|
|
498
|
+
const parsed = classifyOpencodeMessage(row.row);
|
|
499
|
+
if (!parsed)
|
|
500
|
+
continue;
|
|
501
|
+
// One synthetic path per row id, so the delete-and-reinsert that follows is idempotent.
|
|
502
|
+
const key = `${ocDbPath}#${row.id}`;
|
|
503
|
+
if (!push(`INSERT INTO files (path, mtime, size) VALUES (${esc(key)},${row.created},0)` +
|
|
504
|
+
` ON CONFLICT(path) DO UPDATE SET mtime=excluded.mtime;` +
|
|
505
|
+
`DELETE FROM messages WHERE file_id=(SELECT id FROM files WHERE path=${esc(key)});` +
|
|
506
|
+
insertSql(key, opencodeStoredRow(parsed, 'opencode', row.id))))
|
|
211
507
|
return false;
|
|
212
508
|
}
|
|
213
509
|
}
|
|
@@ -216,7 +512,26 @@ export function syncUsageDb() {
|
|
|
216
512
|
}
|
|
217
513
|
if (!push(`INSERT OR REPLACE INTO meta (k, v) VALUES ('last_sync',${esc(String(Date.now()))});`))
|
|
218
514
|
return false;
|
|
219
|
-
|
|
515
|
+
if (!flush())
|
|
516
|
+
return false;
|
|
517
|
+
if (preWipe > 0) {
|
|
518
|
+
const kept = countMessages(db);
|
|
519
|
+
if (kept < preWipe * REPARSE_GUARD_RATIO) {
|
|
520
|
+
const newest = listUsageBackups()[0];
|
|
521
|
+
if (newest && restoreUsageBackup(path.basename(newest.file))) {
|
|
522
|
+
guardReport = `re-parse kept ${kept} of ${preWipe} rows; restored ${path.basename(newest.file)} (set TERSIO_FORCE_REPARSE=1 to accept the loss)`;
|
|
523
|
+
return false;
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
// Reclaim the migration's DELETE storm so the file stays small.
|
|
528
|
+
if (migrated) {
|
|
529
|
+
try {
|
|
530
|
+
run(db, `VACUUM;`);
|
|
531
|
+
}
|
|
532
|
+
catch { /* best-effort */ }
|
|
533
|
+
}
|
|
534
|
+
return true;
|
|
220
535
|
}
|
|
221
536
|
export function readUsageDb() {
|
|
222
537
|
const db = usageDbPath();
|
|
@@ -229,13 +544,13 @@ export function readUsageDb() {
|
|
|
229
544
|
}
|
|
230
545
|
let rows;
|
|
231
546
|
try {
|
|
232
|
-
rows = query(db, `SELECT t, model, i, o, d, cr, cw, usd, st, code, note, tools FROM messages;`);
|
|
547
|
+
rows = query(db, `SELECT t, model, i, o, d, cr, cw, usd, st, code, note, tools, h, id FROM messages;`);
|
|
233
548
|
}
|
|
234
549
|
catch {
|
|
235
550
|
return null;
|
|
236
551
|
}
|
|
237
552
|
const accum = newSessionAccum();
|
|
238
|
-
for (const [t, model, i, o, d, cr, cw, usd, st, code, note, tools] of rows) {
|
|
553
|
+
for (const [t, model, i, o, d, cr, cw, usd, st, code, note, tools, h, id] of rows) {
|
|
239
554
|
const ts = t === '' ? undefined : Number(t);
|
|
240
555
|
let toolNames = [];
|
|
241
556
|
try {
|
|
@@ -259,7 +574,7 @@ export function readUsageDb() {
|
|
|
259
574
|
st: st === 'error' || st === 'aborted' ? st : 'completed',
|
|
260
575
|
code: code === '' ? undefined : Number(code),
|
|
261
576
|
note: note === '' ? undefined : note,
|
|
262
|
-
}, toolNames);
|
|
577
|
+
}, toolNames, h === '' ? undefined : h, id === '' || id === undefined ? undefined : id);
|
|
263
578
|
}
|
|
264
579
|
accum.recent.sort((a, b) => b.t - a.t);
|
|
265
580
|
let syncedAt = 0;
|
|
@@ -278,6 +593,7 @@ export function readUsageDb() {
|
|
|
278
593
|
byDayModel: accum.byDayModel,
|
|
279
594
|
byTool: accum.byTool,
|
|
280
595
|
byModelMessages: accum.byModelMessages,
|
|
596
|
+
byHost: accum.byHost,
|
|
281
597
|
costMeasured: accum.costMeasured,
|
|
282
598
|
recent: accum.recent.slice(0, RECENT_LIMIT),
|
|
283
599
|
},
|