@oh-my-pi/omp-stats 18.7.0 → 18.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/dist/client/index.css +1 -1
- package/dist/client/index.js +96 -96
- package/dist/types/aggregator.d.ts +3 -0
- package/dist/types/client/api.d.ts +5 -1
- package/dist/types/client/data/live.d.ts +16 -11
- package/dist/types/client/data/query-store.d.ts +39 -0
- package/dist/types/client/data/query.d.ts +10 -5
- package/dist/types/client/traces/Minimap.d.ts +2 -0
- package/dist/types/client/traces/TimelineCanvas.d.ts +29 -8
- package/dist/types/client/traces/TranscriptList.d.ts +3 -1
- package/dist/types/db.d.ts +14 -2
- package/dist/types/embedded-client.d.ts +5 -0
- package/dist/types/live.d.ts +5 -3
- package/dist/types/rollup.d.ts +9 -1
- package/dist/types/trace.d.ts +18 -6
- package/package.json +4 -4
- package/src/aggregator.ts +26 -9
- package/src/client/api.ts +40 -5
- package/src/client/app/LiveChip.tsx +2 -2
- package/src/client/app/Shell.tsx +2 -2
- package/src/client/data/live.tsx +85 -45
- package/src/client/data/query-store.ts +108 -0
- package/src/client/data/query.ts +47 -56
- package/src/client/routes/CostsRoute.tsx +4 -2
- package/src/client/routes/ErrorsRoute.tsx +3 -1
- package/src/client/routes/FrustrationRoute.tsx +1 -1
- package/src/client/routes/GainRoute.tsx +3 -1
- package/src/client/routes/ModelsRoute.tsx +9 -2
- package/src/client/routes/OverviewRoute.tsx +2 -2
- package/src/client/routes/ProjectsRoute.tsx +1 -1
- package/src/client/routes/ProvidersRoute.tsx +5 -3
- package/src/client/routes/RequestsRoute.tsx +1 -1
- package/src/client/routes/ToolsRoute.tsx +1 -1
- package/src/client/routes/TracesRoute.tsx +5 -4
- package/src/client/traces/Minimap.tsx +63 -26
- package/src/client/traces/TimelineCanvas.tsx +224 -105
- package/src/client/traces/TraceView.tsx +29 -10
- package/src/client/traces/TranscriptList.tsx +17 -9
- package/src/client/traces/traces.css +26 -0
- package/src/client/ui/RequestDrawer.tsx +3 -1
- package/src/db.ts +106 -64
- package/src/embedded-client.ts +9 -0
- package/src/live.ts +36 -11
- package/src/parser.ts +1 -8
- package/src/rollup.ts +41 -20
- package/src/server.ts +38 -24
- package/src/trace.ts +54 -40
package/src/live.ts
CHANGED
|
@@ -5,15 +5,16 @@
|
|
|
5
5
|
* The dashboard never waits for ingest. {@link StatsLive.start} kicks a full
|
|
6
6
|
* sync in the background and watches the sessions directory; changed
|
|
7
7
|
* transcripts are re-synced (only those files) shortly after they are written.
|
|
8
|
-
* Every
|
|
9
|
-
* pages refetch and fill in while parsing is still
|
|
8
|
+
* Every batch that changes stored rows bumps {@link LiveStatus.version}
|
|
9
|
+
* (throttled), so open pages refetch and fill in while parsing is still
|
|
10
|
+
* running; syncs that change nothing leave the version alone.
|
|
10
11
|
*/
|
|
11
12
|
|
|
12
13
|
import * as fs from "node:fs";
|
|
13
14
|
import * as path from "node:path";
|
|
14
15
|
import { getSessionsDir, logger } from "@oh-my-pi/pi-utils";
|
|
15
16
|
import { syncAllSessions } from "./aggregator";
|
|
16
|
-
import { initDb } from "./db";
|
|
17
|
+
import { getDataVersion, initDb } from "./db";
|
|
17
18
|
import { getRollupStatus, refreshRollups } from "./rollup";
|
|
18
19
|
import type { LiveStatus, LiveSyncStatus } from "./shared-types";
|
|
19
20
|
|
|
@@ -28,7 +29,8 @@ const SYNC_RETRY_MS = 10_000;
|
|
|
28
29
|
/** Minimum spacing of progress-only status events. */
|
|
29
30
|
const PROGRESS_THROTTLE_MS = 150;
|
|
30
31
|
|
|
31
|
-
|
|
32
|
+
/** Receives each distinct status and its JSON serialization (shared by every listener). */
|
|
33
|
+
type Listener = (status: LiveStatus, frame: string) => void;
|
|
32
34
|
|
|
33
35
|
/** One per process; owned by the dashboard server (see `startServer`). */
|
|
34
36
|
export class StatsLive {
|
|
@@ -36,6 +38,10 @@ export class StatsLive {
|
|
|
36
38
|
#sync: LiveSyncStatus = { phase: "idle", current: 0, total: 0, processed: 0, lastSyncedAt: null, error: null };
|
|
37
39
|
#indexingHours = 0;
|
|
38
40
|
#listeners = new Set<Listener>();
|
|
41
|
+
/** Last frame sent to listeners; identical statuses are not re-sent. */
|
|
42
|
+
#lastFrame: string | null = null;
|
|
43
|
+
/** `PRAGMA data_version` at the last sync; null before the first. */
|
|
44
|
+
#dataVersion: number | null = null;
|
|
39
45
|
|
|
40
46
|
#started = false;
|
|
41
47
|
#watcher: fs.FSWatcher | null = null;
|
|
@@ -133,8 +139,9 @@ export class StatsLive {
|
|
|
133
139
|
};
|
|
134
140
|
this.#emitProgress();
|
|
135
141
|
}
|
|
136
|
-
|
|
137
|
-
|
|
142
|
+
// Only committed row changes make pages refetch; unchanged transcripts cost nothing.
|
|
143
|
+
if (event.changes > committed) {
|
|
144
|
+
committed = event.changes;
|
|
138
145
|
this.#changed();
|
|
139
146
|
}
|
|
140
147
|
},
|
|
@@ -159,14 +166,28 @@ export class StatsLive {
|
|
|
159
166
|
}, SYNC_RETRY_MS);
|
|
160
167
|
}
|
|
161
168
|
}
|
|
162
|
-
// Another process may have ingested too;
|
|
163
|
-
this.#changed();
|
|
169
|
+
// Another process may have ingested too; its commits move SQLite's data_version.
|
|
170
|
+
if (this.#externalWrites()) this.#changed();
|
|
164
171
|
this.#emit();
|
|
165
172
|
}
|
|
166
173
|
|
|
174
|
+
/** Whether another connection committed since the last check (always true on the first). */
|
|
175
|
+
#externalWrites(): boolean {
|
|
176
|
+
const current = getDataVersion();
|
|
177
|
+
if (current === null) return false;
|
|
178
|
+
const changed = current !== this.#dataVersion;
|
|
179
|
+
this.#dataVersion = current;
|
|
180
|
+
return changed;
|
|
181
|
+
}
|
|
182
|
+
|
|
167
183
|
/** Stored data changed: refresh rollups and (throttled) bump the version. */
|
|
168
184
|
#changed(): void {
|
|
169
185
|
void this.#refresh();
|
|
186
|
+
this.#bumpVersion();
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/** Bump the data version (throttled), making open pages refetch. */
|
|
190
|
+
#bumpVersion(): void {
|
|
170
191
|
if (this.#versionTimer) return;
|
|
171
192
|
const wait = Math.max(0, this.#lastVersionAt + VERSION_THROTTLE_MS - Date.now());
|
|
172
193
|
this.#versionTimer = setTimeout(() => {
|
|
@@ -190,8 +211,9 @@ export class StatsLive {
|
|
|
190
211
|
await refreshRollups({
|
|
191
212
|
onProgress: remaining => {
|
|
192
213
|
this.#indexingHours = remaining;
|
|
193
|
-
// Each rolled batch fills in more history for open pages
|
|
194
|
-
|
|
214
|
+
// Each rolled batch fills in more history for open pages; the rows
|
|
215
|
+
// themselves are already being rolled, so only the version moves.
|
|
216
|
+
if (remaining > 0) this.#bumpVersion();
|
|
195
217
|
},
|
|
196
218
|
});
|
|
197
219
|
} while (this.#refreshAgain);
|
|
@@ -240,9 +262,12 @@ export class StatsLive {
|
|
|
240
262
|
|
|
241
263
|
#emit(): void {
|
|
242
264
|
const status = this.status();
|
|
265
|
+
const frame = JSON.stringify(status);
|
|
266
|
+
if (frame === this.#lastFrame) return;
|
|
267
|
+
this.#lastFrame = frame;
|
|
243
268
|
for (const listener of this.#listeners) {
|
|
244
269
|
try {
|
|
245
|
-
listener(status);
|
|
270
|
+
listener(status, frame);
|
|
246
271
|
} catch (error) {
|
|
247
272
|
logger.warn("Stats live listener failed", { error: String(error) });
|
|
248
273
|
}
|
package/src/parser.ts
CHANGED
|
@@ -688,14 +688,7 @@ export async function listSessionFiles(folderPath: string): Promise<string[]> {
|
|
|
688
688
|
*/
|
|
689
689
|
export async function listAllSessionFiles(): Promise<string[]> {
|
|
690
690
|
const folders = await listSessionFolders();
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
for (const folder of folders) {
|
|
694
|
-
const files = await listSessionFiles(folder);
|
|
695
|
-
allFiles.push(...files);
|
|
696
|
-
}
|
|
697
|
-
|
|
698
|
-
return allFiles;
|
|
691
|
+
return (await Promise.all(folders.map(listSessionFiles))).flat();
|
|
699
692
|
}
|
|
700
693
|
|
|
701
694
|
/**
|
package/src/rollup.ts
CHANGED
|
@@ -32,6 +32,7 @@ import type {
|
|
|
32
32
|
AgentTypeStats,
|
|
33
33
|
AggregatedStats,
|
|
34
34
|
CostTimeSeriesPoint,
|
|
35
|
+
DailyActivityPoint,
|
|
35
36
|
FolderStats,
|
|
36
37
|
ModelPerformancePoint,
|
|
37
38
|
ModelStats,
|
|
@@ -179,7 +180,7 @@ const EXACT_DIRTY_SESSIONS = 512;
|
|
|
179
180
|
*/
|
|
180
181
|
export function ensureRollupSchema(database: Database): void {
|
|
181
182
|
const readMeta = (key: string) =>
|
|
182
|
-
(database.
|
|
183
|
+
(database.query("SELECT value FROM meta WHERE key = ?").get(key) as { value: string } | undefined)?.value;
|
|
183
184
|
const fresh = readMeta(ROLLUP_VERSION_KEY) !== ROLLUP_VERSION;
|
|
184
185
|
// Steady state touches no schema: DDL on every open would contend with
|
|
185
186
|
// other omp processes writing the same database.
|
|
@@ -273,7 +274,7 @@ export function ensureRollupSchema(database: Database): void {
|
|
|
273
274
|
}
|
|
274
275
|
}
|
|
275
276
|
database
|
|
276
|
-
.
|
|
277
|
+
.query("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)")
|
|
277
278
|
.run(TRIGGER_VERSION_KEY, TRIGGER_VERSION);
|
|
278
279
|
if (fresh) {
|
|
279
280
|
database.run(`
|
|
@@ -286,7 +287,7 @@ export function ensureRollupSchema(database: Database): void {
|
|
|
286
287
|
SELECT DISTINCT session_file FROM messages UNION SELECT DISTINCT session_file FROM tool_calls
|
|
287
288
|
`);
|
|
288
289
|
database
|
|
289
|
-
.
|
|
290
|
+
.query("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)")
|
|
290
291
|
.run(ROLLUP_VERSION_KEY, ROLLUP_VERSION);
|
|
291
292
|
}
|
|
292
293
|
})
|
|
@@ -307,7 +308,7 @@ export function getRollupStatus(): RollupStatus {
|
|
|
307
308
|
const database = currentDb();
|
|
308
309
|
if (!database) return { dirtyHours: 0, dirtySessions: 0 };
|
|
309
310
|
return database
|
|
310
|
-
.
|
|
311
|
+
.query(
|
|
311
312
|
"SELECT (SELECT COUNT(*) FROM rollup_dirty) AS dirtyHours, (SELECT COUNT(*) FROM session_dirty) AS dirtySessions",
|
|
312
313
|
)
|
|
313
314
|
.get() as RollupStatus;
|
|
@@ -322,25 +323,25 @@ export function refreshRollupBatch(limit: number): number {
|
|
|
322
323
|
const database = currentDb();
|
|
323
324
|
if (!database) return 0;
|
|
324
325
|
const run = database.transaction(() => {
|
|
325
|
-
const buckets = database.
|
|
326
|
+
const buckets = database.query("SELECT bucket FROM rollup_dirty ORDER BY bucket DESC LIMIT ?").all(limit) as {
|
|
326
327
|
bucket: number;
|
|
327
328
|
}[];
|
|
328
|
-
const clearMessages = database.
|
|
329
|
-
const clearTools = database.
|
|
330
|
-
const fillMessages = database.
|
|
329
|
+
const clearMessages = database.query("DELETE FROM message_rollup WHERE bucket = ?");
|
|
330
|
+
const clearTools = database.query("DELETE FROM tool_rollup WHERE bucket = ?");
|
|
331
|
+
const fillMessages = database.query(`
|
|
331
332
|
INSERT INTO message_rollup (${MESSAGE_ROLLUP_COLUMNS})
|
|
332
333
|
SELECT ${messageFacts("?1", "")}
|
|
333
334
|
FROM messages WHERE timestamp >= ?1 AND timestamp < ?1 + ${HOUR_MS}
|
|
334
335
|
GROUP BY ${MESSAGE_DIMENSIONS}
|
|
335
336
|
`);
|
|
336
|
-
const fillTools = database.
|
|
337
|
+
const fillTools = database.query(`
|
|
337
338
|
INSERT INTO tool_rollup (${TOOL_ROLLUP_COLUMNS})
|
|
338
339
|
SELECT ${toolFacts("?1")}
|
|
339
340
|
FROM tool_calls t ${TOOL_JOIN}
|
|
340
341
|
WHERE t.timestamp >= ?1 AND t.timestamp < ?1 + ${HOUR_MS}
|
|
341
342
|
GROUP BY t.tool_name, t.model, t.provider
|
|
342
343
|
`);
|
|
343
|
-
const clean = database.
|
|
344
|
+
const clean = database.query("DELETE FROM rollup_dirty WHERE bucket = ?");
|
|
344
345
|
for (const { bucket } of buckets) {
|
|
345
346
|
clearMessages.run(bucket);
|
|
346
347
|
clearTools.run(bucket);
|
|
@@ -351,16 +352,16 @@ export function refreshRollupBatch(limit: number): number {
|
|
|
351
352
|
|
|
352
353
|
// Transcripts are cheap (indexed by session_file); roll many per hour-batch.
|
|
353
354
|
const sessions = database
|
|
354
|
-
.
|
|
355
|
+
.query("SELECT session_file FROM session_dirty LIMIT ?")
|
|
355
356
|
.all(limit * SESSIONS_PER_HOUR) as { session_file: string }[];
|
|
356
|
-
const clearSession = database.
|
|
357
|
-
const fillSession = database.
|
|
357
|
+
const clearSession = database.query("DELETE FROM session_rollup WHERE session_file = ?");
|
|
358
|
+
const fillSession = database.query(`
|
|
358
359
|
INSERT INTO session_rollup (${SESSION_ROLLUP_COLUMNS})
|
|
359
360
|
SELECT ${sessionFacts("m.")}
|
|
360
361
|
FROM messages m WHERE m.session_file = ?1
|
|
361
362
|
GROUP BY m.session_file
|
|
362
363
|
`);
|
|
363
|
-
const cleanSession = database.
|
|
364
|
+
const cleanSession = database.query("DELETE FROM session_dirty WHERE session_file = ?");
|
|
364
365
|
for (const { session_file } of sessions) {
|
|
365
366
|
clearSession.run(session_file);
|
|
366
367
|
fillSession.run(session_file);
|
|
@@ -393,7 +394,7 @@ export async function refreshRollups(opts?: RefreshOptions): Promise<void> {
|
|
|
393
394
|
const elapsed = performance.now() - started;
|
|
394
395
|
if (elapsed < REFRESH_TARGET_MS / 2) batch = Math.min(REFRESH_MAX_BATCH, batch * 2);
|
|
395
396
|
else if (elapsed > REFRESH_TARGET_MS * 2) batch = Math.max(1, Math.floor(batch / 2));
|
|
396
|
-
opts?.onProgress
|
|
397
|
+
if (opts?.onProgress) opts.onProgress(getRollupStatus().dirtyHours);
|
|
397
398
|
await Bun.sleep(0);
|
|
398
399
|
}
|
|
399
400
|
}
|
|
@@ -416,7 +417,7 @@ interface Source {
|
|
|
416
417
|
|
|
417
418
|
function dirtyIsSmall(database: Database): boolean {
|
|
418
419
|
const row = database
|
|
419
|
-
.
|
|
420
|
+
.query(`SELECT COUNT(*) AS n FROM (SELECT 1 FROM rollup_dirty LIMIT ${EXACT_DIRTY_LIMIT + 1})`)
|
|
420
421
|
.get() as {
|
|
421
422
|
n: number;
|
|
422
423
|
};
|
|
@@ -508,7 +509,8 @@ function queryMessages<T>(window: Pick<RangeWindow, "cutoff"> & { bucketMs?: num
|
|
|
508
509
|
if (!database) return [];
|
|
509
510
|
const fine = window.bucketMs !== undefined && window.bucketMs < HOUR_MS;
|
|
510
511
|
const source = messageSource(database, normalizeCutoff(window.cutoff), fine);
|
|
511
|
-
|
|
512
|
+
// `query` caches the compiled statement: the SQL varies only over a small set of shapes.
|
|
513
|
+
return database.query(`SELECT ${select} FROM (${source.sql}) f ${tail}`).all(...source.params) as T[];
|
|
512
514
|
}
|
|
513
515
|
|
|
514
516
|
function queryTools<T>(window: Pick<RangeWindow, "cutoff"> & { bucketMs?: number }, select: string, tail = ""): T[] {
|
|
@@ -516,7 +518,7 @@ function queryTools<T>(window: Pick<RangeWindow, "cutoff"> & { bucketMs?: number
|
|
|
516
518
|
if (!database) return [];
|
|
517
519
|
const fine = window.bucketMs !== undefined && window.bucketMs < HOUR_MS;
|
|
518
520
|
const source = toolSource(database, normalizeCutoff(window.cutoff), fine);
|
|
519
|
-
return database.
|
|
521
|
+
return database.query(`SELECT ${select} FROM (${source.sql}) f ${tail}`).all(...source.params) as T[];
|
|
520
522
|
}
|
|
521
523
|
|
|
522
524
|
function normalizeCutoff(cutoff: number | null | undefined): number | null {
|
|
@@ -804,6 +806,25 @@ export function getProviderHourlyBurn(cutoff: number | null = null): ProviderHou
|
|
|
804
806
|
}));
|
|
805
807
|
}
|
|
806
808
|
|
|
809
|
+
/**
|
|
810
|
+
* Per-local-day request/token/cost totals since `cutoff`, oldest first, read
|
|
811
|
+
* from the hourly rollup. Returns null when the local UTC offset is not a whole
|
|
812
|
+
* hour (now or at `cutoff`), since hourly buckets then straddle local midnight,
|
|
813
|
+
* or while the dirty backlog is too large for exact reads; callers then
|
|
814
|
+
* aggregate the raw rows instead.
|
|
815
|
+
*/
|
|
816
|
+
export function getDailyActivityFromRollup(cutoff: number): DailyActivityPoint[] | null {
|
|
817
|
+
if (new Date().getTimezoneOffset() % 60 !== 0 || new Date(cutoff).getTimezoneOffset() % 60 !== 0) return null;
|
|
818
|
+
const database = currentDb();
|
|
819
|
+
if (!database || !dirtyIsSmall(database)) return null;
|
|
820
|
+
return queryMessages<{ day: string; cost: number; requests: number; total_tokens: number | null }>(
|
|
821
|
+
{ cutoff },
|
|
822
|
+
`date(f.bucket / 1000, 'unixepoch', 'localtime') AS day, TOTAL(f.cost_total) AS cost,
|
|
823
|
+
SUM(f.requests) AS requests, SUM(f.total_tokens) AS total_tokens`,
|
|
824
|
+
"GROUP BY day ORDER BY day",
|
|
825
|
+
).map(row => ({ day: row.day, cost: row.cost, requests: row.requests, totalTokens: row.total_tokens ?? 0 }));
|
|
826
|
+
}
|
|
827
|
+
|
|
807
828
|
/** Tokens/cost per bucket per provider. */
|
|
808
829
|
export function getProviderTimeSeries({ cutoff, bucketMs }: RangeWindow): ProviderTimeSeriesPoint[] {
|
|
809
830
|
return queryMessages<{
|
|
@@ -855,7 +876,7 @@ export function getSessionRollups(): SessionRollupRow[] {
|
|
|
855
876
|
const database = currentDb();
|
|
856
877
|
if (!database) return [];
|
|
857
878
|
const { n } = database
|
|
858
|
-
.
|
|
879
|
+
.query(`SELECT COUNT(*) AS n FROM (SELECT 1 FROM session_dirty LIMIT ${EXACT_DIRTY_SESSIONS + 1})`)
|
|
859
880
|
.get() as { n: number };
|
|
860
881
|
const exact = n <= EXACT_DIRTY_SESSIONS;
|
|
861
882
|
const parts = [
|
|
@@ -869,7 +890,7 @@ export function getSessionRollups(): SessionRollupRow[] {
|
|
|
869
890
|
);
|
|
870
891
|
}
|
|
871
892
|
return database
|
|
872
|
-
.
|
|
893
|
+
.query(
|
|
873
894
|
`SELECT session_file AS sessionFile, requests, started_at AS startedAt, ended_at AS endedAt,
|
|
874
895
|
total_tokens AS totalTokens, cost_total AS costTotal, unpriced AS unpricedRequests, models,
|
|
875
896
|
tool_calls AS toolCalls
|
package/src/server.ts
CHANGED
|
@@ -14,9 +14,11 @@ import {
|
|
|
14
14
|
getRecentErrors,
|
|
15
15
|
getRecentRequests,
|
|
16
16
|
getRequestDetails,
|
|
17
|
+
getTimeRangeConfig,
|
|
17
18
|
getToolDashboardStats,
|
|
18
19
|
} from "./aggregator";
|
|
19
|
-
import {
|
|
20
|
+
import { initDb } from "./db";
|
|
21
|
+
import { decodeEmbeddedClientArchive, hasEmbeddedClientArchive } from "./embedded-client";
|
|
20
22
|
import embeddedClientArchiveTxt from "./embedded-client.generated.txt";
|
|
21
23
|
import {
|
|
22
24
|
cancelFrustrationRun,
|
|
@@ -36,18 +38,17 @@ import {
|
|
|
36
38
|
STATS_DASHBOARD_HOSTNAME_HEADER,
|
|
37
39
|
STATS_DASHBOARD_SECURITY_VERSION,
|
|
38
40
|
} from "./port-conflict";
|
|
39
|
-
import
|
|
41
|
+
import { getStatsByModel, getTimeSeries } from "./rollup";
|
|
40
42
|
import {
|
|
41
43
|
buildSessionTrace,
|
|
42
44
|
getTraceEntry,
|
|
43
45
|
listSessionSummaries,
|
|
44
46
|
TRACE_ETAG_VERSION,
|
|
47
|
+
type TraceFingerprint,
|
|
45
48
|
traceFingerprintForEtag,
|
|
46
49
|
TracePathError,
|
|
47
50
|
} from "./trace";
|
|
48
51
|
|
|
49
|
-
const EMBEDDED_CLIENT_ARCHIVE = decodeEmbeddedClientArchive(embeddedClientArchiveTxt);
|
|
50
|
-
|
|
51
52
|
const CLIENT_DIR = path.join(import.meta.dir, "client");
|
|
52
53
|
const STATIC_DIR = path.join(import.meta.dir, "..", "dist", "client");
|
|
53
54
|
const IS_BUN_COMPILED =
|
|
@@ -60,21 +61,24 @@ const IS_BUN_COMPILED =
|
|
|
60
61
|
// dashboard sources or prebuilt dist/client next to the bundle, so the
|
|
61
62
|
// embedded archive is the only viable asset source.
|
|
62
63
|
const IS_PREBUILT = IS_BUN_COMPILED || Boolean(process.env.PI_BUNDLED || Bun.env.PI_BUNDLED);
|
|
63
|
-
|
|
64
|
+
// Sniff the base64 gzip magic instead of decoding: importing this module (every
|
|
65
|
+
// TUI start) must not pay for a multi-megabyte archive nobody may serve.
|
|
66
|
+
const USE_EMBEDDED_CLIENT = hasEmbeddedClientArchive(embeddedClientArchiveTxt) || IS_PREBUILT;
|
|
64
67
|
|
|
65
68
|
let embeddedClientFilesPromise: Promise<Map<string, Blob>> | null = null;
|
|
66
69
|
|
|
67
70
|
async function getEmbeddedClientFiles(): Promise<Map<string, Blob>> {
|
|
68
71
|
if (embeddedClientFilesPromise) return embeddedClientFilesPromise;
|
|
69
72
|
|
|
70
|
-
|
|
73
|
+
const archive = decodeEmbeddedClientArchive(embeddedClientArchiveTxt);
|
|
74
|
+
if (!archive) {
|
|
71
75
|
throw new Error(
|
|
72
76
|
"Embedded stats client bundle missing. Rebuild the omp binary or npm bundle with embedded stats assets.",
|
|
73
77
|
);
|
|
74
78
|
}
|
|
75
79
|
|
|
76
80
|
// Keep bundled assets in memory so OS temporary-file cleanup cannot break a live dashboard.
|
|
77
|
-
embeddedClientFilesPromise = new Bun.Archive(
|
|
81
|
+
embeddedClientFilesPromise = new Bun.Archive(archive).files().then(files => {
|
|
78
82
|
for (const [name, file] of files) {
|
|
79
83
|
// Archive entries are untyped blobs; infer MIME types just as disk-backed Bun files do.
|
|
80
84
|
files.set(name, new File([file], name, { type: Bun.file(name).type }));
|
|
@@ -241,8 +245,9 @@ export async function handleApi(req: Request): Promise<Response> {
|
|
|
241
245
|
}
|
|
242
246
|
|
|
243
247
|
if (path === "/api/stats/models") {
|
|
244
|
-
|
|
245
|
-
|
|
248
|
+
// Also the port-conflict identity probe: keep it to the one query it serves.
|
|
249
|
+
await initDb();
|
|
250
|
+
return Response.json(getStatsByModel(getTimeRangeConfig(range).cutoff));
|
|
246
251
|
}
|
|
247
252
|
|
|
248
253
|
if (path === "/api/stats/folders") {
|
|
@@ -251,8 +256,8 @@ export async function handleApi(req: Request): Promise<Response> {
|
|
|
251
256
|
}
|
|
252
257
|
|
|
253
258
|
if (path === "/api/stats/timeseries") {
|
|
254
|
-
|
|
255
|
-
return Response.json(
|
|
259
|
+
await initDb();
|
|
260
|
+
return Response.json(getTimeSeries(getTimeRangeConfig(range)));
|
|
256
261
|
}
|
|
257
262
|
|
|
258
263
|
if (path.startsWith("/api/request/")) {
|
|
@@ -293,14 +298,15 @@ export async function handleApi(req: Request): Promise<Response> {
|
|
|
293
298
|
// The fingerprint covers child transcripts, so a subagent-only
|
|
294
299
|
// append changes the ETag and never 304s stale.
|
|
295
300
|
const clientEtag = req.headers.get("if-none-match");
|
|
301
|
+
let fingerprint: TraceFingerprint | undefined;
|
|
296
302
|
if (clientEtag) {
|
|
297
|
-
|
|
303
|
+
fingerprint = await traceFingerprintForEtag(file);
|
|
298
304
|
if (fingerprint !== undefined) {
|
|
299
|
-
const etag = `"${TRACE_ETAG_VERSION}:${fingerprint}"`;
|
|
305
|
+
const etag = `"${TRACE_ETAG_VERSION}:${fingerprint.rootMtimeMs}:${fingerprint.childFingerprint}"`;
|
|
300
306
|
if (clientEtag === etag) return new Response(null, { status: 304 });
|
|
301
307
|
}
|
|
302
308
|
}
|
|
303
|
-
const trace = await buildSessionTrace(file);
|
|
309
|
+
const trace = await buildSessionTrace(file, fingerprint);
|
|
304
310
|
const etag = `"${TRACE_ETAG_VERSION}:${trace.etag}"`;
|
|
305
311
|
if (clientEtag === etag) return new Response(null, { status: 304 });
|
|
306
312
|
return Response.json(trace, { headers: { ETag: etag } });
|
|
@@ -383,7 +389,7 @@ function createDashboardServer(port: number, hostname: string): Server<undefined
|
|
|
383
389
|
if (path === "/api/events") {
|
|
384
390
|
// Long-lived stream: exempt from the idle timeout.
|
|
385
391
|
server.timeout(req, 0);
|
|
386
|
-
return liveEventStream(dashboardHeaders);
|
|
392
|
+
return liveEventStream(dashboardHeaders, req.signal);
|
|
387
393
|
}
|
|
388
394
|
|
|
389
395
|
try {
|
|
@@ -423,8 +429,9 @@ const EVENT_HEARTBEAT_MS = 15_000;
|
|
|
423
429
|
/**
|
|
424
430
|
* Server-sent events of {@link LiveStatus}: the current status immediately,
|
|
425
431
|
* then every change (sync progress, data version bumps, indexing backlog).
|
|
432
|
+
* Disconnects unsubscribe, so ingest winds down once no page is listening.
|
|
426
433
|
*/
|
|
427
|
-
function liveEventStream(headers: Record<string, string
|
|
434
|
+
function liveEventStream(headers: Record<string, string>, signal: AbortSignal): Response {
|
|
428
435
|
const live = statsLive();
|
|
429
436
|
// Ingest starts when the first page connects, not when the server binds.
|
|
430
437
|
live.start();
|
|
@@ -432,19 +439,26 @@ function liveEventStream(headers: Record<string, string>): Response {
|
|
|
432
439
|
let cleanup = () => {};
|
|
433
440
|
const stream = new ReadableStream<Uint8Array>({
|
|
434
441
|
start(controller) {
|
|
435
|
-
const
|
|
436
|
-
|
|
442
|
+
const write = (chunk: string) => {
|
|
443
|
+
try {
|
|
444
|
+
controller.enqueue(encoder.encode(chunk));
|
|
445
|
+
} catch {
|
|
446
|
+
// Stream already closed: the client left without a cancel.
|
|
447
|
+
cleanup();
|
|
448
|
+
}
|
|
437
449
|
};
|
|
438
|
-
|
|
439
|
-
const unsubscribe = live.subscribe(
|
|
440
|
-
const heartbeat = setInterval(
|
|
441
|
-
|
|
442
|
-
EVENT_HEARTBEAT_MS,
|
|
443
|
-
);
|
|
450
|
+
write(`data: ${JSON.stringify(live.status())}\n\n`);
|
|
451
|
+
const unsubscribe = live.subscribe((_status, frame) => write(`data: ${frame}\n\n`));
|
|
452
|
+
const heartbeat = setInterval(() => write(": keep-alive\n\n"), EVENT_HEARTBEAT_MS);
|
|
453
|
+
const onAbort = () => cleanup();
|
|
444
454
|
cleanup = () => {
|
|
455
|
+
cleanup = () => {};
|
|
445
456
|
unsubscribe();
|
|
446
457
|
clearInterval(heartbeat);
|
|
458
|
+
signal.removeEventListener("abort", onAbort);
|
|
447
459
|
};
|
|
460
|
+
if (signal.aborted) cleanup();
|
|
461
|
+
else signal.addEventListener("abort", onAbort, { once: true });
|
|
448
462
|
},
|
|
449
463
|
cancel() {
|
|
450
464
|
cleanup();
|
package/src/trace.ts
CHANGED
|
@@ -750,11 +750,6 @@ async function buildTrackTree(
|
|
|
750
750
|
// ---------------------------------------------------------------------------
|
|
751
751
|
// Public surface
|
|
752
752
|
|
|
753
|
-
/**
|
|
754
|
-
* Assemble the full span tree for one root session transcript.
|
|
755
|
-
* Throws {@link TracePathError} for paths outside the sessions root; ENOENT
|
|
756
|
-
* passes through for the caller's 404 mapping.
|
|
757
|
-
*/
|
|
758
753
|
// Single-entry memo: the dashboard polls one open trace every 15s. Keyed on
|
|
759
754
|
// the root file's mtime PLUS a fingerprint of the child-transcript SET under
|
|
760
755
|
// the root's artifacts tree (one cheap readdir+stat walk, no parse). A
|
|
@@ -788,35 +783,46 @@ async function childTranscriptsFingerprint(rootFile: string, depth = 0): Promise
|
|
|
788
783
|
return "";
|
|
789
784
|
}
|
|
790
785
|
names.sort();
|
|
786
|
+
const dir = transcriptStem(rootFile);
|
|
791
787
|
const transcriptNames = names.filter(name => name.endsWith(".jsonl") || name.endsWith(".jsonl.gz"));
|
|
792
|
-
const
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
}
|
|
804
|
-
|
|
805
|
-
return
|
|
788
|
+
const childParts = await Promise.all(
|
|
789
|
+
transcriptNames.map(async name => {
|
|
790
|
+
const childFile = path.join(dir, name);
|
|
791
|
+
try {
|
|
792
|
+
const stat = await fs.stat(childFile);
|
|
793
|
+
// Recurse like the builder (nested subagents), without parsing.
|
|
794
|
+
const nested = await childTranscriptsFingerprint(childFile, depth + 1);
|
|
795
|
+
return nested ? `${name}@${stat.mtimeMs},${name}/{${nested}}` : `${name}@${stat.mtimeMs}`;
|
|
796
|
+
} catch {
|
|
797
|
+
return `${name}@gone`;
|
|
798
|
+
}
|
|
799
|
+
}),
|
|
800
|
+
);
|
|
801
|
+
return [`n=${transcriptNames.length}`, ...childParts].join(",");
|
|
802
|
+
}
|
|
803
|
+
|
|
804
|
+
/** Freshness of a root trace: root mtime plus the child-transcript set fingerprint. */
|
|
805
|
+
export interface TraceFingerprint {
|
|
806
|
+
rootMtimeMs: number;
|
|
807
|
+
childFingerprint: string;
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
/** Stat the root and walk its child transcripts (readdir+stat, no parse). */
|
|
811
|
+
async function readTraceFingerprint(resolved: string): Promise<TraceFingerprint> {
|
|
812
|
+
const rootMtimeMs = (await fs.stat(resolved)).mtimeMs;
|
|
813
|
+
const childFingerprint = await childTranscriptsFingerprint(resolved);
|
|
814
|
+
return { rootMtimeMs, childFingerprint };
|
|
806
815
|
}
|
|
807
816
|
|
|
808
817
|
/**
|
|
809
|
-
* Freshness fingerprint for the trace ETag pre-check
|
|
810
|
-
*
|
|
811
|
-
*
|
|
812
|
-
*
|
|
818
|
+
* Freshness fingerprint for the trace ETag pre-check; `SessionTrace.etag` is
|
|
819
|
+
* `${rootMtimeMs}:${childFingerprint}` for an unraced build. Returns undefined
|
|
820
|
+
* when the path is rejected or missing (the caller falls through to the full
|
|
821
|
+
* build, which maps those to 400/404).
|
|
813
822
|
*/
|
|
814
|
-
export async function traceFingerprintForEtag(fileParam: string): Promise<
|
|
823
|
+
export async function traceFingerprintForEtag(fileParam: string): Promise<TraceFingerprint | undefined> {
|
|
815
824
|
try {
|
|
816
|
-
|
|
817
|
-
const rootMs = (await fs.stat(resolved)).mtimeMs;
|
|
818
|
-
const childFp = await childTranscriptsFingerprint(resolved);
|
|
819
|
-
return `${rootMs}:${childFp}`;
|
|
825
|
+
return await readTraceFingerprint(resolveSessionPath(fileParam));
|
|
820
826
|
} catch {
|
|
821
827
|
return undefined;
|
|
822
828
|
}
|
|
@@ -827,16 +833,22 @@ export function traceMemoForTests(): { file: string; mtimeMs: number } | undefin
|
|
|
827
833
|
return { file: traceMemo.file, mtimeMs: traceMemo.entry.mtimeMs };
|
|
828
834
|
}
|
|
829
835
|
|
|
830
|
-
|
|
836
|
+
/**
|
|
837
|
+
* Assemble the full span tree for one root session transcript.
|
|
838
|
+
* Throws {@link TracePathError} for paths outside the sessions root; ENOENT
|
|
839
|
+
* passes through for the caller's 404 mapping. `fingerprint` may carry the
|
|
840
|
+
* {@link traceFingerprintForEtag} value the caller just computed for the same
|
|
841
|
+
* file (its ETag pre-check), sparing a second walk.
|
|
842
|
+
*/
|
|
843
|
+
export async function buildSessionTrace(fileParam: string, fingerprint?: TraceFingerprint): Promise<SessionTrace> {
|
|
831
844
|
const resolved = resolveSessionPath(fileParam);
|
|
832
|
-
const rootStat = await fs.stat(resolved);
|
|
833
845
|
const memo = traceMemo?.file === resolved ? traceMemo.entry : undefined;
|
|
834
846
|
// Pre-build child fingerprint on EVERY path (hit check and rebuild
|
|
835
847
|
// alike): the post-build race guard compares against this, so a child
|
|
836
848
|
// append landing mid-parse is detected even on first builds and
|
|
837
849
|
// memo-less rebuilds, where no prior fingerprint exists to compare.
|
|
838
|
-
const preChildFingerprint = await
|
|
839
|
-
if (memo && memo.mtimeMs ===
|
|
850
|
+
const { rootMtimeMs, childFingerprint: preChildFingerprint } = fingerprint ?? (await readTraceFingerprint(resolved));
|
|
851
|
+
if (memo && memo.mtimeMs === rootMtimeMs) {
|
|
840
852
|
// Equality, not <=: a LOWER mark (delete/older-mtime replace) is
|
|
841
853
|
// also stale — only an identical child set may reuse the trace.
|
|
842
854
|
if (preChildFingerprint === memo.childFingerprint) return memo.trace;
|
|
@@ -901,7 +913,7 @@ export async function buildSessionTrace(fileParam: string): Promise<SessionTrace
|
|
|
901
913
|
else if (span.kind === "turn") turns++;
|
|
902
914
|
}
|
|
903
915
|
|
|
904
|
-
const start = startedAt ?? rootScan.headerTs ??
|
|
916
|
+
const start = startedAt ?? rootScan.headerTs ?? rootMtimeMs;
|
|
905
917
|
const end = endedAt ?? start;
|
|
906
918
|
const wallMs = end - start;
|
|
907
919
|
|
|
@@ -925,7 +937,7 @@ export async function buildSessionTrace(fileParam: string): Promise<SessionTrace
|
|
|
925
937
|
// neither the trace nor the key.
|
|
926
938
|
const childFingerprint = await childTranscriptsFingerprint(resolved);
|
|
927
939
|
// Root rewritten mid-build: same no-cache rule as the child race below.
|
|
928
|
-
const rootChanged = (await fs.stat(resolved).catch(() => null))?.mtimeMs !==
|
|
940
|
+
const rootChanged = (await fs.stat(resolved).catch(() => null))?.mtimeMs !== rootMtimeMs;
|
|
929
941
|
const raced = childFingerprint !== preChildFingerprint;
|
|
930
942
|
// Raced builds keep the PRE-build fingerprint as their ETag: the parsed
|
|
931
943
|
// bytes predate the racy write, so claiming the post-write state would
|
|
@@ -941,15 +953,15 @@ export async function buildSessionTrace(fileParam: string): Promise<SessionTrace
|
|
|
941
953
|
cwd: rootScan.cwd,
|
|
942
954
|
startedAt: start,
|
|
943
955
|
endedAt: end,
|
|
944
|
-
mtimeMs:
|
|
945
|
-
etag: `${
|
|
956
|
+
mtimeMs: rootMtimeMs,
|
|
957
|
+
etag: `${rootMtimeMs}:${etagFingerprint}`,
|
|
946
958
|
tracks,
|
|
947
959
|
summary,
|
|
948
960
|
};
|
|
949
961
|
if (!raced && !rootChanged) {
|
|
950
962
|
traceMemo = {
|
|
951
963
|
file: resolved,
|
|
952
|
-
entry: { mtimeMs:
|
|
964
|
+
entry: { mtimeMs: rootMtimeMs, childFingerprint, trace },
|
|
953
965
|
};
|
|
954
966
|
}
|
|
955
967
|
return trace;
|
|
@@ -1068,7 +1080,8 @@ function basenameTimestamp(base: string): number | undefined {
|
|
|
1068
1080
|
// Short-TTL memo for the disk sweep behind /api/sessions (30s poll): the
|
|
1069
1081
|
// session list changes only on create/edit, so readdir+stat of every file per
|
|
1070
1082
|
// poll is pure syscall churn. TTL is deliberately short so a just-created
|
|
1071
|
-
// session appears within seconds.
|
|
1083
|
+
// session appears within seconds. Only the newest `limit` roots are kept, so
|
|
1084
|
+
// the memo stays small however many sessions exist.
|
|
1072
1085
|
let diskRootsMemo:
|
|
1073
1086
|
| {
|
|
1074
1087
|
atMs: number;
|
|
@@ -1117,8 +1130,9 @@ async function scanDiskRoots(limit: number): Promise<Array<{ file: string; mtime
|
|
|
1117
1130
|
}),
|
|
1118
1131
|
);
|
|
1119
1132
|
roots.sort((a, b) => b.mtimeMs - a.mtimeMs);
|
|
1120
|
-
|
|
1121
|
-
|
|
1133
|
+
const newest = roots.slice(0, limit);
|
|
1134
|
+
diskRootsMemo = { atMs: Date.now(), sessionsDir, limit, roots: newest };
|
|
1135
|
+
return newest.slice();
|
|
1122
1136
|
}
|
|
1123
1137
|
|
|
1124
1138
|
/**
|