@oh-my-pi/omp-stats 18.13.2 → 18.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +18 -18
- package/dist/client/index.js +78 -78
- package/dist/types/chunked-insert.d.ts +32 -0
- package/dist/types/client/traces/SpanDrawer.d.ts +9 -3
- package/dist/types/client/traces/TimelineCanvas.d.ts +3 -1
- package/dist/types/client/traces/TraceView.d.ts +13 -0
- package/dist/types/client/traces/index.d.ts +5 -0
- package/dist/types/client/traces/trace-colors.d.ts +3 -1
- package/dist/types/client/ui/SearchInput.d.ts +2 -1
- package/dist/types/db.d.ts +5 -5
- package/dist/types/port-conflict.d.ts +9 -3
- package/dist/types/server.d.ts +7 -3
- package/dist/types/shared-types.d.ts +3 -2
- package/dist/types/trace.d.ts +27 -1
- package/dist/types/types.d.ts +6 -1
- package/package.json +11 -6
- package/src/chunked-insert.ts +89 -0
- package/src/client/traces/SpanDrawer.tsx +56 -9
- package/src/client/traces/TimelineCanvas.tsx +13 -4
- package/src/client/traces/TraceView.tsx +165 -158
- package/src/client/traces/index.ts +5 -0
- package/src/client/traces/trace-colors.ts +14 -10
- package/src/client/ui/SearchInput.tsx +3 -1
- package/src/db.ts +197 -147
- package/src/parser.ts +13 -2
- package/src/port-conflict.ts +67 -30
- package/src/server.ts +39 -31
- package/src/shared-types.ts +3 -2
- package/src/trace.ts +119 -35
- package/src/types.ts +6 -1
- package/THIRD-PARTY-NOTICES.txt +0 -22901
package/src/db.ts
CHANGED
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
} from "@oh-my-pi/pi-catalog/models";
|
|
12
12
|
import type { ModelCost } from "@oh-my-pi/pi-catalog/types";
|
|
13
13
|
import { getConfigRootDir, getStatsDbPath, VERSION } from "@oh-my-pi/pi-utils";
|
|
14
|
+
import { ChunkedInsert } from "./chunked-insert";
|
|
14
15
|
import { classifyAgentType, type ParseSessionResult, type SessionParserState } from "./parser";
|
|
15
16
|
import { ensureRollupSchema, getDailyActivityFromRollup } from "./rollup";
|
|
16
17
|
import type {
|
|
@@ -135,6 +136,17 @@ export async function initDb(): Promise<Database> {
|
|
|
135
136
|
// https://github.com/can1357/oh-my-pi/issues/2421.
|
|
136
137
|
db.run("PRAGMA busy_timeout = 5000");
|
|
137
138
|
db.run("PRAGMA journal_mode = WAL");
|
|
139
|
+
// The database is a derived cache of the transcripts, so WAL's NORMAL sync
|
|
140
|
+
// (fsync only at checkpoints) is safe: a crash can drop the newest batches,
|
|
141
|
+
// which the next sync re-ingests from their unchanged cursors. Rarer, larger
|
|
142
|
+
// checkpoints cut the F_FULLFSYNC stalls macOS pays per checkpoint. The
|
|
143
|
+
// statement journal and temp b-tree every trigger-carrying insert opens stay
|
|
144
|
+
// in memory instead of a temp file, and a 64 MiB page cache (default 8 MiB)
|
|
145
|
+
// keeps more index pages of a multi-GB database resident between batches.
|
|
146
|
+
db.run("PRAGMA synchronous = NORMAL");
|
|
147
|
+
db.run("PRAGMA temp_store = MEMORY");
|
|
148
|
+
db.run("PRAGMA cache_size = -65536");
|
|
149
|
+
db.run("PRAGMA wal_autocheckpoint = 16384");
|
|
138
150
|
|
|
139
151
|
// Whether `messages` predates this init — drives the one-time agent_type
|
|
140
152
|
// backfill below, so it must be sampled before CREATE TABLE adds the table.
|
|
@@ -174,11 +186,17 @@ export async function initDb(): Promise<Database> {
|
|
|
174
186
|
UNIQUE(session_file, entry_id)
|
|
175
187
|
);
|
|
176
188
|
|
|
177
|
-
|
|
178
|
-
|
|
189
|
+
-- Fork-guard identity lookups key on (entry_id, timestamp). Time-first
|
|
190
|
+
-- keeps one transcript's rows in a few neighbouring leaves; entry-first
|
|
191
|
+
-- scattered every insert, delete and lookup across the whole index. The
|
|
192
|
+
-- composite also serves every timestamp range scan.
|
|
193
|
+
DROP INDEX IF EXISTS idx_messages_timestamp;
|
|
194
|
+
DROP INDEX IF EXISTS idx_messages_entry_timestamp;
|
|
195
|
+
CREATE INDEX IF NOT EXISTS idx_messages_timestamp_entry ON messages(timestamp, entry_id);
|
|
179
196
|
CREATE INDEX IF NOT EXISTS idx_messages_model ON messages(model);
|
|
180
197
|
CREATE INDEX IF NOT EXISTS idx_messages_folder ON messages(folder);
|
|
181
|
-
|
|
198
|
+
-- UNIQUE(session_file, entry_id) already indexes session_file lookups.
|
|
199
|
+
DROP INDEX IF EXISTS idx_messages_session;
|
|
182
200
|
CREATE INDEX IF NOT EXISTS idx_messages_timestamp_model_provider ON messages(timestamp, model, provider);
|
|
183
201
|
CREATE INDEX IF NOT EXISTS idx_messages_timestamp_folder ON messages(timestamp, folder);
|
|
184
202
|
CREATE INDEX IF NOT EXISTS idx_messages_stop_reason_timestamp ON messages(stop_reason, timestamp);
|
|
@@ -211,8 +229,9 @@ export async function initDb(): Promise<Database> {
|
|
|
211
229
|
UNIQUE(session_file, entry_id)
|
|
212
230
|
);
|
|
213
231
|
|
|
214
|
-
|
|
215
|
-
|
|
232
|
+
DROP INDEX IF EXISTS idx_user_messages_timestamp;
|
|
233
|
+
DROP INDEX IF EXISTS idx_user_messages_entry_timestamp;
|
|
234
|
+
CREATE INDEX IF NOT EXISTS idx_user_messages_timestamp_entry ON user_messages(timestamp, entry_id);
|
|
216
235
|
CREATE INDEX IF NOT EXISTS idx_user_messages_timestamp_model ON user_messages(timestamp, model, provider);
|
|
217
236
|
|
|
218
237
|
-- Judge verdicts keyed by prose hash: identical messages share one verdict,
|
|
@@ -244,8 +263,9 @@ export async function initDb(): Promise<Database> {
|
|
|
244
263
|
UNIQUE(session_file, tool_call_id)
|
|
245
264
|
);
|
|
246
265
|
|
|
247
|
-
|
|
248
|
-
|
|
266
|
+
DROP INDEX IF EXISTS idx_tool_calls_timestamp;
|
|
267
|
+
DROP INDEX IF EXISTS idx_tool_calls_entry_timestamp;
|
|
268
|
+
CREATE INDEX IF NOT EXISTS idx_tool_calls_timestamp_entry ON tool_calls(timestamp, entry_id);
|
|
249
269
|
CREATE INDEX IF NOT EXISTS idx_tool_calls_tool_timestamp ON tool_calls(tool_name, timestamp);
|
|
250
270
|
|
|
251
271
|
CREATE TABLE IF NOT EXISTS meta (
|
|
@@ -348,7 +368,7 @@ export async function initDb(): Promise<Database> {
|
|
|
348
368
|
prose_hash TEXT NOT NULL DEFAULT '',
|
|
349
369
|
UNIQUE(session_file, entry_id)
|
|
350
370
|
);
|
|
351
|
-
CREATE INDEX IF NOT EXISTS
|
|
371
|
+
CREATE INDEX IF NOT EXISTS idx_user_messages_timestamp_entry ON user_messages(timestamp, entry_id);
|
|
352
372
|
CREATE INDEX IF NOT EXISTS idx_user_messages_timestamp_model ON user_messages(timestamp, model, provider);
|
|
353
373
|
`);
|
|
354
374
|
} else if (!userMessageColumns.some(column => column.name === "prose")) {
|
|
@@ -674,6 +694,17 @@ export function setFileOffset(
|
|
|
674
694
|
stmt.run(sessionFile, offset, lastModified, parserState ? JSON.stringify(parserState) : null);
|
|
675
695
|
}
|
|
676
696
|
|
|
697
|
+
/**
|
|
698
|
+
* Run `fn` atomically: inline when the caller already holds a transaction,
|
|
699
|
+
* otherwise in a fresh one. Nesting `db.transaction` would open a savepoint,
|
|
700
|
+
* which copies every page the batch touches into the statement journal; each
|
|
701
|
+
* statement end then rewinds that ever-growing journal, so a long batch
|
|
702
|
+
* degrades quadratically.
|
|
703
|
+
*/
|
|
704
|
+
function atomically<T>(database: Database, fn: () => T): T {
|
|
705
|
+
return database.inTransaction ? fn() : database.transaction(fn)();
|
|
706
|
+
}
|
|
707
|
+
|
|
677
708
|
/** Parsed transcript queued by the sync loop for an atomic database batch. */
|
|
678
709
|
export interface ParsedSession {
|
|
679
710
|
sessionFile: string;
|
|
@@ -873,7 +904,7 @@ export function completeSessionSync(reconcile: boolean): void {
|
|
|
873
904
|
* `provider`, token counts, and `responseId`. The `UNIQUE(session_file,
|
|
874
905
|
* entry_id)` constraint alone keys each row by file, so without the guard
|
|
875
906
|
* below the same provider request would land twice and inflate every
|
|
876
|
-
* aggregate. The
|
|
907
|
+
* aggregate. The {@link ChunkedInsert} fork guard skips inserts whose
|
|
877
908
|
* `(entry_id, timestamp)` already exists under a different `session_file` —
|
|
878
909
|
* first-write-wins across the lineage. Same-file re-syncs still hit the
|
|
879
910
|
* `ON CONFLICT(session_file, entry_id)` upsert below, which re-derives the
|
|
@@ -882,21 +913,39 @@ export function completeSessionSync(reconcile: boolean): void {
|
|
|
882
913
|
*/
|
|
883
914
|
export function insertMessageStats(stats: Iterable<MessageStatsInput>): number {
|
|
884
915
|
if (!db) return 0;
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
916
|
+
const writer = new ChunkedInsert(db, {
|
|
917
|
+
verb: "INSERT",
|
|
918
|
+
table: "messages",
|
|
919
|
+
columns: [
|
|
920
|
+
"session_file",
|
|
921
|
+
"entry_id",
|
|
922
|
+
"folder",
|
|
923
|
+
"model",
|
|
924
|
+
"provider",
|
|
925
|
+
"api",
|
|
926
|
+
"timestamp",
|
|
927
|
+
"duration",
|
|
928
|
+
"ttft",
|
|
929
|
+
"stop_reason",
|
|
930
|
+
"error_message",
|
|
931
|
+
"input_tokens",
|
|
932
|
+
"output_tokens",
|
|
933
|
+
"cache_read_tokens",
|
|
934
|
+
"cache_write_tokens",
|
|
935
|
+
"total_tokens",
|
|
936
|
+
"premium_requests",
|
|
937
|
+
"cost_input",
|
|
938
|
+
"cost_output",
|
|
939
|
+
"cost_cache_read",
|
|
940
|
+
"cost_cache_write",
|
|
941
|
+
"cost_total",
|
|
942
|
+
"cost_no_cache_input",
|
|
943
|
+
"cost_unpriced",
|
|
944
|
+
"agent_type",
|
|
945
|
+
"service_tier",
|
|
946
|
+
],
|
|
947
|
+
identity: ["entry_id", "timestamp"],
|
|
948
|
+
upsert: `ON CONFLICT(session_file, entry_id) DO UPDATE SET
|
|
900
949
|
premium_requests = MAX(messages.premium_requests, excluded.premium_requests),
|
|
901
950
|
cost_input = excluded.cost_input,
|
|
902
951
|
cost_output = excluded.cost_output,
|
|
@@ -905,53 +954,47 @@ export function insertMessageStats(stats: Iterable<MessageStatsInput>): number {
|
|
|
905
954
|
cost_total = excluded.cost_total,
|
|
906
955
|
cost_no_cache_input = excluded.cost_no_cache_input,
|
|
907
956
|
cost_unpriced = excluded.cost_unpriced,
|
|
908
|
-
service_tier = excluded.service_tier
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
let inserted = 0;
|
|
912
|
-
const insert = db.transaction(() => {
|
|
957
|
+
service_tier = excluded.service_tier`,
|
|
958
|
+
});
|
|
959
|
+
return atomically(db, () => {
|
|
913
960
|
for (const s of stats) {
|
|
914
961
|
const { cost, unpriced } = resolveStoredCost(s);
|
|
915
962
|
const noCacheInputCost = calculateNoCacheInputCost(s.provider, s.model, s.usage, s.timestamp) ?? 0;
|
|
916
|
-
|
|
917
|
-
s.sessionFile,
|
|
918
|
-
s.entryId,
|
|
919
|
-
s.folder,
|
|
920
|
-
s.model,
|
|
921
|
-
s.provider,
|
|
922
|
-
s.api,
|
|
923
|
-
s.timestamp,
|
|
924
|
-
s.duration,
|
|
925
|
-
s.ttft,
|
|
926
|
-
s.stopReason,
|
|
927
|
-
s.errorMessage,
|
|
928
|
-
s.usage.input,
|
|
929
|
-
s.usage.output,
|
|
930
|
-
s.usage.cacheRead,
|
|
931
|
-
s.usage.cacheWrite,
|
|
932
|
-
s.usage.totalTokens,
|
|
933
|
-
s.usage.premiumRequests ?? 0,
|
|
934
|
-
cost.input,
|
|
935
|
-
cost.output,
|
|
936
|
-
cost.cacheRead,
|
|
937
|
-
cost.cacheWrite,
|
|
938
|
-
cost.total,
|
|
939
|
-
noCacheInputCost,
|
|
940
|
-
unpriced ? 1 : 0,
|
|
941
|
-
s.agentType,
|
|
942
|
-
s.serviceTier ?? null,
|
|
943
|
-
// `WHERE NOT EXISTS` binds: skip when a different session_file
|
|
944
|
-
// already holds this (entry_id, timestamp).
|
|
945
|
-
s.entryId,
|
|
946
|
-
s.timestamp,
|
|
963
|
+
writer.add(
|
|
947
964
|
s.sessionFile,
|
|
965
|
+
[s.entryId, s.timestamp],
|
|
966
|
+
[
|
|
967
|
+
s.sessionFile,
|
|
968
|
+
s.entryId,
|
|
969
|
+
s.folder,
|
|
970
|
+
s.model,
|
|
971
|
+
s.provider,
|
|
972
|
+
s.api,
|
|
973
|
+
s.timestamp,
|
|
974
|
+
s.duration,
|
|
975
|
+
s.ttft,
|
|
976
|
+
s.stopReason,
|
|
977
|
+
s.errorMessage,
|
|
978
|
+
s.usage.input,
|
|
979
|
+
s.usage.output,
|
|
980
|
+
s.usage.cacheRead,
|
|
981
|
+
s.usage.cacheWrite,
|
|
982
|
+
s.usage.totalTokens,
|
|
983
|
+
s.usage.premiumRequests ?? 0,
|
|
984
|
+
cost.input,
|
|
985
|
+
cost.output,
|
|
986
|
+
cost.cacheRead,
|
|
987
|
+
cost.cacheWrite,
|
|
988
|
+
cost.total,
|
|
989
|
+
noCacheInputCost,
|
|
990
|
+
unpriced ? 1 : 0,
|
|
991
|
+
s.agentType,
|
|
992
|
+
s.serviceTier ?? null,
|
|
993
|
+
],
|
|
948
994
|
);
|
|
949
|
-
if (result.changes > 0) inserted++;
|
|
950
995
|
}
|
|
996
|
+
return writer.finish();
|
|
951
997
|
});
|
|
952
|
-
|
|
953
|
-
insert();
|
|
954
|
-
return inserted;
|
|
955
998
|
}
|
|
956
999
|
|
|
957
1000
|
/**
|
|
@@ -1333,57 +1376,62 @@ export function markSessionBackfillsComplete(): void {
|
|
|
1333
1376
|
|
|
1334
1377
|
/**
|
|
1335
1378
|
* Insert user-message stats. Idempotent via UNIQUE(session_file, entry_id).
|
|
1336
|
-
* The
|
|
1337
|
-
*
|
|
1338
|
-
*
|
|
1379
|
+
* The fork guard matches {@link insertMessageStats}: forks copy user entries
|
|
1380
|
+
* verbatim into the child JSONL, so the same `(entry_id, timestamp)` must not
|
|
1381
|
+
* land twice across different session files.
|
|
1339
1382
|
*/
|
|
1340
1383
|
export function insertUserMessageStats(stats: Iterable<UserMessageStats>): number {
|
|
1341
1384
|
if (!db) return 0;
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1385
|
+
const writer = new ChunkedInsert(db, {
|
|
1386
|
+
verb: "INSERT OR IGNORE",
|
|
1387
|
+
table: "user_messages",
|
|
1388
|
+
columns: [
|
|
1389
|
+
"session_file",
|
|
1390
|
+
"entry_id",
|
|
1391
|
+
"folder",
|
|
1392
|
+
"timestamp",
|
|
1393
|
+
"model",
|
|
1394
|
+
"provider",
|
|
1395
|
+
"chars",
|
|
1396
|
+
"words",
|
|
1397
|
+
"yelling",
|
|
1398
|
+
"profanity",
|
|
1399
|
+
"anguish",
|
|
1400
|
+
"negation",
|
|
1401
|
+
"repetition",
|
|
1402
|
+
"blame",
|
|
1403
|
+
"prose",
|
|
1404
|
+
"prose_hash",
|
|
1405
|
+
],
|
|
1406
|
+
identity: ["entry_id", "timestamp"],
|
|
1407
|
+
});
|
|
1408
|
+
return atomically(db, () => {
|
|
1358
1409
|
for (const s of stats) {
|
|
1359
|
-
|
|
1360
|
-
s.sessionFile,
|
|
1361
|
-
s.entryId,
|
|
1362
|
-
s.folder,
|
|
1363
|
-
s.timestamp,
|
|
1364
|
-
s.model,
|
|
1365
|
-
s.provider,
|
|
1366
|
-
s.chars,
|
|
1367
|
-
s.words,
|
|
1368
|
-
s.yelling,
|
|
1369
|
-
s.profanity,
|
|
1370
|
-
s.anguish,
|
|
1371
|
-
s.negation,
|
|
1372
|
-
s.repetition,
|
|
1373
|
-
s.blame,
|
|
1374
|
-
s.prose,
|
|
1375
|
-
s.proseHash,
|
|
1376
|
-
// `WHERE NOT EXISTS` binds: skip when a different session_file
|
|
1377
|
-
// already holds this (entry_id, timestamp).
|
|
1378
|
-
s.entryId,
|
|
1379
|
-
s.timestamp,
|
|
1410
|
+
writer.add(
|
|
1380
1411
|
s.sessionFile,
|
|
1412
|
+
[s.entryId, s.timestamp],
|
|
1413
|
+
[
|
|
1414
|
+
s.sessionFile,
|
|
1415
|
+
s.entryId,
|
|
1416
|
+
s.folder,
|
|
1417
|
+
s.timestamp,
|
|
1418
|
+
s.model,
|
|
1419
|
+
s.provider,
|
|
1420
|
+
s.chars,
|
|
1421
|
+
s.words,
|
|
1422
|
+
s.yelling,
|
|
1423
|
+
s.profanity,
|
|
1424
|
+
s.anguish,
|
|
1425
|
+
s.negation,
|
|
1426
|
+
s.repetition,
|
|
1427
|
+
s.blame,
|
|
1428
|
+
s.prose,
|
|
1429
|
+
s.proseHash,
|
|
1430
|
+
],
|
|
1381
1431
|
);
|
|
1382
|
-
if (result.changes > 0) inserted++;
|
|
1383
1432
|
}
|
|
1433
|
+
return writer.finish();
|
|
1384
1434
|
});
|
|
1385
|
-
insert();
|
|
1386
|
-
return inserted;
|
|
1387
1435
|
}
|
|
1388
1436
|
|
|
1389
1437
|
/**
|
|
@@ -1404,13 +1452,12 @@ export function updateUserMessageLinks(links: Iterable<UserMessageLink>): number
|
|
|
1404
1452
|
`);
|
|
1405
1453
|
|
|
1406
1454
|
let updated = 0;
|
|
1407
|
-
|
|
1455
|
+
atomically(db, () => {
|
|
1408
1456
|
for (const link of links) {
|
|
1409
1457
|
const result = stmt.run(link.model, link.provider, link.sessionFile, link.entryId);
|
|
1410
1458
|
if (result.changes > 0) updated++;
|
|
1411
1459
|
}
|
|
1412
1460
|
});
|
|
1413
|
-
apply();
|
|
1414
1461
|
return updated;
|
|
1415
1462
|
}
|
|
1416
1463
|
|
|
@@ -1574,7 +1621,7 @@ export function upsertFrustrationVerdicts(verdicts: readonly FrustrationVerdict[
|
|
|
1574
1621
|
|
|
1575
1622
|
/**
|
|
1576
1623
|
* Insert tool-call rows. Idempotent via UNIQUE(session_file, tool_call_id);
|
|
1577
|
-
* the
|
|
1624
|
+
* the fork guard mirrors {@link insertMessageStats}: forked
|
|
1578
1625
|
* sessions deep-copy assistant entries (same `entry_id`, `timestamp`, and
|
|
1579
1626
|
* tool-call ids under a new file), so first-write-wins across the lineage
|
|
1580
1627
|
* keeps aggregates from double counting. Keyed on the assistant entry
|
|
@@ -1583,46 +1630,50 @@ export function upsertFrustrationVerdicts(verdicts: readonly FrustrationVerdict[
|
|
|
1583
1630
|
*/
|
|
1584
1631
|
export function insertToolCalls(calls: Iterable<ToolCallStats>): number {
|
|
1585
1632
|
if (!db) return 0;
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1633
|
+
const writer = new ChunkedInsert(db, {
|
|
1634
|
+
verb: "INSERT OR IGNORE",
|
|
1635
|
+
table: "tool_calls",
|
|
1636
|
+
columns: [
|
|
1637
|
+
"session_file",
|
|
1638
|
+
"entry_id",
|
|
1639
|
+
"tool_call_id",
|
|
1640
|
+
"folder",
|
|
1641
|
+
"tool_name",
|
|
1642
|
+
"model",
|
|
1643
|
+
"provider",
|
|
1644
|
+
"timestamp",
|
|
1645
|
+
"agent_type",
|
|
1646
|
+
"calls_in_turn",
|
|
1647
|
+
"args_chars",
|
|
1648
|
+
"result_chars",
|
|
1649
|
+
"is_error",
|
|
1650
|
+
],
|
|
1651
|
+
identity: ["entry_id", "timestamp", "tool_call_id"],
|
|
1652
|
+
});
|
|
1653
|
+
return atomically(db, () => {
|
|
1601
1654
|
for (const c of calls) {
|
|
1602
|
-
|
|
1603
|
-
c.sessionFile,
|
|
1604
|
-
c.entryId,
|
|
1605
|
-
c.toolCallId,
|
|
1606
|
-
c.folder,
|
|
1607
|
-
c.toolName,
|
|
1608
|
-
c.model,
|
|
1609
|
-
c.provider,
|
|
1610
|
-
c.timestamp,
|
|
1611
|
-
c.agentType,
|
|
1612
|
-
c.callsInTurn,
|
|
1613
|
-
c.argsChars,
|
|
1614
|
-
// `WHERE NOT EXISTS` binds: skip when a different session_file
|
|
1615
|
-
// already holds this (entry_id, timestamp, tool_call_id).
|
|
1616
|
-
c.entryId,
|
|
1617
|
-
c.timestamp,
|
|
1618
|
-
c.toolCallId,
|
|
1655
|
+
writer.add(
|
|
1619
1656
|
c.sessionFile,
|
|
1657
|
+
[c.entryId, c.timestamp, c.toolCallId],
|
|
1658
|
+
[
|
|
1659
|
+
c.sessionFile,
|
|
1660
|
+
c.entryId,
|
|
1661
|
+
c.toolCallId,
|
|
1662
|
+
c.folder,
|
|
1663
|
+
c.toolName,
|
|
1664
|
+
c.model,
|
|
1665
|
+
c.provider,
|
|
1666
|
+
c.timestamp,
|
|
1667
|
+
c.agentType,
|
|
1668
|
+
c.callsInTurn,
|
|
1669
|
+
c.argsChars,
|
|
1670
|
+
c.resultChars ?? null,
|
|
1671
|
+
c.isError === undefined ? null : c.isError ? 1 : 0,
|
|
1672
|
+
],
|
|
1620
1673
|
);
|
|
1621
|
-
if (result.changes > 0) inserted++;
|
|
1622
1674
|
}
|
|
1675
|
+
return writer.finish();
|
|
1623
1676
|
});
|
|
1624
|
-
insert();
|
|
1625
|
-
return inserted;
|
|
1626
1677
|
}
|
|
1627
1678
|
|
|
1628
1679
|
/**
|
|
@@ -1642,12 +1693,11 @@ export function updateToolResults(links: Iterable<ToolResultLink>): number {
|
|
|
1642
1693
|
`);
|
|
1643
1694
|
|
|
1644
1695
|
let updated = 0;
|
|
1645
|
-
|
|
1696
|
+
atomically(db, () => {
|
|
1646
1697
|
for (const link of links) {
|
|
1647
1698
|
const result = stmt.run(link.resultChars, link.isError ? 1 : 0, link.sessionFile, link.toolCallId);
|
|
1648
1699
|
updated += result.changes;
|
|
1649
1700
|
}
|
|
1650
1701
|
});
|
|
1651
|
-
apply();
|
|
1652
1702
|
return updated;
|
|
1653
1703
|
}
|
package/src/parser.ts
CHANGED
|
@@ -553,6 +553,8 @@ export async function parseSessionFile(
|
|
|
553
553
|
const userLinks: UserMessageLink[] = [];
|
|
554
554
|
const toolCalls: ToolCallStats[] = [];
|
|
555
555
|
const toolResults: ToolResultLink[] = [];
|
|
556
|
+
// First call per id: the row `INSERT OR IGNORE` keeps, so it can be written with its result.
|
|
557
|
+
const callsById = new Map<string, ToolCallStats>();
|
|
556
558
|
|
|
557
559
|
// Reduce each entry immediately so full replays do not retain tool-output/message bodies.
|
|
558
560
|
const visit = (entry: SessionEntry): void => {
|
|
@@ -567,7 +569,13 @@ export async function parseSessionFile(
|
|
|
567
569
|
}
|
|
568
570
|
if (isToolResultMessage(entry)) {
|
|
569
571
|
const link = extractToolResultLink(sessionPath, entry);
|
|
570
|
-
if (link)
|
|
572
|
+
if (!link) return;
|
|
573
|
+
toolResults.push(link);
|
|
574
|
+
const call = callsById.get(link.toolCallId);
|
|
575
|
+
if (call && call.resultChars === undefined) {
|
|
576
|
+
call.resultChars = link.resultChars;
|
|
577
|
+
call.isError = link.isError;
|
|
578
|
+
}
|
|
571
579
|
return;
|
|
572
580
|
}
|
|
573
581
|
if (isModelUsage(entry)) {
|
|
@@ -578,7 +586,10 @@ export async function parseSessionFile(
|
|
|
578
586
|
if (isAssistantMessage(entry)) {
|
|
579
587
|
const msgStats = extractStats(sessionPath, folder, entry, currentServiceTier, agentType);
|
|
580
588
|
if (msgStats) stats.push(msgStats);
|
|
581
|
-
|
|
589
|
+
for (const call of extractToolCalls(sessionPath, folder, entry, agentType)) {
|
|
590
|
+
toolCalls.push(call);
|
|
591
|
+
if (!callsById.has(call.toolCallId)) callsById.set(call.toolCallId, call);
|
|
592
|
+
}
|
|
582
593
|
// Persist links even when the user entry was ingested in an earlier tail read.
|
|
583
594
|
const parentId = entry.parentId;
|
|
584
595
|
const msg = entry.message;
|
package/src/port-conflict.ts
CHANGED
|
@@ -1,10 +1,15 @@
|
|
|
1
1
|
import type { Dirent } from "node:fs";
|
|
2
2
|
import * as fs from "node:fs/promises";
|
|
3
3
|
import * as path from "node:path";
|
|
4
|
-
import { $which } from "@oh-my-pi/pi-utils";
|
|
4
|
+
import { $which, logger } from "@oh-my-pi/pi-utils";
|
|
5
5
|
import { $ } from "bun";
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
/**
|
|
8
|
+
* Upper bound on the identity probe. A live dashboard ingests on its main thread
|
|
9
|
+
* (serially on macOS), which stalls its responses for over a second on large
|
|
10
|
+
* histories; giving up sooner mistakes a busy dashboard for a hung listener.
|
|
11
|
+
*/
|
|
12
|
+
const STATS_PROBE_TIMEOUT_MS = 3_000;
|
|
8
13
|
const PROCESS_EXIT_POLL_MS = 50;
|
|
9
14
|
const PROCESS_EXIT_POLLS = 10;
|
|
10
15
|
const STATS_RUNTIME_IMAGES: Record<string, true> = { bun: true, node: true, omp: true, "omp-stats": true };
|
|
@@ -26,15 +31,29 @@ export const STATS_DASHBOARD_SECURITY_VERSION = "3";
|
|
|
26
31
|
/** IPv4 loopback address shared by the dashboard server and reuse probe. */
|
|
27
32
|
export const STATS_DASHBOARD_HOSTNAME = "127.0.0.1";
|
|
28
33
|
|
|
29
|
-
|
|
34
|
+
/**
|
|
35
|
+
* What answers on the dashboard port:
|
|
36
|
+
* - `reusable`: a current dashboard bound to the requested host.
|
|
37
|
+
* - `replaceable`: an older dashboard this version may upgrade.
|
|
38
|
+
* - `occupied`: any other HTTP answer.
|
|
39
|
+
* - `unresponsive`: accepted the connection but did not answer in time (busy or hung).
|
|
40
|
+
* - `unreachable`: the connection failed, typically because nothing listens there.
|
|
41
|
+
*/
|
|
42
|
+
type StatsDashboardProbe = "reusable" | "replaceable" | "occupied" | "unresponsive" | "unreachable";
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* How {@link startServer} proceeds with the requested port: bind it, reuse the
|
|
46
|
+
* dashboard already serving it, or fall back to an ephemeral port because the
|
|
47
|
+
* holder is neither reusable nor safe to stop.
|
|
48
|
+
*/
|
|
49
|
+
export type StatsPortClaim = "bind" | "reuse" | "fallback";
|
|
30
50
|
|
|
31
51
|
async function probeStatsDashboard(port: number, hostname: string): Promise<StatsDashboardProbe> {
|
|
32
52
|
const probeHostname = hostname === "0.0.0.0" ? STATS_DASHBOARD_HOSTNAME : hostname === "::" ? "::1" : hostname;
|
|
33
53
|
const urlHostname = probeHostname.includes(":") ? `[${probeHostname}]` : probeHostname;
|
|
54
|
+
const signal = AbortSignal.timeout(STATS_PROBE_TIMEOUT_MS);
|
|
34
55
|
try {
|
|
35
|
-
const response = await fetch(`http://${urlHostname}:${port}/api/stats/models`, {
|
|
36
|
-
signal: AbortSignal.timeout(STATS_PROBE_TIMEOUT_MS),
|
|
37
|
-
});
|
|
56
|
+
const response = await fetch(`http://${urlHostname}:${port}/api/stats/models`, { signal });
|
|
38
57
|
const dashboardVersionHeader = response.headers.get(STATS_DASHBOARD_HEADER);
|
|
39
58
|
const dashboardVersion = dashboardVersionHeader === null ? Number.NaN : Number(dashboardVersionHeader);
|
|
40
59
|
// Never replace a newer dashboard: an older CLI must not downgrade it.
|
|
@@ -52,7 +71,7 @@ async function probeStatsDashboard(port: number, hostname: string): Promise<Stat
|
|
|
52
71
|
if (reusable) return "reusable";
|
|
53
72
|
return replaceable ? "replaceable" : "occupied";
|
|
54
73
|
} catch {
|
|
55
|
-
return "unreachable";
|
|
74
|
+
return signal.aborted ? "unresponsive" : "unreachable";
|
|
56
75
|
}
|
|
57
76
|
}
|
|
58
77
|
|
|
@@ -227,13 +246,16 @@ async function terminatePortHolder(holder: PortHolder): Promise<void> {
|
|
|
227
246
|
await Bun.sleep(PROCESS_EXIT_POLL_MS);
|
|
228
247
|
}
|
|
229
248
|
|
|
230
|
-
|
|
249
|
+
/**
|
|
250
|
+
* Stop the stats dashboard answering `probe` on `port` so it can be rebound.
|
|
251
|
+
* Holders that cannot be positively identified as an omp stats dashboard, or
|
|
252
|
+
* that refuse to stop, are left running and yield `"fallback"`.
|
|
253
|
+
*/
|
|
254
|
+
async function reclaimStatsPort(port: number, probe: "replaceable" | "occupied"): Promise<StatsPortClaim> {
|
|
231
255
|
const holder = await findPortHolder(port);
|
|
232
|
-
if (!holder) {
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
if (holder.pid === process.pid) {
|
|
236
|
-
throw new Error(`Port ${port} is held by the current process (${holder.image}, PID ${holder.pid}).`);
|
|
256
|
+
if (!holder || holder.pid === process.pid) {
|
|
257
|
+
logger.debug("Stats dashboard port holder not reclaimable", { port, holder });
|
|
258
|
+
return "fallback";
|
|
237
259
|
}
|
|
238
260
|
|
|
239
261
|
const normalizedImage = holder.image
|
|
@@ -247,33 +269,48 @@ async function reclaimStatsPort(port: number, hasDashboardIdentity = false): Pro
|
|
|
247
269
|
/\/packages\/stats\/src\/index\.ts(?:["'\s]|$)/.test(normalizedCommand) ||
|
|
248
270
|
(normalizedImage === "omp" && /(?:^|\s)stats(?:\s|$)/.test(normalizedCommand)) ||
|
|
249
271
|
/(?:^|\/)omp(?:\.exe)?["'\s]+stats(?:["'\s]|$)/.test(normalizedCommand);
|
|
272
|
+
const hasDashboardIdentity = probe === "replaceable";
|
|
250
273
|
if (!STATS_RUNTIME_IMAGES[normalizedImage] || (!hasStatsIdentity && !hasDashboardIdentity)) {
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
);
|
|
274
|
+
logger.debug("Stats dashboard port held by an unidentified process", { port, holder });
|
|
275
|
+
return "fallback";
|
|
254
276
|
}
|
|
255
277
|
|
|
256
|
-
|
|
257
|
-
|
|
278
|
+
try {
|
|
279
|
+
await terminatePortHolder(holder);
|
|
280
|
+
} catch (error) {
|
|
281
|
+
logger.warn("Failed to stop stale stats dashboard", { port, pid: holder.pid, error: String(error) });
|
|
282
|
+
return "fallback";
|
|
283
|
+
}
|
|
284
|
+
return "bind";
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
async function claimStatsPort(port: number, hostname: string, ifUnreachable: StatsPortClaim): Promise<StatsPortClaim> {
|
|
288
|
+
const probe = await probeStatsDashboard(port, hostname);
|
|
289
|
+
switch (probe) {
|
|
290
|
+
case "reusable":
|
|
291
|
+
return "reuse";
|
|
292
|
+
case "replaceable":
|
|
293
|
+
case "occupied":
|
|
294
|
+
return reclaimStatsPort(port, probe);
|
|
295
|
+
case "unresponsive":
|
|
296
|
+
// Busy or hung: a dashboard stalled mid-ingest must never be stopped.
|
|
297
|
+
return "fallback";
|
|
298
|
+
case "unreachable":
|
|
299
|
+
return ifUnreachable;
|
|
300
|
+
}
|
|
258
301
|
}
|
|
259
302
|
|
|
260
303
|
/**
|
|
261
|
-
* Reuse a
|
|
304
|
+
* Reuse a current dashboard, or upgrade an older one, before binding.
|
|
262
305
|
* The preflight is needed on platforms that permit wildcard and loopback-specific
|
|
263
306
|
* listeners to coexist on one port.
|
|
264
307
|
*/
|
|
265
|
-
export async function prepareStatsPort(port: number, hostname = STATS_DASHBOARD_HOSTNAME): Promise<
|
|
266
|
-
if (port === 0) return "
|
|
267
|
-
|
|
268
|
-
if (probe === "reusable") return "reuse";
|
|
269
|
-
if (probe === "replaceable") return reclaimStatsPort(port, true);
|
|
270
|
-
if (probe === "occupied") return reclaimStatsPort(port);
|
|
271
|
-
return "retry";
|
|
308
|
+
export async function prepareStatsPort(port: number, hostname = STATS_DASHBOARD_HOSTNAME): Promise<StatsPortClaim> {
|
|
309
|
+
if (port === 0) return "bind";
|
|
310
|
+
return claimStatsPort(port, hostname, "bind");
|
|
272
311
|
}
|
|
273
312
|
|
|
274
313
|
/** Reuse or reclaim a listener found after the server bind reports EADDRINUSE. */
|
|
275
|
-
export async function recoverStatsPort(port: number, hostname = STATS_DASHBOARD_HOSTNAME): Promise<
|
|
276
|
-
|
|
277
|
-
if (probe === "reusable") return "reuse";
|
|
278
|
-
return reclaimStatsPort(port, probe === "replaceable");
|
|
314
|
+
export async function recoverStatsPort(port: number, hostname = STATS_DASHBOARD_HOSTNAME): Promise<StatsPortClaim> {
|
|
315
|
+
return claimStatsPort(port, hostname, "fallback");
|
|
279
316
|
}
|