@oh-my-pi/omp-stats 18.6.3 → 18.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/dist/client/index.css +1 -1
- package/dist/client/index.js +96 -96
- package/dist/types/aggregator.d.ts +3 -0
- package/dist/types/client/api.d.ts +5 -1
- package/dist/types/client/data/live.d.ts +16 -11
- package/dist/types/client/data/query-store.d.ts +39 -0
- package/dist/types/client/data/query.d.ts +10 -5
- package/dist/types/client/traces/Minimap.d.ts +2 -0
- package/dist/types/client/traces/TimelineCanvas.d.ts +29 -8
- package/dist/types/client/traces/TranscriptList.d.ts +3 -1
- package/dist/types/db.d.ts +14 -2
- package/dist/types/embedded-client.d.ts +5 -0
- package/dist/types/live.d.ts +5 -3
- package/dist/types/rollup.d.ts +9 -1
- package/dist/types/trace.d.ts +18 -6
- package/dist/types/types.d.ts +8 -0
- package/package.json +4 -4
- package/src/aggregator.ts +26 -9
- package/src/client/api.ts +40 -5
- package/src/client/app/LiveChip.tsx +2 -2
- package/src/client/app/Shell.tsx +2 -2
- package/src/client/data/live.tsx +85 -45
- package/src/client/data/query-store.ts +108 -0
- package/src/client/data/query.ts +47 -56
- package/src/client/routes/CostsRoute.tsx +4 -2
- package/src/client/routes/ErrorsRoute.tsx +3 -1
- package/src/client/routes/FrustrationRoute.tsx +1 -1
- package/src/client/routes/GainRoute.tsx +3 -1
- package/src/client/routes/ModelsRoute.tsx +9 -2
- package/src/client/routes/OverviewRoute.tsx +2 -2
- package/src/client/routes/ProjectsRoute.tsx +1 -1
- package/src/client/routes/ProvidersRoute.tsx +5 -3
- package/src/client/routes/RequestsRoute.tsx +1 -1
- package/src/client/routes/ToolsRoute.tsx +1 -1
- package/src/client/routes/TracesRoute.tsx +5 -4
- package/src/client/styles.css +4 -2
- package/src/client/traces/Minimap.tsx +63 -26
- package/src/client/traces/TimelineCanvas.tsx +224 -105
- package/src/client/traces/TraceView.tsx +29 -10
- package/src/client/traces/TranscriptList.tsx +17 -9
- package/src/client/traces/traces.css +26 -0
- package/src/client/ui/RequestDrawer.tsx +3 -1
- package/src/db.ts +123 -69
- package/src/embedded-client.ts +9 -0
- package/src/live.ts +36 -11
- package/src/parser.ts +18 -17
- package/src/rollup.ts +41 -20
- package/src/server.ts +38 -24
- package/src/trace.ts +54 -40
- package/src/types.ts +8 -0
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* selection. Hand-rolled fixed-row virtualization (32px rows, spacer divs).
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
import { useEffect, useMemo, useRef, useState } from "react";
|
|
7
|
+
import { memo, useEffect, useMemo, useRef, useState } from "react";
|
|
8
8
|
import { formatDurationMs } from "../data/formatters";
|
|
9
9
|
import { EmptyState } from "../ui";
|
|
10
10
|
import type { TraceMarker, TraceSpan, TraceTrack } from "../types";
|
|
@@ -25,6 +25,8 @@ export interface TranscriptListProps {
|
|
|
25
25
|
selection: string | null;
|
|
26
26
|
onSelect: (spanId: string) => void;
|
|
27
27
|
search: string;
|
|
28
|
+
/** Spans matching `search` (from the trace view); `null` when not searching. */
|
|
29
|
+
matchIds: ReadonlySet<string> | null;
|
|
28
30
|
traceStart: number;
|
|
29
31
|
}
|
|
30
32
|
|
|
@@ -49,20 +51,26 @@ export function buildTranscriptRows(tracks: TraceTrack[]): TranscriptRow[] {
|
|
|
49
51
|
return rows;
|
|
50
52
|
}
|
|
51
53
|
|
|
52
|
-
export
|
|
54
|
+
export const TranscriptList = memo(function TranscriptList({
|
|
55
|
+
tracks,
|
|
56
|
+
selection,
|
|
57
|
+
onSelect,
|
|
58
|
+
search,
|
|
59
|
+
matchIds,
|
|
60
|
+
traceStart,
|
|
61
|
+
}: TranscriptListProps) {
|
|
53
62
|
const colors = useTraceTheme();
|
|
54
63
|
const containerRef = useRef<HTMLDivElement>(null);
|
|
55
64
|
const [scrollTop, setScrollTop] = useState(0);
|
|
56
65
|
|
|
57
66
|
const allRows = useMemo(() => buildTranscriptRows(tracks), [tracks]);
|
|
58
67
|
const rows = useMemo(() => {
|
|
68
|
+
if (!matchIds) return allRows;
|
|
59
69
|
const needle = search.trim().toLowerCase();
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
});
|
|
65
|
-
}, [allRows, search]);
|
|
70
|
+
return allRows.filter(row =>
|
|
71
|
+
row.span ? matchIds.has(row.span.id) : (row.marker?.label ?? "").toLowerCase().includes(needle),
|
|
72
|
+
);
|
|
73
|
+
}, [allRows, search, matchIds]);
|
|
66
74
|
|
|
67
75
|
// Scroll the selected row into view when selection changes externally.
|
|
68
76
|
useEffect(() => {
|
|
@@ -135,4 +143,4 @@ export function TranscriptList({ tracks, selection, onSelect, search, traceStart
|
|
|
135
143
|
{rows.length === 0 && <EmptyState title="No matching events" />}
|
|
136
144
|
</div>
|
|
137
145
|
);
|
|
138
|
-
}
|
|
146
|
+
});
|
|
@@ -111,6 +111,32 @@
|
|
|
111
111
|
cursor: grabbing;
|
|
112
112
|
}
|
|
113
113
|
|
|
114
|
+
/* Scene canvas with the hover/selection boxes stacked on top. */
|
|
115
|
+
.traces-canvas-stack {
|
|
116
|
+
position: relative;
|
|
117
|
+
flex-shrink: 0;
|
|
118
|
+
overflow: hidden;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
.traces-canvas-stack > canvas {
|
|
122
|
+
display: block;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
.traces-canvas-hover,
|
|
126
|
+
.traces-canvas-selection {
|
|
127
|
+
position: absolute;
|
|
128
|
+
pointer-events: none;
|
|
129
|
+
box-sizing: border-box;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
.traces-canvas-hover {
|
|
133
|
+
background: rgba(255, 255, 255, 0.16);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
.traces-canvas-selection {
|
|
137
|
+
border: 1.5px solid;
|
|
138
|
+
}
|
|
139
|
+
|
|
114
140
|
.traces-chevron {
|
|
115
141
|
display: inline-flex;
|
|
116
142
|
background: none;
|
|
@@ -34,7 +34,9 @@ export interface RequestDrawerProps {
|
|
|
34
34
|
|
|
35
35
|
/** Full detail sheet for one model request: timing, tokens, cost, ids, error, and the raw session entry. */
|
|
36
36
|
export function RequestDrawer({ id, onClose }: RequestDrawerProps) {
|
|
37
|
-
const query = useQuery(["request", id], () => getRequestDetails(id ?? 0), {
|
|
37
|
+
const query = useQuery(["request", id], ({ signal }) => getRequestDetails(id ?? 0, signal), {
|
|
38
|
+
enabled: id !== null,
|
|
39
|
+
});
|
|
38
40
|
// Never show the previous request's payload under a new id.
|
|
39
41
|
const details = query.stale ? null : query.data;
|
|
40
42
|
const status = details ? REQUEST_STATUS[requestStatus(details)] : null;
|
package/src/db.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Database } from "bun:sqlite";
|
|
2
2
|
import * as fs from "node:fs/promises";
|
|
3
|
-
import type { Usage } from "@oh-my-pi/pi-ai";
|
|
3
|
+
import type { ServiceTier, Usage } from "@oh-my-pi/pi-ai";
|
|
4
4
|
import {
|
|
5
5
|
calculateUncachedInputCost,
|
|
6
6
|
calculateUsageCost,
|
|
@@ -8,9 +8,9 @@ import {
|
|
|
8
8
|
getBundledModel,
|
|
9
9
|
} from "@oh-my-pi/pi-catalog/models";
|
|
10
10
|
import type { ModelCost } from "@oh-my-pi/pi-catalog/types";
|
|
11
|
-
import { getConfigRootDir, getStatsDbPath } from "@oh-my-pi/pi-utils";
|
|
11
|
+
import { getConfigRootDir, getStatsDbPath, VERSION } from "@oh-my-pi/pi-utils";
|
|
12
12
|
import { classifyAgentType, type ParseSessionResult, type SessionParserState } from "./parser";
|
|
13
|
-
import { ensureRollupSchema } from "./rollup";
|
|
13
|
+
import { ensureRollupSchema, getDailyActivityFromRollup } from "./rollup";
|
|
14
14
|
import type {
|
|
15
15
|
AgentType,
|
|
16
16
|
DailyActivityPoint,
|
|
@@ -88,7 +88,10 @@ const BACKFILL_COMPLETE = "complete";
|
|
|
88
88
|
const BACKFILL_PENDING = "pending";
|
|
89
89
|
const USER_MESSAGES_BACKFILL_KEY = "user_messages_v9";
|
|
90
90
|
const USER_MESSAGE_LINKS_REPAIR_KEY = "user_message_links_v1";
|
|
91
|
-
|
|
91
|
+
// v2: the parser also records the served service tier per message, so a full
|
|
92
|
+
// re-parse fills `service_tier` and re-derives ultrafast premium counts that the
|
|
93
|
+
// v1 pass (priority only) left at zero.
|
|
94
|
+
const PRIORITY_PREMIUM_REQUESTS_BACKFILL_KEY = "premium_requests_priority_v2";
|
|
92
95
|
const AGENT_TYPE_BACKFILL_KEY = "agent_type_v1";
|
|
93
96
|
const FORK_DEDUPE_KEY = "fork_dedupe_v1";
|
|
94
97
|
// v2: tool-name sanitization at ingest (see `sanitizeToolName` in parser.ts)
|
|
@@ -106,6 +109,13 @@ const COST_REINGEST_BACKFILL_KEY = "messages_cost_reingest_v1";
|
|
|
106
109
|
// above is already spent for them — without this one, every historical row
|
|
107
110
|
// keeps `cost_unpriced = 0` and unknown scheduled spend reports as free.
|
|
108
111
|
const COST_UNPRICED_BACKFILL_KEY = "messages_cost_unpriced_v1";
|
|
112
|
+
// Rows predating the `cost_no_cache_input` column are NULL; ingest always
|
|
113
|
+
// writes a value, so one pass settles the column for good.
|
|
114
|
+
const NO_CACHE_INPUT_BACKFILL_KEY = "cost_no_cache_input_v1";
|
|
115
|
+
// Zero-cost rows are repriced from the bundled catalog once per release (the
|
|
116
|
+
// value stored is the version that ran it), since only a catalog update can
|
|
117
|
+
// price a model it could not price before.
|
|
118
|
+
const CATALOG_COST_BACKFILL_KEY = "catalog_cost_backfill";
|
|
109
119
|
function shouldResetBackfill(value: string | undefined): boolean {
|
|
110
120
|
return value !== BACKFILL_COMPLETE && value !== BACKFILL_PENDING;
|
|
111
121
|
}
|
|
@@ -127,7 +137,7 @@ export async function initDb(): Promise<Database> {
|
|
|
127
137
|
// Whether `messages` predates this init — drives the one-time agent_type
|
|
128
138
|
// backfill below, so it must be sampled before CREATE TABLE adds the table.
|
|
129
139
|
const messagesTableExisted =
|
|
130
|
-
db.
|
|
140
|
+
db.query("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'messages'").get() !== undefined;
|
|
131
141
|
|
|
132
142
|
// Create tables
|
|
133
143
|
db.run(`
|
|
@@ -158,6 +168,7 @@ export async function initDb(): Promise<Database> {
|
|
|
158
168
|
cost_no_cache_input REAL,
|
|
159
169
|
cost_unpriced INTEGER NOT NULL DEFAULT 0,
|
|
160
170
|
agent_type TEXT NOT NULL DEFAULT 'main',
|
|
171
|
+
service_tier TEXT,
|
|
161
172
|
UNIQUE(session_file, entry_id)
|
|
162
173
|
);
|
|
163
174
|
|
|
@@ -241,23 +252,27 @@ export async function initDb(): Promise<Database> {
|
|
|
241
252
|
);
|
|
242
253
|
`);
|
|
243
254
|
|
|
244
|
-
const offsetColumns = db.
|
|
255
|
+
const offsetColumns = db.query("PRAGMA table_info(file_offsets)").all() as { name: string }[];
|
|
245
256
|
if (!offsetColumns.some(column => column.name === "parser_state")) {
|
|
246
257
|
db.run("ALTER TABLE file_offsets ADD COLUMN parser_state TEXT");
|
|
247
258
|
}
|
|
248
|
-
const messageColumns = db.
|
|
259
|
+
const messageColumns = db.query("PRAGMA table_info(messages)").all() as { name: string }[];
|
|
249
260
|
if (!messageColumns.some(column => column.name === "premium_requests")) {
|
|
250
261
|
db.run("ALTER TABLE messages ADD COLUMN premium_requests REAL NOT NULL DEFAULT 0");
|
|
251
262
|
}
|
|
252
263
|
if (!messageColumns.some(column => column.name === "cost_no_cache_input")) {
|
|
253
264
|
db.run("ALTER TABLE messages ADD COLUMN cost_no_cache_input REAL");
|
|
254
265
|
}
|
|
266
|
+
// Rows ingested before this column existed carry no served tier; a re-parse
|
|
267
|
+
// fills them from the session's assistant messages.
|
|
268
|
+
if (!messageColumns.some(column => column.name === "service_tier")) {
|
|
269
|
+
db.run("ALTER TABLE messages ADD COLUMN service_tier TEXT");
|
|
270
|
+
}
|
|
255
271
|
// Rows ingested before this column existed default to 0 (not unpriced), so
|
|
256
272
|
// their epoch-sentinel zeros read as free until a re-parse rewrites them.
|
|
257
273
|
if (!messageColumns.some(column => column.name === "cost_unpriced")) {
|
|
258
274
|
db.run("ALTER TABLE messages ADD COLUMN cost_unpriced INTEGER NOT NULL DEFAULT 0");
|
|
259
275
|
}
|
|
260
|
-
db.run("UPDATE messages SET premium_requests = 0 WHERE premium_requests IS NULL");
|
|
261
276
|
// Token-usage-by-agent: each message is classified main / subagent / advisor
|
|
262
277
|
// from its transcript path. A brand-new table gets the column from CREATE
|
|
263
278
|
// TABLE and the parser labels rows at insert time; a pre-existing table gets
|
|
@@ -273,7 +288,7 @@ export async function initDb(): Promise<Database> {
|
|
|
273
288
|
// sentinel write (process killed in between) still reclassifies on the next
|
|
274
289
|
// init instead of silently leaving every row as the 'main' default. A
|
|
275
290
|
// brand-new empty table has nothing to reclassify, so it settles COMPLETE.
|
|
276
|
-
db.
|
|
291
|
+
db.query("INSERT OR IGNORE INTO meta (key, value) VALUES (?, ?)").run(
|
|
277
292
|
AGENT_TYPE_BACKFILL_KEY,
|
|
278
293
|
messagesTableExisted ? BACKFILL_PENDING : BACKFILL_COMPLETE,
|
|
279
294
|
);
|
|
@@ -298,7 +313,7 @@ export async function initDb(): Promise<Database> {
|
|
|
298
313
|
// v7 -> v8: `no-op` compounds no longer count as negation; recovered
|
|
299
314
|
// measured false negatives: `:(` emoticons -> anguish,
|
|
300
315
|
// `why (would|did) you` -> blame, `makes no sense` -> negation.
|
|
301
|
-
const userMessageColumns = db.
|
|
316
|
+
const userMessageColumns = db.query("PRAGMA table_info(user_messages)").all() as {
|
|
302
317
|
name: string;
|
|
303
318
|
}[];
|
|
304
319
|
const hasStaleColumn =
|
|
@@ -491,17 +506,20 @@ function calculateNoCacheInputCost(
|
|
|
491
506
|
}
|
|
492
507
|
|
|
493
508
|
function backfillMissingCatalogCosts(database: Database): void {
|
|
509
|
+
const ran = database.query("SELECT value FROM meta WHERE key = ?").get(CATALOG_COST_BACKFILL_KEY) as
|
|
510
|
+
| { value: string }
|
|
511
|
+
| undefined;
|
|
512
|
+
if (ran?.value === VERSION) return;
|
|
513
|
+
const markRan = database.query("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)");
|
|
494
514
|
const rows = database
|
|
495
|
-
.
|
|
515
|
+
.query(`
|
|
496
516
|
SELECT id, provider, model, timestamp, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens
|
|
497
517
|
FROM messages
|
|
498
518
|
WHERE cost_total = 0 AND total_tokens > 0
|
|
499
519
|
`)
|
|
500
520
|
.all() as CostBackfillRow[];
|
|
501
521
|
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
const update = database.prepare(`
|
|
522
|
+
const update = database.query(`
|
|
505
523
|
UPDATE messages
|
|
506
524
|
SET cost_input = ?, cost_output = ?, cost_cache_read = ?, cost_cache_write = ?, cost_total = ?
|
|
507
525
|
WHERE id = ?
|
|
@@ -532,22 +550,27 @@ function backfillMissingCatalogCosts(database: Database): void {
|
|
|
532
550
|
|
|
533
551
|
update.run(cost.input, cost.output, cost.cacheRead, cost.cacheWrite, cost.total, row.id);
|
|
534
552
|
}
|
|
553
|
+
markRan.run(CATALOG_COST_BACKFILL_KEY, VERSION);
|
|
535
554
|
});
|
|
536
555
|
|
|
537
556
|
applyBackfill();
|
|
538
557
|
}
|
|
539
558
|
|
|
540
559
|
function backfillNoCacheInputCosts(database: Database): void {
|
|
560
|
+
const row = database.query("SELECT value FROM meta WHERE key = ?").get(NO_CACHE_INPUT_BACKFILL_KEY) as
|
|
561
|
+
| { value: string }
|
|
562
|
+
| undefined;
|
|
563
|
+
if (row?.value === BACKFILL_COMPLETE) return;
|
|
564
|
+
const markComplete = database.query("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)");
|
|
541
565
|
const rows = database
|
|
542
|
-
.
|
|
566
|
+
.query(`
|
|
543
567
|
SELECT id, provider, model, timestamp, input_tokens, cache_read_tokens, cache_write_tokens
|
|
544
568
|
FROM messages
|
|
545
569
|
WHERE cost_no_cache_input IS NULL
|
|
546
570
|
`)
|
|
547
571
|
.all() as NoCacheInputCostBackfillRow[];
|
|
548
|
-
if (rows.length === 0) return;
|
|
549
572
|
|
|
550
|
-
const update = database.
|
|
573
|
+
const update = database.query("UPDATE messages SET cost_no_cache_input = ? WHERE id = ?");
|
|
551
574
|
const applyBackfill = database.transaction(() => {
|
|
552
575
|
for (const row of rows) {
|
|
553
576
|
const cost = calculateNoCacheInputCost(
|
|
@@ -563,6 +586,7 @@ function backfillNoCacheInputCosts(database: Database): void {
|
|
|
563
586
|
);
|
|
564
587
|
update.run(cost ?? 0, row.id);
|
|
565
588
|
}
|
|
589
|
+
markComplete.run(NO_CACHE_INPUT_BACKFILL_KEY, BACKFILL_COMPLETE);
|
|
566
590
|
});
|
|
567
591
|
applyBackfill();
|
|
568
592
|
}
|
|
@@ -653,18 +677,24 @@ function* parsedRows<T>(
|
|
|
653
677
|
for (const result of results) yield* select(result);
|
|
654
678
|
}
|
|
655
679
|
|
|
656
|
-
/**
|
|
680
|
+
/**
|
|
681
|
+
* Commit distinct transcripts' rows, reconciliation state, and cursors together; failure rolls back the batch.
|
|
682
|
+
* `changes` counts stored rows inserted, updated or deleted (cursor bookkeeping excluded), so callers can
|
|
683
|
+
* tell a batch that changed nothing.
|
|
684
|
+
*/
|
|
657
685
|
export function applySessionParseResults(sessions: ParsedSession[]): {
|
|
658
686
|
processed: number;
|
|
659
687
|
files: number;
|
|
660
688
|
reconcile: boolean;
|
|
689
|
+
changes: number;
|
|
661
690
|
} {
|
|
662
|
-
if (!db) return { processed: 0, files: 0, reconcile: false };
|
|
691
|
+
if (!db) return { processed: 0, files: 0, reconcile: false, changes: 0 };
|
|
663
692
|
const database = db;
|
|
664
693
|
return database.transaction(() => {
|
|
665
694
|
let processed = 0;
|
|
666
695
|
let files = 0;
|
|
667
696
|
let reconcile = false;
|
|
697
|
+
let changes = 0;
|
|
668
698
|
const writes: ParseSessionResult[] = [];
|
|
669
699
|
for (const { sessionFile, result, rebuild, replay } of sessions) {
|
|
670
700
|
const parserState = result.parserState;
|
|
@@ -749,27 +779,34 @@ export function applySessionParseResults(sessions: ParsedSession[]): {
|
|
|
749
779
|
if (absentMessages.size > 0 || absentUsers.size > 0 || absentTools.size > 0) {
|
|
750
780
|
reconcile = true;
|
|
751
781
|
for (const id of absentMessages) {
|
|
752
|
-
database
|
|
782
|
+
changes += database
|
|
753
783
|
.query("DELETE FROM messages WHERE session_file = ? AND entry_id = ?")
|
|
754
|
-
.run(sessionFile, id);
|
|
784
|
+
.run(sessionFile, id).changes;
|
|
755
785
|
}
|
|
756
786
|
for (const id of absentUsers) {
|
|
757
|
-
database
|
|
787
|
+
changes += database
|
|
758
788
|
.query("DELETE FROM user_messages WHERE session_file = ? AND entry_id = ?")
|
|
759
|
-
.run(sessionFile, id);
|
|
789
|
+
.run(sessionFile, id).changes;
|
|
760
790
|
}
|
|
761
791
|
for (const id of absentTools) {
|
|
762
|
-
database
|
|
792
|
+
changes += database
|
|
763
793
|
.query("DELETE FROM tool_calls WHERE session_file = ? AND tool_call_id = ?")
|
|
764
|
-
.run(sessionFile, id);
|
|
794
|
+
.run(sessionFile, id).changes;
|
|
765
795
|
}
|
|
766
796
|
}
|
|
767
797
|
}
|
|
768
798
|
if (replace) {
|
|
769
|
-
if (messages.length > 0)
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
if (
|
|
799
|
+
if (messages.length > 0) {
|
|
800
|
+
changes += database.query("DELETE FROM messages WHERE session_file = ?").run(sessionFile).changes;
|
|
801
|
+
}
|
|
802
|
+
if (users.length > 0) {
|
|
803
|
+
changes += database
|
|
804
|
+
.query("DELETE FROM user_messages WHERE session_file = ?")
|
|
805
|
+
.run(sessionFile).changes;
|
|
806
|
+
}
|
|
807
|
+
if (tools.length > 0) {
|
|
808
|
+
changes += database.query("DELETE FROM tool_calls WHERE session_file = ?").run(sessionFile).changes;
|
|
809
|
+
}
|
|
773
810
|
}
|
|
774
811
|
}
|
|
775
812
|
writes.push(rows);
|
|
@@ -778,11 +815,12 @@ export function applySessionParseResults(sessions: ParsedSession[]): {
|
|
|
778
815
|
if (count > 0) files++;
|
|
779
816
|
}
|
|
780
817
|
// Remove replaced rows before choosing new fork owners, then write each table once per batch.
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
818
|
+
changes +=
|
|
819
|
+
insertMessageStats(parsedRows(writes, result => result.stats)) +
|
|
820
|
+
insertUserMessageStats(parsedRows(writes, result => result.userStats)) +
|
|
821
|
+
updateUserMessageLinks(parsedRows(writes, result => result.userLinks)) +
|
|
822
|
+
insertToolCalls(parsedRows(writes, result => result.toolCalls)) +
|
|
823
|
+
updateToolResults(parsedRows(writes, result => result.toolResults));
|
|
786
824
|
for (const { sessionFile, result } of sessions) {
|
|
787
825
|
if (result.parserState) {
|
|
788
826
|
setFileOffset(sessionFile, result.newOffset, result.parserState.mtimeMs, result.parserState);
|
|
@@ -791,16 +829,26 @@ export function applySessionParseResults(sessions: ParsedSession[]): {
|
|
|
791
829
|
if (reconcile) {
|
|
792
830
|
database.query("INSERT OR REPLACE INTO meta (key, value) VALUES ('session_reconciliation', 'pending')").run();
|
|
793
831
|
}
|
|
794
|
-
return { processed, files, reconcile };
|
|
832
|
+
return { processed, files, reconcile, changes };
|
|
795
833
|
})();
|
|
796
834
|
}
|
|
797
835
|
|
|
836
|
+
/**
|
|
837
|
+
* SQLite `PRAGMA data_version` of the open connection, or null before
|
|
838
|
+
* {@link initDb}. It moves only when another connection (another omp process)
|
|
839
|
+
* commits, so a caller can detect foreign writes without scanning.
|
|
840
|
+
*/
|
|
841
|
+
export function getDataVersion(): number | null {
|
|
842
|
+
if (!db) return null;
|
|
843
|
+
return db.query<{ data_version: number }, []>("PRAGMA data_version").get()?.data_version ?? null;
|
|
844
|
+
}
|
|
845
|
+
|
|
798
846
|
export function prepareSessionSync(): boolean {
|
|
799
|
-
return Boolean(db?.
|
|
847
|
+
return Boolean(db?.query("SELECT 1 FROM meta WHERE key = 'session_reconciliation'").get());
|
|
800
848
|
}
|
|
801
849
|
|
|
802
850
|
export function completeSessionSync(reconcile: boolean): void {
|
|
803
|
-
if (!reconcile) db?.
|
|
851
|
+
if (!reconcile) db?.query("DELETE FROM meta WHERE key = 'session_reconciliation'").run();
|
|
804
852
|
}
|
|
805
853
|
|
|
806
854
|
/**
|
|
@@ -828,9 +876,9 @@ export function insertMessageStats(stats: Iterable<MessageStatsInput>): number {
|
|
|
828
876
|
duration, ttft, stop_reason, error_message,
|
|
829
877
|
input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, total_tokens, premium_requests,
|
|
830
878
|
cost_input, cost_output, cost_cache_read, cost_cache_write, cost_total, cost_no_cache_input,
|
|
831
|
-
cost_unpriced, agent_type
|
|
879
|
+
cost_unpriced, agent_type, service_tier
|
|
832
880
|
)
|
|
833
|
-
SELECT ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
|
|
881
|
+
SELECT ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
|
|
834
882
|
WHERE NOT EXISTS (
|
|
835
883
|
SELECT 1 FROM messages
|
|
836
884
|
WHERE entry_id = ? AND timestamp = ? AND session_file <> ?
|
|
@@ -843,7 +891,8 @@ export function insertMessageStats(stats: Iterable<MessageStatsInput>): number {
|
|
|
843
891
|
cost_cache_write = excluded.cost_cache_write,
|
|
844
892
|
cost_total = excluded.cost_total,
|
|
845
893
|
cost_no_cache_input = excluded.cost_no_cache_input,
|
|
846
|
-
cost_unpriced = excluded.cost_unpriced
|
|
894
|
+
cost_unpriced = excluded.cost_unpriced,
|
|
895
|
+
service_tier = excluded.service_tier
|
|
847
896
|
`);
|
|
848
897
|
|
|
849
898
|
let inserted = 0;
|
|
@@ -877,6 +926,7 @@ export function insertMessageStats(stats: Iterable<MessageStatsInput>): number {
|
|
|
877
926
|
noCacheInputCost,
|
|
878
927
|
unpriced ? 1 : 0,
|
|
879
928
|
s.agentType,
|
|
929
|
+
s.serviceTier ?? null,
|
|
880
930
|
// `WHERE NOT EXISTS` binds: skip when a different session_file
|
|
881
931
|
// already holds this (entry_id, timestamp).
|
|
882
932
|
s.entryId,
|
|
@@ -896,7 +946,7 @@ export function insertMessageStats(stats: Iterable<MessageStatsInput>): number {
|
|
|
896
946
|
*/
|
|
897
947
|
export function getMessageCount(): number {
|
|
898
948
|
if (!db) return 0;
|
|
899
|
-
const stmt = db.
|
|
949
|
+
const stmt = db.query("SELECT COUNT(*) as count FROM messages");
|
|
900
950
|
const row = stmt.get() as { count: number };
|
|
901
951
|
return row.count;
|
|
902
952
|
}
|
|
@@ -941,13 +991,14 @@ function rowToMessageStats(row: any): MessageStats {
|
|
|
941
991
|
},
|
|
942
992
|
},
|
|
943
993
|
agentType: (row.agent_type as AgentType) ?? "main",
|
|
994
|
+
serviceTier: (row.service_tier as ServiceTier | null) ?? null,
|
|
944
995
|
costUnpriced: row.cost_unpriced === 1,
|
|
945
996
|
};
|
|
946
997
|
}
|
|
947
998
|
|
|
948
999
|
export function getRecentRequests(limit = 100): MessageStats[] {
|
|
949
1000
|
if (!db) return [];
|
|
950
|
-
const stmt = db.
|
|
1001
|
+
const stmt = db.query(`
|
|
951
1002
|
SELECT * FROM messages
|
|
952
1003
|
ORDER BY timestamp DESC
|
|
953
1004
|
LIMIT ?
|
|
@@ -958,7 +1009,7 @@ export function getRecentRequests(limit = 100): MessageStats[] {
|
|
|
958
1009
|
export function getRecentErrors(limit = 100, cutoff?: number | null): MessageStats[] {
|
|
959
1010
|
if (!db) return [];
|
|
960
1011
|
const hasCutoff = cutoff !== undefined && cutoff !== null;
|
|
961
|
-
const stmt = db.
|
|
1012
|
+
const stmt = db.query(`
|
|
962
1013
|
SELECT * FROM messages
|
|
963
1014
|
WHERE stop_reason = 'error'
|
|
964
1015
|
${hasCutoff ? "AND timestamp >= ?" : ""}
|
|
@@ -971,7 +1022,7 @@ export function getRecentErrors(limit = 100, cutoff?: number | null): MessageSta
|
|
|
971
1022
|
|
|
972
1023
|
export function getMessageById(id: number): MessageStats | null {
|
|
973
1024
|
if (!db) return null;
|
|
974
|
-
const stmt = db.
|
|
1025
|
+
const stmt = db.query("SELECT * FROM messages WHERE id = ?");
|
|
975
1026
|
const row = stmt.get(id);
|
|
976
1027
|
return row ? rowToMessageStats(row) : null;
|
|
977
1028
|
}
|
|
@@ -980,12 +1031,15 @@ export function getMessageById(id: number): MessageStats | null {
|
|
|
980
1031
|
* Self-initializing (opens the stats DB on first use) so the coding-agent TUI
|
|
981
1032
|
* can query without the dashboard server's init flow. Days use the machine's
|
|
982
1033
|
* timezone — this is a localhost tool, same rationale as
|
|
983
|
-
* {@link getProviderHourlyBurn}.
|
|
1034
|
+
* {@link getProviderHourlyBurn}. Reads the hourly rollup, falling back to the
|
|
1035
|
+
* raw rows while its rebuild backlog is too large for exact reads.
|
|
984
1036
|
*/
|
|
985
1037
|
export async function getDailyActivity(days = 371): Promise<DailyActivityPoint[]> {
|
|
986
1038
|
const database = await initDb();
|
|
987
1039
|
const cutoff = Date.now() - days * 24 * 60 * 60 * 1000;
|
|
988
|
-
const
|
|
1040
|
+
const rolled = getDailyActivityFromRollup(cutoff);
|
|
1041
|
+
if (rolled) return rolled;
|
|
1042
|
+
const stmt = database.query(`
|
|
989
1043
|
SELECT
|
|
990
1044
|
date(timestamp / 1000, 'unixepoch', 'localtime') as day,
|
|
991
1045
|
SUM(cost_total) as cost,
|
|
@@ -1053,7 +1107,7 @@ export async function getDailyActivity(days = 371): Promise<DailyActivityPoint[]
|
|
|
1053
1107
|
* Existing `messages` rows are unaffected - `INSERT OR IGNORE` keeps them.
|
|
1054
1108
|
*/
|
|
1055
1109
|
function backfillUserMessages(database: Database): void {
|
|
1056
|
-
const row = database.
|
|
1110
|
+
const row = database.query("SELECT value FROM meta WHERE key = ?").get(USER_MESSAGES_BACKFILL_KEY) as
|
|
1057
1111
|
| { value: string }
|
|
1058
1112
|
| undefined;
|
|
1059
1113
|
if (!shouldResetBackfill(row?.value)) return;
|
|
@@ -1061,7 +1115,7 @@ function backfillUserMessages(database: Database): void {
|
|
|
1061
1115
|
database.run("DELETE FROM user_messages");
|
|
1062
1116
|
database.run("DELETE FROM file_offsets");
|
|
1063
1117
|
database
|
|
1064
|
-
.
|
|
1118
|
+
.query("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)")
|
|
1065
1119
|
.run(USER_MESSAGES_BACKFILL_KEY, BACKFILL_PENDING);
|
|
1066
1120
|
}
|
|
1067
1121
|
|
|
@@ -1074,7 +1128,7 @@ function backfillUserMessages(database: Database): void {
|
|
|
1074
1128
|
* written here prevents re-wiping on subsequent inits.
|
|
1075
1129
|
*/
|
|
1076
1130
|
function backfillToolCalls(database: Database): void {
|
|
1077
|
-
const row = database.
|
|
1131
|
+
const row = database.query("SELECT value FROM meta WHERE key = ?").get(TOOL_CALLS_BACKFILL_KEY) as
|
|
1078
1132
|
| { value: string }
|
|
1079
1133
|
| undefined;
|
|
1080
1134
|
if (!shouldResetBackfill(row?.value)) return;
|
|
@@ -1082,7 +1136,7 @@ function backfillToolCalls(database: Database): void {
|
|
|
1082
1136
|
database.run("DELETE FROM tool_calls");
|
|
1083
1137
|
database.run("DELETE FROM file_offsets");
|
|
1084
1138
|
database
|
|
1085
|
-
.
|
|
1139
|
+
.query("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)")
|
|
1086
1140
|
.run(TOOL_CALLS_BACKFILL_KEY, BACKFILL_PENDING);
|
|
1087
1141
|
}
|
|
1088
1142
|
|
|
@@ -1098,14 +1152,14 @@ function backfillToolCalls(database: Database): void {
|
|
|
1098
1152
|
* offset reset is safe. Same sentinel protocol as {@link backfillToolCalls}.
|
|
1099
1153
|
*/
|
|
1100
1154
|
function backfillReingestCosts(database: Database): void {
|
|
1101
|
-
const row = database.
|
|
1155
|
+
const row = database.query("SELECT value FROM meta WHERE key = ?").get(COST_REINGEST_BACKFILL_KEY) as
|
|
1102
1156
|
| { value: string }
|
|
1103
1157
|
| undefined;
|
|
1104
1158
|
if (!shouldResetBackfill(row?.value)) return;
|
|
1105
1159
|
|
|
1106
1160
|
database.run("DELETE FROM file_offsets");
|
|
1107
1161
|
database
|
|
1108
|
-
.
|
|
1162
|
+
.query("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)")
|
|
1109
1163
|
.run(COST_REINGEST_BACKFILL_KEY, BACKFILL_PENDING);
|
|
1110
1164
|
}
|
|
1111
1165
|
|
|
@@ -1119,14 +1173,14 @@ function backfillReingestCosts(database: Database): void {
|
|
|
1119
1173
|
* absence as absence. Same sentinel protocol as {@link backfillReingestCosts}.
|
|
1120
1174
|
*/
|
|
1121
1175
|
function backfillUnpricedCosts(database: Database): void {
|
|
1122
|
-
const row = database.
|
|
1176
|
+
const row = database.query("SELECT value FROM meta WHERE key = ?").get(COST_UNPRICED_BACKFILL_KEY) as
|
|
1123
1177
|
| { value: string }
|
|
1124
1178
|
| undefined;
|
|
1125
1179
|
if (!shouldResetBackfill(row?.value)) return;
|
|
1126
1180
|
|
|
1127
1181
|
database.run("DELETE FROM file_offsets");
|
|
1128
1182
|
database
|
|
1129
|
-
.
|
|
1183
|
+
.query("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)")
|
|
1130
1184
|
.run(COST_UNPRICED_BACKFILL_KEY, BACKFILL_PENDING);
|
|
1131
1185
|
}
|
|
1132
1186
|
|
|
@@ -1140,16 +1194,16 @@ function backfillUnpricedCosts(database: Database): void {
|
|
|
1140
1194
|
* interrupted run rolls back and retries on the next init.
|
|
1141
1195
|
*/
|
|
1142
1196
|
function backfillAgentType(database: Database): void {
|
|
1143
|
-
const row = database.
|
|
1197
|
+
const row = database.query("SELECT value FROM meta WHERE key = ?").get(AGENT_TYPE_BACKFILL_KEY) as
|
|
1144
1198
|
| { value: string }
|
|
1145
1199
|
| undefined;
|
|
1146
1200
|
if (row?.value !== BACKFILL_PENDING) return;
|
|
1147
1201
|
|
|
1148
|
-
const sessionFiles = database.
|
|
1202
|
+
const sessionFiles = database.query("SELECT DISTINCT session_file FROM messages").all() as {
|
|
1149
1203
|
session_file: string;
|
|
1150
1204
|
}[];
|
|
1151
|
-
const update = database.
|
|
1152
|
-
const markComplete = database.
|
|
1205
|
+
const update = database.query("UPDATE messages SET agent_type = ? WHERE session_file = ?");
|
|
1206
|
+
const markComplete = database.query("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)");
|
|
1153
1207
|
const apply = database.transaction(() => {
|
|
1154
1208
|
for (const { session_file } of sessionFiles) {
|
|
1155
1209
|
const agentType = classifyAgentType(session_file);
|
|
@@ -1177,12 +1231,12 @@ function backfillAgentType(database: Database): void {
|
|
|
1177
1231
|
* retries on the next init.
|
|
1178
1232
|
*/
|
|
1179
1233
|
function backfillForkDuplicates(database: Database): void {
|
|
1180
|
-
const row = database.
|
|
1234
|
+
const row = database.query("SELECT value FROM meta WHERE key = ?").get(FORK_DEDUPE_KEY) as
|
|
1181
1235
|
| { value: string }
|
|
1182
1236
|
| undefined;
|
|
1183
1237
|
if (row?.value === BACKFILL_COMPLETE) return;
|
|
1184
1238
|
|
|
1185
|
-
const markComplete = database.
|
|
1239
|
+
const markComplete = database.query("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)");
|
|
1186
1240
|
const apply = database.transaction(() => {
|
|
1187
1241
|
database.run(`
|
|
1188
1242
|
DELETE FROM messages
|
|
@@ -1210,14 +1264,14 @@ function backfillForkDuplicates(database: Database): void {
|
|
|
1210
1264
|
* sentinel row in `meta`.
|
|
1211
1265
|
*/
|
|
1212
1266
|
function repairUserMessageLinks(database: Database): void {
|
|
1213
|
-
const row = database.
|
|
1267
|
+
const row = database.query("SELECT value FROM meta WHERE key = ?").get(USER_MESSAGE_LINKS_REPAIR_KEY) as
|
|
1214
1268
|
| { value: string }
|
|
1215
1269
|
| undefined;
|
|
1216
1270
|
if (!shouldResetBackfill(row?.value)) return;
|
|
1217
1271
|
|
|
1218
1272
|
database.run("DELETE FROM file_offsets");
|
|
1219
1273
|
database
|
|
1220
|
-
.
|
|
1274
|
+
.query("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)")
|
|
1221
1275
|
.run(USER_MESSAGE_LINKS_REPAIR_KEY, BACKFILL_PENDING);
|
|
1222
1276
|
}
|
|
1223
1277
|
|
|
@@ -1232,14 +1286,14 @@ function repairUserMessageLinks(database: Database): void {
|
|
|
1232
1286
|
* column. Idempotent: gated by a sentinel row in `meta`.
|
|
1233
1287
|
*/
|
|
1234
1288
|
function backfillPriorityPremiumRequests(database: Database): void {
|
|
1235
|
-
const row = database.
|
|
1289
|
+
const row = database.query("SELECT value FROM meta WHERE key = ?").get(PRIORITY_PREMIUM_REQUESTS_BACKFILL_KEY) as
|
|
1236
1290
|
| { value: string }
|
|
1237
1291
|
| undefined;
|
|
1238
1292
|
if (!shouldResetBackfill(row?.value)) return;
|
|
1239
1293
|
|
|
1240
1294
|
database.run("DELETE FROM file_offsets");
|
|
1241
1295
|
database
|
|
1242
|
-
.
|
|
1296
|
+
.query("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)")
|
|
1243
1297
|
.run(PRIORITY_PREMIUM_REQUESTS_BACKFILL_KEY, BACKFILL_PENDING);
|
|
1244
1298
|
}
|
|
1245
1299
|
|
|
@@ -1248,7 +1302,7 @@ function backfillPriorityPremiumRequests(database: Database): void {
|
|
|
1248
1302
|
*/
|
|
1249
1303
|
export function markSessionBackfillsComplete(): void {
|
|
1250
1304
|
if (!db) return;
|
|
1251
|
-
const markComplete = db.
|
|
1305
|
+
const markComplete = db.query("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)");
|
|
1252
1306
|
const apply = db.transaction(() => {
|
|
1253
1307
|
for (const key of [
|
|
1254
1308
|
USER_MESSAGES_BACKFILL_KEY,
|
|
@@ -1425,7 +1479,7 @@ function toFrustrationCounts(row: FrustrationCountsRow | undefined): Frustration
|
|
|
1425
1479
|
export function getFrustrationOverall(cutoff?: number | null): FrustrationCounts {
|
|
1426
1480
|
if (!db) return toFrustrationCounts(undefined);
|
|
1427
1481
|
const hasCutoff = hasRangeCutoff(cutoff);
|
|
1428
|
-
const stmt = db.
|
|
1482
|
+
const stmt = db.query(`
|
|
1429
1483
|
SELECT ${FRUSTRATION_COUNTS_SQL}
|
|
1430
1484
|
FROM user_messages u
|
|
1431
1485
|
LEFT JOIN frustration_verdicts v ON v.prose_hash = u.prose_hash
|
|
@@ -1442,7 +1496,7 @@ export function getFrustrationOverall(cutoff?: number | null): FrustrationCounts
|
|
|
1442
1496
|
export function getFrustrationByModel(cutoff?: number | null): FrustrationModelRow[] {
|
|
1443
1497
|
if (!db) return [];
|
|
1444
1498
|
const hasCutoff = hasRangeCutoff(cutoff);
|
|
1445
|
-
const stmt = db.
|
|
1499
|
+
const stmt = db.query(`
|
|
1446
1500
|
SELECT u.model AS model, u.provider AS provider, MIN(u.timestamp) AS first_seen, ${FRUSTRATION_COUNTS_SQL}
|
|
1447
1501
|
FROM user_messages u
|
|
1448
1502
|
LEFT JOIN frustration_verdicts v ON v.prose_hash = u.prose_hash
|
|
@@ -1473,7 +1527,7 @@ function pendingProseSql(hasCutoff: boolean): string {
|
|
|
1473
1527
|
export function getPendingFrustrationProse(cutoff?: number | null): PendingProse[] {
|
|
1474
1528
|
if (!db) return [];
|
|
1475
1529
|
const hasCutoff = hasRangeCutoff(cutoff);
|
|
1476
|
-
const stmt = db.
|
|
1530
|
+
const stmt = db.query(pendingProseSql(hasCutoff));
|
|
1477
1531
|
return (hasCutoff ? stmt.all(cutoff) : stmt.all()) as PendingProse[];
|
|
1478
1532
|
}
|
|
1479
1533
|
|
|
@@ -1481,7 +1535,7 @@ export function getPendingFrustrationProse(cutoff?: number | null): PendingProse
|
|
|
1481
1535
|
export function getPendingFrustrationTotals(cutoff?: number | null): { messages: number; chars: number } {
|
|
1482
1536
|
if (!db) return { messages: 0, chars: 0 };
|
|
1483
1537
|
const hasCutoff = hasRangeCutoff(cutoff);
|
|
1484
|
-
const stmt = db.
|
|
1538
|
+
const stmt = db.query(
|
|
1485
1539
|
`SELECT COUNT(*) AS messages, COALESCE(SUM(LENGTH(prose)), 0) AS chars FROM (${pendingProseSql(hasCutoff)})`,
|
|
1486
1540
|
);
|
|
1487
1541
|
const row = (hasCutoff ? stmt.get(cutoff) : stmt.get()) as { messages: number; chars: number } | undefined;
|