@linxiraos/pi-stats 1.1.13 → 1.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -35
- package/dist/client/index.css +1 -1
- package/dist/client/index.js +56 -56
- package/dist/client/styles.css +46 -46
- package/dist/types/client/components/chart-shared.d.ts +21 -21
- package/dist/types/client/components/models-table-shared.d.ts +14 -14
- package/dist/types/client/data/formatters.d.ts +7 -2
- package/dist/types/client/traces/trace-colors.d.ts +1 -1
- package/dist/types/client/types.d.ts +2 -0
- package/dist/types/db.d.ts +24 -2
- package/dist/types/parser.d.ts +45 -2
- package/dist/types/shared-types.d.ts +4 -0
- package/dist/types/trace.d.ts +1 -1
- package/dist/types/types.d.ts +21 -0
- package/package.json +4 -4
- package/src/aggregator.ts +4 -0
- package/src/client/app/AppLayout.tsx +1 -1
- package/src/client/app/NavRail.tsx +1 -1
- package/src/client/components/AgentTokenShare.tsx +6 -6
- package/src/client/components/chart-shared.tsx +22 -22
- package/src/client/data/formatters.ts +15 -3
- package/src/client/routes/BehaviorRoute.tsx +10 -10
- package/src/client/routes/ModelsRoute.tsx +1 -1
- package/src/client/routes/OverviewRoute.tsx +4 -4
- package/src/client/routes/ProvidersRoute.tsx +3 -3
- package/src/client/routes/TracesRoute.tsx +10 -3
- package/src/client/styles.css +54 -52
- package/src/client/traces/SummaryStrip.tsx +2 -2
- package/src/client/traces/trace-colors.ts +13 -13
- package/src/client/types.ts +2 -0
- package/src/db.ts +242 -46
- package/src/parser.ts +131 -18
- package/src/server.ts +1 -1
- package/src/shared-types.ts +4 -0
- package/src/trace.ts +52 -8
- package/src/types.ts +20 -0
package/src/db.ts
CHANGED
|
@@ -22,6 +22,7 @@ import type {
|
|
|
22
22
|
DailyActivityPoint,
|
|
23
23
|
FolderStats,
|
|
24
24
|
MessageStats,
|
|
25
|
+
MessageStatsInput,
|
|
25
26
|
ModelPerformancePoint,
|
|
26
27
|
ModelStats,
|
|
27
28
|
ModelTimeSeriesPoint,
|
|
@@ -49,13 +50,35 @@ const ZERO_USAGE_COST: UsageCost = {
|
|
|
49
50
|
total: 0,
|
|
50
51
|
};
|
|
51
52
|
|
|
52
|
-
|
|
53
|
-
|
|
53
|
+
/**
|
|
54
|
+
* Predicate counting one stored request as "unpriced" — the public rate card
|
|
55
|
+
* has no charge for it, so its zero is unknown spend rather than free usage.
|
|
56
|
+
* Two shapes store that zero:
|
|
57
|
+
* - `xai-oauth` bills through the SuperGrok subscription, so ingestion
|
|
58
|
+
* deliberately records no per-request price;
|
|
59
|
+
* - `cost_unpriced = 1`, set by `insertMessageStats` when `resolveStoredCost`
|
|
60
|
+
* refuses to price a scheduled (time-based) card whose entry carried no
|
|
61
|
+
* recoverable request timestamp — the epoch would silently become the
|
|
62
|
+
* tariff. Such a row keeps its real tokens and its zero until a re-parse
|
|
63
|
+
* recovers the time.
|
|
64
|
+
*
|
|
65
|
+
* Nothing else sets the marker: an explicit recorded zero, a free flat card,
|
|
66
|
+
* and a model with no catalog card at all keep `cost_unpriced = 0` even at the
|
|
67
|
+
* parser's timestamp sentinel, because their zero is a real price.
|
|
68
|
+
* `prefix` qualifies the columns for queries that alias `messages`.
|
|
69
|
+
*/
|
|
70
|
+
function unpricedRequestSql(prefix = ""): string {
|
|
71
|
+
return `CASE WHEN ${prefix}total_tokens > 0 AND ${prefix}cost_total = 0
|
|
72
|
+
AND (${prefix}provider = 'xai-oauth' OR ${prefix}cost_unpriced = 1) THEN 1 ELSE 0 END`;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const UNPRICED_REQUEST_SQL = unpricedRequestSql();
|
|
54
76
|
|
|
55
77
|
interface CostBackfillRow {
|
|
56
78
|
id: number;
|
|
57
79
|
provider: string;
|
|
58
80
|
model: string;
|
|
81
|
+
timestamp: number;
|
|
59
82
|
input_tokens: number;
|
|
60
83
|
output_tokens: number;
|
|
61
84
|
cache_read_tokens: number;
|
|
@@ -66,6 +89,7 @@ interface NoCacheInputCostBackfillRow {
|
|
|
66
89
|
id: number;
|
|
67
90
|
provider: string;
|
|
68
91
|
model: string;
|
|
92
|
+
timestamp: number;
|
|
69
93
|
input_tokens: number;
|
|
70
94
|
cache_read_tokens: number;
|
|
71
95
|
cache_write_tokens: number;
|
|
@@ -128,6 +152,11 @@ const TOOL_CALLS_BACKFILL_KEY = "tool_calls_v1";
|
|
|
128
152
|
// reach the inclusive 200K tier. A one-time full re-parse repairs them through
|
|
129
153
|
// the cost-refreshing UPSERT in `insertMessageStats`.
|
|
130
154
|
const COST_REINGEST_BACKFILL_KEY = "messages_cost_reingest_v1";
|
|
155
|
+
// The absence-aware `resolveStoredCost` and the `cost_unpriced` marker only
|
|
156
|
+
// reach already-ingested rows through a re-parse, and the reingest sentinel
|
|
157
|
+
// above is already spent for them — without this one, every historical row
|
|
158
|
+
// keeps `cost_unpriced = 0` and unknown scheduled spend reports as free.
|
|
159
|
+
const COST_UNPRICED_BACKFILL_KEY = "messages_cost_unpriced_v1";
|
|
131
160
|
function shouldResetBackfill(value: string | undefined): boolean {
|
|
132
161
|
return value !== BACKFILL_COMPLETE && value !== BACKFILL_PENDING;
|
|
133
162
|
}
|
|
@@ -178,6 +207,7 @@ export async function initDb(): Promise<Database> {
|
|
|
178
207
|
cost_cache_write REAL NOT NULL,
|
|
179
208
|
cost_total REAL NOT NULL,
|
|
180
209
|
cost_no_cache_input REAL,
|
|
210
|
+
cost_unpriced INTEGER NOT NULL DEFAULT 0,
|
|
181
211
|
agent_type TEXT NOT NULL DEFAULT 'main',
|
|
182
212
|
UNIQUE(session_file, entry_id)
|
|
183
213
|
);
|
|
@@ -252,6 +282,11 @@ export async function initDb(): Promise<Database> {
|
|
|
252
282
|
if (!messageColumns.some(column => column.name === "cost_no_cache_input")) {
|
|
253
283
|
db.run("ALTER TABLE messages ADD COLUMN cost_no_cache_input REAL");
|
|
254
284
|
}
|
|
285
|
+
// Rows ingested before this column existed default to 0 (not unpriced), so
|
|
286
|
+
// their epoch-sentinel zeros read as free until a re-parse rewrites them.
|
|
287
|
+
if (!messageColumns.some(column => column.name === "cost_unpriced")) {
|
|
288
|
+
db.run("ALTER TABLE messages ADD COLUMN cost_unpriced INTEGER NOT NULL DEFAULT 0");
|
|
289
|
+
}
|
|
255
290
|
db.run("UPDATE messages SET premium_requests = 0 WHERE premium_requests IS NULL");
|
|
256
291
|
// Token-usage-by-agent: each message is classified main / subagent / advisor
|
|
257
292
|
// from its transcript path. A brand-new table gets the column from CREATE
|
|
@@ -331,6 +366,7 @@ export async function initDb(): Promise<Database> {
|
|
|
331
366
|
backfillUserMessages(db);
|
|
332
367
|
backfillToolCalls(db);
|
|
333
368
|
backfillReingestCosts(db);
|
|
369
|
+
backfillUnpricedCosts(db);
|
|
334
370
|
repairUserMessageLinks(db);
|
|
335
371
|
backfillPriorityPremiumRequests(db);
|
|
336
372
|
backfillAgentType(db);
|
|
@@ -366,7 +402,17 @@ function getCatalogCost(provider: string, modelId: string): ModelCost | null {
|
|
|
366
402
|
return null;
|
|
367
403
|
}
|
|
368
404
|
|
|
369
|
-
|
|
405
|
+
/** Whether the catalog prices this model on a time-based (scheduled) card. */
|
|
406
|
+
export function isScheduledCatalogModel(provider: string, modelId: string): boolean {
|
|
407
|
+
return getCatalogCost(provider, modelId)?.timeBased != null;
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
function calculateCatalogCost(
|
|
411
|
+
provider: string,
|
|
412
|
+
modelId: string,
|
|
413
|
+
tokens: CostTokens,
|
|
414
|
+
timestamp: number,
|
|
415
|
+
): UsageCost | null {
|
|
370
416
|
const cost = getCatalogCost(provider, modelId);
|
|
371
417
|
if (!cost) return null;
|
|
372
418
|
|
|
@@ -383,10 +429,10 @@ function calculateCatalogCost(provider: string, modelId: string, tokens: CostTok
|
|
|
383
429
|
(orchestration?.cacheRead ?? 0),
|
|
384
430
|
cost: { ...ZERO_USAGE_COST },
|
|
385
431
|
};
|
|
386
|
-
return calculateUsageCost(cost, usage);
|
|
432
|
+
return calculateUsageCost(cost, usage, timestamp);
|
|
387
433
|
}
|
|
388
434
|
|
|
389
|
-
function normalizeUsageCost(cost: UsageCost): UsageCost {
|
|
435
|
+
function normalizeUsageCost(cost: Partial<UsageCost>): UsageCost {
|
|
390
436
|
const input = cost.input ?? 0;
|
|
391
437
|
const output = cost.output ?? 0;
|
|
392
438
|
const cacheRead = cost.cacheRead ?? 0;
|
|
@@ -395,37 +441,80 @@ function normalizeUsageCost(cost: UsageCost): UsageCost {
|
|
|
395
441
|
return { input, output, cacheRead, cacheWrite, total };
|
|
396
442
|
}
|
|
397
443
|
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
444
|
+
/**
|
|
445
|
+
* The parser records no timestamp for an entry that carries neither a numeric
|
|
446
|
+
* message timestamp nor a parseable entry timestamp, leaving this sentinel.
|
|
447
|
+
* Pricing such a request from the clock would bill it as a 1970 request.
|
|
448
|
+
*/
|
|
449
|
+
function hasRequestTimestamp(timestamp: number): boolean {
|
|
450
|
+
return Number.isFinite(timestamp) && timestamp > 0;
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
interface ResolvedCost {
|
|
454
|
+
cost: UsageCost;
|
|
455
|
+
/**
|
|
456
|
+
* The stored zero is unknown spend rather than a price: a scheduled card
|
|
457
|
+
* with no recoverable request timestamp to select a tariff from. Persisted
|
|
458
|
+
* as `cost_unpriced` so the aggregates do not have to infer it.
|
|
459
|
+
*/
|
|
460
|
+
unpriced: boolean;
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
function resolveStoredCost(stats: MessageStatsInput): ResolvedCost {
|
|
464
|
+
// `usage.cost` is absent when the session entry recorded no price at all, and
|
|
465
|
+
// legacy payloads can carry a partially-populated cost object (e.g. only
|
|
466
|
+
// `total`). The messages table declares every cost_* column as REAL NOT NULL,
|
|
467
|
+
// so any missing field must be normalised here before binding into SQLite.
|
|
468
|
+
const raw: Partial<UsageCost> | undefined = stats.usage.cost;
|
|
404
469
|
const storedCost = raw ? normalizeUsageCost(raw) : undefined;
|
|
470
|
+
const catalogCost = getCatalogCost(stats.provider, stats.model);
|
|
405
471
|
|
|
406
|
-
//
|
|
407
|
-
//
|
|
408
|
-
if (storedCost && (
|
|
472
|
+
// Scheduled prices are frozen per request, including explicitly free usage.
|
|
473
|
+
// Preserve legacy zero-cost subscription correction for unscheduled models.
|
|
474
|
+
if (storedCost && Number.isFinite(storedCost.total) && (storedCost.total !== 0 || catalogCost?.timeBased)) {
|
|
475
|
+
return { cost: storedCost, unpriced: false };
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
// Without a request timestamp a scheduled card has no tariff to select, and
|
|
479
|
+
// the epoch would silently become one. Leave the request unpriced; a re-parse
|
|
480
|
+
// of the session file repairs it once the timestamp is recoverable.
|
|
481
|
+
if (!hasRequestTimestamp(stats.timestamp) && catalogCost?.timeBased) {
|
|
482
|
+
return { cost: storedCost ?? ZERO_USAGE_COST, unpriced: true };
|
|
483
|
+
}
|
|
409
484
|
|
|
410
|
-
return
|
|
485
|
+
return {
|
|
486
|
+
cost:
|
|
487
|
+
calculateCatalogCost(stats.provider, stats.model, stats.usage, stats.timestamp) ??
|
|
488
|
+
storedCost ??
|
|
489
|
+
ZERO_USAGE_COST,
|
|
490
|
+
unpriced: false,
|
|
491
|
+
};
|
|
411
492
|
}
|
|
412
493
|
|
|
413
|
-
function calculateNoCacheInputCost(
|
|
494
|
+
function calculateNoCacheInputCost(
|
|
495
|
+
provider: string,
|
|
496
|
+
modelId: string,
|
|
497
|
+
tokens: CostTokens,
|
|
498
|
+
timestamp: number,
|
|
499
|
+
): number | null {
|
|
414
500
|
const cost = getCatalogCost(provider, modelId);
|
|
415
501
|
if (!cost) return null;
|
|
502
|
+
// Mirrors `resolveStoredCost`: an unpriced scheduled request must not report
|
|
503
|
+
// its whole prompt as cache savings just because the clock could be read.
|
|
504
|
+
if (!hasRequestTimestamp(timestamp) && cost.timeBased) return null;
|
|
416
505
|
const promptInputTokens =
|
|
417
506
|
tokens.input +
|
|
418
507
|
tokens.cacheRead +
|
|
419
508
|
tokens.cacheWrite +
|
|
420
509
|
(tokens.orchestration?.input ?? 0) +
|
|
421
510
|
(tokens.orchestration?.cacheRead ?? 0);
|
|
422
|
-
return calculateUncachedInputCost(cost, promptInputTokens);
|
|
511
|
+
return calculateUncachedInputCost(cost, promptInputTokens, timestamp);
|
|
423
512
|
}
|
|
424
513
|
|
|
425
514
|
function backfillMissingCatalogCosts(database: Database): void {
|
|
426
515
|
const rows = database
|
|
427
516
|
.prepare(`
|
|
428
|
-
SELECT id, provider, model, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens
|
|
517
|
+
SELECT id, provider, model, timestamp, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens
|
|
429
518
|
FROM messages
|
|
430
519
|
WHERE cost_total = 0 AND total_tokens > 0
|
|
431
520
|
`)
|
|
@@ -441,12 +530,24 @@ function backfillMissingCatalogCosts(database: Database): void {
|
|
|
441
530
|
|
|
442
531
|
const applyBackfill = database.transaction(() => {
|
|
443
532
|
for (const row of rows) {
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
533
|
+
// A stored zero cannot distinguish missing historical prices from an
|
|
534
|
+
// explicitly free request. Never reprice recorded scheduled usage: a
|
|
535
|
+
// zero ingested for a scheduled card is permanent, because this
|
|
536
|
+
// backfill skips those rows. Re-parsing the session file is the only
|
|
537
|
+
// repair, and it works only once the entry's absent `cost` reaches
|
|
538
|
+
// `resolveStoredCost` as absence rather than as a synthesized zero.
|
|
539
|
+
if (getCatalogCost(row.provider, row.model)?.timeBased) continue;
|
|
540
|
+
const cost = calculateCatalogCost(
|
|
541
|
+
row.provider,
|
|
542
|
+
row.model,
|
|
543
|
+
{
|
|
544
|
+
input: row.input_tokens,
|
|
545
|
+
output: row.output_tokens,
|
|
546
|
+
cacheRead: row.cache_read_tokens,
|
|
547
|
+
cacheWrite: row.cache_write_tokens,
|
|
548
|
+
},
|
|
549
|
+
row.timestamp,
|
|
550
|
+
);
|
|
450
551
|
|
|
451
552
|
if (!cost || cost.total === 0) continue;
|
|
452
553
|
|
|
@@ -460,7 +561,7 @@ function backfillMissingCatalogCosts(database: Database): void {
|
|
|
460
561
|
function backfillNoCacheInputCosts(database: Database): void {
|
|
461
562
|
const rows = database
|
|
462
563
|
.prepare(`
|
|
463
|
-
SELECT id, provider, model, input_tokens, cache_read_tokens, cache_write_tokens
|
|
564
|
+
SELECT id, provider, model, timestamp, input_tokens, cache_read_tokens, cache_write_tokens
|
|
464
565
|
FROM messages
|
|
465
566
|
WHERE cost_no_cache_input IS NULL
|
|
466
567
|
`)
|
|
@@ -470,12 +571,17 @@ function backfillNoCacheInputCosts(database: Database): void {
|
|
|
470
571
|
const update = database.prepare("UPDATE messages SET cost_no_cache_input = ? WHERE id = ?");
|
|
471
572
|
const applyBackfill = database.transaction(() => {
|
|
472
573
|
for (const row of rows) {
|
|
473
|
-
const cost = calculateNoCacheInputCost(
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
574
|
+
const cost = calculateNoCacheInputCost(
|
|
575
|
+
row.provider,
|
|
576
|
+
row.model,
|
|
577
|
+
{
|
|
578
|
+
input: row.input_tokens,
|
|
579
|
+
output: 0,
|
|
580
|
+
cacheRead: row.cache_read_tokens,
|
|
581
|
+
cacheWrite: row.cache_write_tokens,
|
|
582
|
+
},
|
|
583
|
+
row.timestamp,
|
|
584
|
+
);
|
|
479
585
|
update.run(cost ?? 0, row.id);
|
|
480
586
|
}
|
|
481
587
|
});
|
|
@@ -523,7 +629,7 @@ export function setFileOffset(sessionFile: string, offset: number, lastModified:
|
|
|
523
629
|
* stored cost (orchestration-aware) and keeps `premium_requests` monotonic, so
|
|
524
630
|
* a forced re-parse repairs historical `premium_requests` and cost fix-ups.
|
|
525
631
|
*/
|
|
526
|
-
export function insertMessageStats(stats:
|
|
632
|
+
export function insertMessageStats(stats: MessageStatsInput[]): number {
|
|
527
633
|
if (!db || stats.length === 0) return 0;
|
|
528
634
|
|
|
529
635
|
const stmt = db.prepare(`
|
|
@@ -531,9 +637,10 @@ export function insertMessageStats(stats: MessageStats[]): number {
|
|
|
531
637
|
session_file, entry_id, folder, model, provider, api, timestamp,
|
|
532
638
|
duration, ttft, stop_reason, error_message,
|
|
533
639
|
input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, total_tokens, premium_requests,
|
|
534
|
-
cost_input, cost_output, cost_cache_read, cost_cache_write, cost_total, cost_no_cache_input,
|
|
640
|
+
cost_input, cost_output, cost_cache_read, cost_cache_write, cost_total, cost_no_cache_input,
|
|
641
|
+
cost_unpriced, agent_type
|
|
535
642
|
)
|
|
536
|
-
SELECT ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
|
|
643
|
+
SELECT ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
|
|
537
644
|
WHERE NOT EXISTS (
|
|
538
645
|
SELECT 1 FROM messages
|
|
539
646
|
WHERE entry_id = ? AND timestamp = ? AND session_file <> ?
|
|
@@ -545,14 +652,15 @@ export function insertMessageStats(stats: MessageStats[]): number {
|
|
|
545
652
|
cost_cache_read = excluded.cost_cache_read,
|
|
546
653
|
cost_cache_write = excluded.cost_cache_write,
|
|
547
654
|
cost_total = excluded.cost_total,
|
|
548
|
-
cost_no_cache_input = excluded.cost_no_cache_input
|
|
655
|
+
cost_no_cache_input = excluded.cost_no_cache_input,
|
|
656
|
+
cost_unpriced = excluded.cost_unpriced
|
|
549
657
|
`);
|
|
550
658
|
|
|
551
659
|
let inserted = 0;
|
|
552
660
|
const insert = db.transaction(() => {
|
|
553
661
|
for (const s of stats) {
|
|
554
|
-
const cost = resolveStoredCost(s);
|
|
555
|
-
const noCacheInputCost = calculateNoCacheInputCost(s.provider, s.model, s.usage) ?? 0;
|
|
662
|
+
const { cost, unpriced } = resolveStoredCost(s);
|
|
663
|
+
const noCacheInputCost = calculateNoCacheInputCost(s.provider, s.model, s.usage, s.timestamp) ?? 0;
|
|
556
664
|
const result = stmt.run(
|
|
557
665
|
s.sessionFile,
|
|
558
666
|
s.entryId,
|
|
@@ -577,6 +685,7 @@ export function insertMessageStats(stats: MessageStats[]): number {
|
|
|
577
685
|
cost.cacheWrite,
|
|
578
686
|
cost.total,
|
|
579
687
|
noCacheInputCost,
|
|
688
|
+
unpriced ? 1 : 0,
|
|
580
689
|
s.agentType,
|
|
581
690
|
// `WHERE NOT EXISTS` binds: skip when a different session_file
|
|
582
691
|
// already holds this (entry_id, timestamp).
|
|
@@ -671,7 +780,7 @@ export function getOverallStats(cutoff?: number): AggregatedStats {
|
|
|
671
780
|
SUM(cache_write_tokens) as total_cache_write_tokens,
|
|
672
781
|
SUM(premium_requests) as total_premium_requests,
|
|
673
782
|
SUM(cost_total) as total_cost,
|
|
674
|
-
SUM(${
|
|
783
|
+
SUM(${UNPRICED_REQUEST_SQL}) as unpriced_requests,
|
|
675
784
|
SUM(CASE WHEN cost_no_cache_input > 0
|
|
676
785
|
THEN cost_input + cost_cache_read + cost_cache_write
|
|
677
786
|
ELSE 0 END) as total_cached_prompt_cost,
|
|
@@ -707,7 +816,7 @@ export function getStatsByModel(cutoff?: number): ModelStats[] {
|
|
|
707
816
|
SUM(cache_write_tokens) as total_cache_write_tokens,
|
|
708
817
|
SUM(premium_requests) as total_premium_requests,
|
|
709
818
|
SUM(cost_total) as total_cost,
|
|
710
|
-
SUM(${
|
|
819
|
+
SUM(${UNPRICED_REQUEST_SQL}) as unpriced_requests,
|
|
711
820
|
SUM(CASE WHEN cost_no_cache_input > 0
|
|
712
821
|
THEN cost_input + cost_cache_read + cost_cache_write
|
|
713
822
|
ELSE 0 END) as total_cached_prompt_cost,
|
|
@@ -749,7 +858,7 @@ export function getStatsByFolder(cutoff?: number): FolderStats[] {
|
|
|
749
858
|
SUM(cache_write_tokens) as total_cache_write_tokens,
|
|
750
859
|
SUM(premium_requests) as total_premium_requests,
|
|
751
860
|
SUM(cost_total) as total_cost,
|
|
752
|
-
SUM(${
|
|
861
|
+
SUM(${UNPRICED_REQUEST_SQL}) as unpriced_requests,
|
|
753
862
|
SUM(CASE WHEN cost_no_cache_input > 0
|
|
754
863
|
THEN cost_input + cost_cache_read + cost_cache_write
|
|
755
864
|
ELSE 0 END) as total_cached_prompt_cost,
|
|
@@ -898,7 +1007,7 @@ export function getStatsByProvider(cutoff?: number | null): ProviderAggregate[]
|
|
|
898
1007
|
SUM(cache_write_tokens) as total_cache_write_tokens,
|
|
899
1008
|
SUM(input_tokens + output_tokens + cache_read_tokens + cache_write_tokens) as total_tokens,
|
|
900
1009
|
SUM(cost_total) as total_cost,
|
|
901
|
-
SUM(${
|
|
1010
|
+
SUM(${UNPRICED_REQUEST_SQL}) as unpriced_requests,
|
|
902
1011
|
SUM(premium_requests) as total_premium_requests,
|
|
903
1012
|
AVG(CASE WHEN duration > 0 THEN output_tokens * 1000.0 / duration ELSE NULL END) as avg_tokens_per_second
|
|
904
1013
|
FROM messages
|
|
@@ -996,7 +1105,7 @@ export function getProviderTimeSeries(
|
|
|
996
1105
|
provider,
|
|
997
1106
|
SUM(input_tokens + output_tokens + cache_read_tokens + cache_write_tokens) as total_tokens,
|
|
998
1107
|
SUM(cost_total) as cost,
|
|
999
|
-
SUM(${
|
|
1108
|
+
SUM(${UNPRICED_REQUEST_SQL}) as unpriced_requests,
|
|
1000
1109
|
COUNT(*) as requests
|
|
1001
1110
|
FROM messages
|
|
1002
1111
|
${hasCutoff ? "WHERE timestamp >= ?" : ""}
|
|
@@ -1119,6 +1228,7 @@ function rowToMessageStats(row: any): MessageStats {
|
|
|
1119
1228
|
},
|
|
1120
1229
|
},
|
|
1121
1230
|
agentType: (row.agent_type as AgentType) ?? "main",
|
|
1231
|
+
costUnpriced: row.cost_unpriced === 1,
|
|
1122
1232
|
};
|
|
1123
1233
|
}
|
|
1124
1234
|
|
|
@@ -1160,6 +1270,7 @@ export interface SessionRollupRow {
|
|
|
1160
1270
|
endedAt: number;
|
|
1161
1271
|
totalTokens: number;
|
|
1162
1272
|
costTotal: number;
|
|
1273
|
+
unpricedRequests: number;
|
|
1163
1274
|
/** Comma-joined DISTINCT models. */
|
|
1164
1275
|
models: string;
|
|
1165
1276
|
}
|
|
@@ -1174,6 +1285,7 @@ export function getSessionRollups(): SessionRollupRow[] {
|
|
|
1174
1285
|
MAX(timestamp + COALESCE(duration, 0)) AS endedAt,
|
|
1175
1286
|
SUM(total_tokens) AS totalTokens,
|
|
1176
1287
|
SUM(cost_total) AS costTotal,
|
|
1288
|
+
SUM(${UNPRICED_REQUEST_SQL}) AS unpricedRequests,
|
|
1177
1289
|
GROUP_CONCAT(DISTINCT model) AS models
|
|
1178
1290
|
FROM messages
|
|
1179
1291
|
GROUP BY session_file
|
|
@@ -1209,7 +1321,7 @@ export function getCostTimeSeries(days = 90, cutoff?: number | null): CostTimeSe
|
|
|
1209
1321
|
model,
|
|
1210
1322
|
provider,
|
|
1211
1323
|
SUM(cost_total) as cost,
|
|
1212
|
-
SUM(${
|
|
1324
|
+
SUM(${UNPRICED_REQUEST_SQL}) as unpriced_requests,
|
|
1213
1325
|
SUM(cost_input) as cost_input,
|
|
1214
1326
|
SUM(cost_output) as cost_output,
|
|
1215
1327
|
SUM(cost_cache_read) as cost_cache_read,
|
|
@@ -1367,6 +1479,27 @@ function backfillReingestCosts(database: Database): void {
|
|
|
1367
1479
|
.run(COST_REINGEST_BACKFILL_KEY, BACKFILL_PENDING);
|
|
1368
1480
|
}
|
|
1369
1481
|
|
|
1482
|
+
/**
|
|
1483
|
+
* One-shot `file_offsets` wipe so the next sync re-parses every session and
|
|
1484
|
+
* re-derives `cost_unpriced` from `resolveStoredCost`. Rows ingested before the
|
|
1485
|
+
* marker existed defaulted to 0, and `INSERT ... ON CONFLICT DO UPDATE` only
|
|
1486
|
+
* refreshes them when the session is re-parsed — which the spent reingest
|
|
1487
|
+
* sentinel above will never do again. The re-parse also re-prices the
|
|
1488
|
+
* previously absent legacy `cost` charges, since `resolveStoredCost` now reads
|
|
1489
|
+
* absence as absence. Same sentinel protocol as {@link backfillReingestCosts}.
|
|
1490
|
+
*/
|
|
1491
|
+
function backfillUnpricedCosts(database: Database): void {
|
|
1492
|
+
const row = database.prepare("SELECT value FROM meta WHERE key = ?").get(COST_UNPRICED_BACKFILL_KEY) as
|
|
1493
|
+
| { value: string }
|
|
1494
|
+
| undefined;
|
|
1495
|
+
if (!shouldResetBackfill(row?.value)) return;
|
|
1496
|
+
|
|
1497
|
+
database.run("DELETE FROM file_offsets");
|
|
1498
|
+
database
|
|
1499
|
+
.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)")
|
|
1500
|
+
.run(COST_UNPRICED_BACKFILL_KEY, BACKFILL_PENDING);
|
|
1501
|
+
}
|
|
1502
|
+
|
|
1370
1503
|
/**
|
|
1371
1504
|
* Reclassify pre-existing `messages` rows by agent type once, after the
|
|
1372
1505
|
* `agent_type` column is added to an older database (every prior row defaulted
|
|
@@ -1493,6 +1626,7 @@ export function markSessionBackfillsComplete(): void {
|
|
|
1493
1626
|
USER_MESSAGE_LINKS_REPAIR_KEY,
|
|
1494
1627
|
PRIORITY_PREMIUM_REQUESTS_BACKFILL_KEY,
|
|
1495
1628
|
COST_REINGEST_BACKFILL_KEY,
|
|
1629
|
+
COST_UNPRICED_BACKFILL_KEY,
|
|
1496
1630
|
]) {
|
|
1497
1631
|
markComplete.run(key, BACKFILL_COMPLETE);
|
|
1498
1632
|
}
|
|
@@ -1500,6 +1634,67 @@ export function markSessionBackfillsComplete(): void {
|
|
|
1500
1634
|
apply();
|
|
1501
1635
|
}
|
|
1502
1636
|
|
|
1637
|
+
/** Tables keyed by transcript path; orphans are purged from all of them. */
|
|
1638
|
+
const SESSION_FILE_TABLES = ["messages", "user_messages", "tool_calls", "file_offsets"] as const;
|
|
1639
|
+
|
|
1640
|
+
export interface PruneOrphanResult {
|
|
1641
|
+
/** Distinct session files whose rows were removed. */
|
|
1642
|
+
prunedFiles: number;
|
|
1643
|
+
/** Total rows removed across all session-keyed tables. */
|
|
1644
|
+
deletedRows: number;
|
|
1645
|
+
}
|
|
1646
|
+
|
|
1647
|
+
/**
|
|
1648
|
+
* Delete every row whose `session_file` no longer exists on disk. Deleted
|
|
1649
|
+
* projects (tmp test suites, wiped transcripts) otherwise keep surfacing in
|
|
1650
|
+
* the Operational Feed and models list forever. Runs in one transaction; the
|
|
1651
|
+
* existing-file set is collected by the caller before any DELETE so the
|
|
1652
|
+
* comparison is a single in-memory set probe per stored path.
|
|
1653
|
+
*/
|
|
1654
|
+
export function pruneOrphanSessionsInDb(database: Database, existingFiles: readonly string[]): PruneOrphanResult {
|
|
1655
|
+
const existing = new Set(existingFiles.map(normalizeSessionFilePath));
|
|
1656
|
+
const orphans = new Set<string>();
|
|
1657
|
+
for (const table of SESSION_FILE_TABLES) {
|
|
1658
|
+
const rows = database.prepare(`SELECT DISTINCT session_file FROM ${table}`).all() as {
|
|
1659
|
+
session_file: string;
|
|
1660
|
+
}[];
|
|
1661
|
+
for (const row of rows) {
|
|
1662
|
+
if (!existing.has(normalizeSessionFilePath(row.session_file))) orphans.add(row.session_file);
|
|
1663
|
+
}
|
|
1664
|
+
}
|
|
1665
|
+
if (orphans.size === 0) return { prunedFiles: 0, deletedRows: 0 };
|
|
1666
|
+
|
|
1667
|
+
let deletedRows = 0;
|
|
1668
|
+
const apply = database.transaction(() => {
|
|
1669
|
+
for (const sessionFile of orphans) {
|
|
1670
|
+
for (const table of SESSION_FILE_TABLES) {
|
|
1671
|
+
const result = database.prepare(`DELETE FROM ${table} WHERE session_file = ?`).run(sessionFile);
|
|
1672
|
+
deletedRows += Number(result.changes);
|
|
1673
|
+
}
|
|
1674
|
+
}
|
|
1675
|
+
});
|
|
1676
|
+
apply();
|
|
1677
|
+
return { prunedFiles: orphans.size, deletedRows };
|
|
1678
|
+
}
|
|
1679
|
+
|
|
1680
|
+
/**
|
|
1681
|
+
* {@link pruneOrphanSessionsInDb} against the module DB handle. No-op when
|
|
1682
|
+
* {@link initDb} has not run.
|
|
1683
|
+
*/
|
|
1684
|
+
export function pruneOrphanSessions(existingFiles: readonly string[]): PruneOrphanResult {
|
|
1685
|
+
if (!db) return { prunedFiles: 0, deletedRows: 0 };
|
|
1686
|
+
return pruneOrphanSessionsInDb(db, existingFiles);
|
|
1687
|
+
}
|
|
1688
|
+
|
|
1689
|
+
/**
|
|
1690
|
+
* Comparison key for stored transcript paths: separators unified and case
|
|
1691
|
+
* folded on Windows, where lookups are case-insensitive.
|
|
1692
|
+
*/
|
|
1693
|
+
function normalizeSessionFilePath(sessionFile: string): string {
|
|
1694
|
+
const unified = sessionFile.replaceAll("\\", "/");
|
|
1695
|
+
return process.platform === "win32" ? unified.toLowerCase() : unified;
|
|
1696
|
+
}
|
|
1697
|
+
|
|
1503
1698
|
/**
|
|
1504
1699
|
* Insert user-message stats. Idempotent via UNIQUE(session_file, entry_id).
|
|
1505
1700
|
* The `WHERE NOT EXISTS` clause matches {@link insertMessageStats}: forks
|
|
@@ -1842,7 +2037,9 @@ export function updateToolResults(links: ToolResultLink[]): number {
|
|
|
1842
2037
|
/**
|
|
1843
2038
|
* Shared SELECT list for tool aggregates. Real provider usage comes from the
|
|
1844
2039
|
* invoking assistant turn (`messages` join) divided by `calls_in_turn`, so
|
|
1845
|
-
* per-tool token/cost shares stay additive across tools.
|
|
2040
|
+
* per-tool token/cost shares stay additive across tools. The unpriced share
|
|
2041
|
+
* reuses the request predicate against the joined message, whose stored cost
|
|
2042
|
+
* it attributes.
|
|
1846
2043
|
*/
|
|
1847
2044
|
const TOOL_AGGREGATE_COLUMNS = `
|
|
1848
2045
|
COUNT(*) as calls,
|
|
@@ -1852,8 +2049,7 @@ const TOOL_AGGREGATE_COLUMNS = `
|
|
|
1852
2049
|
SUM(COALESCE(m.total_tokens, 0) * 1.0 / t.calls_in_turn) as total_tokens_share,
|
|
1853
2050
|
SUM(COALESCE(m.output_tokens, 0) * 1.0 / t.calls_in_turn) as output_tokens_share,
|
|
1854
2051
|
SUM(COALESCE(m.cost_total, 0) / t.calls_in_turn) as cost_share,
|
|
1855
|
-
SUM(
|
|
1856
|
-
THEN 1.0 / t.calls_in_turn ELSE 0 END) as unpriced_requests_share,
|
|
2052
|
+
SUM(${unpricedRequestSql("m.")} * 1.0 / t.calls_in_turn) as unpriced_requests_share,
|
|
1857
2053
|
MAX(t.timestamp) as last_used
|
|
1858
2054
|
`;
|
|
1859
2055
|
|