@thehammer/danx-session-telemetry 0.1.47 → 0.1.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.source-hash +1 -1
- package/dist/dispatch-usage.js +22 -3
- package/dist/pricing.d.ts +12 -0
- package/dist/pricing.js +35 -0
- package/dist/session-log-watcher.d.ts +13 -0
- package/dist/session-log-watcher.js +18 -6
- package/dist/stats-wire.d.ts +9 -1
- package/dist/stats-wire.js +57 -0
- package/dist/subagent-live.js +2 -2
- package/dist/transcript-timing.d.ts +9 -1
- package/dist/transcript-timing.js +1 -0
- package/dist/usage-ledger.d.ts +57 -2
- package/dist/usage-ledger.js +84 -3
- package/package.json +1 -1
package/dist/.source-hash
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
b6fe84abf58a5a669072ebe58339676af7f647461c02913638f5756219baa2dc
|
package/dist/dispatch-usage.js
CHANGED
|
@@ -16,9 +16,9 @@
|
|
|
16
16
|
* DB-free: runs in the DB-less worker. The sink decides the transport
|
|
17
17
|
* (HTTP on a registered worker, getPool-direct on the dashboard).
|
|
18
18
|
*/
|
|
19
|
-
import { UsageLedgerCounter, ledgerLineFromRaw, excessUsage, } from "./usage-ledger.js";
|
|
19
|
+
import { UsageLedgerCounter, ledgerLineFromRaw, reportCompactionsOf, excessUsage, } from "./usage-ledger.js";
|
|
20
20
|
import { ClaimTracker, emptyClaimState } from "./claim-tracker.js";
|
|
21
|
-
import { ACTIVITY_BATCH_CAP, DispatchUsageTerminalError, chunkEntries, collapseActivityChanges, } from "./stats-wire.js";
|
|
21
|
+
import { ACTIVITY_BATCH_CAP, COMPACTION_BATCH_CAP, DispatchUsageTerminalError, chunkEntries, collapseActivityChanges, } from "./stats-wire.js";
|
|
22
22
|
import { createUsageWriteQueue } from "./write-queue.js";
|
|
23
23
|
import { createLogger } from "./internal-logger.js";
|
|
24
24
|
const log = createLogger("dispatch-usage");
|
|
@@ -34,6 +34,13 @@ export function zeroUsageTotals() {
|
|
|
34
34
|
toolCallCount: 0,
|
|
35
35
|
};
|
|
36
36
|
}
|
|
37
|
+
/** DX-4922 - the compactions as wire-sized chunks. */
|
|
38
|
+
function chunkCompactions(compactions) {
|
|
39
|
+
const out = [];
|
|
40
|
+
for (let i = 0; i < compactions.length; i += COMPACTION_BATCH_CAP)
|
|
41
|
+
out.push(compactions.slice(i, i + COMPACTION_BATCH_CAP));
|
|
42
|
+
return out;
|
|
43
|
+
}
|
|
37
44
|
export class DispatchUsage {
|
|
38
45
|
counter = new UsageLedgerCounter();
|
|
39
46
|
/** DX-3962 - `null` in `"worker"` mode: the worker can never create a claim, so it can never send an `issueId`. */
|
|
@@ -338,7 +345,8 @@ export class DispatchUsage {
|
|
|
338
345
|
* {@link DispatchUsage.unacknowledged}).
|
|
339
346
|
*/
|
|
340
347
|
observeRawLine(raw, meta) {
|
|
341
|
-
|
|
348
|
+
// DX-4922: a compaction boundary is judged against the session the file belongs to.
|
|
349
|
+
const line = ledgerLineFromRaw(raw, meta.source, meta.byteOffset, reportCompactionsOf(meta.sessionId));
|
|
342
350
|
if (!line)
|
|
343
351
|
return;
|
|
344
352
|
const message = raw.message;
|
|
@@ -418,6 +426,11 @@ export class DispatchUsage {
|
|
|
418
426
|
// starts, so the cap is respected here rather than merely made improbable.
|
|
419
427
|
// Splitting across writes is lossless: each row is a full-state idempotent
|
|
420
428
|
// upsert.
|
|
429
|
+
// DX-4922 - every queued compaction, in chunks of the wire's cap: chunk i rides payload i, so a backlog is
|
|
430
|
+
// drained by this one write (the terminal one included) and never waits for a later tick; a failed payload puts
|
|
431
|
+
// its chunk and every later one back on the counter's queue.
|
|
432
|
+
const compactions = this.counter.takeChangedCompactions();
|
|
433
|
+
const compactionChunks = chunkCompactions(compactions);
|
|
421
434
|
const activities = this.pendingActivities.slice(0, ACTIVITY_BATCH_CAP);
|
|
422
435
|
this.pendingActivities = this.pendingActivities.slice(ACTIVITY_BATCH_CAP);
|
|
423
436
|
// Consume only the resets that reached a report (see `buildCursorReports`).
|
|
@@ -431,11 +444,16 @@ export class DispatchUsage {
|
|
|
431
444
|
const haveSideData = carriesStartCursors ||
|
|
432
445
|
(catchUp ? this.storedAckedLags(reports) : this.cursorsMoved(reports)) ||
|
|
433
446
|
activities.length > 0 ||
|
|
447
|
+
compactions.length > 0 ||
|
|
434
448
|
(this.transcriptPath !== null && !this.transcriptPathSent);
|
|
435
449
|
if (batches.length === 0 && (!this.started || haveSideData))
|
|
436
450
|
batches.push([]);
|
|
451
|
+
while (batches.length < compactionChunks.length)
|
|
452
|
+
batches.push([]);
|
|
437
453
|
for (let i = 0; i < batches.length; i++) {
|
|
438
454
|
const payload = { entries: batches[i] };
|
|
455
|
+
if (i < compactionChunks.length)
|
|
456
|
+
payload.compactions = compactionChunks[i];
|
|
439
457
|
const isLast = i === batches.length - 1;
|
|
440
458
|
if (isLast) {
|
|
441
459
|
if (Object.keys(reports).length > 0)
|
|
@@ -492,6 +510,7 @@ export class DispatchUsage {
|
|
|
492
510
|
}
|
|
493
511
|
this.healthy = false;
|
|
494
512
|
this.counter.requeue(batches.slice(i).flat().map((e) => e.messageId));
|
|
513
|
+
this.counter.requeueCompactions(compactionChunks.slice(i).flat().map((c) => c.id));
|
|
495
514
|
// Re-queue the side data at the FRONT, and leave `ackedCursor` where it
|
|
496
515
|
// was: the next write re-reports all three. Advancing an
|
|
497
516
|
// unacknowledged cursor would skip those lines permanently, and
|
package/dist/pricing.d.ts
CHANGED
|
@@ -73,6 +73,8 @@ export interface PricedTokens {
|
|
|
73
73
|
export declare const SYNTHETIC_MODEL = "<synthetic>";
|
|
74
74
|
/** The `usage.speed` value a fast-mode reply carries. */
|
|
75
75
|
export declare const FAST_SPEED = "fast";
|
|
76
|
+
/** DX-4867 - a prompt longer than this many tokens (input + cache reads + cache writes) is billed at the model's long-prompt card. */
|
|
77
|
+
export declare const LONG_PROMPT_TOKENS = 100000;
|
|
76
78
|
/** DX-4593 - whether a `speed:"fast"` request on `model` is billed at the STANDARD row (see `FAST_BILLS_AT_STANDARD_RATE`): it has no fast row of its own. */
|
|
77
79
|
export declare function billsFastAtStandardRate(model: string): boolean;
|
|
78
80
|
/** One row of the list-price table: a model at a speed and its five per-million-token prices. */
|
|
@@ -119,6 +121,16 @@ export declare function priceUsage(model: string, speed: string, tokens: PricedT
|
|
|
119
121
|
* standard table), or `null` when the tables have none. The default `PriceLookup` of {@link priceLedgerCredit}.
|
|
120
122
|
*/
|
|
121
123
|
export declare function listedPrice(model: string, speed: string): ModelPrice | null;
|
|
124
|
+
/** The prompt tokens of one reply: everything it sent in, cached or not. */
|
|
125
|
+
export declare function promptTokensOf(tokens: PricedTokens): number;
|
|
126
|
+
/** DX-4867 - whether `model` has a second, long-prompt rate card (a model with one card never switches, whatever the prompt length). */
|
|
127
|
+
export declare function hasLongPromptCard(model: string): boolean;
|
|
128
|
+
/**
|
|
129
|
+
* DX-4867 - price ONE reply (not a bucket of replies): like {@link priceUsage}, but a reply whose prompt is longer than
|
|
130
|
+
* {@link LONG_PROMPT_TOKENS} on a model with a long-prompt card is charged that card. A model with one rate card prices
|
|
131
|
+
* exactly as `priceUsage` does.
|
|
132
|
+
*/
|
|
133
|
+
export declare function priceReplyUsage(model: string, speed: string, tokens: PricedTokens): PriceResult;
|
|
122
134
|
/**
|
|
123
135
|
* DX-4595 - dollars for `tokens` at `price` (USD per million tokens): each bucket with its own price. The ONE copy of the formula: `priceUsage`
|
|
124
136
|
* charges the list price with it, and spend pacing charges an account's own (editable) prices with it.
|
package/dist/pricing.js
CHANGED
|
@@ -94,9 +94,23 @@ const PRICING = {
|
|
|
94
94
|
"claude-sonnet-4-6": price(3, 15, 0.3),
|
|
95
95
|
"claude-sonnet-4-5-20250929": price(3, 15, 0.3),
|
|
96
96
|
"claude-sonnet-4-20250514": price(3, 15, 0.3),
|
|
97
|
+
// DX-4867 - Claude Haiku 5.5, the page's row for prompts up to 100K tokens ($0.10 in / $0.50 out, cache hits
|
|
98
|
+
// 0.1x = $0.01, 5m write 1.25x, 1h write 2x). It has a SECOND rate card above 100K prompt tokens
|
|
99
|
+
// (LONG_CONTEXT_PRICING below): this row is the card a usage BUCKET is priced with, because a bucket sums
|
|
100
|
+
// many replies and cannot say which of them crossed the line. `priceReplyUsage` picks the card per reply.
|
|
101
|
+
"claude-haiku-5-5": price(0.1, 0.5, 0.01),
|
|
97
102
|
"claude-haiku-4-5": price(1, 5, 0.1),
|
|
98
103
|
"claude-haiku-4-5-20251001": price(1, 5, 0.1),
|
|
99
104
|
};
|
|
105
|
+
/** DX-4867 - a prompt longer than this many tokens (input + cache reads + cache writes) is billed at the model's long-prompt card. */
|
|
106
|
+
export const LONG_PROMPT_TOKENS = 100_000;
|
|
107
|
+
/**
|
|
108
|
+
* DX-4867 - the rate card of a reply whose prompt is longer than {@link LONG_PROMPT_TOKENS}, for the models that have
|
|
109
|
+
* one (Claude Haiku 5.5: $0.50 in / $2.50 out, cache hits 0.1x). Source: the pricing page's Claude Haiku 5.5 row.
|
|
110
|
+
*/
|
|
111
|
+
const LONG_CONTEXT_PRICING = {
|
|
112
|
+
"claude-haiku-5-5": price(0.5, 2.5, 0.05),
|
|
113
|
+
};
|
|
100
114
|
/**
|
|
101
115
|
* Fast-mode prices (the page's "Fast mode pricing" section): premium
|
|
102
116
|
* input/output with the prompt-caching multipliers applied ON TOP (so a 5m
|
|
@@ -175,6 +189,27 @@ export function listedPrice(model, speed) {
|
|
|
175
189
|
const table = useFastTable ? FAST_PRICING : PRICING;
|
|
176
190
|
return Object.hasOwn(table, model) ? table[model] : null;
|
|
177
191
|
}
|
|
192
|
+
/** The prompt tokens of one reply: everything it sent in, cached or not. */
|
|
193
|
+
export function promptTokensOf(tokens) {
|
|
194
|
+
return tokens.input + tokens.cacheRead + tokens.cacheWrite5m + tokens.cacheWrite1h;
|
|
195
|
+
}
|
|
196
|
+
/** DX-4867 - whether `model` has a second, long-prompt rate card (a model with one card never switches, whatever the prompt length). */
|
|
197
|
+
export function hasLongPromptCard(model) {
|
|
198
|
+
return Object.hasOwn(LONG_CONTEXT_PRICING, model);
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* DX-4867 - price ONE reply (not a bucket of replies): like {@link priceUsage}, but a reply whose prompt is longer than
|
|
202
|
+
* {@link LONG_PROMPT_TOKENS} on a model with a long-prompt card is charged that card. A model with one rate card prices
|
|
203
|
+
* exactly as `priceUsage` does.
|
|
204
|
+
*/
|
|
205
|
+
export function priceReplyUsage(model, speed, tokens) {
|
|
206
|
+
const long = promptTokensOf(tokens) > LONG_PROMPT_TOKENS && Object.hasOwn(LONG_CONTEXT_PRICING, model);
|
|
207
|
+
if (!long)
|
|
208
|
+
return priceUsage(model, speed, tokens);
|
|
209
|
+
if (speed === FAST_SPEED)
|
|
210
|
+
return { kind: "unpriced", model, speed };
|
|
211
|
+
return { kind: "priced", costUsd: costOfTokens(tokens, LONG_CONTEXT_PRICING[model]) };
|
|
212
|
+
}
|
|
178
213
|
/**
|
|
179
214
|
* DX-4595 - dollars for `tokens` at `price` (USD per million tokens): each bucket with its own price. The ONE copy of the formula: `priceUsage`
|
|
180
215
|
* charges the list price with it, and spend pacing charges an account's own (editable) prices with it.
|
|
@@ -197,7 +197,20 @@ export interface EntryMeta {
|
|
|
197
197
|
* skipping them is what would lose data.
|
|
198
198
|
*/
|
|
199
199
|
replay: boolean;
|
|
200
|
+
/**
|
|
201
|
+
* DX-4922 - the session whose transcript this line is read from, taken from the file's own location (never from a
|
|
202
|
+
* line: a forked session's file starts with copies of its parent's lines, which keep the PARENT's `sessionId`). A
|
|
203
|
+
* main transcript is `<session id>.jsonl`; a sub-agent's is `<session id>/subagents/agent-*.jsonl`, and the lines of
|
|
204
|
+
* both carry that session id.
|
|
205
|
+
*/
|
|
206
|
+
sessionId: string;
|
|
200
207
|
}
|
|
208
|
+
/**
|
|
209
|
+
* DX-4922 - the session id a transcript file belongs to: `<session id>.jsonl` for the main transcript, and for a
|
|
210
|
+
* sub-agent file (`<session id>/subagents/agent-*.jsonl`) the directory above `subagents`. Verified on local
|
|
211
|
+
* transcripts: a sub-agent's lines carry its parent session's id, never its own agent id.
|
|
212
|
+
*/
|
|
213
|
+
export declare function transcriptSessionId(filePath: string): string;
|
|
201
214
|
/**
|
|
202
215
|
* Encode a working directory to the subdir name Claude Code writes JSONL
|
|
203
216
|
* under. Replaces `/`, `.`, AND `_` with `-`. Verified empirically against
|
|
@@ -57,6 +57,17 @@ export function skipReplays(consumer) {
|
|
|
57
57
|
consumer(entry);
|
|
58
58
|
};
|
|
59
59
|
}
|
|
60
|
+
/**
|
|
61
|
+
* DX-4922 - the session id a transcript file belongs to: `<session id>.jsonl` for the main transcript, and for a
|
|
62
|
+
* sub-agent file (`<session id>/subagents/agent-*.jsonl`) the directory above `subagents`. Verified on local
|
|
63
|
+
* transcripts: a sub-agent's lines carry its parent session's id, never its own agent id.
|
|
64
|
+
*/
|
|
65
|
+
export function transcriptSessionId(filePath) {
|
|
66
|
+
const dir = dirname(filePath);
|
|
67
|
+
if (basename(dir) === "subagents")
|
|
68
|
+
return basename(dirname(dir));
|
|
69
|
+
return basename(filePath, ".jsonl");
|
|
70
|
+
}
|
|
60
71
|
/**
|
|
61
72
|
* Encode a working directory to the subdir name Claude Code writes JSONL
|
|
62
73
|
* under. Replaces `/`, `.`, AND `_` with `-`. Verified empirically against
|
|
@@ -1192,7 +1203,12 @@ export class SessionLogWatcher {
|
|
|
1192
1203
|
* unread byte), so a line starting exactly AT it is new.
|
|
1193
1204
|
*/
|
|
1194
1205
|
entryMeta(state, byteOffset) {
|
|
1195
|
-
return {
|
|
1206
|
+
return {
|
|
1207
|
+
source: state.source,
|
|
1208
|
+
byteOffset,
|
|
1209
|
+
replay: byteOffset < state.replayUntil,
|
|
1210
|
+
sessionId: transcriptSessionId(state.filePath),
|
|
1211
|
+
};
|
|
1196
1212
|
}
|
|
1197
1213
|
/**
|
|
1198
1214
|
* Emit an entry to registered consumers. For sub-agent streams,
|
|
@@ -1218,11 +1234,7 @@ export class SessionLogWatcher {
|
|
|
1218
1234
|
// `delivered` cursor was already handed to a previous incarnation's
|
|
1219
1235
|
// subscribers. Strictly `<`: `delivered` is an exclusive end (the next
|
|
1220
1236
|
// unread byte), so a line starting exactly AT it is new.
|
|
1221
|
-
const meta =
|
|
1222
|
-
source: state.source,
|
|
1223
|
-
byteOffset,
|
|
1224
|
-
replay: byteOffset < state.replayUntil,
|
|
1225
|
-
};
|
|
1237
|
+
const meta = this.entryMeta(state, byteOffset);
|
|
1226
1238
|
for (const consumer of this.consumers) {
|
|
1227
1239
|
try {
|
|
1228
1240
|
consumer(entry, meta);
|
package/dist/stats-wire.d.ts
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* the turn count and the tool-call count are derived from the ledger on the
|
|
10
10
|
* server, and the response hands them back.
|
|
11
11
|
*/
|
|
12
|
-
import type
|
|
12
|
+
import { type CompactionEntry, type LedgerEntry, type LedgerUsage } from "./usage-ledger.js";
|
|
13
13
|
import { type ClaimState } from "./claim-tracker.js";
|
|
14
14
|
/**
|
|
15
15
|
* DX-2963 — cap on entries per stats write. A from-0 backlog (a reattach, a
|
|
@@ -146,6 +146,12 @@ export interface StatsPayload {
|
|
|
146
146
|
* same deploy-window reason as {@link StatsPayload.sourceCursors}.
|
|
147
147
|
*/
|
|
148
148
|
activities?: ActivityChange[];
|
|
149
|
+
/**
|
|
150
|
+
* DX-4922 — compactions read since the last write, at most {@link COMPACTION_BATCH_CAP} per payload; each an idempotent
|
|
151
|
+
* upsert keyed by the boundary's id. Optional for the same deploy-window reason as {@link StatsPayload.sourceCursors}:
|
|
152
|
+
* a collector that predates compaction detection sends none; absent means "none to report", never "none exist".
|
|
153
|
+
*/
|
|
154
|
+
compactions?: CompactionEntry[];
|
|
149
155
|
/**
|
|
150
156
|
* DX-2954 — the session's own transcript file path, recorded on the
|
|
151
157
|
* dispatch's `jsonl_path`.
|
|
@@ -209,6 +215,8 @@ export declare class StatsPayloadError extends Error {
|
|
|
209
215
|
* it upserts an unbounded list.
|
|
210
216
|
*/
|
|
211
217
|
export declare const ACTIVITY_BATCH_CAP = 500;
|
|
218
|
+
/** DX-4922 — cap on compaction rows per stats write; a session compacts a handful of times, so a longer list is a defect. */
|
|
219
|
+
export declare const COMPACTION_BATCH_CAP = 100;
|
|
212
220
|
/** DX-2954 — cap on sources per stats write, for the same reason. */
|
|
213
221
|
export declare const CURSOR_BATCH_CAP = 500;
|
|
214
222
|
/**
|
package/dist/stats-wire.js
CHANGED
|
@@ -1,3 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* DX-2963 — the stats-write wire contract, shared by the worker (sender) and
|
|
3
|
+
* the dashboard (the one server writer, `usage-ledger.ts`). DB-free so a
|
|
4
|
+
* DB-less worker can import it.
|
|
5
|
+
*
|
|
6
|
+
* A stats write carries ONLY ledger entries (one per reply: the per-field
|
|
7
|
+
* maximum usage seen so far, the reply's tool-use ids) that are new or changed
|
|
8
|
+
* since the last acknowledged write — never totals. Every total, the cost,
|
|
9
|
+
* the turn count and the tool-call count are derived from the ledger on the
|
|
10
|
+
* server, and the response hands them back.
|
|
11
|
+
*/
|
|
12
|
+
import { COMPACTION_TRIGGERS, } from "./usage-ledger.js";
|
|
1
13
|
import { LEDGER_TIMING_FIELDS } from "./ledger-timing.js";
|
|
2
14
|
import { ISSUE_ID_MAX_LENGTH, ISSUE_ID_PATTERN, MAX_HELD_CLAIMS, MAX_PENDING_TRANSITIONS, } from "./claim-tracker.js";
|
|
3
15
|
/**
|
|
@@ -52,6 +64,7 @@ const STATS_PAYLOAD_KEYS = new Set([
|
|
|
52
64
|
"entries",
|
|
53
65
|
"sourceCursors",
|
|
54
66
|
"activities",
|
|
67
|
+
"compactions",
|
|
55
68
|
"transcriptPath",
|
|
56
69
|
]);
|
|
57
70
|
/**
|
|
@@ -61,6 +74,8 @@ const STATS_PAYLOAD_KEYS = new Set([
|
|
|
61
74
|
* it upserts an unbounded list.
|
|
62
75
|
*/
|
|
63
76
|
export const ACTIVITY_BATCH_CAP = 500;
|
|
77
|
+
/** DX-4922 — cap on compaction rows per stats write; a session compacts a handful of times, so a longer list is a defect. */
|
|
78
|
+
export const COMPACTION_BATCH_CAP = 100;
|
|
64
79
|
/** DX-2954 — cap on sources per stats write, for the same reason. */
|
|
65
80
|
export const CURSOR_BATCH_CAP = 500;
|
|
66
81
|
/**
|
|
@@ -343,6 +358,46 @@ function parseActivities(raw) {
|
|
|
343
358
|
};
|
|
344
359
|
});
|
|
345
360
|
}
|
|
361
|
+
const COMPACTION_KEYS = new Set(["id", "source", "trigger", "at", "contextBefore"]);
|
|
362
|
+
function parseCompactions(raw) {
|
|
363
|
+
if (!Array.isArray(raw))
|
|
364
|
+
throw new StatsPayloadError("compactions must be an array");
|
|
365
|
+
if (raw.length > COMPACTION_BATCH_CAP) {
|
|
366
|
+
throw new StatsPayloadError(`compactions exceeds the cap of ${COMPACTION_BATCH_CAP}`);
|
|
367
|
+
}
|
|
368
|
+
const parsed = raw.map((entry, i) => {
|
|
369
|
+
const where = `compactions[${i}]`;
|
|
370
|
+
if (!entry || typeof entry !== "object")
|
|
371
|
+
throw new StatsPayloadError(where);
|
|
372
|
+
const c = entry;
|
|
373
|
+
for (const k of Object.keys(c)) {
|
|
374
|
+
if (!COMPACTION_KEYS.has(k))
|
|
375
|
+
throw new StatsPayloadError(`${where}.${k} is not a compaction field`);
|
|
376
|
+
}
|
|
377
|
+
if (typeof c.id !== "string" || !c.id)
|
|
378
|
+
throw new StatsPayloadError(`${where}.id`);
|
|
379
|
+
if (typeof c.source !== "string" || !c.source)
|
|
380
|
+
throw new StatsPayloadError(`${where}.source`);
|
|
381
|
+
if (!COMPACTION_TRIGGERS.includes(c.trigger)) {
|
|
382
|
+
throw new StatsPayloadError(`${where}.trigger must be one of ${COMPACTION_TRIGGERS.join(", ")}`);
|
|
383
|
+
}
|
|
384
|
+
if (!isNonNegInt(c.at))
|
|
385
|
+
throw new StatsPayloadError(`${where}.at`);
|
|
386
|
+
if (!isNonNegInt(c.contextBefore))
|
|
387
|
+
throw new StatsPayloadError(`${where}.contextBefore`);
|
|
388
|
+
return {
|
|
389
|
+
id: c.id,
|
|
390
|
+
source: c.source,
|
|
391
|
+
trigger: c.trigger,
|
|
392
|
+
at: c.at,
|
|
393
|
+
contextBefore: c.contextBefore,
|
|
394
|
+
};
|
|
395
|
+
});
|
|
396
|
+
// One statement upserts the list, and Postgres refuses a statement that touches a row twice.
|
|
397
|
+
if (new Set(parsed.map((c) => c.id)).size !== parsed.length)
|
|
398
|
+
throw new StatsPayloadError("compactions repeats an id");
|
|
399
|
+
return parsed;
|
|
400
|
+
}
|
|
346
401
|
/**
|
|
347
402
|
* Strict parse of a stats body (fail loud — a malformed entry is a worker
|
|
348
403
|
* bug, never coerced). Refuses more than {@link STATS_BATCH_CAP} entries, and
|
|
@@ -425,6 +480,8 @@ export function parseStatsPayload(body) {
|
|
|
425
480
|
payload.sourceCursors = parseSourceCursors(b.sourceCursors);
|
|
426
481
|
if (b.activities !== undefined)
|
|
427
482
|
payload.activities = parseActivities(b.activities);
|
|
483
|
+
if (b.compactions !== undefined)
|
|
484
|
+
payload.compactions = parseCompactions(b.compactions);
|
|
428
485
|
if (b.transcriptPath !== undefined)
|
|
429
486
|
payload.transcriptPath = b.transcriptPath;
|
|
430
487
|
return payload;
|
package/dist/subagent-live.js
CHANGED
|
@@ -37,7 +37,7 @@
|
|
|
37
37
|
*/
|
|
38
38
|
import { createActivityObserver } from "./activity-observer.js";
|
|
39
39
|
import { SYNTHETIC_MODEL, priceLedgerCredit } from "./pricing.js";
|
|
40
|
-
import { MAIN_SOURCE, UsageLedgerCounter, ledgerLineFromRaw, totalTokensOfUsage } from "./usage-ledger.js";
|
|
40
|
+
import { MAIN_SOURCE, REPORT_NO_COMPACTIONS, UsageLedgerCounter, ledgerLineFromRaw, totalTokensOfUsage } from "./usage-ledger.js";
|
|
41
41
|
/** The line's own timestamp in epoch ms, or null when it carries none that parses. */
|
|
42
42
|
function lineTime(raw) {
|
|
43
43
|
if (typeof raw.timestamp !== "string")
|
|
@@ -126,7 +126,7 @@ export function createSubagentLiveView() {
|
|
|
126
126
|
}
|
|
127
127
|
}
|
|
128
128
|
}
|
|
129
|
-
const line = ledgerLineFromRaw(raw, meta.source, meta.byteOffset);
|
|
129
|
+
const line = ledgerLineFromRaw(raw, meta.source, meta.byteOffset, REPORT_NO_COMPACTIONS);
|
|
130
130
|
if (line !== null && state.counter.observeLine(line))
|
|
131
131
|
dirty.add(meta.source);
|
|
132
132
|
},
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
* line it sees (a stale queued attachment at a restart must not be a gap), and
|
|
16
16
|
* credits only what it derived from lines in its own window.
|
|
17
17
|
*/
|
|
18
|
-
import type { LedgerLine } from "./usage-ledger.js";
|
|
18
|
+
import type { CompactionBoundary, LedgerLine } from "./usage-ledger.js";
|
|
19
19
|
import { type LedgerTiming } from "./ledger-timing.js";
|
|
20
20
|
/** A single gap above this is a suspend or an unanswered prompt, never work. */
|
|
21
21
|
export declare const MAX_CREDITED_GAP_MS = 600000;
|
|
@@ -57,6 +57,14 @@ export type TranscriptLine = {
|
|
|
57
57
|
source: string;
|
|
58
58
|
timestamp: number;
|
|
59
59
|
byteOffset: number;
|
|
60
|
+
}
|
|
61
|
+
/** DX-4922 - a `system` line whose subtype is `compact_boundary`; it feeds the timing reducer exactly as any system line does. */
|
|
62
|
+
| {
|
|
63
|
+
kind: "compaction";
|
|
64
|
+
source: string;
|
|
65
|
+
timestamp: number;
|
|
66
|
+
byteOffset: number;
|
|
67
|
+
boundary: CompactionBoundary;
|
|
60
68
|
};
|
|
61
69
|
export declare function timestampOf(t: TranscriptLine): number;
|
|
62
70
|
export declare class SourceTimingReducer {
|
package/dist/usage-ledger.d.ts
CHANGED
|
@@ -51,6 +51,31 @@ export interface LedgerEntry {
|
|
|
51
51
|
*/
|
|
52
52
|
issueId: string | null;
|
|
53
53
|
}
|
|
54
|
+
/** DX-4922 - what started a compaction: Claude Code's own threshold (`auto`) or the user's `/compact` (`manual`). */
|
|
55
|
+
export declare const COMPACTION_TRIGGERS: readonly ["auto", "manual"];
|
|
56
|
+
export type CompactionTrigger = (typeof COMPACTION_TRIGGERS)[number];
|
|
57
|
+
/**
|
|
58
|
+
* DX-4922 - the facts of one `compact_boundary` transcript line (`type: "system"`, `subtype: "compact_boundary"`;
|
|
59
|
+
* shape taken from real transcripts): its `uuid`, `compactMetadata.trigger` and `compactMetadata.preTokens`, the
|
|
60
|
+
* context size Claude Code measured just before it replaced the history with a summary.
|
|
61
|
+
*/
|
|
62
|
+
export interface CompactionBoundary {
|
|
63
|
+
/** The boundary line's own `uuid`: the compaction's identity, stable across a re-read of the file. */
|
|
64
|
+
id: string;
|
|
65
|
+
trigger: CompactionTrigger;
|
|
66
|
+
contextBefore: number;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* DX-4922 - one compaction as the collector reports it. A resend of the same `id` is an idempotent upsert on the server,
|
|
70
|
+
* which also derives the restart point (the context size of the source's first reply after the boundary) from the
|
|
71
|
+
* replies it stores, so a collector restart can never lose it.
|
|
72
|
+
*/
|
|
73
|
+
export interface CompactionEntry extends CompactionBoundary {
|
|
74
|
+
/** `"main"` or the sub-agent id, as {@link LedgerEntry.source}. */
|
|
75
|
+
source: string;
|
|
76
|
+
/** The boundary line's transcript timestamp, epoch ms. */
|
|
77
|
+
at: number;
|
|
78
|
+
}
|
|
54
79
|
/** The fields of one assistant line the counter needs. */
|
|
55
80
|
export interface LedgerLine {
|
|
56
81
|
messageId: string | undefined;
|
|
@@ -120,6 +145,9 @@ export declare class UsageLedgerCounter {
|
|
|
120
145
|
private readonly changed;
|
|
121
146
|
/** DX-3964 - one timing reducer per source (transcript file); see `transcript-timing.ts`. */
|
|
122
147
|
private readonly reducers;
|
|
148
|
+
/** DX-4922 - every compaction boundary read, by its id. */
|
|
149
|
+
private readonly compactions;
|
|
150
|
+
private readonly compactionsChanged;
|
|
123
151
|
/**
|
|
124
152
|
* DX-3964 - feed one line of the fed set (assistant, user, attachment, system) in FILE ORDER. An assistant
|
|
125
153
|
* line credits usage exactly as {@link observe} does; every kind also feeds its source's timing reducer,
|
|
@@ -127,6 +155,11 @@ export declare class UsageLedgerCounter {
|
|
|
127
155
|
* with no token growth - counts as a change, so {@link takeChanged} re-sends it.
|
|
128
156
|
*/
|
|
129
157
|
observeLine(t: TranscriptLine): boolean;
|
|
158
|
+
private observeCompaction;
|
|
159
|
+
/** DX-4922 - the compactions read since the last call, oldest first, as snapshots; clears the queue. */
|
|
160
|
+
takeChangedCompactions(): CompactionEntry[];
|
|
161
|
+
/** DX-4922 - put compactions back on the changed queue (a write that failed). */
|
|
162
|
+
requeueCompactions(ids: readonly string[]): void;
|
|
130
163
|
/**
|
|
131
164
|
* DX-3964 - a source's file was truncated and is read again from 0: its timing reducer starts fresh and
|
|
132
165
|
* first-line-anchored, so a stale `hwm` of the old file can never credit a gap at the reset. Entries keep their
|
|
@@ -145,19 +178,41 @@ export declare class UsageLedgerCounter {
|
|
|
145
178
|
/** Distinct tool-use ids across all replies. */
|
|
146
179
|
toolUseCount(): number;
|
|
147
180
|
}
|
|
181
|
+
/**
|
|
182
|
+
* DX-4922 - whether a reader reports the compactions it reads. A reader that ships its ledger entries to the server
|
|
183
|
+
* (`DispatchUsage`) reports the compactions of the session it reads; one that only measures locally (the sub-agent live
|
|
184
|
+
* view, the probe, the bench attempt telemetry) says so by name, and a boundary it meets is then a plain system line.
|
|
185
|
+
*/
|
|
186
|
+
export type CompactionReading = {
|
|
187
|
+
readonly report: false;
|
|
188
|
+
} | {
|
|
189
|
+
readonly report: true;
|
|
190
|
+
readonly sessionId: string;
|
|
191
|
+
};
|
|
192
|
+
/** The reader ships nothing, so it reports no compactions. */
|
|
193
|
+
export declare const REPORT_NO_COMPACTIONS: CompactionReading;
|
|
194
|
+
/** The reader reports the compactions of `sessionId`: the session its transcript FILE belongs to. */
|
|
195
|
+
export declare function reportCompactionsOf(sessionId: string): CompactionReading;
|
|
196
|
+
/**
|
|
197
|
+
* The reading for a transcript of a FRESH session read as text: the session id its own lines carry. Only right where
|
|
198
|
+
* nothing is inherited (a bench attempt starts a new session); a fork's file starts with its parent's lines, so a
|
|
199
|
+
* live reader takes the session from the file's location instead (`transcriptSessionId`). A transcript with no line
|
|
200
|
+
* carrying a session id has no compaction boundary either, so it reports none.
|
|
201
|
+
*/
|
|
202
|
+
export declare function reportCompactionsOfFreshTranscript(text: string): CompactionReading;
|
|
148
203
|
/**
|
|
149
204
|
* Parse one raw transcript line (a parsed JSONL object) of the fed set - assistant, user (tool_result and human
|
|
150
205
|
* prompts), attachment, system - into a {@link TranscriptLine}. Returns `null` for every other `type`
|
|
151
206
|
* (`queue-operation`, `last-prompt`, `file-history-*`, `custom-title`, ...). THE one parser for the usage feed:
|
|
152
207
|
* the live watcher subscription, `observeTranscriptText` (claude-probe) and the tests all go through it.
|
|
153
208
|
*/
|
|
154
|
-
export declare function ledgerLineFromRaw(raw: Record<string, unknown>, source: string, byteOffset: number): TranscriptLine | null;
|
|
209
|
+
export declare function ledgerLineFromRaw(raw: Record<string, unknown>, source: string, byteOffset: number, compactions: CompactionReading): TranscriptLine | null;
|
|
155
210
|
/**
|
|
156
211
|
* Count a whole transcript text (one file) into a counter. Byte offsets are
|
|
157
212
|
* computed on UTF-8 bytes so a no-id line keys identically to the live
|
|
158
213
|
* watcher's own offset for the same line.
|
|
159
214
|
*/
|
|
160
|
-
export declare function observeTranscriptText(counter: UsageLedgerCounter, text: string, source: string, startByteOffset?: number): void;
|
|
215
|
+
export declare function observeTranscriptText(counter: UsageLedgerCounter, text: string, source: string, compactions: CompactionReading, startByteOffset?: number): void;
|
|
161
216
|
/**
|
|
162
217
|
* The counter's {@link LedgerLine} for one converted watcher entry
|
|
163
218
|
* (`AgentLogEntry`, as `SessionLogWatcher` emits it: `data.messageId`,
|
package/dist/usage-ledger.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { LEDGER_TIMING_FIELDS, maxTiming, zeroTiming } from "./ledger-timing.js";
|
|
2
2
|
import { SourceTimingReducer, } from "./transcript-timing.js";
|
|
3
|
+
/** DX-4922 - what started a compaction: Claude Code's own threshold (`auto`) or the user's `/compact` (`manual`). */
|
|
4
|
+
export const COMPACTION_TRIGGERS = ["auto", "manual"];
|
|
3
5
|
export const MAIN_SOURCE = "main";
|
|
4
6
|
/**
|
|
5
7
|
* DX-3261 — a producer handed this module a line it cannot safely key or
|
|
@@ -123,6 +125,9 @@ export class UsageLedgerCounter {
|
|
|
123
125
|
changed = new Set();
|
|
124
126
|
/** DX-3964 - one timing reducer per source (transcript file); see `transcript-timing.ts`. */
|
|
125
127
|
reducers = new Map();
|
|
128
|
+
/** DX-4922 - every compaction boundary read, by its id. */
|
|
129
|
+
compactions = new Map();
|
|
130
|
+
compactionsChanged = new Set();
|
|
126
131
|
/**
|
|
127
132
|
* DX-3964 - feed one line of the fed set (assistant, user, attachment, system) in FILE ORDER. An assistant
|
|
128
133
|
* line credits usage exactly as {@link observe} does; every kind also feeds its source's timing reducer,
|
|
@@ -131,6 +136,8 @@ export class UsageLedgerCounter {
|
|
|
131
136
|
*/
|
|
132
137
|
observeLine(t) {
|
|
133
138
|
let didChange = t.kind === "assistant" ? this.observe(t.line) : false;
|
|
139
|
+
if (t.kind === "compaction")
|
|
140
|
+
this.observeCompaction(t);
|
|
134
141
|
const source = t.kind === "assistant" ? t.line.source : t.source;
|
|
135
142
|
let reducer = this.reducers.get(source);
|
|
136
143
|
if (!reducer) {
|
|
@@ -151,6 +158,24 @@ export class UsageLedgerCounter {
|
|
|
151
158
|
}
|
|
152
159
|
return didChange;
|
|
153
160
|
}
|
|
161
|
+
observeCompaction(t) {
|
|
162
|
+
if (this.compactions.has(t.boundary.id))
|
|
163
|
+
return; // a replay of a boundary already read
|
|
164
|
+
this.compactions.set(t.boundary.id, { ...t.boundary, source: t.source, at: t.timestamp });
|
|
165
|
+
this.compactionsChanged.add(t.boundary.id);
|
|
166
|
+
}
|
|
167
|
+
/** DX-4922 - the compactions read since the last call, oldest first, as snapshots; clears the queue. */
|
|
168
|
+
takeChangedCompactions() {
|
|
169
|
+
const out = [...this.compactionsChanged].map((id) => ({ ...this.compactions.get(id) }));
|
|
170
|
+
this.compactionsChanged.clear();
|
|
171
|
+
return out;
|
|
172
|
+
}
|
|
173
|
+
/** DX-4922 - put compactions back on the changed queue (a write that failed). */
|
|
174
|
+
requeueCompactions(ids) {
|
|
175
|
+
for (const id of ids)
|
|
176
|
+
if (this.compactions.has(id))
|
|
177
|
+
this.compactionsChanged.add(id);
|
|
178
|
+
}
|
|
154
179
|
/**
|
|
155
180
|
* DX-3964 - a source's file was truncated and is read again from 0: its timing reducer starts fresh and
|
|
156
181
|
* first-line-anchored, so a stale `hwm` of the old file can never credit a gap at the reset. Entries keep their
|
|
@@ -309,13 +334,42 @@ function toolUsesOf(blocks) {
|
|
|
309
334
|
input: b.input && typeof b.input === "object" && !Array.isArray(b.input) ? b.input : null,
|
|
310
335
|
}));
|
|
311
336
|
}
|
|
337
|
+
/** The reader ships nothing, so it reports no compactions. */
|
|
338
|
+
export const REPORT_NO_COMPACTIONS = { report: false };
|
|
339
|
+
/** The reader reports the compactions of `sessionId`: the session its transcript FILE belongs to. */
|
|
340
|
+
export function reportCompactionsOf(sessionId) {
|
|
341
|
+
return { report: true, sessionId };
|
|
342
|
+
}
|
|
343
|
+
const NEWLINE = "\n";
|
|
344
|
+
/**
|
|
345
|
+
* The reading for a transcript of a FRESH session read as text: the session id its own lines carry. Only right where
|
|
346
|
+
* nothing is inherited (a bench attempt starts a new session); a fork's file starts with its parent's lines, so a
|
|
347
|
+
* live reader takes the session from the file's location instead (`transcriptSessionId`). A transcript with no line
|
|
348
|
+
* carrying a session id has no compaction boundary either, so it reports none.
|
|
349
|
+
*/
|
|
350
|
+
export function reportCompactionsOfFreshTranscript(text) {
|
|
351
|
+
for (const rawLine of text.split(NEWLINE)) {
|
|
352
|
+
const trimmed = rawLine.trim();
|
|
353
|
+
if (!trimmed)
|
|
354
|
+
continue;
|
|
355
|
+
try {
|
|
356
|
+
const sessionId = JSON.parse(trimmed).sessionId;
|
|
357
|
+
if (typeof sessionId === "string" && sessionId.length > 0)
|
|
358
|
+
return reportCompactionsOf(sessionId);
|
|
359
|
+
}
|
|
360
|
+
catch {
|
|
361
|
+
// a truncated line carries no session id; the next one may
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
return REPORT_NO_COMPACTIONS;
|
|
365
|
+
}
|
|
312
366
|
/**
|
|
313
367
|
* Parse one raw transcript line (a parsed JSONL object) of the fed set - assistant, user (tool_result and human
|
|
314
368
|
* prompts), attachment, system - into a {@link TranscriptLine}. Returns `null` for every other `type`
|
|
315
369
|
* (`queue-operation`, `last-prompt`, `file-history-*`, `custom-title`, ...). THE one parser for the usage feed:
|
|
316
370
|
* the live watcher subscription, `observeTranscriptText` (claude-probe) and the tests all go through it.
|
|
317
371
|
*/
|
|
318
|
-
export function ledgerLineFromRaw(raw, source, byteOffset) {
|
|
372
|
+
export function ledgerLineFromRaw(raw, source, byteOffset, compactions) {
|
|
319
373
|
const type = raw.type;
|
|
320
374
|
if (typeof type !== "string" || !FED_TYPES.has(type))
|
|
321
375
|
return null;
|
|
@@ -352,14 +406,41 @@ export function ledgerLineFromRaw(raw, source, byteOffset) {
|
|
|
352
406
|
if (type === "user") {
|
|
353
407
|
return { kind: "user", source, timestamp, byteOffset, results: toolResultsOf(raw, blocksOf(message)) };
|
|
354
408
|
}
|
|
409
|
+
// DX-4922: only the session's OWN compactions are reported. A forked or resumed session's transcript starts with
|
|
410
|
+
// copies of its parent's lines, boundary lines included, and those keep the parent's `sessionId`; the parent reports
|
|
411
|
+
// its own, so a copy is read as the plain system line it also is for timing. A reader that reports no compactions
|
|
412
|
+
// reads every boundary that way.
|
|
413
|
+
if (type === "system" && raw.subtype === "compact_boundary" && compactions.report) {
|
|
414
|
+
if (typeof raw.sessionId !== "string" || raw.sessionId.length === 0) {
|
|
415
|
+
throw new LedgerLineDataError("ledgerLineFromRaw: compact_boundary without a sessionId, so whose compaction it is cannot be told");
|
|
416
|
+
}
|
|
417
|
+
if (raw.sessionId === compactions.sessionId) {
|
|
418
|
+
return { kind: "compaction", source, timestamp, byteOffset, boundary: compactionBoundaryOf(raw) };
|
|
419
|
+
}
|
|
420
|
+
}
|
|
355
421
|
return { kind: type, source, timestamp, byteOffset };
|
|
356
422
|
}
|
|
423
|
+
/** DX-4922 - read a `compact_boundary` line; a boundary this reader cannot identify or measure is a loud error, never a guess. */
|
|
424
|
+
function compactionBoundaryOf(raw) {
|
|
425
|
+
const meta = raw.compactMetadata;
|
|
426
|
+
const trigger = meta?.trigger;
|
|
427
|
+
const preTokens = meta?.preTokens;
|
|
428
|
+
if (typeof raw.uuid !== "string" ||
|
|
429
|
+
raw.uuid.length === 0 ||
|
|
430
|
+
!COMPACTION_TRIGGERS.includes(trigger) ||
|
|
431
|
+
typeof preTokens !== "number" ||
|
|
432
|
+
!Number.isFinite(preTokens) ||
|
|
433
|
+
preTokens < 0) {
|
|
434
|
+
throw new LedgerLineDataError(`ledgerLineFromRaw: compact_boundary without a uuid, a known trigger and a preTokens count (uuid=${JSON.stringify(raw.uuid)}, trigger=${JSON.stringify(trigger)}, preTokens=${JSON.stringify(preTokens)})`);
|
|
435
|
+
}
|
|
436
|
+
return { id: raw.uuid, trigger: trigger, contextBefore: Math.round(preTokens) };
|
|
437
|
+
}
|
|
357
438
|
/**
|
|
358
439
|
* Count a whole transcript text (one file) into a counter. Byte offsets are
|
|
359
440
|
* computed on UTF-8 bytes so a no-id line keys identically to the live
|
|
360
441
|
* watcher's own offset for the same line.
|
|
361
442
|
*/
|
|
362
|
-
export function observeTranscriptText(counter, text, source, startByteOffset = 0) {
|
|
443
|
+
export function observeTranscriptText(counter, text, source, compactions, startByteOffset = 0) {
|
|
363
444
|
const encoder = new TextEncoder();
|
|
364
445
|
let offset = startByteOffset;
|
|
365
446
|
for (const rawLine of text.split("\n")) {
|
|
@@ -376,7 +457,7 @@ export function observeTranscriptText(counter, text, source, startByteOffset = 0
|
|
|
376
457
|
catch {
|
|
377
458
|
continue;
|
|
378
459
|
}
|
|
379
|
-
const line = ledgerLineFromRaw(parsed, source, lineOffset);
|
|
460
|
+
const line = ledgerLineFromRaw(parsed, source, lineOffset, compactions);
|
|
380
461
|
if (line)
|
|
381
462
|
counter.observeLine(line);
|
|
382
463
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@thehammer/danx-session-telemetry",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.49",
|
|
4
4
|
"description": "Shared Claude Code transcript collector (DX-2962) — incremental JSONL tailing for the danxbot worker and the danx-dashboard-mcp MCP server's relay, so both run identical collector code.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|