codexmeter 1.0.18 → 1.0.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/codexmeter.js +38 -0
- package/dist/assets/index-BBdnQ5aF.css +1 -0
- package/dist/assets/index-HV8wVWe1.js +34 -0
- package/dist/index.html +16 -16
- package/package.json +1 -1
- package/server/export-replay.js +2 -2
- package/server/ingest.js +183 -157
- package/server/live-state.js +19 -0
- package/server/rollout-reader.js +185 -26
- package/server/rollout-worker-pool.js +103 -4
- package/server/rollout-worker.js +2 -2
- package/src/utils/animationsDefault.js +4 -6
- package/dist/assets/index-Bbiu1QZp.js +0 -34
- package/dist/assets/index-CVvcsIkn.css +0 -1
package/dist/index.html
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
<!DOCTYPE html>
|
|
2
|
-
<html lang="en">
|
|
3
|
-
<head>
|
|
4
|
-
<meta charset="UTF-8" />
|
|
5
|
-
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
6
|
-
<title>CodexMeter</title>
|
|
7
|
-
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
|
8
|
-
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
|
9
|
-
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet" />
|
|
10
|
-
<script type="module" crossorigin src="/assets/index-
|
|
11
|
-
<link rel="stylesheet" crossorigin href="/assets/index-
|
|
12
|
-
</head>
|
|
13
|
-
<body>
|
|
14
|
-
<div id="root"></div>
|
|
15
|
-
</body>
|
|
16
|
-
</html>
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="UTF-8" />
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
6
|
+
<title>CodexMeter</title>
|
|
7
|
+
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
|
8
|
+
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
|
9
|
+
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet" />
|
|
10
|
+
<script type="module" crossorigin src="/assets/index-HV8wVWe1.js"></script>
|
|
11
|
+
<link rel="stylesheet" crossorigin href="/assets/index-BBdnQ5aF.css">
|
|
12
|
+
</head>
|
|
13
|
+
<body>
|
|
14
|
+
<div id="root"></div>
|
|
15
|
+
</body>
|
|
16
|
+
</html>
|
package/package.json
CHANGED
package/server/export-replay.js
CHANGED
|
@@ -36,7 +36,7 @@ export function beginReplayCapture(replay, ingestId, bootstrapPayload) {
|
|
|
36
36
|
|
|
37
37
|
export function recordReplayEvent(replay, event, payload) {
|
|
38
38
|
if (!replay?.active) return;
|
|
39
|
-
if (!['progress', 'patch', 'complete'].includes(event)) return;
|
|
39
|
+
if (!['progress', 'patch', 'snapshot', 'complete'].includes(event)) return;
|
|
40
40
|
replay.events.push({
|
|
41
41
|
event,
|
|
42
42
|
at_ms: Math.max(0, Date.now() - replay.started_at_ms),
|
|
@@ -73,7 +73,7 @@ export function getReplaySnapshot(replay) {
|
|
|
73
73
|
events: rest.map((event) => ({
|
|
74
74
|
event: event.event,
|
|
75
75
|
at_ms: event.at_ms,
|
|
76
|
-
mode: event.event === 'patch'
|
|
76
|
+
mode: event.event === 'patch' || event.event === 'snapshot' ? event.event : 'progress',
|
|
77
77
|
payload: clonePayload(event.payload),
|
|
78
78
|
})),
|
|
79
79
|
};
|
package/server/ingest.js
CHANGED
|
@@ -7,7 +7,7 @@ import {
|
|
|
7
7
|
import { calculateCostFromUsage, initPricing, priceSession } from './cost-catalog.js';
|
|
8
8
|
import { buildAggregates, buildSessionView } from './aggregator.js';
|
|
9
9
|
import { createDayKeyFormatter } from './day-key.js';
|
|
10
|
-
import { createLiveAggregateState, createEmptyLivePatch, applySessionToLiveState,
|
|
10
|
+
import { createLiveAggregateState, createEmptyLivePatch, applySessionToLiveState, buildLivePatch, buildLiveSnapshot } from './live-state.js';
|
|
11
11
|
import { createRolloutWorkerPool } from './rollout-worker-pool.js';
|
|
12
12
|
import { beginReplayCapture, createReplayCaptureState, failReplayCapture, getReplaySnapshot, recordReplayEvent, resetReplayCapture } from './export-replay.js';
|
|
13
13
|
import { findUsageEntryAtOrBefore, hasUsageTotals, readUsageTimeline, subtractUsageTotals } from './rollout-reader.js';
|
|
@@ -17,17 +17,11 @@ const LIVE_FRAME_INTERVAL_MS = Math.max(
|
|
|
17
17
|
16,
|
|
18
18
|
Math.round(Number(OVERVIEW_INGEST_ANIMATION.live?.frameIntervalMs) || 50)
|
|
19
19
|
);
|
|
20
|
-
const
|
|
21
|
-
const LIVE_OVERVIEW_HZ = Math.max(
|
|
20
|
+
const LIVE_SNAPSHOT_HZ = Math.max(
|
|
22
21
|
1,
|
|
23
|
-
Number(OVERVIEW_INGEST_ANIMATION.live?.
|
|
22
|
+
Number(OVERVIEW_INGEST_ANIMATION.live?.snapshotHz) || 10
|
|
24
23
|
);
|
|
25
|
-
const
|
|
26
|
-
const LIVE_DAY_KEYS_PER_EMIT = Math.max(
|
|
27
|
-
1,
|
|
28
|
-
Math.round(Number(OVERVIEW_INGEST_ANIMATION.live?.dayKeysPerEmit) || 1)
|
|
29
|
-
);
|
|
30
|
-
const LIVE_SESSION_REORDER_BUFFER = 160;
|
|
24
|
+
const LIVE_SNAPSHOT_CADENCE_MS = Math.round(1000 / LIVE_SNAPSHOT_HZ);
|
|
31
25
|
|
|
32
26
|
export function createIngestState() {
|
|
33
27
|
return {
|
|
@@ -51,20 +45,33 @@ export function createIngestState() {
|
|
|
51
45
|
live_subscribers: new Set(),
|
|
52
46
|
live_pump_timer: null,
|
|
53
47
|
live_pending_patch: createEmptyLivePatch(),
|
|
54
|
-
live_session_buffer: [],
|
|
55
48
|
live_progress_dirty: false,
|
|
56
49
|
live_last_emit_at: 0,
|
|
57
|
-
|
|
50
|
+
live_last_snapshot_emit_at: 0,
|
|
58
51
|
replay_capture: createReplayCaptureState(),
|
|
59
52
|
};
|
|
60
53
|
}
|
|
61
54
|
|
|
62
55
|
export async function runIngest(codexHome, state, opts = {}) {
|
|
56
|
+
const timingEnabled = opts.ingestTiming === true;
|
|
57
|
+
const timingStartedAt = performance.now();
|
|
58
|
+
let timingLastLogAt = timingStartedAt;
|
|
63
59
|
const tz = opts.timezone || Intl.DateTimeFormat().resolvedOptions().timeZone;
|
|
64
60
|
const toDayKey = createDayKeyFormatter(tz);
|
|
65
61
|
const runToken = state.run_token;
|
|
66
62
|
const isCurrentRun = () => state.run_token === runToken;
|
|
67
|
-
const
|
|
63
|
+
const fastRolloutReader = opts.fastRolloutReader === true;
|
|
64
|
+
const rgRolloutReader = opts.rgRolloutReader === true;
|
|
65
|
+
const streamRolloutChunks = opts.streamRolloutChunks === true;
|
|
66
|
+
const readerOptions = {
|
|
67
|
+
fastScan: fastRolloutReader || rgRolloutReader,
|
|
68
|
+
rgScan: rgRolloutReader,
|
|
69
|
+
rgMinBytes: opts.rgMinBytes,
|
|
70
|
+
};
|
|
71
|
+
const workerPool = createRolloutWorkerPool({
|
|
72
|
+
size: opts.workerThreads,
|
|
73
|
+
readerOptions,
|
|
74
|
+
});
|
|
68
75
|
|
|
69
76
|
try {
|
|
70
77
|
state.live_state = createLiveAggregateState(tz);
|
|
@@ -74,7 +81,7 @@ export async function runIngest(codexHome, state, opts = {}) {
|
|
|
74
81
|
ingest_id: state.ingest_id,
|
|
75
82
|
seq: 0,
|
|
76
83
|
progress: progressPayload(state),
|
|
77
|
-
data:
|
|
84
|
+
data: buildLiveSnapshot(state.live_state),
|
|
78
85
|
});
|
|
79
86
|
queueLiveProgress(state);
|
|
80
87
|
broadcastBootstrap(state);
|
|
@@ -140,7 +147,7 @@ export async function runIngest(codexHome, state, opts = {}) {
|
|
|
140
147
|
const bootstrapCandidates = sessions.filter((session) => !session.rollout_path);
|
|
141
148
|
if (bootstrapCandidates.length) {
|
|
142
149
|
assignRootThreadIds(sessions);
|
|
143
|
-
const bootstrapSessions =
|
|
150
|
+
const bootstrapSessions = filterLiveSessions(bootstrapCandidates, opts);
|
|
144
151
|
const bootstrapPatch = createEmptyLivePatch();
|
|
145
152
|
for (const session of bootstrapSessions) {
|
|
146
153
|
finalizeSessionMetrics(session, toDayKey);
|
|
@@ -149,24 +156,38 @@ export async function runIngest(codexHome, state, opts = {}) {
|
|
|
149
156
|
queueLivePatch(state, bootstrapPatch);
|
|
150
157
|
}
|
|
151
158
|
|
|
152
|
-
const candidates = selectEnrichmentCandidates(sessions
|
|
159
|
+
const candidates = selectEnrichmentCandidates(sessions, {
|
|
160
|
+
recentFirstDays: opts.recentFirstDays,
|
|
161
|
+
warmupOldestCount: opts.warmupOldestCount,
|
|
162
|
+
});
|
|
153
163
|
|
|
154
164
|
state.needs_enrichment = candidates.length;
|
|
155
165
|
state.percent = candidates.length > 0 ? 0.08 : 0.90;
|
|
156
166
|
const BATCH_SIZE = opts.batchSize || 40;
|
|
167
|
+
const RESULT_CHUNK_SIZE = Math.max(1, Number(opts.resultChunkSize || Math.min(BATCH_SIZE, 16)) || 16);
|
|
157
168
|
const ROOT_REFRESH_EVERY = opts.rootRefreshEvery || 1000;
|
|
158
|
-
const
|
|
169
|
+
const FORK_CORRECTION_CONCURRENCY = Math.max(1, Number(opts.forkCorrectionConcurrency || 1) || 1);
|
|
170
|
+
logIngestTiming(timingEnabled, 'start', {
|
|
171
|
+
rollouts: candidates.length,
|
|
172
|
+
workers: workerPool.size,
|
|
173
|
+
rg: rgRolloutReader,
|
|
174
|
+
stream: streamRolloutChunks,
|
|
175
|
+
forkConcurrency: FORK_CORRECTION_CONCURRENCY,
|
|
176
|
+
});
|
|
177
|
+
const resolveForkUsageSnapshot = createForkUsageSnapshotResolver(sessions, {
|
|
178
|
+
fastScan: fastRolloutReader,
|
|
179
|
+
rgScan: rgRolloutReader,
|
|
180
|
+
rgMinBytes: opts.rgMinBytes,
|
|
181
|
+
});
|
|
159
182
|
let lastRootRefreshCount = 0;
|
|
183
|
+
let enrichedCount = 0;
|
|
160
184
|
|
|
161
|
-
|
|
162
|
-
const batch = candidates.slice(i, i + BATCH_SIZE);
|
|
163
|
-
|
|
185
|
+
const handleEnrichedBatch = async (batch, results) => {
|
|
164
186
|
if (batch[0]?.started_at) {
|
|
165
187
|
const d = new Date(batch[0].started_at * 1000);
|
|
166
188
|
state.current_date_bucket = d.toLocaleDateString('en-CA', { timeZone: tz });
|
|
167
189
|
}
|
|
168
190
|
|
|
169
|
-
const results = await workerPool.mapRollouts(batch.map((session) => session.rollout_path), tz);
|
|
170
191
|
if (!isCurrentRun()) return;
|
|
171
192
|
|
|
172
193
|
const workerError = results.find((result) => result?.ok === false);
|
|
@@ -202,7 +223,7 @@ export async function runIngest(codexHome, state, opts = {}) {
|
|
|
202
223
|
}
|
|
203
224
|
}
|
|
204
225
|
|
|
205
|
-
await applyForkUsageCorrections(batch, resolveForkUsageSnapshot);
|
|
226
|
+
await applyForkUsageCorrections(batch, resolveForkUsageSnapshot, FORK_CORRECTION_CONCURRENCY);
|
|
206
227
|
for (const s of batch) {
|
|
207
228
|
if (s._usage_by_day_raw) {
|
|
208
229
|
s.usage_by_day = buildUsageByDayMetrics(s.model_name, s._usage_by_day_raw);
|
|
@@ -216,32 +237,65 @@ export async function runIngest(codexHome, state, opts = {}) {
|
|
|
216
237
|
s.materialized = true;
|
|
217
238
|
}
|
|
218
239
|
|
|
240
|
+
enrichedCount = Math.min(enrichedCount + batch.length, candidates.length);
|
|
219
241
|
const shouldRefreshRoots =
|
|
220
|
-
|
|
221
|
-
(
|
|
242
|
+
enrichedCount === batch.length ||
|
|
243
|
+
(enrichedCount - lastRootRefreshCount) >= ROOT_REFRESH_EVERY;
|
|
222
244
|
|
|
223
245
|
if (shouldRefreshRoots) {
|
|
224
246
|
assignRootThreadIds(sessions);
|
|
225
|
-
lastRootRefreshCount =
|
|
247
|
+
lastRootRefreshCount = enrichedCount;
|
|
226
248
|
}
|
|
227
|
-
const liveReady =
|
|
228
|
-
state.current_date_bucket = pickVisibleDateBucket(
|
|
229
|
-
filterVisibleSessions(state.live_session_buffer),
|
|
230
|
-
filterVisibleSessions(liveReady)
|
|
231
|
-
);
|
|
249
|
+
const liveReady = filterLiveSessions(batch, opts);
|
|
250
|
+
state.current_date_bucket = pickVisibleDateBucket([], liveReady);
|
|
232
251
|
const livePatch = createEmptyLivePatch();
|
|
233
|
-
for (const session of
|
|
252
|
+
for (const session of liveReady) {
|
|
234
253
|
applySessionToLiveState(state.live_state, session, livePatch);
|
|
235
254
|
}
|
|
236
255
|
queueLivePatch(state, livePatch);
|
|
237
256
|
|
|
238
|
-
state.enriched =
|
|
257
|
+
state.enriched = enrichedCount;
|
|
239
258
|
state.percent = candidates.length > 0
|
|
240
259
|
? 0.08 + (state.enriched / candidates.length) * 0.82
|
|
241
260
|
: 0.90;
|
|
242
261
|
queueLiveProgress(state);
|
|
243
262
|
|
|
263
|
+
const now = performance.now();
|
|
264
|
+
if (timingEnabled && now - timingLastLogAt >= 5000) {
|
|
265
|
+
timingLastLogAt = now;
|
|
266
|
+
const elapsedSeconds = Math.max((now - timingStartedAt) / 1000, 0.001);
|
|
267
|
+
logIngestTiming(true, 'progress', {
|
|
268
|
+
enriched: state.enriched,
|
|
269
|
+
rollouts: candidates.length,
|
|
270
|
+
rate: Math.round(state.enriched / elapsedSeconds),
|
|
271
|
+
currentDate: state.current_date_bucket,
|
|
272
|
+
});
|
|
273
|
+
}
|
|
274
|
+
};
|
|
275
|
+
|
|
276
|
+
if (streamRolloutChunks) {
|
|
277
|
+
await workerPool.mapRolloutsInChunks(
|
|
278
|
+
candidates.map((session) => session.rollout_path),
|
|
279
|
+
tz,
|
|
280
|
+
{
|
|
281
|
+
chunkSize: RESULT_CHUNK_SIZE,
|
|
282
|
+
onChunk: async (chunk) => {
|
|
283
|
+
const orderedChunk = chunk.slice().sort((a, b) => a.index - b.index);
|
|
284
|
+
const batch = orderedChunk.map(({ index }) => candidates[index]);
|
|
285
|
+
const results = orderedChunk.map(({ result }) => result);
|
|
286
|
+
await handleEnrichedBatch(batch, results);
|
|
287
|
+
},
|
|
288
|
+
}
|
|
289
|
+
);
|
|
244
290
|
if (!isCurrentRun()) return;
|
|
291
|
+
} else {
|
|
292
|
+
for (let i = 0; i < candidates.length; i += BATCH_SIZE) {
|
|
293
|
+
const batch = candidates.slice(i, i + BATCH_SIZE);
|
|
294
|
+
const results = await workerPool.mapRollouts(batch.map((session) => session.rollout_path), tz);
|
|
295
|
+
await handleEnrichedBatch(batch, results);
|
|
296
|
+
|
|
297
|
+
if (!isCurrentRun()) return;
|
|
298
|
+
}
|
|
245
299
|
}
|
|
246
300
|
|
|
247
301
|
for (const s of sessions) {
|
|
@@ -249,8 +303,9 @@ export async function runIngest(codexHome, state, opts = {}) {
|
|
|
249
303
|
}
|
|
250
304
|
|
|
251
305
|
assignRootThreadIds(sessions);
|
|
306
|
+
state.live_state = createLiveAggregateState(tz);
|
|
252
307
|
const finalLivePatch = createEmptyLivePatch();
|
|
253
|
-
for (const session of
|
|
308
|
+
for (const session of filterLiveSessions(sessions, opts)) {
|
|
254
309
|
applySessionToLiveState(state.live_state, session, finalLivePatch);
|
|
255
310
|
}
|
|
256
311
|
queueLivePatch(state, finalLivePatch);
|
|
@@ -268,6 +323,11 @@ export async function runIngest(codexHome, state, opts = {}) {
|
|
|
268
323
|
state.current_date_bucket = null;
|
|
269
324
|
queueLiveProgress(state);
|
|
270
325
|
finalizeWithoutSubscribers(state);
|
|
326
|
+
logIngestTiming(timingEnabled, 'finalizing', {
|
|
327
|
+
elapsedSeconds: Number(((performance.now() - timingStartedAt) / 1000).toFixed(1)),
|
|
328
|
+
rollouts: candidates.length,
|
|
329
|
+
events: state.replay_capture.events.length,
|
|
330
|
+
});
|
|
271
331
|
|
|
272
332
|
} catch (err) {
|
|
273
333
|
if (!isCurrentRun()) return;
|
|
@@ -305,10 +365,9 @@ export function restartIngest(codexHome, state, opts = {}) {
|
|
|
305
365
|
state.live_state = null;
|
|
306
366
|
state.live_seq = 0;
|
|
307
367
|
state.live_pending_patch = createEmptyLivePatch();
|
|
308
|
-
state.live_session_buffer = [];
|
|
309
368
|
state.live_progress_dirty = false;
|
|
310
369
|
state.live_last_emit_at = 0;
|
|
311
|
-
state.
|
|
370
|
+
state.live_last_snapshot_emit_at = 0;
|
|
312
371
|
resetReplayCapture(state.replay_capture);
|
|
313
372
|
|
|
314
373
|
return runIngest(codexHome, state, opts);
|
|
@@ -328,10 +387,31 @@ export function filterVisibleSessions(sessions) {
|
|
|
328
387
|
return sessions.filter((session) => !isReviewLauncherSession(session));
|
|
329
388
|
}
|
|
330
389
|
|
|
331
|
-
|
|
332
|
-
return sessions
|
|
390
|
+
function filterLiveSessions(sessions, opts) {
|
|
391
|
+
return applyFilters(filterVisibleSessions(sessions), opts);
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
export function selectEnrichmentCandidates(sessions, opts = {}) {
|
|
395
|
+
const sorted = sessions
|
|
333
396
|
.filter((session) => session.rollout_path)
|
|
334
397
|
.sort((a, b) => (a.started_at || 0) - (b.started_at || 0));
|
|
398
|
+
const recentFirstDays = Number(opts.recentFirstDays || 0);
|
|
399
|
+
const warmupOldestCount = Math.max(0, Number(opts.warmupOldestCount || 0) || 0);
|
|
400
|
+
if (!Number.isFinite(recentFirstDays) || recentFirstDays <= 0) return sorted;
|
|
401
|
+
|
|
402
|
+
const warmup = sorted.slice(0, warmupOldestCount);
|
|
403
|
+
const remaining = sorted.slice(warmupOldestCount);
|
|
404
|
+
const cutoff = (Date.now() / 1000) - recentFirstDays * 86400;
|
|
405
|
+
const recent = [];
|
|
406
|
+
const older = [];
|
|
407
|
+
for (const session of remaining) {
|
|
408
|
+
if ((session.ended_at || session.started_at || 0) >= cutoff) {
|
|
409
|
+
recent.push(session);
|
|
410
|
+
} else {
|
|
411
|
+
older.push(session);
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
return [...warmup, ...recent, ...older];
|
|
335
415
|
}
|
|
336
416
|
|
|
337
417
|
export function pickVisibleDateBucket(bufferedSessions, liveReadySessions) {
|
|
@@ -468,9 +548,14 @@ function buildUsageByDayMetrics(modelName, usageByDay) {
|
|
|
468
548
|
return entries;
|
|
469
549
|
}
|
|
470
550
|
|
|
471
|
-
function createForkUsageSnapshotResolver(sessions) {
|
|
551
|
+
function createForkUsageSnapshotResolver(sessions, opts = {}) {
|
|
472
552
|
const byId = new Map(sessions.map((session) => [session.thread_id, session]));
|
|
473
553
|
const timelineCache = new Map();
|
|
554
|
+
const timelineOptions = {
|
|
555
|
+
fastScan: opts.fastScan === true || opts.rgScan === true,
|
|
556
|
+
rgScan: opts.rgScan === true,
|
|
557
|
+
rgMinBytes: opts.rgMinBytes,
|
|
558
|
+
};
|
|
474
559
|
|
|
475
560
|
return async function resolveForkUsageSnapshot(parentThreadId, boundary) {
|
|
476
561
|
if (!parentThreadId || !boundary) return null;
|
|
@@ -479,7 +564,7 @@ function createForkUsageSnapshotResolver(sessions) {
|
|
|
479
564
|
|
|
480
565
|
let timelinePromise = timelineCache.get(parentThreadId);
|
|
481
566
|
if (!timelinePromise) {
|
|
482
|
-
timelinePromise = readUsageTimeline(parent.rollout_path);
|
|
567
|
+
timelinePromise = readUsageTimeline(parent.rollout_path, timelineOptions);
|
|
483
568
|
timelineCache.set(parentThreadId, timelinePromise);
|
|
484
569
|
}
|
|
485
570
|
|
|
@@ -501,29 +586,38 @@ export function selectForkParentUsageEntry(timeline, {
|
|
|
501
586
|
return firstUsageEntry || startEntry || null;
|
|
502
587
|
}
|
|
503
588
|
|
|
504
|
-
async function applyForkUsageCorrections(sessions, resolveForkUsageSnapshot) {
|
|
505
|
-
|
|
589
|
+
async function applyForkUsageCorrections(sessions, resolveForkUsageSnapshot, concurrency = 1) {
|
|
590
|
+
const targets = sessions.filter((session) => {
|
|
506
591
|
const parentThreadId = getForkLineageParentThreadId(session);
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
592
|
+
return parentThreadId && !session._usage_reset_detected;
|
|
593
|
+
});
|
|
594
|
+
if (!targets.length) return;
|
|
595
|
+
|
|
596
|
+
let index = 0;
|
|
597
|
+
const workerCount = Math.min(Math.max(1, Number(concurrency) || 1), targets.length);
|
|
598
|
+
await Promise.all(Array.from({ length: workerCount }, async () => {
|
|
599
|
+
while (index < targets.length) {
|
|
600
|
+
const session = targets[index++];
|
|
601
|
+
const parentThreadId = getForkLineageParentThreadId(session);
|
|
602
|
+
// Use the child's first observed token snapshot as the primary fork
|
|
603
|
+
// boundary. On the real fork-heavy March chains, this consistently
|
|
604
|
+
// produced better attribution than started_at: the delay from start to
|
|
605
|
+
// first token_count is usually sub-second, while started_at retains
|
|
606
|
+
// slightly more inherited parent usage. We still pass started_at through
|
|
607
|
+
// to the resolver so a parent reset between fork start and first child
|
|
608
|
+
// stream can fall back to the pre-reset parent segment instead of
|
|
609
|
+
// under-subtracting inherited usage. If the child rollout later resets its
|
|
610
|
+
// token counters, we treat the post-reset segment as the authoritative
|
|
611
|
+
// child-only usage and do not subtract the parent again, because that
|
|
612
|
+
// would double-normalize the same inherited baseline and undercount the
|
|
613
|
+
// child.
|
|
614
|
+
const inheritedUsage = await resolveForkUsageSnapshot(parentThreadId, {
|
|
615
|
+
startedAtMs: session.started_at ? session.started_at * 1000 : null,
|
|
616
|
+
firstUsageTimestampMs: session._first_usage_timestamp_ms || null,
|
|
617
|
+
});
|
|
618
|
+
applyForkUsageCorrection(session, inheritedUsage);
|
|
619
|
+
}
|
|
620
|
+
}));
|
|
527
621
|
}
|
|
528
622
|
|
|
529
623
|
export function getForkLineageParentThreadId(session) {
|
|
@@ -612,29 +706,6 @@ function deriveLiveSortDay(session, toDayKey) {
|
|
|
612
706
|
return null;
|
|
613
707
|
}
|
|
614
708
|
|
|
615
|
-
function compareLiveSessionOrder(a, b) {
|
|
616
|
-
const dayCompare = String(a.live_sort_day || '9999-12-31').localeCompare(String(b.live_sort_day || '9999-12-31'));
|
|
617
|
-
if (dayCompare !== 0) return dayCompare;
|
|
618
|
-
const liveTsCompare = (a.live_sort_ts || a.started_at || 0) - (b.live_sort_ts || b.started_at || 0);
|
|
619
|
-
if (liveTsCompare !== 0) return liveTsCompare;
|
|
620
|
-
return String(a.thread_id || '').localeCompare(String(b.thread_id || ''));
|
|
621
|
-
}
|
|
622
|
-
|
|
623
|
-
function bufferLiveSessionsForPresentation(state, sessions) {
|
|
624
|
-
if (!sessions.length) return [];
|
|
625
|
-
state.live_session_buffer.push(...sessions);
|
|
626
|
-
state.live_session_buffer.sort(compareLiveSessionOrder);
|
|
627
|
-
const flushCount = Math.max(0, state.live_session_buffer.length - LIVE_SESSION_REORDER_BUFFER);
|
|
628
|
-
if (flushCount <= 0) return [];
|
|
629
|
-
return state.live_session_buffer.splice(0, flushCount);
|
|
630
|
-
}
|
|
631
|
-
|
|
632
|
-
function flushBufferedLiveSessions(state) {
|
|
633
|
-
if (!state.live_session_buffer.length) return [];
|
|
634
|
-
state.live_session_buffer.sort(compareLiveSessionOrder);
|
|
635
|
-
return state.live_session_buffer.splice(0, state.live_session_buffer.length);
|
|
636
|
-
}
|
|
637
|
-
|
|
638
709
|
function queueLiveProgress(state) {
|
|
639
710
|
state.live_progress_dirty = true;
|
|
640
711
|
ensureLivePump(state);
|
|
@@ -688,11 +759,15 @@ function flushLive(state, forcedEvent = null) {
|
|
|
688
759
|
}
|
|
689
760
|
}
|
|
690
761
|
|
|
691
|
-
const
|
|
692
|
-
const
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
762
|
+
const patchPending = !isPatchEmpty(state.live_pending_patch);
|
|
763
|
+
const readyForSnapshot = forcedEvent || (patchPending && readyForSnapshotEmit(state, Date.now()));
|
|
764
|
+
if (!forcedEvent && !readyForSnapshot && !state.live_progress_dirty && !state.presentation_complete_pending) {
|
|
765
|
+
return;
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
const flushablePatch = readyForSnapshot ? takeAllPendingPatch(state) : createEmptyLivePatch();
|
|
769
|
+
const shouldEmitComplete = !forcedEvent && state.presentation_complete_pending;
|
|
770
|
+
const event = forcedEvent || (shouldEmitComplete ? 'complete' : (readyForSnapshot ? 'snapshot' : 'progress'));
|
|
696
771
|
|
|
697
772
|
if (shouldEmitComplete) {
|
|
698
773
|
state.phase = 'complete';
|
|
@@ -707,10 +782,10 @@ function flushLive(state, forcedEvent = null) {
|
|
|
707
782
|
progress: progressPayload(state),
|
|
708
783
|
};
|
|
709
784
|
|
|
710
|
-
if (event === '
|
|
785
|
+
if (event === 'snapshot' || event === 'complete') {
|
|
786
|
+
payload.data = buildLiveSnapshot(state.live_state);
|
|
787
|
+
} else if (event === 'patch') {
|
|
711
788
|
payload.data = buildLivePatch(state.live_state, flushablePatch);
|
|
712
|
-
} else if (event === 'bootstrap') {
|
|
713
|
-
payload.data = buildLiveBootstrap(state.live_state);
|
|
714
789
|
}
|
|
715
790
|
|
|
716
791
|
recordReplayEvent(state.replay_capture, event, payload);
|
|
@@ -718,8 +793,13 @@ function flushLive(state, forcedEvent = null) {
|
|
|
718
793
|
broadcastLive(state, event, payload);
|
|
719
794
|
}
|
|
720
795
|
state.live_last_emit_at = Date.now();
|
|
721
|
-
|
|
722
|
-
|
|
796
|
+
if (event === 'snapshot' || event === 'complete') {
|
|
797
|
+
state.live_last_snapshot_emit_at = state.live_last_emit_at;
|
|
798
|
+
}
|
|
799
|
+
state.live_progress_dirty = event === 'progress' || event === 'snapshot' || event === 'complete'
|
|
800
|
+
? false
|
|
801
|
+
: state.live_progress_dirty;
|
|
802
|
+
if (forcedEvent || event === 'complete') {
|
|
723
803
|
state.live_pending_patch = createEmptyLivePatch();
|
|
724
804
|
}
|
|
725
805
|
stopLivePumpIfIdle(state);
|
|
@@ -760,52 +840,8 @@ function isPatchEmpty(patch) {
|
|
|
760
840
|
patch.heatmap.size === 0;
|
|
761
841
|
}
|
|
762
842
|
|
|
763
|
-
function
|
|
764
|
-
|
|
765
|
-
const sent = createEmptyLivePatch();
|
|
766
|
-
|
|
767
|
-
const overviewDirty =
|
|
768
|
-
state.live_pending_patch.overview.size > 0 ||
|
|
769
|
-
state.live_pending_patch.repos.total.size > 0 || state.live_pending_patch.repos.d7.size > 0 || state.live_pending_patch.repos.d30.size > 0 ||
|
|
770
|
-
state.live_pending_patch.models.total.size > 0 || state.live_pending_patch.models.d7.size > 0 || state.live_pending_patch.models.d30.size > 0 ||
|
|
771
|
-
state.live_pending_patch.families.total.size > 0 || state.live_pending_patch.families.d7.size > 0 || state.live_pending_patch.families.d30.size > 0 ||
|
|
772
|
-
state.live_pending_patch.daily.size > 0 || state.live_pending_patch.heatmap.size > 0;
|
|
773
|
-
|
|
774
|
-
if (!overviewDirty || !readyForOverview(state, now)) {
|
|
775
|
-
return sent;
|
|
776
|
-
}
|
|
777
|
-
|
|
778
|
-
moveSet(state.live_pending_patch.overview, sent.overview);
|
|
779
|
-
moveRangeSets(state.live_pending_patch.repos, sent.repos);
|
|
780
|
-
moveRangeSets(state.live_pending_patch.models, sent.models);
|
|
781
|
-
moveRangeSets(state.live_pending_patch.families, sent.families);
|
|
782
|
-
|
|
783
|
-
const nextDayKeys = takeNextChronologicalDayKeys(
|
|
784
|
-
state.live_pending_patch.daily,
|
|
785
|
-
state.live_pending_patch.heatmap,
|
|
786
|
-
LIVE_DAY_KEYS_PER_EMIT
|
|
787
|
-
);
|
|
788
|
-
moveSpecificKeys(state.live_pending_patch.daily, sent.daily, nextDayKeys);
|
|
789
|
-
moveSpecificKeys(state.live_pending_patch.heatmap, sent.heatmap, nextDayKeys);
|
|
790
|
-
state.live_last_overview_emit_at = now;
|
|
791
|
-
|
|
792
|
-
return sent;
|
|
793
|
-
}
|
|
794
|
-
|
|
795
|
-
function readyForOverview(state, now) {
|
|
796
|
-
return (now - state.live_last_overview_emit_at) >= getOverviewCadenceMs(state);
|
|
797
|
-
}
|
|
798
|
-
|
|
799
|
-
function getOverviewCadenceMs(state) {
|
|
800
|
-
const tail = OVERVIEW_INGEST_ANIMATION.tail;
|
|
801
|
-
const tailStartPercent = Math.min(Math.max(tail?.startPercent ?? 0.95, 0), 0.999);
|
|
802
|
-
const tailHz = Number.isFinite(tail?.overviewHz) && tail.overviewHz > 0 ? tail.overviewHz : 5;
|
|
803
|
-
const tailCadenceMs = Math.round(1000 / tailHz);
|
|
804
|
-
|
|
805
|
-
if (tail?.enabled && (state.presentation_complete_pending || (state.percent || 0) >= tailStartPercent)) {
|
|
806
|
-
return tailCadenceMs;
|
|
807
|
-
}
|
|
808
|
-
return LIVE_OVERVIEW_CADENCE_MS;
|
|
843
|
+
function readyForSnapshotEmit(state, now) {
|
|
844
|
+
return (now - state.live_last_snapshot_emit_at) >= LIVE_SNAPSHOT_CADENCE_MS;
|
|
809
845
|
}
|
|
810
846
|
|
|
811
847
|
function moveSet(from, to) {
|
|
@@ -830,21 +866,6 @@ function takeAllPendingPatch(state) {
|
|
|
830
866
|
return sent;
|
|
831
867
|
}
|
|
832
868
|
|
|
833
|
-
function moveSpecificKeys(from, to, keys) {
|
|
834
|
-
for (const key of keys) {
|
|
835
|
-
if (!from.has(key)) continue;
|
|
836
|
-
from.delete(key);
|
|
837
|
-
to.add(key);
|
|
838
|
-
}
|
|
839
|
-
}
|
|
840
|
-
|
|
841
|
-
function takeNextChronologicalDayKeys(dailySet, heatmapSet, limit) {
|
|
842
|
-
const allKeys = new Set([...dailySet, ...heatmapSet]);
|
|
843
|
-
return [...allKeys]
|
|
844
|
-
.sort((a, b) => String(a).localeCompare(String(b)))
|
|
845
|
-
.slice(0, limit);
|
|
846
|
-
}
|
|
847
|
-
|
|
848
869
|
function progressPayload(state) {
|
|
849
870
|
return {
|
|
850
871
|
phase: state.phase,
|
|
@@ -860,6 +881,11 @@ function progressPayload(state) {
|
|
|
860
881
|
};
|
|
861
882
|
}
|
|
862
883
|
|
|
884
|
+
function logIngestTiming(enabled, event, payload) {
|
|
885
|
+
if (!enabled) return;
|
|
886
|
+
console.log(`codexmeter ingest timing ${event}: ${JSON.stringify(payload)}`);
|
|
887
|
+
}
|
|
888
|
+
|
|
863
889
|
export function attachLiveSubscriber(state, res) {
|
|
864
890
|
state.live_subscribers.add(res);
|
|
865
891
|
ensureLivePump(state);
|
|
@@ -867,7 +893,7 @@ export function attachLiveSubscriber(state, res) {
|
|
|
867
893
|
ingest_id: state.ingest_id,
|
|
868
894
|
seq: ++state.live_seq,
|
|
869
895
|
progress: progressPayload(state),
|
|
870
|
-
data:
|
|
896
|
+
data: buildLiveSnapshot(state.live_state || createLiveAggregateState(Intl.DateTimeFormat().resolvedOptions().timeZone)),
|
|
871
897
|
};
|
|
872
898
|
res.write(`event: bootstrap\ndata: ${JSON.stringify(payload)}\n\n`);
|
|
873
899
|
}
|
|
@@ -887,7 +913,7 @@ function broadcastBootstrap(state) {
|
|
|
887
913
|
ingest_id: state.ingest_id,
|
|
888
914
|
seq: ++state.live_seq,
|
|
889
915
|
progress: progressPayload(state),
|
|
890
|
-
data:
|
|
916
|
+
data: buildLiveSnapshot(state.live_state || createLiveAggregateState(Intl.DateTimeFormat().resolvedOptions().timeZone)),
|
|
891
917
|
};
|
|
892
918
|
broadcastLive(state, 'bootstrap', payload);
|
|
893
919
|
}
|
package/server/live-state.js
CHANGED
|
@@ -79,6 +79,17 @@ export function buildLiveBootstrap(live) {
|
|
|
79
79
|
};
|
|
80
80
|
}
|
|
81
81
|
|
|
82
|
+
export function buildLiveSnapshot(live) {
|
|
83
|
+
return {
|
|
84
|
+
overview: serializeOverview(live),
|
|
85
|
+
repos: serializeTopRanges(live.repos, live.repoTopKeys, serializeRepoSummary),
|
|
86
|
+
models: serializeTopRanges(live.models, live.modelTopKeys, serializeModelSummary),
|
|
87
|
+
families: serializeTopRanges(live.families, live.familyTopKeys, serializeFamilySummary),
|
|
88
|
+
daily: serializeDailySnapshot(live.daily),
|
|
89
|
+
heatmap: serializeHeatmap(live.heatmap),
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
82
93
|
export function buildLivePatch(live, patch) {
|
|
83
94
|
return {
|
|
84
95
|
overview: Object.fromEntries(
|
|
@@ -464,6 +475,10 @@ function serializeDaily(dayMap) {
|
|
|
464
475
|
return Object.fromEntries([...dayMap.entries()].map(([dayKey, value]) => [dayKey, serializeDailyEntry(value)]));
|
|
465
476
|
}
|
|
466
477
|
|
|
478
|
+
function serializeDailySnapshot(dayMap) {
|
|
479
|
+
return Object.fromEntries([...dayMap.entries()].map(([dayKey, value]) => [dayKey, serializeDailySnapshotEntry(value)]));
|
|
480
|
+
}
|
|
481
|
+
|
|
467
482
|
function serializeDailyEntry(value) {
|
|
468
483
|
return {
|
|
469
484
|
tokens: Math.round(value?.tokens || 0),
|
|
@@ -477,6 +492,10 @@ function serializeDailyEntry(value) {
|
|
|
477
492
|
};
|
|
478
493
|
}
|
|
479
494
|
|
|
495
|
+
function serializeDailySnapshotEntry(value) {
|
|
496
|
+
return serializeDailyEntry(value);
|
|
497
|
+
}
|
|
498
|
+
|
|
480
499
|
function serializeHeatmap(dayMap) {
|
|
481
500
|
return Object.fromEntries([...dayMap.entries()].map(([dayKey, value]) => [dayKey, serializeHeatmapEntry(value)]));
|
|
482
501
|
}
|