@shapeshift-labs/frontier-loom-ui 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client.js +319 -40
- package/dist/client.js.map +1 -1
- package/dist/public/styles.css +272 -12
- package/dist/server.d.ts +5 -2
- package/dist/server.js +2113 -93
- package/dist/server.js.map +1 -1
- package/package.json +2 -2
package/dist/server.js
CHANGED
|
@@ -4,7 +4,7 @@ import fs from 'node:fs/promises';
|
|
|
4
4
|
import http from 'node:http';
|
|
5
5
|
import path from 'node:path';
|
|
6
6
|
import { fileURLToPath } from 'node:url';
|
|
7
|
-
import { readCodexDashboardSnapshot } from '@shapeshift-labs/frontier-swarm-codex';
|
|
7
|
+
import { estimateCodexModelCost, readCodexDashboardSnapshot } from '@shapeshift-labs/frontier-swarm-codex';
|
|
8
8
|
const packageDir = path.dirname(fileURLToPath(import.meta.url));
|
|
9
9
|
const HEALTH_JSON_PARSE_MAX_BYTES = 16 * 1024 * 1024;
|
|
10
10
|
const TASK_DETAIL_PATCH_MAX_BYTES = 512 * 1024;
|
|
@@ -13,13 +13,20 @@ const ARTIFACT_VIEW_MAX_BYTES = 768 * 1024;
|
|
|
13
13
|
const ARTIFACT_DIRECTORY_MAX_ENTRIES = 200;
|
|
14
14
|
const HUMAN_ACTION_ANSWER_MAX_BYTES = 16 * 1024;
|
|
15
15
|
const CODEX_EVENTS_USAGE_MAX_BYTES = 8 * 1024 * 1024;
|
|
16
|
-
const
|
|
16
|
+
const DASHBOARD_SNAPSHOT_CACHE_MS = 5000;
|
|
17
|
+
const LIFETIME_DASHBOARD_MAX_SOURCES = 24;
|
|
17
18
|
const LIFETIME_DASHBOARD_MAX_JOBS = 800;
|
|
18
19
|
const LIFETIME_DASHBOARD_SCAN_MAX_FILES = 600;
|
|
19
20
|
const LIFETIME_DASHBOARD_SCAN_MAX_DEPTH = 5;
|
|
21
|
+
const LIFETIME_DASHBOARD_MAX_AUTONOMOUS_DECISION_FILES = 400;
|
|
22
|
+
const LIFETIME_DASHBOARD_MAX_DRAIN_RUNS = 6;
|
|
23
|
+
const LIFETIME_DASHBOARD_MAX_ACTIVE_PID_RUNS = 32;
|
|
24
|
+
const LIFETIME_DASHBOARD_MAX_QUEUE_TASKS = 500;
|
|
25
|
+
const LIFETIME_DASHBOARD_SOURCE_TIMEOUT_MS = 2500;
|
|
20
26
|
const LIFETIME_DASHBOARD_RESET_FILE = '.loom-ui-reset.json';
|
|
21
27
|
const REVIEW_DECISIONS_FILE = '.loom-ui-review-decisions.json';
|
|
22
28
|
const dashboardStreamListeners = new Set();
|
|
29
|
+
let dashboardSnapshotCache;
|
|
23
30
|
export function createLoomUiServer(options = {}) {
|
|
24
31
|
const normalized = normalizeServerOptions(options);
|
|
25
32
|
const server = http.createServer(async (request, response) => {
|
|
@@ -64,10 +71,10 @@ async function handleRequest(request, response, options) {
|
|
|
64
71
|
await streamDashboard(request, response, options);
|
|
65
72
|
}
|
|
66
73
|
else if (request.method === 'GET' && url.pathname === '/api/dashboard') {
|
|
67
|
-
writeJson(response, 200, await
|
|
74
|
+
writeJson(response, 200, await readDashboardSnapshotCached(options));
|
|
68
75
|
}
|
|
69
76
|
else if (request.method === 'GET' && url.pathname === '/api/task-details') {
|
|
70
|
-
writeJson(response, 200, await readTaskDetails(options, textValue(url.searchParams.get('id'), '')));
|
|
77
|
+
writeJson(response, 200, await readTaskDetails(options, textValue(url.searchParams.get('id'), ''), textValue(url.searchParams.get('sourceRun'), '')));
|
|
71
78
|
}
|
|
72
79
|
else if (request.method === 'GET' && url.pathname === '/api/artifact') {
|
|
73
80
|
writeJson(response, 200, await readArtifact(options, textValue(url.searchParams.get('path'), '')));
|
|
@@ -114,7 +121,7 @@ async function streamDashboard(request, response, options) {
|
|
|
114
121
|
return;
|
|
115
122
|
pending = true;
|
|
116
123
|
try {
|
|
117
|
-
const snapshot = await
|
|
124
|
+
const snapshot = await readDashboardSnapshotCached(options);
|
|
118
125
|
const signature = dashboardStreamSignature(snapshot);
|
|
119
126
|
const body = JSON.stringify(snapshot);
|
|
120
127
|
if (signature !== lastSignature) {
|
|
@@ -172,6 +179,16 @@ function debounce(fn, delayMs) {
|
|
|
172
179
|
}, delayMs);
|
|
173
180
|
};
|
|
174
181
|
}
|
|
182
|
+
function withTimeout(promise, timeoutMs, message) {
|
|
183
|
+
let timer;
|
|
184
|
+
const timeout = new Promise((_resolve, reject) => {
|
|
185
|
+
timer = setTimeout(() => reject(new Error(message)), timeoutMs);
|
|
186
|
+
});
|
|
187
|
+
return Promise.race([promise, timeout]).finally(() => {
|
|
188
|
+
if (timer)
|
|
189
|
+
clearTimeout(timer);
|
|
190
|
+
});
|
|
191
|
+
}
|
|
175
192
|
async function createDashboardWatchers(options, onChange) {
|
|
176
193
|
const roots = await dashboardWatchRoots(options);
|
|
177
194
|
const watchers = [];
|
|
@@ -189,7 +206,10 @@ async function createDashboardWatchers(options, onChange) {
|
|
|
189
206
|
}
|
|
190
207
|
function watchDirectory(root, recursive, onChange) {
|
|
191
208
|
try {
|
|
192
|
-
return watch(root, { recursive },
|
|
209
|
+
return watch(root, { recursive }, () => {
|
|
210
|
+
invalidateDashboardSnapshotCache();
|
|
211
|
+
onChange();
|
|
212
|
+
});
|
|
193
213
|
}
|
|
194
214
|
catch {
|
|
195
215
|
return undefined;
|
|
@@ -209,6 +229,9 @@ async function dashboardWatchRoots(options) {
|
|
|
209
229
|
const agentRuns = path.join(options.cwd, 'agent-runs');
|
|
210
230
|
if (await fileExists(agentRuns))
|
|
211
231
|
roots.push(agentRuns);
|
|
232
|
+
const loomQueues = path.join(options.cwd, '.loom', 'queues');
|
|
233
|
+
if (await fileExists(loomQueues))
|
|
234
|
+
roots.push(loomQueues);
|
|
212
235
|
}
|
|
213
236
|
return uniquePaths(roots);
|
|
214
237
|
}
|
|
@@ -247,10 +270,39 @@ async function readDashboardSnapshot(options) {
|
|
|
247
270
|
return readLifetimeDashboardSnapshot(options);
|
|
248
271
|
return readScopedDashboardSnapshot(options);
|
|
249
272
|
}
|
|
250
|
-
async function
|
|
273
|
+
async function readDashboardSnapshotCached(options) {
|
|
274
|
+
const key = JSON.stringify(dashboardInput(options));
|
|
275
|
+
const now = Date.now();
|
|
276
|
+
if (dashboardSnapshotCache?.key === key) {
|
|
277
|
+
if (dashboardSnapshotCache.value !== undefined && now - dashboardSnapshotCache.at < DASHBOARD_SNAPSHOT_CACHE_MS) {
|
|
278
|
+
return dashboardSnapshotCache.value;
|
|
279
|
+
}
|
|
280
|
+
if (dashboardSnapshotCache.pending)
|
|
281
|
+
return dashboardSnapshotCache.pending;
|
|
282
|
+
}
|
|
283
|
+
const pending = readDashboardSnapshot(options).then((value) => {
|
|
284
|
+
dashboardSnapshotCache = { key, at: Date.now(), value };
|
|
285
|
+
return value;
|
|
286
|
+
}, (error) => {
|
|
287
|
+
if (dashboardSnapshotCache?.key === key) {
|
|
288
|
+
dashboardSnapshotCache = dashboardSnapshotCache.value === undefined
|
|
289
|
+
? undefined
|
|
290
|
+
: { key, at: dashboardSnapshotCache.at, value: dashboardSnapshotCache.value };
|
|
291
|
+
}
|
|
292
|
+
throw error;
|
|
293
|
+
});
|
|
294
|
+
dashboardSnapshotCache = { key, at: now, value: dashboardSnapshotCache?.key === key ? dashboardSnapshotCache.value : undefined, pending };
|
|
295
|
+
return pending;
|
|
296
|
+
}
|
|
297
|
+
function invalidateDashboardSnapshotCache() {
|
|
298
|
+
dashboardSnapshotCache = undefined;
|
|
299
|
+
}
|
|
300
|
+
async function readScopedDashboardSnapshot(options, readOptions = {}) {
|
|
251
301
|
const snapshot = await readCodexDashboardSnapshot(dashboardInput(options));
|
|
252
|
-
const activeRunSnapshot = await readActiveRunSnapshot(options);
|
|
302
|
+
const activeRunSnapshot = readOptions.includeActiveRun === false ? undefined : await readActiveRunSnapshot(options);
|
|
253
303
|
const reviewDecisions = await readCoordinatorReviewDecisions(options.cwd);
|
|
304
|
+
const autonomousDecisions = await readAutonomousMergeDecisions(options.cwd);
|
|
305
|
+
const decisions = mergeReviewDecisionLists(reviewDecisions, autonomousDecisions);
|
|
254
306
|
if (!snapshot || typeof snapshot !== 'object' || Array.isArray(snapshot))
|
|
255
307
|
return activeRunSnapshot ?? snapshot;
|
|
256
308
|
const answers = await readHumanActionAnswers(options);
|
|
@@ -258,15 +310,18 @@ async function readScopedDashboardSnapshot(options) {
|
|
|
258
310
|
const jobs = Array.isArray(record.jobs) ? record.jobs : [];
|
|
259
311
|
const activeJobs = recordArray(activeRunSnapshot?.jobs);
|
|
260
312
|
if (shouldPreferActiveRunSnapshot(jobs, activeJobs)) {
|
|
313
|
+
const activeAgentRows = activeAgentsFromJobs(activeJobs);
|
|
261
314
|
return {
|
|
262
315
|
...activeRunSnapshot,
|
|
263
316
|
collectionJobs: jobs,
|
|
317
|
+
activeAgents: activeAgentRows,
|
|
264
318
|
humanActions: recordArray(record.humanActions),
|
|
265
319
|
humanActionAnswers: answers,
|
|
266
320
|
sources: {
|
|
267
321
|
...recordValue(record.sources),
|
|
268
322
|
...recordValue(activeRunSnapshot?.sources),
|
|
269
323
|
...(reviewDecisions.length ? { coordinatorReviewDecisions: coordinatorReviewDecisionPath(options.cwd) } : {}),
|
|
324
|
+
...(autonomousDecisions.length ? { autonomousMergeDecisions: autonomousDecisionSourceSummary(autonomousDecisions) } : {}),
|
|
270
325
|
...(answers.length ? { humanActionAnswers: await humanActionAnswerLogPath(options) } : {})
|
|
271
326
|
},
|
|
272
327
|
raw: {
|
|
@@ -279,14 +334,16 @@ async function readScopedDashboardSnapshot(options) {
|
|
|
279
334
|
}
|
|
280
335
|
};
|
|
281
336
|
}
|
|
282
|
-
const mergedJobs = applyCoordinatorReviewDecisions(mergeActiveRunJobTelemetry(jobs, activeJobs),
|
|
337
|
+
const mergedJobs = applyCoordinatorReviewDecisions(mergeActiveRunJobTelemetry(jobs, activeJobs), decisions).map(withRecomputedCostFields);
|
|
283
338
|
return {
|
|
284
339
|
...normalizeCoordinatorFacingSnapshot(record),
|
|
285
340
|
jobs: mergedJobs,
|
|
341
|
+
activeAgents: activeAgentsFromJobs(mergedJobs),
|
|
286
342
|
humanActionAnswers: answers,
|
|
287
343
|
sources: {
|
|
288
344
|
...recordValue(record.sources),
|
|
289
345
|
...(reviewDecisions.length ? { coordinatorReviewDecisions: coordinatorReviewDecisionPath(options.cwd) } : {}),
|
|
346
|
+
...(autonomousDecisions.length ? { autonomousMergeDecisions: autonomousDecisionSourceSummary(autonomousDecisions) } : {}),
|
|
290
347
|
...(answers.length ? { humanActionAnswers: await humanActionAnswerLogPath(options) } : {})
|
|
291
348
|
}
|
|
292
349
|
};
|
|
@@ -294,22 +351,39 @@ async function readScopedDashboardSnapshot(options) {
|
|
|
294
351
|
async function readLifetimeDashboardSnapshot(options) {
|
|
295
352
|
const sources = await discoverLifetimeDashboardSources(options.cwd);
|
|
296
353
|
const snapshots = [];
|
|
354
|
+
let skippedSourceCount = 0;
|
|
355
|
+
let timedOutSourceCount = 0;
|
|
297
356
|
for (const source of sources.slice(0, LIFETIME_DASHBOARD_MAX_SOURCES)) {
|
|
298
357
|
try {
|
|
299
|
-
const snapshot =
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
358
|
+
const snapshot = await withTimeout((async () => {
|
|
359
|
+
const scopedSnapshot = await readScopedDashboardSnapshot({
|
|
360
|
+
...options,
|
|
361
|
+
run: source.run,
|
|
362
|
+
collection: source.collection,
|
|
363
|
+
continuation: source.continuation
|
|
364
|
+
}, { includeActiveRun: false });
|
|
365
|
+
return enrichLifetimeRunSnapshotEvidence(options.cwd, source, recordValue(scopedSnapshot));
|
|
366
|
+
})(), LIFETIME_DASHBOARD_SOURCE_TIMEOUT_MS, `lifetime source timed out: ${source.path}`);
|
|
305
367
|
if (Object.keys(snapshot).length)
|
|
306
368
|
snapshots.push({ source, snapshot });
|
|
307
369
|
}
|
|
308
|
-
catch {
|
|
370
|
+
catch (error) {
|
|
371
|
+
skippedSourceCount++;
|
|
372
|
+
if (error instanceof Error && error.message.startsWith('lifetime source timed out:'))
|
|
373
|
+
timedOutSourceCount++;
|
|
309
374
|
continue;
|
|
310
375
|
}
|
|
311
376
|
}
|
|
312
|
-
|
|
377
|
+
const lifetime = await combineLifetimeDashboardSnapshots(options, sources, snapshots, mergeReviewDecisionLists(await readCoordinatorReviewDecisions(options.cwd), await readAutonomousMergeDecisions(options.cwd)), await readLifetimeQueueBacklog(options.cwd));
|
|
378
|
+
const lifetimeWithSourceHealth = {
|
|
379
|
+
...lifetime,
|
|
380
|
+
sources: {
|
|
381
|
+
...recordValue(lifetime.sources),
|
|
382
|
+
skippedSourceCount,
|
|
383
|
+
timedOutSourceCount
|
|
384
|
+
}
|
|
385
|
+
};
|
|
386
|
+
return mergeLifetimeActiveRunSnapshot(mergeLifetimeDrainCoordinatorSnapshot(lifetimeWithSourceHealth, await readLatestDrainCoordinatorSnapshot(options.cwd)), await readLifetimeActiveRunSnapshot(options));
|
|
313
387
|
}
|
|
314
388
|
async function discoverLifetimeDashboardSources(cwd) {
|
|
315
389
|
const root = path.join(cwd, 'agent-runs');
|
|
@@ -341,7 +415,7 @@ async function discoverLifetimeDashboardSources(cwd) {
|
|
|
341
415
|
continue;
|
|
342
416
|
const hasCollection = entry.files.has('collection.json') || entry.files.has('coordinator-query.json');
|
|
343
417
|
const hasContinuation = entry.files.has('continuation.json');
|
|
344
|
-
const hasRun = entry.files.has('swarm-results.json') || entry.files.has('
|
|
418
|
+
const hasRun = entry.files.has('swarm-results.json') || entry.files.has('coordinator-dashboard.json');
|
|
345
419
|
if (hasCollection) {
|
|
346
420
|
out.push({
|
|
347
421
|
id: `collection:${relative}`,
|
|
@@ -378,17 +452,54 @@ async function discoverLifetimeDashboardSources(cwd) {
|
|
|
378
452
|
return dedupeLifetimeDashboardSources(out).sort((left, right) => right.mtimeMs - left.mtimeMs || left.path.localeCompare(right.path));
|
|
379
453
|
}
|
|
380
454
|
function dedupeLifetimeDashboardSources(sources) {
|
|
381
|
-
const
|
|
455
|
+
const preferredCollections = preferredLifetimeCollectionsByFamily(sources);
|
|
382
456
|
const runs = new Set(sources.filter((source) => source.kind === 'run').map(lifetimeRunFamilyKey));
|
|
383
457
|
return sources.filter((source) => {
|
|
384
458
|
const family = lifetimeRunFamilyKey(source);
|
|
385
|
-
if (source.kind === '
|
|
459
|
+
if (source.kind === 'collection' && preferredCollections.get(family) !== source)
|
|
386
460
|
return false;
|
|
387
461
|
if (source.kind === 'collection' && source.path.endsWith('/collected-missing') && runs.has(family))
|
|
388
462
|
return false;
|
|
389
463
|
return true;
|
|
390
464
|
});
|
|
391
465
|
}
|
|
466
|
+
function preferredLifetimeCollectionsByFamily(sources) {
|
|
467
|
+
const out = new Map();
|
|
468
|
+
for (const source of sources) {
|
|
469
|
+
if (source.kind !== 'collection')
|
|
470
|
+
continue;
|
|
471
|
+
const family = lifetimeRunFamilyKey(source);
|
|
472
|
+
const current = out.get(family);
|
|
473
|
+
if (!current || compareLifetimeCollectionPreference(source, current) > 0)
|
|
474
|
+
out.set(family, source);
|
|
475
|
+
}
|
|
476
|
+
return out;
|
|
477
|
+
}
|
|
478
|
+
function compareLifetimeCollectionPreference(left, right) {
|
|
479
|
+
return lifetimeCollectionPreference(left) - lifetimeCollectionPreference(right)
|
|
480
|
+
|| left.mtimeMs - right.mtimeMs
|
|
481
|
+
|| right.path.localeCompare(left.path);
|
|
482
|
+
}
|
|
483
|
+
function lifetimeCollectionPreference(source) {
|
|
484
|
+
const pathLabel = normalized(source.path);
|
|
485
|
+
if (pathLabel.endsWith('/post-coordinator-collected') || pathLabel.includes('/post-coordinator-collected'))
|
|
486
|
+
return 80;
|
|
487
|
+
if (pathLabel.endsWith('/coordinator-collected') || pathLabel.includes('/coordinator-collected'))
|
|
488
|
+
return 70;
|
|
489
|
+
if (pathLabel.endsWith('/collected-resolved') || pathLabel.includes('/collected-resolved-'))
|
|
490
|
+
return 60;
|
|
491
|
+
if (pathLabel.endsWith('/collected-with-decisions') || pathLabel.includes('/collected-with-decisions-'))
|
|
492
|
+
return 55;
|
|
493
|
+
if (pathLabel.endsWith('/collected'))
|
|
494
|
+
return 50;
|
|
495
|
+
if (pathLabel.includes('/collected-current'))
|
|
496
|
+
return 30;
|
|
497
|
+
if (pathLabel.includes('/collected-partial'))
|
|
498
|
+
return 20;
|
|
499
|
+
if (pathLabel.includes('/collected-missing'))
|
|
500
|
+
return 10;
|
|
501
|
+
return 40;
|
|
502
|
+
}
|
|
392
503
|
function lifetimeRunFamilyKey(source) {
|
|
393
504
|
const parts = source.path.split(/[\\/]/g).filter(Boolean);
|
|
394
505
|
if (!parts.length)
|
|
@@ -397,6 +508,283 @@ function lifetimeRunFamilyKey(source) {
|
|
|
397
508
|
const start = agentRunsIndex >= 0 ? agentRunsIndex + 1 : 0;
|
|
398
509
|
return parts[start] ?? source.path;
|
|
399
510
|
}
|
|
511
|
+
function lifetimeRunRootKey(source) {
|
|
512
|
+
const parts = source.path.split(/[\\/]/g).filter(Boolean);
|
|
513
|
+
if (!parts.length)
|
|
514
|
+
return source.path;
|
|
515
|
+
const agentRunsIndex = parts.lastIndexOf('agent-runs');
|
|
516
|
+
const start = agentRunsIndex >= 0 ? agentRunsIndex + 1 : 0;
|
|
517
|
+
const rootParts = parts.slice(start, start + 2);
|
|
518
|
+
return rootParts.length ? rootParts.join('/') : source.path;
|
|
519
|
+
}
|
|
520
|
+
function isDrainedAutonomousRunSnapshot(snapshot) {
|
|
521
|
+
const rawRun = recordValue(recordValue(snapshot.raw).run);
|
|
522
|
+
const autoDrain = recordValue(rawRun.autoDrain);
|
|
523
|
+
const autoDrainSummary = recordValue(autoDrain.summary);
|
|
524
|
+
const artifactSummary = recordValue(recordValue(rawRun.autoDrainArtifacts).summary);
|
|
525
|
+
const summary = Object.keys(autoDrainSummary).length ? autoDrainSummary : artifactSummary;
|
|
526
|
+
if (!Object.keys(summary).length)
|
|
527
|
+
return false;
|
|
528
|
+
if (textValue(summary.rerunManifestTerminalState, '') !== 'drained')
|
|
529
|
+
return false;
|
|
530
|
+
if (numberValue(summary.remainingReadyCount) > 0)
|
|
531
|
+
return false;
|
|
532
|
+
if (numberValue(summary.humanBlockedCount) > 0 || numberValue(summary.humanBlockedDecisionCount) > 0)
|
|
533
|
+
return false;
|
|
534
|
+
if (numberValue(summary.conflictBlockedCount) > 0 || numberValue(summary.rerunTaskCount) > 0)
|
|
535
|
+
return false;
|
|
536
|
+
return numberValue(summary.committedDecisionCount) > 0 || numberValue(summary.terminalCount) > 0 || numberValue(summary.decisionCount) > 0;
|
|
537
|
+
}
|
|
538
|
+
function collapseSupersededLifetimeReviewJobs(jobs) {
|
|
539
|
+
const resolvedAtByJob = new Map();
|
|
540
|
+
for (const job of jobs) {
|
|
541
|
+
if (!isResolvedCoordinatorReviewRecord(job))
|
|
542
|
+
continue;
|
|
543
|
+
const key = lifetimeReviewDedupeKey(job);
|
|
544
|
+
if (!key)
|
|
545
|
+
continue;
|
|
546
|
+
resolvedAtByJob.set(key, Math.max(resolvedAtByJob.get(key) ?? 0, numberValue(job.generatedAt)));
|
|
547
|
+
}
|
|
548
|
+
return jobs.filter((job) => {
|
|
549
|
+
if (!isOpenCoordinatorReviewRecord(job))
|
|
550
|
+
return true;
|
|
551
|
+
const key = lifetimeReviewDedupeKey(job);
|
|
552
|
+
if (!key)
|
|
553
|
+
return true;
|
|
554
|
+
return (resolvedAtByJob.get(key) ?? 0) < numberValue(job.generatedAt);
|
|
555
|
+
});
|
|
556
|
+
}
|
|
557
|
+
function dedupeLifetimeDashboardJobs(jobs) {
|
|
558
|
+
const byTask = new Map();
|
|
559
|
+
for (const job of jobs) {
|
|
560
|
+
const key = lifetimeJobDedupeKey(job);
|
|
561
|
+
if (!key)
|
|
562
|
+
continue;
|
|
563
|
+
const current = byTask.get(key);
|
|
564
|
+
if (!current || compareLifetimeJobPreference(job, current) > 0)
|
|
565
|
+
byTask.set(key, job);
|
|
566
|
+
}
|
|
567
|
+
return Array.from(byTask.values()).sort((left, right) => {
|
|
568
|
+
return numberValue(right.generatedAt) - numberValue(left.generatedAt)
|
|
569
|
+
|| textValue(left.lane, '').localeCompare(textValue(right.lane, ''))
|
|
570
|
+
|| textValue(left.title, '').localeCompare(textValue(right.title, ''));
|
|
571
|
+
});
|
|
572
|
+
}
|
|
573
|
+
function lifetimeJobDedupeKey(job) {
|
|
574
|
+
return canonicalLifetimeTaskKey(textValue(job.originalJobId ?? job.taskId ?? job.id ?? job.jobId, ''));
|
|
575
|
+
}
|
|
576
|
+
function compareLifetimeJobPreference(left, right) {
|
|
577
|
+
return lifetimeJobPreference(left) - lifetimeJobPreference(right)
|
|
578
|
+
|| numberValue(left.generatedAt) - numberValue(right.generatedAt)
|
|
579
|
+
|| textValue(right.sourceLabel, '').localeCompare(textValue(left.sourceLabel, ''));
|
|
580
|
+
}
|
|
581
|
+
function lifetimeJobPreference(job) {
|
|
582
|
+
const status = normalized(job.status);
|
|
583
|
+
const bucket = normalized(job.bucket);
|
|
584
|
+
const liveness = normalized(job.liveness);
|
|
585
|
+
let score = 0;
|
|
586
|
+
if (status === 'running')
|
|
587
|
+
score += 120;
|
|
588
|
+
else if (status === 'completed')
|
|
589
|
+
score += 100;
|
|
590
|
+
else if (status === 'failed' || status === 'blocked')
|
|
591
|
+
score += 70;
|
|
592
|
+
else if (['queued', 'pending', 'todo', 'open'].includes(status) || ['queued', 'todo'].includes(bucket))
|
|
593
|
+
score += 40;
|
|
594
|
+
if (liveness === 'missing' || status === 'planned')
|
|
595
|
+
score -= 60;
|
|
596
|
+
if (numberValue(job.changedPathCount))
|
|
597
|
+
score += 12;
|
|
598
|
+
if (numberValue(job.evidencePathCount))
|
|
599
|
+
score += 8;
|
|
600
|
+
if (numberValue(job.actualInputTokens) || numberValue(job.estimatedInputTokens))
|
|
601
|
+
score += 4;
|
|
602
|
+
return score;
|
|
603
|
+
}
|
|
604
|
+
function lifetimeReviewDedupeKey(job) {
|
|
605
|
+
return canonicalLifetimeTaskKey(textValue(job.originalJobId ?? job.jobId ?? job.taskId, ''));
|
|
606
|
+
}
|
|
607
|
+
function canonicalLifetimeTaskKey(value) {
|
|
608
|
+
return unscopedLifetimeTaskKey(value)
|
|
609
|
+
.trim()
|
|
610
|
+
.replace(/(?:-continuation)?-rerun(?:-\d+)?$/u, '')
|
|
611
|
+
.replace(/(?:-continuation)?-retry(?:-\d+)?$/u, '');
|
|
612
|
+
}
|
|
613
|
+
function unscopedLifetimeTaskKey(value) {
|
|
614
|
+
const trimmed = value.trim();
|
|
615
|
+
const match = /^(?:run|collection|continuation):.+:([^:]+)$/u.exec(trimmed);
|
|
616
|
+
return match?.[1] ?? trimmed;
|
|
617
|
+
}
|
|
618
|
+
function isOpenCoordinatorReviewRecord(job) {
|
|
619
|
+
if (isResolvedCoordinatorReviewRecord(job))
|
|
620
|
+
return false;
|
|
621
|
+
return isCoordinatorPortBucket(job.bucket)
|
|
622
|
+
|| isCoordinatorPortBucket(job.disposition)
|
|
623
|
+
|| isCoordinatorPortBucket(job.mergeReadiness)
|
|
624
|
+
|| normalized(job.status) === 'needs-review';
|
|
625
|
+
}
|
|
626
|
+
function isResolvedCoordinatorReviewRecord(job) {
|
|
627
|
+
const bucket = normalized(job.bucket);
|
|
628
|
+
if (bucket === 'review-resolved' || bucket === 'resolved-review' || job.reviewResolved === true)
|
|
629
|
+
return true;
|
|
630
|
+
const status = textValue(job.coordinatorDecisionStatus ?? recordValue(job.coordinatorDecision).status, '');
|
|
631
|
+
return Boolean(status) && isResolvedCoordinatorDecision(status);
|
|
632
|
+
}
|
|
633
|
+
async function readLifetimeQueueBacklog(cwd) {
|
|
634
|
+
const root = path.join(cwd, '.loom', 'queues');
|
|
635
|
+
const stat = await fs.stat(root).catch(() => undefined);
|
|
636
|
+
if (!stat?.isDirectory())
|
|
637
|
+
return { entries: [], manifests: [], sourceCount: 0, paths: [], generatedAt: 0 };
|
|
638
|
+
const queueDirs = await fs.readdir(root, { withFileTypes: true }).catch(() => []);
|
|
639
|
+
const entriesById = new Map();
|
|
640
|
+
const manifests = [];
|
|
641
|
+
const paths = [];
|
|
642
|
+
let generatedAt = 0;
|
|
643
|
+
for (const queueDir of queueDirs) {
|
|
644
|
+
if (!queueDir.isDirectory())
|
|
645
|
+
continue;
|
|
646
|
+
const dir = path.join(root, queueDir.name);
|
|
647
|
+
const manifestFile = await preferredQueueManifestFile(dir);
|
|
648
|
+
if (manifestFile) {
|
|
649
|
+
const manifestStat = await fs.stat(manifestFile).catch(() => undefined);
|
|
650
|
+
generatedAt = Math.max(generatedAt, manifestStat?.mtimeMs ?? 0);
|
|
651
|
+
const manifest = await readQueueCapacityManifest(cwd, manifestFile);
|
|
652
|
+
if (manifest)
|
|
653
|
+
manifests.push(manifest);
|
|
654
|
+
}
|
|
655
|
+
for (const taskFile of await queueTaskFiles(dir)) {
|
|
656
|
+
const fileStat = await fs.stat(taskFile).catch(() => undefined);
|
|
657
|
+
generatedAt = Math.max(generatedAt, fileStat?.mtimeMs ?? 0);
|
|
658
|
+
paths.push(path.relative(cwd, taskFile));
|
|
659
|
+
const tasks = await readQueueTaskFile(taskFile);
|
|
660
|
+
for (const task of tasks) {
|
|
661
|
+
const id = textValue(task.id ?? task.taskId ?? task.title, '');
|
|
662
|
+
if (!id)
|
|
663
|
+
continue;
|
|
664
|
+
entriesById.set(id, normalizeQueueBacklogEntry(cwd, queueDir.name, taskFile, task));
|
|
665
|
+
if (entriesById.size >= LIFETIME_DASHBOARD_MAX_QUEUE_TASKS)
|
|
666
|
+
break;
|
|
667
|
+
}
|
|
668
|
+
if (entriesById.size >= LIFETIME_DASHBOARD_MAX_QUEUE_TASKS)
|
|
669
|
+
break;
|
|
670
|
+
}
|
|
671
|
+
if (entriesById.size >= LIFETIME_DASHBOARD_MAX_QUEUE_TASKS)
|
|
672
|
+
break;
|
|
673
|
+
}
|
|
674
|
+
return {
|
|
675
|
+
entries: Array.from(entriesById.values()),
|
|
676
|
+
manifests,
|
|
677
|
+
sourceCount: paths.length,
|
|
678
|
+
paths,
|
|
679
|
+
generatedAt
|
|
680
|
+
};
|
|
681
|
+
}
|
|
682
|
+
async function queueTaskFiles(dir) {
|
|
683
|
+
const entries = await fs.readdir(dir, { withFileTypes: true }).catch(() => []);
|
|
684
|
+
const candidates = [];
|
|
685
|
+
for (const entry of entries) {
|
|
686
|
+
if (!entry.isFile() || !/^tasks(?:\.(?:remaining|backlog)-[\w.-]+)?\.json$/.test(entry.name))
|
|
687
|
+
continue;
|
|
688
|
+
const file = path.join(dir, entry.name);
|
|
689
|
+
const stat = await fs.stat(file).catch(() => undefined);
|
|
690
|
+
candidates.push({
|
|
691
|
+
file,
|
|
692
|
+
mtimeMs: stat?.mtimeMs ?? 0,
|
|
693
|
+
preferred: entry.name.startsWith('tasks.backlog-') ? 2 : entry.name.startsWith('tasks.remaining-') ? 1 : 0
|
|
694
|
+
});
|
|
695
|
+
}
|
|
696
|
+
return candidates
|
|
697
|
+
.sort((left, right) => left.mtimeMs - right.mtimeMs || left.preferred - right.preferred || left.file.localeCompare(right.file))
|
|
698
|
+
.map((candidate) => candidate.file);
|
|
699
|
+
}
|
|
700
|
+
async function preferredQueueManifestFile(dir) {
|
|
701
|
+
const entries = await fs.readdir(dir, { withFileTypes: true }).catch(() => []);
|
|
702
|
+
const candidates = [];
|
|
703
|
+
for (const entry of entries) {
|
|
704
|
+
if (!entry.isFile() || !/^manifest(?:\.[\w.-]+)?\.json$/.test(entry.name))
|
|
705
|
+
continue;
|
|
706
|
+
const file = path.join(dir, entry.name);
|
|
707
|
+
const stat = await fs.stat(file).catch(() => undefined);
|
|
708
|
+
candidates.push({
|
|
709
|
+
file,
|
|
710
|
+
mtimeMs: stat?.mtimeMs ?? 0,
|
|
711
|
+
preferred: entry.name.includes('high-concurrency') ? 2 : entry.name === 'manifest.json' ? 1 : 0
|
|
712
|
+
});
|
|
713
|
+
}
|
|
714
|
+
return candidates.sort((left, right) => right.preferred - left.preferred || right.mtimeMs - left.mtimeMs || left.file.localeCompare(right.file))[0]?.file;
|
|
715
|
+
}
|
|
716
|
+
async function readQueueCapacityManifest(cwd, file) {
|
|
717
|
+
const raw = recordValue(await readJsonFile(file));
|
|
718
|
+
if (!Object.keys(raw).length)
|
|
719
|
+
return undefined;
|
|
720
|
+
const computeRows = recordArray(raw.compute);
|
|
721
|
+
const computeById = new Map(computeRows.map((entry) => [textValue(entry.id, ''), entry]));
|
|
722
|
+
const defaultComputeId = textValue(recordValue(raw.policy).defaultCompute, textValue(computeRows[0]?.id, ''));
|
|
723
|
+
const defaultCompute = recordValue(computeById.get(defaultComputeId) ?? computeRows[0]);
|
|
724
|
+
const defaultConcurrency = numberValue(recordValue(raw.policy).defaultConcurrency);
|
|
725
|
+
const computeMaxConcurrency = computeRows.reduce((max, entry) => Math.max(max, numberValue(entry.maxConcurrency)), 0);
|
|
726
|
+
const manifestMaxConcurrency = numberValue(raw.maxConcurrency);
|
|
727
|
+
const lanes = recordArray(raw.lanes).map((lane) => {
|
|
728
|
+
const computeId = textValue(lane.compute, defaultComputeId);
|
|
729
|
+
const compute = recordValue(computeById.get(computeId) ?? defaultCompute);
|
|
730
|
+
return {
|
|
731
|
+
id: textValue(lane.id, 'lane'),
|
|
732
|
+
title: textValue(lane.title ?? lane.id, 'Lane'),
|
|
733
|
+
layer: textValue(lane.layer, ''),
|
|
734
|
+
compute: computeId,
|
|
735
|
+
model: textValue(compute.model, textValue(compute.id, '')),
|
|
736
|
+
maxConcurrency: numberValue(lane.maxConcurrency) || 1
|
|
737
|
+
};
|
|
738
|
+
});
|
|
739
|
+
return {
|
|
740
|
+
path: path.relative(cwd, file),
|
|
741
|
+
id: textValue(raw.id, path.basename(file, '.json')),
|
|
742
|
+
title: textValue(raw.title, 'Swarm manifest'),
|
|
743
|
+
defaultConcurrency,
|
|
744
|
+
computeMaxConcurrency,
|
|
745
|
+
maxConcurrency: manifestMaxConcurrency || defaultConcurrency || computeMaxConcurrency || lanes.reduce((sum, lane) => sum + lane.maxConcurrency, 0),
|
|
746
|
+
lanes
|
|
747
|
+
};
|
|
748
|
+
}
|
|
749
|
+
async function readQueueTaskFile(file) {
|
|
750
|
+
const raw = await readJsonFile(file);
|
|
751
|
+
if (Array.isArray(raw))
|
|
752
|
+
return raw.map(recordValue).filter((entry) => Object.keys(entry).length);
|
|
753
|
+
const record = recordValue(raw);
|
|
754
|
+
return recordArray(record.tasks ?? record.entries ?? record.items);
|
|
755
|
+
}
|
|
756
|
+
function normalizeQueueBacklogEntry(cwd, queueId, file, task) {
|
|
757
|
+
const id = textValue(task.id ?? task.taskId ?? task.title, 'task');
|
|
758
|
+
const queueStatus = textValue(task.status ?? task.state, 'open');
|
|
759
|
+
const status = ['done', 'completed', 'failed', 'blocked'].includes(normalized(queueStatus)) ? queueStatus : 'todo';
|
|
760
|
+
const sourceRefs = stringArray(task.sourceRefs);
|
|
761
|
+
const targetRefs = stringArray(task.targetRefs);
|
|
762
|
+
const allowedWrites = stringArray(task.allowedWrites);
|
|
763
|
+
const files = uniquePaths([...targetRefs, ...allowedWrites, ...sourceRefs]).slice(0, 40);
|
|
764
|
+
return {
|
|
765
|
+
id,
|
|
766
|
+
taskId: id,
|
|
767
|
+
title: textValue(task.title ?? task.objective ?? id, id),
|
|
768
|
+
objective: textValue(task.objective ?? task.summary, ''),
|
|
769
|
+
status,
|
|
770
|
+
queueStatus,
|
|
771
|
+
ready: status === 'todo',
|
|
772
|
+
lane: textValue(task.lane ?? task.groupId ?? task.epicId, queueId),
|
|
773
|
+
group: textValue(task.groupId ?? task.epicId, queueId),
|
|
774
|
+
epicId: textValue(task.epicId, ''),
|
|
775
|
+
priority: numberValue(task.priority),
|
|
776
|
+
changedPaths: files,
|
|
777
|
+
changedPathCount: files.length,
|
|
778
|
+
sourceRefs,
|
|
779
|
+
targetRefs,
|
|
780
|
+
allowedWrites,
|
|
781
|
+
acceptance: stringArray(task.acceptance),
|
|
782
|
+
verification: recordArray(task.verification),
|
|
783
|
+
tags: stringArray(task.tags),
|
|
784
|
+
sourceLabel: path.relative(cwd, file),
|
|
785
|
+
sourceQueue: queueId
|
|
786
|
+
};
|
|
787
|
+
}
|
|
400
788
|
async function readLifetimeDashboardResetCutoff(root) {
|
|
401
789
|
const reset = recordValue(await readJsonFile(path.join(root, LIFETIME_DASHBOARD_RESET_FILE)));
|
|
402
790
|
return numberValue(reset.resetAt ?? reset.generatedAt);
|
|
@@ -431,23 +819,47 @@ async function findLifetimeDashboardArtifactFiles(root, input) {
|
|
|
431
819
|
await walk(root, 0);
|
|
432
820
|
return out;
|
|
433
821
|
}
|
|
434
|
-
function combineLifetimeDashboardSnapshots(options, discoveredSources, snapshots, reviewDecisions) {
|
|
435
|
-
const
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
822
|
+
async function combineLifetimeDashboardSnapshots(options, discoveredSources, snapshots, reviewDecisions, queueBacklog) {
|
|
823
|
+
const drainedRunRoots = new Set(snapshots
|
|
824
|
+
.filter((entry) => entry.source.kind === 'run' && isDrainedAutonomousRunSnapshot(entry.snapshot))
|
|
825
|
+
.map((entry) => lifetimeRunRootKey(entry.source)));
|
|
826
|
+
const visibleSnapshots = snapshots.filter((entry) => {
|
|
827
|
+
if (entry.source.kind !== 'collection')
|
|
828
|
+
return true;
|
|
829
|
+
return !drainedRunRoots.has(lifetimeRunRootKey(entry.source));
|
|
830
|
+
});
|
|
831
|
+
const autoDrainDelays = lifetimeAutoDrainDelayRecords(visibleSnapshots);
|
|
832
|
+
const jobs = dedupeLifetimeDashboardJobs(collapseSupersededLifetimeReviewJobs(applyCoordinatorReviewDecisions(visibleSnapshots.flatMap(({ source, snapshot }) => {
|
|
833
|
+
const autoDrainDelay = lifetimeAutoDrainDelayRecord(source, snapshot);
|
|
834
|
+
return recordArray(snapshot.jobs).map((job) => {
|
|
835
|
+
const sourceJobId = textValue(job.id ?? job.jobId ?? job.taskId, 'job');
|
|
836
|
+
return withRecomputedCostFields({
|
|
837
|
+
...job,
|
|
838
|
+
id: lifetimeScopedId(source, sourceJobId),
|
|
839
|
+
sourceJobId,
|
|
840
|
+
originalJobId: unscopedLifetimeTaskKey(sourceJobId),
|
|
841
|
+
sourceRun: source.run,
|
|
842
|
+
sourceCollection: source.collection,
|
|
843
|
+
sourceContinuation: source.continuation,
|
|
844
|
+
sourceLabel: source.label,
|
|
845
|
+
...(autoDrainDelay ? {
|
|
846
|
+
coordinationDelay: autoDrainDelay.reason,
|
|
847
|
+
autoDrainSkippedReason: autoDrainDelay.skippedReason,
|
|
848
|
+
autoDrainDirtyPathCount: autoDrainDelay.dirtyPathCount
|
|
849
|
+
} : {}),
|
|
850
|
+
generatedAt: numberValue(job.generatedAt) || numberValue(snapshot.generatedAt) || source.mtimeMs
|
|
851
|
+
});
|
|
852
|
+
});
|
|
853
|
+
}), reviewDecisions))).slice(0, LIFETIME_DASHBOARD_MAX_JOBS);
|
|
854
|
+
const humanActionAnswers = await readHumanActionAnswers(options);
|
|
855
|
+
const summary = {
|
|
856
|
+
...lifetimeDashboardSummary(jobs),
|
|
857
|
+
coordinationDelayCount: autoDrainDelays.length,
|
|
858
|
+
dirtyAutoDrainSkipCount: autoDrainDelays.filter((record) => record.skippedReason === 'dirty-worktree').length
|
|
859
|
+
};
|
|
860
|
+
const queueOverlay = lifetimeQueueBacklogOverlay(queueBacklog, jobs);
|
|
861
|
+
const latestGeneratedAt = Math.max(Date.now(), numberValue(queueBacklog.generatedAt), ...visibleSnapshots.map((entry) => numberValue(entry.snapshot.generatedAt)), ...discoveredSources.map((source) => source.mtimeMs));
|
|
862
|
+
const events = visibleSnapshots.flatMap(({ source, snapshot }) => recordArray(snapshot.events).map((event) => ({
|
|
451
863
|
...event,
|
|
452
864
|
sourceLabel: source.label,
|
|
453
865
|
message: textValue(event.message, textValue(event.type, 'event')),
|
|
@@ -456,14 +868,19 @@ function combineLifetimeDashboardSnapshots(options, discoveredSources, snapshots
|
|
|
456
868
|
return {
|
|
457
869
|
kind: 'frontier.loom-ui.lifetime-dashboard',
|
|
458
870
|
version: 1,
|
|
459
|
-
ok:
|
|
871
|
+
ok: true,
|
|
460
872
|
generatedAt: latestGeneratedAt,
|
|
461
873
|
cwd: options.cwd,
|
|
462
874
|
sources: {
|
|
463
875
|
workspace: options.cwd,
|
|
464
876
|
lifetimeRoot: path.join(options.cwd, 'agent-runs'),
|
|
877
|
+
queueRoot: path.join(options.cwd, '.loom', 'queues'),
|
|
465
878
|
sourceCount: discoveredSources.length,
|
|
466
|
-
loadedSourceCount:
|
|
879
|
+
loadedSourceCount: visibleSnapshots.length,
|
|
880
|
+
suppressedCollectionSourceCount: snapshots.length - visibleSnapshots.length,
|
|
881
|
+
queueSourceCount: queueBacklog.sourceCount,
|
|
882
|
+
coordinationDelayCount: autoDrainDelays.length,
|
|
883
|
+
...(humanActionAnswers.length ? { humanActionAnswers: await humanActionAnswerLogPath(options) } : {}),
|
|
467
884
|
...(reviewDecisions.length ? { coordinatorReviewDecisions: coordinatorReviewDecisionPath(options.cwd) } : {})
|
|
468
885
|
},
|
|
469
886
|
summary,
|
|
@@ -472,22 +889,32 @@ function combineLifetimeDashboardSnapshots(options, discoveredSources, snapshots
|
|
|
472
889
|
quality: {},
|
|
473
890
|
timeSeries: lifetimeTimeSeries(jobs, events),
|
|
474
891
|
lanes: lifetimeLaneRows(jobs),
|
|
892
|
+
capacity: lifetimeCapacitySummary(queueBacklog, jobs, queueOverlay.entries),
|
|
475
893
|
jobs,
|
|
476
|
-
humanActions:
|
|
894
|
+
humanActions: visibleSnapshots.flatMap(({ snapshot }) => recordArray(snapshot.humanActions)).slice(-100),
|
|
477
895
|
humanActionAnswers,
|
|
478
896
|
events,
|
|
479
|
-
routing: lifetimeRoutingSummary(
|
|
897
|
+
routing: await lifetimeRoutingSummary(options.cwd, visibleSnapshots),
|
|
480
898
|
backlog: {
|
|
481
899
|
id: 'workspace-lifetime',
|
|
482
|
-
entryCount:
|
|
483
|
-
readyCount:
|
|
900
|
+
entryCount: queueOverlay.totalCount,
|
|
901
|
+
readyCount: queueOverlay.readyCount,
|
|
902
|
+
activeCount: queueOverlay.activeCount,
|
|
903
|
+
doneCount: queueOverlay.doneCount,
|
|
904
|
+
failedCount: queueOverlay.failedCount,
|
|
905
|
+
representedCount: queueOverlay.representedCount,
|
|
906
|
+
entries: queueOverlay.entries
|
|
484
907
|
},
|
|
485
908
|
raw: {
|
|
486
909
|
lifetime: {
|
|
487
910
|
mode: 'workspace',
|
|
488
911
|
sourceCount: discoveredSources.length,
|
|
489
|
-
loadedSourceCount:
|
|
490
|
-
|
|
912
|
+
loadedSourceCount: visibleSnapshots.length,
|
|
913
|
+
suppressedCollectionSourceCount: snapshots.length - visibleSnapshots.length,
|
|
914
|
+
autoDrainDelays,
|
|
915
|
+
sources: discoveredSources.slice(0, LIFETIME_DASHBOARD_MAX_SOURCES),
|
|
916
|
+
manifests: queueBacklog.manifests,
|
|
917
|
+
queueSources: queueBacklog.paths
|
|
491
918
|
}
|
|
492
919
|
}
|
|
493
920
|
};
|
|
@@ -495,6 +922,723 @@ function combineLifetimeDashboardSnapshots(options, discoveredSources, snapshots
|
|
|
495
922
|
function lifetimeScopedId(source, id) {
|
|
496
923
|
return `${source.id}:${id}`.replaceAll(/[^\w:.-]+/g, '-');
|
|
497
924
|
}
|
|
925
|
+
function lifetimeAutoDrainDelayRecords(entries) {
|
|
926
|
+
return entries
|
|
927
|
+
.map(({ source, snapshot }) => lifetimeAutoDrainDelayRecord(source, snapshot))
|
|
928
|
+
.filter((record) => Boolean(record));
|
|
929
|
+
}
|
|
930
|
+
function lifetimeAutoDrainDelayRecord(source, snapshot) {
|
|
931
|
+
const rawRun = recordValue(recordValue(snapshot.raw).run);
|
|
932
|
+
const autoDrain = recordValue(rawRun.autoDrain);
|
|
933
|
+
const skippedReason = textValue(autoDrain.skippedReason, '');
|
|
934
|
+
if (skippedReason !== 'dirty-worktree')
|
|
935
|
+
return undefined;
|
|
936
|
+
const summary = recordValue(autoDrain.summary);
|
|
937
|
+
const dirtyPaths = stringArray(autoDrain.dirtyPaths);
|
|
938
|
+
return {
|
|
939
|
+
source: source.path,
|
|
940
|
+
sourceLabel: source.label,
|
|
941
|
+
reason: 'apply-delayed-by-dirty-worktree',
|
|
942
|
+
skippedReason,
|
|
943
|
+
dirtyPathCount: dirtyPaths.length,
|
|
944
|
+
dirtyPaths: dirtyPaths.slice(0, 12),
|
|
945
|
+
remainingReadyCount: numberValue(summary.remainingReadyCount),
|
|
946
|
+
generatedAt: numberValue(autoDrain.generatedAt) || numberValue(snapshot.generatedAt) || source.mtimeMs
|
|
947
|
+
};
|
|
948
|
+
}
|
|
949
|
+
async function enrichLifetimeRunSnapshotEvidence(cwd, source, snapshot) {
|
|
950
|
+
if (source.kind !== 'run' || !source.run)
|
|
951
|
+
return snapshot;
|
|
952
|
+
const runRoot = safeCwdRelativeDirectory(cwd, source.run);
|
|
953
|
+
if (!runRoot)
|
|
954
|
+
return snapshot;
|
|
955
|
+
const jobs = recordArray(snapshot.jobs);
|
|
956
|
+
if (!jobs.length)
|
|
957
|
+
return snapshot;
|
|
958
|
+
const enrichedJobs = await Promise.all(jobs.map((job) => enrichLifetimeRunJobEvidence(cwd, runRoot, job)));
|
|
959
|
+
return {
|
|
960
|
+
...snapshot,
|
|
961
|
+
jobs: enrichedJobs
|
|
962
|
+
};
|
|
963
|
+
}
|
|
964
|
+
async function enrichLifetimeRunJobEvidence(cwd, runRoot, job) {
|
|
965
|
+
const jobDir = await findBestRawRunJobDir(runRoot, rawRunJobIdCandidates(job));
|
|
966
|
+
if (!jobDir)
|
|
967
|
+
return job;
|
|
968
|
+
const evidenceDir = path.join(jobDir, 'evidence');
|
|
969
|
+
const eventsPath = path.join(jobDir, 'codex-events.jsonl');
|
|
970
|
+
const rootMerge = recordValue(await readJsonFile(path.join(jobDir, 'merge.json')));
|
|
971
|
+
const evidenceMerge = recordValue(await readJsonFile(path.join(evidenceDir, 'merge.json')));
|
|
972
|
+
const evidenceRecord = recordValue(await readJsonFile(path.join(evidenceDir, 'evidence.json')));
|
|
973
|
+
const merge = Object.keys(rootMerge).length ? rootMerge : evidenceMerge;
|
|
974
|
+
const rawPatchPath = await firstExistingRelativePath(cwd, rawRunPatchCandidates(jobDir));
|
|
975
|
+
const usage = await readCodexEventUsageSummary(eventsPath);
|
|
976
|
+
const evidencePaths = await existingRelativePaths(cwd, [
|
|
977
|
+
path.join(jobDir, 'last-message.md'),
|
|
978
|
+
eventsPath,
|
|
979
|
+
path.join(jobDir, 'merge.json'),
|
|
980
|
+
path.join(evidenceDir, 'last-message.md'),
|
|
981
|
+
path.join(evidenceDir, 'handoff.md'),
|
|
982
|
+
path.join(evidenceDir, 'evidence.json'),
|
|
983
|
+
path.join(evidenceDir, 'merge.json'),
|
|
984
|
+
path.join(evidenceDir, 'human-question.json'),
|
|
985
|
+
path.join(evidenceDir, 'resource-allocation.json'),
|
|
986
|
+
path.join(evidenceDir, 'model-availability.json'),
|
|
987
|
+
...rawRunPatchCandidates(jobDir)
|
|
988
|
+
]);
|
|
989
|
+
if (!evidencePaths.length && !rawPatchPath && !Object.keys(merge).length)
|
|
990
|
+
return job;
|
|
991
|
+
const patchChangedPaths = await readPatchChangedPathList(cwd, rawPatchPath);
|
|
992
|
+
const changedPaths = uniquePaths([
|
|
993
|
+
...stringArray(job.changedPaths),
|
|
994
|
+
...stringArray(merge.changedPaths),
|
|
995
|
+
...patchChangedPaths
|
|
996
|
+
]);
|
|
997
|
+
const ownershipViolations = uniquePaths([
|
|
998
|
+
...stringArray(job.ownershipViolations),
|
|
999
|
+
...stringArray(merge.ownershipViolations)
|
|
1000
|
+
]);
|
|
1001
|
+
const status = lifetimeRunEvidenceStatus(job, merge, evidencePaths);
|
|
1002
|
+
const bucket = lifetimeRunEvidenceBucket(job, status, evidencePaths, rawPatchPath);
|
|
1003
|
+
const collectReasonClasses = uniquePaths([
|
|
1004
|
+
...stringArray(job.collectReasonClasses),
|
|
1005
|
+
status === 'failed' && evidencePaths.length ? 'worker failed with evidence' : 'raw run evidence discovered'
|
|
1006
|
+
]);
|
|
1007
|
+
const mergedEvidencePaths = uniquePaths([...stringArray(job.evidencePaths), ...evidencePaths]);
|
|
1008
|
+
const commandEvidence = commandEvidenceFromRecords(job, merge, evidenceRecord);
|
|
1009
|
+
return withRecomputedCostFields({
|
|
1010
|
+
...job,
|
|
1011
|
+
status,
|
|
1012
|
+
bucket,
|
|
1013
|
+
disposition: textValue(merge.disposition, textValue(job.disposition, status)),
|
|
1014
|
+
mergeReadiness: textValue(merge.mergeReadiness, textValue(job.mergeReadiness, status)),
|
|
1015
|
+
...(rawPatchPath ? { patchPath: rawPatchPath, artifactPaths: uniquePaths([rawPatchPath, ...stringArray(job.artifactPaths)]) } : {}),
|
|
1016
|
+
changedPaths,
|
|
1017
|
+
changedPathCount: changedPaths.length || numberValue(job.changedPathCount),
|
|
1018
|
+
ownershipViolations,
|
|
1019
|
+
ownershipViolationCount: ownershipViolations.length || numberValue(job.ownershipViolationCount),
|
|
1020
|
+
...(usage.inputTokens ? { actualInputTokens: usage.inputTokens, inputTokens: usage.inputTokens } : {}),
|
|
1021
|
+
...(!usage.inputTokens && usage.estimatedInputTokens ? { estimatedInputTokens: usage.estimatedInputTokens } : {}),
|
|
1022
|
+
...(usage.cachedInputTokens ? { cachedInputTokens: usage.cachedInputTokens } : {}),
|
|
1023
|
+
...(usage.uncachedInputTokens ? { uncachedInputTokens: usage.uncachedInputTokens } : {}),
|
|
1024
|
+
...(usage.outputTokens ? { actualOutputTokens: usage.outputTokens, outputTokens: usage.outputTokens } : {}),
|
|
1025
|
+
...(usage.reasoningOutputTokens ? { reasoningOutputTokens: usage.reasoningOutputTokens } : {}),
|
|
1026
|
+
...(usage.eventCount || usage.estimatedInputTokens ? {
|
|
1027
|
+
usage: {
|
|
1028
|
+
...recordValue(job.usage),
|
|
1029
|
+
input_tokens: usage.inputTokens,
|
|
1030
|
+
cached_input_tokens: usage.cachedInputTokens,
|
|
1031
|
+
uncached_input_tokens: usage.uncachedInputTokens,
|
|
1032
|
+
output_tokens: usage.outputTokens,
|
|
1033
|
+
reasoning_output_tokens: usage.reasoningOutputTokens,
|
|
1034
|
+
estimated_input_tokens: usage.estimatedInputTokens,
|
|
1035
|
+
estimated_from_event_bytes: usage.estimatedFromEventBytes,
|
|
1036
|
+
source: usage.eventCount ? 'codex-events.jsonl' : 'codex-events.jsonl-estimate',
|
|
1037
|
+
event_count: usage.eventCount
|
|
1038
|
+
}
|
|
1039
|
+
} : {}),
|
|
1040
|
+
evidencePaths: mergedEvidencePaths,
|
|
1041
|
+
evidencePathCount: mergedEvidencePaths.length,
|
|
1042
|
+
reasons: stringArray(job.reasons).length ? stringArray(job.reasons) : stringArray(merge.reasons),
|
|
1043
|
+
commandsPassed: commandEvidence.passed,
|
|
1044
|
+
commandsFailed: commandEvidence.failed,
|
|
1045
|
+
collectReasonClasses,
|
|
1046
|
+
runEvidenceRecovered: true
|
|
1047
|
+
});
|
|
1048
|
+
}
|
|
1049
|
+
function rawRunJobIdCandidates(job) {
|
|
1050
|
+
const values = [
|
|
1051
|
+
textValue(job.originalJobId, ''),
|
|
1052
|
+
textValue(job.jobId, ''),
|
|
1053
|
+
textValue(job.id, ''),
|
|
1054
|
+
textValue(job.taskId, '')
|
|
1055
|
+
].filter(Boolean);
|
|
1056
|
+
const out = new Set();
|
|
1057
|
+
for (const value of values) {
|
|
1058
|
+
out.add(value);
|
|
1059
|
+
const parts = value.split(':').filter(Boolean);
|
|
1060
|
+
if (parts.length)
|
|
1061
|
+
out.add(parts[parts.length - 1]);
|
|
1062
|
+
}
|
|
1063
|
+
return Array.from(out);
|
|
1064
|
+
}
|
|
1065
|
+
async function findBestRawRunJobDir(runRoot, candidates) {
|
|
1066
|
+
const matches = new Map();
|
|
1067
|
+
for (const candidate of candidates) {
|
|
1068
|
+
const direct = path.join(runRoot, candidate);
|
|
1069
|
+
if (await rawRunJobHasArtifacts(direct))
|
|
1070
|
+
matches.set(direct, await rawRunJobEvidenceScore(direct));
|
|
1071
|
+
for (const match of await findRawRunJobDirs(runRoot, candidate, 0)) {
|
|
1072
|
+
matches.set(match, await rawRunJobEvidenceScore(match));
|
|
1073
|
+
}
|
|
1074
|
+
}
|
|
1075
|
+
return Array.from(matches.entries()).sort((left, right) => right[1] - left[1] || right[0].localeCompare(left[0]))[0]?.[0];
|
|
1076
|
+
}
|
|
1077
|
+
async function rawRunJobEvidenceScore(jobDir) {
|
|
1078
|
+
let score = 0;
|
|
1079
|
+
for (const [relative, weight] of [
|
|
1080
|
+
['last-message.md', 100],
|
|
1081
|
+
['evidence/merge.json', 80],
|
|
1082
|
+
['merge.json', 80],
|
|
1083
|
+
['evidence/evidence.json', 50],
|
|
1084
|
+
['evidence/changes.patch', 40],
|
|
1085
|
+
['changes.patch', 40],
|
|
1086
|
+
['codex-events.jsonl', 20]
|
|
1087
|
+
]) {
|
|
1088
|
+
const stat = await fs.stat(path.join(jobDir, relative)).catch(() => undefined);
|
|
1089
|
+
if (stat?.isFile())
|
|
1090
|
+
score += weight + Math.min(10, Math.floor(stat.size / 1024));
|
|
1091
|
+
}
|
|
1092
|
+
const stat = await fs.stat(jobDir).catch(() => undefined);
|
|
1093
|
+
return score + Math.floor((stat?.mtimeMs ?? 0) / 1_000_000_000);
|
|
1094
|
+
}
|
|
1095
|
+
function lifetimeRunEvidenceStatus(job, merge, evidencePaths) {
|
|
1096
|
+
const mergeStatus = textValue(coordinatorFacingMachineLabel(merge.status), '');
|
|
1097
|
+
if (mergeStatus)
|
|
1098
|
+
return mergeStatus;
|
|
1099
|
+
const status = textValue(coordinatorFacingMachineLabel(job.status), '');
|
|
1100
|
+
if (status)
|
|
1101
|
+
return status === 'failed' && evidencePaths.some((entry) => entry.endsWith('last-message.md')) ? 'completed' : status;
|
|
1102
|
+
return evidencePaths.some((entry) => entry.endsWith('last-message.md')) ? 'completed' : 'failed';
|
|
1103
|
+
}
|
|
1104
|
+
function lifetimeRunEvidenceBucket(job, status, evidencePaths, patchPath) {
|
|
1105
|
+
const bucket = textValue(coordinatorFacingMachineLabel(job.bucket), '');
|
|
1106
|
+
if (status === 'running')
|
|
1107
|
+
return 'running';
|
|
1108
|
+
if (status === 'completed')
|
|
1109
|
+
return bucket && bucket !== 'failed-evidence' ? bucket : 'completed';
|
|
1110
|
+
if (status === 'failed' && (evidencePaths.length || patchPath))
|
|
1111
|
+
return 'worker-failed';
|
|
1112
|
+
return bucket || (status === 'failed' ? 'failed-evidence' : status);
|
|
1113
|
+
}
|
|
1114
|
+
async function readLatestDrainCoordinatorSnapshot(cwd) {
|
|
1115
|
+
const root = path.join(cwd, 'agent-runs', 'frontier-swarm-codex');
|
|
1116
|
+
const drains = await findDrainCoordinatorRunDirs(root);
|
|
1117
|
+
const activeDrainRoot = drains[0] ? drainRootForRunDir(drains[0]) : '';
|
|
1118
|
+
const runDirs = drains
|
|
1119
|
+
.filter((runDir) => drainRootForRunDir(runDir) === activeDrainRoot)
|
|
1120
|
+
.slice(0, LIFETIME_DASHBOARD_MAX_DRAIN_RUNS);
|
|
1121
|
+
const jobs = [];
|
|
1122
|
+
for (const runDir of runDirs)
|
|
1123
|
+
jobs.push(...await readDrainCoordinatorJobs(cwd, runDir));
|
|
1124
|
+
if (!jobs.length)
|
|
1125
|
+
return undefined;
|
|
1126
|
+
const generatedAt = Math.max(...jobs.map((job) => numberValue(job.generatedAt)), Date.now());
|
|
1127
|
+
return {
|
|
1128
|
+
ok: true,
|
|
1129
|
+
generatedAt,
|
|
1130
|
+
cwd,
|
|
1131
|
+
sources: {
|
|
1132
|
+
activeDrain: runDirs[0],
|
|
1133
|
+
activeDrainSources: runDirs
|
|
1134
|
+
},
|
|
1135
|
+
summary: lifetimeDashboardSummary(jobs),
|
|
1136
|
+
lanes: lifetimeLaneRows(jobs),
|
|
1137
|
+
jobs,
|
|
1138
|
+
events: activeRunEvents(jobs),
|
|
1139
|
+
raw: {
|
|
1140
|
+
activeDrain: {
|
|
1141
|
+
runDirs,
|
|
1142
|
+
jobCount: jobs.length,
|
|
1143
|
+
runningCount: jobs.filter((job) => textValue(job.status, '') === 'running').length
|
|
1144
|
+
}
|
|
1145
|
+
}
|
|
1146
|
+
};
|
|
1147
|
+
}
|
|
1148
|
+
async function findDrainCoordinatorRunDirs(root) {
|
|
1149
|
+
const out = [];
|
|
1150
|
+
const rootEntries = await fs.readdir(root, { withFileTypes: true }).catch(() => []);
|
|
1151
|
+
for (const drain of rootEntries) {
|
|
1152
|
+
if (!drain.isDirectory() || !drain.name.startsWith('drain-'))
|
|
1153
|
+
continue;
|
|
1154
|
+
const drainDir = path.join(root, drain.name);
|
|
1155
|
+
const iterationEntries = await fs.readdir(drainDir, { withFileTypes: true }).catch(() => []);
|
|
1156
|
+
for (const iteration of iterationEntries) {
|
|
1157
|
+
if (!iteration.isDirectory() || !iteration.name.startsWith('iteration-'))
|
|
1158
|
+
continue;
|
|
1159
|
+
for (const runDirName of ['coordinator-run', 'worker-run']) {
|
|
1160
|
+
const runDir = path.join(drainDir, iteration.name, runDirName);
|
|
1161
|
+
const stat = await fs.stat(runDir).catch(() => undefined);
|
|
1162
|
+
if (stat?.isDirectory())
|
|
1163
|
+
out.push({ dir: runDir, mtimeMs: stat.mtimeMs });
|
|
1164
|
+
}
|
|
1165
|
+
}
|
|
1166
|
+
}
|
|
1167
|
+
return out.sort((left, right) => right.mtimeMs - left.mtimeMs || right.dir.localeCompare(left.dir)).map((entry) => entry.dir);
|
|
1168
|
+
}
|
|
1169
|
+
function drainRootForRunDir(runDir) {
|
|
1170
|
+
return path.dirname(path.dirname(runDir));
|
|
1171
|
+
}
|
|
1172
|
+
async function readDrainCoordinatorJobs(cwd, coordinatorRunDir) {
|
|
1173
|
+
const entries = await fs.readdir(coordinatorRunDir, { withFileTypes: true }).catch(() => []);
|
|
1174
|
+
const liveLines = liveProcessLinesForPath(coordinatorRunDir);
|
|
1175
|
+
const now = Date.now();
|
|
1176
|
+
const jobs = [];
|
|
1177
|
+
const seenJobIds = new Set();
|
|
1178
|
+
for (const entry of entries) {
|
|
1179
|
+
if (!entry.isDirectory() || entry.name === 'streams')
|
|
1180
|
+
continue;
|
|
1181
|
+
seenJobIds.add(entry.name);
|
|
1182
|
+
jobs.push(await readDrainCoordinatorJob(cwd, coordinatorRunDir, path.join(coordinatorRunDir, entry.name), liveLines, now));
|
|
1183
|
+
}
|
|
1184
|
+
const pidEntries = await readRunPidEntries(coordinatorRunDir);
|
|
1185
|
+
const planJobs = await readRunPlanJobs(coordinatorRunDir);
|
|
1186
|
+
for (const entry of pidEntries) {
|
|
1187
|
+
const jobId = textValue(entry.jobId, '');
|
|
1188
|
+
if (!jobId || seenJobIds.has(jobId))
|
|
1189
|
+
continue;
|
|
1190
|
+
jobs.push(readDrainPidManifestJob(cwd, coordinatorRunDir, entry, planJobs.get(jobId), now));
|
|
1191
|
+
}
|
|
1192
|
+
return jobs.sort((left, right) => textValue(left.lane, '').localeCompare(textValue(right.lane, '')));
|
|
1193
|
+
}
|
|
1194
|
+
async function readRunPidEntries(runDir) {
|
|
1195
|
+
const pidManifest = recordValue(await readJsonFile(path.join(runDir, 'pids.json')));
|
|
1196
|
+
return recordArray(pidManifest.entries).filter((entry) => textValue(entry.role, '') === 'codex');
|
|
1197
|
+
}
|
|
1198
|
+
async function readRunPlanJobs(runDir) {
|
|
1199
|
+
const plan = recordValue(await readJsonFile(path.join(runDir, 'swarm-plan.json')));
|
|
1200
|
+
const entries = [];
|
|
1201
|
+
for (const job of recordArray(plan.jobs)) {
|
|
1202
|
+
const id = textValue(job.id, '');
|
|
1203
|
+
if (id)
|
|
1204
|
+
entries.push([id, job]);
|
|
1205
|
+
}
|
|
1206
|
+
return new Map(entries);
|
|
1207
|
+
}
|
|
1208
|
+
function readDrainPidManifestJob(cwd, coordinatorRunDir, entry, planJob, now) {
|
|
1209
|
+
const jobId = textValue(entry.jobId, 'job');
|
|
1210
|
+
const runKind = path.basename(coordinatorRunDir) === 'worker-run' ? 'worker' : 'coordinator';
|
|
1211
|
+
const task = recordValue(planJob?.task);
|
|
1212
|
+
const compute = recordValue(planJob?.compute);
|
|
1213
|
+
const command = stringArray(entry.command);
|
|
1214
|
+
const live = isProcessLive(numberValue(entry.pid), entry);
|
|
1215
|
+
const status = live ? 'running' : 'failed';
|
|
1216
|
+
const startedAt = numberValue(entry.startedAt);
|
|
1217
|
+
const lane = textValue(planJob?.lane ?? task.lane, drainCoordinatorLane(jobId, runKind));
|
|
1218
|
+
return withRecomputedCostFields({
|
|
1219
|
+
id: jobId,
|
|
1220
|
+
originalJobId: jobId,
|
|
1221
|
+
taskId: textValue(planJob?.taskId ?? task.id, jobId),
|
|
1222
|
+
title: textValue(planJob?.title ?? task.title, runKind === 'coordinator' ? `Coordinate lane review for ${lane}` : `Continue ${lane} work`),
|
|
1223
|
+
lane,
|
|
1224
|
+
status,
|
|
1225
|
+
bucket: status === 'running' ? 'running' : 'failed-evidence',
|
|
1226
|
+
disposition: status === 'running' ? 'active' : 'failed',
|
|
1227
|
+
agentId: jobId,
|
|
1228
|
+
workerId: jobId,
|
|
1229
|
+
model: textValue(compute.model, commandOptionValue(command, '--model') || 'gpt-5.5'),
|
|
1230
|
+
computeId: textValue(compute.id, runKind === 'coordinator' ? 'coordinator-agent' : 'continuation-worker'),
|
|
1231
|
+
reasoningEffort: textValue(compute.reasoningEffort, ''),
|
|
1232
|
+
startedAt: startedAt || undefined,
|
|
1233
|
+
durationMs: startedAt ? Math.max(0, now - startedAt) : 0,
|
|
1234
|
+
evidencePaths: [],
|
|
1235
|
+
evidencePathCount: 0,
|
|
1236
|
+
changedPathCount: 0,
|
|
1237
|
+
collectReasonClasses: status === 'running' ? [`active drain ${runKind}`] : [`missing ${runKind} output`],
|
|
1238
|
+
mergeReadiness: status,
|
|
1239
|
+
sourceRun: path.relative(cwd, coordinatorRunDir),
|
|
1240
|
+
sourceLabel: path.relative(cwd, coordinatorRunDir),
|
|
1241
|
+
generatedAt: now
|
|
1242
|
+
});
|
|
1243
|
+
}
|
|
1244
|
+
function commandOptionValue(command, option) {
|
|
1245
|
+
const index = command.indexOf(option);
|
|
1246
|
+
return index >= 0 ? textValue(command[index + 1], '') : '';
|
|
1247
|
+
}
|
|
1248
|
+
async function readDrainCoordinatorJob(cwd, coordinatorRunDir, jobDir, liveLines, now) {
|
|
1249
|
+
const jobId = path.basename(jobDir);
|
|
1250
|
+
const runKind = path.basename(coordinatorRunDir) === 'worker-run' ? 'worker' : 'coordinator';
|
|
1251
|
+
const evidenceDir = path.join(jobDir, 'evidence');
|
|
1252
|
+
const eventsPath = path.join(jobDir, 'codex-events.jsonl');
|
|
1253
|
+
const lastMessagePath = path.join(jobDir, 'last-message.md');
|
|
1254
|
+
const decisionsJson = path.join(evidenceDir, 'coordinator-decisions.json');
|
|
1255
|
+
const decisionsJsonl = path.join(evidenceDir, 'coordinator-decisions.jsonl');
|
|
1256
|
+
const modelAvailability = recordValue(await readJsonFile(path.join(evidenceDir, 'model-availability.json')));
|
|
1257
|
+
const evidenceRecord = recordValue(await readJsonFile(path.join(evidenceDir, 'evidence.json')));
|
|
1258
|
+
const eventStat = await fs.stat(eventsPath).catch(() => undefined);
|
|
1259
|
+
const lastMessageStat = await fs.stat(lastMessagePath).catch(() => undefined);
|
|
1260
|
+
const decisionJsonStat = await fs.stat(decisionsJson).catch(() => undefined);
|
|
1261
|
+
const decisionJsonlStat = await fs.stat(decisionsJsonl).catch(() => undefined);
|
|
1262
|
+
const live = liveLines.some((line) => line.includes(jobDir) || line.includes(jobId));
|
|
1263
|
+
const hasDecision = Boolean(decisionJsonStat?.isFile() || decisionJsonlStat?.isFile());
|
|
1264
|
+
const failed = !live && !lastMessageStat && !hasDecision && await codexEventsHaveFailure(eventsPath);
|
|
1265
|
+
const status = live && !lastMessageStat
|
|
1266
|
+
? 'running'
|
|
1267
|
+
: lastMessageStat || hasDecision
|
|
1268
|
+
? 'completed'
|
|
1269
|
+
: 'failed';
|
|
1270
|
+
const startedAt = numberValue(eventStat?.birthtimeMs ?? eventStat?.ctimeMs ?? eventStat?.mtimeMs);
|
|
1271
|
+
const finishedAt = status === 'running'
|
|
1272
|
+
? undefined
|
|
1273
|
+
: Math.max(numberValue(lastMessageStat?.mtimeMs), numberValue(decisionJsonStat?.mtimeMs), numberValue(decisionJsonlStat?.mtimeMs), numberValue(eventStat?.mtimeMs));
|
|
1274
|
+
const usage = await readCodexEventUsageSummary(eventsPath);
|
|
1275
|
+
const lane = drainCoordinatorLane(jobId, runKind);
|
|
1276
|
+
const evidencePaths = await existingRelativePaths(cwd, [
|
|
1277
|
+
eventsPath,
|
|
1278
|
+
lastMessagePath,
|
|
1279
|
+
decisionsJson,
|
|
1280
|
+
decisionsJsonl,
|
|
1281
|
+
path.join(evidenceDir, 'merge.json'),
|
|
1282
|
+
path.join(evidenceDir, 'evidence.json'),
|
|
1283
|
+
path.join(evidenceDir, 'human-question.json'),
|
|
1284
|
+
path.join(evidenceDir, 'resource-allocation.json'),
|
|
1285
|
+
path.join(evidenceDir, 'model-availability.json')
|
|
1286
|
+
]);
|
|
1287
|
+
const commandEvidence = commandEvidenceFromRecords(evidenceRecord);
|
|
1288
|
+
return withRecomputedCostFields({
|
|
1289
|
+
id: jobId,
|
|
1290
|
+
originalJobId: jobId,
|
|
1291
|
+
taskId: jobId,
|
|
1292
|
+
title: runKind === 'coordinator' ? `Coordinate lane review for ${lane}` : `Continue ${lane} work`,
|
|
1293
|
+
lane,
|
|
1294
|
+
status,
|
|
1295
|
+
bucket: status === 'running' ? 'running' : status === 'completed' ? 'completed' : 'failed-evidence',
|
|
1296
|
+
disposition: status === 'running' ? 'active' : status,
|
|
1297
|
+
agentId: jobId,
|
|
1298
|
+
workerId: jobId,
|
|
1299
|
+
model: textValue(modelAvailability.effectiveModel ?? modelAvailability.requestedModel, 'gpt-5.5'),
|
|
1300
|
+
computeId: runKind === 'coordinator' ? 'coordinator-agent' : 'continuation-worker',
|
|
1301
|
+
startedAt: startedAt || undefined,
|
|
1302
|
+
...(finishedAt ? { finishedAt } : {}),
|
|
1303
|
+
durationMs: startedAt ? Math.max(0, (finishedAt ?? now) - startedAt) : 0,
|
|
1304
|
+
...(usage.inputTokens ? { actualInputTokens: usage.inputTokens, inputTokens: usage.inputTokens } : {}),
|
|
1305
|
+
...(!usage.inputTokens && usage.estimatedInputTokens ? { estimatedInputTokens: usage.estimatedInputTokens } : {}),
|
|
1306
|
+
...(usage.cachedInputTokens ? { cachedInputTokens: usage.cachedInputTokens } : {}),
|
|
1307
|
+
...(usage.uncachedInputTokens ? { uncachedInputTokens: usage.uncachedInputTokens } : {}),
|
|
1308
|
+
...(usage.outputTokens ? { actualOutputTokens: usage.outputTokens, outputTokens: usage.outputTokens } : {}),
|
|
1309
|
+
...(usage.reasoningOutputTokens ? { reasoningOutputTokens: usage.reasoningOutputTokens } : {}),
|
|
1310
|
+
...(usage.eventCount ? { usage: { ...usage, source: 'codex-events.jsonl' } } : {}),
|
|
1311
|
+
evidencePaths,
|
|
1312
|
+
evidencePathCount: evidencePaths.length,
|
|
1313
|
+
commandsPassed: commandEvidence.passed,
|
|
1314
|
+
commandsFailed: commandEvidence.failed,
|
|
1315
|
+
changedPathCount: 0,
|
|
1316
|
+
collectReasonClasses: status === 'running' ? [`active drain ${runKind}`] : [`drain ${runKind}`],
|
|
1317
|
+
mergeReadiness: status,
|
|
1318
|
+
sourceRun: path.relative(cwd, coordinatorRunDir),
|
|
1319
|
+
sourceLabel: path.relative(cwd, coordinatorRunDir),
|
|
1320
|
+
generatedAt: numberValue(finishedAt) || numberValue(eventStat?.mtimeMs) || now
|
|
1321
|
+
});
|
|
1322
|
+
}
|
|
1323
|
+
function liveProcessLinesForPath(needle) {
|
|
1324
|
+
const result = spawnSync('pgrep', ['-fl', needle], { encoding: 'utf8' });
|
|
1325
|
+
if (result.status !== 0 && !result.stdout)
|
|
1326
|
+
return [];
|
|
1327
|
+
return result.stdout.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
|
|
1328
|
+
}
|
|
1329
|
+
async function existingRelativePaths(cwd, files) {
|
|
1330
|
+
const out = [];
|
|
1331
|
+
for (const file of files) {
|
|
1332
|
+
const stat = await fs.stat(file).catch(() => undefined);
|
|
1333
|
+
if (stat?.isFile())
|
|
1334
|
+
out.push(path.relative(cwd, file));
|
|
1335
|
+
}
|
|
1336
|
+
return out;
|
|
1337
|
+
}
|
|
1338
|
+
async function firstExistingRelativePath(cwd, files) {
|
|
1339
|
+
for (const file of files) {
|
|
1340
|
+
const stat = await fs.stat(file).catch(() => undefined);
|
|
1341
|
+
if (stat?.isFile())
|
|
1342
|
+
return path.relative(cwd, file);
|
|
1343
|
+
}
|
|
1344
|
+
return undefined;
|
|
1345
|
+
}
|
|
1346
|
+
function rawRunPatchCandidates(jobDir) {
|
|
1347
|
+
return [
|
|
1348
|
+
path.join(jobDir, 'changes.patch'),
|
|
1349
|
+
path.join(jobDir, 'source.patch'),
|
|
1350
|
+
path.join(jobDir, 'evidence', 'changes.patch'),
|
|
1351
|
+
path.join(jobDir, 'evidence', 'source.patch')
|
|
1352
|
+
];
|
|
1353
|
+
}
|
|
1354
|
+
async function readPatchChangedPathList(cwd, patchPath) {
|
|
1355
|
+
if (!patchPath)
|
|
1356
|
+
return [];
|
|
1357
|
+
const absolute = path.resolve(cwd, patchPath);
|
|
1358
|
+
if (!isPathInside(cwd, absolute))
|
|
1359
|
+
return [];
|
|
1360
|
+
const stat = await fs.stat(absolute).catch(() => undefined);
|
|
1361
|
+
if (!stat?.isFile() || stat.size > TASK_DETAIL_PATCH_MAX_BYTES)
|
|
1362
|
+
return [];
|
|
1363
|
+
const patch = await fs.readFile(absolute, 'utf8');
|
|
1364
|
+
return uniquePaths(parseUnifiedPatchFiles(patch).map((file) => file.path).filter(Boolean));
|
|
1365
|
+
}
|
|
1366
|
+
async function codexEventsHaveFailure(file) {
|
|
1367
|
+
const text = await fs.readFile(file, 'utf8').catch(() => '');
|
|
1368
|
+
return /"type":"(?:error|turn\.failed)"/.test(text);
|
|
1369
|
+
}
|
|
1370
|
+
async function codexEventsHaveQuotaLimit(file) {
|
|
1371
|
+
const text = await fs.readFile(file, 'utf8').catch(() => '');
|
|
1372
|
+
return /usage limit|quota|purchase more credits/i.test(text);
|
|
1373
|
+
}
|
|
1374
|
+
function drainCoordinatorLane(jobId, runKind = 'coordinator') {
|
|
1375
|
+
if (runKind === 'worker') {
|
|
1376
|
+
for (const marker of [
|
|
1377
|
+
'-continuation-rerun-',
|
|
1378
|
+
'-continuation-supersede-',
|
|
1379
|
+
'-continuation-reject-',
|
|
1380
|
+
'-continuation-',
|
|
1381
|
+
'-queue-candidate-package-'
|
|
1382
|
+
]) {
|
|
1383
|
+
const index = jobId.indexOf(marker);
|
|
1384
|
+
if (index > 0)
|
|
1385
|
+
return jobId.slice(0, index);
|
|
1386
|
+
}
|
|
1387
|
+
}
|
|
1388
|
+
const marker = '-coordinator-agent-';
|
|
1389
|
+
const index = jobId.indexOf(marker);
|
|
1390
|
+
return index > 0 ? jobId.slice(0, index) : jobId;
|
|
1391
|
+
}
|
|
1392
|
+
function mergeLifetimeDrainCoordinatorSnapshot(lifetime, drain) {
|
|
1393
|
+
const drainJobs = recordArray(drain?.jobs);
|
|
1394
|
+
if (!drainJobs.length)
|
|
1395
|
+
return lifetime;
|
|
1396
|
+
const existingJobs = recordArray(lifetime.jobs).filter((job) => {
|
|
1397
|
+
const source = textValue(job.sourceRun ?? job.sourceLabel, '');
|
|
1398
|
+
return !/agent-runs\/frontier-swarm-codex\/drain-.*\/(?:coordinator-run|worker-run)/.test(source);
|
|
1399
|
+
});
|
|
1400
|
+
const jobs = [...drainJobs, ...existingJobs].slice(0, LIFETIME_DASHBOARD_MAX_JOBS);
|
|
1401
|
+
const events = [...recordArray(lifetime.events), ...recordArray(drain?.events)]
|
|
1402
|
+
.sort((left, right) => numberValue(left.at) - numberValue(right.at))
|
|
1403
|
+
.slice(-160);
|
|
1404
|
+
return {
|
|
1405
|
+
...lifetime,
|
|
1406
|
+
generatedAt: Math.max(numberValue(lifetime.generatedAt), numberValue(drain?.generatedAt), Date.now()),
|
|
1407
|
+
sources: {
|
|
1408
|
+
...recordValue(lifetime.sources),
|
|
1409
|
+
...recordValue(drain?.sources)
|
|
1410
|
+
},
|
|
1411
|
+
summary: lifetimeDashboardSummary(jobs),
|
|
1412
|
+
health: lifetimeHealthSummary(jobs),
|
|
1413
|
+
lanes: lifetimeLaneRows(jobs),
|
|
1414
|
+
jobs,
|
|
1415
|
+
events,
|
|
1416
|
+
raw: {
|
|
1417
|
+
...recordValue(lifetime.raw),
|
|
1418
|
+
activeDrain: recordValue(recordValue(drain?.raw).activeDrain)
|
|
1419
|
+
}
|
|
1420
|
+
};
|
|
1421
|
+
}
|
|
1422
|
+
function mergeLifetimeActiveRunSnapshot(lifetime, active) {
|
|
1423
|
+
const activeJobs = recordArray(active?.jobs).filter((job) => textValue(job.status, '') === 'running');
|
|
1424
|
+
if (!activeJobs.length)
|
|
1425
|
+
return lifetime;
|
|
1426
|
+
const activeKeys = new Set(activeJobs.map(lifetimeJobDedupeKey).filter(Boolean));
|
|
1427
|
+
const existingJobs = recordArray(lifetime.jobs).filter((job) => !activeKeys.has(lifetimeJobDedupeKey(job)));
|
|
1428
|
+
const jobs = [...activeJobs, ...existingJobs].slice(0, LIFETIME_DASHBOARD_MAX_JOBS);
|
|
1429
|
+
const activeAgents = activeAgentsFromJobs(jobs);
|
|
1430
|
+
const events = [...recordArray(lifetime.events), ...recordArray(active?.events)]
|
|
1431
|
+
.sort((left, right) => numberValue(left.at) - numberValue(right.at))
|
|
1432
|
+
.slice(-160);
|
|
1433
|
+
return {
|
|
1434
|
+
...lifetime,
|
|
1435
|
+
generatedAt: Math.max(numberValue(lifetime.generatedAt), numberValue(active?.generatedAt), Date.now()),
|
|
1436
|
+
sources: {
|
|
1437
|
+
...recordValue(lifetime.sources),
|
|
1438
|
+
...recordValue(active?.sources)
|
|
1439
|
+
},
|
|
1440
|
+
summary: lifetimeDashboardSummary(jobs),
|
|
1441
|
+
health: lifetimeHealthSummary(jobs),
|
|
1442
|
+
lanes: lifetimeLaneRows(jobs),
|
|
1443
|
+
capacity: lifetimeCapacitySummary({
|
|
1444
|
+
entries: recordArray(recordValue(lifetime.backlog).entries),
|
|
1445
|
+
manifests: recordArray(recordValue(recordValue(lifetime.raw).lifetime).manifests),
|
|
1446
|
+
sourceCount: numberValue(recordValue(lifetime.backlog).entryCount),
|
|
1447
|
+
paths: stringArray(recordValue(recordValue(lifetime.raw).lifetime).queueSources),
|
|
1448
|
+
generatedAt: numberValue(lifetime.generatedAt)
|
|
1449
|
+
}, jobs, recordArray(recordValue(lifetime.backlog).entries)),
|
|
1450
|
+
jobs,
|
|
1451
|
+
activeAgents,
|
|
1452
|
+
events,
|
|
1453
|
+
raw: {
|
|
1454
|
+
...recordValue(lifetime.raw),
|
|
1455
|
+
activeRuns: recordValue(recordValue(active?.raw).activeRuns)
|
|
1456
|
+
}
|
|
1457
|
+
};
|
|
1458
|
+
}
|
|
1459
|
+
async function readLifetimeActiveRunSnapshot(options) {
|
|
1460
|
+
const jobs = await readLiveCodexProcessJobs(options.cwd);
|
|
1461
|
+
const sources = uniquePaths(jobs.map((job) => textValue(job.sourceRun, '')).filter(Boolean));
|
|
1462
|
+
if (!jobs.length)
|
|
1463
|
+
return undefined;
|
|
1464
|
+
const generatedAt = Date.now();
|
|
1465
|
+
return {
|
|
1466
|
+
ok: true,
|
|
1467
|
+
generatedAt,
|
|
1468
|
+
cwd: options.cwd,
|
|
1469
|
+
sources: {
|
|
1470
|
+
activeRuns: sources,
|
|
1471
|
+
activeRunCount: sources.length
|
|
1472
|
+
},
|
|
1473
|
+
summary: lifetimeDashboardSummary(jobs),
|
|
1474
|
+
lanes: lifetimeLaneRows(jobs),
|
|
1475
|
+
jobs,
|
|
1476
|
+
activeAgents: activeAgentsFromJobs(jobs),
|
|
1477
|
+
events: activeRunEvents(jobs),
|
|
1478
|
+
raw: {
|
|
1479
|
+
activeRuns: {
|
|
1480
|
+
runDirs: sources,
|
|
1481
|
+
jobCount: jobs.length,
|
|
1482
|
+
runningCount: jobs.length
|
|
1483
|
+
}
|
|
1484
|
+
}
|
|
1485
|
+
};
|
|
1486
|
+
}
|
|
1487
|
+
async function readLiveCodexProcessJobs(cwd) {
|
|
1488
|
+
if (process.platform === 'win32')
|
|
1489
|
+
return [];
|
|
1490
|
+
const result = spawnSync('ps', ['-axo', 'pid,ppid,etime,command'], {
|
|
1491
|
+
encoding: 'utf8',
|
|
1492
|
+
stdio: ['ignore', 'pipe', 'ignore']
|
|
1493
|
+
});
|
|
1494
|
+
if (result.status !== 0 && !result.stdout)
|
|
1495
|
+
return [];
|
|
1496
|
+
const agentWorktreeRoot = `${path.join(cwd, 'agent-worktrees')}/`;
|
|
1497
|
+
const now = Date.now();
|
|
1498
|
+
const byWorker = new Map();
|
|
1499
|
+
for (const line of result.stdout.split(/\r?\n/)) {
|
|
1500
|
+
if (!line.includes('codex ') || !line.includes(' exec ') || !line.includes(agentWorktreeRoot))
|
|
1501
|
+
continue;
|
|
1502
|
+
const fields = /^\s*(\d+)\s+(\d+)\s+(\S+)\s+(.+)$/.exec(line);
|
|
1503
|
+
if (!fields)
|
|
1504
|
+
continue;
|
|
1505
|
+
const command = fields[4];
|
|
1506
|
+
const cd = commandOptionValue(splitCommandWords(command), '--cd') || regexCommandOption(command, '--cd');
|
|
1507
|
+
if (!cd.startsWith(agentWorktreeRoot))
|
|
1508
|
+
continue;
|
|
1509
|
+
const outputLastMessage = commandOptionValue(splitCommandWords(command), '--output-last-message') || regexCommandOption(command, '--output-last-message');
|
|
1510
|
+
const model = commandOptionValue(splitCommandWords(command), '--model') || regexCommandOption(command, '--model') || '';
|
|
1511
|
+
const runDir = outputLastMessage ? path.dirname(path.dirname(outputLastMessage)) : '';
|
|
1512
|
+
const jobId = outputLastMessage ? path.basename(path.dirname(outputLastMessage)) : path.basename(cd);
|
|
1513
|
+
const key = outputLastMessage || cd;
|
|
1514
|
+
const pid = Number(fields[1]);
|
|
1515
|
+
const current = byWorker.get(key);
|
|
1516
|
+
if (!current || pid < current.pid)
|
|
1517
|
+
byWorker.set(key, { pid, etime: fields[3], cd, model, outputLastMessage, runDir, jobId });
|
|
1518
|
+
}
|
|
1519
|
+
const planCache = new Map();
|
|
1520
|
+
const jobs = [];
|
|
1521
|
+
for (const worker of byWorker.values()) {
|
|
1522
|
+
const relativeRun = worker.runDir && isPathInside(cwd, worker.runDir) ? path.relative(cwd, worker.runDir) : '';
|
|
1523
|
+
const planJobs = worker.runDir
|
|
1524
|
+
? await (planCache.get(worker.runDir) ?? planCache.set(worker.runDir, readRunPlanJobs(worker.runDir)).get(worker.runDir))
|
|
1525
|
+
: new Map();
|
|
1526
|
+
const planJob = planJobs.get(worker.jobId);
|
|
1527
|
+
const task = recordValue(planJob?.task);
|
|
1528
|
+
const compute = recordValue(planJob?.compute);
|
|
1529
|
+
const startedAt = now - parsePsElapsedMs(worker.etime);
|
|
1530
|
+
const title = textValue(planJob?.title ?? task.title, humanizeWorkerJobId(worker.jobId));
|
|
1531
|
+
const lane = textValue(planJob?.lane ?? task.lane, inferLaneFromWorkerJobId(worker.jobId));
|
|
1532
|
+
jobs.push({
|
|
1533
|
+
id: relativeRun ? `run:${relativeRun.replaceAll(/[^\w:.-]+/g, '-')}:${worker.jobId}` : worker.jobId,
|
|
1534
|
+
originalJobId: worker.jobId,
|
|
1535
|
+
taskId: textValue(planJob?.taskId ?? task.id, worker.jobId),
|
|
1536
|
+
title,
|
|
1537
|
+
lane,
|
|
1538
|
+
status: 'running',
|
|
1539
|
+
bucket: 'running',
|
|
1540
|
+
disposition: 'active',
|
|
1541
|
+
agentId: worker.jobId,
|
|
1542
|
+
workerId: worker.jobId,
|
|
1543
|
+
model: textValue(compute.model, worker.model),
|
|
1544
|
+
computeId: textValue(compute.id, ''),
|
|
1545
|
+
reasoningEffort: textValue(compute.reasoningEffort, ''),
|
|
1546
|
+
startedAt,
|
|
1547
|
+
durationMs: Math.max(0, now - startedAt),
|
|
1548
|
+
evidencePaths: worker.outputLastMessage && isPathInside(cwd, worker.outputLastMessage) ? [path.relative(cwd, worker.outputLastMessage)] : [],
|
|
1549
|
+
evidencePathCount: worker.outputLastMessage ? 1 : 0,
|
|
1550
|
+
changedPathCount: 0,
|
|
1551
|
+
collectReasonClasses: ['active worker process'],
|
|
1552
|
+
mergeReadiness: 'running',
|
|
1553
|
+
sourceRun: relativeRun,
|
|
1554
|
+
sourceLabel: relativeRun ? lifetimeSourceLabel(relativeRun) : 'active-process',
|
|
1555
|
+
generatedAt: now
|
|
1556
|
+
});
|
|
1557
|
+
}
|
|
1558
|
+
return jobs.sort((left, right) => textValue(left.lane, '').localeCompare(textValue(right.lane, '')) || textValue(left.title, '').localeCompare(textValue(right.title, '')));
|
|
1559
|
+
}
|
|
1560
|
+
function regexCommandOption(command, option) {
|
|
1561
|
+
const escaped = option.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
1562
|
+
const match = new RegExp(`${escaped}\\s+([^\\s]+)`).exec(command);
|
|
1563
|
+
return match?.[1] ?? '';
|
|
1564
|
+
}
|
|
1565
|
+
function splitCommandWords(command) {
|
|
1566
|
+
return command.match(/"[^"]*"|'[^']*'|\S+/g)?.map((word) => word.replace(/^(['"])(.*)\1$/u, '$2')) ?? [];
|
|
1567
|
+
}
|
|
1568
|
+
function parsePsElapsedMs(value) {
|
|
1569
|
+
const daySplit = value.split('-');
|
|
1570
|
+
const days = daySplit.length === 2 ? Number(daySplit[0]) || 0 : 0;
|
|
1571
|
+
const time = daySplit.at(-1) ?? '';
|
|
1572
|
+
const parts = time.split(':').map((part) => Number(part) || 0);
|
|
1573
|
+
const [hours, minutes, seconds] = parts.length === 3 ? parts : [0, parts[0] ?? 0, parts[1] ?? 0];
|
|
1574
|
+
return (((days * 24 + hours) * 60 + minutes) * 60 + seconds) * 1000;
|
|
1575
|
+
}
|
|
1576
|
+
function inferLaneFromWorkerJobId(jobId) {
|
|
1577
|
+
const parts = jobId.split('-');
|
|
1578
|
+
const half = Math.floor(parts.length / 2);
|
|
1579
|
+
if (half > 0 && parts.slice(0, half).join('-') === parts.slice(half, half * 2).join('-'))
|
|
1580
|
+
return parts.slice(0, half).join('-');
|
|
1581
|
+
return parts.slice(0, Math.max(1, parts.length - 1)).join('-');
|
|
1582
|
+
}
|
|
1583
|
+
function humanizeWorkerJobId(jobId) {
|
|
1584
|
+
return inferLaneFromWorkerJobId(jobId).split('-').filter(Boolean).map((part) => part[0]?.toUpperCase() + part.slice(1)).join(' ') || jobId;
|
|
1585
|
+
}
|
|
1586
|
+
async function findLifetimeActiveRunDirs(cwd) {
|
|
1587
|
+
const root = path.join(cwd, 'agent-runs');
|
|
1588
|
+
const stat = await fs.stat(root).catch(() => undefined);
|
|
1589
|
+
if (!stat?.isDirectory())
|
|
1590
|
+
return [];
|
|
1591
|
+
const resetAt = await readLifetimeDashboardResetCutoff(root);
|
|
1592
|
+
const files = await findPidManifestFiles(root, LIFETIME_DASHBOARD_SCAN_MAX_DEPTH, LIFETIME_DASHBOARD_SCAN_MAX_FILES, resetAt);
|
|
1593
|
+
const candidates = await Promise.all(files.map(async (file) => ({
|
|
1594
|
+
file,
|
|
1595
|
+
dir: path.dirname(file),
|
|
1596
|
+
mtimeMs: (await fs.stat(file).catch(() => undefined))?.mtimeMs ?? 0
|
|
1597
|
+
})));
|
|
1598
|
+
const out = [];
|
|
1599
|
+
for (const candidate of candidates.sort((left, right) => right.mtimeMs - left.mtimeMs || right.file.localeCompare(left.file))) {
|
|
1600
|
+
if (out.length >= LIFETIME_DASHBOARD_MAX_ACTIVE_PID_RUNS)
|
|
1601
|
+
break;
|
|
1602
|
+
if (await pidManifestHasLiveCodexEntry(candidate.file))
|
|
1603
|
+
out.push(candidate.dir);
|
|
1604
|
+
}
|
|
1605
|
+
return uniquePaths(out);
|
|
1606
|
+
}
|
|
1607
|
+
async function findPidManifestFiles(root, maxDepth, maxFiles, resetAt) {
|
|
1608
|
+
const out = [];
|
|
1609
|
+
const skipDirs = new Set(['.git', 'node_modules', 'dist', 'coverage', 'evidence', 'streams', 'patch-scores', 'apply-ledger', 'artifact-index']);
|
|
1610
|
+
async function walk(current, depth) {
|
|
1611
|
+
if (out.length >= maxFiles || depth > maxDepth)
|
|
1612
|
+
return;
|
|
1613
|
+
const entries = await fs.readdir(current, { withFileTypes: true }).catch(() => []);
|
|
1614
|
+
for (const entry of entries) {
|
|
1615
|
+
if (out.length >= maxFiles)
|
|
1616
|
+
return;
|
|
1617
|
+
const absolute = path.join(current, entry.name);
|
|
1618
|
+
if (entry.isDirectory()) {
|
|
1619
|
+
if (skipDirs.has(entry.name) || entry.name.startsWith('.'))
|
|
1620
|
+
continue;
|
|
1621
|
+
if (resetAt && depth === 0) {
|
|
1622
|
+
const dirStat = await fs.stat(absolute).catch(() => undefined);
|
|
1623
|
+
if ((dirStat?.mtimeMs ?? 0) <= resetAt)
|
|
1624
|
+
continue;
|
|
1625
|
+
}
|
|
1626
|
+
await walk(absolute, depth + 1);
|
|
1627
|
+
}
|
|
1628
|
+
else if (entry.isFile() && entry.name === 'pids.json') {
|
|
1629
|
+
out.push(absolute);
|
|
1630
|
+
}
|
|
1631
|
+
}
|
|
1632
|
+
}
|
|
1633
|
+
await walk(root, 0);
|
|
1634
|
+
return out;
|
|
1635
|
+
}
|
|
1636
|
+
async function pidManifestHasLiveCodexEntry(file) {
|
|
1637
|
+
const manifest = recordValue(await readJsonFile(file));
|
|
1638
|
+
return recordArray(manifest.entries)
|
|
1639
|
+
.filter((entry) => textValue(entry.role, '') === 'codex')
|
|
1640
|
+
.some((entry) => isProcessLive(numberValue(entry.pid), entry));
|
|
1641
|
+
}
|
|
498
1642
|
async function readCoordinatorReviewDecisions(cwd) {
|
|
499
1643
|
const file = coordinatorReviewDecisionPath(cwd);
|
|
500
1644
|
const raw = await readJsonFile(file);
|
|
@@ -506,6 +1650,127 @@ async function readCoordinatorReviewDecisions(cwd) {
|
|
|
506
1650
|
function coordinatorReviewDecisionPath(cwd) {
|
|
507
1651
|
return path.join(cwd, 'agent-runs', REVIEW_DECISIONS_FILE);
|
|
508
1652
|
}
|
|
1653
|
+
async function readAutonomousMergeDecisions(cwd) {
|
|
1654
|
+
const root = path.join(cwd, 'agent-runs');
|
|
1655
|
+
const stat = await fs.stat(root).catch(() => undefined);
|
|
1656
|
+
if (!stat?.isDirectory())
|
|
1657
|
+
return [];
|
|
1658
|
+
const files = await findAutonomousMergeDecisionFiles(root);
|
|
1659
|
+
const out = [];
|
|
1660
|
+
for (const file of files) {
|
|
1661
|
+
const text = await fs.readFile(file, 'utf8').catch(() => '');
|
|
1662
|
+
for (const line of text.split(/\r?\n/u)) {
|
|
1663
|
+
const trimmed = line.trim();
|
|
1664
|
+
if (!trimmed)
|
|
1665
|
+
continue;
|
|
1666
|
+
const raw = safeJsonObject(trimmed);
|
|
1667
|
+
if (!raw)
|
|
1668
|
+
continue;
|
|
1669
|
+
const normalizedDecision = normalizeAutonomousMergeDecision(cwd, file, raw);
|
|
1670
|
+
if (normalizedDecision)
|
|
1671
|
+
out.push(normalizedDecision);
|
|
1672
|
+
}
|
|
1673
|
+
}
|
|
1674
|
+
return out.sort(compareCoordinatorReviewDecisionRecency);
|
|
1675
|
+
}
|
|
1676
|
+
async function findAutonomousMergeDecisionFiles(root) {
|
|
1677
|
+
const out = [];
|
|
1678
|
+
async function walk(current, depth) {
|
|
1679
|
+
if (out.length >= LIFETIME_DASHBOARD_MAX_AUTONOMOUS_DECISION_FILES || depth > LIFETIME_DASHBOARD_SCAN_MAX_DEPTH)
|
|
1680
|
+
return;
|
|
1681
|
+
const entries = await fs.readdir(current, { withFileTypes: true }).catch(() => []);
|
|
1682
|
+
for (const entry of entries) {
|
|
1683
|
+
if (out.length >= LIFETIME_DASHBOARD_MAX_AUTONOMOUS_DECISION_FILES)
|
|
1684
|
+
return;
|
|
1685
|
+
const absolute = path.join(current, entry.name);
|
|
1686
|
+
if (entry.isDirectory()) {
|
|
1687
|
+
if (entry.name === 'node_modules' || entry.name === '.git' || entry.name === 'artifact-store' || entry.name.startsWith('.'))
|
|
1688
|
+
continue;
|
|
1689
|
+
await walk(absolute, depth + 1);
|
|
1690
|
+
}
|
|
1691
|
+
else if (entry.isFile() && entry.name === 'autonomous-merge-decisions.jsonl') {
|
|
1692
|
+
const stat = await fs.stat(absolute).catch(() => undefined);
|
|
1693
|
+
out.push({ file: absolute, mtimeMs: stat?.mtimeMs ?? 0 });
|
|
1694
|
+
}
|
|
1695
|
+
}
|
|
1696
|
+
}
|
|
1697
|
+
await walk(root, 0);
|
|
1698
|
+
return out.sort((left, right) => right.mtimeMs - left.mtimeMs || right.file.localeCompare(left.file)).map((entry) => entry.file);
|
|
1699
|
+
}
|
|
1700
|
+
function normalizeAutonomousMergeDecision(cwd, file, raw) {
|
|
1701
|
+
const status = autonomousDecisionStatus(textValue(raw.status ?? raw.decision, ''));
|
|
1702
|
+
if (!status)
|
|
1703
|
+
return undefined;
|
|
1704
|
+
const ids = Array.from(new Set([
|
|
1705
|
+
textValue(raw.jobId, ''),
|
|
1706
|
+
textValue(raw.taskId, ''),
|
|
1707
|
+
...stringArray(raw.queueItemIds),
|
|
1708
|
+
...stringArray(raw.queueKeys).map((key) => key.replace(/^(?:queue|task|job):/u, ''))
|
|
1709
|
+
].filter(Boolean)));
|
|
1710
|
+
if (!ids.length)
|
|
1711
|
+
return undefined;
|
|
1712
|
+
const decidedAtMs = Math.max(numberValue(raw.finishedAt), numberValue(raw.startedAt), Date.parse(textValue(raw.finishedAtIso ?? raw.decidedAt ?? raw.generatedAt, '')) || 0);
|
|
1713
|
+
const relativeFile = path.relative(cwd, file);
|
|
1714
|
+
return {
|
|
1715
|
+
id: textValue(raw.id, ids[0]),
|
|
1716
|
+
jobId: textValue(raw.jobId, ''),
|
|
1717
|
+
taskId: textValue(raw.taskId, ''),
|
|
1718
|
+
matchIds: ids,
|
|
1719
|
+
status,
|
|
1720
|
+
decision: status,
|
|
1721
|
+
reason: textValue(raw.reason, ''),
|
|
1722
|
+
decidedAt: decidedAtMs ? new Date(decidedAtMs).toISOString() : '',
|
|
1723
|
+
decidedAtMs,
|
|
1724
|
+
sourceArtifact: relativeFile,
|
|
1725
|
+
sourceKind: 'autonomous-merge-decision',
|
|
1726
|
+
latestPath: textValue(raw.bundlePath ?? raw.patchPath, relativeFile),
|
|
1727
|
+
autonomousDecision: raw
|
|
1728
|
+
};
|
|
1729
|
+
}
|
|
1730
|
+
function autonomousDecisionStatus(value) {
|
|
1731
|
+
const status = normalized(value);
|
|
1732
|
+
if (status === 'committed' || status === 'applied')
|
|
1733
|
+
return status;
|
|
1734
|
+
if (status === 'accepted' || status === 'accepted-applied')
|
|
1735
|
+
return 'applied';
|
|
1736
|
+
if (status === 'rejected' || status === 'rerun' || status === 'superseded')
|
|
1737
|
+
return status;
|
|
1738
|
+
if (status === 'conflict' || status === 'conflict-blocked')
|
|
1739
|
+
return 'conflict-blocked';
|
|
1740
|
+
if (status === 'human-blocked' || status === 'human-question')
|
|
1741
|
+
return 'human-blocked';
|
|
1742
|
+
return status;
|
|
1743
|
+
}
|
|
1744
|
+
function mergeReviewDecisionLists(...groups) {
|
|
1745
|
+
return groups.flat().sort(compareCoordinatorReviewDecisionRecency);
|
|
1746
|
+
}
|
|
1747
|
+
function compareCoordinatorReviewDecisionRecency(left, right) {
|
|
1748
|
+
return decisionTime(right) - decisionTime(left)
|
|
1749
|
+
|| textValue(right.sourceArtifact ?? right.latestPath, '').localeCompare(textValue(left.sourceArtifact ?? left.latestPath, ''));
|
|
1750
|
+
}
|
|
1751
|
+
function decisionTime(decision) {
|
|
1752
|
+
return numberValue(decision.decidedAtMs)
|
|
1753
|
+
|| numberValue(decision.finishedAt)
|
|
1754
|
+
|| numberValue(decision.startedAt)
|
|
1755
|
+
|| Date.parse(textValue(decision.decidedAt, ''))
|
|
1756
|
+
|| 0;
|
|
1757
|
+
}
|
|
1758
|
+
function autonomousDecisionSourceSummary(decisions) {
|
|
1759
|
+
const files = Array.from(new Set(decisions.map((decision) => textValue(decision.sourceArtifact, '')).filter(Boolean)));
|
|
1760
|
+
return {
|
|
1761
|
+
count: decisions.length,
|
|
1762
|
+
fileCount: files.length,
|
|
1763
|
+
files: files.slice(0, 20)
|
|
1764
|
+
};
|
|
1765
|
+
}
|
|
1766
|
+
function safeJsonObject(value) {
|
|
1767
|
+
try {
|
|
1768
|
+
return recordValue(JSON.parse(value));
|
|
1769
|
+
}
|
|
1770
|
+
catch {
|
|
1771
|
+
return undefined;
|
|
1772
|
+
}
|
|
1773
|
+
}
|
|
509
1774
|
function applyCoordinatorReviewDecisions(jobs, decisions) {
|
|
510
1775
|
const records = jobs
|
|
511
1776
|
.map(recordValue)
|
|
@@ -519,25 +1784,103 @@ function applyCoordinatorReviewDecisions(jobs, decisions) {
|
|
|
519
1784
|
return record;
|
|
520
1785
|
const status = textValue(decision.status ?? decision.decision, 'resolved');
|
|
521
1786
|
const resolved = isResolvedCoordinatorDecision(status);
|
|
522
|
-
|
|
1787
|
+
const decided = {
|
|
523
1788
|
...record,
|
|
524
1789
|
coordinatorDecision: decision,
|
|
525
1790
|
coordinatorDecisionStatus: status,
|
|
526
1791
|
coordinatorDecisionAt: textValue(decision.decidedAt, ''),
|
|
527
1792
|
reviewResolved: resolved,
|
|
528
|
-
...(resolved && isCoordinatorPortBucket(record.bucket) ? { bucket: 'review-resolved' } : {}),
|
|
529
1793
|
...(resolved ? { disposition: status } : {})
|
|
530
1794
|
};
|
|
1795
|
+
return resolved ? markCoordinatorReviewResolved(decided, status) : decided;
|
|
531
1796
|
});
|
|
532
1797
|
}
|
|
533
1798
|
function normalizeCoordinatorFacingJob(record) {
|
|
534
|
-
|
|
1799
|
+
const status = coordinatorFacingMachineLabel(record.status);
|
|
1800
|
+
let bucket = coordinatorFacingMachineLabel(record.bucket);
|
|
1801
|
+
if (!bucket && status === 'completed')
|
|
1802
|
+
bucket = 'completed';
|
|
1803
|
+
else if (!bucket && status === 'running')
|
|
1804
|
+
bucket = 'running';
|
|
1805
|
+
else if (!bucket && status === 'failed')
|
|
1806
|
+
bucket = 'failed-evidence';
|
|
1807
|
+
else if (!bucket && status === 'blocked')
|
|
1808
|
+
bucket = 'blocked';
|
|
1809
|
+
let normalizedRecord = {
|
|
535
1810
|
...record,
|
|
536
|
-
bucket
|
|
537
|
-
status
|
|
1811
|
+
bucket,
|
|
1812
|
+
status,
|
|
538
1813
|
disposition: coordinatorFacingMachineLabel(record.disposition),
|
|
539
1814
|
mergeReadiness: coordinatorFacingMachineLabel(record.mergeReadiness)
|
|
540
1815
|
};
|
|
1816
|
+
normalizedRecord = normalizeHistoricalEvidenceFailureJob(normalizedRecord);
|
|
1817
|
+
if (!isResolvedCoordinatorReviewRecord(normalizedRecord))
|
|
1818
|
+
return normalizedRecord;
|
|
1819
|
+
return markCoordinatorReviewResolved(normalizedRecord, textValue(normalizedRecord.coordinatorDecisionStatus ?? normalizedRecord.disposition, 'review-resolved'));
|
|
1820
|
+
}
|
|
1821
|
+
function normalizeHistoricalEvidenceFailureJob(record) {
|
|
1822
|
+
if (!isHistoricalOwnershipRescopeCandidate(record))
|
|
1823
|
+
return record;
|
|
1824
|
+
return {
|
|
1825
|
+
...record,
|
|
1826
|
+
reviewResolved: false,
|
|
1827
|
+
originalBucket: record.originalBucket ?? record.bucket,
|
|
1828
|
+
originalStatus: record.originalStatus ?? record.status,
|
|
1829
|
+
originalDisposition: record.originalDisposition ?? record.disposition,
|
|
1830
|
+
bucket: 'rerun-work',
|
|
1831
|
+
status: 'completed',
|
|
1832
|
+
disposition: 'needs-rerun',
|
|
1833
|
+
mergeReadiness: 'needs-rerun',
|
|
1834
|
+
evidenceFailureNormalized: true,
|
|
1835
|
+
collectReasonClasses: uniquePaths([...stringArray(record.collectReasonClasses), 'ownership-rescope-rerun'])
|
|
1836
|
+
};
|
|
1837
|
+
}
|
|
1838
|
+
function isHistoricalOwnershipRescopeCandidate(record) {
|
|
1839
|
+
const bucket = normalized(record.bucket);
|
|
1840
|
+
const status = normalized(record.status);
|
|
1841
|
+
const disposition = normalized(record.disposition);
|
|
1842
|
+
const readiness = normalized(record.mergeReadiness);
|
|
1843
|
+
const failedEvidence = bucket === 'failed-evidence'
|
|
1844
|
+
|| bucket === 'worker-failed'
|
|
1845
|
+
|| status === 'failed'
|
|
1846
|
+
|| disposition === 'rejected'
|
|
1847
|
+
|| disposition === 'failed'
|
|
1848
|
+
|| readiness === 'rejected'
|
|
1849
|
+
|| readiness === 'blocked';
|
|
1850
|
+
if (!failedEvidence)
|
|
1851
|
+
return false;
|
|
1852
|
+
const ownershipViolationCount = numberValue(record.ownershipViolationCount)
|
|
1853
|
+
|| stringArray(record.ownershipViolations).length;
|
|
1854
|
+
if (!ownershipViolationCount)
|
|
1855
|
+
return false;
|
|
1856
|
+
const changedPathCount = numberValue(record.changedPathCount)
|
|
1857
|
+
|| stringArray(record.changedPaths).length;
|
|
1858
|
+
if (!changedPathCount)
|
|
1859
|
+
return false;
|
|
1860
|
+
return Boolean(textValue(record.patchPath, '') || textValue(record.patchArtifactPath, '') || changedPathCount);
|
|
1861
|
+
}
|
|
1862
|
+
function markCoordinatorReviewResolved(record, disposition) {
|
|
1863
|
+
const existingOriginalReasons = stringArray(record.originalReasons);
|
|
1864
|
+
const originalReasons = existingOriginalReasons.length ? existingOriginalReasons : stringArray(record.reasons);
|
|
1865
|
+
const retainedReasons = originalReasons.filter((reason) => !isOpenCoordinatorReviewReason(reason));
|
|
1866
|
+
const decisionReason = textValue(recordValue(record.coordinatorDecision).reason, '');
|
|
1867
|
+
const reasons = uniquePaths([
|
|
1868
|
+
...retainedReasons,
|
|
1869
|
+
...(decisionReason ? [decisionReason] : [])
|
|
1870
|
+
]);
|
|
1871
|
+
return {
|
|
1872
|
+
...record,
|
|
1873
|
+
reviewResolved: true,
|
|
1874
|
+
originalBucket: record.originalBucket ?? record.bucket,
|
|
1875
|
+
originalStatus: record.originalStatus ?? record.status,
|
|
1876
|
+
...(originalReasons.length ? { originalReasons } : {}),
|
|
1877
|
+
bucket: 'review-resolved',
|
|
1878
|
+
status: 'completed',
|
|
1879
|
+
disposition: disposition || 'review-resolved',
|
|
1880
|
+
mergeReadiness: 'review-resolved',
|
|
1881
|
+
reasons,
|
|
1882
|
+
health: ['failed', 'warning'].includes(normalized(record.health)) ? 'resolved' : record.health
|
|
1883
|
+
};
|
|
541
1884
|
}
|
|
542
1885
|
function normalizeCoordinatorFacingSnapshot(record) {
|
|
543
1886
|
return {
|
|
@@ -583,6 +1926,14 @@ function isCoordinatorPortBucket(value) {
|
|
|
583
1926
|
|| bucket === 'needs-coordinator-review'
|
|
584
1927
|
|| bucket === 'needs-coordinator-decision';
|
|
585
1928
|
}
|
|
1929
|
+
function isOpenCoordinatorReviewReason(value) {
|
|
1930
|
+
const reason = normalized(value);
|
|
1931
|
+
return isCoordinatorPortBucket(reason)
|
|
1932
|
+
|| reason === 'needs-port'
|
|
1933
|
+
|| reason === 'needs-review'
|
|
1934
|
+
|| reason === 'manual-port-required'
|
|
1935
|
+
|| reason === 'manual port required';
|
|
1936
|
+
}
|
|
586
1937
|
function coordinatorFacingMachineKey(value) {
|
|
587
1938
|
return textValue(coordinatorFacingMachineLabel(value), value);
|
|
588
1939
|
}
|
|
@@ -616,6 +1967,8 @@ function coordinatorReviewDecisionMatches(job, decision) {
|
|
|
616
1967
|
const decisionSource = textValue(decision.source ?? decision.sourceCollection ?? decision.sourceRun ?? decision.sourceLabel, '');
|
|
617
1968
|
if (!decisionSource)
|
|
618
1969
|
return true;
|
|
1970
|
+
if (isHistoricalReviewDrainDecision(decision))
|
|
1971
|
+
return historicalReviewDrainDecisionMatches(job, decision);
|
|
619
1972
|
const jobSources = [
|
|
620
1973
|
textValue(job.sourceLabel, ''),
|
|
621
1974
|
textValue(job.sourceCollection, ''),
|
|
@@ -624,6 +1977,46 @@ function coordinatorReviewDecisionMatches(job, decision) {
|
|
|
624
1977
|
].filter(Boolean);
|
|
625
1978
|
return jobSources.some((source) => source === decisionSource || source.endsWith(decisionSource) || decisionSource.endsWith(source));
|
|
626
1979
|
}
|
|
1980
|
+
function isHistoricalReviewDrainDecision(decision) {
|
|
1981
|
+
const sources = [
|
|
1982
|
+
textValue(decision.source, ''),
|
|
1983
|
+
textValue(decision.sourceCollection, ''),
|
|
1984
|
+
textValue(decision.sourceRun, ''),
|
|
1985
|
+
textValue(decision.sourceLabel, ''),
|
|
1986
|
+
textValue(decision.sourceArtifact, '')
|
|
1987
|
+
];
|
|
1988
|
+
return sources.some((source) => normalized(source).includes('historical-review-drain'));
|
|
1989
|
+
}
|
|
1990
|
+
function historicalReviewDrainDecisionMatches(job, decision) {
|
|
1991
|
+
if (normalized(job.status) === 'running' || normalized(job.bucket) === 'running')
|
|
1992
|
+
return false;
|
|
1993
|
+
const latestPath = textValue(decision.latestPath, '');
|
|
1994
|
+
if (!latestPath)
|
|
1995
|
+
return false;
|
|
1996
|
+
const latestRoot = historicalReviewDrainSourceRoot(latestPath);
|
|
1997
|
+
if (!latestRoot)
|
|
1998
|
+
return false;
|
|
1999
|
+
const jobSources = [
|
|
2000
|
+
textValue(job.sourceLabel, ''),
|
|
2001
|
+
textValue(job.sourceCollection, ''),
|
|
2002
|
+
textValue(job.sourceRun, ''),
|
|
2003
|
+
textValue(job.sourceContinuation, '')
|
|
2004
|
+
].filter(Boolean);
|
|
2005
|
+
return jobSources.some((source) => {
|
|
2006
|
+
const jobRoot = historicalReviewDrainSourceRoot(source);
|
|
2007
|
+
return Boolean(jobRoot) && (jobRoot === latestRoot || jobRoot.endsWith(latestRoot) || latestRoot.endsWith(jobRoot));
|
|
2008
|
+
});
|
|
2009
|
+
}
|
|
2010
|
+
function historicalReviewDrainSourceRoot(value) {
|
|
2011
|
+
const normalizedPath = value.replaceAll('\\', '/').replace(/\/(?:queue-overlay|collection|coordinator-query|swarm-results|coordinator-dashboard)\.json$/u, '');
|
|
2012
|
+
const autoDrainIndex = normalizedPath.indexOf('/auto-drain/');
|
|
2013
|
+
if (autoDrainIndex >= 0)
|
|
2014
|
+
return normalizedPath.slice(0, autoDrainIndex);
|
|
2015
|
+
const collectionIndex = normalizedPath.search(/\/(?:collection|collected|post-coordinator-collected|coordinator-collected)[^/]*(?:\/|$)/u);
|
|
2016
|
+
if (collectionIndex >= 0)
|
|
2017
|
+
return normalizedPath.slice(0, collectionIndex);
|
|
2018
|
+
return normalizedPath.replace(/\/$/u, '');
|
|
2019
|
+
}
|
|
627
2020
|
function coordinatorDecisionIds(record) {
|
|
628
2021
|
return Array.from(new Set([
|
|
629
2022
|
textValue(record.id, ''),
|
|
@@ -675,10 +2068,27 @@ function lifetimeDashboardSummary(jobs) {
|
|
|
675
2068
|
averageDurationMs: jobs.length ? Math.round(durationMs / jobs.length) : 0,
|
|
676
2069
|
maxDurationMs: jobs.reduce((max, job) => Math.max(max, numberValue(job.durationMs)), 0),
|
|
677
2070
|
actualInputTokens: jobs.reduce((sum, job) => sum + numberValue(job.actualInputTokens), 0),
|
|
2071
|
+
estimatedInputTokens: jobs.reduce((sum, job) => sum + numberValue(job.estimatedInputTokens), 0),
|
|
678
2072
|
cachedInputTokens: jobs.reduce((sum, job) => sum + numberValue(job.cachedInputTokens), 0),
|
|
679
|
-
uncachedInputTokens: jobs.reduce((sum, job) => sum + numberValue(job.uncachedInputTokens), 0)
|
|
2073
|
+
uncachedInputTokens: jobs.reduce((sum, job) => sum + numberValue(job.uncachedInputTokens), 0),
|
|
2074
|
+
estimatedCostUsd: roundUsd(jobs.reduce((sum, job) => sum + numberValue(job.estimatedCostUsd), 0)),
|
|
2075
|
+
estimatedInputCostUsd: roundUsd(jobs.reduce((sum, job) => sum + numberValue(job.estimatedInputCostUsd), 0)),
|
|
2076
|
+
estimatedCachedInputCostUsd: roundUsd(jobs.reduce((sum, job) => sum + numberValue(job.estimatedCachedInputCostUsd), 0)),
|
|
2077
|
+
estimatedUncachedInputCostUsd: roundUsd(jobs.reduce((sum, job) => sum + numberValue(job.estimatedUncachedInputCostUsd), 0)),
|
|
2078
|
+
estimatedOutputCostUsd: roundUsd(jobs.reduce((sum, job) => sum + numberValue(job.estimatedOutputCostUsd), 0)),
|
|
2079
|
+
estimatedCostMicroUsd: jobs.reduce((sum, job) => sum + numberValue(job.estimatedCostMicroUsd), 0),
|
|
2080
|
+
priceKnownJobCount: jobs.filter((job) => job.priceKnown === true).length,
|
|
2081
|
+
bucketCounts: countJobsByBucket(jobs)
|
|
680
2082
|
};
|
|
681
2083
|
}
|
|
2084
|
+
function countJobsByBucket(jobs) {
|
|
2085
|
+
const counts = {};
|
|
2086
|
+
for (const job of jobs) {
|
|
2087
|
+
const bucket = textValue(job.bucket, 'unknown') || 'unknown';
|
|
2088
|
+
counts[bucket] = (counts[bucket] ?? 0) + 1;
|
|
2089
|
+
}
|
|
2090
|
+
return counts;
|
|
2091
|
+
}
|
|
682
2092
|
function lifetimeHealthSummary(jobs) {
|
|
683
2093
|
const summary = lifetimeDashboardSummary(jobs);
|
|
684
2094
|
const failedJobCount = numberValue(summary.failedCount);
|
|
@@ -719,6 +2129,144 @@ function lifetimeLaneRows(jobs) {
|
|
|
719
2129
|
runningCount: entries.filter((job) => textValue(job.status, '') === 'running').length
|
|
720
2130
|
}));
|
|
721
2131
|
}
|
|
2132
|
+
function lifetimeQueueBacklogOverlay(queueBacklog, jobs) {
|
|
2133
|
+
const jobsByKey = new Map();
|
|
2134
|
+
for (const job of jobs) {
|
|
2135
|
+
for (const key of recordIdentityKeys(job).map(canonicalLifetimeTaskKey).filter(Boolean)) {
|
|
2136
|
+
jobsByKey.set(key, [...(jobsByKey.get(key) ?? []), job]);
|
|
2137
|
+
}
|
|
2138
|
+
}
|
|
2139
|
+
let activeCount = 0;
|
|
2140
|
+
let doneCount = 0;
|
|
2141
|
+
let failedCount = 0;
|
|
2142
|
+
let representedCount = 0;
|
|
2143
|
+
const entries = [];
|
|
2144
|
+
for (const entry of queueBacklog.entries) {
|
|
2145
|
+
const matchedJobs = Array.from(new Set(recordIdentityKeys(entry).map(canonicalLifetimeTaskKey).flatMap((key) => jobsByKey.get(key) ?? [])));
|
|
2146
|
+
if (!matchedJobs.length) {
|
|
2147
|
+
entries.push(entry);
|
|
2148
|
+
continue;
|
|
2149
|
+
}
|
|
2150
|
+
representedCount += 1;
|
|
2151
|
+
if (matchedJobs.some((job) => textValue(job.status, '') === 'running'))
|
|
2152
|
+
activeCount += 1;
|
|
2153
|
+
else if (matchedJobs.some(isLifetimeFailedJob))
|
|
2154
|
+
failedCount += 1;
|
|
2155
|
+
else if (matchedJobs.some((job) => textValue(job.status, '') === 'completed' || isResolvedCoordinatorReviewRecord(job)))
|
|
2156
|
+
doneCount += 1;
|
|
2157
|
+
else
|
|
2158
|
+
activeCount += 1;
|
|
2159
|
+
}
|
|
2160
|
+
return {
|
|
2161
|
+
entries,
|
|
2162
|
+
totalCount: queueBacklog.entries.length,
|
|
2163
|
+
readyCount: entries.filter((entry) => textValue(entry.status, '') === 'todo').length,
|
|
2164
|
+
activeCount,
|
|
2165
|
+
doneCount,
|
|
2166
|
+
failedCount,
|
|
2167
|
+
representedCount
|
|
2168
|
+
};
|
|
2169
|
+
}
|
|
2170
|
+
function lifetimeCapacitySummary(queueBacklog, jobs, openQueueEntries = queueBacklog.entries) {
|
|
2171
|
+
const manifest = queueBacklog.manifests[0];
|
|
2172
|
+
const laneRows = new Map();
|
|
2173
|
+
const terminalTaskIds = new Set(jobs
|
|
2174
|
+
.filter((job) => ['completed', 'failed', 'blocked'].includes(textValue(job.status, '').toLowerCase()))
|
|
2175
|
+
.flatMap((job) => recordIdentityKeys(job).map(canonicalLifetimeTaskKey)));
|
|
2176
|
+
const representedTaskIds = new Set(jobs.flatMap((job) => recordIdentityKeys(job).map(canonicalLifetimeTaskKey)));
|
|
2177
|
+
const openEntries = openQueueEntries.filter((entry) => {
|
|
2178
|
+
const ids = recordIdentityKeys(entry);
|
|
2179
|
+
return !ids.some((id) => terminalTaskIds.has(canonicalLifetimeTaskKey(id)) || representedTaskIds.has(canonicalLifetimeTaskKey(id)));
|
|
2180
|
+
});
|
|
2181
|
+
const queuedByLane = groupRecordsByText(openEntries, (entry) => textValue(entry.lane ?? entry.group ?? entry.sourceQueue, 'unassigned'));
|
|
2182
|
+
const jobsByLane = groupRecordsByText(jobs, (job) => textValue(job.lane, 'unassigned'));
|
|
2183
|
+
const manifestLanes = manifest?.lanes ?? [];
|
|
2184
|
+
for (const lane of manifestLanes) {
|
|
2185
|
+
laneRows.set(lane.id, capacityLaneRow(lane, queuedByLane.get(lane.id) ?? [], jobsByLane.get(lane.id) ?? []));
|
|
2186
|
+
}
|
|
2187
|
+
for (const [laneId, entries] of queuedByLane) {
|
|
2188
|
+
if (!laneRows.has(laneId))
|
|
2189
|
+
laneRows.set(laneId, capacityLaneRow({ id: laneId, title: laneId, layer: '', compute: '', model: '', maxConcurrency: 1 }, entries, jobsByLane.get(laneId) ?? []));
|
|
2190
|
+
}
|
|
2191
|
+
for (const [laneId, entries] of jobsByLane) {
|
|
2192
|
+
if (!laneRows.has(laneId))
|
|
2193
|
+
laneRows.set(laneId, capacityLaneRow({ id: laneId, title: laneId, layer: '', compute: '', model: '', maxConcurrency: 1 }, queuedByLane.get(laneId) ?? [], entries));
|
|
2194
|
+
}
|
|
2195
|
+
const lanes = Array.from(laneRows.values()).sort((left, right) => {
|
|
2196
|
+
const pressure = numberValue(right.runningCount) - numberValue(left.runningCount)
|
|
2197
|
+
|| numberValue(right.queuedTaskCount) - numberValue(left.queuedTaskCount);
|
|
2198
|
+
return pressure || textValue(left.id, '').localeCompare(textValue(right.id, ''));
|
|
2199
|
+
});
|
|
2200
|
+
const runningAgentCount = jobs.filter((job) => textValue(job.status, '') === 'running').length;
|
|
2201
|
+
const queuedTaskCount = lanes.reduce((sum, lane) => sum + numberValue(lane.queuedTaskCount), 0);
|
|
2202
|
+
return {
|
|
2203
|
+
manifestPath: manifest?.path ?? '',
|
|
2204
|
+
manifestId: manifest?.id ?? '',
|
|
2205
|
+
title: manifest?.title ?? 'Swarm capacity',
|
|
2206
|
+
defaultConcurrency: manifest?.defaultConcurrency ?? 0,
|
|
2207
|
+
computeMaxConcurrency: manifest?.computeMaxConcurrency ?? 0,
|
|
2208
|
+
maxConcurrency: manifest?.maxConcurrency ?? 0,
|
|
2209
|
+
laneCount: lanes.length,
|
|
2210
|
+
openLaneCount: lanes.filter((lane) => numberValue(lane.queuedTaskCount) > 0 || numberValue(lane.runningCount) > 0).length,
|
|
2211
|
+
activeLaneCount: lanes.filter((lane) => numberValue(lane.runningCount) > 0).length,
|
|
2212
|
+
runningAgentCount,
|
|
2213
|
+
assignedAgentCount: lanes.reduce((sum, lane) => sum + numberValue(lane.assignedAgentCount), 0),
|
|
2214
|
+
queuedTaskCount,
|
|
2215
|
+
totalTaskCount: queueBacklog.entries.length,
|
|
2216
|
+
completedTaskCount: jobs.filter((job) => textValue(job.status, '') === 'completed').length,
|
|
2217
|
+
lanes,
|
|
2218
|
+
queueSources: queueBacklog.paths
|
|
2219
|
+
};
|
|
2220
|
+
}
|
|
2221
|
+
function capacityLaneRow(lane, queuedEntries, laneJobs) {
|
|
2222
|
+
const runningJobs = laneJobs.filter((job) => textValue(job.status, '') === 'running');
|
|
2223
|
+
const queuedJobs = laneJobs.filter((job) => textValue(job.status, '') === 'queued');
|
|
2224
|
+
const assignedAgents = Array.from(new Set(runningJobs.map((job) => textValue(job.agentId ?? job.workerId ?? job.id, '')).filter(Boolean))).slice(0, 6);
|
|
2225
|
+
const queuedKeys = new Set([
|
|
2226
|
+
...queuedEntries.filter((entry) => {
|
|
2227
|
+
const status = normalized(entry.status ?? entry.queueStatus);
|
|
2228
|
+
return !status || ['todo', 'queued', 'pending', 'ready', 'open'].includes(status);
|
|
2229
|
+
}).map(primaryRecordIdentityKey).filter(Boolean),
|
|
2230
|
+
...queuedJobs.map(primaryRecordIdentityKey).filter(Boolean)
|
|
2231
|
+
]);
|
|
2232
|
+
return {
|
|
2233
|
+
id: lane.id,
|
|
2234
|
+
title: lane.title || lane.id,
|
|
2235
|
+
layer: lane.layer,
|
|
2236
|
+
compute: lane.compute,
|
|
2237
|
+
model: lane.model,
|
|
2238
|
+
maxConcurrency: lane.maxConcurrency,
|
|
2239
|
+
queuedTaskCount: queuedKeys.size,
|
|
2240
|
+
totalTaskCount: queuedEntries.length,
|
|
2241
|
+
runningCount: runningJobs.length,
|
|
2242
|
+
completedCount: laneJobs.filter((job) => textValue(job.status, '') === 'completed').length,
|
|
2243
|
+
failedCount: laneJobs.filter(isLifetimeFailedJob).length,
|
|
2244
|
+
assignedAgentCount: assignedAgents.length,
|
|
2245
|
+
assignedAgents
|
|
2246
|
+
};
|
|
2247
|
+
}
|
|
2248
|
+
function recordIdentityKeys(record) {
|
|
2249
|
+
return Array.from(new Set([
|
|
2250
|
+
textValue(record.id, ''),
|
|
2251
|
+
textValue(record.originalJobId, ''),
|
|
2252
|
+
textValue(record.jobId, ''),
|
|
2253
|
+
textValue(record.taskId, '')
|
|
2254
|
+
].filter(Boolean)));
|
|
2255
|
+
}
|
|
2256
|
+
function primaryRecordIdentityKey(record) {
|
|
2257
|
+
return canonicalLifetimeTaskKey(textValue(record.taskId, ''))
|
|
2258
|
+
|| canonicalLifetimeTaskKey(textValue(record.originalJobId, ''))
|
|
2259
|
+
|| canonicalLifetimeTaskKey(textValue(record.jobId, ''))
|
|
2260
|
+
|| canonicalLifetimeTaskKey(textValue(record.id, ''));
|
|
2261
|
+
}
|
|
2262
|
+
function groupRecordsByText(records, keyFor) {
|
|
2263
|
+
const groups = new Map();
|
|
2264
|
+
for (const record of records) {
|
|
2265
|
+
const key = keyFor(record) || 'unassigned';
|
|
2266
|
+
groups.set(key, [...(groups.get(key) ?? []), record]);
|
|
2267
|
+
}
|
|
2268
|
+
return groups;
|
|
2269
|
+
}
|
|
722
2270
|
function lifetimeTimeSeries(jobs, events) {
|
|
723
2271
|
const bucketMs = 24 * 60 * 60 * 1000;
|
|
724
2272
|
const buckets = new Map();
|
|
@@ -727,7 +2275,7 @@ function lifetimeTimeSeries(jobs, events) {
|
|
|
727
2275
|
if (!at)
|
|
728
2276
|
continue;
|
|
729
2277
|
const bucketAt = startOfLocalDay(at);
|
|
730
|
-
const bucket = buckets.get(bucketAt) ??
|
|
2278
|
+
const bucket = buckets.get(bucketAt) ?? emptyLifetimeTimeBucket(bucketAt);
|
|
731
2279
|
if (['completed', 'failed', 'blocked'].includes(textValue(job.status, '').toLowerCase()))
|
|
732
2280
|
bucket.terminalJobCount += 1;
|
|
733
2281
|
if (textValue(job.health, '') === 'warning')
|
|
@@ -736,7 +2284,10 @@ function lifetimeTimeSeries(jobs, events) {
|
|
|
736
2284
|
bucket.failureJobCount += 1;
|
|
737
2285
|
bucket.durationMs += numberValue(job.durationMs);
|
|
738
2286
|
bucket.actualInputTokens += numberValue(job.actualInputTokens);
|
|
2287
|
+
bucket.estimatedInputTokens += numberValue(job.estimatedInputTokens);
|
|
739
2288
|
bucket.uncachedInputTokens += numberValue(job.uncachedInputTokens);
|
|
2289
|
+
bucket.estimatedCostUsd = roundUsd(bucket.estimatedCostUsd + numberValue(job.estimatedCostUsd));
|
|
2290
|
+
bucket.estimatedCostMicroUsd += numberValue(job.estimatedCostMicroUsd);
|
|
740
2291
|
buckets.set(bucketAt, bucket);
|
|
741
2292
|
}
|
|
742
2293
|
for (const event of events) {
|
|
@@ -744,7 +2295,7 @@ function lifetimeTimeSeries(jobs, events) {
|
|
|
744
2295
|
if (!at)
|
|
745
2296
|
continue;
|
|
746
2297
|
const bucketAt = startOfLocalDay(at);
|
|
747
|
-
const bucket = buckets.get(bucketAt) ??
|
|
2298
|
+
const bucket = buckets.get(bucketAt) ?? emptyLifetimeTimeBucket(bucketAt);
|
|
748
2299
|
bucket.eventCount += 1;
|
|
749
2300
|
buckets.set(bucketAt, bucket);
|
|
750
2301
|
}
|
|
@@ -759,11 +2310,29 @@ function lifetimeTimeSeries(jobs, events) {
|
|
|
759
2310
|
failureJobCount: points.reduce((sum, point) => sum + point.failureJobCount, 0),
|
|
760
2311
|
durationMs: points.reduce((sum, point) => sum + point.durationMs, 0),
|
|
761
2312
|
actualInputTokens: points.reduce((sum, point) => sum + point.actualInputTokens, 0),
|
|
2313
|
+
estimatedInputTokens: points.reduce((sum, point) => sum + point.estimatedInputTokens, 0),
|
|
762
2314
|
uncachedInputTokens: points.reduce((sum, point) => sum + point.uncachedInputTokens, 0),
|
|
2315
|
+
estimatedCostUsd: roundUsd(points.reduce((sum, point) => sum + point.estimatedCostUsd, 0)),
|
|
2316
|
+
estimatedCostMicroUsd: points.reduce((sum, point) => sum + point.estimatedCostMicroUsd, 0),
|
|
763
2317
|
missingTimestampJobCount: jobs.filter((job) => !numberValue(job.finishedAt) && !numberValue(job.generatedAt) && !numberValue(job.startedAt)).length
|
|
764
2318
|
}
|
|
765
2319
|
};
|
|
766
2320
|
}
|
|
2321
|
+
function emptyLifetimeTimeBucket(at) {
|
|
2322
|
+
return {
|
|
2323
|
+
at,
|
|
2324
|
+
terminalJobCount: 0,
|
|
2325
|
+
warningJobCount: 0,
|
|
2326
|
+
failureJobCount: 0,
|
|
2327
|
+
durationMs: 0,
|
|
2328
|
+
actualInputTokens: 0,
|
|
2329
|
+
estimatedInputTokens: 0,
|
|
2330
|
+
uncachedInputTokens: 0,
|
|
2331
|
+
estimatedCostUsd: 0,
|
|
2332
|
+
estimatedCostMicroUsd: 0,
|
|
2333
|
+
eventCount: 0
|
|
2334
|
+
};
|
|
2335
|
+
}
|
|
767
2336
|
function startOfLocalDay(value) {
|
|
768
2337
|
const date = new Date(value);
|
|
769
2338
|
return new Date(date.getFullYear(), date.getMonth(), date.getDate()).getTime();
|
|
@@ -777,28 +2346,92 @@ function lifetimeSemanticSummary(jobs) {
|
|
|
777
2346
|
conflicts: jobs.filter((job) => textValue(job.semanticReadiness, '') === 'blocked').length
|
|
778
2347
|
};
|
|
779
2348
|
}
|
|
780
|
-
function lifetimeRoutingSummary(
|
|
781
|
-
const routingRows =
|
|
782
|
-
|
|
2349
|
+
async function lifetimeRoutingSummary(cwd, entries) {
|
|
2350
|
+
const routingRows = entries.map(({ snapshot }) => recordValue(snapshot.routing)).filter((entry) => Object.keys(entry).length);
|
|
2351
|
+
const sidecars = await readLifetimeRoutingSidecars(cwd, entries.map(({ source }) => source));
|
|
2352
|
+
if (!routingRows.length && !sidecars.tournamentCount && !sidecars.feedbackArtifactCount && !sidecars.historyCount)
|
|
783
2353
|
return undefined;
|
|
784
2354
|
return {
|
|
785
2355
|
policyId: 'workspace-lifetime',
|
|
786
2356
|
preferenceCount: routingRows.reduce((sum, row) => sum + numberValue(row.preferenceCount), 0),
|
|
787
2357
|
preferCount: routingRows.reduce((sum, row) => sum + numberValue(row.preferCount), 0),
|
|
788
2358
|
avoidCount: routingRows.reduce((sum, row) => sum + numberValue(row.avoidCount), 0),
|
|
789
|
-
|
|
790
|
-
|
|
2359
|
+
feedbackCount: routingRows.reduce((sum, row) => sum + numberValue(row.feedbackCount), 0) + sidecars.feedbackSignalCount,
|
|
2360
|
+
tournamentObservationCount: routingRows.reduce((sum, row) => sum + numberValue(row.tournamentObservationCount), 0) + sidecars.tournamentObservationCount,
|
|
2361
|
+
tournamentRecommendationCount: routingRows.reduce((sum, row) => sum + numberValue(row.tournamentRecommendationCount), 0) + sidecars.tournamentRecommendationCount,
|
|
2362
|
+
tournamentCount: sidecars.tournamentCount,
|
|
2363
|
+
tournamentMatchCount: sidecars.tournamentMatchCount,
|
|
2364
|
+
tournamentVerifiedCount: sidecars.tournamentVerifiedCount,
|
|
2365
|
+
tournamentTopStrategyId: sidecars.topStrategyId,
|
|
2366
|
+
tournamentDecisionGrade: sidecars.decisionGrade,
|
|
2367
|
+
strategyHistoryCount: sidecars.historyCount,
|
|
2368
|
+
feedbackArtifactCount: sidecars.feedbackArtifactCount
|
|
2369
|
+
};
|
|
2370
|
+
}
|
|
2371
|
+
async function readLifetimeRoutingSidecars(cwd, sources) {
|
|
2372
|
+
const out = {
|
|
2373
|
+
tournamentCount: 0,
|
|
2374
|
+
tournamentMatchCount: 0,
|
|
2375
|
+
tournamentVerifiedCount: 0,
|
|
2376
|
+
tournamentObservationCount: 0,
|
|
2377
|
+
tournamentRecommendationCount: 0,
|
|
2378
|
+
feedbackSignalCount: 0,
|
|
2379
|
+
feedbackArtifactCount: 0,
|
|
2380
|
+
historyCount: 0,
|
|
2381
|
+
topStrategyId: '',
|
|
2382
|
+
decisionGrade: ''
|
|
791
2383
|
};
|
|
2384
|
+
const seen = new Set();
|
|
2385
|
+
for (const source of sources) {
|
|
2386
|
+
const dir = lifetimeRoutingSidecarDir(cwd, source);
|
|
2387
|
+
if (!dir || seen.has(dir))
|
|
2388
|
+
continue;
|
|
2389
|
+
seen.add(dir);
|
|
2390
|
+
const tournament = recordValue(await readJsonFile(path.join(dir, 'strategy-tournament.json')));
|
|
2391
|
+
const tournamentSummary = recordValue(tournament.summary);
|
|
2392
|
+
if (Object.keys(tournament).length) {
|
|
2393
|
+
out.tournamentCount += 1;
|
|
2394
|
+
out.tournamentMatchCount += numberValue(tournamentSummary.matchCount);
|
|
2395
|
+
out.tournamentVerifiedCount += numberValue(tournamentSummary.verifiedCount);
|
|
2396
|
+
out.tournamentObservationCount += numberValue(tournamentSummary.matchCount) || recordArray(tournament.matches).length;
|
|
2397
|
+
out.tournamentRecommendationCount += textValue(tournament.winnerId ?? tournamentSummary.topStrategyId, '') ? 1 : 0;
|
|
2398
|
+
if (!out.topStrategyId)
|
|
2399
|
+
out.topStrategyId = textValue(tournamentSummary.topStrategyId ?? tournament.winnerId, '');
|
|
2400
|
+
if (!out.decisionGrade)
|
|
2401
|
+
out.decisionGrade = textValue(tournamentSummary.decisionGrade, '');
|
|
2402
|
+
}
|
|
2403
|
+
const feedback = recordValue(await readJsonFile(path.join(dir, 'tournament-adaptive-feedback.json')));
|
|
2404
|
+
const feedbackSummary = recordValue(feedback.summary);
|
|
2405
|
+
if (Object.keys(feedback).length) {
|
|
2406
|
+
out.feedbackArtifactCount += 1;
|
|
2407
|
+
out.tournamentObservationCount += numberValue(feedbackSummary.observationCount) || recordArray(feedback.observations).length;
|
|
2408
|
+
out.tournamentRecommendationCount += numberValue(feedbackSummary.recommendationCount) || recordArray(feedback.recommendations).length;
|
|
2409
|
+
out.feedbackSignalCount += numberValue(feedbackSummary.reduceSignals) + numberValue(feedbackSummary.increaseSignals) + numberValue(feedbackSummary.holdSignals);
|
|
2410
|
+
}
|
|
2411
|
+
const history = recordValue(await readJsonFile(path.join(dir, 'strategy-history.json')));
|
|
2412
|
+
if (Object.keys(history).length) {
|
|
2413
|
+
out.historyCount += recordArray(history.tournaments).length || 1;
|
|
2414
|
+
}
|
|
2415
|
+
}
|
|
2416
|
+
return out;
|
|
2417
|
+
}
|
|
2418
|
+
function lifetimeRoutingSidecarDir(cwd, source) {
|
|
2419
|
+
const relative = source.collection ?? source.continuation ?? source.run;
|
|
2420
|
+
if (!relative)
|
|
2421
|
+
return undefined;
|
|
2422
|
+
const absolute = path.resolve(cwd, relative);
|
|
2423
|
+
return isPathInside(cwd, absolute) ? absolute : undefined;
|
|
792
2424
|
}
|
|
793
2425
|
function isLifetimeFailedJob(job) {
|
|
2426
|
+
if (isResolvedCoordinatorReviewRecord(job))
|
|
2427
|
+
return false;
|
|
794
2428
|
const status = textValue(job.status, '').toLowerCase();
|
|
795
2429
|
const health = textValue(job.health, '').toLowerCase();
|
|
796
2430
|
const bucket = textValue(job.bucket, '').toLowerCase();
|
|
2431
|
+
if (bucket === 'rerun-work')
|
|
2432
|
+
return false;
|
|
797
2433
|
return status === 'failed' || health === 'failed' || bucket === 'failed-evidence';
|
|
798
2434
|
}
|
|
799
|
-
function awaitNoop(value) {
|
|
800
|
-
return value;
|
|
801
|
-
}
|
|
802
2435
|
function shouldPreferActiveRunSnapshot(jobs, activeJobs) {
|
|
803
2436
|
if (!activeJobs.length)
|
|
804
2437
|
return false;
|
|
@@ -807,24 +2440,27 @@ function shouldPreferActiveRunSnapshot(jobs, activeJobs) {
|
|
|
807
2440
|
return true;
|
|
808
2441
|
return activeJobs.length > jobs.length;
|
|
809
2442
|
}
|
|
810
|
-
async function readActiveRunSnapshot(options) {
|
|
2443
|
+
async function readActiveRunSnapshot(options, readOptions = {}) {
|
|
811
2444
|
const runDir = await resolveRunDirectory(options);
|
|
812
2445
|
if (!runDir)
|
|
813
2446
|
return undefined;
|
|
814
2447
|
const pidPath = path.join(runDir, 'pids.json');
|
|
815
2448
|
const pidManifest = recordValue(await readJsonFile(pidPath));
|
|
816
|
-
|
|
2449
|
+
let entries = recordArray(pidManifest.entries).filter((entry) => textValue(entry.role, '') === 'codex');
|
|
2450
|
+
if (readOptions.runningOnly)
|
|
2451
|
+
entries = entries.filter((entry) => isProcessLive(numberValue(entry.pid), entry));
|
|
817
2452
|
if (!entries.length)
|
|
818
2453
|
return undefined;
|
|
819
2454
|
const planPath = path.join(runDir, 'swarm-plan.json');
|
|
820
2455
|
const plan = recordValue(await readJsonFile(planPath));
|
|
821
2456
|
const planJobs = new Map(recordArray(plan.jobs).map((job) => [textValue(job.id, ''), job]));
|
|
822
2457
|
const now = Date.now();
|
|
823
|
-
const jobs = await Promise.all(entries.map((entry) => activeRunJob(runDir, entry, planJobs.get(textValue(entry.jobId, '')), now)));
|
|
2458
|
+
const jobs = await Promise.all(entries.map((entry) => activeRunJob(options.cwd, runDir, entry, planJobs.get(textValue(entry.jobId, '')), now, readOptions)));
|
|
824
2459
|
const runningCount = jobs.filter((job) => textValue(job.status, '') === 'running').length;
|
|
825
2460
|
const completedCount = jobs.filter((job) => textValue(job.status, '') === 'completed').length;
|
|
826
2461
|
const failedCount = jobs.filter((job) => textValue(job.status, '') === 'failed').length;
|
|
827
2462
|
const actualInputTokens = jobs.reduce((sum, job) => sum + numberValue(job.actualInputTokens), 0);
|
|
2463
|
+
const estimatedInputTokens = jobs.reduce((sum, job) => sum + numberValue(job.estimatedInputTokens), 0);
|
|
828
2464
|
const cachedInputTokens = jobs.reduce((sum, job) => sum + numberValue(job.cachedInputTokens), 0);
|
|
829
2465
|
return {
|
|
830
2466
|
ok: true,
|
|
@@ -837,6 +2473,7 @@ async function readActiveRunSnapshot(options) {
|
|
|
837
2473
|
runningCount,
|
|
838
2474
|
blockedCount: 0,
|
|
839
2475
|
actualInputTokens,
|
|
2476
|
+
estimatedInputTokens,
|
|
840
2477
|
cachedInputTokens,
|
|
841
2478
|
uncachedInputTokens: Math.max(0, actualInputTokens - cachedInputTokens),
|
|
842
2479
|
durationMs: jobs.reduce((sum, job) => Math.max(sum, numberValue(job.durationMs)), 0),
|
|
@@ -849,6 +2486,7 @@ async function readActiveRunSnapshot(options) {
|
|
|
849
2486
|
},
|
|
850
2487
|
lanes: activeRunLanes(jobs),
|
|
851
2488
|
jobs,
|
|
2489
|
+
activeAgents: activeAgentsFromJobs(jobs),
|
|
852
2490
|
events: activeRunEvents(jobs),
|
|
853
2491
|
sources: {
|
|
854
2492
|
run: runDir,
|
|
@@ -864,35 +2502,53 @@ async function readActiveRunSnapshot(options) {
|
|
|
864
2502
|
}
|
|
865
2503
|
};
|
|
866
2504
|
}
|
|
867
|
-
async function activeRunJob(runDir, entry, planJob, now) {
|
|
2505
|
+
async function activeRunJob(cwd, runDir, entry, planJob, now, readOptions = {}) {
|
|
868
2506
|
const jobId = textValue(entry.jobId, 'job');
|
|
869
2507
|
const jobDir = path.join(runDir, jobId);
|
|
870
2508
|
const lastMessagePath = path.join(jobDir, 'last-message.md');
|
|
871
2509
|
const mergePath = path.join(jobDir, 'merge.json');
|
|
2510
|
+
const eventsPath = path.join(jobDir, 'codex-events.jsonl');
|
|
2511
|
+
const evidenceRecord = recordValue(await readJsonFile(path.join(jobDir, 'evidence', 'evidence.json')));
|
|
872
2512
|
const lastMessage = await fs.stat(lastMessagePath).catch(() => undefined);
|
|
873
2513
|
const merge = recordValue(await readJsonFile(mergePath));
|
|
874
2514
|
const live = isProcessLive(numberValue(entry.pid), entry);
|
|
875
|
-
const
|
|
2515
|
+
const quotaDeferred = !live && !lastMessage && !Object.keys(merge).length && await codexEventsHaveQuotaLimit(eventsPath);
|
|
2516
|
+
const status = live && !lastMessage ? 'running' : quotaDeferred ? 'completed' : lastMessage || Object.keys(merge).length ? 'completed' : 'failed';
|
|
876
2517
|
const startedAt = numberValue(entry.startedAt);
|
|
877
2518
|
const finishedAt = status === 'running' ? undefined : Math.max(numberValue(lastMessage?.mtimeMs), numberValue(merge.generatedAt));
|
|
878
|
-
const
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
2519
|
+
const rawPatchPath = await firstExistingRelativePath(cwd, rawRunPatchCandidates(jobDir));
|
|
2520
|
+
const evidencePaths = await existingRelativePaths(cwd, [
|
|
2521
|
+
lastMessagePath,
|
|
2522
|
+
eventsPath,
|
|
2523
|
+
path.join(jobDir, 'evidence', 'last-message.md'),
|
|
2524
|
+
path.join(jobDir, 'evidence', 'handoff.md'),
|
|
2525
|
+
path.join(jobDir, 'evidence', 'evidence.json'),
|
|
2526
|
+
path.join(jobDir, 'evidence', 'resource-allocation.json'),
|
|
2527
|
+
...rawRunPatchCandidates(jobDir),
|
|
2528
|
+
...(Object.keys(merge).length ? [mergePath] : [])
|
|
2529
|
+
]);
|
|
2530
|
+
const usage = readOptions.includeUsage === false ? emptyCodexEventUsageSummary() : await readCodexEventUsageSummary(eventsPath);
|
|
885
2531
|
const task = recordValue(planJob?.task);
|
|
886
2532
|
const compute = recordValue(planJob?.compute);
|
|
887
|
-
const changedPaths =
|
|
888
|
-
|
|
2533
|
+
const changedPaths = uniquePaths([
|
|
2534
|
+
...stringArray(merge.changedPaths),
|
|
2535
|
+
...await readPatchChangedPathList(cwd, rawPatchPath)
|
|
2536
|
+
]);
|
|
2537
|
+
const commandEvidence = commandEvidenceFromRecords(merge, evidenceRecord);
|
|
2538
|
+
return withRecomputedCostFields({
|
|
889
2539
|
id: jobId,
|
|
890
2540
|
taskId: textValue(planJob?.taskId ?? task.id, jobId),
|
|
891
2541
|
title: textValue(planJob?.title ?? task.title, jobId),
|
|
892
2542
|
lane: textValue(planJob?.lane ?? task.lane, 'active-run'),
|
|
893
2543
|
status,
|
|
894
|
-
bucket: status === 'running' ? 'running' : status === 'completed' ? 'completed' : 'failed-evidence',
|
|
895
|
-
disposition: status === 'running' ? 'active' : status,
|
|
2544
|
+
bucket: quotaDeferred ? 'review-resolved' : status === 'running' ? 'running' : status === 'completed' ? 'completed' : 'failed-evidence',
|
|
2545
|
+
disposition: status === 'running' ? 'active' : quotaDeferred ? 'quota-deferred' : status,
|
|
2546
|
+
...(quotaDeferred ? {
|
|
2547
|
+
reviewResolved: true,
|
|
2548
|
+
coordinatorDecisionStatus: 'quota-deferred',
|
|
2549
|
+
originalStatus: 'queued',
|
|
2550
|
+
originalBucket: 'queued'
|
|
2551
|
+
} : {}),
|
|
896
2552
|
agentId: jobId,
|
|
897
2553
|
workerId: jobId,
|
|
898
2554
|
model: textValue(compute.model, ''),
|
|
@@ -902,6 +2558,7 @@ async function activeRunJob(runDir, entry, planJob, now) {
|
|
|
902
2558
|
...(finishedAt ? { finishedAt } : {}),
|
|
903
2559
|
durationMs: startedAt ? Math.max(0, (finishedAt ?? now) - startedAt) : 0,
|
|
904
2560
|
...(usage.inputTokens ? { actualInputTokens: usage.inputTokens, inputTokens: usage.inputTokens } : {}),
|
|
2561
|
+
...(!usage.inputTokens && usage.estimatedInputTokens ? { estimatedInputTokens: usage.estimatedInputTokens } : {}),
|
|
905
2562
|
...(usage.cachedInputTokens ? { cachedInputTokens: usage.cachedInputTokens } : {}),
|
|
906
2563
|
...(usage.uncachedInputTokens ? { uncachedInputTokens: usage.uncachedInputTokens } : {}),
|
|
907
2564
|
...(usage.outputTokens ? { actualOutputTokens: usage.outputTokens, outputTokens: usage.outputTokens } : {}),
|
|
@@ -913,19 +2570,22 @@ async function activeRunJob(runDir, entry, planJob, now) {
|
|
|
913
2570
|
uncached_input_tokens: usage.uncachedInputTokens,
|
|
914
2571
|
output_tokens: usage.outputTokens,
|
|
915
2572
|
reasoning_output_tokens: usage.reasoningOutputTokens,
|
|
2573
|
+
estimated_input_tokens: usage.estimatedInputTokens,
|
|
2574
|
+
estimated_from_event_bytes: usage.estimatedFromEventBytes,
|
|
916
2575
|
source: 'codex-events.jsonl',
|
|
917
2576
|
event_count: usage.eventCount
|
|
918
2577
|
}
|
|
919
2578
|
} : {}),
|
|
920
2579
|
changedPaths,
|
|
921
2580
|
changedPathCount: changedPaths.length || numberValue(merge.changedPathCount),
|
|
2581
|
+
...(rawPatchPath ? { patchPath: rawPatchPath, artifactPaths: [rawPatchPath] } : {}),
|
|
922
2582
|
evidencePaths,
|
|
923
2583
|
evidencePathCount: evidencePaths.length,
|
|
924
|
-
commandsPassed:
|
|
925
|
-
commandsFailed:
|
|
926
|
-
collectReasonClasses: status === 'running' ? ['active worker'] : [],
|
|
927
|
-
mergeReadiness: textValue(merge.mergeReadiness, status)
|
|
928
|
-
};
|
|
2584
|
+
commandsPassed: commandEvidence.passed,
|
|
2585
|
+
commandsFailed: commandEvidence.failed,
|
|
2586
|
+
collectReasonClasses: status === 'running' ? ['active worker'] : quotaDeferred ? ['quota deferred'] : [],
|
|
2587
|
+
mergeReadiness: quotaDeferred ? 'quota-deferred' : textValue(merge.mergeReadiness, status)
|
|
2588
|
+
});
|
|
929
2589
|
}
|
|
930
2590
|
function activeRunLanes(jobs) {
|
|
931
2591
|
const byLane = new Map();
|
|
@@ -952,6 +2612,39 @@ function activeRunEvents(jobs) {
|
|
|
952
2612
|
message: `${textValue(job.title, 'worker')} ${textValue(job.status, 'running')}`
|
|
953
2613
|
}));
|
|
954
2614
|
}
|
|
2615
|
+
function activeAgentsFromJobs(jobs) {
|
|
2616
|
+
return jobs
|
|
2617
|
+
.map(recordValue)
|
|
2618
|
+
.filter((job) => textValue(job.status, '') === 'running')
|
|
2619
|
+
.map((job) => {
|
|
2620
|
+
const id = textValue(job.agentId ?? job.workerId ?? job.originalJobId ?? job.id, 'agent');
|
|
2621
|
+
return {
|
|
2622
|
+
id,
|
|
2623
|
+
agentId: id,
|
|
2624
|
+
workerId: textValue(job.workerId ?? job.agentId ?? id, id),
|
|
2625
|
+
jobId: textValue(job.originalJobId ?? job.id ?? job.taskId, id),
|
|
2626
|
+
taskId: textValue(job.taskId ?? job.originalJobId ?? job.id, id),
|
|
2627
|
+
title: textValue(job.title, id),
|
|
2628
|
+
lane: textValue(job.lane, ''),
|
|
2629
|
+
status: 'active',
|
|
2630
|
+
model: textValue(job.model, ''),
|
|
2631
|
+
computeId: textValue(job.computeId, ''),
|
|
2632
|
+
reasoningEffort: textValue(job.reasoningEffort, ''),
|
|
2633
|
+
startedAt: numberValue(job.startedAt) || undefined,
|
|
2634
|
+
durationMs: numberValue(job.durationMs),
|
|
2635
|
+
inputTokens: numberValue(job.inputTokens || job.actualInputTokens || job.estimatedInputTokens),
|
|
2636
|
+
uncachedInputTokens: numberValue(job.uncachedInputTokens),
|
|
2637
|
+
cachedInputTokens: numberValue(job.cachedInputTokens),
|
|
2638
|
+
outputTokens: numberValue(job.outputTokens || job.actualOutputTokens),
|
|
2639
|
+
changedPaths: stringArray(job.changedPaths),
|
|
2640
|
+
changedPathCount: numberValue(job.changedPathCount) || stringArray(job.changedPaths).length,
|
|
2641
|
+
evidencePaths: stringArray(job.evidencePaths),
|
|
2642
|
+
evidencePathCount: numberValue(job.evidencePathCount) || stringArray(job.evidencePaths).length,
|
|
2643
|
+
sourceRun: textValue(job.sourceRun, ''),
|
|
2644
|
+
sourceLabel: textValue(job.sourceLabel, '')
|
|
2645
|
+
};
|
|
2646
|
+
});
|
|
2647
|
+
}
|
|
955
2648
|
function mergeActiveRunJobTelemetry(jobs, activeJobs) {
|
|
956
2649
|
if (!activeJobs.length)
|
|
957
2650
|
return jobs;
|
|
@@ -975,6 +2668,7 @@ function mergeActiveRunJobTelemetry(jobs, activeJobs) {
|
|
|
975
2668
|
...record,
|
|
976
2669
|
...(numberValue(activeJob.actualInputTokens) ? { actualInputTokens: numberValue(activeJob.actualInputTokens) } : {}),
|
|
977
2670
|
...(numberValue(activeJob.inputTokens) ? { inputTokens: numberValue(activeJob.inputTokens) } : {}),
|
|
2671
|
+
...(numberValue(activeJob.estimatedInputTokens) ? { estimatedInputTokens: numberValue(activeJob.estimatedInputTokens) } : {}),
|
|
978
2672
|
...(numberValue(activeJob.cachedInputTokens) ? { cachedInputTokens: numberValue(activeJob.cachedInputTokens) } : {}),
|
|
979
2673
|
...(numberValue(activeJob.uncachedInputTokens) ? { uncachedInputTokens: numberValue(activeJob.uncachedInputTokens) } : {}),
|
|
980
2674
|
...(numberValue(activeJob.actualOutputTokens) ? { actualOutputTokens: numberValue(activeJob.actualOutputTokens) } : {}),
|
|
@@ -999,6 +2693,7 @@ function jobTelemetryKeys(job) {
|
|
|
999
2693
|
function hasTokenTelemetry(job) {
|
|
1000
2694
|
return numberValue(job.actualInputTokens)
|
|
1001
2695
|
+ numberValue(job.inputTokens)
|
|
2696
|
+
+ numberValue(job.estimatedInputTokens)
|
|
1002
2697
|
+ numberValue(job.cachedInputTokens)
|
|
1003
2698
|
+ numberValue(job.uncachedInputTokens)
|
|
1004
2699
|
+ numberValue(job.outputTokens)
|
|
@@ -1013,6 +2708,7 @@ async function readCodexEventUsageSummary(file) {
|
|
|
1013
2708
|
if (!text)
|
|
1014
2709
|
return empty;
|
|
1015
2710
|
const summary = emptyCodexEventUsageSummary();
|
|
2711
|
+
summary.estimatedFromEventBytes = Buffer.byteLength(text, 'utf8');
|
|
1016
2712
|
for (const line of text.split(/\r?\n/g)) {
|
|
1017
2713
|
const trimmed = line.trim();
|
|
1018
2714
|
if (!trimmed)
|
|
@@ -1040,6 +2736,9 @@ async function readCodexEventUsageSummary(file) {
|
|
|
1040
2736
|
if (summary.inputTokens && !summary.uncachedInputTokens) {
|
|
1041
2737
|
summary.uncachedInputTokens = Math.max(0, summary.inputTokens - summary.cachedInputTokens);
|
|
1042
2738
|
}
|
|
2739
|
+
if (!hasCodexUsageValues(summary)) {
|
|
2740
|
+
summary.estimatedInputTokens = estimateInputTokensFromEventText(text);
|
|
2741
|
+
}
|
|
1043
2742
|
return summary;
|
|
1044
2743
|
}
|
|
1045
2744
|
function emptyCodexEventUsageSummary() {
|
|
@@ -1049,9 +2748,17 @@ function emptyCodexEventUsageSummary() {
|
|
|
1049
2748
|
uncachedInputTokens: 0,
|
|
1050
2749
|
outputTokens: 0,
|
|
1051
2750
|
reasoningOutputTokens: 0,
|
|
2751
|
+
estimatedInputTokens: 0,
|
|
2752
|
+
estimatedFromEventBytes: 0,
|
|
1052
2753
|
eventCount: 0
|
|
1053
2754
|
};
|
|
1054
2755
|
}
|
|
2756
|
+
function estimateInputTokensFromEventText(text) {
|
|
2757
|
+
const compactText = text.replace(/\s+/g, ' ').trim();
|
|
2758
|
+
if (!compactText)
|
|
2759
|
+
return 0;
|
|
2760
|
+
return Math.max(1, Math.ceil(compactText.length / 4));
|
|
2761
|
+
}
|
|
1055
2762
|
function collectCodexUsageRecords(value, depth = 0) {
|
|
1056
2763
|
if (depth > 5 || !value || typeof value !== 'object')
|
|
1057
2764
|
return [];
|
|
@@ -1090,7 +2797,9 @@ function normalizeCodexUsageRecord(record) {
|
|
|
1090
2797
|
uncachedInputTokens,
|
|
1091
2798
|
outputTokens,
|
|
1092
2799
|
reasoningOutputTokens,
|
|
1093
|
-
|
|
2800
|
+
estimatedInputTokens: 0,
|
|
2801
|
+
estimatedFromEventBytes: 0,
|
|
2802
|
+
eventCount: hasCodexUsageValues({ inputTokens, cachedInputTokens, uncachedInputTokens, outputTokens, reasoningOutputTokens }) ? 1 : 0
|
|
1094
2803
|
};
|
|
1095
2804
|
}
|
|
1096
2805
|
function hasCodexUsageValues(usage) {
|
|
@@ -1123,6 +2832,196 @@ async function readJsonFile(file) {
|
|
|
1123
2832
|
return undefined;
|
|
1124
2833
|
}
|
|
1125
2834
|
}
|
|
2835
|
+
function withRecomputedCostFields(record) {
|
|
2836
|
+
const model = textValue(record.model ?? record.pricingModel ?? record.pricingMatchedModel, '');
|
|
2837
|
+
if (!model || !hasCostTokenEvidence(record))
|
|
2838
|
+
return record;
|
|
2839
|
+
const cost = estimateCodexModelCost({
|
|
2840
|
+
model,
|
|
2841
|
+
estimatedInputTokens: firstCostTokenNumber(record.estimatedInputTokens, record.estimated_input_tokens),
|
|
2842
|
+
actualInputTokens: firstCostTokenNumber(record.actualInputTokens, record.inputTokens, record.promptTokens, record.actual_input_tokens, record.input_tokens, record.prompt_tokens),
|
|
2843
|
+
cachedInputTokens: firstCostTokenNumber(record.cachedInputTokens, record.cachedPromptTokens, record.cached_input_tokens, record.cached_prompt_tokens),
|
|
2844
|
+
uncachedInputTokens: firstCostTokenNumber(record.uncachedInputTokens, record.uncached_input_tokens),
|
|
2845
|
+
outputTokens: optionalCostTokenNumber(record.actualOutputTokens, record.outputTokens, record.completionTokens, record.responseTokens, record.actual_output_tokens, record.output_tokens, record.completion_tokens, record.response_tokens)
|
|
2846
|
+
});
|
|
2847
|
+
return {
|
|
2848
|
+
...record,
|
|
2849
|
+
billableInputTokens: cost.billableInputTokens,
|
|
2850
|
+
priceKnown: cost.priceKnown,
|
|
2851
|
+
...(cost.pricingModel ? { pricingModel: cost.pricingModel } : {}),
|
|
2852
|
+
...(cost.pricingMatchedModel ? { pricingMatchedModel: cost.pricingMatchedModel } : {}),
|
|
2853
|
+
...(cost.pricingSource ? { pricingSource: cost.pricingSource } : {}),
|
|
2854
|
+
...(cost.pricingUpdatedAt ? { pricingUpdatedAt: cost.pricingUpdatedAt } : {}),
|
|
2855
|
+
estimatedCostUsd: cost.estimatedCostUsd,
|
|
2856
|
+
estimatedInputCostUsd: cost.estimatedInputCostUsd,
|
|
2857
|
+
estimatedCachedInputCostUsd: cost.estimatedCachedInputCostUsd,
|
|
2858
|
+
estimatedUncachedInputCostUsd: cost.estimatedUncachedInputCostUsd,
|
|
2859
|
+
estimatedOutputCostUsd: cost.estimatedOutputCostUsd,
|
|
2860
|
+
estimatedCostMicroUsd: cost.estimatedCostMicroUsd,
|
|
2861
|
+
costEstimateInputOnly: cost.costEstimateInputOnly,
|
|
2862
|
+
costEstimateEstimatedInput: cost.costEstimateEstimatedInput,
|
|
2863
|
+
costEstimateMissingOutputTokens: cost.costEstimateMissingOutputTokens,
|
|
2864
|
+
costEstimateLongContext: cost.costEstimateLongContext,
|
|
2865
|
+
...(cost.unknownPricingReason ? { unknownPricingReason: cost.unknownPricingReason } : { unknownPricingReason: undefined })
|
|
2866
|
+
};
|
|
2867
|
+
}
|
|
2868
|
+
function hasCostTokenEvidence(record) {
|
|
2869
|
+
return [
|
|
2870
|
+
record.actualInputTokens,
|
|
2871
|
+
record.inputTokens,
|
|
2872
|
+
record.promptTokens,
|
|
2873
|
+
record.estimatedInputTokens,
|
|
2874
|
+
record.cachedInputTokens,
|
|
2875
|
+
record.uncachedInputTokens,
|
|
2876
|
+
record.actualOutputTokens,
|
|
2877
|
+
record.outputTokens,
|
|
2878
|
+
record.completionTokens,
|
|
2879
|
+
record.responseTokens,
|
|
2880
|
+
record.actual_input_tokens,
|
|
2881
|
+
record.input_tokens,
|
|
2882
|
+
record.prompt_tokens,
|
|
2883
|
+
record.estimated_input_tokens,
|
|
2884
|
+
record.cached_input_tokens,
|
|
2885
|
+
record.uncached_input_tokens,
|
|
2886
|
+
record.actual_output_tokens,
|
|
2887
|
+
record.output_tokens,
|
|
2888
|
+
record.completion_tokens,
|
|
2889
|
+
record.response_tokens
|
|
2890
|
+
].some((value) => optionalCostTokenNumber(value) !== undefined && optionalCostTokenNumber(value) > 0);
|
|
2891
|
+
}
|
|
2892
|
+
function firstCostTokenNumber(...values) {
|
|
2893
|
+
for (const value of values) {
|
|
2894
|
+
const number = optionalCostTokenNumber(value);
|
|
2895
|
+
if (number !== undefined && number > 0)
|
|
2896
|
+
return number;
|
|
2897
|
+
}
|
|
2898
|
+
return 0;
|
|
2899
|
+
}
|
|
2900
|
+
function optionalCostTokenNumber(...values) {
|
|
2901
|
+
for (const value of values) {
|
|
2902
|
+
if (value === undefined || value === null || value === '')
|
|
2903
|
+
continue;
|
|
2904
|
+
const number = Number(value);
|
|
2905
|
+
if (Number.isFinite(number) && number >= 0)
|
|
2906
|
+
return number;
|
|
2907
|
+
}
|
|
2908
|
+
return undefined;
|
|
2909
|
+
}
|
|
2910
|
+
function roundUsd(value) {
|
|
2911
|
+
return Math.round(value * 1_000_000_000) / 1_000_000_000;
|
|
2912
|
+
}
|
|
2913
|
+
async function commandEvidenceFromArtifactPaths(cwd, evidencePaths, outputDir = '') {
|
|
2914
|
+
let out = emptyCommandEvidence();
|
|
2915
|
+
for (const evidencePath of evidencePaths.slice(0, 60)) {
|
|
2916
|
+
if (!evidencePath.endsWith('.json'))
|
|
2917
|
+
continue;
|
|
2918
|
+
const displayPath = resolveRelativeArtifactPath(outputDir, evidencePath);
|
|
2919
|
+
const absolute = path.isAbsolute(displayPath) ? displayPath : path.resolve(cwd, displayPath);
|
|
2920
|
+
const roots = uniquePaths([cwd, outputDir].filter(Boolean).map((root) => path.resolve(root)));
|
|
2921
|
+
if (!roots.some((root) => isPathInside(root, absolute)))
|
|
2922
|
+
continue;
|
|
2923
|
+
out = mergeCommandEvidence(out, commandEvidenceFromRecords(recordValue(await readJsonFile(absolute))));
|
|
2924
|
+
}
|
|
2925
|
+
return out;
|
|
2926
|
+
}
|
|
2927
|
+
function commandEvidenceFromRecords(...records) {
|
|
2928
|
+
let out = emptyCommandEvidence();
|
|
2929
|
+
for (const record of records) {
|
|
2930
|
+
out = mergeCommandEvidence(out, {
|
|
2931
|
+
passed: [
|
|
2932
|
+
...commandRecordsFromKnownBucket(record.commandsPassed, 'passed'),
|
|
2933
|
+
...commandRecordsFromKnownBucket(record.passedCommands, 'passed')
|
|
2934
|
+
],
|
|
2935
|
+
failed: [
|
|
2936
|
+
...commandRecordsFromKnownBucket(record.commandsFailed, 'failed'),
|
|
2937
|
+
...commandRecordsFromKnownBucket(record.failedCommands, 'failed')
|
|
2938
|
+
]
|
|
2939
|
+
});
|
|
2940
|
+
for (const key of ['verification', 'commands', 'checks', 'testResults', 'results']) {
|
|
2941
|
+
const classified = classifyCommandRecords(recordArray(record[key]));
|
|
2942
|
+
out = mergeCommandEvidence(out, classified);
|
|
2943
|
+
}
|
|
2944
|
+
}
|
|
2945
|
+
return out;
|
|
2946
|
+
}
|
|
2947
|
+
function commandRecordsFromKnownBucket(value, fallbackStatus) {
|
|
2948
|
+
return recordArray(value)
|
|
2949
|
+
.map((record) => normalizedCommandRecord(record, fallbackStatus))
|
|
2950
|
+
.filter((record) => Boolean(record));
|
|
2951
|
+
}
|
|
2952
|
+
function classifyCommandRecords(records) {
|
|
2953
|
+
const out = emptyCommandEvidence();
|
|
2954
|
+
for (const record of records) {
|
|
2955
|
+
const classified = classifyCommandRecord(record);
|
|
2956
|
+
if (classified.status === 'passed')
|
|
2957
|
+
out.passed.push(classified.record);
|
|
2958
|
+
else if (classified.status === 'failed')
|
|
2959
|
+
out.failed.push(classified.record);
|
|
2960
|
+
}
|
|
2961
|
+
return normalizeCommandEvidence(out);
|
|
2962
|
+
}
|
|
2963
|
+
function classifyCommandRecord(record) {
|
|
2964
|
+
const statusText = normalized(record.status ?? record.result ?? record.outcome ?? record.state);
|
|
2965
|
+
const exitCode = hasOwnKey(record, 'exitCode') ? numberValue(record.exitCode) : undefined;
|
|
2966
|
+
const status = commandStatus(statusText, exitCode);
|
|
2967
|
+
return { status, record: normalizedCommandRecord(record, status || undefined) ?? record };
|
|
2968
|
+
}
|
|
2969
|
+
function commandStatus(statusText, exitCode) {
|
|
2970
|
+
if (['passed', 'pass', 'ok', 'success', 'succeeded', 'completed', 'green'].includes(statusText))
|
|
2971
|
+
return 'passed';
|
|
2972
|
+
if (['failed', 'fail', 'failure', 'error', 'errored', 'red', 'timeout', 'timed-out', 'blocked', 'nonzero'].includes(statusText))
|
|
2973
|
+
return 'failed';
|
|
2974
|
+
if (exitCode !== undefined)
|
|
2975
|
+
return exitCode === 0 ? 'passed' : 'failed';
|
|
2976
|
+
return '';
|
|
2977
|
+
}
|
|
2978
|
+
function normalizedCommandRecord(record, fallbackStatus) {
|
|
2979
|
+
const command = textValue(record.command ?? record.cmd ?? record.name ?? record.label, '');
|
|
2980
|
+
const cwd = textValue(record.cwd ?? record.dir, '');
|
|
2981
|
+
const status = textValue(record.status ?? record.result ?? record.outcome ?? record.state, fallbackStatus ?? '');
|
|
2982
|
+
if (!command && !cwd && !status)
|
|
2983
|
+
return undefined;
|
|
2984
|
+
return {
|
|
2985
|
+
...record,
|
|
2986
|
+
...(command && !record.command ? { command } : {}),
|
|
2987
|
+
...(cwd && !record.cwd ? { cwd } : {}),
|
|
2988
|
+
...(status && !record.status ? { status } : {})
|
|
2989
|
+
};
|
|
2990
|
+
}
|
|
2991
|
+
function mergeCommandEvidence(...entries) {
|
|
2992
|
+
return normalizeCommandEvidence({
|
|
2993
|
+
passed: entries.flatMap((entry) => entry.passed),
|
|
2994
|
+
failed: entries.flatMap((entry) => entry.failed)
|
|
2995
|
+
});
|
|
2996
|
+
}
|
|
2997
|
+
function normalizeCommandEvidence(evidence) {
|
|
2998
|
+
return {
|
|
2999
|
+
passed: uniqueCommandRecords(evidence.passed),
|
|
3000
|
+
failed: uniqueCommandRecords(evidence.failed)
|
|
3001
|
+
};
|
|
3002
|
+
}
|
|
3003
|
+
function uniqueCommandRecords(records) {
|
|
3004
|
+
const seen = new Set();
|
|
3005
|
+
const out = [];
|
|
3006
|
+
for (const record of records) {
|
|
3007
|
+
const key = [
|
|
3008
|
+
textValue(record.command ?? record.name, ''),
|
|
3009
|
+
textValue(record.cwd, ''),
|
|
3010
|
+
normalized(record.status ?? record.result ?? record.outcome)
|
|
3011
|
+
].join('\0');
|
|
3012
|
+
if (seen.has(key))
|
|
3013
|
+
continue;
|
|
3014
|
+
seen.add(key);
|
|
3015
|
+
out.push(record);
|
|
3016
|
+
}
|
|
3017
|
+
return out;
|
|
3018
|
+
}
|
|
3019
|
+
function emptyCommandEvidence() {
|
|
3020
|
+
return { passed: [], failed: [] };
|
|
3021
|
+
}
|
|
3022
|
+
function hasOwnKey(record, key) {
|
|
3023
|
+
return Object.prototype.hasOwnProperty.call(record, key);
|
|
3024
|
+
}
|
|
1126
3025
|
function isProcessLive(pid, entry) {
|
|
1127
3026
|
if (!pid)
|
|
1128
3027
|
return false;
|
|
@@ -1164,28 +3063,116 @@ function processCommandMatchesPidManifest(command, entry) {
|
|
|
1164
3063
|
return false;
|
|
1165
3064
|
return true;
|
|
1166
3065
|
}
|
|
1167
|
-
function
|
|
1168
|
-
return path.dirname(path.dirname(runDir));
|
|
1169
|
-
}
|
|
1170
|
-
async function readTaskDetails(options, jobId) {
|
|
3066
|
+
async function readTaskDetails(options, jobId, sourceRun = '') {
|
|
1171
3067
|
if (!jobId)
|
|
1172
3068
|
return { ok: false, jobId, files: [], commandsPassed: [], commandsFailed: [], evidenceArtifacts: [], error: 'missing job id' };
|
|
1173
|
-
const entry = await findCollectionBundle(options, jobId);
|
|
3069
|
+
const entry = await findCollectionBundle(options, jobId) ?? await findRawRunTaskBundle(options, jobId, sourceRun);
|
|
1174
3070
|
if (!entry)
|
|
1175
3071
|
return { ok: false, jobId, files: [], commandsPassed: [], commandsFailed: [], evidenceArtifacts: [], error: 'task not found in collection' };
|
|
1176
3072
|
const { bundle, outputDir } = entry;
|
|
1177
3073
|
const patchPath = textValue(bundle.patchPath, '');
|
|
1178
3074
|
const evidencePaths = stringArray(bundle.evidencePaths).slice(0, 40);
|
|
3075
|
+
const evidenceCommands = await commandEvidenceFromArtifactPaths(options.cwd, evidencePaths, outputDir);
|
|
3076
|
+
const commandEvidence = mergeCommandEvidence(commandEvidenceFromRecords(bundle), evidenceCommands);
|
|
1179
3077
|
return {
|
|
1180
3078
|
ok: true,
|
|
1181
3079
|
jobId,
|
|
1182
3080
|
...(patchPath ? { patchArtifact: artifactRecord(patchPath) } : {}),
|
|
1183
3081
|
files: patchPath ? await readPatchFiles(options, patchPath) : [],
|
|
1184
|
-
commandsPassed:
|
|
1185
|
-
commandsFailed:
|
|
3082
|
+
commandsPassed: commandEvidence.passed.slice(0, 20),
|
|
3083
|
+
commandsFailed: commandEvidence.failed.slice(0, 20),
|
|
1186
3084
|
evidenceArtifacts: evidencePaths.map((evidencePath) => artifactRecord(resolveRelativeArtifactPath(outputDir, evidencePath), evidencePath))
|
|
1187
3085
|
};
|
|
1188
3086
|
}
|
|
3087
|
+
async function findRawRunTaskBundle(options, jobId, sourceRun = '') {
|
|
3088
|
+
const sourceRunRoot = sourceRun ? safeCwdRelativeDirectory(options.cwd, sourceRun) : undefined;
|
|
3089
|
+
const hintedRoot = sourceRunRoot ?? rawRunSourceHint(options.cwd, jobId);
|
|
3090
|
+
const root = hintedRoot ?? path.join(options.cwd, 'agent-runs');
|
|
3091
|
+
const stat = await fs.stat(root).catch(() => undefined);
|
|
3092
|
+
if (!stat?.isDirectory())
|
|
3093
|
+
return undefined;
|
|
3094
|
+
const matches = await findRawRunJobDirs(root, jobId, 0);
|
|
3095
|
+
if (!matches.length)
|
|
3096
|
+
return undefined;
|
|
3097
|
+
const scoredMatches = await Promise.all(matches.map(async (match) => {
|
|
3098
|
+
const patchPath = await firstExistingRelativePath(options.cwd, rawRunPatchCandidates(match));
|
|
3099
|
+
const matchStat = await fs.stat(match).catch(() => undefined);
|
|
3100
|
+
return { match, patchPath, mtimeMs: matchStat?.mtimeMs ?? 0 };
|
|
3101
|
+
}));
|
|
3102
|
+
scoredMatches.sort((left, right) => {
|
|
3103
|
+
const patchScore = Number(Boolean(right.patchPath)) - Number(Boolean(left.patchPath));
|
|
3104
|
+
if (patchScore)
|
|
3105
|
+
return patchScore;
|
|
3106
|
+
const timeScore = right.mtimeMs - left.mtimeMs;
|
|
3107
|
+
if (timeScore)
|
|
3108
|
+
return timeScore;
|
|
3109
|
+
return right.match.localeCompare(left.match);
|
|
3110
|
+
});
|
|
3111
|
+
const { match, patchPath } = scoredMatches[0];
|
|
3112
|
+
const evidencePaths = await existingRelativePaths(options.cwd, [
|
|
3113
|
+
path.join(match, 'last-message.md'),
|
|
3114
|
+
path.join(match, 'codex-events.jsonl'),
|
|
3115
|
+
path.join(match, 'evidence', 'last-message.md'),
|
|
3116
|
+
path.join(match, 'evidence', 'handoff.md'),
|
|
3117
|
+
path.join(match, 'evidence', 'evidence.json'),
|
|
3118
|
+
path.join(match, 'evidence', 'resource-allocation.json'),
|
|
3119
|
+
...rawRunPatchCandidates(match)
|
|
3120
|
+
]);
|
|
3121
|
+
const evidenceCommands = await commandEvidenceFromArtifactPaths(options.cwd, evidencePaths);
|
|
3122
|
+
return {
|
|
3123
|
+
bundle: {
|
|
3124
|
+
jobId: path.basename(match),
|
|
3125
|
+
patchPath,
|
|
3126
|
+
changedPaths: await readPatchChangedPathList(options.cwd, patchPath),
|
|
3127
|
+
evidencePaths,
|
|
3128
|
+
commandsPassed: evidenceCommands.passed,
|
|
3129
|
+
commandsFailed: evidenceCommands.failed
|
|
3130
|
+
}
|
|
3131
|
+
};
|
|
3132
|
+
}
|
|
3133
|
+
function rawRunSourceHint(cwd, jobId) {
|
|
3134
|
+
const match = /(?:^|:)(agent-runs\/[^:]+)/.exec(jobId);
|
|
3135
|
+
if (!match)
|
|
3136
|
+
return undefined;
|
|
3137
|
+
const absolute = path.resolve(cwd, match[1]);
|
|
3138
|
+
return isPathInside(cwd, absolute) ? absolute : undefined;
|
|
3139
|
+
}
|
|
3140
|
+
function safeCwdRelativeDirectory(cwd, input) {
|
|
3141
|
+
const absolute = path.resolve(cwd, input);
|
|
3142
|
+
if (!isPathInside(cwd, absolute))
|
|
3143
|
+
return undefined;
|
|
3144
|
+
return absolute;
|
|
3145
|
+
}
|
|
3146
|
+
async function findRawRunJobDirs(root, jobId, depth) {
|
|
3147
|
+
if (depth > 5)
|
|
3148
|
+
return [];
|
|
3149
|
+
const entries = await fs.readdir(root, { withFileTypes: true }).catch(() => []);
|
|
3150
|
+
const matches = [];
|
|
3151
|
+
for (const entry of entries) {
|
|
3152
|
+
if (!entry.isDirectory() || entry.name === 'streams' || entry.name === 'artifact-store' || entry.name.startsWith('.'))
|
|
3153
|
+
continue;
|
|
3154
|
+
const absolute = path.join(root, entry.name);
|
|
3155
|
+
if (rawRunJobIdMatches(jobId, entry.name) && await rawRunJobHasArtifacts(absolute))
|
|
3156
|
+
matches.push(absolute);
|
|
3157
|
+
matches.push(...await findRawRunJobDirs(absolute, jobId, depth + 1));
|
|
3158
|
+
}
|
|
3159
|
+
return matches;
|
|
3160
|
+
}
|
|
3161
|
+
async function rawRunJobHasArtifacts(jobDir) {
|
|
3162
|
+
for (const file of [
|
|
3163
|
+
path.join(jobDir, 'last-message.md'),
|
|
3164
|
+
path.join(jobDir, 'codex-events.jsonl'),
|
|
3165
|
+
...rawRunPatchCandidates(jobDir)
|
|
3166
|
+
]) {
|
|
3167
|
+
const stat = await fs.stat(file).catch(() => undefined);
|
|
3168
|
+
if (stat?.isFile())
|
|
3169
|
+
return true;
|
|
3170
|
+
}
|
|
3171
|
+
return false;
|
|
3172
|
+
}
|
|
3173
|
+
function rawRunJobIdMatches(requestedId, jobId) {
|
|
3174
|
+
return requestedId === jobId || requestedId.endsWith(`:${jobId}`) || requestedId.endsWith(`-${jobId}`);
|
|
3175
|
+
}
|
|
1189
3176
|
async function findCollectionBundle(options, jobId) {
|
|
1190
3177
|
const collectionFile = await resolveArtifactFile(options.cwd, options.collection, ['collection.json', 'coordinator-query.json']);
|
|
1191
3178
|
if (!collectionFile)
|
|
@@ -1253,6 +3240,7 @@ async function writeHumanActionAnswer(options, body) {
|
|
|
1253
3240
|
return { ok: true, code, answerPath };
|
|
1254
3241
|
}
|
|
1255
3242
|
function notifyDashboardStreams() {
|
|
3243
|
+
invalidateDashboardSnapshotCache();
|
|
1256
3244
|
for (const listener of dashboardStreamListeners)
|
|
1257
3245
|
listener();
|
|
1258
3246
|
}
|
|
@@ -1321,7 +3309,7 @@ async function readPatchFiles(options, patchPath) {
|
|
|
1321
3309
|
return parseUnifiedPatchFiles(patch).slice(0, 40);
|
|
1322
3310
|
}
|
|
1323
3311
|
function parseUnifiedPatchFiles(patch) {
|
|
1324
|
-
const sections = patch
|
|
3312
|
+
const sections = splitUnifiedPatchSections(patch);
|
|
1325
3313
|
return sections.flatMap((section) => {
|
|
1326
3314
|
const lines = section.split('\n');
|
|
1327
3315
|
const pathLine = lines.find((line) => line.startsWith('+++ ')) ?? lines.find((line) => line.startsWith('diff --git '));
|
|
@@ -1343,6 +3331,27 @@ function parseUnifiedPatchFiles(patch) {
|
|
|
1343
3331
|
}];
|
|
1344
3332
|
});
|
|
1345
3333
|
}
|
|
3334
|
+
function splitUnifiedPatchSections(patch) {
|
|
3335
|
+
if (/^diff --git /m.test(patch)) {
|
|
3336
|
+
return patch.split(/\n(?=diff --git )/g).filter((section) => section.trim().length > 0);
|
|
3337
|
+
}
|
|
3338
|
+
const lines = patch.split('\n');
|
|
3339
|
+
const sections = [];
|
|
3340
|
+
let current = [];
|
|
3341
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
3342
|
+
const line = lines[index];
|
|
3343
|
+
const next = lines[index + 1] ?? '';
|
|
3344
|
+
const startsPlainFile = line.startsWith('--- ') && next.startsWith('+++ ');
|
|
3345
|
+
if (startsPlainFile && current.length) {
|
|
3346
|
+
sections.push(current.join('\n'));
|
|
3347
|
+
current = [];
|
|
3348
|
+
}
|
|
3349
|
+
current.push(line);
|
|
3350
|
+
}
|
|
3351
|
+
if (current.some((line) => line.trim().length > 0))
|
|
3352
|
+
sections.push(current.join('\n'));
|
|
3353
|
+
return sections;
|
|
3354
|
+
}
|
|
1346
3355
|
function parseUnifiedPatchHunks(section) {
|
|
1347
3356
|
const hunks = [];
|
|
1348
3357
|
let current = { header: 'File header', lines: [] };
|
|
@@ -1384,9 +3393,20 @@ function parseUnifiedPatchHunks(section) {
|
|
|
1384
3393
|
function patchFilePath(line) {
|
|
1385
3394
|
const plus = /^\+\+\+\s+(?:b\/)?(.+)$/.exec(line);
|
|
1386
3395
|
if (plus && plus[1] !== '/dev/null')
|
|
1387
|
-
return plus[1];
|
|
3396
|
+
return normalizePatchDisplayPath(plus[1]);
|
|
1388
3397
|
const diff = /^diff --git\s+a\/(.+?)\s+b\/(.+)$/.exec(line);
|
|
1389
|
-
return diff?.[2] ?? '';
|
|
3398
|
+
return normalizePatchDisplayPath(diff?.[2] ?? '');
|
|
3399
|
+
}
|
|
3400
|
+
function normalizePatchDisplayPath(value) {
|
|
3401
|
+
let clean = value.trim().split(/\t/g)[0]?.trim() ?? '';
|
|
3402
|
+
clean = clean.replace(/^(?:a|b)\//, '');
|
|
3403
|
+
const packageIndex = clean.indexOf('/packages/');
|
|
3404
|
+
if (packageIndex >= 0)
|
|
3405
|
+
clean = clean.slice(packageIndex + 1);
|
|
3406
|
+
const repoPackageIndex = clean.indexOf('packages/');
|
|
3407
|
+
if (repoPackageIndex > 0)
|
|
3408
|
+
clean = clean.slice(repoPackageIndex);
|
|
3409
|
+
return clean === '/dev/null' ? '' : clean;
|
|
1390
3410
|
}
|
|
1391
3411
|
function artifactRecord(pathValue, label = pathValue) {
|
|
1392
3412
|
return {
|