@shapeshift-labs/frontier-loom-ui 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client.js +296 -31
- package/dist/client.js.map +1 -1
- package/dist/public/styles.css +266 -12
- package/dist/server.d.ts +5 -2
- package/dist/server.js +2078 -94
- package/dist/server.js.map +1 -1
- package/package.json +2 -2
package/dist/server.js
CHANGED
|
@@ -4,7 +4,7 @@ import fs from 'node:fs/promises';
|
|
|
4
4
|
import http from 'node:http';
|
|
5
5
|
import path from 'node:path';
|
|
6
6
|
import { fileURLToPath } from 'node:url';
|
|
7
|
-
import { readCodexDashboardSnapshot } from '@shapeshift-labs/frontier-swarm-codex';
|
|
7
|
+
import { estimateCodexModelCost, readCodexDashboardSnapshot } from '@shapeshift-labs/frontier-swarm-codex';
|
|
8
8
|
const packageDir = path.dirname(fileURLToPath(import.meta.url));
|
|
9
9
|
const HEALTH_JSON_PARSE_MAX_BYTES = 16 * 1024 * 1024;
|
|
10
10
|
const TASK_DETAIL_PATCH_MAX_BYTES = 512 * 1024;
|
|
@@ -13,13 +13,20 @@ const ARTIFACT_VIEW_MAX_BYTES = 768 * 1024;
|
|
|
13
13
|
const ARTIFACT_DIRECTORY_MAX_ENTRIES = 200;
|
|
14
14
|
const HUMAN_ACTION_ANSWER_MAX_BYTES = 16 * 1024;
|
|
15
15
|
const CODEX_EVENTS_USAGE_MAX_BYTES = 8 * 1024 * 1024;
|
|
16
|
-
const
|
|
16
|
+
const DASHBOARD_SNAPSHOT_CACHE_MS = 5000;
|
|
17
|
+
const LIFETIME_DASHBOARD_MAX_SOURCES = 24;
|
|
17
18
|
const LIFETIME_DASHBOARD_MAX_JOBS = 800;
|
|
18
19
|
const LIFETIME_DASHBOARD_SCAN_MAX_FILES = 600;
|
|
19
20
|
const LIFETIME_DASHBOARD_SCAN_MAX_DEPTH = 5;
|
|
21
|
+
const LIFETIME_DASHBOARD_MAX_AUTONOMOUS_DECISION_FILES = 400;
|
|
22
|
+
const LIFETIME_DASHBOARD_MAX_DRAIN_RUNS = 6;
|
|
23
|
+
const LIFETIME_DASHBOARD_MAX_ACTIVE_PID_RUNS = 32;
|
|
24
|
+
const LIFETIME_DASHBOARD_MAX_QUEUE_TASKS = 500;
|
|
25
|
+
const LIFETIME_DASHBOARD_SOURCE_TIMEOUT_MS = 2500;
|
|
20
26
|
const LIFETIME_DASHBOARD_RESET_FILE = '.loom-ui-reset.json';
|
|
21
27
|
const REVIEW_DECISIONS_FILE = '.loom-ui-review-decisions.json';
|
|
22
28
|
const dashboardStreamListeners = new Set();
|
|
29
|
+
let dashboardSnapshotCache;
|
|
23
30
|
export function createLoomUiServer(options = {}) {
|
|
24
31
|
const normalized = normalizeServerOptions(options);
|
|
25
32
|
const server = http.createServer(async (request, response) => {
|
|
@@ -64,10 +71,10 @@ async function handleRequest(request, response, options) {
|
|
|
64
71
|
await streamDashboard(request, response, options);
|
|
65
72
|
}
|
|
66
73
|
else if (request.method === 'GET' && url.pathname === '/api/dashboard') {
|
|
67
|
-
writeJson(response, 200, await
|
|
74
|
+
writeJson(response, 200, await readDashboardSnapshotCached(options));
|
|
68
75
|
}
|
|
69
76
|
else if (request.method === 'GET' && url.pathname === '/api/task-details') {
|
|
70
|
-
writeJson(response, 200, await readTaskDetails(options, textValue(url.searchParams.get('id'), '')));
|
|
77
|
+
writeJson(response, 200, await readTaskDetails(options, textValue(url.searchParams.get('id'), ''), textValue(url.searchParams.get('sourceRun'), '')));
|
|
71
78
|
}
|
|
72
79
|
else if (request.method === 'GET' && url.pathname === '/api/artifact') {
|
|
73
80
|
writeJson(response, 200, await readArtifact(options, textValue(url.searchParams.get('path'), '')));
|
|
@@ -114,7 +121,7 @@ async function streamDashboard(request, response, options) {
|
|
|
114
121
|
return;
|
|
115
122
|
pending = true;
|
|
116
123
|
try {
|
|
117
|
-
const snapshot = await
|
|
124
|
+
const snapshot = await readDashboardSnapshotCached(options);
|
|
118
125
|
const signature = dashboardStreamSignature(snapshot);
|
|
119
126
|
const body = JSON.stringify(snapshot);
|
|
120
127
|
if (signature !== lastSignature) {
|
|
@@ -172,6 +179,16 @@ function debounce(fn, delayMs) {
|
|
|
172
179
|
}, delayMs);
|
|
173
180
|
};
|
|
174
181
|
}
|
|
182
|
+
function withTimeout(promise, timeoutMs, message) {
|
|
183
|
+
let timer;
|
|
184
|
+
const timeout = new Promise((_resolve, reject) => {
|
|
185
|
+
timer = setTimeout(() => reject(new Error(message)), timeoutMs);
|
|
186
|
+
});
|
|
187
|
+
return Promise.race([promise, timeout]).finally(() => {
|
|
188
|
+
if (timer)
|
|
189
|
+
clearTimeout(timer);
|
|
190
|
+
});
|
|
191
|
+
}
|
|
175
192
|
async function createDashboardWatchers(options, onChange) {
|
|
176
193
|
const roots = await dashboardWatchRoots(options);
|
|
177
194
|
const watchers = [];
|
|
@@ -189,7 +206,10 @@ async function createDashboardWatchers(options, onChange) {
|
|
|
189
206
|
}
|
|
190
207
|
function watchDirectory(root, recursive, onChange) {
|
|
191
208
|
try {
|
|
192
|
-
return watch(root, { recursive },
|
|
209
|
+
return watch(root, { recursive }, () => {
|
|
210
|
+
invalidateDashboardSnapshotCache();
|
|
211
|
+
onChange();
|
|
212
|
+
});
|
|
193
213
|
}
|
|
194
214
|
catch {
|
|
195
215
|
return undefined;
|
|
@@ -209,6 +229,9 @@ async function dashboardWatchRoots(options) {
|
|
|
209
229
|
const agentRuns = path.join(options.cwd, 'agent-runs');
|
|
210
230
|
if (await fileExists(agentRuns))
|
|
211
231
|
roots.push(agentRuns);
|
|
232
|
+
const loomQueues = path.join(options.cwd, '.loom', 'queues');
|
|
233
|
+
if (await fileExists(loomQueues))
|
|
234
|
+
roots.push(loomQueues);
|
|
212
235
|
}
|
|
213
236
|
return uniquePaths(roots);
|
|
214
237
|
}
|
|
@@ -247,10 +270,39 @@ async function readDashboardSnapshot(options) {
|
|
|
247
270
|
return readLifetimeDashboardSnapshot(options);
|
|
248
271
|
return readScopedDashboardSnapshot(options);
|
|
249
272
|
}
|
|
250
|
-
async function
|
|
273
|
+
async function readDashboardSnapshotCached(options) {
|
|
274
|
+
const key = JSON.stringify(dashboardInput(options));
|
|
275
|
+
const now = Date.now();
|
|
276
|
+
if (dashboardSnapshotCache?.key === key) {
|
|
277
|
+
if (dashboardSnapshotCache.value !== undefined && now - dashboardSnapshotCache.at < DASHBOARD_SNAPSHOT_CACHE_MS) {
|
|
278
|
+
return dashboardSnapshotCache.value;
|
|
279
|
+
}
|
|
280
|
+
if (dashboardSnapshotCache.pending)
|
|
281
|
+
return dashboardSnapshotCache.pending;
|
|
282
|
+
}
|
|
283
|
+
const pending = readDashboardSnapshot(options).then((value) => {
|
|
284
|
+
dashboardSnapshotCache = { key, at: Date.now(), value };
|
|
285
|
+
return value;
|
|
286
|
+
}, (error) => {
|
|
287
|
+
if (dashboardSnapshotCache?.key === key) {
|
|
288
|
+
dashboardSnapshotCache = dashboardSnapshotCache.value === undefined
|
|
289
|
+
? undefined
|
|
290
|
+
: { key, at: dashboardSnapshotCache.at, value: dashboardSnapshotCache.value };
|
|
291
|
+
}
|
|
292
|
+
throw error;
|
|
293
|
+
});
|
|
294
|
+
dashboardSnapshotCache = { key, at: now, value: dashboardSnapshotCache?.key === key ? dashboardSnapshotCache.value : undefined, pending };
|
|
295
|
+
return pending;
|
|
296
|
+
}
|
|
297
|
+
function invalidateDashboardSnapshotCache() {
|
|
298
|
+
dashboardSnapshotCache = undefined;
|
|
299
|
+
}
|
|
300
|
+
async function readScopedDashboardSnapshot(options, readOptions = {}) {
|
|
251
301
|
const snapshot = await readCodexDashboardSnapshot(dashboardInput(options));
|
|
252
|
-
const activeRunSnapshot = await readActiveRunSnapshot(options);
|
|
302
|
+
const activeRunSnapshot = readOptions.includeActiveRun === false ? undefined : await readActiveRunSnapshot(options);
|
|
253
303
|
const reviewDecisions = await readCoordinatorReviewDecisions(options.cwd);
|
|
304
|
+
const autonomousDecisions = await readAutonomousMergeDecisions(options.cwd);
|
|
305
|
+
const decisions = mergeReviewDecisionLists(reviewDecisions, autonomousDecisions);
|
|
254
306
|
if (!snapshot || typeof snapshot !== 'object' || Array.isArray(snapshot))
|
|
255
307
|
return activeRunSnapshot ?? snapshot;
|
|
256
308
|
const answers = await readHumanActionAnswers(options);
|
|
@@ -258,15 +310,18 @@ async function readScopedDashboardSnapshot(options) {
|
|
|
258
310
|
const jobs = Array.isArray(record.jobs) ? record.jobs : [];
|
|
259
311
|
const activeJobs = recordArray(activeRunSnapshot?.jobs);
|
|
260
312
|
if (shouldPreferActiveRunSnapshot(jobs, activeJobs)) {
|
|
313
|
+
const activeAgentRows = activeAgentsFromJobs(activeJobs);
|
|
261
314
|
return {
|
|
262
315
|
...activeRunSnapshot,
|
|
263
316
|
collectionJobs: jobs,
|
|
317
|
+
activeAgents: activeAgentRows,
|
|
264
318
|
humanActions: recordArray(record.humanActions),
|
|
265
319
|
humanActionAnswers: answers,
|
|
266
320
|
sources: {
|
|
267
321
|
...recordValue(record.sources),
|
|
268
322
|
...recordValue(activeRunSnapshot?.sources),
|
|
269
323
|
...(reviewDecisions.length ? { coordinatorReviewDecisions: coordinatorReviewDecisionPath(options.cwd) } : {}),
|
|
324
|
+
...(autonomousDecisions.length ? { autonomousMergeDecisions: autonomousDecisionSourceSummary(autonomousDecisions) } : {}),
|
|
270
325
|
...(answers.length ? { humanActionAnswers: await humanActionAnswerLogPath(options) } : {})
|
|
271
326
|
},
|
|
272
327
|
raw: {
|
|
@@ -279,14 +334,16 @@ async function readScopedDashboardSnapshot(options) {
|
|
|
279
334
|
}
|
|
280
335
|
};
|
|
281
336
|
}
|
|
282
|
-
const mergedJobs = applyCoordinatorReviewDecisions(mergeActiveRunJobTelemetry(jobs, activeJobs),
|
|
337
|
+
const mergedJobs = applyCoordinatorReviewDecisions(mergeActiveRunJobTelemetry(jobs, activeJobs), decisions).map(withRecomputedCostFields);
|
|
283
338
|
return {
|
|
284
339
|
...normalizeCoordinatorFacingSnapshot(record),
|
|
285
340
|
jobs: mergedJobs,
|
|
341
|
+
activeAgents: activeAgentsFromJobs(mergedJobs),
|
|
286
342
|
humanActionAnswers: answers,
|
|
287
343
|
sources: {
|
|
288
344
|
...recordValue(record.sources),
|
|
289
345
|
...(reviewDecisions.length ? { coordinatorReviewDecisions: coordinatorReviewDecisionPath(options.cwd) } : {}),
|
|
346
|
+
...(autonomousDecisions.length ? { autonomousMergeDecisions: autonomousDecisionSourceSummary(autonomousDecisions) } : {}),
|
|
290
347
|
...(answers.length ? { humanActionAnswers: await humanActionAnswerLogPath(options) } : {})
|
|
291
348
|
}
|
|
292
349
|
};
|
|
@@ -294,22 +351,39 @@ async function readScopedDashboardSnapshot(options) {
|
|
|
294
351
|
async function readLifetimeDashboardSnapshot(options) {
|
|
295
352
|
const sources = await discoverLifetimeDashboardSources(options.cwd);
|
|
296
353
|
const snapshots = [];
|
|
354
|
+
let skippedSourceCount = 0;
|
|
355
|
+
let timedOutSourceCount = 0;
|
|
297
356
|
for (const source of sources.slice(0, LIFETIME_DASHBOARD_MAX_SOURCES)) {
|
|
298
357
|
try {
|
|
299
|
-
const snapshot =
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
358
|
+
const snapshot = await withTimeout((async () => {
|
|
359
|
+
const scopedSnapshot = await readScopedDashboardSnapshot({
|
|
360
|
+
...options,
|
|
361
|
+
run: source.run,
|
|
362
|
+
collection: source.collection,
|
|
363
|
+
continuation: source.continuation
|
|
364
|
+
}, { includeActiveRun: false });
|
|
365
|
+
return enrichLifetimeRunSnapshotEvidence(options.cwd, source, recordValue(scopedSnapshot));
|
|
366
|
+
})(), LIFETIME_DASHBOARD_SOURCE_TIMEOUT_MS, `lifetime source timed out: ${source.path}`);
|
|
305
367
|
if (Object.keys(snapshot).length)
|
|
306
368
|
snapshots.push({ source, snapshot });
|
|
307
369
|
}
|
|
308
|
-
catch {
|
|
370
|
+
catch (error) {
|
|
371
|
+
skippedSourceCount++;
|
|
372
|
+
if (error instanceof Error && error.message.startsWith('lifetime source timed out:'))
|
|
373
|
+
timedOutSourceCount++;
|
|
309
374
|
continue;
|
|
310
375
|
}
|
|
311
376
|
}
|
|
312
|
-
|
|
377
|
+
const lifetime = await combineLifetimeDashboardSnapshots(options, sources, snapshots, mergeReviewDecisionLists(await readCoordinatorReviewDecisions(options.cwd), await readAutonomousMergeDecisions(options.cwd)), await readLifetimeQueueBacklog(options.cwd));
|
|
378
|
+
const lifetimeWithSourceHealth = {
|
|
379
|
+
...lifetime,
|
|
380
|
+
sources: {
|
|
381
|
+
...recordValue(lifetime.sources),
|
|
382
|
+
skippedSourceCount,
|
|
383
|
+
timedOutSourceCount
|
|
384
|
+
}
|
|
385
|
+
};
|
|
386
|
+
return mergeLifetimeActiveRunSnapshot(mergeLifetimeDrainCoordinatorSnapshot(lifetimeWithSourceHealth, await readLatestDrainCoordinatorSnapshot(options.cwd)), await readLifetimeActiveRunSnapshot(options));
|
|
313
387
|
}
|
|
314
388
|
async function discoverLifetimeDashboardSources(cwd) {
|
|
315
389
|
const root = path.join(cwd, 'agent-runs');
|
|
@@ -341,7 +415,7 @@ async function discoverLifetimeDashboardSources(cwd) {
|
|
|
341
415
|
continue;
|
|
342
416
|
const hasCollection = entry.files.has('collection.json') || entry.files.has('coordinator-query.json');
|
|
343
417
|
const hasContinuation = entry.files.has('continuation.json');
|
|
344
|
-
const hasRun = entry.files.has('swarm-results.json') || entry.files.has('
|
|
418
|
+
const hasRun = entry.files.has('swarm-results.json') || entry.files.has('coordinator-dashboard.json');
|
|
345
419
|
if (hasCollection) {
|
|
346
420
|
out.push({
|
|
347
421
|
id: `collection:${relative}`,
|
|
@@ -378,13 +452,10 @@ async function discoverLifetimeDashboardSources(cwd) {
|
|
|
378
452
|
return dedupeLifetimeDashboardSources(out).sort((left, right) => right.mtimeMs - left.mtimeMs || left.path.localeCompare(right.path));
|
|
379
453
|
}
|
|
380
454
|
function dedupeLifetimeDashboardSources(sources) {
|
|
381
|
-
const collections = new Set(sources.filter((source) => source.kind === 'collection').map(lifetimeRunFamilyKey));
|
|
382
455
|
const preferredCollections = preferredLifetimeCollectionsByFamily(sources);
|
|
383
456
|
const runs = new Set(sources.filter((source) => source.kind === 'run').map(lifetimeRunFamilyKey));
|
|
384
457
|
return sources.filter((source) => {
|
|
385
458
|
const family = lifetimeRunFamilyKey(source);
|
|
386
|
-
if (source.kind === 'run' && collections.has(family))
|
|
387
|
-
return false;
|
|
388
459
|
if (source.kind === 'collection' && preferredCollections.get(family) !== source)
|
|
389
460
|
return false;
|
|
390
461
|
if (source.kind === 'collection' && source.path.endsWith('/collected-missing') && runs.has(family))
|
|
@@ -411,6 +482,10 @@ function compareLifetimeCollectionPreference(left, right) {
|
|
|
411
482
|
}
|
|
412
483
|
function lifetimeCollectionPreference(source) {
|
|
413
484
|
const pathLabel = normalized(source.path);
|
|
485
|
+
if (pathLabel.endsWith('/post-coordinator-collected') || pathLabel.includes('/post-coordinator-collected'))
|
|
486
|
+
return 80;
|
|
487
|
+
if (pathLabel.endsWith('/coordinator-collected') || pathLabel.includes('/coordinator-collected'))
|
|
488
|
+
return 70;
|
|
414
489
|
if (pathLabel.endsWith('/collected-resolved') || pathLabel.includes('/collected-resolved-'))
|
|
415
490
|
return 60;
|
|
416
491
|
if (pathLabel.endsWith('/collected-with-decisions') || pathLabel.includes('/collected-with-decisions-'))
|
|
@@ -433,6 +508,283 @@ function lifetimeRunFamilyKey(source) {
|
|
|
433
508
|
const start = agentRunsIndex >= 0 ? agentRunsIndex + 1 : 0;
|
|
434
509
|
return parts[start] ?? source.path;
|
|
435
510
|
}
|
|
511
|
+
function lifetimeRunRootKey(source) {
|
|
512
|
+
const parts = source.path.split(/[\\/]/g).filter(Boolean);
|
|
513
|
+
if (!parts.length)
|
|
514
|
+
return source.path;
|
|
515
|
+
const agentRunsIndex = parts.lastIndexOf('agent-runs');
|
|
516
|
+
const start = agentRunsIndex >= 0 ? agentRunsIndex + 1 : 0;
|
|
517
|
+
const rootParts = parts.slice(start, start + 2);
|
|
518
|
+
return rootParts.length ? rootParts.join('/') : source.path;
|
|
519
|
+
}
|
|
520
|
+
function isDrainedAutonomousRunSnapshot(snapshot) {
|
|
521
|
+
const rawRun = recordValue(recordValue(snapshot.raw).run);
|
|
522
|
+
const autoDrain = recordValue(rawRun.autoDrain);
|
|
523
|
+
const autoDrainSummary = recordValue(autoDrain.summary);
|
|
524
|
+
const artifactSummary = recordValue(recordValue(rawRun.autoDrainArtifacts).summary);
|
|
525
|
+
const summary = Object.keys(autoDrainSummary).length ? autoDrainSummary : artifactSummary;
|
|
526
|
+
if (!Object.keys(summary).length)
|
|
527
|
+
return false;
|
|
528
|
+
if (textValue(summary.rerunManifestTerminalState, '') !== 'drained')
|
|
529
|
+
return false;
|
|
530
|
+
if (numberValue(summary.remainingReadyCount) > 0)
|
|
531
|
+
return false;
|
|
532
|
+
if (numberValue(summary.humanBlockedCount) > 0 || numberValue(summary.humanBlockedDecisionCount) > 0)
|
|
533
|
+
return false;
|
|
534
|
+
if (numberValue(summary.conflictBlockedCount) > 0 || numberValue(summary.rerunTaskCount) > 0)
|
|
535
|
+
return false;
|
|
536
|
+
return numberValue(summary.committedDecisionCount) > 0 || numberValue(summary.terminalCount) > 0 || numberValue(summary.decisionCount) > 0;
|
|
537
|
+
}
|
|
538
|
+
function collapseSupersededLifetimeReviewJobs(jobs) {
|
|
539
|
+
const resolvedAtByJob = new Map();
|
|
540
|
+
for (const job of jobs) {
|
|
541
|
+
if (!isResolvedCoordinatorReviewRecord(job))
|
|
542
|
+
continue;
|
|
543
|
+
const key = lifetimeReviewDedupeKey(job);
|
|
544
|
+
if (!key)
|
|
545
|
+
continue;
|
|
546
|
+
resolvedAtByJob.set(key, Math.max(resolvedAtByJob.get(key) ?? 0, numberValue(job.generatedAt)));
|
|
547
|
+
}
|
|
548
|
+
return jobs.filter((job) => {
|
|
549
|
+
if (!isOpenCoordinatorReviewRecord(job))
|
|
550
|
+
return true;
|
|
551
|
+
const key = lifetimeReviewDedupeKey(job);
|
|
552
|
+
if (!key)
|
|
553
|
+
return true;
|
|
554
|
+
return (resolvedAtByJob.get(key) ?? 0) < numberValue(job.generatedAt);
|
|
555
|
+
});
|
|
556
|
+
}
|
|
557
|
+
function dedupeLifetimeDashboardJobs(jobs) {
|
|
558
|
+
const byTask = new Map();
|
|
559
|
+
for (const job of jobs) {
|
|
560
|
+
const key = lifetimeJobDedupeKey(job);
|
|
561
|
+
if (!key)
|
|
562
|
+
continue;
|
|
563
|
+
const current = byTask.get(key);
|
|
564
|
+
if (!current || compareLifetimeJobPreference(job, current) > 0)
|
|
565
|
+
byTask.set(key, job);
|
|
566
|
+
}
|
|
567
|
+
return Array.from(byTask.values()).sort((left, right) => {
|
|
568
|
+
return numberValue(right.generatedAt) - numberValue(left.generatedAt)
|
|
569
|
+
|| textValue(left.lane, '').localeCompare(textValue(right.lane, ''))
|
|
570
|
+
|| textValue(left.title, '').localeCompare(textValue(right.title, ''));
|
|
571
|
+
});
|
|
572
|
+
}
|
|
573
|
+
function lifetimeJobDedupeKey(job) {
|
|
574
|
+
return canonicalLifetimeTaskKey(textValue(job.originalJobId ?? job.taskId ?? job.id ?? job.jobId, ''));
|
|
575
|
+
}
|
|
576
|
+
function compareLifetimeJobPreference(left, right) {
|
|
577
|
+
return lifetimeJobPreference(left) - lifetimeJobPreference(right)
|
|
578
|
+
|| numberValue(left.generatedAt) - numberValue(right.generatedAt)
|
|
579
|
+
|| textValue(right.sourceLabel, '').localeCompare(textValue(left.sourceLabel, ''));
|
|
580
|
+
}
|
|
581
|
+
function lifetimeJobPreference(job) {
|
|
582
|
+
const status = normalized(job.status);
|
|
583
|
+
const bucket = normalized(job.bucket);
|
|
584
|
+
const liveness = normalized(job.liveness);
|
|
585
|
+
let score = 0;
|
|
586
|
+
if (status === 'running')
|
|
587
|
+
score += 120;
|
|
588
|
+
else if (status === 'completed')
|
|
589
|
+
score += 100;
|
|
590
|
+
else if (status === 'failed' || status === 'blocked')
|
|
591
|
+
score += 70;
|
|
592
|
+
else if (['queued', 'pending', 'todo', 'open'].includes(status) || ['queued', 'todo'].includes(bucket))
|
|
593
|
+
score += 40;
|
|
594
|
+
if (liveness === 'missing' || status === 'planned')
|
|
595
|
+
score -= 60;
|
|
596
|
+
if (numberValue(job.changedPathCount))
|
|
597
|
+
score += 12;
|
|
598
|
+
if (numberValue(job.evidencePathCount))
|
|
599
|
+
score += 8;
|
|
600
|
+
if (numberValue(job.actualInputTokens) || numberValue(job.estimatedInputTokens))
|
|
601
|
+
score += 4;
|
|
602
|
+
return score;
|
|
603
|
+
}
|
|
604
|
+
function lifetimeReviewDedupeKey(job) {
|
|
605
|
+
return canonicalLifetimeTaskKey(textValue(job.originalJobId ?? job.jobId ?? job.taskId, ''));
|
|
606
|
+
}
|
|
607
|
+
function canonicalLifetimeTaskKey(value) {
|
|
608
|
+
return unscopedLifetimeTaskKey(value)
|
|
609
|
+
.trim()
|
|
610
|
+
.replace(/(?:-continuation)?-rerun(?:-\d+)?$/u, '')
|
|
611
|
+
.replace(/(?:-continuation)?-retry(?:-\d+)?$/u, '');
|
|
612
|
+
}
|
|
613
|
+
function unscopedLifetimeTaskKey(value) {
|
|
614
|
+
const trimmed = value.trim();
|
|
615
|
+
const match = /^(?:run|collection|continuation):.+:([^:]+)$/u.exec(trimmed);
|
|
616
|
+
return match?.[1] ?? trimmed;
|
|
617
|
+
}
|
|
618
|
+
function isOpenCoordinatorReviewRecord(job) {
|
|
619
|
+
if (isResolvedCoordinatorReviewRecord(job))
|
|
620
|
+
return false;
|
|
621
|
+
return isCoordinatorPortBucket(job.bucket)
|
|
622
|
+
|| isCoordinatorPortBucket(job.disposition)
|
|
623
|
+
|| isCoordinatorPortBucket(job.mergeReadiness)
|
|
624
|
+
|| normalized(job.status) === 'needs-review';
|
|
625
|
+
}
|
|
626
|
+
function isResolvedCoordinatorReviewRecord(job) {
|
|
627
|
+
const bucket = normalized(job.bucket);
|
|
628
|
+
if (bucket === 'review-resolved' || bucket === 'resolved-review' || job.reviewResolved === true)
|
|
629
|
+
return true;
|
|
630
|
+
const status = textValue(job.coordinatorDecisionStatus ?? recordValue(job.coordinatorDecision).status, '');
|
|
631
|
+
return Boolean(status) && isResolvedCoordinatorDecision(status);
|
|
632
|
+
}
|
|
633
|
+
async function readLifetimeQueueBacklog(cwd) {
|
|
634
|
+
const root = path.join(cwd, '.loom', 'queues');
|
|
635
|
+
const stat = await fs.stat(root).catch(() => undefined);
|
|
636
|
+
if (!stat?.isDirectory())
|
|
637
|
+
return { entries: [], manifests: [], sourceCount: 0, paths: [], generatedAt: 0 };
|
|
638
|
+
const queueDirs = await fs.readdir(root, { withFileTypes: true }).catch(() => []);
|
|
639
|
+
const entriesById = new Map();
|
|
640
|
+
const manifests = [];
|
|
641
|
+
const paths = [];
|
|
642
|
+
let generatedAt = 0;
|
|
643
|
+
for (const queueDir of queueDirs) {
|
|
644
|
+
if (!queueDir.isDirectory())
|
|
645
|
+
continue;
|
|
646
|
+
const dir = path.join(root, queueDir.name);
|
|
647
|
+
const manifestFile = await preferredQueueManifestFile(dir);
|
|
648
|
+
if (manifestFile) {
|
|
649
|
+
const manifestStat = await fs.stat(manifestFile).catch(() => undefined);
|
|
650
|
+
generatedAt = Math.max(generatedAt, manifestStat?.mtimeMs ?? 0);
|
|
651
|
+
const manifest = await readQueueCapacityManifest(cwd, manifestFile);
|
|
652
|
+
if (manifest)
|
|
653
|
+
manifests.push(manifest);
|
|
654
|
+
}
|
|
655
|
+
for (const taskFile of await queueTaskFiles(dir)) {
|
|
656
|
+
const fileStat = await fs.stat(taskFile).catch(() => undefined);
|
|
657
|
+
generatedAt = Math.max(generatedAt, fileStat?.mtimeMs ?? 0);
|
|
658
|
+
paths.push(path.relative(cwd, taskFile));
|
|
659
|
+
const tasks = await readQueueTaskFile(taskFile);
|
|
660
|
+
for (const task of tasks) {
|
|
661
|
+
const id = textValue(task.id ?? task.taskId ?? task.title, '');
|
|
662
|
+
if (!id)
|
|
663
|
+
continue;
|
|
664
|
+
entriesById.set(id, normalizeQueueBacklogEntry(cwd, queueDir.name, taskFile, task));
|
|
665
|
+
if (entriesById.size >= LIFETIME_DASHBOARD_MAX_QUEUE_TASKS)
|
|
666
|
+
break;
|
|
667
|
+
}
|
|
668
|
+
if (entriesById.size >= LIFETIME_DASHBOARD_MAX_QUEUE_TASKS)
|
|
669
|
+
break;
|
|
670
|
+
}
|
|
671
|
+
if (entriesById.size >= LIFETIME_DASHBOARD_MAX_QUEUE_TASKS)
|
|
672
|
+
break;
|
|
673
|
+
}
|
|
674
|
+
return {
|
|
675
|
+
entries: Array.from(entriesById.values()),
|
|
676
|
+
manifests,
|
|
677
|
+
sourceCount: paths.length,
|
|
678
|
+
paths,
|
|
679
|
+
generatedAt
|
|
680
|
+
};
|
|
681
|
+
}
|
|
682
|
+
async function queueTaskFiles(dir) {
|
|
683
|
+
const entries = await fs.readdir(dir, { withFileTypes: true }).catch(() => []);
|
|
684
|
+
const candidates = [];
|
|
685
|
+
for (const entry of entries) {
|
|
686
|
+
if (!entry.isFile() || !/^tasks(?:\.(?:remaining|backlog)-[\w.-]+)?\.json$/.test(entry.name))
|
|
687
|
+
continue;
|
|
688
|
+
const file = path.join(dir, entry.name);
|
|
689
|
+
const stat = await fs.stat(file).catch(() => undefined);
|
|
690
|
+
candidates.push({
|
|
691
|
+
file,
|
|
692
|
+
mtimeMs: stat?.mtimeMs ?? 0,
|
|
693
|
+
preferred: entry.name.startsWith('tasks.backlog-') ? 2 : entry.name.startsWith('tasks.remaining-') ? 1 : 0
|
|
694
|
+
});
|
|
695
|
+
}
|
|
696
|
+
return candidates
|
|
697
|
+
.sort((left, right) => left.mtimeMs - right.mtimeMs || left.preferred - right.preferred || left.file.localeCompare(right.file))
|
|
698
|
+
.map((candidate) => candidate.file);
|
|
699
|
+
}
|
|
700
|
+
async function preferredQueueManifestFile(dir) {
|
|
701
|
+
const entries = await fs.readdir(dir, { withFileTypes: true }).catch(() => []);
|
|
702
|
+
const candidates = [];
|
|
703
|
+
for (const entry of entries) {
|
|
704
|
+
if (!entry.isFile() || !/^manifest(?:\.[\w.-]+)?\.json$/.test(entry.name))
|
|
705
|
+
continue;
|
|
706
|
+
const file = path.join(dir, entry.name);
|
|
707
|
+
const stat = await fs.stat(file).catch(() => undefined);
|
|
708
|
+
candidates.push({
|
|
709
|
+
file,
|
|
710
|
+
mtimeMs: stat?.mtimeMs ?? 0,
|
|
711
|
+
preferred: entry.name.includes('high-concurrency') ? 2 : entry.name === 'manifest.json' ? 1 : 0
|
|
712
|
+
});
|
|
713
|
+
}
|
|
714
|
+
return candidates.sort((left, right) => right.preferred - left.preferred || right.mtimeMs - left.mtimeMs || left.file.localeCompare(right.file))[0]?.file;
|
|
715
|
+
}
|
|
716
|
+
async function readQueueCapacityManifest(cwd, file) {
|
|
717
|
+
const raw = recordValue(await readJsonFile(file));
|
|
718
|
+
if (!Object.keys(raw).length)
|
|
719
|
+
return undefined;
|
|
720
|
+
const computeRows = recordArray(raw.compute);
|
|
721
|
+
const computeById = new Map(computeRows.map((entry) => [textValue(entry.id, ''), entry]));
|
|
722
|
+
const defaultComputeId = textValue(recordValue(raw.policy).defaultCompute, textValue(computeRows[0]?.id, ''));
|
|
723
|
+
const defaultCompute = recordValue(computeById.get(defaultComputeId) ?? computeRows[0]);
|
|
724
|
+
const defaultConcurrency = numberValue(recordValue(raw.policy).defaultConcurrency);
|
|
725
|
+
const computeMaxConcurrency = computeRows.reduce((max, entry) => Math.max(max, numberValue(entry.maxConcurrency)), 0);
|
|
726
|
+
const manifestMaxConcurrency = numberValue(raw.maxConcurrency);
|
|
727
|
+
const lanes = recordArray(raw.lanes).map((lane) => {
|
|
728
|
+
const computeId = textValue(lane.compute, defaultComputeId);
|
|
729
|
+
const compute = recordValue(computeById.get(computeId) ?? defaultCompute);
|
|
730
|
+
return {
|
|
731
|
+
id: textValue(lane.id, 'lane'),
|
|
732
|
+
title: textValue(lane.title ?? lane.id, 'Lane'),
|
|
733
|
+
layer: textValue(lane.layer, ''),
|
|
734
|
+
compute: computeId,
|
|
735
|
+
model: textValue(compute.model, textValue(compute.id, '')),
|
|
736
|
+
maxConcurrency: numberValue(lane.maxConcurrency) || 1
|
|
737
|
+
};
|
|
738
|
+
});
|
|
739
|
+
return {
|
|
740
|
+
path: path.relative(cwd, file),
|
|
741
|
+
id: textValue(raw.id, path.basename(file, '.json')),
|
|
742
|
+
title: textValue(raw.title, 'Swarm manifest'),
|
|
743
|
+
defaultConcurrency,
|
|
744
|
+
computeMaxConcurrency,
|
|
745
|
+
maxConcurrency: manifestMaxConcurrency || defaultConcurrency || computeMaxConcurrency || lanes.reduce((sum, lane) => sum + lane.maxConcurrency, 0),
|
|
746
|
+
lanes
|
|
747
|
+
};
|
|
748
|
+
}
|
|
749
|
+
async function readQueueTaskFile(file) {
|
|
750
|
+
const raw = await readJsonFile(file);
|
|
751
|
+
if (Array.isArray(raw))
|
|
752
|
+
return raw.map(recordValue).filter((entry) => Object.keys(entry).length);
|
|
753
|
+
const record = recordValue(raw);
|
|
754
|
+
return recordArray(record.tasks ?? record.entries ?? record.items);
|
|
755
|
+
}
|
|
756
|
+
function normalizeQueueBacklogEntry(cwd, queueId, file, task) {
|
|
757
|
+
const id = textValue(task.id ?? task.taskId ?? task.title, 'task');
|
|
758
|
+
const queueStatus = textValue(task.status ?? task.state, 'open');
|
|
759
|
+
const status = ['done', 'completed', 'failed', 'blocked'].includes(normalized(queueStatus)) ? queueStatus : 'todo';
|
|
760
|
+
const sourceRefs = stringArray(task.sourceRefs);
|
|
761
|
+
const targetRefs = stringArray(task.targetRefs);
|
|
762
|
+
const allowedWrites = stringArray(task.allowedWrites);
|
|
763
|
+
const files = uniquePaths([...targetRefs, ...allowedWrites, ...sourceRefs]).slice(0, 40);
|
|
764
|
+
return {
|
|
765
|
+
id,
|
|
766
|
+
taskId: id,
|
|
767
|
+
title: textValue(task.title ?? task.objective ?? id, id),
|
|
768
|
+
objective: textValue(task.objective ?? task.summary, ''),
|
|
769
|
+
status,
|
|
770
|
+
queueStatus,
|
|
771
|
+
ready: status === 'todo',
|
|
772
|
+
lane: textValue(task.lane ?? task.groupId ?? task.epicId, queueId),
|
|
773
|
+
group: textValue(task.groupId ?? task.epicId, queueId),
|
|
774
|
+
epicId: textValue(task.epicId, ''),
|
|
775
|
+
priority: numberValue(task.priority),
|
|
776
|
+
changedPaths: files,
|
|
777
|
+
changedPathCount: files.length,
|
|
778
|
+
sourceRefs,
|
|
779
|
+
targetRefs,
|
|
780
|
+
allowedWrites,
|
|
781
|
+
acceptance: stringArray(task.acceptance),
|
|
782
|
+
verification: recordArray(task.verification),
|
|
783
|
+
tags: stringArray(task.tags),
|
|
784
|
+
sourceLabel: path.relative(cwd, file),
|
|
785
|
+
sourceQueue: queueId
|
|
786
|
+
};
|
|
787
|
+
}
|
|
436
788
|
async function readLifetimeDashboardResetCutoff(root) {
|
|
437
789
|
const reset = recordValue(await readJsonFile(path.join(root, LIFETIME_DASHBOARD_RESET_FILE)));
|
|
438
790
|
return numberValue(reset.resetAt ?? reset.generatedAt);
|
|
@@ -467,23 +819,47 @@ async function findLifetimeDashboardArtifactFiles(root, input) {
|
|
|
467
819
|
await walk(root, 0);
|
|
468
820
|
return out;
|
|
469
821
|
}
|
|
470
|
-
function combineLifetimeDashboardSnapshots(options, discoveredSources, snapshots, reviewDecisions) {
|
|
471
|
-
const
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
822
|
+
async function combineLifetimeDashboardSnapshots(options, discoveredSources, snapshots, reviewDecisions, queueBacklog) {
|
|
823
|
+
const drainedRunRoots = new Set(snapshots
|
|
824
|
+
.filter((entry) => entry.source.kind === 'run' && isDrainedAutonomousRunSnapshot(entry.snapshot))
|
|
825
|
+
.map((entry) => lifetimeRunRootKey(entry.source)));
|
|
826
|
+
const visibleSnapshots = snapshots.filter((entry) => {
|
|
827
|
+
if (entry.source.kind !== 'collection')
|
|
828
|
+
return true;
|
|
829
|
+
return !drainedRunRoots.has(lifetimeRunRootKey(entry.source));
|
|
830
|
+
});
|
|
831
|
+
const autoDrainDelays = lifetimeAutoDrainDelayRecords(visibleSnapshots);
|
|
832
|
+
const jobs = dedupeLifetimeDashboardJobs(collapseSupersededLifetimeReviewJobs(applyCoordinatorReviewDecisions(visibleSnapshots.flatMap(({ source, snapshot }) => {
|
|
833
|
+
const autoDrainDelay = lifetimeAutoDrainDelayRecord(source, snapshot);
|
|
834
|
+
return recordArray(snapshot.jobs).map((job) => {
|
|
835
|
+
const sourceJobId = textValue(job.id ?? job.jobId ?? job.taskId, 'job');
|
|
836
|
+
return withRecomputedCostFields({
|
|
837
|
+
...job,
|
|
838
|
+
id: lifetimeScopedId(source, sourceJobId),
|
|
839
|
+
sourceJobId,
|
|
840
|
+
originalJobId: unscopedLifetimeTaskKey(sourceJobId),
|
|
841
|
+
sourceRun: source.run,
|
|
842
|
+
sourceCollection: source.collection,
|
|
843
|
+
sourceContinuation: source.continuation,
|
|
844
|
+
sourceLabel: source.label,
|
|
845
|
+
...(autoDrainDelay ? {
|
|
846
|
+
coordinationDelay: autoDrainDelay.reason,
|
|
847
|
+
autoDrainSkippedReason: autoDrainDelay.skippedReason,
|
|
848
|
+
autoDrainDirtyPathCount: autoDrainDelay.dirtyPathCount
|
|
849
|
+
} : {}),
|
|
850
|
+
generatedAt: numberValue(job.generatedAt) || numberValue(snapshot.generatedAt) || source.mtimeMs
|
|
851
|
+
});
|
|
852
|
+
});
|
|
853
|
+
}), reviewDecisions))).slice(0, LIFETIME_DASHBOARD_MAX_JOBS);
|
|
854
|
+
const humanActionAnswers = await readHumanActionAnswers(options);
|
|
855
|
+
const summary = {
|
|
856
|
+
...lifetimeDashboardSummary(jobs),
|
|
857
|
+
coordinationDelayCount: autoDrainDelays.length,
|
|
858
|
+
dirtyAutoDrainSkipCount: autoDrainDelays.filter((record) => record.skippedReason === 'dirty-worktree').length
|
|
859
|
+
};
|
|
860
|
+
const queueOverlay = lifetimeQueueBacklogOverlay(queueBacklog, jobs);
|
|
861
|
+
const latestGeneratedAt = Math.max(Date.now(), numberValue(queueBacklog.generatedAt), ...visibleSnapshots.map((entry) => numberValue(entry.snapshot.generatedAt)), ...discoveredSources.map((source) => source.mtimeMs));
|
|
862
|
+
const events = visibleSnapshots.flatMap(({ source, snapshot }) => recordArray(snapshot.events).map((event) => ({
|
|
487
863
|
...event,
|
|
488
864
|
sourceLabel: source.label,
|
|
489
865
|
message: textValue(event.message, textValue(event.type, 'event')),
|
|
@@ -492,14 +868,19 @@ function combineLifetimeDashboardSnapshots(options, discoveredSources, snapshots
|
|
|
492
868
|
return {
|
|
493
869
|
kind: 'frontier.loom-ui.lifetime-dashboard',
|
|
494
870
|
version: 1,
|
|
495
|
-
ok:
|
|
871
|
+
ok: true,
|
|
496
872
|
generatedAt: latestGeneratedAt,
|
|
497
873
|
cwd: options.cwd,
|
|
498
874
|
sources: {
|
|
499
875
|
workspace: options.cwd,
|
|
500
876
|
lifetimeRoot: path.join(options.cwd, 'agent-runs'),
|
|
877
|
+
queueRoot: path.join(options.cwd, '.loom', 'queues'),
|
|
501
878
|
sourceCount: discoveredSources.length,
|
|
502
|
-
loadedSourceCount:
|
|
879
|
+
loadedSourceCount: visibleSnapshots.length,
|
|
880
|
+
suppressedCollectionSourceCount: snapshots.length - visibleSnapshots.length,
|
|
881
|
+
queueSourceCount: queueBacklog.sourceCount,
|
|
882
|
+
coordinationDelayCount: autoDrainDelays.length,
|
|
883
|
+
...(humanActionAnswers.length ? { humanActionAnswers: await humanActionAnswerLogPath(options) } : {}),
|
|
503
884
|
...(reviewDecisions.length ? { coordinatorReviewDecisions: coordinatorReviewDecisionPath(options.cwd) } : {})
|
|
504
885
|
},
|
|
505
886
|
summary,
|
|
@@ -508,22 +889,32 @@ function combineLifetimeDashboardSnapshots(options, discoveredSources, snapshots
|
|
|
508
889
|
quality: {},
|
|
509
890
|
timeSeries: lifetimeTimeSeries(jobs, events),
|
|
510
891
|
lanes: lifetimeLaneRows(jobs),
|
|
892
|
+
capacity: lifetimeCapacitySummary(queueBacklog, jobs, queueOverlay.entries),
|
|
511
893
|
jobs,
|
|
512
|
-
humanActions:
|
|
894
|
+
humanActions: visibleSnapshots.flatMap(({ snapshot }) => recordArray(snapshot.humanActions)).slice(-100),
|
|
513
895
|
humanActionAnswers,
|
|
514
896
|
events,
|
|
515
|
-
routing: lifetimeRoutingSummary(
|
|
897
|
+
routing: await lifetimeRoutingSummary(options.cwd, visibleSnapshots),
|
|
516
898
|
backlog: {
|
|
517
899
|
id: 'workspace-lifetime',
|
|
518
|
-
entryCount:
|
|
519
|
-
readyCount:
|
|
900
|
+
entryCount: queueOverlay.totalCount,
|
|
901
|
+
readyCount: queueOverlay.readyCount,
|
|
902
|
+
activeCount: queueOverlay.activeCount,
|
|
903
|
+
doneCount: queueOverlay.doneCount,
|
|
904
|
+
failedCount: queueOverlay.failedCount,
|
|
905
|
+
representedCount: queueOverlay.representedCount,
|
|
906
|
+
entries: queueOverlay.entries
|
|
520
907
|
},
|
|
521
908
|
raw: {
|
|
522
909
|
lifetime: {
|
|
523
910
|
mode: 'workspace',
|
|
524
911
|
sourceCount: discoveredSources.length,
|
|
525
|
-
loadedSourceCount:
|
|
526
|
-
|
|
912
|
+
loadedSourceCount: visibleSnapshots.length,
|
|
913
|
+
suppressedCollectionSourceCount: snapshots.length - visibleSnapshots.length,
|
|
914
|
+
autoDrainDelays,
|
|
915
|
+
sources: discoveredSources.slice(0, LIFETIME_DASHBOARD_MAX_SOURCES),
|
|
916
|
+
manifests: queueBacklog.manifests,
|
|
917
|
+
queueSources: queueBacklog.paths
|
|
527
918
|
}
|
|
528
919
|
}
|
|
529
920
|
};
|
|
@@ -531,6 +922,723 @@ function combineLifetimeDashboardSnapshots(options, discoveredSources, snapshots
|
|
|
531
922
|
function lifetimeScopedId(source, id) {
|
|
532
923
|
return `${source.id}:${id}`.replaceAll(/[^\w:.-]+/g, '-');
|
|
533
924
|
}
|
|
925
|
+
function lifetimeAutoDrainDelayRecords(entries) {
|
|
926
|
+
return entries
|
|
927
|
+
.map(({ source, snapshot }) => lifetimeAutoDrainDelayRecord(source, snapshot))
|
|
928
|
+
.filter((record) => Boolean(record));
|
|
929
|
+
}
|
|
930
|
+
function lifetimeAutoDrainDelayRecord(source, snapshot) {
|
|
931
|
+
const rawRun = recordValue(recordValue(snapshot.raw).run);
|
|
932
|
+
const autoDrain = recordValue(rawRun.autoDrain);
|
|
933
|
+
const skippedReason = textValue(autoDrain.skippedReason, '');
|
|
934
|
+
if (skippedReason !== 'dirty-worktree')
|
|
935
|
+
return undefined;
|
|
936
|
+
const summary = recordValue(autoDrain.summary);
|
|
937
|
+
const dirtyPaths = stringArray(autoDrain.dirtyPaths);
|
|
938
|
+
return {
|
|
939
|
+
source: source.path,
|
|
940
|
+
sourceLabel: source.label,
|
|
941
|
+
reason: 'apply-delayed-by-dirty-worktree',
|
|
942
|
+
skippedReason,
|
|
943
|
+
dirtyPathCount: dirtyPaths.length,
|
|
944
|
+
dirtyPaths: dirtyPaths.slice(0, 12),
|
|
945
|
+
remainingReadyCount: numberValue(summary.remainingReadyCount),
|
|
946
|
+
generatedAt: numberValue(autoDrain.generatedAt) || numberValue(snapshot.generatedAt) || source.mtimeMs
|
|
947
|
+
};
|
|
948
|
+
}
|
|
949
|
+
async function enrichLifetimeRunSnapshotEvidence(cwd, source, snapshot) {
|
|
950
|
+
if (source.kind !== 'run' || !source.run)
|
|
951
|
+
return snapshot;
|
|
952
|
+
const runRoot = safeCwdRelativeDirectory(cwd, source.run);
|
|
953
|
+
if (!runRoot)
|
|
954
|
+
return snapshot;
|
|
955
|
+
const jobs = recordArray(snapshot.jobs);
|
|
956
|
+
if (!jobs.length)
|
|
957
|
+
return snapshot;
|
|
958
|
+
const enrichedJobs = await Promise.all(jobs.map((job) => enrichLifetimeRunJobEvidence(cwd, runRoot, job)));
|
|
959
|
+
return {
|
|
960
|
+
...snapshot,
|
|
961
|
+
jobs: enrichedJobs
|
|
962
|
+
};
|
|
963
|
+
}
|
|
964
|
+
async function enrichLifetimeRunJobEvidence(cwd, runRoot, job) {
|
|
965
|
+
const jobDir = await findBestRawRunJobDir(runRoot, rawRunJobIdCandidates(job));
|
|
966
|
+
if (!jobDir)
|
|
967
|
+
return job;
|
|
968
|
+
const evidenceDir = path.join(jobDir, 'evidence');
|
|
969
|
+
const eventsPath = path.join(jobDir, 'codex-events.jsonl');
|
|
970
|
+
const rootMerge = recordValue(await readJsonFile(path.join(jobDir, 'merge.json')));
|
|
971
|
+
const evidenceMerge = recordValue(await readJsonFile(path.join(evidenceDir, 'merge.json')));
|
|
972
|
+
const evidenceRecord = recordValue(await readJsonFile(path.join(evidenceDir, 'evidence.json')));
|
|
973
|
+
const merge = Object.keys(rootMerge).length ? rootMerge : evidenceMerge;
|
|
974
|
+
const rawPatchPath = await firstExistingRelativePath(cwd, rawRunPatchCandidates(jobDir));
|
|
975
|
+
const usage = await readCodexEventUsageSummary(eventsPath);
|
|
976
|
+
const evidencePaths = await existingRelativePaths(cwd, [
|
|
977
|
+
path.join(jobDir, 'last-message.md'),
|
|
978
|
+
eventsPath,
|
|
979
|
+
path.join(jobDir, 'merge.json'),
|
|
980
|
+
path.join(evidenceDir, 'last-message.md'),
|
|
981
|
+
path.join(evidenceDir, 'handoff.md'),
|
|
982
|
+
path.join(evidenceDir, 'evidence.json'),
|
|
983
|
+
path.join(evidenceDir, 'merge.json'),
|
|
984
|
+
path.join(evidenceDir, 'human-question.json'),
|
|
985
|
+
path.join(evidenceDir, 'resource-allocation.json'),
|
|
986
|
+
path.join(evidenceDir, 'model-availability.json'),
|
|
987
|
+
...rawRunPatchCandidates(jobDir)
|
|
988
|
+
]);
|
|
989
|
+
if (!evidencePaths.length && !rawPatchPath && !Object.keys(merge).length)
|
|
990
|
+
return job;
|
|
991
|
+
const patchChangedPaths = await readPatchChangedPathList(cwd, rawPatchPath);
|
|
992
|
+
const changedPaths = uniquePaths([
|
|
993
|
+
...stringArray(job.changedPaths),
|
|
994
|
+
...stringArray(merge.changedPaths),
|
|
995
|
+
...patchChangedPaths
|
|
996
|
+
]);
|
|
997
|
+
const ownershipViolations = uniquePaths([
|
|
998
|
+
...stringArray(job.ownershipViolations),
|
|
999
|
+
...stringArray(merge.ownershipViolations)
|
|
1000
|
+
]);
|
|
1001
|
+
const status = lifetimeRunEvidenceStatus(job, merge, evidencePaths);
|
|
1002
|
+
const bucket = lifetimeRunEvidenceBucket(job, status, evidencePaths, rawPatchPath);
|
|
1003
|
+
const collectReasonClasses = uniquePaths([
|
|
1004
|
+
...stringArray(job.collectReasonClasses),
|
|
1005
|
+
status === 'failed' && evidencePaths.length ? 'worker failed with evidence' : 'raw run evidence discovered'
|
|
1006
|
+
]);
|
|
1007
|
+
const mergedEvidencePaths = uniquePaths([...stringArray(job.evidencePaths), ...evidencePaths]);
|
|
1008
|
+
const commandEvidence = commandEvidenceFromRecords(job, merge, evidenceRecord);
|
|
1009
|
+
return withRecomputedCostFields({
|
|
1010
|
+
...job,
|
|
1011
|
+
status,
|
|
1012
|
+
bucket,
|
|
1013
|
+
disposition: textValue(merge.disposition, textValue(job.disposition, status)),
|
|
1014
|
+
mergeReadiness: textValue(merge.mergeReadiness, textValue(job.mergeReadiness, status)),
|
|
1015
|
+
...(rawPatchPath ? { patchPath: rawPatchPath, artifactPaths: uniquePaths([rawPatchPath, ...stringArray(job.artifactPaths)]) } : {}),
|
|
1016
|
+
changedPaths,
|
|
1017
|
+
changedPathCount: changedPaths.length || numberValue(job.changedPathCount),
|
|
1018
|
+
ownershipViolations,
|
|
1019
|
+
ownershipViolationCount: ownershipViolations.length || numberValue(job.ownershipViolationCount),
|
|
1020
|
+
...(usage.inputTokens ? { actualInputTokens: usage.inputTokens, inputTokens: usage.inputTokens } : {}),
|
|
1021
|
+
...(!usage.inputTokens && usage.estimatedInputTokens ? { estimatedInputTokens: usage.estimatedInputTokens } : {}),
|
|
1022
|
+
...(usage.cachedInputTokens ? { cachedInputTokens: usage.cachedInputTokens } : {}),
|
|
1023
|
+
...(usage.uncachedInputTokens ? { uncachedInputTokens: usage.uncachedInputTokens } : {}),
|
|
1024
|
+
...(usage.outputTokens ? { actualOutputTokens: usage.outputTokens, outputTokens: usage.outputTokens } : {}),
|
|
1025
|
+
...(usage.reasoningOutputTokens ? { reasoningOutputTokens: usage.reasoningOutputTokens } : {}),
|
|
1026
|
+
...(usage.eventCount || usage.estimatedInputTokens ? {
|
|
1027
|
+
usage: {
|
|
1028
|
+
...recordValue(job.usage),
|
|
1029
|
+
input_tokens: usage.inputTokens,
|
|
1030
|
+
cached_input_tokens: usage.cachedInputTokens,
|
|
1031
|
+
uncached_input_tokens: usage.uncachedInputTokens,
|
|
1032
|
+
output_tokens: usage.outputTokens,
|
|
1033
|
+
reasoning_output_tokens: usage.reasoningOutputTokens,
|
|
1034
|
+
estimated_input_tokens: usage.estimatedInputTokens,
|
|
1035
|
+
estimated_from_event_bytes: usage.estimatedFromEventBytes,
|
|
1036
|
+
source: usage.eventCount ? 'codex-events.jsonl' : 'codex-events.jsonl-estimate',
|
|
1037
|
+
event_count: usage.eventCount
|
|
1038
|
+
}
|
|
1039
|
+
} : {}),
|
|
1040
|
+
evidencePaths: mergedEvidencePaths,
|
|
1041
|
+
evidencePathCount: mergedEvidencePaths.length,
|
|
1042
|
+
reasons: stringArray(job.reasons).length ? stringArray(job.reasons) : stringArray(merge.reasons),
|
|
1043
|
+
commandsPassed: commandEvidence.passed,
|
|
1044
|
+
commandsFailed: commandEvidence.failed,
|
|
1045
|
+
collectReasonClasses,
|
|
1046
|
+
runEvidenceRecovered: true
|
|
1047
|
+
});
|
|
1048
|
+
}
|
|
1049
|
+
function rawRunJobIdCandidates(job) {
|
|
1050
|
+
const values = [
|
|
1051
|
+
textValue(job.originalJobId, ''),
|
|
1052
|
+
textValue(job.jobId, ''),
|
|
1053
|
+
textValue(job.id, ''),
|
|
1054
|
+
textValue(job.taskId, '')
|
|
1055
|
+
].filter(Boolean);
|
|
1056
|
+
const out = new Set();
|
|
1057
|
+
for (const value of values) {
|
|
1058
|
+
out.add(value);
|
|
1059
|
+
const parts = value.split(':').filter(Boolean);
|
|
1060
|
+
if (parts.length)
|
|
1061
|
+
out.add(parts[parts.length - 1]);
|
|
1062
|
+
}
|
|
1063
|
+
return Array.from(out);
|
|
1064
|
+
}
|
|
1065
|
+
async function findBestRawRunJobDir(runRoot, candidates) {
|
|
1066
|
+
const matches = new Map();
|
|
1067
|
+
for (const candidate of candidates) {
|
|
1068
|
+
const direct = path.join(runRoot, candidate);
|
|
1069
|
+
if (await rawRunJobHasArtifacts(direct))
|
|
1070
|
+
matches.set(direct, await rawRunJobEvidenceScore(direct));
|
|
1071
|
+
for (const match of await findRawRunJobDirs(runRoot, candidate, 0)) {
|
|
1072
|
+
matches.set(match, await rawRunJobEvidenceScore(match));
|
|
1073
|
+
}
|
|
1074
|
+
}
|
|
1075
|
+
return Array.from(matches.entries()).sort((left, right) => right[1] - left[1] || right[0].localeCompare(left[0]))[0]?.[0];
|
|
1076
|
+
}
|
|
1077
|
+
async function rawRunJobEvidenceScore(jobDir) {
|
|
1078
|
+
let score = 0;
|
|
1079
|
+
for (const [relative, weight] of [
|
|
1080
|
+
['last-message.md', 100],
|
|
1081
|
+
['evidence/merge.json', 80],
|
|
1082
|
+
['merge.json', 80],
|
|
1083
|
+
['evidence/evidence.json', 50],
|
|
1084
|
+
['evidence/changes.patch', 40],
|
|
1085
|
+
['changes.patch', 40],
|
|
1086
|
+
['codex-events.jsonl', 20]
|
|
1087
|
+
]) {
|
|
1088
|
+
const stat = await fs.stat(path.join(jobDir, relative)).catch(() => undefined);
|
|
1089
|
+
if (stat?.isFile())
|
|
1090
|
+
score += weight + Math.min(10, Math.floor(stat.size / 1024));
|
|
1091
|
+
}
|
|
1092
|
+
const stat = await fs.stat(jobDir).catch(() => undefined);
|
|
1093
|
+
return score + Math.floor((stat?.mtimeMs ?? 0) / 1_000_000_000);
|
|
1094
|
+
}
|
|
1095
|
+
function lifetimeRunEvidenceStatus(job, merge, evidencePaths) {
|
|
1096
|
+
const mergeStatus = textValue(coordinatorFacingMachineLabel(merge.status), '');
|
|
1097
|
+
if (mergeStatus)
|
|
1098
|
+
return mergeStatus;
|
|
1099
|
+
const status = textValue(coordinatorFacingMachineLabel(job.status), '');
|
|
1100
|
+
if (status)
|
|
1101
|
+
return status === 'failed' && evidencePaths.some((entry) => entry.endsWith('last-message.md')) ? 'completed' : status;
|
|
1102
|
+
return evidencePaths.some((entry) => entry.endsWith('last-message.md')) ? 'completed' : 'failed';
|
|
1103
|
+
}
|
|
1104
|
+
function lifetimeRunEvidenceBucket(job, status, evidencePaths, patchPath) {
|
|
1105
|
+
const bucket = textValue(coordinatorFacingMachineLabel(job.bucket), '');
|
|
1106
|
+
if (status === 'running')
|
|
1107
|
+
return 'running';
|
|
1108
|
+
if (status === 'completed')
|
|
1109
|
+
return bucket && bucket !== 'failed-evidence' ? bucket : 'completed';
|
|
1110
|
+
if (status === 'failed' && (evidencePaths.length || patchPath))
|
|
1111
|
+
return 'worker-failed';
|
|
1112
|
+
return bucket || (status === 'failed' ? 'failed-evidence' : status);
|
|
1113
|
+
}
|
|
1114
|
+
async function readLatestDrainCoordinatorSnapshot(cwd) {
|
|
1115
|
+
const root = path.join(cwd, 'agent-runs', 'frontier-swarm-codex');
|
|
1116
|
+
const drains = await findDrainCoordinatorRunDirs(root);
|
|
1117
|
+
const activeDrainRoot = drains[0] ? drainRootForRunDir(drains[0]) : '';
|
|
1118
|
+
const runDirs = drains
|
|
1119
|
+
.filter((runDir) => drainRootForRunDir(runDir) === activeDrainRoot)
|
|
1120
|
+
.slice(0, LIFETIME_DASHBOARD_MAX_DRAIN_RUNS);
|
|
1121
|
+
const jobs = [];
|
|
1122
|
+
for (const runDir of runDirs)
|
|
1123
|
+
jobs.push(...await readDrainCoordinatorJobs(cwd, runDir));
|
|
1124
|
+
if (!jobs.length)
|
|
1125
|
+
return undefined;
|
|
1126
|
+
const generatedAt = Math.max(...jobs.map((job) => numberValue(job.generatedAt)), Date.now());
|
|
1127
|
+
return {
|
|
1128
|
+
ok: true,
|
|
1129
|
+
generatedAt,
|
|
1130
|
+
cwd,
|
|
1131
|
+
sources: {
|
|
1132
|
+
activeDrain: runDirs[0],
|
|
1133
|
+
activeDrainSources: runDirs
|
|
1134
|
+
},
|
|
1135
|
+
summary: lifetimeDashboardSummary(jobs),
|
|
1136
|
+
lanes: lifetimeLaneRows(jobs),
|
|
1137
|
+
jobs,
|
|
1138
|
+
events: activeRunEvents(jobs),
|
|
1139
|
+
raw: {
|
|
1140
|
+
activeDrain: {
|
|
1141
|
+
runDirs,
|
|
1142
|
+
jobCount: jobs.length,
|
|
1143
|
+
runningCount: jobs.filter((job) => textValue(job.status, '') === 'running').length
|
|
1144
|
+
}
|
|
1145
|
+
}
|
|
1146
|
+
};
|
|
1147
|
+
}
|
|
1148
|
+
async function findDrainCoordinatorRunDirs(root) {
|
|
1149
|
+
const out = [];
|
|
1150
|
+
const rootEntries = await fs.readdir(root, { withFileTypes: true }).catch(() => []);
|
|
1151
|
+
for (const drain of rootEntries) {
|
|
1152
|
+
if (!drain.isDirectory() || !drain.name.startsWith('drain-'))
|
|
1153
|
+
continue;
|
|
1154
|
+
const drainDir = path.join(root, drain.name);
|
|
1155
|
+
const iterationEntries = await fs.readdir(drainDir, { withFileTypes: true }).catch(() => []);
|
|
1156
|
+
for (const iteration of iterationEntries) {
|
|
1157
|
+
if (!iteration.isDirectory() || !iteration.name.startsWith('iteration-'))
|
|
1158
|
+
continue;
|
|
1159
|
+
for (const runDirName of ['coordinator-run', 'worker-run']) {
|
|
1160
|
+
const runDir = path.join(drainDir, iteration.name, runDirName);
|
|
1161
|
+
const stat = await fs.stat(runDir).catch(() => undefined);
|
|
1162
|
+
if (stat?.isDirectory())
|
|
1163
|
+
out.push({ dir: runDir, mtimeMs: stat.mtimeMs });
|
|
1164
|
+
}
|
|
1165
|
+
}
|
|
1166
|
+
}
|
|
1167
|
+
return out.sort((left, right) => right.mtimeMs - left.mtimeMs || right.dir.localeCompare(left.dir)).map((entry) => entry.dir);
|
|
1168
|
+
}
|
|
1169
|
+
function drainRootForRunDir(runDir) {
|
|
1170
|
+
return path.dirname(path.dirname(runDir));
|
|
1171
|
+
}
|
|
1172
|
+
async function readDrainCoordinatorJobs(cwd, coordinatorRunDir) {
|
|
1173
|
+
const entries = await fs.readdir(coordinatorRunDir, { withFileTypes: true }).catch(() => []);
|
|
1174
|
+
const liveLines = liveProcessLinesForPath(coordinatorRunDir);
|
|
1175
|
+
const now = Date.now();
|
|
1176
|
+
const jobs = [];
|
|
1177
|
+
const seenJobIds = new Set();
|
|
1178
|
+
for (const entry of entries) {
|
|
1179
|
+
if (!entry.isDirectory() || entry.name === 'streams')
|
|
1180
|
+
continue;
|
|
1181
|
+
seenJobIds.add(entry.name);
|
|
1182
|
+
jobs.push(await readDrainCoordinatorJob(cwd, coordinatorRunDir, path.join(coordinatorRunDir, entry.name), liveLines, now));
|
|
1183
|
+
}
|
|
1184
|
+
const pidEntries = await readRunPidEntries(coordinatorRunDir);
|
|
1185
|
+
const planJobs = await readRunPlanJobs(coordinatorRunDir);
|
|
1186
|
+
for (const entry of pidEntries) {
|
|
1187
|
+
const jobId = textValue(entry.jobId, '');
|
|
1188
|
+
if (!jobId || seenJobIds.has(jobId))
|
|
1189
|
+
continue;
|
|
1190
|
+
jobs.push(readDrainPidManifestJob(cwd, coordinatorRunDir, entry, planJobs.get(jobId), now));
|
|
1191
|
+
}
|
|
1192
|
+
return jobs.sort((left, right) => textValue(left.lane, '').localeCompare(textValue(right.lane, '')));
|
|
1193
|
+
}
|
|
1194
|
+
async function readRunPidEntries(runDir) {
|
|
1195
|
+
const pidManifest = recordValue(await readJsonFile(path.join(runDir, 'pids.json')));
|
|
1196
|
+
return recordArray(pidManifest.entries).filter((entry) => textValue(entry.role, '') === 'codex');
|
|
1197
|
+
}
|
|
1198
|
+
async function readRunPlanJobs(runDir) {
|
|
1199
|
+
const plan = recordValue(await readJsonFile(path.join(runDir, 'swarm-plan.json')));
|
|
1200
|
+
const entries = [];
|
|
1201
|
+
for (const job of recordArray(plan.jobs)) {
|
|
1202
|
+
const id = textValue(job.id, '');
|
|
1203
|
+
if (id)
|
|
1204
|
+
entries.push([id, job]);
|
|
1205
|
+
}
|
|
1206
|
+
return new Map(entries);
|
|
1207
|
+
}
|
|
1208
|
+
function readDrainPidManifestJob(cwd, coordinatorRunDir, entry, planJob, now) {
|
|
1209
|
+
const jobId = textValue(entry.jobId, 'job');
|
|
1210
|
+
const runKind = path.basename(coordinatorRunDir) === 'worker-run' ? 'worker' : 'coordinator';
|
|
1211
|
+
const task = recordValue(planJob?.task);
|
|
1212
|
+
const compute = recordValue(planJob?.compute);
|
|
1213
|
+
const command = stringArray(entry.command);
|
|
1214
|
+
const live = isProcessLive(numberValue(entry.pid), entry);
|
|
1215
|
+
const status = live ? 'running' : 'failed';
|
|
1216
|
+
const startedAt = numberValue(entry.startedAt);
|
|
1217
|
+
const lane = textValue(planJob?.lane ?? task.lane, drainCoordinatorLane(jobId, runKind));
|
|
1218
|
+
return withRecomputedCostFields({
|
|
1219
|
+
id: jobId,
|
|
1220
|
+
originalJobId: jobId,
|
|
1221
|
+
taskId: textValue(planJob?.taskId ?? task.id, jobId),
|
|
1222
|
+
title: textValue(planJob?.title ?? task.title, runKind === 'coordinator' ? `Coordinate lane review for ${lane}` : `Continue ${lane} work`),
|
|
1223
|
+
lane,
|
|
1224
|
+
status,
|
|
1225
|
+
bucket: status === 'running' ? 'running' : 'failed-evidence',
|
|
1226
|
+
disposition: status === 'running' ? 'active' : 'failed',
|
|
1227
|
+
agentId: jobId,
|
|
1228
|
+
workerId: jobId,
|
|
1229
|
+
model: textValue(compute.model, commandOptionValue(command, '--model') || 'gpt-5.5'),
|
|
1230
|
+
computeId: textValue(compute.id, runKind === 'coordinator' ? 'coordinator-agent' : 'continuation-worker'),
|
|
1231
|
+
reasoningEffort: textValue(compute.reasoningEffort, ''),
|
|
1232
|
+
startedAt: startedAt || undefined,
|
|
1233
|
+
durationMs: startedAt ? Math.max(0, now - startedAt) : 0,
|
|
1234
|
+
evidencePaths: [],
|
|
1235
|
+
evidencePathCount: 0,
|
|
1236
|
+
changedPathCount: 0,
|
|
1237
|
+
collectReasonClasses: status === 'running' ? [`active drain ${runKind}`] : [`missing ${runKind} output`],
|
|
1238
|
+
mergeReadiness: status,
|
|
1239
|
+
sourceRun: path.relative(cwd, coordinatorRunDir),
|
|
1240
|
+
sourceLabel: path.relative(cwd, coordinatorRunDir),
|
|
1241
|
+
generatedAt: now
|
|
1242
|
+
});
|
|
1243
|
+
}
|
|
1244
|
+
function commandOptionValue(command, option) {
|
|
1245
|
+
const index = command.indexOf(option);
|
|
1246
|
+
return index >= 0 ? textValue(command[index + 1], '') : '';
|
|
1247
|
+
}
|
|
1248
|
+
async function readDrainCoordinatorJob(cwd, coordinatorRunDir, jobDir, liveLines, now) {
|
|
1249
|
+
const jobId = path.basename(jobDir);
|
|
1250
|
+
const runKind = path.basename(coordinatorRunDir) === 'worker-run' ? 'worker' : 'coordinator';
|
|
1251
|
+
const evidenceDir = path.join(jobDir, 'evidence');
|
|
1252
|
+
const eventsPath = path.join(jobDir, 'codex-events.jsonl');
|
|
1253
|
+
const lastMessagePath = path.join(jobDir, 'last-message.md');
|
|
1254
|
+
const decisionsJson = path.join(evidenceDir, 'coordinator-decisions.json');
|
|
1255
|
+
const decisionsJsonl = path.join(evidenceDir, 'coordinator-decisions.jsonl');
|
|
1256
|
+
const modelAvailability = recordValue(await readJsonFile(path.join(evidenceDir, 'model-availability.json')));
|
|
1257
|
+
const evidenceRecord = recordValue(await readJsonFile(path.join(evidenceDir, 'evidence.json')));
|
|
1258
|
+
const eventStat = await fs.stat(eventsPath).catch(() => undefined);
|
|
1259
|
+
const lastMessageStat = await fs.stat(lastMessagePath).catch(() => undefined);
|
|
1260
|
+
const decisionJsonStat = await fs.stat(decisionsJson).catch(() => undefined);
|
|
1261
|
+
const decisionJsonlStat = await fs.stat(decisionsJsonl).catch(() => undefined);
|
|
1262
|
+
const live = liveLines.some((line) => line.includes(jobDir) || line.includes(jobId));
|
|
1263
|
+
const hasDecision = Boolean(decisionJsonStat?.isFile() || decisionJsonlStat?.isFile());
|
|
1264
|
+
const failed = !live && !lastMessageStat && !hasDecision && await codexEventsHaveFailure(eventsPath);
|
|
1265
|
+
const status = live && !lastMessageStat
|
|
1266
|
+
? 'running'
|
|
1267
|
+
: lastMessageStat || hasDecision
|
|
1268
|
+
? 'completed'
|
|
1269
|
+
: 'failed';
|
|
1270
|
+
const startedAt = numberValue(eventStat?.birthtimeMs ?? eventStat?.ctimeMs ?? eventStat?.mtimeMs);
|
|
1271
|
+
const finishedAt = status === 'running'
|
|
1272
|
+
? undefined
|
|
1273
|
+
: Math.max(numberValue(lastMessageStat?.mtimeMs), numberValue(decisionJsonStat?.mtimeMs), numberValue(decisionJsonlStat?.mtimeMs), numberValue(eventStat?.mtimeMs));
|
|
1274
|
+
const usage = await readCodexEventUsageSummary(eventsPath);
|
|
1275
|
+
const lane = drainCoordinatorLane(jobId, runKind);
|
|
1276
|
+
const evidencePaths = await existingRelativePaths(cwd, [
|
|
1277
|
+
eventsPath,
|
|
1278
|
+
lastMessagePath,
|
|
1279
|
+
decisionsJson,
|
|
1280
|
+
decisionsJsonl,
|
|
1281
|
+
path.join(evidenceDir, 'merge.json'),
|
|
1282
|
+
path.join(evidenceDir, 'evidence.json'),
|
|
1283
|
+
path.join(evidenceDir, 'human-question.json'),
|
|
1284
|
+
path.join(evidenceDir, 'resource-allocation.json'),
|
|
1285
|
+
path.join(evidenceDir, 'model-availability.json')
|
|
1286
|
+
]);
|
|
1287
|
+
const commandEvidence = commandEvidenceFromRecords(evidenceRecord);
|
|
1288
|
+
return withRecomputedCostFields({
|
|
1289
|
+
id: jobId,
|
|
1290
|
+
originalJobId: jobId,
|
|
1291
|
+
taskId: jobId,
|
|
1292
|
+
title: runKind === 'coordinator' ? `Coordinate lane review for ${lane}` : `Continue ${lane} work`,
|
|
1293
|
+
lane,
|
|
1294
|
+
status,
|
|
1295
|
+
bucket: status === 'running' ? 'running' : status === 'completed' ? 'completed' : 'failed-evidence',
|
|
1296
|
+
disposition: status === 'running' ? 'active' : status,
|
|
1297
|
+
agentId: jobId,
|
|
1298
|
+
workerId: jobId,
|
|
1299
|
+
model: textValue(modelAvailability.effectiveModel ?? modelAvailability.requestedModel, 'gpt-5.5'),
|
|
1300
|
+
computeId: runKind === 'coordinator' ? 'coordinator-agent' : 'continuation-worker',
|
|
1301
|
+
startedAt: startedAt || undefined,
|
|
1302
|
+
...(finishedAt ? { finishedAt } : {}),
|
|
1303
|
+
durationMs: startedAt ? Math.max(0, (finishedAt ?? now) - startedAt) : 0,
|
|
1304
|
+
...(usage.inputTokens ? { actualInputTokens: usage.inputTokens, inputTokens: usage.inputTokens } : {}),
|
|
1305
|
+
...(!usage.inputTokens && usage.estimatedInputTokens ? { estimatedInputTokens: usage.estimatedInputTokens } : {}),
|
|
1306
|
+
...(usage.cachedInputTokens ? { cachedInputTokens: usage.cachedInputTokens } : {}),
|
|
1307
|
+
...(usage.uncachedInputTokens ? { uncachedInputTokens: usage.uncachedInputTokens } : {}),
|
|
1308
|
+
...(usage.outputTokens ? { actualOutputTokens: usage.outputTokens, outputTokens: usage.outputTokens } : {}),
|
|
1309
|
+
...(usage.reasoningOutputTokens ? { reasoningOutputTokens: usage.reasoningOutputTokens } : {}),
|
|
1310
|
+
...(usage.eventCount ? { usage: { ...usage, source: 'codex-events.jsonl' } } : {}),
|
|
1311
|
+
evidencePaths,
|
|
1312
|
+
evidencePathCount: evidencePaths.length,
|
|
1313
|
+
commandsPassed: commandEvidence.passed,
|
|
1314
|
+
commandsFailed: commandEvidence.failed,
|
|
1315
|
+
changedPathCount: 0,
|
|
1316
|
+
collectReasonClasses: status === 'running' ? [`active drain ${runKind}`] : [`drain ${runKind}`],
|
|
1317
|
+
mergeReadiness: status,
|
|
1318
|
+
sourceRun: path.relative(cwd, coordinatorRunDir),
|
|
1319
|
+
sourceLabel: path.relative(cwd, coordinatorRunDir),
|
|
1320
|
+
generatedAt: numberValue(finishedAt) || numberValue(eventStat?.mtimeMs) || now
|
|
1321
|
+
});
|
|
1322
|
+
}
|
|
1323
|
+
function liveProcessLinesForPath(needle) {
|
|
1324
|
+
const result = spawnSync('pgrep', ['-fl', needle], { encoding: 'utf8' });
|
|
1325
|
+
if (result.status !== 0 && !result.stdout)
|
|
1326
|
+
return [];
|
|
1327
|
+
return result.stdout.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
|
|
1328
|
+
}
|
|
1329
|
+
async function existingRelativePaths(cwd, files) {
|
|
1330
|
+
const out = [];
|
|
1331
|
+
for (const file of files) {
|
|
1332
|
+
const stat = await fs.stat(file).catch(() => undefined);
|
|
1333
|
+
if (stat?.isFile())
|
|
1334
|
+
out.push(path.relative(cwd, file));
|
|
1335
|
+
}
|
|
1336
|
+
return out;
|
|
1337
|
+
}
|
|
1338
|
+
async function firstExistingRelativePath(cwd, files) {
|
|
1339
|
+
for (const file of files) {
|
|
1340
|
+
const stat = await fs.stat(file).catch(() => undefined);
|
|
1341
|
+
if (stat?.isFile())
|
|
1342
|
+
return path.relative(cwd, file);
|
|
1343
|
+
}
|
|
1344
|
+
return undefined;
|
|
1345
|
+
}
|
|
1346
|
+
function rawRunPatchCandidates(jobDir) {
|
|
1347
|
+
return [
|
|
1348
|
+
path.join(jobDir, 'changes.patch'),
|
|
1349
|
+
path.join(jobDir, 'source.patch'),
|
|
1350
|
+
path.join(jobDir, 'evidence', 'changes.patch'),
|
|
1351
|
+
path.join(jobDir, 'evidence', 'source.patch')
|
|
1352
|
+
];
|
|
1353
|
+
}
|
|
1354
|
+
async function readPatchChangedPathList(cwd, patchPath) {
|
|
1355
|
+
if (!patchPath)
|
|
1356
|
+
return [];
|
|
1357
|
+
const absolute = path.resolve(cwd, patchPath);
|
|
1358
|
+
if (!isPathInside(cwd, absolute))
|
|
1359
|
+
return [];
|
|
1360
|
+
const stat = await fs.stat(absolute).catch(() => undefined);
|
|
1361
|
+
if (!stat?.isFile() || stat.size > TASK_DETAIL_PATCH_MAX_BYTES)
|
|
1362
|
+
return [];
|
|
1363
|
+
const patch = await fs.readFile(absolute, 'utf8');
|
|
1364
|
+
return uniquePaths(parseUnifiedPatchFiles(patch).map((file) => file.path).filter(Boolean));
|
|
1365
|
+
}
|
|
1366
|
+
async function codexEventsHaveFailure(file) {
|
|
1367
|
+
const text = await fs.readFile(file, 'utf8').catch(() => '');
|
|
1368
|
+
return /"type":"(?:error|turn\.failed)"/.test(text);
|
|
1369
|
+
}
|
|
1370
|
+
async function codexEventsHaveQuotaLimit(file) {
|
|
1371
|
+
const text = await fs.readFile(file, 'utf8').catch(() => '');
|
|
1372
|
+
return /usage limit|quota|purchase more credits/i.test(text);
|
|
1373
|
+
}
|
|
1374
|
+
function drainCoordinatorLane(jobId, runKind = 'coordinator') {
|
|
1375
|
+
if (runKind === 'worker') {
|
|
1376
|
+
for (const marker of [
|
|
1377
|
+
'-continuation-rerun-',
|
|
1378
|
+
'-continuation-supersede-',
|
|
1379
|
+
'-continuation-reject-',
|
|
1380
|
+
'-continuation-',
|
|
1381
|
+
'-queue-candidate-package-'
|
|
1382
|
+
]) {
|
|
1383
|
+
const index = jobId.indexOf(marker);
|
|
1384
|
+
if (index > 0)
|
|
1385
|
+
return jobId.slice(0, index);
|
|
1386
|
+
}
|
|
1387
|
+
}
|
|
1388
|
+
const marker = '-coordinator-agent-';
|
|
1389
|
+
const index = jobId.indexOf(marker);
|
|
1390
|
+
return index > 0 ? jobId.slice(0, index) : jobId;
|
|
1391
|
+
}
|
|
1392
|
+
function mergeLifetimeDrainCoordinatorSnapshot(lifetime, drain) {
|
|
1393
|
+
const drainJobs = recordArray(drain?.jobs);
|
|
1394
|
+
if (!drainJobs.length)
|
|
1395
|
+
return lifetime;
|
|
1396
|
+
const existingJobs = recordArray(lifetime.jobs).filter((job) => {
|
|
1397
|
+
const source = textValue(job.sourceRun ?? job.sourceLabel, '');
|
|
1398
|
+
return !/agent-runs\/frontier-swarm-codex\/drain-.*\/(?:coordinator-run|worker-run)/.test(source);
|
|
1399
|
+
});
|
|
1400
|
+
const jobs = [...drainJobs, ...existingJobs].slice(0, LIFETIME_DASHBOARD_MAX_JOBS);
|
|
1401
|
+
const events = [...recordArray(lifetime.events), ...recordArray(drain?.events)]
|
|
1402
|
+
.sort((left, right) => numberValue(left.at) - numberValue(right.at))
|
|
1403
|
+
.slice(-160);
|
|
1404
|
+
return {
|
|
1405
|
+
...lifetime,
|
|
1406
|
+
generatedAt: Math.max(numberValue(lifetime.generatedAt), numberValue(drain?.generatedAt), Date.now()),
|
|
1407
|
+
sources: {
|
|
1408
|
+
...recordValue(lifetime.sources),
|
|
1409
|
+
...recordValue(drain?.sources)
|
|
1410
|
+
},
|
|
1411
|
+
summary: lifetimeDashboardSummary(jobs),
|
|
1412
|
+
health: lifetimeHealthSummary(jobs),
|
|
1413
|
+
lanes: lifetimeLaneRows(jobs),
|
|
1414
|
+
jobs,
|
|
1415
|
+
events,
|
|
1416
|
+
raw: {
|
|
1417
|
+
...recordValue(lifetime.raw),
|
|
1418
|
+
activeDrain: recordValue(recordValue(drain?.raw).activeDrain)
|
|
1419
|
+
}
|
|
1420
|
+
};
|
|
1421
|
+
}
|
|
1422
|
+
function mergeLifetimeActiveRunSnapshot(lifetime, active) {
|
|
1423
|
+
const activeJobs = recordArray(active?.jobs).filter((job) => textValue(job.status, '') === 'running');
|
|
1424
|
+
if (!activeJobs.length)
|
|
1425
|
+
return lifetime;
|
|
1426
|
+
const activeKeys = new Set(activeJobs.map(lifetimeJobDedupeKey).filter(Boolean));
|
|
1427
|
+
const existingJobs = recordArray(lifetime.jobs).filter((job) => !activeKeys.has(lifetimeJobDedupeKey(job)));
|
|
1428
|
+
const jobs = [...activeJobs, ...existingJobs].slice(0, LIFETIME_DASHBOARD_MAX_JOBS);
|
|
1429
|
+
const activeAgents = activeAgentsFromJobs(jobs);
|
|
1430
|
+
const events = [...recordArray(lifetime.events), ...recordArray(active?.events)]
|
|
1431
|
+
.sort((left, right) => numberValue(left.at) - numberValue(right.at))
|
|
1432
|
+
.slice(-160);
|
|
1433
|
+
return {
|
|
1434
|
+
...lifetime,
|
|
1435
|
+
generatedAt: Math.max(numberValue(lifetime.generatedAt), numberValue(active?.generatedAt), Date.now()),
|
|
1436
|
+
sources: {
|
|
1437
|
+
...recordValue(lifetime.sources),
|
|
1438
|
+
...recordValue(active?.sources)
|
|
1439
|
+
},
|
|
1440
|
+
summary: lifetimeDashboardSummary(jobs),
|
|
1441
|
+
health: lifetimeHealthSummary(jobs),
|
|
1442
|
+
lanes: lifetimeLaneRows(jobs),
|
|
1443
|
+
capacity: lifetimeCapacitySummary({
|
|
1444
|
+
entries: recordArray(recordValue(lifetime.backlog).entries),
|
|
1445
|
+
manifests: recordArray(recordValue(recordValue(lifetime.raw).lifetime).manifests),
|
|
1446
|
+
sourceCount: numberValue(recordValue(lifetime.backlog).entryCount),
|
|
1447
|
+
paths: stringArray(recordValue(recordValue(lifetime.raw).lifetime).queueSources),
|
|
1448
|
+
generatedAt: numberValue(lifetime.generatedAt)
|
|
1449
|
+
}, jobs, recordArray(recordValue(lifetime.backlog).entries)),
|
|
1450
|
+
jobs,
|
|
1451
|
+
activeAgents,
|
|
1452
|
+
events,
|
|
1453
|
+
raw: {
|
|
1454
|
+
...recordValue(lifetime.raw),
|
|
1455
|
+
activeRuns: recordValue(recordValue(active?.raw).activeRuns)
|
|
1456
|
+
}
|
|
1457
|
+
};
|
|
1458
|
+
}
|
|
1459
|
+
async function readLifetimeActiveRunSnapshot(options) {
|
|
1460
|
+
const jobs = await readLiveCodexProcessJobs(options.cwd);
|
|
1461
|
+
const sources = uniquePaths(jobs.map((job) => textValue(job.sourceRun, '')).filter(Boolean));
|
|
1462
|
+
if (!jobs.length)
|
|
1463
|
+
return undefined;
|
|
1464
|
+
const generatedAt = Date.now();
|
|
1465
|
+
return {
|
|
1466
|
+
ok: true,
|
|
1467
|
+
generatedAt,
|
|
1468
|
+
cwd: options.cwd,
|
|
1469
|
+
sources: {
|
|
1470
|
+
activeRuns: sources,
|
|
1471
|
+
activeRunCount: sources.length
|
|
1472
|
+
},
|
|
1473
|
+
summary: lifetimeDashboardSummary(jobs),
|
|
1474
|
+
lanes: lifetimeLaneRows(jobs),
|
|
1475
|
+
jobs,
|
|
1476
|
+
activeAgents: activeAgentsFromJobs(jobs),
|
|
1477
|
+
events: activeRunEvents(jobs),
|
|
1478
|
+
raw: {
|
|
1479
|
+
activeRuns: {
|
|
1480
|
+
runDirs: sources,
|
|
1481
|
+
jobCount: jobs.length,
|
|
1482
|
+
runningCount: jobs.length
|
|
1483
|
+
}
|
|
1484
|
+
}
|
|
1485
|
+
};
|
|
1486
|
+
}
|
|
1487
|
+
async function readLiveCodexProcessJobs(cwd) {
|
|
1488
|
+
if (process.platform === 'win32')
|
|
1489
|
+
return [];
|
|
1490
|
+
const result = spawnSync('ps', ['-axo', 'pid,ppid,etime,command'], {
|
|
1491
|
+
encoding: 'utf8',
|
|
1492
|
+
stdio: ['ignore', 'pipe', 'ignore']
|
|
1493
|
+
});
|
|
1494
|
+
if (result.status !== 0 && !result.stdout)
|
|
1495
|
+
return [];
|
|
1496
|
+
const agentWorktreeRoot = `${path.join(cwd, 'agent-worktrees')}/`;
|
|
1497
|
+
const now = Date.now();
|
|
1498
|
+
const byWorker = new Map();
|
|
1499
|
+
for (const line of result.stdout.split(/\r?\n/)) {
|
|
1500
|
+
if (!line.includes('codex ') || !line.includes(' exec ') || !line.includes(agentWorktreeRoot))
|
|
1501
|
+
continue;
|
|
1502
|
+
const fields = /^\s*(\d+)\s+(\d+)\s+(\S+)\s+(.+)$/.exec(line);
|
|
1503
|
+
if (!fields)
|
|
1504
|
+
continue;
|
|
1505
|
+
const command = fields[4];
|
|
1506
|
+
const cd = commandOptionValue(splitCommandWords(command), '--cd') || regexCommandOption(command, '--cd');
|
|
1507
|
+
if (!cd.startsWith(agentWorktreeRoot))
|
|
1508
|
+
continue;
|
|
1509
|
+
const outputLastMessage = commandOptionValue(splitCommandWords(command), '--output-last-message') || regexCommandOption(command, '--output-last-message');
|
|
1510
|
+
const model = commandOptionValue(splitCommandWords(command), '--model') || regexCommandOption(command, '--model') || '';
|
|
1511
|
+
const runDir = outputLastMessage ? path.dirname(path.dirname(outputLastMessage)) : '';
|
|
1512
|
+
const jobId = outputLastMessage ? path.basename(path.dirname(outputLastMessage)) : path.basename(cd);
|
|
1513
|
+
const key = outputLastMessage || cd;
|
|
1514
|
+
const pid = Number(fields[1]);
|
|
1515
|
+
const current = byWorker.get(key);
|
|
1516
|
+
if (!current || pid < current.pid)
|
|
1517
|
+
byWorker.set(key, { pid, etime: fields[3], cd, model, outputLastMessage, runDir, jobId });
|
|
1518
|
+
}
|
|
1519
|
+
const planCache = new Map();
|
|
1520
|
+
const jobs = [];
|
|
1521
|
+
for (const worker of byWorker.values()) {
|
|
1522
|
+
const relativeRun = worker.runDir && isPathInside(cwd, worker.runDir) ? path.relative(cwd, worker.runDir) : '';
|
|
1523
|
+
const planJobs = worker.runDir
|
|
1524
|
+
? await (planCache.get(worker.runDir) ?? planCache.set(worker.runDir, readRunPlanJobs(worker.runDir)).get(worker.runDir))
|
|
1525
|
+
: new Map();
|
|
1526
|
+
const planJob = planJobs.get(worker.jobId);
|
|
1527
|
+
const task = recordValue(planJob?.task);
|
|
1528
|
+
const compute = recordValue(planJob?.compute);
|
|
1529
|
+
const startedAt = now - parsePsElapsedMs(worker.etime);
|
|
1530
|
+
const title = textValue(planJob?.title ?? task.title, humanizeWorkerJobId(worker.jobId));
|
|
1531
|
+
const lane = textValue(planJob?.lane ?? task.lane, inferLaneFromWorkerJobId(worker.jobId));
|
|
1532
|
+
jobs.push({
|
|
1533
|
+
id: relativeRun ? `run:${relativeRun.replaceAll(/[^\w:.-]+/g, '-')}:${worker.jobId}` : worker.jobId,
|
|
1534
|
+
originalJobId: worker.jobId,
|
|
1535
|
+
taskId: textValue(planJob?.taskId ?? task.id, worker.jobId),
|
|
1536
|
+
title,
|
|
1537
|
+
lane,
|
|
1538
|
+
status: 'running',
|
|
1539
|
+
bucket: 'running',
|
|
1540
|
+
disposition: 'active',
|
|
1541
|
+
agentId: worker.jobId,
|
|
1542
|
+
workerId: worker.jobId,
|
|
1543
|
+
model: textValue(compute.model, worker.model),
|
|
1544
|
+
computeId: textValue(compute.id, ''),
|
|
1545
|
+
reasoningEffort: textValue(compute.reasoningEffort, ''),
|
|
1546
|
+
startedAt,
|
|
1547
|
+
durationMs: Math.max(0, now - startedAt),
|
|
1548
|
+
evidencePaths: worker.outputLastMessage && isPathInside(cwd, worker.outputLastMessage) ? [path.relative(cwd, worker.outputLastMessage)] : [],
|
|
1549
|
+
evidencePathCount: worker.outputLastMessage ? 1 : 0,
|
|
1550
|
+
changedPathCount: 0,
|
|
1551
|
+
collectReasonClasses: ['active worker process'],
|
|
1552
|
+
mergeReadiness: 'running',
|
|
1553
|
+
sourceRun: relativeRun,
|
|
1554
|
+
sourceLabel: relativeRun ? lifetimeSourceLabel(relativeRun) : 'active-process',
|
|
1555
|
+
generatedAt: now
|
|
1556
|
+
});
|
|
1557
|
+
}
|
|
1558
|
+
return jobs.sort((left, right) => textValue(left.lane, '').localeCompare(textValue(right.lane, '')) || textValue(left.title, '').localeCompare(textValue(right.title, '')));
|
|
1559
|
+
}
|
|
1560
|
+
function regexCommandOption(command, option) {
|
|
1561
|
+
const escaped = option.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
1562
|
+
const match = new RegExp(`${escaped}\\s+([^\\s]+)`).exec(command);
|
|
1563
|
+
return match?.[1] ?? '';
|
|
1564
|
+
}
|
|
1565
|
+
function splitCommandWords(command) {
|
|
1566
|
+
return command.match(/"[^"]*"|'[^']*'|\S+/g)?.map((word) => word.replace(/^(['"])(.*)\1$/u, '$2')) ?? [];
|
|
1567
|
+
}
|
|
1568
|
+
function parsePsElapsedMs(value) {
|
|
1569
|
+
const daySplit = value.split('-');
|
|
1570
|
+
const days = daySplit.length === 2 ? Number(daySplit[0]) || 0 : 0;
|
|
1571
|
+
const time = daySplit.at(-1) ?? '';
|
|
1572
|
+
const parts = time.split(':').map((part) => Number(part) || 0);
|
|
1573
|
+
const [hours, minutes, seconds] = parts.length === 3 ? parts : [0, parts[0] ?? 0, parts[1] ?? 0];
|
|
1574
|
+
return (((days * 24 + hours) * 60 + minutes) * 60 + seconds) * 1000;
|
|
1575
|
+
}
|
|
1576
|
+
function inferLaneFromWorkerJobId(jobId) {
|
|
1577
|
+
const parts = jobId.split('-');
|
|
1578
|
+
const half = Math.floor(parts.length / 2);
|
|
1579
|
+
if (half > 0 && parts.slice(0, half).join('-') === parts.slice(half, half * 2).join('-'))
|
|
1580
|
+
return parts.slice(0, half).join('-');
|
|
1581
|
+
return parts.slice(0, Math.max(1, parts.length - 1)).join('-');
|
|
1582
|
+
}
|
|
1583
|
+
function humanizeWorkerJobId(jobId) {
|
|
1584
|
+
return inferLaneFromWorkerJobId(jobId).split('-').filter(Boolean).map((part) => part[0]?.toUpperCase() + part.slice(1)).join(' ') || jobId;
|
|
1585
|
+
}
|
|
1586
|
+
async function findLifetimeActiveRunDirs(cwd) {
|
|
1587
|
+
const root = path.join(cwd, 'agent-runs');
|
|
1588
|
+
const stat = await fs.stat(root).catch(() => undefined);
|
|
1589
|
+
if (!stat?.isDirectory())
|
|
1590
|
+
return [];
|
|
1591
|
+
const resetAt = await readLifetimeDashboardResetCutoff(root);
|
|
1592
|
+
const files = await findPidManifestFiles(root, LIFETIME_DASHBOARD_SCAN_MAX_DEPTH, LIFETIME_DASHBOARD_SCAN_MAX_FILES, resetAt);
|
|
1593
|
+
const candidates = await Promise.all(files.map(async (file) => ({
|
|
1594
|
+
file,
|
|
1595
|
+
dir: path.dirname(file),
|
|
1596
|
+
mtimeMs: (await fs.stat(file).catch(() => undefined))?.mtimeMs ?? 0
|
|
1597
|
+
})));
|
|
1598
|
+
const out = [];
|
|
1599
|
+
for (const candidate of candidates.sort((left, right) => right.mtimeMs - left.mtimeMs || right.file.localeCompare(left.file))) {
|
|
1600
|
+
if (out.length >= LIFETIME_DASHBOARD_MAX_ACTIVE_PID_RUNS)
|
|
1601
|
+
break;
|
|
1602
|
+
if (await pidManifestHasLiveCodexEntry(candidate.file))
|
|
1603
|
+
out.push(candidate.dir);
|
|
1604
|
+
}
|
|
1605
|
+
return uniquePaths(out);
|
|
1606
|
+
}
|
|
1607
|
+
async function findPidManifestFiles(root, maxDepth, maxFiles, resetAt) {
|
|
1608
|
+
const out = [];
|
|
1609
|
+
const skipDirs = new Set(['.git', 'node_modules', 'dist', 'coverage', 'evidence', 'streams', 'patch-scores', 'apply-ledger', 'artifact-index']);
|
|
1610
|
+
async function walk(current, depth) {
|
|
1611
|
+
if (out.length >= maxFiles || depth > maxDepth)
|
|
1612
|
+
return;
|
|
1613
|
+
const entries = await fs.readdir(current, { withFileTypes: true }).catch(() => []);
|
|
1614
|
+
for (const entry of entries) {
|
|
1615
|
+
if (out.length >= maxFiles)
|
|
1616
|
+
return;
|
|
1617
|
+
const absolute = path.join(current, entry.name);
|
|
1618
|
+
if (entry.isDirectory()) {
|
|
1619
|
+
if (skipDirs.has(entry.name) || entry.name.startsWith('.'))
|
|
1620
|
+
continue;
|
|
1621
|
+
if (resetAt && depth === 0) {
|
|
1622
|
+
const dirStat = await fs.stat(absolute).catch(() => undefined);
|
|
1623
|
+
if ((dirStat?.mtimeMs ?? 0) <= resetAt)
|
|
1624
|
+
continue;
|
|
1625
|
+
}
|
|
1626
|
+
await walk(absolute, depth + 1);
|
|
1627
|
+
}
|
|
1628
|
+
else if (entry.isFile() && entry.name === 'pids.json') {
|
|
1629
|
+
out.push(absolute);
|
|
1630
|
+
}
|
|
1631
|
+
}
|
|
1632
|
+
}
|
|
1633
|
+
await walk(root, 0);
|
|
1634
|
+
return out;
|
|
1635
|
+
}
|
|
1636
|
+
async function pidManifestHasLiveCodexEntry(file) {
|
|
1637
|
+
const manifest = recordValue(await readJsonFile(file));
|
|
1638
|
+
return recordArray(manifest.entries)
|
|
1639
|
+
.filter((entry) => textValue(entry.role, '') === 'codex')
|
|
1640
|
+
.some((entry) => isProcessLive(numberValue(entry.pid), entry));
|
|
1641
|
+
}
|
|
534
1642
|
async function readCoordinatorReviewDecisions(cwd) {
|
|
535
1643
|
const file = coordinatorReviewDecisionPath(cwd);
|
|
536
1644
|
const raw = await readJsonFile(file);
|
|
@@ -542,6 +1650,127 @@ async function readCoordinatorReviewDecisions(cwd) {
|
|
|
542
1650
|
function coordinatorReviewDecisionPath(cwd) {
|
|
543
1651
|
return path.join(cwd, 'agent-runs', REVIEW_DECISIONS_FILE);
|
|
544
1652
|
}
|
|
1653
|
+
async function readAutonomousMergeDecisions(cwd) {
|
|
1654
|
+
const root = path.join(cwd, 'agent-runs');
|
|
1655
|
+
const stat = await fs.stat(root).catch(() => undefined);
|
|
1656
|
+
if (!stat?.isDirectory())
|
|
1657
|
+
return [];
|
|
1658
|
+
const files = await findAutonomousMergeDecisionFiles(root);
|
|
1659
|
+
const out = [];
|
|
1660
|
+
for (const file of files) {
|
|
1661
|
+
const text = await fs.readFile(file, 'utf8').catch(() => '');
|
|
1662
|
+
for (const line of text.split(/\r?\n/u)) {
|
|
1663
|
+
const trimmed = line.trim();
|
|
1664
|
+
if (!trimmed)
|
|
1665
|
+
continue;
|
|
1666
|
+
const raw = safeJsonObject(trimmed);
|
|
1667
|
+
if (!raw)
|
|
1668
|
+
continue;
|
|
1669
|
+
const normalizedDecision = normalizeAutonomousMergeDecision(cwd, file, raw);
|
|
1670
|
+
if (normalizedDecision)
|
|
1671
|
+
out.push(normalizedDecision);
|
|
1672
|
+
}
|
|
1673
|
+
}
|
|
1674
|
+
return out.sort(compareCoordinatorReviewDecisionRecency);
|
|
1675
|
+
}
|
|
1676
|
+
async function findAutonomousMergeDecisionFiles(root) {
|
|
1677
|
+
const out = [];
|
|
1678
|
+
async function walk(current, depth) {
|
|
1679
|
+
if (out.length >= LIFETIME_DASHBOARD_MAX_AUTONOMOUS_DECISION_FILES || depth > LIFETIME_DASHBOARD_SCAN_MAX_DEPTH)
|
|
1680
|
+
return;
|
|
1681
|
+
const entries = await fs.readdir(current, { withFileTypes: true }).catch(() => []);
|
|
1682
|
+
for (const entry of entries) {
|
|
1683
|
+
if (out.length >= LIFETIME_DASHBOARD_MAX_AUTONOMOUS_DECISION_FILES)
|
|
1684
|
+
return;
|
|
1685
|
+
const absolute = path.join(current, entry.name);
|
|
1686
|
+
if (entry.isDirectory()) {
|
|
1687
|
+
if (entry.name === 'node_modules' || entry.name === '.git' || entry.name === 'artifact-store' || entry.name.startsWith('.'))
|
|
1688
|
+
continue;
|
|
1689
|
+
await walk(absolute, depth + 1);
|
|
1690
|
+
}
|
|
1691
|
+
else if (entry.isFile() && entry.name === 'autonomous-merge-decisions.jsonl') {
|
|
1692
|
+
const stat = await fs.stat(absolute).catch(() => undefined);
|
|
1693
|
+
out.push({ file: absolute, mtimeMs: stat?.mtimeMs ?? 0 });
|
|
1694
|
+
}
|
|
1695
|
+
}
|
|
1696
|
+
}
|
|
1697
|
+
await walk(root, 0);
|
|
1698
|
+
return out.sort((left, right) => right.mtimeMs - left.mtimeMs || right.file.localeCompare(left.file)).map((entry) => entry.file);
|
|
1699
|
+
}
|
|
1700
|
+
function normalizeAutonomousMergeDecision(cwd, file, raw) {
|
|
1701
|
+
const status = autonomousDecisionStatus(textValue(raw.status ?? raw.decision, ''));
|
|
1702
|
+
if (!status)
|
|
1703
|
+
return undefined;
|
|
1704
|
+
const ids = Array.from(new Set([
|
|
1705
|
+
textValue(raw.jobId, ''),
|
|
1706
|
+
textValue(raw.taskId, ''),
|
|
1707
|
+
...stringArray(raw.queueItemIds),
|
|
1708
|
+
...stringArray(raw.queueKeys).map((key) => key.replace(/^(?:queue|task|job):/u, ''))
|
|
1709
|
+
].filter(Boolean)));
|
|
1710
|
+
if (!ids.length)
|
|
1711
|
+
return undefined;
|
|
1712
|
+
const decidedAtMs = Math.max(numberValue(raw.finishedAt), numberValue(raw.startedAt), Date.parse(textValue(raw.finishedAtIso ?? raw.decidedAt ?? raw.generatedAt, '')) || 0);
|
|
1713
|
+
const relativeFile = path.relative(cwd, file);
|
|
1714
|
+
return {
|
|
1715
|
+
id: textValue(raw.id, ids[0]),
|
|
1716
|
+
jobId: textValue(raw.jobId, ''),
|
|
1717
|
+
taskId: textValue(raw.taskId, ''),
|
|
1718
|
+
matchIds: ids,
|
|
1719
|
+
status,
|
|
1720
|
+
decision: status,
|
|
1721
|
+
reason: textValue(raw.reason, ''),
|
|
1722
|
+
decidedAt: decidedAtMs ? new Date(decidedAtMs).toISOString() : '',
|
|
1723
|
+
decidedAtMs,
|
|
1724
|
+
sourceArtifact: relativeFile,
|
|
1725
|
+
sourceKind: 'autonomous-merge-decision',
|
|
1726
|
+
latestPath: textValue(raw.bundlePath ?? raw.patchPath, relativeFile),
|
|
1727
|
+
autonomousDecision: raw
|
|
1728
|
+
};
|
|
1729
|
+
}
|
|
1730
|
+
function autonomousDecisionStatus(value) {
|
|
1731
|
+
const status = normalized(value);
|
|
1732
|
+
if (status === 'committed' || status === 'applied')
|
|
1733
|
+
return status;
|
|
1734
|
+
if (status === 'accepted' || status === 'accepted-applied')
|
|
1735
|
+
return 'applied';
|
|
1736
|
+
if (status === 'rejected' || status === 'rerun' || status === 'superseded')
|
|
1737
|
+
return status;
|
|
1738
|
+
if (status === 'conflict' || status === 'conflict-blocked')
|
|
1739
|
+
return 'conflict-blocked';
|
|
1740
|
+
if (status === 'human-blocked' || status === 'human-question')
|
|
1741
|
+
return 'human-blocked';
|
|
1742
|
+
return status;
|
|
1743
|
+
}
|
|
1744
|
+
function mergeReviewDecisionLists(...groups) {
|
|
1745
|
+
return groups.flat().sort(compareCoordinatorReviewDecisionRecency);
|
|
1746
|
+
}
|
|
1747
|
+
function compareCoordinatorReviewDecisionRecency(left, right) {
|
|
1748
|
+
return decisionTime(right) - decisionTime(left)
|
|
1749
|
+
|| textValue(right.sourceArtifact ?? right.latestPath, '').localeCompare(textValue(left.sourceArtifact ?? left.latestPath, ''));
|
|
1750
|
+
}
|
|
1751
|
+
function decisionTime(decision) {
|
|
1752
|
+
return numberValue(decision.decidedAtMs)
|
|
1753
|
+
|| numberValue(decision.finishedAt)
|
|
1754
|
+
|| numberValue(decision.startedAt)
|
|
1755
|
+
|| Date.parse(textValue(decision.decidedAt, ''))
|
|
1756
|
+
|| 0;
|
|
1757
|
+
}
|
|
1758
|
+
function autonomousDecisionSourceSummary(decisions) {
|
|
1759
|
+
const files = Array.from(new Set(decisions.map((decision) => textValue(decision.sourceArtifact, '')).filter(Boolean)));
|
|
1760
|
+
return {
|
|
1761
|
+
count: decisions.length,
|
|
1762
|
+
fileCount: files.length,
|
|
1763
|
+
files: files.slice(0, 20)
|
|
1764
|
+
};
|
|
1765
|
+
}
|
|
1766
|
+
function safeJsonObject(value) {
|
|
1767
|
+
try {
|
|
1768
|
+
return recordValue(JSON.parse(value));
|
|
1769
|
+
}
|
|
1770
|
+
catch {
|
|
1771
|
+
return undefined;
|
|
1772
|
+
}
|
|
1773
|
+
}
|
|
545
1774
|
function applyCoordinatorReviewDecisions(jobs, decisions) {
|
|
546
1775
|
const records = jobs
|
|
547
1776
|
.map(recordValue)
|
|
@@ -555,25 +1784,103 @@ function applyCoordinatorReviewDecisions(jobs, decisions) {
|
|
|
555
1784
|
return record;
|
|
556
1785
|
const status = textValue(decision.status ?? decision.decision, 'resolved');
|
|
557
1786
|
const resolved = isResolvedCoordinatorDecision(status);
|
|
558
|
-
|
|
1787
|
+
const decided = {
|
|
559
1788
|
...record,
|
|
560
1789
|
coordinatorDecision: decision,
|
|
561
1790
|
coordinatorDecisionStatus: status,
|
|
562
1791
|
coordinatorDecisionAt: textValue(decision.decidedAt, ''),
|
|
563
1792
|
reviewResolved: resolved,
|
|
564
|
-
...(resolved && isCoordinatorPortBucket(record.bucket) ? { bucket: 'review-resolved' } : {}),
|
|
565
1793
|
...(resolved ? { disposition: status } : {})
|
|
566
1794
|
};
|
|
1795
|
+
return resolved ? markCoordinatorReviewResolved(decided, status) : decided;
|
|
567
1796
|
});
|
|
568
1797
|
}
|
|
569
1798
|
function normalizeCoordinatorFacingJob(record) {
|
|
570
|
-
|
|
1799
|
+
const status = coordinatorFacingMachineLabel(record.status);
|
|
1800
|
+
let bucket = coordinatorFacingMachineLabel(record.bucket);
|
|
1801
|
+
if (!bucket && status === 'completed')
|
|
1802
|
+
bucket = 'completed';
|
|
1803
|
+
else if (!bucket && status === 'running')
|
|
1804
|
+
bucket = 'running';
|
|
1805
|
+
else if (!bucket && status === 'failed')
|
|
1806
|
+
bucket = 'failed-evidence';
|
|
1807
|
+
else if (!bucket && status === 'blocked')
|
|
1808
|
+
bucket = 'blocked';
|
|
1809
|
+
let normalizedRecord = {
|
|
571
1810
|
...record,
|
|
572
|
-
bucket
|
|
573
|
-
status
|
|
1811
|
+
bucket,
|
|
1812
|
+
status,
|
|
574
1813
|
disposition: coordinatorFacingMachineLabel(record.disposition),
|
|
575
1814
|
mergeReadiness: coordinatorFacingMachineLabel(record.mergeReadiness)
|
|
576
1815
|
};
|
|
1816
|
+
normalizedRecord = normalizeHistoricalEvidenceFailureJob(normalizedRecord);
|
|
1817
|
+
if (!isResolvedCoordinatorReviewRecord(normalizedRecord))
|
|
1818
|
+
return normalizedRecord;
|
|
1819
|
+
return markCoordinatorReviewResolved(normalizedRecord, textValue(normalizedRecord.coordinatorDecisionStatus ?? normalizedRecord.disposition, 'review-resolved'));
|
|
1820
|
+
}
|
|
1821
|
+
function normalizeHistoricalEvidenceFailureJob(record) {
|
|
1822
|
+
if (!isHistoricalOwnershipRescopeCandidate(record))
|
|
1823
|
+
return record;
|
|
1824
|
+
return {
|
|
1825
|
+
...record,
|
|
1826
|
+
reviewResolved: false,
|
|
1827
|
+
originalBucket: record.originalBucket ?? record.bucket,
|
|
1828
|
+
originalStatus: record.originalStatus ?? record.status,
|
|
1829
|
+
originalDisposition: record.originalDisposition ?? record.disposition,
|
|
1830
|
+
bucket: 'rerun-work',
|
|
1831
|
+
status: 'completed',
|
|
1832
|
+
disposition: 'needs-rerun',
|
|
1833
|
+
mergeReadiness: 'needs-rerun',
|
|
1834
|
+
evidenceFailureNormalized: true,
|
|
1835
|
+
collectReasonClasses: uniquePaths([...stringArray(record.collectReasonClasses), 'ownership-rescope-rerun'])
|
|
1836
|
+
};
|
|
1837
|
+
}
|
|
1838
|
+
function isHistoricalOwnershipRescopeCandidate(record) {
|
|
1839
|
+
const bucket = normalized(record.bucket);
|
|
1840
|
+
const status = normalized(record.status);
|
|
1841
|
+
const disposition = normalized(record.disposition);
|
|
1842
|
+
const readiness = normalized(record.mergeReadiness);
|
|
1843
|
+
const failedEvidence = bucket === 'failed-evidence'
|
|
1844
|
+
|| bucket === 'worker-failed'
|
|
1845
|
+
|| status === 'failed'
|
|
1846
|
+
|| disposition === 'rejected'
|
|
1847
|
+
|| disposition === 'failed'
|
|
1848
|
+
|| readiness === 'rejected'
|
|
1849
|
+
|| readiness === 'blocked';
|
|
1850
|
+
if (!failedEvidence)
|
|
1851
|
+
return false;
|
|
1852
|
+
const ownershipViolationCount = numberValue(record.ownershipViolationCount)
|
|
1853
|
+
|| stringArray(record.ownershipViolations).length;
|
|
1854
|
+
if (!ownershipViolationCount)
|
|
1855
|
+
return false;
|
|
1856
|
+
const changedPathCount = numberValue(record.changedPathCount)
|
|
1857
|
+
|| stringArray(record.changedPaths).length;
|
|
1858
|
+
if (!changedPathCount)
|
|
1859
|
+
return false;
|
|
1860
|
+
return Boolean(textValue(record.patchPath, '') || textValue(record.patchArtifactPath, '') || changedPathCount);
|
|
1861
|
+
}
|
|
1862
|
+
function markCoordinatorReviewResolved(record, disposition) {
|
|
1863
|
+
const existingOriginalReasons = stringArray(record.originalReasons);
|
|
1864
|
+
const originalReasons = existingOriginalReasons.length ? existingOriginalReasons : stringArray(record.reasons);
|
|
1865
|
+
const retainedReasons = originalReasons.filter((reason) => !isOpenCoordinatorReviewReason(reason));
|
|
1866
|
+
const decisionReason = textValue(recordValue(record.coordinatorDecision).reason, '');
|
|
1867
|
+
const reasons = uniquePaths([
|
|
1868
|
+
...retainedReasons,
|
|
1869
|
+
...(decisionReason ? [decisionReason] : [])
|
|
1870
|
+
]);
|
|
1871
|
+
return {
|
|
1872
|
+
...record,
|
|
1873
|
+
reviewResolved: true,
|
|
1874
|
+
originalBucket: record.originalBucket ?? record.bucket,
|
|
1875
|
+
originalStatus: record.originalStatus ?? record.status,
|
|
1876
|
+
...(originalReasons.length ? { originalReasons } : {}),
|
|
1877
|
+
bucket: 'review-resolved',
|
|
1878
|
+
status: 'completed',
|
|
1879
|
+
disposition: disposition || 'review-resolved',
|
|
1880
|
+
mergeReadiness: 'review-resolved',
|
|
1881
|
+
reasons,
|
|
1882
|
+
health: ['failed', 'warning'].includes(normalized(record.health)) ? 'resolved' : record.health
|
|
1883
|
+
};
|
|
577
1884
|
}
|
|
578
1885
|
function normalizeCoordinatorFacingSnapshot(record) {
|
|
579
1886
|
return {
|
|
@@ -619,6 +1926,14 @@ function isCoordinatorPortBucket(value) {
|
|
|
619
1926
|
|| bucket === 'needs-coordinator-review'
|
|
620
1927
|
|| bucket === 'needs-coordinator-decision';
|
|
621
1928
|
}
|
|
1929
|
+
function isOpenCoordinatorReviewReason(value) {
|
|
1930
|
+
const reason = normalized(value);
|
|
1931
|
+
return isCoordinatorPortBucket(reason)
|
|
1932
|
+
|| reason === 'needs-port'
|
|
1933
|
+
|| reason === 'needs-review'
|
|
1934
|
+
|| reason === 'manual-port-required'
|
|
1935
|
+
|| reason === 'manual port required';
|
|
1936
|
+
}
|
|
622
1937
|
function coordinatorFacingMachineKey(value) {
|
|
623
1938
|
return textValue(coordinatorFacingMachineLabel(value), value);
|
|
624
1939
|
}
|
|
@@ -652,6 +1967,8 @@ function coordinatorReviewDecisionMatches(job, decision) {
|
|
|
652
1967
|
const decisionSource = textValue(decision.source ?? decision.sourceCollection ?? decision.sourceRun ?? decision.sourceLabel, '');
|
|
653
1968
|
if (!decisionSource)
|
|
654
1969
|
return true;
|
|
1970
|
+
if (isHistoricalReviewDrainDecision(decision))
|
|
1971
|
+
return historicalReviewDrainDecisionMatches(job, decision);
|
|
655
1972
|
const jobSources = [
|
|
656
1973
|
textValue(job.sourceLabel, ''),
|
|
657
1974
|
textValue(job.sourceCollection, ''),
|
|
@@ -660,6 +1977,46 @@ function coordinatorReviewDecisionMatches(job, decision) {
|
|
|
660
1977
|
].filter(Boolean);
|
|
661
1978
|
return jobSources.some((source) => source === decisionSource || source.endsWith(decisionSource) || decisionSource.endsWith(source));
|
|
662
1979
|
}
|
|
1980
|
+
function isHistoricalReviewDrainDecision(decision) {
|
|
1981
|
+
const sources = [
|
|
1982
|
+
textValue(decision.source, ''),
|
|
1983
|
+
textValue(decision.sourceCollection, ''),
|
|
1984
|
+
textValue(decision.sourceRun, ''),
|
|
1985
|
+
textValue(decision.sourceLabel, ''),
|
|
1986
|
+
textValue(decision.sourceArtifact, '')
|
|
1987
|
+
];
|
|
1988
|
+
return sources.some((source) => normalized(source).includes('historical-review-drain'));
|
|
1989
|
+
}
|
|
1990
|
+
function historicalReviewDrainDecisionMatches(job, decision) {
|
|
1991
|
+
if (normalized(job.status) === 'running' || normalized(job.bucket) === 'running')
|
|
1992
|
+
return false;
|
|
1993
|
+
const latestPath = textValue(decision.latestPath, '');
|
|
1994
|
+
if (!latestPath)
|
|
1995
|
+
return false;
|
|
1996
|
+
const latestRoot = historicalReviewDrainSourceRoot(latestPath);
|
|
1997
|
+
if (!latestRoot)
|
|
1998
|
+
return false;
|
|
1999
|
+
const jobSources = [
|
|
2000
|
+
textValue(job.sourceLabel, ''),
|
|
2001
|
+
textValue(job.sourceCollection, ''),
|
|
2002
|
+
textValue(job.sourceRun, ''),
|
|
2003
|
+
textValue(job.sourceContinuation, '')
|
|
2004
|
+
].filter(Boolean);
|
|
2005
|
+
return jobSources.some((source) => {
|
|
2006
|
+
const jobRoot = historicalReviewDrainSourceRoot(source);
|
|
2007
|
+
return Boolean(jobRoot) && (jobRoot === latestRoot || jobRoot.endsWith(latestRoot) || latestRoot.endsWith(jobRoot));
|
|
2008
|
+
});
|
|
2009
|
+
}
|
|
2010
|
+
function historicalReviewDrainSourceRoot(value) {
|
|
2011
|
+
const normalizedPath = value.replaceAll('\\', '/').replace(/\/(?:queue-overlay|collection|coordinator-query|swarm-results|coordinator-dashboard)\.json$/u, '');
|
|
2012
|
+
const autoDrainIndex = normalizedPath.indexOf('/auto-drain/');
|
|
2013
|
+
if (autoDrainIndex >= 0)
|
|
2014
|
+
return normalizedPath.slice(0, autoDrainIndex);
|
|
2015
|
+
const collectionIndex = normalizedPath.search(/\/(?:collection|collected|post-coordinator-collected|coordinator-collected)[^/]*(?:\/|$)/u);
|
|
2016
|
+
if (collectionIndex >= 0)
|
|
2017
|
+
return normalizedPath.slice(0, collectionIndex);
|
|
2018
|
+
return normalizedPath.replace(/\/$/u, '');
|
|
2019
|
+
}
|
|
663
2020
|
function coordinatorDecisionIds(record) {
|
|
664
2021
|
return Array.from(new Set([
|
|
665
2022
|
textValue(record.id, ''),
|
|
@@ -711,10 +2068,27 @@ function lifetimeDashboardSummary(jobs) {
|
|
|
711
2068
|
averageDurationMs: jobs.length ? Math.round(durationMs / jobs.length) : 0,
|
|
712
2069
|
maxDurationMs: jobs.reduce((max, job) => Math.max(max, numberValue(job.durationMs)), 0),
|
|
713
2070
|
actualInputTokens: jobs.reduce((sum, job) => sum + numberValue(job.actualInputTokens), 0),
|
|
2071
|
+
estimatedInputTokens: jobs.reduce((sum, job) => sum + numberValue(job.estimatedInputTokens), 0),
|
|
714
2072
|
cachedInputTokens: jobs.reduce((sum, job) => sum + numberValue(job.cachedInputTokens), 0),
|
|
715
|
-
uncachedInputTokens: jobs.reduce((sum, job) => sum + numberValue(job.uncachedInputTokens), 0)
|
|
2073
|
+
uncachedInputTokens: jobs.reduce((sum, job) => sum + numberValue(job.uncachedInputTokens), 0),
|
|
2074
|
+
estimatedCostUsd: roundUsd(jobs.reduce((sum, job) => sum + numberValue(job.estimatedCostUsd), 0)),
|
|
2075
|
+
estimatedInputCostUsd: roundUsd(jobs.reduce((sum, job) => sum + numberValue(job.estimatedInputCostUsd), 0)),
|
|
2076
|
+
estimatedCachedInputCostUsd: roundUsd(jobs.reduce((sum, job) => sum + numberValue(job.estimatedCachedInputCostUsd), 0)),
|
|
2077
|
+
estimatedUncachedInputCostUsd: roundUsd(jobs.reduce((sum, job) => sum + numberValue(job.estimatedUncachedInputCostUsd), 0)),
|
|
2078
|
+
estimatedOutputCostUsd: roundUsd(jobs.reduce((sum, job) => sum + numberValue(job.estimatedOutputCostUsd), 0)),
|
|
2079
|
+
estimatedCostMicroUsd: jobs.reduce((sum, job) => sum + numberValue(job.estimatedCostMicroUsd), 0),
|
|
2080
|
+
priceKnownJobCount: jobs.filter((job) => job.priceKnown === true).length,
|
|
2081
|
+
bucketCounts: countJobsByBucket(jobs)
|
|
716
2082
|
};
|
|
717
2083
|
}
|
|
2084
|
+
function countJobsByBucket(jobs) {
|
|
2085
|
+
const counts = {};
|
|
2086
|
+
for (const job of jobs) {
|
|
2087
|
+
const bucket = textValue(job.bucket, 'unknown') || 'unknown';
|
|
2088
|
+
counts[bucket] = (counts[bucket] ?? 0) + 1;
|
|
2089
|
+
}
|
|
2090
|
+
return counts;
|
|
2091
|
+
}
|
|
718
2092
|
function lifetimeHealthSummary(jobs) {
|
|
719
2093
|
const summary = lifetimeDashboardSummary(jobs);
|
|
720
2094
|
const failedJobCount = numberValue(summary.failedCount);
|
|
@@ -755,6 +2129,144 @@ function lifetimeLaneRows(jobs) {
|
|
|
755
2129
|
runningCount: entries.filter((job) => textValue(job.status, '') === 'running').length
|
|
756
2130
|
}));
|
|
757
2131
|
}
|
|
2132
|
+
function lifetimeQueueBacklogOverlay(queueBacklog, jobs) {
|
|
2133
|
+
const jobsByKey = new Map();
|
|
2134
|
+
for (const job of jobs) {
|
|
2135
|
+
for (const key of recordIdentityKeys(job).map(canonicalLifetimeTaskKey).filter(Boolean)) {
|
|
2136
|
+
jobsByKey.set(key, [...(jobsByKey.get(key) ?? []), job]);
|
|
2137
|
+
}
|
|
2138
|
+
}
|
|
2139
|
+
let activeCount = 0;
|
|
2140
|
+
let doneCount = 0;
|
|
2141
|
+
let failedCount = 0;
|
|
2142
|
+
let representedCount = 0;
|
|
2143
|
+
const entries = [];
|
|
2144
|
+
for (const entry of queueBacklog.entries) {
|
|
2145
|
+
const matchedJobs = Array.from(new Set(recordIdentityKeys(entry).map(canonicalLifetimeTaskKey).flatMap((key) => jobsByKey.get(key) ?? [])));
|
|
2146
|
+
if (!matchedJobs.length) {
|
|
2147
|
+
entries.push(entry);
|
|
2148
|
+
continue;
|
|
2149
|
+
}
|
|
2150
|
+
representedCount += 1;
|
|
2151
|
+
if (matchedJobs.some((job) => textValue(job.status, '') === 'running'))
|
|
2152
|
+
activeCount += 1;
|
|
2153
|
+
else if (matchedJobs.some(isLifetimeFailedJob))
|
|
2154
|
+
failedCount += 1;
|
|
2155
|
+
else if (matchedJobs.some((job) => textValue(job.status, '') === 'completed' || isResolvedCoordinatorReviewRecord(job)))
|
|
2156
|
+
doneCount += 1;
|
|
2157
|
+
else
|
|
2158
|
+
activeCount += 1;
|
|
2159
|
+
}
|
|
2160
|
+
return {
|
|
2161
|
+
entries,
|
|
2162
|
+
totalCount: queueBacklog.entries.length,
|
|
2163
|
+
readyCount: entries.filter((entry) => textValue(entry.status, '') === 'todo').length,
|
|
2164
|
+
activeCount,
|
|
2165
|
+
doneCount,
|
|
2166
|
+
failedCount,
|
|
2167
|
+
representedCount
|
|
2168
|
+
};
|
|
2169
|
+
}
|
|
2170
|
+
function lifetimeCapacitySummary(queueBacklog, jobs, openQueueEntries = queueBacklog.entries) {
|
|
2171
|
+
const manifest = queueBacklog.manifests[0];
|
|
2172
|
+
const laneRows = new Map();
|
|
2173
|
+
const terminalTaskIds = new Set(jobs
|
|
2174
|
+
.filter((job) => ['completed', 'failed', 'blocked'].includes(textValue(job.status, '').toLowerCase()))
|
|
2175
|
+
.flatMap((job) => recordIdentityKeys(job).map(canonicalLifetimeTaskKey)));
|
|
2176
|
+
const representedTaskIds = new Set(jobs.flatMap((job) => recordIdentityKeys(job).map(canonicalLifetimeTaskKey)));
|
|
2177
|
+
const openEntries = openQueueEntries.filter((entry) => {
|
|
2178
|
+
const ids = recordIdentityKeys(entry);
|
|
2179
|
+
return !ids.some((id) => terminalTaskIds.has(canonicalLifetimeTaskKey(id)) || representedTaskIds.has(canonicalLifetimeTaskKey(id)));
|
|
2180
|
+
});
|
|
2181
|
+
const queuedByLane = groupRecordsByText(openEntries, (entry) => textValue(entry.lane ?? entry.group ?? entry.sourceQueue, 'unassigned'));
|
|
2182
|
+
const jobsByLane = groupRecordsByText(jobs, (job) => textValue(job.lane, 'unassigned'));
|
|
2183
|
+
const manifestLanes = manifest?.lanes ?? [];
|
|
2184
|
+
for (const lane of manifestLanes) {
|
|
2185
|
+
laneRows.set(lane.id, capacityLaneRow(lane, queuedByLane.get(lane.id) ?? [], jobsByLane.get(lane.id) ?? []));
|
|
2186
|
+
}
|
|
2187
|
+
for (const [laneId, entries] of queuedByLane) {
|
|
2188
|
+
if (!laneRows.has(laneId))
|
|
2189
|
+
laneRows.set(laneId, capacityLaneRow({ id: laneId, title: laneId, layer: '', compute: '', model: '', maxConcurrency: 1 }, entries, jobsByLane.get(laneId) ?? []));
|
|
2190
|
+
}
|
|
2191
|
+
for (const [laneId, entries] of jobsByLane) {
|
|
2192
|
+
if (!laneRows.has(laneId))
|
|
2193
|
+
laneRows.set(laneId, capacityLaneRow({ id: laneId, title: laneId, layer: '', compute: '', model: '', maxConcurrency: 1 }, queuedByLane.get(laneId) ?? [], entries));
|
|
2194
|
+
}
|
|
2195
|
+
const lanes = Array.from(laneRows.values()).sort((left, right) => {
|
|
2196
|
+
const pressure = numberValue(right.runningCount) - numberValue(left.runningCount)
|
|
2197
|
+
|| numberValue(right.queuedTaskCount) - numberValue(left.queuedTaskCount);
|
|
2198
|
+
return pressure || textValue(left.id, '').localeCompare(textValue(right.id, ''));
|
|
2199
|
+
});
|
|
2200
|
+
const runningAgentCount = jobs.filter((job) => textValue(job.status, '') === 'running').length;
|
|
2201
|
+
const queuedTaskCount = lanes.reduce((sum, lane) => sum + numberValue(lane.queuedTaskCount), 0);
|
|
2202
|
+
return {
|
|
2203
|
+
manifestPath: manifest?.path ?? '',
|
|
2204
|
+
manifestId: manifest?.id ?? '',
|
|
2205
|
+
title: manifest?.title ?? 'Swarm capacity',
|
|
2206
|
+
defaultConcurrency: manifest?.defaultConcurrency ?? 0,
|
|
2207
|
+
computeMaxConcurrency: manifest?.computeMaxConcurrency ?? 0,
|
|
2208
|
+
maxConcurrency: manifest?.maxConcurrency ?? 0,
|
|
2209
|
+
laneCount: lanes.length,
|
|
2210
|
+
openLaneCount: lanes.filter((lane) => numberValue(lane.queuedTaskCount) > 0 || numberValue(lane.runningCount) > 0).length,
|
|
2211
|
+
activeLaneCount: lanes.filter((lane) => numberValue(lane.runningCount) > 0).length,
|
|
2212
|
+
runningAgentCount,
|
|
2213
|
+
assignedAgentCount: lanes.reduce((sum, lane) => sum + numberValue(lane.assignedAgentCount), 0),
|
|
2214
|
+
queuedTaskCount,
|
|
2215
|
+
totalTaskCount: queueBacklog.entries.length,
|
|
2216
|
+
completedTaskCount: jobs.filter((job) => textValue(job.status, '') === 'completed').length,
|
|
2217
|
+
lanes,
|
|
2218
|
+
queueSources: queueBacklog.paths
|
|
2219
|
+
};
|
|
2220
|
+
}
|
|
2221
|
+
function capacityLaneRow(lane, queuedEntries, laneJobs) {
|
|
2222
|
+
const runningJobs = laneJobs.filter((job) => textValue(job.status, '') === 'running');
|
|
2223
|
+
const queuedJobs = laneJobs.filter((job) => textValue(job.status, '') === 'queued');
|
|
2224
|
+
const assignedAgents = Array.from(new Set(runningJobs.map((job) => textValue(job.agentId ?? job.workerId ?? job.id, '')).filter(Boolean))).slice(0, 6);
|
|
2225
|
+
const queuedKeys = new Set([
|
|
2226
|
+
...queuedEntries.filter((entry) => {
|
|
2227
|
+
const status = normalized(entry.status ?? entry.queueStatus);
|
|
2228
|
+
return !status || ['todo', 'queued', 'pending', 'ready', 'open'].includes(status);
|
|
2229
|
+
}).map(primaryRecordIdentityKey).filter(Boolean),
|
|
2230
|
+
...queuedJobs.map(primaryRecordIdentityKey).filter(Boolean)
|
|
2231
|
+
]);
|
|
2232
|
+
return {
|
|
2233
|
+
id: lane.id,
|
|
2234
|
+
title: lane.title || lane.id,
|
|
2235
|
+
layer: lane.layer,
|
|
2236
|
+
compute: lane.compute,
|
|
2237
|
+
model: lane.model,
|
|
2238
|
+
maxConcurrency: lane.maxConcurrency,
|
|
2239
|
+
queuedTaskCount: queuedKeys.size,
|
|
2240
|
+
totalTaskCount: queuedEntries.length,
|
|
2241
|
+
runningCount: runningJobs.length,
|
|
2242
|
+
completedCount: laneJobs.filter((job) => textValue(job.status, '') === 'completed').length,
|
|
2243
|
+
failedCount: laneJobs.filter(isLifetimeFailedJob).length,
|
|
2244
|
+
assignedAgentCount: assignedAgents.length,
|
|
2245
|
+
assignedAgents
|
|
2246
|
+
};
|
|
2247
|
+
}
|
|
2248
|
+
function recordIdentityKeys(record) {
|
|
2249
|
+
return Array.from(new Set([
|
|
2250
|
+
textValue(record.id, ''),
|
|
2251
|
+
textValue(record.originalJobId, ''),
|
|
2252
|
+
textValue(record.jobId, ''),
|
|
2253
|
+
textValue(record.taskId, '')
|
|
2254
|
+
].filter(Boolean)));
|
|
2255
|
+
}
|
|
2256
|
+
function primaryRecordIdentityKey(record) {
|
|
2257
|
+
return canonicalLifetimeTaskKey(textValue(record.taskId, ''))
|
|
2258
|
+
|| canonicalLifetimeTaskKey(textValue(record.originalJobId, ''))
|
|
2259
|
+
|| canonicalLifetimeTaskKey(textValue(record.jobId, ''))
|
|
2260
|
+
|| canonicalLifetimeTaskKey(textValue(record.id, ''));
|
|
2261
|
+
}
|
|
2262
|
+
function groupRecordsByText(records, keyFor) {
|
|
2263
|
+
const groups = new Map();
|
|
2264
|
+
for (const record of records) {
|
|
2265
|
+
const key = keyFor(record) || 'unassigned';
|
|
2266
|
+
groups.set(key, [...(groups.get(key) ?? []), record]);
|
|
2267
|
+
}
|
|
2268
|
+
return groups;
|
|
2269
|
+
}
|
|
758
2270
|
function lifetimeTimeSeries(jobs, events) {
|
|
759
2271
|
const bucketMs = 24 * 60 * 60 * 1000;
|
|
760
2272
|
const buckets = new Map();
|
|
@@ -763,7 +2275,7 @@ function lifetimeTimeSeries(jobs, events) {
|
|
|
763
2275
|
if (!at)
|
|
764
2276
|
continue;
|
|
765
2277
|
const bucketAt = startOfLocalDay(at);
|
|
766
|
-
const bucket = buckets.get(bucketAt) ??
|
|
2278
|
+
const bucket = buckets.get(bucketAt) ?? emptyLifetimeTimeBucket(bucketAt);
|
|
767
2279
|
if (['completed', 'failed', 'blocked'].includes(textValue(job.status, '').toLowerCase()))
|
|
768
2280
|
bucket.terminalJobCount += 1;
|
|
769
2281
|
if (textValue(job.health, '') === 'warning')
|
|
@@ -772,7 +2284,10 @@ function lifetimeTimeSeries(jobs, events) {
|
|
|
772
2284
|
bucket.failureJobCount += 1;
|
|
773
2285
|
bucket.durationMs += numberValue(job.durationMs);
|
|
774
2286
|
bucket.actualInputTokens += numberValue(job.actualInputTokens);
|
|
2287
|
+
bucket.estimatedInputTokens += numberValue(job.estimatedInputTokens);
|
|
775
2288
|
bucket.uncachedInputTokens += numberValue(job.uncachedInputTokens);
|
|
2289
|
+
bucket.estimatedCostUsd = roundUsd(bucket.estimatedCostUsd + numberValue(job.estimatedCostUsd));
|
|
2290
|
+
bucket.estimatedCostMicroUsd += numberValue(job.estimatedCostMicroUsd);
|
|
776
2291
|
buckets.set(bucketAt, bucket);
|
|
777
2292
|
}
|
|
778
2293
|
for (const event of events) {
|
|
@@ -780,7 +2295,7 @@ function lifetimeTimeSeries(jobs, events) {
|
|
|
780
2295
|
if (!at)
|
|
781
2296
|
continue;
|
|
782
2297
|
const bucketAt = startOfLocalDay(at);
|
|
783
|
-
const bucket = buckets.get(bucketAt) ??
|
|
2298
|
+
const bucket = buckets.get(bucketAt) ?? emptyLifetimeTimeBucket(bucketAt);
|
|
784
2299
|
bucket.eventCount += 1;
|
|
785
2300
|
buckets.set(bucketAt, bucket);
|
|
786
2301
|
}
|
|
@@ -795,11 +2310,29 @@ function lifetimeTimeSeries(jobs, events) {
|
|
|
795
2310
|
failureJobCount: points.reduce((sum, point) => sum + point.failureJobCount, 0),
|
|
796
2311
|
durationMs: points.reduce((sum, point) => sum + point.durationMs, 0),
|
|
797
2312
|
actualInputTokens: points.reduce((sum, point) => sum + point.actualInputTokens, 0),
|
|
2313
|
+
estimatedInputTokens: points.reduce((sum, point) => sum + point.estimatedInputTokens, 0),
|
|
798
2314
|
uncachedInputTokens: points.reduce((sum, point) => sum + point.uncachedInputTokens, 0),
|
|
2315
|
+
estimatedCostUsd: roundUsd(points.reduce((sum, point) => sum + point.estimatedCostUsd, 0)),
|
|
2316
|
+
estimatedCostMicroUsd: points.reduce((sum, point) => sum + point.estimatedCostMicroUsd, 0),
|
|
799
2317
|
missingTimestampJobCount: jobs.filter((job) => !numberValue(job.finishedAt) && !numberValue(job.generatedAt) && !numberValue(job.startedAt)).length
|
|
800
2318
|
}
|
|
801
2319
|
};
|
|
802
2320
|
}
|
|
2321
|
+
function emptyLifetimeTimeBucket(at) {
|
|
2322
|
+
return {
|
|
2323
|
+
at,
|
|
2324
|
+
terminalJobCount: 0,
|
|
2325
|
+
warningJobCount: 0,
|
|
2326
|
+
failureJobCount: 0,
|
|
2327
|
+
durationMs: 0,
|
|
2328
|
+
actualInputTokens: 0,
|
|
2329
|
+
estimatedInputTokens: 0,
|
|
2330
|
+
uncachedInputTokens: 0,
|
|
2331
|
+
estimatedCostUsd: 0,
|
|
2332
|
+
estimatedCostMicroUsd: 0,
|
|
2333
|
+
eventCount: 0
|
|
2334
|
+
};
|
|
2335
|
+
}
|
|
803
2336
|
function startOfLocalDay(value) {
|
|
804
2337
|
const date = new Date(value);
|
|
805
2338
|
return new Date(date.getFullYear(), date.getMonth(), date.getDate()).getTime();
|
|
@@ -813,28 +2346,92 @@ function lifetimeSemanticSummary(jobs) {
|
|
|
813
2346
|
conflicts: jobs.filter((job) => textValue(job.semanticReadiness, '') === 'blocked').length
|
|
814
2347
|
};
|
|
815
2348
|
}
|
|
816
|
-
function lifetimeRoutingSummary(
|
|
817
|
-
const routingRows =
|
|
818
|
-
|
|
2349
|
+
async function lifetimeRoutingSummary(cwd, entries) {
|
|
2350
|
+
const routingRows = entries.map(({ snapshot }) => recordValue(snapshot.routing)).filter((entry) => Object.keys(entry).length);
|
|
2351
|
+
const sidecars = await readLifetimeRoutingSidecars(cwd, entries.map(({ source }) => source));
|
|
2352
|
+
if (!routingRows.length && !sidecars.tournamentCount && !sidecars.feedbackArtifactCount && !sidecars.historyCount)
|
|
819
2353
|
return undefined;
|
|
820
2354
|
return {
|
|
821
2355
|
policyId: 'workspace-lifetime',
|
|
822
2356
|
preferenceCount: routingRows.reduce((sum, row) => sum + numberValue(row.preferenceCount), 0),
|
|
823
2357
|
preferCount: routingRows.reduce((sum, row) => sum + numberValue(row.preferCount), 0),
|
|
824
2358
|
avoidCount: routingRows.reduce((sum, row) => sum + numberValue(row.avoidCount), 0),
|
|
825
|
-
|
|
826
|
-
|
|
2359
|
+
feedbackCount: routingRows.reduce((sum, row) => sum + numberValue(row.feedbackCount), 0) + sidecars.feedbackSignalCount,
|
|
2360
|
+
tournamentObservationCount: routingRows.reduce((sum, row) => sum + numberValue(row.tournamentObservationCount), 0) + sidecars.tournamentObservationCount,
|
|
2361
|
+
tournamentRecommendationCount: routingRows.reduce((sum, row) => sum + numberValue(row.tournamentRecommendationCount), 0) + sidecars.tournamentRecommendationCount,
|
|
2362
|
+
tournamentCount: sidecars.tournamentCount,
|
|
2363
|
+
tournamentMatchCount: sidecars.tournamentMatchCount,
|
|
2364
|
+
tournamentVerifiedCount: sidecars.tournamentVerifiedCount,
|
|
2365
|
+
tournamentTopStrategyId: sidecars.topStrategyId,
|
|
2366
|
+
tournamentDecisionGrade: sidecars.decisionGrade,
|
|
2367
|
+
strategyHistoryCount: sidecars.historyCount,
|
|
2368
|
+
feedbackArtifactCount: sidecars.feedbackArtifactCount
|
|
2369
|
+
};
|
|
2370
|
+
}
|
|
2371
|
+
async function readLifetimeRoutingSidecars(cwd, sources) {
|
|
2372
|
+
const out = {
|
|
2373
|
+
tournamentCount: 0,
|
|
2374
|
+
tournamentMatchCount: 0,
|
|
2375
|
+
tournamentVerifiedCount: 0,
|
|
2376
|
+
tournamentObservationCount: 0,
|
|
2377
|
+
tournamentRecommendationCount: 0,
|
|
2378
|
+
feedbackSignalCount: 0,
|
|
2379
|
+
feedbackArtifactCount: 0,
|
|
2380
|
+
historyCount: 0,
|
|
2381
|
+
topStrategyId: '',
|
|
2382
|
+
decisionGrade: ''
|
|
827
2383
|
};
|
|
2384
|
+
const seen = new Set();
|
|
2385
|
+
for (const source of sources) {
|
|
2386
|
+
const dir = lifetimeRoutingSidecarDir(cwd, source);
|
|
2387
|
+
if (!dir || seen.has(dir))
|
|
2388
|
+
continue;
|
|
2389
|
+
seen.add(dir);
|
|
2390
|
+
const tournament = recordValue(await readJsonFile(path.join(dir, 'strategy-tournament.json')));
|
|
2391
|
+
const tournamentSummary = recordValue(tournament.summary);
|
|
2392
|
+
if (Object.keys(tournament).length) {
|
|
2393
|
+
out.tournamentCount += 1;
|
|
2394
|
+
out.tournamentMatchCount += numberValue(tournamentSummary.matchCount);
|
|
2395
|
+
out.tournamentVerifiedCount += numberValue(tournamentSummary.verifiedCount);
|
|
2396
|
+
out.tournamentObservationCount += numberValue(tournamentSummary.matchCount) || recordArray(tournament.matches).length;
|
|
2397
|
+
out.tournamentRecommendationCount += textValue(tournament.winnerId ?? tournamentSummary.topStrategyId, '') ? 1 : 0;
|
|
2398
|
+
if (!out.topStrategyId)
|
|
2399
|
+
out.topStrategyId = textValue(tournamentSummary.topStrategyId ?? tournament.winnerId, '');
|
|
2400
|
+
if (!out.decisionGrade)
|
|
2401
|
+
out.decisionGrade = textValue(tournamentSummary.decisionGrade, '');
|
|
2402
|
+
}
|
|
2403
|
+
const feedback = recordValue(await readJsonFile(path.join(dir, 'tournament-adaptive-feedback.json')));
|
|
2404
|
+
const feedbackSummary = recordValue(feedback.summary);
|
|
2405
|
+
if (Object.keys(feedback).length) {
|
|
2406
|
+
out.feedbackArtifactCount += 1;
|
|
2407
|
+
out.tournamentObservationCount += numberValue(feedbackSummary.observationCount) || recordArray(feedback.observations).length;
|
|
2408
|
+
out.tournamentRecommendationCount += numberValue(feedbackSummary.recommendationCount) || recordArray(feedback.recommendations).length;
|
|
2409
|
+
out.feedbackSignalCount += numberValue(feedbackSummary.reduceSignals) + numberValue(feedbackSummary.increaseSignals) + numberValue(feedbackSummary.holdSignals);
|
|
2410
|
+
}
|
|
2411
|
+
const history = recordValue(await readJsonFile(path.join(dir, 'strategy-history.json')));
|
|
2412
|
+
if (Object.keys(history).length) {
|
|
2413
|
+
out.historyCount += recordArray(history.tournaments).length || 1;
|
|
2414
|
+
}
|
|
2415
|
+
}
|
|
2416
|
+
return out;
|
|
2417
|
+
}
|
|
2418
|
+
function lifetimeRoutingSidecarDir(cwd, source) {
|
|
2419
|
+
const relative = source.collection ?? source.continuation ?? source.run;
|
|
2420
|
+
if (!relative)
|
|
2421
|
+
return undefined;
|
|
2422
|
+
const absolute = path.resolve(cwd, relative);
|
|
2423
|
+
return isPathInside(cwd, absolute) ? absolute : undefined;
|
|
828
2424
|
}
|
|
829
2425
|
function isLifetimeFailedJob(job) {
|
|
2426
|
+
if (isResolvedCoordinatorReviewRecord(job))
|
|
2427
|
+
return false;
|
|
830
2428
|
const status = textValue(job.status, '').toLowerCase();
|
|
831
2429
|
const health = textValue(job.health, '').toLowerCase();
|
|
832
2430
|
const bucket = textValue(job.bucket, '').toLowerCase();
|
|
2431
|
+
if (bucket === 'rerun-work')
|
|
2432
|
+
return false;
|
|
833
2433
|
return status === 'failed' || health === 'failed' || bucket === 'failed-evidence';
|
|
834
2434
|
}
|
|
835
|
-
function awaitNoop(value) {
|
|
836
|
-
return value;
|
|
837
|
-
}
|
|
838
2435
|
function shouldPreferActiveRunSnapshot(jobs, activeJobs) {
|
|
839
2436
|
if (!activeJobs.length)
|
|
840
2437
|
return false;
|
|
@@ -843,24 +2440,27 @@ function shouldPreferActiveRunSnapshot(jobs, activeJobs) {
|
|
|
843
2440
|
return true;
|
|
844
2441
|
return activeJobs.length > jobs.length;
|
|
845
2442
|
}
|
|
846
|
-
async function readActiveRunSnapshot(options) {
|
|
2443
|
+
async function readActiveRunSnapshot(options, readOptions = {}) {
|
|
847
2444
|
const runDir = await resolveRunDirectory(options);
|
|
848
2445
|
if (!runDir)
|
|
849
2446
|
return undefined;
|
|
850
2447
|
const pidPath = path.join(runDir, 'pids.json');
|
|
851
2448
|
const pidManifest = recordValue(await readJsonFile(pidPath));
|
|
852
|
-
|
|
2449
|
+
let entries = recordArray(pidManifest.entries).filter((entry) => textValue(entry.role, '') === 'codex');
|
|
2450
|
+
if (readOptions.runningOnly)
|
|
2451
|
+
entries = entries.filter((entry) => isProcessLive(numberValue(entry.pid), entry));
|
|
853
2452
|
if (!entries.length)
|
|
854
2453
|
return undefined;
|
|
855
2454
|
const planPath = path.join(runDir, 'swarm-plan.json');
|
|
856
2455
|
const plan = recordValue(await readJsonFile(planPath));
|
|
857
2456
|
const planJobs = new Map(recordArray(plan.jobs).map((job) => [textValue(job.id, ''), job]));
|
|
858
2457
|
const now = Date.now();
|
|
859
|
-
const jobs = await Promise.all(entries.map((entry) => activeRunJob(runDir, entry, planJobs.get(textValue(entry.jobId, '')), now)));
|
|
2458
|
+
const jobs = await Promise.all(entries.map((entry) => activeRunJob(options.cwd, runDir, entry, planJobs.get(textValue(entry.jobId, '')), now, readOptions)));
|
|
860
2459
|
const runningCount = jobs.filter((job) => textValue(job.status, '') === 'running').length;
|
|
861
2460
|
const completedCount = jobs.filter((job) => textValue(job.status, '') === 'completed').length;
|
|
862
2461
|
const failedCount = jobs.filter((job) => textValue(job.status, '') === 'failed').length;
|
|
863
2462
|
const actualInputTokens = jobs.reduce((sum, job) => sum + numberValue(job.actualInputTokens), 0);
|
|
2463
|
+
const estimatedInputTokens = jobs.reduce((sum, job) => sum + numberValue(job.estimatedInputTokens), 0);
|
|
864
2464
|
const cachedInputTokens = jobs.reduce((sum, job) => sum + numberValue(job.cachedInputTokens), 0);
|
|
865
2465
|
return {
|
|
866
2466
|
ok: true,
|
|
@@ -873,6 +2473,7 @@ async function readActiveRunSnapshot(options) {
|
|
|
873
2473
|
runningCount,
|
|
874
2474
|
blockedCount: 0,
|
|
875
2475
|
actualInputTokens,
|
|
2476
|
+
estimatedInputTokens,
|
|
876
2477
|
cachedInputTokens,
|
|
877
2478
|
uncachedInputTokens: Math.max(0, actualInputTokens - cachedInputTokens),
|
|
878
2479
|
durationMs: jobs.reduce((sum, job) => Math.max(sum, numberValue(job.durationMs)), 0),
|
|
@@ -885,6 +2486,7 @@ async function readActiveRunSnapshot(options) {
|
|
|
885
2486
|
},
|
|
886
2487
|
lanes: activeRunLanes(jobs),
|
|
887
2488
|
jobs,
|
|
2489
|
+
activeAgents: activeAgentsFromJobs(jobs),
|
|
888
2490
|
events: activeRunEvents(jobs),
|
|
889
2491
|
sources: {
|
|
890
2492
|
run: runDir,
|
|
@@ -900,35 +2502,53 @@ async function readActiveRunSnapshot(options) {
|
|
|
900
2502
|
}
|
|
901
2503
|
};
|
|
902
2504
|
}
|
|
903
|
-
async function activeRunJob(runDir, entry, planJob, now) {
|
|
2505
|
+
async function activeRunJob(cwd, runDir, entry, planJob, now, readOptions = {}) {
|
|
904
2506
|
const jobId = textValue(entry.jobId, 'job');
|
|
905
2507
|
const jobDir = path.join(runDir, jobId);
|
|
906
2508
|
const lastMessagePath = path.join(jobDir, 'last-message.md');
|
|
907
2509
|
const mergePath = path.join(jobDir, 'merge.json');
|
|
2510
|
+
const eventsPath = path.join(jobDir, 'codex-events.jsonl');
|
|
2511
|
+
const evidenceRecord = recordValue(await readJsonFile(path.join(jobDir, 'evidence', 'evidence.json')));
|
|
908
2512
|
const lastMessage = await fs.stat(lastMessagePath).catch(() => undefined);
|
|
909
2513
|
const merge = recordValue(await readJsonFile(mergePath));
|
|
910
2514
|
const live = isProcessLive(numberValue(entry.pid), entry);
|
|
911
|
-
const
|
|
2515
|
+
const quotaDeferred = !live && !lastMessage && !Object.keys(merge).length && await codexEventsHaveQuotaLimit(eventsPath);
|
|
2516
|
+
const status = live && !lastMessage ? 'running' : quotaDeferred ? 'completed' : lastMessage || Object.keys(merge).length ? 'completed' : 'failed';
|
|
912
2517
|
const startedAt = numberValue(entry.startedAt);
|
|
913
2518
|
const finishedAt = status === 'running' ? undefined : Math.max(numberValue(lastMessage?.mtimeMs), numberValue(merge.generatedAt));
|
|
914
|
-
const
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
2519
|
+
const rawPatchPath = await firstExistingRelativePath(cwd, rawRunPatchCandidates(jobDir));
|
|
2520
|
+
const evidencePaths = await existingRelativePaths(cwd, [
|
|
2521
|
+
lastMessagePath,
|
|
2522
|
+
eventsPath,
|
|
2523
|
+
path.join(jobDir, 'evidence', 'last-message.md'),
|
|
2524
|
+
path.join(jobDir, 'evidence', 'handoff.md'),
|
|
2525
|
+
path.join(jobDir, 'evidence', 'evidence.json'),
|
|
2526
|
+
path.join(jobDir, 'evidence', 'resource-allocation.json'),
|
|
2527
|
+
...rawRunPatchCandidates(jobDir),
|
|
2528
|
+
...(Object.keys(merge).length ? [mergePath] : [])
|
|
2529
|
+
]);
|
|
2530
|
+
const usage = readOptions.includeUsage === false ? emptyCodexEventUsageSummary() : await readCodexEventUsageSummary(eventsPath);
|
|
921
2531
|
const task = recordValue(planJob?.task);
|
|
922
2532
|
const compute = recordValue(planJob?.compute);
|
|
923
|
-
const changedPaths =
|
|
924
|
-
|
|
2533
|
+
const changedPaths = uniquePaths([
|
|
2534
|
+
...stringArray(merge.changedPaths),
|
|
2535
|
+
...await readPatchChangedPathList(cwd, rawPatchPath)
|
|
2536
|
+
]);
|
|
2537
|
+
const commandEvidence = commandEvidenceFromRecords(merge, evidenceRecord);
|
|
2538
|
+
return withRecomputedCostFields({
|
|
925
2539
|
id: jobId,
|
|
926
2540
|
taskId: textValue(planJob?.taskId ?? task.id, jobId),
|
|
927
2541
|
title: textValue(planJob?.title ?? task.title, jobId),
|
|
928
2542
|
lane: textValue(planJob?.lane ?? task.lane, 'active-run'),
|
|
929
2543
|
status,
|
|
930
|
-
bucket: status === 'running' ? 'running' : status === 'completed' ? 'completed' : 'failed-evidence',
|
|
931
|
-
disposition: status === 'running' ? 'active' : status,
|
|
2544
|
+
bucket: quotaDeferred ? 'review-resolved' : status === 'running' ? 'running' : status === 'completed' ? 'completed' : 'failed-evidence',
|
|
2545
|
+
disposition: status === 'running' ? 'active' : quotaDeferred ? 'quota-deferred' : status,
|
|
2546
|
+
...(quotaDeferred ? {
|
|
2547
|
+
reviewResolved: true,
|
|
2548
|
+
coordinatorDecisionStatus: 'quota-deferred',
|
|
2549
|
+
originalStatus: 'queued',
|
|
2550
|
+
originalBucket: 'queued'
|
|
2551
|
+
} : {}),
|
|
932
2552
|
agentId: jobId,
|
|
933
2553
|
workerId: jobId,
|
|
934
2554
|
model: textValue(compute.model, ''),
|
|
@@ -938,6 +2558,7 @@ async function activeRunJob(runDir, entry, planJob, now) {
|
|
|
938
2558
|
...(finishedAt ? { finishedAt } : {}),
|
|
939
2559
|
durationMs: startedAt ? Math.max(0, (finishedAt ?? now) - startedAt) : 0,
|
|
940
2560
|
...(usage.inputTokens ? { actualInputTokens: usage.inputTokens, inputTokens: usage.inputTokens } : {}),
|
|
2561
|
+
...(!usage.inputTokens && usage.estimatedInputTokens ? { estimatedInputTokens: usage.estimatedInputTokens } : {}),
|
|
941
2562
|
...(usage.cachedInputTokens ? { cachedInputTokens: usage.cachedInputTokens } : {}),
|
|
942
2563
|
...(usage.uncachedInputTokens ? { uncachedInputTokens: usage.uncachedInputTokens } : {}),
|
|
943
2564
|
...(usage.outputTokens ? { actualOutputTokens: usage.outputTokens, outputTokens: usage.outputTokens } : {}),
|
|
@@ -949,19 +2570,22 @@ async function activeRunJob(runDir, entry, planJob, now) {
|
|
|
949
2570
|
uncached_input_tokens: usage.uncachedInputTokens,
|
|
950
2571
|
output_tokens: usage.outputTokens,
|
|
951
2572
|
reasoning_output_tokens: usage.reasoningOutputTokens,
|
|
2573
|
+
estimated_input_tokens: usage.estimatedInputTokens,
|
|
2574
|
+
estimated_from_event_bytes: usage.estimatedFromEventBytes,
|
|
952
2575
|
source: 'codex-events.jsonl',
|
|
953
2576
|
event_count: usage.eventCount
|
|
954
2577
|
}
|
|
955
2578
|
} : {}),
|
|
956
2579
|
changedPaths,
|
|
957
2580
|
changedPathCount: changedPaths.length || numberValue(merge.changedPathCount),
|
|
2581
|
+
...(rawPatchPath ? { patchPath: rawPatchPath, artifactPaths: [rawPatchPath] } : {}),
|
|
958
2582
|
evidencePaths,
|
|
959
2583
|
evidencePathCount: evidencePaths.length,
|
|
960
|
-
commandsPassed:
|
|
961
|
-
commandsFailed:
|
|
962
|
-
collectReasonClasses: status === 'running' ? ['active worker'] : [],
|
|
963
|
-
mergeReadiness: textValue(merge.mergeReadiness, status)
|
|
964
|
-
};
|
|
2584
|
+
commandsPassed: commandEvidence.passed,
|
|
2585
|
+
commandsFailed: commandEvidence.failed,
|
|
2586
|
+
collectReasonClasses: status === 'running' ? ['active worker'] : quotaDeferred ? ['quota deferred'] : [],
|
|
2587
|
+
mergeReadiness: quotaDeferred ? 'quota-deferred' : textValue(merge.mergeReadiness, status)
|
|
2588
|
+
});
|
|
965
2589
|
}
|
|
966
2590
|
function activeRunLanes(jobs) {
|
|
967
2591
|
const byLane = new Map();
|
|
@@ -988,6 +2612,39 @@ function activeRunEvents(jobs) {
|
|
|
988
2612
|
message: `${textValue(job.title, 'worker')} ${textValue(job.status, 'running')}`
|
|
989
2613
|
}));
|
|
990
2614
|
}
|
|
2615
|
+
function activeAgentsFromJobs(jobs) {
|
|
2616
|
+
return jobs
|
|
2617
|
+
.map(recordValue)
|
|
2618
|
+
.filter((job) => textValue(job.status, '') === 'running')
|
|
2619
|
+
.map((job) => {
|
|
2620
|
+
const id = textValue(job.agentId ?? job.workerId ?? job.originalJobId ?? job.id, 'agent');
|
|
2621
|
+
return {
|
|
2622
|
+
id,
|
|
2623
|
+
agentId: id,
|
|
2624
|
+
workerId: textValue(job.workerId ?? job.agentId ?? id, id),
|
|
2625
|
+
jobId: textValue(job.originalJobId ?? job.id ?? job.taskId, id),
|
|
2626
|
+
taskId: textValue(job.taskId ?? job.originalJobId ?? job.id, id),
|
|
2627
|
+
title: textValue(job.title, id),
|
|
2628
|
+
lane: textValue(job.lane, ''),
|
|
2629
|
+
status: 'active',
|
|
2630
|
+
model: textValue(job.model, ''),
|
|
2631
|
+
computeId: textValue(job.computeId, ''),
|
|
2632
|
+
reasoningEffort: textValue(job.reasoningEffort, ''),
|
|
2633
|
+
startedAt: numberValue(job.startedAt) || undefined,
|
|
2634
|
+
durationMs: numberValue(job.durationMs),
|
|
2635
|
+
inputTokens: numberValue(job.inputTokens || job.actualInputTokens || job.estimatedInputTokens),
|
|
2636
|
+
uncachedInputTokens: numberValue(job.uncachedInputTokens),
|
|
2637
|
+
cachedInputTokens: numberValue(job.cachedInputTokens),
|
|
2638
|
+
outputTokens: numberValue(job.outputTokens || job.actualOutputTokens),
|
|
2639
|
+
changedPaths: stringArray(job.changedPaths),
|
|
2640
|
+
changedPathCount: numberValue(job.changedPathCount) || stringArray(job.changedPaths).length,
|
|
2641
|
+
evidencePaths: stringArray(job.evidencePaths),
|
|
2642
|
+
evidencePathCount: numberValue(job.evidencePathCount) || stringArray(job.evidencePaths).length,
|
|
2643
|
+
sourceRun: textValue(job.sourceRun, ''),
|
|
2644
|
+
sourceLabel: textValue(job.sourceLabel, '')
|
|
2645
|
+
};
|
|
2646
|
+
});
|
|
2647
|
+
}
|
|
991
2648
|
function mergeActiveRunJobTelemetry(jobs, activeJobs) {
|
|
992
2649
|
if (!activeJobs.length)
|
|
993
2650
|
return jobs;
|
|
@@ -1011,6 +2668,7 @@ function mergeActiveRunJobTelemetry(jobs, activeJobs) {
|
|
|
1011
2668
|
...record,
|
|
1012
2669
|
...(numberValue(activeJob.actualInputTokens) ? { actualInputTokens: numberValue(activeJob.actualInputTokens) } : {}),
|
|
1013
2670
|
...(numberValue(activeJob.inputTokens) ? { inputTokens: numberValue(activeJob.inputTokens) } : {}),
|
|
2671
|
+
...(numberValue(activeJob.estimatedInputTokens) ? { estimatedInputTokens: numberValue(activeJob.estimatedInputTokens) } : {}),
|
|
1014
2672
|
...(numberValue(activeJob.cachedInputTokens) ? { cachedInputTokens: numberValue(activeJob.cachedInputTokens) } : {}),
|
|
1015
2673
|
...(numberValue(activeJob.uncachedInputTokens) ? { uncachedInputTokens: numberValue(activeJob.uncachedInputTokens) } : {}),
|
|
1016
2674
|
...(numberValue(activeJob.actualOutputTokens) ? { actualOutputTokens: numberValue(activeJob.actualOutputTokens) } : {}),
|
|
@@ -1035,6 +2693,7 @@ function jobTelemetryKeys(job) {
|
|
|
1035
2693
|
function hasTokenTelemetry(job) {
|
|
1036
2694
|
return numberValue(job.actualInputTokens)
|
|
1037
2695
|
+ numberValue(job.inputTokens)
|
|
2696
|
+
+ numberValue(job.estimatedInputTokens)
|
|
1038
2697
|
+ numberValue(job.cachedInputTokens)
|
|
1039
2698
|
+ numberValue(job.uncachedInputTokens)
|
|
1040
2699
|
+ numberValue(job.outputTokens)
|
|
@@ -1049,6 +2708,7 @@ async function readCodexEventUsageSummary(file) {
|
|
|
1049
2708
|
if (!text)
|
|
1050
2709
|
return empty;
|
|
1051
2710
|
const summary = emptyCodexEventUsageSummary();
|
|
2711
|
+
summary.estimatedFromEventBytes = Buffer.byteLength(text, 'utf8');
|
|
1052
2712
|
for (const line of text.split(/\r?\n/g)) {
|
|
1053
2713
|
const trimmed = line.trim();
|
|
1054
2714
|
if (!trimmed)
|
|
@@ -1076,6 +2736,9 @@ async function readCodexEventUsageSummary(file) {
|
|
|
1076
2736
|
if (summary.inputTokens && !summary.uncachedInputTokens) {
|
|
1077
2737
|
summary.uncachedInputTokens = Math.max(0, summary.inputTokens - summary.cachedInputTokens);
|
|
1078
2738
|
}
|
|
2739
|
+
if (!hasCodexUsageValues(summary)) {
|
|
2740
|
+
summary.estimatedInputTokens = estimateInputTokensFromEventText(text);
|
|
2741
|
+
}
|
|
1079
2742
|
return summary;
|
|
1080
2743
|
}
|
|
1081
2744
|
function emptyCodexEventUsageSummary() {
|
|
@@ -1085,9 +2748,17 @@ function emptyCodexEventUsageSummary() {
|
|
|
1085
2748
|
uncachedInputTokens: 0,
|
|
1086
2749
|
outputTokens: 0,
|
|
1087
2750
|
reasoningOutputTokens: 0,
|
|
2751
|
+
estimatedInputTokens: 0,
|
|
2752
|
+
estimatedFromEventBytes: 0,
|
|
1088
2753
|
eventCount: 0
|
|
1089
2754
|
};
|
|
1090
2755
|
}
|
|
2756
|
+
function estimateInputTokensFromEventText(text) {
|
|
2757
|
+
const compactText = text.replace(/\s+/g, ' ').trim();
|
|
2758
|
+
if (!compactText)
|
|
2759
|
+
return 0;
|
|
2760
|
+
return Math.max(1, Math.ceil(compactText.length / 4));
|
|
2761
|
+
}
|
|
1091
2762
|
function collectCodexUsageRecords(value, depth = 0) {
|
|
1092
2763
|
if (depth > 5 || !value || typeof value !== 'object')
|
|
1093
2764
|
return [];
|
|
@@ -1126,7 +2797,9 @@ function normalizeCodexUsageRecord(record) {
|
|
|
1126
2797
|
uncachedInputTokens,
|
|
1127
2798
|
outputTokens,
|
|
1128
2799
|
reasoningOutputTokens,
|
|
1129
|
-
|
|
2800
|
+
estimatedInputTokens: 0,
|
|
2801
|
+
estimatedFromEventBytes: 0,
|
|
2802
|
+
eventCount: hasCodexUsageValues({ inputTokens, cachedInputTokens, uncachedInputTokens, outputTokens, reasoningOutputTokens }) ? 1 : 0
|
|
1130
2803
|
};
|
|
1131
2804
|
}
|
|
1132
2805
|
function hasCodexUsageValues(usage) {
|
|
@@ -1159,6 +2832,196 @@ async function readJsonFile(file) {
|
|
|
1159
2832
|
return undefined;
|
|
1160
2833
|
}
|
|
1161
2834
|
}
|
|
2835
|
+
function withRecomputedCostFields(record) {
|
|
2836
|
+
const model = textValue(record.model ?? record.pricingModel ?? record.pricingMatchedModel, '');
|
|
2837
|
+
if (!model || !hasCostTokenEvidence(record))
|
|
2838
|
+
return record;
|
|
2839
|
+
const cost = estimateCodexModelCost({
|
|
2840
|
+
model,
|
|
2841
|
+
estimatedInputTokens: firstCostTokenNumber(record.estimatedInputTokens, record.estimated_input_tokens),
|
|
2842
|
+
actualInputTokens: firstCostTokenNumber(record.actualInputTokens, record.inputTokens, record.promptTokens, record.actual_input_tokens, record.input_tokens, record.prompt_tokens),
|
|
2843
|
+
cachedInputTokens: firstCostTokenNumber(record.cachedInputTokens, record.cachedPromptTokens, record.cached_input_tokens, record.cached_prompt_tokens),
|
|
2844
|
+
uncachedInputTokens: firstCostTokenNumber(record.uncachedInputTokens, record.uncached_input_tokens),
|
|
2845
|
+
outputTokens: optionalCostTokenNumber(record.actualOutputTokens, record.outputTokens, record.completionTokens, record.responseTokens, record.actual_output_tokens, record.output_tokens, record.completion_tokens, record.response_tokens)
|
|
2846
|
+
});
|
|
2847
|
+
return {
|
|
2848
|
+
...record,
|
|
2849
|
+
billableInputTokens: cost.billableInputTokens,
|
|
2850
|
+
priceKnown: cost.priceKnown,
|
|
2851
|
+
...(cost.pricingModel ? { pricingModel: cost.pricingModel } : {}),
|
|
2852
|
+
...(cost.pricingMatchedModel ? { pricingMatchedModel: cost.pricingMatchedModel } : {}),
|
|
2853
|
+
...(cost.pricingSource ? { pricingSource: cost.pricingSource } : {}),
|
|
2854
|
+
...(cost.pricingUpdatedAt ? { pricingUpdatedAt: cost.pricingUpdatedAt } : {}),
|
|
2855
|
+
estimatedCostUsd: cost.estimatedCostUsd,
|
|
2856
|
+
estimatedInputCostUsd: cost.estimatedInputCostUsd,
|
|
2857
|
+
estimatedCachedInputCostUsd: cost.estimatedCachedInputCostUsd,
|
|
2858
|
+
estimatedUncachedInputCostUsd: cost.estimatedUncachedInputCostUsd,
|
|
2859
|
+
estimatedOutputCostUsd: cost.estimatedOutputCostUsd,
|
|
2860
|
+
estimatedCostMicroUsd: cost.estimatedCostMicroUsd,
|
|
2861
|
+
costEstimateInputOnly: cost.costEstimateInputOnly,
|
|
2862
|
+
costEstimateEstimatedInput: cost.costEstimateEstimatedInput,
|
|
2863
|
+
costEstimateMissingOutputTokens: cost.costEstimateMissingOutputTokens,
|
|
2864
|
+
costEstimateLongContext: cost.costEstimateLongContext,
|
|
2865
|
+
...(cost.unknownPricingReason ? { unknownPricingReason: cost.unknownPricingReason } : { unknownPricingReason: undefined })
|
|
2866
|
+
};
|
|
2867
|
+
}
|
|
2868
|
+
function hasCostTokenEvidence(record) {
|
|
2869
|
+
return [
|
|
2870
|
+
record.actualInputTokens,
|
|
2871
|
+
record.inputTokens,
|
|
2872
|
+
record.promptTokens,
|
|
2873
|
+
record.estimatedInputTokens,
|
|
2874
|
+
record.cachedInputTokens,
|
|
2875
|
+
record.uncachedInputTokens,
|
|
2876
|
+
record.actualOutputTokens,
|
|
2877
|
+
record.outputTokens,
|
|
2878
|
+
record.completionTokens,
|
|
2879
|
+
record.responseTokens,
|
|
2880
|
+
record.actual_input_tokens,
|
|
2881
|
+
record.input_tokens,
|
|
2882
|
+
record.prompt_tokens,
|
|
2883
|
+
record.estimated_input_tokens,
|
|
2884
|
+
record.cached_input_tokens,
|
|
2885
|
+
record.uncached_input_tokens,
|
|
2886
|
+
record.actual_output_tokens,
|
|
2887
|
+
record.output_tokens,
|
|
2888
|
+
record.completion_tokens,
|
|
2889
|
+
record.response_tokens
|
|
2890
|
+
].some((value) => optionalCostTokenNumber(value) !== undefined && optionalCostTokenNumber(value) > 0);
|
|
2891
|
+
}
|
|
2892
|
+
function firstCostTokenNumber(...values) {
|
|
2893
|
+
for (const value of values) {
|
|
2894
|
+
const number = optionalCostTokenNumber(value);
|
|
2895
|
+
if (number !== undefined && number > 0)
|
|
2896
|
+
return number;
|
|
2897
|
+
}
|
|
2898
|
+
return 0;
|
|
2899
|
+
}
|
|
2900
|
+
function optionalCostTokenNumber(...values) {
|
|
2901
|
+
for (const value of values) {
|
|
2902
|
+
if (value === undefined || value === null || value === '')
|
|
2903
|
+
continue;
|
|
2904
|
+
const number = Number(value);
|
|
2905
|
+
if (Number.isFinite(number) && number >= 0)
|
|
2906
|
+
return number;
|
|
2907
|
+
}
|
|
2908
|
+
return undefined;
|
|
2909
|
+
}
|
|
2910
|
+
function roundUsd(value) {
|
|
2911
|
+
return Math.round(value * 1_000_000_000) / 1_000_000_000;
|
|
2912
|
+
}
|
|
2913
|
+
async function commandEvidenceFromArtifactPaths(cwd, evidencePaths, outputDir = '') {
|
|
2914
|
+
let out = emptyCommandEvidence();
|
|
2915
|
+
for (const evidencePath of evidencePaths.slice(0, 60)) {
|
|
2916
|
+
if (!evidencePath.endsWith('.json'))
|
|
2917
|
+
continue;
|
|
2918
|
+
const displayPath = resolveRelativeArtifactPath(outputDir, evidencePath);
|
|
2919
|
+
const absolute = path.isAbsolute(displayPath) ? displayPath : path.resolve(cwd, displayPath);
|
|
2920
|
+
const roots = uniquePaths([cwd, outputDir].filter(Boolean).map((root) => path.resolve(root)));
|
|
2921
|
+
if (!roots.some((root) => isPathInside(root, absolute)))
|
|
2922
|
+
continue;
|
|
2923
|
+
out = mergeCommandEvidence(out, commandEvidenceFromRecords(recordValue(await readJsonFile(absolute))));
|
|
2924
|
+
}
|
|
2925
|
+
return out;
|
|
2926
|
+
}
|
|
2927
|
+
function commandEvidenceFromRecords(...records) {
|
|
2928
|
+
let out = emptyCommandEvidence();
|
|
2929
|
+
for (const record of records) {
|
|
2930
|
+
out = mergeCommandEvidence(out, {
|
|
2931
|
+
passed: [
|
|
2932
|
+
...commandRecordsFromKnownBucket(record.commandsPassed, 'passed'),
|
|
2933
|
+
...commandRecordsFromKnownBucket(record.passedCommands, 'passed')
|
|
2934
|
+
],
|
|
2935
|
+
failed: [
|
|
2936
|
+
...commandRecordsFromKnownBucket(record.commandsFailed, 'failed'),
|
|
2937
|
+
...commandRecordsFromKnownBucket(record.failedCommands, 'failed')
|
|
2938
|
+
]
|
|
2939
|
+
});
|
|
2940
|
+
for (const key of ['verification', 'commands', 'checks', 'testResults', 'results']) {
|
|
2941
|
+
const classified = classifyCommandRecords(recordArray(record[key]));
|
|
2942
|
+
out = mergeCommandEvidence(out, classified);
|
|
2943
|
+
}
|
|
2944
|
+
}
|
|
2945
|
+
return out;
|
|
2946
|
+
}
|
|
2947
|
+
function commandRecordsFromKnownBucket(value, fallbackStatus) {
|
|
2948
|
+
return recordArray(value)
|
|
2949
|
+
.map((record) => normalizedCommandRecord(record, fallbackStatus))
|
|
2950
|
+
.filter((record) => Boolean(record));
|
|
2951
|
+
}
|
|
2952
|
+
function classifyCommandRecords(records) {
|
|
2953
|
+
const out = emptyCommandEvidence();
|
|
2954
|
+
for (const record of records) {
|
|
2955
|
+
const classified = classifyCommandRecord(record);
|
|
2956
|
+
if (classified.status === 'passed')
|
|
2957
|
+
out.passed.push(classified.record);
|
|
2958
|
+
else if (classified.status === 'failed')
|
|
2959
|
+
out.failed.push(classified.record);
|
|
2960
|
+
}
|
|
2961
|
+
return normalizeCommandEvidence(out);
|
|
2962
|
+
}
|
|
2963
|
+
function classifyCommandRecord(record) {
|
|
2964
|
+
const statusText = normalized(record.status ?? record.result ?? record.outcome ?? record.state);
|
|
2965
|
+
const exitCode = hasOwnKey(record, 'exitCode') ? numberValue(record.exitCode) : undefined;
|
|
2966
|
+
const status = commandStatus(statusText, exitCode);
|
|
2967
|
+
return { status, record: normalizedCommandRecord(record, status || undefined) ?? record };
|
|
2968
|
+
}
|
|
2969
|
+
function commandStatus(statusText, exitCode) {
|
|
2970
|
+
if (['passed', 'pass', 'ok', 'success', 'succeeded', 'completed', 'green'].includes(statusText))
|
|
2971
|
+
return 'passed';
|
|
2972
|
+
if (['failed', 'fail', 'failure', 'error', 'errored', 'red', 'timeout', 'timed-out', 'blocked', 'nonzero'].includes(statusText))
|
|
2973
|
+
return 'failed';
|
|
2974
|
+
if (exitCode !== undefined)
|
|
2975
|
+
return exitCode === 0 ? 'passed' : 'failed';
|
|
2976
|
+
return '';
|
|
2977
|
+
}
|
|
2978
|
+
function normalizedCommandRecord(record, fallbackStatus) {
|
|
2979
|
+
const command = textValue(record.command ?? record.cmd ?? record.name ?? record.label, '');
|
|
2980
|
+
const cwd = textValue(record.cwd ?? record.dir, '');
|
|
2981
|
+
const status = textValue(record.status ?? record.result ?? record.outcome ?? record.state, fallbackStatus ?? '');
|
|
2982
|
+
if (!command && !cwd && !status)
|
|
2983
|
+
return undefined;
|
|
2984
|
+
return {
|
|
2985
|
+
...record,
|
|
2986
|
+
...(command && !record.command ? { command } : {}),
|
|
2987
|
+
...(cwd && !record.cwd ? { cwd } : {}),
|
|
2988
|
+
...(status && !record.status ? { status } : {})
|
|
2989
|
+
};
|
|
2990
|
+
}
|
|
2991
|
+
function mergeCommandEvidence(...entries) {
|
|
2992
|
+
return normalizeCommandEvidence({
|
|
2993
|
+
passed: entries.flatMap((entry) => entry.passed),
|
|
2994
|
+
failed: entries.flatMap((entry) => entry.failed)
|
|
2995
|
+
});
|
|
2996
|
+
}
|
|
2997
|
+
function normalizeCommandEvidence(evidence) {
|
|
2998
|
+
return {
|
|
2999
|
+
passed: uniqueCommandRecords(evidence.passed),
|
|
3000
|
+
failed: uniqueCommandRecords(evidence.failed)
|
|
3001
|
+
};
|
|
3002
|
+
}
|
|
3003
|
+
function uniqueCommandRecords(records) {
|
|
3004
|
+
const seen = new Set();
|
|
3005
|
+
const out = [];
|
|
3006
|
+
for (const record of records) {
|
|
3007
|
+
const key = [
|
|
3008
|
+
textValue(record.command ?? record.name, ''),
|
|
3009
|
+
textValue(record.cwd, ''),
|
|
3010
|
+
normalized(record.status ?? record.result ?? record.outcome)
|
|
3011
|
+
].join('\0');
|
|
3012
|
+
if (seen.has(key))
|
|
3013
|
+
continue;
|
|
3014
|
+
seen.add(key);
|
|
3015
|
+
out.push(record);
|
|
3016
|
+
}
|
|
3017
|
+
return out;
|
|
3018
|
+
}
|
|
3019
|
+
function emptyCommandEvidence() {
|
|
3020
|
+
return { passed: [], failed: [] };
|
|
3021
|
+
}
|
|
3022
|
+
function hasOwnKey(record, key) {
|
|
3023
|
+
return Object.prototype.hasOwnProperty.call(record, key);
|
|
3024
|
+
}
|
|
1162
3025
|
function isProcessLive(pid, entry) {
|
|
1163
3026
|
if (!pid)
|
|
1164
3027
|
return false;
|
|
@@ -1200,28 +3063,116 @@ function processCommandMatchesPidManifest(command, entry) {
|
|
|
1200
3063
|
return false;
|
|
1201
3064
|
return true;
|
|
1202
3065
|
}
|
|
1203
|
-
function
|
|
1204
|
-
return path.dirname(path.dirname(runDir));
|
|
1205
|
-
}
|
|
1206
|
-
async function readTaskDetails(options, jobId) {
|
|
3066
|
+
async function readTaskDetails(options, jobId, sourceRun = '') {
|
|
1207
3067
|
if (!jobId)
|
|
1208
3068
|
return { ok: false, jobId, files: [], commandsPassed: [], commandsFailed: [], evidenceArtifacts: [], error: 'missing job id' };
|
|
1209
|
-
const entry = await findCollectionBundle(options, jobId);
|
|
3069
|
+
const entry = await findCollectionBundle(options, jobId) ?? await findRawRunTaskBundle(options, jobId, sourceRun);
|
|
1210
3070
|
if (!entry)
|
|
1211
3071
|
return { ok: false, jobId, files: [], commandsPassed: [], commandsFailed: [], evidenceArtifacts: [], error: 'task not found in collection' };
|
|
1212
3072
|
const { bundle, outputDir } = entry;
|
|
1213
3073
|
const patchPath = textValue(bundle.patchPath, '');
|
|
1214
3074
|
const evidencePaths = stringArray(bundle.evidencePaths).slice(0, 40);
|
|
3075
|
+
const evidenceCommands = await commandEvidenceFromArtifactPaths(options.cwd, evidencePaths, outputDir);
|
|
3076
|
+
const commandEvidence = mergeCommandEvidence(commandEvidenceFromRecords(bundle), evidenceCommands);
|
|
1215
3077
|
return {
|
|
1216
3078
|
ok: true,
|
|
1217
3079
|
jobId,
|
|
1218
3080
|
...(patchPath ? { patchArtifact: artifactRecord(patchPath) } : {}),
|
|
1219
3081
|
files: patchPath ? await readPatchFiles(options, patchPath) : [],
|
|
1220
|
-
commandsPassed:
|
|
1221
|
-
commandsFailed:
|
|
3082
|
+
commandsPassed: commandEvidence.passed.slice(0, 20),
|
|
3083
|
+
commandsFailed: commandEvidence.failed.slice(0, 20),
|
|
1222
3084
|
evidenceArtifacts: evidencePaths.map((evidencePath) => artifactRecord(resolveRelativeArtifactPath(outputDir, evidencePath), evidencePath))
|
|
1223
3085
|
};
|
|
1224
3086
|
}
|
|
3087
|
+
async function findRawRunTaskBundle(options, jobId, sourceRun = '') {
|
|
3088
|
+
const sourceRunRoot = sourceRun ? safeCwdRelativeDirectory(options.cwd, sourceRun) : undefined;
|
|
3089
|
+
const hintedRoot = sourceRunRoot ?? rawRunSourceHint(options.cwd, jobId);
|
|
3090
|
+
const root = hintedRoot ?? path.join(options.cwd, 'agent-runs');
|
|
3091
|
+
const stat = await fs.stat(root).catch(() => undefined);
|
|
3092
|
+
if (!stat?.isDirectory())
|
|
3093
|
+
return undefined;
|
|
3094
|
+
const matches = await findRawRunJobDirs(root, jobId, 0);
|
|
3095
|
+
if (!matches.length)
|
|
3096
|
+
return undefined;
|
|
3097
|
+
const scoredMatches = await Promise.all(matches.map(async (match) => {
|
|
3098
|
+
const patchPath = await firstExistingRelativePath(options.cwd, rawRunPatchCandidates(match));
|
|
3099
|
+
const matchStat = await fs.stat(match).catch(() => undefined);
|
|
3100
|
+
return { match, patchPath, mtimeMs: matchStat?.mtimeMs ?? 0 };
|
|
3101
|
+
}));
|
|
3102
|
+
scoredMatches.sort((left, right) => {
|
|
3103
|
+
const patchScore = Number(Boolean(right.patchPath)) - Number(Boolean(left.patchPath));
|
|
3104
|
+
if (patchScore)
|
|
3105
|
+
return patchScore;
|
|
3106
|
+
const timeScore = right.mtimeMs - left.mtimeMs;
|
|
3107
|
+
if (timeScore)
|
|
3108
|
+
return timeScore;
|
|
3109
|
+
return right.match.localeCompare(left.match);
|
|
3110
|
+
});
|
|
3111
|
+
const { match, patchPath } = scoredMatches[0];
|
|
3112
|
+
const evidencePaths = await existingRelativePaths(options.cwd, [
|
|
3113
|
+
path.join(match, 'last-message.md'),
|
|
3114
|
+
path.join(match, 'codex-events.jsonl'),
|
|
3115
|
+
path.join(match, 'evidence', 'last-message.md'),
|
|
3116
|
+
path.join(match, 'evidence', 'handoff.md'),
|
|
3117
|
+
path.join(match, 'evidence', 'evidence.json'),
|
|
3118
|
+
path.join(match, 'evidence', 'resource-allocation.json'),
|
|
3119
|
+
...rawRunPatchCandidates(match)
|
|
3120
|
+
]);
|
|
3121
|
+
const evidenceCommands = await commandEvidenceFromArtifactPaths(options.cwd, evidencePaths);
|
|
3122
|
+
return {
|
|
3123
|
+
bundle: {
|
|
3124
|
+
jobId: path.basename(match),
|
|
3125
|
+
patchPath,
|
|
3126
|
+
changedPaths: await readPatchChangedPathList(options.cwd, patchPath),
|
|
3127
|
+
evidencePaths,
|
|
3128
|
+
commandsPassed: evidenceCommands.passed,
|
|
3129
|
+
commandsFailed: evidenceCommands.failed
|
|
3130
|
+
}
|
|
3131
|
+
};
|
|
3132
|
+
}
|
|
3133
|
+
function rawRunSourceHint(cwd, jobId) {
|
|
3134
|
+
const match = /(?:^|:)(agent-runs\/[^:]+)/.exec(jobId);
|
|
3135
|
+
if (!match)
|
|
3136
|
+
return undefined;
|
|
3137
|
+
const absolute = path.resolve(cwd, match[1]);
|
|
3138
|
+
return isPathInside(cwd, absolute) ? absolute : undefined;
|
|
3139
|
+
}
|
|
3140
|
+
function safeCwdRelativeDirectory(cwd, input) {
|
|
3141
|
+
const absolute = path.resolve(cwd, input);
|
|
3142
|
+
if (!isPathInside(cwd, absolute))
|
|
3143
|
+
return undefined;
|
|
3144
|
+
return absolute;
|
|
3145
|
+
}
|
|
3146
|
+
async function findRawRunJobDirs(root, jobId, depth) {
|
|
3147
|
+
if (depth > 5)
|
|
3148
|
+
return [];
|
|
3149
|
+
const entries = await fs.readdir(root, { withFileTypes: true }).catch(() => []);
|
|
3150
|
+
const matches = [];
|
|
3151
|
+
for (const entry of entries) {
|
|
3152
|
+
if (!entry.isDirectory() || entry.name === 'streams' || entry.name === 'artifact-store' || entry.name.startsWith('.'))
|
|
3153
|
+
continue;
|
|
3154
|
+
const absolute = path.join(root, entry.name);
|
|
3155
|
+
if (rawRunJobIdMatches(jobId, entry.name) && await rawRunJobHasArtifacts(absolute))
|
|
3156
|
+
matches.push(absolute);
|
|
3157
|
+
matches.push(...await findRawRunJobDirs(absolute, jobId, depth + 1));
|
|
3158
|
+
}
|
|
3159
|
+
return matches;
|
|
3160
|
+
}
|
|
3161
|
+
async function rawRunJobHasArtifacts(jobDir) {
|
|
3162
|
+
for (const file of [
|
|
3163
|
+
path.join(jobDir, 'last-message.md'),
|
|
3164
|
+
path.join(jobDir, 'codex-events.jsonl'),
|
|
3165
|
+
...rawRunPatchCandidates(jobDir)
|
|
3166
|
+
]) {
|
|
3167
|
+
const stat = await fs.stat(file).catch(() => undefined);
|
|
3168
|
+
if (stat?.isFile())
|
|
3169
|
+
return true;
|
|
3170
|
+
}
|
|
3171
|
+
return false;
|
|
3172
|
+
}
|
|
3173
|
+
function rawRunJobIdMatches(requestedId, jobId) {
|
|
3174
|
+
return requestedId === jobId || requestedId.endsWith(`:${jobId}`) || requestedId.endsWith(`-${jobId}`);
|
|
3175
|
+
}
|
|
1225
3176
|
async function findCollectionBundle(options, jobId) {
|
|
1226
3177
|
const collectionFile = await resolveArtifactFile(options.cwd, options.collection, ['collection.json', 'coordinator-query.json']);
|
|
1227
3178
|
if (!collectionFile)
|
|
@@ -1289,6 +3240,7 @@ async function writeHumanActionAnswer(options, body) {
|
|
|
1289
3240
|
return { ok: true, code, answerPath };
|
|
1290
3241
|
}
|
|
1291
3242
|
function notifyDashboardStreams() {
|
|
3243
|
+
invalidateDashboardSnapshotCache();
|
|
1292
3244
|
for (const listener of dashboardStreamListeners)
|
|
1293
3245
|
listener();
|
|
1294
3246
|
}
|
|
@@ -1357,7 +3309,7 @@ async function readPatchFiles(options, patchPath) {
|
|
|
1357
3309
|
return parseUnifiedPatchFiles(patch).slice(0, 40);
|
|
1358
3310
|
}
|
|
1359
3311
|
function parseUnifiedPatchFiles(patch) {
|
|
1360
|
-
const sections = patch
|
|
3312
|
+
const sections = splitUnifiedPatchSections(patch);
|
|
1361
3313
|
return sections.flatMap((section) => {
|
|
1362
3314
|
const lines = section.split('\n');
|
|
1363
3315
|
const pathLine = lines.find((line) => line.startsWith('+++ ')) ?? lines.find((line) => line.startsWith('diff --git '));
|
|
@@ -1379,6 +3331,27 @@ function parseUnifiedPatchFiles(patch) {
|
|
|
1379
3331
|
}];
|
|
1380
3332
|
});
|
|
1381
3333
|
}
|
|
3334
|
+
function splitUnifiedPatchSections(patch) {
|
|
3335
|
+
if (/^diff --git /m.test(patch)) {
|
|
3336
|
+
return patch.split(/\n(?=diff --git )/g).filter((section) => section.trim().length > 0);
|
|
3337
|
+
}
|
|
3338
|
+
const lines = patch.split('\n');
|
|
3339
|
+
const sections = [];
|
|
3340
|
+
let current = [];
|
|
3341
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
3342
|
+
const line = lines[index];
|
|
3343
|
+
const next = lines[index + 1] ?? '';
|
|
3344
|
+
const startsPlainFile = line.startsWith('--- ') && next.startsWith('+++ ');
|
|
3345
|
+
if (startsPlainFile && current.length) {
|
|
3346
|
+
sections.push(current.join('\n'));
|
|
3347
|
+
current = [];
|
|
3348
|
+
}
|
|
3349
|
+
current.push(line);
|
|
3350
|
+
}
|
|
3351
|
+
if (current.some((line) => line.trim().length > 0))
|
|
3352
|
+
sections.push(current.join('\n'));
|
|
3353
|
+
return sections;
|
|
3354
|
+
}
|
|
1382
3355
|
function parseUnifiedPatchHunks(section) {
|
|
1383
3356
|
const hunks = [];
|
|
1384
3357
|
let current = { header: 'File header', lines: [] };
|
|
@@ -1420,9 +3393,20 @@ function parseUnifiedPatchHunks(section) {
|
|
|
1420
3393
|
function patchFilePath(line) {
|
|
1421
3394
|
const plus = /^\+\+\+\s+(?:b\/)?(.+)$/.exec(line);
|
|
1422
3395
|
if (plus && plus[1] !== '/dev/null')
|
|
1423
|
-
return plus[1];
|
|
3396
|
+
return normalizePatchDisplayPath(plus[1]);
|
|
1424
3397
|
const diff = /^diff --git\s+a\/(.+?)\s+b\/(.+)$/.exec(line);
|
|
1425
|
-
return diff?.[2] ?? '';
|
|
3398
|
+
return normalizePatchDisplayPath(diff?.[2] ?? '');
|
|
3399
|
+
}
|
|
3400
|
+
function normalizePatchDisplayPath(value) {
|
|
3401
|
+
let clean = value.trim().split(/\t/g)[0]?.trim() ?? '';
|
|
3402
|
+
clean = clean.replace(/^(?:a|b)\//, '');
|
|
3403
|
+
const packageIndex = clean.indexOf('/packages/');
|
|
3404
|
+
if (packageIndex >= 0)
|
|
3405
|
+
clean = clean.slice(packageIndex + 1);
|
|
3406
|
+
const repoPackageIndex = clean.indexOf('packages/');
|
|
3407
|
+
if (repoPackageIndex > 0)
|
|
3408
|
+
clean = clean.slice(repoPackageIndex);
|
|
3409
|
+
return clean === '/dev/null' ? '' : clean;
|
|
1426
3410
|
}
|
|
1427
3411
|
function artifactRecord(pathValue, label = pathValue) {
|
|
1428
3412
|
return {
|