@shapeshift-labs/frontier-swarm-codex 0.5.31 → 0.5.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -6
- package/dist/cli.js +27 -0
- package/dist/cli.js.map +1 -1
- package/dist/index.d.ts +168 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +707 -53
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -10,7 +10,7 @@ import { spawn } from 'node:child_process';
|
|
|
10
10
|
import fs from 'node:fs/promises';
|
|
11
11
|
import os from 'node:os';
|
|
12
12
|
import path from 'node:path';
|
|
13
|
-
import { FRONTIER_SWARM_DEFAULT_MODEL, FRONTIER_SWARM_DEFAULT_REASONING_EFFORT, checkSwarmOwnership, completeSwarmJob, FRONTIER_SWARM_MERGE_BUNDLE_KIND, FRONTIER_SWARM_MERGE_BUNDLE_VERSION, createSwarmMergeBundle, createSwarmMergeIndex, createSwarmQueueOverlay, matchesGlob, createSwarmEventStream, createSwarmLeases, createSwarmPlan, createSwarmProof, createSwarmRun, createSwarmSchedule, recordSwarmEvent, routeSwarmEventToMailboxes } from '@shapeshift-labs/frontier-swarm';
|
|
13
|
+
import { FRONTIER_SWARM_DEFAULT_MODEL, FRONTIER_SWARM_DEFAULT_REASONING_EFFORT, checkSwarmOwnership, completeSwarmJob, createSwarmCoordinatorDashboard, createSwarmAdaptiveLoadPlan, createSwarmEvidenceIndex, createSwarmMergeAdmission, FRONTIER_SWARM_MERGE_BUNDLE_KIND, FRONTIER_SWARM_MERGE_BUNDLE_VERSION, createSwarmMergeBundle, createSwarmMergeIndex, createSwarmQueueOverlay, matchesGlob, createSwarmEventStream, createSwarmLeases, createSwarmPlan, createSwarmProof, createSwarmRun, createSwarmSchedule, createSwarmScheduleInputFromAdaptiveLoadPlan, recordSwarmEvent, routeSwarmEventToMailboxes } from '@shapeshift-labs/frontier-swarm';
|
|
14
14
|
export const FRONTIER_SWARM_CODEX_DEFAULT_MODEL = FRONTIER_SWARM_DEFAULT_MODEL;
|
|
15
15
|
export const FRONTIER_SWARM_CODEX_DEFAULT_REASONING_EFFORT = FRONTIER_SWARM_DEFAULT_REASONING_EFFORT;
|
|
16
16
|
export const FRONTIER_SWARM_CODEX_WORKSPACE_MANIFEST_KIND = 'frontier.swarm-codex.workspace-manifest';
|
|
@@ -27,6 +27,12 @@ export const FRONTIER_SWARM_CODEX_PATCH_SCORE_KIND = 'frontier.swarm-codex.patch
|
|
|
27
27
|
export const FRONTIER_SWARM_CODEX_PATCH_SCORE_VERSION = 1;
|
|
28
28
|
export const FRONTIER_SWARM_CODEX_SEMANTIC_IMPORT_KIND = 'frontier.swarm-codex.semantic-imports';
|
|
29
29
|
export const FRONTIER_SWARM_CODEX_SEMANTIC_IMPORT_VERSION = 1;
|
|
30
|
+
export const FRONTIER_SWARM_CODEX_JOB_EVIDENCE_KIND = 'frontier.swarm-codex.job-evidence';
|
|
31
|
+
export const FRONTIER_SWARM_CODEX_JOB_EVIDENCE_VERSION = 1;
|
|
32
|
+
export const FRONTIER_SWARM_CODEX_PATCH_INTENT_KIND = 'frontier.swarm-codex.patch-intent';
|
|
33
|
+
export const FRONTIER_SWARM_CODEX_PATCH_INTENT_VERSION = 1;
|
|
34
|
+
export const FRONTIER_SWARM_CODEX_COMPACT_DASHBOARD_KIND = 'frontier.swarm-codex.compact-dashboard';
|
|
35
|
+
export const FRONTIER_SWARM_CODEX_COMPACT_DASHBOARD_VERSION = 1;
|
|
30
36
|
const DEFAULT_WORKSPACE_INCLUDES = ['AGENTS.md', 'package.json', 'package-lock.json', 'pnpm-lock.yaml', 'yarn.lock', 'config'];
|
|
31
37
|
const DEFAULT_WORKSPACE_EXCLUDES = [
|
|
32
38
|
'.git',
|
|
@@ -133,7 +139,12 @@ export async function runCodexSwarm(plan, options) {
|
|
|
133
139
|
run = recordSwarmEvent(run, startedEvent);
|
|
134
140
|
await appendFileSwarmEvent(eventStream, startedEvent);
|
|
135
141
|
const runOptions = { ...options, eventStream, pidManifestPath };
|
|
136
|
-
const results = await runScheduledJobPool(plan,
|
|
142
|
+
const results = await runScheduledJobPool(plan, {
|
|
143
|
+
concurrency: Math.max(1, options.maxConcurrency ?? 1),
|
|
144
|
+
adaptive: options.adaptiveConcurrency,
|
|
145
|
+
outDir,
|
|
146
|
+
eventStream
|
|
147
|
+
}, (job, lease) => runCodexJob(job, runOptions, outDir, lease));
|
|
137
148
|
for (const result of results) {
|
|
138
149
|
const job = plan.jobs.find((entry) => entry.id === result.jobId);
|
|
139
150
|
if (job) {
|
|
@@ -229,8 +240,12 @@ export async function runCodexJob(job, options, outDir, lease) {
|
|
|
229
240
|
paths,
|
|
230
241
|
resourceAllocation,
|
|
231
242
|
env: resourceAllocation.env,
|
|
232
|
-
timeoutMs: job.compute.timeoutMs ?? options.jobTimeoutMs ?? 7200000
|
|
243
|
+
timeoutMs: job.compute.timeoutMs ?? options.jobTimeoutMs ?? 7200000,
|
|
244
|
+
compactLogs: normalizeCompactLogOptions(options.compactLogs)
|
|
233
245
|
});
|
|
246
|
+
const logSummary = execution.logSummary ?? createEmptyCodexLogSummary(paths);
|
|
247
|
+
if (!execution.logSummary)
|
|
248
|
+
await fs.writeFile(paths.logSummaryPath, JSON.stringify(logSummary, null, 2) + '\n');
|
|
234
249
|
const collected = execution.changedPaths
|
|
235
250
|
? filterWorkspaceChangedPaths(execution.changedPaths, workspacePlan)
|
|
236
251
|
: options.collectGitStatus === false
|
|
@@ -259,13 +274,17 @@ export async function runCodexJob(job, options, outDir, lease) {
|
|
|
259
274
|
options: options.semanticImport
|
|
260
275
|
});
|
|
261
276
|
const handoffArtifacts = await discoverCodexHandoffArtifacts({ root: paths.jobDir });
|
|
277
|
+
const evidenceSummaryPath = path.join(paths.evidenceDir, 'evidence.json');
|
|
262
278
|
const evidencePaths = uniqueStrings([
|
|
263
279
|
paths.evidenceDir,
|
|
280
|
+
evidenceSummaryPath,
|
|
264
281
|
paths.resourceAllocationPath,
|
|
265
282
|
paths.workspaceProofPath,
|
|
266
283
|
paths.mergeBundlePath,
|
|
267
284
|
...(patchPath ? [patchPath] : []),
|
|
268
285
|
...(semanticImport ? [semanticImport.path] : []),
|
|
286
|
+
paths.patchIntentPath,
|
|
287
|
+
paths.logSummaryPath,
|
|
269
288
|
...handoffArtifacts.map((artifact) => artifact.path)
|
|
270
289
|
]);
|
|
271
290
|
const result = {
|
|
@@ -288,6 +307,7 @@ export async function runCodexJob(job, options, outDir, lease) {
|
|
|
288
307
|
metadata: {
|
|
289
308
|
...(lease ? { leaseId: lease.id, leaseToken: lease.token, fencingToken: lease.fencingToken } : {}),
|
|
290
309
|
resourceAllocation,
|
|
310
|
+
logSummary,
|
|
291
311
|
...(semanticImport ? { semanticImport: semanticImport.sidecar.summary } : {}),
|
|
292
312
|
codexHandoffArtifacts: handoffArtifacts
|
|
293
313
|
}
|
|
@@ -299,8 +319,11 @@ export async function runCodexJob(job, options, outDir, lease) {
|
|
|
299
319
|
...(patchPath ? { patchPath } : {}),
|
|
300
320
|
evidencePaths: uniqueStrings([
|
|
301
321
|
paths.evidenceDir,
|
|
322
|
+
evidenceSummaryPath,
|
|
302
323
|
paths.resourceAllocationPath,
|
|
303
324
|
paths.workspaceProofPath,
|
|
325
|
+
paths.patchIntentPath,
|
|
326
|
+
paths.logSummaryPath,
|
|
304
327
|
...(semanticImport ? [semanticImport.path] : []),
|
|
305
328
|
...handoffArtifacts.map((artifact) => artifact.path)
|
|
306
329
|
]),
|
|
@@ -309,13 +332,174 @@ export async function runCodexJob(job, options, outDir, lease) {
|
|
|
309
332
|
...(semanticImport ? { metadata: { semanticImport: semanticImport.sidecar.summary } } : {})
|
|
310
333
|
});
|
|
311
334
|
await fs.writeFile(paths.mergeBundlePath, JSON.stringify(mergeBundle, null, 2) + '\n');
|
|
335
|
+
await writeCodexPatchIntent({
|
|
336
|
+
file: paths.patchIntentPath,
|
|
337
|
+
job,
|
|
338
|
+
result,
|
|
339
|
+
mergeBundle,
|
|
340
|
+
patchPath,
|
|
341
|
+
semanticImport: semanticImport?.sidecar,
|
|
342
|
+
semanticImportExpected: options.semanticImportExpected ?? semanticImportEnabled(options.semanticImport),
|
|
343
|
+
evidencePaths
|
|
344
|
+
});
|
|
345
|
+
await writeCodexJobEvidenceSummary({
|
|
346
|
+
file: evidenceSummaryPath,
|
|
347
|
+
job,
|
|
348
|
+
result,
|
|
349
|
+
mergeBundle,
|
|
350
|
+
mergeBundlePath: paths.mergeBundlePath,
|
|
351
|
+
patchPath,
|
|
352
|
+
patchIntentPath: paths.patchIntentPath,
|
|
353
|
+
logSummary,
|
|
354
|
+
semanticImportPath: semanticImport?.path,
|
|
355
|
+
semanticImport: semanticImport?.sidecar,
|
|
356
|
+
handoffArtifacts
|
|
357
|
+
});
|
|
312
358
|
return result;
|
|
313
359
|
}
|
|
360
|
+
async function writeCodexJobEvidenceSummary(input) {
|
|
361
|
+
const patchHunks = input.patchPath ? await readPatchHunks(input.patchPath) : [];
|
|
362
|
+
const sourceCitations = createCodexEvidenceSourceCitations(input.mergeBundle, input.semanticImport);
|
|
363
|
+
const evidence = {
|
|
364
|
+
kind: FRONTIER_SWARM_CODEX_JOB_EVIDENCE_KIND,
|
|
365
|
+
version: FRONTIER_SWARM_CODEX_JOB_EVIDENCE_VERSION,
|
|
366
|
+
generatedAt: Date.now(),
|
|
367
|
+
jobId: input.job.id,
|
|
368
|
+
taskId: input.job.taskId,
|
|
369
|
+
lane: input.job.lane,
|
|
370
|
+
status: input.result.status ?? 'unknown',
|
|
371
|
+
mergeReadiness: input.mergeBundle.mergeReadiness,
|
|
372
|
+
disposition: input.mergeBundle.disposition,
|
|
373
|
+
riskLevel: input.mergeBundle.riskLevel,
|
|
374
|
+
changedPaths: [...input.mergeBundle.changedPaths],
|
|
375
|
+
changedRegions: [...input.mergeBundle.changedRegions],
|
|
376
|
+
ownershipViolations: [...input.mergeBundle.ownershipViolations],
|
|
377
|
+
...(input.patchPath ? { patchPath: input.patchPath } : {}),
|
|
378
|
+
mergeBundlePath: input.mergeBundlePath,
|
|
379
|
+
...(input.patchIntentPath ? { patchIntentPath: input.patchIntentPath } : {}),
|
|
380
|
+
...(input.semanticImportPath ? { semanticImportPath: input.semanticImportPath } : {}),
|
|
381
|
+
evidencePaths: uniqueStrings(input.mergeBundle.evidencePaths),
|
|
382
|
+
handoffArtifacts: input.handoffArtifacts.map((artifact) => ({ ...artifact })),
|
|
383
|
+
commands: {
|
|
384
|
+
passed: input.mergeBundle.commandsPassed.map((command) => ({
|
|
385
|
+
name: command.name,
|
|
386
|
+
command: [...command.command],
|
|
387
|
+
...(command.status !== undefined ? { status: command.status } : {})
|
|
388
|
+
})),
|
|
389
|
+
failed: input.mergeBundle.commandsFailed.map((command) => ({
|
|
390
|
+
name: command.name,
|
|
391
|
+
command: [...command.command],
|
|
392
|
+
...(command.status !== undefined ? { status: command.status } : {})
|
|
393
|
+
}))
|
|
394
|
+
},
|
|
395
|
+
patchHunks,
|
|
396
|
+
readyToPortHunkCount: input.mergeBundle.disposition === 'needs-port' || input.mergeBundle.disposition === 'auto-mergeable' ? patchHunks.length : 0,
|
|
397
|
+
...(input.semanticImport ? { semanticImport: input.semanticImport.summary } : {}),
|
|
398
|
+
sourceCitations,
|
|
399
|
+
metadata: {
|
|
400
|
+
autoMergeable: input.mergeBundle.autoMergeable,
|
|
401
|
+
staleAgainstHead: input.mergeBundle.staleAgainstHead,
|
|
402
|
+
...(input.logSummary ? { logSummary: input.logSummary } : {}),
|
|
403
|
+
reasons: input.mergeBundle.reasons
|
|
404
|
+
}
|
|
405
|
+
};
|
|
406
|
+
await fs.writeFile(input.file, JSON.stringify(evidence, null, 2) + '\n');
|
|
407
|
+
}
|
|
408
|
+
async function writeCodexPatchIntent(input) {
|
|
409
|
+
const patchHunks = input.patchPath ? await readPatchHunks(input.patchPath) : [];
|
|
410
|
+
const semanticImportQuality = summarizeCodexSemanticImportQuality(input.semanticImport?.summary, input.semanticImportExpected);
|
|
411
|
+
const warnings = uniqueStrings([
|
|
412
|
+
...semanticImportQuality.warnings,
|
|
413
|
+
...(input.mergeBundle.staleAgainstHead ? ['stale against coordinator head'] : []),
|
|
414
|
+
...(input.mergeBundle.ownershipViolations.length ? ['ownership violations present'] : []),
|
|
415
|
+
...(input.mergeBundle.commandsFailed.length ? ['verification commands failed'] : []),
|
|
416
|
+
...(input.mergeBundle.disposition === 'discovery-only' ? ['discovery-only output'] : [])
|
|
417
|
+
]);
|
|
418
|
+
const intent = {
|
|
419
|
+
kind: FRONTIER_SWARM_CODEX_PATCH_INTENT_KIND,
|
|
420
|
+
version: FRONTIER_SWARM_CODEX_PATCH_INTENT_VERSION,
|
|
421
|
+
generatedAt: Date.now(),
|
|
422
|
+
jobId: input.job.id,
|
|
423
|
+
taskId: input.job.taskId,
|
|
424
|
+
lane: input.job.lane,
|
|
425
|
+
changedPaths: [...input.mergeBundle.changedPaths],
|
|
426
|
+
changedRegions: [...input.mergeBundle.changedRegions],
|
|
427
|
+
intent: input.mergeBundle.changedPaths.length
|
|
428
|
+
? `Patch ${input.mergeBundle.changedPaths.slice(0, 5).join(', ')}`
|
|
429
|
+
: 'No source patch produced',
|
|
430
|
+
why: input.result.lastMessage ? firstNonEmptyLine(input.result.lastMessage) ?? input.job.task.objective : input.job.task.objective,
|
|
431
|
+
riskLevel: input.mergeBundle.riskLevel,
|
|
432
|
+
mergeReadiness: input.mergeBundle.mergeReadiness,
|
|
433
|
+
disposition: input.mergeBundle.disposition,
|
|
434
|
+
safeToPortManually: input.mergeBundle.commandsFailed.length === 0
|
|
435
|
+
&& input.mergeBundle.ownershipViolations.length === 0
|
|
436
|
+
&& !input.mergeBundle.staleAgainstHead
|
|
437
|
+
&& input.mergeBundle.disposition !== 'rejected'
|
|
438
|
+
&& input.mergeBundle.disposition !== 'blocked',
|
|
439
|
+
verification: input.mergeBundle.commandsPassed.concat(input.mergeBundle.commandsFailed).map((command) => ({
|
|
440
|
+
name: command.name,
|
|
441
|
+
command: [...command.command],
|
|
442
|
+
...(command.status !== undefined ? { status: command.status } : {}),
|
|
443
|
+
required: command.required
|
|
444
|
+
})),
|
|
445
|
+
evidencePaths: uniqueStrings(input.evidencePaths),
|
|
446
|
+
semanticImportQuality,
|
|
447
|
+
patchHunks,
|
|
448
|
+
warnings
|
|
449
|
+
};
|
|
450
|
+
await fs.writeFile(input.file, JSON.stringify(intent, null, 2) + '\n');
|
|
451
|
+
}
|
|
452
|
+
async function readPatchHunks(file) {
|
|
453
|
+
const text = await fs.readFile(file, 'utf8').catch(() => '');
|
|
454
|
+
if (!text.trim())
|
|
455
|
+
return [];
|
|
456
|
+
const hunks = [];
|
|
457
|
+
let currentFile;
|
|
458
|
+
for (const line of text.split(/\r?\n/)) {
|
|
459
|
+
if (line.startsWith('+++ ')) {
|
|
460
|
+
currentFile = line.slice(4).replace(/^b\//, '').trim();
|
|
461
|
+
continue;
|
|
462
|
+
}
|
|
463
|
+
if (!line.startsWith('@@'))
|
|
464
|
+
continue;
|
|
465
|
+
const match = /^@@\s+-(\d+)(?:,(\d+))?\s+\+(\d+)(?:,(\d+))?/.exec(line);
|
|
466
|
+
hunks.push({
|
|
467
|
+
...(currentFile ? { file: currentFile } : {}),
|
|
468
|
+
header: line,
|
|
469
|
+
...(match ? {
|
|
470
|
+
oldStart: Number(match[1]),
|
|
471
|
+
oldLines: Number(match[2] ?? '1'),
|
|
472
|
+
newStart: Number(match[3]),
|
|
473
|
+
newLines: Number(match[4] ?? '1')
|
|
474
|
+
} : {})
|
|
475
|
+
});
|
|
476
|
+
}
|
|
477
|
+
return hunks;
|
|
478
|
+
}
|
|
479
|
+
function createCodexEvidenceSourceCitations(bundle, semanticImport) {
|
|
480
|
+
const citations = [
|
|
481
|
+
...bundle.changedPaths.map((file) => ({ path: file, kind: 'changed-source' })),
|
|
482
|
+
...(semanticImport?.records ?? []).map((record) => ({
|
|
483
|
+
path: record.path,
|
|
484
|
+
kind: 'semantic-import',
|
|
485
|
+
...(record.language ? { language: record.language } : {}),
|
|
486
|
+
...(record.universalAstHash ? { hash: record.universalAstHash } : {})
|
|
487
|
+
}))
|
|
488
|
+
];
|
|
489
|
+
const seen = new Set();
|
|
490
|
+
return citations.filter((citation) => {
|
|
491
|
+
const key = `${citation.kind}:${citation.path}:${citation.language ?? ''}:${citation.hash ?? ''}`;
|
|
492
|
+
if (seen.has(key))
|
|
493
|
+
return false;
|
|
494
|
+
seen.add(key);
|
|
495
|
+
return true;
|
|
496
|
+
});
|
|
497
|
+
}
|
|
314
498
|
async function createCodexSemanticImportSidecar(input) {
|
|
315
499
|
const options = normalizeSemanticImportOptions(input.options);
|
|
316
500
|
if (!options)
|
|
317
501
|
return undefined;
|
|
318
|
-
const selection = selectSemanticImportPaths(input.changedPaths, options);
|
|
502
|
+
const selection = selectSemanticImportPaths(semanticImportCandidatePaths(input.job, input.changedPaths), options);
|
|
319
503
|
const selected = selection.selected;
|
|
320
504
|
const records = [];
|
|
321
505
|
const importPath = path.join(input.evidenceDir, 'semantic-imports.json');
|
|
@@ -648,6 +832,19 @@ export function renderCodexPrompt(job, input) {
|
|
|
648
832
|
export async function spawnCodexExecutor(input) {
|
|
649
833
|
await fs.writeFile(input.paths.eventsPath, '');
|
|
650
834
|
await fs.writeFile(input.paths.stderrPath, '');
|
|
835
|
+
const logOptions = normalizeCompactLogOptions(input.compactLogs);
|
|
836
|
+
const eventLimit = logOptions.enabled === false ? Number.POSITIVE_INFINITY : logOptions.maxEventBytes ?? 1_000_000;
|
|
837
|
+
const stderrLimit = logOptions.enabled === false ? Number.POSITIVE_INFINITY : logOptions.maxStderrBytes ?? 256_000;
|
|
838
|
+
const logSummary = {
|
|
839
|
+
eventsPath: input.paths.eventsPath,
|
|
840
|
+
stderrPath: input.paths.stderrPath,
|
|
841
|
+
eventBytes: 0,
|
|
842
|
+
stderrBytes: 0,
|
|
843
|
+
eventBytesWritten: 0,
|
|
844
|
+
stderrBytesWritten: 0,
|
|
845
|
+
eventBytesTruncated: 0,
|
|
846
|
+
stderrBytesTruncated: 0
|
|
847
|
+
};
|
|
651
848
|
return new Promise((resolve) => {
|
|
652
849
|
const child = spawn(input.codexPath, input.args, {
|
|
653
850
|
cwd: input.cwd,
|
|
@@ -664,23 +861,52 @@ export async function spawnCodexExecutor(input) {
|
|
|
664
861
|
}).catch(() => { });
|
|
665
862
|
}
|
|
666
863
|
const timer = setTimeout(() => child.kill('SIGTERM'), input.timeoutMs);
|
|
667
|
-
child.stdout.on('data', (chunk) =>
|
|
668
|
-
child.stderr.on('data', (chunk) =>
|
|
864
|
+
child.stdout.on('data', (chunk) => appendLimitedLogChunk(input.paths.eventsPath, chunk, eventLimit, logSummary, 'event').catch(() => { }));
|
|
865
|
+
child.stderr.on('data', (chunk) => appendLimitedLogChunk(input.paths.stderrPath, chunk, stderrLimit, logSummary, 'stderr').catch(() => { }));
|
|
669
866
|
child.stdin.end(input.prompt);
|
|
670
867
|
child.on('close', async (code, signal) => {
|
|
671
868
|
clearTimeout(timer);
|
|
869
|
+
await fs.writeFile(input.paths.logSummaryPath, JSON.stringify(logSummary, null, 2) + '\n').catch(() => { });
|
|
672
870
|
resolve({
|
|
673
871
|
exitCode: code ?? 1,
|
|
674
872
|
...(signal ? { signal } : {}),
|
|
675
|
-
lastMessage: await readOptionalText(input.paths.lastMessagePath)
|
|
873
|
+
lastMessage: await readOptionalText(input.paths.lastMessagePath),
|
|
874
|
+
logSummary
|
|
676
875
|
});
|
|
677
876
|
});
|
|
678
877
|
child.on('error', (error) => {
|
|
679
878
|
clearTimeout(timer);
|
|
680
|
-
|
|
879
|
+
fs.writeFile(input.paths.logSummaryPath, JSON.stringify(logSummary, null, 2) + '\n').catch(() => { });
|
|
880
|
+
resolve({ exitCode: 1, logSummary, error });
|
|
681
881
|
});
|
|
682
882
|
});
|
|
683
883
|
}
|
|
884
|
+
async function appendLimitedLogChunk(file, chunk, limit, summary, kind) {
|
|
885
|
+
const bytes = chunk.byteLength;
|
|
886
|
+
if (kind === 'event')
|
|
887
|
+
summary.eventBytes += bytes;
|
|
888
|
+
else
|
|
889
|
+
summary.stderrBytes += bytes;
|
|
890
|
+
const written = kind === 'event' ? summary.eventBytesWritten : summary.stderrBytesWritten;
|
|
891
|
+
const available = Math.max(0, limit - written);
|
|
892
|
+
if (available <= 0) {
|
|
893
|
+
if (kind === 'event')
|
|
894
|
+
summary.eventBytesTruncated += bytes;
|
|
895
|
+
else
|
|
896
|
+
summary.stderrBytesTruncated += bytes;
|
|
897
|
+
return;
|
|
898
|
+
}
|
|
899
|
+
const slice = bytes > available ? chunk.subarray(0, available) : chunk;
|
|
900
|
+
await fs.appendFile(file, slice);
|
|
901
|
+
if (kind === 'event') {
|
|
902
|
+
summary.eventBytesWritten += slice.byteLength;
|
|
903
|
+
summary.eventBytesTruncated += bytes - slice.byteLength;
|
|
904
|
+
}
|
|
905
|
+
else {
|
|
906
|
+
summary.stderrBytesWritten += slice.byteLength;
|
|
907
|
+
summary.stderrBytesTruncated += bytes - slice.byteLength;
|
|
908
|
+
}
|
|
909
|
+
}
|
|
684
910
|
async function createJobPaths(outDir, job, options) {
|
|
685
911
|
const jobDir = path.join(outDir, job.id);
|
|
686
912
|
const paths = {
|
|
@@ -694,6 +920,8 @@ async function createJobPaths(outDir, job, options) {
|
|
|
694
920
|
workspaceProofPath: path.join(jobDir, 'evidence', 'workspace-proof.json'),
|
|
695
921
|
patchPath: path.join(jobDir, 'evidence', 'changes.patch'),
|
|
696
922
|
mergeBundlePath: path.join(jobDir, 'evidence', 'merge.json'),
|
|
923
|
+
patchIntentPath: path.join(jobDir, 'evidence', 'patch-intent.json'),
|
|
924
|
+
logSummaryPath: path.join(jobDir, 'evidence', 'log-summary.json'),
|
|
697
925
|
pidManifestPath: path.resolve(options.cwd ?? process.cwd(), options.pidManifestPath ?? path.join(outDir, 'pids.json'))
|
|
698
926
|
};
|
|
699
927
|
await fs.mkdir(paths.evidenceDir, { recursive: true });
|
|
@@ -895,38 +1123,19 @@ export async function appendFileSwarmEvent(stream, event) {
|
|
|
895
1123
|
}));
|
|
896
1124
|
}
|
|
897
1125
|
export async function writeSwarmCoordinatorSnapshot(file, input) {
|
|
898
|
-
const
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
}
|
|
911
|
-
const mergeReadiness = input.run.results.reduce((acc, result) => {
|
|
912
|
-
acc[result.mergeReadiness] = (acc[result.mergeReadiness] ?? 0) + 1;
|
|
913
|
-
return acc;
|
|
914
|
-
}, {});
|
|
915
|
-
const dashboard = {
|
|
916
|
-
kind: 'frontier.swarm-codex.coordinator-dashboard',
|
|
917
|
-
version: 1,
|
|
918
|
-
generatedAt: new Date().toISOString(),
|
|
919
|
-
ok: input.ok,
|
|
920
|
-
outDir: input.outDir,
|
|
921
|
-
runId: input.run.id,
|
|
922
|
-
planId: input.plan.id,
|
|
923
|
-
summary: input.run.summary,
|
|
924
|
-
byLane,
|
|
925
|
-
mergeReadiness,
|
|
926
|
-
eventStream: input.eventStream ?? null,
|
|
927
|
-
pidManifestPath: input.pidManifestPath ?? null,
|
|
928
|
-
proof: input.proof
|
|
929
|
-
};
|
|
1126
|
+
const processes = input.pidManifestPath ? await readCodexPidProcesses(input.pidManifestPath).catch(() => []) : [];
|
|
1127
|
+
const dashboard = createSwarmCoordinatorDashboard({
|
|
1128
|
+
plan: input.plan,
|
|
1129
|
+
run: input.run,
|
|
1130
|
+
processes,
|
|
1131
|
+
metadata: {
|
|
1132
|
+
ok: input.ok,
|
|
1133
|
+
outDir: input.outDir,
|
|
1134
|
+
eventStream: input.eventStream ?? null,
|
|
1135
|
+
pidManifestPath: input.pidManifestPath ?? null,
|
|
1136
|
+
proof: input.proof
|
|
1137
|
+
}
|
|
1138
|
+
});
|
|
930
1139
|
await fs.mkdir(path.dirname(file), { recursive: true });
|
|
931
1140
|
await fs.writeFile(file, JSON.stringify(dashboard, null, 2) + '\n');
|
|
932
1141
|
}
|
|
@@ -993,7 +1202,9 @@ export async function collectCodexSwarmRun(input) {
|
|
|
993
1202
|
'stale-against-head': []
|
|
994
1203
|
};
|
|
995
1204
|
const collectedBundles = [];
|
|
1205
|
+
const evidenceEntries = [];
|
|
996
1206
|
const patchStatuses = {};
|
|
1207
|
+
const processes = await readCodexPidProcesses(path.join(runDir, 'pids.json')).catch(() => []);
|
|
997
1208
|
const mergePaths = (await findFilesByName(runDir, 'merge.json'))
|
|
998
1209
|
.filter((mergePath) => !pathHasIgnoredSegment(path.relative(runDir, mergePath), [
|
|
999
1210
|
'collected',
|
|
@@ -1015,23 +1226,38 @@ export async function collectCodexSwarmRun(input) {
|
|
|
1015
1226
|
for (const { mergePath, bundle } of mergeRecords) {
|
|
1016
1227
|
const patchPath = resolveBundlePatchPath(bundle, mergePath);
|
|
1017
1228
|
const patchExists = !!patchPath && await pathExists(patchPath);
|
|
1018
|
-
const
|
|
1229
|
+
const staleness = input.checkStale === false
|
|
1230
|
+
? { stale: false, patchStatus: patchExists ? 'unknown' : 'missing', reasons: ['stale check disabled'] }
|
|
1231
|
+
: await bundlePatchStaleness(bundle, mergePath, cwd);
|
|
1232
|
+
const staleAgainstHead = staleness.stale;
|
|
1019
1233
|
const bucket = classifyCodexCollectBucket(bundle, staleAgainstHead);
|
|
1020
1234
|
const branchName = input.branchPrefix ? `${input.branchPrefix}/${slug(bundle.jobId)}` : bundle.branchName;
|
|
1235
|
+
const outputDir = path.join(outDir, bucket, slug(bundle.jobId));
|
|
1236
|
+
const collectedEvidencePath = path.join(outputDir, 'evidence.json');
|
|
1021
1237
|
const nextBundle = {
|
|
1022
1238
|
...bundle,
|
|
1023
1239
|
...(branchName ? { branchName } : {}),
|
|
1024
1240
|
staleAgainstHead: bundle.staleAgainstHead || staleAgainstHead,
|
|
1025
1241
|
disposition: staleAgainstHead ? 'stale-against-head' : bundle.disposition,
|
|
1026
|
-
autoMergeable: bucket === 'ready-to-apply' && bundle.autoMergeable
|
|
1242
|
+
autoMergeable: bucket === 'ready-to-apply' && bundle.autoMergeable,
|
|
1243
|
+
evidencePaths: uniqueStrings([...bundle.evidencePaths, collectedEvidencePath])
|
|
1027
1244
|
};
|
|
1028
1245
|
collectedBundles.push(nextBundle);
|
|
1029
|
-
patchStatuses[nextBundle.jobId] =
|
|
1030
|
-
const outputDir = path.join(outDir, bucket, slug(bundle.jobId));
|
|
1246
|
+
patchStatuses[nextBundle.jobId] = staleness.patchStatus;
|
|
1031
1247
|
await fs.mkdir(outputDir, { recursive: true });
|
|
1032
1248
|
await fs.writeFile(path.join(outputDir, 'merge.json'), JSON.stringify(nextBundle, null, 2) + '\n');
|
|
1033
1249
|
if (patchPath && await pathExists(patchPath))
|
|
1034
1250
|
await fs.copyFile(patchPath, path.join(outputDir, 'changes.patch')).catch(() => { });
|
|
1251
|
+
await copyOrWriteCollectedEvidenceSummary({
|
|
1252
|
+
file: collectedEvidencePath,
|
|
1253
|
+
bundle: nextBundle,
|
|
1254
|
+
bucket,
|
|
1255
|
+
mergePath,
|
|
1256
|
+
patchPath,
|
|
1257
|
+
patchStatus: patchStatuses[nextBundle.jobId],
|
|
1258
|
+
staleReasons: staleness.reasons
|
|
1259
|
+
});
|
|
1260
|
+
evidenceEntries.push(...createCollectedEvidenceEntries(nextBundle, collectedEvidencePath, bucket));
|
|
1035
1261
|
buckets[bucket].push({ bucket, jobId: bundle.jobId, mergePath, outputDir, bundle: nextBundle });
|
|
1036
1262
|
}
|
|
1037
1263
|
const mergeIndex = createSwarmMergeIndex({
|
|
@@ -1043,6 +1269,33 @@ export async function collectCodexSwarmRun(input) {
|
|
|
1043
1269
|
runId: path.basename(runDir),
|
|
1044
1270
|
bundles: collectedBundles
|
|
1045
1271
|
});
|
|
1272
|
+
const evidenceIndex = createSwarmEvidenceIndex({
|
|
1273
|
+
id: `codex-evidence-index:${path.basename(runDir)}`,
|
|
1274
|
+
entries: evidenceEntries,
|
|
1275
|
+
generatedAt
|
|
1276
|
+
});
|
|
1277
|
+
const admission = createSwarmMergeAdmission({
|
|
1278
|
+
index: mergeIndex,
|
|
1279
|
+
maxReady: Math.max(mergeIndex.summary.readyToApplyCount, 1),
|
|
1280
|
+
allowRisks: ['low', 'medium', 'unknown'],
|
|
1281
|
+
generatedAt
|
|
1282
|
+
});
|
|
1283
|
+
const dashboard = createSwarmCoordinatorDashboard({
|
|
1284
|
+
bundles: collectedBundles,
|
|
1285
|
+
mergeIndex,
|
|
1286
|
+
queueOverlay,
|
|
1287
|
+
evidenceIndex,
|
|
1288
|
+
admission,
|
|
1289
|
+
processes,
|
|
1290
|
+
generatedAt,
|
|
1291
|
+
metadata: { runDir, outDir }
|
|
1292
|
+
});
|
|
1293
|
+
const compactDashboard = createCodexCompactDashboard({
|
|
1294
|
+
runDir,
|
|
1295
|
+
dashboard,
|
|
1296
|
+
semanticImportExpected: input.semanticImportExpected ?? false,
|
|
1297
|
+
generatedAt
|
|
1298
|
+
});
|
|
1046
1299
|
const summary = {
|
|
1047
1300
|
total: mergeRecords.length,
|
|
1048
1301
|
'ready-to-apply': buckets['ready-to-apply'].length,
|
|
@@ -1060,14 +1313,181 @@ export async function collectCodexSwarmRun(input) {
|
|
|
1060
1313
|
buckets,
|
|
1061
1314
|
mergeIndex,
|
|
1062
1315
|
queueOverlay,
|
|
1316
|
+
evidenceIndex,
|
|
1317
|
+
admission,
|
|
1318
|
+
dashboard,
|
|
1319
|
+
compactDashboard,
|
|
1063
1320
|
summary
|
|
1064
1321
|
};
|
|
1065
1322
|
await fs.mkdir(outDir, { recursive: true });
|
|
1066
1323
|
await fs.writeFile(path.join(outDir, 'collection.json'), JSON.stringify(result, null, 2) + '\n');
|
|
1067
1324
|
await fs.writeFile(path.join(outDir, 'merge-index.json'), JSON.stringify(mergeIndex, null, 2) + '\n');
|
|
1068
1325
|
await fs.writeFile(path.join(outDir, 'queue-overlay.json'), JSON.stringify(queueOverlay, null, 2) + '\n');
|
|
1326
|
+
await fs.writeFile(path.join(outDir, 'evidence-index.json'), JSON.stringify(evidenceIndex, null, 2) + '\n');
|
|
1327
|
+
await fs.writeFile(path.join(outDir, 'merge-admission.json'), JSON.stringify(admission, null, 2) + '\n');
|
|
1328
|
+
await fs.writeFile(path.join(outDir, 'coordinator-query.json'), JSON.stringify(dashboard, null, 2) + '\n');
|
|
1329
|
+
await fs.writeFile(path.join(outDir, 'compact-dashboard.json'), JSON.stringify(compactDashboard, null, 2) + '\n');
|
|
1069
1330
|
return result;
|
|
1070
1331
|
}
|
|
1332
|
+
async function readCodexPidProcesses(file) {
|
|
1333
|
+
const manifest = await readCodexPidManifest(file);
|
|
1334
|
+
return Promise.all(manifest.entries.map(async (entry) => ({
|
|
1335
|
+
pid: entry.pid,
|
|
1336
|
+
role: entry.role,
|
|
1337
|
+
...(entry.jobId ? { jobId: entry.jobId } : {}),
|
|
1338
|
+
...(entry.runId ? { runId: entry.runId } : {}),
|
|
1339
|
+
status: await pidIsAlive(entry.pid) ? 'running' : 'missing',
|
|
1340
|
+
startedAt: entry.startedAt,
|
|
1341
|
+
...(entry.command ? { command: entry.command } : {})
|
|
1342
|
+
})));
|
|
1343
|
+
}
|
|
1344
|
+
async function pidIsAlive(pid) {
|
|
1345
|
+
try {
|
|
1346
|
+
process.kill(pid, 0);
|
|
1347
|
+
return true;
|
|
1348
|
+
}
|
|
1349
|
+
catch {
|
|
1350
|
+
return false;
|
|
1351
|
+
}
|
|
1352
|
+
}
|
|
1353
|
+
async function copyOrWriteCollectedEvidenceSummary(input) {
|
|
1354
|
+
const existing = input.bundle.evidencePaths.find((entry) => path.basename(entry) === 'evidence.json' && entry !== input.file);
|
|
1355
|
+
await fs.mkdir(path.dirname(input.file), { recursive: true });
|
|
1356
|
+
if (existing && await pathExists(existing)) {
|
|
1357
|
+
await fs.copyFile(existing, input.file).catch(() => { });
|
|
1358
|
+
if (await pathExists(input.file))
|
|
1359
|
+
return;
|
|
1360
|
+
}
|
|
1361
|
+
const patchHunks = input.patchPath ? await readPatchHunks(input.patchPath) : [];
|
|
1362
|
+
const evidence = {
|
|
1363
|
+
kind: FRONTIER_SWARM_CODEX_JOB_EVIDENCE_KIND,
|
|
1364
|
+
version: FRONTIER_SWARM_CODEX_JOB_EVIDENCE_VERSION,
|
|
1365
|
+
generatedAt: Date.now(),
|
|
1366
|
+
jobId: input.bundle.jobId,
|
|
1367
|
+
taskId: input.bundle.taskId ?? input.bundle.jobId,
|
|
1368
|
+
lane: input.bundle.lane ?? 'unknown',
|
|
1369
|
+
status: input.bundle.status,
|
|
1370
|
+
mergeReadiness: input.bundle.mergeReadiness,
|
|
1371
|
+
disposition: input.bundle.disposition,
|
|
1372
|
+
riskLevel: input.bundle.riskLevel,
|
|
1373
|
+
changedPaths: [...input.bundle.changedPaths],
|
|
1374
|
+
changedRegions: [...input.bundle.changedRegions],
|
|
1375
|
+
ownershipViolations: [...input.bundle.ownershipViolations],
|
|
1376
|
+
...(input.patchPath ? { patchPath: input.patchPath } : {}),
|
|
1377
|
+
mergeBundlePath: input.mergePath,
|
|
1378
|
+
evidencePaths: uniqueStrings(input.bundle.evidencePaths),
|
|
1379
|
+
handoffArtifacts: input.bundle.evidencePaths.map((entry) => ({ path: entry, kind: classifyCodexHandoffArtifact(entry) ?? 'evidence' })),
|
|
1380
|
+
commands: {
|
|
1381
|
+
passed: input.bundle.commandsPassed.map((command) => ({ name: command.name, command: [...command.command], ...(command.status !== undefined ? { status: command.status } : {}) })),
|
|
1382
|
+
failed: input.bundle.commandsFailed.map((command) => ({ name: command.name, command: [...command.command], ...(command.status !== undefined ? { status: command.status } : {}) }))
|
|
1383
|
+
},
|
|
1384
|
+
patchHunks,
|
|
1385
|
+
readyToPortHunkCount: input.bucket === 'needs-human-port' || input.bucket === 'ready-to-apply' ? patchHunks.length : 0,
|
|
1386
|
+
...(input.bundle.semanticImport ? { semanticImport: input.bundle.semanticImport } : {}),
|
|
1387
|
+
sourceCitations: createCodexEvidenceSourceCitations(input.bundle),
|
|
1388
|
+
metadata: {
|
|
1389
|
+
bucket: input.bucket,
|
|
1390
|
+
patchStatus: input.patchStatus,
|
|
1391
|
+
staleReasons: input.staleReasons ?? [],
|
|
1392
|
+
autoMergeable: input.bundle.autoMergeable,
|
|
1393
|
+
staleAgainstHead: input.bundle.staleAgainstHead,
|
|
1394
|
+
reasons: input.bundle.reasons
|
|
1395
|
+
}
|
|
1396
|
+
};
|
|
1397
|
+
await fs.writeFile(input.file, JSON.stringify(evidence, null, 2) + '\n');
|
|
1398
|
+
}
|
|
1399
|
+
function createCollectedEvidenceEntries(bundle, collectedEvidencePath, bucket) {
|
|
1400
|
+
const confidence = bucket === 'ready-to-apply' ? 0.95 : bucket === 'needs-human-port' ? 0.7 : bucket === 'failed-evidence' ? 0.25 : 0.2;
|
|
1401
|
+
const entries = [{
|
|
1402
|
+
jobId: bundle.jobId,
|
|
1403
|
+
queueItemId: bundle.queueItemIds[0],
|
|
1404
|
+
lane: bundle.lane,
|
|
1405
|
+
topic: 'merge-admission',
|
|
1406
|
+
path: collectedEvidencePath,
|
|
1407
|
+
kind: 'evidence',
|
|
1408
|
+
status: bucket,
|
|
1409
|
+
confidence,
|
|
1410
|
+
tags: ['coordinator-query', bucket],
|
|
1411
|
+
facets: {
|
|
1412
|
+
bucket,
|
|
1413
|
+
disposition: bundle.disposition,
|
|
1414
|
+
riskLevel: bundle.riskLevel,
|
|
1415
|
+
autoMergeable: bundle.autoMergeable,
|
|
1416
|
+
staleAgainstHead: bundle.staleAgainstHead,
|
|
1417
|
+
semanticSymbols: bundle.semanticImport?.semanticIndex.symbols ?? 0,
|
|
1418
|
+
semanticRegions: bundle.semanticImport?.semanticSidecars.ownershipRegions ?? 0
|
|
1419
|
+
}
|
|
1420
|
+
}];
|
|
1421
|
+
for (const file of bundle.evidencePaths) {
|
|
1422
|
+
if (file === collectedEvidencePath)
|
|
1423
|
+
continue;
|
|
1424
|
+
entries.push({
|
|
1425
|
+
jobId: bundle.jobId,
|
|
1426
|
+
queueItemId: bundle.queueItemIds[0],
|
|
1427
|
+
lane: bundle.lane,
|
|
1428
|
+
topic: path.basename(file),
|
|
1429
|
+
path: file,
|
|
1430
|
+
kind: classifyCodexHandoffArtifact(file) ?? 'evidence',
|
|
1431
|
+
status: bucket,
|
|
1432
|
+
confidence,
|
|
1433
|
+
tags: [bucket],
|
|
1434
|
+
facets: { bucket, disposition: bundle.disposition }
|
|
1435
|
+
});
|
|
1436
|
+
}
|
|
1437
|
+
return entries;
|
|
1438
|
+
}
|
|
1439
|
+
function createCodexCompactDashboard(input) {
|
|
1440
|
+
const qualities = new Map(input.dashboard.jobs.map((job) => [
|
|
1441
|
+
job.jobId,
|
|
1442
|
+
summarizeCodexSemanticImportQuality(job.semanticImport, input.semanticImportExpected)
|
|
1443
|
+
]));
|
|
1444
|
+
const semanticQualities = Array.from(qualities.values());
|
|
1445
|
+
const usefulPatchJobs = input.dashboard.jobs.filter((job) => ((job.disposition === 'auto-mergeable' || job.disposition === 'needs-port')
|
|
1446
|
+
&& job.changedPaths.length > 0
|
|
1447
|
+
&& job.tests.requiredFailed === 0));
|
|
1448
|
+
const topJobs = [...input.dashboard.jobs]
|
|
1449
|
+
.filter((job) => job.changedPaths.length > 0 || job.evidencePaths.length > 0)
|
|
1450
|
+
.sort((left, right) => right.mergeScore - left.mergeScore || left.jobId.localeCompare(right.jobId))
|
|
1451
|
+
.slice(0, 20)
|
|
1452
|
+
.map((job) => ({
|
|
1453
|
+
jobId: job.jobId,
|
|
1454
|
+
...(job.lane ? { lane: job.lane } : {}),
|
|
1455
|
+
disposition: job.disposition,
|
|
1456
|
+
mergeScore: job.mergeScore,
|
|
1457
|
+
changedPaths: job.changedPaths.slice(0, 12),
|
|
1458
|
+
semanticImportQuality: qualities.get(job.jobId),
|
|
1459
|
+
staleAgainstHead: job.staleAgainstHead,
|
|
1460
|
+
...(job.duplicateGroupId ? { duplicateGroupId: job.duplicateGroupId } : {}),
|
|
1461
|
+
evidencePaths: job.evidencePaths.slice(0, 12)
|
|
1462
|
+
}));
|
|
1463
|
+
return {
|
|
1464
|
+
kind: FRONTIER_SWARM_CODEX_COMPACT_DASHBOARD_KIND,
|
|
1465
|
+
version: FRONTIER_SWARM_CODEX_COMPACT_DASHBOARD_VERSION,
|
|
1466
|
+
generatedAt: input.generatedAt,
|
|
1467
|
+
runDir: input.runDir,
|
|
1468
|
+
total: input.dashboard.summary.jobCount,
|
|
1469
|
+
activeJobs: input.dashboard.jobs.filter((job) => job.liveness === 'running').length,
|
|
1470
|
+
usefulPatchCount: usefulPatchJobs.length,
|
|
1471
|
+
stalePatchCount: input.dashboard.summary.staleAgainstHeadCount,
|
|
1472
|
+
duplicateDiscoveryCount: input.dashboard.duplicateGroups.length,
|
|
1473
|
+
semanticImport: {
|
|
1474
|
+
expected: input.semanticImportExpected,
|
|
1475
|
+
presentCount: semanticQualities.filter((entry) => entry.present).length,
|
|
1476
|
+
emptyCount: semanticQualities.filter((entry) => entry.empty).length,
|
|
1477
|
+
weakCount: semanticQualities.filter((entry) => entry.present && entry.warnings.length > 0).length,
|
|
1478
|
+
symbolCount: semanticQualities.reduce((sum, entry) => sum + entry.symbols, 0),
|
|
1479
|
+
ownershipRegionCount: semanticQualities.reduce((sum, entry) => sum + entry.ownershipRegions, 0),
|
|
1480
|
+
patchHintCount: semanticQualities.reduce((sum, entry) => sum + entry.patchHints, 0)
|
|
1481
|
+
},
|
|
1482
|
+
evidence: {
|
|
1483
|
+
readyToApply: input.dashboard.summary.readyToApplyCount,
|
|
1484
|
+
needsHumanPort: input.dashboard.summary.needsHumanPortCount,
|
|
1485
|
+
failedEvidence: input.dashboard.summary.failedEvidenceCount,
|
|
1486
|
+
averageMergeScore: input.dashboard.summary.averageMergeScore
|
|
1487
|
+
},
|
|
1488
|
+
topJobs
|
|
1489
|
+
};
|
|
1490
|
+
}
|
|
1071
1491
|
export async function applyCodexSwarmCollection(input) {
|
|
1072
1492
|
const generatedAt = Date.now();
|
|
1073
1493
|
const cwd = path.resolve(input.cwd ?? process.cwd());
|
|
@@ -1388,12 +1808,132 @@ function summarizePatchScoreSemanticEvidence(bundle) {
|
|
|
1388
1808
|
reasons: uniqueStrings(reasons)
|
|
1389
1809
|
};
|
|
1390
1810
|
}
|
|
1811
|
+
function summarizeCodexSemanticImportQuality(summary, expected = false) {
|
|
1812
|
+
const selected = nonNegativeNumber(summary?.selected);
|
|
1813
|
+
const eligible = nonNegativeNumber(summary?.eligible);
|
|
1814
|
+
const imported = nonNegativeNumber(summary?.imported);
|
|
1815
|
+
const symbols = nonNegativeNumber(summary?.semanticIndex?.symbols);
|
|
1816
|
+
const ownershipRegions = nonNegativeNumber(summary?.semanticSidecars?.ownershipRegions);
|
|
1817
|
+
const patchHints = nonNegativeNumber(summary?.semanticSidecars?.patchHints);
|
|
1818
|
+
const sourceMapMappings = nonNegativeNumber(summary?.sourceMapMappingCount);
|
|
1819
|
+
const present = !!summary;
|
|
1820
|
+
const empty = present && (nonNegativeNumber(summary?.total) === 0 || selected === 0 && eligible === 0 && imported === 0 && symbols === 0);
|
|
1821
|
+
const warnings = [];
|
|
1822
|
+
if (expected && !present)
|
|
1823
|
+
warnings.push('semantic import expected but missing');
|
|
1824
|
+
if (expected && empty)
|
|
1825
|
+
warnings.push('semantic import expected but empty');
|
|
1826
|
+
if (present && imported === 0)
|
|
1827
|
+
warnings.push('semantic import imported no files');
|
|
1828
|
+
if (present && selected > 0 && symbols === 0)
|
|
1829
|
+
warnings.push('semantic import has no symbols');
|
|
1830
|
+
if (present && selected > 0 && ownershipRegions === 0)
|
|
1831
|
+
warnings.push('semantic import has no ownership regions');
|
|
1832
|
+
if (present && selected > 0 && sourceMapMappings === 0)
|
|
1833
|
+
warnings.push('semantic import has no source-map mappings');
|
|
1834
|
+
return {
|
|
1835
|
+
expected,
|
|
1836
|
+
present,
|
|
1837
|
+
empty,
|
|
1838
|
+
selected,
|
|
1839
|
+
eligible,
|
|
1840
|
+
imported,
|
|
1841
|
+
symbols,
|
|
1842
|
+
ownershipRegions,
|
|
1843
|
+
patchHints,
|
|
1844
|
+
sourceMapMappings,
|
|
1845
|
+
warnings: uniqueStrings(warnings)
|
|
1846
|
+
};
|
|
1847
|
+
}
|
|
1391
1848
|
function semanticImportSummaryFromBundle(bundle) {
|
|
1392
1849
|
if (bundle.semanticImport)
|
|
1393
1850
|
return bundle.semanticImport;
|
|
1394
1851
|
const metadata = bundle.metadata;
|
|
1395
1852
|
return metadata?.semanticImport;
|
|
1396
1853
|
}
|
|
1854
|
+
function semanticImportEnabled(input) {
|
|
1855
|
+
if (input === true)
|
|
1856
|
+
return true;
|
|
1857
|
+
if (!input)
|
|
1858
|
+
return false;
|
|
1859
|
+
return input.enabled !== false;
|
|
1860
|
+
}
|
|
1861
|
+
function normalizeCompactLogOptions(input) {
|
|
1862
|
+
if (input === false)
|
|
1863
|
+
return { enabled: false };
|
|
1864
|
+
if (input === true || input === undefined)
|
|
1865
|
+
return { enabled: true, maxEventBytes: 1_000_000, maxStderrBytes: 256_000 };
|
|
1866
|
+
return {
|
|
1867
|
+
enabled: input.enabled ?? true,
|
|
1868
|
+
maxEventBytes: positiveInteger(input.maxEventBytes, 1_000_000),
|
|
1869
|
+
maxStderrBytes: positiveInteger(input.maxStderrBytes, 256_000)
|
|
1870
|
+
};
|
|
1871
|
+
}
|
|
1872
|
+
function normalizeAdaptiveConcurrencyOptions(input, maxConcurrency) {
|
|
1873
|
+
if (input === false || input === undefined) {
|
|
1874
|
+
return { enabled: false, mode: 'balanced', minConcurrency: 1, maxConcurrency, writePlan: true };
|
|
1875
|
+
}
|
|
1876
|
+
if (input === true) {
|
|
1877
|
+
return { enabled: true, mode: 'balanced', minConcurrency: 1, maxConcurrency, writePlan: true };
|
|
1878
|
+
}
|
|
1879
|
+
return {
|
|
1880
|
+
enabled: input.enabled ?? true,
|
|
1881
|
+
mode: input.mode ?? 'balanced',
|
|
1882
|
+
minConcurrency: Math.max(1, Math.min(maxConcurrency, Math.floor(input.minConcurrency ?? 1))),
|
|
1883
|
+
maxConcurrency: Math.max(1, Math.min(maxConcurrency, Math.floor(input.maxConcurrency ?? maxConcurrency))),
|
|
1884
|
+
writePlan: input.writePlan ?? true
|
|
1885
|
+
};
|
|
1886
|
+
}
|
|
1887
|
+
function createCodexAdaptiveObservations(results) {
|
|
1888
|
+
const observations = [];
|
|
1889
|
+
for (const result of results) {
|
|
1890
|
+
const metadata = result.metadata && typeof result.metadata === 'object' ? result.metadata : {};
|
|
1891
|
+
const logSummary = metadata.logSummary;
|
|
1892
|
+
if (logSummary && (logSummary.eventBytesTruncated > 0 || logSummary.stderrBytesTruncated > 0 || logSummary.eventBytes > 1_000_000 || logSummary.stderrBytes > 256_000)) {
|
|
1893
|
+
observations.push({
|
|
1894
|
+
kind: 'log-noise',
|
|
1895
|
+
severity: logSummary.eventBytesTruncated > 0 || logSummary.stderrBytesTruncated > 0 ? 'warning' : 'info',
|
|
1896
|
+
jobId: result.jobId,
|
|
1897
|
+
value: logSummary.eventBytes + logSummary.stderrBytes,
|
|
1898
|
+
reason: 'worker output exceeded compact log threshold',
|
|
1899
|
+
metadata: logSummary
|
|
1900
|
+
});
|
|
1901
|
+
}
|
|
1902
|
+
if (result.mergeDisposition === 'stale-against-head') {
|
|
1903
|
+
observations.push({ kind: 'stale-patch', severity: 'warning', jobId: result.jobId, reason: 'worker result is stale against head' });
|
|
1904
|
+
}
|
|
1905
|
+
if (result.mergeDisposition === 'discovery-only' || result.mergeReadiness === 'discovery-only') {
|
|
1906
|
+
observations.push({ kind: 'discovery-only-output', severity: 'info', jobId: result.jobId, reason: 'worker produced discovery-only output' });
|
|
1907
|
+
}
|
|
1908
|
+
const semanticQuality = summarizeCodexSemanticImportQuality(result.semanticImport ?? metadata.semanticImport, false);
|
|
1909
|
+
if (semanticQuality.present && semanticQuality.empty) {
|
|
1910
|
+
observations.push({ kind: 'semantic-empty', severity: 'warning', jobId: result.jobId, reason: 'worker semantic sidecar is empty' });
|
|
1911
|
+
}
|
|
1912
|
+
else if (semanticQuality.present && semanticQuality.warnings.length > 0) {
|
|
1913
|
+
observations.push({ kind: 'semantic-weak', severity: 'info', jobId: result.jobId, reasons: semanticQuality.warnings });
|
|
1914
|
+
}
|
|
1915
|
+
}
|
|
1916
|
+
return observations;
|
|
1917
|
+
}
|
|
1918
|
+
function createEmptyCodexLogSummary(paths) {
|
|
1919
|
+
return {
|
|
1920
|
+
eventsPath: paths.eventsPath,
|
|
1921
|
+
stderrPath: paths.stderrPath,
|
|
1922
|
+
eventBytes: 0,
|
|
1923
|
+
stderrBytes: 0,
|
|
1924
|
+
eventBytesWritten: 0,
|
|
1925
|
+
stderrBytesWritten: 0,
|
|
1926
|
+
eventBytesTruncated: 0,
|
|
1927
|
+
stderrBytesTruncated: 0
|
|
1928
|
+
};
|
|
1929
|
+
}
|
|
1930
|
+
function positiveInteger(value, fallback) {
|
|
1931
|
+
const number = Number(value);
|
|
1932
|
+
return Number.isFinite(number) && number > 0 ? Math.floor(number) : fallback;
|
|
1933
|
+
}
|
|
1934
|
+
function firstNonEmptyLine(text) {
|
|
1935
|
+
return text.split(/\r?\n/).map((line) => line.trim()).find(Boolean);
|
|
1936
|
+
}
|
|
1397
1937
|
function numberRecord(value) {
|
|
1398
1938
|
if (!value || typeof value !== 'object')
|
|
1399
1939
|
return {};
|
|
@@ -1709,24 +2249,65 @@ async function runVerification(commands, cwd) {
|
|
|
1709
2249
|
}
|
|
1710
2250
|
return results;
|
|
1711
2251
|
}
|
|
1712
|
-
async function runScheduledJobPool(plan,
|
|
2252
|
+
async function runScheduledJobPool(plan, input, worker) {
|
|
2253
|
+
const concurrency = Math.max(1, Math.floor(input.concurrency));
|
|
2254
|
+
const adaptiveOptions = normalizeAdaptiveConcurrencyOptions(input.adaptive, concurrency);
|
|
1713
2255
|
const results = [];
|
|
1714
2256
|
const active = new Map();
|
|
1715
2257
|
const leases = [];
|
|
1716
2258
|
const completed = new Set();
|
|
1717
2259
|
const resultByJob = new Map();
|
|
2260
|
+
const adaptiveHistory = [];
|
|
2261
|
+
let currentAdaptiveLimits;
|
|
1718
2262
|
while (resultByJob.size < plan.jobs.length) {
|
|
1719
2263
|
const run = createSwarmRun({ plan, status: 'running', results });
|
|
1720
2264
|
run.jobs = run.jobs.map((job) => active.has(job.id) ? { ...job, status: 'running' } : job);
|
|
1721
|
-
const
|
|
2265
|
+
const adaptivePlan = adaptiveOptions.enabled ? createSwarmAdaptiveLoadPlan({
|
|
1722
2266
|
plan,
|
|
1723
2267
|
run,
|
|
1724
|
-
|
|
2268
|
+
mode: adaptiveOptions.mode,
|
|
2269
|
+
maxLimits: { maxReadyJobs: adaptiveOptions.maxConcurrency },
|
|
2270
|
+
minLimits: { maxReadyJobs: adaptiveOptions.minConcurrency },
|
|
2271
|
+
currentLimits: currentAdaptiveLimits ?? { maxReadyJobs: adaptiveOptions.maxConcurrency },
|
|
2272
|
+
observations: createCodexAdaptiveObservations(results)
|
|
2273
|
+
}) : undefined;
|
|
2274
|
+
if (adaptivePlan) {
|
|
2275
|
+
currentAdaptiveLimits = adaptivePlan.effectiveLimits;
|
|
2276
|
+
adaptiveHistory.push(adaptivePlan);
|
|
2277
|
+
if (adaptiveOptions.writePlan !== false && input.outDir) {
|
|
2278
|
+
await writeJsonAtomic(path.join(input.outDir, 'adaptive-load.json'), {
|
|
2279
|
+
latest: adaptivePlan,
|
|
2280
|
+
history: adaptiveHistory.slice(-50)
|
|
2281
|
+
}).catch(() => { });
|
|
2282
|
+
}
|
|
2283
|
+
await appendFileSwarmEvent(input.eventStream, {
|
|
2284
|
+
type: 'swarm.adaptive-load',
|
|
2285
|
+
runId: run.id,
|
|
2286
|
+
data: {
|
|
2287
|
+
mode: adaptivePlan.mode,
|
|
2288
|
+
effectiveMaxReadyJobs: adaptivePlan.effectiveLimits.maxReadyJobs,
|
|
2289
|
+
bottleneckCount: adaptivePlan.summary.bottleneckCount,
|
|
2290
|
+
decisions: adaptivePlan.decisions.map((decision) => ({
|
|
2291
|
+
action: decision.action,
|
|
2292
|
+
target: decision.target,
|
|
2293
|
+
key: decision.key,
|
|
2294
|
+
previous: decision.previous,
|
|
2295
|
+
next: decision.next,
|
|
2296
|
+
reason: decision.reason
|
|
2297
|
+
}))
|
|
2298
|
+
}
|
|
2299
|
+
});
|
|
2300
|
+
}
|
|
2301
|
+
const effectiveConcurrency = Math.max(1, Math.min(concurrency, adaptivePlan?.effectiveLimits.maxReadyJobs ?? concurrency));
|
|
2302
|
+
const readyWindow = Math.max(0, effectiveConcurrency - active.size);
|
|
2303
|
+
const schedule = createSwarmSchedule({
|
|
2304
|
+
...(adaptivePlan ? createSwarmScheduleInputFromAdaptiveLoadPlan(plan, adaptivePlan, { run }) : { plan, run }),
|
|
2305
|
+
maxReadyJobs: readyWindow
|
|
1725
2306
|
});
|
|
1726
2307
|
const nextLeases = createSwarmLeases({
|
|
1727
2308
|
schedule,
|
|
1728
2309
|
workerId: 'frontier-swarm-codex',
|
|
1729
|
-
count:
|
|
2310
|
+
count: readyWindow,
|
|
1730
2311
|
existingLeases: leases
|
|
1731
2312
|
});
|
|
1732
2313
|
for (const lease of nextLeases) {
|
|
@@ -1882,6 +2463,15 @@ function selectSemanticImportPaths(changedPaths, options) {
|
|
|
1882
2463
|
maxFiles
|
|
1883
2464
|
};
|
|
1884
2465
|
}
|
|
2466
|
+
function semanticImportCandidatePaths(job, changedPaths) {
|
|
2467
|
+
const concreteRefs = job.task.sourceRefs.concat(job.task.targetRefs).filter((file) => {
|
|
2468
|
+
const normalized = normalizeWorkspacePath(file);
|
|
2469
|
+
return normalized
|
|
2470
|
+
&& !normalized.includes('*')
|
|
2471
|
+
&& path.extname(normalized).length > 0;
|
|
2472
|
+
});
|
|
2473
|
+
return uniqueWorkspacePaths([...changedPaths, ...concreteRefs]);
|
|
2474
|
+
}
|
|
1885
2475
|
function inferSemanticImportLanguage(file, overrides) {
|
|
1886
2476
|
const ext = path.extname(file).toLowerCase();
|
|
1887
2477
|
return overrides?.[file] ?? overrides?.[ext] ?? {
|
|
@@ -2200,15 +2790,77 @@ async function findFilesByName(root, name) {
|
|
|
2200
2790
|
await walk(root);
|
|
2201
2791
|
return out;
|
|
2202
2792
|
}
|
|
2203
|
-
async function
|
|
2793
|
+
async function bundlePatchStaleness(bundle, mergePath, cwd) {
|
|
2204
2794
|
const patchPath = resolveBundlePatchPath(bundle, mergePath);
|
|
2205
2795
|
if (!patchPath || !await pathExists(patchPath))
|
|
2206
|
-
return false;
|
|
2796
|
+
return { stale: false, patchStatus: 'missing', reasons: ['missing patch'] };
|
|
2207
2797
|
const patch = await fs.readFile(patchPath, 'utf8').catch(() => '');
|
|
2208
2798
|
if (!patch.trim())
|
|
2209
|
-
return false;
|
|
2799
|
+
return { stale: false, patchStatus: 'missing', reasons: ['empty patch'] };
|
|
2210
2800
|
const result = await runProcess('git', ['apply', '--check', patchPath], { cwd, allowFailure: true });
|
|
2211
|
-
|
|
2801
|
+
if (result.status === 0)
|
|
2802
|
+
return { stale: false, patchStatus: 'applies', reasons: ['patch applies to working tree'] };
|
|
2803
|
+
const cached = await runProcess('git', ['apply', '--check', '--cached', patchPath], { cwd, allowFailure: true });
|
|
2804
|
+
if (cached.status === 0) {
|
|
2805
|
+
return {
|
|
2806
|
+
stale: false,
|
|
2807
|
+
patchStatus: 'dirty-workspace-conflict',
|
|
2808
|
+
reasons: ['patch applies to index but not dirty working tree']
|
|
2809
|
+
};
|
|
2810
|
+
}
|
|
2811
|
+
const baseStatus = await patchBaseHashStatus(patch, cwd);
|
|
2812
|
+
if (baseStatus.known && baseStatus.mismatched === 0) {
|
|
2813
|
+
return {
|
|
2814
|
+
stale: false,
|
|
2815
|
+
patchStatus: 'needs-port',
|
|
2816
|
+
reasons: ['patch base hashes match HEAD but textual apply failed', ...baseStatus.reasons]
|
|
2817
|
+
};
|
|
2818
|
+
}
|
|
2819
|
+
return {
|
|
2820
|
+
stale: true,
|
|
2821
|
+
patchStatus: 'stale',
|
|
2822
|
+
reasons: uniqueStrings(['git apply --check failed', ...baseStatus.reasons, ...tail(result.stderr || result.stdout, 3)])
|
|
2823
|
+
};
|
|
2824
|
+
}
|
|
2825
|
+
async function patchBaseHashStatus(patch, cwd) {
|
|
2826
|
+
const entries = parsePatchBaseHashes(patch);
|
|
2827
|
+
if (entries.length === 0)
|
|
2828
|
+
return { known: false, mismatched: 0, reasons: ['no patch base hashes available'] };
|
|
2829
|
+
let mismatched = 0;
|
|
2830
|
+
const reasons = [];
|
|
2831
|
+
for (const entry of entries) {
|
|
2832
|
+
const head = await runProcess('git', ['rev-parse', `HEAD:${entry.path}`], { cwd, allowFailure: true });
|
|
2833
|
+
if (head.status !== 0) {
|
|
2834
|
+
mismatched += 1;
|
|
2835
|
+
reasons.push(`missing HEAD blob for ${entry.path}`);
|
|
2836
|
+
continue;
|
|
2837
|
+
}
|
|
2838
|
+
const headHash = head.stdout.trim();
|
|
2839
|
+
if (!headHash.startsWith(entry.oldHash)) {
|
|
2840
|
+
mismatched += 1;
|
|
2841
|
+
reasons.push(`base hash mismatch for ${entry.path}`);
|
|
2842
|
+
}
|
|
2843
|
+
}
|
|
2844
|
+
return { known: true, mismatched, reasons };
|
|
2845
|
+
}
|
|
2846
|
+
function parsePatchBaseHashes(patch) {
|
|
2847
|
+
const lines = patch.split(/\r?\n/);
|
|
2848
|
+
const entries = [];
|
|
2849
|
+
let currentPath;
|
|
2850
|
+
for (const line of lines) {
|
|
2851
|
+
if (line.startsWith('diff --git ')) {
|
|
2852
|
+
const parts = line.split(/\s+/);
|
|
2853
|
+
const right = parts[3] ?? parts[2];
|
|
2854
|
+
currentPath = right?.startsWith('b/') ? right.slice(2) : right;
|
|
2855
|
+
continue;
|
|
2856
|
+
}
|
|
2857
|
+
if (!currentPath || !line.startsWith('index '))
|
|
2858
|
+
continue;
|
|
2859
|
+
const match = /^index\s+([0-9a-f]+)\.\.([0-9a-f]+)/i.exec(line);
|
|
2860
|
+
if (match?.[1] && match[1] !== '0000000')
|
|
2861
|
+
entries.push({ path: currentPath, oldHash: match[1] });
|
|
2862
|
+
}
|
|
2863
|
+
return entries;
|
|
2212
2864
|
}
|
|
2213
2865
|
function resolveBundlePatchPath(bundle, mergePath) {
|
|
2214
2866
|
if (!bundle.patchPath)
|
|
@@ -2256,7 +2908,9 @@ function normalizeCollectedMergeBundle(value, mergePath) {
|
|
|
2256
2908
|
...(typeof input.branchName === 'string' ? { branchName: input.branchName } : {}),
|
|
2257
2909
|
...(typeof input.commit === 'string' ? { commit: input.commit } : {}),
|
|
2258
2910
|
staleAgainstHead: Boolean(input.staleAgainstHead),
|
|
2259
|
-
reasons: stringArray(input.reasons)
|
|
2911
|
+
reasons: stringArray(input.reasons),
|
|
2912
|
+
...(isObject(input.semanticImport) ? { semanticImport: input.semanticImport } : {}),
|
|
2913
|
+
...(isObject(input.metadata) ? { metadata: input.metadata } : {})
|
|
2260
2914
|
};
|
|
2261
2915
|
}
|
|
2262
2916
|
function mergeRecordScore(record) {
|