@wildorder/nightshift 0.7.2 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/as-built.d.ts +125 -0
- package/dist/as-built.d.ts.map +1 -0
- package/dist/as-built.js +322 -0
- package/dist/as-built.js.map +1 -0
- package/dist/cli.js +15 -0
- package/dist/cli.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/manifest.d.ts +2 -2
- package/dist/run-program.d.ts +76 -2
- package/dist/run-program.d.ts.map +1 -1
- package/dist/run-program.js +473 -41
- package/dist/run-program.js.map +1 -1
- package/dist/whole-program-review.d.ts +165 -0
- package/dist/whole-program-review.d.ts.map +1 -0
- package/dist/whole-program-review.js +580 -0
- package/dist/whole-program-review.js.map +1 -0
- package/package.json +2 -2
package/dist/run-program.js
CHANGED
|
@@ -17,6 +17,7 @@ import { detectDefaultBranch, programBranchName } from "./program-branch.js";
|
|
|
17
17
|
import { restoreProgramsDir, snapshotProgramsDir } from "./programs-dir.js";
|
|
18
18
|
import { CouldNotStartError } from "./exit-codes.js";
|
|
19
19
|
import { runReportPath } from "./report-path.js";
|
|
20
|
+
import { AS_BUILT_PATH, runWholeProgramReview, renderWholeProgramReview, } from "./whole-program-review.js";
|
|
20
21
|
const execFileAsync = promisify(execFile);
|
|
21
22
|
/** Matches every wording git uses to report an empty commit attempt. */
|
|
22
23
|
const NOTHING_TO_COMMIT = /nothing to commit|nothing added to commit|no changes added to commit/u;
|
|
@@ -111,6 +112,15 @@ export const defaultGitOps = {
|
|
|
111
112
|
return false;
|
|
112
113
|
}
|
|
113
114
|
},
|
|
115
|
+
async mergeBase(cwd, a, b) {
|
|
116
|
+
try {
|
|
117
|
+
const { stdout } = await execFileAsync("git", ["merge-base", a, b], { cwd });
|
|
118
|
+
return stdout.trim();
|
|
119
|
+
}
|
|
120
|
+
catch {
|
|
121
|
+
return undefined;
|
|
122
|
+
}
|
|
123
|
+
},
|
|
114
124
|
async resetHard(cwd, commit) {
|
|
115
125
|
await execFileAsync("git", ["reset", "--hard", commit], { cwd });
|
|
116
126
|
},
|
|
@@ -174,7 +184,7 @@ function findingsRuledOnSection(records) {
|
|
|
174
184
|
return [];
|
|
175
185
|
return ["## Findings ruled on", "", ...lines, ""];
|
|
176
186
|
}
|
|
177
|
-
function implementerBrief(manifest, workstream, spec, ledger, priorFailure) {
|
|
187
|
+
function implementerBrief(manifest, workstream, spec, ledger, priorFailure, priorDiagnosis) {
|
|
178
188
|
const roster = manifest.workstreams
|
|
179
189
|
.map((entry) => {
|
|
180
190
|
const scope = entry.scope?.summary ?? entry.name;
|
|
@@ -201,10 +211,25 @@ function implementerBrief(manifest, workstream, spec, ledger, priorFailure) {
|
|
|
201
211
|
? [
|
|
202
212
|
"## Previous attempt failed",
|
|
203
213
|
"",
|
|
204
|
-
"A previous attempt at this workstream failed.
|
|
214
|
+
"A previous attempt at this workstream failed. The verify output:",
|
|
205
215
|
"",
|
|
206
216
|
priorFailure,
|
|
207
217
|
"",
|
|
218
|
+
...(priorDiagnosis
|
|
219
|
+
? [
|
|
220
|
+
"An independent reviewer read the failing tree before you were",
|
|
221
|
+
"spawned. Its diagnosis:",
|
|
222
|
+
"",
|
|
223
|
+
priorDiagnosis,
|
|
224
|
+
"",
|
|
225
|
+
"The tests in the tree were written by the failed attempt, not",
|
|
226
|
+
"by a human. When the diagnosis says an assertion is wrong,",
|
|
227
|
+
"rewrite or delete that assertion rather than bending the",
|
|
228
|
+
"implementation to satisfy it — the spec, not the failing",
|
|
229
|
+
"test, is the contract.",
|
|
230
|
+
"",
|
|
231
|
+
]
|
|
232
|
+
: []),
|
|
208
233
|
"Start from the diagnosis; the working tree may already contain",
|
|
209
234
|
"partial work from that attempt.",
|
|
210
235
|
"",
|
|
@@ -284,8 +309,8 @@ export async function runProgram(options) {
|
|
|
284
309
|
? `decider: ${describeAgent(decider)}`
|
|
285
310
|
: "decider: none configured — implementer defaults will stand unratified");
|
|
286
311
|
log(reviewer
|
|
287
|
-
? `test critique
|
|
288
|
-
: "
|
|
312
|
+
? `reviewer (test critique, whole-program review): ${describeAgent(reviewer)}`
|
|
313
|
+
: "reviewer: none configured — test critique and the whole-program review are disabled");
|
|
289
314
|
const cycles = findCycles(manifest.workstreams);
|
|
290
315
|
if (cycles.length > 0) {
|
|
291
316
|
// A cyclic graph cannot be ordered; this is a planning defect, not a
|
|
@@ -341,6 +366,11 @@ export async function runProgram(options) {
|
|
|
341
366
|
}
|
|
342
367
|
}
|
|
343
368
|
}
|
|
369
|
+
// HEAD as it stood before the run's first agent ran — the whole-program
|
|
370
|
+
// review stage's fallback diff base (see resolveProgramDiffBase) when a
|
|
371
|
+
// resumed run's merge-base with the default branch is unavailable or
|
|
372
|
+
// equal to HEAD.
|
|
373
|
+
const runStartCommit = isRepository ? await git.currentCommit(root) : undefined;
|
|
344
374
|
// Subject ids already sent to the decider this run — shared across the
|
|
345
375
|
// authoring and build stages (a run is one process) and across the
|
|
346
376
|
// decision and finding paths, so "once per subject per run" (SC-10) is
|
|
@@ -415,23 +445,51 @@ export async function runProgram(options) {
|
|
|
415
445
|
}
|
|
416
446
|
}
|
|
417
447
|
}
|
|
418
|
-
const ledger = await readDecisionLedger(root, options.programId);
|
|
419
|
-
const escalations = escalatedRecords(ledger);
|
|
420
448
|
const complete = results.every((result) => result.outcome.status === "complete" ||
|
|
421
449
|
result.outcome.status === "skipped");
|
|
422
450
|
manifest.program.status = complete ? "complete" : "partial";
|
|
423
451
|
await saveManifest(root, options.programId, manifest);
|
|
452
|
+
const wholeProgram = await runWholeProgramStage({
|
|
453
|
+
root,
|
|
454
|
+
programId: options.programId,
|
|
455
|
+
manifest,
|
|
456
|
+
config,
|
|
457
|
+
agentRunner,
|
|
458
|
+
git,
|
|
459
|
+
isRepository,
|
|
460
|
+
results,
|
|
461
|
+
authorResult,
|
|
462
|
+
runStartCommit,
|
|
463
|
+
decider,
|
|
464
|
+
reviewed,
|
|
465
|
+
triaged,
|
|
466
|
+
now,
|
|
467
|
+
log,
|
|
468
|
+
});
|
|
469
|
+
const ledger = await readDecisionLedger(root, options.programId);
|
|
470
|
+
const escalations = escalatedRecords(ledger);
|
|
424
471
|
const reportPath = runReportPath(root, options.programId);
|
|
425
|
-
|
|
426
|
-
//
|
|
427
|
-
//
|
|
428
|
-
//
|
|
429
|
-
//
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
472
|
+
// Every workstream verdict, `complete`, and the exit-code mapping are
|
|
473
|
+
// already settled above; writing and committing the report is bookkeeping
|
|
474
|
+
// that must not be able to reject `runProgram` after the fact (SC-05,
|
|
475
|
+
// SC-12) — a full disk or an EISDIR here is a logged line, not a thrown
|
|
476
|
+
// run.
|
|
477
|
+
try {
|
|
478
|
+
await writeFile(reportPath,
|
|
479
|
+
// `triaged` is the run-local set of subject ids the decider actually
|
|
480
|
+
// ruled on this run (built up across the authoring, build, and
|
|
481
|
+
// whole-program stages, see its declaration above) — exactly the
|
|
482
|
+
// `triagedThisRun` basis the "this run" triage ratio needs, since the
|
|
483
|
+
// projected ledger carries no run identifier of its own. It excludes
|
|
484
|
+
// ids that were merely sent but whose invocation failed or returned no
|
|
485
|
+
// valid verdict.
|
|
486
|
+
renderRunReport(manifest, results, ledger, triaged, authorResult, now(), wholeProgram), "utf8");
|
|
487
|
+
if (isRepository) {
|
|
488
|
+
await git.commitPaths(root, `nightshift(${options.programId}): run report and decision ledger`, ["docs/programs"]);
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
catch (error) {
|
|
492
|
+
log(`run report: could not write or commit ${reportPath}: ${error.message}`);
|
|
435
493
|
}
|
|
436
494
|
log(`run report: ${reportPath}`);
|
|
437
495
|
return {
|
|
@@ -440,6 +498,7 @@ export async function runProgram(options) {
|
|
|
440
498
|
workstreams: results,
|
|
441
499
|
escalations,
|
|
442
500
|
reportPath,
|
|
501
|
+
wholeProgramReview: wholeProgram,
|
|
443
502
|
};
|
|
444
503
|
async function runWorkstream(workstream) {
|
|
445
504
|
const base = {
|
|
@@ -466,15 +525,25 @@ export async function runProgram(options) {
|
|
|
466
525
|
? await git.currentCommit(root)
|
|
467
526
|
: undefined;
|
|
468
527
|
let priorFailure;
|
|
528
|
+
let priorDiagnosis;
|
|
529
|
+
let implementerFingerprint;
|
|
469
530
|
const attempts = [
|
|
470
531
|
{ agent, label: "implementer" },
|
|
471
532
|
];
|
|
472
533
|
if (recovery && !recovery.borrowedImplementer) {
|
|
473
534
|
attempts.push({ agent: recovery.agent, label: "recovery" });
|
|
535
|
+
// The third seat exists only when a reviewer can inform it. An
|
|
536
|
+
// uninformed retry has already been spent (recovery); running the
|
|
537
|
+
// roster again blind is a coin flip the ledger should hear about
|
|
538
|
+
// instead. Alternation is deliberate — the implementer returns with
|
|
539
|
+
// the reviewer's diagnosis in hand, a composition neither prior
|
|
540
|
+
// attempt had.
|
|
541
|
+
if (reviewer)
|
|
542
|
+
attempts.push({ agent, label: "informed retry" });
|
|
474
543
|
}
|
|
475
544
|
for (const [index, attempt] of attempts.entries()) {
|
|
476
545
|
log(`${workstream.id} ${workstream.name}: ${attempt.label} attempt`);
|
|
477
|
-
const brief = implementerBrief(manifest, workstream, spec, ledgerAtStart, priorFailure);
|
|
546
|
+
const brief = implementerBrief(manifest, workstream, spec, ledgerAtStart, priorFailure, priorDiagnosis);
|
|
478
547
|
const invocation = await invokeAgent(agentRunner, attempt.agent, brief, root);
|
|
479
548
|
const summary = resolveSummary(invocation.output);
|
|
480
549
|
base.summary = summary.text;
|
|
@@ -562,15 +631,57 @@ export async function runProgram(options) {
|
|
|
562
631
|
}
|
|
563
632
|
priorFailure = failure;
|
|
564
633
|
log(`${workstream.id}: ${attempt.label} attempt failed — ${failure}`);
|
|
565
|
-
|
|
634
|
+
// The informed retry is spent only when the diagnosed recovery attempt
|
|
635
|
+
// moved the failure at all. A failure reproduced identically after a
|
|
636
|
+
// diagnosis says the roster is stuck, not unlucky — that belongs in
|
|
637
|
+
// the ledger, not in a third spawn.
|
|
638
|
+
const stuck = attempt.label === "recovery" &&
|
|
639
|
+
index < attempts.length - 1 &&
|
|
640
|
+
implementerFingerprint !== undefined &&
|
|
641
|
+
failureFingerprint(failure) === implementerFingerprint;
|
|
642
|
+
if (index === attempts.length - 1 || stuck) {
|
|
566
643
|
workstream.status = "failed";
|
|
567
644
|
await saveManifest(root, options.programId, manifest);
|
|
568
645
|
// The work stays in the tree for a resume; decisions made on the way
|
|
569
646
|
// to a failure are still journaled and still reviewable.
|
|
570
647
|
await reviewWorkstreamDecisions(workstream.id, parsed.decisions, baseCommit);
|
|
571
|
-
base.outcome = {
|
|
648
|
+
base.outcome = {
|
|
649
|
+
status: "failed",
|
|
650
|
+
reason: stuck
|
|
651
|
+
? failure +
|
|
652
|
+
"\n\nThe recovery attempt, briefed with a reviewer diagnosis, " +
|
|
653
|
+
"reproduced the implementer's failure (identical up to counts " +
|
|
654
|
+
"and timings), so the final retry was not spent."
|
|
655
|
+
: failure,
|
|
656
|
+
};
|
|
572
657
|
return base;
|
|
573
658
|
}
|
|
659
|
+
if (attempt.label === "implementer") {
|
|
660
|
+
implementerFingerprint = failureFingerprint(failure);
|
|
661
|
+
}
|
|
662
|
+
// Diagnose the failure for the next attempt's brief. Overwrites any
|
|
663
|
+
// prior diagnosis — a read of an older failure must not be pinned to
|
|
664
|
+
// a newer one.
|
|
665
|
+
priorDiagnosis = reviewer
|
|
666
|
+
? await diagnoseFailure({
|
|
667
|
+
root,
|
|
668
|
+
manifest,
|
|
669
|
+
workstream,
|
|
670
|
+
spec,
|
|
671
|
+
agentRunner,
|
|
672
|
+
reviewer,
|
|
673
|
+
git,
|
|
674
|
+
baseCommit,
|
|
675
|
+
failure,
|
|
676
|
+
})
|
|
677
|
+
: undefined;
|
|
678
|
+
if (priorDiagnosis !== undefined) {
|
|
679
|
+
(base.failureDiagnoses ??= []).push({
|
|
680
|
+
attempt: attempt.label,
|
|
681
|
+
verdict: priorDiagnosis,
|
|
682
|
+
});
|
|
683
|
+
log(`${workstream.id}: reviewer diagnosed the ${attempt.label} failure`);
|
|
684
|
+
}
|
|
574
685
|
}
|
|
575
686
|
return base;
|
|
576
687
|
}
|
|
@@ -695,6 +806,192 @@ export async function runProgram(options) {
|
|
|
695
806
|
return kept ? fix.greenCommit : greenCommit;
|
|
696
807
|
}
|
|
697
808
|
}
|
|
809
|
+
/**
|
|
810
|
+
* Resolves the program's diff base for the whole-program review: the
|
|
811
|
+
* merge-base with the default branch, when it exists and differs from HEAD
|
|
812
|
+
* (the program's real branch point — the same range a pull request shows a
|
|
813
|
+
* human); otherwise `runStartCommit`, when it differs from HEAD (a
|
|
814
|
+
* `--force` run made directly on the default branch, where the merge-base is
|
|
815
|
+
* HEAD itself); otherwise `undefined` — an ordinary outcome, not a failure.
|
|
816
|
+
*/
|
|
817
|
+
export async function resolveProgramDiffBase(args) {
|
|
818
|
+
const { root, git, isRepository, runStartCommit } = args;
|
|
819
|
+
if (!isRepository)
|
|
820
|
+
return undefined;
|
|
821
|
+
const head = await git.currentCommit(root);
|
|
822
|
+
if (head === undefined)
|
|
823
|
+
return undefined;
|
|
824
|
+
const defaultBranch = await detectDefaultBranch(root);
|
|
825
|
+
const base = await git.mergeBase(root, defaultBranch, head);
|
|
826
|
+
if (base !== undefined && base !== head)
|
|
827
|
+
return base;
|
|
828
|
+
if (runStartCommit !== undefined && runStartCommit !== head)
|
|
829
|
+
return runStartCommit;
|
|
830
|
+
return undefined;
|
|
831
|
+
}
|
|
832
|
+
/** The synthetic workstream id whole-program findings are journaled under (SC-07). */
|
|
833
|
+
export const WHOLE_PROGRAM_SUBJECT = "whole-program";
|
|
834
|
+
function emptyWholeProgramSeverityCounts() {
|
|
835
|
+
return { blocker: 0, major: 0, minor: 0, advisory: 0 };
|
|
836
|
+
}
|
|
837
|
+
/** The placeholder outcome for the (practically unreachable) case where the stage's own body throws before the pass ever runs. */
|
|
838
|
+
function stageFailureOutcome(reason) {
|
|
839
|
+
return {
|
|
840
|
+
ran: false,
|
|
841
|
+
status: "reviewer-error",
|
|
842
|
+
refreshed: false,
|
|
843
|
+
writtenPaths: [],
|
|
844
|
+
findings: [],
|
|
845
|
+
errors: [],
|
|
846
|
+
severityCounts: emptyWholeProgramSeverityCounts(),
|
|
847
|
+
lengthOverrun: false,
|
|
848
|
+
missingLimitations: [],
|
|
849
|
+
inputClipped: false,
|
|
850
|
+
reason,
|
|
851
|
+
};
|
|
852
|
+
}
|
|
853
|
+
/** Failed and parked workstreams only (SC-08) — a `skipped` result means already complete, i.e. built. */
|
|
854
|
+
function notBuiltRoster(results, authorResult) {
|
|
855
|
+
const authorById = new Map(authorResult.results.map((entry) => [entry.id, entry]));
|
|
856
|
+
const notBuilt = [];
|
|
857
|
+
for (const result of results) {
|
|
858
|
+
const outcome = result.outcome;
|
|
859
|
+
if (outcome.status !== "failed" && outcome.status !== "parked")
|
|
860
|
+
continue;
|
|
861
|
+
const authorOutcome = authorById.get(result.id)?.outcome;
|
|
862
|
+
const reason = authorOutcome !== undefined &&
|
|
863
|
+
(authorOutcome.status === "failed" || authorOutcome.status === "parked")
|
|
864
|
+
? `${outcome.reason} (spec authoring: ${authorOutcome.reason})`
|
|
865
|
+
: outcome.reason;
|
|
866
|
+
notBuilt.push({
|
|
867
|
+
id: result.id,
|
|
868
|
+
name: result.name,
|
|
869
|
+
status: outcome.status,
|
|
870
|
+
reason,
|
|
871
|
+
});
|
|
872
|
+
}
|
|
873
|
+
return notBuilt;
|
|
874
|
+
}
|
|
875
|
+
/**
|
|
876
|
+
* The end-of-run whole-program review stage: establishes the diff base,
|
|
877
|
+
* calls WS-01's never-throwing pass exactly once, commits exactly what it
|
|
878
|
+
* wrote, and routes its findings through the existing findings path. Every
|
|
879
|
+
* risky step around the pass gets its own catch so a stage failure is a
|
|
880
|
+
* sentence in the report, never a change to the run's outcome (SC-05,
|
|
881
|
+
* SC-12) — see the outer catch below for the belt-and-braces case where a
|
|
882
|
+
* bug in this function's own body throws before the pass has even run.
|
|
883
|
+
*/
|
|
884
|
+
export async function runWholeProgramStage(args) {
|
|
885
|
+
const { root, programId, manifest, config, agentRunner, git, isRepository, results, authorResult, runStartCommit, decider, reviewed, triaged, now, log, fs, } = args;
|
|
886
|
+
const errors = [];
|
|
887
|
+
let commit;
|
|
888
|
+
let commitSkipped = false;
|
|
889
|
+
let findingIds = [];
|
|
890
|
+
let outcome = stageFailureOutcome("the whole-program review stage did not complete");
|
|
891
|
+
// The caller's log callback is itself a risky step — it is not this
|
|
892
|
+
// stage's own code and can throw (a full log file, a broken transport).
|
|
893
|
+
// Both of the stage's own lines go through this wrapper rather than `log`
|
|
894
|
+
// directly, so a throwing logger costs one sentence in `errors` and never
|
|
895
|
+
// aborts the commit or the findings routing that follow it.
|
|
896
|
+
const safeLog = (line) => {
|
|
897
|
+
try {
|
|
898
|
+
log(line);
|
|
899
|
+
}
|
|
900
|
+
catch (error) {
|
|
901
|
+
errors.push(`the whole-program review logger failed: ${error.message}`);
|
|
902
|
+
}
|
|
903
|
+
};
|
|
904
|
+
try {
|
|
905
|
+
safeLog("whole-program review: starting");
|
|
906
|
+
const base = await resolveProgramDiffBase({ root, git, isRepository, runStartCommit });
|
|
907
|
+
const diff = base === undefined ? "" : await git.diffSince(root, base);
|
|
908
|
+
const notBuilt = notBuiltRoster(results, authorResult);
|
|
909
|
+
const reviewCommit = isRepository ? await git.currentCommit(root) : undefined;
|
|
910
|
+
outcome = await runWholeProgramReview({
|
|
911
|
+
root,
|
|
912
|
+
programId,
|
|
913
|
+
manifest,
|
|
914
|
+
config,
|
|
915
|
+
agentRunner,
|
|
916
|
+
diff,
|
|
917
|
+
...(base === undefined ? {} : { baseCommit: base }),
|
|
918
|
+
...(notBuilt.length === 0 ? {} : { notBuilt }),
|
|
919
|
+
...(fs === undefined ? {} : { fs }),
|
|
920
|
+
log,
|
|
921
|
+
});
|
|
922
|
+
safeLog(outcome.status === "refreshed"
|
|
923
|
+
? `whole-program review: refreshed ${AS_BUILT_PATH}` +
|
|
924
|
+
(outcome.archivedTo ? ` (archived to ${outcome.archivedTo})` : "")
|
|
925
|
+
: `whole-program review: ${outcome.status}` +
|
|
926
|
+
(outcome.reason ? ` — ${outcome.reason}` : ""));
|
|
927
|
+
if (outcome.writtenPaths.length > 0) {
|
|
928
|
+
if (isRepository) {
|
|
929
|
+
try {
|
|
930
|
+
commit = await git.commitPaths(root, `nightshift(${programId}): as-built snapshot`, outcome.writtenPaths);
|
|
931
|
+
}
|
|
932
|
+
catch (error) {
|
|
933
|
+
errors.push(`could not commit the as-built snapshot: ${error.message}`);
|
|
934
|
+
}
|
|
935
|
+
}
|
|
936
|
+
else {
|
|
937
|
+
commitSkipped = true;
|
|
938
|
+
}
|
|
939
|
+
}
|
|
940
|
+
const events = findingsToLedgerEvents({
|
|
941
|
+
workstreamId: WHOLE_PROGRAM_SUBJECT,
|
|
942
|
+
findings: outcome.findings.filter(hasRoutableEvidence),
|
|
943
|
+
...(reviewCommit === undefined ? {} : { baseCommit: reviewCommit }),
|
|
944
|
+
now,
|
|
945
|
+
});
|
|
946
|
+
let journaled = false;
|
|
947
|
+
if (events.length === 0) {
|
|
948
|
+
journaled = true;
|
|
949
|
+
}
|
|
950
|
+
else {
|
|
951
|
+
try {
|
|
952
|
+
await appendLedgerEvents(root, programId, events);
|
|
953
|
+
journaled = true;
|
|
954
|
+
findingIds = events.map((event) => event.id);
|
|
955
|
+
}
|
|
956
|
+
catch (error) {
|
|
957
|
+
errors.push(`could not journal the whole-program findings: ${error.message}`);
|
|
958
|
+
}
|
|
959
|
+
}
|
|
960
|
+
if (journaled && events.length > 0) {
|
|
961
|
+
try {
|
|
962
|
+
await triageFindings({
|
|
963
|
+
root,
|
|
964
|
+
programId,
|
|
965
|
+
manifest,
|
|
966
|
+
workstreamId: WHOLE_PROGRAM_SUBJECT,
|
|
967
|
+
findings: events,
|
|
968
|
+
baseCommit: reviewCommit,
|
|
969
|
+
decider,
|
|
970
|
+
agentRunner,
|
|
971
|
+
git,
|
|
972
|
+
isRepository,
|
|
973
|
+
reviewed,
|
|
974
|
+
triaged,
|
|
975
|
+
now,
|
|
976
|
+
log,
|
|
977
|
+
});
|
|
978
|
+
}
|
|
979
|
+
catch (error) {
|
|
980
|
+
errors.push(`the decider failed to triage the whole-program findings: ${error.message}`);
|
|
981
|
+
}
|
|
982
|
+
}
|
|
983
|
+
}
|
|
984
|
+
catch (error) {
|
|
985
|
+
errors.push(`the whole-program review stage failed unexpectedly: ${error.message}`);
|
|
986
|
+
}
|
|
987
|
+
return {
|
|
988
|
+
outcome,
|
|
989
|
+
...(commit === undefined ? {} : { commit }),
|
|
990
|
+
commitSkipped,
|
|
991
|
+
findingIds,
|
|
992
|
+
errors,
|
|
993
|
+
};
|
|
994
|
+
}
|
|
698
995
|
/** Undefined means the attempt verified clean; otherwise the diagnosis. */
|
|
699
996
|
async function verifyAttempt(config, verifyRunner, root, agentExitCode) {
|
|
700
997
|
if (agentExitCode !== 0) {
|
|
@@ -730,6 +1027,73 @@ function clipForReview(text, label) {
|
|
|
730
1027
|
function hasFindingsBlock(output) {
|
|
731
1028
|
return /```findings/u.test(output);
|
|
732
1029
|
}
|
|
1030
|
+
/**
|
|
1031
|
+
* Two verify failures are "the same failure" when they differ only in
|
|
1032
|
+
* numbers — counts, timings, durations, and line offsets are the volatile
|
|
1033
|
+
* parts of test-runner output. Deterministic on purpose: whether a retry
|
|
1034
|
+
* changed anything is not a judgment call to hand a model.
|
|
1035
|
+
*/
|
|
1036
|
+
function failureFingerprint(failure) {
|
|
1037
|
+
return failure.replace(/\d+/gu, "#");
|
|
1038
|
+
}
|
|
1039
|
+
function failureDiagnosisBrief(manifest, workstream, spec, diff, failure) {
|
|
1040
|
+
return [
|
|
1041
|
+
`# Failure diagnosis: ${workstream.id} ${workstream.name}`,
|
|
1042
|
+
"",
|
|
1043
|
+
`Program: ${manifest.program.id} — ${manifest.program.name}`,
|
|
1044
|
+
"",
|
|
1045
|
+
"An implementation attempt for this workstream just failed verification.",
|
|
1046
|
+
"You wrote none of it. Before another attempt is spent, decide what is",
|
|
1047
|
+
"actually wrong: the tests, the implementation, or both. The attempt",
|
|
1048
|
+
"wrote its own tests, so a failing assertion is exactly as suspect as",
|
|
1049
|
+
"the code under it.",
|
|
1050
|
+
"",
|
|
1051
|
+
"Begin your reply with one line:",
|
|
1052
|
+
"",
|
|
1053
|
+
" Verdict: test-wrong | code-wrong | both | unclear",
|
|
1054
|
+
"",
|
|
1055
|
+
"then explain in at most three short paragraphs, quoting the specific",
|
|
1056
|
+
"assertion or code at fault. If a test asserts something the spec never",
|
|
1057
|
+
"promises — a literal string that reads like a description of a",
|
|
1058
|
+
"requirement rather than output the code could produce — say so plainly",
|
|
1059
|
+
"and name the line. Do not modify any file; your reply is the only",
|
|
1060
|
+
"artifact.",
|
|
1061
|
+
"",
|
|
1062
|
+
"## Specification",
|
|
1063
|
+
"",
|
|
1064
|
+
spec.trim(),
|
|
1065
|
+
"",
|
|
1066
|
+
"## Diff of the failing attempt (uncommitted, against the pre-workstream tree)",
|
|
1067
|
+
"",
|
|
1068
|
+
"```diff",
|
|
1069
|
+
diff,
|
|
1070
|
+
"```",
|
|
1071
|
+
"",
|
|
1072
|
+
"## Verify output",
|
|
1073
|
+
"",
|
|
1074
|
+
failure,
|
|
1075
|
+
"",
|
|
1076
|
+
].join("\n");
|
|
1077
|
+
}
|
|
1078
|
+
/**
|
|
1079
|
+
* The test-critique reviewer's read of a *failing* tree, taken before a
|
|
1080
|
+
* retry is spent. The post-green critique can only ever see tests that
|
|
1081
|
+
* already passed, so a wrong assertion written by the implementer is
|
|
1082
|
+
* invisible to every party on the red path: both attempts treat the test as
|
|
1083
|
+
* ground truth and try to satisfy it. This read is what makes a retry
|
|
1084
|
+
* informed rather than blind. Fails open: any reviewer misbehavior returns
|
|
1085
|
+
* undefined and the retry proceeds on the raw verify output alone.
|
|
1086
|
+
*/
|
|
1087
|
+
async function diagnoseFailure(options) {
|
|
1088
|
+
const { root, manifest, workstream, spec, agentRunner, reviewer, git, baseCommit, failure } = options;
|
|
1089
|
+
const rawDiff = baseCommit !== undefined ? await git.diffSince(root, baseCommit) : "";
|
|
1090
|
+
const brief = failureDiagnosisBrief(manifest, workstream, clipForReview(spec, "spec").text, clipForReview(rawDiff, "diff").text, failure);
|
|
1091
|
+
const invocation = await invokeAgent(agentRunner, reviewer, brief, root);
|
|
1092
|
+
if (invocation.exitCode !== 0)
|
|
1093
|
+
return undefined;
|
|
1094
|
+
const text = invocation.output.trim();
|
|
1095
|
+
return text === "" ? undefined : tail(text, 2000);
|
|
1096
|
+
}
|
|
733
1097
|
function successCriteriaLines(manifest) {
|
|
734
1098
|
if (manifest.successCriteria.length === 0) {
|
|
735
1099
|
return ["None recorded in the manifest."];
|
|
@@ -1039,36 +1403,36 @@ function renderRawFinding(finding, workstreamLabel, sinceFixed) {
|
|
|
1039
1403
|
return `- **${finding.subject}** (${finding.severity}, ${workstreamLabel}) — ${finding.message}${suffix}`;
|
|
1040
1404
|
}
|
|
1041
1405
|
/**
|
|
1042
|
-
*
|
|
1043
|
-
*
|
|
1044
|
-
*
|
|
1045
|
-
* WS-02's own dedup ever let that happen) is not double-listed.
|
|
1406
|
+
* Minor/advisory findings from one list, deduped by subject/message/location.
|
|
1407
|
+
* Shared by `collectPassLimitations` (a pass's `open` and `resolved` findings)
|
|
1408
|
+
* and the whole-program review's flat `Finding[]`.
|
|
1046
1409
|
*/
|
|
1047
|
-
function
|
|
1048
|
-
if (!outcome)
|
|
1049
|
-
return [];
|
|
1410
|
+
function collectLimitationLines(findings, label, sinceFixed = false) {
|
|
1050
1411
|
const seen = new Set();
|
|
1051
1412
|
const lines = [];
|
|
1052
|
-
for (const finding of
|
|
1053
|
-
if (finding.severity !== "minor" && finding.severity !== "advisory")
|
|
1054
|
-
continue;
|
|
1055
|
-
const key = limitationKey(finding);
|
|
1056
|
-
if (seen.has(key))
|
|
1057
|
-
continue;
|
|
1058
|
-
seen.add(key);
|
|
1059
|
-
lines.push(renderRawFinding(finding, workstreamLabel, false));
|
|
1060
|
-
}
|
|
1061
|
-
for (const finding of outcome.resolved) {
|
|
1413
|
+
for (const finding of findings) {
|
|
1062
1414
|
if (finding.severity !== "minor" && finding.severity !== "advisory")
|
|
1063
1415
|
continue;
|
|
1064
1416
|
const key = limitationKey(finding);
|
|
1065
1417
|
if (seen.has(key))
|
|
1066
1418
|
continue;
|
|
1067
1419
|
seen.add(key);
|
|
1068
|
-
lines.push(renderRawFinding(finding,
|
|
1420
|
+
lines.push(renderRawFinding(finding, label, sinceFixed));
|
|
1069
1421
|
}
|
|
1070
1422
|
return lines;
|
|
1071
1423
|
}
|
|
1424
|
+
/**
|
|
1425
|
+
* A pass's minor/advisory findings, `open` and `resolved` alike (SC-07's
|
|
1426
|
+
* "every ... raised" — a finding the writer fixed was still raised).
|
|
1427
|
+
*/
|
|
1428
|
+
function collectPassLimitations(outcome, workstreamLabel) {
|
|
1429
|
+
if (!outcome)
|
|
1430
|
+
return [];
|
|
1431
|
+
return [
|
|
1432
|
+
...collectLimitationLines(outcome.open, workstreamLabel),
|
|
1433
|
+
...collectLimitationLines(outcome.resolved, workstreamLabel, true),
|
|
1434
|
+
];
|
|
1435
|
+
}
|
|
1072
1436
|
/**
|
|
1073
1437
|
* One aggregated section (SC-07): every minor/advisory finding raised
|
|
1074
1438
|
* anywhere in the run, every finding the decider accepted, any finding
|
|
@@ -1076,11 +1440,14 @@ function collectPassLimitations(outcome, workstreamLabel) {
|
|
|
1076
1440
|
* every fix-now attempt, and the run-local triage ratio. Fail-open
|
|
1077
1441
|
* throughout — an empty run renders a plain statement, never nothing.
|
|
1078
1442
|
*/
|
|
1079
|
-
function renderKnownLimitations(results, authorResult, ledger, escalated, triagedThisRun, programId) {
|
|
1443
|
+
function renderKnownLimitations(results, authorResult, ledger, escalated, triagedThisRun, programId, wholeProgram) {
|
|
1080
1444
|
const lines = ["## Known limitations", ""];
|
|
1081
1445
|
const passLimitations = [
|
|
1082
1446
|
...results.flatMap((result) => collectPassLimitations(result.testCritique, `${result.id} test critique`)),
|
|
1083
1447
|
...authorResult.results.flatMap((entry) => collectPassLimitations(entry.specCritique, `${entry.id} spec critique`)),
|
|
1448
|
+
...(wholeProgram
|
|
1449
|
+
? collectLimitationLines(wholeProgram.outcome.findings, "whole-program review")
|
|
1450
|
+
: []),
|
|
1084
1451
|
];
|
|
1085
1452
|
const acceptedFindings = ledger.findings.filter((record) => record.status === "accepted");
|
|
1086
1453
|
const openFindings = ledger.findings.filter((record) => record.status === "open");
|
|
@@ -1125,6 +1492,10 @@ function renderKnownLimitations(results, authorResult, ledger, escalated, triage
|
|
|
1125
1492
|
: "";
|
|
1126
1493
|
lines.push(`- **${record.subject}** — fixed in the run${commitPart}: ${record.fixAttempt.note}`, "");
|
|
1127
1494
|
}
|
|
1495
|
+
else if (record.workstream === WHOLE_PROGRAM_SUBJECT) {
|
|
1496
|
+
lines.push(`- **${record.subject}** — from the whole-program review, triaged ` +
|
|
1497
|
+
"`fix-now`: carried to the next run; nothing was fixed in this one.", "");
|
|
1498
|
+
}
|
|
1128
1499
|
else {
|
|
1129
1500
|
lines.push(`- **${record.subject}** — fix-now triaged; outcome pending.`, "");
|
|
1130
1501
|
}
|
|
@@ -1136,7 +1507,54 @@ function renderKnownLimitations(results, authorResult, ledger, escalated, triage
|
|
|
1136
1507
|
`${escalatedThisRunCount} ${escalatedThisRunCount === 1 ? "was" : "were"} escalated.`, "");
|
|
1137
1508
|
return lines;
|
|
1138
1509
|
}
|
|
1139
|
-
|
|
1510
|
+
/** The commit fact WS-01's outcome cannot know — sha and paths, or why there is none. */
|
|
1511
|
+
function commitFactLines(wholeProgram) {
|
|
1512
|
+
const { commit, commitSkipped, outcome } = wholeProgram;
|
|
1513
|
+
if (commit !== undefined) {
|
|
1514
|
+
return [`Committed as \`${commit}\`: ${outcome.writtenPaths.join(", ")}.`];
|
|
1515
|
+
}
|
|
1516
|
+
if (outcome.writtenPaths.length === 0) {
|
|
1517
|
+
return ["No commit — nothing was written."];
|
|
1518
|
+
}
|
|
1519
|
+
if (commitSkipped) {
|
|
1520
|
+
return [
|
|
1521
|
+
"Written to the working tree but not committed: this is not a git repository.",
|
|
1522
|
+
];
|
|
1523
|
+
}
|
|
1524
|
+
return ["The snapshot was written, but committing it failed — see the stage error below."];
|
|
1525
|
+
}
|
|
1526
|
+
/**
|
|
1527
|
+
* A whole-program finding this stage journaled that the decider triaged
|
|
1528
|
+
* `fix-now` this run. Keys on `findingIds` (this run's own journaling), not
|
|
1529
|
+
* on `record.workstream` — that broader statement is §3.7's, in
|
|
1530
|
+
* `renderKnownLimitations`, and applies across runs.
|
|
1531
|
+
*/
|
|
1532
|
+
function carriedFixNowLines(wholeProgram, ledger) {
|
|
1533
|
+
if (wholeProgram.findingIds.length === 0)
|
|
1534
|
+
return [];
|
|
1535
|
+
const ids = new Set(wholeProgram.findingIds);
|
|
1536
|
+
const carried = ledger.findings.filter((record) => ids.has(record.id) && record.status === "fix-now");
|
|
1537
|
+
if (carried.length === 0)
|
|
1538
|
+
return [];
|
|
1539
|
+
return carried.map((record) => `Triaged \`fix-now\`: **${record.subject}** — carried to the next run; nothing was fixed in this one.`);
|
|
1540
|
+
}
|
|
1541
|
+
/**
|
|
1542
|
+
* The `## Whole-program review` section: delegates its body to WS-01's own
|
|
1543
|
+
* renderer verbatim, and adds only what that module cannot know — the commit
|
|
1544
|
+
* fact, this run's carried fix-now findings, and this stage's own errors.
|
|
1545
|
+
*/
|
|
1546
|
+
function renderWholeProgramSection(wholeProgram, ledger) {
|
|
1547
|
+
const lines = ["## Whole-program review", ""];
|
|
1548
|
+
lines.push(...renderWholeProgramReview(wholeProgram.outcome));
|
|
1549
|
+
lines.push(...commitFactLines(wholeProgram));
|
|
1550
|
+
lines.push(...carriedFixNowLines(wholeProgram, ledger));
|
|
1551
|
+
if (wholeProgram.errors.length > 0) {
|
|
1552
|
+
lines.push(...wholeProgram.errors.map((error) => `Stage error: ${error}`));
|
|
1553
|
+
}
|
|
1554
|
+
lines.push("");
|
|
1555
|
+
return lines;
|
|
1556
|
+
}
|
|
1557
|
+
export function renderRunReport(manifest, results, ledger, triagedThisRun, authorResult, at, wholeProgram) {
|
|
1140
1558
|
const programId = manifest.program.id;
|
|
1141
1559
|
const built = results.filter((result) => result.outcome.status === "complete" ||
|
|
1142
1560
|
result.outcome.status === "skipped").length;
|
|
@@ -1168,7 +1586,11 @@ export function renderRunReport(manifest, results, ledger, triagedThisRun, autho
|
|
|
1168
1586
|
const reason = result.outcome.status === "failed" || result.outcome.status === "parked"
|
|
1169
1587
|
? result.outcome.reason
|
|
1170
1588
|
: "";
|
|
1171
|
-
lines.push(`### ${result.id} ${result.name} — ${result.outcome.status}`, "", reason, ""
|
|
1589
|
+
lines.push(`### ${result.id} ${result.name} — ${result.outcome.status}`, "", reason, "");
|
|
1590
|
+
for (const diagnosis of result.failureDiagnoses ?? []) {
|
|
1591
|
+
lines.push(`**Reviewer diagnosis (after the ${diagnosis.attempt} attempt):**`, "", diagnosis.verdict, "");
|
|
1592
|
+
}
|
|
1593
|
+
lines.push(`Retry after fixing: \`npx --yes @wildorder/nightshift run ${manifest.program.id}\``, "(completed workstreams are skipped automatically).", "");
|
|
1172
1594
|
}
|
|
1173
1595
|
}
|
|
1174
1596
|
lines.push("## Workstreams", "");
|
|
@@ -1177,6 +1599,13 @@ export function renderRunReport(manifest, results, ledger, triagedThisRun, autho
|
|
|
1177
1599
|
if (result.summary) {
|
|
1178
1600
|
lines.push(` - ${result.summary.replace(/\s+/gu, " ").trim()}`);
|
|
1179
1601
|
}
|
|
1602
|
+
if (result.outcome.status === "complete") {
|
|
1603
|
+
for (const diagnosis of result.failureDiagnoses ?? []) {
|
|
1604
|
+
const oneLine = diagnosis.verdict.replace(/\s+/gu, " ").trim();
|
|
1605
|
+
const head = oneLine.length > 300 ? `${oneLine.slice(0, 300)}…` : oneLine;
|
|
1606
|
+
lines.push(` - Recovered: the ${diagnosis.attempt} attempt failed and the reviewer diagnosed it before a later attempt went green — ${head}`);
|
|
1607
|
+
}
|
|
1608
|
+
}
|
|
1180
1609
|
if (result.testCritique) {
|
|
1181
1610
|
for (const line of renderPassReport("Test critique", result.testCritique)) {
|
|
1182
1611
|
lines.push(line === "" ? "" : ` ${line}`);
|
|
@@ -1187,6 +1616,9 @@ export function renderRunReport(manifest, results, ledger, triagedThisRun, autho
|
|
|
1187
1616
|
}
|
|
1188
1617
|
}
|
|
1189
1618
|
lines.push("");
|
|
1619
|
+
if (wholeProgram !== undefined) {
|
|
1620
|
+
lines.push(...renderWholeProgramSection(wholeProgram, ledger));
|
|
1621
|
+
}
|
|
1190
1622
|
const settledDecisions = ledger.decisions.filter((record) => record.status !== "escalated");
|
|
1191
1623
|
lines.push("## Decisions made along the way", "", "Every judgment call an agent surfaced, with its review status.", "Anything here can be revisited: roll back to the anchor commit and", "re-run, or just say which option you want changed.", "");
|
|
1192
1624
|
if (settledDecisions.length === 0) {
|
|
@@ -1197,7 +1629,7 @@ export function renderRunReport(manifest, results, ledger, triagedThisRun, autho
|
|
|
1197
1629
|
lines.push(...renderRecord(record, { density: "compact", programId }), "");
|
|
1198
1630
|
}
|
|
1199
1631
|
}
|
|
1200
|
-
lines.push(...renderKnownLimitations(results, authorResult, ledger, escalated, triagedThisRun, programId));
|
|
1632
|
+
lines.push(...renderKnownLimitations(results, authorResult, ledger, escalated, triagedThisRun, programId, wholeProgram));
|
|
1201
1633
|
const decisionErrors = results.flatMap((result) => result.decisionErrors);
|
|
1202
1634
|
if (decisionErrors.length > 0) {
|
|
1203
1635
|
lines.push("## Decision blocks the runner could not read", "", ...decisionErrors.map((error) => `- ${error}`), "");
|