@wildorder/nightshift 0.16.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +289 -1
- package/dist/agent-runner.d.ts +95 -5
- package/dist/agent-runner.d.ts.map +1 -1
- package/dist/agent-runner.js +442 -32
- package/dist/agent-runner.js.map +1 -1
- package/dist/agent-session.d.ts +123 -0
- package/dist/agent-session.d.ts.map +1 -0
- package/dist/agent-session.js +114 -0
- package/dist/agent-session.js.map +1 -0
- package/dist/atomic-write.d.ts +18 -0
- package/dist/atomic-write.d.ts.map +1 -0
- package/dist/atomic-write.js +74 -0
- package/dist/atomic-write.js.map +1 -0
- package/dist/author.d.ts +118 -4
- package/dist/author.d.ts.map +1 -1
- package/dist/author.js +650 -163
- package/dist/author.js.map +1 -1
- package/dist/causal-analysis.d.ts +216 -0
- package/dist/causal-analysis.d.ts.map +1 -0
- package/dist/causal-analysis.js +786 -0
- package/dist/causal-analysis.js.map +1 -0
- package/dist/cli.js +22 -2
- package/dist/cli.js.map +1 -1
- package/dist/concurrency.d.ts +24 -0
- package/dist/concurrency.d.ts.map +1 -0
- package/dist/concurrency.js +28 -0
- package/dist/concurrency.js.map +1 -0
- package/dist/decider-review.d.ts +13 -1
- package/dist/decider-review.d.ts.map +1 -1
- package/dist/decider-review.js +21 -7
- package/dist/decider-review.js.map +1 -1
- package/dist/decision-ledger.d.ts +7 -0
- package/dist/decision-ledger.d.ts.map +1 -1
- package/dist/decision-ledger.js +25 -10
- package/dist/decision-ledger.js.map +1 -1
- package/dist/detect-package-manager.d.ts +10 -0
- package/dist/detect-package-manager.d.ts.map +1 -1
- package/dist/detect-package-manager.js +12 -0
- package/dist/detect-package-manager.js.map +1 -1
- package/dist/index.d.ts +8 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -1
- package/dist/index.js.map +1 -1
- package/dist/manifest.d.ts +24 -5
- package/dist/manifest.d.ts.map +1 -1
- package/dist/manifest.js +254 -108
- package/dist/manifest.js.map +1 -1
- package/dist/permits.d.ts +18 -0
- package/dist/permits.d.ts.map +1 -1
- package/dist/permits.js +155 -0
- package/dist/permits.js.map +1 -1
- package/dist/preflight.d.ts +3 -0
- package/dist/preflight.d.ts.map +1 -1
- package/dist/preflight.js +73 -57
- package/dist/preflight.js.map +1 -1
- package/dist/prompt-telemetry.d.ts +64 -0
- package/dist/prompt-telemetry.d.ts.map +1 -0
- package/dist/prompt-telemetry.js +112 -0
- package/dist/prompt-telemetry.js.map +1 -0
- package/dist/provider-telemetry.d.ts +106 -0
- package/dist/provider-telemetry.d.ts.map +1 -0
- package/dist/provider-telemetry.js +423 -0
- package/dist/provider-telemetry.js.map +1 -0
- package/dist/review-pass.d.ts +52 -10
- package/dist/review-pass.d.ts.map +1 -1
- package/dist/review-pass.js +114 -30
- package/dist/review-pass.js.map +1 -1
- package/dist/run-analytics-report.d.ts +286 -0
- package/dist/run-analytics-report.d.ts.map +1 -0
- package/dist/run-analytics-report.js +1242 -0
- package/dist/run-analytics-report.js.map +1 -0
- package/dist/run-analytics.d.ts +844 -0
- package/dist/run-analytics.d.ts.map +1 -0
- package/dist/run-analytics.js +777 -0
- package/dist/run-analytics.js.map +1 -0
- package/dist/run-log.d.ts +16 -0
- package/dist/run-log.d.ts.map +1 -0
- package/dist/run-log.js +32 -0
- package/dist/run-log.js.map +1 -0
- package/dist/run-program.d.ts +268 -4
- package/dist/run-program.d.ts.map +1 -1
- package/dist/run-program.js +2508 -762
- package/dist/run-program.js.map +1 -1
- package/dist/scheduler.d.ts +75 -0
- package/dist/scheduler.d.ts.map +1 -0
- package/dist/scheduler.js +126 -0
- package/dist/scheduler.js.map +1 -0
- package/dist/verify-cache.d.ts +64 -0
- package/dist/verify-cache.d.ts.map +1 -0
- package/dist/verify-cache.js +56 -0
- package/dist/verify-cache.js.map +1 -0
- package/dist/whole-program-review.d.ts +3 -0
- package/dist/whole-program-review.d.ts.map +1 -1
- package/dist/whole-program-review.js +8 -1
- package/dist/whole-program-review.js.map +1 -1
- package/dist/worktree.d.ts +124 -0
- package/dist/worktree.d.ts.map +1 -0
- package/dist/worktree.js +397 -0
- package/dist/worktree.js.map +1 -0
- package/dist/write-queue.d.ts +25 -0
- package/dist/write-queue.d.ts.map +1 -0
- package/dist/write-queue.js +52 -0
- package/dist/write-queue.js.map +1 -0
- package/package.json +2 -2
- package/skills/plan-program/SKILL.md +37 -5
- package/dist/worktree-guard.d.ts +0 -40
- package/dist/worktree-guard.d.ts.map +0 -1
- package/dist/worktree-guard.js +0 -91
- package/dist/worktree-guard.js.map +0 -1
package/dist/run-program.js
CHANGED
|
@@ -1,26 +1,46 @@
|
|
|
1
1
|
import { execFile } from "node:child_process";
|
|
2
|
-
import {
|
|
2
|
+
import { randomBytes } from "node:crypto";
|
|
3
|
+
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
3
5
|
import { join, resolve } from "node:path";
|
|
4
6
|
import { promisify } from "node:util";
|
|
5
7
|
import { defaultAgentRunner, defaultVerifyRunner, describeAgent, invokeAgent, resolveAgent, resolveAuthorAgent, resolveDeciderAgent, resolveRecoveryAgent, resolveReviewerAgent, tail, } from "./agent-runner.js";
|
|
8
|
+
import { describeSessionDecline, spawnWithSessionContinuity, SESSION_CONTINUATION_PREAMBLE, } from "./agent-session.js";
|
|
6
9
|
import { resolveSummary, summaryContract } from "./agent-summary.js";
|
|
7
|
-
import {
|
|
10
|
+
import { prepareAuthoring } from "./author.js";
|
|
8
11
|
import { decisionContract, decisionFingerprint, extractDecisions, } from "./decision.js";
|
|
9
12
|
import { appendLedgerEvents, denialFingerprint, readDecisionLedger, } from "./decision-ledger.js";
|
|
10
13
|
import { escalatedRecords, renderDenial, renderRecord, } from "./decision-view.js";
|
|
11
14
|
import { reviewDecisions, triageFindings } from "./decider-review.js";
|
|
12
15
|
import { fingerprint } from "./findings.js";
|
|
13
16
|
import { findCycles, stableTopologicalOrder } from "./graph.js";
|
|
14
|
-
import { extractFindings, findingsContract, findingsToLedgerEvents, hasRoutableEvidence, locateInRepo, renderPassReport, reviewerAbsentOutcome, runReviewPass, verifyEvidence, } from "./review-pass.js";
|
|
17
|
+
import { extractFindings, findingsContract, findingsToLedgerEvents, hasRoutableEvidence, locateInRepo, renderPassReport, reviewerAbsentOutcome, runReviewPass, verifyEvidence, TEST_CRITIQUE_CYCLES, TEST_CRITIQUE_CYCLE_NOTE, } from "./review-pass.js";
|
|
18
|
+
import { createVerifyCache, runnerOwnedVerifyPaths } from "./verify-cache.js";
|
|
15
19
|
import { loadManifest, manifestMergeCount, saveManifest, } from "./manifest.js";
|
|
16
20
|
import { detectDefaultBranch, programBranchName } from "./program-branch.js";
|
|
17
21
|
import { defaultPrerequisiteRunner, runPreflight, } from "./preflight.js";
|
|
18
|
-
import { restoreProgramsDir, snapshotProgramsDir } from "./programs-dir.js";
|
|
19
22
|
import { buildPermitsContext } from "./permits.js";
|
|
20
23
|
import { CouldNotStartError } from "./exit-codes.js";
|
|
24
|
+
import { clippedInputPoint, createBriefBuilder, promptComponentSizePoints, } from "./prompt-telemetry.js";
|
|
25
|
+
import { NOOP_RUN_RECORDER, readPriorRunId, recordWorkstreamReady, recordWorkstreamScheduled, recordWorkstreamWaiting, RunRecorder, } from "./run-analytics.js";
|
|
26
|
+
import { renderAnalyticsSection } from "./run-analytics-report.js";
|
|
27
|
+
import { drainProgramWrites, programWriteKey } from "./write-queue.js";
|
|
28
|
+
import { schedulerLog, workstreamLog } from "./run-log.js";
|
|
29
|
+
import { createMutex } from "./concurrency.js";
|
|
30
|
+
import { normalizeMaxParallel, runReadySet } from "./scheduler.js";
|
|
31
|
+
import { defaultWorktreeOps, isRunnerOwnedArtifactPath, resolveWorktreeBase, retainedRefName, worktreeBranchName, worktreeBranchPrefix, worktreeInstallCommand, worktreePathFor, } from "./worktree.js";
|
|
21
32
|
import { runReportPath } from "./report-path.js";
|
|
22
33
|
import { AS_BUILT_PATH, runWholeProgramReview, renderWholeProgramReview, } from "./whole-program-review.js";
|
|
34
|
+
import { runCausalAnalysis, renderCausalAnalysisSection, } from "./causal-analysis.js";
|
|
23
35
|
const execFileAsync = promisify(execFile);
|
|
36
|
+
/** The attempt-loop seat label to its stage (WS-02) — the role-derived
|
|
37
|
+
* `STAGE_FOR_ROLE` default in `agent-runner.ts` cannot distinguish these
|
|
38
|
+
* three, since `agent` drives both `implementer` and `informed-retry`. */
|
|
39
|
+
const STAGE_FOR_SEAT = {
|
|
40
|
+
implementer: "implementer",
|
|
41
|
+
recovery: "recovery",
|
|
42
|
+
"informed retry": "informed-retry",
|
|
43
|
+
};
|
|
24
44
|
/** Matches every wording git uses to report an empty commit attempt. */
|
|
25
45
|
const NOTHING_TO_COMMIT = /nothing to commit|nothing added to commit|no changes added to commit/u;
|
|
26
46
|
/**
|
|
@@ -133,8 +153,13 @@ export const defaultGitOps = {
|
|
|
133
153
|
return undefined;
|
|
134
154
|
}
|
|
135
155
|
},
|
|
136
|
-
async commitAll(cwd, message) {
|
|
137
|
-
|
|
156
|
+
async commitAll(cwd, message, excludePaths) {
|
|
157
|
+
if (excludePaths && excludePaths.length > 0) {
|
|
158
|
+
await execFileAsync("git", ["add", "-A", "--", ":(top)", ...excludePaths.map((path) => `:(top,exclude)${path}`)], { cwd });
|
|
159
|
+
}
|
|
160
|
+
else {
|
|
161
|
+
await execFileAsync("git", ["add", "-A"], { cwd });
|
|
162
|
+
}
|
|
138
163
|
try {
|
|
139
164
|
await execFileAsync("git", ["commit", "-m", message], { cwd });
|
|
140
165
|
}
|
|
@@ -151,7 +176,12 @@ export const defaultGitOps = {
|
|
|
151
176
|
async commitPaths(cwd, message, paths) {
|
|
152
177
|
await execFileAsync("git", ["add", "--", ...paths], { cwd });
|
|
153
178
|
try {
|
|
154
|
-
|
|
179
|
+
// The trailing pathspec is load-bearing, not decoration: without it
|
|
180
|
+
// this commits the *whole* index, sweeping in anything else staged in
|
|
181
|
+
// `cwd` at the time (an unrelated human `git add`, or the runner's own
|
|
182
|
+
// residue from a prior failed commit) — the same hazard squashIntegrate
|
|
183
|
+
// guards against for the workstream integration commit (WS-02 §2, §3.5).
|
|
184
|
+
await execFileAsync("git", ["commit", "-m", message, "--", ...paths], { cwd });
|
|
155
185
|
}
|
|
156
186
|
catch (error) {
|
|
157
187
|
const output = String(error.stdout ?? "");
|
|
@@ -168,7 +198,10 @@ export const defaultGitOps = {
|
|
|
168
198
|
},
|
|
169
199
|
async diffSince(cwd, commit) {
|
|
170
200
|
try {
|
|
171
|
-
|
|
201
|
+
// `:(top)`-anchored pathspecs are repo-root-relative regardless of
|
|
202
|
+
// `cwd` — in a worktree, "repo root" is the worktree root, exactly the
|
|
203
|
+
// tree a reviewer or decider spawned there should see (WS-02).
|
|
204
|
+
const { stdout } = await execFileAsync("git", ["diff", commit, "--", ":(top)", ":(top,exclude)docs/programs"], { cwd, maxBuffer: 10 * 1024 * 1024 });
|
|
172
205
|
return stdout;
|
|
173
206
|
}
|
|
174
207
|
catch {
|
|
@@ -177,11 +210,20 @@ export const defaultGitOps = {
|
|
|
177
210
|
},
|
|
178
211
|
async dirtyPaths(cwd) {
|
|
179
212
|
try {
|
|
180
|
-
const { stdout } = await execFileAsync("git", ["status", "--porcelain"], { cwd });
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
213
|
+
const { stdout } = await execFileAsync("git", ["status", "--porcelain", "-z"], { cwd });
|
|
214
|
+
// NUL-delimited so quoted and non-ASCII names come back verbatim and
|
|
215
|
+
// compare equal to `changedPaths`' `-z` output. A rename or copy entry
|
|
216
|
+
// carries its original path as a second token, which is skipped.
|
|
217
|
+
const tokens = stdout.split("\0").filter((token) => token !== "");
|
|
218
|
+
const paths = [];
|
|
219
|
+
for (let index = 0; index < tokens.length; index += 1) {
|
|
220
|
+
const token = tokens[index];
|
|
221
|
+
const code = token.slice(0, 2);
|
|
222
|
+
paths.push(token.slice(3));
|
|
223
|
+
if (code.includes("R") || code.includes("C"))
|
|
224
|
+
index += 1;
|
|
225
|
+
}
|
|
226
|
+
return paths;
|
|
185
227
|
}
|
|
186
228
|
catch {
|
|
187
229
|
return [];
|
|
@@ -223,7 +265,76 @@ export const defaultGitOps = {
|
|
|
223
265
|
async createRef(cwd, ref, commit) {
|
|
224
266
|
await execFileAsync("git", ["update-ref", ref, commit], { cwd });
|
|
225
267
|
},
|
|
268
|
+
async deleteRef(cwd, ref) {
|
|
269
|
+
try {
|
|
270
|
+
await execFileAsync("git", ["update-ref", "-d", ref], { cwd });
|
|
271
|
+
}
|
|
272
|
+
catch {
|
|
273
|
+
// Already absent — nothing to delete.
|
|
274
|
+
}
|
|
275
|
+
},
|
|
276
|
+
async treeHash(cwd, exclude) {
|
|
277
|
+
let tmpDir;
|
|
278
|
+
try {
|
|
279
|
+
tmpDir = await mkdtemp(join(tmpdir(), "nightshift-verify-cache-"));
|
|
280
|
+
const indexFile = join(tmpDir, "index");
|
|
281
|
+
const env = { ...process.env, GIT_INDEX_FILE: indexFile };
|
|
282
|
+
await execFileAsync("git", ["read-tree", "HEAD"], { cwd, env });
|
|
283
|
+
await execFileAsync("git", ["add", "-A", "--", ":/"], { cwd, env });
|
|
284
|
+
if (exclude.length > 0) {
|
|
285
|
+
await execFileAsync("git", ["rm", "--cached", "-q", "--ignore-unmatch", "--", ...exclude], { cwd, env });
|
|
286
|
+
}
|
|
287
|
+
const { stdout } = await execFileAsync("git", ["write-tree"], { cwd, env });
|
|
288
|
+
return stdout.trim();
|
|
289
|
+
}
|
|
290
|
+
catch {
|
|
291
|
+
return undefined;
|
|
292
|
+
}
|
|
293
|
+
finally {
|
|
294
|
+
if (tmpDir !== undefined) {
|
|
295
|
+
try {
|
|
296
|
+
await rm(tmpDir, { recursive: true, force: true });
|
|
297
|
+
}
|
|
298
|
+
catch {
|
|
299
|
+
// Best-effort cleanup; a leftover temp dir is not a correctness
|
|
300
|
+
// issue and must not mask the hash result above.
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
},
|
|
226
305
|
};
|
|
306
|
+
/**
|
|
307
|
+
* Fail-open span wrapper for a runner-owned git-and-persistence operation
|
|
308
|
+
* (WS-02): opens a span, awaits `fn`, closes it `success` on completion or
|
|
309
|
+
* `interrupted` on a throw (re-thrown unchanged, via `finally`) — this
|
|
310
|
+
* module's stage functions already guard the *run* error a wrapped call can
|
|
311
|
+
* raise, so this wrapper only ever adds an observation, never changes what
|
|
312
|
+
* the caller sees. `evidenceFor`, when given, derives an evidence reference
|
|
313
|
+
* from the successful result (e.g. a commit sha) rather than every call
|
|
314
|
+
* needing to know the schema's evidence-kind vocabulary itself. Shared by
|
|
315
|
+
* `run-program.ts`'s own git/manifest/ledger call sites and imported by
|
|
316
|
+
* `author.ts` and `decider-review.ts` for theirs, so every runner-owned
|
|
317
|
+
* commit, large diff, and manifest/ledger persist is instrumented through
|
|
318
|
+
* one implementation.
|
|
319
|
+
*/
|
|
320
|
+
export async function timed(recorder, dims, fn, evidenceFor) {
|
|
321
|
+
const span = recorder.span(dims);
|
|
322
|
+
let result;
|
|
323
|
+
try {
|
|
324
|
+
result = await fn();
|
|
325
|
+
}
|
|
326
|
+
catch (error) {
|
|
327
|
+
span.close({ outcome: "interrupted" });
|
|
328
|
+
throw error;
|
|
329
|
+
}
|
|
330
|
+
const evidence = evidenceFor?.(result);
|
|
331
|
+
span.close({ outcome: "success", ...(evidence && evidence.length > 0 ? { evidence } : {}) });
|
|
332
|
+
return result;
|
|
333
|
+
}
|
|
334
|
+
/** `timed`'s `evidenceFor` for a git commit call: a defined, non-empty sha becomes a `commit` evidence ref; "nothing to commit" (`undefined`) carries none. */
|
|
335
|
+
export function commitEvidence(sha) {
|
|
336
|
+
return sha === undefined ? undefined : [{ kind: "commit", locality: "local", ref: sha }];
|
|
337
|
+
}
|
|
227
338
|
/**
|
|
228
339
|
* Ledger decisions for one workstream, projected into its brief. A
|
|
229
340
|
* human-decided record is binding — the implementer is told to build that
|
|
@@ -311,6 +422,13 @@ function shellPolicySection(shellPolicy) {
|
|
|
311
422
|
"",
|
|
312
423
|
];
|
|
313
424
|
}
|
|
425
|
+
/**
|
|
426
|
+
* Builds the implementer's brief, byte-for-byte identical to the plain
|
|
427
|
+
* `array.join("\n")` this produced before WS-03 — but through a
|
|
428
|
+
* `BriefBuilder` so every line is also classified into an SC-07 prompt
|
|
429
|
+
* component, with `components` summing exactly to the joined brief's byte
|
|
430
|
+
* length.
|
|
431
|
+
*/
|
|
314
432
|
function implementerBrief(manifest, workstream, spec, ledger, shellPolicy, priorFailure, priorDiagnosis) {
|
|
315
433
|
const roster = manifest.workstreams
|
|
316
434
|
.map((entry) => {
|
|
@@ -321,64 +439,46 @@ function implementerBrief(manifest, workstream, spec, ledger, shellPolicy, prior
|
|
|
321
439
|
const relevantDecisions = ledger.decisions.filter((record) => record.workstream === workstream.id &&
|
|
322
440
|
(record.status === "human-decided" || record.status === "ratified"));
|
|
323
441
|
const relevantFindings = ledger.findings.filter((record) => record.workstream === workstream.id && record.status === "human-decided");
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
: []),
|
|
365
|
-
"## Specification",
|
|
366
|
-
"",
|
|
367
|
-
spec.trim(),
|
|
368
|
-
"",
|
|
369
|
-
"## Rules",
|
|
370
|
-
"",
|
|
371
|
-
"- Never commit; the runner owns commits and commits only verified work.",
|
|
372
|
-
"- Follow the repository's AGENTS.md if present.",
|
|
373
|
-
"- Run the project's checks yourself as you work; the runner verifies",
|
|
374
|
-
" independently afterwards and your claim of success is never trusted",
|
|
375
|
-
" without it.",
|
|
376
|
-
"",
|
|
377
|
-
...shellPolicySection(shellPolicy),
|
|
378
|
-
decisionContract(),
|
|
379
|
-
"",
|
|
380
|
-
summaryContract(),
|
|
381
|
-
].join("\n");
|
|
442
|
+
const builder = createBriefBuilder();
|
|
443
|
+
builder.push("framing", `# Workstream ${workstream.id}: ${workstream.name}`, "");
|
|
444
|
+
builder.push("program-narrative", `Program: ${manifest.program.id} — ${manifest.program.name}`);
|
|
445
|
+
builder.push("framing", "", "You are implementing exactly one workstream of a larger program. The", "full roster, so you know what exists and do not rebuild another", "workstream's work (conform only to your declared dependencies):", "");
|
|
446
|
+
builder.push("roster", roster);
|
|
447
|
+
builder.push("framing", "");
|
|
448
|
+
builder.push("ledger-rulings", ...decisionsRuledOnSection(relevantDecisions), ...findingsRuledOnSection(relevantFindings));
|
|
449
|
+
builder.push("prior-diagnostics", ...(priorFailure
|
|
450
|
+
? [
|
|
451
|
+
"## Previous attempt failed",
|
|
452
|
+
"",
|
|
453
|
+
"A previous attempt at this workstream failed. The verify output:",
|
|
454
|
+
"",
|
|
455
|
+
priorFailure,
|
|
456
|
+
"",
|
|
457
|
+
...(priorDiagnosis
|
|
458
|
+
? [
|
|
459
|
+
"An independent reviewer read the failing tree before you were",
|
|
460
|
+
"spawned. Its diagnosis:",
|
|
461
|
+
"",
|
|
462
|
+
priorDiagnosis,
|
|
463
|
+
"",
|
|
464
|
+
"The tests in the tree were written by the failed attempt, not",
|
|
465
|
+
"by a human. When the diagnosis says an assertion is wrong,",
|
|
466
|
+
"rewrite or delete that assertion rather than bending the",
|
|
467
|
+
"implementation to satisfy it — the spec, not the failing",
|
|
468
|
+
"test, is the contract.",
|
|
469
|
+
"",
|
|
470
|
+
]
|
|
471
|
+
: []),
|
|
472
|
+
"Start from the diagnosis; the working tree may already contain",
|
|
473
|
+
"partial work from that attempt.",
|
|
474
|
+
"",
|
|
475
|
+
]
|
|
476
|
+
: []));
|
|
477
|
+
builder.push("framing", "## Specification", "");
|
|
478
|
+
builder.push("workstream-spec", spec.trim());
|
|
479
|
+
builder.push("framing", "", "## Rules", "", "- Never commit; the runner owns commits and commits only verified work.", "- Follow the repository's AGENTS.md if present, with one exception,", " stated next.", "- Run only the checks that cover what you changed — the tests for the", " code you touched, and the command that was failing if you were given", " a failure above. Do not run the project's full build/type-check/test/", " lint suite: the runner runs it in full after you and gates on the", " result, so a full run of your own is duplicated work, not extra", " safety. This overrides AGENTS.md's \"verify before claiming", " completion\" directive for this spawn only. Your claim of success is", " never trusted either way, so you are not being asked to prove", " anything here — only to catch your own mistakes early.", "- You are working in a dedicated git worktree on a per-workstream", " branch. Do not switch branches, and do not edit `docs/programs/` —", " those are the runner's own artifacts and changes to them are", " discarded.", "");
|
|
480
|
+
builder.push("runner-instructions", ...shellPolicySection(shellPolicy), decisionContract(), "", summaryContract());
|
|
481
|
+
return { brief: builder.join(), components: builder.components() };
|
|
382
482
|
}
|
|
383
483
|
/** Known `docs/programs/` artifact suffixes, longest-specific first is not required — each is checked independently. */
|
|
384
484
|
const DOCS_PROGRAMS_SUFFIXES = [
|
|
@@ -386,6 +486,7 @@ const DOCS_PROGRAMS_SUFFIXES = [
|
|
|
386
486
|
"-manifest.json",
|
|
387
487
|
"-run-report.md",
|
|
388
488
|
"-decisions.jsonl",
|
|
489
|
+
"-run-analytics.json",
|
|
389
490
|
];
|
|
390
491
|
/**
|
|
391
492
|
* The program id a planning-artifact path belongs to, or undefined when the
|
|
@@ -492,6 +593,522 @@ async function permissionsStartupLines(config, permits, agent, recovery, decider
|
|
|
492
593
|
}
|
|
493
594
|
return lines;
|
|
494
595
|
}
|
|
596
|
+
/**
|
|
597
|
+
* Reaps every worktree and per-workstream branch this program created,
|
|
598
|
+
* driven by `git worktree list --porcelain` filtered on the branch prefix —
|
|
599
|
+
* never on path, since git realpath-resolves the paths it reports (WS-02,
|
|
600
|
+
* §3.7). Runs on every run, not only a resume: a crashed prior run leaves the
|
|
601
|
+
* same debris. Retained `refs/nightshift/…` commits are never touched — they
|
|
602
|
+
* are the forensic record a human needs to salvage a parked integration.
|
|
603
|
+
*/
|
|
604
|
+
export async function reapProgramWorktrees(root, programId, worktrees, log,
|
|
605
|
+
/** Workstreams already `complete` in the manifest: their leftover
|
|
606
|
+
* worktree (a crash between integration and cleanup) holds nothing the
|
|
607
|
+
* program branch lacks, so it is reaped without salvage. */
|
|
608
|
+
completeWorkstreamIds = new Set()) {
|
|
609
|
+
const prefix = worktreeBranchPrefix(programId);
|
|
610
|
+
const reaped = [];
|
|
611
|
+
const salvaged = [];
|
|
612
|
+
const list = await worktrees.list(root);
|
|
613
|
+
for (const info of list) {
|
|
614
|
+
const short = info.branch?.startsWith("refs/heads/")
|
|
615
|
+
? info.branch.slice("refs/heads/".length)
|
|
616
|
+
: info.branch;
|
|
617
|
+
if (short === undefined || !short.startsWith(prefix))
|
|
618
|
+
continue;
|
|
619
|
+
const ref = await salvageWorktree(root, programId, short, info, worktrees, completeWorkstreamIds);
|
|
620
|
+
if (ref !== undefined)
|
|
621
|
+
salvaged.push(ref);
|
|
622
|
+
await worktrees.remove(root, info.path);
|
|
623
|
+
await worktrees.deleteBranch(root, short);
|
|
624
|
+
reaped.push(short);
|
|
625
|
+
}
|
|
626
|
+
await worktrees.prune(root);
|
|
627
|
+
const orphanBranches = await worktrees.branchesUnder(root, prefix);
|
|
628
|
+
for (const branch of orphanBranches) {
|
|
629
|
+
await worktrees.deleteBranch(root, branch);
|
|
630
|
+
if (!reaped.includes(branch))
|
|
631
|
+
reaped.push(branch);
|
|
632
|
+
}
|
|
633
|
+
if (salvaged.length > 0) {
|
|
634
|
+
log(`worktrees: salvaged unintegrated work from a prior run before reaping — inspect with ` +
|
|
635
|
+
`\`git log <ref>\` or \`git worktree add <path> <ref>\`: ${salvaged.join(", ")}`);
|
|
636
|
+
}
|
|
637
|
+
if (reaped.length > 0) {
|
|
638
|
+
log(`worktrees: reaped ${reaped.length} leftover branch(es) from a prior run: ${reaped.join(", ")}`);
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
/**
|
|
642
|
+
* Preserves a leftover worktree's work before it is reaped — the
|
|
643
|
+
* reversibility contract: a resume must never destroy the only copy of a
|
|
644
|
+
* failed attempt that the run report just told the human to inspect.
|
|
645
|
+
* Everything in the worktree is committed onto its own branch and, when
|
|
646
|
+
* that head is not already reachable from the program branch, pinned under
|
|
647
|
+
* `refs/nightshift/<program>/<workstream>-wip`. Returns the ref, or undefined
|
|
648
|
+
* when nothing needed salvaging (the workstream is complete, the directory
|
|
649
|
+
* is gone, or the `WorktreeOps` double has no salvage seam). Never throws:
|
|
650
|
+
* salvage is best-effort and must not block the run.
|
|
651
|
+
*/
|
|
652
|
+
async function salvageWorktree(root, programId, shortBranch, info, worktrees, completeWorkstreamIds) {
|
|
653
|
+
if (!worktrees.commitEverything || !worktrees.updateRef || !worktrees.isAncestor)
|
|
654
|
+
return undefined;
|
|
655
|
+
if (info.prunable === true)
|
|
656
|
+
return undefined;
|
|
657
|
+
const workstreamId = shortBranch.slice(worktreeBranchPrefix(programId).length);
|
|
658
|
+
if (workstreamId === "")
|
|
659
|
+
return undefined;
|
|
660
|
+
const lowered = new Set([...completeWorkstreamIds].map((id) => id.toLowerCase()));
|
|
661
|
+
if (lowered.has(workstreamId))
|
|
662
|
+
return undefined;
|
|
663
|
+
try {
|
|
664
|
+
const head = await worktrees.commitEverything(info.path, `nightshift(${programId}): salvaged work in progress for ${workstreamId}`);
|
|
665
|
+
if (head === undefined)
|
|
666
|
+
return undefined;
|
|
667
|
+
if (await worktrees.isAncestor(root, head, "HEAD"))
|
|
668
|
+
return undefined;
|
|
669
|
+
const ref = `${retainedRefName(programId, workstreamId)}-wip`;
|
|
670
|
+
await worktrees.updateRef(root, ref, head);
|
|
671
|
+
return ref;
|
|
672
|
+
}
|
|
673
|
+
catch {
|
|
674
|
+
return undefined;
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
const INSTALL_TRIGGER_NAMES = [
|
|
678
|
+
"package.json",
|
|
679
|
+
"package-lock.json",
|
|
680
|
+
"pnpm-lock.yaml",
|
|
681
|
+
"yarn.lock",
|
|
682
|
+
"bun.lockb",
|
|
683
|
+
"bun.lock",
|
|
684
|
+
];
|
|
685
|
+
/** True when any path in `paths` is (or ends with) a file that can change a worktree's installed dependencies (WS-02, §3.5 step 1). */
|
|
686
|
+
function touchesInstallFiles(paths) {
|
|
687
|
+
return paths.some((path) => INSTALL_TRIGGER_NAMES.some((name) => path === name || path.endsWith(`/${name}`)));
|
|
688
|
+
}
|
|
689
|
+
/**
|
|
690
|
+
* Runs a worktree's dependency install command, when a lockfile is present
|
|
691
|
+
* in the worktree root itself (WS-02, §3.3) — attributed to the caller's
|
|
692
|
+
* open `worktree-setup` span. A worktree with no lockfile installs nothing
|
|
693
|
+
* and is not a failure.
|
|
694
|
+
*/
|
|
695
|
+
async function installWorktreeDependencies(worktreeRoot, setupRunner, log) {
|
|
696
|
+
const installCommand = worktreeInstallCommand(worktreeRoot);
|
|
697
|
+
if (installCommand === undefined) {
|
|
698
|
+
log("worktree: no lockfile — dependencies not installed");
|
|
699
|
+
return { ok: true };
|
|
700
|
+
}
|
|
701
|
+
let result;
|
|
702
|
+
try {
|
|
703
|
+
result = await setupRunner(installCommand, worktreeRoot);
|
|
704
|
+
}
|
|
705
|
+
catch (error) {
|
|
706
|
+
return { ok: false, reason: `dependency install could not run: ${error.message}` };
|
|
707
|
+
}
|
|
708
|
+
if (result.exitCode !== 0) {
|
|
709
|
+
return {
|
|
710
|
+
ok: false,
|
|
711
|
+
reason: `dependency install (\`${installCommand}\`) failed:\n${tail(result.output, 1500)}`,
|
|
712
|
+
};
|
|
713
|
+
}
|
|
714
|
+
log(`worktree: installed dependencies (${installCommand})`);
|
|
715
|
+
return { ok: true };
|
|
716
|
+
}
|
|
717
|
+
export async function prepareWorktree(args) {
|
|
718
|
+
const { root, programId, workstreamId, worktrees, worktreeBase, setupRunner, log, programTip } = args;
|
|
719
|
+
const branch = worktreeBranchName(programId, workstreamId);
|
|
720
|
+
const path = worktreePathFor(worktreeBase.base, programId, workstreamId, worktreeBase.repoHash);
|
|
721
|
+
try {
|
|
722
|
+
// Idempotent belt-and-braces against a crash-leftover from this same
|
|
723
|
+
// run; cross-run leftovers are reaped at run start.
|
|
724
|
+
await worktrees.remove(root, path);
|
|
725
|
+
await worktrees.deleteBranch(root, branch);
|
|
726
|
+
// `remove` is a no-op for a directory git no longer lists (a prior
|
|
727
|
+
// `worktree prune` after a crash); `add` would then refuse with
|
|
728
|
+
// "already exists". The path is program-owned, so clearing it is safe.
|
|
729
|
+
await rm(path, { recursive: true, force: true });
|
|
730
|
+
await worktrees.add(root, path, branch, programTip);
|
|
731
|
+
}
|
|
732
|
+
catch (error) {
|
|
733
|
+
await worktrees.remove(root, path);
|
|
734
|
+
await worktrees.deleteBranch(root, branch);
|
|
735
|
+
return { ok: false, reason: `could not create the worktree: ${error.message}`, created: false };
|
|
736
|
+
}
|
|
737
|
+
log(`worktree: created at ${path}`);
|
|
738
|
+
const install = await installWorktreeDependencies(path, setupRunner, log);
|
|
739
|
+
if (!install.ok) {
|
|
740
|
+
return { ok: false, reason: install.reason, created: true, cwd: path, branch };
|
|
741
|
+
}
|
|
742
|
+
return { ok: true, cwd: path, branch };
|
|
743
|
+
}
|
|
744
|
+
/**
|
|
745
|
+
* One integration attempt: discard runner debris, merge the program branch
|
|
746
|
+
* forward when its tip has moved, re-verify the merged tree, then squash
|
|
747
|
+
* onto the program branch as one path-limited commit (WS-02, §3.5). Never
|
|
748
|
+
* throws; every failure mode is a typed result the caller decides what to do
|
|
749
|
+
* with (retain and recover, or report).
|
|
750
|
+
*/
|
|
751
|
+
async function attemptIntegration(args) {
|
|
752
|
+
const { root, programId, workstreamId, workstreamName, cwd, branch, finalCommit, worktrees, git, config, verifyRunner, setupRunner, scope, log, rootIndexMutex, rosterTaskFiles, } = args;
|
|
753
|
+
// The full refname, not the short branch name: `refs/nightshift/<p>/<ws>`
|
|
754
|
+
// (the retained ref) and `refs/heads/nightshift/<p>/<ws>` (this branch)
|
|
755
|
+
// are distinct refs that never collide, but git's own short-name
|
|
756
|
+
// disambiguation tries `refs/<name>` *before* `refs/heads/<name>` — a bare
|
|
757
|
+
// `nightshift/<p>/<ws>` passed to `git diff`/`git merge --squash` resolves
|
|
758
|
+
// ambiguously and can silently pick the retained ref instead of the
|
|
759
|
+
// branch. Every revision-parsing use below is spelled out in full; only
|
|
760
|
+
// `deleteBranch`, which operates on `refs/heads/` by construction, uses
|
|
761
|
+
// the short name.
|
|
762
|
+
const branchRef = `refs/heads/${branch}`;
|
|
763
|
+
// Steps 0-1: discard runner debris, then merge the program branch forward
|
|
764
|
+
// when its tip is not already an ancestor of this worktree's own commit.
|
|
765
|
+
const mergeSpan = scope.span({ stage: "integration", workstream: workstreamId });
|
|
766
|
+
const discarded = await worktrees.discardRunnerArtifacts(cwd);
|
|
767
|
+
if (discarded.length > 0) {
|
|
768
|
+
log(`integration: discarded agent edit(s) under docs/programs: ${discarded.join(", ")}`);
|
|
769
|
+
}
|
|
770
|
+
const tipNow = await worktrees.resolveCommit(root, "HEAD");
|
|
771
|
+
if (tipNow === undefined) {
|
|
772
|
+
mergeSpan.close({ outcome: "failed" });
|
|
773
|
+
return { ok: false, kind: "tip-unresolvable" };
|
|
774
|
+
}
|
|
775
|
+
let mergeRan = false;
|
|
776
|
+
if (!(await git.isAncestor(cwd, tipNow, finalCommit))) {
|
|
777
|
+
mergeRan = true;
|
|
778
|
+
const mergeResult = await worktrees.mergeInto(cwd, tipNow, `nightshift(${programId}): merge program branch into ${workstreamId}`);
|
|
779
|
+
if (!mergeResult.ok) {
|
|
780
|
+
await worktrees.abortMerge(cwd);
|
|
781
|
+
mergeSpan.close({ outcome: "failed" });
|
|
782
|
+
return { ok: false, kind: "conflict", output: mergeResult.output, tipNow };
|
|
783
|
+
}
|
|
784
|
+
log("integration: merged the program branch forward");
|
|
785
|
+
const changedByMerge = await worktrees.changedPaths(cwd, finalCommit, "HEAD");
|
|
786
|
+
if (touchesInstallFiles(changedByMerge)) {
|
|
787
|
+
const install = await installWorktreeDependencies(cwd, setupRunner, log);
|
|
788
|
+
if (!install.ok) {
|
|
789
|
+
mergeSpan.close({ outcome: "failed" });
|
|
790
|
+
return { ok: false, kind: "install", output: install.reason, tipNow };
|
|
791
|
+
}
|
|
792
|
+
}
|
|
793
|
+
}
|
|
794
|
+
mergeSpan.close({ outcome: "success" });
|
|
795
|
+
// Step 2: re-verify, but only when the merge actually ran — an unmoved tip
|
|
796
|
+
// means the tree is byte-identical to the one that just verified green.
|
|
797
|
+
if (mergeRan) {
|
|
798
|
+
log("integration: verifying the merged tree");
|
|
799
|
+
const verifyFailure = await verifyAttempt(config, verifyRunner, cwd, root, 0, scope, "verification-command", {
|
|
800
|
+
programId,
|
|
801
|
+
phase: "integration",
|
|
802
|
+
workstream: workstreamId,
|
|
803
|
+
...(args.verifyCache ? { cache: args.verifyCache } : {}),
|
|
804
|
+
});
|
|
805
|
+
if (verifyFailure !== undefined) {
|
|
806
|
+
return { ok: false, kind: "verify", output: verifyFailure, tipNow };
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
// Step 3: squash onto the program branch, in the root checkout.
|
|
810
|
+
const squashSpan = scope.span({ stage: "integration", workstream: workstreamId });
|
|
811
|
+
const incoming = await worktrees.changedPaths(root, tipNow, branchRef);
|
|
812
|
+
if (incoming.length === 0) {
|
|
813
|
+
squashSpan.close({ outcome: "success" });
|
|
814
|
+
log("integration: nothing to integrate — the branch matches the tip");
|
|
815
|
+
return { ok: true };
|
|
816
|
+
}
|
|
817
|
+
// WS-05 §3.7.1: everything from here on runs inside `rootIndexMutex`,
|
|
818
|
+
// which (together with `integrationMutex`, already held by the caller for
|
|
819
|
+
// this whole flow) makes the re-resolved tip below stable for the rest of
|
|
820
|
+
// this step. The only thing that can have moved the program-branch tip
|
|
821
|
+
// since `tipNow` was resolved above is an authoring commit — it takes no
|
|
822
|
+
// `integrationMutex` and must not (§3.7) — so a movement here is always
|
|
823
|
+
// exactly that, never a sibling integration.
|
|
824
|
+
return await rootIndexMutex.runExclusive(async () => {
|
|
825
|
+
const tipAtSquash = await worktrees.resolveCommit(root, "HEAD");
|
|
826
|
+
if (tipAtSquash === undefined) {
|
|
827
|
+
squashSpan.close({ outcome: "failed" });
|
|
828
|
+
return { ok: false, kind: "tip-unresolvable" };
|
|
829
|
+
}
|
|
830
|
+
const movedPaths = tipAtSquash === tipNow ? [] : await worktrees.changedPaths(root, tipNow, tipAtSquash);
|
|
831
|
+
if (movedPaths.length > 0) {
|
|
832
|
+
// Branch 3: the movement collides with this workstream's own paths, or
|
|
833
|
+
// touches something an authoring commit has no business touching —
|
|
834
|
+
// never proceed past a movement like that; land it unverified would be
|
|
835
|
+
// exactly the unproven integration SC-02 forbids.
|
|
836
|
+
const intersecting = movedPaths.filter((path) => incoming.includes(path));
|
|
837
|
+
const disallowed = movedPaths.filter((path) => !isRunnerOwnedArtifactPath(path) && !rosterTaskFiles.has(path));
|
|
838
|
+
if (intersecting.length > 0 || disallowed.length > 0) {
|
|
839
|
+
squashSpan.close({ outcome: "failed" });
|
|
840
|
+
const parts = [];
|
|
841
|
+
if (intersecting.length > 0) {
|
|
842
|
+
parts.push(`path(s) this workstream also touched: ${intersecting.join(", ")}`);
|
|
843
|
+
}
|
|
844
|
+
if (disallowed.length > 0) {
|
|
845
|
+
parts.push(`path(s) outside the runner's own artifacts or a roster spec file: ${disallowed.join(", ")}`);
|
|
846
|
+
}
|
|
847
|
+
return {
|
|
848
|
+
ok: false,
|
|
849
|
+
kind: "moved",
|
|
850
|
+
tipNow,
|
|
851
|
+
tipAtSquash,
|
|
852
|
+
output: `the program branch moved from \`${tipNow}\` to \`${tipAtSquash}\` while this ` +
|
|
853
|
+
`workstream was integrating (an authoring commit landed); ${parts.join("; ")}.`,
|
|
854
|
+
};
|
|
855
|
+
}
|
|
856
|
+
}
|
|
857
|
+
// Branch 2 (movedPaths non-empty but disjoint from `incoming` and every
|
|
858
|
+
// path runner-owned or a roster spec file) falls through to the same
|
|
859
|
+
// squash as branch 1 (no movement) — only the post-condition below
|
|
860
|
+
// differs, relaxed to the subset form §3.7.1 requires.
|
|
861
|
+
const dirtyAtRoot = new Set(await git.dirtyPaths(root));
|
|
862
|
+
const overlap = incoming.filter((path) => dirtyAtRoot.has(path));
|
|
863
|
+
if (overlap.length > 0) {
|
|
864
|
+
squashSpan.close({ outcome: "failed" });
|
|
865
|
+
return {
|
|
866
|
+
ok: false,
|
|
867
|
+
kind: "refused",
|
|
868
|
+
output: `the workstream changed path(s) already uncommitted at the root checkout: ${overlap.join(", ")}`,
|
|
869
|
+
tipNow,
|
|
870
|
+
};
|
|
871
|
+
}
|
|
872
|
+
const squash = await worktrees.squashIntegrate(root, branchRef, `nightshift(${programId}): ${workstreamId} ${workstreamName}`, incoming);
|
|
873
|
+
if (!squash.ok) {
|
|
874
|
+
squashSpan.close({ outcome: "failed" });
|
|
875
|
+
return { ok: false, kind: "refused", output: squash.output, tipNow };
|
|
876
|
+
}
|
|
877
|
+
if (squash.commit !== undefined) {
|
|
878
|
+
// Post-condition: the integrated tree equals the branch's tree, up to
|
|
879
|
+
// paths the concurrent tip movement (if any) already explains — a
|
|
880
|
+
// path list that produced the commit was incomplete, or a movement
|
|
881
|
+
// that collided, is worse than a failed integration.
|
|
882
|
+
const remaining = await worktrees.changedPaths(root, squash.commit, branchRef);
|
|
883
|
+
const unexplained = remaining.filter((path) => !movedPaths.includes(path));
|
|
884
|
+
if (unexplained.length > 0) {
|
|
885
|
+
squashSpan.close({ outcome: "failed" });
|
|
886
|
+
return {
|
|
887
|
+
ok: false,
|
|
888
|
+
kind: "refused",
|
|
889
|
+
output: `the integration commit's tree does not match the branch's: ${unexplained.join(", ")}`,
|
|
890
|
+
tipNow,
|
|
891
|
+
};
|
|
892
|
+
}
|
|
893
|
+
}
|
|
894
|
+
squashSpan.close({ outcome: "success" });
|
|
895
|
+
log(squash.commit !== undefined
|
|
896
|
+
? `integration: integrated as \`${squash.commit}\``
|
|
897
|
+
: "integration: nothing to commit");
|
|
898
|
+
return { ok: true, ...(squash.commit === undefined ? {} : { commit: squash.commit }) };
|
|
899
|
+
});
|
|
900
|
+
}
|
|
901
|
+
function integrationFailureText(result, finalCommit) {
|
|
902
|
+
if (result.kind === "tip-unresolvable") {
|
|
903
|
+
return "cannot resolve the program-branch tip";
|
|
904
|
+
}
|
|
905
|
+
// Every branch below names both commits the recovery brief and the
|
|
906
|
+
// outcome reason need: the verified commit this worktree built, and the
|
|
907
|
+
// program-branch tip it could not be reconciled with (WS-02 §3.6 step 6).
|
|
908
|
+
// "conflict" was the only kind that did this until the coverage review
|
|
909
|
+
// caught that a red integration verify, a failed post-merge reinstall, or
|
|
910
|
+
// a refused squash all left a diagnosis that named neither.
|
|
911
|
+
const verb = result.kind === "conflict"
|
|
912
|
+
? `merging the program branch (now at \`${result.tipNow}\`) into the verified commit \`${finalCommit}\` conflicted`
|
|
913
|
+
: result.kind === "verify"
|
|
914
|
+
? `the merged tree (program branch now at \`${result.tipNow}\`, verified commit \`${finalCommit}\`) failed re-verification`
|
|
915
|
+
: result.kind === "install"
|
|
916
|
+
? `reinstalling dependencies after merging the program branch (now at \`${result.tipNow}\`) into the verified commit \`${finalCommit}\` failed`
|
|
917
|
+
// "moved" (WS-05 §3.7.1): a concurrent authoring commit moved the
|
|
918
|
+
// tip during this integration's own squash step, colliding with
|
|
919
|
+
// this workstream's paths or touching something it should not
|
|
920
|
+
// have — named by both shas, never retried in place.
|
|
921
|
+
: result.kind === "moved"
|
|
922
|
+
? `landing the verified commit \`${finalCommit}\` (scheduled from tip \`${result.tipNow}\`) conflicted with an authoring commit that landed during integration (program branch now at \`${result.tipAtSquash}\`)`
|
|
923
|
+
: `the squash integration of the verified commit \`${finalCommit}\` onto the program branch (now at \`${result.tipNow}\`) failed`;
|
|
924
|
+
return `${verb}:\n${result.output}`;
|
|
925
|
+
}
|
|
926
|
+
function integrationRecoveryBrief(workstream, finalCommit, retainedRef, tipNow, failureText) {
|
|
927
|
+
return [
|
|
928
|
+
`# Integration recovery: ${workstream.id} ${workstream.name}`,
|
|
929
|
+
"",
|
|
930
|
+
"This workstream's build already verified green, as commit",
|
|
931
|
+
`\`${finalCommit}\`, reachable from this worktree as \`${retainedRef}\` — `,
|
|
932
|
+
"`git show`, `git diff`, and `git cherry-pick` against that ref all",
|
|
933
|
+
"work from here even though it is not checked out. Since then, the",
|
|
934
|
+
`program branch moved to a new tip, \`${tipNow}\`, and landing the`,
|
|
935
|
+
"verified work on top of it failed:",
|
|
936
|
+
"",
|
|
937
|
+
failureText,
|
|
938
|
+
"",
|
|
939
|
+
"Your job is to land the workstream's *intent* on top of the current",
|
|
940
|
+
"tip — not to reproduce the old commit byte for byte. Resolve whatever",
|
|
941
|
+
"conflicted, or fix whatever the merged tree's verification found,",
|
|
942
|
+
"however that takes: editing the code, editing the tests the failed",
|
|
943
|
+
"attempt wrote, or both.",
|
|
944
|
+
"",
|
|
945
|
+
"You are working in a dedicated git worktree on a per-workstream",
|
|
946
|
+
"branch. Never commit — the runner owns commits and verifies your fix",
|
|
947
|
+
"itself. Do not switch branches, and do not edit `docs/programs/` —",
|
|
948
|
+
"those are the runner's own artifacts and changes to them are",
|
|
949
|
+
"discarded.",
|
|
950
|
+
"",
|
|
951
|
+
summaryContract(),
|
|
952
|
+
].join("\n");
|
|
953
|
+
}
|
|
954
|
+
/**
|
|
955
|
+
* Integrates one green worktree onto the program branch (WS-02, §3.5-§3.6):
|
|
956
|
+
* one attempt, and on failure, exactly one bounded recovery attempt in a
|
|
957
|
+
* fresh worktree from the current tip, briefed with both commits and the
|
|
958
|
+
* failure. The verified commit is retained under `refs/nightshift/` on
|
|
959
|
+
* every path, before any mutation, and deleted again only once this
|
|
960
|
+
* function returns success.
|
|
961
|
+
*/
|
|
962
|
+
export async function integrateWorkstream(args) {
|
|
963
|
+
const { root, programId, workstream, cwd, branch, finalCommit, worktrees, git, config, verifyRunner, setupRunner, agentRunner, permits, recoveryAgent, worktreeBase, scope, log, transcriptSink, rootIndexMutex, rosterTaskFiles, } = args;
|
|
964
|
+
const ref = retainedRefName(programId, workstream.id);
|
|
965
|
+
const cleanup = [{ path: cwd, branch }];
|
|
966
|
+
// Full refname, not the short branch name: see attemptIntegration's own
|
|
967
|
+
// note on why a bare short name can resolve ambiguously once a
|
|
968
|
+
// same-named `refs/nightshift/...` ref exists (e.g. left over from a
|
|
969
|
+
// prior run's failed integration of this same workstream).
|
|
970
|
+
await git.createRef(root, ref, `refs/heads/${branch}`);
|
|
971
|
+
const first = await attemptIntegration({
|
|
972
|
+
root,
|
|
973
|
+
programId,
|
|
974
|
+
workstreamId: workstream.id,
|
|
975
|
+
workstreamName: workstream.name,
|
|
976
|
+
cwd,
|
|
977
|
+
branch,
|
|
978
|
+
finalCommit,
|
|
979
|
+
worktrees,
|
|
980
|
+
git,
|
|
981
|
+
config,
|
|
982
|
+
verifyRunner,
|
|
983
|
+
setupRunner,
|
|
984
|
+
...(args.verifyCache ? { verifyCache: args.verifyCache } : {}),
|
|
985
|
+
scope,
|
|
986
|
+
log,
|
|
987
|
+
rootIndexMutex,
|
|
988
|
+
rosterTaskFiles,
|
|
989
|
+
});
|
|
990
|
+
if (first.ok) {
|
|
991
|
+
return { ok: true, ...(first.commit === undefined ? {} : { commit: first.commit }), transcripts: [], cleanup };
|
|
992
|
+
}
|
|
993
|
+
const firstReason = integrationFailureText(first, finalCommit);
|
|
994
|
+
log(`integration: ${firstReason}`);
|
|
995
|
+
const recoveryBranch = `${worktreeBranchName(programId, workstream.id)}-integration`;
|
|
996
|
+
const recoveryPath = worktreePathFor(worktreeBase.base, programId, `${workstream.id}-integration`, worktreeBase.repoHash);
|
|
997
|
+
const transcripts = [];
|
|
998
|
+
const recoveryTip = await worktrees.resolveCommit(root, "HEAD");
|
|
999
|
+
if (recoveryTip === undefined) {
|
|
1000
|
+
return {
|
|
1001
|
+
ok: false,
|
|
1002
|
+
reason: `${firstReason}\n\nRecovery could not start: the program-branch tip could not be resolved.`,
|
|
1003
|
+
retainedRef: ref,
|
|
1004
|
+
worktreePaths: [cwd],
|
|
1005
|
+
transcripts,
|
|
1006
|
+
};
|
|
1007
|
+
}
|
|
1008
|
+
try {
|
|
1009
|
+
await worktrees.remove(root, recoveryPath);
|
|
1010
|
+
await worktrees.deleteBranch(root, recoveryBranch);
|
|
1011
|
+
await worktrees.add(root, recoveryPath, recoveryBranch, recoveryTip);
|
|
1012
|
+
}
|
|
1013
|
+
catch (error) {
|
|
1014
|
+
return {
|
|
1015
|
+
ok: false,
|
|
1016
|
+
reason: `${firstReason}\n\nRecovery could not start: the recovery worktree could not be ` +
|
|
1017
|
+
`created (${error.message}).`,
|
|
1018
|
+
retainedRef: ref,
|
|
1019
|
+
worktreePaths: [cwd],
|
|
1020
|
+
transcripts,
|
|
1021
|
+
};
|
|
1022
|
+
}
|
|
1023
|
+
cleanup.push({ path: recoveryPath, branch: recoveryBranch });
|
|
1024
|
+
const install = await installWorktreeDependencies(recoveryPath, setupRunner, log);
|
|
1025
|
+
if (!install.ok) {
|
|
1026
|
+
return {
|
|
1027
|
+
ok: false,
|
|
1028
|
+
reason: `${firstReason}\n\nRecovery could not start: ${install.reason}`,
|
|
1029
|
+
retainedRef: ref,
|
|
1030
|
+
worktreePaths: [cwd, recoveryPath],
|
|
1031
|
+
transcripts,
|
|
1032
|
+
};
|
|
1033
|
+
}
|
|
1034
|
+
const cause = first.kind === "verify" || first.kind === "install" ? "integration-verify" : "integration-conflict";
|
|
1035
|
+
const failureTipNow = first.kind === "tip-unresolvable" ? recoveryTip : first.tipNow;
|
|
1036
|
+
const brief = integrationRecoveryBrief(workstream, finalCommit, ref, failureTipNow, firstReason);
|
|
1037
|
+
let invocation;
|
|
1038
|
+
try {
|
|
1039
|
+
invocation = await invokeAgent(agentRunner, recoveryAgent, brief, recoveryPath, permits, "recoveryAgent", transcriptSink("integration-recovery"), scope, { stage: "recovery", workstream: workstream.id, attemptSeat: "integration recovery", attemptReason: cause });
|
|
1040
|
+
if (invocation.transcript)
|
|
1041
|
+
transcripts.push(invocation.transcript);
|
|
1042
|
+
}
|
|
1043
|
+
catch (error) {
|
|
1044
|
+
return {
|
|
1045
|
+
ok: false,
|
|
1046
|
+
reason: `${firstReason}\n\nThe recovery agent could not be spawned: ${error.message}`,
|
|
1047
|
+
retainedRef: ref,
|
|
1048
|
+
worktreePaths: [cwd, recoveryPath],
|
|
1049
|
+
transcripts,
|
|
1050
|
+
};
|
|
1051
|
+
}
|
|
1052
|
+
const recoveryVerifyFailure = await verifyAttempt(config, verifyRunner, recoveryPath, root, invocation.exitCode, scope, "verification-command", {
|
|
1053
|
+
programId,
|
|
1054
|
+
phase: "integration-recovery",
|
|
1055
|
+
workstream: workstream.id,
|
|
1056
|
+
role: "recoveryAgent",
|
|
1057
|
+
attemptSeat: "integration recovery",
|
|
1058
|
+
});
|
|
1059
|
+
if (recoveryVerifyFailure !== undefined) {
|
|
1060
|
+
return {
|
|
1061
|
+
ok: false,
|
|
1062
|
+
reason: `${firstReason}\n\nThe recovery attempt failed verification: ${recoveryVerifyFailure}`,
|
|
1063
|
+
retainedRef: ref,
|
|
1064
|
+
worktreePaths: [cwd, recoveryPath],
|
|
1065
|
+
transcripts,
|
|
1066
|
+
};
|
|
1067
|
+
}
|
|
1068
|
+
const recoveryCommit = await git.commitAll(recoveryPath, `nightshift(${programId}): ${workstream.id} integration recovery`, ["docs/programs"]);
|
|
1069
|
+
if (recoveryCommit === undefined) {
|
|
1070
|
+
return {
|
|
1071
|
+
ok: false,
|
|
1072
|
+
reason: `${firstReason}\n\nThe recovery attempt verified green but produced no change.`,
|
|
1073
|
+
retainedRef: ref,
|
|
1074
|
+
worktreePaths: [cwd, recoveryPath],
|
|
1075
|
+
transcripts,
|
|
1076
|
+
};
|
|
1077
|
+
}
|
|
1078
|
+
await git.createRef(root, ref, `refs/heads/${recoveryBranch}`);
|
|
1079
|
+
const second = await attemptIntegration({
|
|
1080
|
+
root,
|
|
1081
|
+
programId,
|
|
1082
|
+
workstreamId: workstream.id,
|
|
1083
|
+
workstreamName: workstream.name,
|
|
1084
|
+
cwd: recoveryPath,
|
|
1085
|
+
branch: recoveryBranch,
|
|
1086
|
+
finalCommit: recoveryCommit,
|
|
1087
|
+
worktrees,
|
|
1088
|
+
git,
|
|
1089
|
+
config,
|
|
1090
|
+
verifyRunner,
|
|
1091
|
+
setupRunner,
|
|
1092
|
+
...(args.verifyCache ? { verifyCache: args.verifyCache } : {}),
|
|
1093
|
+
scope,
|
|
1094
|
+
log,
|
|
1095
|
+
rootIndexMutex,
|
|
1096
|
+
rosterTaskFiles,
|
|
1097
|
+
});
|
|
1098
|
+
if (second.ok) {
|
|
1099
|
+
return { ok: true, ...(second.commit === undefined ? {} : { commit: second.commit }), transcripts, cleanup };
|
|
1100
|
+
}
|
|
1101
|
+
// Do not recurse — one bounded recovery attempt only (§3.6).
|
|
1102
|
+
const secondReason = integrationFailureText(second, recoveryCommit);
|
|
1103
|
+
return {
|
|
1104
|
+
ok: false,
|
|
1105
|
+
reason: `${firstReason}\n\nThe recovery attempt also failed to integrate: ${secondReason}\n\n` +
|
|
1106
|
+
`The recovery's own verified commit is \`${recoveryCommit}\`, retained as \`${ref}\`.`,
|
|
1107
|
+
retainedRef: ref,
|
|
1108
|
+
worktreePaths: [cwd, recoveryPath],
|
|
1109
|
+
transcripts,
|
|
1110
|
+
};
|
|
1111
|
+
}
|
|
495
1112
|
/**
|
|
496
1113
|
* The decide-and-continue runner.
|
|
497
1114
|
*
|
|
@@ -525,8 +1142,67 @@ async function runProgramWith(options, permits) {
|
|
|
525
1142
|
const agentRunner = options.agentRunner ?? defaultAgentRunner;
|
|
526
1143
|
const verifyRunner = options.verifyRunner ?? defaultVerifyRunner;
|
|
527
1144
|
const git = options.git ?? defaultGitOps;
|
|
528
|
-
|
|
1145
|
+
// One cache per run (WS-09, SC-14): an identical tree — everything the
|
|
1146
|
+
// verify commands can read except this program's own six bookkeeping
|
|
1147
|
+
// files (§3.3.2) — is never verified twice. No global, no module state: a
|
|
1148
|
+
// second `runProgram` in the same process gets its own cache.
|
|
1149
|
+
const verifyCache = createVerifyCache({
|
|
1150
|
+
git,
|
|
1151
|
+
commands: config.verify,
|
|
1152
|
+
exclude: runnerOwnedVerifyPaths(options.programId),
|
|
1153
|
+
});
|
|
1154
|
+
const worktrees = options.worktrees ?? defaultWorktreeOps;
|
|
1155
|
+
// A separate default from `verifyRunner` on purpose (WS-02, §3.3): tests
|
|
1156
|
+
// stub `verifyRunner` to always return green, and reusing it for the
|
|
1157
|
+
// worktree install step would make a failed install look like a success.
|
|
1158
|
+
const setupRunner = options.setupRunner ?? defaultVerifyRunner;
|
|
1159
|
+
// Resolved once, inside the isRepository branch below, and read from every
|
|
1160
|
+
// workstream flow via closure (WS-02) — undefined when this is not a git
|
|
1161
|
+
// repository, where no worktree is ever created (§3.9).
|
|
1162
|
+
let worktreeBase;
|
|
1163
|
+
const baseLog = options.log ?? ((line) => console.log(line));
|
|
1164
|
+
const log = baseLog;
|
|
529
1165
|
const now = options.now ?? (() => new Date());
|
|
1166
|
+
const monotonic = options.monotonic ?? (() => performance.now());
|
|
1167
|
+
// Read before anything durable happens (a plain read, safe before the
|
|
1168
|
+
// could-not-start gate): a prior finalized artifact's run id becomes this
|
|
1169
|
+
// run's `parentRunId`, so a resumed invocation carries explicit lineage
|
|
1170
|
+
// instead of silently blending its elapsed time with a prior attempt
|
|
1171
|
+
// (SC-14). Created here — before the gate — so pre-gate stages (preflight,
|
|
1172
|
+
// baseline verification) can be buffered in memory; it stays disarmed,
|
|
1173
|
+
// writing nothing durable, until `arm()` is called below.
|
|
1174
|
+
const priorRunId = await readPriorRunId(root, options.programId);
|
|
1175
|
+
const recorder = options.recorder ??
|
|
1176
|
+
RunRecorder.create({
|
|
1177
|
+
clock: { now, monotonic },
|
|
1178
|
+
root,
|
|
1179
|
+
programId: options.programId,
|
|
1180
|
+
...(priorRunId === undefined ? {} : { parentRunId: priorRunId }),
|
|
1181
|
+
log,
|
|
1182
|
+
});
|
|
1183
|
+
// The run-level reference span (WS-01's own; the fourth reference span
|
|
1184
|
+
// alongside the three named seams — see the WS-01/WS-02 emission-boundary
|
|
1185
|
+
// decision). Opened here, at the very first monotonic tick this function
|
|
1186
|
+
// can observe — before preflight, before baseline verification, before the
|
|
1187
|
+
// could-not-start gate — so it covers the true run start, not merely the
|
|
1188
|
+
// portion after arming. `span()` buffers in memory regardless of arm
|
|
1189
|
+
// state, so a could-not-start throw before `arm()` simply discards it,
|
|
1190
|
+
// same as every other pre-gate observation (SC-02). Its bucket is
|
|
1191
|
+
// `unattributed` by construction — see STAGE_BUCKET's doc comment.
|
|
1192
|
+
//
|
|
1193
|
+
// It is deliberately never closed on the normal-completion path (see the
|
|
1194
|
+
// `report-commit` span below for why): the reconciled numbers stop at the
|
|
1195
|
+
// pre-render snapshot (SC-03, SC-14), but the run's true end is the report
|
|
1196
|
+
// commit, and closing this span before the report is written and
|
|
1197
|
+
// committed would stamp an end that has not happened — exactly the defect
|
|
1198
|
+
// an earlier version of
|
|
1199
|
+
// this design had, closing here and silently excluding the trailing write
|
|
1200
|
+
// and commit from every span instead of from just this one. Leaving it
|
|
1201
|
+
// open is not a special case for that fix: it is the same "true end not
|
|
1202
|
+
// observed" semantics the schema already uses for a crashed run, applied
|
|
1203
|
+
// honestly to the one boundary this recorder can never observe closing
|
|
1204
|
+
// even on success.
|
|
1205
|
+
const runSpan = recorder.span({ stage: "run" });
|
|
530
1206
|
let manifest;
|
|
531
1207
|
try {
|
|
532
1208
|
manifest = await loadManifest(root, options.programId);
|
|
@@ -613,7 +1289,7 @@ async function runProgramWith(options, permits) {
|
|
|
613
1289
|
// interrupted, whose own leftover work is exactly what a resume needs.
|
|
614
1290
|
// The runner's own artifacts under docs/programs/ are exempt: they are
|
|
615
1291
|
// output, not work in progress.
|
|
616
|
-
const dirty = (await git.dirtyPaths(root)).filter((path) => !path
|
|
1292
|
+
const dirty = (await git.dirtyPaths(root)).filter((path) => !isRunnerOwnedArtifactPath(path));
|
|
617
1293
|
if (dirty.length > 0 && !resuming) {
|
|
618
1294
|
throw new CouldNotStartError(`The working tree has uncommitted changes the run would sweep into its commits:\n` +
|
|
619
1295
|
dirty.map((path) => ` ${path}`).join("\n") +
|
|
@@ -622,7 +1298,7 @@ async function runProgramWith(options, permits) {
|
|
|
622
1298
|
// A run always commits to the program's own branch — never the default
|
|
623
1299
|
// branch, and never another program's branch, since the runner commits
|
|
624
1300
|
// with `git add -A` and a mismatched worktree would sweep this program's
|
|
625
|
-
// work onto the wrong branch. These refuse before
|
|
1301
|
+
// work onto the wrong branch. These refuse before the pipeline below
|
|
626
1302
|
// spawns the first agent; --force is the explicit escape hatch.
|
|
627
1303
|
if (!options.force) {
|
|
628
1304
|
const currentBranch = await git.currentBranch(root);
|
|
@@ -645,6 +1321,26 @@ async function runProgramWith(options, permits) {
|
|
|
645
1321
|
`with \`${branchCommand}\`, or re-run with --force.`);
|
|
646
1322
|
}
|
|
647
1323
|
}
|
|
1324
|
+
// Every worktree and per-workstream branch this program left behind —
|
|
1325
|
+
// a crashed prior run, not only an ordinary resume — is reaped before
|
|
1326
|
+
// anything else runs (WS-02, §3.7). Retained `refs/nightshift/…` commits
|
|
1327
|
+
// are the forensic record a human needs to salvage a parked
|
|
1328
|
+
// integration and are never touched here.
|
|
1329
|
+
await reapProgramWorktrees(root, options.programId, worktrees, log, new Set(manifest.workstreams.filter((w) => w.status === "complete").map((w) => w.id)));
|
|
1330
|
+
worktreeBase = await resolveWorktreeBase(root, options.worktreeBaseDir);
|
|
1331
|
+
log(`worktrees: ${worktreeBase.base}` +
|
|
1332
|
+
(worktreeBase.usedFallback ? " (OS-temp fallback — the sibling location was not writable)" : ""));
|
|
1333
|
+
// A resumed run's own leftover work no longer sits in the root tree a
|
|
1334
|
+
// workstream's build reads from — that tree is now a fresh worktree from
|
|
1335
|
+
// the program-branch tip. Said once, plainly, rather than discovered
|
|
1336
|
+
// later as "my edit vanished" (WS-02, §3.7).
|
|
1337
|
+
if (resuming && dirty.length > 0) {
|
|
1338
|
+
log("warning: the working tree has uncommitted change(s) that a " +
|
|
1339
|
+
"worktree-isolated build will not see (it starts from the " +
|
|
1340
|
+
"program-branch tip), and an integration may refuse if a " +
|
|
1341
|
+
"workstream also touches the same path(s):\n" +
|
|
1342
|
+
dirty.map((path) => ` ${path}`).join("\n"));
|
|
1343
|
+
}
|
|
648
1344
|
}
|
|
649
1345
|
// HEAD as it stood before the run's first agent ran — the whole-program
|
|
650
1346
|
// review stage's fallback diff base (see resolveProgramDiffBase) when a
|
|
@@ -654,8 +1350,10 @@ async function runProgramWith(options, permits) {
|
|
|
654
1350
|
// The run's own identity — distinct from runStartCommit, which two
|
|
655
1351
|
// invocations can share (the same commit) or lack entirely (no
|
|
656
1352
|
// repository). Stamped on each `prerequisite-verified` ledger event so the
|
|
657
|
-
// ledger records which run verified a prerequisite.
|
|
658
|
-
|
|
1353
|
+
// ledger records which run verified a prerequisite. Promoted from the
|
|
1354
|
+
// recorder's identity (SC-14) rather than minted separately, so the ledger
|
|
1355
|
+
// and the analytics artifact always carry the same run id.
|
|
1356
|
+
const runId = recorder.runId;
|
|
659
1357
|
// Preflight: every *pending* prerequisite's verifyCommand executes here —
|
|
660
1358
|
// before baseline verification and before any agent spawns (SC-03). This
|
|
661
1359
|
// mutates manifest.prerequisites in memory (pending -> satisfied on a met
|
|
@@ -670,6 +1368,7 @@ async function runProgramWith(options, permits) {
|
|
|
670
1368
|
runId,
|
|
671
1369
|
runStartCommit,
|
|
672
1370
|
log,
|
|
1371
|
+
recorder,
|
|
673
1372
|
});
|
|
674
1373
|
// Baseline verification: the verify suite on the untouched tree, before
|
|
675
1374
|
// the first agent is spawned. A red baseline on a fresh start means the
|
|
@@ -690,7 +1389,7 @@ async function runProgramWith(options, permits) {
|
|
|
690
1389
|
let baselineFingerprint;
|
|
691
1390
|
if (Object.keys(config.verify).length > 0) {
|
|
692
1391
|
log("baseline: running the verify commands on the untouched tree");
|
|
693
|
-
const baselineFailure = await verifyAttempt(config, verifyRunner, root, 0);
|
|
1392
|
+
const baselineFailure = await verifyAttempt(config, verifyRunner, root, root, 0, recorder, "baseline-verification", { programId: options.programId, phase: "baseline", cache: verifyCache });
|
|
694
1393
|
if (baselineFailure === undefined) {
|
|
695
1394
|
log("baseline: clean");
|
|
696
1395
|
}
|
|
@@ -715,10 +1414,14 @@ async function runProgramWith(options, permits) {
|
|
|
715
1414
|
// makes the only crash-between-writes residue a harmless idempotent
|
|
716
1415
|
// duplicate event, never a lost one. A could-not-start throw above has
|
|
717
1416
|
// already unwound the process before reaching here, so an exit-3 run
|
|
718
|
-
// mutates neither the manifest nor the ledger
|
|
1417
|
+
// mutates neither the manifest nor the ledger — nor, from this line on, the
|
|
1418
|
+
// analytics artifact: `arm()` enables the recorder's durable writes, so a
|
|
1419
|
+
// could-not-start run leaves no working log and no finalized artifact
|
|
1420
|
+
// (SC-02).
|
|
1421
|
+
recorder.arm();
|
|
719
1422
|
if (preflight.events.length > 0) {
|
|
720
|
-
await appendLedgerEvents(root, options.programId, preflight.events);
|
|
721
|
-
await saveManifest(root, options.programId, manifest, { log });
|
|
1423
|
+
await timed(recorder, { stage: "ledger-persist" }, () => appendLedgerEvents(root, options.programId, preflight.events));
|
|
1424
|
+
await timed(recorder, { stage: "manifest-persist" }, () => saveManifest(root, options.programId, manifest, { log }));
|
|
722
1425
|
}
|
|
723
1426
|
// Subject ids already sent to the decider this run — shared across the
|
|
724
1427
|
// authoring and build stages (a run is one process) and across the
|
|
@@ -732,671 +1435,1166 @@ async function runProgramWith(options, permits) {
|
|
|
732
1435
|
// report's "this run" triage ratio — see decider-review.ts's doc comment
|
|
733
1436
|
// on `triaged`.
|
|
734
1437
|
const triaged = new Set();
|
|
735
|
-
//
|
|
736
|
-
//
|
|
737
|
-
//
|
|
738
|
-
//
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
log,
|
|
747
|
-
now,
|
|
748
|
-
reviewed,
|
|
749
|
-
triaged,
|
|
750
|
-
});
|
|
751
|
-
manifest = await loadManifest(root, options.programId);
|
|
752
|
-
// Loaded once so every brief in this run projects the same picture of
|
|
753
|
-
// human-decided and ratified choices; decisions this run itself journals
|
|
754
|
-
// are picked up fresh by `readDecisionLedger` at the end, for escalations.
|
|
755
|
-
const ledgerAtStart = await readDecisionLedger(root, options.programId);
|
|
756
|
-
const ordered = stableTopologicalOrder(manifest.workstreams);
|
|
757
|
-
const results = [];
|
|
758
|
-
// Every workstream whose spec authoring failed or parked. Seeds `blocked`
|
|
759
|
-
// below (their briefs would be missing a producer's spec) and also guards
|
|
760
|
-
// the awaiting_human branch in the build loop: an authoring failure must
|
|
761
|
-
// never be repainted as a planned wait — see that branch's own comment.
|
|
762
|
-
const authoringFailed = new Set(authorResult.results
|
|
763
|
-
.filter((entry) => entry.outcome.status === "failed" || entry.outcome.status === "parked")
|
|
764
|
-
.map((entry) => entry.id));
|
|
765
|
-
const blocked = new Set(authoringFailed);
|
|
766
|
-
// The awaiting cone: every workstream directly referencing an unmet
|
|
767
|
-
// prerequisite, plus everything transitively downstream of one — the same
|
|
768
|
-
// downstream-cone machinery failure isolation uses, with the semantics
|
|
769
|
-
// inverted (no agent spawned, no retry, no diagnosis; see the build loop).
|
|
770
|
-
const unmetPrerequisiteIds = new Set(preflight.checks.filter((check) => !check.met).map((check) => check.id));
|
|
771
|
-
const directlyAwaiting = new Set(manifest.workstreams
|
|
772
|
-
.filter((workstream) => workstream.prerequisites.some((id) => unmetPrerequisiteIds.has(id)))
|
|
773
|
-
.map((workstream) => workstream.id));
|
|
774
|
-
const awaitingCone = new Set([
|
|
775
|
-
...directlyAwaiting,
|
|
776
|
-
...downstreamCone(manifest.workstreams, [...directlyAwaiting]),
|
|
777
|
-
]);
|
|
778
|
-
// Set when a workstream's verify failure reproduced the pre-run baseline:
|
|
779
|
-
// the environment is broken, so every remaining workstream parks instead
|
|
780
|
-
// of spending its budget on the same crash.
|
|
781
|
-
let environmentalHalt = false;
|
|
782
|
-
for (const workstream of ordered) {
|
|
783
|
-
if (workstream.status === "complete") {
|
|
784
|
-
results.push({
|
|
785
|
-
id: workstream.id,
|
|
786
|
-
name: workstream.name,
|
|
787
|
-
outcome: { status: "skipped", reason: "already complete" },
|
|
788
|
-
decisionIds: [],
|
|
789
|
-
decisionErrors: [],
|
|
790
|
-
});
|
|
791
|
-
continue;
|
|
792
|
-
}
|
|
793
|
-
if (environmentalHalt) {
|
|
794
|
-
workstream.status = "parked";
|
|
795
|
-
results.push({
|
|
796
|
-
id: workstream.id,
|
|
797
|
-
name: workstream.name,
|
|
798
|
-
outcome: {
|
|
799
|
-
status: "parked",
|
|
800
|
-
reason: "the run halted on an environmental verification failure; parked, not attempted",
|
|
801
|
-
},
|
|
802
|
-
decisionIds: [],
|
|
803
|
-
decisionErrors: [],
|
|
804
|
-
});
|
|
805
|
-
continue;
|
|
806
|
-
}
|
|
807
|
-
// Authoring failures take precedence over awaiting: a workstream whose
|
|
808
|
-
// spec authoring failed or parked must fall through to the `blocked`
|
|
809
|
-
// branch below and render as the genuine failure it is, never be
|
|
810
|
-
// repainted `awaiting_human` — an authoring failure has no other
|
|
811
|
-
// representation in `results` than that branch's own `parked` result, so
|
|
812
|
-
// rewriting it here would erase the only trace of it and the run would
|
|
813
|
-
// wrongly look like a pure intermission. A workstream that is merely
|
|
814
|
-
// downstream of a *build* failure and also in the awaiting cone still
|
|
815
|
-
// resolves to awaiting_human here; that is safe because the build
|
|
816
|
-
// failure self-reports `failed` in `results`, so the run is classified a
|
|
817
|
-
// partial regardless (see the intermission classification below).
|
|
818
|
-
if (awaitingCone.has(workstream.id) && !authoringFailed.has(workstream.id)) {
|
|
819
|
-
workstream.status = "awaiting_human";
|
|
820
|
-
await saveManifest(root, options.programId, manifest, { log });
|
|
821
|
-
const ownUnmet = workstream.prerequisites.filter((id) => unmetPrerequisiteIds.has(id));
|
|
822
|
-
const reason = ownUnmet.length > 0
|
|
823
|
-
? `waiting on human prerequisite ${ownUnmet.join(", ")}; not attempted.`
|
|
824
|
-
: "an upstream workstream is awaiting a human prerequisite; not attempted.";
|
|
825
|
-
results.push({
|
|
826
|
-
id: workstream.id,
|
|
827
|
-
name: workstream.name,
|
|
828
|
-
outcome: { status: "awaiting_human", reason },
|
|
829
|
-
decisionIds: [],
|
|
830
|
-
decisionErrors: [],
|
|
831
|
-
});
|
|
832
|
-
continue;
|
|
833
|
-
}
|
|
834
|
-
if (blocked.has(workstream.id)) {
|
|
835
|
-
workstream.status = "parked";
|
|
836
|
-
results.push({
|
|
837
|
-
id: workstream.id,
|
|
838
|
-
name: workstream.name,
|
|
839
|
-
outcome: {
|
|
840
|
-
status: "parked",
|
|
841
|
-
reason: "an upstream dependency failed; parked, not attempted",
|
|
842
|
-
},
|
|
843
|
-
decisionIds: [],
|
|
844
|
-
decisionErrors: [],
|
|
845
|
-
});
|
|
846
|
-
continue;
|
|
847
|
-
}
|
|
848
|
-
const result = await runWorkstream(workstream);
|
|
849
|
-
results.push(result);
|
|
850
|
-
if (result.environmental === true)
|
|
851
|
-
environmentalHalt = true;
|
|
852
|
-
if (result.outcome.status === "failed") {
|
|
853
|
-
// Failure isolation: park the downstream cone, keep building the rest.
|
|
854
|
-
for (const id of downstreamCone(manifest.workstreams, [workstream.id])) {
|
|
855
|
-
blocked.add(id);
|
|
856
|
-
}
|
|
857
|
-
}
|
|
858
|
-
}
|
|
859
|
-
const complete = results.every((result) => result.outcome.status === "complete" ||
|
|
860
|
-
result.outcome.status === "skipped");
|
|
861
|
-
const anyAwaiting = results.some((result) => result.outcome.status === "awaiting_human");
|
|
862
|
-
const anyFailedOrParked = results.some((result) => result.outcome.status === "failed" || result.outcome.status === "parked");
|
|
863
|
-
// Author-stage failures are counted too: an authoring failure the awaiting
|
|
864
|
-
// guard did NOT catch — e.g. a workstream that authored fine but sits
|
|
865
|
-
// downstream of an authoring failure — must still sink the intermission.
|
|
866
|
-
// Same set that seeded `blocked`/guarded the awaiting branch above;
|
|
867
|
-
// classifying on it here as well makes "any genuine failure => ordinary
|
|
868
|
-
// partial" true independent of how the cone painted it.
|
|
869
|
-
const anyAuthoringFailure = authorResult.results.some((entry) => entry.outcome.status === "failed" || entry.outcome.status === "parked");
|
|
870
|
-
// A planned intermission: nothing failed anywhere (build or authoring), at
|
|
871
|
-
// least one workstream is waiting on a human, and not everything built. A
|
|
872
|
-
// genuine failure makes it an ordinary partial instead — the human should
|
|
873
|
-
// read a diagnosis, not a checklist.
|
|
874
|
-
const intermission = !complete && anyAwaiting && !anyFailedOrParked && !anyAuthoringFailure;
|
|
875
|
-
manifest.program.status = complete
|
|
876
|
-
? "complete"
|
|
877
|
-
: intermission
|
|
878
|
-
? "awaiting_human"
|
|
879
|
-
: "partial";
|
|
880
|
-
await saveManifest(root, options.programId, manifest, { log });
|
|
881
|
-
const wholeProgram = await runWholeProgramStage({
|
|
882
|
-
root,
|
|
883
|
-
programId: options.programId,
|
|
884
|
-
manifest,
|
|
885
|
-
config,
|
|
886
|
-
agentRunner,
|
|
887
|
-
permits,
|
|
888
|
-
git,
|
|
889
|
-
isRepository,
|
|
890
|
-
results,
|
|
891
|
-
authorResult,
|
|
892
|
-
runStartCommit,
|
|
893
|
-
decider,
|
|
894
|
-
reviewed,
|
|
895
|
-
triaged,
|
|
896
|
-
now,
|
|
897
|
-
log,
|
|
898
|
-
});
|
|
899
|
-
const ledger = await readDecisionLedger(root, options.programId);
|
|
900
|
-
const escalations = escalatedRecords(ledger);
|
|
901
|
-
const reportPath = runReportPath(root, options.programId);
|
|
902
|
-
// Every workstream verdict, `complete`, and the exit-code mapping are
|
|
903
|
-
// already settled above; writing and committing the report is bookkeeping
|
|
904
|
-
// that must not be able to reject `runProgram` after the fact (SC-05,
|
|
905
|
-
// SC-12) — a full disk or an EISDIR here is a logged line, not a thrown
|
|
906
|
-
// run.
|
|
1438
|
+
// Everything from here through the run's return is wrapped in one more
|
|
1439
|
+
// level of function nesting so a thrown crash still finalizes the
|
|
1440
|
+
// analytics artifact before control leaves (SC-02) without pulling any of
|
|
1441
|
+
// this region's locals out of scope of the stage functions declared after
|
|
1442
|
+
// its `return` (they are hoisted declarations inside `runProgramBody`,
|
|
1443
|
+
// exactly as they were inside `runProgramWith` before this wrapping).
|
|
1444
|
+
// `finalize()` is fail-open and cannot itself throw or mask the original
|
|
1445
|
+
// error. On the normal path the artifact is finalized explicitly, below,
|
|
1446
|
+
// before the report is written and committed, so it rides into the same
|
|
1447
|
+
// `docs/programs` commit as the report; this catch exists only for the
|
|
1448
|
+
// crash path.
|
|
907
1449
|
try {
|
|
908
|
-
await
|
|
909
|
-
// `triaged` is the run-local set of subject ids the decider actually
|
|
910
|
-
// ruled on this run (built up across the authoring, build, and
|
|
911
|
-
// whole-program stages, see its declaration above) — exactly the
|
|
912
|
-
// `triagedThisRun` basis the "this run" triage ratio needs, since the
|
|
913
|
-
// projected ledger carries no run identifier of its own. It excludes
|
|
914
|
-
// ids that were merely sent but whose invocation failed or returned no
|
|
915
|
-
// valid verdict.
|
|
916
|
-
renderRunReport(manifest, results, ledger, triaged, authorResult, now(), wholeProgram, preflight, options.resumeCommand, manifestMergeCount(root, options.programId) > mergesAtStart), "utf8");
|
|
917
|
-
if (isRepository) {
|
|
918
|
-
await git.commitPaths(root, `nightshift(${options.programId}): run report and decision ledger`, ["docs/programs"]);
|
|
919
|
-
}
|
|
1450
|
+
return await runProgramBody();
|
|
920
1451
|
}
|
|
921
1452
|
catch (error) {
|
|
922
|
-
|
|
1453
|
+
const finalizeStart = monotonic();
|
|
1454
|
+
await recorder.finalize();
|
|
1455
|
+
recorder.point({
|
|
1456
|
+
kind: "analytics-overhead",
|
|
1457
|
+
coverage: "observed",
|
|
1458
|
+
value: monotonic() - finalizeStart,
|
|
1459
|
+
unit: "ms",
|
|
1460
|
+
label: "finalize",
|
|
1461
|
+
});
|
|
1462
|
+
throw error;
|
|
923
1463
|
}
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
const
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
1464
|
+
async function runProgramBody() {
|
|
1465
|
+
// WS-05 §3.7: the total lock order for this whole run. `integrationMutex`
|
|
1466
|
+
// (WS-04's, unchanged) is held across a whole integration attempt,
|
|
1467
|
+
// including its re-verify pass, deliberately for minutes at a time.
|
|
1468
|
+
// `rootIndexMutex` (new) is held only for the git-index operations at
|
|
1469
|
+
// `root`: authoring's spec commits (inside `prepareAuthoring`), an
|
|
1470
|
+
// integration's step-3 tip re-resolve and squash (§3.7.1), and the final
|
|
1471
|
+
// report commit below. Beneath both sits the WS-01 write queue
|
|
1472
|
+
// (`enqueueProgramWrite`). Never taken in any other order, and never
|
|
1473
|
+
// nested with `src/author.ts`'s own `graphMutex` (its merge critical
|
|
1474
|
+
// section), which takes no git operation at all.
|
|
1475
|
+
const integrationMutex = createMutex();
|
|
1476
|
+
const rootIndexMutex = createMutex();
|
|
1477
|
+
// WS-05 §3.1: one dependency graph with two kinds of node instead of a
|
|
1478
|
+
// barrier between two loops. `prepareAuthoring` throws for exactly the
|
|
1479
|
+
// reasons `authorProgram` threw before this workstream — no author agent
|
|
1480
|
+
// configured, a cyclic manifest, an unknown `--from` id — and does so
|
|
1481
|
+
// before any agent spawns. The manifest object is this run's single
|
|
1482
|
+
// shared one (§3.4): authoring mutates and persists *this* object, so the
|
|
1483
|
+
// build scheduler sees a discovered dependency edge or a `specStatus`
|
|
1484
|
+
// transition the instant an author makes it, and no post-authoring reload
|
|
1485
|
+
// is needed or performed.
|
|
1486
|
+
const pipeline = await prepareAuthoring({
|
|
1487
|
+
cwd: options.cwd,
|
|
1488
|
+
programId: options.programId,
|
|
1489
|
+
config,
|
|
1490
|
+
agentRunner,
|
|
1491
|
+
permits,
|
|
1492
|
+
git,
|
|
1493
|
+
log,
|
|
1494
|
+
now,
|
|
1495
|
+
reviewed,
|
|
1496
|
+
triaged,
|
|
1497
|
+
recorder,
|
|
1498
|
+
runSpan,
|
|
1499
|
+
monotonic,
|
|
1500
|
+
manifest,
|
|
1501
|
+
rootIndexMutex,
|
|
1502
|
+
...(options.maxParallel === undefined ? {} : { maxParallel: options.maxParallel }),
|
|
1503
|
+
});
|
|
1504
|
+
// Every workstream's `taskFile` — one of the two path shapes a concurrent
|
|
1505
|
+
// authoring commit may safely have touched while an integration was in
|
|
1506
|
+
// flight (§3.7.1); fixed for the whole run, since only `dependencies`
|
|
1507
|
+
// mutate during authoring, never a workstream's own `taskFile`.
|
|
1508
|
+
const rosterTaskFiles = new Set(manifest.workstreams.map((workstream) => workstream.taskFile));
|
|
1509
|
+
const rosterIds = new Set(manifest.workstreams.map((workstream) => workstream.id));
|
|
1510
|
+
// The awaiting cone: every workstream directly referencing an unmet
|
|
1511
|
+
// prerequisite, plus everything transitively downstream of one — the same
|
|
1512
|
+
// downstream-cone machinery failure isolation uses, with the semantics
|
|
1513
|
+
// inverted (no agent spawned, no retry, no diagnosis; see `dispositionOf`).
|
|
1514
|
+
// `directlyAwaiting` is fixed for the run (authoring never mutates
|
|
1515
|
+
// `prerequisites`); its downstream cone is recomputed live on every call
|
|
1516
|
+
// (§3.5) — a dependency edge an author discovers mid-run can still route a
|
|
1517
|
+
// workstream into this cone after the run started.
|
|
1518
|
+
const unmetPrerequisiteIds = new Set(preflight.checks.filter((check) => !check.met).map((check) => check.id));
|
|
1519
|
+
const directlyAwaiting = new Set(manifest.workstreams
|
|
1520
|
+
.filter((workstream) => workstream.prerequisites.some((id) => unmetPrerequisiteIds.has(id)))
|
|
1521
|
+
.map((workstream) => workstream.id));
|
|
1522
|
+
function isAwaiting(id) {
|
|
1523
|
+
return (directlyAwaiting.has(id) ||
|
|
1524
|
+
downstreamCone(manifest.workstreams, [...directlyAwaiting]).has(id));
|
|
956
1525
|
}
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
1526
|
+
// Every workstream whose spec authoring failed or parked, and every
|
|
1527
|
+
// workstream whose *build* failed — grown live as author and build items
|
|
1528
|
+
// settle (§3.5), never frozen at a barrier that no longer exists.
|
|
1529
|
+
// `isBlocked` recomputes the downstream cone of both sets on every call:
|
|
1530
|
+
// by the time `build:X` is admissible, every ancestor's `author` item
|
|
1531
|
+
// *and* `build` item has already settled (`build:X` depends on `build:D`
|
|
1532
|
+
// for every roster dependency `D`, which itself depends on `author:D`), so
|
|
1533
|
+
// the live sets contain everything that could possibly block `X` at that
|
|
1534
|
+
// instant — the correctness lemma this replaces WS-04's frozen `blocked`
|
|
1535
|
+
// set with.
|
|
1536
|
+
const authoringFailed = new Set();
|
|
1537
|
+
const buildFailed = new Set();
|
|
1538
|
+
function isBlocked(id) {
|
|
1539
|
+
return (authoringFailed.has(id) ||
|
|
1540
|
+
downstreamCone(manifest.workstreams, [...authoringFailed, ...buildFailed]).has(id));
|
|
963
1541
|
}
|
|
964
|
-
workstream
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
let
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
let spawnFailure;
|
|
991
|
-
try {
|
|
992
|
-
invocation = await invokeAgent(agentRunner, attempt.agent, brief, root, permits, attempt.role, transcriptSink(attempt.label.replaceAll(" ", "-")));
|
|
993
|
-
}
|
|
994
|
-
catch (error) {
|
|
995
|
-
// The agent process never started — its command is wrong, missing,
|
|
996
|
-
// or unrunnable. Nothing was built; this is the captured
|
|
997
|
-
// silent-exit incident (SC-12). Make it a diagnosed failure via the
|
|
998
|
-
// existing retry/park machinery below, never a process exit.
|
|
999
|
-
spawnFailure =
|
|
1000
|
-
`the ${attempt.label} agent could not be spawned: ` +
|
|
1001
|
-
`${error.message}. The configured command ` +
|
|
1002
|
-
`\`${describeAgent(attempt.agent)}\` did not start, so nothing was built.`;
|
|
1003
|
-
invocation = { exitCode: 1, output: "" };
|
|
1004
|
-
}
|
|
1005
|
-
recordTranscript(invocation.transcript);
|
|
1006
|
-
const summary = resolveSummary(invocation.output);
|
|
1007
|
-
base.summary = summary.text;
|
|
1008
|
-
await journalDenials(workstream.id, attempt.label, invocation.transcript, summary.needsPermission);
|
|
1009
|
-
const parsed = extractDecisions(invocation.output);
|
|
1010
|
-
base.decisionErrors.push(...parsed.errors);
|
|
1011
|
-
await journalDecisions(workstream, parsed.decisions, baseCommit);
|
|
1012
|
-
base.decisionIds = parsed.decisions.map((decision) => decisionFingerprint(workstream.id, decision));
|
|
1013
|
-
let failure = await verifyAttempt(config, verifyRunner, root, invocation.exitCode);
|
|
1014
|
-
// Deterministic protocol failures the runner can see without a model:
|
|
1015
|
-
// no diagnosis is spent on them, because no read of the tree could add
|
|
1016
|
-
// anything to what the runner already knows.
|
|
1017
|
-
let deterministicFailure = false;
|
|
1018
|
-
// A spawn failure takes precedence over whatever verifyAttempt saw
|
|
1019
|
-
// (the untouched tree can verify green on its own) — no reviewer
|
|
1020
|
-
// diagnosis is spent reading an empty diff.
|
|
1021
|
-
if (spawnFailure !== undefined) {
|
|
1022
|
-
failure = spawnFailure;
|
|
1023
|
-
deterministicFailure = true;
|
|
1024
|
-
}
|
|
1025
|
-
// A brief that never reached the agent's stdin means the process
|
|
1026
|
-
// crashed at startup — whatever it printed was not an answer to the
|
|
1027
|
-
// brief, even when it exited 0.
|
|
1028
|
-
if (failure === undefined && invocation.inputError !== undefined) {
|
|
1029
|
-
failure =
|
|
1030
|
-
`the brief could not be delivered to the agent's stdin ` +
|
|
1031
|
-
`(${invocation.inputError}) — the agent likely crashed at startup, ` +
|
|
1032
|
-
`so its reply answered nothing.`;
|
|
1033
|
-
deterministicFailure = true;
|
|
1034
|
-
}
|
|
1035
|
-
// A green verify over an unchanged tree is just the baseline passing
|
|
1036
|
-
// again, not built work. Without this, an agent that crashes at
|
|
1037
|
-
// startup while exiting 0 (or replies without working) sweeps only
|
|
1038
|
-
// the runner's own in_progress manifest write into a metadata-only
|
|
1039
|
-
// commit and walks through the gate as "complete". A reply that never
|
|
1040
|
-
// produces the work is a protocol failure, not an empty result.
|
|
1041
|
-
if (failure === undefined && isRepository) {
|
|
1042
|
-
const dirty = await git.dirtyPaths(root);
|
|
1043
|
-
const meaningful = dirty.some((path) => !path.replaceAll("\\", "/").startsWith("docs/programs/"));
|
|
1044
|
-
if (!meaningful) {
|
|
1045
|
-
failure =
|
|
1046
|
-
"the attempt verified green but changed nothing outside " +
|
|
1047
|
-
"docs/programs/ — nothing was built, and the passing verify was " +
|
|
1048
|
-
"the untouched baseline. A no-op attempt is a protocol failure " +
|
|
1049
|
-
"(the agent may have crashed at startup or replied without " +
|
|
1050
|
-
"working), not a completed workstream.";
|
|
1051
|
-
deterministicFailure = true;
|
|
1052
|
-
}
|
|
1542
|
+
// Set when a workstream's verify failure reproduced the pre-run baseline:
|
|
1543
|
+
// the environment is broken, so every remaining workstream parks instead
|
|
1544
|
+
// of spending its budget on the same crash — in *both* phases (§3.6):
|
|
1545
|
+
// in-flight authoring finishes and commits its spec, in-flight builds
|
|
1546
|
+
// finish and integrate, but no not-yet-dispatched author or build
|
|
1547
|
+
// launches. Mutated only from the scheduler's `launchBuild`, after a
|
|
1548
|
+
// result is in hand — the same place today's serial loop mutated it.
|
|
1549
|
+
let environmentalHalt = false;
|
|
1550
|
+
/**
|
|
1551
|
+
* WS-04's five-branch cascade (minus the `runWorkstream` branch, which is
|
|
1552
|
+
* the absence of a short-circuit), extracted so the scheduler's `classify`
|
|
1553
|
+
* can consult it on every pass without re-implementing it. Precedence is
|
|
1554
|
+
* load-bearing and unchanged: complete, then environmental halt, then
|
|
1555
|
+
* awaiting (guarded against repainting an authoring failure), then
|
|
1556
|
+
* blocked. Evaluated at dispatch time, when every dependency's `build`
|
|
1557
|
+
* item has already settled (admissibility requires it), so `isBlocked` is
|
|
1558
|
+
* as authoritative here as it was over WS-04's frozen `blocked` set
|
|
1559
|
+
* (WS-05 §3.5).
|
|
1560
|
+
*/
|
|
1561
|
+
function dispositionOf(workstream) {
|
|
1562
|
+
if (workstream.status === "complete") {
|
|
1563
|
+
return {
|
|
1564
|
+
kind: "short-circuit",
|
|
1565
|
+
outcome: { status: "skipped", reason: "already complete" },
|
|
1566
|
+
persist: false,
|
|
1567
|
+
};
|
|
1053
1568
|
}
|
|
1054
|
-
if (
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
}
|
|
1064
|
-
const critique = isRepository && c0 !== undefined
|
|
1065
|
-
? await runTestCritique({
|
|
1066
|
-
root,
|
|
1067
|
-
manifest,
|
|
1068
|
-
workstream,
|
|
1069
|
-
spec,
|
|
1070
|
-
config,
|
|
1071
|
-
agentRunner,
|
|
1072
|
-
permits,
|
|
1073
|
-
verifyRunner,
|
|
1074
|
-
git,
|
|
1075
|
-
reviewer,
|
|
1076
|
-
agent,
|
|
1077
|
-
baseCommit,
|
|
1078
|
-
greenCommit: c0,
|
|
1079
|
-
now,
|
|
1080
|
-
log,
|
|
1081
|
-
transcriptSink,
|
|
1082
|
-
})
|
|
1083
|
-
: undefined;
|
|
1084
|
-
// The manifest's single commit field records the workstream's final
|
|
1085
|
-
// verified state — after any kept fix, that is the fix commit, not
|
|
1086
|
-
// the earlier green one.
|
|
1087
|
-
let finalCommit = critique?.finalCommit ?? c0;
|
|
1088
|
-
if (critique && critique.stageErrors.length > 0) {
|
|
1089
|
-
(base.stageErrors ??= []).push(...critique.stageErrors);
|
|
1090
|
-
}
|
|
1091
|
-
for (const transcript of critique?.transcripts ?? [])
|
|
1092
|
-
recordTranscript(transcript);
|
|
1093
|
-
// Findings anchor to c0 (the green, pre-critique commit) — the
|
|
1094
|
-
// honest rollback point — while the decider below diffs from
|
|
1095
|
-
// baseCommit (pre-workstream), so it sees the whole workstream.
|
|
1096
|
-
const findingEvents = findingsToLedgerEvents({
|
|
1097
|
-
workstreamId: workstream.id,
|
|
1098
|
-
findings: (critique?.outcome.open ?? []).filter(hasRoutableEvidence),
|
|
1099
|
-
...(c0 === undefined ? {} : { baseCommit: c0 }),
|
|
1100
|
-
now,
|
|
1101
|
-
});
|
|
1102
|
-
await appendLedgerEvents(root, options.programId, findingEvents);
|
|
1103
|
-
// The commit's own sha cannot be part of the tree it commits, so the
|
|
1104
|
-
// manifest records it only now — swept forward into whatever commits
|
|
1105
|
-
// next. Replay reads the manifest's current state, not the commit
|
|
1106
|
-
// that last touched it, so this lag is harmless.
|
|
1107
|
-
workstream.status = "complete";
|
|
1108
|
-
if (finalCommit !== undefined)
|
|
1109
|
-
workstream.commit = finalCommit;
|
|
1110
|
-
await saveManifest(root, options.programId, manifest, { log });
|
|
1111
|
-
{
|
|
1112
|
-
const spawnErrors = await reviewWorkstreamDecisions(workstream.id, parsed.decisions, baseCommit);
|
|
1113
|
-
if (spawnErrors.length > 0)
|
|
1114
|
-
(base.stageErrors ??= []).push(...spawnErrors);
|
|
1115
|
-
}
|
|
1116
|
-
// Findings are triaged against the pre-workstream diff (baseCommit),
|
|
1117
|
-
// the same one the decider reviews decisions against — c0/finalCommit
|
|
1118
|
-
// are the fix loop's own rollback anchors, a different thing.
|
|
1119
|
-
{
|
|
1120
|
-
const spawnErrors = await reviewWorkstreamFindings(workstream.id, findingEvents, baseCommit);
|
|
1121
|
-
if (spawnErrors.length > 0)
|
|
1122
|
-
(base.stageErrors ??= []).push(...spawnErrors);
|
|
1123
|
-
}
|
|
1124
|
-
// A fix-now triage drives exactly one bounded fix attempt, here —
|
|
1125
|
-
// after the triage that produced it, before the run advances. Only
|
|
1126
|
-
// reachable when there is a green commit to attempt from and a
|
|
1127
|
-
// decider that could have produced a fix-now verdict in the first
|
|
1128
|
-
// place (SC-13).
|
|
1129
|
-
if (finalCommit !== undefined && decider) {
|
|
1130
|
-
const driven = await driveFixNowFindings(workstream, spec, critique?.outcome.open ?? [], finalCommit, transcriptSink("fix-now-fix"));
|
|
1131
|
-
recordTranscript(driven.transcript);
|
|
1132
|
-
if (driven.spawnError !== undefined) {
|
|
1133
|
-
(base.stageErrors ??= []).push(driven.spawnError);
|
|
1134
|
-
}
|
|
1135
|
-
if (driven.commit !== finalCommit) {
|
|
1136
|
-
finalCommit = driven.commit;
|
|
1137
|
-
workstream.commit = finalCommit;
|
|
1138
|
-
await saveManifest(root, options.programId, manifest, { log });
|
|
1139
|
-
}
|
|
1140
|
-
}
|
|
1141
|
-
if (critique)
|
|
1142
|
-
base.testCritique = critique.outcome;
|
|
1143
|
-
base.testCritiqueDiffClipped = critique?.diffClipped ?? false;
|
|
1144
|
-
base.outcome = {
|
|
1145
|
-
status: "complete",
|
|
1146
|
-
...(finalCommit === undefined ? {} : { commit: finalCommit }),
|
|
1569
|
+
if (environmentalHalt) {
|
|
1570
|
+
return {
|
|
1571
|
+
kind: "short-circuit",
|
|
1572
|
+
status: "parked",
|
|
1573
|
+
outcome: {
|
|
1574
|
+
status: "parked",
|
|
1575
|
+
reason: "the run halted on an environmental verification failure; parked, not attempted",
|
|
1576
|
+
},
|
|
1577
|
+
persist: false,
|
|
1147
1578
|
};
|
|
1148
|
-
return base;
|
|
1149
1579
|
}
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
//
|
|
1153
|
-
//
|
|
1154
|
-
//
|
|
1155
|
-
//
|
|
1156
|
-
//
|
|
1157
|
-
//
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
reason: failure +
|
|
1172
|
-
"\n\nThis failure is identical (up to counts and timings) to the " +
|
|
1173
|
-
"verification failure recorded on the untouched tree before the " +
|
|
1174
|
-
"run began — the environment was broken before this workstream " +
|
|
1175
|
-
"ran. No retry was spent, and the run halted here.",
|
|
1580
|
+
// Authoring failures take precedence over awaiting: a workstream whose
|
|
1581
|
+
// spec authoring failed or parked must fall through to the `isBlocked`
|
|
1582
|
+
// branch below and render as the genuine failure it is, never be
|
|
1583
|
+
// repainted `awaiting_human` — an authoring failure has no other
|
|
1584
|
+
// representation in `results` than that branch's own `parked` result, so
|
|
1585
|
+
// rewriting it here would erase the only trace of it and the run would
|
|
1586
|
+
// wrongly look like a pure intermission. A workstream that is merely
|
|
1587
|
+
// downstream of a *build* failure and also in the awaiting cone still
|
|
1588
|
+
// resolves to awaiting_human here; that is safe because the build
|
|
1589
|
+
// failure self-reports `failed` in `results`, so the run is classified a
|
|
1590
|
+
// partial regardless (see the intermission classification below).
|
|
1591
|
+
if (isAwaiting(workstream.id) && !authoringFailed.has(workstream.id)) {
|
|
1592
|
+
const ownUnmet = workstream.prerequisites.filter((id) => unmetPrerequisiteIds.has(id));
|
|
1593
|
+
const reason = ownUnmet.length > 0
|
|
1594
|
+
? `waiting on human prerequisite ${ownUnmet.join(", ")}; not attempted.`
|
|
1595
|
+
: "an upstream workstream is awaiting a human prerequisite; not attempted.";
|
|
1596
|
+
return {
|
|
1597
|
+
kind: "short-circuit",
|
|
1598
|
+
status: "awaiting_human",
|
|
1599
|
+
outcome: { status: "awaiting_human", reason },
|
|
1600
|
+
persist: true,
|
|
1176
1601
|
};
|
|
1177
|
-
return base;
|
|
1178
1602
|
}
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
workstream.status = "failed";
|
|
1189
|
-
await saveManifest(root, options.programId, manifest, { log });
|
|
1190
|
-
// The work stays in the tree for a resume; decisions made on the way
|
|
1191
|
-
// to a failure are still journaled and still reviewable.
|
|
1192
|
-
{
|
|
1193
|
-
const spawnErrors = await reviewWorkstreamDecisions(workstream.id, parsed.decisions, baseCommit);
|
|
1194
|
-
if (spawnErrors.length > 0)
|
|
1195
|
-
(base.stageErrors ??= []).push(...spawnErrors);
|
|
1196
|
-
}
|
|
1197
|
-
base.outcome = {
|
|
1198
|
-
status: "failed",
|
|
1199
|
-
reason: stuck
|
|
1200
|
-
? failure +
|
|
1201
|
-
"\n\nThe recovery attempt, briefed with a reviewer diagnosis, " +
|
|
1202
|
-
"reproduced the implementer's failure (identical up to counts " +
|
|
1203
|
-
"and timings), so the final retry was not spent."
|
|
1204
|
-
: failure,
|
|
1603
|
+
if (isBlocked(workstream.id)) {
|
|
1604
|
+
return {
|
|
1605
|
+
kind: "short-circuit",
|
|
1606
|
+
status: "parked",
|
|
1607
|
+
outcome: {
|
|
1608
|
+
status: "parked",
|
|
1609
|
+
reason: "an upstream dependency failed; parked, not attempted",
|
|
1610
|
+
},
|
|
1611
|
+
persist: false,
|
|
1205
1612
|
};
|
|
1206
|
-
return base;
|
|
1207
1613
|
}
|
|
1208
|
-
|
|
1209
|
-
|
|
1614
|
+
return { kind: "build" };
|
|
1615
|
+
}
|
|
1616
|
+
// `classify` and `settle` are two separate scheduler calls, and `settle`
|
|
1617
|
+
// runs on a later microtask — a naive `settle` that recomputes
|
|
1618
|
+
// `dispositionOf` could observe a different answer than the one that sent
|
|
1619
|
+
// it there (a sibling can flip `environmentalHalt` in between). Memoising
|
|
1620
|
+
// the dispatching pass's answer and having `settle` read it back avoids
|
|
1621
|
+
// the hazard entirely rather than reasoning about it (WS-04, §3.2).
|
|
1622
|
+
const dispositionCache = new Map();
|
|
1623
|
+
function dispositionFor(workstream) {
|
|
1624
|
+
const fresh = dispositionOf(workstream);
|
|
1625
|
+
dispositionCache.set(workstream.id, fresh);
|
|
1626
|
+
return fresh;
|
|
1627
|
+
}
|
|
1628
|
+
async function applyShortCircuit(workstream) {
|
|
1629
|
+
const disposition = dispositionCache.get(workstream.id);
|
|
1630
|
+
if (disposition === undefined || disposition.kind !== "short-circuit") {
|
|
1631
|
+
// Unreachable: the scheduler only calls `settle` for an item whose
|
|
1632
|
+
// `classify` call (via `dispositionFor`) just returned "settle".
|
|
1633
|
+
throw new Error(`applyShortCircuit called for ${workstream.id} without a cached short-circuit disposition`);
|
|
1210
1634
|
}
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
// and a reviewer read of an empty diff could only muddy it.
|
|
1216
|
-
if (reviewer && !deterministicFailure) {
|
|
1217
|
-
const diagnosed = await diagnoseFailure({
|
|
1218
|
-
root,
|
|
1219
|
-
manifest,
|
|
1220
|
-
workstream,
|
|
1221
|
-
spec,
|
|
1222
|
-
agentRunner,
|
|
1223
|
-
permits,
|
|
1224
|
-
reviewer,
|
|
1225
|
-
git,
|
|
1226
|
-
baseCommit,
|
|
1227
|
-
failure,
|
|
1228
|
-
observe: transcriptSink(`diagnose-${attempt.label.replaceAll(" ", "-")}`),
|
|
1229
|
-
});
|
|
1230
|
-
priorDiagnosis = diagnosed.diagnosis;
|
|
1231
|
-
if (diagnosed.spawnError !== undefined) {
|
|
1232
|
-
(base.stageErrors ??= []).push(diagnosed.spawnError);
|
|
1233
|
-
log(`${workstream.id}: ${diagnosed.spawnError}`);
|
|
1234
|
-
}
|
|
1235
|
-
await journalDenials(workstream.id, "reviewer", diagnosed.transcript);
|
|
1635
|
+
if (disposition.status !== undefined)
|
|
1636
|
+
workstream.status = disposition.status;
|
|
1637
|
+
if (disposition.persist) {
|
|
1638
|
+
await timed(recorder, { stage: "manifest-persist" }, () => saveManifest(root, options.programId, manifest, { log }));
|
|
1236
1639
|
}
|
|
1237
|
-
|
|
1238
|
-
|
|
1640
|
+
return {
|
|
1641
|
+
id: workstream.id,
|
|
1642
|
+
name: workstream.name,
|
|
1643
|
+
outcome: disposition.outcome,
|
|
1644
|
+
decisionIds: [],
|
|
1645
|
+
decisionErrors: [],
|
|
1646
|
+
};
|
|
1647
|
+
}
|
|
1648
|
+
const byId = new Map();
|
|
1649
|
+
const cap = normalizeMaxParallel(options.maxParallel);
|
|
1650
|
+
const schedLog = schedulerLog(baseLog);
|
|
1651
|
+
schedLog(`pipelining authoring and building over ${manifest.workstreams.length} workstream(s)` +
|
|
1652
|
+
(cap === Infinity ? "" : `, at most ${cap} flow(s) at a time`));
|
|
1653
|
+
const authorItems = manifest.workstreams.map((workstream) => ({
|
|
1654
|
+
id: `author:${workstream.id}`,
|
|
1655
|
+
phase: "author",
|
|
1656
|
+
workstream,
|
|
1657
|
+
get dependencies() {
|
|
1658
|
+
return workstream.dependencies.map((id) => `author:${id}`);
|
|
1659
|
+
},
|
|
1660
|
+
}));
|
|
1661
|
+
const buildItems = manifest.workstreams.map((workstream) => ({
|
|
1662
|
+
id: `build:${workstream.id}`,
|
|
1663
|
+
phase: "build",
|
|
1664
|
+
workstream,
|
|
1665
|
+
get dependencies() {
|
|
1666
|
+
return [`author:${workstream.id}`, ...workstream.dependencies.map((id) => `build:${id}`)];
|
|
1667
|
+
},
|
|
1668
|
+
}));
|
|
1669
|
+
// Authors first, then builds, both in manifest order (§3.1): `runReadySet`
|
|
1670
|
+
// breaks ties by this input order, so at `cap: 1` every author drains
|
|
1671
|
+
// before any build is even considered — the pipeline rebuilds today's
|
|
1672
|
+
// barrier exactly rather than approximating it (§3.10).
|
|
1673
|
+
const items = [...authorItems, ...buildItems];
|
|
1674
|
+
// Settle instants, keyed by phased-item id, for the `spec-pending` wait
|
|
1675
|
+
// measurement below (§3.8); the scheduler does not expose these itself.
|
|
1676
|
+
const settledAt = new Map();
|
|
1677
|
+
const schedulerStart = monotonic();
|
|
1678
|
+
async function launchAuthor(item) {
|
|
1679
|
+
const result = await pipeline.authorWorkstream(item.workstream);
|
|
1680
|
+
settledAt.set(item.id, monotonic());
|
|
1681
|
+
if (result.outcome.status === "failed" || result.outcome.status === "parked") {
|
|
1682
|
+
authoringFailed.add(item.workstream.id);
|
|
1239
1683
|
}
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1684
|
+
}
|
|
1685
|
+
// The only path by which an author item settles without running (§3.6): a
|
|
1686
|
+
// halt-parked author never ran, so `specStatus` is left exactly as it
|
|
1687
|
+
// stood, and it does not enter `authoringFailed` — a downstream workstream
|
|
1688
|
+
// reports the halt reason, never "an upstream dependency's authoring
|
|
1689
|
+
// failed".
|
|
1690
|
+
async function settleHaltedAuthor(item) {
|
|
1691
|
+
pipeline.haltAuthorWorkstream(item.workstream, "the run halted on an environmental verification failure; parked, not attempted");
|
|
1692
|
+
settledAt.set(item.id, monotonic());
|
|
1693
|
+
}
|
|
1694
|
+
async function settleBuild(item) {
|
|
1695
|
+
byId.set(item.workstream.id, await applyShortCircuit(item.workstream));
|
|
1696
|
+
settledAt.set(item.id, monotonic());
|
|
1697
|
+
}
|
|
1698
|
+
async function launchBuild(item) {
|
|
1699
|
+
const workstream = item.workstream;
|
|
1700
|
+
// The scheduling instant (WS-04, §3.5): resolved first, with no
|
|
1701
|
+
// `await` between the read and the `workstream-scheduled` emission, so
|
|
1702
|
+
// no sibling can integrate in between — the point and the sha
|
|
1703
|
+
// therefore describe the same instant by construction. The per-flow
|
|
1704
|
+
// ledger read (§3.4) goes strictly after this emission, and nothing
|
|
1705
|
+
// else is inserted between them, so WS-04's
|
|
1706
|
+
// `scheduled − ready === concurrency-cap` identity stays exact.
|
|
1707
|
+
const scheduledFrom = isRepository
|
|
1708
|
+
? await worktrees.resolveCommit(root, "HEAD")
|
|
1709
|
+
: undefined;
|
|
1710
|
+
recordWorkstreamScheduled(recorder, workstream.id);
|
|
1711
|
+
const ledger = await readDecisionLedger(root, options.programId);
|
|
1712
|
+
const result = await runWorkstream(workstream, ledger, scheduledFrom === undefined ? {} : { scheduledFrom });
|
|
1713
|
+
byId.set(workstream.id, result);
|
|
1714
|
+
settledAt.set(item.id, monotonic());
|
|
1715
|
+
if (result.environmental === true && !environmentalHalt) {
|
|
1716
|
+
environmentalHalt = true;
|
|
1717
|
+
schedLog("environmental verification failure — halting further dispatch");
|
|
1718
|
+
schedLog("[run] the halt stops dispatch in both phases — in-flight authoring finishes " +
|
|
1719
|
+
"and commits its spec, in-flight builds finish and integrate, and every " +
|
|
1720
|
+
"not-yet-dispatched author or build parks with the halt reason");
|
|
1721
|
+
}
|
|
1722
|
+
if (result.outcome.status === "failed" || result.outcome.status === "parked") {
|
|
1723
|
+
// A parked build (no spec on disk) leaves its dependents without a
|
|
1724
|
+
// producer just as a failure does; both seed the downstream cone.
|
|
1725
|
+
buildFailed.add(workstream.id);
|
|
1246
1726
|
}
|
|
1247
1727
|
}
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1728
|
+
/** Emits `spec-pending` (§3.8): the wait between when this workstream's
|
|
1729
|
+
* build *could* have become ready — the latest settle of any roster
|
|
1730
|
+
* dependency's build item, or the scheduler's start when it has none —
|
|
1731
|
+
* and the instant it actually did. Authors emit no `workstream-ready`;
|
|
1732
|
+
* that point means "a build is ready" and must stay unique per
|
|
1733
|
+
* workstream. */
|
|
1734
|
+
function onBuildReady(item) {
|
|
1735
|
+
const readyAt = monotonic();
|
|
1736
|
+
const depsReadyAt = item.workstream.dependencies
|
|
1737
|
+
.filter((id) => rosterIds.has(id))
|
|
1738
|
+
.reduce((latest, id) => Math.max(latest, settledAt.get(`build:${id}`) ?? schedulerStart), schedulerStart);
|
|
1739
|
+
recordWorkstreamReady(recorder, item.workstream.id);
|
|
1740
|
+
const waited = readyAt - depsReadyAt;
|
|
1741
|
+
if (waited > 0) {
|
|
1742
|
+
recordWorkstreamWaiting(recorder, item.workstream.id, { cause: "spec-pending", waitedMs: waited });
|
|
1743
|
+
}
|
|
1744
|
+
}
|
|
1745
|
+
/** Emits `author-concurrency-cap` (§3.8) — the authoring half of the
|
|
1746
|
+
* shared `--max-parallel` budget, distinct from `concurrency-cap` so
|
|
1747
|
+
* per-(workstream, cause) uniqueness and WS-04's
|
|
1748
|
+
* `scheduled − ready === concurrency-cap` identity both stay exact. */
|
|
1749
|
+
function onAuthorLaunch(item, cappedMs) {
|
|
1750
|
+
if (cappedMs > 0) {
|
|
1751
|
+
recordWorkstreamWaiting(recorder, item.workstream.id, {
|
|
1752
|
+
cause: "author-concurrency-cap",
|
|
1753
|
+
waitedMs: cappedMs,
|
|
1754
|
+
});
|
|
1755
|
+
}
|
|
1756
|
+
}
|
|
1757
|
+
await runReadySet({
|
|
1758
|
+
items,
|
|
1759
|
+
cap,
|
|
1760
|
+
monotonic,
|
|
1761
|
+
classify: (item) => item.phase === "author"
|
|
1762
|
+
// The one short-circuit an author item has: the environmental halt
|
|
1763
|
+
// (§3.6). Authoring has no other short-circuit — the cone parks are
|
|
1764
|
+
// WS-03's own, inside `handle`.
|
|
1765
|
+
? environmentalHalt
|
|
1766
|
+
? "settle"
|
|
1767
|
+
: "launch"
|
|
1768
|
+
: dispositionFor(item.workstream).kind === "build"
|
|
1769
|
+
? "launch"
|
|
1770
|
+
: "settle",
|
|
1771
|
+
launch: (item) => (item.phase === "author" ? launchAuthor(item) : launchBuild(item)),
|
|
1772
|
+
settle: (item) => (item.phase === "author" ? settleHaltedAuthor(item) : settleBuild(item)),
|
|
1773
|
+
onAdmissible: (item) => {
|
|
1774
|
+
if (item.phase === "build")
|
|
1775
|
+
onBuildReady(item);
|
|
1776
|
+
},
|
|
1777
|
+
onLaunch: (item, cappedMs) => {
|
|
1778
|
+
if (item.phase === "author") {
|
|
1779
|
+
onAuthorLaunch(item, cappedMs);
|
|
1780
|
+
}
|
|
1781
|
+
else if (cappedMs > 0) {
|
|
1782
|
+
recordWorkstreamWaiting(recorder, item.workstream.id, { cause: "concurrency-cap", waitedMs: cappedMs });
|
|
1783
|
+
}
|
|
1784
|
+
},
|
|
1785
|
+
});
|
|
1786
|
+
const authorResult = pipeline.finish();
|
|
1787
|
+
// Wall-clock completion order is not deterministic; `ordered` is.
|
|
1788
|
+
// Computed only now, after the drain (WS-05 §3.9) — never before it, since
|
|
1789
|
+
// an author discovering a dependency mid-run can still change the graph
|
|
1790
|
+
// this walks. Rebuilding `results` from it is what keeps every existing
|
|
1791
|
+
// positional assertion (`results[0]`, `results[1]`, …) green under an
|
|
1792
|
+
// unbounded default. `runReadySet`'s settle-in-`finally` makes a missing
|
|
1793
|
+
// id impossible; filtered defensively anyway per SC-12's fail-open rule.
|
|
1794
|
+
const ordered = stableTopologicalOrder(manifest.workstreams);
|
|
1795
|
+
const results = [];
|
|
1796
|
+
for (const workstream of ordered) {
|
|
1797
|
+
const result = byId.get(workstream.id);
|
|
1798
|
+
if (result === undefined) {
|
|
1799
|
+
schedLog(`warning: ${workstream.id} produced no result — omitted from the report`);
|
|
1278
1800
|
continue;
|
|
1279
|
-
|
|
1801
|
+
}
|
|
1802
|
+
results.push(result);
|
|
1280
1803
|
}
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1804
|
+
const complete = results.every((result) => result.outcome.status === "complete" ||
|
|
1805
|
+
result.outcome.status === "skipped");
|
|
1806
|
+
const anyAwaiting = results.some((result) => result.outcome.status === "awaiting_human");
|
|
1807
|
+
const anyFailedOrParked = results.some((result) => result.outcome.status === "failed" || result.outcome.status === "parked");
|
|
1808
|
+
// Author-stage failures are counted too: an authoring failure the awaiting
|
|
1809
|
+
// guard did NOT catch — e.g. a workstream that authored fine but sits
|
|
1810
|
+
// downstream of an authoring failure — must still sink the intermission.
|
|
1811
|
+
// Same set that seeded `blocked`/guarded the awaiting branch above;
|
|
1812
|
+
// classifying on it here as well makes "any genuine failure => ordinary
|
|
1813
|
+
// partial" true independent of how the cone painted it.
|
|
1814
|
+
const anyAuthoringFailure = authorResult.results.some((entry) => entry.outcome.status === "failed" || entry.outcome.status === "parked");
|
|
1815
|
+
// A planned intermission: nothing failed anywhere (build or authoring), at
|
|
1816
|
+
// least one workstream is waiting on a human, and not everything built. A
|
|
1817
|
+
// genuine failure makes it an ordinary partial instead — the human should
|
|
1818
|
+
// read a diagnosis, not a checklist.
|
|
1819
|
+
const intermission = !complete && anyAwaiting && !anyFailedOrParked && !anyAuthoringFailure;
|
|
1820
|
+
manifest.program.status = complete
|
|
1821
|
+
? "complete"
|
|
1822
|
+
: intermission
|
|
1823
|
+
? "awaiting_human"
|
|
1824
|
+
: "partial";
|
|
1825
|
+
await timed(recorder, { stage: "manifest-persist" }, () => saveManifest(root, options.programId, manifest, { log }));
|
|
1826
|
+
const wholeProgram = await runWholeProgramStage({
|
|
1284
1827
|
root,
|
|
1285
1828
|
programId: options.programId,
|
|
1286
1829
|
manifest,
|
|
1287
|
-
|
|
1288
|
-
decisions,
|
|
1289
|
-
baseCommit,
|
|
1290
|
-
decider,
|
|
1830
|
+
config,
|
|
1291
1831
|
agentRunner,
|
|
1292
1832
|
permits,
|
|
1293
1833
|
git,
|
|
1294
1834
|
isRepository,
|
|
1835
|
+
results,
|
|
1836
|
+
authorResult,
|
|
1837
|
+
runStartCommit,
|
|
1838
|
+
decider,
|
|
1295
1839
|
reviewed,
|
|
1296
|
-
onTranscript: (transcript) => journalDenials(workstreamId, "decider", transcript),
|
|
1297
1840
|
triaged,
|
|
1298
1841
|
now,
|
|
1299
1842
|
log,
|
|
1300
|
-
|
|
1843
|
+
recorder,
|
|
1301
1844
|
});
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1845
|
+
const ledger = await readDecisionLedger(root, options.programId);
|
|
1846
|
+
const escalations = escalatedRecords(ledger);
|
|
1847
|
+
// Runs after the ledger read above, so its brief already sees this run's
|
|
1848
|
+
// own whole-program findings/rulings, and before report assembly, so its
|
|
1849
|
+
// section renders next to WS-04's own (§3.2 of the WS-05 spec). Read-only
|
|
1850
|
+
// and fail-open: nothing here can change `results`, `ledger`, the run's
|
|
1851
|
+
// outcome, or the exit code.
|
|
1852
|
+
const causalAnalysis = await runCausalAnalysisStage({
|
|
1305
1853
|
root,
|
|
1306
1854
|
programId: options.programId,
|
|
1307
1855
|
manifest,
|
|
1308
|
-
|
|
1309
|
-
findings,
|
|
1310
|
-
baseCommit,
|
|
1311
|
-
decider,
|
|
1856
|
+
config,
|
|
1312
1857
|
agentRunner,
|
|
1313
1858
|
permits,
|
|
1314
1859
|
git,
|
|
1315
1860
|
isRepository,
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
triaged,
|
|
1320
|
-
now,
|
|
1861
|
+
results,
|
|
1862
|
+
runStartCommit,
|
|
1863
|
+
ledger,
|
|
1321
1864
|
log,
|
|
1865
|
+
recorder,
|
|
1322
1866
|
});
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1867
|
+
const reportPath = runReportPath(root, options.programId);
|
|
1868
|
+
// Under concurrency (WS-02+) this is what guarantees no manifest or
|
|
1869
|
+
// ledger write is still in flight while docs/programs is staged below;
|
|
1870
|
+
// today, with nothing concurrent, it is a no-op that must exist anyway.
|
|
1871
|
+
await drainProgramWrites(programWriteKey(root, options.programId));
|
|
1872
|
+
// Rendering is the last stage the recorder can actually observe (SC-03):
|
|
1873
|
+
// captured as its own span, nested under the run span, rather than folded
|
|
1874
|
+
// silently into it. Kept in its own try/catch, separate from the write and
|
|
1875
|
+
// commit below, so a rendering bug still lets `finalize()` run and still
|
|
1876
|
+
// lets the write/commit attempt proceed against whatever report content
|
|
1877
|
+
// *is* available — a rendering failure is a logged line, not a thrown run,
|
|
1878
|
+
// exactly like the write/commit failure path below (SC-05, SC-12).
|
|
1879
|
+
let reportContent;
|
|
1880
|
+
try {
|
|
1881
|
+
const reportSpan = recorder.span({ stage: "report-assembly" });
|
|
1882
|
+
// `triaged` is the run-local set of subject ids the decider actually
|
|
1883
|
+
// ruled on this run (built up across the authoring, build, and
|
|
1884
|
+
// whole-program stages, see its declaration above) — exactly the
|
|
1885
|
+
// `triagedThisRun` basis the "this run" triage ratio needs, since the
|
|
1886
|
+
// projected ledger carries no run identifier of its own. It excludes
|
|
1887
|
+
// ids that were merely sent but whose invocation failed or returned no
|
|
1888
|
+
// valid verdict.
|
|
1889
|
+
// Computed before renderRunReport (§3.2/§3.9): renderAnalyticsSection is
|
|
1890
|
+
// WS-04's own never-throwing isolation boundary, so an analytics bug can
|
|
1891
|
+
// never propagate into this try's catch below and leave reportContent
|
|
1892
|
+
// undefined — which would suppress the entire report (SC-13). The
|
|
1893
|
+
// recorder's snapshot, not the finalized file, is the only artifact that
|
|
1894
|
+
// exists at this point in the sequence (finalize() runs later, below).
|
|
1895
|
+
// Passed as a thunk, not called eagerly here: `recorder.snapshot()` is
|
|
1896
|
+
// documented as non-throwing, but evaluating it as a plain argument would
|
|
1897
|
+
// run it *before* renderAnalyticsSection's own try/catch is reached, so a
|
|
1898
|
+
// violation of that contract would still escape this boundary.
|
|
1899
|
+
const analyticsSection = renderAnalyticsSection(() => recorder.snapshot());
|
|
1900
|
+
// The causal section's own "observed measurements" layer is rendered
|
|
1901
|
+
// from a *later* snapshot than the one the analyzer was briefed with
|
|
1902
|
+
// (`causalAnalysis` above) — one that now includes the analyzer's own
|
|
1903
|
+
// `causal-analysis` span — exactly as WS-04's own section above it does.
|
|
1904
|
+
// The two differ by exactly the analyzer's own cost; the section states
|
|
1905
|
+
// that boundary explicitly (§3.2).
|
|
1906
|
+
const causalAnalysisSection = renderCausalAnalysisSection(causalAnalysis.outcome, () => recorder.snapshot());
|
|
1907
|
+
reportContent = renderRunReport(manifest, results, ledger, triaged, authorResult, now(), wholeProgram, preflight, options.resumeCommand, manifestMergeCount(root, options.programId) > mergesAtStart, analyticsSection, causalAnalysisSection);
|
|
1908
|
+
reportSpan.close({ outcome: "success" });
|
|
1349
1909
|
}
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1910
|
+
catch (error) {
|
|
1911
|
+
log(`run report: could not render ${reportPath}: ${error.message}`);
|
|
1912
|
+
}
|
|
1913
|
+
// The `report-commit` span (the stage STAGE_BUCKET already reserves for
|
|
1914
|
+
// exactly this) covers writing the report to disk and committing
|
|
1915
|
+
// docs/programs, nested explicitly under `runSpan` rather than the
|
|
1916
|
+
// implicit stack, since `reportSpan` above has already closed and popped
|
|
1917
|
+
// itself. Both this span and its parent `runSpan` are opened/left open
|
|
1918
|
+
// rather than closed here, because the write and the commit they describe
|
|
1919
|
+
// happen *after* `finalize()` below persists this very artifact — closing
|
|
1920
|
+
// them now would stamp an `endOffsetMs` that is a lie the moment the write
|
|
1921
|
+
// or commit takes any measurable time at all. An absent `endOffsetMs` is
|
|
1922
|
+
// exactly the schema's existing "true end not observed" semantics; the
|
|
1923
|
+
// run's actual outcome classification is recorded as a point observation
|
|
1924
|
+
// instead of a span-close result, since it — unlike an interval's end — is
|
|
1925
|
+
// a fact already known at this instant.
|
|
1926
|
+
runSpan.child({ stage: "report-commit" });
|
|
1927
|
+
recorder.point({
|
|
1928
|
+
kind: "run-outcome",
|
|
1929
|
+
coverage: "observed",
|
|
1930
|
+
dimensions: { stage: "run", outcome: complete ? "success" : intermission ? "no-op" : "failed" },
|
|
1371
1931
|
});
|
|
1372
|
-
//
|
|
1373
|
-
//
|
|
1374
|
-
//
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1932
|
+
// Finalized once every stage the recorder can observe closing has closed
|
|
1933
|
+
// (report-assembly above; report-commit and run intentionally remain
|
|
1934
|
+
// open — see above). Still runs before the report is written and
|
|
1935
|
+
// committed, so the artifact rides into the same `docs/programs` commit
|
|
1936
|
+
// (SC-02, SC-14). Fail-open: never throws, never delays or blocks the
|
|
1937
|
+
// report below.
|
|
1938
|
+
const finalizeStart = monotonic();
|
|
1939
|
+
await recorder.finalize();
|
|
1940
|
+
// `finalize()` seals the point/span arrays before its own duration is
|
|
1941
|
+
// known, so an overhead measure around it can never live in the artifact
|
|
1942
|
+
// it measures (§3.4.d) — emitted after, as a non-conserved sidecar-only
|
|
1943
|
+
// point: no `stage`, never summed into any bucket.
|
|
1944
|
+
recorder.point({
|
|
1945
|
+
kind: "analytics-overhead",
|
|
1946
|
+
coverage: "observed",
|
|
1947
|
+
value: monotonic() - finalizeStart,
|
|
1948
|
+
unit: "ms",
|
|
1949
|
+
label: "finalize",
|
|
1950
|
+
});
|
|
1951
|
+
// Every workstream verdict, `complete`, and the exit-code mapping are
|
|
1952
|
+
// already settled above; writing and committing the report is bookkeeping
|
|
1953
|
+
// that must not be able to reject `runProgram` after the fact (SC-05,
|
|
1954
|
+
// SC-12) — a full disk or an EISDIR here is a logged line, not a thrown
|
|
1955
|
+
// run.
|
|
1956
|
+
if (reportContent !== undefined) {
|
|
1957
|
+
try {
|
|
1958
|
+
await writeFile(reportPath, reportContent, "utf8");
|
|
1959
|
+
if (isRepository) {
|
|
1960
|
+
// Guarded by `rootIndexMutex` (WS-05 §3.7) like every other git-index
|
|
1961
|
+
// operation at root; by this point the unified scheduler has fully
|
|
1962
|
+
// drained, so nothing else can be contending for it, but the lock
|
|
1963
|
+
// discipline is stated once and applied everywhere rather than
|
|
1964
|
+
// special-cased at the one call site that "happens to" run alone.
|
|
1965
|
+
await rootIndexMutex.runExclusive(() => git.commitPaths(root, `nightshift(${options.programId}): run report and decision ledger`, ["docs/programs"]));
|
|
1966
|
+
}
|
|
1967
|
+
// A best-effort forensic breadcrumb for the one boundary the
|
|
1968
|
+
// already-committed canonical artifact can never describe: `point()`
|
|
1969
|
+
// still appends to the gitignored sidecar once armed even though
|
|
1970
|
+
// `finalize()` has already run, and never touches the canonical JSON
|
|
1971
|
+
// again, so the working tree stays clean after a normal run.
|
|
1972
|
+
recorder.point({
|
|
1973
|
+
kind: "report-commit-result",
|
|
1974
|
+
coverage: "observed",
|
|
1975
|
+
dimensions: { stage: "report-commit", outcome: "success" },
|
|
1976
|
+
});
|
|
1977
|
+
}
|
|
1978
|
+
catch (error) {
|
|
1979
|
+
log(`run report: could not write or commit ${reportPath}: ${error.message}`);
|
|
1980
|
+
recorder.point({
|
|
1981
|
+
kind: "report-commit-result",
|
|
1982
|
+
coverage: "observed",
|
|
1983
|
+
dimensions: { stage: "report-commit", outcome: "failed" },
|
|
1984
|
+
});
|
|
1985
|
+
}
|
|
1986
|
+
}
|
|
1987
|
+
log(`run report: ${reportPath}`);
|
|
1395
1988
|
return {
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1989
|
+
programId: options.programId,
|
|
1990
|
+
complete,
|
|
1991
|
+
intermission,
|
|
1992
|
+
workstreams: results,
|
|
1993
|
+
escalations,
|
|
1994
|
+
reportPath,
|
|
1995
|
+
wholeProgramReview: wholeProgram,
|
|
1996
|
+
causalAnalysis,
|
|
1399
1997
|
};
|
|
1998
|
+
async function runWorkstream(workstream,
|
|
1999
|
+
/**
|
|
2000
|
+
* The decision ledger, read once per build flow (WS-05 §3.4) rather
|
|
2001
|
+
* than once for the whole run — a build's brief then sees every
|
|
2002
|
+
* decision journalled before it started, including its own
|
|
2003
|
+
* dependencies' authoring, without breaking WS-04's
|
|
2004
|
+
* `scheduled − ready === concurrency-cap` identity (the read happens
|
|
2005
|
+
* strictly after the `workstream-scheduled` emission, in the caller).
|
|
2006
|
+
*/
|
|
2007
|
+
ledger, flowOptions = {}) {
|
|
2008
|
+
// Explicit parenting (WS-01 §3.3): every span/point this flow opens
|
|
2009
|
+
// parents to the run span itself, never to whatever happens to be open
|
|
2010
|
+
// on the implicit stack — the seam future concurrent workstreams (WS-04)
|
|
2011
|
+
// stand on. `[WS-xx] `-prefixes every line this flow logs.
|
|
2012
|
+
const scope = runSpan.scope({ workstream: workstream.id });
|
|
2013
|
+
const log = workstreamLog(baseLog, workstream.id);
|
|
2014
|
+
const base = {
|
|
2015
|
+
id: workstream.id,
|
|
2016
|
+
name: workstream.name,
|
|
2017
|
+
outcome: { status: "failed", reason: "not attempted" },
|
|
2018
|
+
decisionIds: [],
|
|
2019
|
+
decisionErrors: [],
|
|
2020
|
+
buildAgentCommand: describeAgent(agent),
|
|
2021
|
+
};
|
|
2022
|
+
/** Every build-role spawn's transcript, for the report's per-workstream
|
|
2023
|
+
* Commands subsection (WS-02). Reviewer/decider spawns are excluded. */
|
|
2024
|
+
function recordTranscript(transcript) {
|
|
2025
|
+
if (transcript)
|
|
2026
|
+
(base.transcripts ??= []).push(transcript);
|
|
2027
|
+
}
|
|
2028
|
+
/** Names the JSONL file under build-logs/<programId>/, distinctly per spawn label. */
|
|
2029
|
+
function transcriptSink(label) {
|
|
2030
|
+
return { root, programId: options.programId, label: `${workstream.id}-${label}`, log };
|
|
2031
|
+
}
|
|
2032
|
+
let spec;
|
|
2033
|
+
try {
|
|
2034
|
+
spec = await readFile(join(root, workstream.taskFile), "utf8");
|
|
2035
|
+
}
|
|
2036
|
+
catch {
|
|
2037
|
+
base.outcome = {
|
|
2038
|
+
status: "parked",
|
|
2039
|
+
reason: `spec not found at ${workstream.taskFile}`,
|
|
2040
|
+
};
|
|
2041
|
+
return base;
|
|
2042
|
+
}
|
|
2043
|
+
workstream.status = "in_progress";
|
|
2044
|
+
await timed(scope, { stage: "manifest-persist" }, () => saveManifest(root, options.programId, manifest, { log }));
|
|
2045
|
+
// One read, not two (WS-04, §3.5): the program-branch tip is the
|
|
2046
|
+
// worktree's base *and* the decision anchor (`baseCommit`) — the same
|
|
2047
|
+
// fact under two names, resolved through `WorktreeOps`, never
|
|
2048
|
+
// `GitOps.currentCommit`. Concurrently, a second independent read here
|
|
2049
|
+
// could observe a sibling's integration land in between, giving this
|
|
2050
|
+
// workstream a `baseCommit` newer than the tree its worktree was
|
|
2051
|
+
// actually built from — the reviewer and decider would then see a
|
|
2052
|
+
// sibling's change as though this workstream had deleted it. Preferring
|
|
2053
|
+
// `flowOptions.scheduledFrom` when present means the scheduler's own
|
|
2054
|
+
// read (taken with no `await` before its `workstream-scheduled`
|
|
2055
|
+
// emission) is the only read on every scheduled path; the fallback here
|
|
2056
|
+
// exists only for the one caller with no scheduling instant to offer.
|
|
2057
|
+
const programTip = isRepository
|
|
2058
|
+
? (flowOptions.scheduledFrom ?? (await worktrees.resolveCommit(root, "HEAD")))
|
|
2059
|
+
: undefined;
|
|
2060
|
+
const baseCommit = programTip;
|
|
2061
|
+
// The worktree lifecycle (WS-02, §3.3): every in-progress workstream
|
|
2062
|
+
// builds in its own dedicated worktree on its own branch, created from
|
|
2063
|
+
// the program-branch tip, never in the root checkout. `cwd` is what
|
|
2064
|
+
// every build-role spawn, verify command, and workstream-owned git
|
|
2065
|
+
// operation below uses from here on; `root` stays reserved for the
|
|
2066
|
+
// manifest, ledger, transcripts, and verify-output evidence.
|
|
2067
|
+
let cwd = root;
|
|
2068
|
+
let worktreeBranchOfThis;
|
|
2069
|
+
if (isRepository) {
|
|
2070
|
+
if (worktreeBase === undefined) {
|
|
2071
|
+
// Unreachable in production (worktreeBase is always resolved inside
|
|
2072
|
+
// the isRepository branch above, before any workstream runs) —
|
|
2073
|
+
// guarded rather than asserted so a future refactor fails loudly.
|
|
2074
|
+
throw new Error("worktree base was never resolved for a git repository run");
|
|
2075
|
+
}
|
|
2076
|
+
// Routed through the same `PrepareWorktreeResult` shape and the same
|
|
2077
|
+
// failure handling below as a real `prepareWorktree` failure, rather
|
|
2078
|
+
// than a second early-return with its own `saveManifest` call —
|
|
2079
|
+
// the tip could not resolve, so there is nothing to build from, but
|
|
2080
|
+
// that is still exactly one more reason a worktree could not be
|
|
2081
|
+
// prepared (WS-04, §3.5).
|
|
2082
|
+
const prepared = programTip === undefined
|
|
2083
|
+
? { ok: false, reason: "cannot resolve the program-branch tip", created: false }
|
|
2084
|
+
: await prepareWorktree({
|
|
2085
|
+
root,
|
|
2086
|
+
programId: options.programId,
|
|
2087
|
+
workstreamId: workstream.id,
|
|
2088
|
+
worktrees,
|
|
2089
|
+
worktreeBase,
|
|
2090
|
+
setupRunner,
|
|
2091
|
+
log,
|
|
2092
|
+
programTip,
|
|
2093
|
+
});
|
|
2094
|
+
if (!prepared.ok) {
|
|
2095
|
+
workstream.status = "failed";
|
|
2096
|
+
if (prepared.created)
|
|
2097
|
+
base.worktreePath = prepared.cwd;
|
|
2098
|
+
base.outcome = { status: "failed", reason: prepared.reason };
|
|
2099
|
+
await timed(scope, { stage: "manifest-persist" }, () => saveManifest(root, options.programId, manifest, { log }));
|
|
2100
|
+
return base;
|
|
2101
|
+
}
|
|
2102
|
+
cwd = prepared.cwd;
|
|
2103
|
+
worktreeBranchOfThis = prepared.branch;
|
|
2104
|
+
base.worktreePath = cwd;
|
|
2105
|
+
}
|
|
2106
|
+
let priorFailure;
|
|
2107
|
+
let priorDiagnosis;
|
|
2108
|
+
let implementerFingerprint;
|
|
2109
|
+
// WS-08: last-wins across every attempt in this workstream's build —
|
|
2110
|
+
// the fix seats resume whichever attempt most recently reported a
|
|
2111
|
+
// session id, which is the attempt whose commit they are fixing.
|
|
2112
|
+
// Capturing on every attempt (including recovery) is not resuming: the
|
|
2113
|
+
// recovery seat still spawns fresh; its handle is only ever a candidate
|
|
2114
|
+
// for the fix seats, subject to invokeAgent's provider/invocation checks.
|
|
2115
|
+
let buildSession;
|
|
2116
|
+
const attempts = [
|
|
2117
|
+
{ agent, label: "implementer", role: "agent" },
|
|
2118
|
+
];
|
|
2119
|
+
if (recovery && !recovery.borrowedImplementer) {
|
|
2120
|
+
attempts.push({ agent: recovery.agent, label: "recovery", role: "recoveryAgent" });
|
|
2121
|
+
// The third seat exists only when a reviewer can inform it. An
|
|
2122
|
+
// uninformed retry has already been spent (recovery); running the
|
|
2123
|
+
// roster again blind is a coin flip the ledger should hear about
|
|
2124
|
+
// instead. Alternation is deliberate — the implementer returns with
|
|
2125
|
+
// the reviewer's diagnosis in hand, a composition neither prior
|
|
2126
|
+
// attempt had.
|
|
2127
|
+
if (reviewer)
|
|
2128
|
+
attempts.push({ agent, label: "informed retry", role: "agent" });
|
|
2129
|
+
}
|
|
2130
|
+
for (const [index, attempt] of attempts.entries()) {
|
|
2131
|
+
log(`${workstream.name}: ${attempt.label} attempt`);
|
|
2132
|
+
const { brief, components } = implementerBrief(manifest, workstream, spec, ledger, config.permits?.policy, priorFailure, priorDiagnosis);
|
|
2133
|
+
for (const point of promptComponentSizePoints(components, { role: attempt.role })) {
|
|
2134
|
+
scope.point(point);
|
|
2135
|
+
}
|
|
2136
|
+
// Why this repetition happened (WS-02, SC-05/SC-06): the first attempt
|
|
2137
|
+
// is always `initial`; a later seat is `informed-by-diagnosis` when the
|
|
2138
|
+
// prior attempt's failure was read by the reviewer, else a plain
|
|
2139
|
+
// `verify-failure` retry (a deterministic failure skips diagnosis —
|
|
2140
|
+
// see below — so the next seat still spawns, just undiagnosed).
|
|
2141
|
+
const seatStage = STAGE_FOR_SEAT[attempt.label] ?? "implementer";
|
|
2142
|
+
const attemptReason = index === 0 ? "initial" : priorDiagnosis !== undefined ? "informed-by-diagnosis" : "verify-failure";
|
|
2143
|
+
/** This attempt's eventual `attempt-outcome` disposition (WS-02, §3.3)
|
|
2144
|
+
* — distinct from the spawn span's own success/failed outcome (an
|
|
2145
|
+
* undelivered-brief or green-but-empty attempt spawns "successfully"
|
|
2146
|
+
* but disposes as `no-op`/`discarded`). Set once, at the point each
|
|
2147
|
+
* failure reason is detected below; `undefined` means the attempt was
|
|
2148
|
+
* kept. */
|
|
2149
|
+
let dispositionOutcome;
|
|
2150
|
+
const emitAttemptOutcome = (outcome, reasonOverride) => {
|
|
2151
|
+
scope.point({
|
|
2152
|
+
kind: "attempt-outcome",
|
|
2153
|
+
coverage: "observed",
|
|
2154
|
+
dimensions: {
|
|
2155
|
+
stage: seatStage,
|
|
2156
|
+
workstream: workstream.id,
|
|
2157
|
+
role: attempt.role,
|
|
2158
|
+
attemptSeat: attempt.label,
|
|
2159
|
+
attemptIndex: index,
|
|
2160
|
+
attemptReason: reasonOverride ?? attemptReason,
|
|
2161
|
+
outcome,
|
|
2162
|
+
},
|
|
2163
|
+
});
|
|
2164
|
+
};
|
|
2165
|
+
let invocation;
|
|
2166
|
+
let spawnFailure;
|
|
2167
|
+
try {
|
|
2168
|
+
invocation = await invokeAgent(agentRunner, attempt.agent, brief, cwd, permits, attempt.role, transcriptSink(attempt.label.replaceAll(" ", "-")), scope, { stage: seatStage, workstream: workstream.id, attemptSeat: attempt.label, attemptIndex: index, attemptReason });
|
|
2169
|
+
if (invocation.session)
|
|
2170
|
+
buildSession = invocation.session;
|
|
2171
|
+
}
|
|
2172
|
+
catch (error) {
|
|
2173
|
+
// The agent process never started — its command is wrong, missing,
|
|
2174
|
+
// or unrunnable. Nothing was built; this is the captured
|
|
2175
|
+
// silent-exit incident (SC-12). Make it a diagnosed failure via the
|
|
2176
|
+
// existing retry/park machinery below, never a process exit.
|
|
2177
|
+
spawnFailure =
|
|
2178
|
+
`the ${attempt.label} agent could not be spawned: ` +
|
|
2179
|
+
`${error.message}. The configured command ` +
|
|
2180
|
+
`\`${describeAgent(attempt.agent)}\` did not start, so nothing was built.`;
|
|
2181
|
+
invocation = { exitCode: 1, output: "" };
|
|
2182
|
+
}
|
|
2183
|
+
recordTranscript(invocation.transcript);
|
|
2184
|
+
const summary = resolveSummary(invocation.output);
|
|
2185
|
+
base.summary = summary.text;
|
|
2186
|
+
await journalDenials(workstream.id, attempt.label, invocation.transcript, summary.needsPermission, scope);
|
|
2187
|
+
const parsed = extractDecisions(invocation.output);
|
|
2188
|
+
base.decisionErrors.push(...parsed.errors);
|
|
2189
|
+
await journalDecisions(workstream, parsed.decisions, baseCommit, scope);
|
|
2190
|
+
base.decisionIds = parsed.decisions.map((decision) => decisionFingerprint(workstream.id, decision));
|
|
2191
|
+
let failure = await verifyAttempt(config, verifyRunner, cwd, root, invocation.exitCode, scope, "verification-command", {
|
|
2192
|
+
programId: options.programId,
|
|
2193
|
+
phase: `post-${attempt.label.replaceAll(" ", "-")}`,
|
|
2194
|
+
workstream: workstream.id,
|
|
2195
|
+
role: attempt.role,
|
|
2196
|
+
attemptSeat: attempt.label,
|
|
2197
|
+
attemptIndex: index,
|
|
2198
|
+
cache: verifyCache,
|
|
2199
|
+
});
|
|
2200
|
+
// Deterministic protocol failures the runner can see without a model:
|
|
2201
|
+
// no diagnosis is spent on them, because no read of the tree could add
|
|
2202
|
+
// anything to what the runner already knows.
|
|
2203
|
+
let deterministicFailure = false;
|
|
2204
|
+
// A spawn failure takes precedence over whatever verifyAttempt saw
|
|
2205
|
+
// (the untouched tree can verify green on its own) — no reviewer
|
|
2206
|
+
// diagnosis is spent reading an empty diff.
|
|
2207
|
+
if (spawnFailure !== undefined) {
|
|
2208
|
+
failure = spawnFailure;
|
|
2209
|
+
deterministicFailure = true;
|
|
2210
|
+
dispositionOutcome = "failed";
|
|
2211
|
+
}
|
|
2212
|
+
// A brief that never reached the agent's stdin means the process
|
|
2213
|
+
// crashed at startup — whatever it printed was not an answer to the
|
|
2214
|
+
// brief, even when it exited 0.
|
|
2215
|
+
if (failure === undefined && invocation.inputError !== undefined) {
|
|
2216
|
+
failure =
|
|
2217
|
+
`the brief could not be delivered to the agent's stdin ` +
|
|
2218
|
+
`(${invocation.inputError}) — the agent likely crashed at startup, ` +
|
|
2219
|
+
`so its reply answered nothing.`;
|
|
2220
|
+
deterministicFailure = true;
|
|
2221
|
+
dispositionOutcome = "no-op";
|
|
2222
|
+
}
|
|
2223
|
+
// A green verify over an unchanged tree is just the baseline passing
|
|
2224
|
+
// again, not built work. Without this, an agent that crashes at
|
|
2225
|
+
// startup while exiting 0 (or replies without working) sweeps only
|
|
2226
|
+
// the runner's own in_progress manifest write into a metadata-only
|
|
2227
|
+
// commit and walks through the gate as "complete". A reply that never
|
|
2228
|
+
// produces the work is a protocol failure, not an empty result.
|
|
2229
|
+
if (failure === undefined && isRepository) {
|
|
2230
|
+
const dirty = await git.dirtyPaths(cwd);
|
|
2231
|
+
const meaningful = dirty.some((path) => !isRunnerOwnedArtifactPath(path));
|
|
2232
|
+
if (!meaningful) {
|
|
2233
|
+
failure =
|
|
2234
|
+
"the attempt verified green but changed nothing outside " +
|
|
2235
|
+
"docs/programs/ — nothing was built, and the passing verify was " +
|
|
2236
|
+
"the untouched baseline. A no-op attempt is a protocol failure " +
|
|
2237
|
+
"(the agent may have crashed at startup or replied without " +
|
|
2238
|
+
"working), not a completed workstream.";
|
|
2239
|
+
deterministicFailure = true;
|
|
2240
|
+
dispositionOutcome = "discarded";
|
|
2241
|
+
}
|
|
2242
|
+
}
|
|
2243
|
+
// A genuine verify failure (not spawn/inputError/green-empty) disposes
|
|
2244
|
+
// as `failed`, same as every other undetermined failure reason.
|
|
2245
|
+
if (failure !== undefined && dispositionOutcome === undefined) {
|
|
2246
|
+
dispositionOutcome = "failed";
|
|
2247
|
+
}
|
|
2248
|
+
if (failure === undefined) {
|
|
2249
|
+
let c0;
|
|
2250
|
+
if (isRepository) {
|
|
2251
|
+
// Unlike the authoring, run-report, and replay commits, this one
|
|
2252
|
+
// stays a whole-tree sweep: an implementing agent touches whatever
|
|
2253
|
+
// files the work required, and that set is exactly what the
|
|
2254
|
+
// runner cannot know in advance. c0 is the green anchor the test
|
|
2255
|
+
// critique's fix loop resets to on a failing fix (SC-07).
|
|
2256
|
+
c0 = await timed(scope, { stage: "git-commit", workstream: workstream.id }, () => git.commitAll(cwd, `nightshift(${options.programId}): ${workstream.id} ${workstream.name}`, ["docs/programs"]), commitEvidence);
|
|
2257
|
+
}
|
|
2258
|
+
// Captured before the pass runs so a fallback's report line can name
|
|
2259
|
+
// the producer's command (§3.11) — an approximation good for the
|
|
2260
|
+
// common one-round case; a later round's true producer is whichever
|
|
2261
|
+
// round most recently resumed, which the pass itself tracks
|
|
2262
|
+
// internally but does not need to surface here.
|
|
2263
|
+
const criticEnteringSession = buildSession;
|
|
2264
|
+
const critique = isRepository && c0 !== undefined
|
|
2265
|
+
? await runTestCritique({
|
|
2266
|
+
root,
|
|
2267
|
+
cwd,
|
|
2268
|
+
manifest,
|
|
2269
|
+
workstream,
|
|
2270
|
+
spec,
|
|
2271
|
+
config,
|
|
2272
|
+
agentRunner,
|
|
2273
|
+
permits,
|
|
2274
|
+
verifyRunner,
|
|
2275
|
+
git,
|
|
2276
|
+
reviewer,
|
|
2277
|
+
agent,
|
|
2278
|
+
deciderConfigured: decider !== undefined,
|
|
2279
|
+
baseCommit,
|
|
2280
|
+
greenCommit: c0,
|
|
2281
|
+
now,
|
|
2282
|
+
log,
|
|
2283
|
+
transcriptSink,
|
|
2284
|
+
recorder: scope,
|
|
2285
|
+
verifyCache,
|
|
2286
|
+
triage: (events, triageBaseCommit) => reviewWorkstreamFindings(workstream.id, cwd, events, triageBaseCommit, scope),
|
|
2287
|
+
...(buildSession ? { buildSession } : {}),
|
|
2288
|
+
})
|
|
2289
|
+
: undefined;
|
|
2290
|
+
if (critique?.continuity) {
|
|
2291
|
+
(base.sessionContinuity ??= []).push(...critique.continuity.map((continuity) => ({
|
|
2292
|
+
seat: "test critique fix",
|
|
2293
|
+
continuity,
|
|
2294
|
+
producerCommand: criticEnteringSession?.command,
|
|
2295
|
+
seatCommand: agent.command,
|
|
2296
|
+
})));
|
|
2297
|
+
}
|
|
2298
|
+
// The workstream's own final verified commit, in its worktree —
|
|
2299
|
+
// after any kept fix, that is the fix commit, not the earlier green
|
|
2300
|
+
// one; already includes any triaged fix-now commit, since the merged
|
|
2301
|
+
// fix runs inside the critique loop itself (WS-09). Not what lands
|
|
2302
|
+
// in the manifest: under WS-02 that is the *integration* commit,
|
|
2303
|
+
// recorded only after integration succeeds, below.
|
|
2304
|
+
const finalCommit = critique?.finalCommit ?? c0;
|
|
2305
|
+
if (critique && critique.stageErrors.length > 0) {
|
|
2306
|
+
(base.stageErrors ??= []).push(...critique.stageErrors);
|
|
2307
|
+
}
|
|
2308
|
+
for (const transcript of critique?.transcripts ?? [])
|
|
2309
|
+
recordTranscript(transcript);
|
|
2310
|
+
{
|
|
2311
|
+
const spawnErrors = await reviewWorkstreamDecisions(workstream.id, cwd, parsed.decisions, baseCommit, scope);
|
|
2312
|
+
if (spawnErrors.length > 0)
|
|
2313
|
+
(base.stageErrors ??= []).push(...spawnErrors);
|
|
2314
|
+
}
|
|
2315
|
+
if (critique)
|
|
2316
|
+
base.testCritique = critique.outcome;
|
|
2317
|
+
base.testCritiqueDiffClipped = critique?.diffClipped ?? false;
|
|
2318
|
+
// Integration (WS-02, §3.5-§3.6): the last thing this flow does on
|
|
2319
|
+
// the green path. Not reached when this is not a git repository —
|
|
2320
|
+
// there is nothing to integrate, and `finalCommit`/`cwd` are already
|
|
2321
|
+
// the root's own.
|
|
2322
|
+
if (!isRepository || finalCommit === undefined) {
|
|
2323
|
+
workstream.status = "complete";
|
|
2324
|
+
if (finalCommit !== undefined)
|
|
2325
|
+
workstream.commit = finalCommit;
|
|
2326
|
+
await timed(scope, { stage: "manifest-persist" }, () => saveManifest(root, options.programId, manifest, { log }));
|
|
2327
|
+
base.outcome = {
|
|
2328
|
+
status: "complete",
|
|
2329
|
+
...(finalCommit === undefined ? {} : { commit: finalCommit }),
|
|
2330
|
+
};
|
|
2331
|
+
emitAttemptOutcome("success");
|
|
2332
|
+
return base;
|
|
2333
|
+
}
|
|
2334
|
+
const recoveryAgentForIntegration = recovery && !recovery.borrowedImplementer ? recovery.agent : agent;
|
|
2335
|
+
// Serialised through the one run-scoped integration mutex (WS-04,
|
|
2336
|
+
// §3.5): the mutex spans the whole call, including the merge-forward
|
|
2337
|
+
// and its re-verify inside `integrateWorkstream` — not just the
|
|
2338
|
+
// final squash — because the merge-forward is the proof that the
|
|
2339
|
+
// squash is a fast-forward, and a sibling landing between them would
|
|
2340
|
+
// void that proof.
|
|
2341
|
+
const mutexWaitStart = monotonic();
|
|
2342
|
+
const integration = await integrationMutex.runExclusive(async () => {
|
|
2343
|
+
const mutexWaitedMs = monotonic() - mutexWaitStart;
|
|
2344
|
+
if (mutexWaitedMs > 0) {
|
|
2345
|
+
recordWorkstreamWaiting(recorder, workstream.id, {
|
|
2346
|
+
cause: "integration-mutex",
|
|
2347
|
+
waitedMs: mutexWaitedMs,
|
|
2348
|
+
});
|
|
2349
|
+
}
|
|
2350
|
+
return integrateWorkstream({
|
|
2351
|
+
root,
|
|
2352
|
+
programId: options.programId,
|
|
2353
|
+
workstream,
|
|
2354
|
+
cwd,
|
|
2355
|
+
branch: worktreeBranchOfThis ?? worktreeBranchName(options.programId, workstream.id),
|
|
2356
|
+
finalCommit,
|
|
2357
|
+
worktrees,
|
|
2358
|
+
git,
|
|
2359
|
+
config,
|
|
2360
|
+
verifyRunner,
|
|
2361
|
+
setupRunner,
|
|
2362
|
+
verifyCache,
|
|
2363
|
+
agentRunner,
|
|
2364
|
+
permits,
|
|
2365
|
+
recoveryAgent: recoveryAgentForIntegration,
|
|
2366
|
+
worktreeBase: worktreeBase,
|
|
2367
|
+
scope,
|
|
2368
|
+
log,
|
|
2369
|
+
transcriptSink,
|
|
2370
|
+
rootIndexMutex,
|
|
2371
|
+
rosterTaskFiles,
|
|
2372
|
+
});
|
|
2373
|
+
});
|
|
2374
|
+
for (const transcript of integration.transcripts)
|
|
2375
|
+
recordTranscript(transcript);
|
|
2376
|
+
if (integration.ok) {
|
|
2377
|
+
workstream.status = "complete";
|
|
2378
|
+
if (integration.commit !== undefined)
|
|
2379
|
+
workstream.commit = integration.commit;
|
|
2380
|
+
await timed(scope, { stage: "manifest-persist" }, () => saveManifest(root, options.programId, manifest, { log }));
|
|
2381
|
+
for (const { path, branch: cleanupBranch } of integration.cleanup) {
|
|
2382
|
+
await worktrees.remove(root, path);
|
|
2383
|
+
await worktrees.deleteBranch(root, cleanupBranch);
|
|
2384
|
+
}
|
|
2385
|
+
await git.deleteRef(root, retainedRefName(options.programId, workstream.id));
|
|
2386
|
+
delete base.worktreePath;
|
|
2387
|
+
base.outcome = {
|
|
2388
|
+
status: "complete",
|
|
2389
|
+
...(integration.commit === undefined ? {} : { commit: integration.commit }),
|
|
2390
|
+
};
|
|
2391
|
+
emitAttemptOutcome("success");
|
|
2392
|
+
return base;
|
|
2393
|
+
}
|
|
2394
|
+
workstream.status = "failed";
|
|
2395
|
+
await timed(scope, { stage: "manifest-persist" }, () => saveManifest(root, options.programId, manifest, { log }));
|
|
2396
|
+
base.retainedRef = integration.retainedRef;
|
|
2397
|
+
base.outcome = {
|
|
2398
|
+
status: "failed",
|
|
2399
|
+
reason: `${integration.reason}\n\nWorktree(s) retained: ${integration.worktreePaths.join(", ")}.`,
|
|
2400
|
+
};
|
|
2401
|
+
emitAttemptOutcome("success");
|
|
2402
|
+
return base;
|
|
2403
|
+
}
|
|
2404
|
+
priorFailure = failure;
|
|
2405
|
+
log(`${attempt.label} attempt failed — ${failure}`);
|
|
2406
|
+
// A failure identical to the pre-run baseline (up to counts and
|
|
2407
|
+
// timings) was there before any workstream ran: environmental, not
|
|
2408
|
+
// this workstream's work. No retry or diagnosis is spent on it, and
|
|
2409
|
+
// the caller halts the run — every later workstream would be charged
|
|
2410
|
+
// for the same broken environment. Workstreams resumed with their own
|
|
2411
|
+
// leftover failure in the tree are exempt (see resumedIds).
|
|
2412
|
+
if (baselineFingerprint !== undefined &&
|
|
2413
|
+
!resumedIds.has(workstream.id) &&
|
|
2414
|
+
failureFingerprint(failure) === baselineFingerprint) {
|
|
2415
|
+
workstream.status = "failed";
|
|
2416
|
+
await timed(scope, { stage: "manifest-persist" }, () => saveManifest(root, options.programId, manifest, { log }));
|
|
2417
|
+
{
|
|
2418
|
+
const spawnErrors = await reviewWorkstreamDecisions(workstream.id, cwd, parsed.decisions, baseCommit, scope);
|
|
2419
|
+
if (spawnErrors.length > 0)
|
|
2420
|
+
(base.stageErrors ??= []).push(...spawnErrors);
|
|
2421
|
+
}
|
|
2422
|
+
base.environmental = true;
|
|
2423
|
+
base.outcome = {
|
|
2424
|
+
status: "failed",
|
|
2425
|
+
reason: failure +
|
|
2426
|
+
"\n\nThis failure is identical (up to counts and timings) to the " +
|
|
2427
|
+
"verification failure recorded on the untouched tree before the " +
|
|
2428
|
+
"run began — the environment was broken before this workstream " +
|
|
2429
|
+
"ran. No retry was spent, and the run halted here.",
|
|
2430
|
+
};
|
|
2431
|
+
emitAttemptOutcome("failed", "environmental");
|
|
2432
|
+
return base;
|
|
2433
|
+
}
|
|
2434
|
+
// The informed retry is spent only when the diagnosed recovery attempt
|
|
2435
|
+
// moved the failure at all. A failure reproduced identically after a
|
|
2436
|
+
// diagnosis says the roster is stuck, not unlucky — that belongs in
|
|
2437
|
+
// the ledger, not in a third spawn.
|
|
2438
|
+
const stuck = attempt.label === "recovery" &&
|
|
2439
|
+
index < attempts.length - 1 &&
|
|
2440
|
+
implementerFingerprint !== undefined &&
|
|
2441
|
+
failureFingerprint(failure) === implementerFingerprint;
|
|
2442
|
+
if (index === attempts.length - 1 || stuck) {
|
|
2443
|
+
workstream.status = "failed";
|
|
2444
|
+
await timed(scope, { stage: "manifest-persist" }, () => saveManifest(root, options.programId, manifest, { log }));
|
|
2445
|
+
// The work stays in the retained worktree (a resume salvages it to a
|
|
2446
|
+
// `refs/nightshift/…-wip` ref before reaping); decisions made on the
|
|
2447
|
+
// way to a failure are still journaled and still reviewable.
|
|
2448
|
+
{
|
|
2449
|
+
const spawnErrors = await reviewWorkstreamDecisions(workstream.id, cwd, parsed.decisions, baseCommit, scope);
|
|
2450
|
+
if (spawnErrors.length > 0)
|
|
2451
|
+
(base.stageErrors ??= []).push(...spawnErrors);
|
|
2452
|
+
}
|
|
2453
|
+
base.outcome = {
|
|
2454
|
+
status: "failed",
|
|
2455
|
+
reason: stuck
|
|
2456
|
+
? failure +
|
|
2457
|
+
"\n\nThe recovery attempt, briefed with a reviewer diagnosis, " +
|
|
2458
|
+
"reproduced the implementer's failure (identical up to counts " +
|
|
2459
|
+
"and timings), so the final retry was not spent."
|
|
2460
|
+
: failure,
|
|
2461
|
+
};
|
|
2462
|
+
emitAttemptOutcome(stuck ? "failed" : (dispositionOutcome ?? "failed"), stuck ? "stuck" : undefined);
|
|
2463
|
+
return base;
|
|
2464
|
+
}
|
|
2465
|
+
if (attempt.label === "implementer") {
|
|
2466
|
+
implementerFingerprint = failureFingerprint(failure);
|
|
2467
|
+
}
|
|
2468
|
+
// Diagnose the failure for the next attempt's brief. Overwrites any
|
|
2469
|
+
// prior diagnosis — a read of an older failure must not be pinned to
|
|
2470
|
+
// a newer one. Deterministic failures (no-op, undelivered brief) are
|
|
2471
|
+
// never diagnosed: the runner already knows exactly what happened,
|
|
2472
|
+
// and a reviewer read of an empty diff could only muddy it.
|
|
2473
|
+
if (reviewer && !deterministicFailure) {
|
|
2474
|
+
const diagnosed = await diagnoseFailure({
|
|
2475
|
+
cwd,
|
|
2476
|
+
manifest,
|
|
2477
|
+
workstream,
|
|
2478
|
+
spec,
|
|
2479
|
+
agentRunner,
|
|
2480
|
+
permits,
|
|
2481
|
+
reviewer,
|
|
2482
|
+
git,
|
|
2483
|
+
baseCommit,
|
|
2484
|
+
failure,
|
|
2485
|
+
attemptSeat: attempt.label,
|
|
2486
|
+
attemptIndex: index,
|
|
2487
|
+
observe: transcriptSink(`diagnose-${attempt.label.replaceAll(" ", "-")}`),
|
|
2488
|
+
recorder: scope,
|
|
2489
|
+
});
|
|
2490
|
+
priorDiagnosis = diagnosed.diagnosis;
|
|
2491
|
+
if (diagnosed.spawnError !== undefined) {
|
|
2492
|
+
(base.stageErrors ??= []).push(diagnosed.spawnError);
|
|
2493
|
+
log(diagnosed.spawnError);
|
|
2494
|
+
}
|
|
2495
|
+
await journalDenials(workstream.id, "reviewer", diagnosed.transcript, [], scope);
|
|
2496
|
+
}
|
|
2497
|
+
else {
|
|
2498
|
+
priorDiagnosis = undefined;
|
|
2499
|
+
}
|
|
2500
|
+
if (priorDiagnosis !== undefined) {
|
|
2501
|
+
(base.failureDiagnoses ??= []).push({
|
|
2502
|
+
attempt: attempt.label,
|
|
2503
|
+
verdict: priorDiagnosis,
|
|
2504
|
+
});
|
|
2505
|
+
log(`reviewer diagnosed the ${attempt.label} failure`);
|
|
2506
|
+
}
|
|
2507
|
+
emitAttemptOutcome(dispositionOutcome ?? "failed");
|
|
2508
|
+
}
|
|
2509
|
+
return base;
|
|
2510
|
+
}
|
|
2511
|
+
async function journalDecisions(workstream, decisions, baseCommit,
|
|
2512
|
+
/** The workstream flow's span scope (SC-09) — see reviewWorkstreamDecisions. */
|
|
2513
|
+
flowScope = recorder) {
|
|
2514
|
+
const log = workstreamLog(baseLog, workstream.id);
|
|
2515
|
+
const events = decisions.map((decision) => ({
|
|
2516
|
+
kind: "decision-recorded",
|
|
2517
|
+
at: now().toISOString(),
|
|
2518
|
+
id: decisionFingerprint(workstream.id, decision),
|
|
2519
|
+
workstream: workstream.id,
|
|
2520
|
+
decision,
|
|
2521
|
+
...(baseCommit === undefined ? {} : { baseCommit }),
|
|
2522
|
+
decidedBy: "implementer",
|
|
2523
|
+
}));
|
|
2524
|
+
await timed(flowScope, { stage: "ledger-persist", workstream: workstream.id }, () => appendLedgerEvents(root, options.programId, events));
|
|
2525
|
+
for (const decision of decisions) {
|
|
2526
|
+
log(`decision: ${decision.title} -> ${decision.chosen}`);
|
|
2527
|
+
}
|
|
2528
|
+
}
|
|
2529
|
+
/**
|
|
2530
|
+
* Journals `permission-denied` events for one spawn (WS-03) — every
|
|
2531
|
+
* workstream-attributed spawn calls this once, passing its own transcript
|
|
2532
|
+
* and (for build roles only) the agent's `needsPermission` self-reports.
|
|
2533
|
+
* A no-op when the spawn produced neither.
|
|
2534
|
+
*/
|
|
2535
|
+
async function journalDenials(workstreamId, attempt, transcript, needsPermission = [], flowScope = recorder) {
|
|
2536
|
+
const log = workstreamLog(baseLog, workstreamId);
|
|
2537
|
+
const events = buildDenialEvents(workstreamId, attempt, transcript?.denials ?? [], needsPermission, config.permits?.deny ?? [], now().toISOString());
|
|
2538
|
+
if (events.length === 0)
|
|
2539
|
+
return;
|
|
2540
|
+
await timed(flowScope, { stage: "ledger-persist", workstream: workstreamId }, () => appendLedgerEvents(root, options.programId, events));
|
|
2541
|
+
for (const event of events) {
|
|
2542
|
+
if (event.kind !== "permission-denied")
|
|
2543
|
+
continue;
|
|
2544
|
+
log(`${attempt}: denied \`${event.command}\` (${event.source})`);
|
|
2545
|
+
}
|
|
2546
|
+
}
|
|
2547
|
+
function reviewWorkstreamDecisions(workstreamId, cwd, decisions, baseCommit,
|
|
2548
|
+
/** The workstream flow's span scope (SC-09): decider spans parent here,
|
|
2549
|
+
* never to whatever span another concurrent flow happens to have open. */
|
|
2550
|
+
flowScope = recorder) {
|
|
2551
|
+
const log = workstreamLog(baseLog, workstreamId);
|
|
2552
|
+
return reviewDecisions({
|
|
2553
|
+
root,
|
|
2554
|
+
cwd,
|
|
2555
|
+
programId: options.programId,
|
|
2556
|
+
manifest,
|
|
2557
|
+
workstreamId,
|
|
2558
|
+
decisions,
|
|
2559
|
+
baseCommit,
|
|
2560
|
+
decider,
|
|
2561
|
+
agentRunner,
|
|
2562
|
+
permits,
|
|
2563
|
+
git,
|
|
2564
|
+
isRepository,
|
|
2565
|
+
reviewed,
|
|
2566
|
+
onTranscript: (transcript) => journalDenials(workstreamId, "decider", transcript, [], flowScope),
|
|
2567
|
+
triaged,
|
|
2568
|
+
now,
|
|
2569
|
+
log,
|
|
2570
|
+
observe: { root, programId: options.programId, label: `${workstreamId}-decider-decision`, log },
|
|
2571
|
+
recorder: flowScope,
|
|
2572
|
+
});
|
|
2573
|
+
}
|
|
2574
|
+
function reviewWorkstreamFindings(workstreamId, cwd, findings, baseCommit, flowScope = recorder) {
|
|
2575
|
+
const log = workstreamLog(baseLog, workstreamId);
|
|
2576
|
+
return triageFindings({
|
|
2577
|
+
root,
|
|
2578
|
+
cwd,
|
|
2579
|
+
programId: options.programId,
|
|
2580
|
+
manifest,
|
|
2581
|
+
workstreamId,
|
|
2582
|
+
findings,
|
|
2583
|
+
baseCommit,
|
|
2584
|
+
decider,
|
|
2585
|
+
agentRunner,
|
|
2586
|
+
permits,
|
|
2587
|
+
git,
|
|
2588
|
+
isRepository,
|
|
2589
|
+
reviewed,
|
|
2590
|
+
observe: { root, programId: options.programId, label: `${workstreamId}-decider-finding`, log },
|
|
2591
|
+
onTranscript: (transcript) => journalDenials(workstreamId, "decider", transcript, [], flowScope),
|
|
2592
|
+
triaged,
|
|
2593
|
+
now,
|
|
2594
|
+
log,
|
|
2595
|
+
recorder: flowScope,
|
|
2596
|
+
});
|
|
2597
|
+
}
|
|
1400
2598
|
}
|
|
1401
2599
|
}
|
|
1402
2600
|
/**
|
|
@@ -1487,6 +2685,7 @@ function notBuiltRoster(results, authorResult) {
|
|
|
1487
2685
|
*/
|
|
1488
2686
|
export async function runWholeProgramStage(args) {
|
|
1489
2687
|
const { root, programId, manifest, config, agentRunner, permits, git, isRepository, results, authorResult, runStartCommit, decider, reviewed, triaged, now, log, fs, } = args;
|
|
2688
|
+
const recorder = args.recorder ?? NOOP_RUN_RECORDER;
|
|
1490
2689
|
const errors = [];
|
|
1491
2690
|
let commit;
|
|
1492
2691
|
let commitSkipped = false;
|
|
@@ -1508,7 +2707,9 @@ export async function runWholeProgramStage(args) {
|
|
|
1508
2707
|
try {
|
|
1509
2708
|
safeLog("whole-program review: starting");
|
|
1510
2709
|
const base = await resolveProgramDiffBase({ root, git, isRepository, runStartCommit });
|
|
1511
|
-
const diff = base === undefined
|
|
2710
|
+
const diff = base === undefined
|
|
2711
|
+
? ""
|
|
2712
|
+
: await timed(recorder, { stage: "git-diff" }, () => git.diffSince(root, base));
|
|
1512
2713
|
const notBuilt = notBuiltRoster(results, authorResult);
|
|
1513
2714
|
const reviewCommit = isRepository ? await git.currentCommit(root) : undefined;
|
|
1514
2715
|
outcome = await runWholeProgramReview({
|
|
@@ -1523,6 +2724,7 @@ export async function runWholeProgramStage(args) {
|
|
|
1523
2724
|
...(notBuilt.length === 0 ? {} : { notBuilt }),
|
|
1524
2725
|
...(fs === undefined ? {} : { fs }),
|
|
1525
2726
|
log,
|
|
2727
|
+
recorder,
|
|
1526
2728
|
});
|
|
1527
2729
|
safeLog(outcome.status === "refreshed"
|
|
1528
2730
|
? `whole-program review: refreshed ${AS_BUILT_PATH}` +
|
|
@@ -1532,7 +2734,7 @@ export async function runWholeProgramStage(args) {
|
|
|
1532
2734
|
if (outcome.writtenPaths.length > 0) {
|
|
1533
2735
|
if (isRepository) {
|
|
1534
2736
|
try {
|
|
1535
|
-
commit = await git.commitPaths(root, `nightshift(${programId}): as-built snapshot`, outcome.writtenPaths);
|
|
2737
|
+
commit = await timed(recorder, { stage: "as-built-snapshot" }, () => git.commitPaths(root, `nightshift(${programId}): as-built snapshot`, outcome.writtenPaths), commitEvidence);
|
|
1536
2738
|
}
|
|
1537
2739
|
catch (error) {
|
|
1538
2740
|
errors.push(`could not commit the as-built snapshot: ${error.message}`);
|
|
@@ -1554,7 +2756,7 @@ export async function runWholeProgramStage(args) {
|
|
|
1554
2756
|
}
|
|
1555
2757
|
else {
|
|
1556
2758
|
try {
|
|
1557
|
-
await appendLedgerEvents(root, programId, events);
|
|
2759
|
+
await timed(recorder, { stage: "ledger-persist", workstream: WHOLE_PROGRAM_SUBJECT }, () => appendLedgerEvents(root, programId, events));
|
|
1558
2760
|
journaled = true;
|
|
1559
2761
|
findingIds = events.map((event) => event.id);
|
|
1560
2762
|
}
|
|
@@ -1580,6 +2782,7 @@ export async function runWholeProgramStage(args) {
|
|
|
1580
2782
|
triaged,
|
|
1581
2783
|
now,
|
|
1582
2784
|
log,
|
|
2785
|
+
recorder,
|
|
1583
2786
|
});
|
|
1584
2787
|
errors.push(...spawnErrors);
|
|
1585
2788
|
}
|
|
@@ -1599,16 +2802,263 @@ export async function runWholeProgramStage(args) {
|
|
|
1599
2802
|
errors,
|
|
1600
2803
|
};
|
|
1601
2804
|
}
|
|
1602
|
-
|
|
1603
|
-
|
|
2805
|
+
function causalAnalysisStageFailure(reason) {
|
|
2806
|
+
return {
|
|
2807
|
+
ran: false,
|
|
2808
|
+
status: "analyzer-error",
|
|
2809
|
+
reason,
|
|
2810
|
+
unresolvedQuestions: [],
|
|
2811
|
+
parseErrors: [],
|
|
2812
|
+
inputClipped: false,
|
|
2813
|
+
};
|
|
2814
|
+
}
|
|
2815
|
+
/**
|
|
2816
|
+
* Projects the run's own `WorkstreamResult[]` into WS-05's narrow,
|
|
2817
|
+
* causal-analysis-owned timeline shape (mirrors `notBuiltRoster`'s mapping
|
|
2818
|
+
* pattern) — `causal-analysis.ts` cannot import `WorkstreamResult` itself
|
|
2819
|
+
* without creating a cycle back into this module.
|
|
2820
|
+
*/
|
|
2821
|
+
function causalAnalysisTimeline(results) {
|
|
2822
|
+
return results.map((result) => ({
|
|
2823
|
+
id: result.id,
|
|
2824
|
+
name: result.name,
|
|
2825
|
+
status: result.outcome.status,
|
|
2826
|
+
...(result.outcome.status === "complete" ? {} : { reason: result.outcome.reason }),
|
|
2827
|
+
...(result.failureDiagnoses === undefined ? {} : { failureDiagnoses: result.failureDiagnoses }),
|
|
2828
|
+
...(result.testCritique === undefined
|
|
2829
|
+
? {}
|
|
2830
|
+
: {
|
|
2831
|
+
testCritique: {
|
|
2832
|
+
stopReason: result.testCritique.stopReason,
|
|
2833
|
+
roundsRun: result.testCritique.roundsRun,
|
|
2834
|
+
openSubjects: result.testCritique.open.map((finding) => finding.subject),
|
|
2835
|
+
resolvedSubjects: result.testCritique.resolved.map((finding) => finding.subject),
|
|
2836
|
+
fixesRun: result.testCritique.fixesRun,
|
|
2837
|
+
cyclesAllotted: result.testCritique.cyclesAllotted,
|
|
2838
|
+
},
|
|
2839
|
+
}),
|
|
2840
|
+
}));
|
|
2841
|
+
}
|
|
2842
|
+
/**
|
|
2843
|
+
* The end-of-run causal-analysis stage (WS-05): resolves the same diff base
|
|
2844
|
+
* the whole-program stage uses, takes the analytics snapshot *before*
|
|
2845
|
+
* spawning the analyzer (so it never reasons about its own cost, §3.2),
|
|
2846
|
+
* calls the never-throwing pass exactly once, and returns its outcome. Every
|
|
2847
|
+
* risky step gets its own catch so a stage failure is a sentence in the
|
|
2848
|
+
* report, never a change to the run's outcome (SC-13) — see the outer catch
|
|
2849
|
+
* below for the belt-and-braces case where a bug in this function's own body
|
|
2850
|
+
* throws before the pass has even run. Read-only: writes no file, journals
|
|
2851
|
+
* no ledger event, and is never a gate.
|
|
2852
|
+
*/
|
|
2853
|
+
export async function runCausalAnalysisStage(args) {
|
|
2854
|
+
const { root, programId, manifest, config, agentRunner, permits, git, isRepository, results, runStartCommit, ledger, log, } = args;
|
|
2855
|
+
const recorder = args.recorder ?? NOOP_RUN_RECORDER;
|
|
2856
|
+
const errors = [];
|
|
2857
|
+
let outcome = causalAnalysisStageFailure("the causal analysis stage did not complete");
|
|
2858
|
+
const safeLog = (line) => {
|
|
2859
|
+
try {
|
|
2860
|
+
log(line);
|
|
2861
|
+
}
|
|
2862
|
+
catch (error) {
|
|
2863
|
+
errors.push(`the causal analysis logger failed: ${error.message}`);
|
|
2864
|
+
}
|
|
2865
|
+
};
|
|
2866
|
+
try {
|
|
2867
|
+
safeLog("causal analysis: starting");
|
|
2868
|
+
const base = await resolveProgramDiffBase({ root, git, isRepository, runStartCommit });
|
|
2869
|
+
const diff = base === undefined
|
|
2870
|
+
? ""
|
|
2871
|
+
: await timed(recorder, { stage: "git-diff" }, () => git.diffSince(root, base));
|
|
2872
|
+
// Taken before the analyzer spawns (§3.2 of the WS-05 spec): the pass is
|
|
2873
|
+
// briefed from the run as it stood at this instant, so it never reasons
|
|
2874
|
+
// about its own review time.
|
|
2875
|
+
const snapshot = recorder.snapshot();
|
|
2876
|
+
// The program document's own narrative — SC-12 requires the analyzer to
|
|
2877
|
+
// consume the plan, not just the manifest's roster/success-criteria
|
|
2878
|
+
// fields. Read fail-open: a missing or unreadable document is an honest
|
|
2879
|
+
// gap in the brief, never a thrown stage (SC-13).
|
|
2880
|
+
const programNarrative = await readFile(join(root, "docs", "programs", `${programId}-program.md`), "utf8").catch(() => undefined);
|
|
2881
|
+
// Every workstream's own spec file — SC-12 requires the analyzer to
|
|
2882
|
+
// consume the specifications themselves, not merely a path to them. Read
|
|
2883
|
+
// fail-open per workstream: a missing or unreadable spec is an honest
|
|
2884
|
+
// gap for that workstream, never a thrown stage (SC-13).
|
|
2885
|
+
const workstreamSpecs = await Promise.all(manifest.workstreams.map(async (workstream) => {
|
|
2886
|
+
const spec = await readFile(join(root, workstream.taskFile), "utf8").catch(() => undefined);
|
|
2887
|
+
return {
|
|
2888
|
+
id: workstream.id,
|
|
2889
|
+
name: workstream.name,
|
|
2890
|
+
taskFile: workstream.taskFile,
|
|
2891
|
+
...(spec === undefined ? {} : { spec }),
|
|
2892
|
+
};
|
|
2893
|
+
}));
|
|
2894
|
+
outcome = await runCausalAnalysis({
|
|
2895
|
+
root,
|
|
2896
|
+
programId,
|
|
2897
|
+
manifest,
|
|
2898
|
+
config,
|
|
2899
|
+
agentRunner,
|
|
2900
|
+
permits,
|
|
2901
|
+
recorder,
|
|
2902
|
+
snapshot,
|
|
2903
|
+
ledger,
|
|
2904
|
+
workstreams: causalAnalysisTimeline(results),
|
|
2905
|
+
diff,
|
|
2906
|
+
workstreamSpecs,
|
|
2907
|
+
...(base === undefined ? {} : { baseCommit: base }),
|
|
2908
|
+
...(programNarrative === undefined ? {} : { programNarrative }),
|
|
2909
|
+
log,
|
|
2910
|
+
});
|
|
2911
|
+
safeLog(`causal analysis: ${outcome.status}${outcome.reason ? ` — ${outcome.reason}` : ""}`);
|
|
2912
|
+
}
|
|
2913
|
+
catch (error) {
|
|
2914
|
+
errors.push(`the causal analysis stage failed unexpectedly: ${error.message}`);
|
|
2915
|
+
}
|
|
2916
|
+
return { outcome, errors };
|
|
2917
|
+
}
|
|
2918
|
+
/**
|
|
2919
|
+
* Fail-open durable write of one *executed* verify command's captured output
|
|
2920
|
+
* (WS-02, SC-05) to a gitignored `build-logs/<programId>/verify/` file,
|
|
2921
|
+
* returned as a `transcript` evidence ref for that command's span. Never
|
|
2922
|
+
* throws: a write failure (full disk, EACCES) simply omits the evidence ref,
|
|
2923
|
+
* never the span or the run (SC-13).
|
|
2924
|
+
*/
|
|
2925
|
+
async function writeVerifyOutputEvidence(root, programId, label, output) {
|
|
2926
|
+
try {
|
|
2927
|
+
const dir = join(root, "build-logs", programId, "verify");
|
|
2928
|
+
await mkdir(dir, { recursive: true });
|
|
2929
|
+
const fileName = `${label}-${randomBytes(4).toString("hex")}.log`;
|
|
2930
|
+
await writeFile(join(dir, fileName), output, "utf8");
|
|
2931
|
+
return {
|
|
2932
|
+
kind: "transcript",
|
|
2933
|
+
locality: "local",
|
|
2934
|
+
ref: `build-logs/${programId}/verify/${fileName}`,
|
|
2935
|
+
note: "verify output",
|
|
2936
|
+
};
|
|
2937
|
+
}
|
|
2938
|
+
catch {
|
|
2939
|
+
return undefined;
|
|
2940
|
+
}
|
|
2941
|
+
}
|
|
2942
|
+
/**
|
|
2943
|
+
* Runs every configured verify command once, first non-zero wins — same
|
|
2944
|
+
* behavior as before WS-02, but the emission model changed (§3.4.a): rather
|
|
2945
|
+
* than one span for the whole pass, this emits one `verification-command` /
|
|
2946
|
+
* `baseline-verification` span per *executed* command, as depth-1 tiling
|
|
2947
|
+
* siblings (never a pass wrapper), so repeated full-suite cost and its
|
|
2948
|
+
* evidence are visible per command. A non-zero `agentExitCode` runs no
|
|
2949
|
+
* commands and emits no span at all — that attempt's failure is already
|
|
2950
|
+
* carried by its own `attempt-outcome` point (§3.3), so there is no
|
|
2951
|
+
* verification observation to fabricate. A command short-circuited by an
|
|
2952
|
+
* earlier command's failure is likewise absent, never a measured zero.
|
|
2953
|
+
* `stage` distinguishes the untouched-tree baseline check (preflight bucket)
|
|
2954
|
+
* from every configured command that follows an agent's own attempt
|
|
2955
|
+
* (verification bucket); `phase` names the invocation (`baseline`,
|
|
2956
|
+
* `post-implementer`, `test-critique-fix`, …) as each command span's
|
|
2957
|
+
* `attemptReason`. `programId`, when given, durably writes each executed
|
|
2958
|
+
* command's captured output as `transcript` evidence; a caller with no
|
|
2959
|
+
* `programId` (a bare test double) simply carries no evidence ref.
|
|
2960
|
+
* `role`/`attemptSeat`/`attemptIndex` carry the triggering attempt's own
|
|
2961
|
+
* dimensions (SC-05/SC-06) so a repeated verify pass can be uniquely
|
|
2962
|
+
* correlated back to the attempt that provoked it, not just its phase.
|
|
2963
|
+
*/
|
|
2964
|
+
async function verifyAttempt(config, verifyRunner,
|
|
2965
|
+
/** The tree the verify commands actually run against (WS-02) — a
|
|
2966
|
+
* workstream's worktree during a build, `root` for the baseline. Also the
|
|
2967
|
+
* tree the verify cache keys on (WS-09) — see the `cwd-in-key` decision. */
|
|
2968
|
+
cwd,
|
|
2969
|
+
/** Where verify-output evidence is written (WS-02) — always `root`, so a
|
|
2970
|
+
* worktree never contains runner-owned transcripts. */
|
|
2971
|
+
evidenceRoot, agentExitCode, recorder = NOOP_RUN_RECORDER, stage = "verification-command", opts = {}) {
|
|
1604
2972
|
if (agentExitCode !== 0) {
|
|
1605
2973
|
return `agent exited with code ${agentExitCode}`;
|
|
1606
2974
|
}
|
|
1607
|
-
|
|
1608
|
-
|
|
2975
|
+
const { programId, phase, workstream, role, attemptSeat, attemptIndex, cache } = opts;
|
|
2976
|
+
const commands = Object.entries(config.verify);
|
|
2977
|
+
if (commands.length === 0)
|
|
2978
|
+
return undefined;
|
|
2979
|
+
const cacheDims = {
|
|
2980
|
+
stage,
|
|
2981
|
+
...(phase === undefined ? {} : { attemptReason: phase }),
|
|
2982
|
+
...(workstream === undefined ? {} : { workstream }),
|
|
2983
|
+
};
|
|
2984
|
+
const key = await cache?.keyFor(cwd);
|
|
2985
|
+
const hit = key === undefined ? undefined : cache?.get(key);
|
|
2986
|
+
if (hit !== undefined) {
|
|
2987
|
+
for (const [name] of commands) {
|
|
2988
|
+
const span = recorder.span({
|
|
2989
|
+
stage,
|
|
2990
|
+
verifyCommand: name,
|
|
2991
|
+
...(phase === undefined ? {} : { attemptReason: phase }),
|
|
2992
|
+
...(workstream === undefined ? {} : { workstream }),
|
|
2993
|
+
...(role === undefined ? {} : { role }),
|
|
2994
|
+
...(attemptSeat === undefined ? {} : { attemptSeat }),
|
|
2995
|
+
...(attemptIndex === undefined ? {} : { attemptIndex }),
|
|
2996
|
+
});
|
|
2997
|
+
const cached = hit.evidence.get(name);
|
|
2998
|
+
const evidence = cached && cached.length > 0
|
|
2999
|
+
? cached.map((ref) => ({
|
|
3000
|
+
...ref,
|
|
3001
|
+
note: `verify output (cached from the ${hit.phase} pass)`,
|
|
3002
|
+
}))
|
|
3003
|
+
: undefined;
|
|
3004
|
+
span.close({ outcome: "skipped", ...(evidence ? { evidence } : {}) });
|
|
3005
|
+
}
|
|
3006
|
+
recorder.point({
|
|
3007
|
+
kind: "verify-cache",
|
|
3008
|
+
coverage: "observed",
|
|
3009
|
+
label: "hit",
|
|
3010
|
+
detail: hit.phase,
|
|
3011
|
+
dimensions: cacheDims,
|
|
3012
|
+
});
|
|
3013
|
+
return undefined;
|
|
3014
|
+
}
|
|
3015
|
+
if (key !== undefined) {
|
|
3016
|
+
recorder.point({ kind: "verify-cache", coverage: "observed", label: "miss", dimensions: cacheDims });
|
|
3017
|
+
}
|
|
3018
|
+
const evidenceByCommand = new Map();
|
|
3019
|
+
for (const [name, command] of commands) {
|
|
3020
|
+
const span = recorder.span({
|
|
3021
|
+
stage,
|
|
3022
|
+
verifyCommand: name,
|
|
3023
|
+
...(phase === undefined ? {} : { attemptReason: phase }),
|
|
3024
|
+
...(workstream === undefined ? {} : { workstream }),
|
|
3025
|
+
...(role === undefined ? {} : { role }),
|
|
3026
|
+
...(attemptSeat === undefined ? {} : { attemptSeat }),
|
|
3027
|
+
...(attemptIndex === undefined ? {} : { attemptIndex }),
|
|
3028
|
+
});
|
|
3029
|
+
let result;
|
|
3030
|
+
try {
|
|
3031
|
+
result = await verifyRunner(command, cwd);
|
|
3032
|
+
}
|
|
3033
|
+
catch (error) {
|
|
3034
|
+
span.close({ outcome: "failed" });
|
|
3035
|
+
throw error;
|
|
3036
|
+
}
|
|
3037
|
+
const evidenceRef = programId === undefined
|
|
3038
|
+
? undefined
|
|
3039
|
+
: await writeVerifyOutputEvidence(evidenceRoot, programId, `${phase ?? stage}-${name}`, result.output);
|
|
3040
|
+
const evidence = evidenceRef ? [evidenceRef] : undefined;
|
|
1609
3041
|
if (result.exitCode !== 0) {
|
|
3042
|
+
span.close({ outcome: "failed", ...(evidence ? { evidence } : {}) });
|
|
1610
3043
|
return `verify \`${name}\` (${command}) exited ${result.exitCode}:\n${tail(result.output, 1500)}`;
|
|
1611
3044
|
}
|
|
3045
|
+
span.close({ outcome: "success", ...(evidence ? { evidence } : {}) });
|
|
3046
|
+
evidenceByCommand.set(name, evidence ?? []);
|
|
3047
|
+
}
|
|
3048
|
+
if (key !== undefined && cache !== undefined) {
|
|
3049
|
+
const settled = await cache.keyFor(cwd);
|
|
3050
|
+
if (settled === key) {
|
|
3051
|
+
cache.set(key, { evidence: evidenceByCommand, phase: phase ?? stage });
|
|
3052
|
+
}
|
|
3053
|
+
else {
|
|
3054
|
+
recorder.point({
|
|
3055
|
+
kind: "verify-cache",
|
|
3056
|
+
coverage: "observed",
|
|
3057
|
+
label: "not-cached",
|
|
3058
|
+
detail: "commands-changed-the-tree",
|
|
3059
|
+
dimensions: cacheDims,
|
|
3060
|
+
});
|
|
3061
|
+
}
|
|
1612
3062
|
}
|
|
1613
3063
|
return undefined;
|
|
1614
3064
|
}
|
|
@@ -1692,12 +3142,24 @@ function failureDiagnosisBrief(manifest, workstream, spec, diff, failure) {
|
|
|
1692
3142
|
* undefined and the retry proceeds on the raw verify output alone.
|
|
1693
3143
|
*/
|
|
1694
3144
|
async function diagnoseFailure(options) {
|
|
1695
|
-
const {
|
|
1696
|
-
const
|
|
1697
|
-
const
|
|
3145
|
+
const { cwd, manifest, workstream, spec, agentRunner, permits, reviewer, git, baseCommit, failure, observe } = options;
|
|
3146
|
+
const recorder = options.recorder ?? NOOP_RUN_RECORDER;
|
|
3147
|
+
const rawDiff = baseCommit !== undefined
|
|
3148
|
+
? await timed(recorder, { stage: "git-diff", workstream: workstream.id }, () => git.diffSince(cwd, baseCommit))
|
|
3149
|
+
: "";
|
|
3150
|
+
const clippedSpec = clipForReview(spec, "spec");
|
|
3151
|
+
const clippedDiff = clipForReview(rawDiff, "diff");
|
|
3152
|
+
recorder.point(clippedInputPoint("failure-diagnosis-spec", spec, clippedSpec.text, { role: "reviewerAgent" }));
|
|
3153
|
+
recorder.point(clippedInputPoint("failure-diagnosis-diff", rawDiff, clippedDiff.text, { role: "reviewerAgent" }));
|
|
3154
|
+
const brief = failureDiagnosisBrief(manifest, workstream, clippedSpec.text, clippedDiff.text, failure);
|
|
1698
3155
|
let invocation;
|
|
1699
3156
|
try {
|
|
1700
|
-
invocation = await invokeAgent(agentRunner, reviewer, brief,
|
|
3157
|
+
invocation = await invokeAgent(agentRunner, reviewer, brief, cwd, permits, "reviewerAgent", observe, recorder, {
|
|
3158
|
+
stage: "failure-diagnosis",
|
|
3159
|
+
workstream: workstream.id,
|
|
3160
|
+
...(options.attemptSeat === undefined ? {} : { attemptSeat: options.attemptSeat }),
|
|
3161
|
+
...(options.attemptIndex === undefined ? {} : { attemptIndex: options.attemptIndex }),
|
|
3162
|
+
});
|
|
1701
3163
|
}
|
|
1702
3164
|
catch (error) {
|
|
1703
3165
|
return {
|
|
@@ -1828,27 +3290,61 @@ function findingEvidenceLine(finding) {
|
|
|
1828
3290
|
.join("; ");
|
|
1829
3291
|
return evidence === "" ? "" : ` Evidence: ${evidence}`;
|
|
1830
3292
|
}
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
* inviting judgment rather than demanding compliance — the manual workflow's
|
|
1834
|
-
* own framing (WS-03 design §2). Never asks the implementer to commit or to
|
|
1835
|
-
* echo a block back.
|
|
1836
|
-
*/
|
|
1837
|
-
function testCritiqueFixBrief(workstream, spec, findings) {
|
|
1838
|
-
const findingsList = findings.map((finding) => [
|
|
3293
|
+
function findingsListLines(findings) {
|
|
3294
|
+
return findings.map((finding) => [
|
|
1839
3295
|
`- **${finding.severity}** (${finding.category}) ${finding.subject}: ${finding.message}`,
|
|
1840
3296
|
findingEvidenceLine(finding),
|
|
1841
3297
|
]
|
|
1842
3298
|
.filter((line) => line !== "")
|
|
1843
3299
|
.join("\n"));
|
|
3300
|
+
}
|
|
3301
|
+
/**
|
|
3302
|
+
* The implementer's fix re-brief (WS-09 §3.5.1): `binding` is the decider's
|
|
3303
|
+
* `fix-now` triage — a required correction — and `advisory` is everything
|
|
3304
|
+
* else the reviewer raised this cycle, left to the fixer's judgment. Never
|
|
3305
|
+
* asks the implementer to commit or to echo a block back.
|
|
3306
|
+
*/
|
|
3307
|
+
function testCritiqueFixBrief(workstream, spec, binding, advisory, resumed, deciderConfigured) {
|
|
3308
|
+
const bindingSection = binding.length > 0
|
|
3309
|
+
? [
|
|
3310
|
+
"## The decider ruled: fix these now",
|
|
3311
|
+
"",
|
|
3312
|
+
"An independent reviewer raised these and the decider triaged them",
|
|
3313
|
+
"as worth one bounded fix attempt before the run proceeds. Apply",
|
|
3314
|
+
"them.",
|
|
3315
|
+
"",
|
|
3316
|
+
...findingsListLines(binding),
|
|
3317
|
+
"",
|
|
3318
|
+
]
|
|
3319
|
+
: [];
|
|
3320
|
+
const advisorySection = advisory.length > 0
|
|
3321
|
+
? [
|
|
3322
|
+
"## Also raised, for your judgment",
|
|
3323
|
+
"",
|
|
3324
|
+
...(deciderConfigured
|
|
3325
|
+
? [
|
|
3326
|
+
"The reviewer raised these too. The decider did not mark them",
|
|
3327
|
+
"for an immediate fix. Weigh them and apply what you agree",
|
|
3328
|
+
"with.",
|
|
3329
|
+
]
|
|
3330
|
+
: [
|
|
3331
|
+
"No decider is configured for this run, so none of these were",
|
|
3332
|
+
"triaged. Weigh them and apply what you agree with.",
|
|
3333
|
+
]),
|
|
3334
|
+
"",
|
|
3335
|
+
...findingsListLines(advisory),
|
|
3336
|
+
"",
|
|
3337
|
+
]
|
|
3338
|
+
: [];
|
|
1844
3339
|
return [
|
|
1845
3340
|
`# Your tests for ${workstream.id}: ${workstream.name} were reviewed`,
|
|
1846
3341
|
"",
|
|
3342
|
+
...(resumed ? [SESSION_CONTINUATION_PREAMBLE] : []),
|
|
1847
3343
|
"An independent reviewer read the diff you produced — implementation",
|
|
1848
|
-
"and tests together
|
|
1849
|
-
"",
|
|
1850
|
-
...findingsList,
|
|
3344
|
+
"and tests together.",
|
|
1851
3345
|
"",
|
|
3346
|
+
...bindingSection,
|
|
3347
|
+
...advisorySection,
|
|
1852
3348
|
"## Specification",
|
|
1853
3349
|
"",
|
|
1854
3350
|
spec.trim(),
|
|
@@ -1862,12 +3358,26 @@ function testCritiqueFixBrief(workstream, spec, findings) {
|
|
|
1862
3358
|
"",
|
|
1863
3359
|
"Never commit — the runner owns commits, verifies your fix itself, and",
|
|
1864
3360
|
"either keeps it or discards it depending on whether it actually",
|
|
1865
|
-
"verifies.
|
|
1866
|
-
"
|
|
3361
|
+
"verifies. Run only the tests that cover what you changed; the runner",
|
|
3362
|
+
"runs the project's full verify suite after you and gates on it, which",
|
|
3363
|
+
"overrides AGENTS.md's \"verify before claiming completion\" directive",
|
|
3364
|
+
"for this spawn. Reply in prose, not a block: what you fixed and what",
|
|
3365
|
+
"you declined, and why.",
|
|
3366
|
+
"",
|
|
3367
|
+
"You are working in a dedicated git worktree on a per-workstream",
|
|
3368
|
+
"branch. Do not switch branches, and do not edit `docs/programs/` —",
|
|
3369
|
+
"those are the runner's own artifacts and changes to them are",
|
|
3370
|
+
"discarded.",
|
|
1867
3371
|
"",
|
|
1868
3372
|
summaryContract(),
|
|
1869
3373
|
].join("\n");
|
|
1870
3374
|
}
|
|
3375
|
+
/** Sorted `dirtyPaths` + `untrackedPaths`, joined, for the §3.6
|
|
3376
|
+
* before/after comparison a fix seat's resumed-spawn fallback guards on. */
|
|
3377
|
+
async function treeFingerprint(git, cwd) {
|
|
3378
|
+
const [dirty, untracked] = await Promise.all([git.dirtyPaths(cwd), git.untrackedPaths(cwd)]);
|
|
3379
|
+
return [...dirty].sort().join("\n") + "\u0000" + [...untracked].sort().join("\n");
|
|
3380
|
+
}
|
|
1871
3381
|
/**
|
|
1872
3382
|
* The fix seam, in full: one brief carrying every finding, one implementer
|
|
1873
3383
|
* invocation, one verification, one commit on green or one reset on
|
|
@@ -1875,21 +3385,80 @@ function testCritiqueFixBrief(workstream, spec, findings) {
|
|
|
1875
3385
|
* whether (and how many times) this is called.
|
|
1876
3386
|
*/
|
|
1877
3387
|
async function attemptFix(options) {
|
|
1878
|
-
const { root, programId, workstream, spec, config, agentRunner, permits, verifyRunner, git, agent,
|
|
1879
|
-
const
|
|
3388
|
+
const { root, cwd, programId, workstream, spec, config, agentRunner, permits, verifyRunner, git, agent, binding, advisory, deciderConfigured, greenCommit, label, log, now, observe, stage, attemptReason, attemptIndex, } = options;
|
|
3389
|
+
const recorder = options.recorder ?? NOOP_RUN_RECORDER;
|
|
1880
3390
|
let invocation;
|
|
3391
|
+
let session;
|
|
3392
|
+
let continuity;
|
|
3393
|
+
const preSpawnTree = options.buildSession ? await treeFingerprint(git, cwd) : undefined;
|
|
1881
3394
|
try {
|
|
1882
|
-
|
|
3395
|
+
const spawned = await spawnWithSessionContinuity({
|
|
3396
|
+
resume: options.buildSession,
|
|
3397
|
+
spawn: (resume, resumed) => {
|
|
3398
|
+
const brief = testCritiqueFixBrief(workstream, spec, binding, advisory, resumed, deciderConfigured);
|
|
3399
|
+
return invokeAgent(agentRunner, agent, brief, cwd, permits, "agent", observe, recorder, { stage, workstream: workstream.id, attemptSeat: label, attemptIndex, attemptReason }, resume === undefined ? {} : { handle: resume });
|
|
3400
|
+
},
|
|
3401
|
+
untouched: async () => preSpawnTree !== undefined && (await treeFingerprint(git, cwd)) === preSpawnTree,
|
|
3402
|
+
log,
|
|
3403
|
+
observeContinuity: (result) => {
|
|
3404
|
+
continuity = result;
|
|
3405
|
+
try {
|
|
3406
|
+
recorder.point({
|
|
3407
|
+
kind: "session-continuity",
|
|
3408
|
+
coverage: "observed",
|
|
3409
|
+
label: result.mode,
|
|
3410
|
+
...(result.declineReason ? { detail: result.declineReason } : {}),
|
|
3411
|
+
dimensions: { stage, workstream: workstream.id, role: "agent", attemptSeat: label, attemptIndex, attemptReason },
|
|
3412
|
+
});
|
|
3413
|
+
}
|
|
3414
|
+
catch (error) {
|
|
3415
|
+
log(`run-analytics: session continuity point failed: ${error.message}`);
|
|
3416
|
+
}
|
|
3417
|
+
},
|
|
3418
|
+
});
|
|
3419
|
+
invocation = spawned.result;
|
|
3420
|
+
session = spawned.result.session;
|
|
3421
|
+
continuity = spawned.continuity;
|
|
1883
3422
|
}
|
|
1884
3423
|
catch (error) {
|
|
3424
|
+
// A rethrow out of `spawnWithSessionContinuity` (WS-08 §3.6) means a
|
|
3425
|
+
// resumed spawn threw *and* it had already touched the tree — the one
|
|
3426
|
+
// case where reaching this catch does not mean "nothing was built".
|
|
3427
|
+
// Reset defensively so a half-applied fix is never left standing; a
|
|
3428
|
+
// plain spawn failure (no resume attempted, or the tree is unchanged)
|
|
3429
|
+
// makes this a no-op reset to the state already there.
|
|
3430
|
+
if (preSpawnTree !== undefined) {
|
|
3431
|
+
try {
|
|
3432
|
+
await git.resetHard(cwd, greenCommit);
|
|
3433
|
+
}
|
|
3434
|
+
catch {
|
|
3435
|
+
// Best-effort: the green state may already be intact, or the reset
|
|
3436
|
+
// itself failed for an unrelated reason; either way this must not
|
|
3437
|
+
// mask the original spawn error being reported below.
|
|
3438
|
+
}
|
|
3439
|
+
}
|
|
1885
3440
|
const spawnError = `the ${label} implementer could not be spawned: ${error.message}; ` +
|
|
1886
3441
|
"the fix was not attempted and the green state was preserved.";
|
|
3442
|
+
recorder.point({
|
|
3443
|
+
kind: "attempt-outcome",
|
|
3444
|
+
coverage: "observed",
|
|
3445
|
+
dimensions: {
|
|
3446
|
+
stage,
|
|
3447
|
+
workstream: workstream.id,
|
|
3448
|
+
role: "agent",
|
|
3449
|
+
attemptSeat: label,
|
|
3450
|
+
attemptIndex,
|
|
3451
|
+
attemptReason,
|
|
3452
|
+
outcome: "failed",
|
|
3453
|
+
},
|
|
3454
|
+
});
|
|
1887
3455
|
return {
|
|
1888
3456
|
outcome: "failed",
|
|
1889
3457
|
greenCommit,
|
|
1890
3458
|
summary: "(fix not attempted — the implementer could not be spawned)",
|
|
1891
3459
|
failure: spawnError,
|
|
1892
3460
|
spawnError,
|
|
3461
|
+
...(continuity ? { continuity } : {}),
|
|
1893
3462
|
};
|
|
1894
3463
|
}
|
|
1895
3464
|
const transcript = invocation.transcript;
|
|
@@ -1897,14 +3466,37 @@ async function attemptFix(options) {
|
|
|
1897
3466
|
const summary = resolvedSummary.text;
|
|
1898
3467
|
const denialEvents = buildDenialEvents(workstream.id, label, transcript?.denials ?? [], resolvedSummary.needsPermission, config.permits?.deny ?? [], now().toISOString());
|
|
1899
3468
|
if (denialEvents.length > 0) {
|
|
1900
|
-
await appendLedgerEvents(root, programId, denialEvents);
|
|
3469
|
+
await timed(recorder, { stage: "ledger-persist", workstream: workstream.id }, () => appendLedgerEvents(root, programId, denialEvents));
|
|
1901
3470
|
for (const event of denialEvents) {
|
|
1902
3471
|
if (event.kind !== "permission-denied")
|
|
1903
3472
|
continue;
|
|
1904
|
-
log(`${
|
|
3473
|
+
log(`${label}: denied \`${event.command}\` (${event.source})`);
|
|
1905
3474
|
}
|
|
1906
3475
|
}
|
|
1907
|
-
const failure = await verifyAttempt(config, verifyRunner, root, invocation.exitCode
|
|
3476
|
+
const failure = await verifyAttempt(config, verifyRunner, cwd, root, invocation.exitCode, recorder, "verification-command", {
|
|
3477
|
+
programId,
|
|
3478
|
+
phase: stage,
|
|
3479
|
+
workstream: workstream.id,
|
|
3480
|
+
role: "agent",
|
|
3481
|
+
attemptSeat: label,
|
|
3482
|
+
attemptIndex,
|
|
3483
|
+
...(options.verifyCache ? { cache: options.verifyCache } : {}),
|
|
3484
|
+
});
|
|
3485
|
+
const emitOutcome = (outcome) => {
|
|
3486
|
+
recorder.point({
|
|
3487
|
+
kind: "attempt-outcome",
|
|
3488
|
+
coverage: "observed",
|
|
3489
|
+
dimensions: {
|
|
3490
|
+
stage,
|
|
3491
|
+
workstream: workstream.id,
|
|
3492
|
+
role: "agent",
|
|
3493
|
+
attemptSeat: label,
|
|
3494
|
+
attemptIndex,
|
|
3495
|
+
attemptReason,
|
|
3496
|
+
outcome,
|
|
3497
|
+
},
|
|
3498
|
+
});
|
|
3499
|
+
};
|
|
1908
3500
|
if (failure === undefined) {
|
|
1909
3501
|
// The runner's own pending ledger and manifest writes under
|
|
1910
3502
|
// docs/programs sit uncommitted in the tree until the run's final
|
|
@@ -1915,33 +3507,38 @@ async function attemptFix(options) {
|
|
|
1915
3507
|
// outside docs/programs counts as the implementer having done anything;
|
|
1916
3508
|
// with none, skip committing — the pending bookkeeping stays for a
|
|
1917
3509
|
// later commit to pick up, nothing is lost.
|
|
1918
|
-
const dirty = await git.dirtyPaths(
|
|
1919
|
-
const meaningfulChange = dirty.some((path) => !path
|
|
3510
|
+
const dirty = await git.dirtyPaths(cwd);
|
|
3511
|
+
const meaningfulChange = dirty.some((path) => !isRunnerOwnedArtifactPath(path));
|
|
1920
3512
|
const commit = meaningfulChange
|
|
1921
|
-
? await git.commitAll(
|
|
3513
|
+
? await timed(recorder, { stage: "git-commit", workstream: workstream.id }, () => git.commitAll(cwd, `nightshift(${programId}): ${workstream.id} ${label}`, ["docs/programs"]), commitEvidence)
|
|
1922
3514
|
: undefined;
|
|
3515
|
+
emitOutcome(commit === undefined ? "discarded" : "success");
|
|
1923
3516
|
return {
|
|
1924
3517
|
outcome: "kept",
|
|
1925
3518
|
greenCommit: commit ?? greenCommit,
|
|
1926
3519
|
...(commit === undefined ? {} : { commit }),
|
|
1927
3520
|
summary,
|
|
1928
3521
|
...(transcript === undefined ? {} : { transcript }),
|
|
3522
|
+
...(session === undefined ? {} : { session }),
|
|
3523
|
+
...(continuity === undefined ? {} : { continuity }),
|
|
1929
3524
|
};
|
|
1930
3525
|
}
|
|
1931
|
-
//
|
|
1932
|
-
//
|
|
1933
|
-
//
|
|
1934
|
-
//
|
|
1935
|
-
//
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
3526
|
+
// A worktree hazard the pre-WS-02 code guarded against with a
|
|
3527
|
+
// snapshot/restore of docs/programs no longer exists: the runner writes
|
|
3528
|
+
// nothing under docs/programs in a worktree, so a whole-tree
|
|
3529
|
+
// `git reset --hard` here throws away nothing the run's journal at `root`
|
|
3530
|
+
// depends on.
|
|
3531
|
+
await git.resetHard(cwd, greenCommit);
|
|
3532
|
+
emitOutcome("failed");
|
|
3533
|
+
return {
|
|
3534
|
+
outcome: "failed",
|
|
3535
|
+
greenCommit,
|
|
3536
|
+
summary,
|
|
3537
|
+
failure,
|
|
3538
|
+
...(transcript === undefined ? {} : { transcript }),
|
|
3539
|
+
...(session === undefined ? {} : { session }),
|
|
3540
|
+
...(continuity === undefined ? {} : { continuity }),
|
|
3541
|
+
};
|
|
1945
3542
|
}
|
|
1946
3543
|
/**
|
|
1947
3544
|
* The test-critique driver: wires WS-01's bounded review loop to the two
|
|
@@ -1951,11 +3548,16 @@ async function attemptFix(options) {
|
|
|
1951
3548
|
* failure), so a review can never turn green work red.
|
|
1952
3549
|
*/
|
|
1953
3550
|
async function runTestCritique(options) {
|
|
1954
|
-
const { root, manifest, workstream, spec, config, agentRunner, permits, verifyRunner, git, reviewer, agent, baseCommit, now, log, transcriptSink, } = options;
|
|
3551
|
+
const { root, cwd, manifest, workstream, spec, config, agentRunner, permits, verifyRunner, git, reviewer, agent, deciderConfigured, baseCommit, now, log, transcriptSink, triage, } = options;
|
|
3552
|
+
const recorder = options.recorder ?? NOOP_RUN_RECORDER;
|
|
1955
3553
|
let greenCommit = options.greenCommit;
|
|
1956
3554
|
let diffClipped = false;
|
|
1957
3555
|
const stageErrors = [];
|
|
1958
3556
|
const transcripts = [];
|
|
3557
|
+
// WS-08: last-wins across every fix round this pass runs — the next
|
|
3558
|
+
// round resumes whichever round most recently reported a session id.
|
|
3559
|
+
let buildSession = options.buildSession;
|
|
3560
|
+
const continuity = [];
|
|
1959
3561
|
if (!reviewer) {
|
|
1960
3562
|
return {
|
|
1961
3563
|
outcome: reviewerAbsentOutcome(),
|
|
@@ -1963,19 +3565,24 @@ async function runTestCritique(options) {
|
|
|
1963
3565
|
diffClipped,
|
|
1964
3566
|
stageErrors,
|
|
1965
3567
|
transcripts,
|
|
3568
|
+
...(buildSession ? { buildSession } : {}),
|
|
1966
3569
|
};
|
|
1967
3570
|
}
|
|
1968
|
-
const locate = (file) => locateInRepo(
|
|
1969
|
-
const review = async (
|
|
1970
|
-
const rawDiff = baseCommit !== undefined
|
|
3571
|
+
const locate = (file) => locateInRepo(cwd, file);
|
|
3572
|
+
const review = async (round, priorOpen) => {
|
|
3573
|
+
const rawDiff = baseCommit !== undefined
|
|
3574
|
+
? await timed(recorder, { stage: "git-diff", workstream: workstream.id }, () => git.diffSince(cwd, baseCommit))
|
|
3575
|
+
: "";
|
|
1971
3576
|
const diff = clipForReview(rawDiff, "diff");
|
|
1972
3577
|
const clippedSpec = clipForReview(spec, "spec");
|
|
1973
3578
|
if (diff.clipped || clippedSpec.clipped)
|
|
1974
3579
|
diffClipped = true;
|
|
3580
|
+
recorder.point(clippedInputPoint("test-critique-diff", rawDiff, diff.text, { role: "reviewerAgent" }));
|
|
3581
|
+
recorder.point(clippedInputPoint("test-critique-spec", spec, clippedSpec.text, { role: "reviewerAgent" }));
|
|
1975
3582
|
const brief = testCritiqueReviewerBrief(manifest, workstream, clippedSpec.text, diff.text, priorOpen);
|
|
1976
3583
|
let invocation;
|
|
1977
3584
|
try {
|
|
1978
|
-
invocation = await invokeAgent(agentRunner, reviewer, brief,
|
|
3585
|
+
invocation = await invokeAgent(agentRunner, reviewer, brief, cwd, permits, "reviewerAgent", transcriptSink("test-critique-reviewer"), recorder, { workstream: workstream.id, attemptIndex: round });
|
|
1979
3586
|
}
|
|
1980
3587
|
catch (error) {
|
|
1981
3588
|
// Fail open, matching the existing reviewer-error path: no findings
|
|
@@ -1987,11 +3594,11 @@ async function runTestCritique(options) {
|
|
|
1987
3594
|
if (invocation.transcript) {
|
|
1988
3595
|
const denialEvents = buildDenialEvents(workstream.id, "reviewer", invocation.transcript.denials, [], config.permits?.deny ?? [], now().toISOString());
|
|
1989
3596
|
if (denialEvents.length > 0) {
|
|
1990
|
-
await appendLedgerEvents(root, manifest.program.id, denialEvents);
|
|
3597
|
+
await timed(recorder, { stage: "ledger-persist", workstream: workstream.id }, () => appendLedgerEvents(root, manifest.program.id, denialEvents));
|
|
1991
3598
|
for (const event of denialEvents) {
|
|
1992
3599
|
if (event.kind !== "permission-denied")
|
|
1993
3600
|
continue;
|
|
1994
|
-
log(
|
|
3601
|
+
log(`reviewer: denied \`${event.command}\` (${event.source})`);
|
|
1995
3602
|
}
|
|
1996
3603
|
}
|
|
1997
3604
|
}
|
|
@@ -2000,9 +3607,56 @@ async function runTestCritique(options) {
|
|
|
2000
3607
|
const ran = invocation.exitCode === 0 && hasFindingsBlock(invocation.output);
|
|
2001
3608
|
return { findings, errors: parsed.errors, ran };
|
|
2002
3609
|
};
|
|
2003
|
-
const respond = async (
|
|
3610
|
+
const respond = async (cycle, findings) => {
|
|
3611
|
+
// Step 1: record. Every routable finding this cycle raised reaches the
|
|
3612
|
+
// ledger — not a post-fix residue (WS-09 §3.5), anchored to the same
|
|
3613
|
+
// durable pre-workstream commit the decider's own diff uses, so a
|
|
3614
|
+
// rendered reference still resolves after this workstream's worktree
|
|
3615
|
+
// and branch are gone (see the comment on `diffSince`'s docs/programs
|
|
3616
|
+
// exclusion for the same worktree-lifetime concern).
|
|
3617
|
+
const routable = findings.filter(hasRoutableEvidence);
|
|
3618
|
+
const events = findingsToLedgerEvents({
|
|
3619
|
+
workstreamId: workstream.id,
|
|
3620
|
+
findings: routable,
|
|
3621
|
+
...(baseCommit === undefined ? {} : { baseCommit }),
|
|
3622
|
+
now,
|
|
3623
|
+
});
|
|
3624
|
+
await timed(recorder, { stage: "ledger-persist", workstream: workstream.id }, () => appendLedgerEvents(root, manifest.program.id, events));
|
|
3625
|
+
// Step 2: triage — one decider spawn per routed finding, before the fix.
|
|
3626
|
+
stageErrors.push(...(await triage(events, baseCommit)));
|
|
3627
|
+
// Step 3: split. Read the ledger projection *after* triage so this
|
|
3628
|
+
// cycle's `finding-triaged` events are visible. A `human-decided`
|
|
3629
|
+
// finding is dropped entirely — never handed to the fix seam
|
|
3630
|
+
// (`run-program.ts` §2.3/§3.6's invariant, preserved); a `fix-now`
|
|
3631
|
+
// finding with no fix attempt on it yet is binding; everything else
|
|
3632
|
+
// (open, accepted, escalated, or no ledger record at all — an
|
|
3633
|
+
// advisory-only or unverifiable finding) is advisory.
|
|
3634
|
+
const ledger = await readDecisionLedger(root, manifest.program.id);
|
|
3635
|
+
const recordsById = new Map(ledger.findings.map((record) => [record.id, record]));
|
|
3636
|
+
const binding = [];
|
|
3637
|
+
const bindingIds = [];
|
|
3638
|
+
const advisory = [];
|
|
3639
|
+
for (const finding of findings) {
|
|
3640
|
+
const id = fingerprint({ ...finding, workstreamId: workstream.id });
|
|
3641
|
+
const record = recordsById.get(id);
|
|
3642
|
+
if (record?.status === "human-decided")
|
|
3643
|
+
continue;
|
|
3644
|
+
if (record?.status === "fix-now" && record.fixAttempt === undefined) {
|
|
3645
|
+
binding.push(finding);
|
|
3646
|
+
bindingIds.push(id);
|
|
3647
|
+
}
|
|
3648
|
+
else {
|
|
3649
|
+
advisory.push(finding);
|
|
3650
|
+
}
|
|
3651
|
+
}
|
|
3652
|
+
// Step 4: fix. Skipped only when every finding this cycle raised was
|
|
3653
|
+
// already human-decided.
|
|
3654
|
+
if (binding.length === 0 && advisory.length === 0) {
|
|
3655
|
+
return { note: "every finding this cycle raised was already human-decided; no fix pass ran" };
|
|
3656
|
+
}
|
|
2004
3657
|
const fix = await attemptFix({
|
|
2005
3658
|
root,
|
|
3659
|
+
cwd,
|
|
2006
3660
|
programId: manifest.program.id,
|
|
2007
3661
|
workstream,
|
|
2008
3662
|
spec,
|
|
@@ -2012,27 +3666,83 @@ async function runTestCritique(options) {
|
|
|
2012
3666
|
verifyRunner,
|
|
2013
3667
|
git,
|
|
2014
3668
|
agent,
|
|
2015
|
-
|
|
3669
|
+
binding,
|
|
3670
|
+
advisory,
|
|
3671
|
+
deciderConfigured,
|
|
2016
3672
|
greenCommit,
|
|
2017
3673
|
label: "test critique fix",
|
|
2018
3674
|
log,
|
|
2019
3675
|
now,
|
|
2020
3676
|
observe: transcriptSink("test-critique-fix"),
|
|
3677
|
+
stage: "test-critique-fix",
|
|
3678
|
+
attemptReason: "triaged-findings",
|
|
3679
|
+
attemptIndex: cycle,
|
|
3680
|
+
recorder,
|
|
3681
|
+
...(options.verifyCache ? { verifyCache: options.verifyCache } : {}),
|
|
3682
|
+
...(buildSession ? { buildSession } : {}),
|
|
2021
3683
|
});
|
|
2022
3684
|
greenCommit = fix.greenCommit;
|
|
2023
3685
|
if (fix.transcript)
|
|
2024
3686
|
transcripts.push(fix.transcript);
|
|
2025
3687
|
if (fix.spawnError !== undefined)
|
|
2026
3688
|
stageErrors.push(fix.spawnError);
|
|
3689
|
+
if (fix.session)
|
|
3690
|
+
buildSession = fix.session;
|
|
3691
|
+
if (fix.continuity)
|
|
3692
|
+
continuity.push(fix.continuity);
|
|
3693
|
+
// Step 5: journal the fix-now outcome. One `finding-fix-attempted` per
|
|
3694
|
+
// binding id, exactly `driveFixNowFindings`'s old rule: "kept" requires
|
|
3695
|
+
// both a clean verify and a landed commit — a clean-but-empty attempt is
|
|
3696
|
+
// a decline, not a fix. The merged fix also covers `advisory` findings
|
|
3697
|
+
// in the same spawn, so a commit does not prove any one binding finding
|
|
3698
|
+
// was addressed (the ledger projection already disclaims per-finding
|
|
3699
|
+
// proof); the note is prefixed with the binding subjects so a human
|
|
3700
|
+
// reading the ledger sees both the instruction and the reply, never
|
|
3701
|
+
// inferred from the fixer's prose.
|
|
3702
|
+
if (bindingIds.length > 0) {
|
|
3703
|
+
const kept = fix.outcome === "kept" && fix.commit !== undefined;
|
|
3704
|
+
const bindingSubjects = binding.map((finding) => finding.subject).join(", ");
|
|
3705
|
+
const note = kept
|
|
3706
|
+
? `(fix-now: ${bindingSubjects}) ${fix.summary}`
|
|
3707
|
+
: fix.outcome === "kept"
|
|
3708
|
+
? `(fix-now: ${bindingSubjects}) ${fix.summary} (the attempt verified clean but made no change; no fix landed)`
|
|
3709
|
+
: `(fix-now: ${bindingSubjects}) ${fix.summary} (fix failed verification and was discarded: ${fix.failure})`;
|
|
3710
|
+
const fixNowEvents = bindingIds.map((id) => ({
|
|
3711
|
+
kind: "finding-fix-attempted",
|
|
3712
|
+
at: now().toISOString(),
|
|
3713
|
+
id,
|
|
3714
|
+
outcome: kept ? "kept" : "failed",
|
|
3715
|
+
note,
|
|
3716
|
+
...(kept && fix.commit !== undefined ? { commit: fix.commit } : {}),
|
|
3717
|
+
attemptedBy: "implementer",
|
|
3718
|
+
}));
|
|
3719
|
+
await timed(recorder, { stage: "ledger-persist", workstream: workstream.id }, () => appendLedgerEvents(root, manifest.program.id, fixNowEvents));
|
|
3720
|
+
log(kept
|
|
3721
|
+
? "fix-now finding(s) verified and committed"
|
|
3722
|
+
: `fix-now finding(s) failed and were escalated — ${note}`);
|
|
3723
|
+
}
|
|
2027
3724
|
if (fix.outcome === "kept") {
|
|
2028
|
-
log(
|
|
3725
|
+
log("test critique fix verified and committed");
|
|
2029
3726
|
return { note: `${fix.summary} (fix verified and committed)` };
|
|
2030
3727
|
}
|
|
2031
|
-
log(
|
|
3728
|
+
log(`test critique fix failed verification and was discarded — ${fix.failure}`);
|
|
2032
3729
|
return { note: `${fix.summary} (fix failed verification and was discarded; the green state was preserved)` };
|
|
2033
3730
|
};
|
|
2034
|
-
const outcome = await runReviewPass({
|
|
2035
|
-
|
|
3731
|
+
const outcome = await runReviewPass({
|
|
3732
|
+
cycles: TEST_CRITIQUE_CYCLES,
|
|
3733
|
+
cycleNote: TEST_CRITIQUE_CYCLE_NOTE,
|
|
3734
|
+
review,
|
|
3735
|
+
respond,
|
|
3736
|
+
});
|
|
3737
|
+
return {
|
|
3738
|
+
outcome,
|
|
3739
|
+
finalCommit: greenCommit,
|
|
3740
|
+
diffClipped,
|
|
3741
|
+
stageErrors,
|
|
3742
|
+
transcripts,
|
|
3743
|
+
...(buildSession ? { buildSession } : {}),
|
|
3744
|
+
...(continuity.length > 0 ? { continuity } : {}),
|
|
3745
|
+
};
|
|
2036
3746
|
}
|
|
2037
3747
|
function renderSpecsSection(authorResult) {
|
|
2038
3748
|
if (authorResult.results.length === 0)
|
|
@@ -2067,7 +3777,7 @@ function renderSpecsSection(authorResult) {
|
|
|
2067
3777
|
lines.push(...notes.map((note) => `- ${note}`), "");
|
|
2068
3778
|
}
|
|
2069
3779
|
if (entry.specCritique) {
|
|
2070
|
-
lines.push(...renderPassReport("Spec critique", entry.specCritique), "");
|
|
3780
|
+
lines.push(...renderPassReport("Spec critique", entry.specCritique, "runner spec check"), "");
|
|
2071
3781
|
}
|
|
2072
3782
|
}
|
|
2073
3783
|
return lines;
|
|
@@ -2314,6 +4024,25 @@ function renderWorkstreamCommands(result) {
|
|
|
2314
4024
|
}
|
|
2315
4025
|
return lines;
|
|
2316
4026
|
}
|
|
4027
|
+
/**
|
|
4028
|
+
* States a fix seat's session-resume fallback where a human will see it
|
|
4029
|
+
* (WS-08 §3.11) — nothing is added when every requesting seat resumed
|
|
4030
|
+
* cleanly, so a clean run's report is unchanged.
|
|
4031
|
+
*/
|
|
4032
|
+
function renderSessionContinuityFallbacks(result) {
|
|
4033
|
+
const lines = [];
|
|
4034
|
+
for (const entry of result.sessionContinuity ?? []) {
|
|
4035
|
+
if (entry.continuity.mode !== "fresh" || entry.continuity.declineReason === undefined)
|
|
4036
|
+
continue;
|
|
4037
|
+
const reason = describeSessionDecline(entry.continuity.declineReason, {
|
|
4038
|
+
producerCommand: entry.producerCommand,
|
|
4039
|
+
seatCommand: entry.seatCommand,
|
|
4040
|
+
});
|
|
4041
|
+
lines.push(` - The ${entry.seat} could not continue the implementer's session (${reason}); ` +
|
|
4042
|
+
"a fresh agent applied the findings instead.");
|
|
4043
|
+
}
|
|
4044
|
+
return lines;
|
|
4045
|
+
}
|
|
2317
4046
|
/**
|
|
2318
4047
|
* The program-level `## Commands denied` section (SC-12): every
|
|
2319
4048
|
* `permission-denied` ledger record, grouped by workstream in the run's own
|
|
@@ -2350,7 +4079,11 @@ function renderCommandsDeniedSection(results, ledger, programId) {
|
|
|
2350
4079
|
}
|
|
2351
4080
|
return lines;
|
|
2352
4081
|
}
|
|
2353
|
-
export function renderRunReport(manifest, results, ledger, triagedThisRun, authorResult, at, wholeProgram, preflight, resumeCommand, manifestAmended
|
|
4082
|
+
export function renderRunReport(manifest, results, ledger, triagedThisRun, authorResult, at, wholeProgram, preflight, resumeCommand, manifestAmended,
|
|
4083
|
+
/** Pre-rendered "Where the time went" lines (WS-04), computed and isolated by the caller via `renderAnalyticsSection`. Omitted entirely, this function's output is unchanged from before WS-04. */
|
|
4084
|
+
analyticsSection,
|
|
4085
|
+
/** Pre-rendered "Why the time went there" lines (WS-05), computed and isolated by the caller via `renderCausalAnalysisSection`. Omitted entirely, this function's output is unchanged from before WS-05. */
|
|
4086
|
+
causalAnalysisSection) {
|
|
2354
4087
|
const programId = manifest.program.id;
|
|
2355
4088
|
const built = results.filter((result) => result.outcome.status === "complete" ||
|
|
2356
4089
|
result.outcome.status === "skipped").length;
|
|
@@ -2395,6 +4128,12 @@ export function renderRunReport(manifest, results, ledger, triagedThisRun, autho
|
|
|
2395
4128
|
for (const diagnosis of result.failureDiagnoses ?? []) {
|
|
2396
4129
|
lines.push(`**Reviewer diagnosis (after the ${diagnosis.attempt} attempt):**`, "", diagnosis.verdict, "");
|
|
2397
4130
|
}
|
|
4131
|
+
if (result.worktreePath !== undefined) {
|
|
4132
|
+
lines.push(`Worktree retained at \`${result.worktreePath}\`.`, "");
|
|
4133
|
+
}
|
|
4134
|
+
if (result.retainedRef !== undefined) {
|
|
4135
|
+
lines.push(`Verified commit retained at \`${result.retainedRef}\`.`, "");
|
|
4136
|
+
}
|
|
2398
4137
|
const workstreamDenials = ledger.denials.filter((denial) => denial.workstream === result.id);
|
|
2399
4138
|
if (workstreamDenials.length > 0) {
|
|
2400
4139
|
lines.push("Commands denied: " +
|
|
@@ -2428,6 +4167,7 @@ export function renderRunReport(manifest, results, ledger, triagedThisRun, autho
|
|
|
2428
4167
|
if (result.outcome.status === "complete" || result.outcome.status === "failed") {
|
|
2429
4168
|
lines.push(...renderWorkstreamCommands(result));
|
|
2430
4169
|
}
|
|
4170
|
+
lines.push(...renderSessionContinuityFallbacks(result));
|
|
2431
4171
|
}
|
|
2432
4172
|
lines.push("");
|
|
2433
4173
|
if (wholeProgram !== undefined) {
|
|
@@ -2459,6 +4199,12 @@ export function renderRunReport(manifest, results, ledger, triagedThisRun, autho
|
|
|
2459
4199
|
"workstream's own outcome is unaffected, but the reason it fell " +
|
|
2460
4200
|
"open belongs here.", "", ...stageErrors.map((error) => `- ${error}`), "");
|
|
2461
4201
|
}
|
|
4202
|
+
if (analyticsSection !== undefined) {
|
|
4203
|
+
lines.push(...analyticsSection);
|
|
4204
|
+
}
|
|
4205
|
+
if (causalAnalysisSection !== undefined) {
|
|
4206
|
+
lines.push(...causalAnalysisSection);
|
|
4207
|
+
}
|
|
2462
4208
|
return lines.join("\n");
|
|
2463
4209
|
}
|
|
2464
4210
|
//# sourceMappingURL=run-program.js.map
|