faberun 0.27.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -3
- package/package.json +1 -1
- package/skills/faberun/SKILL.md +3 -3
- package/skills/faberun/references/contract.md +9 -9
- package/skills/faberun/references/local-env.md +11 -0
- package/skills/faberun/references/operations.md +26 -33
- package/src/campaign/brief.mjs +41 -9
- package/src/campaign/campaign-brief.mjs +18 -3
- package/src/campaign/index.mjs +22 -9
- package/src/campaign/ledger.mjs +72 -4
- package/src/campaign/metrics.mjs +29 -2
- package/src/campaign/record.mjs +34 -0
- package/src/cli/campaign.mjs +35 -12
- package/src/cli/contract.mjs +11 -1
- package/src/cli/launch.mjs +31 -4
- package/src/cli/manual.mjs +24 -8
- package/src/cli/plan.mjs +56 -5
- package/src/cli/setup.mjs +5 -0
- package/src/cli/skills.mjs +18 -7
- package/src/cli/uninstall.mjs +542 -0
- package/src/cli/walkthrough.mjs +462 -0
- package/src/cli.mjs +72 -8
- package/src/contract/articles.mjs +30 -0
- package/src/contract/definition-of-done.mjs +164 -0
- package/src/contract/index.mjs +8 -5
- package/src/contract/judge-independence.mjs +18 -0
- package/src/contract/runtime.mjs +20 -6
- package/src/contract/scope-findings.mjs +120 -0
- package/src/contract/snapshot.mjs +19 -5
- package/src/contract/task-packet.mjs +49 -1
- package/src/contract/worker-result.mjs +37 -11
- package/src/engine/assignment.mjs +45 -0
- package/src/engine/detach.mjs +18 -0
- package/src/engine/gate.mjs +10 -2
- package/src/engine/judge-gate.mjs +24 -9
- package/src/engine/live-preflight.mjs +117 -18
- package/src/engine/prompts.mjs +58 -4
- package/src/engine/resume.mjs +119 -12
- package/src/engine/retry.mjs +11 -11
- package/src/engine/run-command.mjs +80 -4
- package/src/engine/run-identity.mjs +27 -14
- package/src/engine/runtime-discovery.mjs +15 -9
- package/src/engine/scheduler.mjs +8 -1
- package/src/engine/worker-env.mjs +179 -0
- package/src/harnesses/agy/index.mjs +11 -0
- package/src/harnesses/catalogue.mjs +29 -0
- package/src/harnesses/claude/index.mjs +98 -1
- package/src/harnesses/codex/index.mjs +15 -0
- package/src/harnesses/dsh/index.mjs +14 -0
- package/src/harnesses/exec-jsonl/index.mjs +12 -0
- package/src/harnesses/fx/index.mjs +13 -0
- package/src/harnesses/index.mjs +21 -10
- package/src/harnesses/replay/index.mjs +12 -0
- package/src/harnesses/zcode/index.mjs +19 -0
- package/src/host/config.mjs +8 -2
- package/src/host/environment-report.mjs +85 -0
- package/src/host/home.mjs +27 -0
- package/src/host/install-registry.mjs +511 -0
- package/src/host/preflight.mjs +32 -9
- package/src/host/uninstall-roots.mjs +196 -0
- package/src/plan/freeze.mjs +2 -0
- package/src/plan/pipeline.mjs +56 -4
- package/src/plan/progress.mjs +101 -0
- package/src/plan/proof-check.mjs +27 -0
- package/src/plan/proof-scope.mjs +1 -1
- package/src/plan/repo-facts.mjs +129 -14
- package/src/plan/rounds.mjs +27 -0
- package/src/plan/spec.mjs +82 -6
- package/src/plan/template.mjs +9 -0
- package/src/repo/declared-paths.mjs +74 -6
- package/src/repo/integrate.mjs +1 -1
- package/src/repo/scope-closure.mjs +59 -0
- package/src/repo/signal-block.mjs +26 -0
- package/src/repo/signal.mjs +9 -24
- package/src/repo/workspace.mjs +189 -14
- package/src/repo/worktree-prune.mjs +13 -5
- package/src/repo/worktree.mjs +54 -0
- package/src/report/campaign-brief.mjs +4 -0
- package/src/report/message.mjs +37 -6
- package/src/report/progress.mjs +13 -2
- package/src/run/paths.mjs +14 -1
- package/src/run/usage.mjs +66 -1
- package/src/util.mjs +18 -2
package/src/engine/resume.mjs
CHANGED
|
@@ -12,13 +12,14 @@
|
|
|
12
12
|
* contract left open, and only from runtimes that are available now: a resume
|
|
13
13
|
* after an exhausted provider is exactly when that matters.
|
|
14
14
|
*/
|
|
15
|
-
import { JUDGE_SCHEMA, SETTLED } from "./prompts.mjs";
|
|
15
|
+
import { JUDGE_SCHEMA, SETTLED, TERMINAL } from "./prompts.mjs";
|
|
16
16
|
import { Buffer } from "node:buffer";
|
|
17
17
|
import { acquire as acquireLock } from "../run/lock.mjs";
|
|
18
18
|
import { applyInvalidWorkerResult, assertRunMutable, handleProviderExhaustion } from "./lifecycle.mjs";
|
|
19
19
|
import { applyJudgeResult } from "./review.mjs";
|
|
20
20
|
import { assertSourceUnchanged, captureRunIdentity, recordedBaseRef, statesFingerprint } from "./run-identity.mjs";
|
|
21
21
|
import { attemptWorkspace, gitHead, removeWorktree, runRefName } from "../repo/worktree.mjs";
|
|
22
|
+
import { replacePacketInContract, widenTaskPacket } from "../repo/scope-closure.mjs";
|
|
22
23
|
import { attemptWorktreePath } from "../run/paths.mjs";
|
|
23
24
|
import { canonicalWorkerResultText, isResultMaterializationInvocation, materializeAttemptResult, recoverWorkerResult } from "./result-file.mjs";
|
|
24
25
|
import { checkPersistedWorkerScope, persistedScopeBoundary, reconcileAmbiguousWorkerRestart, resolveUnknownEffect } from "./scope.mjs";
|
|
@@ -35,16 +36,17 @@ import { hasOperationIntent, hasOperationSettlement, operationNeedsRecovery, pro
|
|
|
35
36
|
import { isUnknownEffectStop, planResumeRetry, renderPreviousAttemptSection } from "./retry.mjs";
|
|
36
37
|
import { join, resolve } from "node:path";
|
|
37
38
|
import { parseDiscoveryResult, parseWorkerResult } from "../contract/worker-result.mjs";
|
|
38
|
-
import { readJson, writeJsonAtomic } from "../run/store.mjs";
|
|
39
|
+
import { readJson, writeJsonAtomic, appendJsonl } from "../run/store.mjs";
|
|
39
40
|
import { recoverIntegrations } from "../repo/integrate.mjs";
|
|
40
41
|
import { registerRun, resolveCampaign } from "../campaign/index.mjs";
|
|
41
42
|
import { syncAgentSignal } from "../repo/signal.mjs";
|
|
42
43
|
import { validateContract } from "../contract/index.mjs";
|
|
43
44
|
import { validateRunMetadata, OPERATOR_ANSWER_MAX_BYTES } from "../contract/snapshot.mjs";
|
|
44
45
|
import { verificationFailureVerdict } from "./judge-gate.mjs";
|
|
45
|
-
import { verificationFailureWithScope } from "../contract/scope-findings.mjs";
|
|
46
|
+
import { verificationFailureWithScope, reauthorProposalRecord, reauthorWriteConflicts, reauthorRiskTier, reauthorApproved } from "../contract/scope-findings.mjs";
|
|
46
47
|
import { applyRejection, applyVerificationFailure, raiseNodeAttention, settleDone } from "./settle.mjs";
|
|
47
48
|
import { isHumanStepPending } from "../contract/human-step.mjs";
|
|
49
|
+
import { runBoundedRounds } from "../plan/rounds.mjs";
|
|
48
50
|
|
|
49
51
|
/** @typedef {import("../repo/integrate.mjs").IntegrationResult} IntegrationResult */
|
|
50
52
|
/** @typedef {import("./lifecycle.mjs").Invocation} Invocation */
|
|
@@ -84,11 +86,14 @@ export function recoveryExhaustionEnvelope(recovery, invocation) {
|
|
|
84
86
|
|
|
85
87
|
/**
|
|
86
88
|
* @param {string} runDirPath
|
|
87
|
-
* @param {{node?: string, reconcile?: string, answer?: {node: string, path: string}, detachedBootstrap?: boolean}} [options]
|
|
89
|
+
* @param {{node?: string, reconcile?: string, answer?: {node: string, path: string}, detachedBootstrap?: boolean, reauthor?: {node: string, discover: (context: {round: number, node: import("../contract/index.mjs").ValidatedNode, state: NodeSnapshot, missingContext: string[], findings: string[]}) => (import("../contract/scope-findings.mjs").ReauthorAdditions|Promise<import("../contract/scope-findings.mjs").ReauthorAdditions>), rounds?: number, approve?: boolean, approveBelow?: string}}} [options]
|
|
88
90
|
* `node` limits the retry in place to one node and its dependants,
|
|
89
91
|
* `reconcile` acknowledges a node stopped as `unknown_effect_reconciled`,
|
|
90
|
-
* `answer` records an operator's answer for
|
|
91
|
-
*
|
|
92
|
+
* `answer` records an operator's answer for any non-terminal node and
|
|
93
|
+
* re-dispatches it,
|
|
94
|
+
* `reauthor` widens one `context_missing` node's packet from a bounded
|
|
95
|
+
* discovery pass and re-dispatches it once the widening validates and is
|
|
96
|
+
* approved, and
|
|
92
97
|
* `detachedBootstrap` is set only by the CLI entry when this process is its
|
|
93
98
|
* own detached child
|
|
94
99
|
* @returns {Promise<RunOutcome>}
|
|
@@ -100,10 +105,16 @@ export async function resumeRun(runDirPath, options = {}) {
|
|
|
100
105
|
if (options.node && options.answer && options.node !== options.answer.node) {
|
|
101
106
|
throw new Error(`--answer ${options.answer.node} conflicts with --node ${options.node}`);
|
|
102
107
|
}
|
|
108
|
+
if (options.reauthor && options.node && options.reauthor.node !== options.node) {
|
|
109
|
+
throw new Error(`--reauthor ${options.reauthor.node} conflicts with --node ${options.node}`);
|
|
110
|
+
}
|
|
111
|
+
if (options.reauthor && options.answer) {
|
|
112
|
+
throw new Error(`--reauthor ${options.reauthor.node} cannot be combined with --answer ${options.answer.node}`);
|
|
113
|
+
}
|
|
103
114
|
const runDir = resolve(runDirPath);
|
|
104
115
|
assertRunMutable(runDir);
|
|
105
116
|
const contractPath = join(runDir, "contract.json");
|
|
106
|
-
|
|
117
|
+
let contract = validateContract(JSON.parse(readFileSync(contractPath, "utf8")), contractPath, { persisted: true });
|
|
107
118
|
const lock = acquireLock(runDir);
|
|
108
119
|
try {
|
|
109
120
|
const storedMetadata = validateRunMetadata(readJson(join(runDir, "run.json")), { requireSourceIdentity: true });
|
|
@@ -161,15 +172,51 @@ export async function resumeRun(runDirPath, options = {}) {
|
|
|
161
172
|
result: { status: "done", summary: boundedSummary(answerText.trim() || answerNode.humanStep.step), verification: [], artifacts: [], missingContext: [] },
|
|
162
173
|
}, lock);
|
|
163
174
|
} else {
|
|
164
|
-
|
|
165
|
-
|
|
175
|
+
// Any non-terminal node accepts an operator answer, not only the
|
|
176
|
+
// `context_missing` boundary. A node that already reached a terminal
|
|
177
|
+
// status has no attempt left to receive the answer, so it is refused
|
|
178
|
+
// here -- in the engine, where the resume decisions are made and the
|
|
179
|
+
// engine test can reach the gate.
|
|
180
|
+
if (!answerState || TERMINAL.has(answerState.status)) {
|
|
181
|
+
throw new Error(`node ${options.answer.node} is not blocked on missing context and is terminal, so it cannot accept an operator answer`);
|
|
166
182
|
}
|
|
167
183
|
const answerOverride = /** @type {import("../contract/index.mjs").ExecutionOverride} */ (/** @type {unknown} */ ({
|
|
168
184
|
kind: "operator-answer",
|
|
169
|
-
reason: `operator answered
|
|
185
|
+
reason: `operator answered node ${options.answer.node}`,
|
|
170
186
|
text: answerText,
|
|
171
187
|
}));
|
|
172
188
|
recordExecutionOverride(runDir, answerState, answerOverride, lock);
|
|
189
|
+
// Render the answer into the bounded `Previous attempt` section before
|
|
190
|
+
// classification. The retry path below renders it too, but a parked
|
|
191
|
+
// node the recover path re-dispatches would otherwise carry the
|
|
192
|
+
// override without ever showing it to the worker.
|
|
193
|
+
answerState.previousAttempt = renderPreviousAttemptSection(answerState) ?? answerState.previousAttempt;
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
/** @type {string|null} */
|
|
197
|
+
let reauthorAppliedNode = null;
|
|
198
|
+
if (options.reauthor) {
|
|
199
|
+
const targetNode = contract.nodes.find((node) => node.id === options.reauthor?.node);
|
|
200
|
+
const targetState = states.get(options.reauthor.node);
|
|
201
|
+
if (!targetNode) throw new Error(`unknown node id: ${options.reauthor.node}`);
|
|
202
|
+
if (!targetState) throw new Error(`missing persisted node snapshot for ${options.reauthor.node}`);
|
|
203
|
+
const reauthorBudget = options.reauthor.rounds ?? DEFAULT_REAUTHOR_ROUNDS;
|
|
204
|
+
const outcome = await reauthorRefusedNode({
|
|
205
|
+
contract, contractPath, runDir, node: targetNode, state: targetState,
|
|
206
|
+
discover: options.reauthor.discover,
|
|
207
|
+
rounds: reauthorBudget,
|
|
208
|
+
approve: options.reauthor.approve === true,
|
|
209
|
+
approveBelow: options.reauthor.approveBelow,
|
|
210
|
+
});
|
|
211
|
+
process.stdout.write(`[resume] ${contract.id} · reauthor ${targetNode.id} · ${outcome.outcome} (${outcome.roundsUsed}/${reauthorBudget} round${reauthorBudget === 1 ? "" : "s"})\n`);
|
|
212
|
+
if (outcome.applied && outcome.contract && outcome.raw) {
|
|
213
|
+
contract = outcome.contract;
|
|
214
|
+
const replacement = contract.nodes.find((node) => node.id === targetNode.id);
|
|
215
|
+
if (!replacement) throw new Error(`widened contract lost node ${targetNode.id}`);
|
|
216
|
+
targetState.packetHash = replacement.packetHash;
|
|
217
|
+
targetState.sourceIdentity = replacement.sourceIdentity;
|
|
218
|
+
writeJsonAtomic(contractPath, outcome.raw);
|
|
219
|
+
reauthorAppliedNode = targetNode.id;
|
|
173
220
|
}
|
|
174
221
|
}
|
|
175
222
|
const plan = planResumeRetry(contract, states, { node: options.node, reconcile: options.reconcile, answer: options.answer?.node });
|
|
@@ -185,9 +232,9 @@ export async function resumeRun(runDirPath, options = {}) {
|
|
|
185
232
|
const state = states.get(node.id);
|
|
186
233
|
if (!state) continue;
|
|
187
234
|
if (state.status === "done") continue;
|
|
188
|
-
if (isBlockedContextTerminal(state) && node.id !== options.answer?.node) continue;
|
|
235
|
+
if (isBlockedContextTerminal(state) && node.id !== options.answer?.node && node.id !== reauthorAppliedNode) continue;
|
|
189
236
|
if (isHumanStepPending(state) && node.id !== options.answer?.node) continue;
|
|
190
|
-
const action = plan.actions.get(node.id) ?? "recover";
|
|
237
|
+
const action = node.id === reauthorAppliedNode ? "retry" : plan.actions.get(node.id) ?? "recover";
|
|
191
238
|
// Adoption before retry: an unresolved blocking review is re-judged from
|
|
192
239
|
// the preserved worker result, never reset to a fresh worker attempt.
|
|
193
240
|
if (action === "rejudge") {
|
|
@@ -546,6 +593,66 @@ export async function resumeRun(runDirPath, options = {}) {
|
|
|
546
593
|
export function isBlockedContextTerminal(state) {
|
|
547
594
|
return state.status === "blocked" && state.error?.code === "context_missing";
|
|
548
595
|
}
|
|
596
|
+
/**
|
|
597
|
+
* The default number of discovery rounds a `resume --reauthor` spends widening
|
|
598
|
+
* one refused packet. The budget is hard: the round after this one never runs.
|
|
599
|
+
*/
|
|
600
|
+
export const DEFAULT_REAUTHOR_ROUNDS = 1;
|
|
601
|
+
/** One round of the reauthor budget: its proposal, candidate, verdict, findings. */
|
|
602
|
+
/** @typedef {{accepted: boolean, round: number, additions: import("../contract/scope-findings.mjs").ReauthorAdditions, packet: import("../contract/index.mjs").TaskPacket, candidate: Record<string, unknown>, widened: ValidatedContract|null, findings: string[]}} ReauthorRound */
|
|
603
|
+
/**
|
|
604
|
+
* Run the bounded discovery pass for one `context_missing` node, validate the
|
|
605
|
+
* widened packet against the whole contract (per-node closure, the cross-node
|
|
606
|
+
* pair, and the write-overlap check no single packet can see), and return the
|
|
607
|
+
* decision. The caller owns applying it: an accepted packet below the approval
|
|
608
|
+
* threshold, or any packet the operator approved, is persisted and the node
|
|
609
|
+
* re-dispatched; everything else leaves the node refused.
|
|
610
|
+
*
|
|
611
|
+
* @param {{contract: ValidatedContract, contractPath: string, runDir: string, node: import("../contract/index.mjs").ValidatedNode, state: NodeSnapshot, discover: (context: {round: number, node: import("../contract/index.mjs").ValidatedNode, state: NodeSnapshot, missingContext: string[], findings: string[]}) => (import("../contract/scope-findings.mjs").ReauthorAdditions|Promise<import("../contract/scope-findings.mjs").ReauthorAdditions>), rounds?: number, approve?: boolean, approveBelow?: string}} options
|
|
612
|
+
* @returns {Promise<{accepted: boolean, roundsUsed: number, last: ReauthorRound|null, history: ReauthorRound[], applied: boolean, outcome: string, record: Record<string, unknown>, contract: ValidatedContract|null, raw: Record<string, unknown>|null}>}
|
|
613
|
+
*/
|
|
614
|
+
export async function reauthorRefusedNode(options) {
|
|
615
|
+
const { contract, contractPath, runDir, node, state, discover, rounds = DEFAULT_REAUTHOR_ROUNDS, approve = false, approveBelow } = options;
|
|
616
|
+
if (!isBlockedContextTerminal(state)) throw new Error(`node ${node.id} is not blocked on missing context, so it cannot be reauthored`);
|
|
617
|
+
if (typeof discover !== "function") throw new TypeError("resume --reauthor requires a discovery runner");
|
|
618
|
+
const raw = JSON.parse(readFileSync(contractPath, "utf8"));
|
|
619
|
+
const refused = /** @type {{missingContext?: string[]}|null|undefined} */ (state.result);
|
|
620
|
+
const missingContext = Array.isArray(refused?.missingContext) ? refused.missingContext : [];
|
|
621
|
+
const budget = await runBoundedRounds(rounds, async (round, history) => {
|
|
622
|
+
const additions = (await discover({ round, node, state, missingContext, findings: history.flatMap((item) => item.findings) })) ?? {};
|
|
623
|
+
const packet = widenTaskPacket(node.taskPacket, additions);
|
|
624
|
+
const candidate = replacePacketInContract(raw, node.id, packet);
|
|
625
|
+
const findings = reauthorWriteConflicts(contract.nodes, node.id, additions.writeFiles).map((conflict) => conflict.reason);
|
|
626
|
+
/** @type {ValidatedContract|null} */
|
|
627
|
+
let widened = null;
|
|
628
|
+
try {
|
|
629
|
+
widened = validateContract(candidate, contractPath);
|
|
630
|
+
} catch (error) {
|
|
631
|
+
findings.push(errorMessage(error));
|
|
632
|
+
}
|
|
633
|
+
const accepted = widened !== null && findings.length === 0;
|
|
634
|
+
if (accepted && widened) {
|
|
635
|
+
const replacement = widened.nodes.find((item) => item.id === node.id);
|
|
636
|
+
if (replacement) /** @type {any} */ (candidate).nodes.find((/** @type {any} */ item) => item.id === node.id).packetHash = replacement.packetHash;
|
|
637
|
+
}
|
|
638
|
+
return { accepted, round, additions, packet, candidate, widened, findings };
|
|
639
|
+
});
|
|
640
|
+
const last = budget.last;
|
|
641
|
+
const outcome = budget.accepted ? (reauthorApproved(node, { approve, approveBelow }) ? "applied" : "approval_required") : "rounds_exhausted";
|
|
642
|
+
const record = reauthorProposalRecord({
|
|
643
|
+
nodeId: node.id, outcome, riskTier: reauthorRiskTier(node), budget: rounds, roundsUsed: budget.roundsUsed,
|
|
644
|
+
additions: last?.additions, packet: last?.packet, findings: last?.findings, history: budget.history,
|
|
645
|
+
});
|
|
646
|
+
appendJsonl(join(runDir, "reauthor.jsonl"), record);
|
|
647
|
+
return {
|
|
648
|
+
...budget,
|
|
649
|
+
applied: outcome === "applied",
|
|
650
|
+
outcome,
|
|
651
|
+
record,
|
|
652
|
+
contract: budget.accepted ? last?.widened ?? null : null,
|
|
653
|
+
raw: budget.accepted ? last?.candidate ?? null : null,
|
|
654
|
+
};
|
|
655
|
+
}
|
|
549
656
|
/**
|
|
550
657
|
* The operator-answer result summary for a human-step node's `done`
|
|
551
658
|
* transition, bounded to the worker-result protocol's own summary ceiling
|
package/src/engine/retry.mjs
CHANGED
|
@@ -15,12 +15,11 @@
|
|
|
15
15
|
* exhausted run budget needs `--max-input-tokens <n>`.
|
|
16
16
|
*/
|
|
17
17
|
import { Buffer } from "node:buffer";
|
|
18
|
+
import { PREVIOUS_ATTEMPT_EVIDENCE_MAX_BYTES } from "../contract/snapshot.mjs";
|
|
18
19
|
|
|
19
20
|
/** Heading of the section appended to a retried attempt's worker and judge prompt. */
|
|
20
21
|
const PREVIOUS_ATTEMPT_HEADING = "Previous attempt";
|
|
21
22
|
|
|
22
|
-
/** Hard ceiling for the whole `Previous attempt` section, in bytes. */
|
|
23
|
-
const PREVIOUS_ATTEMPT_MAX_BYTES = 8 * 1024;
|
|
24
23
|
|
|
25
24
|
/** Heading under which an operator's answer is rendered inside the section. */
|
|
26
25
|
const OPERATOR_ANSWER_HEADING = "Operator answer";
|
|
@@ -267,11 +266,6 @@ export function renderPreviousAttemptSection(state) {
|
|
|
267
266
|
if (state.error?.code || state.error?.message) {
|
|
268
267
|
parts.push(`Error: ${state.error?.code ?? "unknown"}${state.error?.message ? ` — ${bounded(state.error.message, 1024)}` : ""}`);
|
|
269
268
|
}
|
|
270
|
-
const answer = mostRecentOperatorAnswer(state);
|
|
271
|
-
if (answer !== null) {
|
|
272
|
-
parts.push(`${OPERATOR_ANSWER_HEADING}:`);
|
|
273
|
-
parts.push(bounded(answer, 1024));
|
|
274
|
-
}
|
|
275
269
|
const findings = state.gate?.findings ?? [];
|
|
276
270
|
if (findings.length) {
|
|
277
271
|
parts.push(`Judge findings (verdict ${state.gate?.verdict ?? "unknown"}, maxSeverity ${state.gate?.maxSeverity ?? "none"}):`);
|
|
@@ -293,8 +287,14 @@ export function renderPreviousAttemptSection(state) {
|
|
|
293
287
|
if (typeof last?.exitCode === "number" && last.exitCode !== 0) parts.push(` exit code: ${last.exitCode}`);
|
|
294
288
|
}
|
|
295
289
|
}
|
|
296
|
-
|
|
297
|
-
|
|
290
|
+
const answer = mostRecentOperatorAnswer(state);
|
|
291
|
+
if (parts.length === 1 && answer === null) return null;
|
|
292
|
+
const evidence = boundSection([`## ${PREVIOUS_ATTEMPT_HEADING}`, "", ...parts].join("\n"));
|
|
293
|
+
// The answer was bounded where it was accepted (OPERATOR_ANSWER_MAX_BYTES),
|
|
294
|
+
// so it is carried whole, after the evidence and outside its budget.
|
|
295
|
+
// Measured 2026-09-28: bounded here at 1 KiB inside an 8 KiB section, an
|
|
296
|
+
// answer the engine had accepted and recorded reached the worker cut short.
|
|
297
|
+
return answer === null ? evidence : `${evidence}\n${OPERATOR_ANSWER_HEADING}:\n${answer}`;
|
|
298
298
|
}
|
|
299
299
|
|
|
300
300
|
/**
|
|
@@ -315,9 +315,9 @@ export function appendPreviousAttempt(prompt, section) {
|
|
|
315
315
|
* @returns {string} the text bounded to the section ceiling, with a marker
|
|
316
316
|
*/
|
|
317
317
|
function boundSection(text) {
|
|
318
|
-
if (Buffer.byteLength(text, "utf8") <=
|
|
318
|
+
if (Buffer.byteLength(text, "utf8") <= PREVIOUS_ATTEMPT_EVIDENCE_MAX_BYTES) return text;
|
|
319
319
|
const marker = "\n… (previous attempt section truncated)";
|
|
320
|
-
const room =
|
|
320
|
+
const room = PREVIOUS_ATTEMPT_EVIDENCE_MAX_BYTES - Buffer.byteLength(marker, "utf8") - 1;
|
|
321
321
|
const cut = Buffer.from(text, "utf8").subarray(0, room).toString("utf8");
|
|
322
322
|
return `${cut}${marker}`;
|
|
323
323
|
}
|
|
@@ -16,10 +16,35 @@ import { mkdirSync, writeFileSync } from "node:fs";
|
|
|
16
16
|
import { processStartToken } from "../run/lock.mjs";
|
|
17
17
|
import { randomUUID } from "node:crypto";
|
|
18
18
|
import { runMutation } from "./mutation.mjs";
|
|
19
|
+
import { SANDBOX_BLOCKED_WRITE } from "../contract/worker-result.mjs";
|
|
20
|
+
import { isAbsolute, resolve } from "node:path";
|
|
21
|
+
import { isContained } from "../util.mjs";
|
|
19
22
|
import { spawn } from "node:child_process";
|
|
20
23
|
import { killTarget, spawnInvocation } from "../host/platform.mjs";
|
|
21
24
|
/** @typedef {import("../contract/verification.mjs").VerificationOptions} VerificationOptions */
|
|
22
25
|
|
|
26
|
+
/**
|
|
27
|
+
* The run-level options plus the sandbox mode the caller knows. The mode is not
|
|
28
|
+
* part of the shared verification contract: it is controller environment, read
|
|
29
|
+
* here only to name a refusal the command itself cannot.
|
|
30
|
+
*
|
|
31
|
+
* @typedef {VerificationOptions & {sandboxMode?: string|null}} RunVerificationOptions
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* The named classification of a `workspace-write` refusal outside the worktree,
|
|
36
|
+
* carrying the mode that caused it and the path that was denied.
|
|
37
|
+
*
|
|
38
|
+
* @typedef {{classification: "sandbox_blocked_write", mode: string, path: string}} SandboxBlockedWrite
|
|
39
|
+
*/
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* A bounded attempt result plus the controller's sandbox classification, which
|
|
43
|
+
* rides beside the captured output only when a refusal was recognized.
|
|
44
|
+
*
|
|
45
|
+
* @typedef {VerificationAttemptResult & {attempt: number, sandboxBlockedWrite?: SandboxBlockedWrite}} SandboxAwareAttemptResult
|
|
46
|
+
*/
|
|
47
|
+
|
|
23
48
|
/** @typedef {import("node:child_process").ChildProcess} ChildProcess */
|
|
24
49
|
/** @typedef {import("../contract/verification.mjs").VerificationAttempt} VerificationAttempt */
|
|
25
50
|
/** @typedef {import("../contract/verification.mjs").VerificationAttemptResult} VerificationAttemptResult */
|
|
@@ -62,12 +87,51 @@ function verificationEnv(command) {
|
|
|
62
87
|
}
|
|
63
88
|
return env;
|
|
64
89
|
}
|
|
90
|
+
|
|
91
|
+
// `workspace-write` denies a toolchain's cache in `$HOME` and the tool reports
|
|
92
|
+
// a read-only filesystem: observed 2026-09-22 with Zig under `dsh`,
|
|
93
|
+
// `manifest_create ReadOnlyFileSystem` before its first source file. The
|
|
94
|
+
// sandbox, not the command, is the cause, so the controller names it. `EROFS`
|
|
95
|
+
// and the POSIX message cover the same refusal from other runners.
|
|
96
|
+
const SANDBOX_READ_ONLY_PATTERN = /ReadOnlyFileSystem|EROFS|read-only file system/iu;
|
|
97
|
+
// A denied path is usually quoted (`unable to load '/path': ReadOnlyFileSystem`)
|
|
98
|
+
// or bare after the operation name; either way it is absolute, since only an
|
|
99
|
+
// effect outside the worktree gets refused.
|
|
100
|
+
const ABSOLUTE_PATH_PATTERNS = Object.freeze([/['"]((?:[A-Za-z]:[\\/]|\/)[^'"]+)['"]/gu, /((?:[A-Za-z]:[\\/]|\/)[^\s'",;:)\]]+)/gu]);
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Classify a failed command whose output names a read-only-filesystem refusal
|
|
104
|
+
* outside the worktree, under `workspace-write`. Returns null for any other
|
|
105
|
+
* mode, a refusal inside the worktree, or output with no path to name.
|
|
106
|
+
*
|
|
107
|
+
* @param {{text?: string, workspace: string, mode?: string|null}} args
|
|
108
|
+
* @returns {SandboxBlockedWrite|null}
|
|
109
|
+
*/
|
|
110
|
+
export function classifySandboxBlockedWrite({ text, workspace, mode }) {
|
|
111
|
+
if (mode !== "workspace-write" || typeof text !== "string") return null;
|
|
112
|
+
const lines = text.split(/\r?\n/u);
|
|
113
|
+
const markerLines = lines.filter((line) => SANDBOX_READ_ONLY_PATTERN.test(line));
|
|
114
|
+
if (markerLines.length === 0) return null;
|
|
115
|
+
// Prefer a path the refusal line itself names; fall back to any path in the
|
|
116
|
+
// output, since `manifest_create ReadOnlyFileSystem` can put the path on a
|
|
117
|
+
// neighbouring line.
|
|
118
|
+
for (const line of [...markerLines, ...lines]) {
|
|
119
|
+
for (const pattern of ABSOLUTE_PATH_PATTERNS) {
|
|
120
|
+
for (const match of line.matchAll(pattern)) {
|
|
121
|
+
const candidate = match[1];
|
|
122
|
+
const path = isAbsolute(candidate) ? candidate : resolve(workspace, candidate);
|
|
123
|
+
if (!isContained(workspace, path)) return { classification: SANDBOX_BLOCKED_WRITE, mode, path };
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
return null;
|
|
128
|
+
}
|
|
65
129
|
/**
|
|
66
130
|
* Run every declared command `repeat` times inside the workspace.
|
|
67
131
|
*
|
|
68
132
|
* @param {unknown} commands
|
|
69
133
|
* @param {string} baseCwd
|
|
70
|
-
* @param {
|
|
134
|
+
* @param {RunVerificationOptions} options
|
|
71
135
|
* @returns {Promise<VerificationResult>}
|
|
72
136
|
*/
|
|
73
137
|
export async function runVerification(commands, baseCwd, options = {}) {
|
|
@@ -92,7 +156,7 @@ export async function runVerification(commands, baseCwd, options = {}) {
|
|
|
92
156
|
* @param {VerificationCommand} command
|
|
93
157
|
* @param {string} baseCwd
|
|
94
158
|
* @param {number} commandIndex
|
|
95
|
-
* @param {
|
|
159
|
+
* @param {RunVerificationOptions} options
|
|
96
160
|
* @returns {Promise<VerificationCommandResult>}
|
|
97
161
|
*/
|
|
98
162
|
async function runRepeatedCommand(command, baseCwd, commandIndex, options) {
|
|
@@ -132,7 +196,7 @@ function signalDeathRetry(result, signal) {
|
|
|
132
196
|
* @param {VerificationCommand} command
|
|
133
197
|
* @param {string} baseCwd
|
|
134
198
|
* @param {number} commandIndex
|
|
135
|
-
* @param {
|
|
199
|
+
* @param {RunVerificationOptions} options
|
|
136
200
|
* @returns {Promise<VerificationCommandResult>}
|
|
137
201
|
*/
|
|
138
202
|
async function runMutationCommand(command, baseCwd, commandIndex, options) {
|
|
@@ -149,7 +213,7 @@ async function runMutationCommand(command, baseCwd, commandIndex, options) {
|
|
|
149
213
|
* @param {string} commandCwd
|
|
150
214
|
* @param {number} attempt
|
|
151
215
|
* @param {AbortSignal|undefined} signal
|
|
152
|
-
* @param {
|
|
216
|
+
* @param {RunVerificationOptions} options
|
|
153
217
|
* @param {number} commandIndex
|
|
154
218
|
* @returns {Promise<VerificationAttemptResult>}
|
|
155
219
|
*/
|
|
@@ -181,6 +245,7 @@ function runCommand(command, baseCwd, commandCwd, attempt, signal, options, comm
|
|
|
181
245
|
if (abortHandler) signal?.removeEventListener("abort", abortHandler);
|
|
182
246
|
if (child?.pid && !error && !signalName && !timedOut) terminateGroup(child);
|
|
183
247
|
const durationMs = Number(process.hrtime.bigint() - started) / 1e6;
|
|
248
|
+
/** @type {SandboxAwareAttemptResult} */
|
|
184
249
|
const result = {
|
|
185
250
|
attempt,
|
|
186
251
|
stdout: stdout.value(),
|
|
@@ -192,6 +257,17 @@ function runCommand(command, baseCwd, commandCwd, attempt, signal, options, comm
|
|
|
192
257
|
error: error ? String(error.message ?? error) : null,
|
|
193
258
|
passed: !error && !timedOut && exitCode === 0 && !signalName,
|
|
194
259
|
};
|
|
260
|
+
// The controller's own sandbox mode bounds where a toolchain may write;
|
|
261
|
+
// a caller may name it, and a controller launched under `dsh` inherits it
|
|
262
|
+
// in `DSH_PERMISSION_MODE`. Only `workspace-write` classifies.
|
|
263
|
+
if (!result.passed) {
|
|
264
|
+
const sandboxBlockedWrite = classifySandboxBlockedWrite({
|
|
265
|
+
text: `${result.stdout}\n${result.stderr}`,
|
|
266
|
+
workspace: resolve(baseCwd, commandCwd),
|
|
267
|
+
mode: options?.sandboxMode ?? process.env.DSH_PERMISSION_MODE ?? null,
|
|
268
|
+
});
|
|
269
|
+
if (sandboxBlockedWrite) result.sandboxBlockedWrite = sandboxBlockedWrite;
|
|
270
|
+
}
|
|
195
271
|
if (!completionReported && identity) {
|
|
196
272
|
completionReported = true;
|
|
197
273
|
try { options?.onAttemptComplete?.({ ...identity, status: result.passed ? "closed" : "failed", completedAt: new Date().toISOString(), result }); } catch {
|
|
@@ -21,7 +21,7 @@ import { createHash } from "node:crypto";
|
|
|
21
21
|
import { lstatSync, readFileSync, readdirSync } from "node:fs";
|
|
22
22
|
import { join, relative, sep } from "node:path";
|
|
23
23
|
import { fileURLToPath } from "node:url";
|
|
24
|
-
import { stableJson } from "../util.mjs";
|
|
24
|
+
import { stableJson, errorCode } from "../util.mjs";
|
|
25
25
|
import { validateRunMetadata } from "../contract/snapshot.mjs";
|
|
26
26
|
import { contractDigest } from "../contract/index.mjs";
|
|
27
27
|
import { RUNS_DIR_NAME, runDirectory } from "../run/paths.mjs";
|
|
@@ -97,6 +97,26 @@ function runDirFor(sourceIdentity) {
|
|
|
97
97
|
return runDirectory(sourceIdentity.cwd, sourceIdentity.contractId);
|
|
98
98
|
}
|
|
99
99
|
|
|
100
|
+
/**
|
|
101
|
+
* Whether a run directory never materialized a node. The directory, its
|
|
102
|
+
* frozen contract and its empty `nodes/` folder are written before runtime
|
|
103
|
+
* assignment and identity capture; node states are written after. A launch
|
|
104
|
+
* refused in that window leaves a directory that never held a node, so
|
|
105
|
+
* nothing can resume it and it would refuse the next attempt as an existing
|
|
106
|
+
* run.
|
|
107
|
+
*
|
|
108
|
+
* @param {string} runDir
|
|
109
|
+
* @returns {boolean}
|
|
110
|
+
*/
|
|
111
|
+
export function runNeverHeldNode(runDir) {
|
|
112
|
+
try {
|
|
113
|
+
return readdirSync(join(runDir, "nodes")).every((name) => !name.endsWith(".json"));
|
|
114
|
+
} catch (error) {
|
|
115
|
+
if (errorCode(error) === "ENOENT") return true;
|
|
116
|
+
throw error;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
100
120
|
/**
|
|
101
121
|
* The records a prior run.json already froze, when one exists. On creation
|
|
102
122
|
* this is empty; on every later metadata rewrite it is the source of truth.
|
|
@@ -506,14 +526,10 @@ export async function assertEnvironmentReady(contract, runDir, sourceIdentity) {
|
|
|
506
526
|
// state reads done, this launch replays an accepted transaction, starts no
|
|
507
527
|
// worker and no judge, and can spend no availability.
|
|
508
528
|
if (blocking === null && launchMayDispatch(runDir)) {
|
|
509
|
-
//
|
|
510
|
-
//
|
|
511
|
-
//
|
|
512
|
-
|
|
513
|
-
// runtime; preflightContract validates it, so only a valid number is lifted.
|
|
514
|
-
const override = Number(process.env.FABERUN_PREFLIGHT_TIMEOUT_SEC);
|
|
515
|
-
const timeoutSec = process.env.FABERUN_PREFLIGHT_TIMEOUT_SEC !== undefined && Number.isFinite(override) && override > 0 ? override : undefined;
|
|
516
|
-
const probes = await livePreflightProbes(contract, runDir, sourceIdentity, timeoutSec);
|
|
529
|
+
// The live budget is `live-preflight.mjs`'s measured default (60s, 180s at xhigh;
|
|
530
|
+
// `FABERUN_PREFLIGHT_TIMEOUT_SEC` the operator override). It is not
|
|
531
|
+
// restated here: a second copy is a second default to keep honest.
|
|
532
|
+
const probes = await livePreflightProbes(contract, runDir, sourceIdentity);
|
|
517
533
|
const silent = probes.filter((probe) => liveSilenceCause(probe) !== null);
|
|
518
534
|
writeJsonAtomic(join(runDir, "env-preflight.json"), evidence(silent.length === 0, probes));
|
|
519
535
|
if (silent.length > 0) {
|
|
@@ -537,8 +553,6 @@ export async function assertEnvironmentReady(contract, runDir, sourceIdentity) {
|
|
|
537
553
|
throw Object.assign(new Error(`env_preflight_failed: ${blocking} · the run stays resumable: fix the environment and resume ${runDir}`), { code: "env_preflight_failed" });
|
|
538
554
|
}
|
|
539
555
|
|
|
540
|
-
/** No routed runtime is given less than this by the dispatch gate. */
|
|
541
|
-
const GATE_MINIMUM_PREFLIGHT_SEC = 60;
|
|
542
556
|
/**
|
|
543
557
|
* The live half of the gate for one launch. Every routed runtime either holds
|
|
544
558
|
* a verdict this machine recorded inside its freshness window -- reused, the
|
|
@@ -555,10 +569,9 @@ const GATE_MINIMUM_PREFLIGHT_SEC = 60;
|
|
|
555
569
|
* @param {ValidatedContract} contract
|
|
556
570
|
* @param {string} runDir
|
|
557
571
|
* @param {SourceIdentity|undefined} sourceIdentity
|
|
558
|
-
* @param {number|undefined} timeoutSec
|
|
559
572
|
* @returns {Promise<ProbeResult[]>}
|
|
560
573
|
*/
|
|
561
|
-
async function livePreflightProbes(contract, runDir, sourceIdentity
|
|
574
|
+
async function livePreflightProbes(contract, runDir, sourceIdentity) {
|
|
562
575
|
const routed = reachableRuntimes(contract);
|
|
563
576
|
/** @type {Map<string, RuntimeAvailability>} */
|
|
564
577
|
const fresh = new Map();
|
|
@@ -631,7 +644,7 @@ async function livePreflightProbes(contract, runDir, sourceIdentity, timeoutSec)
|
|
|
631
644
|
};
|
|
632
645
|
});
|
|
633
646
|
}
|
|
634
|
-
const probes = await preflightContract(join(runDir, "contract.json"), {
|
|
647
|
+
const probes = await preflightContract(join(runDir, "contract.json"), { persisted: true });
|
|
635
648
|
// What this launch bought is durable from here on: every ask that reached a
|
|
636
649
|
// provider -- a refusal included, an answer being an answer -- is recorded
|
|
637
650
|
// under the provider's own identity. Silence and a command that never
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { normalizeProviderAvailability, probeRuntime } from "../harnesses/index.mjs";
|
|
2
2
|
import { effectiveProvider } from "../contract/provider.mjs";
|
|
3
|
-
import { SAME_VENDOR_REVIEW_MODE, sameVendorTierRefusal } from "../contract/judge-independence.mjs";
|
|
3
|
+
import { SAME_VENDOR_REVIEW_MODE, effectiveJudgeIndependence, sameVendorTierRefusal } from "../contract/judge-independence.mjs";
|
|
4
4
|
|
|
5
5
|
// Availability normalization belongs to the adapter registry, which is where
|
|
6
6
|
// each provider's own exhaustion, balance, and authentication wording is
|
|
@@ -176,10 +176,12 @@ export function availableCandidates(runtimes, availability = {}) {
|
|
|
176
176
|
* copied exactly; callers persist the returned pair in run state.
|
|
177
177
|
*
|
|
178
178
|
* An optional `options.config` narrows the candidate set to the harnesses the
|
|
179
|
-
* operator enabled
|
|
180
|
-
*
|
|
181
|
-
* `
|
|
182
|
-
*
|
|
179
|
+
* operator enabled, prefers its named worker and judge, and -- when the
|
|
180
|
+
* contract declares none (R36) -- supplies the machine-wide
|
|
181
|
+
* `judgeIndependence` opt-in. An empty narrowed set falls back to the
|
|
182
|
+
* unrestricted candidates and is reported through `options.onWarning`; config
|
|
183
|
+
* never overrides an explicit node, gate or runtime-default declaration, and
|
|
184
|
+
* the cross-vendor judge rule still applies.
|
|
183
185
|
*
|
|
184
186
|
* `options.listJudge` (R18) is asked for an omitted judge before the single
|
|
185
187
|
* `config.judge` preference and the strongest-candidate default: it is the
|
|
@@ -194,7 +196,7 @@ export function availableCandidates(runtimes, availability = {}) {
|
|
|
194
196
|
* @param {RuntimeContract} contract
|
|
195
197
|
* @param {Record<string, RuntimeAvailability>} availability
|
|
196
198
|
* @param {ComposeOptions} [options]
|
|
197
|
-
* @returns {Record<string, {worker: string, judge: string}>}
|
|
199
|
+
* @returns {Record<string, {worker: string, judge: string, judgeIndependence?: "same-vendor"}>}
|
|
198
200
|
*/
|
|
199
201
|
export function composeAssignments(contract, availability = {}, options = {}) {
|
|
200
202
|
const allCandidates = availableCandidates(contract.runtimes, availability);
|
|
@@ -206,7 +208,7 @@ export function composeAssignments(contract, availability = {}, options = {}) {
|
|
|
206
208
|
if (restricted.length) candidates = restricted;
|
|
207
209
|
else if (allCandidates.length) options.onWarning?.(`config harnesses · none of ${config.harnesses.join(", ")} is available · using all available runtimes`);
|
|
208
210
|
}
|
|
209
|
-
/** @type {Record<string, {worker: string, judge: string}>} */
|
|
211
|
+
/** @type {Record<string, {worker: string, judge: string, judgeIndependence?: "same-vendor"}>} */
|
|
210
212
|
const assignments = {};
|
|
211
213
|
for (const node of contract.nodes ?? []) {
|
|
212
214
|
const workerOmitted = node.runtime === undefined && contract.runtimeDefaults?.worker === undefined;
|
|
@@ -235,7 +237,11 @@ export function composeAssignments(contract, availability = {}, options = {}) {
|
|
|
235
237
|
// R20: in same-vendor mode a judge of the worker's own provider is
|
|
236
238
|
// admissible when it is another model at an equal or higher declared tier;
|
|
237
239
|
// the strongest such candidate is the default only when no other vendor has one.
|
|
238
|
-
|
|
240
|
+
// R36: a contract that declares no mode inherits the machine config's, and
|
|
241
|
+
// the effective mode is recorded on the assignment so the launch decision
|
|
242
|
+
// survives in the run snapshot.
|
|
243
|
+
const judgeIndependence = effectiveJudgeIndependence(contract, config);
|
|
244
|
+
const sameVendorMode = judgeIndependence === SAME_VENDOR_REVIEW_MODE;
|
|
239
245
|
const judge = declaredJudge
|
|
240
246
|
?? judgeListPick?.chosen
|
|
241
247
|
?? (preferredJudge && effectiveProvider(preferredJudge.runtime) !== workerProvider ? preferredJudge.id : undefined)
|
|
@@ -251,7 +257,7 @@ export function composeAssignments(contract, availability = {}, options = {}) {
|
|
|
251
257
|
const refusal = sameVendorTierRefusal(/** @type {{harness: string, model: string}} */ (workerRuntime), /** @type {{harness: string, model: string}} */ (judgeRuntime));
|
|
252
258
|
if (refusal) throw new Error(`runtime_assignment_judge_unavailable: ${refusal.message} for node ${node.id}`);
|
|
253
259
|
}
|
|
254
|
-
assignments[node.id] = { worker, judge: judge ?? worker };
|
|
260
|
+
assignments[node.id] = { worker, judge: judge ?? worker, ...(judgeIndependence === undefined ? {} : { judgeIndependence }) };
|
|
255
261
|
}
|
|
256
262
|
return assignments;
|
|
257
263
|
}
|
package/src/engine/scheduler.mjs
CHANGED
|
@@ -759,7 +759,14 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
|
|
|
759
759
|
heartbeat.stop();
|
|
760
760
|
lock.release();
|
|
761
761
|
}
|
|
762
|
-
|
|
762
|
+
// Forced, and after the lock is released: the change-detected render here
|
|
763
|
+
// was a no-op whenever the last tick had already written the same node
|
|
764
|
+
// states, which is the usual case, so the stored status.json kept the
|
|
765
|
+
// `controller: active` the last tick wrote while the lock was still held.
|
|
766
|
+
// Measured 2026-09-27 (AP12): `safe-to-hand-to-a-friend-phase-2` finished
|
|
767
|
+
// with `state: "active"` and pid 43002, and a watcher waiting for the
|
|
768
|
+
// controller to leave spun for two hours on a pid that no longer existed.
|
|
769
|
+
renderStatusIfChanged(true, null);
|
|
763
770
|
renderCampaignHandoffSafely(campaign, runsDir, runDir);
|
|
764
771
|
writeFindingsArtifact(runDir, contract, states);
|
|
765
772
|
const failed = [...states.values()].filter((state) => state.status !== "done");
|