faberun 0.29.0 → 0.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/campaign/campaign-brief.mjs +17 -6
- package/src/campaign/chain.mjs +124 -8
- package/src/campaign/ledger.mjs +19 -7
- package/src/campaign/metrics-command.mjs +68 -9
- package/src/campaign/metrics.mjs +75 -45
- package/src/campaign/record.mjs +8 -1
- package/src/campaign/reserve.mjs +340 -0
- package/src/campaign/spend.mjs +229 -0
- package/src/campaign/unpark.mjs +54 -1
- package/src/campaign/watch.mjs +314 -37
- package/src/contract/judge-envelope.mjs +17 -2
- package/src/contract/scope-findings.mjs +3 -1
- package/src/contract/task-packet.mjs +40 -5
- package/src/contract/verification.mjs +67 -1
- package/src/engine/aux-admission.mjs +124 -0
- package/src/engine/capacity.mjs +78 -0
- package/src/engine/dispatch.mjs +116 -23
- package/src/engine/failure-cause.mjs +59 -0
- package/src/engine/lifecycle.mjs +3 -2
- package/src/engine/phase-session.mjs +56 -10
- package/src/engine/prompts.mjs +57 -2
- package/src/engine/reserve-admission.mjs +99 -0
- package/src/engine/resume.mjs +3 -1
- package/src/engine/review.mjs +21 -0
- package/src/engine/run-command.mjs +166 -11
- package/src/engine/scheduler.mjs +71 -88
- package/src/engine/settle.mjs +15 -8
- package/src/engine/settlement-overlap.mjs +143 -0
- package/src/engine/verify.mjs +131 -7
- package/src/harnesses/session-metrics.mjs +112 -20
- package/src/notify/index.mjs +86 -10
- package/src/plan/freeze.mjs +55 -15
- package/src/plan/human-step.mjs +36 -0
- package/src/plan/pipeline.mjs +120 -38
- package/src/plan/preflight.mjs +81 -5
- package/src/plan/progress.mjs +101 -1
- package/src/plan/repo-facts.mjs +431 -55
- package/src/plan/resolve.mjs +13 -4
- package/src/plan/revise-context.mjs +108 -0
- package/src/plan/rounds.mjs +209 -50
- package/src/plan/sizing.mjs +53 -5
- package/src/plan/template.mjs +297 -49
- package/src/repo/integrate.mjs +47 -5
- package/src/repo/scope-closure.mjs +22 -0
- package/src/report/campaign-brief-estimate.mjs +47 -1
- package/src/report/campaign-brief.mjs +27 -0
- package/src/report/locale.mjs +20 -0
- package/src/report/metrics-report.mjs +38 -0
- package/src/report/progress.mjs +194 -10
- package/src/run/usage-windows.mjs +99 -4
- package/src/run/usage.mjs +69 -7
- package/src/web/api.mjs +179 -24
- package/src/web/app.mjs +27 -2
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "faberun",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.31.0",
|
|
4
4
|
"description": "Faberun is a development orchestration system that turns intent into verified software: harness- and model-agnostic, it keeps the intent, coordinates the work, verifies the result and decides what happens next.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -45,8 +45,14 @@ import { projectCampaignDecisions } from "./projection.mjs";
|
|
|
45
45
|
/** @typedef {import("./campaign-brief-graph.mjs").BriefGraph} BriefGraph */
|
|
46
46
|
/** @typedef {{status: "range"|"insufficient data", min: number|null, max: number|null, samples: number|null, reason: string|null, sourceRuns: string[], method: string|null, provenance: string|null}} BriefMeasure */
|
|
47
47
|
/** @typedef {{status?: "range"|"insufficient data", min?: number|null, max?: number|null, samples?: number|null, reason?: string|null, sourceRuns?: string[], method?: string|null, provenance?: string|null}} BriefMeasureInput */
|
|
48
|
-
/** @typedef {{cost?: BriefMeasureInput, duration?: BriefMeasureInput, runtimes?: string[], models?: string[], effectiveConcurrency?: number, nodeCount?: number, workerCount?: number, sampleCutoff?: string|null, method?: string[], assumptions?: string[]}} BriefEstimateInput */
|
|
49
|
-
/**
|
|
48
|
+
/** @typedef {{cost?: BriefMeasureInput, costPerAccepted?: BriefMeasureInput, duration?: BriefMeasureInput, runtimes?: string[], models?: string[], effectiveConcurrency?: number, nodeCount?: number, workerCount?: number, sampleCutoff?: string|null, method?: string[], assumptions?: string[]}} BriefEstimateInput */
|
|
49
|
+
/**
|
|
50
|
+
* `costPerAccepted` is optional because models built before the measure
|
|
51
|
+
* existed (test fixtures, older snapshots) may omit it: the renderer and the
|
|
52
|
+
* decision gate treat an absent measure as unrecorded evidence, never as a
|
|
53
|
+
* crash. `buildEstimate` itself always records it.
|
|
54
|
+
* @typedef {{cost: BriefMeasure, costPerAccepted?: BriefMeasure, duration: BriefMeasure, runtimes: string[], models: string[], effectiveConcurrency: number, nodeCount: number, workerCount: number, sampleCutoff: string|null, method: string[], assumptions: string[], gaps: string[]}} BriefEstimate
|
|
55
|
+
*/
|
|
50
56
|
/** @typedef {{measure: string, target: string, evidence: string}} BriefSuccessCriterion */
|
|
51
57
|
/** @typedef {{risk: string, impact: string, mitigation: string}} BriefRisk */
|
|
52
58
|
/** @typedef {{intent: string|null, expectedOutcome: string|null, successCriteria: BriefSuccessCriterion[], humanFacts: string[], calculatedFacts: string[], gaps: string[]}} BriefOpening */
|
|
@@ -193,9 +199,11 @@ export function buildBriefModel(options) {
|
|
|
193
199
|
|
|
194
200
|
/**
|
|
195
201
|
* `gaps to resolve` when the coverage matrix, the work graph or the opening's
|
|
196
|
-
* source facts report a gap, or when
|
|
202
|
+
* source facts report a gap, or when any estimate measure lacks its required
|
|
197
203
|
* evidence; otherwise `ready for human review`. Neither state approves
|
|
198
|
-
* execution.
|
|
204
|
+
* execution. Cost per accepted result is an estimate measure like cost and
|
|
205
|
+
* duration: without it the brief cannot support an outcome-vs-price-tag
|
|
206
|
+
* comparison (phase-4 R5), so it gates readiness the same way.
|
|
199
207
|
*
|
|
200
208
|
* @param {{opening: BriefOpening, coverage: BriefCoverage, graph: BriefGraph, decisions: BriefDecisions, estimate: BriefEstimate}} model
|
|
201
209
|
* @returns {BriefDecisionState}
|
|
@@ -205,7 +213,7 @@ export function decideBriefState(model) {
|
|
|
205
213
|
if (model.graph.gaps.length > 0) return "gaps to resolve";
|
|
206
214
|
if (model.decisions.gaps.length > 0) return "gaps to resolve";
|
|
207
215
|
if (model.opening.gaps.length > 0) return "gaps to resolve";
|
|
208
|
-
if (model.estimate.cost.status !== "range" || model.estimate.duration.status !== "range") return "gaps to resolve";
|
|
216
|
+
if (model.estimate.cost.status !== "range" || model.estimate.costPerAccepted?.status !== "range" || model.estimate.duration.status !== "range") return "gaps to resolve";
|
|
209
217
|
return "ready for human review";
|
|
210
218
|
}
|
|
211
219
|
|
|
@@ -492,16 +500,19 @@ function normalizeDecisionText(text) {
|
|
|
492
500
|
*/
|
|
493
501
|
function buildEstimate(input, graph, sampleCutoff) {
|
|
494
502
|
const cost = normalizeMeasure(input?.cost, "cost");
|
|
503
|
+
const costPerAccepted = normalizeMeasure(input?.costPerAccepted, "cost per accepted result");
|
|
495
504
|
const duration = normalizeMeasure(input?.duration, "duration");
|
|
496
505
|
/** @type {string[]} */
|
|
497
506
|
const gaps = [];
|
|
498
507
|
if (cost.status !== "range") gaps.push(`cost estimate reports insufficient data: ${cost.reason ?? "no reason recorded"}`);
|
|
508
|
+
if (costPerAccepted.status !== "range") gaps.push(`cost per accepted result estimate reports insufficient data: ${costPerAccepted.reason ?? "no reason recorded"}`);
|
|
499
509
|
if (duration.status !== "range") gaps.push(`duration estimate reports insufficient data: ${duration.reason ?? "no reason recorded"}`);
|
|
500
510
|
if (typeof sampleCutoff !== "string" || !sampleCutoff.trim()) {
|
|
501
511
|
gaps.push("usage sample cutoff is not recorded");
|
|
502
512
|
}
|
|
503
513
|
return {
|
|
504
514
|
cost,
|
|
515
|
+
costPerAccepted,
|
|
505
516
|
duration,
|
|
506
517
|
runtimes: unique(input?.runtimes ?? graph.nodes.map((node) => node.runtimeId).filter((id) => id !== null).map(String)),
|
|
507
518
|
models: unique(input?.models ?? graph.nodes.map((node) => node.model).filter((model) => model !== null).map(String)),
|
|
@@ -584,7 +595,7 @@ function buildOpening(parsedSpec, coverage, graph, estimate, decisions) {
|
|
|
584
595
|
const calculatedFacts = [
|
|
585
596
|
`Coverage: ${coverage.covered} covered, ${coverage.uncovered} uncovered, ${coverage.outside} outside this plan, ${coverage.traceabilityMissing} traceability missing of ${coverage.total} spec requirements.`,
|
|
586
597
|
`Work graph: ${graph.nodes.length} nodes, ${graph.edges.length} dependency edges, ${graph.independent.length} dependency-independent, effective concurrency ${graph.effectiveConcurrency} under maxParallel ${graph.maxParallel}.`,
|
|
587
|
-
`Estimate: cost ${estimate.cost.status}, duration ${estimate.duration.status}; sample cutoff ${estimate.sampleCutoff ?? "not recorded"}.`,
|
|
598
|
+
`Estimate: cost ${estimate.cost.status}, cost per accepted result ${estimate.costPerAccepted?.status ?? "insufficient data"}, duration ${estimate.duration.status}; sample cutoff ${estimate.sampleCutoff ?? "not recorded"}.`,
|
|
588
599
|
];
|
|
589
600
|
return { intent, expectedOutcome, successCriteria, humanFacts, calculatedFacts, gaps };
|
|
590
601
|
}
|
package/src/campaign/chain.mjs
CHANGED
|
@@ -15,7 +15,17 @@
|
|
|
15
15
|
* Advancement is `runOutcome`, never node counts: `succeeded` advances,
|
|
16
16
|
* `waiting` is a self-resuming tier exhaustion and is neither advanced nor
|
|
17
17
|
* parked, and `parked`/`canceled` park the campaign with attention. An
|
|
18
|
-
* in-flight run (`state: unfinished`/`unknown`) is
|
|
18
|
+
* in-flight run (`state: unfinished`/`unknown`) whose controller is alive is
|
|
19
|
+
* awaited, not relaunched.
|
|
20
|
+
*
|
|
21
|
+
* A run whose controller is gone is the other half of that rule, and the reason
|
|
22
|
+
* R2 exists: the coordinator used to wait on it forever, so a `run` that died
|
|
23
|
+
* after its bootstrap held the whole campaign. The chain resumes that same run
|
|
24
|
+
* through the engine's own `resume` -- it does not re-launch, so no node is
|
|
25
|
+
* dispatched twice -- bounded by `MAX_RUN_RECOVERY_ATTEMPTS` consecutive resumes
|
|
26
|
+
* without progress, after which the campaign records attention instead of
|
|
27
|
+
* retrying. A run carrying a requested pause is never resumed: taking the pause
|
|
28
|
+
* back is the operator's `resume`, not the chain's.
|
|
19
29
|
*
|
|
20
30
|
* Validation of contract N+1 happens here, at launch, against the `landBranch`
|
|
21
31
|
* N's promotion just advanced. Validating the manifest up front would reject
|
|
@@ -24,9 +34,11 @@
|
|
|
24
34
|
* checked first so tampering between authoring and launch is caught before the
|
|
25
35
|
* branch-aware validation runs.
|
|
26
36
|
*
|
|
27
|
-
* The chain
|
|
28
|
-
* `coordinator.lock`, the campaign `heartbeat.json`, and the
|
|
29
|
-
* record. Every run lock
|
|
37
|
+
* The chain writes no node state and no run metadata of its own. Its own
|
|
38
|
+
* artifacts are `coordinator.lock`, the campaign `heartbeat.json`, and the
|
|
39
|
+
* campaign attention record. Every run lock belongs to a controller: the
|
|
40
|
+
* detached child the chain launched, or the controller `resumeRun` becomes
|
|
41
|
+
* while the chain waits on it. The one writer of a run's nodes stays the engine.
|
|
30
42
|
*/
|
|
31
43
|
import { execFileSync } from "node:child_process";
|
|
32
44
|
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
@@ -37,7 +49,7 @@ import { readCampaign } from "./record.mjs";
|
|
|
37
49
|
import { validateContract } from "../contract/index.mjs";
|
|
38
50
|
import { defaultControllerIdentity, storedContractDigest, verifyControllerIdentity } from "../engine/run-identity.mjs";
|
|
39
51
|
import { refuseSelfSignal } from "../engine/process-identity.mjs";
|
|
40
|
-
import { HEARTBEAT_INTERVAL_MS, createHeartbeat, groupAlive, heartbeatBreach, readHeartbeat, runProgress, waitForGroupGone } from "../engine/supervise.mjs";
|
|
52
|
+
import { HEARTBEAT_INTERVAL_MS, MAX_CONSECUTIVE_LAUNCH_FAILURES, controllerAlive, createHeartbeat, groupAlive, heartbeatBreach, readHeartbeat, runProgress, waitForGroupGone } from "../engine/supervise.mjs";
|
|
41
53
|
import { pidAlive, processStartToken } from "../run/lock.mjs";
|
|
42
54
|
import { delay, errorCode, errorMessage } from "../util.mjs";
|
|
43
55
|
import { writeJsonAtomic } from "../run/store.mjs";
|
|
@@ -64,6 +76,21 @@ export const DEFAULT_COORDINATOR_TERMINATE_GRACE_MS = 5_000;
|
|
|
64
76
|
/** How long the group gets after `SIGKILL` before the takeover stops waiting. */
|
|
65
77
|
export const DEFAULT_COORDINATOR_KILL_GRACE_MS = 5_000;
|
|
66
78
|
|
|
79
|
+
/**
|
|
80
|
+
* How many consecutive resumes of one run the chain starts before it records
|
|
81
|
+
* attention instead of trying again. The number is `supervise.mjs`'s
|
|
82
|
+
* `MAX_CONSECUTIVE_LAUNCH_FAILURES`, where the reason is recorded: a run that
|
|
83
|
+
* refuses to resume refuses forever, and a loop that keeps trying hides that
|
|
84
|
+
* from the operator.
|
|
85
|
+
*
|
|
86
|
+
* The budget is the coordinator invocation's own memory, and it resets the
|
|
87
|
+
* moment the run's own heartbeat shows progress, so a run that dies repeatedly
|
|
88
|
+
* over hours is still recovered each time. What has to survive a crash is the
|
|
89
|
+
* attention written when the budget is spent, and the campaign record carries
|
|
90
|
+
* that: the next invocation reads it and resumes nothing.
|
|
91
|
+
*/
|
|
92
|
+
export const MAX_RUN_RECOVERY_ATTEMPTS = MAX_CONSECUTIVE_LAUNCH_FAILURES;
|
|
93
|
+
|
|
67
94
|
/**
|
|
68
95
|
* @param {string} campaignPath
|
|
69
96
|
* @returns {string}
|
|
@@ -328,6 +355,68 @@ export function classifyRunProgress(progress) {
|
|
|
328
355
|
return "unfinished";
|
|
329
356
|
}
|
|
330
357
|
|
|
358
|
+
/**
|
|
359
|
+
* The chain's own count of how many resumes it has started for one run since
|
|
360
|
+
* that run last showed progress, and the reason the last one failed.
|
|
361
|
+
*
|
|
362
|
+
* @typedef {{attempts: number, progressAt: string|null, reason: string|null}} RecoveryAttempts
|
|
363
|
+
*/
|
|
364
|
+
|
|
365
|
+
/**
|
|
366
|
+
* The decision for one in-flight run whose controller is not alive. This is the
|
|
367
|
+
* one place the chain may cause work on a run it did not launch, and it is
|
|
368
|
+
* deliberately narrow: a requested pause is never overridden, the resume is the
|
|
369
|
+
* engine's own (which takes the run lock, so two controllers cannot both run
|
|
370
|
+
* the run), the run's own progress resets the budget, and a spent budget ends in
|
|
371
|
+
* attention rather than in a loop that hides a refusal.
|
|
372
|
+
*
|
|
373
|
+
* Returns the attention the campaign parks with, or null when the caller should
|
|
374
|
+
* keep polling.
|
|
375
|
+
*
|
|
376
|
+
* @param {{
|
|
377
|
+
* runId: string,
|
|
378
|
+
* runDir: string,
|
|
379
|
+
* contractPath: string,
|
|
380
|
+
* progress: RunProgress,
|
|
381
|
+
* attempts: Map<string, RecoveryAttempts>,
|
|
382
|
+
* resume: (runDir: string, context: {runId: string, attempt: number}) => Promise<void>|void,
|
|
383
|
+
* emit: (line: string) => void,
|
|
384
|
+
* }} context
|
|
385
|
+
* @returns {Promise<Omit<CampaignAttention, "at">|null>}
|
|
386
|
+
*/
|
|
387
|
+
async function resumeDeadRun(context) {
|
|
388
|
+
const { runId, runDir, contractPath, progress, attempts, resume, emit } = context;
|
|
389
|
+
const first = progress.outcomeNodes?.[0];
|
|
390
|
+
const naming = { contractPath, contractId: runId, runId, node: first?.id ?? null, status: first?.status ?? "unfinished" };
|
|
391
|
+
if (existsSync(join(runDir, "cancel.request.json"))) {
|
|
392
|
+
// Re-checked here and not only in the classification above: an operator can
|
|
393
|
+
// request the pause between those two reads, and a resume consumes a cancel
|
|
394
|
+
// request as an instruction to continue, which is exactly what must not
|
|
395
|
+
// happen to a pause the chain decided to honour.
|
|
396
|
+
return { ...naming, code: "run_canceled", message: `contract ${runId} run carries a requested pause (cancel.request.json) and no controller; \`faberun resume ${runDir}\` continues it when the operator says so`, resume: `resume ${runDir}` };
|
|
397
|
+
}
|
|
398
|
+
const progressAt = readHeartbeat(runDir)?.lastProgressAt ?? null;
|
|
399
|
+
const previous = attempts.get(runId);
|
|
400
|
+
// Same rule `supervise.mjs`'s `nextRelaunch` applies to the run-level
|
|
401
|
+
// supervisor: progress since the last resume is the evidence the recovery
|
|
402
|
+
// worked, so the budget starts again.
|
|
403
|
+
const spent = previous && previous.progressAt === progressAt ? previous.attempts : 0;
|
|
404
|
+
if (spent >= MAX_RUN_RECOVERY_ATTEMPTS) {
|
|
405
|
+
const reason = previous?.reason ? `; last failure: ${previous.reason}` : "";
|
|
406
|
+
return { ...naming, code: "run_recovery_exhausted", message: `contract ${runId} run is unfinished with no live controller and ${spent} resume attempts made no progress${reason}; \`faberun resume ${runDir}\` retries it from the operator side`, resume: `resume ${runDir}` };
|
|
407
|
+
}
|
|
408
|
+
attempts.set(runId, { attempts: spent + 1, progressAt, reason: null });
|
|
409
|
+
try {
|
|
410
|
+
await resume(runDir, { runId, attempt: spent + 1 });
|
|
411
|
+
emit(`[campaign] resumed run ${runId} · attempt ${spent + 1}/${MAX_RUN_RECOVERY_ATTEMPTS} · ${runDir}`);
|
|
412
|
+
} catch (error) {
|
|
413
|
+
const reason = errorMessage(error);
|
|
414
|
+
attempts.set(runId, { attempts: spent + 1, progressAt, reason });
|
|
415
|
+
emit(`[campaign] run ${runId} resume failed · attempt ${spent + 1}/${MAX_RUN_RECOVERY_ATTEMPTS} · ${reason}`);
|
|
416
|
+
}
|
|
417
|
+
return null;
|
|
418
|
+
}
|
|
419
|
+
|
|
331
420
|
/**
|
|
332
421
|
* @param {string} contractPath
|
|
333
422
|
* @returns {{id: string, cwd: string, runDir: string}}
|
|
@@ -388,6 +477,7 @@ function storedContractId(stored) {
|
|
|
388
477
|
* controllerIdentity?: ControllerIdentity,
|
|
389
478
|
* refreshController?: boolean,
|
|
390
479
|
* launch?: (contractPath: string, context: {baseRef: string|undefined, controllerIdentity: ControllerIdentity, runDir: string, contract: ValidatedContract}) => Promise<void>|void,
|
|
480
|
+
* resume?: (runDir: string, context: {runId: string, attempt: number}) => Promise<void>|void,
|
|
391
481
|
* validate?: (entry: CampaignContract, context: {repo: string|undefined, baseRef: string|undefined, contractPath: string}) => ValidatedContract,
|
|
392
482
|
* progress?: (runDir: string) => RunProgress,
|
|
393
483
|
* heartbeat?: {progress: (nodeId?: string, budgetBasis?: number) => void, setActive: (nodes: {nodeId: string, budgetBasis: number}[]) => void, stop: () => void},
|
|
@@ -412,6 +502,16 @@ export async function driveCampaignChain(campaignPath, options = {}) {
|
|
|
412
502
|
const progressOf = options.progress ?? ((runDir) => runProgress(runDir, now()));
|
|
413
503
|
const validate = options.validate ?? ((entry, context) => validateManifestEntryAtLaunch(entry, context));
|
|
414
504
|
const launch = options.launch ?? (() => { throw new Error("driveCampaignChain requires a launch seam"); });
|
|
505
|
+
// The resume seam is the engine's own resume by default: it takes the run's
|
|
506
|
+
// controller lock and verifies the run's recorded identity before dispatching
|
|
507
|
+
// anything, which is what keeps the chain from becoming a second controller.
|
|
508
|
+
// The engine is reached lazily because `engine/scheduler.mjs` imports this
|
|
509
|
+
// module's `validateContractForLaunch`: a static edge from here to
|
|
510
|
+
// `engine/resume.mjs` would close a runtime import cycle through the chain.
|
|
511
|
+
const resume = options.resume ?? (async (runDir) => {
|
|
512
|
+
const { resumeRun } = await import("../engine/resume.mjs");
|
|
513
|
+
await resumeRun(runDir, {});
|
|
514
|
+
});
|
|
415
515
|
|
|
416
516
|
/** @type {{role: "observed"|"took-over"|"became", lock: Record<string, unknown>}|null} */
|
|
417
517
|
let acquired = null;
|
|
@@ -437,6 +537,8 @@ export async function driveCampaignChain(campaignPath, options = {}) {
|
|
|
437
537
|
let validatedIndex = -1;
|
|
438
538
|
/** @type {ValidatedContract|null} */
|
|
439
539
|
let validatedContract = null;
|
|
540
|
+
/** @type {Map<string, RecoveryAttempts>} */
|
|
541
|
+
const recovery = new Map();
|
|
440
542
|
|
|
441
543
|
/** @param {Omit<CampaignAttention, "at">} attention */
|
|
442
544
|
const park = (attention) => {
|
|
@@ -542,9 +644,23 @@ export async function driveCampaignChain(campaignPath, options = {}) {
|
|
|
542
644
|
resume: `resume ${runDir}`,
|
|
543
645
|
});
|
|
544
646
|
}
|
|
545
|
-
// waiting
|
|
546
|
-
//
|
|
547
|
-
//
|
|
647
|
+
// A `waiting` run resumes itself at the instant its provider recorded:
|
|
648
|
+
// it needs no controller before then, and `lastProgressAt` is
|
|
649
|
+
// deliberately not refreshed, because a wait is not work. Anything else
|
|
650
|
+
// still in flight is either owned by a live controller -- which is
|
|
651
|
+
// awaited -- or orphaned, which is resumed here, bounded.
|
|
652
|
+
if (classification !== "waiting" && !controllerAlive(runDir, { now: now(), heartbeatIntervalMs })) {
|
|
653
|
+
const attention = await resumeDeadRun({
|
|
654
|
+
runId: id,
|
|
655
|
+
runDir,
|
|
656
|
+
contractPath: entry.path,
|
|
657
|
+
progress,
|
|
658
|
+
attempts: recovery,
|
|
659
|
+
resume,
|
|
660
|
+
emit,
|
|
661
|
+
});
|
|
662
|
+
if (attention) return park(attention);
|
|
663
|
+
}
|
|
548
664
|
await sleep(pollMs);
|
|
549
665
|
continue;
|
|
550
666
|
}
|
package/src/campaign/ledger.mjs
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
import { copyFileSync, cpSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
|
|
6
6
|
import { dirname, join, resolve } from "node:path";
|
|
7
7
|
|
|
8
|
-
import { readMetricNodeSnapshots } from "./metrics-command.mjs";
|
|
8
|
+
import { isPlanningRun, readMetricNodeSnapshots, stringEntriesOf } from "./metrics-command.mjs";
|
|
9
9
|
import { writeJsonAtomic } from "../run/store.mjs";
|
|
10
10
|
import { CAMPAIGN_FILE, JOURNAL_FILE, LEDGER_SOURCE_MANIFEST_FILE, campaignsDir } from "./layout.mjs";
|
|
11
11
|
import { readCampaign } from "./record.mjs";
|
|
@@ -79,6 +79,13 @@ export function unpreservedLedgerCampaigns(runsDir) {
|
|
|
79
79
|
* sources are reported so a pruned run cannot make the close fail or look
|
|
80
80
|
* complete by accident. Repeated calls overwrite the same paths.
|
|
81
81
|
*
|
|
82
|
+
* The manifest also records which linked runs were planning runs (R5): the
|
|
83
|
+
* ledger preserves no contracts, so a run that was launched to plan has no
|
|
84
|
+
* other evidence left of what it was, and the projection reads that list back
|
|
85
|
+
* to separate planning spend from worker and judge spend. A run recorded as
|
|
86
|
+
* planning stays recorded, because reledgering a pruned run must not silently
|
|
87
|
+
* reclassify the usage it already preserved.
|
|
88
|
+
*
|
|
82
89
|
* `ledgerDir` is the exact destination. It defaults to the versioned ledger
|
|
83
90
|
* under `repoRoot/docs/campaigns/<id>/ledger`; `closeCampaign` passes the
|
|
84
91
|
* campaign directory in the operator home instead when `--ledger-in-repo` was
|
|
@@ -104,6 +111,7 @@ export function preserveCampaignLedger(campaignPath, repoRoot, ledgerDir = undef
|
|
|
104
111
|
const priorManifest = readSourceManifest(ledgerDir);
|
|
105
112
|
const absent = new Set(priorManifest.absent);
|
|
106
113
|
const preserved = new Set(priorManifest.preserved);
|
|
114
|
+
const planning = new Set(priorManifest.planning);
|
|
107
115
|
for (const name of [JOURNAL_FILE, CAMPAIGN_FILE]) {
|
|
108
116
|
const source = join(campaignPath, name);
|
|
109
117
|
if (!existsSync(source)) continue;
|
|
@@ -113,6 +121,7 @@ export function preserveCampaignLedger(campaignPath, repoRoot, ledgerDir = undef
|
|
|
113
121
|
}
|
|
114
122
|
for (const runId of campaign.linkedRunIds) {
|
|
115
123
|
const runDir = join(runsDir, runId);
|
|
124
|
+
if (isPlanningRun(runDir)) planning.add(runId);
|
|
116
125
|
for (const [sourceName, destinationSuffix] of RUN_SOURCES) {
|
|
117
126
|
const source = join(runDir, sourceName);
|
|
118
127
|
const destinationName = `${runId}.${destinationSuffix}`;
|
|
@@ -150,7 +159,7 @@ export function preserveCampaignLedger(campaignPath, repoRoot, ledgerDir = undef
|
|
|
150
159
|
written.push(...filesUnder(proposalsSource).map((path) => join(proposalsDestination, path)));
|
|
151
160
|
}
|
|
152
161
|
const manifest = join(ledgerDir, LEDGER_SOURCE_MANIFEST_FILE);
|
|
153
|
-
writeFileSync(manifest, `${JSON.stringify({ schemaVersion: 1, absent: [...absent].sort(), preserved: [...preserved].sort() })}\n`);
|
|
162
|
+
writeFileSync(manifest, `${JSON.stringify({ schemaVersion: 1, absent: [...absent].sort(), preserved: [...preserved].sort(), planning: [...planning].sort() })}\n`);
|
|
154
163
|
written.push(manifest);
|
|
155
164
|
// The ledger exists, so a versioned preservation now marks the record. The
|
|
156
165
|
// already-copied campaign.json is refreshed afterwards so the durable copy
|
|
@@ -186,8 +195,10 @@ export function reledgerCampaignLedger(campaignPath, repoRoot) {
|
|
|
186
195
|
const sourceManifest = readSourceManifest(ledgerDir);
|
|
187
196
|
const preserved = new Set(sourceManifest.preserved);
|
|
188
197
|
const absent = new Set(sourceManifest.absent);
|
|
198
|
+
const planning = new Set(sourceManifest.planning);
|
|
189
199
|
for (const runId of campaign.linkedRunIds) {
|
|
190
200
|
const runDir = join(runsDir, runId);
|
|
201
|
+
if (isPlanningRun(runDir)) planning.add(runId);
|
|
191
202
|
for (const [sourceName, destinationSuffix] of RUN_SOURCES) {
|
|
192
203
|
const source = join(runDir, sourceName);
|
|
193
204
|
const change = { runId, source: sourceName };
|
|
@@ -241,7 +252,7 @@ export function reledgerCampaignLedger(campaignPath, repoRoot) {
|
|
|
241
252
|
} else if (existsSync(join(campaignPath, CAMPAIGN_FILE))) {
|
|
242
253
|
copyFileSync(join(campaignPath, CAMPAIGN_FILE), ledgerCampaign);
|
|
243
254
|
}
|
|
244
|
-
writeFileSync(join(ledgerDir, LEDGER_SOURCE_MANIFEST_FILE), `${JSON.stringify({ schemaVersion: 1, absent: [...absent].sort(), preserved: [...preserved].sort() })}\n`);
|
|
255
|
+
writeFileSync(join(ledgerDir, LEDGER_SOURCE_MANIFEST_FILE), `${JSON.stringify({ schemaVersion: 1, absent: [...absent].sort(), preserved: [...preserved].sort(), planning: [...planning].sort() })}\n`);
|
|
245
256
|
return { ledgerDir, copied, gone };
|
|
246
257
|
}
|
|
247
258
|
|
|
@@ -299,17 +310,18 @@ function projectedField(value, field) {
|
|
|
299
310
|
return Object.hasOwn(object, "kind") ? { kind: object.kind } : {};
|
|
300
311
|
}
|
|
301
312
|
|
|
302
|
-
/** @param {string} root @returns {{absent: string[], preserved: string[]}} */
|
|
313
|
+
/** @param {string} root @returns {{absent: string[], preserved: string[], planning: string[]}} */
|
|
303
314
|
function readSourceManifest(root) {
|
|
304
315
|
try {
|
|
305
316
|
const value = JSON.parse(readFileSync(join(root, LEDGER_SOURCE_MANIFEST_FILE), "utf8"));
|
|
306
317
|
return {
|
|
307
|
-
absent:
|
|
308
|
-
preserved:
|
|
318
|
+
absent: stringEntriesOf(value?.absent),
|
|
319
|
+
preserved: stringEntriesOf(value?.preserved),
|
|
320
|
+
planning: stringEntriesOf(value?.planning),
|
|
309
321
|
};
|
|
310
322
|
} catch {
|
|
311
323
|
// An old ledger has no origin manifest; its files remain subject to the legacy missing-source rules.
|
|
312
|
-
return { absent: [], preserved: [] };
|
|
324
|
+
return { absent: [], preserved: [], planning: [] };
|
|
313
325
|
}
|
|
314
326
|
}
|
|
315
327
|
|
|
@@ -2,6 +2,11 @@
|
|
|
2
2
|
* Filesystem and command layer for `faberun metrics`. This module owns the
|
|
3
3
|
* command's flags and reads the campaign's run or ledger artefacts, while
|
|
4
4
|
* `metrics.mjs` stays a pure function over parsed records.
|
|
5
|
+
*
|
|
6
|
+
* It also answers which of those runs were planning runs, because only a
|
|
7
|
+
* filesystem read can: a run directory still carries the contract that proves
|
|
8
|
+
* it, and a versioned ledger carries the classification `preserveCampaignLedger`
|
|
9
|
+
* recorded, since it preserves no contracts.
|
|
5
10
|
*/
|
|
6
11
|
|
|
7
12
|
import { existsSync, readFileSync } from "node:fs";
|
|
@@ -50,6 +55,7 @@ const NOTIFY_LOG_FILE = "notify.jsonl";
|
|
|
50
55
|
* journal: unknown[],
|
|
51
56
|
* campaign: JsonObject,
|
|
52
57
|
* excludedRunIds: string[],
|
|
58
|
+
* planningRunIds?: string[],
|
|
53
59
|
* missingSources: string[],
|
|
54
60
|
* }} MetricsSources
|
|
55
61
|
*/
|
|
@@ -78,19 +84,25 @@ export function readMetricsSources(campaignPath, { runsDir = join(campaignPath,
|
|
|
78
84
|
/** @type {unknown[]} */
|
|
79
85
|
const journal = [];
|
|
80
86
|
/** @type {string[]} */
|
|
87
|
+
const planningRunIds = [];
|
|
88
|
+
/** @type {string[]} */
|
|
81
89
|
const missingSources = [];
|
|
82
|
-
const
|
|
90
|
+
const ledgerManifest = ledgerDir === undefined
|
|
91
|
+
? { absent: /** @type {Set<string>} */ (new Set()), planning: /** @type {Set<string>} */ (new Set()) }
|
|
92
|
+
: readLedgerManifest(ledgerDir);
|
|
83
93
|
const journalPath = join(ledgerDir ?? campaignPath, "journal.jsonl");
|
|
84
94
|
if (existsSync(journalPath)) journal.push(...readJsonlRecords(journalPath));
|
|
85
95
|
else missingSources.push("journal.jsonl");
|
|
86
96
|
for (const runId of campaign.linkedRunIds) {
|
|
87
97
|
if (ledgerDir === undefined) {
|
|
98
|
+
if (isPlanningRun(join(runsDir, runId))) planningRunIds.push(runId);
|
|
88
99
|
for (const record of readJsonlRecords(join(runsDir, runId, RUN_EVENTS_FILE))) events.push({ ...jsonObjectOf(record), runId });
|
|
89
100
|
for (const record of readJsonlRecords(join(runsDir, runId, USAGE_LOG_FILE))) usageRecords.push({ ...jsonObjectOf(record), runId });
|
|
90
101
|
for (const record of readJsonlRecords(join(runsDir, runId, NOTIFY_LOG_FILE))) notifications.push({ ...jsonObjectOf(record), runId });
|
|
91
102
|
for (const node of readMetricNodeSnapshots(join(runsDir, runId))) nodes.push({ ...node, runId });
|
|
92
103
|
continue;
|
|
93
104
|
}
|
|
105
|
+
if (ledgerManifest.planning.has(runId)) planningRunIds.push(runId);
|
|
94
106
|
/** @type {[string, unknown[]][]} */
|
|
95
107
|
const sources = [
|
|
96
108
|
[RUN_EVENTS_FILE, events],
|
|
@@ -100,14 +112,14 @@ export function readMetricsSources(campaignPath, { runsDir = join(campaignPath,
|
|
|
100
112
|
for (const [suffix, target] of sources) {
|
|
101
113
|
const path = join(ledgerDir, `${runId}.${suffix}`);
|
|
102
114
|
if (!existsSync(path)) {
|
|
103
|
-
if (!
|
|
115
|
+
if (!ledgerManifest.absent.has(`${runId}.${suffix}`)) missingSources.push(`${runId}.${suffix}`);
|
|
104
116
|
continue;
|
|
105
117
|
}
|
|
106
118
|
for (const record of readJsonlRecords(path)) target.push({ ...jsonObjectOf(record), runId });
|
|
107
119
|
}
|
|
108
120
|
const nodesPath = join(ledgerDir, `${runId}.nodes.json`);
|
|
109
121
|
if (!existsSync(nodesPath)) {
|
|
110
|
-
if (!
|
|
122
|
+
if (!ledgerManifest.absent.has(`${runId}.nodes.json`)) missingSources.push(`${runId}.nodes.json`);
|
|
111
123
|
continue;
|
|
112
124
|
}
|
|
113
125
|
for (const node of readLedgerNodeSnapshots(nodesPath)) nodes.push({ ...node, runId });
|
|
@@ -123,22 +135,69 @@ export function readMetricsSources(campaignPath, { runsDir = join(campaignPath,
|
|
|
123
135
|
journal,
|
|
124
136
|
campaign: campaignObject,
|
|
125
137
|
excludedRunIds: excludedRunIdsFor(campaignObject, []),
|
|
138
|
+
planningRunIds,
|
|
126
139
|
missingSources,
|
|
127
140
|
};
|
|
128
141
|
}
|
|
129
142
|
|
|
130
|
-
/**
|
|
131
|
-
|
|
143
|
+
/**
|
|
144
|
+
* Whether a run is a planning run: every node of its contract is a discovery
|
|
145
|
+
* packet, the rule the expense pool already uses to keep planning out of an
|
|
146
|
+
* execution estimate (`collectCompletedExecutionNodes`). A contract that is
|
|
147
|
+
* gone, torn, or carries no nodes proves nothing about how the run was
|
|
148
|
+
* launched, so the run is not classified as planning on that evidence alone.
|
|
149
|
+
*
|
|
150
|
+
* @param {string} runDir
|
|
151
|
+
* @returns {boolean}
|
|
152
|
+
*/
|
|
153
|
+
export function isPlanningRun(runDir) {
|
|
154
|
+
let contract;
|
|
155
|
+
try {
|
|
156
|
+
contract = JSON.parse(readFileSync(join(runDir, "contract.json"), "utf8"));
|
|
157
|
+
} catch {
|
|
158
|
+
return false;
|
|
159
|
+
}
|
|
160
|
+
const contractNodes = /** @type {unknown[]} */ (Array.isArray(contract?.nodes) ? contract.nodes : []);
|
|
161
|
+
return contractNodes.length > 0 && contractNodes.every((node) => taskPacketModeOf(node) === "discovery");
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** @param {unknown} node @returns {unknown} */
|
|
165
|
+
function taskPacketModeOf(node) {
|
|
166
|
+
return jsonObjectOf(jsonObjectOf(node)?.taskPacket)?.mode;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* @param {string} ledgerDir
|
|
171
|
+
* @returns {{absent: Set<string>, planning: Set<string>}}
|
|
172
|
+
*/
|
|
173
|
+
function readLedgerManifest(ledgerDir) {
|
|
132
174
|
try {
|
|
133
175
|
const manifest = JSON.parse(readFileSync(join(ledgerDir, LEDGER_SOURCE_MANIFEST_FILE), "utf8"));
|
|
134
|
-
const preserved =
|
|
135
|
-
return
|
|
176
|
+
const preserved = new Set(stringEntriesOf(manifest?.preserved));
|
|
177
|
+
return {
|
|
178
|
+
absent: new Set(stringEntriesOf(manifest?.absent).filter((source) => !preserved.has(source))),
|
|
179
|
+
planning: new Set(stringEntriesOf(manifest?.planning)),
|
|
180
|
+
};
|
|
136
181
|
} catch {
|
|
137
|
-
// An old ledger has no origin manifest, so an absent file remains
|
|
138
|
-
|
|
182
|
+
// An old ledger has no origin manifest, so an absent file remains
|
|
183
|
+
// incomplete evidence and no run can be proven a planning run.
|
|
184
|
+
return { absent: new Set(), planning: new Set() };
|
|
139
185
|
}
|
|
140
186
|
}
|
|
141
187
|
|
|
188
|
+
/**
|
|
189
|
+
* The string entries of one `sources.json` list field, and an empty list for
|
|
190
|
+
* anything else. The ledger's writer merges and sorts these lists and the
|
|
191
|
+
* reader above subtracts preserved names from absent ones, so both need the
|
|
192
|
+
* same tolerant read of the file's shape; it has one home here.
|
|
193
|
+
*
|
|
194
|
+
* @param {unknown} value
|
|
195
|
+
* @returns {string[]}
|
|
196
|
+
*/
|
|
197
|
+
export function stringEntriesOf(value) {
|
|
198
|
+
return Array.isArray(value) ? value.filter((entry) => typeof entry === "string") : [];
|
|
199
|
+
}
|
|
200
|
+
|
|
142
201
|
/** @param {string} path @returns {Omit<RunNode, "runId">[]} */
|
|
143
202
|
function readLedgerNodeSnapshots(path) {
|
|
144
203
|
try {
|