faberun 0.24.0 → 0.25.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/skills/faberun/references/contract.md +4 -3
- package/src/campaign/index.mjs +12 -2
- package/src/cli/campaign.mjs +1 -0
- package/src/contract/task-packet.mjs +4 -1
- package/src/contract/worker-result.mjs +35 -4
- package/src/engine/lifecycle.mjs +16 -8
- package/src/engine/live-preflight.mjs +4 -1
- package/src/engine/resume.mjs +3 -3
- package/src/engine/run-identity.mjs +43 -5
- package/src/engine/runtime-discovery.mjs +4 -1
- package/src/harnesses/codex/usage-window.mjs +76 -0
- package/src/harnesses/zcode/index.mjs +17 -3
- package/src/host/preflight.mjs +33 -2
- package/src/plan/pipeline.mjs +2 -1
- package/src/plan/preflight.mjs +42 -1
- package/src/repo/worktree.mjs +49 -2
- package/src/run/availability.mjs +73 -0
- package/src/run/paths.mjs +6 -1
- package/src/run/usage-windows.mjs +82 -0
- package/src/run/usage.mjs +7 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "faberun",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.25.1",
|
|
4
4
|
"description": "Faberun is a development orchestration system that turns intent into verified software: harness- and model-agnostic, it keeps the intent, coordinates the work, verifies the result and decides what happens next.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -25,7 +25,6 @@ typecheck`). Schema version is `3`.
|
|
|
25
25
|
"sol": { "harness": "codex", "model": "gpt-5.6-sol", "reasoning": "xhigh", "vendor": "openai-sol" },
|
|
26
26
|
"opus": { "harness": "claude", "model": "opus", "permissionMode": "acceptEdits" },
|
|
27
27
|
"zcode-flash": { "harness": "zcode", "model": "glm-5.3-flash", "vendor": "zhipu-flash", "permissionMode": "edit" },
|
|
28
|
-
"zcode-pro": { "harness": "zcode", "model": "glm-5.3", "vendor": "zhipu-pro", "permissionMode": "plan" },
|
|
29
28
|
"agy-flash": { "harness": "agy", "model": "gemini-3.8-flash-low" }
|
|
30
29
|
},
|
|
31
30
|
"nodes": [
|
|
@@ -141,8 +140,10 @@ missing something, never repository-wide exploration. Bounded: 32 KiB total,
|
|
|
141
140
|
4 KiB summary, 32 entries each in `changedFiles`/`verification`/`artifacts`
|
|
142
141
|
(16 in `missingContext`), 2 KiB per entry (16 KiB per artifact). Unknown
|
|
143
142
|
provider-added fields are dropped; missing/malformed canonical fields are
|
|
144
|
-
rejected (`worker-result.mjs`). A discovery node
|
|
145
|
-
one `artifacts` entry
|
|
143
|
+
rejected (`worker-result.mjs`). A discovery node without `readFiles` returns
|
|
144
|
+
`done` with one `artifacts` entry, the next node's execution packet; one closed
|
|
145
|
+
to its `readFiles` (every planning stage) delivers through `output`, `artifacts`
|
|
146
|
+
`[]`.
|
|
146
147
|
|
|
147
148
|
## Runtimes and routing
|
|
148
149
|
|
package/src/campaign/index.mjs
CHANGED
|
@@ -10,6 +10,7 @@ import { writeJsonAtomic } from "../run/store.mjs";
|
|
|
10
10
|
import { projectIdForRunsDir, repositoryForRunsDir } from "../run/paths.mjs";
|
|
11
11
|
import { requireId, requirePacketHash, requireString, requireTimestamp } from "../contract/assert.mjs";
|
|
12
12
|
import { promoteRun } from "../repo/integrate.mjs";
|
|
13
|
+
import { releaseRunWorktrees } from "../repo/worktree.mjs";
|
|
13
14
|
import { CAMPAIGN_FILE, GOAL_TEXT_BYTES, PROJECTION_FILE, campaignDir, campaignsDir } from "./layout.mjs";
|
|
14
15
|
import { readCampaign } from "./record.mjs";
|
|
15
16
|
import { appendJournal, normalizeText, readJournalForDedupe } from "./journal.mjs";
|
|
@@ -130,7 +131,7 @@ export function resolveCampaign(runsDir, campaignId) {
|
|
|
130
131
|
/**
|
|
131
132
|
* @param {string} campaignPath
|
|
132
133
|
* @param {{at?: string, eventId?: string}} options
|
|
133
|
-
* @returns {{path: string, campaign: Campaign, ledgerFiles: string[], ledgerSkipped: {runId: string, source: string}[]}}
|
|
134
|
+
* @returns {{path: string, campaign: Campaign, ledgerFiles: string[], ledgerSkipped: {runId: string, source: string}[], worktrees: {removed: number, archived: string[]}}}
|
|
134
135
|
*/
|
|
135
136
|
export function closeCampaign(campaignPath, { at = new Date().toISOString(), eventId = randomUUID() } = {}) {
|
|
136
137
|
requireTimestamp(at, "at");
|
|
@@ -154,7 +155,16 @@ export function closeCampaign(campaignPath, { at = new Date().toISOString(), eve
|
|
|
154
155
|
// Preserve after the closed record exists: the ledger is the recomputable
|
|
155
156
|
// closed record, including requirements used by the north-star projector.
|
|
156
157
|
const ledger = preserveCampaignLedger(campaignPath, campaignRepoRoot(campaignPath));
|
|
157
|
-
|
|
158
|
+
// A closed campaign resumes nothing, so the attempt worktrees its runs left
|
|
159
|
+
// go now, each archived under a ref first (releaseRunWorktrees).
|
|
160
|
+
const runsDir = resolve(campaignPath, "..", "..");
|
|
161
|
+
const worktrees = { removed: 0, archived: /** @type {string[]} */ ([]) };
|
|
162
|
+
for (const runId of campaign.linkedRunIds) {
|
|
163
|
+
const released = releaseRunWorktrees(campaignRepoRoot(campaignPath), join(runsDir, runId), runId);
|
|
164
|
+
worktrees.removed += released.removed;
|
|
165
|
+
worktrees.archived.push(...released.archived);
|
|
166
|
+
}
|
|
167
|
+
return { path: campaignPath, campaign: closed, ledgerFiles: ledger.written, ledgerSkipped: ledger.skipped, worktrees };
|
|
158
168
|
}
|
|
159
169
|
|
|
160
170
|
/**
|
package/src/cli/campaign.mjs
CHANGED
|
@@ -320,6 +320,7 @@ function close(campaignId, values) {
|
|
|
320
320
|
process.stdout.write(`[campaign] ledger skipped · ${skipped.runId}/${skipped.source}\n`);
|
|
321
321
|
}
|
|
322
322
|
reportRequirementClosure(closed.campaign.requirements ?? []);
|
|
323
|
+
if (closed.worktrees.removed > 0) process.stdout.write(`[campaign] worktrees released · ${closed.worktrees.removed} · archived under refs/faberun-archive/\n`);
|
|
323
324
|
if (syncAgentSignal(runsDir)) process.stdout.write(`[campaign] AGENTS.md signal updated\n`);
|
|
324
325
|
}
|
|
325
326
|
|
|
@@ -426,8 +426,11 @@ function pathInside(path, root) {
|
|
|
426
426
|
* @returns {string}
|
|
427
427
|
*/
|
|
428
428
|
function renderDiscoveryPrompt(packet, nodeId) {
|
|
429
|
+
// The closed variant says what settlement accepts (engine/lifecycle.mjs):
|
|
430
|
+
// only a discovery packet with no read files owes an execution packet in
|
|
431
|
+
// artifacts[0]; one closed to its read files delivers through `output`.
|
|
429
432
|
const closedContext = packet.readFiles.length
|
|
430
|
-
? "This discovery context is read-only and closed to the listed read files. Inspect only those files and do not perform repository-wide exploration. Return the worker-result JSON object below;
|
|
433
|
+
? "This discovery context is read-only and closed to the listed read files. Inspect only those files and do not perform repository-wide exploration. Return the worker-result JSON object below; deliver your result in `output` and send `artifacts` as []."
|
|
431
434
|
: "This discovery context is read-only and is the one exception to closed inspection: no read files were pre-supplied, so you may inspect the repository read-only only as needed to produce the packet. Do not edit the repository. Return the worker-result JSON object below; put exactly one execution task packet JSON string in artifacts[0].";
|
|
432
435
|
const lines = [
|
|
433
436
|
`# Node ${nodeId} (discovery)`,
|
|
@@ -25,6 +25,23 @@ const RESULT_LIMITS = Object.freeze({
|
|
|
25
25
|
*/
|
|
26
26
|
export const DERIVED_WORKER_RESULT_FIELDS = Object.freeze(["changedFiles"]);
|
|
27
27
|
|
|
28
|
+
/**
|
|
29
|
+
* A worker result that is well-formed JSON but breaks one of the byte
|
|
30
|
+
* ceilings above. Typed, not a plain TypeError, so the repair prompt can name
|
|
31
|
+
* the ceiling that was broken: observed 2026-09-25, a planning draft that
|
|
32
|
+
* copied its 38 KiB plan into `artifacts[0]` was told twice to drop markdown
|
|
33
|
+
* fences it never wrote, and repeated the copy.
|
|
34
|
+
*/
|
|
35
|
+
export class WorkerResultSizeError extends TypeError {
|
|
36
|
+
/** @param {string} field @param {number} limit */
|
|
37
|
+
constructor(field, limit) {
|
|
38
|
+
super(`${field} exceeds ${limit} bytes`);
|
|
39
|
+
this.name = "WorkerResultSizeError";
|
|
40
|
+
this.field = field;
|
|
41
|
+
this.limit = limit;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
28
45
|
/** @typedef {"done"|"blocked_context"} WorkerResultStatus */
|
|
29
46
|
|
|
30
47
|
/**
|
|
@@ -37,6 +54,20 @@ export const DERIVED_WORKER_RESULT_FIELDS = Object.freeze(["changedFiles"]);
|
|
|
37
54
|
* @typedef {{status: WorkerResultStatus, summary: string, verification: string[], artifacts: string[], missingContext: string[], output?: Record<string, unknown>}} WorkerResult
|
|
38
55
|
*/
|
|
39
56
|
|
|
57
|
+
/**
|
|
58
|
+
* The repair instruction for an invalid worker result: a broken byte ceiling
|
|
59
|
+
* is named, since re-emitting the same object without fences cannot fix it.
|
|
60
|
+
*
|
|
61
|
+
* @param {unknown} cause
|
|
62
|
+
* @returns {string}
|
|
63
|
+
*/
|
|
64
|
+
export function invalidResultRepair(cause) {
|
|
65
|
+
if (cause instanceof WorkerResultSizeError) {
|
|
66
|
+
return `the result breaks a size ceiling: ${cause.field} exceeds ${cause.limit} bytes. Keep every field within its ceiling; a node that delivers through \`output\` sends \`artifacts\` as [] and never copies \`output\` into \`artifacts\`.`;
|
|
67
|
+
}
|
|
68
|
+
return "the entire final message must be exactly the required JSON object: no markdown fences, no prose before or after it. Return it as the only content of the final message.";
|
|
69
|
+
}
|
|
70
|
+
|
|
40
71
|
/**
|
|
41
72
|
* @param {string} value
|
|
42
73
|
* @returns {WorkerResult}
|
|
@@ -45,7 +76,7 @@ export function parseWorkerResult(value) {
|
|
|
45
76
|
if (typeof value !== "string") throw new TypeError("worker result must be JSON text");
|
|
46
77
|
const maxRawBytes = RESULT_LIMITS.bytes + RESULT_LIMITS.outputBytes;
|
|
47
78
|
if (Buffer.byteLength(value, "utf8") > maxRawBytes) {
|
|
48
|
-
throw new
|
|
79
|
+
throw new WorkerResultSizeError("worker result", maxRawBytes);
|
|
49
80
|
}
|
|
50
81
|
let parsed;
|
|
51
82
|
try {
|
|
@@ -100,7 +131,7 @@ export function validateWorkerResult(value) {
|
|
|
100
131
|
if (Object.hasOwn(record, "output") && record.output !== undefined) {
|
|
101
132
|
assertObject(record.output, "worker result.output");
|
|
102
133
|
if (Buffer.byteLength(JSON.stringify(record.output), "utf8") > RESULT_LIMITS.outputBytes) {
|
|
103
|
-
throw new
|
|
134
|
+
throw new WorkerResultSizeError("worker result.output", RESULT_LIMITS.outputBytes);
|
|
104
135
|
}
|
|
105
136
|
output = /** @type {Record<string, unknown>} */ (record.output);
|
|
106
137
|
}
|
|
@@ -115,7 +146,7 @@ export function validateWorkerResult(value) {
|
|
|
115
146
|
// it is a discovery node's deliverable, not incidental prose competing with
|
|
116
147
|
// summary/verification for the same 32 KiB budget.
|
|
117
148
|
if (Buffer.byteLength(JSON.stringify(envelope), "utf8") > RESULT_LIMITS.bytes) {
|
|
118
|
-
throw new
|
|
149
|
+
throw new WorkerResultSizeError("worker result", RESULT_LIMITS.bytes);
|
|
119
150
|
}
|
|
120
151
|
return output === undefined ? envelope : { ...envelope, output };
|
|
121
152
|
}
|
|
@@ -165,6 +196,6 @@ function requireList(value, label, maxItems, itemBytes) {
|
|
|
165
196
|
if (!Array.isArray(value) || value.length > maxItems) throw new TypeError(`${label} must be an array with at most ${maxItems} items`);
|
|
166
197
|
for (const [index, item] of value.entries()) {
|
|
167
198
|
if (typeof item !== "string") throw new TypeError(`${label}[${index}] must be a string`);
|
|
168
|
-
if (Buffer.byteLength(item, "utf8") > itemBytes) throw new
|
|
199
|
+
if (Buffer.byteLength(item, "utf8") > itemBytes) throw new WorkerResultSizeError(`${label}[${index}]`, itemBytes);
|
|
169
200
|
}
|
|
170
201
|
}
|
package/src/engine/lifecycle.mjs
CHANGED
|
@@ -33,9 +33,11 @@ import {
|
|
|
33
33
|
upsertTierExhaustionCandidate,
|
|
34
34
|
} from "./backoff.mjs";
|
|
35
35
|
import { exhaustedUntilOf } from "./runtime-discovery.mjs";
|
|
36
|
+
import { getHarness, normalizeProviderAvailability } from "../harnesses/index.mjs";
|
|
37
|
+
import { availabilityKey, recordRefusal, refusalOfAvailability } from "../run/availability.mjs";
|
|
36
38
|
|
|
37
39
|
import { acquire as acquireLock } from "../run/lock.mjs";
|
|
38
|
-
import { parseDiscoveryResult } from "../contract/worker-result.mjs";
|
|
40
|
+
import { invalidResultRepair, parseDiscoveryResult } from "../contract/worker-result.mjs";
|
|
39
41
|
import { boundedUtf8, errorCode, errorMessage, excerpt } from "../util.mjs";
|
|
40
42
|
import { readJson, writeJsonAtomic } from "../run/store.mjs";
|
|
41
43
|
import { invocationAlive } from "./process.mjs";
|
|
@@ -441,7 +443,7 @@ export async function finalizeClosedJobs(contract, runDir, states, closed, lock,
|
|
|
441
443
|
// path fails it today.
|
|
442
444
|
if (workerResultError) {
|
|
443
445
|
if (envelope.status === "done") {
|
|
444
|
-
await applyInvalidWorkerResult(contract, job.node, state, runDir, running, lock,
|
|
446
|
+
await applyInvalidWorkerResult(contract, job.node, state, runDir, running, lock, workerResultError, states, campaignPath);
|
|
445
447
|
continue;
|
|
446
448
|
}
|
|
447
449
|
adoptedWorkerResult = null;
|
|
@@ -547,7 +549,7 @@ export async function finalizeClosedJobs(contract, runDir, states, closed, lock,
|
|
|
547
549
|
}, lock);
|
|
548
550
|
continue;
|
|
549
551
|
}
|
|
550
|
-
await applyInvalidWorkerResult(contract, job.node, state, runDir, running, lock,
|
|
552
|
+
await applyInvalidWorkerResult(contract, job.node, state, runDir, running, lock, error, states, campaignPath);
|
|
551
553
|
continue;
|
|
552
554
|
}
|
|
553
555
|
// The artifact demand follows the documented discovery contract, not the
|
|
@@ -560,7 +562,7 @@ export async function finalizeClosedJobs(contract, runDir, states, closed, lock,
|
|
|
560
562
|
try {
|
|
561
563
|
parseDiscoveryResult(workerResult, attemptWorkspace(state) ?? contract.cwd);
|
|
562
564
|
} catch (error) {
|
|
563
|
-
await applyInvalidWorkerResult(contract, job.node, state, runDir, running, lock,
|
|
565
|
+
await applyInvalidWorkerResult(contract, job.node, state, runDir, running, lock, error, states, campaignPath);
|
|
564
566
|
continue;
|
|
565
567
|
}
|
|
566
568
|
}
|
|
@@ -635,6 +637,10 @@ export function handleProviderExhaustion(contract, runDir, node, state, role, en
|
|
|
635
637
|
now,
|
|
636
638
|
});
|
|
637
639
|
const exhaustedUntil = exhaustedUntilOf(envelope);
|
|
640
|
+
// Every other process on this machine learns the refusal now, not from its own failed call.
|
|
641
|
+
const runtime = contract.runtimes?.[current];
|
|
642
|
+
const refusal = runtime ? refusalOfAvailability(normalizeProviderAvailability(runtime, envelope)) : null;
|
|
643
|
+
if (runtime && refusal) recordRefusal(availabilityKey({ harness: runtime.harness, model: runtime.model, executable: getHarness(runtime.harness).executable(runtime) }), refusal);
|
|
638
644
|
const plan = planRoute(contract, node, state, role, error, current, schedule, now, exhaustedUntil);
|
|
639
645
|
const status = envelope.status === "failed" ? "failed" : "exhausted";
|
|
640
646
|
if (plan.blocked) {
|
|
@@ -731,19 +737,21 @@ function applyRoute(contract, runDir, state, lock, { role, error, current, plan,
|
|
|
731
737
|
* told to return. A second one is evidence about the provider rather than the
|
|
732
738
|
* packet, so the node records a protocol_failure and takes its failover edge —
|
|
733
739
|
* and when no edge remains, it blocks and raises attention rather than filing
|
|
734
|
-
* a quiet exhaustion nobody reads.
|
|
740
|
+
* a quiet exhaustion nobody reads. The repair names what was wrong
|
|
741
|
+
* (invalidResultRepair), so a broken byte ceiling is not told about fences.
|
|
735
742
|
*
|
|
736
|
-
* @param {ValidatedContract} contract @param {ValidatedNode} node @param {NodeSnapshot} state @param {string} runDir @param {Map<string, Job>|null} running @param {LockHandle} lock @param {
|
|
743
|
+
* @param {ValidatedContract} contract @param {ValidatedNode} node @param {NodeSnapshot} state @param {string} runDir @param {Map<string, Job>|null} running @param {LockHandle} lock @param {unknown} cause @param {Map<string, NodeSnapshot>} states @param {string} campaignPath
|
|
737
744
|
*/
|
|
738
|
-
export async function applyInvalidWorkerResult(contract, node, state, runDir, running, lock,
|
|
745
|
+
export async function applyInvalidWorkerResult(contract, node, state, runDir, running, lock, cause, states, campaignPath) {
|
|
739
746
|
clearTierExhaustion(state);
|
|
747
|
+
const message = errorMessage(cause);
|
|
740
748
|
const verdict = /** @type {JudgeVerdict} */ ({
|
|
741
749
|
verdict: "fail",
|
|
742
750
|
maxSeverity: "critical",
|
|
743
751
|
summary: "worker result did not match the structured result protocol",
|
|
744
752
|
findings: [{
|
|
745
753
|
severity: "critical",
|
|
746
|
-
description:
|
|
754
|
+
description: invalidResultRepair(cause),
|
|
747
755
|
evidence: boundedUtf8(message, 4 * 1024),
|
|
748
756
|
}],
|
|
749
757
|
});
|
|
@@ -16,7 +16,7 @@ import { emptyUsage } from "../run/usage.mjs";
|
|
|
16
16
|
import { errorMessage } from "../util.mjs";
|
|
17
17
|
import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs";
|
|
18
18
|
import { join, resolve } from "node:path";
|
|
19
|
-
import { normalizeProviderResult, probeRuntime, providerCommand } from "../harnesses/index.mjs";
|
|
19
|
+
import { normalizeProviderAvailability, normalizeProviderResult, probeRuntime, providerCommand } from "../harnesses/index.mjs";
|
|
20
20
|
import { killTarget, spawnInvocation } from "../host/platform.mjs";
|
|
21
21
|
import { reachableRuntimes } from "../host/preflight.mjs";
|
|
22
22
|
import { spawn } from "node:child_process";
|
|
@@ -106,6 +106,9 @@ export async function preflightRuntimes(entries, options = {}) {
|
|
|
106
106
|
liveStatus: live.status,
|
|
107
107
|
usage: live.usage,
|
|
108
108
|
costUsd: live.costUsd,
|
|
109
|
+
// A failed ask carries its classified cause (quota, balance, model) and
|
|
110
|
+
// reset instant, so a refusal can be recorded as one instead of as an answer.
|
|
111
|
+
...(live.status === "done" ? {} : { availability: normalizeProviderAvailability(/** @type {any} */ (runtimes[index]), live) }),
|
|
109
112
|
detail: `${check.detail ?? "static probe failed"} · ${liveDetail}`,
|
|
110
113
|
};
|
|
111
114
|
}));
|
package/src/engine/resume.mjs
CHANGED
|
@@ -284,7 +284,7 @@ export async function resumeRun(runDirPath, options = {}) {
|
|
|
284
284
|
? recoverWorkerResult(runDir, state, contract, node)
|
|
285
285
|
: canonicalWorkerResultText(runDir, node.id) ?? recovery.result;
|
|
286
286
|
} catch (error) {
|
|
287
|
-
await applyInvalidWorkerResult(contract, node, state, runDir, null, lock,
|
|
287
|
+
await applyInvalidWorkerResult(contract, node, state, runDir, null, lock, error, states, campaign.path);
|
|
288
288
|
continue;
|
|
289
289
|
}
|
|
290
290
|
if (recovery.phase === "worker" && workerResult !== null && workerResult !== undefined) {
|
|
@@ -301,7 +301,7 @@ export async function resumeRun(runDirPath, options = {}) {
|
|
|
301
301
|
try {
|
|
302
302
|
parsedWorkerResult = parseWorkerResult(String(extractJson(workerResult) ?? workerResult));
|
|
303
303
|
} catch (error) {
|
|
304
|
-
await applyInvalidWorkerResult(contract, node, state, runDir, null, lock,
|
|
304
|
+
await applyInvalidWorkerResult(contract, node, state, runDir, null, lock, error, states, campaign.path);
|
|
305
305
|
continue;
|
|
306
306
|
}
|
|
307
307
|
// Same discriminator as lifecycle.mjs: the artifact is owed only by
|
|
@@ -311,7 +311,7 @@ export async function resumeRun(runDirPath, options = {}) {
|
|
|
311
311
|
try {
|
|
312
312
|
parseDiscoveryResult(parsedWorkerResult, attemptWorkspace(state) ?? contract.cwd);
|
|
313
313
|
} catch (error) {
|
|
314
|
-
await applyInvalidWorkerResult(contract, node, state, runDir, null, lock,
|
|
314
|
+
await applyInvalidWorkerResult(contract, node, state, runDir, null, lock, error, states, campaign.path);
|
|
315
315
|
continue;
|
|
316
316
|
}
|
|
317
317
|
}
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
*/
|
|
14
14
|
import { CONTRACT_VERSION, PROTOCOL_SCHEMA_VERSION, getHarness, harnessCapabilities, probeRuntime } from "../harnesses/index.mjs";
|
|
15
15
|
import { appendJsonl, writeJsonAtomic } from "../run/store.mjs";
|
|
16
|
-
import { availabilityKey, readAvailability,
|
|
16
|
+
import { availabilityKey, readAvailability, readRefusal, recordProbeVerdicts } from "../run/availability.mjs";
|
|
17
17
|
import { blockingChecks, environmentPreflight, reachableRuntimes } from "../host/preflight.mjs";
|
|
18
18
|
import { captureSourceIdentity } from "../repo/source-identity.mjs";
|
|
19
19
|
import { boundedGitSync } from "../repo/worktree.mjs";
|
|
@@ -569,6 +569,46 @@ async function livePreflightProbes(contract, runDir, sourceIdentity, timeoutSec)
|
|
|
569
569
|
if (verdict) fresh.set(id, verdict);
|
|
570
570
|
}
|
|
571
571
|
}
|
|
572
|
+
// A refusal another process met on this machine (quota, balance, model) is
|
|
573
|
+
// not asked again before its reset: the launch is refused on it, spending
|
|
574
|
+
// nothing, and the operator is told when the provider is back.
|
|
575
|
+
if (!pendingFreshPreflight) {
|
|
576
|
+
const refused = [...routed.entries()].map(([id, { runtime }]) => {
|
|
577
|
+
const executable = getHarness(runtime.harness).executable(runtime);
|
|
578
|
+
const held = readRefusal(availabilityKey({ harness: runtime.harness, model: runtime.model, executable }));
|
|
579
|
+
return held ? {
|
|
580
|
+
id,
|
|
581
|
+
harness: runtime.harness,
|
|
582
|
+
executable,
|
|
583
|
+
model: runtime.model,
|
|
584
|
+
version: sourceIdentity?.harnessVersions?.[id] ?? null,
|
|
585
|
+
capabilities: harnessCapabilities(runtime),
|
|
586
|
+
requiredCapabilities: {},
|
|
587
|
+
requiredCapabilitySets: [],
|
|
588
|
+
ok: false,
|
|
589
|
+
live: true,
|
|
590
|
+
liveStatus: "exhausted",
|
|
591
|
+
availability: { available: false, exhaustedUntil: held.exhaustedUntil, reason: held.reason },
|
|
592
|
+
detail: `live exhausted · ${held.reason}: refusal recorded on this machine at ${held.observedAt}, back at ${held.exhaustedUntil}`,
|
|
593
|
+
} : null;
|
|
594
|
+
});
|
|
595
|
+
if (refused.some(Boolean)) {
|
|
596
|
+
return [...routed.entries()].map(([id, { runtime }], index) => refused[index] ?? {
|
|
597
|
+
id,
|
|
598
|
+
harness: runtime.harness,
|
|
599
|
+
executable: getHarness(runtime.harness).executable(runtime),
|
|
600
|
+
model: runtime.model,
|
|
601
|
+
version: sourceIdentity?.harnessVersions?.[id] ?? null,
|
|
602
|
+
capabilities: harnessCapabilities(runtime),
|
|
603
|
+
requiredCapabilities: {},
|
|
604
|
+
requiredCapabilitySets: [],
|
|
605
|
+
ok: true,
|
|
606
|
+
live: false,
|
|
607
|
+
liveStatus: "not-asked",
|
|
608
|
+
detail: "not asked: another routed runtime is refused on this machine",
|
|
609
|
+
});
|
|
610
|
+
}
|
|
611
|
+
}
|
|
572
612
|
if (routed.size > 0 && fresh.size === routed.size) {
|
|
573
613
|
return [...routed.entries()].map(([id, { runtime }]) => {
|
|
574
614
|
const verdict = /** @type {RuntimeAvailability} */ (fresh.get(id));
|
|
@@ -594,10 +634,8 @@ async function livePreflightProbes(contract, runDir, sourceIdentity, timeoutSec)
|
|
|
594
634
|
// under the provider's own identity. Silence and a command that never
|
|
595
635
|
// reached a provider are verdicts of nothing and are never recorded, so the
|
|
596
636
|
// operator who fixes the host is never told the fix "already answered".
|
|
597
|
-
|
|
598
|
-
probes
|
|
599
|
-
.filter((probe) => probe.live === true && probe.liveStatus !== "reused" && liveVerdictRecorded(probe))
|
|
600
|
-
.map((probe) => availabilityKey(probe)),
|
|
637
|
+
recordProbeVerdicts(
|
|
638
|
+
probes.filter((probe) => probe.live === true && probe.liveStatus !== "reused" && liveVerdictRecorded(probe)),
|
|
601
639
|
Date.now(),
|
|
602
640
|
);
|
|
603
641
|
return probes;
|
|
@@ -51,9 +51,12 @@ export const DISCOVERY_RUNTIME_DEFINITIONS = Object.freeze({
|
|
|
51
51
|
tier: 1,
|
|
52
52
|
costRank: 1,
|
|
53
53
|
},
|
|
54
|
+
// The flash tier, not the unsuffixed pro: measured 2026-09-24 in the judge
|
|
55
|
+
// canary, glm-5.3 recalled 0.79 against glm-5.3-flash's 0.85 at about nine
|
|
56
|
+
// times the price, which the owner confirmed matches the public benchmarks.
|
|
54
57
|
"zcode-glm": {
|
|
55
58
|
harness: "zcode",
|
|
56
|
-
model: "glm-5.3",
|
|
59
|
+
model: "glm-5.3-flash",
|
|
57
60
|
vendor: "zhipu",
|
|
58
61
|
config: { "auth_token.env_key": "ZAI_API_KEY" },
|
|
59
62
|
tier: 1,
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The usage windows a Codex account reports, read from the session file the
|
|
3
|
+
* CLI writes for every thread. It is separate from the adapter because it is
|
|
4
|
+
* not part of an invocation's result: `codex exec --json` streams no limits,
|
|
5
|
+
* while `$CODEX_HOME/sessions/YYYY/MM/DD/rollout-*-<thread>.jsonl` carries a
|
|
6
|
+
* `rate_limits` record on every turn (measured 2026-09-24: `primary` is the
|
|
7
|
+
* 300-minute window, `secondary` the 10080-minute week, each with
|
|
8
|
+
* `used_percent` and `resets_at` in epoch seconds).
|
|
9
|
+
*/
|
|
10
|
+
import { existsSync, readFileSync, readdirSync } from "node:fs";
|
|
11
|
+
import { homedir } from "node:os";
|
|
12
|
+
import { join } from "node:path";
|
|
13
|
+
|
|
14
|
+
/** @typedef {{window: string, usedPercent: number, resetsAt: string}} UsageWindow */
|
|
15
|
+
|
|
16
|
+
/** @param {Record<string, string|undefined>} [env] @returns {string} the account's Codex home */
|
|
17
|
+
export function codexHome(env = process.env) {
|
|
18
|
+
return env.CODEX_HOME || join(homedir(), ".codex");
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* The account's windows as the thread's session file last reported them, or
|
|
23
|
+
* an empty list when the file or the record cannot be found.
|
|
24
|
+
*
|
|
25
|
+
* @param {string} threadId
|
|
26
|
+
* @param {{env?: Record<string, string|undefined>, now?: number}} [options]
|
|
27
|
+
* @returns {UsageWindow[]}
|
|
28
|
+
*/
|
|
29
|
+
export function codexUsageWindows(threadId, options = {}) {
|
|
30
|
+
const now = options.now ?? Date.now();
|
|
31
|
+
const sessions = join(codexHome(options.env), "sessions");
|
|
32
|
+
for (const day of [0, 1]) {
|
|
33
|
+
const date = new Date(now - day * 86_400_000);
|
|
34
|
+
const dir = join(sessions, String(date.getFullYear()), String(date.getMonth() + 1).padStart(2, "0"), String(date.getDate()).padStart(2, "0"));
|
|
35
|
+
if (!existsSync(dir)) continue;
|
|
36
|
+
const file = readdirSync(dir).find((name) => name.endsWith(`-${threadId}.jsonl`));
|
|
37
|
+
if (file) return lastWindows(readFileSync(join(dir, file), "utf8"));
|
|
38
|
+
}
|
|
39
|
+
return [];
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** @param {string} text @returns {UsageWindow[]} */
|
|
43
|
+
function lastWindows(text) {
|
|
44
|
+
const lines = text.split("\n");
|
|
45
|
+
for (let index = lines.length - 1; index >= 0; index -= 1) {
|
|
46
|
+
if (!lines[index].includes("\"rate_limits\"")) continue;
|
|
47
|
+
let limits;
|
|
48
|
+
try {
|
|
49
|
+
limits = findRateLimits(JSON.parse(lines[index]));
|
|
50
|
+
} catch {
|
|
51
|
+
// A line cut by a writer still flushing is skipped; an earlier one answers.
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
if (!limits) continue;
|
|
55
|
+
return [["primary", limits.primary], ["secondary", limits.secondary]]
|
|
56
|
+
.filter(([, entry]) => entry && typeof entry.used_percent === "number" && typeof entry.resets_at === "number")
|
|
57
|
+
.map(([name, entry]) => ({
|
|
58
|
+
window: entry.window_minutes === 10080 ? "seven_day" : entry.window_minutes === 300 ? "five_hour" : String(name),
|
|
59
|
+
usedPercent: entry.used_percent,
|
|
60
|
+
resetsAt: new Date(entry.resets_at * 1000).toISOString(),
|
|
61
|
+
}));
|
|
62
|
+
}
|
|
63
|
+
return [];
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** @param {unknown} value @returns {any} the first `rate_limits` object anywhere in a record */
|
|
67
|
+
function findRateLimits(value) {
|
|
68
|
+
if (!value || typeof value !== "object") return null;
|
|
69
|
+
const record = /** @type {Record<string, unknown>} */ (value);
|
|
70
|
+
if (record.rate_limits && typeof record.rate_limits === "object") return record.rate_limits;
|
|
71
|
+
for (const child of Object.values(record)) {
|
|
72
|
+
const found = findRateLimits(child);
|
|
73
|
+
if (found) return found;
|
|
74
|
+
}
|
|
75
|
+
return null;
|
|
76
|
+
}
|
|
@@ -208,7 +208,7 @@ function authToken(runtime) {
|
|
|
208
208
|
* each one, `doctor` checks binaries), so a permissions or disk failure has to
|
|
209
209
|
* degrade into "not found" — never into an aborted run or a crashed report.
|
|
210
210
|
*
|
|
211
|
-
* @param {{env?: Record<string, string|undefined>, pathDirs?: string[], home?: string, bundle?: {electron: string, cli: string}}} [options]
|
|
211
|
+
* @param {{env?: Record<string, string|undefined>, pathDirs?: string[], home?: string, bundle?: {electron: string, cli: string}, host?: {arch: string, nodeMajor: number, node: string}}} [options]
|
|
212
212
|
* @returns {void}
|
|
213
213
|
*/
|
|
214
214
|
export function ensureZcodeAvailable(options = {}) {
|
|
@@ -218,7 +218,7 @@ export function ensureZcodeAvailable(options = {}) {
|
|
|
218
218
|
if (resolvesOnPath(pathDirs, ZCODE_BIN_NAME)) return;
|
|
219
219
|
const bundle = options.bundle ?? zcodeBundle(env);
|
|
220
220
|
if (!bundle || !existsSync(bundle.electron) || !existsSync(bundle.cli)) return;
|
|
221
|
-
const body = zcodeShim(bundle);
|
|
221
|
+
const body = zcodeShim(bundle, options.host);
|
|
222
222
|
for (const dir of shimDirs(options.home ?? homedir())) {
|
|
223
223
|
if (!pathDirs.includes(dir)) continue;
|
|
224
224
|
if (settleShim(join(dir, ZCODE_BIN_NAME), body)) return;
|
|
@@ -252,9 +252,23 @@ function zcodeBundle(env) {
|
|
|
252
252
|
* outlives it.
|
|
253
253
|
*
|
|
254
254
|
* @param {{electron: string, cli: string}} bundle
|
|
255
|
+
* @param {{arch: string, nodeMajor: number, node: string}} [host]
|
|
255
256
|
* @returns {string}
|
|
256
257
|
*/
|
|
257
|
-
function zcodeShim(bundle) {
|
|
258
|
+
function zcodeShim(bundle, host = { arch: process.arch, nodeMajor: Number(process.versions.node.split(".")[0]), node: process.execPath }) {
|
|
259
|
+
// Measured 2026-09-24 on darwin arm64 with node 26.8.1: the bundled CLI
|
|
260
|
+
// answers a 3.2 KB response intact under the system node, and every process
|
|
261
|
+
// it spawns under the app's Electron shows an icon in the macOS Dock. The
|
|
262
|
+
// Electron path stays for every other host, where the detached-ArrayBuffer
|
|
263
|
+
// failure (node 24 x64) is the only measurement there is.
|
|
264
|
+
if (host.arch === "arm64" && host.nodeMajor >= 26) {
|
|
265
|
+
return `#!/usr/bin/env bash
|
|
266
|
+
set -euo pipefail
|
|
267
|
+
# Written by the faberun zcode harness; the ZCode app owns the CLI path.
|
|
268
|
+
exec ${host.node} \\
|
|
269
|
+
${bundle.cli} "$@"
|
|
270
|
+
`;
|
|
271
|
+
}
|
|
258
272
|
return `#!/usr/bin/env bash
|
|
259
273
|
set -euo pipefail
|
|
260
274
|
# Written by the faberun zcode harness; the ZCode app owns both paths.
|
package/src/host/preflight.mjs
CHANGED
|
@@ -43,7 +43,8 @@ import { NOTIFY_SESSION_ENV, sessionWakeNotice } from "../notify/session.mjs";
|
|
|
43
43
|
import { findExecutable } from "./platform.mjs";
|
|
44
44
|
import { colorLevel, statusToken } from "../cli/brand.mjs";
|
|
45
45
|
import { RUNS_DIR_NAME } from "../run/paths.mjs";
|
|
46
|
-
import { availabilityKey, readAvailability,
|
|
46
|
+
import { availabilityKey, readAvailability, recordProbeVerdicts } from "../run/availability.mjs";
|
|
47
|
+
import { readUsageWindows, usageAccountOf } from "../run/usage-windows.mjs";
|
|
47
48
|
|
|
48
49
|
/** @typedef {import("../contract/index.mjs").ValidatedContract} ValidatedContract */
|
|
49
50
|
/** @typedef {import("../contract/index.mjs").RuntimeSnapshot} RuntimeSnapshot */
|
|
@@ -222,6 +223,35 @@ export function checkRuntimeBinaries(runtimes, harnessVersions, cwd = ".") {
|
|
|
222
223
|
return pass("runtime binaries", resolved.length ? `${resolved.join(" · ")}${aside}` : "no routed runtime");
|
|
223
224
|
}
|
|
224
225
|
|
|
226
|
+
/**
|
|
227
|
+
* How much of each routed account's provider-reported window is spent. A
|
|
228
|
+
* window at 100% blocks the dispatch until it resets, because every call would
|
|
229
|
+
* be refused; one at or above 90% is an advisory warning with the reset
|
|
230
|
+
* instant, so the operator can ration before the wall. The 90% line is a first
|
|
231
|
+
* guess, not a measurement. No check is added when no routed runtime spends
|
|
232
|
+
* from an account that reports windows.
|
|
233
|
+
*
|
|
234
|
+
* @param {ReachableRuntimes} runtimes
|
|
235
|
+
* @param {NodeJS.ProcessEnv} env
|
|
236
|
+
* @param {number} [now]
|
|
237
|
+
* @returns {EnvCheck[]}
|
|
238
|
+
*/
|
|
239
|
+
export function checkUsageWindows(runtimes, env, now = Date.now()) {
|
|
240
|
+
const accounts = new Set([...runtimes.values()].filter((entry) => entry.routed).map((entry) => usageAccountOf(entry.runtime, env)).filter((account) => account !== null));
|
|
241
|
+
/** @type {EnvCheck[]} */
|
|
242
|
+
const checks = [];
|
|
243
|
+
for (const account of accounts) {
|
|
244
|
+
const windows = readUsageWindows(/** @type {string} */ (account), now);
|
|
245
|
+
if (windows.length === 0) continue;
|
|
246
|
+
const worst = windows.reduce((a, b) => (b.usedPercent > a.usedPercent ? b : a));
|
|
247
|
+
const detail = `${account.split(":")[0]} ${windows.map((window) => `${window.window} ${window.usedPercent}% used, resets ${window.resetsAt}`).join(" · ")}`;
|
|
248
|
+
if (worst.usedPercent >= 100) checks.push(fail("usage window", `${detail}: every call would be refused until ${worst.resetsAt}`));
|
|
249
|
+
else if (worst.usedPercent >= 90) checks.push(fail("usage window", detail, true));
|
|
250
|
+
else checks.push(pass("usage window", detail));
|
|
251
|
+
}
|
|
252
|
+
return checks;
|
|
253
|
+
}
|
|
254
|
+
|
|
225
255
|
/**
|
|
226
256
|
* @param {{cwd: string, runtimes: ReachableRuntimes, harnessVersions?: Record<string, string|null>, env?: NodeJS.ProcessEnv}} options
|
|
227
257
|
* @returns {EnvReport}
|
|
@@ -234,6 +264,7 @@ export function environmentPreflight(options) {
|
|
|
234
264
|
checkGit(cwd),
|
|
235
265
|
checkWorktree(cwd, env.FABERUN_REQUIRE_CLEAN_WORKTREE === "1"),
|
|
236
266
|
checkRuntimeBinaries(options.runtimes, options.harnessVersions ?? {}, cwd),
|
|
267
|
+
...checkUsageWindows(options.runtimes, env),
|
|
237
268
|
];
|
|
238
269
|
return { schemaVersion: ENV_PREFLIGHT_SCHEMA_VERSION, ok: checks.every((check) => check.ok || check.advisory), checks };
|
|
239
270
|
}
|
|
@@ -517,7 +548,7 @@ async function liveAvailabilityChecks(contractPath, runtimes) {
|
|
|
517
548
|
const override = Number(process.env.FABERUN_PREFLIGHT_TIMEOUT_SEC);
|
|
518
549
|
const timeoutSec = process.env.FABERUN_PREFLIGHT_TIMEOUT_SEC !== undefined && Number.isFinite(override) && override > 0 ? override : 60;
|
|
519
550
|
const probes = await preflightContract(contractPath, { liveTimeoutSec: timeoutSec });
|
|
520
|
-
|
|
551
|
+
recordProbeVerdicts(probes.filter((probe) => liveVerdict(probe).recorded));
|
|
521
552
|
return probes.map((probe) => {
|
|
522
553
|
const verdict = liveVerdict(probe);
|
|
523
554
|
return {
|
package/src/plan/pipeline.mjs
CHANGED
|
@@ -24,7 +24,7 @@ import { campaignCli } from "../cli/campaign.mjs";
|
|
|
24
24
|
import { appendJsonl, writeJsonAtomic } from "../run/store.mjs";
|
|
25
25
|
import { stableJson } from "../util.mjs";
|
|
26
26
|
import { allowanceDelta, allowanceEventFields, sampleAllowance } from "../seat/allowance.mjs";
|
|
27
|
-
import { askPlanningRuntimes, refusePlanningSilence } from "./preflight.mjs";
|
|
27
|
+
import { askPlanningRuntimes, refusePlanningSilence, refuseUnplannableRuntimes } from "./preflight.mjs";
|
|
28
28
|
import { parseSpec, validateSpec } from "./spec.mjs";
|
|
29
29
|
import { collectRepoFacts } from "./repo-facts.mjs";
|
|
30
30
|
import { RISK_TIERS, TASK_KIND_CATALOGUE_FILE, buildPlanningContract, renderTaskKindCatalogue, validateFindings, validatePlanOutput } from "./template.mjs";
|
|
@@ -111,6 +111,7 @@ export async function runPlanningPipeline(options) {
|
|
|
111
111
|
const campaignPath = campaignTree(cwd, campaignId);
|
|
112
112
|
const campaign = readCampaign(campaignPath);
|
|
113
113
|
if (campaign.status !== "active") throw new Error(`campaign is closed: ${campaignId}`);
|
|
114
|
+
refuseUnplannableRuntimes(runtimes, runtimeDefaults, packageMode);
|
|
114
115
|
refusePlanningSilence(await ask(runtimes, runtimeDefaults, cwd), cwd);
|
|
115
116
|
|
|
116
117
|
const relativeSpecPath = repoRelativePath(cwd, specPath, "specPath");
|
package/src/plan/preflight.mjs
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
import { preflightRuntimes } from "../engine/live-preflight.mjs";
|
|
10
10
|
import { liveSilenceCause } from "../engine/live-silence.mjs";
|
|
11
11
|
import { harnessCapabilities } from "../harnesses/index.mjs";
|
|
12
|
-
import { validateRuntime } from "../contract/runtime.mjs";
|
|
12
|
+
import { assertRuntimeExecutesCommands, validateRuntime } from "../contract/runtime.mjs";
|
|
13
13
|
|
|
14
14
|
/**
|
|
15
15
|
* The runtimes a planning run will spend, asked once before its first stage.
|
|
@@ -75,3 +75,44 @@ export function refusePlanningSilence(checks, cwd) {
|
|
|
75
75
|
{ code: "env_preflight_failed" },
|
|
76
76
|
);
|
|
77
77
|
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Refuse, before anything is spent, a runtime choice no frozen contract could
|
|
81
|
+
* carry. Both halves were measured on 2026-09-24 in `choose-the-judges` R5,
|
|
82
|
+
* where each surfaced only at freeze, after four rounds: a worker whose
|
|
83
|
+
* permission mode cannot run commands cannot carry an implementation node's
|
|
84
|
+
* verification (US$ 7.64 and an hour, contested), and a judge that shares the
|
|
85
|
+
* worker's vendor, or the vendor of any runtime its fallback reaches, is
|
|
86
|
+
* refused by the vendor rule on every frozen node (`runtime_routing_unmet` in
|
|
87
|
+
* all four rounds). The pipeline's defaults name both roles, so both are
|
|
88
|
+
* decidable here.
|
|
89
|
+
*
|
|
90
|
+
* @param {Record<string, Record<string, unknown>>} runtimes
|
|
91
|
+
* @param {{worker?: string, judge?: string}} runtimeDefaults
|
|
92
|
+
* @param {"implementation"|"exploratory"} packageMode
|
|
93
|
+
* @returns {void}
|
|
94
|
+
*/
|
|
95
|
+
export function refuseUnplannableRuntimes(runtimes, runtimeDefaults, packageMode) {
|
|
96
|
+
const worker = runtimeDefaults.worker;
|
|
97
|
+
if (!worker || !runtimes[worker]) return;
|
|
98
|
+
if (packageMode === "implementation") {
|
|
99
|
+
try {
|
|
100
|
+
assertRuntimeExecutesCommands(/** @type {any} */ (runtimes), worker, 0, "every implementation node", "worker runtime");
|
|
101
|
+
} catch (error) {
|
|
102
|
+
throw new Error(`--runtime-defaults worker=${worker} cannot run the verification an implementation plan carries: ${error instanceof Error ? error.message : String(error)}`);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
const judge = runtimeDefaults.judge;
|
|
106
|
+
if (!judge || !runtimes[judge]) return;
|
|
107
|
+
/** @type {Set<string>} */
|
|
108
|
+
const vendors = new Set();
|
|
109
|
+
const seen = new Set();
|
|
110
|
+
for (let id = /** @type {string|undefined} */ (worker); id && runtimes[id] && !seen.has(id); id = /** @type {string|undefined} */ (runtimes[id].fallback)) {
|
|
111
|
+
seen.add(id);
|
|
112
|
+
if (typeof runtimes[id].vendor === "string") vendors.add(/** @type {string} */ (runtimes[id].vendor));
|
|
113
|
+
}
|
|
114
|
+
const judgeVendor = runtimes[judge].vendor;
|
|
115
|
+
if (typeof judgeVendor === "string" && vendors.has(judgeVendor)) {
|
|
116
|
+
throw new Error(`--runtime-defaults judge=${judge} shares vendor ${judgeVendor} with worker ${worker} or its fallback, so no frozen node could route its judge; name a judge of another vendor`);
|
|
117
|
+
}
|
|
118
|
+
}
|
package/src/repo/worktree.mjs
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { spawnSync } from "node:child_process";
|
|
2
2
|
import { gitArguments } from "../host/platform.mjs";
|
|
3
|
-
import { existsSync, lstatSync, mkdirSync, readFileSync, realpathSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, realpathSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs";
|
|
4
4
|
import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
5
|
-
import { RUNS_DIR_NAME, attemptWorktreePath, candidateWorktreePath } from "../run/paths.mjs";
|
|
5
|
+
import { RUNS_DIR_NAME, attemptWorktreePath, candidateWorktreePath, worktreeRoot } from "../run/paths.mjs";
|
|
6
6
|
|
|
7
7
|
/** @typedef {import("../contract/index.mjs").NodeSnapshot} NodeSnapshot */
|
|
8
8
|
|
|
@@ -481,3 +481,50 @@ export function attemptWorkspace(state) {
|
|
|
481
481
|
const path = state?.worktree?.path;
|
|
482
482
|
return path && state?.worktree?.status !== "removed" && existsSync(path) ? path : null;
|
|
483
483
|
}
|
|
484
|
+
|
|
485
|
+
/**
|
|
486
|
+
* Release every attempt worktree a finished run left behind, keeping what it
|
|
487
|
+
* held. Measured 2026-09-24: attempts that were never accepted (a failed or
|
|
488
|
+
* blocked try, the first of two, a discovery node) kept their worktree and
|
|
489
|
+
* branch forever, and one machine held 41 registered and 42 orphaned
|
|
490
|
+
* worktrees (909 MB) from campaigns long closed. Each worktree's HEAD, and its
|
|
491
|
+
* uncommitted delta when it has one (`git stash create`, which touches no
|
|
492
|
+
* file), is kept under `refs/faberun-archive/<run>/<attempt>` before the
|
|
493
|
+
* worktree and its branch go, so removal loses nothing git can hold.
|
|
494
|
+
*
|
|
495
|
+
* @param {string} repo
|
|
496
|
+
* @param {string} runDir
|
|
497
|
+
* @param {string} runId
|
|
498
|
+
* @returns {{removed: number, archived: string[]}}
|
|
499
|
+
*/
|
|
500
|
+
export function releaseRunWorktrees(repo, runDir, runId) {
|
|
501
|
+
const root = worktreeRoot(runDir, runId);
|
|
502
|
+
if (!existsSync(root)) return { removed: 0, archived: [] };
|
|
503
|
+
/** @type {string[]} */
|
|
504
|
+
const archived = [];
|
|
505
|
+
let removed = 0;
|
|
506
|
+
for (const entry of readdirSync(root)) {
|
|
507
|
+
const path = join(root, entry);
|
|
508
|
+
/** @param {string[]} args @returns {string} */
|
|
509
|
+
const inTree = (args) => {
|
|
510
|
+
try {
|
|
511
|
+
return git(path, args);
|
|
512
|
+
} catch {
|
|
513
|
+
// An orphaned directory git no longer knows answers nothing; it is removed below all the same.
|
|
514
|
+
return "";
|
|
515
|
+
}
|
|
516
|
+
};
|
|
517
|
+
const branch = inTree(["symbolic-ref", "--quiet", "--short", "HEAD"]);
|
|
518
|
+
const keep = inTree(["stash", "create"]) || inTree(["rev-parse", "HEAD"]);
|
|
519
|
+
if (keep) {
|
|
520
|
+
const ref = `refs/faberun-archive/${runId}/${entry}`;
|
|
521
|
+
runGit(["-C", repo, "update-ref", ref, keep]);
|
|
522
|
+
archived.push(ref);
|
|
523
|
+
}
|
|
524
|
+
removeWorktree(repo, path);
|
|
525
|
+
if (branch) deleteRef(repo, `refs/heads/${branch}`);
|
|
526
|
+
removed += 1;
|
|
527
|
+
}
|
|
528
|
+
rmSync(root, { recursive: true, force: true });
|
|
529
|
+
return { removed, archived };
|
|
530
|
+
}
|
package/src/run/availability.mjs
CHANGED
|
@@ -25,6 +25,16 @@ import { writeJsonAtomic } from "./store.mjs";
|
|
|
25
25
|
|
|
26
26
|
/** @typedef {import("../engine/runtime-discovery.mjs").RuntimeAvailability} RuntimeAvailability */
|
|
27
27
|
|
|
28
|
+
/**
|
|
29
|
+
* Answers a provider gives before doing any work that no retry inside their
|
|
30
|
+
* window can change. Measured 2026-09-24: a probe answered
|
|
31
|
+
* `quota_exhausted` was stored as "answered", the next launches reused it
|
|
32
|
+
* without asking, and six processes spent into two exhausted accounts.
|
|
33
|
+
*/
|
|
34
|
+
const REFUSAL_REASONS = new Set(["quota_exhausted", "insufficient_balance", "model_not_supported"]);
|
|
35
|
+
/** How long a refusal that names no reset instant is held. A first guess, not a measurement. */
|
|
36
|
+
const REFUSAL_HOLD_MS = 15 * 60 * 1000;
|
|
37
|
+
|
|
28
38
|
/** The store shape this module reads and writes; a foreign shape reads as empty. */
|
|
29
39
|
const STORE_SCHEMA_VERSION = 1;
|
|
30
40
|
|
|
@@ -96,6 +106,69 @@ export function recordAvailability(keys, now = Date.now()) {
|
|
|
96
106
|
writeJsonAtomic(path, store);
|
|
97
107
|
}
|
|
98
108
|
|
|
109
|
+
/**
|
|
110
|
+
* The refusal a probe or a provider envelope's classified availability
|
|
111
|
+
* carries, or null when it is anything else.
|
|
112
|
+
*
|
|
113
|
+
* @param {{available?: boolean, exhaustedUntil?: string|null, reason?: string}|null|undefined} availability
|
|
114
|
+
* @returns {{reason: string, exhaustedUntil: string|null}|null}
|
|
115
|
+
*/
|
|
116
|
+
export function refusalOfAvailability(availability) {
|
|
117
|
+
if (!availability || availability.available !== false || !REFUSAL_REASONS.has(String(availability.reason))) return null;
|
|
118
|
+
return { reason: String(availability.reason), exhaustedUntil: availability.exhaustedUntil ?? null };
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Record that a provider refused the work, so every process on this machine
|
|
123
|
+
* knows until the reset: the launch gate asks again instead of reusing an old
|
|
124
|
+
* answer, and a class that spends one call per case stops before its next.
|
|
125
|
+
*
|
|
126
|
+
* @param {string} key
|
|
127
|
+
* @param {{reason: string, exhaustedUntil: string|null}} refusal
|
|
128
|
+
* @param {number} [now]
|
|
129
|
+
* @returns {void}
|
|
130
|
+
*/
|
|
131
|
+
export function recordRefusal(key, refusal, now = Date.now()) {
|
|
132
|
+
const store = loadStore();
|
|
133
|
+
store.verdicts[key] = {
|
|
134
|
+
available: false,
|
|
135
|
+
exhaustedUntil: refusal.exhaustedUntil ?? new Date(now + REFUSAL_HOLD_MS).toISOString(),
|
|
136
|
+
reason: refusal.reason,
|
|
137
|
+
observedAt: new Date(now).toISOString(),
|
|
138
|
+
window: PREFLIGHT_WINDOW,
|
|
139
|
+
};
|
|
140
|
+
const path = availabilityPath();
|
|
141
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
142
|
+
writeJsonAtomic(path, store);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* The refusal recorded for a provider that has not yet reset, or null.
|
|
147
|
+
*
|
|
148
|
+
* @param {string} key
|
|
149
|
+
* @param {number} [now]
|
|
150
|
+
* @returns {RuntimeAvailability|null}
|
|
151
|
+
*/
|
|
152
|
+
export function readRefusal(key, now = Date.now()) {
|
|
153
|
+
const record = loadVerdicts()[key];
|
|
154
|
+
if (!record || record.available !== false || !record.exhaustedUntil) return null;
|
|
155
|
+
return Date.parse(record.exhaustedUntil) > now ? record : null;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Record what a set of live probes learned: an answer as an answer, a refusal
|
|
160
|
+
* as a refusal until its reset.
|
|
161
|
+
*
|
|
162
|
+
* @param {{harness: string, model: string, executable: string, availability?: {available: boolean, exhaustedUntil: string|null, reason: string}}[]} probes
|
|
163
|
+
* @param {number} [now]
|
|
164
|
+
* @returns {void}
|
|
165
|
+
*/
|
|
166
|
+
export function recordProbeVerdicts(probes, now = Date.now()) {
|
|
167
|
+
const refused = probes.filter((probe) => refusalOfAvailability(probe.availability));
|
|
168
|
+
recordAvailability(probes.filter((probe) => !refused.includes(probe)).map((probe) => availabilityKey(probe)), now);
|
|
169
|
+
for (const probe of refused) recordRefusal(availabilityKey(probe), /** @type {{reason: string, exhaustedUntil: string|null}} */ (refusalOfAvailability(probe.availability)), now);
|
|
170
|
+
}
|
|
171
|
+
|
|
99
172
|
/**
|
|
100
173
|
* @returns {{schemaVersion: number, verdicts: Record<string, RuntimeAvailability>}}
|
|
101
174
|
*/
|
package/src/run/paths.mjs
CHANGED
|
@@ -147,6 +147,11 @@ export function availabilityPath() {
|
|
|
147
147
|
return join(faberunHome(), "availability.json");
|
|
148
148
|
}
|
|
149
149
|
|
|
150
|
+
/** @returns {string} the machine-wide record of each account's provider-reported usage windows */
|
|
151
|
+
export function usageWindowsPath() {
|
|
152
|
+
return join(faberunHome(), "usage-windows.json");
|
|
153
|
+
}
|
|
154
|
+
|
|
150
155
|
/**
|
|
151
156
|
* The campaigns directory beneath a runs root: `<runs>/campaigns`.
|
|
152
157
|
* Composes `campaign/layout.mjs`'s `campaignsDir`, which owns the shape given
|
|
@@ -173,7 +178,7 @@ export function campaignTree(cwd, campaignId) {
|
|
|
173
178
|
}
|
|
174
179
|
|
|
175
180
|
/** @param {string} runDir @param {string} runId @returns {string} */
|
|
176
|
-
function worktreeRoot(runDir, runId) {
|
|
181
|
+
export function worktreeRoot(runDir, runId) {
|
|
177
182
|
return join(dirname(runDir), "worktrees", runId);
|
|
178
183
|
}
|
|
179
184
|
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The machine-wide record of how much of each account's provider-reported
|
|
3
|
+
* usage window is spent. It is separate from the availability store because
|
|
4
|
+
* it answers a different question: availability says whether a provider
|
|
5
|
+
* answered, this says how close the account is to refusing. Measured
|
|
6
|
+
* 2026-09-24: the Codex week went from 86% to 94% in one afternoon of judge
|
|
7
|
+
* readings, and nothing read it until the owner looked.
|
|
8
|
+
*/
|
|
9
|
+
import { mkdirSync, readFileSync } from "node:fs";
|
|
10
|
+
import { dirname } from "node:path";
|
|
11
|
+
import { usageWindowsPath } from "./paths.mjs";
|
|
12
|
+
import { writeJsonAtomic } from "./store.mjs";
|
|
13
|
+
import { codexHome, codexUsageWindows } from "../harnesses/codex/usage-window.mjs";
|
|
14
|
+
|
|
15
|
+
/** @typedef {import("../harnesses/codex/usage-window.mjs").UsageWindow} UsageWindow */
|
|
16
|
+
/** @typedef {{windows: UsageWindow[], observedAt: string}} AccountWindows */
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* @param {string} account the account key, e.g. `codex:<CODEX_HOME>`
|
|
20
|
+
* @param {UsageWindow[]} windows
|
|
21
|
+
* @param {number} [now]
|
|
22
|
+
* @returns {void}
|
|
23
|
+
*/
|
|
24
|
+
export function recordUsageWindows(account, windows, now = Date.now()) {
|
|
25
|
+
if (windows.length === 0) return;
|
|
26
|
+
const store = loadStore();
|
|
27
|
+
store.accounts[account] = { windows, observedAt: new Date(now).toISOString() };
|
|
28
|
+
const path = usageWindowsPath();
|
|
29
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
30
|
+
writeJsonAtomic(path, store);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* The account's windows that have not reset yet, or an empty list.
|
|
35
|
+
*
|
|
36
|
+
* @param {string} account
|
|
37
|
+
* @param {number} [now]
|
|
38
|
+
* @returns {UsageWindow[]}
|
|
39
|
+
*/
|
|
40
|
+
export function readUsageWindows(account, now = Date.now()) {
|
|
41
|
+
const record = loadStore().accounts[account];
|
|
42
|
+
if (!record || !Array.isArray(record.windows)) return [];
|
|
43
|
+
return record.windows.filter((window) => typeof window.usedPercent === "number" && Date.parse(window.resetsAt) > now);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** @returns {{schemaVersion: number, accounts: Record<string, AccountWindows>}} */
|
|
47
|
+
function loadStore() {
|
|
48
|
+
try {
|
|
49
|
+
const parsed = JSON.parse(readFileSync(usageWindowsPath(), "utf8"));
|
|
50
|
+
if (parsed && parsed.schemaVersion === 1 && parsed.accounts && typeof parsed.accounts === "object") return parsed;
|
|
51
|
+
} catch {
|
|
52
|
+
// ENOENT on a machine that never recorded a window, or a store a truncated
|
|
53
|
+
// write left unreadable: either way nothing is known, which reads as no warning.
|
|
54
|
+
}
|
|
55
|
+
return { schemaVersion: 1, accounts: {} };
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* The account a runtime spends from, when its provider reports windows; null
|
|
60
|
+
* for every harness that reports none (only Codex is measured to).
|
|
61
|
+
*
|
|
62
|
+
* @param {{harness: string}} runtime
|
|
63
|
+
* @param {Record<string, string|undefined>} [env]
|
|
64
|
+
* @returns {string|null}
|
|
65
|
+
*/
|
|
66
|
+
export function usageAccountOf(runtime, env = process.env) {
|
|
67
|
+
return runtime.harness === "codex" ? `codex:${codexHome(env)}` : null;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Record what a finished invocation's provider reported about its account.
|
|
72
|
+
*
|
|
73
|
+
* @param {{harness: string}} runtime
|
|
74
|
+
* @param {string|null|undefined} continuationId the provider's thread id
|
|
75
|
+
* @param {{env?: Record<string, string|undefined>, now?: number}} [options]
|
|
76
|
+
* @returns {void}
|
|
77
|
+
*/
|
|
78
|
+
export function recordInvocationWindows(runtime, continuationId, options = {}) {
|
|
79
|
+
const account = usageAccountOf(runtime, options.env);
|
|
80
|
+
if (!account || !continuationId) return;
|
|
81
|
+
recordUsageWindows(account, codexUsageWindows(continuationId, options), options.now);
|
|
82
|
+
}
|
package/src/run/usage.mjs
CHANGED
|
@@ -16,6 +16,7 @@ import { priceUsage } from "../engine/process.mjs";
|
|
|
16
16
|
import { readBoundedTail, sessionLedger } from "../engine/transcript.mjs";
|
|
17
17
|
import { writeNode } from "../engine/state.mjs";
|
|
18
18
|
import { harnessCapabilities, normalizeProviderResult } from "../harnesses/index.mjs";
|
|
19
|
+
import { recordInvocationWindows } from "./usage-windows.mjs";
|
|
19
20
|
|
|
20
21
|
// `priceUsage` is defined beside `invocationResult`, the second source point,
|
|
21
22
|
// and re-exported here so the ledger's public surface is unchanged. This module
|
|
@@ -88,6 +89,12 @@ export function recordInvocationUsage(job, options = {}) {
|
|
|
88
89
|
// Price only after the backfill has run: the counters this function persists
|
|
89
90
|
// and returns are the ones the price is derived from, and the envelope becomes
|
|
90
91
|
// the single priced object every later copy spreads from.
|
|
92
|
+
try {
|
|
93
|
+
recordInvocationWindows(job.runtime, envelope.continuationId);
|
|
94
|
+
} catch {
|
|
95
|
+
// The window record is telemetry about the account, never a reason to fail
|
|
96
|
+
// the invocation it rode on: a missing or unreadable session file means no reading.
|
|
97
|
+
}
|
|
91
98
|
const priced = priceUsage(job.runtime, envelope.usage, envelope.costUsd);
|
|
92
99
|
envelope = { ...envelope, costUsd: priced.costUsd, costProvenance: priced.costProvenance };
|
|
93
100
|
// The per-request ledger is read here, with the usage, for the same reason
|