faberun 0.17.2 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/integrations/claude-code/statusline.sh +12 -2
- package/package.json +2 -2
- package/skills/faberun/references/operations.md +14 -7
- package/skills/init-agentkit/scripts/install-agentkit.sh +10 -2
- package/src/campaign/chain.mjs +5 -4
- package/src/campaign/metrics.mjs +3 -1
- package/src/cli/launch.mjs +10 -1
- package/src/cli.mjs +10 -4
- package/src/contract/index.mjs +16 -3
- package/src/contract/task-packet.mjs +13 -1
- package/src/engine/bulk-read.mjs +7 -1
- package/src/engine/gate.mjs +17 -3
- package/src/engine/judge-gate.mjs +2 -2
- package/src/engine/live-preflight.mjs +43 -10
- package/src/engine/live-silence.mjs +49 -0
- package/src/engine/process-identity.mjs +12 -1
- package/src/engine/process.mjs +2 -1
- package/src/engine/run-command.mjs +8 -5
- package/src/engine/run-identity.mjs +213 -14
- package/src/engine/runtime-discovery.mjs +26 -5
- package/src/engine/scheduler.mjs +1 -1
- package/src/engine/supervise.mjs +2 -1
- package/src/harnesses/catalogue.mjs +8 -1
- package/src/harnesses/dsh/runner.mjs +6 -1
- package/src/harnesses/index.mjs +35 -7
- package/src/host/platform.mjs +204 -1
- package/src/host/preflight.mjs +142 -7
- package/src/notify/index.mjs +77 -11
- package/src/notify/session.mjs +64 -24
- package/src/plan/pipeline.mjs +5 -1
- package/src/plan/preflight.mjs +77 -0
- package/src/repo/declared-paths.mjs +87 -6
- package/src/repo/signal.mjs +56 -21
- package/src/repo/workspace.mjs +3 -2
- package/src/repo/worktree.mjs +2 -1
- package/src/report/locale.mjs +20 -0
- package/src/report/message.mjs +2 -2
- package/src/report/next.mjs +15 -1
- package/src/run/availability.mjs +138 -0
- package/src/run/disk-gc.mjs +16 -2
- package/src/run/lock.mjs +8 -0
- package/src/run/paths.mjs +13 -0
- package/src/seat/allowance.mjs +4 -1
- package/src/seat/tmux.mjs +9 -1
|
@@ -17,6 +17,7 @@ import { processStartToken } from "../run/lock.mjs";
|
|
|
17
17
|
import { randomUUID } from "node:crypto";
|
|
18
18
|
import { runMutation } from "./mutation.mjs";
|
|
19
19
|
import { spawn } from "node:child_process";
|
|
20
|
+
import { killTarget, spawnInvocation } from "../host/platform.mjs";
|
|
20
21
|
/** @typedef {import("../contract/verification.mjs").VerificationOptions} VerificationOptions */
|
|
21
22
|
|
|
22
23
|
/** @typedef {import("node:child_process").ChildProcess} ChildProcess */
|
|
@@ -228,7 +229,11 @@ function runCommand(command, baseCwd, commandCwd, attempt, signal, options, comm
|
|
|
228
229
|
};
|
|
229
230
|
options?.onAttemptStart?.({ ...identity });
|
|
230
231
|
const env = verificationEnv(command);
|
|
231
|
-
|
|
232
|
+
// A verification command names a binary the same way a harness runtime
|
|
233
|
+
// does, and on Windows `npm test` is `npm.cmd`: the invocation, not the
|
|
234
|
+
// raw argv, is what can actually be spawned there.
|
|
235
|
+
const invocation = spawnInvocation(command.argv[0], command.argv.slice(1), { cwd });
|
|
236
|
+
child = spawn(invocation.command, invocation.args, { cwd, env, detached: process.platform !== "win32", stdio: ["ignore", "pipe", "pipe"], ...invocation.options });
|
|
232
237
|
const pid = child.pid ?? null;
|
|
233
238
|
let paused = false;
|
|
234
239
|
if (process.platform !== "win32" && pid) {
|
|
@@ -268,8 +273,7 @@ function runCommand(command, baseCwd, commandCwd, attempt, signal, options, comm
|
|
|
268
273
|
*/
|
|
269
274
|
function terminateGroup(child) {
|
|
270
275
|
try {
|
|
271
|
-
|
|
272
|
-
else child.kill("SIGTERM");
|
|
276
|
+
killTarget(process.platform === "win32" ? /** @type {number} */ (child.pid) : -/** @type {number} */ (child.pid), "SIGTERM");
|
|
273
277
|
} catch {
|
|
274
278
|
try { child.kill("SIGTERM"); } catch {
|
|
275
279
|
// ESRCH: the group kill failed and the leader was already gone.
|
|
@@ -277,8 +281,7 @@ function terminateGroup(child) {
|
|
|
277
281
|
}
|
|
278
282
|
setTimeout(() => {
|
|
279
283
|
try {
|
|
280
|
-
|
|
281
|
-
else child.kill("SIGKILL");
|
|
284
|
+
killTarget(process.platform === "win32" ? /** @type {number} */ (child.pid) : -/** @type {number} */ (child.pid), "SIGKILL");
|
|
282
285
|
} catch {
|
|
283
286
|
try { child.kill("SIGKILL"); } catch {
|
|
284
287
|
// ESRCH: the SIGKILL fallback found no leader left to kill.
|
|
@@ -11,8 +11,9 @@
|
|
|
11
11
|
* version on its first call, and a missing version is indistinguishable from a
|
|
12
12
|
* changed one.
|
|
13
13
|
*/
|
|
14
|
-
import { CONTRACT_VERSION, PROTOCOL_SCHEMA_VERSION, probeRuntime } from "../harnesses/index.mjs";
|
|
14
|
+
import { CONTRACT_VERSION, PROTOCOL_SCHEMA_VERSION, getHarness, harnessCapabilities, probeRuntime } from "../harnesses/index.mjs";
|
|
15
15
|
import { appendJsonl, writeJsonAtomic } from "../run/store.mjs";
|
|
16
|
+
import { availabilityKey, readAvailability, recordAvailability } from "../run/availability.mjs";
|
|
16
17
|
import { blockingChecks, environmentPreflight, reachableRuntimes } from "../host/preflight.mjs";
|
|
17
18
|
import { captureSourceIdentity } from "../repo/source-identity.mjs";
|
|
18
19
|
import { boundedGitSync } from "../repo/worktree.mjs";
|
|
@@ -24,8 +25,12 @@ import { stableJson } from "../util.mjs";
|
|
|
24
25
|
import { validateRunMetadata } from "../contract/snapshot.mjs";
|
|
25
26
|
import { contractDigest } from "../contract/index.mjs";
|
|
26
27
|
import { RUNS_DIR_NAME, runDirectory } from "../run/paths.mjs";
|
|
28
|
+
import { preflightContract } from "./live-preflight.mjs";
|
|
29
|
+
import { liveSilenceCause } from "./live-silence.mjs";
|
|
27
30
|
|
|
28
31
|
/** @typedef {import("../harnesses/index.mjs").HarnessRuntime} HarnessRuntime */
|
|
32
|
+
/** @typedef {import("../harnesses/index.mjs").ProbeResult} ProbeResult */
|
|
33
|
+
/** @typedef {import("../engine/runtime-discovery.mjs").RuntimeAvailability} RuntimeAvailability */
|
|
29
34
|
/** @typedef {import("../cli.mjs").LockHandle} LockHandle */
|
|
30
35
|
/** @typedef {import("../contract/index.mjs").NodeSnapshot} NodeSnapshot */
|
|
31
36
|
/** @typedef {import("../contract/index.mjs").RunMetadata} RunMetadata */
|
|
@@ -182,6 +187,25 @@ export function setLaunchBaseRef(baseRef) {
|
|
|
182
187
|
pendingLaunchBaseRef = baseRef ?? null;
|
|
183
188
|
}
|
|
184
189
|
|
|
190
|
+
/**
|
|
191
|
+
* Whether this launch must ask every routed runtime again even where the
|
|
192
|
+
* verdict store holds a fresh answer (`--fresh-preflight`). A module rather
|
|
193
|
+
* than an option for the same reason `setLaunchBaseRef` is: the gate is
|
|
194
|
+
* reached through `runContract` and `resumeRun`, which thread no launch
|
|
195
|
+
* options of their own. A forced launch still records what it observes.
|
|
196
|
+
*
|
|
197
|
+
* @type {boolean}
|
|
198
|
+
*/
|
|
199
|
+
let pendingFreshPreflight = false;
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* @param {boolean} force
|
|
203
|
+
* @returns {void}
|
|
204
|
+
*/
|
|
205
|
+
export function setFreshPreflight(force) {
|
|
206
|
+
pendingFreshPreflight = force === true;
|
|
207
|
+
}
|
|
208
|
+
|
|
185
209
|
/**
|
|
186
210
|
* The base ref a run was launched against, when it was launched with
|
|
187
211
|
* `--base-ref`. A run recorded before this field existed, or launched
|
|
@@ -401,32 +425,207 @@ export function serializableContract(contract) {
|
|
|
401
425
|
};
|
|
402
426
|
}
|
|
403
427
|
/**
|
|
404
|
-
*
|
|
405
|
-
*
|
|
406
|
-
*
|
|
407
|
-
*
|
|
428
|
+
* Live failure codes that name the pipeline rather than the provider, so they
|
|
429
|
+
* are verdicts of nothing and are never recorded: the two silences above, and
|
|
430
|
+
* `command_invalid`, which never reached a provider at all. The recordable
|
|
431
|
+
* set is the complement of this one, not of `LIVE_SILENCE_CAUSES` -- a
|
|
432
|
+
* command that could not be constructed passes the gate (validation owns
|
|
433
|
+
* that defect) but learned nothing about availability, so there is no verdict
|
|
434
|
+
* to persist.
|
|
435
|
+
*/
|
|
436
|
+
const LIVE_NO_VERDICT_CAUSES = new Set(["preflight_timeout", "spawn_error", "command_invalid"]);
|
|
437
|
+
|
|
438
|
+
/**
|
|
439
|
+
* Whether an asked probe reached a provider and so carries a verdict worth
|
|
440
|
+
* recording. `done` reached it and completed; a failure whose detail names a
|
|
441
|
+
* live error code reached it too unless the code is one of the pipeline's own
|
|
442
|
+
* (see `LIVE_NO_VERDICT_CAUSES`). A refusal, an auth failure, unparsable
|
|
443
|
+
* output -- anything a provider itself produced -- is an answer.
|
|
444
|
+
*
|
|
445
|
+
* @param {ProbeResult} probe
|
|
446
|
+
* @returns {boolean}
|
|
447
|
+
*/
|
|
448
|
+
function liveVerdictRecorded(probe) {
|
|
449
|
+
if (probe.liveStatus === "done") return true;
|
|
450
|
+
const match = / · live \S+ · ([a-z_]+):/u.exec(probe.detail ?? "");
|
|
451
|
+
return match !== null && !LIVE_NO_VERDICT_CAUSES.has(match[1]);
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
/**
|
|
455
|
+
* The dispatch gate: no node starts until the host can carry the run and the
|
|
456
|
+
* runtimes it routes to have answered. The static half proves the host facts
|
|
457
|
+
* — disk, git, worktree, a versioned binary per routed runtime. The live half
|
|
458
|
+
* asks every routed runtime one trivial prompt through `preflightContract`,
|
|
459
|
+
* read-only in a throwaway repository, because a present, versioned binary
|
|
460
|
+
* can still hold a dead credential, a spent quota, or a model that no longer
|
|
461
|
+
* answers — and each of those fails a run minutes in, after a worktree and a
|
|
462
|
+
* campaign event already exist.
|
|
463
|
+
*
|
|
464
|
+
* Answered means answered, not healthy: any verdict a provider returns,
|
|
465
|
+
* a quota refusal included, counts as having answered, and the run proceeds
|
|
466
|
+
* onto whatever the contract declares. Only pipeline silence blocks, and a
|
|
467
|
+
* silent runtime is named with cause unknown — the verdict
|
|
468
|
+
* `normalizeProviderAvailability` reserves for a probe that named no cause.
|
|
469
|
+
*
|
|
470
|
+
* The report is written as run evidence either way, and a blocking failure
|
|
471
|
+
* leaves the materialized run untouched — the operator fixes the host and
|
|
472
|
+
* resumes, so a run is never silently restarted and already-paid nodes are
|
|
473
|
+
* not redone.
|
|
474
|
+
*
|
|
475
|
+
* A verdict is no longer single-launch property: before asking, the gate
|
|
476
|
+
* reads the verdict store under the operator's home (`run/availability.mjs`),
|
|
477
|
+
* and a provider answered inside its window is reused, the evidence naming
|
|
478
|
+
* the reuse and the instant the verdict was observed. Every ask that reached
|
|
479
|
+
* a provider is recorded there for the next launch. `--fresh-preflight`
|
|
480
|
+
* skips the read; it never skips the record.
|
|
408
481
|
*
|
|
409
482
|
* @param {ValidatedContract} contract
|
|
410
483
|
* @param {string} runDir
|
|
411
484
|
* @param {SourceIdentity|undefined} sourceIdentity
|
|
412
485
|
*/
|
|
413
|
-
export function assertEnvironmentReady(contract, runDir, sourceIdentity) {
|
|
486
|
+
export async function assertEnvironmentReady(contract, runDir, sourceIdentity) {
|
|
487
|
+
const at = new Date().toISOString();
|
|
414
488
|
const report = environmentPreflight({
|
|
415
489
|
cwd: contract.cwd,
|
|
416
490
|
runtimes: reachableRuntimes(contract),
|
|
417
491
|
harnessVersions: sourceIdentity?.harnessVersions ?? {},
|
|
418
492
|
});
|
|
419
|
-
|
|
493
|
+
/** @param {boolean} ok @param {import("../harnesses/index.mjs").ProbeResult[]} [probes] @returns {Record<string, unknown>} */
|
|
494
|
+
const evidence = (ok, probes) => ({
|
|
420
495
|
schemaVersion: report.schemaVersion,
|
|
421
496
|
contractVersion: CONTRACT_VERSION,
|
|
422
|
-
at
|
|
497
|
+
at,
|
|
423
498
|
contractId: contract.id,
|
|
424
|
-
ok
|
|
499
|
+
ok,
|
|
425
500
|
checks: report.checks,
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
501
|
+
...(probes === undefined ? {} : { runtimes: probes }),
|
|
502
|
+
});
|
|
503
|
+
writeJsonAtomic(join(runDir, "env-preflight.json"), evidence(report.ok));
|
|
504
|
+
let blocking = report.ok ? null : blockingChecks(report).map((check) => `${check.name}: ${check.detail}`).join(" · ");
|
|
505
|
+
// A launch with nothing left to dispatch asks nothing: when every persisted
|
|
506
|
+
// state reads done, this launch replays an accepted transaction, starts no
|
|
507
|
+
// worker and no judge, and can spend no availability.
|
|
508
|
+
if (blocking === null && launchMayDispatch(runDir)) {
|
|
509
|
+
// measured 2026-09-22: asking four routed runtimes in parallel took about
|
|
510
|
+
// 18s, so the default budget is 60s. FABERUN_PREFLIGHT_TIMEOUT_SEC stays
|
|
511
|
+
// the operator override; preflightContract validates it, so only a valid
|
|
512
|
+
// number is lifted here.
|
|
513
|
+
const override = Number(process.env.FABERUN_PREFLIGHT_TIMEOUT_SEC);
|
|
514
|
+
const timeoutSec = process.env.FABERUN_PREFLIGHT_TIMEOUT_SEC !== undefined && Number.isFinite(override) && override > 0 ? override : 60;
|
|
515
|
+
const probes = await livePreflightProbes(contract, runDir, sourceIdentity, timeoutSec);
|
|
516
|
+
const silent = probes.filter((probe) => liveSilenceCause(probe) !== null);
|
|
517
|
+
writeJsonAtomic(join(runDir, "env-preflight.json"), evidence(silent.length === 0, probes));
|
|
518
|
+
if (silent.length > 0) {
|
|
519
|
+
blocking = `no runtime answered the live preflight: ${silent.map((probe) => `${probe.id ?? probe.harness} (cause unknown)`).join(" · ")}`;
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
if (blocking === null) return;
|
|
523
|
+
// The blocking failure is durable run evidence. The event carries exactly
|
|
524
|
+
// these seven fields — the live ProbeResults stay in env-preflight.json and
|
|
525
|
+
// never enter the event stream — and the append stays in this function: the
|
|
526
|
+
// field-ownership document names assertEnvironmentReady as the writer.
|
|
527
|
+
appendJsonl(join(runDir, "events.jsonl"), {
|
|
528
|
+
type: "run.env-preflight-failed",
|
|
529
|
+
schemaVersion: report.schemaVersion,
|
|
530
|
+
contractVersion: CONTRACT_VERSION,
|
|
531
|
+
at,
|
|
532
|
+
contractId: contract.id,
|
|
533
|
+
ok: false,
|
|
534
|
+
checks: report.checks,
|
|
535
|
+
});
|
|
431
536
|
throw Object.assign(new Error(`env_preflight_failed: ${blocking} · the run stays resumable: fix the environment and resume ${runDir}`), { code: "env_preflight_failed" });
|
|
432
537
|
}
|
|
538
|
+
|
|
539
|
+
/**
|
|
540
|
+
* The live half of the gate for one launch. Every routed runtime either holds
|
|
541
|
+
* a verdict this machine recorded inside its freshness window -- reused, the
|
|
542
|
+
* evidence naming the instant it was observed -- or is asked now, and every
|
|
543
|
+
* ask that reached a provider is recorded for the next launch. Nothing here
|
|
544
|
+
* decides what an answer means: reuse changes only whether the provider is
|
|
545
|
+
* asked, never whether a pass is a pass.
|
|
546
|
+
*
|
|
547
|
+
* The read keys on what identifies the provider -- harness, model, the
|
|
548
|
+
* executable the harness adapter itself resolves -- the same resolution a
|
|
549
|
+
* probe reports, so a runtime re-labelled between contracts is still one
|
|
550
|
+
* provider and a provider pointing at another binary is a new question.
|
|
551
|
+
*
|
|
552
|
+
* @param {ValidatedContract} contract
|
|
553
|
+
* @param {string} runDir
|
|
554
|
+
* @param {SourceIdentity|undefined} sourceIdentity
|
|
555
|
+
* @param {number} timeoutSec
|
|
556
|
+
* @returns {Promise<ProbeResult[]>}
|
|
557
|
+
*/
|
|
558
|
+
async function livePreflightProbes(contract, runDir, sourceIdentity, timeoutSec) {
|
|
559
|
+
const routed = reachableRuntimes(contract);
|
|
560
|
+
/** @type {Map<string, RuntimeAvailability>} */
|
|
561
|
+
const fresh = new Map();
|
|
562
|
+
if (!pendingFreshPreflight) {
|
|
563
|
+
for (const [id, { runtime }] of routed) {
|
|
564
|
+
const verdict = readAvailability(availabilityKey({
|
|
565
|
+
harness: runtime.harness,
|
|
566
|
+
model: runtime.model,
|
|
567
|
+
executable: getHarness(runtime.harness).executable(runtime),
|
|
568
|
+
}));
|
|
569
|
+
if (verdict) fresh.set(id, verdict);
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
if (routed.size > 0 && fresh.size === routed.size) {
|
|
573
|
+
return [...routed.entries()].map(([id, { runtime }]) => {
|
|
574
|
+
const verdict = /** @type {RuntimeAvailability} */ (fresh.get(id));
|
|
575
|
+
return {
|
|
576
|
+
id,
|
|
577
|
+
harness: runtime.harness,
|
|
578
|
+
executable: getHarness(runtime.harness).executable(runtime),
|
|
579
|
+
model: runtime.model,
|
|
580
|
+
version: sourceIdentity?.harnessVersions?.[id] ?? null,
|
|
581
|
+
capabilities: harnessCapabilities(runtime),
|
|
582
|
+
requiredCapabilities: {},
|
|
583
|
+
requiredCapabilitySets: [],
|
|
584
|
+
ok: true,
|
|
585
|
+
live: true,
|
|
586
|
+
liveStatus: "reused",
|
|
587
|
+
detail: `live verdict reused · observed ${verdict.observedAt ?? "unknown instant"}`,
|
|
588
|
+
};
|
|
589
|
+
});
|
|
590
|
+
}
|
|
591
|
+
const probes = await preflightContract(join(runDir, "contract.json"), { liveTimeoutSec: timeoutSec, persisted: true });
|
|
592
|
+
// What this launch bought is durable from here on: every ask that reached a
|
|
593
|
+
// provider -- a refusal included, an answer being an answer -- is recorded
|
|
594
|
+
// under the provider's own identity. Silence and a command that never
|
|
595
|
+
// reached a provider are verdicts of nothing and are never recorded, so the
|
|
596
|
+
// operator who fixes the host is never told the fix "already answered".
|
|
597
|
+
recordAvailability(
|
|
598
|
+
probes
|
|
599
|
+
.filter((probe) => probe.live === true && probe.liveStatus !== "reused" && liveVerdictRecorded(probe))
|
|
600
|
+
.map((probe) => availabilityKey(probe)),
|
|
601
|
+
Date.now(),
|
|
602
|
+
);
|
|
603
|
+
return probes;
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
/**
|
|
607
|
+
* Whether this launch can dispatch anything. Read defensively: a missing
|
|
608
|
+
* nodes directory or an unparseable state means the launch may dispatch, so
|
|
609
|
+
* the runtimes are asked.
|
|
610
|
+
*
|
|
611
|
+
* @param {string} runDir
|
|
612
|
+
* @returns {boolean}
|
|
613
|
+
*/
|
|
614
|
+
function launchMayDispatch(runDir) {
|
|
615
|
+
let names;
|
|
616
|
+
try {
|
|
617
|
+
names = readdirSync(join(runDir, "nodes"));
|
|
618
|
+
} catch {
|
|
619
|
+
return true;
|
|
620
|
+
}
|
|
621
|
+
const states = names.filter((name) => name.endsWith(".json"));
|
|
622
|
+
if (states.length === 0) return true;
|
|
623
|
+
return states.some((name) => {
|
|
624
|
+
try {
|
|
625
|
+
return JSON.parse(readFileSync(join(runDir, "nodes", name), "utf8")).status !== "done";
|
|
626
|
+
} catch {
|
|
627
|
+
return true;
|
|
628
|
+
}
|
|
629
|
+
});
|
|
630
|
+
}
|
|
631
|
+
|
|
@@ -234,21 +234,42 @@ function tierOrder(runtime) {
|
|
|
234
234
|
}
|
|
235
235
|
|
|
236
236
|
/**
|
|
237
|
-
*
|
|
238
|
-
*
|
|
239
|
-
*
|
|
237
|
+
* How long a recorded live-preflight verdict stays fresh, in seconds. This is
|
|
238
|
+
* the hello's own clock and is never derived from the quota windows below: a
|
|
239
|
+
* spend allowance expires when the provider resets it, a hello expires
|
|
240
|
+
* because whatever it proved -- a working credential, a spawning binary, a
|
|
241
|
+
* model that answers -- has stopped holding. Measured 2026-09-22: the ask
|
|
242
|
+
* costs about 18s for four runtimes in parallel, so every launch that reuses
|
|
243
|
+
* instead of asking saves about that. The window bets that a provider which
|
|
244
|
+
* answered still answers for the next quarter hour -- long enough to cover a
|
|
245
|
+
* burst of relaunches and retries, short enough that whatever died in
|
|
246
|
+
* between is bought again within fifteen minutes.
|
|
247
|
+
*/
|
|
248
|
+
const PREFLIGHT_FRESH_SEC = 15 * 60;
|
|
249
|
+
|
|
250
|
+
/** The window label a persisted live-preflight verdict carries. */
|
|
251
|
+
export const PREFLIGHT_WINDOW = "preflight";
|
|
252
|
+
|
|
253
|
+
/**
|
|
254
|
+
* Span in seconds of every window label a catalogue or stored record may
|
|
255
|
+
* carry. `five_hour` and `seven_day` are claude's rate-limit labels (measured
|
|
256
|
+
* 2026-09-17, the `rate_limit_event` line recorded in
|
|
257
|
+
* `src/harnesses/protocol.mjs`); `preflight` is the hello's own clock, whose
|
|
258
|
+
* duration and reasoning live in `PREFLIGHT_FRESH_SEC` above -- the two kinds
|
|
259
|
+
* of window expire for different reasons and are never one clock. A label
|
|
240
260
|
* missing here cannot prove staleness, so its observation never self-expires.
|
|
241
261
|
*
|
|
242
262
|
* @type {Readonly<Record<string, number>>}
|
|
243
263
|
*/
|
|
244
|
-
const AVAILABILITY_WINDOW_SEC = Object.freeze({ five_hour: 5 * 3600, seven_day: 7 * 86400 });
|
|
264
|
+
const AVAILABILITY_WINDOW_SEC = Object.freeze({ five_hour: 5 * 3600, seven_day: 7 * 86400, [PREFLIGHT_WINDOW]: PREFLIGHT_FRESH_SEC });
|
|
245
265
|
|
|
246
266
|
/**
|
|
247
267
|
* May a runtime be admitted on this catalogue record? Exhaustion is waited
|
|
248
268
|
* out on `exhaustedUntil`; an observation older than its own window reads as
|
|
249
269
|
* unknown and admits nothing, because unknown must not look rested. This is
|
|
250
270
|
* the one home of the rule: plan routing and engine composition both read it,
|
|
251
|
-
*
|
|
271
|
+
* as does the verdict store's reuse decision in `run/availability.mjs`, so
|
|
272
|
+
* the null and staleness semantics cannot drift between readers. The
|
|
252
273
|
* parameter is typed on the fields the rule reads, not on the full record --
|
|
253
274
|
* the plan's table copy names no `reason`.
|
|
254
275
|
*
|
package/src/engine/scheduler.mjs
CHANGED
|
@@ -315,7 +315,7 @@ export async function runContract(contractPath, options = {}) {
|
|
|
315
315
|
*/
|
|
316
316
|
export async function driveRun(contract, runDir, states, campaign, lock, sourceIdentity, resume = {}, options = {}) {
|
|
317
317
|
lock.assert();
|
|
318
|
-
assertEnvironmentReady(contract, runDir, sourceIdentity);
|
|
318
|
+
await assertEnvironmentReady(contract, runDir, sourceIdentity);
|
|
319
319
|
const runsDir = runsRoot(contract.cwd);
|
|
320
320
|
const bootstrapNonce = bootstrapNonceForProcess();
|
|
321
321
|
// Only the CLI entry can answer this: a nonce inherited by evals/run.mjs or
|
package/src/engine/supervise.mjs
CHANGED
|
@@ -42,6 +42,7 @@ import { readJson, writeJsonAtomic } from "../run/store.mjs";
|
|
|
42
42
|
import { delay, errorCode, errorMessage } from "../util.mjs";
|
|
43
43
|
import { loadPersistedContract } from "../contract/index.mjs";
|
|
44
44
|
import { emitScheduledAttention } from "./notify-queue.mjs";
|
|
45
|
+
import { killTarget } from "../host/platform.mjs";
|
|
45
46
|
|
|
46
47
|
/** What `--interval` defaults to, in seconds: often enough that a dead controller costs a minute of wall clock, rare enough to be free. */
|
|
47
48
|
export const DEFAULT_SUPERVISE_INTERVAL_SEC = 30;
|
|
@@ -532,7 +533,7 @@ export async function terminateControllerGroup(runDir, options = {}) {
|
|
|
532
533
|
if (errorCode(groupError) !== "ESRCH" && errorCode(groupError) !== "EPERM") throw groupError;
|
|
533
534
|
}
|
|
534
535
|
}
|
|
535
|
-
|
|
536
|
+
killTarget(target, signal);
|
|
536
537
|
} catch (error) {
|
|
537
538
|
if (errorCode(error) !== "ESRCH" && errorCode(error) !== "EPERM") throw error;
|
|
538
539
|
}
|
|
@@ -2,6 +2,7 @@ import { spawnSync } from "node:child_process";
|
|
|
2
2
|
import { getHarness, probeRuntime, registeredHarnesses, resolveVendor } from "./index.mjs";
|
|
3
3
|
import { DISCOVERY_RUNTIME_DEFINITIONS, composeAssignments } from "../engine/runtime-discovery.mjs";
|
|
4
4
|
import { errorMessage } from "../util.mjs";
|
|
5
|
+
import { spawnInvocation } from "../host/platform.mjs";
|
|
5
6
|
|
|
6
7
|
/**
|
|
7
8
|
* Model catalogue report: which models each registered harness can run, the
|
|
@@ -292,8 +293,14 @@ function displayOrder() {
|
|
|
292
293
|
*/
|
|
293
294
|
function agyCliCatalogue(cwd) {
|
|
294
295
|
const executable = getHarness("agy").executable({ harness: "agy", model: "agy-models" });
|
|
295
|
-
|
|
296
|
+
// The provider CLI is reached the way every other one here is: a name
|
|
297
|
+
// through PATHEXT, a `.cmd` through the interpreter, a script through the
|
|
298
|
+
// interpreter its shebang names. A raw spawn of it answers ENOENT on
|
|
299
|
+
// Windows and the declared catalogue silently wins.
|
|
300
|
+
const invocation = spawnInvocation(executable, ["models"], { cwd });
|
|
301
|
+
const result = spawnSync(invocation.command, invocation.args, {
|
|
296
302
|
cwd,
|
|
303
|
+
...invocation.options,
|
|
297
304
|
encoding: "utf8",
|
|
298
305
|
stdio: ["ignore", "pipe", "ignore"],
|
|
299
306
|
timeout: AGY_CATALOGUE_TIMEOUT_MS,
|
|
@@ -23,6 +23,7 @@
|
|
|
23
23
|
*/
|
|
24
24
|
|
|
25
25
|
import { spawn } from "node:child_process";
|
|
26
|
+
import { spawnInvocation } from "../../host/platform.mjs";
|
|
26
27
|
import { writeSync } from "node:fs";
|
|
27
28
|
|
|
28
29
|
const HANDSHAKE_ID = 1;
|
|
@@ -145,7 +146,11 @@ if (options.sandbox) env.DSH_PERMISSION_MODE = options.sandbox;
|
|
|
145
146
|
|
|
146
147
|
const args = ["--profile", "sdk"];
|
|
147
148
|
for (const patch of options.patches) args.push("--patch", patch);
|
|
148
|
-
|
|
149
|
+
// The harness binary, reached the way this platform reaches one: a name
|
|
150
|
+
// through PATHEXT, a `.cmd` through the command interpreter, a POSIX script
|
|
151
|
+
// through the interpreter its shebang names.
|
|
152
|
+
const invocation = spawnInvocation(options.dsh, args);
|
|
153
|
+
const child = spawn(invocation.command, invocation.args, { cwd: process.cwd(), env, stdio: ["pipe", "pipe", "pipe"], ...invocation.options });
|
|
149
154
|
|
|
150
155
|
const usage = { inputTokens: 0, outputTokens: 0, cacheReadInputTokens: 0 };
|
|
151
156
|
let sawUsage = false;
|
package/src/harnesses/index.mjs
CHANGED
|
@@ -7,6 +7,7 @@ import { zcodeHarness } from "./zcode/index.mjs";
|
|
|
7
7
|
import { execJsonlHarness } from "./exec-jsonl/index.mjs";
|
|
8
8
|
import { replayHarness } from "./replay/index.mjs";
|
|
9
9
|
import { withoutNotifyEnv } from "../notify/index.mjs";
|
|
10
|
+
import { spawnInvocation } from "../host/platform.mjs";
|
|
10
11
|
|
|
11
12
|
/** Current wire-contract version for runner protocol artifacts. */
|
|
12
13
|
export const PROTOCOL_SCHEMA_VERSION = 3;
|
|
@@ -231,7 +232,9 @@ export function normalizeProviderResult(runtimeOrHarness, stdout, exitCode, sign
|
|
|
231
232
|
|
|
232
233
|
/**
|
|
233
234
|
* Normalize a provider envelope or recorded response into the availability
|
|
234
|
-
* shape used by doctor and runtime assignment.
|
|
235
|
+
* shape used by doctor and runtime assignment. The reason names the cause the
|
|
236
|
+
* provider actually gave; a probe that timed out naming nothing reports
|
|
237
|
+
* `unknown`, never a guessed cause.
|
|
235
238
|
*
|
|
236
239
|
* @param {string|{harness: string}} runtimeOrHarness
|
|
237
240
|
* @param {unknown} response
|
|
@@ -261,7 +264,12 @@ export function normalizeProviderAvailability(runtimeOrHarness, response, exitCo
|
|
|
261
264
|
if (envelope.status === "exhausted" || classified?.reason === "quota_exhausted") {
|
|
262
265
|
return { available: false, exhaustedUntil: exhaustedUntilOf(envelope), reason: code || "quota_exhausted" };
|
|
263
266
|
}
|
|
264
|
-
if (classified
|
|
267
|
+
if (classified) return classified;
|
|
268
|
+
// A probe that timed out names no cause at all, so any of the four would be
|
|
269
|
+
// invented from a slow answer. measured 2026-09-22: agy reported
|
|
270
|
+
// preflight_timeout after 15s and the real cause -- no configuration
|
|
271
|
+
// directory -- was visible only when a human looked at the filesystem.
|
|
272
|
+
if (/timeout|timed.?out/iu.test(text)) return { available: false, exhaustedUntil: null, reason: "unknown" };
|
|
265
273
|
return { available: false, exhaustedUntil: null, reason: code || "provider_unavailable" };
|
|
266
274
|
}
|
|
267
275
|
|
|
@@ -288,9 +296,10 @@ export function exhaustedUntilOf(envelope, now = Date.now()) {
|
|
|
288
296
|
/**
|
|
289
297
|
* Classify raw provider-produced text (a structured envelope's `error.code
|
|
290
298
|
* error.message`, or a probe's raw stderr on a non-zero exit) into the same
|
|
291
|
-
* insufficient-balance/quota/
|
|
292
|
-
* recognizes. Shared so a CLI-missing exit
|
|
293
|
-
* message are classified by one set of
|
|
299
|
+
* insufficient-balance/quota/credential/model reasons
|
|
300
|
+
* `normalizeProviderAvailability` recognizes. Shared so a CLI-missing exit
|
|
301
|
+
* and a raw stderr balance/quota message are classified by one set of
|
|
302
|
+
* patterns, never two drifting copies.
|
|
294
303
|
*
|
|
295
304
|
* @param {string} text
|
|
296
305
|
* @returns {{available: false, exhaustedUntil: string|null, reason: string}|null} null when text names none of the known patterns
|
|
@@ -302,9 +311,22 @@ function classifyAvailabilityText(text) {
|
|
|
302
311
|
if (/quota|rate.?limit|usage limit|session limit|limit exhausted|1310/iu.test(text)) {
|
|
303
312
|
return { available: false, exhaustedUntil: resetTimestamp(text), reason: "quota_exhausted" };
|
|
304
313
|
}
|
|
314
|
+
// measured 2026-09-22, a Codex account asked for a model it may not use:
|
|
315
|
+
// "The <model> model is not supported when using Codex with a ChatGPT
|
|
316
|
+
// account". The remedy is declaring another model, not a wait or a retry.
|
|
317
|
+
if (/model .*not supported|unsupported model/iu.test(text)) {
|
|
318
|
+
return { available: false, exhaustedUntil: null, reason: "model_not_supported" };
|
|
319
|
+
}
|
|
305
320
|
if (/auth|credential|unauthori[sz]ed|forbidden|invalid.*(?:key|token)|(?:api|access) key|login/iu.test(text)) {
|
|
306
321
|
return { available: false, exhaustedUntil: null, reason: "authentication_failed" };
|
|
307
322
|
}
|
|
323
|
+
// A harness whose configuration directory does not exist has no credential
|
|
324
|
+
// to even fail with (measured 2026-09-22: agy), so this sits last: only text
|
|
325
|
+
// no earlier pattern named reaches it, and no classification that used to
|
|
326
|
+
// land elsewhere changes.
|
|
327
|
+
if (/config(?:uration)? (?:directory|dir)\b/iu.test(text)) {
|
|
328
|
+
return { available: false, exhaustedUntil: null, reason: "credentials_missing" };
|
|
329
|
+
}
|
|
308
330
|
return null;
|
|
309
331
|
}
|
|
310
332
|
|
|
@@ -450,13 +472,15 @@ export function probeRuntime(runtime, options = {}) {
|
|
|
450
472
|
return new Promise((settle) => {
|
|
451
473
|
let child;
|
|
452
474
|
try {
|
|
453
|
-
|
|
475
|
+
const invocation = spawnInvocation(executable, args, { cwd: options.cwd });
|
|
476
|
+
child = spawn(invocation.command, invocation.args, {
|
|
454
477
|
cwd: options.cwd,
|
|
455
478
|
// A worker or judge never delivers a notification; the controller does.
|
|
456
479
|
// In this repository a worker runs the test suite, whose fixture
|
|
457
480
|
// controllers would otherwise inherit a live transport and deliver.
|
|
458
481
|
env: withoutNotifyEnv(process.env),
|
|
459
482
|
stdio: ["ignore", "pipe", "pipe"],
|
|
483
|
+
...invocation.options,
|
|
460
484
|
});
|
|
461
485
|
} catch (error) {
|
|
462
486
|
settle({
|
|
@@ -485,7 +509,11 @@ export function probeRuntime(runtime, options = {}) {
|
|
|
485
509
|
child.kill("SIGTERM");
|
|
486
510
|
finish({
|
|
487
511
|
...base,
|
|
488
|
-
|
|
512
|
+
// A probe that ran out of time said nothing about why: `unknown` is
|
|
513
|
+
// the honest verdict, not one of the four causes. measured 2026-09-22:
|
|
514
|
+
// agy timed out after 15s while the real cause -- no configuration
|
|
515
|
+
// directory -- was visible only on the filesystem.
|
|
516
|
+
availability: { available: false, exhaustedUntil: null, reason: "unknown" },
|
|
489
517
|
detail: `${identity(null)} · ${[missingEnvironmentDetail, `no response within ${timeoutSec}s`].filter(Boolean).join(" · ")}`,
|
|
490
518
|
});
|
|
491
519
|
}, timeoutSec * 1_000);
|