@staix/agent-hub 0.12.20 → 0.12.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/README.md +17 -9
- package/docs/agent-notes/adapters.md +13 -3
- package/docs/agent-notes/benchmarks.md +1 -1
- package/docs/agent-notes/bus.md +7 -3
- package/docs/agent-notes/daemon.md +12 -2
- package/docs/agent-notes/local-worker.md +5 -1
- package/docs/agent-notes/tasks.md +4 -2
- package/docs/agent-notes/tests.md +5 -2
- package/docs/bench.md +143 -0
- package/docs/events.md +15 -5
- package/docs/operations.md +510 -54
- package/docs/quickstart.md +16 -11
- package/docs/research.md +92 -0
- package/docs/security.md +49 -11
- package/docs/smoke.md +138 -0
- package/docs/specs/2026-09-19-agent-hub-design.md +227 -20
- package/docs/specs/2026-09-20-upgrade-recovery-design.md +152 -18
- package/docs/verified.json +34 -17
- package/package.json +1 -1
- package/plugins/agent-hub/.claude-plugin/plugin.json +1 -1
- package/plugins/agent-hub/server.js +3 -2
- package/src/adapters/acp.ts +77 -10
- package/src/adapters/codex-appserver.ts +72 -2
- package/src/adapters/local-worker.ts +46 -11
- package/src/adapters/pi.ts +215 -22
- package/src/cli/bench.ts +287 -0
- package/src/cli/console-state.ts +226 -28
- package/src/cli/console.ts +16 -4
- package/src/cli/facts-hook.ts +50 -8
- package/src/cli/help.ts +16 -5
- package/src/cli/identity.ts +10 -4
- package/src/cli/launch.ts +272 -18
- package/src/cli/main.ts +290 -30
- package/src/cli/preview.ts +13 -3
- package/src/cli/recovery-package.ts +17 -0
- package/src/cli/status-lines.ts +2 -1
- package/src/cli/terminal-recovery.ts +83 -31
- package/src/cli/upgrade-interactive.ts +297 -0
- package/src/cli/upgrade-runtime.ts +123 -20
- package/src/cli/upgrade.ts +108 -19
- package/src/hub/bench.ts +211 -0
- package/src/hub/bus.ts +24 -7
- package/src/hub/conductor.ts +22 -21
- package/src/hub/config-trust.ts +1 -0
- package/src/hub/control-client.ts +4 -1
- package/src/hub/daemon.ts +477 -86
- package/src/hub/delivery-journal.ts +6 -6
- package/src/hub/events.ts +4 -2
- package/src/hub/lifecycle.ts +8 -13
- package/src/hub/manager.ts +16 -14
- package/src/hub/peers.ts +2 -2
- package/src/hub/permission-mode.ts +44 -0
- package/src/hub/recovery-store.ts +59 -9
- package/src/hub/registry.ts +36 -22
- package/src/hub/research.ts +255 -0
- package/src/hub/tasks.ts +99 -55
- package/src/hub/tool-refusal.ts +7 -0
- package/src/hub/ui.ts +4 -1
- package/src/local/deny.ts +48 -1
- package/src/local/sandbox.ts +6 -3
- package/src/local/tools.ts +34 -11
- package/src/pi/extension.ts +43 -20
- package/src/pi/process-signature.ts +23 -0
- package/src/pi/tool-receipts.ts +24 -2
- package/src/ui/index.html +98 -17
- package/src/ui/task-progress.ts +33 -0
package/src/hub/bench.ts
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { appendFileSync, chmodSync, existsSync, mkdirSync, readdirSync, readFileSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { hubHome } from "./project.ts";
|
|
5
|
+
import { processLiveness } from "../pi/process-signature.ts";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* #251: benchmark suites, runs and their comparison. A run's records hold ids, outcomes, exit codes, counts, tokens
|
|
9
|
+
* and times, never the suite's text, a command's output or anything an agent wrote.
|
|
10
|
+
*/
|
|
11
|
+
export const BENCH_SCHEMA = "agent-hub.bench/v1";
|
|
12
|
+
|
|
13
|
+
export interface SuiteTask { id: string; title: string; detail?: string; class?: string; owner?: string; ref: string; setup?: string; verify: string; timeout_s: number }
|
|
14
|
+
export interface Suite { name: string; tasks: SuiteTask[] }
|
|
15
|
+
|
|
16
|
+
const TASK_FIELDS = new Set(["id", "title", "detail", "class", "owner", "ref", "setup", "verify", "timeout_s"]);
|
|
17
|
+
const CLASSES = new Set(["plan", "implement", "bulk_edit", "test", "review", "summarize", "triage"]);
|
|
18
|
+
/** A suite file, checked field by field; every refusal names the task and the field. */
|
|
19
|
+
export function parseSuite(text: string): Suite {
|
|
20
|
+
let raw: unknown;
|
|
21
|
+
try { raw = JSON.parse(text); } catch { throw new Error("suite: not valid JSON"); }
|
|
22
|
+
const suite = raw as Record<string, unknown>;
|
|
23
|
+
if (!suite || typeof suite !== "object" || Array.isArray(suite)) throw new Error("suite: expected an object with name and tasks");
|
|
24
|
+
for (const key of Object.keys(suite)) if (key !== "name" && key !== "tasks") throw new Error(`suite: unknown field ${key}`);
|
|
25
|
+
if (typeof suite.name !== "string" || !suite.name.trim()) throw new Error("suite: name is required");
|
|
26
|
+
if (!Array.isArray(suite.tasks) || !suite.tasks.length) throw new Error("suite: tasks must be a non-empty list");
|
|
27
|
+
const ids = new Set<string>();
|
|
28
|
+
const tasks = suite.tasks.map((item, i): SuiteTask => {
|
|
29
|
+
const t = item as Record<string, unknown>;
|
|
30
|
+
const where = `suite task ${typeof t?.id === "string" ? t.id : `#${i + 1}`}`;
|
|
31
|
+
if (!t || typeof t !== "object" || Array.isArray(t)) throw new Error(`${where}: expected an object`);
|
|
32
|
+
for (const key of Object.keys(t)) if (!TASK_FIELDS.has(key)) throw new Error(`${where}: unknown field ${key}`);
|
|
33
|
+
for (const key of ["id", "title", "ref", "verify"]) if (typeof t[key] !== "string" || !(t[key] as string).trim()) throw new Error(`${where}: ${key} is required`);
|
|
34
|
+
for (const key of ["detail", "class", "owner", "setup"]) if (t[key] !== undefined && typeof t[key] !== "string") throw new Error(`${where}: ${key} must be text`);
|
|
35
|
+
if (!/^[A-Za-z0-9._-]{1,64}$/.test(t.id as string)) throw new Error(`${where}: id may hold letters, digits, dot, dash and underscore`);
|
|
36
|
+
if (ids.has(t.id as string)) throw new Error(`${where}: duplicate id`);
|
|
37
|
+
ids.add(t.id as string);
|
|
38
|
+
if (t.class !== undefined && !CLASSES.has(t.class as string)) throw new Error(`${where}: class must be one of ${[...CLASSES].join(", ")}`);
|
|
39
|
+
if (typeof t.timeout_s !== "number" || !Number.isFinite(t.timeout_s) || t.timeout_s < 10 || t.timeout_s > 86_400) throw new Error(`${where}: timeout_s must be a number of seconds from 10 to 86400`);
|
|
40
|
+
return t as unknown as SuiteTask;
|
|
41
|
+
});
|
|
42
|
+
return { name: suite.name, tasks };
|
|
43
|
+
}
|
|
44
|
+
export const suiteHash = (text: string): string => createHash("sha256").update(text).digest("hex");
|
|
45
|
+
|
|
46
|
+
export interface Fingerprint { peers: { id: string; model?: string; permissionMode?: string }[]; routingHash: string | null; hubVersion: string }
|
|
47
|
+
export interface RunHeader {
|
|
48
|
+
schema: typeof BENCH_SCHEMA; kind: "run"; run: string; suite: string; suiteHash: string; arm: string; fingerprint: Fingerprint;
|
|
49
|
+
version: string; startedAt: string; tasks: number; repeats: number; runner: { pid: number; signature: string | null };
|
|
50
|
+
/** Task ids in run order, so a running run can name the attempt in progress. */
|
|
51
|
+
order: string[];
|
|
52
|
+
}
|
|
53
|
+
export type Outcome = "pass" | "fail" | "timeout" | "error";
|
|
54
|
+
/** Closed list: a reason never quotes a command or its output. */
|
|
55
|
+
export type AttemptError = "setup failed" | "setup timeout" | "propose refused" | "hub stopped" | "verify timeout" | "reset failed" | "interrupted";
|
|
56
|
+
export interface AttemptMetrics {
|
|
57
|
+
tokens: number; wallMs: number | null; activeMs: number; reviewRounds: number; changesRequested: number; checkFailed: number;
|
|
58
|
+
firstPass: boolean; turns: number; filesChanged: number; models: string[];
|
|
59
|
+
}
|
|
60
|
+
export interface Attempt {
|
|
61
|
+
schema: typeof BENCH_SCHEMA; kind: "attempt"; run: string; task: string; repeat: number; outcome: Outcome; error?: AttemptError;
|
|
62
|
+
verifyExit: number | null; hubTask: number | null; startedAt: string; endedAt: string; ms: number; metrics: AttemptMetrics | null;
|
|
63
|
+
}
|
|
64
|
+
export interface RunEnd { schema: typeof BENCH_SCHEMA; kind: "end"; run: string; endedAt: string; stopped?: "timeout" | "interrupted" | "error" }
|
|
65
|
+
export type BenchRecord = RunHeader | Attempt | RunEnd;
|
|
66
|
+
export interface BenchRun { header: RunHeader; attempts: Attempt[]; end?: RunEnd; state: "running" | "finished" | "stopped" | "interrupted" | "unknown" }
|
|
67
|
+
|
|
68
|
+
export const benchDir = (home = hubHome()): string => join(home, "bench");
|
|
69
|
+
export const runFile = (run: string, home = hubHome()): string => join(benchDir(home), `${run}.jsonl`);
|
|
70
|
+
export function appendBench(record: BenchRecord, home = hubHome()): void {
|
|
71
|
+
mkdirSync(benchDir(home), { recursive: true, mode: 0o700 });
|
|
72
|
+
chmodSync(benchDir(home), 0o700); // mkdir's mode applies only when it creates the directory
|
|
73
|
+
const file = runFile(record.run, home);
|
|
74
|
+
appendFileSync(file, JSON.stringify(record) + "\n", { mode: 0o600 });
|
|
75
|
+
chmodSync(file, 0o600);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** Every run on this machine, newest first. A run with no end record is running while its runner lives. */
|
|
79
|
+
export function readRuns(home = hubHome()): BenchRun[] {
|
|
80
|
+
const dir = benchDir(home);
|
|
81
|
+
if (!existsSync(dir)) return [];
|
|
82
|
+
const runs: BenchRun[] = [];
|
|
83
|
+
for (const name of readdirSync(dir).filter((n) => n.endsWith(".jsonl"))) try {
|
|
84
|
+
let header: RunHeader | undefined, end: RunEnd | undefined;
|
|
85
|
+
const attempts: Attempt[] = [];
|
|
86
|
+
for (const line of readFileSync(join(dir, name), "utf8").split("\n")) {
|
|
87
|
+
if (!line.trim()) continue;
|
|
88
|
+
let r: BenchRecord;
|
|
89
|
+
try { r = JSON.parse(line); } catch { continue; }
|
|
90
|
+
if (r?.schema !== BENCH_SCHEMA) continue;
|
|
91
|
+
if (r.kind === "run") header = r; else if (r.kind === "attempt") attempts.push(r); else if (r.kind === "end") end = r;
|
|
92
|
+
}
|
|
93
|
+
if (!header || typeof header.startedAt !== "string" || !header.runner) continue;
|
|
94
|
+
const owner = end ? undefined : processLiveness(header.runner.pid, header.runner.signature);
|
|
95
|
+
const state = end ? (end.stopped ? "stopped" : "finished") : owner === "live" ? "running" : owner === "gone" ? "interrupted" : "unknown";
|
|
96
|
+
runs.push({ header, attempts, ...(end ? { end } : {}), state });
|
|
97
|
+
} catch { /* a file that cannot be read or has no valid header is not a run; the others still are */ }
|
|
98
|
+
return runs.sort((a, b) => b.header.startedAt.localeCompare(a.header.startedAt));
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
const median = (values: number[]): number | null => {
|
|
102
|
+
if (!values.length) return null;
|
|
103
|
+
const s = [...values].sort((a, b) => a - b), m = Math.floor(s.length / 2);
|
|
104
|
+
return s.length % 2 ? s[m]! : (s[m - 1]! + s[m]!) / 2;
|
|
105
|
+
};
|
|
106
|
+
export interface Measures {
|
|
107
|
+
attempts: number; pass: number; fail: number; timeout: number; error: number;
|
|
108
|
+
/** What the measures are computed over: attempts that did not end in an error; attempts that have #247 measures. */
|
|
109
|
+
scored: number; measured: number;
|
|
110
|
+
passRate: number | null; firstPassRate: number | null; tokensMedian: number | null; wallMsMedian: number | null; reworkMean: number | null;
|
|
111
|
+
}
|
|
112
|
+
export function measures(attempts: Attempt[]): Measures {
|
|
113
|
+
const count = (o: Outcome) => attempts.filter((a) => a.outcome === o).length;
|
|
114
|
+
const scored = attempts.filter((a) => a.outcome !== "error");
|
|
115
|
+
const withMetrics = attempts.flatMap((a) => a.metrics ? [a.metrics] : []);
|
|
116
|
+
return { attempts: attempts.length, pass: count("pass"), fail: count("fail"), timeout: count("timeout"), error: count("error"), scored: scored.length, measured: withMetrics.length,
|
|
117
|
+
passRate: scored.length ? count("pass") / scored.length : null,
|
|
118
|
+
firstPassRate: withMetrics.length ? withMetrics.filter((m) => m.firstPass).length / withMetrics.length : null,
|
|
119
|
+
tokensMedian: median(withMetrics.map((m) => m.tokens)), wallMsMedian: median(scored.map((a) => a.ms)),
|
|
120
|
+
reworkMean: withMetrics.length ? withMetrics.reduce((n, m) => n + m.changesRequested, 0) / withMetrics.length : null };
|
|
121
|
+
}
|
|
122
|
+
export interface RunSummary extends Measures { run: string; suite: string; arm: string; state: BenchRun["state"]; startedAt: string; done: number; total: number; current?: string }
|
|
123
|
+
export function runSummary(r: BenchRun): RunSummary {
|
|
124
|
+
const total = r.header.tasks * r.header.repeats;
|
|
125
|
+
const current = r.state === "running" && r.attempts.length < total ? r.header.order?.[r.attempts.length % r.header.order.length] : undefined;
|
|
126
|
+
return { run: r.header.run, suite: r.header.suite, arm: r.header.arm, state: r.state, startedAt: r.header.startedAt, done: r.attempts.length, total, ...(current ? { current } : {}), ...measures(r.attempts) };
|
|
127
|
+
}
|
|
128
|
+
/** Per task and overall for one run (or several runs of one arm). */
|
|
129
|
+
export function benchReport(runs: BenchRun[]): { overall: Measures; tasks: Record<string, Measures> } {
|
|
130
|
+
const attempts = runs.flatMap((r) => r.attempts);
|
|
131
|
+
// A Map: a task id may be `constructor` or `__proto__`, which a plain object answers with an inherited value.
|
|
132
|
+
const tasks = new Map<string, Attempt[]>();
|
|
133
|
+
for (const a of attempts) { const id = String(a.task); tasks.set(id, [...(tasks.get(id) ?? []), a]); } // a store line is not trusted to hold a string
|
|
134
|
+
return { overall: measures(attempts), tasks: Object.fromEntries([...tasks].sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => [k, measures(v)])) };
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** A seeded generator, so a comparison prints the same interval every time for the same records. */
|
|
138
|
+
function mulberry32(seed: number): () => number {
|
|
139
|
+
let a = seed >>> 0;
|
|
140
|
+
return () => { a = (a + 0x6d2b79f5) >>> 0; let t = a; t = Math.imul(t ^ (t >>> 15), t | 1); t ^= t + Math.imul(t ^ (t >>> 7), t | 61); return ((t ^ (t >>> 14)) >>> 0) / 4294967296; };
|
|
141
|
+
}
|
|
142
|
+
export const MIN_ATTEMPTS = 5;
|
|
143
|
+
type MeasureName = "passRate" | "firstPassRate" | "tokensMedian" | "wallMsMedian";
|
|
144
|
+
export interface Comparison {
|
|
145
|
+
arms: { arm: string; runs: number; measures: Measures }[];
|
|
146
|
+
/** Each other arm against the first: the difference and its 95% bootstrap interval; inconclusive below MIN_ATTEMPTS. */
|
|
147
|
+
differences: { arm: string; measure: MeasureName; diff: number | null; low: number | null; high: number | null; inconclusive: boolean }[];
|
|
148
|
+
}
|
|
149
|
+
export function benchCompare(groups: { arm: string; runs: BenchRun[] }[], resamples = 1000, seed = 251): Comparison {
|
|
150
|
+
const arms = groups.map((g) => ({ arm: g.arm, runs: g.runs.length, attempts: g.runs.flatMap((r) => r.attempts) }));
|
|
151
|
+
const base = arms[0];
|
|
152
|
+
const differences: Comparison["differences"] = [];
|
|
153
|
+
const names: MeasureName[] = ["passRate", "firstPassRate", "tokensMedian", "wallMsMedian"];
|
|
154
|
+
if (base) for (const other of arms.slice(1)) {
|
|
155
|
+
const random = mulberry32(seed);
|
|
156
|
+
const pick = (xs: Attempt[]) => Array.from({ length: xs.length }, () => xs[Math.floor(random() * xs.length)]!);
|
|
157
|
+
const samples: Record<MeasureName, number[]> = { passRate: [], firstPassRate: [], tokensMedian: [], wallMsMedian: [] };
|
|
158
|
+
if (base.attempts.length && other.attempts.length) for (let i = 0; i < resamples; i++) {
|
|
159
|
+
const a = measures(pick(base.attempts)), b = measures(pick(other.attempts));
|
|
160
|
+
for (const n of names) if (a[n] !== null && b[n] !== null) samples[n].push(b[n]! - a[n]!);
|
|
161
|
+
}
|
|
162
|
+
const ma = measures(base.attempts), mb = measures(other.attempts);
|
|
163
|
+
// What each measure is computed over: pass rate and wall time over attempts that did not end in an error, the
|
|
164
|
+
// #247 measures over attempts that have them. Below MIN_ATTEMPTS of those in either arm a difference is inconclusive.
|
|
165
|
+
const counted = (xs: Attempt[], n: MeasureName) => n === "passRate" || n === "wallMsMedian" ? xs.filter((a) => a.outcome !== "error").length : xs.filter((a) => a.metrics).length;
|
|
166
|
+
for (const n of names) {
|
|
167
|
+
const s = samples[n].sort((x, y) => x - y);
|
|
168
|
+
const diff = ma[n] !== null && mb[n] !== null ? mb[n]! - ma[n]! : null;
|
|
169
|
+
const rank = (q: number) => s[Math.min(s.length - 1, Math.max(0, Math.ceil(q * s.length) - 1))]!; // nearest rank
|
|
170
|
+
differences.push({ arm: other.arm, measure: n, diff, low: s.length ? rank(0.025) : null, high: s.length ? rank(0.975) : null,
|
|
171
|
+
inconclusive: counted(base.attempts, n) < MIN_ATTEMPTS || counted(other.attempts, n) < MIN_ATTEMPTS || diff === null });
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
return { arms: arms.map((a) => ({ arm: a.arm, runs: a.runs, measures: measures(a.attempts) })), differences };
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/** The runs an arm label stands for: every run of it that is over, a stopped one included (its attempts carry their own outcomes). */
|
|
178
|
+
export const armRuns = (runs: BenchRun[], arm: string): BenchRun[] => runs.filter((r) => r.header.arm === arm && r.state !== "running" && r.state !== "unknown");
|
|
179
|
+
/** Groups to compare must run the same suite (name and file hash), or their numbers measure different work. */
|
|
180
|
+
export function sameSuite(groups: { arm: string; runs: BenchRun[] }[]): string | null {
|
|
181
|
+
const keys = new Set(groups.flatMap((g) => g.runs.map((r) => `${r.header.suite} (${String(r.header.suiteHash ?? "").slice(0, 12)})`)));
|
|
182
|
+
return keys.size > 1 ? `these runs come from different suites or suite versions: ${[...keys].join(", ")}` : null;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
const pct = (v: number | null) => v === null ? "-" : `${Math.round(v * 100)}%`;
|
|
186
|
+
const num = (v: number | null) => v === null ? "-" : String(Math.round(v));
|
|
187
|
+
const dur = (ms: number | null) => ms === null ? "-" : ms < 120_000 ? `${Math.round(ms / 1000)}s` : `${Math.round(ms / 60_000)}m`;
|
|
188
|
+
const row = (name: string, m: Measures) => `${name.padEnd(28)} ${String(m.attempts).padStart(8)} ${pct(m.passRate).padStart(5)} ${`${m.pass}/${m.fail}/${m.timeout}/${m.error}`.padStart(11)} ${pct(m.firstPassRate).padStart(10)} ${(m.reworkMean === null ? "-" : m.reworkMean.toFixed(2)).padStart(6)} ${num(m.tokensMedian).padStart(10)} ${dur(m.wallMsMedian).padStart(8)}`;
|
|
189
|
+
const HEAD = `${"".padEnd(28)} ${"attempts".padStart(8)} ${"pass".padStart(5)} ${"p/f/t/e".padStart(11)} ${"first pass".padStart(10)} ${"rework".padStart(6)} ${"tokens p50".padStart(10)} ${"wall p50".padStart(8)}`;
|
|
190
|
+
export function formatRuns(runs: BenchRun[]): string[] {
|
|
191
|
+
if (!runs.length) return ["no benchmark runs on this machine (ahub bench run <suite.json> --arm <label>)"];
|
|
192
|
+
return runs.map(runSummary).map((s) => `${s.run} ${s.state.padEnd(11)} ${s.suite} [${s.arm}] ${s.done}/${s.total} pass ${pct(s.passRate)} tokens p50 ${num(s.tokensMedian)} wall p50 ${dur(s.wallMsMedian)} ${s.startedAt}`);
|
|
193
|
+
}
|
|
194
|
+
export function formatReport(r: ReturnType<typeof benchReport>): string[] {
|
|
195
|
+
return [HEAD, row("all tasks", r.overall), ...Object.entries(r.tasks).map(([k, m]) => row(k, m))];
|
|
196
|
+
}
|
|
197
|
+
export function formatCompare(c: Comparison): string[] {
|
|
198
|
+
const lines = [HEAD, ...c.arms.map((a) => row(`${a.arm} (${a.runs} run${a.runs === 1 ? "" : "s"})`, a.measures))];
|
|
199
|
+
const fmt = (n: MeasureName, v: number | null) => v === null ? "-" : n.endsWith("Rate") ? `${v >= 0 ? "+" : ""}${Math.round(v * 100)}pt` : n === "wallMsMedian" ? `${v >= 0 ? "+" : "-"}${dur(Math.abs(v))}` : `${v >= 0 ? "+" : ""}${Math.round(v)}`;
|
|
200
|
+
for (const d of c.differences) lines.push(`${d.arm} vs ${c.arms[0]!.arm}: ${d.measure} ${fmt(d.measure, d.diff)} (95% ${fmt(d.measure, d.low)} to ${fmt(d.measure, d.high)})${d.inconclusive ? `, inconclusive: fewer than ${MIN_ATTEMPTS} attempts counted for it in an arm` : ""}`);
|
|
201
|
+
return lines;
|
|
202
|
+
}
|
|
203
|
+
export const CSV_COLUMNS = ["run", "suite", "arm", "task", "repeat", "outcome", "error", "verifyExit", "hubTask", "startedAt", "endedAt", "ms", "tokens", "wallMs", "activeMs",
|
|
204
|
+
"reviewRounds", "changesRequested", "checkFailed", "firstPass", "turns", "filesChanged", "models"] as const;
|
|
205
|
+
export function benchCsv(runs: BenchRun[]): string {
|
|
206
|
+
const cell = (v: unknown) => { const s = v === null || v === undefined ? "" : String(v); return /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s; };
|
|
207
|
+
const rows = runs.flatMap((r) => r.attempts.map((a) => [r.header.run, r.header.suite, r.header.arm, a.task, a.repeat, a.outcome, a.error, a.verifyExit, a.hubTask, a.startedAt, a.endedAt, a.ms,
|
|
208
|
+
a.metrics?.tokens, a.metrics?.wallMs, a.metrics?.activeMs, a.metrics?.reviewRounds, a.metrics?.changesRequested, a.metrics?.checkFailed, a.metrics?.firstPass, a.metrics?.turns,
|
|
209
|
+
a.metrics?.filesChanged, a.metrics?.models.join(" ")].map(cell).join(",")));
|
|
210
|
+
return [CSV_COLUMNS.join(","), ...rows].join("\n") + "\n";
|
|
211
|
+
}
|
package/src/hub/bus.ts
CHANGED
|
@@ -17,6 +17,8 @@ export type BusEvent =
|
|
|
17
17
|
| { t: "state"; peer: PeerId; state: PeerState };
|
|
18
18
|
|
|
19
19
|
export interface BusOptions {
|
|
20
|
+
/** Transient owner lifecycle fence: retain new work without changing console/budget pauses. */
|
|
21
|
+
deliveryHeld?: (peer: PeerId) => boolean;
|
|
20
22
|
/** Observe original envelopes immediately before a transport starts its turn. */
|
|
21
23
|
onDeliver?: (peer: PeerId, originals: Envelope[]) => void;
|
|
22
24
|
retryMs: number;
|
|
@@ -295,7 +297,14 @@ export class Bus {
|
|
|
295
297
|
if (current.revision === 0) {
|
|
296
298
|
this.journal.importSnapshot(snapshot.journal ?? { schemaVersion: 1, revision: 1, instanceId: this.journal.instanceId, bus: snapshot, deliveries: [], manualPaused: snapshot.manualPaused ?? [] }, _operationId);
|
|
297
299
|
}
|
|
300
|
+
const committedQueues = snapshot.queues;
|
|
298
301
|
snapshot = this.journal.snapshot().bus;
|
|
302
|
+
// Offline peers need not reattach. Their empty queue keys still belong to the
|
|
303
|
+
// committed integrity view, even when no delivery ever persisted those keys (#234).
|
|
304
|
+
// The journal remains authoritative for work: never resurrect a non-empty queue.
|
|
305
|
+
for (const [id, queue] of Object.entries(committedQueues)) {
|
|
306
|
+
if (queue.length === 0 && !Object.hasOwn(snapshot.queues, id)) snapshot.queues[id] = [];
|
|
307
|
+
}
|
|
299
308
|
}
|
|
300
309
|
this.loadSnapshot(snapshot);
|
|
301
310
|
this.persist();
|
|
@@ -348,7 +357,10 @@ export class Bus {
|
|
|
348
357
|
if (record.revision !== revision) throw new Error("stale delivery revision");
|
|
349
358
|
if (record.state === "queued" && action === "retry") throw new Error("queued delivery is already scheduled");
|
|
350
359
|
if (action === "retry" && (this.queues.get(record.peer)?.length ?? 0) + record.originals.length > this.opts.queueCap) throw new Error("recipient queue is full");
|
|
351
|
-
const
|
|
360
|
+
const queue = this.queues.get(record.peer) ?? [];
|
|
361
|
+
const hadQueue = this.queues.has(record.peer);
|
|
362
|
+
const before = [...queue];
|
|
363
|
+
const preface = this.prefaces.get(record.peer);
|
|
352
364
|
try {
|
|
353
365
|
this.journal.transaction(() => {
|
|
354
366
|
if (!this.journal!.get(id)) this.journal!.createDelivery({ ...record });
|
|
@@ -356,10 +368,9 @@ export class Bus {
|
|
|
356
368
|
this.journal!.resolve(id, current.revision, action, reason, revision);
|
|
357
369
|
if (record.state === "queued") {
|
|
358
370
|
const ids = new Set(record.originals.map((env) => env.id));
|
|
359
|
-
|
|
371
|
+
queue.splice(0, queue.length, ...queue.filter((env) => !ids.has(env.id)));
|
|
360
372
|
}
|
|
361
373
|
if (action === "retry") {
|
|
362
|
-
const queue = this.queues.get(record.peer) ?? [];
|
|
363
374
|
for (const env of record.originals) {
|
|
364
375
|
if (env.from === HUB && env.kind === "presence") this.restorePreface(record.peer, env);
|
|
365
376
|
else if (!queue.some((old) => old.id === env.id)) queue.push(env);
|
|
@@ -368,7 +379,13 @@ export class Bus {
|
|
|
368
379
|
}
|
|
369
380
|
this.persist();
|
|
370
381
|
});
|
|
371
|
-
} catch (error) {
|
|
382
|
+
} catch (error) {
|
|
383
|
+
queue.splice(0, queue.length, ...before);
|
|
384
|
+
if (!hadQueue) this.queues.delete(record.peer);
|
|
385
|
+
if (preface) this.prefaces.set(record.peer, preface);
|
|
386
|
+
else this.prefaces.delete(record.peer);
|
|
387
|
+
throw error;
|
|
388
|
+
}
|
|
372
389
|
this.refreshRecoveryHold(record.peer);
|
|
373
390
|
this.onQueues?.();
|
|
374
391
|
void this.drain(record.peer);
|
|
@@ -533,7 +550,7 @@ export class Bus {
|
|
|
533
550
|
try { for (const id of targets) {
|
|
534
551
|
const peer = this.peers.get(id);
|
|
535
552
|
if (!peer) { this.enqueue(id, env); continue; }
|
|
536
|
-
if (!this.journal && !this.recoveryHeld && !this.recoveryHeldPeers.has(id) && env.priority === "important" && peer.steer && this.stateOf(id) === "busy") {
|
|
553
|
+
if (!this.journal && !this.recoveryHeld && !this.recoveryHeldPeers.has(id) && !this.opts.deliveryHeld?.(id) && env.priority === "important" && peer.steer && this.stateOf(id) === "busy") {
|
|
537
554
|
// Not queued while the steer is in flight, or an idle transition would deliver it a second time.
|
|
538
555
|
this.steering++;
|
|
539
556
|
const deliveryId = crypto.randomUUID();
|
|
@@ -553,7 +570,7 @@ export class Bus {
|
|
|
553
570
|
this.onQueues?.();
|
|
554
571
|
for (const id of targets) {
|
|
555
572
|
const peer = this.peers.get(id);
|
|
556
|
-
if (!this.recoveryHeld && !this.recoveryHeldPeers.has(id) && env.priority === "important" && peer?.steer && this.stateOf(id) === "busy") void this.steerQueued(id, env);
|
|
573
|
+
if (!this.recoveryHeld && !this.recoveryHeldPeers.has(id) && !this.opts.deliveryHeld?.(id) && env.priority === "important" && peer?.steer && this.stateOf(id) === "busy") void this.steerQueued(id, env);
|
|
557
574
|
else void this.drain(id);
|
|
558
575
|
}
|
|
559
576
|
}
|
|
@@ -717,7 +734,7 @@ export class Bus {
|
|
|
717
734
|
|
|
718
735
|
/** Holds on a peer's queue, for a drain and a pull alike: recovery, an uncertain delivery, a pause (console, budget, conductor). */
|
|
719
736
|
private held(id: PeerId): boolean {
|
|
720
|
-
return this.recoveryHeld || this.recoveryHeldPeers.has(id) || this.paused.has(id);
|
|
737
|
+
return this.recoveryHeld || this.recoveryHeldPeers.has(id) || this.paused.has(id) || this.opts.deliveryHeld?.(id) === true;
|
|
721
738
|
}
|
|
722
739
|
|
|
723
740
|
/** What a peer was just handed stays resolvable for `reply_to`: moved to the newest end, the stored envelope kept. */
|
package/src/hub/conductor.ts
CHANGED
|
@@ -3,6 +3,7 @@ import type { Task } from "./board.ts";
|
|
|
3
3
|
import { CONDUCTOR_TOOL_NAMES } from "./hub-tools.ts";
|
|
4
4
|
import { HUB, USER } from "./envelope.ts";
|
|
5
5
|
import { OWNERSHIP_EVENTS } from "./tasks.ts";
|
|
6
|
+
import { PreEffectToolRefusal } from "./tool-refusal.ts";
|
|
6
7
|
import type { HubEvent } from "./events.ts";
|
|
7
8
|
import type { SupervisionFeed } from "./supervision.ts";
|
|
8
9
|
import type { Budget } from "./budget.ts";
|
|
@@ -24,8 +25,8 @@ export function publicPeerBudget(status: ReturnType<Budget["status"]>, nameable:
|
|
|
24
25
|
}
|
|
25
26
|
|
|
26
27
|
const PEER = /^[a-z][a-z0-9-]{0,31}$/;
|
|
27
|
-
const peerId = (value: unknown): string => {
|
|
28
|
-
if (typeof value !== "string" || /\s/.test(value) || !PEER.test(value) || ["user", "hub", "digest"].includes(value)) throw new Error("peer must be a valid agent peer id");
|
|
28
|
+
const peerId = (value: unknown, preEffect = false): string => {
|
|
29
|
+
if (typeof value !== "string" || /\s/.test(value) || !PEER.test(value) || ["user", "hub", "digest"].includes(value)) throw new (preEffect ? PreEffectToolRefusal : Error)("peer must be a valid agent peer id");
|
|
29
30
|
return value;
|
|
30
31
|
};
|
|
31
32
|
|
|
@@ -45,16 +46,16 @@ export function conductorPeer(roles: unknown): string | null {
|
|
|
45
46
|
}
|
|
46
47
|
|
|
47
48
|
/** The role check always comes first; default-allow capabilities cannot grant a role. */
|
|
48
|
-
export function requireConductor(peer: string, roles: unknown, capabilities: Record<string, unknown> = {}, assign = false): void {
|
|
49
|
-
if (conductorPeer(roles) !== peer) throw new Error("this operation requires the explicit conductor role");
|
|
50
|
-
if (assign) requireAssign(peer, capabilities);
|
|
49
|
+
export function requireConductor(peer: string, roles: unknown, capabilities: Record<string, unknown> = {}, assign = false, preEffect = false): void {
|
|
50
|
+
if (conductorPeer(roles) !== peer) throw new (preEffect ? PreEffectToolRefusal : Error)("this operation requires the explicit conductor role");
|
|
51
|
+
if (assign) requireAssign(peer, capabilities, preEffect);
|
|
51
52
|
}
|
|
52
53
|
|
|
53
54
|
/** A peer with an explicit capabilities list needs `assign` in it to move work. */
|
|
54
|
-
function requireAssign(peer: string, capabilities: Record<string, unknown
|
|
55
|
+
function requireAssign(peer: string, capabilities: Record<string, unknown>, preEffect = false): void {
|
|
55
56
|
if (!Object.hasOwn(capabilities, peer)) return;
|
|
56
57
|
const caps = capabilities[peer];
|
|
57
|
-
if (!Array.isArray(caps) || !caps.includes("assign")) throw new Error(`${peer} requires assign capability for this operation`);
|
|
58
|
+
if (!Array.isArray(caps) || !caps.includes("assign")) throw new (preEffect ? PreEffectToolRefusal : Error)(`${peer} requires assign capability for this operation`);
|
|
58
59
|
}
|
|
59
60
|
|
|
60
61
|
export interface ConductorHold { peer: string; actor: string; since: number }
|
|
@@ -101,11 +102,11 @@ export class ConductorHolds {
|
|
|
101
102
|
|
|
102
103
|
export type ConductorStart = { peer: "local" | "kimi" | "pi"; mode: "headless" } | { peer: "claude" | "codex" | "pi"; mode: "tui"; command: string };
|
|
103
104
|
/** Preview commands come from the same launch planner the CLI uses, never caller-supplied shell text. */
|
|
104
|
-
export function conductorStart(peer: unknown, mode: unknown, preview: (peer: "claude" | "codex" | "pi") => string): ConductorStart {
|
|
105
|
-
if (mode !== undefined && mode !== "headless" && mode !== "tui") throw new Error("mode must be headless or tui");
|
|
105
|
+
export function conductorStart(peer: unknown, mode: unknown, preview: (peer: "claude" | "codex" | "pi") => string, preEffect = false): ConductorStart {
|
|
106
|
+
if (mode !== undefined && mode !== "headless" && mode !== "tui") throw new (preEffect ? PreEffectToolRefusal : Error)("mode must be headless or tui");
|
|
106
107
|
if (peer === "claude" || peer === "codex" || (peer === "pi" && mode === "tui")) return { peer, mode: "tui", command: preview(peer) };
|
|
107
108
|
if ((peer === "local" || peer === "kimi" || peer === "pi") && mode !== "tui") return { peer, mode: "headless" };
|
|
108
|
-
throw new Error("only local, kimi and headless pi may be started by the conductor");
|
|
109
|
+
throw new (preEffect ? PreEffectToolRefusal : Error)("only local, kimi and headless pi may be started by the conductor");
|
|
109
110
|
}
|
|
110
111
|
|
|
111
112
|
const numberOrNull = (v: unknown): number | null => typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : null;
|
|
@@ -169,7 +170,7 @@ const taskId = (v: unknown): number | undefined => {
|
|
|
169
170
|
export class Conductor {
|
|
170
171
|
constructor(private readonly holds: ConductorHolds, private readonly hooks: ConductorHooks) {}
|
|
171
172
|
async execute(actor: string, tool: string, args: Record<string, unknown>): Promise<unknown> {
|
|
172
|
-
if (!CONDUCTOR_TOOL_NAMES.has(tool)) throw new
|
|
173
|
+
if (!CONDUCTOR_TOOL_NAMES.has(tool)) throw new PreEffectToolRefusal("unknown conductor tool");
|
|
173
174
|
if ((tool === "hub_task_show" || tool === "hub_task_assign") && conductorPeer(this.hooks.roles()) !== actor) {
|
|
174
175
|
const id = taskId(args.id);
|
|
175
176
|
const task = id === undefined ? undefined : this.hooks.task(id);
|
|
@@ -182,14 +183,14 @@ export class Conductor {
|
|
|
182
183
|
const moved = task?.history.findLast((h) => h.by !== HUB && (OWNERSHIP_EVENTS.has(h.event) || h.event === "reserved"));
|
|
183
184
|
const accepted = task?.history.some((h) => h.event === "accepted");
|
|
184
185
|
if (task && tool === "hub_task_assign" && task.history[0]?.by === actor && task.state === "proposed" && !accepted && moved?.by !== USER) {
|
|
185
|
-
const peer = peerId(args.peer);
|
|
186
|
-
if (peer !== actor) requireAssign(actor, this.hooks.capabilities());
|
|
186
|
+
const peer = peerId(args.peer, true);
|
|
187
|
+
if (peer !== actor) requireAssign(actor, this.hooks.capabilities(), true);
|
|
187
188
|
await this.hooks.assign(actor, task.id, peer);
|
|
188
189
|
const updated = this.hooks.task(task.id);
|
|
189
190
|
return updated ? publicConductorTask(updated, this.hooks.publicView) : { id: task.id };
|
|
190
191
|
}
|
|
191
192
|
}
|
|
192
|
-
requireConductor(actor, this.hooks.roles(), this.hooks.capabilities(), tool === "hub_task_assign" || tool === "hub_task_escalate");
|
|
193
|
+
requireConductor(actor, this.hooks.roles(), this.hooks.capabilities(), tool === "hub_task_assign" || tool === "hub_task_escalate", true);
|
|
193
194
|
const action = tool.slice(4);
|
|
194
195
|
const emit = (extra: Pick<ConductEvent, "task" | "peer"> = {}) => {
|
|
195
196
|
try { this.hooks.audit({ kind: "conduct", actor, action, ...extra }); } catch { /* never throw after a task or hold write */ }
|
|
@@ -197,12 +198,12 @@ export class Conductor {
|
|
|
197
198
|
if (tool === "hub_status") { const result = publicConductorStatus(this.hooks.status()); emit(); return result; }
|
|
198
199
|
if (tool.startsWith("hub_task_")) {
|
|
199
200
|
const id = taskId(args.id);
|
|
200
|
-
if (id === undefined) throw new
|
|
201
|
+
if (id === undefined) throw new PreEffectToolRefusal("id must be a positive integer");
|
|
201
202
|
const task = this.hooks.task(id);
|
|
202
|
-
if (!task) throw new
|
|
203
|
+
if (!task) throw new PreEffectToolRefusal(`no task #${args.id}`);
|
|
203
204
|
if (tool === "hub_task_show") { const result = publicConductorTask(task, this.hooks.publicView); emit({ task: task.id }); return result; }
|
|
204
205
|
if (tool === "hub_task_assign") {
|
|
205
|
-
const peer = peerId(args.peer);
|
|
206
|
+
const peer = peerId(args.peer, true);
|
|
206
207
|
await this.hooks.assign(actor, task.id, peer); emit({ task: task.id, peer });
|
|
207
208
|
} else { await this.hooks.escalate(actor, task.id); emit({ task: task.id }); }
|
|
208
209
|
// Task callbacks may return raw objects; always re-read and apply the public view.
|
|
@@ -210,15 +211,15 @@ export class Conductor {
|
|
|
210
211
|
return updated ? publicConductorTask(updated, this.hooks.publicView) : { id: task.id };
|
|
211
212
|
}
|
|
212
213
|
if (tool === "hub_peer_start") {
|
|
213
|
-
const start = conductorStart(args.peer, args.mode, this.hooks.preview);
|
|
214
|
+
const start = conductorStart(args.peer, args.mode, this.hooks.preview, true);
|
|
214
215
|
if (start.mode === "headless") await this.hooks.start(start.peer);
|
|
215
216
|
emit({ peer: start.peer }); return start;
|
|
216
217
|
}
|
|
217
|
-
const peer = peerId(args.peer);
|
|
218
|
-
if (!this.hooks.known(peer)) throw new
|
|
218
|
+
const peer = peerId(args.peer, true);
|
|
219
|
+
if (!this.hooks.known(peer)) throw new PreEffectToolRefusal(`unknown peer: ${peer}`);
|
|
219
220
|
if (tool === "hub_peer_hold") { this.holds.hold(peer, actor); this.hooks.pause(peer); }
|
|
220
221
|
else {
|
|
221
|
-
if (this.holds.get(peer)?.actor !== actor) throw new
|
|
222
|
+
if (this.holds.get(peer)?.actor !== actor) throw new PreEffectToolRefusal("you may release only a conductor hold you placed");
|
|
222
223
|
await this.hooks.validateRelease?.(peer);
|
|
223
224
|
requireConductor(actor, this.hooks.roles(), this.hooks.capabilities());
|
|
224
225
|
this.holds.release(peer, actor); this.hooks.release(peer);
|
package/src/hub/config-trust.ts
CHANGED
|
@@ -30,6 +30,8 @@ export interface ControlDescriptor {
|
|
|
30
30
|
cwd?: string;
|
|
31
31
|
protocol?: number;
|
|
32
32
|
pid?: number;
|
|
33
|
+
/** #226: the daemon's processSignature, written into status.json only (not the status reply); absent before 0.12.21. */
|
|
34
|
+
pidSignature?: string;
|
|
33
35
|
}
|
|
34
36
|
|
|
35
37
|
/** Where the daemon of this state dir listens. Re-read before every connect: port and token change per run. */
|
|
@@ -39,7 +41,8 @@ export function readControl(stateDir: string): ControlDescriptor | undefined {
|
|
|
39
41
|
const token = readFileSync(join(stateDir, "control-token"), "utf8").trim();
|
|
40
42
|
if (!Number.isInteger(status.controlPort) || status.controlPort < 1 || status.controlPort > 65535 || !token) return undefined;
|
|
41
43
|
return { url: `ws://127.0.0.1:${status.controlPort}`, token, projectId: status.projectId,
|
|
42
|
-
instanceId: status.instanceId, cwd: status.cwd, protocol: status.protocol, pid: status.pid
|
|
44
|
+
instanceId: status.instanceId, cwd: status.cwd, protocol: status.protocol, pid: status.pid,
|
|
45
|
+
...(typeof status.pidSignature === "string" && status.pidSignature ? { pidSignature: status.pidSignature } : {}) };
|
|
43
46
|
} catch {
|
|
44
47
|
return undefined;
|
|
45
48
|
}
|