foldrun 2.5.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/foldrun.mjs +23 -5
- package/docs/agents.md +74 -0
- package/docs/api.md +12 -4
- package/docs/checks.md +10 -0
- package/docs/cli.md +22 -6
- package/docs/evals.md +151 -1
- package/docs/flows.md +63 -2
- package/docs/four-jobs.md +1 -1
- package/docs/hooks.md +2 -2
- package/docs/network.md +6 -2
- package/docs/runs.md +28 -0
- package/docs/secrets.md +8 -1
- package/docs/tools.md +1 -1
- package/package.json +6 -3
- package/src/commands.mjs +566 -21
- package/src/guide.mjs +1 -1
package/src/commands.mjs
CHANGED
|
@@ -1340,7 +1340,8 @@ async function checkProblems(workspace, flags = {}) {
|
|
|
1340
1340
|
note("warn", `agents/${a.name}`, `subagents: [${s}] — an agent delegating to itself; it is skipped at run time`);
|
|
1341
1341
|
} else if (!sub.description?.trim()) {
|
|
1342
1342
|
note("error", `agents/${s}`, `no description — ${a.name} delegates to it (subagents:), and the model picks a sub-agent by its description`);
|
|
1343
|
-
} else if (sub.tools.length && !checkOverlap(a.tools, sub.tools).length) {
|
|
1343
|
+
} else if (!ownSandboxOf(sub) && sub.tools.length && !checkOverlap(a.tools, sub.tools).length) {
|
|
1344
|
+
// sandbox: own — its tools are its own, not cut to the parent's.
|
|
1344
1345
|
note("warn", `agents/${a.name}`, `subagents: [${s}] — none of ${s}'s tools are in ${a.name}'s, and a sub-agent never gets more than its parent: it would have no tools`);
|
|
1345
1346
|
}
|
|
1346
1347
|
}
|
|
@@ -1888,13 +1889,23 @@ async function probeCmd(modelArg) {
|
|
|
1888
1889
|
async function runEvals(name, flags = {}) {
|
|
1889
1890
|
assertCredentials();
|
|
1890
1891
|
await useGallery(flags);
|
|
1891
|
-
const { listEvals, runEval } = await core();
|
|
1892
|
+
const { listEvals, runEval, workspaceDir } = await core();
|
|
1892
1893
|
const T = "default";
|
|
1893
1894
|
const P = "workspace";
|
|
1895
|
+
// Where this workspace's evals are, for telling a grader by its file when
|
|
1896
|
+
// the installed core predates graders — the workspace chosen, not the cwd.
|
|
1897
|
+
const evalsDir = path.join(workspaceDir(T, P), "evals");
|
|
1894
1898
|
// An `inputs: true` file is a flow's saved inputs, not a test: it has
|
|
1895
1899
|
// nothing to assert, so it is never run as one (see evals.md).
|
|
1896
|
-
const
|
|
1900
|
+
const named = listEvals(T, P).filter((e) => (!name || e.name === name) && !e.inputs);
|
|
1901
|
+
// A grader (`trigger: runs`) grades real runs; it has no cases to run here.
|
|
1902
|
+
const all = named.filter((e) => !isGrader(e, evalsDir));
|
|
1903
|
+
for (const g of named.filter((e) => isGrader(e, evalsDir))) {
|
|
1904
|
+
console.log(`\n ${c.bold(g.name)} ${c.dim(`grades live runs — \`foldrun eval grades ${g.name} --to <workspace>\` shows what it found`)}`);
|
|
1905
|
+
}
|
|
1897
1906
|
if (all.length === 0) {
|
|
1907
|
+
// Only graders here (or the one named is a grader): listed above, nothing to run — not an error.
|
|
1908
|
+
if (named.length) return 0;
|
|
1898
1909
|
if (name && listEvals(T, P).some((e) => e.name === name)) {
|
|
1899
1910
|
throw new Error(`"${name}" is a file of saved inputs, not an eval — \`foldrun invoke <flow> --inputs <set>\` runs one`);
|
|
1900
1911
|
}
|
|
@@ -3609,13 +3620,38 @@ const CONNECT_PORT = 8642;
|
|
|
3609
3620
|
* a provider that issues none (GitHub, most LinkedIn apps) gets its access
|
|
3610
3621
|
* token stored as a static value and the expiry is said out loud.
|
|
3611
3622
|
*/
|
|
3623
|
+
/**
|
|
3624
|
+
* `--authorize-param name=value`, repeatable: extra query parameters on the
|
|
3625
|
+
* provider's authorize URL — what a provider wants beyond OAuth's own
|
|
3626
|
+
* (Google's access_type=offline and prompt=consent, for a refresh token).
|
|
3627
|
+
* foldrun knows no provider; the person types what theirs documents.
|
|
3628
|
+
*/
|
|
3629
|
+
function authorizeParams(flags) {
|
|
3630
|
+
const out = {};
|
|
3631
|
+
for (const p of [flags["authorize-param"]].flat().filter((x) => x !== undefined)) {
|
|
3632
|
+
const i = typeof p === "string" ? p.indexOf("=") : -1;
|
|
3633
|
+
if (i < 1) throw new Error(`--authorize-param takes name=value, got "${p}"`);
|
|
3634
|
+
out[p.slice(0, i).trim()] = p.slice(i + 1).trim();
|
|
3635
|
+
}
|
|
3636
|
+
return out;
|
|
3637
|
+
}
|
|
3638
|
+
|
|
3639
|
+
/** --provider is deprecated: a runner keeps no provider list. Said once,
|
|
3640
|
+
* on stderr, so a script reading stdout is not disturbed. */
|
|
3641
|
+
function warnProviderDeprecated(flags) {
|
|
3642
|
+
if (flags.provider !== undefined) {
|
|
3643
|
+
console.error(c.dim(" --provider is deprecated — pass the provider's own --authorize-url and --token-url, and any --authorize-param it documents"));
|
|
3644
|
+
}
|
|
3645
|
+
}
|
|
3646
|
+
|
|
3612
3647
|
async function connect(positional, flags) {
|
|
3613
3648
|
const name = positional[0];
|
|
3614
|
-
if (!name) throw new Error("usage: foldrun connect NAME
|
|
3649
|
+
if (!name) throw new Error("usage: foldrun connect NAME --authorize-url <url> --token-url <url> [--authorize-param name=value …] [--scopes …] [--workspace <name>] [--new-client]");
|
|
3615
3650
|
if (!/^[A-Z][A-Z0-9_]*$/.test(name)) throw new Error(`secret name "${name}" must be UPPER_SNAKE_CASE`);
|
|
3616
3651
|
const { OAUTH_PRESETS } = await core();
|
|
3617
3652
|
|
|
3618
3653
|
const providerName = flags.provider;
|
|
3654
|
+
warnProviderDeprecated(flags);
|
|
3619
3655
|
const preset = providerName ? OAUTH_PRESETS[providerName] : undefined;
|
|
3620
3656
|
if (providerName && !preset) {
|
|
3621
3657
|
throw new Error(`unknown provider "${providerName}" — one of ${Object.keys(OAUTH_PRESETS).join(", ")}, or pass --authorize-url and --token-url`);
|
|
@@ -3668,7 +3704,8 @@ async function connect(positional, flags) {
|
|
|
3668
3704
|
}
|
|
3669
3705
|
}
|
|
3670
3706
|
|
|
3671
|
-
if (!authorizeUrl || !tokenUrl) throw new Error("--
|
|
3707
|
+
if (!authorizeUrl || !tokenUrl) throw new Error("--authorize-url and --token-url — the provider's own addresses, from its OAuth docs");
|
|
3708
|
+
const extra = { ...(preset?.authorize_extra ?? {}), ...authorizeParams(flags) };
|
|
3672
3709
|
if (!/^https:\/\//.test(tokenUrl) && !/^http:\/\/(127\.0\.0\.1|localhost)[:/]/.test(tokenUrl)) {
|
|
3673
3710
|
throw new Error("token URL must be https — a refresh token over http is a leaked one");
|
|
3674
3711
|
}
|
|
@@ -3698,7 +3735,7 @@ async function connect(positional, flags) {
|
|
|
3698
3735
|
const verifier = pkce ? crypto.randomBytes(32).toString("base64url") : null;
|
|
3699
3736
|
const authorize = new URL(authorizeUrl);
|
|
3700
3737
|
// Extras first: the flow's own parameters are set after, and win.
|
|
3701
|
-
for (const [k, v] of Object.entries(
|
|
3738
|
+
for (const [k, v] of Object.entries(extra)) authorize.searchParams.set(k, v);
|
|
3702
3739
|
for (const k of ["client_id", "redirect_uri", "response_type", "scope", "state", "code_challenge", "code_challenge_method"]) authorize.searchParams.delete(k);
|
|
3703
3740
|
authorize.searchParams.set("response_type", "code");
|
|
3704
3741
|
authorize.searchParams.set("client_id", clientId);
|
|
@@ -6297,6 +6334,57 @@ function issueDoctor(issue) {
|
|
|
6297
6334
|
* on one — the dashboard's Issues page and its buttons, same route.
|
|
6298
6335
|
* `foldrun issues doctor <id>` is the card's Doctor: the run doctor on the
|
|
6299
6336
|
* issue's newest run, in that run's workspace. */
|
|
6337
|
+
/**
|
|
6338
|
+
* `foldrun inbox` — what is waiting on you across your workspaces (GET
|
|
6339
|
+
* /api/me/inbox): gates, questions agents ask, open issues, proposed fixes,
|
|
6340
|
+
* your runs that went wrong this week. Assigned to you and started by you by
|
|
6341
|
+
* default; `--team` for the rest of what waits in your workspaces, `--all`
|
|
6342
|
+
* for every bucket, `--dismissed` for what you set aside, `--kind
|
|
6343
|
+
* approval|question|issue|proposal|run`, `--unread`, `--json`. `read`,
|
|
6344
|
+
* `unread`, `dismiss` (set aside — it still waits where it lives) and
|
|
6345
|
+
* `restore` take item ids. A key reads the inbox of the person who minted it.
|
|
6346
|
+
* The dashboard's Inbox. Exit 1 when anything assigned to you is unread.
|
|
6347
|
+
*/
|
|
6348
|
+
async function inboxCmd(positional, flags) {
|
|
6349
|
+
const url = platformFor(flags, "inbox");
|
|
6350
|
+
const [verb, ...ids] = positional;
|
|
6351
|
+
const ACTIONS = { read: "read", unread: "unread", dismiss: "dismiss", restore: "restore" };
|
|
6352
|
+
if (verb) {
|
|
6353
|
+
const action = ACTIONS[verb];
|
|
6354
|
+
if (!action) throw new Error(`\`foldrun inbox\` lists what waits on you; its verbs are read, unread, dismiss and restore — not ${verb}`);
|
|
6355
|
+
if (!ids.length) throw new Error(`\`foldrun inbox ${verb} <id>…\` — the ids are the first column of \`foldrun inbox\``);
|
|
6356
|
+
await remoteCall(url, flags, "/api/me/inbox", { method: "POST", body: JSON.stringify({ ids, action }) });
|
|
6357
|
+
console.log(`\n ${c.green(action === "dismiss" ? "set aside" : action === "restore" ? "put back" : `marked ${action}`)} ${ids.length} item${ids.length === 1 ? "" : "s"}\n`);
|
|
6358
|
+
return 0;
|
|
6359
|
+
}
|
|
6360
|
+
const q = new URLSearchParams();
|
|
6361
|
+
if (flags.all === true) q.set("bucket", "all");
|
|
6362
|
+
else if (flags.team === true) q.set("bucket", "team");
|
|
6363
|
+
else if (flags.started === true) q.set("bucket", "started");
|
|
6364
|
+
else if (flags.mine === true) q.set("bucket", "mine");
|
|
6365
|
+
if (flags.dismissed === true) q.set("dismissed", "1");
|
|
6366
|
+
if (typeof flags.kind === "string") q.set("kind", flags.kind);
|
|
6367
|
+
if (typeof flags.workspace === "string") q.set("workspace", flags.workspace);
|
|
6368
|
+
const view = await remoteCall(url, flags, `/api/me/inbox${q.size ? `?${q}` : ""}`);
|
|
6369
|
+
const items = (view.items ?? []).filter((i) => (flags.unread === true ? !i.read : true));
|
|
6370
|
+
if (flags.json) {
|
|
6371
|
+
console.log(JSON.stringify({ ...view, items }, null, 2));
|
|
6372
|
+
return view.counts?.unread ? 1 : 0;
|
|
6373
|
+
}
|
|
6374
|
+
const k = view.counts ?? {};
|
|
6375
|
+
console.log(`\n ${c.bold(`inbox of ${view.person}`)} ${c.dim(`· for you ${k.mine ?? 0} · started by you ${k.started ?? 0} · team ${k.team ?? 0} · ${k.unread ?? 0} unread`)}`);
|
|
6376
|
+
if (!items.length) console.log(`\n ${c.dim(flags.dismissed ? "nothing set aside" : "nothing waiting")}`);
|
|
6377
|
+
const BUCKET = { mine: "for you", started: "yours", team: "team" };
|
|
6378
|
+
for (const i of items) {
|
|
6379
|
+
const dot = i.read ? " " : c.amber("•");
|
|
6380
|
+
console.log(`\n ${dot} ${c.bold(i.title)} ${c.dim(`· ${i.kind} · ${BUCKET[i.bucket] ?? i.bucket} · ${i.workspace} · ${String(i.at).slice(0, 16).replace("T", " ")}`)}`);
|
|
6381
|
+
if (i.detail) console.log(` ${i.detail}`);
|
|
6382
|
+
console.log(` ${c.dim(`${i.id} — ${i.why}`)}`);
|
|
6383
|
+
}
|
|
6384
|
+
console.log("");
|
|
6385
|
+
return k.unread ? 1 : 0;
|
|
6386
|
+
}
|
|
6387
|
+
|
|
6300
6388
|
async function issuesCmd(positional, flags, layout) {
|
|
6301
6389
|
const url = platformFor(flags, "issues");
|
|
6302
6390
|
const [verb, id] = positional;
|
|
@@ -8156,6 +8244,35 @@ async function flowRmStepCmd(positional, flags, layout) {
|
|
|
8156
8244
|
}
|
|
8157
8245
|
|
|
8158
8246
|
/** Whether an agent file says what the agent is for. */
|
|
8247
|
+
/** `sandbox: own` on an agent as core's listAgents reports it — from core
|
|
8248
|
+
* 2.8 on; an older core does not say, and the agent reads as not own. */
|
|
8249
|
+
function ownSandboxOf(agent) {
|
|
8250
|
+
return /** @type {{ sandbox?: string } | undefined} */ (agent)?.sandbox === "own";
|
|
8251
|
+
}
|
|
8252
|
+
|
|
8253
|
+
/** An agent file's frontmatter as a run reads it, from its text. A run folds
|
|
8254
|
+
* the four job blocks' keys up to the top (core's configMatter, kept private
|
|
8255
|
+
* — not in @foldrun/core's publicApi), so `guardrail: { sandbox: own }` reads
|
|
8256
|
+
* as `sandbox: own`. The CLI reads only two foldable keys off this — `sandbox`
|
|
8257
|
+
* and `network`, both in an agent's `guardrail:` block — so it lifts just
|
|
8258
|
+
* those, the same way a run does: a top-level value wins, else the block's is
|
|
8259
|
+
* lifted. {} when it does not parse. */
|
|
8260
|
+
async function effectiveFront(raw) {
|
|
8261
|
+
const { readFrontmatter } = await core();
|
|
8262
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "foldrun-front-"));
|
|
8263
|
+
try {
|
|
8264
|
+
fs.writeFileSync(path.join(dir, "agent.md"), raw);
|
|
8265
|
+
const front = /** @type {Record<string, any>} */ (readFrontmatter(path.join(dir, "agent.md")) ?? {});
|
|
8266
|
+
const guardrail = front.guardrail && typeof front.guardrail === "object" ? front.guardrail : {};
|
|
8267
|
+
for (const key of ["sandbox", "network"]) if (!(key in front) && key in guardrail) front[key] = guardrail[key];
|
|
8268
|
+
return front;
|
|
8269
|
+
} catch {
|
|
8270
|
+
return {};
|
|
8271
|
+
} finally {
|
|
8272
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
8273
|
+
}
|
|
8274
|
+
}
|
|
8275
|
+
|
|
8159
8276
|
function hasDescription(raw) {
|
|
8160
8277
|
const front = /^---\r?\n([\s\S]*?)\r?\n---/.exec(raw)?.[1] ?? "";
|
|
8161
8278
|
const lines = front.split(/\r?\n/);
|
|
@@ -8267,7 +8384,7 @@ async function agentImportCmd(positional, flags, layout) {
|
|
|
8267
8384
|
}
|
|
8268
8385
|
|
|
8269
8386
|
/**
|
|
8270
|
-
* `foldrun agent link <agent> --subagent <worker> | --consult <agent> |
|
|
8387
|
+
* `foldrun agent link <agent> --subagent <worker> [--isolated] | --consult <agent> |
|
|
8271
8388
|
* --can-ask`, and `unlink` — an agent's team, edited in its frontmatter
|
|
8272
8389
|
* list and nowhere else: `subagents:`, `agents:`, `ask` in `tools:`.
|
|
8273
8390
|
*/
|
|
@@ -8275,7 +8392,7 @@ async function agentLinkCmd(positional, flags, layout) {
|
|
|
8275
8392
|
const verb = positional[0];
|
|
8276
8393
|
const add = verb === "link";
|
|
8277
8394
|
const name = positional[1];
|
|
8278
|
-
const usage = `\`foldrun agent ${verb} <agent> --subagent <worker> | --consult <agent> | --can-ask\``;
|
|
8395
|
+
const usage = `\`foldrun agent ${verb} <agent> --subagent <worker> [--isolated] | --consult <agent> | --can-ask\``;
|
|
8279
8396
|
if (!name) throw new Error(usage);
|
|
8280
8397
|
const picked = [
|
|
8281
8398
|
typeof flags.subagent === "string" ? ["subagents", flags.subagent] : null,
|
|
@@ -8284,6 +8401,10 @@ async function agentLinkCmd(positional, flags, layout) {
|
|
|
8284
8401
|
].filter(Boolean);
|
|
8285
8402
|
if (picked.length !== 1) throw new Error(`${usage} — exactly one of them`);
|
|
8286
8403
|
const [key, member] = /** @type {[string, string]} */ (picked[0]);
|
|
8404
|
+
// --isolated: the worker runs each job in its own sandbox (sandbox: own on
|
|
8405
|
+
// its file) — its own tools, secrets and network, not cut to the parent's.
|
|
8406
|
+
const isolated = flags.isolated === true;
|
|
8407
|
+
if (isolated && (key !== "subagents" || !add)) throw new Error("--isolated goes with `agent link <agent> --subagent <worker>`");
|
|
8287
8408
|
|
|
8288
8409
|
const desk = await openDesk(flags, layout, `agent ${verb} ${name}`);
|
|
8289
8410
|
const { editFrontmatterList, setFrontmatterScalar } = await patterns();
|
|
@@ -8297,8 +8418,11 @@ async function agentLinkCmd(positional, flags, layout) {
|
|
|
8297
8418
|
|
|
8298
8419
|
const raw = desk.files.get(fileOf(name));
|
|
8299
8420
|
const next = editFrontmatterList(raw, key, member, add ? "add" : "remove");
|
|
8300
|
-
const said = key === "subagents" ? `${member} as a sub-agent` : key === "agents" ? `${member} as a consult` : "asking you mid-step";
|
|
8301
|
-
|
|
8421
|
+
const said = key === "subagents" ? `${member} as a sub-agent${isolated ? " in its own sandbox" : ""}` : key === "agents" ? `${member} as a consult` : "asking you mid-step";
|
|
8422
|
+
// What a run reads, through core's parser: a `guardrail:` block counts,
|
|
8423
|
+
// and "sandbox: own" inside a description does not.
|
|
8424
|
+
const workerOwn = isolated && (await effectiveFront(desk.files.get(fileOf(member)) ?? "")).sandbox === "own";
|
|
8425
|
+
if (next === raw && (!isolated || workerOwn)) {
|
|
8302
8426
|
console.log(`\n ${c.dim(add ? `${name} already has ${said}` : `${name} does not have ${said} — nothing to remove`)}\n`);
|
|
8303
8427
|
return 0;
|
|
8304
8428
|
}
|
|
@@ -8316,9 +8440,22 @@ async function agentLinkCmd(positional, flags, layout) {
|
|
|
8316
8440
|
if (!description) throw new Error("no description given — nothing written");
|
|
8317
8441
|
}
|
|
8318
8442
|
const w = desk.files.get(fileOf(member));
|
|
8319
|
-
edits.push({ agent: member, rel: fileOf(member), before: w, after: setFrontmatterScalar(w, "description", description),
|
|
8443
|
+
edits.push({ agent: member, rel: fileOf(member), before: w, after: setFrontmatterScalar(w, "description", description), bodies: [{ set: { key: "description", value: description } }] });
|
|
8444
|
+
}
|
|
8445
|
+
if (isolated && !workerOwn) {
|
|
8446
|
+
// The worker's file once, whatever is set on it: one diff, one write,
|
|
8447
|
+
// one PATCH per key on the platform.
|
|
8448
|
+
const prior = edits.find((e) => e.agent === member);
|
|
8449
|
+
const set = { set: { key: "sandbox", value: "own" } };
|
|
8450
|
+
if (prior) {
|
|
8451
|
+
prior.after = setFrontmatterScalar(prior.after, "sandbox", "own");
|
|
8452
|
+
prior.bodies.push(set);
|
|
8453
|
+
} else {
|
|
8454
|
+
const w = desk.files.get(fileOf(member));
|
|
8455
|
+
edits.push({ agent: member, rel: fileOf(member), before: w, after: setFrontmatterScalar(w, "sandbox", "own"), bodies: [set] });
|
|
8456
|
+
}
|
|
8320
8457
|
}
|
|
8321
|
-
edits.push({ agent: name, rel: fileOf(name), before: raw, after: next,
|
|
8458
|
+
if (next !== raw) edits.push({ agent: name, rel: fileOf(name), before: raw, after: next, bodies: [{ list: { key, name: member, action: add ? "add" : "remove" } }] });
|
|
8322
8459
|
|
|
8323
8460
|
console.log(`\n ${c.bold(name)} ${c.dim(`· ${add ? "add" : "remove"} ${said} · ${desk.label}`)}\n`);
|
|
8324
8461
|
for (const e of edits) {
|
|
@@ -8341,7 +8478,9 @@ async function agentLinkCmd(positional, flags, layout) {
|
|
|
8341
8478
|
if (!desk.platform) for (const e of edits) assertUnchanged(desk, e.rel, e.before);
|
|
8342
8479
|
for (const e of edits) {
|
|
8343
8480
|
if (desk.platform) {
|
|
8344
|
-
|
|
8481
|
+
for (const body of e.bodies) {
|
|
8482
|
+
await remoteCall(desk.url, flags, `/api/workspaces/${enc(desk.ws)}/agents/${enc(e.agent)}`, { method: "PATCH", body: JSON.stringify(body) });
|
|
8483
|
+
}
|
|
8345
8484
|
} else {
|
|
8346
8485
|
writeIfUnchanged(desk, e.rel, e.before, e.after);
|
|
8347
8486
|
}
|
|
@@ -8373,10 +8512,12 @@ async function agentNetworkCmd(positional, flags, layout) {
|
|
|
8373
8512
|
}
|
|
8374
8513
|
const resolve = (files) =>
|
|
8375
8514
|
inTree(desk, files, async (dir) => {
|
|
8376
|
-
const { listAgents
|
|
8515
|
+
const { listAgents } = await core();
|
|
8516
|
+
// The folded view a run reads: `network` set inside a `guardrail:` block
|
|
8517
|
+
// counts the same as a top-level one (effectiveFront lifts it).
|
|
8377
8518
|
return {
|
|
8378
8519
|
info: listAgents("default", "workspace").find((a) => a.name === name),
|
|
8379
|
-
own: net.readNetwork(
|
|
8520
|
+
own: net.readNetwork((await effectiveFront(fs.readFileSync(path.join(dir, rel), "utf8"))).network).setting,
|
|
8380
8521
|
};
|
|
8381
8522
|
});
|
|
8382
8523
|
const show = (info) => {
|
|
@@ -8526,7 +8667,7 @@ async function flowShowCmd(positional, flags, layout) {
|
|
|
8526
8667
|
const a = byAgent.get(s.agent);
|
|
8527
8668
|
if (!a) return;
|
|
8528
8669
|
if (a.model) console.log(`${under}${p.dim(`model ${a.model}${a.tools?.length ? ` · ${a.tools.length} tool${a.tools.length === 1 ? "" : "s"}` : ""}`)}`);
|
|
8529
|
-
for (const w of a.subagents ?? []) console.log(`${under}↳ sub-agent ${w}`);
|
|
8670
|
+
for (const w of a.subagents ?? []) console.log(`${under}↳ sub-agent ${w}${ownSandboxOf(byAgent.get(w)) ? p.dim(" (own sandbox)") : ""}`);
|
|
8530
8671
|
for (const w of a.consults ?? []) console.log(`${under}· consults ${w}`);
|
|
8531
8672
|
if ((a.tools ?? []).includes("ask")) console.log(`${under}${p.amber("may ask you")}`);
|
|
8532
8673
|
for (const x of ofAgent(s.agent)) console.log(`${under}${mark(x)}`);
|
|
@@ -8647,10 +8788,16 @@ async function remoteEvalsCmd(name, flags, layout) {
|
|
|
8647
8788
|
if (!name) throw new Error("--run reads one eval's run — name it: `foldrun eval <name> --to <ws> --run <id>`");
|
|
8648
8789
|
}
|
|
8649
8790
|
const { evals = [] } = await remoteCall(url, flags, `/api/workspaces/${enc(ws)}/evals`);
|
|
8650
|
-
const
|
|
8651
|
-
if (!
|
|
8791
|
+
const named = evals.filter((e) => !name || e.name === name);
|
|
8792
|
+
if (!named.length) {
|
|
8652
8793
|
throw new Error(name ? `no eval "${name}" in ${ws} — it has ${evals.map((e) => e.name).join(", ") || "none"}` : `no evals in ${ws} on ${url}`);
|
|
8653
8794
|
}
|
|
8795
|
+
// Graders grade real runs; `eval grades` shows what they found.
|
|
8796
|
+
const all = named.filter((e) => e.trigger !== "runs");
|
|
8797
|
+
for (const g of named.filter((e) => e.trigger === "runs")) {
|
|
8798
|
+
console.log(`\n ${c.bold(g.name)} ${c.dim(`grades live runs — \`foldrun eval grades ${g.name} --to ${ws}\``)}`);
|
|
8799
|
+
}
|
|
8800
|
+
if (!all.length) return 0;
|
|
8654
8801
|
let failed = 0;
|
|
8655
8802
|
for (const info of all) {
|
|
8656
8803
|
console.log(`\n ${c.bold(info.name)} ${c.dim(`${info.cases?.length ?? "?"} cases · ${ws}`)}`);
|
|
@@ -8691,6 +8838,341 @@ async function remoteEvalsCmd(name, flags, layout) {
|
|
|
8691
8838
|
return failed ? 1 : 0;
|
|
8692
8839
|
}
|
|
8693
8840
|
|
|
8841
|
+
/** Is this eval a grader (`trigger: runs`)? A core that predates graders
|
|
8842
|
+
* reads the trigger as `deploy`, so the file is asked too. */
|
|
8843
|
+
function isGrader(info, dir) {
|
|
8844
|
+
if (info.trigger === "runs") return true;
|
|
8845
|
+
try {
|
|
8846
|
+
const raw = fs.readFileSync(path.join(dir, info.file), "utf8");
|
|
8847
|
+
const front = /^---\r?\n([\s\S]*?)\r?\n---/.exec(raw)?.[1] ?? "";
|
|
8848
|
+
return /^trigger:\s*["']?runs["']?\s*$/m.test(front);
|
|
8849
|
+
} catch {
|
|
8850
|
+
return false;
|
|
8851
|
+
}
|
|
8852
|
+
}
|
|
8853
|
+
|
|
8854
|
+
/** One grade, the way `eval grades` and `eval grade` print it. */
|
|
8855
|
+
function printGrade(g) {
|
|
8856
|
+
const mark = g.status === "pass" ? c.green("✓ pass") : g.status === "fail" ? c.red("✗ fail") : c.amber("? not decided");
|
|
8857
|
+
const miss = (g.checks ?? []).find((x) => !x.passed);
|
|
8858
|
+
const why = miss ? ` — ${miss.name}: ${miss.error ?? (miss.assertions ?? []).find((a) => !a.passed)?.detail ?? "failed"}` : "";
|
|
8859
|
+
console.log(` ${mark} ${c.bold(g.runId)} ${c.dim(String(g.at ?? "").slice(0, 16).replace("T", " "))}${why}`);
|
|
8860
|
+
}
|
|
8861
|
+
|
|
8862
|
+
/**
|
|
8863
|
+
* `foldrun eval grades <eval> --to <workspace>` — what a grader (an eval with
|
|
8864
|
+
* `trigger: runs`) found in the real runs it graded: the pass rate over
|
|
8865
|
+
* `--days` (default 30), then each graded run, newest first (`--status
|
|
8866
|
+
* fail` for the failures alone, `--limit`). GET …/evals/<e>/grades — the
|
|
8867
|
+
* Evals page's grader card. Exit 1 when any graded run in the window failed.
|
|
8868
|
+
*/
|
|
8869
|
+
async function evalGradesCmd(name, flags, layout) {
|
|
8870
|
+
if (!name) throw new Error("which grader? `foldrun eval grades <eval> --to <workspace>`");
|
|
8871
|
+
const url = platformFor(flags, "eval grades");
|
|
8872
|
+
const ws = platformWorkspace(flags, layout, `eval grades ${name}`);
|
|
8873
|
+
const q = new URLSearchParams();
|
|
8874
|
+
if (flags.days !== undefined) q.set("days", String(flags.days));
|
|
8875
|
+
if (typeof flags.status === "string") q.set("status", flags.status);
|
|
8876
|
+
if (flags.limit !== undefined) q.set("limit", String(flags.limit));
|
|
8877
|
+
const r = await remoteCall(url, flags, `/api/workspaces/${enc(ws)}/evals/${enc(name)}/grades${q.size ? `?${q}` : ""}`);
|
|
8878
|
+
if (flags.json) {
|
|
8879
|
+
console.log(JSON.stringify(r, null, 2));
|
|
8880
|
+
return r.summary?.failed ? 1 : 0;
|
|
8881
|
+
}
|
|
8882
|
+
const s = r.summary ?? {};
|
|
8883
|
+
const rate = s.rate === null || s.rate === undefined ? "—" : `${Math.round(s.rate * 100)}%`;
|
|
8884
|
+
console.log(`\n ${c.bold(r.eval)} ${c.dim(`grades ${r.sample}% of ${r.flow ? `flow ${r.flow}` : `agent ${r.agent}`} runs · ${ws}`)}`);
|
|
8885
|
+
console.log(` ${rate} passing over ${r.days} days — ${s.graded ?? 0} graded, ${s.passed ?? 0} passed, ${s.failed ?? 0} failed${s.errors ? `, ${s.errors} not decided` : ""} · judges $${Number(s.costUsd ?? 0).toFixed(4)}\n`);
|
|
8886
|
+
for (const g of r.grades ?? []) printGrade(g);
|
|
8887
|
+
if (!(r.grades ?? []).length) console.log(` ${c.dim("no graded runs in the window")}`);
|
|
8888
|
+
console.log("");
|
|
8889
|
+
return s.failed ? 1 : 0;
|
|
8890
|
+
}
|
|
8891
|
+
|
|
8892
|
+
/**
|
|
8893
|
+
* `foldrun eval grade <eval> --run <id> --to <workspace>` — grade one finished
|
|
8894
|
+
* run now, sampled or not (POST …/evals/<e>/grade { run }). The grade is kept
|
|
8895
|
+
* like any other; a run graded again shows its newer grade. Exit 1 unless it
|
|
8896
|
+
* passed.
|
|
8897
|
+
*/
|
|
8898
|
+
async function evalGradeCmd(name, flags, layout) {
|
|
8899
|
+
if (!name || typeof flags.run !== "string" || !flags.run.trim()) {
|
|
8900
|
+
throw new Error("`foldrun eval grade <eval> --run <run-id> --to <workspace>`");
|
|
8901
|
+
}
|
|
8902
|
+
const url = platformFor(flags, "eval grade");
|
|
8903
|
+
const ws = platformWorkspace(flags, layout, `eval grade ${name}`);
|
|
8904
|
+
const g = await remoteCall(url, flags, `/api/workspaces/${enc(ws)}/evals/${enc(name)}/grade`, { method: "POST", body: JSON.stringify({ run: flags.run.trim() }) });
|
|
8905
|
+
if (flags.json) console.log(JSON.stringify(g, null, 2));
|
|
8906
|
+
else {
|
|
8907
|
+
console.log("");
|
|
8908
|
+
printGrade(g);
|
|
8909
|
+
for (const ch of g.checks ?? []) {
|
|
8910
|
+
console.log(` ${ch.passed ? c.green("✓") : ch.error ? c.amber("?") : c.red("✗")} ${ch.name}${ch.error ? c.dim(` — ${ch.error}`) : ""}`);
|
|
8911
|
+
for (const a of ch.assertions ?? []) if (!a.passed) console.log(` ${c.dim(`${a.type}: ${a.value}`)} — ${a.detail}`);
|
|
8912
|
+
}
|
|
8913
|
+
console.log(` ${c.dim(`judges $${Number(g.costUsd ?? 0).toFixed(4)}`)}\n`);
|
|
8914
|
+
}
|
|
8915
|
+
return g.status === "pass" ? 0 : 1;
|
|
8916
|
+
}
|
|
8917
|
+
|
|
8918
|
+
/** One comparison, as a table on a terminal. */
|
|
8919
|
+
function printComparison(r) {
|
|
8920
|
+
const pct = (x) => (x === null || x === undefined ? "—" : `${Math.round(x * 100)}%`);
|
|
8921
|
+
const w = Math.max(...(r.configs ?? []).map((x) => String(x.label).length), 7);
|
|
8922
|
+
console.log(`\n ${c.bold(r.eval)} ${c.dim(`judge ${r.judge?.model ?? "?"} · spent $${Number(r.spentUsd ?? 0).toFixed(2)} of $${Number(r.maxUsd ?? 0).toFixed(2)}${r.repeat > 1 ? ` · ${r.repeat}× each` : ""}`)}`);
|
|
8923
|
+
if (r.stopped) console.log(` ${c.amber(`stopped early: ${r.stopped}`)}`);
|
|
8924
|
+
console.log(`\n ${c.dim(`${"setting".padEnd(w)} passing cost per run time`)}`);
|
|
8925
|
+
for (const x of r.configs ?? []) {
|
|
8926
|
+
const decided = x.trials - (x.notRun ?? 0);
|
|
8927
|
+
console.log(` ${String(x.label).padEnd(w)} ${pct(x.rate).padStart(4)} ${`${x.passed}/${decided}`.padEnd(7)} ${`$${Number(x.costUsd).toFixed(4)}`.padStart(9)} ${x.avgCostUsd === null ? " —" : `$${Number(x.avgCostUsd).toFixed(4)}`.padStart(8)} ${x.avgSecs === null ? "—" : `${x.avgSecs}s`}${x.notRun ? c.dim(` (${x.notRun} not run)`) : ""}`);
|
|
8928
|
+
}
|
|
8929
|
+
const differ = (r.cases ?? []).filter((row) => new Set(row.passed.map((p, i) => p / (row.trials[i] || 1))).size > 1);
|
|
8930
|
+
if (differ.length) {
|
|
8931
|
+
console.log(`\n ${c.dim("where they differ:")}`);
|
|
8932
|
+
for (const row of differ) console.log(` ${row.case}: ${row.passed.map((p, i) => `${r.configs[i].label} ${p}/${row.trials[i]}`).join(" · ")}`);
|
|
8933
|
+
}
|
|
8934
|
+
console.log("");
|
|
8935
|
+
}
|
|
8936
|
+
|
|
8937
|
+
/**
|
|
8938
|
+
* `foldrun eval compare <eval> --models fast,default,max --to <workspace>` —
|
|
8939
|
+
* an eval's cases under two to four models, side by side (POST
|
|
8940
|
+
* …/evals/<e>/compare, then polled). `--models` takes model[:effort] items
|
|
8941
|
+
* (`fast:low,max:high`); `--effort` is the effort for the items that name
|
|
8942
|
+
* none; `--agent <name>` compares only that agent's steps of a flow eval;
|
|
8943
|
+
* `--repeat` (1–5) runs each case more than once; `--max-usd` is the spend
|
|
8944
|
+
* limit (default 5). Every run is a test run. Waits for the result
|
|
8945
|
+
* (`--timeout <s>`, default 3600; past it, the id to read it with).
|
|
8946
|
+
* `--run <id>` reads one comparison; `--list` the kept ones. The Evals
|
|
8947
|
+
* page's Compare models.
|
|
8948
|
+
*/
|
|
8949
|
+
async function evalCompareCmd(name, flags, layout) {
|
|
8950
|
+
if (!name) throw new Error("which eval? `foldrun eval compare <eval> --models fast,default --to <workspace>`");
|
|
8951
|
+
const url = platformFor(flags, "eval compare");
|
|
8952
|
+
const ws = platformWorkspace(flags, layout, `eval compare ${name}`);
|
|
8953
|
+
const base = `/api/workspaces/${enc(ws)}/evals/${enc(name)}/compare`;
|
|
8954
|
+
if (flags.list === true) {
|
|
8955
|
+
const r = await remoteCall(url, flags, base);
|
|
8956
|
+
if (flags.json) return console.log(JSON.stringify(r, null, 2)), 0;
|
|
8957
|
+
if (r.running) console.log(`\n ${c.dim(`running: ${r.runId}, since ${r.startedAt}${r.progress ? ` — ${r.progress.done}/${r.progress.total} runs` : ""}`)}`);
|
|
8958
|
+
for (const p of r.comparisons ?? []) {
|
|
8959
|
+
const line = p.status === "done" && p.result ? p.result.configs.map((x) => `${x.label} ${x.rate === null ? "—" : `${Math.round(x.rate * 100)}%`}`).join(" · ") : c.red(p.error ?? "failed");
|
|
8960
|
+
console.log(` ${c.bold(p.runId)} ${c.dim(String(p.startedAt).slice(0, 16).replace("T", " "))} ${line}`);
|
|
8961
|
+
}
|
|
8962
|
+
if (!(r.comparisons ?? []).length && !r.running) console.log(`\n ${c.dim(`no comparisons of ${name} yet`)}`);
|
|
8963
|
+
console.log("");
|
|
8964
|
+
return 0;
|
|
8965
|
+
}
|
|
8966
|
+
const finish = (view) => {
|
|
8967
|
+
if (view.state === "done" && view.result) {
|
|
8968
|
+
if (flags.json) console.log(JSON.stringify(view.result, null, 2));
|
|
8969
|
+
else printComparison(view.result);
|
|
8970
|
+
return 0;
|
|
8971
|
+
}
|
|
8972
|
+
throw new Error(view.error ?? `the comparison ended as ${view.state ?? "unknown"}, with no result`);
|
|
8973
|
+
};
|
|
8974
|
+
// --run reads one comparison and starts nothing: a bare --run (an empty
|
|
8975
|
+
// variable in a script) is refused, never read as "start a new one".
|
|
8976
|
+
if (flags.run !== undefined && (typeof flags.run !== "string" || !flags.run.trim())) {
|
|
8977
|
+
throw new Error("--run takes the comparison id it printed — `foldrun eval compare <eval> --to <ws> --run <id>`");
|
|
8978
|
+
}
|
|
8979
|
+
if (typeof flags.run === "string") {
|
|
8980
|
+
const view = await remoteCall(url, flags, `${base}?run=${encodeURIComponent(flags.run)}`);
|
|
8981
|
+
if (view.state === "running") {
|
|
8982
|
+
if (flags.json) console.log(JSON.stringify(view, null, 2));
|
|
8983
|
+
else console.log(`\n ${c.dim(`still running, since ${view.startedAt}${view.progress ? ` — ${view.progress.done}/${view.progress.total} runs` : ""}`)}\n`);
|
|
8984
|
+
return 1;
|
|
8985
|
+
}
|
|
8986
|
+
return finish(view);
|
|
8987
|
+
}
|
|
8988
|
+
if (typeof flags.models !== "string" || !flags.models.trim()) {
|
|
8989
|
+
throw new Error("--models takes two to four settings, e.g. --models fast,default or --models fast:low,max:high");
|
|
8990
|
+
}
|
|
8991
|
+
const effort = typeof flags.effort === "string" ? flags.effort : undefined;
|
|
8992
|
+
const agent = typeof flags.agent === "string" ? flags.agent : undefined;
|
|
8993
|
+
const configs = flags.models.split(",").map((item) => item.trim()).filter(Boolean).map((item) => {
|
|
8994
|
+
// Only a trailing known effort is an effort: llama3.2:3b stays one model id.
|
|
8995
|
+
const m = /^(.*):(low|medium|high|xhigh|max)$/.exec(item);
|
|
8996
|
+
const model = m ? m[1] : item;
|
|
8997
|
+
const e = m ? m[2] : undefined;
|
|
8998
|
+
return { model, ...((e ?? effort) ? { effort: e ?? effort } : {}), ...(agent ? { agent } : {}) };
|
|
8999
|
+
});
|
|
9000
|
+
const body = { configs, ...(flags.repeat !== undefined ? { repeat: Number(flags.repeat) } : {}), ...(flags["max-usd"] !== undefined ? { maxUsd: Number(flags["max-usd"]) } : {}) };
|
|
9001
|
+
const started = await remoteCall(url, flags, base, { method: "POST", body: JSON.stringify(body) });
|
|
9002
|
+
// --json is one JSON document on stdout; what happens on the way goes to stderr.
|
|
9003
|
+
const say = flags.json ? (s) => console.error(s) : (s) => console.log(s);
|
|
9004
|
+
say(`\n ${c.dim(`comparison ${started.runId}: ${started.trials} test runs, at most $${Number(started.maxUsd).toFixed(2)}`)}`);
|
|
9005
|
+
const wait = explicitTimeout(flags) ?? 3600;
|
|
9006
|
+
const deadline = Date.now() + wait * 1000;
|
|
9007
|
+
let said = -1;
|
|
9008
|
+
for (;;) {
|
|
9009
|
+
if (Date.now() > deadline) {
|
|
9010
|
+
throw new Error(`still running after ${wait}s — \`foldrun eval compare ${name} --to ${ws} --run ${started.runId}\` reads it when it is done`);
|
|
9011
|
+
}
|
|
9012
|
+
await new Promise((r) => setTimeout(r, Number(process.env.FOLDRUN_EVAL_POLL_MS) || 5000));
|
|
9013
|
+
const view = await remoteCall(url, flags, `${base}?run=${encodeURIComponent(started.runId)}`);
|
|
9014
|
+
if (view.state === "running") {
|
|
9015
|
+
if (view.progress && view.progress.done !== said) {
|
|
9016
|
+
said = view.progress.done;
|
|
9017
|
+
say(` ${c.dim(`${view.progress.done}/${view.progress.total} runs done`)}`);
|
|
9018
|
+
}
|
|
9019
|
+
continue;
|
|
9020
|
+
}
|
|
9021
|
+
return finish(view);
|
|
9022
|
+
}
|
|
9023
|
+
}
|
|
9024
|
+
|
|
9025
|
+
// ------------------------------------------------------------- proposals
|
|
9026
|
+
|
|
9027
|
+
/**
|
|
9028
|
+
* A file's change, as ordered -/+ lines with a little context — the same
|
|
9029
|
+
* reading the dashboard's diff gives. An LCS over the lines, so a reordered
|
|
9030
|
+
* step or a removed duplicate shows; a file too long for that (over 2,000
|
|
9031
|
+
* lines a side) is shown whole as removed and added rather than guessed at.
|
|
9032
|
+
*/
|
|
9033
|
+
function lineChanges(before, after) {
|
|
9034
|
+
const a = before === null || before === undefined ? [] : String(before).split("\n");
|
|
9035
|
+
const b = String(after ?? "").split("\n");
|
|
9036
|
+
if (a.length > 2000 || b.length > 2000) return [...a.map((l) => ["-", l]), ...b.map((l) => ["+", l])];
|
|
9037
|
+
const n = a.length;
|
|
9038
|
+
const m = b.length;
|
|
9039
|
+
const lcs = Array.from({ length: n + 1 }, () => new Uint16Array(m + 1));
|
|
9040
|
+
for (let i = n - 1; i >= 0; i--) for (let j = m - 1; j >= 0; j--) lcs[i][j] = a[i] === b[j] ? lcs[i + 1][j + 1] + 1 : Math.max(lcs[i + 1][j], lcs[i][j + 1]);
|
|
9041
|
+
const out = [];
|
|
9042
|
+
let i = 0;
|
|
9043
|
+
let j = 0;
|
|
9044
|
+
while (i < n || j < m) {
|
|
9045
|
+
if (i < n && j < m && a[i] === b[j]) {
|
|
9046
|
+
out.push([" ", a[i]]);
|
|
9047
|
+
i++;
|
|
9048
|
+
j++;
|
|
9049
|
+
} else if (j < m && (i === n || lcs[i][j + 1] >= lcs[i + 1][j])) out.push(["+", b[j++]]);
|
|
9050
|
+
else out.push(["-", a[i++]]);
|
|
9051
|
+
}
|
|
9052
|
+
return out;
|
|
9053
|
+
}
|
|
9054
|
+
|
|
9055
|
+
function printFileChange(f) {
|
|
9056
|
+
console.log(`\n ${c.bold(f.path)}${f.before === null ? c.dim(" (new)") : ""}`);
|
|
9057
|
+
const ops = lineChanges(f.before, f.content);
|
|
9058
|
+
const near = (k) => ops.slice(Math.max(0, k - 2), k + 3).some(([op]) => op !== " ");
|
|
9059
|
+
let skipped = false;
|
|
9060
|
+
ops.forEach(([op, line], k) => {
|
|
9061
|
+
if (op === " " && !near(k)) {
|
|
9062
|
+
if (!skipped) console.log(` ${c.dim(" …")}`);
|
|
9063
|
+
skipped = true;
|
|
9064
|
+
return;
|
|
9065
|
+
}
|
|
9066
|
+
skipped = false;
|
|
9067
|
+
console.log(` ${op === "+" ? c.green(`+ ${line}`) : op === "-" ? c.red(`- ${line}`) : c.dim(` ${line}`)}`);
|
|
9068
|
+
});
|
|
9069
|
+
}
|
|
9070
|
+
|
|
9071
|
+
const PROPOSAL_LABEL = { drafting: "being drafted", ready: "ready to review", applying: "applying", applied: "applied", partial: "partly applied", failed: "draft failed", dismissed: "dismissed" };
|
|
9072
|
+
|
|
9073
|
+
/**
|
|
9074
|
+
* `foldrun proposals --to <workspace>` — the proposed fixes in a workspace
|
|
9075
|
+
* (GET …/proposals), newest first: status, the eval, how many failures each
|
|
9076
|
+
* covers. `--status ready` for the ones waiting on a person, `--eval <name>`,
|
|
9077
|
+
* `--json`. `show <id>` prints one (the assistant's reasoning, the failures,
|
|
9078
|
+
* the change per file); `apply <id>` writes its files after asking (`--yes`
|
|
9079
|
+
* without a terminal); `dismiss <id>` sets it aside. The Evals page's
|
|
9080
|
+
* Proposed fixes.
|
|
9081
|
+
*/
|
|
9082
|
+
async function proposalsCmd(positional, flags, layout) {
|
|
9083
|
+
const url = platformFor(flags, "proposals");
|
|
9084
|
+
const ws = platformWorkspace(flags, layout, "proposals");
|
|
9085
|
+
const verb = positional[0];
|
|
9086
|
+
const id = positional[1];
|
|
9087
|
+
if (verb === "show" || verb === "apply" || verb === "dismiss") {
|
|
9088
|
+
if (!id) throw new Error(`which proposal? \`foldrun proposals ${verb} <id> --to ${ws}\``);
|
|
9089
|
+
const p = await remoteCall(url, flags, `/api/workspaces/${enc(ws)}/proposals/${enc(id)}`);
|
|
9090
|
+
if (verb === "show") {
|
|
9091
|
+
if (flags.json) return console.log(JSON.stringify(p, null, 2)), 0;
|
|
9092
|
+
console.log(`\n ${c.bold(p.id)} ${PROPOSAL_LABEL[p.status] ?? p.status} · ${p.eval} · ${c.dim(`${String(p.createdAt).slice(0, 16).replace("T", " ")} · ${p.by === "auto" ? "opened after repeated failures" : `asked by ${p.by}`}`)}`);
|
|
9093
|
+
if (p.error) console.log(` ${c.red(p.error)}`);
|
|
9094
|
+
for (const n of p.notes ?? []) console.log(` ${c.amber(n)}`);
|
|
9095
|
+
console.log(`\n ${c.dim("covers:")}`);
|
|
9096
|
+
for (const f of p.failures ?? []) console.log(` · ${f.runId ?? f.case} — ${f.check}: ${f.detail}`);
|
|
9097
|
+
if (p.reply) console.log(`\n${p.reply.split("\n").map((l) => ` ${l}`).join("\n")}`);
|
|
9098
|
+
for (const f of p.files ?? []) printFileChange(f);
|
|
9099
|
+
for (const i of p.issues ?? []) console.log(` ${i.level === "error" ? c.red("✗") : c.amber("!")} ${i.where}: ${i.message}`);
|
|
9100
|
+
console.log("");
|
|
9101
|
+
return 0;
|
|
9102
|
+
}
|
|
9103
|
+
if (verb === "apply") {
|
|
9104
|
+
if (p.status !== "ready") throw new Error(`${id} is ${p.status} — only a ready proposal can be applied`);
|
|
9105
|
+
// With --json the change is still shown before the yes — on stderr, so stdout stays one document.
|
|
9106
|
+
if (flags.json) {
|
|
9107
|
+
const log = console.log;
|
|
9108
|
+
console.log = (...a) => console.error(...a);
|
|
9109
|
+
try {
|
|
9110
|
+
for (const f of p.files ?? []) printFileChange(f);
|
|
9111
|
+
} finally {
|
|
9112
|
+
console.log = log;
|
|
9113
|
+
}
|
|
9114
|
+
} else for (const f of p.files ?? []) printFileChange(f);
|
|
9115
|
+
const files = (p.files ?? []).map((f) => f.path).join(", ");
|
|
9116
|
+
if (!(await confirmed(flags, "proposals apply", `\n Write ${files} in ${ws}? [y/N] `))) {
|
|
9117
|
+
console.log(`\n ${c.dim("nothing written")}\n`);
|
|
9118
|
+
return 1;
|
|
9119
|
+
}
|
|
9120
|
+
}
|
|
9121
|
+
const r = await remoteCall(url, flags, `/api/workspaces/${enc(ws)}/proposals/${enc(id)}`, { method: "POST", body: JSON.stringify({ action: verb }) });
|
|
9122
|
+
if (flags.json) console.log(JSON.stringify(r, null, 2));
|
|
9123
|
+
else console.log(`\n ${c.green(PROPOSAL_LABEL[r.status] ?? r.status)} ${r.id}${r.applied?.length ? c.dim(` — wrote ${r.applied.join(", ")}`) : ""}\n`);
|
|
9124
|
+
return 0;
|
|
9125
|
+
}
|
|
9126
|
+
if (verb && verb !== "ls" && verb !== "list") throw new Error(`\`foldrun proposals\` lists them; show, apply and dismiss take an id — not "${verb}"`);
|
|
9127
|
+
const q = new URLSearchParams();
|
|
9128
|
+
if (typeof flags.status === "string") q.set("status", flags.status);
|
|
9129
|
+
if (typeof flags.eval === "string") q.set("eval", flags.eval);
|
|
9130
|
+
const { proposals = [] } = await remoteCall(url, flags, `/api/workspaces/${enc(ws)}/proposals${q.size ? `?${q}` : ""}`);
|
|
9131
|
+
if (flags.json) return console.log(JSON.stringify(proposals, null, 2)), 0;
|
|
9132
|
+
if (!proposals.length) {
|
|
9133
|
+
console.log(`\n ${c.dim(`no proposed fixes in ${ws}`)}\n`);
|
|
9134
|
+
return 0;
|
|
9135
|
+
}
|
|
9136
|
+
console.log("");
|
|
9137
|
+
for (const p of proposals) {
|
|
9138
|
+
const tone = p.status === "ready" ? c.amber : p.status === "applied" ? c.green : p.status === "failed" || p.status === "partial" ? c.red : c.dim;
|
|
9139
|
+
console.log(` ${c.bold(p.id)} ${tone((PROPOSAL_LABEL[p.status] ?? p.status).padEnd(15))} ${p.eval} ${c.dim(`· ${p.failures?.length ?? 0} failure(s) · ${(p.files ?? []).map((f) => f.path).join(", ") || "no files"} · ${String(p.createdAt).slice(0, 10)}`)}`);
|
|
9140
|
+
}
|
|
9141
|
+
console.log("");
|
|
9142
|
+
return 0;
|
|
9143
|
+
}
|
|
9144
|
+
|
|
9145
|
+
/**
|
|
9146
|
+
* `foldrun eval improve <eval> --to <workspace>` — ask for a proposed fix now
|
|
9147
|
+
* (POST …/evals/<e>/improve): from a grader's failed grades no proposal has
|
|
9148
|
+
* covered, or an eval's last failed cases. Waits for the draft (`--timeout`,
|
|
9149
|
+
* default 300) and prints it; `--no-wait` returns at once with the id.
|
|
9150
|
+
*/
|
|
9151
|
+
async function evalImproveCmd(name, flags, layout) {
|
|
9152
|
+
if (!name) throw new Error("which eval? `foldrun eval improve <eval> --to <workspace>`");
|
|
9153
|
+
const url = platformFor(flags, "eval improve");
|
|
9154
|
+
const ws = platformWorkspace(flags, layout, `eval improve ${name}`);
|
|
9155
|
+
const p = await remoteCall(url, flags, `/api/workspaces/${enc(ws)}/evals/${enc(name)}/improve`, { method: "POST", body: "{}" });
|
|
9156
|
+
const say = flags.json ? (s) => console.error(s) : (s) => console.log(s);
|
|
9157
|
+
say(`\n ${c.dim(`proposal ${p.id} — the assistant is drafting it from ${p.failures?.length ?? 0} failure(s)`)}`);
|
|
9158
|
+
if (flags.wait === false || flags["no-wait"] === true) {
|
|
9159
|
+
if (flags.json) console.log(JSON.stringify(p, null, 2));
|
|
9160
|
+
else say(` ${c.dim(`\`foldrun proposals show ${p.id} --to ${ws}\` when it is ready`)}\n`);
|
|
9161
|
+
return 0;
|
|
9162
|
+
}
|
|
9163
|
+
const deadline = Date.now() + (explicitTimeout(flags) ?? 300) * 1000;
|
|
9164
|
+
for (;;) {
|
|
9165
|
+
await new Promise((r) => setTimeout(r, Number(process.env.FOLDRUN_EVAL_POLL_MS) || 4000));
|
|
9166
|
+
const now = await remoteCall(url, flags, `/api/workspaces/${enc(ws)}/proposals/${enc(p.id)}`);
|
|
9167
|
+
if (now.status !== "drafting") return proposalsCmd(["show", p.id], flags, layout).then(() => (now.status === "ready" ? 0 : 1));
|
|
9168
|
+
if (Date.now() > deadline) {
|
|
9169
|
+
if (flags.json) console.log(JSON.stringify(now, null, 2));
|
|
9170
|
+
else say(` ${c.dim(`still drafting — \`foldrun proposals show ${p.id} --to ${ws}\` later`)}\n`);
|
|
9171
|
+
return 1;
|
|
9172
|
+
}
|
|
9173
|
+
}
|
|
9174
|
+
}
|
|
9175
|
+
|
|
8694
9176
|
/**
|
|
8695
9177
|
* `foldrun promote <run-id>` — a finished run, kept as a regression case:
|
|
8696
9178
|
* its task, and what the next run must still say.
|
|
@@ -9910,6 +10392,54 @@ async function flowTriggerCmd(positional, flags, layout) {
|
|
|
9910
10392
|
});
|
|
9911
10393
|
}
|
|
9912
10394
|
|
|
10395
|
+
/**
|
|
10396
|
+
* `foldrun flow filter <flow> --payload <file|-|json> [--to <workspace>]` —
|
|
10397
|
+
* would this delivery start the flow? Its `filter:` (core trigger-filter.ts)
|
|
10398
|
+
* against a payload: a webhook's JSON body, an email's { from, to, subject,
|
|
10399
|
+
* text }, a chained run's { run, flow, status, ending, verdict, summary }, a
|
|
10400
|
+
* storage event's { by, paths }. With --to, the deployed flow answers (POST
|
|
10401
|
+
* …/flows/<flow>/filter — the Triggers page's Test box); without, the local
|
|
10402
|
+
* file. Nothing starts. Exit 0 when it would start a run, 1 when not.
|
|
10403
|
+
*/
|
|
10404
|
+
async function flowFilterCmd(positional, flags, layout) {
|
|
10405
|
+
const flow = positional[1];
|
|
10406
|
+
if (!flow || flags.payload === undefined || flags.payload === true) {
|
|
10407
|
+
throw new Error("`foldrun flow filter <flow> --payload <file.json|->` (or inline JSON); --to <workspace> tests the deployed flow");
|
|
10408
|
+
}
|
|
10409
|
+
const spec = String(flags.payload);
|
|
10410
|
+
const text = spec === "-" ? fs.readFileSync(0, "utf8") : /^\s*[{[]/.test(spec) ? spec : fs.readFileSync(spec, "utf8");
|
|
10411
|
+
let payload;
|
|
10412
|
+
try {
|
|
10413
|
+
payload = JSON.parse(text);
|
|
10414
|
+
} catch {
|
|
10415
|
+
payload = text; // read as a webhook body: not JSON, so no filter matches
|
|
10416
|
+
}
|
|
10417
|
+
let answer;
|
|
10418
|
+
if (namesPlatform(flags)) {
|
|
10419
|
+
const url = platformFor(flags, "flow filter");
|
|
10420
|
+
const ws = platformWorkspace(flags, layout, "flow filter");
|
|
10421
|
+
answer = await remoteCall(url, flags, `/api/workspaces/${enc(ws)}/flows/${enc(flow)}/filter`, { method: "POST", body: JSON.stringify({ payload }) });
|
|
10422
|
+
} else {
|
|
10423
|
+
const desk = await openDesk(flags, layout, `flow filter ${flow}`, flow);
|
|
10424
|
+
const rel = await flowFileOf(desk, flow);
|
|
10425
|
+
const lib = /** @type {any} */ (await core());
|
|
10426
|
+
if (typeof lib.matchFilter !== "function") {
|
|
10427
|
+
throw new Error("this install's @foldrun/core predates filter: — update foldrun, or test the deployed flow with --to <workspace>");
|
|
10428
|
+
}
|
|
10429
|
+
const f = lib.parseFlow(path.basename(rel), desk.files.get(rel));
|
|
10430
|
+
const read = typeof payload === "string" ? lib.webhookFilterPayload(payload) : payload;
|
|
10431
|
+
answer = { flow: f.name, trigger: f.trigger, filter: f.filter, applies: lib.FILTERABLE_TRIGGERS.includes(f.trigger), ...(f.filter != null ? lib.matchFilter(f.filter, read) : { match: true, reason: "this flow has no filter: — every delivery starts it" }) };
|
|
10432
|
+
}
|
|
10433
|
+
if (flags.json) {
|
|
10434
|
+
console.log(JSON.stringify(answer, null, 2));
|
|
10435
|
+
} else {
|
|
10436
|
+
console.log(`\n ${answer.match ? c.green("would start a run") : c.amber("would not start a run")} — ${answer.reason}`);
|
|
10437
|
+
if (answer.applies === false) console.log(` ${c.dim(`trigger: ${answer.trigger} delivers no payload, so its filter never applies`)}`);
|
|
10438
|
+
console.log("");
|
|
10439
|
+
}
|
|
10440
|
+
return answer.match ? 0 : 1;
|
|
10441
|
+
}
|
|
10442
|
+
|
|
9913
10443
|
/**
|
|
9914
10444
|
* `foldrun flow move-step <flow> --step <n|agent> --group <g> | --after <g>`
|
|
9915
10445
|
* — dragging a card on the board: into group g, in parallel with what is
|
|
@@ -10524,13 +11054,15 @@ async function oauthClientsCmd(positional, flags) {
|
|
|
10524
11054
|
return 0;
|
|
10525
11055
|
}
|
|
10526
11056
|
if (verb === "add") {
|
|
10527
|
-
if (!name || !/^[a-z0-9][a-z0-9-]*$/.test(name)) throw new Error("`foldrun secrets clients add <name> --
|
|
11057
|
+
if (!name || !/^[a-z0-9][a-z0-9-]*$/.test(name)) throw new Error("`foldrun secrets clients add <name> --authorize-url … --token-url … --client-id … [--authorize-param name=value …] [--scopes …]` — the name in kebab-case");
|
|
11058
|
+
warnProviderDeprecated(flags);
|
|
10528
11059
|
const { OAUTH_PRESETS } = await core();
|
|
10529
11060
|
const preset = flags.provider ? OAUTH_PRESETS[flags.provider] : undefined;
|
|
10530
11061
|
if (flags.provider && !preset) throw new Error(`unknown provider "${flags.provider}" — one of ${Object.keys(OAUTH_PRESETS).join(", ")}, or pass --authorize-url and --token-url`);
|
|
10531
11062
|
const authorize_url = flags["authorize-url"] ?? preset?.authorize_url;
|
|
10532
11063
|
const token_url = flags["token-url"] ?? preset?.token_url;
|
|
10533
|
-
if (!authorize_url || !token_url) throw new Error("--
|
|
11064
|
+
if (!authorize_url || !token_url) throw new Error("--authorize-url and --token-url — the provider's own addresses, from its OAuth docs");
|
|
11065
|
+
const authorize_extra = { ...(preset?.authorize_extra ?? {}), ...authorizeParams(flags) };
|
|
10534
11066
|
const client_id = flags["client-id"] ?? env("OAUTH_CLIENT_ID") ?? (await promptVisible(" client_id: "));
|
|
10535
11067
|
const pkce = flags["no-pkce"] !== true;
|
|
10536
11068
|
const client_secret = flags["client-secret"] ?? env("OAUTH_CLIENT_SECRET") ?? (await promptHidden(` client_secret${pkce ? " (empty for a public app)" : ""}: `));
|
|
@@ -10539,7 +11071,7 @@ async function oauthClientsCmd(positional, flags) {
|
|
|
10539
11071
|
if (flags.issuer !== undefined && (typeof flags.issuer !== "string" || !/^https:\/\/[^?#]+$/.test(flags.issuer))) throw new Error("--issuer is the provider's https issuer URL, no query or fragment");
|
|
10540
11072
|
const config = {
|
|
10541
11073
|
authorize_url, token_url, client_id, client_secret: client_secret ?? "", scopes: flags.scopes ?? preset?.scopes_example ?? "",
|
|
10542
|
-
...(
|
|
11074
|
+
...(Object.keys(authorize_extra).length ? { authorize_extra } : {}),
|
|
10543
11075
|
...(pkce ? {} : { pkce: false }),
|
|
10544
11076
|
...(typeof flags.issuer === "string" ? { issuer: flags.issuer } : {}),
|
|
10545
11077
|
};
|
|
@@ -11218,6 +11750,13 @@ export async function run(command, positional, flags, workspace, layout) {
|
|
|
11218
11750
|
// `eval new <name>` makes one on the platform; an eval called "new"
|
|
11219
11751
|
// still runs as `foldrun eval new` with nothing after it.
|
|
11220
11752
|
if (positional[0] === "new" && positional[1]) return evalNewCmd(positional[1], flags, layout);
|
|
11753
|
+
// `eval grades <name>` / `eval grade <name> --run <id>`: a grader's
|
|
11754
|
+
// findings, or one run graded now (an eval called "grades" or "grade"
|
|
11755
|
+
// still runs as `foldrun eval grades` with nothing after it).
|
|
11756
|
+
if (positional[0] === "grades" && positional[1]) return evalGradesCmd(positional[1], flags, layout);
|
|
11757
|
+
if (positional[0] === "grade" && positional[1]) return evalGradeCmd(positional[1], flags, layout);
|
|
11758
|
+
if (positional[0] === "compare" && positional[1]) return evalCompareCmd(positional[1], flags, layout);
|
|
11759
|
+
if (positional[0] === "improve" && positional[1]) return evalImproveCmd(positional[1], flags, layout);
|
|
11221
11760
|
// --to names a deployed workspace: its evals run there.
|
|
11222
11761
|
if (typeof flags.to === "string") return remoteEvalsCmd(positional[0], flags, layout);
|
|
11223
11762
|
// --last reads the platform's results; a local eval has none to read,
|
|
@@ -11227,6 +11766,9 @@ export async function run(command, positional, flags, workspace, layout) {
|
|
|
11227
11766
|
return runEvals(positional[0], flags);
|
|
11228
11767
|
case "promote":
|
|
11229
11768
|
return promoteCmd(positional[0], flags, layout);
|
|
11769
|
+
case "proposals":
|
|
11770
|
+
case "proposal":
|
|
11771
|
+
return proposalsCmd(positional, flags, layout);
|
|
11230
11772
|
case "observe":
|
|
11231
11773
|
return observeCmd(flags, layout);
|
|
11232
11774
|
case "usage":
|
|
@@ -11288,6 +11830,8 @@ export async function run(command, positional, flags, workspace, layout) {
|
|
|
11288
11830
|
return approvalsCmd(flags, layout);
|
|
11289
11831
|
case "issues":
|
|
11290
11832
|
return issuesCmd(positional, flags, layout);
|
|
11833
|
+
case "inbox":
|
|
11834
|
+
return inboxCmd(positional, flags);
|
|
11291
11835
|
case "insights":
|
|
11292
11836
|
return insightsCmd(flags);
|
|
11293
11837
|
case "approve":
|
|
@@ -11365,6 +11909,7 @@ export async function run(command, positional, flags, workspace, layout) {
|
|
|
11365
11909
|
if (positional[0] === "hook") return flowHookCmd(positional, flags, layout);
|
|
11366
11910
|
if (positional[0] === "set") return flowSetCmd(positional, flags, layout);
|
|
11367
11911
|
if (positional[0] === "trigger") return flowTriggerCmd(positional, flags, layout);
|
|
11912
|
+
if (positional[0] === "filter") return flowFilterCmd(positional, flags, layout);
|
|
11368
11913
|
if (positional[0] === "move-step") return flowMoveStepCmd(positional, flags, layout);
|
|
11369
11914
|
if (positional[0] === "copy-step") return flowCopyStepCmd(positional, flags, layout);
|
|
11370
11915
|
if (positional[0] === "paste") return flowPasteCmd(positional, flags, layout);
|