foldrun 2.3.0 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/foldrun.mjs +16 -7
- package/docs/agents.md +22 -0
- package/docs/api.md +9 -6
- package/docs/budgets-and-billing.md +1 -1
- package/docs/checks.md +18 -3
- package/docs/cli.md +8 -6
- package/docs/evals.md +2 -2
- package/docs/flows.md +10 -1
- package/docs/four-jobs.md +95 -0
- package/docs/network.md +1 -1
- package/docs/notifications.md +6 -1
- package/docs/runs.md +1 -1
- package/docs/secrets.md +99 -0
- package/docs/workspaces.md +6 -1
- package/package.json +3 -3
- package/src/commands.mjs +300 -21
package/bin/foldrun.mjs
CHANGED
|
@@ -135,7 +135,7 @@ const HELP = `foldrun — agents are just folders
|
|
|
135
135
|
foldrun hook rm <name> delete a hook folder — says which agents still turn it on, asks (--yes, --library)
|
|
136
136
|
foldrun extract [dir] move single-file script tools into folders (tool.md + run.*)
|
|
137
137
|
foldrun run <target> run an agent or flow HERE (target: name, or flow:name) — on the platform: flow run / invoke a flow, agent run one agent
|
|
138
|
-
foldrun eval [name] run one eval, or all of them (--to <workspace>: the deployed evals, run on the platform)
|
|
138
|
+
foldrun eval [name] run one eval, or all of them (--to <workspace>: the deployed evals, run on the platform; --last reads their latest results, --run <id> one run's verdict, running nothing)
|
|
139
139
|
foldrun eval new <name> a new eval in the deployed workspace, its agent the first one there — edit its cases, then eval <name> --to (--to)
|
|
140
140
|
foldrun promote <run-id> keep a finished run as a regression case in evals/ (--eval <name>, --expect "contains: …" repeatable, --case <name>); a run that went wrong takes --should "what it should have done"
|
|
141
141
|
foldrun probe <model> live check: can this model hold a tool loop here?
|
|
@@ -183,10 +183,12 @@ const HELP = `foldrun — agents are just folders
|
|
|
183
183
|
foldrun billing card Stripe's page for saving a card for auto top-up (owner only)
|
|
184
184
|
foldrun billing credit <usd> put credit on the balance by hand (--note "…") — only on an install without Stripe (self-hosted, dev); refused once billing is live
|
|
185
185
|
foldrun gallery the platform's built-in tools (web) and whether you keep your own copy — also pull (a copy for offline runs), upgrade <tool>
|
|
186
|
-
foldrun
|
|
186
|
+
foldrun identity what your cloud must trust for keyless identities: the issuer and each agent's subject (--to <workspace>, --json)
|
|
187
|
+
foldrun jobs the four jobs — governance, guardrail, observability, evaluation — and what each has set (--to <workspace>, --json)
|
|
188
|
+
foldrun account the account's defaults — also set <key> <value>, clear <key> (timezone, notify, budget, concurrency, deploy_block none|secrets|security, network_enforce on|off, owner, alerts <threshold> <value>, judge model|effort <value>; or <job>.<field>, e.g. governance.owner), pause [--reason] [--stop] / resume (the kill switch), telemetry [set <url> --header name=\${SECRET} | clear | test] (runs as OpenTelemetry traces), providers (--check) (singular; accounts lists logins)
|
|
187
189
|
foldrun account export everything the platform holds about the account as one JSON file (owner only; --file <path>, --force)
|
|
188
190
|
foldrun account model the account's own model key — set <provider> --key <api key> (--key - reads stdin; custom --base-url --format), remove
|
|
189
|
-
foldrun workspace one deployed workspace's settings (--to) — also set <name|description|timezone|budget|notify|network_enforce> <value
|
|
191
|
+
foldrun workspace one deployed workspace's settings (--to) — also set <name|description|timezone|budget|notify|network_enforce|owner|alerts|judge> <value> (or <job>.<field>), clear <key>, pause [--reason] [--stop] / resume (the kill switch)
|
|
190
192
|
foldrun workspace vocabulary the names the dashboard's editor completes in a workspace: agents, flows, skills, tools, secrets, scripts, types, documents (--to, --json)
|
|
191
193
|
foldrun notify test send one test notification from a workspace and say what happened (--to <workspace>)
|
|
192
194
|
foldrun webhooks deliveries the notify: webhooks a workspace sent, newest first — status, attempts, last answer (--failed, --status <s>, --event <e>, --limit, --offset, --to)
|
|
@@ -276,7 +278,7 @@ Options
|
|
|
276
278
|
--on-fail <agent> flow add: who takes the step over when it fails
|
|
277
279
|
--off flow add: take the pattern off the step instead (fan-out, loop, approval, ask, wait, rescue)
|
|
278
280
|
--schedule "<cron>" flow trigger: the cron line a schedule fires on (5 fields, or @daily); --timezone <zone> the zone it is read in
|
|
279
|
-
--kind <k> secrets set: file (--file <path>), ssh (--host, --user, --port, --key-file or a prompted password), api (--base-url, --header "Name: value" repeatable), service-account (--file <key.json>, --scopes), m2m (--token-url, --client-id, --scopes; secret prompted)
|
|
281
|
+
--kind <k> secrets set: file (--file <path>), ssh (--host, --user, --port, --key-file or a prompted password), api (--base-url, --header "Name: value" repeatable, or --basic-user <u> with the password prompted or --basic-password), service-account (--file <key.json>, --scopes, --subject <user to act as>), client-cert (--cert-file, --key-file, --passphrase, --export to put it in the sandbox as files too), identity (--audience: keyless cloud access, no credential stored), m2m (--token-url, --client-id, --scopes; secret prompted)
|
|
280
282
|
--client <name> connect: run the consent from that OAuth client saved on the platform (secrets clients lists them)
|
|
281
283
|
--subagent <name> agent link, agent unlink: a worker it delegates to (subagents:)
|
|
282
284
|
--consult <name> agent link, agent unlink: a colleague it asks (agents:)
|
|
@@ -304,6 +306,7 @@ Options
|
|
|
304
306
|
--step <n> approve, reject: decide only that step; default is every step that is waiting
|
|
305
307
|
--payload <json|text|@f> approve: release a step waiting on wait: event with this body, recorded as you
|
|
306
308
|
--note "<text>" approve, reject: guidance the agent reads — or the reason for a refusal; billing credit: the ledger line's note
|
|
309
|
+
--run <id> eval --to: that eval run's verdict, by the id the run printed; runs nothing
|
|
307
310
|
--stale [days] agent ls: only agents not run in that many days (never run included); bare, those past alerts: stale_days
|
|
308
311
|
--reason "<text>" account pause, workspace pause: why — shown to everyone who opens it
|
|
309
312
|
--stop account pause, workspace pause: also stop every run going now (asks; --yes to mean it)
|
|
@@ -332,7 +335,9 @@ Options
|
|
|
332
335
|
--from <n> invoke, flow run, rerun: start at step n; earlier steps are recorded as skipped (a bulk rerun defaults to 1)
|
|
333
336
|
--no-browser login, connect, billing top-up/card/plan: print the address instead of opening it
|
|
334
337
|
--provider <name> connect: google, github, microsoft or linkedin (fills the URLs)
|
|
335
|
-
--
|
|
338
|
+
--no-pkce connect, secrets clients add: leave PKCE off (only for an app with a secret whose provider refuses it)
|
|
339
|
+
--issuer <url> secrets clients add: the provider's issuer, checked against the iss it sends back (RFC 9207)
|
|
340
|
+
--scopes "<a b c>" connect: space-separated scopes (default: the provider's example; with --client, the saved client's default — the connection keeps the ones asked for)
|
|
336
341
|
--client-id / --client-secret connect: the OAuth app (prompted if omitted; secret never echoed)
|
|
337
342
|
--port <n> connect: loopback port for the redirect (default 8642 — register http://localhost:8642/callback on the app)
|
|
338
343
|
--authorize-url / --token-url connect: a provider with no preset
|
|
@@ -379,7 +384,7 @@ if (!command || command === "--help" || command === "-h") {
|
|
|
379
384
|
// `--value` as the account's argument and stored an empty secret; `--force
|
|
380
385
|
// ./dir` swallowed the directory. A flag followed by another flag is also
|
|
381
386
|
// boolean, so an unlisted switch at least does not eat its neighbour.
|
|
382
|
-
const BOOLEAN_FLAGS = new Set(["account", "follow", "force", "oauth2", "wait", "watch", "print", "dry-run", "help", "no-browser", "local", "test", "flat", "yes", "platform", "json", "forever", "all", "check", "new-client", "quiet", "off", "can-ask", "preview", "csv", "failed", "starter", "history", "list", "library", "clear", "security", "changed", "write-baseline", "no-baseline", "rules", "calls", "tree", "tool-error", "stop"]);
|
|
387
|
+
const BOOLEAN_FLAGS = new Set(["account", "follow", "force", "oauth2", "wait", "watch", "print", "dry-run", "help", "no-browser", "local", "test", "flat", "yes", "platform", "json", "forever", "all", "check", "new-client", "quiet", "off", "can-ask", "preview", "csv", "failed", "starter", "history", "list", "library", "clear", "security", "changed", "write-baseline", "no-baseline", "rules", "calls", "tree", "tool-error", "stop", "last", "no-pkce", "export"]);
|
|
383
388
|
// Flags said more than once collect into a list: `--tag a --tag b`.
|
|
384
389
|
const REPEATABLE = new Set(["tag", "expect", "header", "site", "add", "remove", "rule"]);
|
|
385
390
|
// `--wait` is a switch everywhere but `flow add <flow> wait`, where it takes
|
|
@@ -491,7 +496,11 @@ function pinned() {
|
|
|
491
496
|
return null;
|
|
492
497
|
}
|
|
493
498
|
|
|
494
|
-
|
|
499
|
+
// `init` makes a new account in the directory it is given. It never resolves
|
|
500
|
+
// to a workspace above it: a home directory holding some unrelated `agents/`
|
|
501
|
+
// (a git repo of that name) passes the workspace test, and `init` anywhere
|
|
502
|
+
// under it used to scaffold into the home directory itself (2026-10-21).
|
|
503
|
+
const workspace = (command === "init" ? null : pinned()) ?? here;
|
|
495
504
|
if (!ACCOUNT_WIDE.has(command)) process.env.FOLDRUN_WORKSPACE = workspace;
|
|
496
505
|
// The account scope, decided once here and honoured by the core's
|
|
497
506
|
// singleAccountRoot — so `library/` beside `workspaces/` is what an agent's
|
package/docs/agents.md
CHANGED
|
@@ -43,6 +43,28 @@ by deleting its folder and redeploying.
|
|
|
43
43
|
Only `name` and the body are required. Every other field widens or narrows
|
|
44
44
|
what the agent can reach, and the default is narrow.
|
|
45
45
|
|
|
46
|
+
### Grouped by job
|
|
47
|
+
|
|
48
|
+
The fields that govern an agent or guard it may be grouped under the job
|
|
49
|
+
they do. `governance:` holds `owner` and `budget`; `guardrail:` holds
|
|
50
|
+
`limits`, `network`, `hooks`, `disallowedTools`, `permissionMode` and
|
|
51
|
+
`secrets`:
|
|
52
|
+
|
|
53
|
+
```yaml
|
|
54
|
+
---
|
|
55
|
+
name: researcher
|
|
56
|
+
tools: [web, read]
|
|
57
|
+
governance:
|
|
58
|
+
owner: ana@acme.io
|
|
59
|
+
guardrail:
|
|
60
|
+
network: [api.example.com]
|
|
61
|
+
limits: {web.search: 40, repeat: 5}
|
|
62
|
+
---
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
It reads exactly as the same fields at the top. A field written both ways is
|
|
66
|
+
refused. See [The four jobs](four-jobs).
|
|
67
|
+
|
|
46
68
|
## Which model, and how hard it thinks
|
|
47
69
|
|
|
48
70
|
| field | values | |
|
package/docs/api.md
CHANGED
|
@@ -179,7 +179,7 @@ history with each message.
|
|
|
179
179
|
| `/api/workspaces/<ws>/export` | GET | this workspace, one flow or one agent as a `.zip` — `?kind=flow&name=<flow>` (the flow, every flow it runs as a step, every agent those steps and their colleagues name, and the workspace tools, skills and scripts they grant), `?kind=agent&name=<agent>` (its folder and what it grants), or neither for the whole authored tree. Never memory/, state/, outputs/, storage, runs or a secret's value; `foldrun-package.json` inside says what it is. Any role that can read the workspace |
|
|
180
180
|
| `/api/workspaces/<ws>/import` | POST | take a package in; the body is the `.zip` (`content-type: application/zip`, at most 20 MB). `?dryRun=1` answers `{ kind, name, from, creates, added, overwritten, unchanged, needs: { secrets, tools, skills, scripts, agents, flows } }` plus `skipped` (entries it will not write: memory, state, anything outside the authored folders) and writes nothing. Without it, applies as one revision; 409 when it would replace a file with different text unless `?overwrite=1`, or when a file cannot be written there (all or nothing). Never deletes. A `Content-Length` is required (411 without). A workspace package into a name that does not exist makes that workspace (`workspace:manage`, the plan's workspace count). 400 for anything not a package, or one holding memory/, state/ or outputs/. Idempotent |
|
|
181
181
|
| `/api/workspaces/<ws>/evals` | GET, POST | list; create (`name`) |
|
|
182
|
-
| `/api/workspaces/<ws>/evals/<eval>/run` | POST |
|
|
182
|
+
| `/api/workspaces/<ws>/evals/<eval>/run` | GET, POST | POST runs one eval and answers with its result; with `{ async: true }` it answers `202 { started, eval, runId, startedAt }` at once and the eval runs on — an eval of slow cases outlives a proxy's read timeout (Cloudflare's is 125 s by default). One run of an eval at a time: a second is `409` naming the run that holds it. `GET ?run=<runId>` is what to poll — that run alone, whatever else runs or writes the eval's latest result meanwhile: `{ runId, state, running }` where `state` is `running`, `done` (with its `result`), `failed` (with `error`), `lost` (it stopped reporting before saving a result, e.g. a restart) or `unavailable` (an id not on record — the last 20 runs are kept). Plain `GET` answers who holds the eval now (`running`, `runId`, `startedAt`) and the latest `result`. The dashboard and `foldrun eval --to` start async and poll by run id |
|
|
183
183
|
| `/api/workspaces/<ws>/tools/<tool>/test` | POST | exercise one tool (`args`, `path`). An http tool with an `operations:` allowlist is probed only with a GET to one of its operations (no `path`: the first GET without parameters), and fails if the allowlist resolves nothing, as a run does |
|
|
184
184
|
| `/api/workspaces/<ws>/hooks` | GET | the [hooks](hooks) this workspace can turn on — its own and the library's (shadowing marked), invalid ones with the reason, and who turns each on — plus `recent`: every deny, rewrite and failure hooks wrote in the last `?runs=` runs (1–500, default 50), newest first; `?all=1` includes notes and context |
|
|
185
185
|
| `/api/workspaces/<ws>/hooks/<name>/test` | POST | run one [hook](hooks) on a sample event: body `{ event }`, answer the hook's own (`ok`, `decision`, `reason`, `input`, `context`, `reply`, `note`, `failure`, `ms`). Shares the `hooks/` path with flow webhooks below — `/test` is only ever a hook's. Answers `404` for no such hook, `422` for an invalid `hook.md`, `400` for an event that is not an object. On an isolated install hook code never runs in the platform's own process, so this answers with where to test it instead (`foldrun hook test` locally, or a test run) |
|
|
@@ -319,7 +319,7 @@ its header and requires a URL it can GET for itself.
|
|
|
319
319
|
| `/api/secrets?workspace=` | GET, PUT, PATCH, DELETE | the vault: list which secrets exist (never a value), each with the `sites` it is bound to; set one (`name` plus one shape, below, and optionally `sites`); bind one to sites with PATCH `{ name, sites }` — `sites` a list like `["api.example.com", "*.example.com"]`, `[]` unbinds, the value untouched, 404 for a secret that is not there, 400 for a site with a scheme, path or port and for a file, ssh or api secret ([bound to sites](secrets#bound-to-sites)); delete one (`name`). `workspace` names a workspace's own store; unsaid is the account's, which a key scoped to some workspaces may not touch. PUT is idempotent; PATCH and a PUT with `sites` are audited as `secret.sites` / `secret.set` |
|
|
320
320
|
|
|
321
321
|
`PUT` takes `name` plus exactly one shape: `value` (a string), `file` (a PEM,
|
|
322
|
-
a cert), `oauth2`, `service_account`, `ssh`, or `
|
|
322
|
+
a cert), `oauth2`, `service_account`, `ssh`, `api`, `client_cert` (`{ cert, key, passphrase?, export? }` — mutual TLS, presented by the egress proxy to the exact sites the secret is bound to), or `identity` (`{ audience }` — keyless cloud access). They go to the vault;
|
|
323
323
|
agents name them in `secrets:` and never see them in a file.
|
|
324
324
|
|
|
325
325
|
Reads return which secrets exist, never their values.
|
|
@@ -343,6 +343,7 @@ Reads return which secrets exist, never their values.
|
|
|
343
343
|
| `/api/issues/<id>` | GET POST | one issue; POST acts on it: `{ action: "acknowledge" | "snooze" | "resolve" | "reopen", for?: "30m" | "4h" | "1d" | "1w", until?: ISO, times?: n, note? }` (editor or above), kept in its history with who did it, and answers `{ issue }` as it now stands. A snooze needs `for`, a future `until`, or `times` (until it happens that many more times; with a time too, whichever comes first). A platform issue is the account's: closing it needs access to every workspace. A run that goes wrong the same way after a resolve opens the issue again |
|
|
344
344
|
| `/api/account/export` | GET | the owner's copy of what the platform holds about the account, as one JSON file (`foldrun-<account>-export.json`): `workspaces` (with their flows), `members` (email, role, scope, since), `runs` (an index — id, workspace, flow, status, times, headline) and the whole `ledger` (each line's charge; the platform's provider cost is left out). `billing:manage`, so the owner; audited as `account.exported`. The markdown itself comes from `foldrun pull` or the source API |
|
|
345
345
|
| `/api/account/telemetry/test` | POST | send one made-up run as an OpenTelemetry trace to the collector `telemetry:` names — the account's, or `{ workspace }`'s — with its header secrets filled from the vault; `{ ok, status?, error?, spans?, endpoint }`. `409` when no `telemetry:` is set. Needs `workspace:manage` |
|
|
346
|
+
| `/api/account/jobs?workspace=` | GET | the four jobs (governance, guardrail, observability, evaluation) and each `AGENTS.md` field of each: `{ jobs: [{ name, title, what, fields: [{ key, account, workspace?, combine, writable }] }] }` — the account's value and the workspace's when `workspace` is given (null: unset), as written with blocks folded; `combine` is how the two meet at run time (`workspace replaces`, `either turns it on`, `both apply`, `per key`, `per field`, `level nearest, sites from both`, `account only`); `writable` is whether that scope's settings PATCH (`/api/account`, or the workspace's) writes it — the rest are written in `AGENTS.md`. See [The four jobs](four-jobs) |
|
|
346
347
|
| `/api/account/pause` | GET, POST, DELETE | the account's kill switch: the same as a workspace's `/pause`, for every workspace at once. GET `{ paused, account, workspaces }` (`workspaces`: those paused on their own). POST `{ reason?, stop?: true }`, DELETE resumes the account — a workspace paused on its own stays paused. Needs `workspace:manage` and access to every workspace |
|
|
347
348
|
| `/api/account/close` | POST | the owner asks to close the account — in person, never a key, with `confirm` set to the account's name and an optional `reason`. Not a delete: the account is suspended at once (runs in flight finish, sign-in and the export still work), `closeRequested` is set on its admin state, and the super admin is mailed to settle the last invoice and any refund. Returns `{ ok, suspended, notified }`; 409 when already asked. Hosted installs only |
|
|
348
349
|
| `/api/account/backups` | GET | what is backed up for this account: `policy` (`schedule`, `keepLocal`, `keepOffsiteDays`, `encrypted`, `covers`, `excludes`), `last`, `backups` (each nightly archive that holds this account, newest first: `id`, `at`, `bytes` (the encrypted object), `sha256` (the archive inside the encryption — what a decrypt must reproduce), `kinds` — files, database, keys — `verified`, `offsite`, `keepDays`, `encrypted` (sealed with age before it left the server; false for archives taken before October 2026), `recipient` (the key's fingerprint, `sha256:<16 hex>`, or null)), `requests` (earlier restore requests) and `workspaces` (`name`, `head`, `history`, `lastChange`) — what a restore can reach. See [Backups and restore](backups-and-restore.md) |
|
|
@@ -350,7 +351,7 @@ Reads return which secrets exist, never their values.
|
|
|
350
351
|
| `/api/account/api` | GET, PATCH | how this account meets the API — GET `{ version: { current, pinned, effective, supported, supportMonths }, limits: { plan, windowSeconds, perCredential: { read, write }, perAccount: { read, write } }, usage }`. `usage` is the last 30 days — `{ days, versions: [{ version, requests, lastSeen }], deprecated: [{ route, requests, lastSeen, deprecatedAt, sunsetAt, link, callers: [{ credential, requests, lastSeen }] }] }`, `credential` naming a key by its label and prefix — or `null` when it could not be read; PATCH `{ version: "YYYY-MM-DD" \| null }` pins the version API-key requests without a `Foldrun-Version` header get (null: always current); the dashboard always gets current. `keys:manage`. See [api-usage](api-usage.md) |
|
|
351
352
|
| `/api/account/flags` | GET | the feature flags as they stand for this account — `{ flags: [{ name, description, owner, default, on, source, rollout }] }`, `source` being what decided it: `account` (an override for this account), `global` (the install-wide setting), `rollout` (a percentage of accounts, by a stable hash of account and flag; `rollout` is the percentage) or `default`. Read-only, every role; only the super admin changes a flag. See [Feature flags](feature-flags.md) |
|
|
352
353
|
| `/api/openapi.json` | GET | this API as an OpenAPI 3.1 document — every route and method, bearer auth, the rate-limit, version and Idempotency-Key headers, and request/response schemas for runs, flow runs, deploy, source, secrets, keys and billing. No key needed. Generated from the route files by `scripts/gen-openapi.mjs`; the dashboard renders it at Docs → API reference, `foldrun api spec` saves it |
|
|
353
|
-
| `/api/account` | GET, PATCH | the account's defaults from its AGENTS.md frontmatter — `timezone`, `budget` (a USD cap — a number for a month, `"60/day"`, `"60/week"`, or `{ usd, period }`; null is no limit), `notify` (`{ url?, email?, events }`), `concurrency`, `deploy_block` (`none`, `secrets` or `security` — answered as `deployBlock`), `network_enforce` (answered as `networkEnforce`), `telemetry` (`{ otlp, headers }` — header values are `${SECRET}` names; answered as `{ endpoint, headers }`); PATCH sets the keys sent, null clears one, and edits the file in place |
|
|
354
|
+
| `/api/account` | GET, PATCH | the account's defaults from its AGENTS.md frontmatter — `timezone`, `budget` (a USD cap — a number for a month, `"60/day"`, `"60/week"`, or `{ usd, period }`; null is no limit), `notify` (`{ url?, email?, events }`), `concurrency`, `deploy_block` (`none`, `secrets` or `security` — answered as `deployBlock`), `network_enforce` (answered as `networkEnforce`), `owner`, `alerts` (as written), `judge` (`{ model?, effort? }`), `telemetry` (`{ otlp, headers }` — header values are `${SECRET}` names; answered as `{ endpoint, headers }`); PATCH sets the keys sent, null clears one, and edits the file in place |
|
|
354
355
|
| `/api/audit?since&until&action&actor&workspace&limit&cursor&format` | GET | the account's audit log, newest first — the caller's account only, whatever a header, query or body names. The owner and unscoped admins (people, or `admin` keys); `audit:read`, and refused to a member or key scoped to some workspaces, because the log is the whole account's. `since`/`until`: `7d`, `24h`, `30m` or a date; `action`: exact (`apikey.created`) or a family (`apikey`); `actor`: exact (an email, `key:<label>`, `support:<email>`); `workspace`: entries about one; `limit` 1–1000 (100; 1000 for CSV); `cursor`: the `next` of the page before. JSON `{ entries: [{ id, at, actor, action, subject, detail }], next, retention }`; `format=csv` downloads the same rows (`x-next-cursor` header when there are more). Records sign-ins and refusals, sign-outs, session revokes, password and email changes, MFA on/off, key mint/rotate/revoke, invites, joins, role/scope changes, removals, ownership transfers, secrets set/deleted (by name — never a value), deploys, workspace deletes, plan and billing changes, export and close requests, and `support.viewed`/`support.ended`. The super admin's own levers (`operator.*`) are the platform's record and not listed. Kept for the life of the account |
|
|
355
356
|
| `/api/avatars/<kind>/<id>` | GET, PUT, DELETE | a picture for a `user` (their own id), a `workspace` (its name) or the `account` (the tenant). PUT the bytes of a PNG, JPEG or WebP under 1 MB; a person sets their own, an admin a workspace's or the account's. GET is cached for a year — the URL's `?v=` changes with the picture |
|
|
356
357
|
| `/api/search?q=` | GET | find anything in the account by name — workspaces, agents, flows, tools, skills, knowledge, memory, runs (by id, flow or headline); ranked, at most six per kind. The dashboard's Find palette reads it |
|
|
@@ -462,9 +463,11 @@ signed-in person, never by a key — mints an ordinary API key at their role.
|
|
|
462
463
|
| Route | Methods | |
|
|
463
464
|
|---|---|---|
|
|
464
465
|
| `/api/oauth/clients` | GET, POST, DELETE | registered providers (`name`, `client`, `config`) |
|
|
465
|
-
| `/api/oauth/start` | POST | begin a connection (`client`, `secret`, `workspace`) |
|
|
466
|
-
| `/api/oauth/callback` | GET | where the provider returns |
|
|
467
|
-
| `/api/
|
|
466
|
+
| `/api/oauth/start` | POST | begin a connection (`client`, `secret`, `workspace`, optional `scopes`). From a signed-in browser: `{ url }` is the provider's consent, bound to that browser by a cookie the callback checks, with PKCE. With an API key: `{ url, handoff: true }` is a dashboard page where a signed-in person accepts first (`foldrun connect --client` opens it). `scopes` (space-separated, as the provider spells them) is what this connection asks for instead of the client's default, and the connection keeps it for its reconnects — one saved app serves any number of connections at different scopes |
|
|
467
|
+
| `/api/oauth/callback` | GET | where the provider returns — refuses a consent that comes back to another browser, and an `iss` that is not the app's issuer |
|
|
468
|
+
| `/api/identity?workspace=` | GET | keyless cloud identity: `{ enabled, issuer, subject, tokenSeconds }` — the issuer a cloud must trust and the subject pattern each agent's tokens carry (`account:<a>:workspace:<w>:agent:<agent>`). Open (no key), outside `/api`: `/.well-known/openid-configuration` and `/.well-known/jwks.json`, 404 when the install does not issue tokens |
|
|
469
|
+
| `/api/oauth/handoff/<id>` | POST | accept a connection started from a terminal: a signed-in person only, never a key; answers `{ url }`, the provider's consent bound to this browser. Used once, ten minutes |
|
|
470
|
+
| `/api/oauth/reconnect` | POST | reconnect a secret with the client it was connected with — no client id (`secret`, `workspace`, `provider`, optional `scopes`) → `{ url }`; it asks for `scopes` when given, else the scopes the secret was connected with, else the client's default |
|
|
468
471
|
| `/api/oauth/connections` | GET | each oauth2 secret's grant: saved client, `status` (ok, expiring, expired, failed, no-client), days left, last error |
|
|
469
472
|
|
|
470
473
|
## Triggers
|
|
@@ -25,7 +25,7 @@ mistake the editor names rather than a cap it quietly keeps.
|
|
|
25
25
|
When an agent's cap and the flow's both apply to one step, the tighter one
|
|
26
26
|
wins, and the step's error says which line set it: `budget: in the flow
|
|
27
27
|
file` or `budget: on the writer agent`.
|
|
28
|
-
| **runs at once** | `concurrency: 3` in the account's `AGENTS.md`, or Settings →
|
|
28
|
+
| **runs at once** | `concurrency: 3` in the account's `AGENTS.md`, or Settings → Governance | the fourth run waits its turn in the queue — never refused, only later. Empty means the install's share (`FOLDRUN_ACCOUNT_CONCURRENCY`) or plan number (`FOLDRUN_PLAN_CONCURRENCY`, which applies only where billing is on), or no cap. Your own number can lower that, never raise it; a higher number comes with a plan |
|
|
29
29
|
| **the queue lane** | the plan's lane (`high` \| `normal` \| `low`); `priority: low` in the account's `AGENTS.md` lowers it, and `high` there counts only with a plan that gives it | when slots are short, accounts take turns, weighted by lane — high 3, normal 2, low 1: every account's first run goes before anyone's second, a fast lane takes three turns to a slow lane's one and never all of them, an account with runs going waits behind one with none, and a run that has waited ten minutes moves up a turn. A flow's own `priority:` orders runs within that. The cap above still holds |
|
|
30
30
|
| **room to start** | the install's `FOLDRUN_RUN_MEMORY_GIB` and `FOLDRUN_ACCOUNT_PODS` | a step whose sandbox would not fit, or whose account already has its most sandboxes running, waits to start — `room: waiting to start — …` on its trail — instead of failing. The wait is not billed; the run page's start-up chip shows it as `room`. A step still waiting after 30 minutes (`FOLDRUN_ROOM_WAIT_SEC`) fails, saying why |
|
|
31
31
|
|
package/docs/checks.md
CHANGED
|
@@ -81,7 +81,7 @@ listed with it. `deploy_block:` in `AGENTS.md` turns them into refusals:
|
|
|
81
81
|
| `security` | any security finding: `secret/*`, `content/*`, `security/*`, `supply-chain/*`, and `flow/outward-ungated` (a step that acts outside with no check or approval) |
|
|
82
82
|
|
|
83
83
|
Set it in the account's `AGENTS.md` for every workspace, or in one
|
|
84
|
-
workspace's `AGENTS.md`, which wins. Settings →
|
|
84
|
+
workspace's `AGENTS.md`, which wins. Settings → Governance → **Refuse a
|
|
85
85
|
deploy on**, or `foldrun account set deploy_block secrets`. A refused
|
|
86
86
|
deploy names each finding and its rule; a `foldrun-ignore` with a reason
|
|
87
87
|
(below) still accepts one.
|
|
@@ -351,6 +351,21 @@ place in `@foldrun/core` (`frontmatter-fields.ts`). The editor's completions
|
|
|
351
351
|
come from the same list, so what the editor suggests and what `check`
|
|
352
352
|
accepts cannot drift apart.
|
|
353
353
|
|
|
354
|
+
### config/conflict
|
|
355
|
+
|
|
356
|
+
*error.* A field written twice: at the top of the frontmatter and inside its
|
|
357
|
+
job block (`limits:` and `guardrail: {limits: …}`). Only the top one would be
|
|
358
|
+
read, so the file is refused until one is removed. See
|
|
359
|
+
[The four jobs](four-jobs).
|
|
360
|
+
|
|
361
|
+
### config/misplaced
|
|
362
|
+
|
|
363
|
+
*error.* A setting inside the wrong job's block — `network:` under
|
|
364
|
+
`governance:` — is not read, so a protection you wrote would not apply; the
|
|
365
|
+
file is refused until it moves. The message names the block it belongs in, or
|
|
366
|
+
says to write it at the top when it does none of the four jobs. A key foldrun
|
|
367
|
+
does not read at all, inside a block, is only a note.
|
|
368
|
+
|
|
354
369
|
### config/invalid-frontmatter
|
|
355
370
|
|
|
356
371
|
*warn.* Frontmatter that YAML cannot read. The commonest cause is a value
|
|
@@ -381,10 +396,10 @@ filtered, ignored and baselined. Each problem keeps its own level.
|
|
|
381
396
|
| Rule | Covers |
|
|
382
397
|
|---|---|
|
|
383
398
|
| `agent/config` | frontmatter that cannot be read; a tool, skill, colleague, sub-agent or hook that does not exist; retired tool names |
|
|
384
|
-
| `flow/config` | a step naming nothing, an option the parser cannot read, `verify:` written as prose, an arrangement that cannot have what its instruction asks for |
|
|
399
|
+
| `flow/config` | a step naming nothing, an option the parser cannot read, `verify:` written as prose, a `verify: matches:` that is not a valid JavaScript regular expression, an arrangement that cannot have what its instruction asks for |
|
|
385
400
|
| `tool/config` | a `run:` program that is missing, an import nothing declares, a leftover `type:` |
|
|
386
401
|
| `skill/config` | a `SKILL.md` without a usable name or description |
|
|
387
|
-
| `eval/config` | an eval that names nothing,
|
|
402
|
+
| `eval/config` | an eval that names nothing, has no cases, or has a `matches:` that is not a valid JavaScript regular expression |
|
|
388
403
|
| `hook/config` | a hook folder that is not a valid hook; a `hooks:` name that resolves to nothing |
|
|
389
404
|
| `knowledge/okf` | a knowledge or memory document that is not a conformant bundle entry, an unreadable date, a stale document |
|
|
390
405
|
| `workspace/config` | an `AGENTS.md` setting that cannot be read; a path a deploy refuses |
|
package/docs/cli.md
CHANGED
|
@@ -70,7 +70,7 @@ claude.ai or Claude Code there, nor any MCP configuration on disk.
|
|
|
70
70
|
| `foldrun hook test <name>` | run one hook HERE on one event, exactly as a step runs it (the event on stdin, a stripped environment, its `timeout:`), and print its answer — decision, reason, a rewritten input, context, reply or note — and how long it took. `--event '<json>'` or `--event @file`; without it, a sample event for the hook's `on:`. A deny is an answer (exit 0); a hook that crashes, times out or answers in the wrong shape is a failure (exit 1) — which, on a gate, refuses the call. `--to <workspace>` runs the DEPLOYED hook on the platform instead (`POST …/hooks/<name>/test { event }`) |
|
|
71
71
|
| `foldrun hook rm <name>` | delete a hook folder (`--library` for the account's). Says which agents and AGENTS.md files still turn it on — their steps are refused until it goes from their `hooks:` — and asks; `--yes` on a pipe |
|
|
72
72
|
| `foldrun run <target>` | run an agent or a flow HERE, on this machine (`name`, or `flow:name`) — `--test` for a [test run](runs#test-runs). On a platform: `flow run`/`invoke` for a flow, `agent run` for one agent |
|
|
73
|
-
| `foldrun eval [name]` | one eval, or all of them. `--to <workspace>` runs the DEPLOYED workspace's evals on the platform (`POST …/evals/<name>/run`), waiting for each verdict, and prints the same lines; exits 1 when a case fails |
|
|
73
|
+
| `foldrun eval [name]` | one eval, or all of them. `--to <workspace>` runs the DEPLOYED workspace's evals on the platform (`POST …/evals/<name>/run` with `async: true`, then polling `GET …/run?run=<id>` — an eval longer than a proxy's timeout still gets its own verdict), waiting for each (`--timeout <s>`, default 900; past it, the run id to read it with). `--run <id>` prints that run's verdict and runs nothing — a run that was lost or is unknown exits 1; `--last` prints each eval's latest result and runs nothing, and prints the same lines; exits 1 when a case fails |
|
|
74
74
|
| `foldrun eval new <name>` | the Evals page's New, on the deployed workspace (`POST …/evals { name }`): an eval whose `agent:` is the workspace's first agent and whose case is a placeholder — edit it (`source cat`/`put`), then `foldrun eval <name> --to <workspace>` runs it. `--to <workspace>` |
|
|
75
75
|
| `foldrun promote <run-id>` | keep a finished run as a regression case: its task and what the next run must still say go into `evals/<target>-regressions.md` (or `--eval <name>`), appended when the file exists. `--expect "contains: $34"` — repeatable, eval syntax — replaces the default judge line that holds the next run to the recorded conclusion; `--case <name>` names the case. A run that went wrong (failed, BLOCKED, BAD) takes `--should "what it should have done"` — the judge's claim — or `--expect` |
|
|
76
76
|
| `foldrun probe <model>` | can this model hold a tool loop here? A live check |
|
|
@@ -96,10 +96,10 @@ claude.ai or Claude Code there, nor any MCP configuration on disk.
|
|
|
96
96
|
| `foldrun message <run-id> "…"` | say something to a running agent. It reaches the model after its next tool call, as context — it changes none of the step's tools. `--step <n>` picks one when a parallel group has several running. Platform only |
|
|
97
97
|
| `foldrun secrets set NAME` | store a secret — also `ls`, `rm`, and `status` (OAuth grant health). `--site <domain>` (repeatable, or `a,b`) binds it as it is stored |
|
|
98
98
|
| `foldrun secrets sites NAME [site…]` | bind a secret to the sites it is for: the egress proxy fills it in requests to those sites alone, and it never enters a sandbox ([bound to sites](secrets#bound-to-sites)). No sites prints what it is bound to; `--clear` unbinds. On the platform, or this machine's vault with `--local` |
|
|
99
|
-
| `foldrun secrets set NAME --kind <k>` | the other shapes the Connections form stores: `file` (`--file <path>`), `ssh` (`--host`, `--user`, `--port`, `--key-file <path>` or a prompted password), `api` (`--base-url`, `--header "Name: value"` repeatable), `service-account` (`--file <key.json>`, `--scopes`), `m2m` (a client-credentials app: `--token-url`, `--client-id`, `--scopes`, the secret prompted). On the platform, or this machine's vault with `--local` |
|
|
99
|
+
| `foldrun secrets set NAME --kind <k>` | the other shapes the Connections form stores: `file` (`--file <path>`), `ssh` (`--host`, `--user`, `--port`, `--key-file <path>` or a prompted password), `api` (`--base-url`, `--header "Name: value"` repeatable, or `--basic-user <u>` with the password prompted or `--basic-password`), `service-account` (`--file <key.json>`, `--scopes`, `--subject <user>` to act as a user where the provider allows it), `client-cert` (`--cert-file`, `--key-file`, `--passphrase`, `--export` to put it in the sandbox as files too; bind it with `secrets sites`), `identity` (`--audience`: keyless cloud access — nothing secret is stored), `m2m` (a client-credentials app: `--token-url`, `--client-id`, `--scopes`, the secret prompted). On the platform, or this machine's vault with `--local` |
|
|
100
100
|
| `foldrun secrets clients` | the OAuth clients saved on the platform — what a consent runs from without typing the client again; the client secret is never shown. `clients add <name> --provider google\|github\|microsoft\|linkedin` (or `--authorize-url` / `--token-url`) `--client-id … --scopes …` saves one (secret prompted), `clients rm <name>` removes it |
|
|
101
|
-
| `foldrun connect NAME` | OAuth sign-in from the terminal, stored as an auto-refreshing secret — `--provider google\|github\|microsoft\|linkedin`; reconnecting a secret whose client is saved asks for nothing (`--new-client` to enter one) |
|
|
102
|
-
| `foldrun connect NAME --client <saved>` | the Connections page's **Connect**: a consent from a client saved on the platform, returning to the platform's own callback, which stores the grant as NAME (`--to <workspace>` for a workspace's). Opens the browser (`--no-browser` prints the address) and waits for it to land |
|
|
101
|
+
| `foldrun connect NAME` | OAuth sign-in from the terminal, stored as an auto-refreshing secret — `--provider google\|github\|microsoft\|linkedin`; PKCE is on (`--no-pkce` only for an app with a secret whose provider refuses it), and an app with no client secret is a public app; reconnecting a secret whose client is saved asks for nothing (`--new-client` to enter one) |
|
|
102
|
+
| `foldrun connect NAME --client <saved>` | the Connections page's **Connect**: a consent from a client saved on the platform (`--scopes "<a b>"` asks for this connection's own scopes instead of the client's default; the connection keeps them). It opens a page in the dashboard where you, signed in, accept the connection before the provider is asked, returning to the platform's own callback, which stores the grant as NAME (`--to <workspace>` for a workspace's). Opens the browser (`--no-browser` prints the address) and waits for it to land |
|
|
103
103
|
| `foldrun deploy [dir]` | push this account into an installation — every workspace and the shared library. `foldrun deploy <workspace>` pushes one. Never removes a workspace the platform has and the folder does not. It first prints where it is going and as whom (`deploying to <url> as <profile> (<email>)`); a deploy that would delete files the folder no longer has lists them and asks — what runs wrote (`storage/`, `state/`, the trigger log) is never deleted, so never listed. The confirmed list goes with the deploy; if the platform would now remove anything else it refuses and the CLI asks again. With no terminal (a script, a CI job, a coding agent) it refuses unless `--yes`. A deploy that only adds or changes files asks nothing |
|
|
104
104
|
| `foldrun help <command>` | that command's usage and options. `foldrun <command> --help` and `-h` print the same and never run the command |
|
|
105
105
|
| `foldrun pull [workspace]` | bring the platform's account down into this folder — refuses to overwrite a locally edited file, and names every one it would have taken (`--force` takes them) |
|
|
@@ -142,8 +142,10 @@ claude.ai or Claude Code there, nor any MCP configuration on disk.
|
|
|
142
142
|
| `foldrun billing top-up <usd>` | the Payment page's top-up: Stripe's checkout page for that amount, opened in the browser (`--no-browser`, or no terminal, prints it). The credit lands when Stripe confirms the payment. `billing card` is the same for saving a card for auto top-up. Both need `billing:manage` |
|
|
143
143
|
| `foldrun billing credit <usd>` | put credit on the balance by hand (`POST /api/billing { usd, note }`, `--note "…"` on the ledger line) — only on an install without Stripe, a self-hosted or dev platform; once billing is live the platform refuses it and top-ups go through `billing top-up`. The dashboard's Billing → Add credit does the same while billing is off. Needs `billing:manage` |
|
|
144
144
|
| `foldrun gallery` | the tools the platform ships to every account (`web`), and for each whether this account keeps its own copy and whether that copy still matches. `pull` lays the shelf down under `~/.foldrun/gallery/<host>/` for offline runs (`foldrun run` and `foldrun eval` refresh it themselves before each run); `upgrade <tool>` replaces the account's copy with the current gallery version, the old one a revision back |
|
|
145
|
-
| `foldrun
|
|
146
|
-
| `foldrun
|
|
145
|
+
| `foldrun identity` | keyless cloud identity: the issuer your cloud must trust and the subject each agent's tokens carry (`GET /api/identity`); `--to <workspace>` fills the workspace in, `--json` |
|
|
146
|
+
| `foldrun jobs` | the four jobs — governance, guardrail, observability, evaluation — and every `AGENTS.md` field of each, with what is set (`GET /api/account/jobs`). `--to <workspace>` shows the workspace's value beside the account's and how the two combine; `--json` the document. `account set <job>.<field>` and `workspace set <job>.<field>` refuse a field that scope cannot set and name the `AGENTS.md` to write it in. See [The four jobs](four-jobs) |
|
|
147
|
+
| `foldrun account` | the account's own defaults — the AGENTS.md frontmatter every workspace under it inherits: `timezone`, `notify`, `budget`, `concurrency`, `deploy_block`, `network_enforce`. `foldrun account set timezone Australia/Sydney`, `set notify email you@example.com` (`--events failed,awaiting-approval,completed`), `set notify url https://…`, `set budget 60/month`, `set concurrency 4`, `set deploy_block secrets` (`none`, `secrets` or `security` — what the scanner's findings do to a deploy), `set network_enforce on` (every workspace refuses a site off an agent's `network:` list; `off` reports only), `set owner <email or name>`, `set alerts <threshold> <value>` (merged into what is set; `off` removes one), `set judge model|effort <value>`. Any setting may be named by its job — `set governance.deploy_block secrets`, `set observability.alerts stale_days 30` — and one named under the wrong job is refused with the right one, `pause` (`--reason "…"`, `--stop` to also stop every run going now — asks, or `--yes`) and `resume`: the kill switch, no new run starts in any workspace while paused, `telemetry` — where each finished run goes as an OpenTelemetry trace: bare shows it, `set <url> --header authorization='${OTEL_TOKEN}'` (repeatable; a value is a vault secret's name), `clear`, `test` (one made-up run, and what the collector answered; exit 1 when not delivered), and `clear <key>` to unset one. `set notify` merges against what is there, because the platform replaces the whole block. `providers` lists the model providers the account's files use and the last daily key check — `--check` asks now, and a dead key is a non-zero exit. `model` is the account's own model key, what every step runs on unless a `provider:` block names another: `foldrun account model` shows it (never the key, only its last four; exit 1 when there is none and runs are refused), `model set <provider> --key <api key>` sets it — a preset name (anthropic, openrouter, openai, groq, …) or `custom --base-url https://… --format anthropic|openai`, `--key -` reads the key from stdin — and `model remove` removes it; a Claude login token is refused for any account not allowed the platform's own credential. `export` downloads everything the platform holds about the account — workspaces, runs, members and the whole ledger — as one JSON file (owner only; `--file <path>`, `--force`). Singular — `accounts` below is the list of logins on this machine |
|
|
148
|
+
| `foldrun workspace` | one deployed workspace's own settings (`--to <workspace>`): description, timezone, budget and notify, read from its `AGENTS.md` — unset means the account's. `set timezone <tz>`, `set budget 20/week`, `set notify email <address>` / `set notify url <https://…>` (`--events`; merged with what the file says), `set network_enforce on` (refuse a site off an agent's `network:` list here; `off` or `clear` goes back to the account's), `set owner`, `set alerts <threshold> <value>`, `set judge model|effort <value>`, and `<job>.<field>` as on `account`, `pause` (`--reason`, `--stop`) and `resume` — this workspace's kill switch, `set description "…"`, `set name <new>` — a rename changes every webhook URL, so it asks first and names the flows whose URLs changed — and `clear <key>`. Singular — `workspaces` is the list |
|
|
147
149
|
| `foldrun workspace vocabulary` | what the dashboard's editor completes as you type in a workspace (`--to`): agents, flows, skills, tools, secrets (names only), scripts, document types and linkable documents (`GET …/vocabulary`; `--json`) |
|
|
148
150
|
| `foldrun notify test` | send one test notification to whatever the workspace's `notify:` resolves to, and print what happened — the provider's own words when it failed ("domain not verified"). Exits 1 when it did not go. `--to <workspace>` |
|
|
149
151
|
| `foldrun webhooks deliveries` | the `notify:` webhooks a workspace sent, newest first: event, status (`delivered`, `retrying`, `failed`, `skipped`), attempts, the last answer and the delivery id — and a switched-off endpoint first, with the way back. `--failed` for the ones that gave up, `--status <s>`, `--event <e>`, `--limit`, `--offset`, `--json`. `--to <workspace>` |
|
package/docs/evals.md
CHANGED
|
@@ -33,7 +33,7 @@ satisfy. The heading is the case's name, so make it the thing being checked.
|
|
|
33
33
|
|---|---|
|
|
34
34
|
| `name` | the eval's identity |
|
|
35
35
|
| `agent` **or** `flow` | what runs. A flow eval runs the whole flow per case |
|
|
36
|
-
| `model` | the **judge's** tier, not the subject's — cases run the agent on its own model |
|
|
36
|
+
| `model` | the **judge's** tier, not the subject's — cases run the agent on its own model. Unset, the workspace's `judge:` in `AGENTS.md` (see [The four jobs](four-jobs#judge--the-evaluation-default)), else the fast tier |
|
|
37
37
|
| `effort` | the judge's effort. PASS/FAIL against one sentence is not deep work |
|
|
38
38
|
| `trigger` | `deploy` (default) or `manual` |
|
|
39
39
|
| `inputs` | `true` makes the file a flow's saved inputs, not a test: its cases are named tasks for [Run with…](flows#run-with-saved-inputs) and `foldrun invoke --inputs`, and nothing runs them as an eval |
|
|
@@ -45,7 +45,7 @@ satisfy. The heading is the case's name, so make it the thing being checked.
|
|
|
45
45
|
|---|---|
|
|
46
46
|
| `contains:` | the text must appear |
|
|
47
47
|
| `not-contains:` | the text must not appear |
|
|
48
|
-
| `matches:` | a regular expression over the reply |
|
|
48
|
+
| `matches:` | a JavaScript regular expression over the reply, case-insensitive. No inline flags such as `(?m)`: write `(^\|\n)` for the start of a line. `foldrun check` reports one that does not compile |
|
|
49
49
|
| `file:` | the path exists and is non-empty |
|
|
50
50
|
| `run:` | a shell command that must exit 0 |
|
|
51
51
|
| `judge:` | a model grades the reply against this sentence |
|
package/docs/flows.md
CHANGED
|
@@ -291,7 +291,7 @@ decided by a number belongs in a command that can be tested:
|
|
|
291
291
|
verify: judge: quotes the real price
|
|
292
292
|
```
|
|
293
293
|
|
|
294
|
-
`contains:`, `not-contains:` and `matches:` test the step's **conclusion** —
|
|
294
|
+
`contains:`, `not-contains:` and `matches:` (a JavaScript regular expression, case-insensitive; no inline `(?m)`, write `(^|\n)` for a line start) test the step's **conclusion** —
|
|
295
295
|
its final reply, the same text the run's headline is read from — so a step
|
|
296
296
|
that narrated between tool calls and then opened its answer with `BAD — …`
|
|
297
297
|
passes `matches: ^BAD\b`. `judge:` grades the same conclusion as "the reply", with everything the step wrote before it shown alongside as context, so a claim about the work can still be checked.
|
|
@@ -492,6 +492,15 @@ before `low`. It adds to the account's lane, so a high flow on a low account
|
|
|
492
492
|
is normal. The account's `concurrency:` still bounds how many of its runs are
|
|
493
493
|
live at once.
|
|
494
494
|
|
|
495
|
+
## Grouped by job
|
|
496
|
+
|
|
497
|
+
A flow's frontmatter may group its fields by the job they do:
|
|
498
|
+
`governance:` (`owner`, `approvers`, `approve_within`, `budget`, `priority`),
|
|
499
|
+
`guardrail:` (`throttle`, `debounce`, `overlap`, `disable_after`, `live`) and
|
|
500
|
+
`observability:` (`issues`, `sla`). It reads exactly as the same fields at
|
|
501
|
+
the top; a field written both ways is refused. Step options stay under their
|
|
502
|
+
step. See [The four jobs](four-jobs).
|
|
503
|
+
|
|
495
504
|
## Model resolution
|
|
496
505
|
|
|
497
506
|
Nearest wins: the step, then the flow, then the agent's own frontmatter. The
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
# The four jobs
|
|
2
|
+
|
|
3
|
+
Running agents is four jobs, and every setting that controls them does one:
|
|
4
|
+
|
|
5
|
+
| Job | What it decides |
|
|
6
|
+
|---|---|
|
|
7
|
+
| **Governance** | who answers for it, and what it may spend, decide and ship |
|
|
8
|
+
| **Guardrail** | what it may reach and call, and what stops it mid-run |
|
|
9
|
+
| **Observability** | what is recorded, and who is told |
|
|
10
|
+
| **Evaluation** | how its work is judged |
|
|
11
|
+
|
|
12
|
+
These are the same settings foldrun has always had. You can write each one at
|
|
13
|
+
the top of the file, as before, or group them under the job they do:
|
|
14
|
+
|
|
15
|
+
```yaml
|
|
16
|
+
---
|
|
17
|
+
name: blog-desk
|
|
18
|
+
governance:
|
|
19
|
+
owner: ops@acme.io
|
|
20
|
+
budget: 60/month
|
|
21
|
+
deploy_block: secrets
|
|
22
|
+
guardrail:
|
|
23
|
+
limits: {calls: 300, repeat: 5}
|
|
24
|
+
network: trusted
|
|
25
|
+
observability:
|
|
26
|
+
notify: {email: ops@acme.io, events: [failed]}
|
|
27
|
+
alerts: {failure_rate: 30%, new_host: true}
|
|
28
|
+
evaluation:
|
|
29
|
+
judge: {model: fast}
|
|
30
|
+
---
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
reads exactly as `owner:`, `budget:`, `limits:` and the rest written at the
|
|
34
|
+
top. Either way works, and the two can be mixed in one file.
|
|
35
|
+
|
|
36
|
+
## Which field belongs to which job
|
|
37
|
+
|
|
38
|
+
| File | `governance:` | `guardrail:` | `observability:` | `evaluation:` |
|
|
39
|
+
|---|---|---|---|---|
|
|
40
|
+
| `AGENTS.md` | `owner`, `budget`, `priority`, `concurrency`, `deploy_block` | `limits`, `network`, `network_enforce`, `hooks` | `alerts`, `telemetry`, `notify`, `capture` | `judge` |
|
|
41
|
+
| an agent | `owner`, `budget` | `limits`, `network`, `hooks`, `disallowedTools`, `permissionMode`, `secrets` | — | — |
|
|
42
|
+
| a flow | `owner`, `approvers`, `approve_within`, `budget`, `priority` | `throttle`, `debounce`, `overlap`, `disable_after`, `live` | `issues`, `sla` | — |
|
|
43
|
+
|
|
44
|
+
A field that does none of the four, such as `name`, `model`, `tools`,
|
|
45
|
+
`trigger` or `schedule`, is written at the top. Step options in a flow, such
|
|
46
|
+
as `verify:` and `limits:` under a step, stay where they are.
|
|
47
|
+
|
|
48
|
+
## The rules
|
|
49
|
+
|
|
50
|
+
- **One place, not both.** A field written at the top *and* inside its block
|
|
51
|
+
is refused: `foldrun check` and a deploy stop on
|
|
52
|
+
[`config/conflict`](checks#config-conflict) until one is removed.
|
|
53
|
+
- **The right block.** A setting inside another job's block is not read —
|
|
54
|
+
a guardrail under `governance:` would guard nothing — so check refuses it
|
|
55
|
+
and says which block it belongs in
|
|
56
|
+
([`config/misplaced`](checks#config-misplaced)).
|
|
57
|
+
- **Editing keeps the form.** Settings, the CLI and the flow canvas edit a
|
|
58
|
+
field where it is written, inside its block or at the top, and add a new
|
|
59
|
+
one at the top. A block written on one line (`guardrail: {network: [a.com]}`)
|
|
60
|
+
is not rewritten for you: the edit is refused (`409`) and asks for the block
|
|
61
|
+
one key per line.
|
|
62
|
+
- **Only the three settings files.** Blocks are read in an agent's
|
|
63
|
+
`agent.md`, a flow and `AGENTS.md`. In knowledge, memory, skills and every
|
|
64
|
+
other document, `governance:` and the rest are that document's own words
|
|
65
|
+
and are left as written.
|
|
66
|
+
- **The cascade is unchanged.** The account's `AGENTS.md`, then the
|
|
67
|
+
workspace's, then the agent, then the step: a block at one level and a flat
|
|
68
|
+
key at another combine exactly as two flat keys would.
|
|
69
|
+
|
|
70
|
+
## `judge:` — the evaluation default
|
|
71
|
+
|
|
72
|
+
`judge: {model, effort}` in `AGENTS.md` is the model, and how hard it
|
|
73
|
+
thinks, for every grader in the workspace: each eval's `judge:` checks and
|
|
74
|
+
every flow step's `verify: judge:`. An eval that names its own `model:` keeps
|
|
75
|
+
it. Unset, judges run on the fast tier. Field by field, the workspace's wins:
|
|
76
|
+
a workspace that sets only `effort` keeps the account's `model`.
|
|
77
|
+
|
|
78
|
+
## Where each job shows
|
|
79
|
+
|
|
80
|
+
| Job | In the dashboard | In the CLI |
|
|
81
|
+
|---|---|---|
|
|
82
|
+
| Governance | Settings → Governance: owner, budget, deploy rules, pause, team, API | `foldrun jobs`, `foldrun account set governance.<field>` |
|
|
83
|
+
| Guardrail | Settings → Guardrail: network enforcement; each agent's page | `foldrun check --security`, `foldrun agent network` |
|
|
84
|
+
| Observability | Settings → Observability: notify, alerts, telemetry; Observe, Issues | `foldrun issues`, `foldrun account telemetry` |
|
|
85
|
+
| Evaluation | Settings → Evaluation: the judge; the Evals page | `foldrun eval --to <workspace>` |
|
|
86
|
+
|
|
87
|
+
`foldrun jobs` prints all four; with `--to <workspace>`, each field's value
|
|
88
|
+
there, the account's, and how the two combine. They do not all simply
|
|
89
|
+
override: `network_enforce` is on when either turns it on, a workspace's
|
|
90
|
+
`budget` and the account's both apply, `hooks` run from both, `network`
|
|
91
|
+
takes its level from the nearest setting and its sites from every level, `limits` merge key by key, `judge`
|
|
92
|
+
field by field, and `priority` and `concurrency` are read from the account
|
|
93
|
+
alone. The rest — `owner`, `notify`, `alerts`, `telemetry` and others — the
|
|
94
|
+
workspace's replaces the account's whole. Fields the CLI cannot set (`limits`,
|
|
95
|
+
`network`, `hooks`, `capture`, …) are written in `AGENTS.md`.
|
package/docs/network.md
CHANGED
|
@@ -144,7 +144,7 @@ The list is refused when any one of three says so:
|
|
|
144
144
|
|
|
145
145
|
Each only switches it on. A workspace cannot turn off what its account
|
|
146
146
|
turned on, and nothing in an account turns off the install's. Settings →
|
|
147
|
-
|
|
147
|
+
Guardrail, on the account's and a workspace's Settings, has the switch, and so does
|
|
148
148
|
`foldrun account set network_enforce on` / `foldrun workspace set
|
|
149
149
|
network_enforce on --to <workspace>`.
|
|
150
150
|
|
package/docs/notifications.md
CHANGED
|
@@ -112,6 +112,11 @@ alerts:
|
|
|
112
112
|
new_host: true # an agent reached a site it never reached before
|
|
113
113
|
```
|
|
114
114
|
|
|
115
|
+
A workspace's `alerts:` replaces the account's whole: once it has its own,
|
|
116
|
+
the account's thresholds no longer apply there, and `alerts: {}` means none
|
|
117
|
+
for that workspace (`new_host: false` turns off one the account has on).
|
|
118
|
+
Without an `alerts:` of its own it takes the account's.
|
|
119
|
+
|
|
115
120
|
Each comes to `notify:` as event `alert` — like the three above it follows
|
|
116
121
|
`failed`, or name `alert` in `events:` — at most once a day per rule, and once
|
|
117
122
|
per slow run: "⚠ blog-desk: 6 of its last 15 runs failed", "⚠ blog-desk was
|
|
@@ -135,7 +140,7 @@ Three watch for behaviour that is out of character rather than broken:
|
|
|
135
140
|
usual".
|
|
136
141
|
- `runs_per_hour` — one flow started that many times in the last hour: a
|
|
137
142
|
loop, a webhook sender retrying, a schedule written wrong. At most once a
|
|
138
|
-
day per flow. `throttle:` caps it for good; Settings → Pause stops it now.
|
|
143
|
+
day per flow. `throttle:` caps it for good; Settings → Governance → Pause stops it now.
|
|
139
144
|
- `new_host` — after each run, the sites every agent reached (from the
|
|
140
145
|
egress lines on its trail) are compared with what it reached before, and a
|
|
141
146
|
new one is told: "blog-desk: agent researcher reached a site it never
|
package/docs/runs.md
CHANGED
|
@@ -153,7 +153,7 @@ telemetry:
|
|
|
153
153
|
authorization: ${OTEL_TOKEN} # a vault secret, by name
|
|
154
154
|
```
|
|
155
155
|
|
|
156
|
-
In the account's `AGENTS.md` (Settings →
|
|
156
|
+
In the account's `AGENTS.md` (Settings → Observability → *Send runs to your
|
|
157
157
|
tracing*, or `foldrun account telemetry set <url> --header
|
|
158
158
|
authorization='${OTEL_TOKEN}'`), or a workspace's, which replaces the
|
|
159
159
|
account's whole. Every run that finishes is sent as one trace: a span for
|
package/docs/secrets.md
CHANGED
|
@@ -30,9 +30,36 @@ PUT /api/secrets { "name": "STRIPE_KEY", "value": "sk_live_…" }
|
|
|
30
30
|
| `file` | a PEM, a certificate, a service-account JSON |
|
|
31
31
|
| `oauth2` | a connected third-party account (see below) |
|
|
32
32
|
| `service_account`, `ssh`, `api` | structured credentials the platform knows how to present |
|
|
33
|
+
| `client_cert` | a certificate and key for mutual TLS, presented by the egress proxy |
|
|
34
|
+
| `identity` | keyless cloud access: an audience, and tokens the platform signs (below) |
|
|
33
35
|
|
|
34
36
|
Reads return which secrets exist, never their values.
|
|
35
37
|
|
|
38
|
+
A **service account** is a key file that signs its own token request. It
|
|
39
|
+
acts as itself, or — with **Act as** (`subject`) — as a user the provider
|
|
40
|
+
lets it act for (RFC 7523's grant subject; for Google, domain-wide
|
|
41
|
+
delegation, which a Workspace admin grants in their own console). Each
|
|
42
|
+
subject gets its own token.
|
|
43
|
+
|
|
44
|
+
An **`api`** connection is a base URL and the headers that go with it. For
|
|
45
|
+
an old system that wants **HTTP Basic**, give a username and password
|
|
46
|
+
instead of an `Authorization` header: the platform builds the header (RFC
|
|
47
|
+
7617, UTF-8), refuses a colon in the username, needs an https base URL, and
|
|
48
|
+
masks the password and the encoded header in every trace — as it does every
|
|
49
|
+
header value of an `api` connection.
|
|
50
|
+
|
|
51
|
+
A **client certificate** (mutual TLS) is a PEM certificate, its key and an
|
|
52
|
+
optional passphrase; the key must belong to the certificate or it is
|
|
53
|
+
refused. Bind it to the **exact hosts** it is for (`foldrun secrets sites
|
|
54
|
+
NAME api.bank.example` — no wildcards): the egress proxy presents it on its
|
|
55
|
+
own connection to those hosts, and the sandbox never holds the key. Two
|
|
56
|
+
certificates bound to one host is refused, not guessed. A redirect to
|
|
57
|
+
another site never carries it. Unbound, it stays out of the sandbox; save it
|
|
58
|
+
with **export** only for a step that runs without the egress proxy, and it
|
|
59
|
+
arrives as files (`$NAME_CERT`, `$NAME_KEY`, `$NAME_PASSPHRASE_FILE`). This
|
|
60
|
+
is client authentication with a certificate — not OAuth's certificate-bound
|
|
61
|
+
tokens (RFC 8705), which foldrun does not do.
|
|
62
|
+
|
|
36
63
|
## Scopes
|
|
37
64
|
|
|
38
65
|
A secret lives at the **account** or in one **workspace**. An agent looks in
|
|
@@ -173,6 +200,78 @@ anything OAuth. Settings → Connections walks the consent flow once and stores
|
|
|
173
200
|
the result as an `oauth2` secret; the platform refreshes it. An agent names
|
|
174
201
|
it in `secrets:` like any other.
|
|
175
202
|
|
|
203
|
+
You bring the OAuth app: register it once with the provider (its client id
|
|
204
|
+
and secret, and the redirect URL the form shows), save it, and connect. Every
|
|
205
|
+
later connection can use the same saved app — choose it under **App** and
|
|
206
|
+
type the scopes this connection needs, as the provider spells them. Each
|
|
207
|
+
connection keeps its own scopes, so one app serves any number of APIs from
|
|
208
|
+
the same provider (an ads API in one secret, a search API in another), and
|
|
209
|
+
**Reconnect** asks for that connection's scopes again. The provider decides
|
|
210
|
+
what the app may ask for; foldrun only carries the words. From a terminal:
|
|
211
|
+
`foldrun connect NAME --client <saved app> --scopes "<scopes>" --to <workspace>`.
|
|
212
|
+
|
|
213
|
+
How a connection is kept safe, whatever the provider:
|
|
214
|
+
|
|
215
|
+
- **PKCE** (S256) is on for every connection: a code stolen on its way back
|
|
216
|
+
is useless without the verifier only foldrun holds. An app with no client
|
|
217
|
+
secret (a *public* app) is allowed — PKCE is then what protects it, and
|
|
218
|
+
its refreshes send no secret. Turn PKCE off only for an app with a secret
|
|
219
|
+
whose provider refuses the parameters.
|
|
220
|
+
- **The browser that starts a connection is the one it must come back to.**
|
|
221
|
+
The consent link carries a cookie the callback checks; a link opened in
|
|
222
|
+
someone else's browser connects nothing.
|
|
223
|
+
- **From a terminal**, `foldrun connect --client` opens a page in the
|
|
224
|
+
dashboard where you, signed in, see the secret, the app, the scopes and
|
|
225
|
+
where it will be stored, and accept before the provider is asked.
|
|
226
|
+
- **Issuer.** Give a saved app its provider's issuer and the `iss` the
|
|
227
|
+
provider sends back (RFC 9207) must match it; without one, a returned
|
|
228
|
+
`iss` must at least be the authorize URL's own site.
|
|
229
|
+
- The flow's own parameters (`client_id`, `redirect_uri`, `scope`, `state`,
|
|
230
|
+
the PKCE pair) cannot be overridden by extra parameters or by a query
|
|
231
|
+
string in the authorize URL.
|
|
232
|
+
- A provider that **rotates refresh tokens** sends a new one with each
|
|
233
|
+
refresh; it replaces the old one in the vault, where that secret is kept.
|
|
234
|
+
|
|
235
|
+
## Keyless cloud identity
|
|
236
|
+
|
|
237
|
+
The safest credential is none at all. A cloud that supports workload
|
|
238
|
+
identity federation — AWS (`AssumeRoleWithWebIdentity`), Google Cloud
|
|
239
|
+
(Workload Identity Federation), Microsoft Entra (federated credentials) —
|
|
240
|
+
can trust **foldrun's signature** instead of a key you store: a step proves
|
|
241
|
+
which account, workspace and agent it is, and the cloud gives it a
|
|
242
|
+
short-lived credential of its own.
|
|
243
|
+
|
|
244
|
+
1. In your cloud, trust this install as an OIDC provider: its issuer is
|
|
245
|
+
shown by `foldrun identity` (and in the Connections form). Pin the
|
|
246
|
+
subject — `account:<account>:workspace:<workspace>:agent:<agent>` — at
|
|
247
|
+
least to your account, and the audience you choose.
|
|
248
|
+
2. `foldrun secrets set AWS_ID --kind identity --audience sts.amazonaws.com`
|
|
249
|
+
(or Connections → Identity). Nothing secret is stored: only the audience.
|
|
250
|
+
3. Name it in an agent's `secrets:`. The step gets **`$AWS_ID_FILE`**: a
|
|
251
|
+
token signed RS256, valid ten minutes, renewed while the step runs. Point
|
|
252
|
+
your cloud's SDK at it — for AWS, `AWS_ROLE_ARN` and
|
|
253
|
+
`AWS_WEB_IDENTITY_TOKEN_FILE=$AWS_ID_FILE`; for Google, an
|
|
254
|
+
`external_account` credential file whose `credential_source.file` is it;
|
|
255
|
+
for Entra, the token as the client assertion.
|
|
256
|
+
|
|
257
|
+
What it is and is not:
|
|
258
|
+
|
|
259
|
+
- A **workload-federation issuer**, not a sign-in provider: discovery
|
|
260
|
+
publishes what federation reads (issuer, JWKS, RS256, public subjects),
|
|
261
|
+
and nothing more.
|
|
262
|
+
- The subject and claims come from the platform's own record of the run,
|
|
263
|
+
never from the sandbox. A token is issued only to the step the identity
|
|
264
|
+
was granted to, only while its driver checks in, and never after the step
|
|
265
|
+
stops or ends.
|
|
266
|
+
- Stopping a step stops renewal; it cannot recall a token already issued,
|
|
267
|
+
which expires within ten minutes on its own.
|
|
268
|
+
- Within an account, a workspace or agent recreated under the same name
|
|
269
|
+
is the same subject: pin your trust policy to the account at least.
|
|
270
|
+
- Who may run or edit an agent that names the identity may use it — as
|
|
271
|
+
with every secret.
|
|
272
|
+
- Not on a laptop: only a sandboxed step on a platform whose operator set
|
|
273
|
+
`FOLDRUN_OIDC_ISSUER` gets a token.
|
|
274
|
+
|
|
176
275
|
## How they are kept
|
|
177
276
|
|
|
178
277
|
Every account has its own encryption key. Secrets are sealed under it, and
|
package/docs/workspaces.md
CHANGED
|
@@ -163,6 +163,11 @@ workspace's, and a workspace that does not takes the account's.
|
|
|
163
163
|
trace; a workspace's block replaces the account's whole. See
|
|
164
164
|
[Runs](runs#sending-the-trace-to-your-own-tracing).
|
|
165
165
|
|
|
166
|
+
The frontmatter may also group these defaults by the job they do —
|
|
167
|
+
`governance:`, `guardrail:`, `observability:` and `evaluation:` (where
|
|
168
|
+
`judge:` sets the model every grader here uses). A block reads exactly as the
|
|
169
|
+
same keys at the top. See [The four jobs](four-jobs) for which key goes where.
|
|
170
|
+
|
|
166
171
|
`network_enforce: true` refuses a site off a step's [`network:`](network)
|
|
167
172
|
list instead of only naming it in the trace. The account's `true` holds for
|
|
168
173
|
every workspace; a workspace can add it for itself but not drop the
|
|
@@ -210,7 +215,7 @@ to the next level with one line in the trace rather than failing the step.
|
|
|
210
215
|
|
|
211
216
|
When something is going wrong — an agent posting what it should not, a
|
|
212
217
|
loop spending money — pause the workspace, or the whole account, from its
|
|
213
|
-
Settings → Pause, `foldrun workspace pause` / `foldrun account pause`, or
|
|
218
|
+
Settings → Governance → Pause, `foldrun workspace pause` / `foldrun account pause`, or
|
|
214
219
|
the API (`POST /api/workspaces/<ws>/pause`, `POST /api/account/pause`).
|
|
215
220
|
|
|
216
221
|
While paused, nothing new starts: a schedule's fire, a webhook delivery, a
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "foldrun",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.5.0",
|
|
4
4
|
"description": "Agents are just folders. Write them, check them, run them — on your machine.",
|
|
5
5
|
"license": "SEE LICENSE IN LICENSE",
|
|
6
6
|
"type": "module",
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
"node": ">=22"
|
|
12
12
|
},
|
|
13
13
|
"dependencies": {
|
|
14
|
-
"@foldrun/core": "^2.
|
|
14
|
+
"@foldrun/core": "^2.7.0"
|
|
15
15
|
},
|
|
16
16
|
"files": [
|
|
17
17
|
"bin",
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
],
|
|
22
22
|
"scripts": {
|
|
23
23
|
"e2e": "FOLDRUN_E2E=1 FOLDRUN_DATA=$(mktemp -d /tmp/foldrun-e2e-XXXXXX) node --test --test-timeout=300000 tests/e2e.test.ts",
|
|
24
|
-
"test": "FOLDRUN_DATA=$(mktemp -d /tmp/foldrun-test-XXXXXX) node --test tests/cli-account.test.ts tests/model-credential.test.ts tests/cli-approvals.test.ts tests/cli-issues.test.ts tests/cli-capture.test.ts tests/cli-stop.test.ts tests/cli-account-settings.test.ts tests/cli-schedule.test.ts tests/cli-billing.test.ts tests/cli-storage.test.ts tests/cli-scaffold.test.ts tests/cli-secrets.test.ts tests/cli-site-login.test.ts tests/cli-check-library.test.ts tests/cli-check-timezone.test.ts tests/cli-check-frontmatter.test.ts tests/cli-agent-import.test.ts tests/cli-check-subagents.test.ts tests/cli-connect.test.ts tests/cli-accounts.test.ts tests/cli-remote.test.ts tests/cli-stream.test.ts tests/cli-doctor.test.ts tests/cli-tool-test.test.ts tests/cli-agent-run.test.ts tests/cli-check-remote.test.ts tests/cli-shares.test.ts tests/cli-triggers.test.ts tests/cli-providers.test.ts tests/cli-rerun-put.test.ts tests/cli-guide.test.ts tests/cli-runtimes.test.ts tests/cli-deploy-safety.test.ts tests/cli-operator.test.ts tests/cli-flow-patterns.test.ts tests/cli-flow-rm-step.test.ts tests/cli-flow-dup-step.test.ts tests/cli-flow-write-stale.test.ts tests/cli-agent-link-stale.test.ts tests/cli-parity.test.ts tests/cli-invoke-inputs.test.ts tests/cli-audit-keys.test.ts tests/cli-version.test.ts tests/cli-check-web-warnings.test.ts tests/cli-report-image.test.ts tests/cli-report-cache.test.ts tests/cli-check-workspace-paths.test.ts tests/cli-parity-dashboard.test.ts tests/cli-api-standards.test.ts tests/cli-backups-onboarding.test.ts tests/cli-check-limits.test.ts tests/cli-gallery.test.ts tests/cli-status-platform.test.ts tests/cli-webhooks-notifications.test.ts tests/cli-flags.test.ts tests/cli-api-coverage.test.ts tests/cli-hooks.test.ts tests/cli-release.test.ts tests/cli-terminal.test.ts tests/cli-network.test.ts tests/cli-assistant.test.ts tests/cli-check-scan.test.ts tests/cli-published.test.ts",
|
|
24
|
+
"test": "FOLDRUN_DATA=$(mktemp -d /tmp/foldrun-test-XXXXXX) node --test tests/cli-account.test.ts tests/model-credential.test.ts tests/cli-approvals.test.ts tests/cli-issues.test.ts tests/cli-capture.test.ts tests/cli-stop.test.ts tests/cli-account-settings.test.ts tests/cli-schedule.test.ts tests/cli-billing.test.ts tests/cli-storage.test.ts tests/cli-scaffold.test.ts tests/cli-secrets.test.ts tests/cli-site-login.test.ts tests/cli-check-library.test.ts tests/cli-check-timezone.test.ts tests/cli-check-frontmatter.test.ts tests/cli-agent-import.test.ts tests/cli-check-subagents.test.ts tests/cli-connect.test.ts tests/cli-accounts.test.ts tests/cli-remote.test.ts tests/cli-stream.test.ts tests/cli-doctor.test.ts tests/cli-tool-test.test.ts tests/cli-agent-run.test.ts tests/cli-check-remote.test.ts tests/cli-shares.test.ts tests/cli-triggers.test.ts tests/cli-providers.test.ts tests/cli-rerun-put.test.ts tests/cli-guide.test.ts tests/cli-runtimes.test.ts tests/cli-deploy-safety.test.ts tests/cli-operator.test.ts tests/cli-flow-patterns.test.ts tests/cli-flow-rm-step.test.ts tests/cli-flow-dup-step.test.ts tests/cli-flow-write-stale.test.ts tests/cli-agent-link-stale.test.ts tests/cli-parity.test.ts tests/cli-invoke-inputs.test.ts tests/cli-audit-keys.test.ts tests/cli-version.test.ts tests/cli-check-web-warnings.test.ts tests/cli-report-image.test.ts tests/cli-report-cache.test.ts tests/cli-check-workspace-paths.test.ts tests/cli-parity-dashboard.test.ts tests/cli-api-standards.test.ts tests/cli-backups-onboarding.test.ts tests/cli-check-limits.test.ts tests/cli-gallery.test.ts tests/cli-status-platform.test.ts tests/cli-webhooks-notifications.test.ts tests/cli-flags.test.ts tests/cli-api-coverage.test.ts tests/cli-hooks.test.ts tests/cli-release.test.ts tests/cli-terminal.test.ts tests/cli-network.test.ts tests/cli-assistant.test.ts tests/cli-check-scan.test.ts tests/cli-published.test.ts tests/cli-init-target.test.ts",
|
|
25
25
|
"release": "node scripts/release.mjs",
|
|
26
26
|
"typecheck": "tsc -p tsconfig.json",
|
|
27
27
|
"docs:sync": "node scripts/sync-docs.mjs",
|
package/src/commands.mjs
CHANGED
|
@@ -3674,8 +3674,11 @@ async function connect(positional, flags) {
|
|
|
3674
3674
|
}
|
|
3675
3675
|
|
|
3676
3676
|
const clientId = flags["client-id"] ?? env("OAUTH_CLIENT_ID") ?? (await promptVisible(" client_id: "));
|
|
3677
|
-
const
|
|
3678
|
-
|
|
3677
|
+
const pkce = flags["no-pkce"] !== true;
|
|
3678
|
+
const clientSecret = flags["client-secret"] ?? env("OAUTH_CLIENT_SECRET") ?? (await promptHidden(` client_secret${pkce ? " (empty for a public app)" : ""}: `));
|
|
3679
|
+
if (!clientId) throw new Error("client_id is required");
|
|
3680
|
+
// No secret is a public app, which PKCE alone protects (RFC 9700 §2.1.1).
|
|
3681
|
+
if (!clientSecret && !pkce) throw new Error("an app with no client_secret needs PKCE — drop --no-pkce");
|
|
3679
3682
|
const scopes = flags.scopes ?? preset?.scopes_example ?? "";
|
|
3680
3683
|
|
|
3681
3684
|
// One fixed loopback address, the same on every developer's machine, so an
|
|
@@ -3691,13 +3694,21 @@ async function connect(positional, flags) {
|
|
|
3691
3694
|
const crypto = await import("node:crypto");
|
|
3692
3695
|
const http = await import("node:http");
|
|
3693
3696
|
const state = crypto.randomBytes(24).toString("hex");
|
|
3697
|
+
// PKCE (RFC 7636, S256): 32 random bytes as the verifier, its hash as the challenge.
|
|
3698
|
+
const verifier = pkce ? crypto.randomBytes(32).toString("base64url") : null;
|
|
3694
3699
|
const authorize = new URL(authorizeUrl);
|
|
3700
|
+
// Extras first: the flow's own parameters are set after, and win.
|
|
3701
|
+
for (const [k, v] of Object.entries(preset?.authorize_extra ?? {})) authorize.searchParams.set(k, v);
|
|
3702
|
+
for (const k of ["client_id", "redirect_uri", "response_type", "scope", "state", "code_challenge", "code_challenge_method"]) authorize.searchParams.delete(k);
|
|
3695
3703
|
authorize.searchParams.set("response_type", "code");
|
|
3696
3704
|
authorize.searchParams.set("client_id", clientId);
|
|
3697
3705
|
authorize.searchParams.set("redirect_uri", redirectUri);
|
|
3698
3706
|
authorize.searchParams.set("state", state);
|
|
3699
3707
|
if (scopes) authorize.searchParams.set("scope", scopes);
|
|
3700
|
-
|
|
3708
|
+
if (verifier) {
|
|
3709
|
+
authorize.searchParams.set("code_challenge", crypto.createHash("sha256").update(verifier).digest("base64url"));
|
|
3710
|
+
authorize.searchParams.set("code_challenge_method", "S256");
|
|
3711
|
+
}
|
|
3701
3712
|
|
|
3702
3713
|
// One request, then the listener closes. The state is the whole authority:
|
|
3703
3714
|
// a callback with any other value is answered and ignored.
|
|
@@ -3733,7 +3744,11 @@ async function connect(positional, flags) {
|
|
|
3733
3744
|
const exchange = await fetch(tokenUrl, {
|
|
3734
3745
|
method: "POST",
|
|
3735
3746
|
headers: { "content-type": "application/x-www-form-urlencoded", accept: "application/json" },
|
|
3736
|
-
body: new URLSearchParams({
|
|
3747
|
+
body: new URLSearchParams({
|
|
3748
|
+
grant_type: "authorization_code", code, redirect_uri: redirectUri, client_id: clientId,
|
|
3749
|
+
...(clientSecret ? { client_secret: clientSecret } : {}),
|
|
3750
|
+
...(verifier ? { code_verifier: verifier } : {}),
|
|
3751
|
+
}).toString(),
|
|
3737
3752
|
});
|
|
3738
3753
|
const payload = /** @type {any} */ (await exchange.json().catch(() => ({})));
|
|
3739
3754
|
if (!exchange.ok || (!payload.refresh_token && !payload.access_token)) {
|
|
@@ -3743,7 +3758,7 @@ async function connect(positional, flags) {
|
|
|
3743
3758
|
// Store: the refresh recipe when there is one, the bare token otherwise.
|
|
3744
3759
|
/** @type {any} */
|
|
3745
3760
|
const body = payload.refresh_token
|
|
3746
|
-
? { name, oauth2: { token_url: tokenUrl, client_id: clientId, client_secret: clientSecret, refresh_token: payload.refresh_token }, workspace: workspaceName, scopes, ...(payload.refresh_token_expires_in ? { refresh_token_expires_in: payload.refresh_token_expires_in } : {}) }
|
|
3761
|
+
? { name, oauth2: { token_url: tokenUrl, client_id: clientId, ...(clientSecret ? { client_secret: clientSecret } : {}), refresh_token: payload.refresh_token }, workspace: workspaceName, scopes, ...(payload.refresh_token_expires_in ? { refresh_token_expires_in: payload.refresh_token_expires_in } : {}) }
|
|
3747
3762
|
: { name, value: payload.access_token, workspace: workspaceName };
|
|
3748
3763
|
if (url) {
|
|
3749
3764
|
// The same PUT `secrets set` makes. A store that fails here has spent the
|
|
@@ -5656,6 +5671,145 @@ async function telemetryCmd(url, args, flags) {
|
|
|
5656
5671
|
return 0;
|
|
5657
5672
|
}
|
|
5658
5673
|
|
|
5674
|
+
// ---------------------------------------------------------- the four jobs
|
|
5675
|
+
//
|
|
5676
|
+
// governance, guardrail, observability, evaluation — what each AGENTS.md
|
|
5677
|
+
// field does, as the platform maps it (GET /api/account/jobs). The CLI reads
|
|
5678
|
+
// the map from the platform rather than from @foldrun/core, so it always
|
|
5679
|
+
// matches the platform it is talking to.
|
|
5680
|
+
|
|
5681
|
+
/** GET /api/account/jobs, for the account or one workspace. */
|
|
5682
|
+
function jobsOf(url, flags, workspace) {
|
|
5683
|
+
return remoteCall(url, flags, `/api/account/jobs${workspace ? `?workspace=${enc(workspace)}` : ""}`);
|
|
5684
|
+
}
|
|
5685
|
+
|
|
5686
|
+
/**
|
|
5687
|
+
* `governance.deploy_block` → `deploy_block`, checked against the map: a
|
|
5688
|
+
* field written under the wrong job is refused with the right one named.
|
|
5689
|
+
* A bare key comes back as it was, with no request.
|
|
5690
|
+
*/
|
|
5691
|
+
async function jobKey(url, flags, key, workspace) {
|
|
5692
|
+
if (typeof key !== "string" || !key.includes(".")) return { key, view: null };
|
|
5693
|
+
const job = key.slice(0, key.indexOf("."));
|
|
5694
|
+
const field = key.slice(key.indexOf(".") + 1);
|
|
5695
|
+
const view = await jobsOf(url, flags, workspace);
|
|
5696
|
+
const j = view.jobs.find((x) => x.name === job);
|
|
5697
|
+
if (!j) throw new Error(`"${job}" is not one of the four jobs — governance, guardrail, observability, evaluation`);
|
|
5698
|
+
const f = j.fields.find((x) => x.key === field);
|
|
5699
|
+
if (!f) {
|
|
5700
|
+
const right = view.jobs.find((x) => x.fields.some((y) => y.key === field));
|
|
5701
|
+
throw new Error(right ? `${field} is a ${right.name} setting, not a ${job} one — \`${right.name}.${field}\`` : `${job} has no setting "${field}" — it holds ${j.fields.map((y) => y.key).join(", ")}`);
|
|
5702
|
+
}
|
|
5703
|
+
// `writable` says what this scope's settings can write; the rest live in
|
|
5704
|
+
// AGENTS.md itself. A server too old to say leaves it to the PATCH.
|
|
5705
|
+
if (f.writable === false) {
|
|
5706
|
+
throw new Error(`${job}.${field} is not set from here — write \`${field}:\` in ${workspace ? `workspaces/${workspace}/AGENTS.md` : "the account's AGENTS.md"} (under \`${job}:\` or at the top) and deploy`);
|
|
5707
|
+
}
|
|
5708
|
+
return { key: field, view };
|
|
5709
|
+
}
|
|
5710
|
+
|
|
5711
|
+
/** The value a field has at one level (the account's, or the workspace's). */
|
|
5712
|
+
async function jobValue(url, flags, key, workspace, view) {
|
|
5713
|
+
const v = view ?? (await jobsOf(url, flags, workspace));
|
|
5714
|
+
for (const j of v.jobs) {
|
|
5715
|
+
const f = j.fields.find((x) => x.key === key);
|
|
5716
|
+
if (f) return workspace ? f.workspace : f.account;
|
|
5717
|
+
}
|
|
5718
|
+
return null;
|
|
5719
|
+
}
|
|
5720
|
+
|
|
5721
|
+
/**
|
|
5722
|
+
* `set alerts <threshold> <value>` and `set judge model|effort <value>` —
|
|
5723
|
+
* one entry merged into the block as it stands (the platform replaces a
|
|
5724
|
+
* block whole). `off` removes the entry. A workspace's alerts, once its own,
|
|
5725
|
+
* stay its own: the last threshold removed leaves `{}` ("none here"), and
|
|
5726
|
+
* `new_host off` is written as false so the account's `on` does not apply;
|
|
5727
|
+
* `clear alerts` goes back to the account's.
|
|
5728
|
+
*/
|
|
5729
|
+
async function mergedEntry(url, flags, key, args, workspace, view, how) {
|
|
5730
|
+
const [entry, ...words] = args;
|
|
5731
|
+
const raw = words.join(" ").trim();
|
|
5732
|
+
const allowed = key === "alerts"
|
|
5733
|
+
? ["failure_rate", "min_runs", "credits_day", "run_minutes", "stale_days", "cost_spike", "runs_per_hour", "new_host"]
|
|
5734
|
+
: ["model", "effort"];
|
|
5735
|
+
if (!allowed.includes(entry)) throw new Error(`${how} <${allowed.join("|")}> <value> — not "${entry ?? ""}"`);
|
|
5736
|
+
if (!raw) throw new Error(`${how} ${entry} <value> — or \`${entry} off\` to remove it`);
|
|
5737
|
+
const current = { ...((await jobValue(url, flags, key, workspace, view)) ?? {}) };
|
|
5738
|
+
if (raw === "off") delete current[entry];
|
|
5739
|
+
else if (entry === "new_host") current.new_host = onOff("new_host", raw, `${how} new_host on`);
|
|
5740
|
+
else current[entry] = key === "alerts" && entry !== "failure_rate" && /^\d+(\.\d+)?$/.test(raw) ? Number(raw) : raw;
|
|
5741
|
+
const own = key === "alerts" && workspace;
|
|
5742
|
+
// A workspace's `new_host: false` is an opt-out from the account's on —
|
|
5743
|
+
// kept whatever else is edited; at the account false is just unset.
|
|
5744
|
+
if (own && entry === "new_host" && raw === "off") current.new_host = false;
|
|
5745
|
+
else if (!own && current.new_host === false) delete current.new_host;
|
|
5746
|
+
if (own) return current;
|
|
5747
|
+
return Object.keys(current).length ? current : null;
|
|
5748
|
+
}
|
|
5749
|
+
|
|
5750
|
+
/** A setting's value as one short line. */
|
|
5751
|
+
function shortValue(v) {
|
|
5752
|
+
if (v === null || v === undefined) return null;
|
|
5753
|
+
return typeof v === "object" ? JSON.stringify(v).replace(/"([a-z_]+)":/g, "$1: ").replace(/,/g, ", ") : String(v);
|
|
5754
|
+
}
|
|
5755
|
+
|
|
5756
|
+
/**
|
|
5757
|
+
* `foldrun identity [--to <workspace>]` — what a cloud must trust to accept
|
|
5758
|
+
* this account's keyless identities: the issuer, and each agent's subject.
|
|
5759
|
+
*/
|
|
5760
|
+
async function identityCmd(flags) {
|
|
5761
|
+
const url = platformFor(flags, "identity");
|
|
5762
|
+
const ws = typeof flags.to === "string" ? flags.to : null;
|
|
5763
|
+
const info = await remoteCall(url, flags, `/api/identity${ws ? `?workspace=${enc(ws)}` : ""}`);
|
|
5764
|
+
if (flags.json === true) {
|
|
5765
|
+
console.log(JSON.stringify(info, null, 2));
|
|
5766
|
+
return 0;
|
|
5767
|
+
}
|
|
5768
|
+
if (!info.enabled) {
|
|
5769
|
+
console.log(`\n ${c.yellow("This install does not issue identity tokens")} ${c.dim("— its operator sets FOLDRUN_OIDC_ISSUER")}\n`);
|
|
5770
|
+
return 1;
|
|
5771
|
+
}
|
|
5772
|
+
console.log(`\n ${c.bold("Keyless cloud identity")}\n`);
|
|
5773
|
+
console.log(` issuer ${info.issuer}`);
|
|
5774
|
+
console.log(` subject ${info.subject}`);
|
|
5775
|
+
console.log(` tokens ${Math.round(info.tokenSeconds / 60)} minutes, renewed while a step runs`);
|
|
5776
|
+
console.log(`\n ${c.dim("Trust the issuer in your cloud and pin at least the account in the subject; then `foldrun secrets set NAME --kind identity --audience <aud>` and name it in an agent's secrets:")}\n`);
|
|
5777
|
+
return 0;
|
|
5778
|
+
}
|
|
5779
|
+
|
|
5780
|
+
/**
|
|
5781
|
+
* `foldrun jobs [--to <workspace>]` — the four jobs and every AGENTS.md
|
|
5782
|
+
* field of each: the account's value, the workspace's, and how the two
|
|
5783
|
+
* combine (they do not all simply override — network_enforce is on if
|
|
5784
|
+
* either turns it on, both budgets apply).
|
|
5785
|
+
*/
|
|
5786
|
+
async function jobsCmd(flags, layout) {
|
|
5787
|
+
const url = platformFor(flags, "jobs");
|
|
5788
|
+
const ws = typeof flags.to === "string" ? flags.to : null;
|
|
5789
|
+
const view = await jobsOf(url, flags, ws);
|
|
5790
|
+
if (flags.json === true) {
|
|
5791
|
+
console.log(JSON.stringify(view, null, 2));
|
|
5792
|
+
return 0;
|
|
5793
|
+
}
|
|
5794
|
+
console.log(`\n ${c.bold("The four jobs")} ${c.dim(ws ? `workspace ${ws}, over the account's` : "the account — every workspace inherits these")}`);
|
|
5795
|
+
for (const j of view.jobs) {
|
|
5796
|
+
console.log(`\n ${c.bold(j.title)} ${c.dim(j.what)}`);
|
|
5797
|
+
const w = Math.max(...j.fields.map((f) => f.key.length));
|
|
5798
|
+
for (const f of j.fields) {
|
|
5799
|
+
const account = shortValue(f.account);
|
|
5800
|
+
if (!ws) {
|
|
5801
|
+
console.log(` ${pad(f.key, w)} ${account ?? c.dim("unset")}`);
|
|
5802
|
+
continue;
|
|
5803
|
+
}
|
|
5804
|
+
const here = shortValue(f.workspace);
|
|
5805
|
+
const how = f.combine ? c.dim(` · ${f.combine}`) : "";
|
|
5806
|
+
console.log(` ${pad(f.key, w)} ${here ?? c.dim("unset")}${c.dim(` account: ${account ?? "unset"}`)}${how}`);
|
|
5807
|
+
}
|
|
5808
|
+
}
|
|
5809
|
+
console.log(`\n ${c.dim(`set one: foldrun ${ws ? `workspace set <job>.<field> … --to ${ws}` : "account set <job>.<field> …"} — e.g. governance.deploy_block secrets`)}\n`);
|
|
5810
|
+
return 0;
|
|
5811
|
+
}
|
|
5812
|
+
|
|
5659
5813
|
/** A GET …/pause, or null from a platform older than the kill switch —
|
|
5660
5814
|
* the settings still print, without the line. */
|
|
5661
5815
|
async function pauseState(request) {
|
|
@@ -5748,15 +5902,23 @@ async function accountCmd(positional, flags) {
|
|
|
5748
5902
|
throw new Error(`unknown account verb "${verb}" — \`foldrun account\` shows them, \`set\` and \`clear\` change one, \`model\` is the account's own model key, \`providers\` checks the model keys, \`export\` downloads everything, \`pause\`/\`resume\` the kill switch`);
|
|
5749
5903
|
}
|
|
5750
5904
|
|
|
5751
|
-
|
|
5752
|
-
const
|
|
5905
|
+
// `governance.deploy_block` names the job; checked, then sent as the field.
|
|
5906
|
+
const { key, view } = await jobKey(url, flags, positional[1], null);
|
|
5907
|
+
// telemetry: is set with its own verb (an endpoint and headers, not one
|
|
5908
|
+
// value); `clear` works here as for any other key.
|
|
5909
|
+
if (key === "telemetry" && verb === "set") {
|
|
5910
|
+
throw new Error("telemetry has its own verb — `foldrun account telemetry set <otlp-url> [--header name=value]`; `foldrun account clear telemetry` turns it off");
|
|
5911
|
+
}
|
|
5912
|
+
const KEYS = ["timezone", "notify", "budget", "concurrency", "deploy_block", "network_enforce", "owner", "alerts", "judge", ...(verb === "clear" ? ["telemetry"] : [])];
|
|
5753
5913
|
if (!KEYS.includes(key)) {
|
|
5754
|
-
throw new Error(`which setting? \`foldrun account ${verb} <${KEYS.join("|")}>
|
|
5914
|
+
throw new Error(`which setting? \`foldrun account ${verb} <${KEYS.join("|")}> …\` — or <job>.<field>, e.g. governance.owner (\`foldrun jobs\` lists them)`);
|
|
5755
5915
|
}
|
|
5756
5916
|
|
|
5757
5917
|
let patch;
|
|
5758
5918
|
if (verb === "clear") {
|
|
5759
5919
|
patch = { [key]: null };
|
|
5920
|
+
} else if (key === "alerts" || key === "judge") {
|
|
5921
|
+
patch = { [key]: await mergedEntry(url, flags, key, positional.slice(2), null, view, `foldrun account set ${key}`) };
|
|
5760
5922
|
} else if (key === "notify") {
|
|
5761
5923
|
// `set notify email you@example.com` / `set notify url https://…`, and
|
|
5762
5924
|
// --events to change what is notified on.
|
|
@@ -8440,9 +8602,50 @@ function printEvalResult(result) {
|
|
|
8440
8602
|
* run there. Each run waits for its verdict: an eval is something you wait
|
|
8441
8603
|
* for, and the platform answers when every case has.
|
|
8442
8604
|
*/
|
|
8605
|
+
/**
|
|
8606
|
+
* One eval on the platform, started and polled (POST …/run { async: true },
|
|
8607
|
+
* then GET …/run) — a long eval outlives the proxy in front of the platform,
|
|
8608
|
+
* which cut the old waiting request at 100 s. A platform that answers the
|
|
8609
|
+
* POST with the result itself (older ones) is read as before.
|
|
8610
|
+
*/
|
|
8611
|
+
async function remoteEvalRun(url, flags, ws, name) {
|
|
8612
|
+
const path = `/api/workspaces/${enc(ws)}/evals/${enc(name)}/run`;
|
|
8613
|
+
const started = await remoteCall(url, flags, path, { method: "POST", body: JSON.stringify({ async: true }) }, { seconds: explicitTimeout(flags) ?? 900 });
|
|
8614
|
+
if (!started?.started) return started;
|
|
8615
|
+
console.log(` ${c.dim(`run ${started.runId}`)}`);
|
|
8616
|
+
const deadline = Date.now() + (explicitTimeout(flags) ?? 900) * 1000;
|
|
8617
|
+
for (;;) {
|
|
8618
|
+
if (Date.now() > deadline) {
|
|
8619
|
+
throw new Error(`${name} is still running after ${explicitTimeout(flags) ?? 900}s — \`foldrun eval ${name} --to ${ws} --run ${started.runId}\` reads this run's verdict when it is done, without running it again`);
|
|
8620
|
+
}
|
|
8621
|
+
await new Promise((r) => setTimeout(r, Number(process.env.FOLDRUN_EVAL_POLL_MS) || 4000));
|
|
8622
|
+
const view = await remoteEvalRunView(url, flags, path, started.runId);
|
|
8623
|
+
if (view.state === "running") continue;
|
|
8624
|
+
return evalRunVerdict(name, view);
|
|
8625
|
+
}
|
|
8626
|
+
}
|
|
8627
|
+
|
|
8628
|
+
/** GET …/run?run=<id> — that one run's state, and nothing else's. */
|
|
8629
|
+
function remoteEvalRunView(url, flags, path, runId) {
|
|
8630
|
+
return remoteCall(url, flags, `${path}?run=${encodeURIComponent(runId)}`);
|
|
8631
|
+
}
|
|
8632
|
+
|
|
8633
|
+
/** A run's view as the result printEvalResult reads — a failure, a lost
|
|
8634
|
+
* lease or an unknown id are an error, never a pass. */
|
|
8635
|
+
function evalRunVerdict(name, view) {
|
|
8636
|
+
if (view.state === "done" && view.result) return view.result;
|
|
8637
|
+
return { eval: name, passed: 0, failed: 0, cases: [], error: view.error ?? `the run ended as ${view.state ?? "unknown"}, with no result` };
|
|
8638
|
+
}
|
|
8639
|
+
|
|
8443
8640
|
async function remoteEvalsCmd(name, flags, layout) {
|
|
8444
8641
|
const url = platformFor(flags, "eval --to <workspace>");
|
|
8445
8642
|
const ws = platformWorkspace(flags, layout, "eval");
|
|
8643
|
+
// --run reads one run of one eval: a bare --run, or one with no eval
|
|
8644
|
+
// named, would otherwise start runs or read the id against every eval.
|
|
8645
|
+
if (flags.run !== undefined) {
|
|
8646
|
+
if (typeof flags.run !== "string" || !flags.run.trim()) throw new Error("--run takes the run id the eval printed — `foldrun eval <name> --to <ws> --run <id>`");
|
|
8647
|
+
if (!name) throw new Error("--run reads one eval's run — name it: `foldrun eval <name> --to <ws> --run <id>`");
|
|
8648
|
+
}
|
|
8446
8649
|
const { evals = [] } = await remoteCall(url, flags, `/api/workspaces/${enc(ws)}/evals`);
|
|
8447
8650
|
const all = evals.filter((e) => !name || e.name === name);
|
|
8448
8651
|
if (!all.length) {
|
|
@@ -8451,7 +8654,36 @@ async function remoteEvalsCmd(name, flags, layout) {
|
|
|
8451
8654
|
let failed = 0;
|
|
8452
8655
|
for (const info of all) {
|
|
8453
8656
|
console.log(`\n ${c.bold(info.name)} ${c.dim(`${info.cases?.length ?? "?"} cases · ${ws}`)}`);
|
|
8454
|
-
|
|
8657
|
+
// --run <id>: that run's verdict, run nothing (GET …/run?run=).
|
|
8658
|
+
if (typeof flags.run === "string") {
|
|
8659
|
+
const view = await remoteEvalRunView(url, flags, `/api/workspaces/${enc(ws)}/evals/${enc(info.name)}/run`, flags.run);
|
|
8660
|
+
if (view.state === "running") {
|
|
8661
|
+
console.log(` ${c.dim(`run ${flags.run} is still running, since ${view.startedAt}`)}`);
|
|
8662
|
+
failed++;
|
|
8663
|
+
continue;
|
|
8664
|
+
}
|
|
8665
|
+
const result = evalRunVerdict(info.name, view);
|
|
8666
|
+
printEvalResult(result);
|
|
8667
|
+
failed += Number(result.failed ?? 0) + (result.error ? 1 : 0);
|
|
8668
|
+
continue;
|
|
8669
|
+
}
|
|
8670
|
+
// --last: what the platform has, run nothing (GET …/run).
|
|
8671
|
+
if (flags.last === true) {
|
|
8672
|
+
const state = await remoteCall(url, flags, `/api/workspaces/${enc(ws)}/evals/${enc(info.name)}/run`);
|
|
8673
|
+
if (state.running) console.log(` ${c.dim(`running since ${state.startedAt} — no result yet`)}`);
|
|
8674
|
+
if (state.error) {
|
|
8675
|
+
console.log(` ${c.red("✗")} the last run could not finish — ${state.error}`);
|
|
8676
|
+
failed++;
|
|
8677
|
+
}
|
|
8678
|
+
if (!state.result) {
|
|
8679
|
+
if (!state.running && !state.error) console.log(` ${c.dim("never run")}`);
|
|
8680
|
+
continue;
|
|
8681
|
+
}
|
|
8682
|
+
printEvalResult(state.result);
|
|
8683
|
+
failed += Number(state.result.failed ?? 0) + (state.result.error ? 1 : 0);
|
|
8684
|
+
continue;
|
|
8685
|
+
}
|
|
8686
|
+
const result = await remoteEvalRun(url, flags, ws, info.name);
|
|
8455
8687
|
printEvalResult(result);
|
|
8456
8688
|
failed += Number(result.failed ?? 0) + (result.error ? 1 : 0);
|
|
8457
8689
|
}
|
|
@@ -9065,8 +9297,10 @@ async function workspaceDefaults(url, flags, ws) {
|
|
|
9065
9297
|
async function workspaceCmd(positional, flags, layout) {
|
|
9066
9298
|
const url = platformFor(flags, "workspace");
|
|
9067
9299
|
const ws = platformWorkspace(flags, layout, "workspace");
|
|
9068
|
-
const [verb,
|
|
9069
|
-
|
|
9300
|
+
const [verb, written, ...rest] = positional;
|
|
9301
|
+
// `observability.notify` names the job; checked, then sent as the field.
|
|
9302
|
+
const { key, view } = verb === "set" || verb === "clear" ? await jobKey(url, flags, written, ws) : { key: written, view: null };
|
|
9303
|
+
const KEYS = ["name", "description", "timezone", "budget", "notify", "network_enforce", "owner", "alerts", "judge"];
|
|
9070
9304
|
const show = (d, title) => {
|
|
9071
9305
|
const notify = d.notify ? [d.notify.email, d.notify.url].filter(Boolean).join(", ") + c.dim(` on ${(d.notify.events ?? []).join(", ") || "nothing"}`) : c.dim("the account's");
|
|
9072
9306
|
const budget = d.budget == null ? c.dim("the account's") : typeof d.budget === "object" ? `$${d.budget.usd} per ${d.budget.period}` : String(d.budget).includes("/") ? `$${String(d.budget).replace("/", " per ")}` : `$${d.budget} per month`;
|
|
@@ -9100,7 +9334,9 @@ async function workspaceCmd(positional, flags, layout) {
|
|
|
9100
9334
|
/** @type {Record<string, any>} */
|
|
9101
9335
|
let patch;
|
|
9102
9336
|
if (verb === "clear") patch = { [key]: key === "description" ? "" : null };
|
|
9103
|
-
else if (key === "
|
|
9337
|
+
else if (key === "alerts" || key === "judge") {
|
|
9338
|
+
patch = { [key]: await mergedEntry(url, flags, key, rest, ws, view, `foldrun workspace set ${key}`) };
|
|
9339
|
+
} else if (key === "notify") {
|
|
9104
9340
|
const [channel, value] = rest;
|
|
9105
9341
|
if (channel !== "email" && channel !== "url") throw new Error("`foldrun workspace set notify email <address>` or `set notify url <https://…>` — and --events failed,awaiting-approval,completed");
|
|
9106
9342
|
if (!value && flags.events === undefined) throw new Error(`\`foldrun workspace set notify ${channel} <value>\` — or --events alone to change only what is notified on`);
|
|
@@ -10296,9 +10532,17 @@ async function oauthClientsCmd(positional, flags) {
|
|
|
10296
10532
|
const token_url = flags["token-url"] ?? preset?.token_url;
|
|
10297
10533
|
if (!authorize_url || !token_url) throw new Error("--provider, or both --authorize-url and --token-url");
|
|
10298
10534
|
const client_id = flags["client-id"] ?? env("OAUTH_CLIENT_ID") ?? (await promptVisible(" client_id: "));
|
|
10299
|
-
const
|
|
10300
|
-
|
|
10301
|
-
|
|
10535
|
+
const pkce = flags["no-pkce"] !== true;
|
|
10536
|
+
const client_secret = flags["client-secret"] ?? env("OAUTH_CLIENT_SECRET") ?? (await promptHidden(` client_secret${pkce ? " (empty for a public app)" : ""}: `));
|
|
10537
|
+
if (!client_id) throw new Error("client_id is required");
|
|
10538
|
+
if (!client_secret && !pkce) throw new Error("an app with no client_secret needs PKCE — drop --no-pkce");
|
|
10539
|
+
if (flags.issuer !== undefined && (typeof flags.issuer !== "string" || !/^https:\/\/[^?#]+$/.test(flags.issuer))) throw new Error("--issuer is the provider's https issuer URL, no query or fragment");
|
|
10540
|
+
const config = {
|
|
10541
|
+
authorize_url, token_url, client_id, client_secret: client_secret ?? "", scopes: flags.scopes ?? preset?.scopes_example ?? "",
|
|
10542
|
+
...(preset?.authorize_extra ? { authorize_extra: preset.authorize_extra } : {}),
|
|
10543
|
+
...(pkce ? {} : { pkce: false }),
|
|
10544
|
+
...(typeof flags.issuer === "string" ? { issuer: flags.issuer } : {}),
|
|
10545
|
+
};
|
|
10302
10546
|
await remoteCall(url, flags, "/api/oauth/clients", { method: "POST", body: JSON.stringify({ name, config }) });
|
|
10303
10547
|
console.log(`\n ${c.green("✓")} saved ${name} ${c.dim(`— \`foldrun connect NAME --client ${name}\` runs a consent from it`)}\n`);
|
|
10304
10548
|
return 0;
|
|
@@ -10315,9 +10559,15 @@ async function connectFromClient(name, flags) {
|
|
|
10315
10559
|
const url = platformFor(flags, "connect --client");
|
|
10316
10560
|
const workspace = typeof flags.to === "string" ? flags.to : undefined;
|
|
10317
10561
|
const before = (await remoteCall(url, flags, "/api/oauth/connections")).connections?.find((s) => s.name === name && (s.workspace ?? undefined) === workspace)?.connectedAt ?? null;
|
|
10318
|
-
|
|
10562
|
+
// --scopes: this connection's own, instead of the saved client's default;
|
|
10563
|
+
// the platform keeps them with the connection for its reconnects.
|
|
10564
|
+
const scopes = typeof flags.scopes === "string" && flags.scopes.trim() ? { scopes: flags.scopes } : {};
|
|
10565
|
+
const started = await remoteCall(url, flags, "/api/oauth/start", { method: "POST", body: JSON.stringify({ client: flags.client, secret: name, ...(workspace ? { workspace } : {}), ...scopes }) });
|
|
10319
10566
|
const opened = flags["no-browser"] === true || !process.stdout.isTTY ? false : await openInBrowser(started.url);
|
|
10320
|
-
|
|
10567
|
+
// The platform answers a hand-off page: a signed-in person accepts the
|
|
10568
|
+
// connection there, then the provider's consent runs in that browser.
|
|
10569
|
+
const what = started.handoff ? "accept the connection, then consent" : "consent";
|
|
10570
|
+
console.log(opened ? `\n Opened your browser to ${what} with the saved ${c.bold(flags.client)} app.\n` : `\n Open this address in your browser to ${what}:\n\n ${started.url}\n`);
|
|
10321
10571
|
console.log(` ${c.dim("Waiting for the platform to store it…")}`);
|
|
10322
10572
|
const deadline = Date.now() + 10 * 60 * 1000;
|
|
10323
10573
|
while (Date.now() < deadline) {
|
|
@@ -10352,7 +10602,11 @@ async function secretOfKind(kind, flags) {
|
|
|
10352
10602
|
}
|
|
10353
10603
|
case "api": {
|
|
10354
10604
|
const lines = [].concat(flags.header ?? []);
|
|
10355
|
-
|
|
10605
|
+
// --basic-user: HTTP Basic from a username and password (prompted, or
|
|
10606
|
+
// --basic-password); the platform builds the Authorization header.
|
|
10607
|
+
const basicUser = typeof flags["basic-user"] === "string" ? flags["basic-user"] : null;
|
|
10608
|
+
const basic = basicUser ? { basic: { username: basicUser, password: typeof flags["basic-password"] === "string" ? flags["basic-password"] : await promptHidden(" password: ") } } : {};
|
|
10609
|
+
if (!lines.length && !basicUser) lines.push(await promptHidden(" header (Name: value): "));
|
|
10356
10610
|
const headers = Object.fromEntries(
|
|
10357
10611
|
lines.filter(Boolean).map((l) => {
|
|
10358
10612
|
const i = String(l).indexOf(":");
|
|
@@ -10361,7 +10615,23 @@ async function secretOfKind(kind, flags) {
|
|
|
10361
10615
|
}),
|
|
10362
10616
|
);
|
|
10363
10617
|
const base = typeof flags["base-url"] === "string" ? { base_url: flags["base-url"] } : {};
|
|
10364
|
-
return { body: { api: { ...base, headers } }, local: (k, n, b, s) => k.setApiSecret("default", n, b.api, s) };
|
|
10618
|
+
return { body: { api: { ...base, headers, ...basic } }, local: (k, n, b, s) => k.setApiSecret("default", n, b.api, s) };
|
|
10619
|
+
}
|
|
10620
|
+
case "identity": {
|
|
10621
|
+
// A keyless cloud identity: only the audience; the platform signs the
|
|
10622
|
+
// tokens (`foldrun identity` prints what the cloud must trust).
|
|
10623
|
+
if (typeof flags.audience !== "string" || !flags.audience.trim()) throw new Error("--kind identity needs --audience <value your cloud's trust expects in aud>");
|
|
10624
|
+
return { body: { identity: { audience: flags.audience } }, local: (k, n, b, s) => k.setIdentitySecret("default", n, b.identity, s) };
|
|
10625
|
+
}
|
|
10626
|
+
case "client-cert": {
|
|
10627
|
+
// A client certificate for mutual TLS. Bind it to the exact hosts it is
|
|
10628
|
+
// for with `foldrun secrets sites NAME host`; the egress proxy presents
|
|
10629
|
+
// it there. --export also puts it in the sandbox as files.
|
|
10630
|
+
const cert = read(flags["cert-file"], "--cert-file <cert.pem> — the certificate, chain included");
|
|
10631
|
+
const key = read(flags["key-file"], "--key-file <key.pem> — its private key");
|
|
10632
|
+
const passphrase = typeof flags.passphrase === "string" ? flags.passphrase : undefined;
|
|
10633
|
+
const client_cert = { cert, key, ...(passphrase ? { passphrase } : {}), ...(flags.export === true ? { export: true } : {}) };
|
|
10634
|
+
return { body: { client_cert }, local: (k, n, b, s) => k.setClientCertSecret("default", n, b.client_cert, s) };
|
|
10365
10635
|
}
|
|
10366
10636
|
case "service-account": {
|
|
10367
10637
|
let j;
|
|
@@ -10371,7 +10641,9 @@ async function secretOfKind(kind, flags) {
|
|
|
10371
10641
|
if (err instanceof SyntaxError) throw new Error(`${flags.file} is not JSON — the key file Google gives you`);
|
|
10372
10642
|
throw err;
|
|
10373
10643
|
}
|
|
10374
|
-
|
|
10644
|
+
// --subject: the user it acts as (domain-wide delegation, granted by the
|
|
10645
|
+
// provider's admin); without it the account acts as itself.
|
|
10646
|
+
const sa = { token_url: j.token_uri ?? "https://oauth2.googleapis.com/token", issuer: j.client_email, private_key: j.private_key, scope: flags.scopes ?? "", ...(typeof flags.subject === "string" ? { subject: flags.subject } : {}) };
|
|
10375
10647
|
return { body: { service_account: sa }, local: (k, n, b, s) => k.setServiceAccountSecret("default", n, b.service_account, s) };
|
|
10376
10648
|
}
|
|
10377
10649
|
case "m2m": {
|
|
@@ -10382,7 +10654,7 @@ async function secretOfKind(kind, flags) {
|
|
|
10382
10654
|
return { body: { oauth2 }, local: (k, n, b, s) => k.setOAuth2Secret("default", n, b.oauth2, s) };
|
|
10383
10655
|
}
|
|
10384
10656
|
default:
|
|
10385
|
-
throw new Error(`unknown --kind "${kind}" — file, ssh, api, service-account or
|
|
10657
|
+
throw new Error(`unknown --kind "${kind}" — file, ssh, api, service-account, m2m, client-cert or identity (a plain value needs no --kind; --oauth2 a refresh token; \`foldrun connect\` a browser sign-in)`);
|
|
10386
10658
|
}
|
|
10387
10659
|
}
|
|
10388
10660
|
|
|
@@ -10948,6 +11220,9 @@ export async function run(command, positional, flags, workspace, layout) {
|
|
|
10948
11220
|
if (positional[0] === "new" && positional[1]) return evalNewCmd(positional[1], flags, layout);
|
|
10949
11221
|
// --to names a deployed workspace: its evals run there.
|
|
10950
11222
|
if (typeof flags.to === "string") return remoteEvalsCmd(positional[0], flags, layout);
|
|
11223
|
+
// --last reads the platform's results; a local eval has none to read,
|
|
11224
|
+
// and running the cases instead would break its promise to run nothing.
|
|
11225
|
+
if (flags.last === true || flags.run !== undefined) throw new Error("--last and --run read a deployed workspace's results — add --to <workspace>");
|
|
10951
11226
|
needsOne(layout, "eval");
|
|
10952
11227
|
return runEvals(positional[0], flags);
|
|
10953
11228
|
case "promote":
|
|
@@ -11035,6 +11310,10 @@ export async function run(command, positional, flags, workspace, layout) {
|
|
|
11035
11310
|
return rerunCmd(positional[0], flags, layout);
|
|
11036
11311
|
case "account":
|
|
11037
11312
|
return accountCmd(positional, flags);
|
|
11313
|
+
case "identity":
|
|
11314
|
+
return identityCmd(flags);
|
|
11315
|
+
case "jobs":
|
|
11316
|
+
return jobsCmd(flags, layout);
|
|
11038
11317
|
case "schedule":
|
|
11039
11318
|
if (positional[0] === "tick") return scheduleTickCmd(flags);
|
|
11040
11319
|
return scheduleCmd(flags);
|