@foldrun/core 2.3.0 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/agent-activity.d.ts +5 -0
- package/dist/src/agent-activity.js +31 -0
- package/dist/src/agent-import.js +1 -1
- package/dist/src/agent-tools.js +1 -1
- package/dist/src/agents-md.js +1 -1
- package/dist/src/alerts.d.ts +60 -0
- package/dist/src/alerts.js +161 -0
- package/dist/src/browser-pod.js +4 -0
- package/dist/src/capture.d.ts +23 -0
- package/dist/src/capture.js +80 -0
- package/dist/src/compare.d.ts +57 -0
- package/dist/src/compare.js +116 -0
- package/dist/src/completions.js +1 -0
- package/dist/src/deploy.d.ts +36 -1
- package/dist/src/deploy.js +170 -11
- package/dist/src/egress.d.ts +16 -1
- package/dist/src/evals.d.ts +5 -1
- package/dist/src/evals.js +26 -18
- package/dist/src/frontmatter-fields.js +9 -0
- package/dist/src/frontmatter-schema.js +3 -0
- package/dist/src/history.d.ts +4 -0
- package/dist/src/history.js +15 -0
- package/dist/src/hooks.js +1 -1
- package/dist/src/insights.d.ts +30 -0
- package/dist/src/insights.js +152 -0
- package/dist/src/issues.d.ts +140 -0
- package/dist/src/issues.js +302 -0
- package/dist/src/library.js +1 -1
- package/dist/src/limits.d.ts +8 -0
- package/dist/src/limits.js +36 -5
- package/dist/src/linkable.js +1 -1
- package/dist/src/matter.d.ts +8 -0
- package/dist/src/matter.js +28 -0
- package/dist/src/model-credential.d.ts +14 -0
- package/dist/src/model-credential.js +40 -0
- package/dist/src/model-env.d.ts +14 -2
- package/dist/src/model-env.js +34 -9
- package/dist/src/network.d.ts +12 -0
- package/dist/src/network.js +25 -1
- package/dist/src/notify.d.ts +16 -0
- package/dist/src/notify.js +17 -7
- package/dist/src/okf.js +1 -1
- package/dist/src/online-checks.d.ts +76 -0
- package/dist/src/online-checks.js +432 -0
- package/dist/src/operator.d.ts +277 -0
- package/dist/src/operator.js +176 -2
- package/dist/src/otel-config.d.ts +18 -0
- package/dist/src/otel-config.js +69 -0
- package/dist/src/otel.d.ts +38 -0
- package/dist/src/otel.js +85 -0
- package/dist/src/outcome.d.ts +40 -0
- package/dist/src/outcome.js +77 -0
- package/dist/src/package.js +1 -1
- package/dist/src/platform.d.ts +1 -1
- package/dist/src/prompt-layers.d.ts +30 -0
- package/dist/src/prompt-layers.js +77 -0
- package/dist/src/provider-health.js +1 -1
- package/dist/src/providers.d.ts +5 -0
- package/dist/src/providers.js +2 -2
- package/dist/src/run-container.d.ts +16 -2
- package/dist/src/run-container.js +57 -5
- package/dist/src/runner.d.ts +53 -2
- package/dist/src/runner.js +356 -70
- package/dist/src/runs-filter.d.ts +48 -0
- package/dist/src/runs-filter.js +73 -0
- package/dist/src/scan.d.ts +19 -1
- package/dist/src/scan.js +60 -7
- package/dist/src/starter.js +4 -2
- package/dist/src/step-exec.d.ts +29 -1
- package/dist/src/step-exec.js +106 -6
- package/dist/src/store.d.ts +130 -3
- package/dist/src/store.js +145 -6
- package/dist/src/tokens.d.ts +38 -0
- package/dist/src/tokens.js +57 -0
- package/dist/src/tool-names.js +1 -1
- package/dist/src/tool-programs.js +1 -1
- package/dist/src/trace-tree.d.ts +40 -0
- package/dist/src/trace-tree.js +156 -0
- package/dist/src/translator.d.ts +8 -0
- package/dist/src/translator.js +7 -2
- package/dist/src/trigger-log.d.ts +1 -1
- package/dist/src/workspace-check.d.ts +18 -0
- package/dist/src/workspace-check.js +34 -0
- package/package.json +2 -2
- package/src/agent-activity.ts +32 -0
- package/src/agent-import.ts +1 -1
- package/src/agent-tools.ts +1 -1
- package/src/agents-md.ts +1 -1
- package/src/alerts.ts +190 -0
- package/src/browser-pod.ts +3 -0
- package/src/capture.ts +88 -0
- package/src/compare.ts +146 -0
- package/src/completions.ts +1 -0
- package/src/deploy.ts +184 -10
- package/src/egress.ts +12 -2
- package/src/evals.ts +26 -18
- package/src/frontmatter-fields.ts +9 -0
- package/src/frontmatter-schema.ts +3 -0
- package/src/history.ts +13 -0
- package/src/hooks.ts +1 -1
- package/src/insights.ts +174 -0
- package/src/issues.ts +396 -0
- package/src/library.ts +1 -1
- package/src/limits.ts +38 -5
- package/src/linkable.ts +1 -1
- package/src/matter.ts +33 -0
- package/src/model-credential.ts +45 -0
- package/src/model-env.ts +42 -10
- package/src/network.ts +25 -1
- package/src/notify.ts +22 -7
- package/src/okf.ts +1 -1
- package/src/online-checks.ts +450 -0
- package/src/operator.ts +213 -3
- package/src/otel-config.ts +67 -0
- package/src/otel.ts +100 -0
- package/src/outcome.ts +90 -0
- package/src/package.ts +1 -1
- package/src/platform.ts +1 -1
- package/src/prompt-layers.ts +90 -0
- package/src/provider-health.ts +1 -1
- package/src/providers.ts +7 -2
- package/src/run-container.ts +66 -7
- package/src/runner.ts +363 -46
- package/src/runs-filter.ts +88 -0
- package/src/scan.ts +68 -7
- package/src/starter.ts +4 -2
- package/src/step-exec.ts +147 -9
- package/src/store.ts +254 -8
- package/src/tokens.ts +68 -0
- package/src/tool-names.ts +1 -1
- package/src/tool-programs.ts +1 -1
- package/src/trace-tree.ts +187 -0
- package/src/translator.ts +13 -2
- package/src/trigger-log.ts +1 -0
- package/src/workspace-check.ts +41 -0
package/dist/src/deploy.js
CHANGED
|
@@ -20,20 +20,24 @@
|
|
|
20
20
|
import fs from "node:fs";
|
|
21
21
|
import os from "node:os";
|
|
22
22
|
import path from "node:path";
|
|
23
|
-
import { saveWorkspace, keptOnDeploy, workspaceDir, listRunSummaries, listWorkspaceFiles, parseFlow, parseToolDef, WORKSPACE_DIRS, assertSafeName, } from "./store.js";
|
|
23
|
+
import { saveWorkspace, keptOnDeploy, workspaceDir, listRunSummaries, listWorkspaceFiles, parseFlow, parseToolDef, WORKSPACE_DIRS, assertSafeName, accountDir, frontmatterErrorText, } from "./store.js";
|
|
24
|
+
import { readAgentsMdKeys } from "./agents-md.js";
|
|
25
|
+
import { libraryTools } from "./library.js";
|
|
24
26
|
import { lintFlow } from "./flow-lint.js";
|
|
25
27
|
import { refNames } from "./refs.js";
|
|
26
|
-
import matter from "
|
|
28
|
+
import matter from "./matter.js";
|
|
27
29
|
import { conformanceIssues } from "./okf.js";
|
|
28
30
|
import { timezoneProblem } from "./clock.js";
|
|
29
31
|
import { webProblems, webWarnings, runnerEngines } from "./providers.js";
|
|
30
32
|
import { readLimits } from "./limits.js";
|
|
33
|
+
import { parseAlerts } from "./alerts.js";
|
|
34
|
+
import { parseTelemetry } from "./otel-config.js";
|
|
31
35
|
import { networkProblems } from "./network.js";
|
|
32
36
|
import { actionProblems } from "./web-actions.js";
|
|
33
37
|
import { languageProblem } from "./language.js";
|
|
34
38
|
import { localeProblems } from "./locale.js";
|
|
35
39
|
import { entriesNoFollow } from "./paths.js";
|
|
36
|
-
import { filterIgnored, findingFingerprint, ruleForWhere, scanFiles } from "./scan.js";
|
|
40
|
+
import { filterIgnored, findingFingerprint, ruleForWhere, scanFiles, scanRule } from "./scan.js";
|
|
37
41
|
/** Directories a deploy never reads out of a source tree. */
|
|
38
42
|
// `.cache` is what a run's tools leave in an agent's folder (HOME is that
|
|
39
43
|
// folder): logs and font caches, never authored.
|
|
@@ -73,9 +77,40 @@ export function isSourcePath(rel) {
|
|
|
73
77
|
// head. Shared by the deploy import (readTree) and the CLI checker so the two
|
|
74
78
|
// agree on what counts as an agent.
|
|
75
79
|
export const SUBAGENT_IMPORT_DIRS = [".agents/agents", ".claude/agents"];
|
|
76
|
-
|
|
80
|
+
/** What agents write as they run, not what anyone authored: state/ and
|
|
81
|
+
* memory/, the workspace's or one agent's. */
|
|
82
|
+
const DATA_PATH = /^(?:agents\/[^/]+\/)?(?:state|memory)\//;
|
|
83
|
+
export class TreeTooLarge extends Error {
|
|
84
|
+
/** What an HTTP route answers with it. */
|
|
85
|
+
status = 413;
|
|
86
|
+
}
|
|
87
|
+
export function readTree(dir, opts = {}) {
|
|
77
88
|
const root = path.resolve(dir);
|
|
78
89
|
const out = [];
|
|
90
|
+
let dataBytes = 0;
|
|
91
|
+
let dataFiles = 0;
|
|
92
|
+
let sourceBytes = 0;
|
|
93
|
+
/** May this file be read? Data over its budget is left out; source over
|
|
94
|
+
* its budget ends the read. */
|
|
95
|
+
const admit = (rel, bytes) => {
|
|
96
|
+
if (DATA_PATH.test(rel)) {
|
|
97
|
+
const over = (opts.maxDataFileBytes !== undefined && bytes > opts.maxDataFileBytes) ||
|
|
98
|
+
(opts.maxDataBytes !== undefined && dataBytes + bytes > opts.maxDataBytes) ||
|
|
99
|
+
(opts.maxDataFiles !== undefined && dataFiles >= opts.maxDataFiles);
|
|
100
|
+
if (over) {
|
|
101
|
+
opts.onSkip?.(rel, bytes);
|
|
102
|
+
return false;
|
|
103
|
+
}
|
|
104
|
+
dataBytes += bytes;
|
|
105
|
+
dataFiles++;
|
|
106
|
+
return true;
|
|
107
|
+
}
|
|
108
|
+
sourceBytes += bytes;
|
|
109
|
+
if (opts.maxSourceBytes !== undefined && sourceBytes > opts.maxSourceBytes) {
|
|
110
|
+
throw new TreeTooLarge(`the workspace's files other than state/ and memory/ are over ${Math.round(opts.maxSourceBytes / 1048576)} MiB — not read`);
|
|
111
|
+
}
|
|
112
|
+
return true;
|
|
113
|
+
};
|
|
79
114
|
const walk = (abs, rel) => {
|
|
80
115
|
let entries;
|
|
81
116
|
try {
|
|
@@ -104,6 +139,8 @@ export function readTree(dir) {
|
|
|
104
139
|
walk(full, next);
|
|
105
140
|
}
|
|
106
141
|
else if (next === "AGENTS.md" || next === "project.md" || IN_WORKSPACE_DIR.test(next)) {
|
|
142
|
+
if (!admit(next, stat.size))
|
|
143
|
+
continue;
|
|
107
144
|
out.push({ path: next, content: fs.readFileSync(full, "utf8") });
|
|
108
145
|
}
|
|
109
146
|
}
|
|
@@ -143,11 +180,16 @@ export function readTree(dir) {
|
|
|
143
180
|
continue;
|
|
144
181
|
const full = path.join(root, importDir, entry);
|
|
145
182
|
try {
|
|
146
|
-
|
|
183
|
+
const st = fs.lstatSync(full);
|
|
184
|
+
if (st.isSymbolicLink())
|
|
185
|
+
continue;
|
|
186
|
+
if (!admit(target, st.size))
|
|
147
187
|
continue;
|
|
148
188
|
out.push({ path: target, content: fs.readFileSync(full, "utf8") });
|
|
149
189
|
}
|
|
150
|
-
catch {
|
|
190
|
+
catch (err) {
|
|
191
|
+
if (err instanceof TreeTooLarge)
|
|
192
|
+
throw err;
|
|
151
193
|
// an unreadable import is skipped, not fatal
|
|
152
194
|
}
|
|
153
195
|
}
|
|
@@ -195,6 +237,89 @@ export function unopenedFrontmatter(raw) {
|
|
|
195
237
|
* advice (an agent that could leak, an unpinned package) is for `foldrun
|
|
196
238
|
* check` and the Checks page, not for every deploy's output.
|
|
197
239
|
*/
|
|
240
|
+
/**
|
|
241
|
+
* What `deploy_block:` in AGENTS.md refuses (FOL-58). `none`, the default:
|
|
242
|
+
* the scanner's findings are warnings and the deploy goes ahead, as it always
|
|
243
|
+
* did. `secrets`: a key or password written into a file is refused.
|
|
244
|
+
* `security`: every security finding is — secrets, hidden or injected text,
|
|
245
|
+
* the lethal trifecta, pipe-to-shell, an unpinned MCP server, a step that
|
|
246
|
+
* acts outside with no check or approval. A
|
|
247
|
+
* `foldrun-ignore` with a reason still accepts one.
|
|
248
|
+
*/
|
|
249
|
+
export const DEPLOY_BLOCK_MODES = ["none", "secrets", "security"];
|
|
250
|
+
const blockMode = (v) => typeof v === "string" && DEPLOY_BLOCK_MODES.includes(v.trim()) ? v.trim() : null;
|
|
251
|
+
/** The workspace's own AGENTS.md, else the account's, else none. */
|
|
252
|
+
export function deployBlockMode(files, accountKeys) {
|
|
253
|
+
// project.md is the older name readAgentsMdKeys still reads (Codex review).
|
|
254
|
+
const own = files.find((f) => f.path === "AGENTS.md") ?? files.find((f) => f.path === "project.md");
|
|
255
|
+
let fromWorkspace = null;
|
|
256
|
+
if (own) {
|
|
257
|
+
try {
|
|
258
|
+
fromWorkspace = blockMode(matter(own.content).data.deploy_block);
|
|
259
|
+
}
|
|
260
|
+
catch {
|
|
261
|
+
// an AGENTS.md that does not parse is reported elsewhere
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
return fromWorkspace ?? blockMode(accountKeys.deploy_block) ?? "none";
|
|
265
|
+
}
|
|
266
|
+
function blockersFor(tenant, given) {
|
|
267
|
+
// The tree as it will be written: `./AGENTS.md` is saved as AGENTS.md, so
|
|
268
|
+
// it is read as that here too (Codex review).
|
|
269
|
+
// Two entries for one path: the save keeps the last, so this does too.
|
|
270
|
+
const byPath = new Map();
|
|
271
|
+
for (const f of given) {
|
|
272
|
+
// path.normalize, as saveWorkspace writes it (a backslash is a filename
|
|
273
|
+
// character here, not a separator — Codex review).
|
|
274
|
+
const p = path.normalize(f.path);
|
|
275
|
+
byPath.set(p, { ...f, path: p });
|
|
276
|
+
}
|
|
277
|
+
const files = [...byPath.values()];
|
|
278
|
+
const account = readAgentsMdKeys(accountDir(tenant));
|
|
279
|
+
const mode = deployBlockMode(files, account);
|
|
280
|
+
if (mode === "none")
|
|
281
|
+
return [];
|
|
282
|
+
// Nearest wins: a workspace tool of the same name replaces the library's,
|
|
283
|
+
// and the scanner adds the workspace's own outward tools itself. Named as
|
|
284
|
+
// the runtime names them — the definition's name:, else the file's.
|
|
285
|
+
const ownTools = new Set();
|
|
286
|
+
for (const f of files) {
|
|
287
|
+
const m = /^tools\/([^/]+?)(?:\/tool)?\.md$/.exec(f.path);
|
|
288
|
+
if (!m)
|
|
289
|
+
continue;
|
|
290
|
+
let data = {};
|
|
291
|
+
try {
|
|
292
|
+
data = matter(f.content).data;
|
|
293
|
+
}
|
|
294
|
+
catch {
|
|
295
|
+
continue; // unreadable: the runtime skips it and keeps the library's
|
|
296
|
+
}
|
|
297
|
+
// Only a definition the runtime accepts replaces the library's; an
|
|
298
|
+
// invalid one is skipped there, and the library tool stays (Codex review).
|
|
299
|
+
const def = parseToolDef(data, m[1], f.content);
|
|
300
|
+
if (def)
|
|
301
|
+
ownTools.add(def.name ?? m[1]);
|
|
302
|
+
}
|
|
303
|
+
const outwardTools = Object.entries(libraryTools(tenant))
|
|
304
|
+
.filter(([n, d]) => d.outward === true && !ownTools.has(n))
|
|
305
|
+
.map(([n]) => n);
|
|
306
|
+
return deployBlockers(files, mode, { accountNetwork: account.network, outwardTools });
|
|
307
|
+
}
|
|
308
|
+
/** The findings `mode` turns into refusals, as deploy issues. */
|
|
309
|
+
export function deployBlockers(files, mode, options = {}) {
|
|
310
|
+
if (mode === "none")
|
|
311
|
+
return [];
|
|
312
|
+
// Every finding checkFiles reports — the scanner's and the structural
|
|
313
|
+
// ones, flow/outward-ungated among them (Codex review) — with the same
|
|
314
|
+
// ignores honoured, kept by its rule's category.
|
|
315
|
+
const categories = new Set(mode === "secrets" ? ["secret"] : ["secret", "content", "security", "supply-chain"]);
|
|
316
|
+
return checkFiles(files, options)
|
|
317
|
+
.filter((f) => f.level !== "info" && categories.has(scanRule(f.rule)?.category ?? ""))
|
|
318
|
+
.map((f) => ({
|
|
319
|
+
where: `${f.file}${f.line ? `:${f.line}` : ""}`,
|
|
320
|
+
message: `${f.message} [${f.rule}] — refused: deploy_block: ${mode} in AGENTS.md`,
|
|
321
|
+
}));
|
|
322
|
+
}
|
|
198
323
|
export function deployWarnings(files) {
|
|
199
324
|
const scanned = scanFiles(files, { only: ["secret", "content"] })
|
|
200
325
|
.filter((f) => f.level !== "info")
|
|
@@ -217,11 +342,11 @@ export function checkFiles(files, options = {}) {
|
|
|
217
342
|
};
|
|
218
343
|
const own = [
|
|
219
344
|
...deployIssues(files).map((i) => finding(i, "error")),
|
|
220
|
-
...structuralWarnings(files).map((i) => finding(i, "warn", i.kind)),
|
|
345
|
+
...structuralWarnings(files, options.outwardTools).map((i) => finding(i, "warn", i.kind)),
|
|
221
346
|
];
|
|
222
347
|
return [...filterIgnored(own, files), ...scanFiles(files, options)].sort((a, b) => a.file.localeCompare(b.file) || (a.line ?? 0) - (b.line ?? 0) || a.rule.localeCompare(b.rule));
|
|
223
348
|
}
|
|
224
|
-
function structuralWarnings(files) {
|
|
349
|
+
function structuralWarnings(files, extraOutward = []) {
|
|
225
350
|
const front = (content) => {
|
|
226
351
|
try {
|
|
227
352
|
return matter(content).data;
|
|
@@ -230,8 +355,9 @@ function structuralWarnings(files) {
|
|
|
230
355
|
return {};
|
|
231
356
|
}
|
|
232
357
|
};
|
|
233
|
-
//
|
|
234
|
-
|
|
358
|
+
// Tools that change something outside the workspace: the workspace's own,
|
|
359
|
+
// and the library's the caller resolved (nearest wins already applied).
|
|
360
|
+
const outwardTools = new Set(extraOutward);
|
|
235
361
|
for (const f of files) {
|
|
236
362
|
const m = f.path.match(/^tools\/([^/]+?)(?:\/tool)?\.md$/);
|
|
237
363
|
if (!m)
|
|
@@ -242,6 +368,9 @@ function structuralWarnings(files) {
|
|
|
242
368
|
// and the CLI use; parseToolDef does not fold it onto the def.
|
|
243
369
|
if (def && data.outward === true)
|
|
244
370
|
outwardTools.add(def.name ?? m[1]);
|
|
371
|
+
// A workspace tool that is not outward replaces a library one that was.
|
|
372
|
+
else if (def)
|
|
373
|
+
outwardTools.delete(def.name ?? m[1]);
|
|
245
374
|
}
|
|
246
375
|
// Agents that grant one of them.
|
|
247
376
|
const outwardAgents = [];
|
|
@@ -311,6 +440,26 @@ export function deployIssues(files) {
|
|
|
311
440
|
if (m)
|
|
312
441
|
agents.add(m[1]);
|
|
313
442
|
}
|
|
443
|
+
// Frontmatter whose YAML does not parse is a file the runtime cannot read:
|
|
444
|
+
// an agent with no tools and the default model, a flow that never fires,
|
|
445
|
+
// a tool that is skipped. Refused where it was written. Until 20 Oct 2026
|
|
446
|
+
// nothing said so — gray-matter's cache let a second parse pass — and one
|
|
447
|
+
// such agent file kept the worker from booting for every account.
|
|
448
|
+
// Definition files only: a skill is read leniently (an unquoted colon is
|
|
449
|
+
// quoted and retried, runner.ts safeMatter), and the markdown beside a
|
|
450
|
+
// definition — references, a tool's README — is prose, not frontmatter.
|
|
451
|
+
for (const f of files) {
|
|
452
|
+
if (!/^(AGENTS\.md|agents\/[^/]+\/agent\.md|flows\/[^/]+\.md|tools\/[^/]+\.md|tools\/[^/]+\/tool\.md|evals\/[^/]+\.md|hooks\/[^/]+\/hook\.md|\.claude\/agents\/[^/]+\.md)$/.test(f.path))
|
|
453
|
+
continue;
|
|
454
|
+
if (!/^\uFEFF?---\r?\n/.test(f.content))
|
|
455
|
+
continue;
|
|
456
|
+
try {
|
|
457
|
+
matter(f.content);
|
|
458
|
+
}
|
|
459
|
+
catch (err) {
|
|
460
|
+
at(f.path, frontmatterErrorText(err));
|
|
461
|
+
}
|
|
462
|
+
}
|
|
314
463
|
// `timezone:` is read at four levels and a value nothing can parse falls
|
|
315
464
|
// silently through to the level above — so it is refused where it was
|
|
316
465
|
// written, not discovered as a date that is a day out.
|
|
@@ -346,6 +495,13 @@ export function deployIssues(files) {
|
|
|
346
495
|
// above it — wider or narrower than the author wrote.
|
|
347
496
|
for (const w of networkProblems(front.network))
|
|
348
497
|
at(f.path, w);
|
|
498
|
+
// `alerts:` that cannot be read would never alert (FOL-72).
|
|
499
|
+
for (const w of parseAlerts(front.alerts).problems)
|
|
500
|
+
at(f.path, w);
|
|
501
|
+
// `telemetry:` that cannot be read would never send — and a token
|
|
502
|
+
// written into it would ship in the file.
|
|
503
|
+
for (const w of parseTelemetry(front.telemetry).problems)
|
|
504
|
+
at(f.path, w);
|
|
349
505
|
const lang = languageProblem(front.language);
|
|
350
506
|
if (lang)
|
|
351
507
|
at(f.path, lang);
|
|
@@ -489,7 +645,10 @@ export function planDeploy(tenant, workspace, files, opts = {}) {
|
|
|
489
645
|
added: added.sort(),
|
|
490
646
|
updated: updated.sort(),
|
|
491
647
|
removed: removed.sort(),
|
|
492
|
-
|
|
648
|
+
// deploy_block: (FOL-58) turns the scanner's findings into refusals —
|
|
649
|
+
// scanned with what the Checks page passes too: the account's network:
|
|
650
|
+
// and the library's outward tools (Codex review).
|
|
651
|
+
issues: [...deployIssues(files), ...blockersFor(tenant, files)],
|
|
493
652
|
warnings: deployWarnings(files),
|
|
494
653
|
blockedBy: runsInFlight(tenant, workspace),
|
|
495
654
|
};
|
package/dist/src/egress.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { PeerInfo } from "./operator.ts";
|
|
1
2
|
import type { TestEffect } from "./test-mode.ts";
|
|
2
3
|
import type { NetworkLevel } from "./network.ts";
|
|
3
4
|
/** `${NAME}` — the shape a secret reference takes everywhere in foldrun. */
|
|
@@ -92,14 +93,28 @@ export interface EgressHooks {
|
|
|
92
93
|
* policy in test-mode.ts to every request on this lease. Carried on the
|
|
93
94
|
* lease, not the request, because the pod is the party not trusted to
|
|
94
95
|
* say so. */
|
|
95
|
-
/** `step` — the step's index in run.steps; keys that step's operator inbox.
|
|
96
|
+
/** `step` — the step's index in run.steps; keys that step's operator inbox.
|
|
97
|
+
* `agent` and `peers` — this step's agent and the other steps of its
|
|
98
|
+
* parallel group (tools: [peers]): the proxy serves the roster and
|
|
99
|
+
* addresses a colleague's message to the right inbox. */
|
|
96
100
|
lease(args: {
|
|
97
101
|
tenant: string;
|
|
98
102
|
runId: string;
|
|
99
103
|
grant: EgressGrant;
|
|
100
104
|
test?: boolean;
|
|
101
105
|
step?: number;
|
|
106
|
+
agent?: string;
|
|
107
|
+
peers?: PeerInfo[];
|
|
102
108
|
}): Promise<EgressLease | null>;
|
|
109
|
+
/** The step is over — every attempt, however it ended (tools: [peers]).
|
|
110
|
+
* A lease comes and goes per attempt, so it cannot say this: a sibling
|
|
111
|
+
* between retries, or waiting for a slot, holds none and is not done.
|
|
112
|
+
* Optional: a platform without it keeps every sibling reachable. */
|
|
113
|
+
stepEnded?(args: {
|
|
114
|
+
tenant: string;
|
|
115
|
+
runId: string;
|
|
116
|
+
step: number;
|
|
117
|
+
}): Promise<void>;
|
|
103
118
|
}
|
|
104
119
|
/** Strip a header set of the proxy's own and hop-by-hop headers. */
|
|
105
120
|
export declare const HOP_BY_HOP: Set<string>;
|
package/dist/src/evals.d.ts
CHANGED
|
@@ -62,10 +62,14 @@ export declare function listEvals(tenant: string, workspace: string): EvalInfo[]
|
|
|
62
62
|
* regressions file when there is one, so a flow collects its cases in one
|
|
63
63
|
* place instead of one file per press.
|
|
64
64
|
*/
|
|
65
|
-
export declare function promoteRunToEval(tenant: string, workspace: string, runId: string,
|
|
65
|
+
export declare function promoteRunToEval(tenant: string, workspace: string, runId: string,
|
|
66
|
+
/** `should`: what a run that went wrong should have done, in a sentence —
|
|
67
|
+
* the judge's claim for the case (FOL-72 step 10). */
|
|
68
|
+
opts?: {
|
|
66
69
|
evalName?: string;
|
|
67
70
|
caseName?: string;
|
|
68
71
|
expect?: string[];
|
|
72
|
+
should?: string;
|
|
69
73
|
by?: string;
|
|
70
74
|
}): {
|
|
71
75
|
file: string;
|
package/dist/src/evals.js
CHANGED
|
@@ -30,16 +30,16 @@
|
|
|
30
30
|
// Deterministic checks run first and cost nothing. The judge only runs if
|
|
31
31
|
// they pass — no point paying a model to grade output already known to be
|
|
32
32
|
// wrong.
|
|
33
|
-
import { accountModelEnv } from "./model-env.js";
|
|
33
|
+
import { accountModelEnv, withModelEnv } from "./model-env.js";
|
|
34
34
|
import { sdkEnv } from "./step-exec.js";
|
|
35
35
|
import fs from "node:fs";
|
|
36
36
|
import path from "node:path";
|
|
37
37
|
import { deployedCommit } from "./deploy.js";
|
|
38
38
|
import { latestRevisionId } from "./history.js";
|
|
39
|
-
import matter from "
|
|
39
|
+
import matter from "./matter.js";
|
|
40
40
|
import { spawn } from "node:child_process";
|
|
41
41
|
import { query } from "@anthropic-ai/claude-agent-sdk";
|
|
42
|
-
import { workspaceDir, readRun, writeWorkspaceFile, resolveModel, resolveEffort, assertSafeName, } from "./store.js";
|
|
42
|
+
import { workspaceDir, readRun, runVerdict, writeWorkspaceFile, resolveModel, resolveEffort, assertSafeName, } from "./store.js";
|
|
43
43
|
import { startFlowRun, loadFlow } from "./runner.js";
|
|
44
44
|
import { platform, platformQueues } from "./platform.js";
|
|
45
45
|
const ASSERTION_TYPES = ["contains", "not-contains", "matches", "file", "run", "judge"];
|
|
@@ -198,16 +198,27 @@ function layoutTask(task) {
|
|
|
198
198
|
* regressions file when there is one, so a flow collects its cases in one
|
|
199
199
|
* place instead of one file per press.
|
|
200
200
|
*/
|
|
201
|
-
export function promoteRunToEval(tenant, workspace, runId,
|
|
201
|
+
export function promoteRunToEval(tenant, workspace, runId,
|
|
202
|
+
/** `should`: what a run that went wrong should have done, in a sentence —
|
|
203
|
+
* the judge's claim for the case (FOL-72 step 10). */
|
|
204
|
+
opts = {}) {
|
|
202
205
|
const run = readRun(tenant, workspace, runId);
|
|
203
206
|
if (!run)
|
|
204
207
|
throw refuse(`run ${runId} not found`, 404);
|
|
205
|
-
if (run.status !== "completed") {
|
|
206
|
-
throw refuse(`only a
|
|
208
|
+
if (run.status !== "completed" && run.status !== "failed") {
|
|
209
|
+
throw refuse(`only a finished run can become an eval — this one is ${run.status}`, 409);
|
|
210
|
+
}
|
|
211
|
+
// A run that went wrong is the test worth having most — but its own answer
|
|
212
|
+
// is the wrong one to hold the next run to, so it needs to be told what
|
|
213
|
+
// was right (FOL-72 step 10).
|
|
214
|
+
const should = oneLine(opts.should ?? "");
|
|
215
|
+
const wentWrong = run.status === "failed" || ["BAD", "BLOCKED"].includes(runVerdict(run) ?? "");
|
|
216
|
+
if (wentWrong && !should && !(opts.expect && opts.expect.length)) {
|
|
217
|
+
throw refuse("this run went wrong, so its answer is not the one to keep — say what it should have done (`should`, a sentence for the judge) or give `expect` lines", 409);
|
|
207
218
|
}
|
|
208
219
|
const produced = run.steps.filter((s) => (s.result ?? "").trim());
|
|
209
220
|
const last = produced[produced.length - 1];
|
|
210
|
-
if (!last)
|
|
221
|
+
if (!last && !should && !(opts.expect && opts.expect.length))
|
|
211
222
|
throw refuse("this run produced no result to hold the next one to", 409);
|
|
212
223
|
if (run.steps.length === 0 || !run.steps[0].instruction?.trim()) {
|
|
213
224
|
throw refuse("this run has no task on record", 409);
|
|
@@ -239,6 +250,9 @@ export function promoteRunToEval(tenant, workspace, runId, opts = {}) {
|
|
|
239
250
|
return `${m[1]}: ${m[2]}`;
|
|
240
251
|
});
|
|
241
252
|
}
|
|
253
|
+
else if (should) {
|
|
254
|
+
expect = [`judge: ${should.slice(0, 400)}`];
|
|
255
|
+
}
|
|
242
256
|
else {
|
|
243
257
|
const conclusion = oneLine(run.summary?.trim() || (last.result ?? "")).slice(0, 200);
|
|
244
258
|
expect = [`judge: reaches the same conclusion as the recorded run: ${conclusion}`];
|
|
@@ -382,7 +396,7 @@ async function judge(rubric, output, model, effort, env) {
|
|
|
382
396
|
}
|
|
383
397
|
return text.trim();
|
|
384
398
|
}
|
|
385
|
-
async function checkAssertion(assertion, output, agentDir, model, effort, judgeEnv = () => sdkEnv(undefined)) {
|
|
399
|
+
async function checkAssertion(assertion, output, agentDir, model, effort, judgeEnv = () => ({ env: sdkEnv(undefined) })) {
|
|
386
400
|
const hay = output.toLowerCase();
|
|
387
401
|
switch (assertion.type) {
|
|
388
402
|
case "contains": {
|
|
@@ -417,14 +431,14 @@ async function checkAssertion(assertion, output, agentDir, model, effort, judgeE
|
|
|
417
431
|
case "judge": {
|
|
418
432
|
// The judge runs on the account's own model key, never quietly on the
|
|
419
433
|
// platform's: a customer's eval is not the operator's bill.
|
|
420
|
-
let
|
|
434
|
+
let m;
|
|
421
435
|
try {
|
|
422
|
-
|
|
436
|
+
m = judgeEnv();
|
|
423
437
|
}
|
|
424
438
|
catch (err) {
|
|
425
439
|
return { assertion, passed: false, detail: `judge not run — ${err instanceof Error ? err.message : String(err)}` };
|
|
426
440
|
}
|
|
427
|
-
const verdict = await judge(assertion.value, output, model, effort, env);
|
|
441
|
+
const verdict = await withModelEnv(m, (env) => judge(assertion.value, output, model, effort, env));
|
|
428
442
|
const passed = /^\s*PASS\b/i.test(verdict);
|
|
429
443
|
return { assertion, passed, detail: verdict.slice(0, 300) || "judge returned nothing" };
|
|
430
444
|
}
|
|
@@ -495,13 +509,7 @@ export async function runEval(tenant, workspace, info) {
|
|
|
495
509
|
for (const a of cheap)
|
|
496
510
|
assertions.push(await checkAssertion(a, output, agentDir, info.model, info.effort));
|
|
497
511
|
if (assertions.every((r) => r.passed)) {
|
|
498
|
-
const judgeEnv = () =>
|
|
499
|
-
const m = accountModelEnv(tenant, workspace, () => sdkEnv(undefined));
|
|
500
|
-
if (m.format !== "anthropic") {
|
|
501
|
-
throw new Error(`this account's provider speaks ${m.format}; a judge needs an Anthropic-format provider`);
|
|
502
|
-
}
|
|
503
|
-
return m.env;
|
|
504
|
-
};
|
|
512
|
+
const judgeEnv = () => accountModelEnv(tenant, workspace, () => sdkEnv(undefined));
|
|
505
513
|
for (const a of judges)
|
|
506
514
|
assertions.push(await checkAssertion(a, output, agentDir, info.model, info.effort, judgeEnv));
|
|
507
515
|
}
|
|
@@ -24,6 +24,7 @@ const LOCALE = { validated: true };
|
|
|
24
24
|
const AGENT = [
|
|
25
25
|
{ key: "name", type: "string", help: "kebab-case; the folder is the identity" },
|
|
26
26
|
{ key: "description", type: "string", help: "what it does and when — other agents and people read it" },
|
|
27
|
+
{ key: "owner", type: "string", help: "the person accountable for it — an email or a name; else the workspace's, else the account's" },
|
|
27
28
|
{ key: "model", type: "string", help: "fast | default | max, an alias, or a full model id" },
|
|
28
29
|
{ key: "effort", type: "string", values: EFFORT_LEVELS, help: "how long it thinks" },
|
|
29
30
|
{ key: "size", type: "string", values: ["small", "large", "heavy"], help: "the sandbox it rents; anything else becomes large" },
|
|
@@ -59,6 +60,7 @@ const AGENTS_MD = [
|
|
|
59
60
|
{ key: "mdagent_version", type: ["string", "number"], legacy: true, help: "older spelling of foldrun_version" },
|
|
60
61
|
{ key: "name", type: "string", help: "the workspace's name" },
|
|
61
62
|
{ key: "description", type: "string", help: "shown in the workspace list" },
|
|
63
|
+
{ key: "owner", type: "string", help: "the person accountable for every agent and flow here that names none" },
|
|
62
64
|
{ key: "provider", type: "object", help: "the model endpoint every agent here uses unless it names its own" },
|
|
63
65
|
{ key: "runtime", type: "object", help: "packages for agents that declare none" },
|
|
64
66
|
{ key: "web", type: "object", validated: true, help: "web.browse defaults" },
|
|
@@ -75,17 +77,24 @@ const AGENTS_MD = [
|
|
|
75
77
|
{ key: "budget", type: ["number", "string"], help: "USD per month, or N/day | N/week | N/month" },
|
|
76
78
|
{ key: "priority", type: "string", values: ["high", "normal", "low"], help: "account: where its runs stand in the queue" },
|
|
77
79
|
{ key: "concurrency", type: ["number", "string"], help: "account: runs at once" },
|
|
80
|
+
{ key: "alerts", type: "object", validated: true, help: "thresholds that tell notify: — failure_rate (30%), min_runs, credits_day, run_minutes, stale_days, cost_spike, runs_per_hour, new_host" },
|
|
81
|
+
{ key: "capture", type: "string", values: ["full", "metadata"], help: "what runs record of each tool call: full (its input and output, secrets masked) or metadata (name, size and time only)" },
|
|
82
|
+
{ key: "network_enforce", type: "boolean", help: "true: a site off a step's network: list is refused, not only named in the trace. The account's or the workspace's true is enough" },
|
|
83
|
+
{ key: "telemetry", type: "object", validated: true, help: "send each run as an OpenTelemetry trace — otlp: <https …/v1/traces>, headers: { authorization: ${SECRET} }" },
|
|
84
|
+
{ key: "deploy_block", type: "string", values: ["none", "secrets", "security"], help: "refuse a deploy on scanner findings: secrets (a key written in) or security (any security warning)" },
|
|
78
85
|
{ key: "schedule", type: ["string", "object"], validated: true, legacy: true, help: "refused: only a flow has a clock" },
|
|
79
86
|
];
|
|
80
87
|
const FLOW = [
|
|
81
88
|
{ key: "name", type: "string", help: "the flow's name" },
|
|
82
89
|
{ key: "description", type: "string", help: "what it is for" },
|
|
90
|
+
{ key: "owner", type: "string", help: "the person accountable for it — an email or a name; else the workspace's, else the account's" },
|
|
83
91
|
{ key: "trigger", type: "string", values: FLOW_TRIGGERS, help: "what starts it; any other value never fires" },
|
|
84
92
|
{ key: "schedule", type: "string", help: "cron, when trigger: schedule" },
|
|
85
93
|
{ key: "timezone", type: "string", validated: true, help: "the zone its schedule reads in" },
|
|
86
94
|
{ key: "model", type: "string", help: "every step's model unless the step says" },
|
|
87
95
|
{ key: "effort", type: "string", values: EFFORT_LEVELS, help: "every step's effort unless the step says" },
|
|
88
96
|
{ key: "overlap", type: "string", validated: true, help: "skip | queue" },
|
|
97
|
+
{ key: "issues", type: "string", values: ["stay-open", "auto-close"], help: "what a good run does to this flow's open issues: stay-open (the default — a person resolves them) or auto-close" },
|
|
89
98
|
{ key: "priority", type: "string", validated: true, help: "high | normal | low" },
|
|
90
99
|
{ key: "catchup", type: "string", validated: true, help: "last | none" },
|
|
91
100
|
{ key: "on", type: "string", validated: true, help: "trigger: flow — completed | failed | blocked | any" },
|
|
@@ -5,6 +5,7 @@ import { FRONTMATTER } from "./frontmatter-fields.js";
|
|
|
5
5
|
import { resolveEffort, parseDuration, parseInstant, parseRate, STEP_OPTION_KEYS } from "./store.js";
|
|
6
6
|
import { budgetProblem } from "./budget.js";
|
|
7
7
|
import { cronProblem } from "./cron.js";
|
|
8
|
+
import { modelSecretCollision } from "./model-credential.js";
|
|
8
9
|
export { FRONTMATTER, schemaKindOf } from "./frontmatter-fields.js";
|
|
9
10
|
/** A value for a message: a scalar as written, anything else by its shape.
|
|
10
11
|
* Never String() of a map — its `toString` is whatever the YAML says. */
|
|
@@ -29,6 +30,7 @@ const CHECKS = {
|
|
|
29
30
|
apis: apisProblem,
|
|
30
31
|
budget: (v) => budgetProblem(v, ["run"]),
|
|
31
32
|
ask: (v) => askProblem(v),
|
|
33
|
+
secrets: modelSecretCollision,
|
|
32
34
|
},
|
|
33
35
|
"agents-md": {
|
|
34
36
|
budget: (v) => budgetProblem(v, ["day", "week", "month"]),
|
|
@@ -52,6 +54,7 @@ const CHECKS = {
|
|
|
52
54
|
transport: (v) => (["http", "script", "mcp"].includes(String(v).toLowerCase()) ? null : `"${String(v)}" is not a transport — http, script or mcp`),
|
|
53
55
|
methods: methodsProblem,
|
|
54
56
|
rate: (v) => (parseRate(v) ? null : `"${shown(v)}" is not a rate — N/s, N/minute, N/hour, or a number per second, above 0`),
|
|
57
|
+
secrets: modelSecretCollision,
|
|
55
58
|
},
|
|
56
59
|
"eval": {
|
|
57
60
|
effort: (v) => (resolveEffort(v) ? null : `"${String(v)}" is not an effort — low, medium, high, xhigh or max; the judge runs at none`),
|
package/dist/src/history.d.ts
CHANGED
|
@@ -86,6 +86,10 @@ export declare function listRevisions(tenant: string, scope: string, opts?: {
|
|
|
86
86
|
limit?: number;
|
|
87
87
|
}): RevisionSummary[];
|
|
88
88
|
export declare function readRevision(tenant: string, scope: string, id: string, forPath?: string): Revision | null;
|
|
89
|
+
/** The files changed after revision `from` up to and including `to`, in a
|
|
90
|
+
* scope — what differs between the versions two runs ran (FOL-72). Null
|
|
91
|
+
* when either is unknown or `to` is not after `from`. */
|
|
92
|
+
export declare function changedPathsBetween(tenant: string, scope: string, from: string | null, to: string | null): string[] | null;
|
|
89
93
|
/** The most recent revision id for a scope — what a change is attributed
|
|
90
94
|
* to when nothing more specific (a deployed commit) is on record. */
|
|
91
95
|
export declare function latestRevisionId(tenant: string, scope: string): string | null;
|
package/dist/src/history.js
CHANGED
|
@@ -344,6 +344,21 @@ export function readRevision(tenant, scope, id, forPath) {
|
|
|
344
344
|
const rev = JSON.parse(fs.readFileSync(file, "utf8"));
|
|
345
345
|
return forPath ? { ...rev, files: rev.files.filter((f) => f.path === forPath) } : rev;
|
|
346
346
|
}
|
|
347
|
+
/** The files changed after revision `from` up to and including `to`, in a
|
|
348
|
+
* scope — what differs between the versions two runs ran (FOL-72). Null
|
|
349
|
+
* when either is unknown or `to` is not after `from`. */
|
|
350
|
+
export function changedPathsBetween(tenant, scope, from, to) {
|
|
351
|
+
if (!from || !to)
|
|
352
|
+
return null;
|
|
353
|
+
if (from === to)
|
|
354
|
+
return [];
|
|
355
|
+
const revs = listRevisions(tenant, scope, { limit: 500 }); // newest first
|
|
356
|
+
const ti = revs.findIndex((r) => r.id === to || r.commit === to);
|
|
357
|
+
const fi = revs.findIndex((r) => r.id === from || r.commit === from);
|
|
358
|
+
if (ti < 0 || fi < 0 || ti > fi)
|
|
359
|
+
return null;
|
|
360
|
+
return [...new Set(revs.slice(ti, fi).flatMap((r) => r.paths))].sort();
|
|
361
|
+
}
|
|
347
362
|
/** The most recent revision id for a scope — what a change is attributed
|
|
348
363
|
* to when nothing more specific (a deployed commit) is on record. */
|
|
349
364
|
export function latestRevisionId(tenant, scope) {
|
package/dist/src/hooks.js
CHANGED
|
@@ -30,7 +30,7 @@ import fs from "node:fs";
|
|
|
30
30
|
import path from "node:path";
|
|
31
31
|
import os from "node:os";
|
|
32
32
|
import { spawn } from "node:child_process";
|
|
33
|
-
import matter from "
|
|
33
|
+
import matter from "./matter.js";
|
|
34
34
|
export const HOOK_EVENTS = ["tool.call", "tool.result", "step.start", "step.end"];
|
|
35
35
|
/** Events that decide something before it happens: a failure refuses. */
|
|
36
36
|
const GATES = new Set(["tool.call", "step.start"]);
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { RunRecord } from "./store.ts";
|
|
2
|
+
export type InsightKind = "cache" | "big-result" | "retry-loop" | "timeout";
|
|
3
|
+
export interface Insight {
|
|
4
|
+
id: string;
|
|
5
|
+
kind: InsightKind;
|
|
6
|
+
workspace: string;
|
|
7
|
+
agent: string;
|
|
8
|
+
tool?: string;
|
|
9
|
+
title: string;
|
|
10
|
+
/** Why it happens, in a sentence. */
|
|
11
|
+
why: string;
|
|
12
|
+
fix: string;
|
|
13
|
+
/** About how many credits it cost in the window; null when not money. */
|
|
14
|
+
credits: number | null;
|
|
15
|
+
/** How many steps showed it. */
|
|
16
|
+
steps: number;
|
|
17
|
+
/** The newest runs it was seen in, at most 5. */
|
|
18
|
+
evidence: {
|
|
19
|
+
workspace: string;
|
|
20
|
+
runId: string;
|
|
21
|
+
step: number;
|
|
22
|
+
}[];
|
|
23
|
+
}
|
|
24
|
+
type Run = RunRecord & {
|
|
25
|
+
workspace: string;
|
|
26
|
+
};
|
|
27
|
+
export declare function findInsights(runs: Run[], opts?: {
|
|
28
|
+
minSteps?: number;
|
|
29
|
+
}): Insight[];
|
|
30
|
+
export {};
|