@foldrun/core 2.3.0 → 2.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. package/dist/src/agent-activity.d.ts +5 -0
  2. package/dist/src/agent-activity.js +31 -0
  3. package/dist/src/agent-import.js +1 -1
  4. package/dist/src/agent-tools.js +1 -1
  5. package/dist/src/agents-md.js +1 -1
  6. package/dist/src/alerts.d.ts +60 -0
  7. package/dist/src/alerts.js +161 -0
  8. package/dist/src/browser-pod.js +4 -0
  9. package/dist/src/capture.d.ts +23 -0
  10. package/dist/src/capture.js +80 -0
  11. package/dist/src/compare.d.ts +57 -0
  12. package/dist/src/compare.js +116 -0
  13. package/dist/src/completions.js +1 -0
  14. package/dist/src/deploy.d.ts +36 -1
  15. package/dist/src/deploy.js +170 -11
  16. package/dist/src/egress.d.ts +16 -1
  17. package/dist/src/evals.d.ts +5 -1
  18. package/dist/src/evals.js +26 -18
  19. package/dist/src/frontmatter-fields.js +9 -0
  20. package/dist/src/frontmatter-schema.js +3 -0
  21. package/dist/src/history.d.ts +4 -0
  22. package/dist/src/history.js +15 -0
  23. package/dist/src/hooks.js +1 -1
  24. package/dist/src/insights.d.ts +30 -0
  25. package/dist/src/insights.js +152 -0
  26. package/dist/src/issues.d.ts +140 -0
  27. package/dist/src/issues.js +302 -0
  28. package/dist/src/library.js +1 -1
  29. package/dist/src/limits.d.ts +8 -0
  30. package/dist/src/limits.js +36 -5
  31. package/dist/src/linkable.js +1 -1
  32. package/dist/src/matter.d.ts +8 -0
  33. package/dist/src/matter.js +28 -0
  34. package/dist/src/model-credential.d.ts +14 -0
  35. package/dist/src/model-credential.js +40 -0
  36. package/dist/src/model-env.d.ts +14 -2
  37. package/dist/src/model-env.js +34 -9
  38. package/dist/src/network.d.ts +12 -0
  39. package/dist/src/network.js +25 -1
  40. package/dist/src/notify.d.ts +16 -0
  41. package/dist/src/notify.js +17 -7
  42. package/dist/src/okf.js +1 -1
  43. package/dist/src/online-checks.d.ts +76 -0
  44. package/dist/src/online-checks.js +432 -0
  45. package/dist/src/operator.d.ts +277 -0
  46. package/dist/src/operator.js +176 -2
  47. package/dist/src/otel-config.d.ts +18 -0
  48. package/dist/src/otel-config.js +69 -0
  49. package/dist/src/otel.d.ts +38 -0
  50. package/dist/src/otel.js +85 -0
  51. package/dist/src/outcome.d.ts +40 -0
  52. package/dist/src/outcome.js +77 -0
  53. package/dist/src/package.js +1 -1
  54. package/dist/src/platform.d.ts +1 -1
  55. package/dist/src/prompt-layers.d.ts +30 -0
  56. package/dist/src/prompt-layers.js +77 -0
  57. package/dist/src/provider-health.js +1 -1
  58. package/dist/src/providers.d.ts +5 -0
  59. package/dist/src/providers.js +2 -2
  60. package/dist/src/run-container.d.ts +16 -2
  61. package/dist/src/run-container.js +57 -5
  62. package/dist/src/runner.d.ts +53 -2
  63. package/dist/src/runner.js +356 -70
  64. package/dist/src/runs-filter.d.ts +48 -0
  65. package/dist/src/runs-filter.js +73 -0
  66. package/dist/src/scan.d.ts +19 -1
  67. package/dist/src/scan.js +60 -7
  68. package/dist/src/starter.js +4 -2
  69. package/dist/src/step-exec.d.ts +29 -1
  70. package/dist/src/step-exec.js +106 -6
  71. package/dist/src/store.d.ts +130 -3
  72. package/dist/src/store.js +145 -6
  73. package/dist/src/tokens.d.ts +38 -0
  74. package/dist/src/tokens.js +57 -0
  75. package/dist/src/tool-names.js +1 -1
  76. package/dist/src/tool-programs.js +1 -1
  77. package/dist/src/trace-tree.d.ts +40 -0
  78. package/dist/src/trace-tree.js +156 -0
  79. package/dist/src/translator.d.ts +8 -0
  80. package/dist/src/translator.js +7 -2
  81. package/dist/src/trigger-log.d.ts +1 -1
  82. package/dist/src/workspace-check.d.ts +18 -0
  83. package/dist/src/workspace-check.js +34 -0
  84. package/package.json +2 -2
  85. package/src/agent-activity.ts +32 -0
  86. package/src/agent-import.ts +1 -1
  87. package/src/agent-tools.ts +1 -1
  88. package/src/agents-md.ts +1 -1
  89. package/src/alerts.ts +190 -0
  90. package/src/browser-pod.ts +3 -0
  91. package/src/capture.ts +88 -0
  92. package/src/compare.ts +146 -0
  93. package/src/completions.ts +1 -0
  94. package/src/deploy.ts +184 -10
  95. package/src/egress.ts +12 -2
  96. package/src/evals.ts +26 -18
  97. package/src/frontmatter-fields.ts +9 -0
  98. package/src/frontmatter-schema.ts +3 -0
  99. package/src/history.ts +13 -0
  100. package/src/hooks.ts +1 -1
  101. package/src/insights.ts +174 -0
  102. package/src/issues.ts +396 -0
  103. package/src/library.ts +1 -1
  104. package/src/limits.ts +38 -5
  105. package/src/linkable.ts +1 -1
  106. package/src/matter.ts +33 -0
  107. package/src/model-credential.ts +45 -0
  108. package/src/model-env.ts +42 -10
  109. package/src/network.ts +25 -1
  110. package/src/notify.ts +22 -7
  111. package/src/okf.ts +1 -1
  112. package/src/online-checks.ts +450 -0
  113. package/src/operator.ts +213 -3
  114. package/src/otel-config.ts +67 -0
  115. package/src/otel.ts +100 -0
  116. package/src/outcome.ts +90 -0
  117. package/src/package.ts +1 -1
  118. package/src/platform.ts +1 -1
  119. package/src/prompt-layers.ts +90 -0
  120. package/src/provider-health.ts +1 -1
  121. package/src/providers.ts +7 -2
  122. package/src/run-container.ts +66 -7
  123. package/src/runner.ts +363 -46
  124. package/src/runs-filter.ts +88 -0
  125. package/src/scan.ts +68 -7
  126. package/src/starter.ts +4 -2
  127. package/src/step-exec.ts +147 -9
  128. package/src/store.ts +254 -8
  129. package/src/tokens.ts +68 -0
  130. package/src/tool-names.ts +1 -1
  131. package/src/tool-programs.ts +1 -1
  132. package/src/trace-tree.ts +187 -0
  133. package/src/translator.ts +13 -2
  134. package/src/trigger-log.ts +1 -0
  135. package/src/workspace-check.ts +41 -0
@@ -20,20 +20,24 @@
20
20
  import fs from "node:fs";
21
21
  import os from "node:os";
22
22
  import path from "node:path";
23
- import { saveWorkspace, keptOnDeploy, workspaceDir, listRunSummaries, listWorkspaceFiles, parseFlow, parseToolDef, WORKSPACE_DIRS, assertSafeName, } from "./store.js";
23
+ import { saveWorkspace, keptOnDeploy, workspaceDir, listRunSummaries, listWorkspaceFiles, parseFlow, parseToolDef, WORKSPACE_DIRS, assertSafeName, accountDir, frontmatterErrorText, } from "./store.js";
24
+ import { readAgentsMdKeys } from "./agents-md.js";
25
+ import { libraryTools } from "./library.js";
24
26
  import { lintFlow } from "./flow-lint.js";
25
27
  import { refNames } from "./refs.js";
26
- import matter from "gray-matter";
28
+ import matter from "./matter.js";
27
29
  import { conformanceIssues } from "./okf.js";
28
30
  import { timezoneProblem } from "./clock.js";
29
31
  import { webProblems, webWarnings, runnerEngines } from "./providers.js";
30
32
  import { readLimits } from "./limits.js";
33
+ import { parseAlerts } from "./alerts.js";
34
+ import { parseTelemetry } from "./otel-config.js";
31
35
  import { networkProblems } from "./network.js";
32
36
  import { actionProblems } from "./web-actions.js";
33
37
  import { languageProblem } from "./language.js";
34
38
  import { localeProblems } from "./locale.js";
35
39
  import { entriesNoFollow } from "./paths.js";
36
- import { filterIgnored, findingFingerprint, ruleForWhere, scanFiles } from "./scan.js";
40
+ import { filterIgnored, findingFingerprint, ruleForWhere, scanFiles, scanRule } from "./scan.js";
37
41
  /** Directories a deploy never reads out of a source tree. */
38
42
  // `.cache` is what a run's tools leave in an agent's folder (HOME is that
39
43
  // folder): logs and font caches, never authored.
@@ -73,9 +77,40 @@ export function isSourcePath(rel) {
73
77
  // head. Shared by the deploy import (readTree) and the CLI checker so the two
74
78
  // agree on what counts as an agent.
75
79
  export const SUBAGENT_IMPORT_DIRS = [".agents/agents", ".claude/agents"];
76
- export function readTree(dir) {
80
+ /** What agents write as they run, not what anyone authored: state/ and
81
+ * memory/, the workspace's or one agent's. */
82
+ const DATA_PATH = /^(?:agents\/[^/]+\/)?(?:state|memory)\//;
83
+ export class TreeTooLarge extends Error {
84
+ /** What an HTTP route answers with it. */
85
+ status = 413;
86
+ }
87
+ export function readTree(dir, opts = {}) {
77
88
  const root = path.resolve(dir);
78
89
  const out = [];
90
+ let dataBytes = 0;
91
+ let dataFiles = 0;
92
+ let sourceBytes = 0;
93
+ /** May this file be read? Data over its budget is left out; source over
94
+ * its budget ends the read. */
95
+ const admit = (rel, bytes) => {
96
+ if (DATA_PATH.test(rel)) {
97
+ const over = (opts.maxDataFileBytes !== undefined && bytes > opts.maxDataFileBytes) ||
98
+ (opts.maxDataBytes !== undefined && dataBytes + bytes > opts.maxDataBytes) ||
99
+ (opts.maxDataFiles !== undefined && dataFiles >= opts.maxDataFiles);
100
+ if (over) {
101
+ opts.onSkip?.(rel, bytes);
102
+ return false;
103
+ }
104
+ dataBytes += bytes;
105
+ dataFiles++;
106
+ return true;
107
+ }
108
+ sourceBytes += bytes;
109
+ if (opts.maxSourceBytes !== undefined && sourceBytes > opts.maxSourceBytes) {
110
+ throw new TreeTooLarge(`the workspace's files other than state/ and memory/ are over ${Math.round(opts.maxSourceBytes / 1048576)} MiB — not read`);
111
+ }
112
+ return true;
113
+ };
79
114
  const walk = (abs, rel) => {
80
115
  let entries;
81
116
  try {
@@ -104,6 +139,8 @@ export function readTree(dir) {
104
139
  walk(full, next);
105
140
  }
106
141
  else if (next === "AGENTS.md" || next === "project.md" || IN_WORKSPACE_DIR.test(next)) {
142
+ if (!admit(next, stat.size))
143
+ continue;
107
144
  out.push({ path: next, content: fs.readFileSync(full, "utf8") });
108
145
  }
109
146
  }
@@ -143,11 +180,16 @@ export function readTree(dir) {
143
180
  continue;
144
181
  const full = path.join(root, importDir, entry);
145
182
  try {
146
- if (fs.lstatSync(full).isSymbolicLink())
183
+ const st = fs.lstatSync(full);
184
+ if (st.isSymbolicLink())
185
+ continue;
186
+ if (!admit(target, st.size))
147
187
  continue;
148
188
  out.push({ path: target, content: fs.readFileSync(full, "utf8") });
149
189
  }
150
- catch {
190
+ catch (err) {
191
+ if (err instanceof TreeTooLarge)
192
+ throw err;
151
193
  // an unreadable import is skipped, not fatal
152
194
  }
153
195
  }
@@ -195,6 +237,89 @@ export function unopenedFrontmatter(raw) {
195
237
  * advice (an agent that could leak, an unpinned package) is for `foldrun
196
238
  * check` and the Checks page, not for every deploy's output.
197
239
  */
240
+ /**
241
+ * What `deploy_block:` in AGENTS.md refuses (FOL-58). `none`, the default:
242
+ * the scanner's findings are warnings and the deploy goes ahead, as it always
243
+ * did. `secrets`: a key or password written into a file is refused.
244
+ * `security`: every security finding is — secrets, hidden or injected text,
245
+ * the lethal trifecta, pipe-to-shell, an unpinned MCP server, a step that
246
+ * acts outside with no check or approval. A
247
+ * `foldrun-ignore` with a reason still accepts one.
248
+ */
249
+ export const DEPLOY_BLOCK_MODES = ["none", "secrets", "security"];
250
+ const blockMode = (v) => typeof v === "string" && DEPLOY_BLOCK_MODES.includes(v.trim()) ? v.trim() : null;
251
+ /** The workspace's own AGENTS.md, else the account's, else none. */
252
+ export function deployBlockMode(files, accountKeys) {
253
+ // project.md is the older name readAgentsMdKeys still reads (Codex review).
254
+ const own = files.find((f) => f.path === "AGENTS.md") ?? files.find((f) => f.path === "project.md");
255
+ let fromWorkspace = null;
256
+ if (own) {
257
+ try {
258
+ fromWorkspace = blockMode(matter(own.content).data.deploy_block);
259
+ }
260
+ catch {
261
+ // an AGENTS.md that does not parse is reported elsewhere
262
+ }
263
+ }
264
+ return fromWorkspace ?? blockMode(accountKeys.deploy_block) ?? "none";
265
+ }
266
+ function blockersFor(tenant, given) {
267
+ // The tree as it will be written: `./AGENTS.md` is saved as AGENTS.md, so
268
+ // it is read as that here too (Codex review).
269
+ // Two entries for one path: the save keeps the last, so this does too.
270
+ const byPath = new Map();
271
+ for (const f of given) {
272
+ // path.normalize, as saveWorkspace writes it (a backslash is a filename
273
+ // character here, not a separator — Codex review).
274
+ const p = path.normalize(f.path);
275
+ byPath.set(p, { ...f, path: p });
276
+ }
277
+ const files = [...byPath.values()];
278
+ const account = readAgentsMdKeys(accountDir(tenant));
279
+ const mode = deployBlockMode(files, account);
280
+ if (mode === "none")
281
+ return [];
282
+ // Nearest wins: a workspace tool of the same name replaces the library's,
283
+ // and the scanner adds the workspace's own outward tools itself. Named as
284
+ // the runtime names them — the definition's name:, else the file's.
285
+ const ownTools = new Set();
286
+ for (const f of files) {
287
+ const m = /^tools\/([^/]+?)(?:\/tool)?\.md$/.exec(f.path);
288
+ if (!m)
289
+ continue;
290
+ let data = {};
291
+ try {
292
+ data = matter(f.content).data;
293
+ }
294
+ catch {
295
+ continue; // unreadable: the runtime skips it and keeps the library's
296
+ }
297
+ // Only a definition the runtime accepts replaces the library's; an
298
+ // invalid one is skipped there, and the library tool stays (Codex review).
299
+ const def = parseToolDef(data, m[1], f.content);
300
+ if (def)
301
+ ownTools.add(def.name ?? m[1]);
302
+ }
303
+ const outwardTools = Object.entries(libraryTools(tenant))
304
+ .filter(([n, d]) => d.outward === true && !ownTools.has(n))
305
+ .map(([n]) => n);
306
+ return deployBlockers(files, mode, { accountNetwork: account.network, outwardTools });
307
+ }
308
+ /** The findings `mode` turns into refusals, as deploy issues. */
309
+ export function deployBlockers(files, mode, options = {}) {
310
+ if (mode === "none")
311
+ return [];
312
+ // Every finding checkFiles reports — the scanner's and the structural
313
+ // ones, flow/outward-ungated among them (Codex review) — with the same
314
+ // ignores honoured, kept by its rule's category.
315
+ const categories = new Set(mode === "secrets" ? ["secret"] : ["secret", "content", "security", "supply-chain"]);
316
+ return checkFiles(files, options)
317
+ .filter((f) => f.level !== "info" && categories.has(scanRule(f.rule)?.category ?? ""))
318
+ .map((f) => ({
319
+ where: `${f.file}${f.line ? `:${f.line}` : ""}`,
320
+ message: `${f.message} [${f.rule}] — refused: deploy_block: ${mode} in AGENTS.md`,
321
+ }));
322
+ }
198
323
  export function deployWarnings(files) {
199
324
  const scanned = scanFiles(files, { only: ["secret", "content"] })
200
325
  .filter((f) => f.level !== "info")
@@ -217,11 +342,11 @@ export function checkFiles(files, options = {}) {
217
342
  };
218
343
  const own = [
219
344
  ...deployIssues(files).map((i) => finding(i, "error")),
220
- ...structuralWarnings(files).map((i) => finding(i, "warn", i.kind)),
345
+ ...structuralWarnings(files, options.outwardTools).map((i) => finding(i, "warn", i.kind)),
221
346
  ];
222
347
  return [...filterIgnored(own, files), ...scanFiles(files, options)].sort((a, b) => a.file.localeCompare(b.file) || (a.line ?? 0) - (b.line ?? 0) || a.rule.localeCompare(b.rule));
223
348
  }
224
- function structuralWarnings(files) {
349
+ function structuralWarnings(files, extraOutward = []) {
225
350
  const front = (content) => {
226
351
  try {
227
352
  return matter(content).data;
@@ -230,8 +355,9 @@ function structuralWarnings(files) {
230
355
  return {};
231
356
  }
232
357
  };
233
- // Workspace tools that change something outside the workspace.
234
- const outwardTools = new Set();
358
+ // Tools that change something outside the workspace: the workspace's own,
359
+ // and the library's the caller resolved (nearest wins already applied).
360
+ const outwardTools = new Set(extraOutward);
235
361
  for (const f of files) {
236
362
  const m = f.path.match(/^tools\/([^/]+?)(?:\/tool)?\.md$/);
237
363
  if (!m)
@@ -242,6 +368,9 @@ function structuralWarnings(files) {
242
368
  // and the CLI use; parseToolDef does not fold it onto the def.
243
369
  if (def && data.outward === true)
244
370
  outwardTools.add(def.name ?? m[1]);
371
+ // A workspace tool that is not outward replaces a library one that was.
372
+ else if (def)
373
+ outwardTools.delete(def.name ?? m[1]);
245
374
  }
246
375
  // Agents that grant one of them.
247
376
  const outwardAgents = [];
@@ -311,6 +440,26 @@ export function deployIssues(files) {
311
440
  if (m)
312
441
  agents.add(m[1]);
313
442
  }
443
+ // Frontmatter whose YAML does not parse is a file the runtime cannot read:
444
+ // an agent with no tools and the default model, a flow that never fires,
445
+ // a tool that is skipped. Refused where it was written. Until 20 Oct 2026
446
+ // nothing said so — gray-matter's cache let a second parse pass — and one
447
+ // such agent file kept the worker from booting for every account.
448
+ // Definition files only: a skill is read leniently (an unquoted colon is
449
+ // quoted and retried, runner.ts safeMatter), and the markdown beside a
450
+ // definition — references, a tool's README — is prose, not frontmatter.
451
+ for (const f of files) {
452
+ if (!/^(AGENTS\.md|agents\/[^/]+\/agent\.md|flows\/[^/]+\.md|tools\/[^/]+\.md|tools\/[^/]+\/tool\.md|evals\/[^/]+\.md|hooks\/[^/]+\/hook\.md|\.claude\/agents\/[^/]+\.md)$/.test(f.path))
453
+ continue;
454
+ if (!/^\uFEFF?---\r?\n/.test(f.content))
455
+ continue;
456
+ try {
457
+ matter(f.content);
458
+ }
459
+ catch (err) {
460
+ at(f.path, frontmatterErrorText(err));
461
+ }
462
+ }
314
463
  // `timezone:` is read at four levels and a value nothing can parse falls
315
464
  // silently through to the level above — so it is refused where it was
316
465
  // written, not discovered as a date that is a day out.
@@ -346,6 +495,13 @@ export function deployIssues(files) {
346
495
  // above it — wider or narrower than the author wrote.
347
496
  for (const w of networkProblems(front.network))
348
497
  at(f.path, w);
498
+ // `alerts:` that cannot be read would never alert (FOL-72).
499
+ for (const w of parseAlerts(front.alerts).problems)
500
+ at(f.path, w);
501
+ // `telemetry:` that cannot be read would never send — and a token
502
+ // written into it would ship in the file.
503
+ for (const w of parseTelemetry(front.telemetry).problems)
504
+ at(f.path, w);
349
505
  const lang = languageProblem(front.language);
350
506
  if (lang)
351
507
  at(f.path, lang);
@@ -489,7 +645,10 @@ export function planDeploy(tenant, workspace, files, opts = {}) {
489
645
  added: added.sort(),
490
646
  updated: updated.sort(),
491
647
  removed: removed.sort(),
492
- issues: deployIssues(files),
648
+ // deploy_block: (FOL-58) turns the scanner's findings into refusals —
649
+ // scanned with what the Checks page passes too: the account's network:
650
+ // and the library's outward tools (Codex review).
651
+ issues: [...deployIssues(files), ...blockersFor(tenant, files)],
493
652
  warnings: deployWarnings(files),
494
653
  blockedBy: runsInFlight(tenant, workspace),
495
654
  };
@@ -1,3 +1,4 @@
1
+ import type { PeerInfo } from "./operator.ts";
1
2
  import type { TestEffect } from "./test-mode.ts";
2
3
  import type { NetworkLevel } from "./network.ts";
3
4
  /** `${NAME}` — the shape a secret reference takes everywhere in foldrun. */
@@ -92,14 +93,28 @@ export interface EgressHooks {
92
93
  * policy in test-mode.ts to every request on this lease. Carried on the
93
94
  * lease, not the request, because the pod is the party not trusted to
94
95
  * say so. */
95
- /** `step` — the step's index in run.steps; keys that step's operator inbox. */
96
+ /** `step` — the step's index in run.steps; keys that step's operator inbox.
97
+ * `agent` and `peers` — this step's agent and the other steps of its
98
+ * parallel group (tools: [peers]): the proxy serves the roster and
99
+ * addresses a colleague's message to the right inbox. */
96
100
  lease(args: {
97
101
  tenant: string;
98
102
  runId: string;
99
103
  grant: EgressGrant;
100
104
  test?: boolean;
101
105
  step?: number;
106
+ agent?: string;
107
+ peers?: PeerInfo[];
102
108
  }): Promise<EgressLease | null>;
109
+ /** The step is over — every attempt, however it ended (tools: [peers]).
110
+ * A lease comes and goes per attempt, so it cannot say this: a sibling
111
+ * between retries, or waiting for a slot, holds none and is not done.
112
+ * Optional: a platform without it keeps every sibling reachable. */
113
+ stepEnded?(args: {
114
+ tenant: string;
115
+ runId: string;
116
+ step: number;
117
+ }): Promise<void>;
103
118
  }
104
119
  /** Strip a header set of the proxy's own and hop-by-hop headers. */
105
120
  export declare const HOP_BY_HOP: Set<string>;
@@ -62,10 +62,14 @@ export declare function listEvals(tenant: string, workspace: string): EvalInfo[]
62
62
  * regressions file when there is one, so a flow collects its cases in one
63
63
  * place instead of one file per press.
64
64
  */
65
- export declare function promoteRunToEval(tenant: string, workspace: string, runId: string, opts?: {
65
+ export declare function promoteRunToEval(tenant: string, workspace: string, runId: string,
66
+ /** `should`: what a run that went wrong should have done, in a sentence —
67
+ * the judge's claim for the case (FOL-72 step 10). */
68
+ opts?: {
66
69
  evalName?: string;
67
70
  caseName?: string;
68
71
  expect?: string[];
72
+ should?: string;
69
73
  by?: string;
70
74
  }): {
71
75
  file: string;
package/dist/src/evals.js CHANGED
@@ -30,16 +30,16 @@
30
30
  // Deterministic checks run first and cost nothing. The judge only runs if
31
31
  // they pass — no point paying a model to grade output already known to be
32
32
  // wrong.
33
- import { accountModelEnv } from "./model-env.js";
33
+ import { accountModelEnv, withModelEnv } from "./model-env.js";
34
34
  import { sdkEnv } from "./step-exec.js";
35
35
  import fs from "node:fs";
36
36
  import path from "node:path";
37
37
  import { deployedCommit } from "./deploy.js";
38
38
  import { latestRevisionId } from "./history.js";
39
- import matter from "gray-matter";
39
+ import matter from "./matter.js";
40
40
  import { spawn } from "node:child_process";
41
41
  import { query } from "@anthropic-ai/claude-agent-sdk";
42
- import { workspaceDir, readRun, writeWorkspaceFile, resolveModel, resolveEffort, assertSafeName, } from "./store.js";
42
+ import { workspaceDir, readRun, runVerdict, writeWorkspaceFile, resolveModel, resolveEffort, assertSafeName, } from "./store.js";
43
43
  import { startFlowRun, loadFlow } from "./runner.js";
44
44
  import { platform, platformQueues } from "./platform.js";
45
45
  const ASSERTION_TYPES = ["contains", "not-contains", "matches", "file", "run", "judge"];
@@ -198,16 +198,27 @@ function layoutTask(task) {
198
198
  * regressions file when there is one, so a flow collects its cases in one
199
199
  * place instead of one file per press.
200
200
  */
201
- export function promoteRunToEval(tenant, workspace, runId, opts = {}) {
201
+ export function promoteRunToEval(tenant, workspace, runId,
202
+ /** `should`: what a run that went wrong should have done, in a sentence —
203
+ * the judge's claim for the case (FOL-72 step 10). */
204
+ opts = {}) {
202
205
  const run = readRun(tenant, workspace, runId);
203
206
  if (!run)
204
207
  throw refuse(`run ${runId} not found`, 404);
205
- if (run.status !== "completed") {
206
- throw refuse(`only a completed run can become an eval — this one is ${run.status}`, 409);
208
+ if (run.status !== "completed" && run.status !== "failed") {
209
+ throw refuse(`only a finished run can become an eval — this one is ${run.status}`, 409);
210
+ }
211
+ // A run that went wrong is the test worth having most — but its own answer
212
+ // is the wrong one to hold the next run to, so it needs to be told what
213
+ // was right (FOL-72 step 10).
214
+ const should = oneLine(opts.should ?? "");
215
+ const wentWrong = run.status === "failed" || ["BAD", "BLOCKED"].includes(runVerdict(run) ?? "");
216
+ if (wentWrong && !should && !(opts.expect && opts.expect.length)) {
217
+ throw refuse("this run went wrong, so its answer is not the one to keep — say what it should have done (`should`, a sentence for the judge) or give `expect` lines", 409);
207
218
  }
208
219
  const produced = run.steps.filter((s) => (s.result ?? "").trim());
209
220
  const last = produced[produced.length - 1];
210
- if (!last)
221
+ if (!last && !should && !(opts.expect && opts.expect.length))
211
222
  throw refuse("this run produced no result to hold the next one to", 409);
212
223
  if (run.steps.length === 0 || !run.steps[0].instruction?.trim()) {
213
224
  throw refuse("this run has no task on record", 409);
@@ -239,6 +250,9 @@ export function promoteRunToEval(tenant, workspace, runId, opts = {}) {
239
250
  return `${m[1]}: ${m[2]}`;
240
251
  });
241
252
  }
253
+ else if (should) {
254
+ expect = [`judge: ${should.slice(0, 400)}`];
255
+ }
242
256
  else {
243
257
  const conclusion = oneLine(run.summary?.trim() || (last.result ?? "")).slice(0, 200);
244
258
  expect = [`judge: reaches the same conclusion as the recorded run: ${conclusion}`];
@@ -382,7 +396,7 @@ async function judge(rubric, output, model, effort, env) {
382
396
  }
383
397
  return text.trim();
384
398
  }
385
- async function checkAssertion(assertion, output, agentDir, model, effort, judgeEnv = () => sdkEnv(undefined)) {
399
+ async function checkAssertion(assertion, output, agentDir, model, effort, judgeEnv = () => ({ env: sdkEnv(undefined) })) {
386
400
  const hay = output.toLowerCase();
387
401
  switch (assertion.type) {
388
402
  case "contains": {
@@ -417,14 +431,14 @@ async function checkAssertion(assertion, output, agentDir, model, effort, judgeE
417
431
  case "judge": {
418
432
  // The judge runs on the account's own model key, never quietly on the
419
433
  // platform's: a customer's eval is not the operator's bill.
420
- let env;
434
+ let m;
421
435
  try {
422
- env = judgeEnv();
436
+ m = judgeEnv();
423
437
  }
424
438
  catch (err) {
425
439
  return { assertion, passed: false, detail: `judge not run — ${err instanceof Error ? err.message : String(err)}` };
426
440
  }
427
- const verdict = await judge(assertion.value, output, model, effort, env);
441
+ const verdict = await withModelEnv(m, (env) => judge(assertion.value, output, model, effort, env));
428
442
  const passed = /^\s*PASS\b/i.test(verdict);
429
443
  return { assertion, passed, detail: verdict.slice(0, 300) || "judge returned nothing" };
430
444
  }
@@ -495,13 +509,7 @@ export async function runEval(tenant, workspace, info) {
495
509
  for (const a of cheap)
496
510
  assertions.push(await checkAssertion(a, output, agentDir, info.model, info.effort));
497
511
  if (assertions.every((r) => r.passed)) {
498
- const judgeEnv = () => {
499
- const m = accountModelEnv(tenant, workspace, () => sdkEnv(undefined));
500
- if (m.format !== "anthropic") {
501
- throw new Error(`this account's provider speaks ${m.format}; a judge needs an Anthropic-format provider`);
502
- }
503
- return m.env;
504
- };
512
+ const judgeEnv = () => accountModelEnv(tenant, workspace, () => sdkEnv(undefined));
505
513
  for (const a of judges)
506
514
  assertions.push(await checkAssertion(a, output, agentDir, info.model, info.effort, judgeEnv));
507
515
  }
@@ -24,6 +24,7 @@ const LOCALE = { validated: true };
24
24
  const AGENT = [
25
25
  { key: "name", type: "string", help: "kebab-case; the folder is the identity" },
26
26
  { key: "description", type: "string", help: "what it does and when — other agents and people read it" },
27
+ { key: "owner", type: "string", help: "the person accountable for it — an email or a name; else the workspace's, else the account's" },
27
28
  { key: "model", type: "string", help: "fast | default | max, an alias, or a full model id" },
28
29
  { key: "effort", type: "string", values: EFFORT_LEVELS, help: "how long it thinks" },
29
30
  { key: "size", type: "string", values: ["small", "large", "heavy"], help: "the sandbox it rents; anything else becomes large" },
@@ -59,6 +60,7 @@ const AGENTS_MD = [
59
60
  { key: "mdagent_version", type: ["string", "number"], legacy: true, help: "older spelling of foldrun_version" },
60
61
  { key: "name", type: "string", help: "the workspace's name" },
61
62
  { key: "description", type: "string", help: "shown in the workspace list" },
63
+ { key: "owner", type: "string", help: "the person accountable for every agent and flow here that names none" },
62
64
  { key: "provider", type: "object", help: "the model endpoint every agent here uses unless it names its own" },
63
65
  { key: "runtime", type: "object", help: "packages for agents that declare none" },
64
66
  { key: "web", type: "object", validated: true, help: "web.browse defaults" },
@@ -75,17 +77,24 @@ const AGENTS_MD = [
75
77
  { key: "budget", type: ["number", "string"], help: "USD per month, or N/day | N/week | N/month" },
76
78
  { key: "priority", type: "string", values: ["high", "normal", "low"], help: "account: where its runs stand in the queue" },
77
79
  { key: "concurrency", type: ["number", "string"], help: "account: runs at once" },
80
+ { key: "alerts", type: "object", validated: true, help: "thresholds that tell notify: — failure_rate (30%), min_runs, credits_day, run_minutes, stale_days, cost_spike, runs_per_hour, new_host" },
81
+ { key: "capture", type: "string", values: ["full", "metadata"], help: "what runs record of each tool call: full (its input and output, secrets masked) or metadata (name, size and time only)" },
82
+ { key: "network_enforce", type: "boolean", help: "true: a site off a step's network: list is refused, not only named in the trace. The account's or the workspace's true is enough" },
83
+ { key: "telemetry", type: "object", validated: true, help: "send each run as an OpenTelemetry trace — otlp: <https …/v1/traces>, headers: { authorization: ${SECRET} }" },
84
+ { key: "deploy_block", type: "string", values: ["none", "secrets", "security"], help: "refuse a deploy on scanner findings: secrets (a key written in) or security (any security warning)" },
78
85
  { key: "schedule", type: ["string", "object"], validated: true, legacy: true, help: "refused: only a flow has a clock" },
79
86
  ];
80
87
  const FLOW = [
81
88
  { key: "name", type: "string", help: "the flow's name" },
82
89
  { key: "description", type: "string", help: "what it is for" },
90
+ { key: "owner", type: "string", help: "the person accountable for it — an email or a name; else the workspace's, else the account's" },
83
91
  { key: "trigger", type: "string", values: FLOW_TRIGGERS, help: "what starts it; any other value never fires" },
84
92
  { key: "schedule", type: "string", help: "cron, when trigger: schedule" },
85
93
  { key: "timezone", type: "string", validated: true, help: "the zone its schedule reads in" },
86
94
  { key: "model", type: "string", help: "every step's model unless the step says" },
87
95
  { key: "effort", type: "string", values: EFFORT_LEVELS, help: "every step's effort unless the step says" },
88
96
  { key: "overlap", type: "string", validated: true, help: "skip | queue" },
97
+ { key: "issues", type: "string", values: ["stay-open", "auto-close"], help: "what a good run does to this flow's open issues: stay-open (the default — a person resolves them) or auto-close" },
89
98
  { key: "priority", type: "string", validated: true, help: "high | normal | low" },
90
99
  { key: "catchup", type: "string", validated: true, help: "last | none" },
91
100
  { key: "on", type: "string", validated: true, help: "trigger: flow — completed | failed | blocked | any" },
@@ -5,6 +5,7 @@ import { FRONTMATTER } from "./frontmatter-fields.js";
5
5
  import { resolveEffort, parseDuration, parseInstant, parseRate, STEP_OPTION_KEYS } from "./store.js";
6
6
  import { budgetProblem } from "./budget.js";
7
7
  import { cronProblem } from "./cron.js";
8
+ import { modelSecretCollision } from "./model-credential.js";
8
9
  export { FRONTMATTER, schemaKindOf } from "./frontmatter-fields.js";
9
10
  /** A value for a message: a scalar as written, anything else by its shape.
10
11
  * Never String() of a map — its `toString` is whatever the YAML says. */
@@ -29,6 +30,7 @@ const CHECKS = {
29
30
  apis: apisProblem,
30
31
  budget: (v) => budgetProblem(v, ["run"]),
31
32
  ask: (v) => askProblem(v),
33
+ secrets: modelSecretCollision,
32
34
  },
33
35
  "agents-md": {
34
36
  budget: (v) => budgetProblem(v, ["day", "week", "month"]),
@@ -52,6 +54,7 @@ const CHECKS = {
52
54
  transport: (v) => (["http", "script", "mcp"].includes(String(v).toLowerCase()) ? null : `"${String(v)}" is not a transport — http, script or mcp`),
53
55
  methods: methodsProblem,
54
56
  rate: (v) => (parseRate(v) ? null : `"${shown(v)}" is not a rate — N/s, N/minute, N/hour, or a number per second, above 0`),
57
+ secrets: modelSecretCollision,
55
58
  },
56
59
  "eval": {
57
60
  effort: (v) => (resolveEffort(v) ? null : `"${String(v)}" is not an effort — low, medium, high, xhigh or max; the judge runs at none`),
@@ -86,6 +86,10 @@ export declare function listRevisions(tenant: string, scope: string, opts?: {
86
86
  limit?: number;
87
87
  }): RevisionSummary[];
88
88
  export declare function readRevision(tenant: string, scope: string, id: string, forPath?: string): Revision | null;
89
+ /** The files changed after revision `from` up to and including `to`, in a
90
+ * scope — what differs between the versions two runs ran (FOL-72). Null
91
+ * when either is unknown or `to` is not after `from`. */
92
+ export declare function changedPathsBetween(tenant: string, scope: string, from: string | null, to: string | null): string[] | null;
89
93
  /** The most recent revision id for a scope — what a change is attributed
90
94
  * to when nothing more specific (a deployed commit) is on record. */
91
95
  export declare function latestRevisionId(tenant: string, scope: string): string | null;
@@ -344,6 +344,21 @@ export function readRevision(tenant, scope, id, forPath) {
344
344
  const rev = JSON.parse(fs.readFileSync(file, "utf8"));
345
345
  return forPath ? { ...rev, files: rev.files.filter((f) => f.path === forPath) } : rev;
346
346
  }
347
+ /** The files changed after revision `from` up to and including `to`, in a
348
+ * scope — what differs between the versions two runs ran (FOL-72). Null
349
+ * when either is unknown or `to` is not after `from`. */
350
+ export function changedPathsBetween(tenant, scope, from, to) {
351
+ if (!from || !to)
352
+ return null;
353
+ if (from === to)
354
+ return [];
355
+ const revs = listRevisions(tenant, scope, { limit: 500 }); // newest first
356
+ const ti = revs.findIndex((r) => r.id === to || r.commit === to);
357
+ const fi = revs.findIndex((r) => r.id === from || r.commit === from);
358
+ if (ti < 0 || fi < 0 || ti > fi)
359
+ return null;
360
+ return [...new Set(revs.slice(ti, fi).flatMap((r) => r.paths))].sort();
361
+ }
347
362
  /** The most recent revision id for a scope — what a change is attributed
348
363
  * to when nothing more specific (a deployed commit) is on record. */
349
364
  export function latestRevisionId(tenant, scope) {
package/dist/src/hooks.js CHANGED
@@ -30,7 +30,7 @@ import fs from "node:fs";
30
30
  import path from "node:path";
31
31
  import os from "node:os";
32
32
  import { spawn } from "node:child_process";
33
- import matter from "gray-matter";
33
+ import matter from "./matter.js";
34
34
  export const HOOK_EVENTS = ["tool.call", "tool.result", "step.start", "step.end"];
35
35
  /** Events that decide something before it happens: a failure refuses. */
36
36
  const GATES = new Set(["tool.call", "step.start"]);
@@ -0,0 +1,30 @@
1
+ import type { RunRecord } from "./store.ts";
2
+ export type InsightKind = "cache" | "big-result" | "retry-loop" | "timeout";
3
+ export interface Insight {
4
+ id: string;
5
+ kind: InsightKind;
6
+ workspace: string;
7
+ agent: string;
8
+ tool?: string;
9
+ title: string;
10
+ /** Why it happens, in a sentence. */
11
+ why: string;
12
+ fix: string;
13
+ /** About how many credits it cost in the window; null when not money. */
14
+ credits: number | null;
15
+ /** How many steps showed it. */
16
+ steps: number;
17
+ /** The newest runs it was seen in, at most 5. */
18
+ evidence: {
19
+ workspace: string;
20
+ runId: string;
21
+ step: number;
22
+ }[];
23
+ }
24
+ type Run = RunRecord & {
25
+ workspace: string;
26
+ };
27
+ export declare function findInsights(runs: Run[], opts?: {
28
+ minSteps?: number;
29
+ }): Insight[];
30
+ export {};