@foldrun/core 2.5.0 → 2.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/dist/src/agent-import.js +2 -2
  2. package/dist/src/agent-tools.js +2 -2
  3. package/dist/src/agents-md.d.ts +5 -1
  4. package/dist/src/agents-md.js +42 -18
  5. package/dist/src/completions.js +25 -3
  6. package/dist/src/deploy.js +72 -7
  7. package/dist/src/evals.d.ts +22 -1
  8. package/dist/src/evals.js +43 -14
  9. package/dist/src/flow-lint.d.ts +14 -0
  10. package/dist/src/flow-lint.js +35 -0
  11. package/dist/src/flow-patterns.js +29 -14
  12. package/dist/src/frontmatter-fields.d.ts +27 -0
  13. package/dist/src/frontmatter-fields.js +51 -3
  14. package/dist/src/frontmatter-guard.d.ts +19 -0
  15. package/dist/src/frontmatter-guard.js +67 -0
  16. package/dist/src/frontmatter-schema.d.ts +2 -2
  17. package/dist/src/frontmatter-schema.js +30 -2
  18. package/dist/src/frontmatter-text.d.ts +31 -0
  19. package/dist/src/frontmatter-text.js +102 -0
  20. package/dist/src/judge-verdict.d.ts +19 -0
  21. package/dist/src/judge-verdict.js +54 -0
  22. package/dist/src/matter.d.ts +34 -0
  23. package/dist/src/matter.js +104 -0
  24. package/dist/src/network.d.ts +0 -1
  25. package/dist/src/network.js +10 -7
  26. package/dist/src/package.js +9 -2
  27. package/dist/src/provider-health.js +2 -2
  28. package/dist/src/run-container.d.ts +9 -0
  29. package/dist/src/run-container.js +4 -1
  30. package/dist/src/runner.js +119 -36
  31. package/dist/src/scan.d.ts +11 -0
  32. package/dist/src/scan.js +86 -8
  33. package/dist/src/starter.js +23 -0
  34. package/dist/src/step-exec.d.ts +10 -2
  35. package/dist/src/step-exec.js +12 -9
  36. package/dist/src/store.d.ts +17 -3
  37. package/dist/src/store.js +14 -10
  38. package/package.json +1 -1
  39. package/src/agent-import.ts +2 -2
  40. package/src/agent-tools.ts +2 -2
  41. package/src/agents-md.ts +39 -14
  42. package/src/completions.ts +26 -3
  43. package/src/deploy.ts +59 -7
  44. package/src/evals.ts +45 -13
  45. package/src/flow-lint.ts +36 -0
  46. package/src/flow-patterns.ts +26 -13
  47. package/src/frontmatter-fields.ts +74 -3
  48. package/src/frontmatter-guard.ts +70 -0
  49. package/src/frontmatter-schema.ts +30 -4
  50. package/src/frontmatter-text.ts +109 -0
  51. package/src/judge-verdict.ts +52 -0
  52. package/src/matter.ts +112 -0
  53. package/src/network.ts +10 -6
  54. package/src/package.ts +8 -2
  55. package/src/provider-health.ts +2 -2
  56. package/src/run-container.ts +11 -1
  57. package/src/runner.ts +118 -37
  58. package/src/scan.ts +86 -8
  59. package/src/starter.ts +23 -0
  60. package/src/step-exec.ts +21 -8
  61. package/src/store.ts +23 -10
@@ -14,7 +14,7 @@
14
14
  // were about somebody else's desk.
15
15
  import fs from "node:fs";
16
16
  import path from "node:path";
17
- import matter from "./matter.js";
17
+ import { configMatter } from "./matter.js";
18
18
  import { assertSafeName, listWorkspaceFiles, readWorkspaceFile, writeWorkspaceFile, workspaceDir, workspaceTools, listWorkspaces, } from "./store.js";
19
19
  import { libraryDir, libraryTools } from "./library.js";
20
20
  import { isRuntimeTool, toolEntryName } from "./tool-names.js";
@@ -64,7 +64,7 @@ export function importAgent(tenant, from, agent, to, opts = {}) {
64
64
  let content = readWorkspaceFile(tenant, from, src);
65
65
  const rel = src.slice(`agents/${agent}/`.length);
66
66
  if (rel === "agent.md") {
67
- front = matter(content).data;
67
+ front = configMatter(content).data;
68
68
  // The folder is the name flows address; a `name:` that disagrees with
69
69
  // it is the first thing check would complain about.
70
70
  if (name !== agent)
@@ -16,7 +16,7 @@
16
16
  import fs from "node:fs";
17
17
  import os from "node:os";
18
18
  import path from "node:path";
19
- import matter from "./matter.js";
19
+ import { configMatter } from "./matter.js";
20
20
  import { z } from "zod";
21
21
  import { createSdkMcpServer, tool } from "@anthropic-ai/claude-agent-sdk";
22
22
  import { resolveModel, resolveEffort } from "./store.js";
@@ -36,7 +36,7 @@ export function gatherConsults(workspaceRoot, names) {
36
36
  missing.push(name);
37
37
  continue;
38
38
  }
39
- const parsed = matter(fs.readFileSync(file, "utf8"));
39
+ const parsed = configMatter(fs.readFileSync(file, "utf8"));
40
40
  consults.push({
41
41
  name,
42
42
  systemPrompt: `${parsed.content.trim()}\n\n` +
@@ -1,8 +1,12 @@
1
+ import { keySpan } from "./frontmatter-text.ts";
2
+ export { keySpan };
1
3
  /** The frontmatter of a directory's AGENTS.md (or project.md), or {}. */
2
4
  export declare function readAgentsMdKeys(dir: string): Record<string, unknown>;
3
5
  /** The text of `content` with each key in `patch` replaced (or removed for
4
6
  * null), everything else byte for byte as it was. A file with no
5
- * frontmatter gets one. */
7
+ * frontmatter gets one. Read back before it is returned (frontmatter-guard
8
+ * .ts): an edit that would not parse, would change another key or would
9
+ * double one into its block throws UnsafeEditError (409). */
6
10
  export declare function patchFrontmatter(content: string, patch: Record<string, unknown | null>): string;
7
11
  /** Apply a frontmatter patch to a directory's AGENTS.md, creating the file
8
12
  * when the directory has none. Returns the new content. */
@@ -8,7 +8,11 @@
8
8
  // else in the file is untouched. Null removes the key.
9
9
  import fs from "node:fs";
10
10
  import path from "node:path";
11
- import matter from "./matter.js";
11
+ import matter, { configMatter } from "./matter.js";
12
+ import { keySpan, dropEmptyBlock } from "./frontmatter-text.js";
13
+ import { guardEdit } from "./frontmatter-guard.js";
14
+ import { JOBS, FOLDABLE } from "./frontmatter-fields.js";
15
+ export { keySpan };
12
16
  const FRONT = /^(---\r?\n)([\s\S]*?)(\r?\n---)(\r?\n|$)/;
13
17
  /** The frontmatter of a directory's AGENTS.md (or project.md), or {}. */
14
18
  export function readAgentsMdKeys(dir) {
@@ -17,7 +21,7 @@ export function readAgentsMdKeys(dir) {
17
21
  if (!fs.existsSync(file))
18
22
  continue;
19
23
  try {
20
- return (matter(fs.readFileSync(file, "utf8")).data ?? {});
24
+ return (configMatter(fs.readFileSync(file, "utf8")).data ?? {});
21
25
  }
22
26
  catch {
23
27
  return {};
@@ -34,8 +38,33 @@ function dumpKey(key, value) {
34
38
  }
35
39
  /** The text of `content` with each key in `patch` replaced (or removed for
36
40
  * null), everything else byte for byte as it was. A file with no
37
- * frontmatter gets one. */
41
+ * frontmatter gets one. Read back before it is returned (frontmatter-guard
42
+ * .ts): an edit that would not parse, would change another key or would
43
+ * double one into its block throws UnsafeEditError (409). */
38
44
  export function patchFrontmatter(content, patch) {
45
+ // The guard reads the result folded (matter.ts configMatter), so a patch
46
+ // that writes a whole job block is checked as the keys it holds: each
47
+ // field the block may hold may change, and each one it sets reads back.
48
+ const changed = new Set();
49
+ const expect = {};
50
+ for (const [key, value] of Object.entries(patch)) {
51
+ changed.add(key);
52
+ const job = JOBS.find((j) => j.name === key)?.name;
53
+ if (!job) {
54
+ expect[key] = value;
55
+ continue;
56
+ }
57
+ for (const k of FOLDABLE[job])
58
+ changed.add(k);
59
+ if (value && typeof value === "object" && !Array.isArray(value)) {
60
+ for (const [k, v] of Object.entries(value))
61
+ if (FOLDABLE[job].has(k))
62
+ expect[k] = v;
63
+ }
64
+ }
65
+ return guardEdit(content, patchFrontmatterText(content, patch), [...changed], expect);
66
+ }
67
+ function patchFrontmatterText(content, patch) {
39
68
  const m = content.match(FRONT);
40
69
  let open = "---\n";
41
70
  let body = "";
@@ -50,22 +79,17 @@ export function patchFrontmatter(content, patch) {
50
79
  }
51
80
  let lines = body === "" ? [] : body.split("\n");
52
81
  for (const [key, value] of Object.entries(patch)) {
53
- const at = lines.findIndex((l) => new RegExp(`^${key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\s*:`).test(l));
54
- let end = at;
55
- if (at !== -1) {
56
- end = at + 1;
57
- // The block continues while lines are indented or blank-inside-block.
58
- while (end < lines.length && (/^[ \t]/.test(lines[end]) || lines[end].trim() === ""))
59
- end++;
60
- // A blank line at the block's foot belongs to what follows.
61
- while (end > at + 1 && lines[end - 1].trim() === "")
62
- end--;
63
- }
64
- const replacement = value === null || value === undefined ? [] : dumpKey(key, value);
65
- if (at === -1)
82
+ // Where it is written now — at the top or inside its job block — is
83
+ // where it is edited; a key written nowhere is added at the top.
84
+ const where = keySpan(lines, key);
85
+ const replacement = value === null || value === undefined ? [] : dumpKey(key, value).map((l) => `${where?.indent ?? ""}${l}`);
86
+ if (!where)
66
87
  lines = [...lines, ...replacement];
67
- else
68
- lines = [...lines.slice(0, at), ...replacement, ...lines.slice(end)];
88
+ else {
89
+ lines = [...lines.slice(0, where.at), ...replacement, ...lines.slice(where.end)];
90
+ if (!replacement.length && where.block !== null)
91
+ lines = dropEmptyBlock(lines, where.block);
92
+ }
69
93
  }
70
94
  const front = lines.length ? `${open}${lines.join("\n")}${close}` : "";
71
95
  if (!front)
@@ -9,7 +9,7 @@
9
9
  // go here. No React, no fetch — so the rules are testable and one component
10
10
  // renders them.
11
11
  import { PROVIDERS } from "./providers.js";
12
- import { FRONTMATTER } from "./frontmatter-fields.js";
12
+ import { FRONTMATTER, JOBS, fieldsOfJob } from "./frontmatter-fields.js";
13
13
  const MODELS = [
14
14
  { label: "fast", hint: "haiku — cheap, for checks and triage" },
15
15
  { label: "default", hint: "sonnet — the working default" },
@@ -587,8 +587,30 @@ export function completionsAt(path, text, cursor, vocab) {
587
587
  };
588
588
  }
589
589
  }
590
- // 6. An indented option under a flow step.
591
- if (kind === "flow" && /^\s+[a-z]*$/.test(line) && line.trim().length >= 0) {
590
+ // 5b. An indented line inside one of the four job blocks (governance,
591
+ // guardrail, observability, evaluation): that job's fields for this
592
+ // kind of file — the block takes nothing else.
593
+ const fmClose = text.indexOf("\n---", text.indexOf("---") + 3);
594
+ const insideFrontmatter = text.startsWith("---") && (fmClose === -1 || cursor <= fmClose);
595
+ const nested = line.match(/^(\s+)([a-zA-Z_]*)$/);
596
+ if (insideFrontmatter && nested) {
597
+ const header = before.slice(0, lineStart).split("\n").reverse().find((l) => /^\S/.test(l));
598
+ const job = JOBS.find((j) => new RegExp(`^${j.name}\\s*:\\s*$`).test(header ?? ""));
599
+ if (job) {
600
+ const schemaKind = /(^|\/)AGENTS\.md$/.test(path) || path === "project.md" ? "agents-md" : SCHEMA_OF[kind];
601
+ const keys = new Set(schemaKind ? fieldsOfJob(schemaKind, job.name) : []);
602
+ const partial = nested[2] ?? "";
603
+ return {
604
+ from: cursor - partial.length,
605
+ query: partial,
606
+ items: filter(fieldsFor(path, kind).filter((c) => keys.has(c.label.split(".")[0])), partial),
607
+ title: `${job.title} — ${job.what}`,
608
+ };
609
+ }
610
+ }
611
+ // 6. An indented option under a flow step — the body's steps, never a
612
+ // line inside the frontmatter.
613
+ if (kind === "flow" && !insideFrontmatter && /^\s+[a-z]*$/.test(line) && line.trim().length >= 0) {
592
614
  const partial = line.trim();
593
615
  return {
594
616
  from: cursor - partial.length,
@@ -23,21 +23,24 @@ import path from "node:path";
23
23
  import { saveWorkspace, keptOnDeploy, workspaceDir, listRunSummaries, listWorkspaceFiles, parseFlow, parseToolDef, WORKSPACE_DIRS, assertSafeName, accountDir, frontmatterErrorText, } from "./store.js";
24
24
  import { readAgentsMdKeys } from "./agents-md.js";
25
25
  import { libraryTools } from "./library.js";
26
- import { lintFlow } from "./flow-lint.js";
26
+ import { lintFlow, regexProblem } from "./flow-lint.js";
27
+ import { parseEval } from "./evals.js";
27
28
  import { refNames } from "./refs.js";
28
- import matter from "./matter.js";
29
+ import matter, { configMatter } from "./matter.js";
30
+ import { frontmatterProblems, schemaKindOf } from "./frontmatter-schema.js";
29
31
  import { conformanceIssues } from "./okf.js";
30
32
  import { timezoneProblem } from "./clock.js";
31
33
  import { webProblems, webWarnings, runnerEngines } from "./providers.js";
32
34
  import { readLimits } from "./limits.js";
33
35
  import { parseAlerts } from "./alerts.js";
34
36
  import { parseTelemetry } from "./otel-config.js";
37
+ import { parseJudge } from "./judge-verdict.js";
35
38
  import { networkProblems } from "./network.js";
36
39
  import { actionProblems } from "./web-actions.js";
37
40
  import { languageProblem } from "./language.js";
38
41
  import { localeProblems } from "./locale.js";
39
42
  import { entriesNoFollow } from "./paths.js";
40
- import { filterIgnored, findingFingerprint, ruleForWhere, scanFiles, scanRule } from "./scan.js";
43
+ import { filterIgnored, findingFingerprint, ruleForWhere, scanFiles, scanRule, STRICT_FRONTMATTER_PATH } from "./scan.js";
41
44
  /** Directories a deploy never reads out of a source tree. */
42
45
  // `.cache` is what a run's tools leave in an agent's folder (HOME is that
43
46
  // folder): logs and font caches, never authored.
@@ -255,7 +258,7 @@ export function deployBlockMode(files, accountKeys) {
255
258
  let fromWorkspace = null;
256
259
  if (own) {
257
260
  try {
258
- fromWorkspace = blockMode(matter(own.content).data.deploy_block);
261
+ fromWorkspace = blockMode(configMatter(own.content).data.deploy_block);
259
262
  }
260
263
  catch {
261
264
  // an AGENTS.md that does not parse is reported elsewhere
@@ -394,6 +397,41 @@ function structuralWarnings(files, extraOutward = []) {
394
397
  for (const w of webWarnings(data, { chrome, chromium }))
395
398
  out.push({ where: f.path, message: w });
396
399
  }
400
+ // An eval's `matches:` is the same JavaScript RegExp as a verify's; one
401
+ // that does not compile fails its case on every run. Read through
402
+ // parseEval, so a block scalar (`matches: |`) or a continued line is
403
+ // tested as the runtime will run it. The line is the item's own: the
404
+ // parsed assertions and the `- matches:` lines come in the same order.
405
+ for (const f of files) {
406
+ if (!/^evals\/[^/]+\.md$/.test(f.path))
407
+ continue;
408
+ let info;
409
+ try {
410
+ info = parseEval(f.path, f.content);
411
+ }
412
+ catch {
413
+ continue;
414
+ }
415
+ const lines = [];
416
+ f.content.split("\n").forEach((line, n) => { if (/^\s*-\s*matches:/.test(line))
417
+ lines.push(n + 1); });
418
+ // When a `- matches:` line sits outside a case (an example in the
419
+ // prose), the order no longer pairs up: name the file, not a wrong line.
420
+ const parsed = info.cases.reduce((n, c) => n + c.expect.filter((a) => a.type === "matches").length, 0);
421
+ const pairs = lines.length === parsed;
422
+ let k = 0;
423
+ for (const c of info.cases) {
424
+ for (const a of c.expect) {
425
+ if (a.type !== "matches")
426
+ continue;
427
+ const at = pairs ? lines[k] : undefined;
428
+ k++;
429
+ const problem = regexProblem(a.value);
430
+ if (problem)
431
+ out.push({ where: at ? `${f.path}:${at}` : f.path, message: `case "${c.name}": matches: ${problem}` });
432
+ }
433
+ }
434
+ }
397
435
  if (!outwardAgents.length)
398
436
  return out;
399
437
  for (const f of files) {
@@ -446,10 +484,11 @@ export function deployIssues(files) {
446
484
  // nothing said so — gray-matter's cache let a second parse pass — and one
447
485
  // such agent file kept the worker from booting for every account.
448
486
  // Definition files only: a skill is read leniently (an unquoted colon is
449
- // quoted and retried, runner.ts safeMatter), and the markdown beside a
487
+ // quoted and retried, matter.ts parseLenient), and the markdown beside a
450
488
  // definition — references, a tool's README — is prose, not frontmatter.
489
+ // The scanner warns about skills and library files (config/invalid-frontmatter).
451
490
  for (const f of files) {
452
- if (!/^(AGENTS\.md|agents\/[^/]+\/agent\.md|flows\/[^/]+\.md|tools\/[^/]+\.md|tools\/[^/]+\/tool\.md|evals\/[^/]+\.md|hooks\/[^/]+\/hook\.md|\.claude\/agents\/[^/]+\.md)$/.test(f.path))
491
+ if (!STRICT_FRONTMATTER_PATH.test(f.path))
453
492
  continue;
454
493
  if (!/^\uFEFF?---\r?\n/.test(f.content))
455
494
  continue;
@@ -460,6 +499,29 @@ export function deployIssues(files) {
460
499
  at(f.path, frontmatterErrorText(err));
461
500
  }
462
501
  }
502
+ // The four job blocks (frontmatter-fields.ts JOB_FIELDS): a setting
503
+ // written both at the top and in its block, or in another job's block, is
504
+ // a setting that does not do what the file says — a guardrail under
505
+ // governance: guards nothing — so the deploy refuses it as check does
506
+ // (config/conflict, config/misplaced; Codex debate, 21 Oct 2026).
507
+ for (const f of files) {
508
+ const kind = schemaKindOf(f.path);
509
+ if (kind !== "agent" && kind !== "flow" && kind !== "agents-md")
510
+ continue;
511
+ if (!/^\uFEFF?---\r?\n/.test(f.content))
512
+ continue;
513
+ let front;
514
+ try {
515
+ front = configMatter(f.content).data;
516
+ }
517
+ catch {
518
+ continue; // unparseable frontmatter is its own report
519
+ }
520
+ for (const p of frontmatterProblems(kind, front, () => null)) {
521
+ if (p.level === "error" && (p.rule === "config/conflict" || p.rule === "config/misplaced"))
522
+ at(f.path, `${p.message} [${p.rule}]`);
523
+ }
524
+ }
463
525
  // `timezone:` is read at four levels and a value nothing can parse falls
464
526
  // silently through to the level above — so it is refused where it was
465
527
  // written, not discovered as a date that is a day out.
@@ -471,7 +533,7 @@ export function deployIssues(files) {
471
533
  at(f.path, unopened);
472
534
  let front = {};
473
535
  try {
474
- front = matter(f.content).data;
536
+ front = configMatter(f.content).data;
475
537
  }
476
538
  catch {
477
539
  continue; // unparseable frontmatter is its own report
@@ -502,6 +564,9 @@ export function deployIssues(files) {
502
564
  // written into it would ship in the file.
503
565
  for (const w of parseTelemetry(front.telemetry).problems)
504
566
  at(f.path, w);
567
+ // `judge:` that cannot be read would leave every judge on the default.
568
+ for (const w of parseJudge(front.judge).problems)
569
+ at(f.path, w);
505
570
  const lang = languageProblem(front.language);
506
571
  if (lang)
507
572
  at(f.path, lang);
@@ -49,8 +49,20 @@ export interface EvalInfo {
49
49
  inputs: boolean;
50
50
  cases: EvalCase[];
51
51
  }
52
- export declare function parseEval(file: string, raw: string): EvalInfo;
52
+ /** `defaults`: the workspace's `judge:` (AGENTS.md), used where the eval
53
+ * names no model or effort of its own. */
54
+ export declare function parseEval(file: string, raw: string, defaults?: {
55
+ model?: string;
56
+ effort?: string;
57
+ }): EvalInfo;
53
58
  export declare function listEvals(tenant: string, workspace: string): EvalInfo[];
59
+ /** `judge:` from AGENTS.md, field by field: the workspace's model (or
60
+ * effort), else the account's — so a workspace that sets only an effort
61
+ * keeps the account's model (Codex debate: the form says so). */
62
+ export declare function judgeDefaults(tenant: string, workspace: string): {
63
+ model?: string;
64
+ effort?: string;
65
+ };
54
66
  /**
55
67
  * Turn a finished run into a regression case.
56
68
  *
@@ -105,6 +117,15 @@ export interface EvalResult {
105
117
  * answers. */
106
118
  error?: string;
107
119
  }
120
+ /**
121
+ * The judge's environment: this process's, with the platform's model
122
+ * credential read live. The process's own CLAUDE_CODE_OAUTH_TOKEN is the copy
123
+ * it was handed at boot; the box refreshes the login every four hours and the
124
+ * old token is then revoked, so a judge on the boot copy failed with "OAuth
125
+ * token revoked" from the first refresh until the pod restarted, while the
126
+ * agent steps (which read the live file) kept working (2026-10-21).
127
+ */
128
+ export declare function judgeProcessEnv(): Record<string, string | undefined>;
108
129
  export declare function runEval(tenant: string, workspace: string, info: EvalInfo): Promise<EvalResult>;
109
130
  /**
110
131
  * The latest result, and one line of history.
package/dist/src/evals.js CHANGED
@@ -32,14 +32,17 @@
32
32
  // wrong.
33
33
  import { accountModelEnv, withModelEnv } from "./model-env.js";
34
34
  import { sdkEnv } from "./step-exec.js";
35
+ import { platformModelEnv } from "./runner.js";
35
36
  import fs from "node:fs";
36
37
  import path from "node:path";
37
38
  import { deployedCommit } from "./deploy.js";
38
39
  import { latestRevisionId } from "./history.js";
39
40
  import matter from "./matter.js";
40
41
  import { spawn } from "node:child_process";
42
+ import { judgeVerdict, JUDGE_VERDICT_INSTRUCTION, parseJudge } from "./judge-verdict.js";
43
+ import { readAgentsMdKeys } from "./agents-md.js";
41
44
  import { query } from "@anthropic-ai/claude-agent-sdk";
42
- import { workspaceDir, readRun, runVerdict, writeWorkspaceFile, resolveModel, resolveEffort, assertSafeName, } from "./store.js";
45
+ import { workspaceDir, accountDir, readRun, runVerdict, writeWorkspaceFile, resolveModel, resolveEffort, assertSafeName, } from "./store.js";
43
46
  import { startFlowRun, loadFlow } from "./runner.js";
44
47
  import { platform, platformQueues } from "./platform.js";
45
48
  const ASSERTION_TYPES = ["contains", "not-contains", "matches", "file", "run", "judge"];
@@ -85,7 +88,9 @@ function readScalar(lines, start, inline, indent) {
85
88
  const first = inline.trim().replace(/^["']|["']$/g, "");
86
89
  return { value: [first, ...parts].join(" ").trim(), end: i };
87
90
  }
88
- export function parseEval(file, raw) {
91
+ /** `defaults`: the workspace's `judge:` (AGENTS.md), used where the eval
92
+ * names no model or effort of its own. */
93
+ export function parseEval(file, raw, defaults = {}) {
89
94
  const { data, content } = matter(raw);
90
95
  const cases = [];
91
96
  let current = null;
@@ -137,8 +142,8 @@ export function parseEval(file, raw) {
137
142
  file,
138
143
  agent: typeof data.agent === "string" ? data.agent : null,
139
144
  flow: typeof data.flow === "string" ? data.flow : null,
140
- model: resolveModel(data.model ?? "fast"),
141
- effort: resolveEffort(data.effort ?? "low"),
145
+ model: resolveModel(data.model ?? defaults.model ?? "fast"),
146
+ effort: resolveEffort(data.effort ?? defaults.effort ?? "low"),
142
147
  trigger: data.trigger === "manual" ? "manual" : "deploy",
143
148
  live: data.live === true,
144
149
  inputs: data.inputs === true,
@@ -153,7 +158,17 @@ export function listEvals(tenant, workspace) {
153
158
  .readdirSync(dir)
154
159
  .filter((f) => f.endsWith(".md"))
155
160
  .sort()
156
- .map((f) => parseEval(f, fs.readFileSync(path.join(dir, f), "utf8")));
161
+ .map((f) => parseEval(f, fs.readFileSync(path.join(dir, f), "utf8"), judgeDefaults(tenant, workspace)));
162
+ }
163
+ /** `judge:` from AGENTS.md, field by field: the workspace's model (or
164
+ * effort), else the account's — so a workspace that sets only an effort
165
+ * keeps the account's model (Codex debate: the form says so). */
166
+ export function judgeDefaults(tenant, workspace) {
167
+ const ws = parseJudge(readAgentsMdKeys(workspaceDir(tenant, workspace)).judge);
168
+ const account = parseJudge(readAgentsMdKeys(accountDir(tenant)).judge);
169
+ const model = ws.model ?? account.model;
170
+ const effort = ws.effort ?? account.effort;
171
+ return { ...(model ? { model } : {}), ...(effort ? { effort } : {}) };
157
172
  }
158
173
  // ---------- promoting a run ----------
159
174
  /** An error a route can turn into the right status without knowing HTTP. */
@@ -365,12 +380,14 @@ function shell(command, cwd) {
365
380
  child.on("error", (err) => resolve({ code: null, out: String(err) }));
366
381
  });
367
382
  }
368
- // The judge gets the rubric and the output, and must answer with one word.
383
+ // The judge gets the rubric and the output, and ends with one word.
369
384
  // Constrained output beats asking for a score: "PASS" or "FAIL" can't be
370
- // misread, where "7/10" needs a threshold nobody agreed on.
385
+ // misread, where "7/10" needs a threshold nobody agreed on. The word comes
386
+ // LAST, after the reasoning: asked for it first, the judge committed before
387
+ // it had looked, then argued the other way — "FAIL … which meets the
388
+ // criterion" (vaza evals, 21 Oct 2026).
371
389
  async function judge(rubric, output, model, effort, env) {
372
- const prompt = `You are grading one piece of output against one criterion. Answer with exactly ` +
373
- `PASS or FAIL on the first line, then one short sentence of reason.\n\n` +
390
+ const prompt = `You are grading one piece of output against one criterion. ${JUDGE_VERDICT_INSTRUCTION}\n\n` +
374
391
  `Criterion: ${rubric}\n\n<output>\n${output.slice(0, 20000)}\n</output>`;
375
392
  let text = "";
376
393
  const q = query({
@@ -396,7 +413,19 @@ async function judge(rubric, output, model, effort, env) {
396
413
  }
397
414
  return text.trim();
398
415
  }
399
- async function checkAssertion(assertion, output, agentDir, model, effort, judgeEnv = () => ({ env: sdkEnv(undefined) })) {
416
+ /**
417
+ * The judge's environment: this process's, with the platform's model
418
+ * credential read live. The process's own CLAUDE_CODE_OAUTH_TOKEN is the copy
419
+ * it was handed at boot; the box refreshes the login every four hours and the
420
+ * old token is then revoked, so a judge on the boot copy failed with "OAuth
421
+ * token revoked" from the first refresh until the pod restarted, while the
422
+ * agent steps (which read the live file) kept working (2026-10-21).
423
+ */
424
+ export function judgeProcessEnv() {
425
+ const live = Object.fromEntries(Object.entries(platformModelEnv()).filter(([, v]) => v !== undefined));
426
+ return sdkEnv({ ...process.env, ...live });
427
+ }
428
+ async function checkAssertion(assertion, output, agentDir, model, effort, judgeEnv = () => ({ env: judgeProcessEnv() })) {
400
429
  const hay = output.toLowerCase();
401
430
  switch (assertion.type) {
402
431
  case "contains": {
@@ -438,9 +467,9 @@ async function checkAssertion(assertion, output, agentDir, model, effort, judgeE
438
467
  catch (err) {
439
468
  return { assertion, passed: false, detail: `judge not run — ${err instanceof Error ? err.message : String(err)}` };
440
469
  }
441
- const verdict = await withModelEnv(m, (env) => judge(assertion.value, output, model, effort, env));
442
- const passed = /^\s*PASS\b/i.test(verdict);
443
- return { assertion, passed, detail: verdict.slice(0, 300) || "judge returned nothing" };
470
+ const text = await withModelEnv(m, (env) => judge(assertion.value, output, model, effort, env));
471
+ const { passed, detail } = judgeVerdict(text);
472
+ return { assertion, passed, detail };
444
473
  }
445
474
  }
446
475
  }
@@ -509,7 +538,7 @@ export async function runEval(tenant, workspace, info) {
509
538
  for (const a of cheap)
510
539
  assertions.push(await checkAssertion(a, output, agentDir, info.model, info.effort));
511
540
  if (assertions.every((r) => r.passed)) {
512
- const judgeEnv = () => accountModelEnv(tenant, workspace, () => sdkEnv(undefined));
541
+ const judgeEnv = () => accountModelEnv(tenant, workspace, judgeProcessEnv);
513
542
  for (const a of judges)
514
543
  assertions.push(await checkAssertion(a, output, agentDir, info.model, info.effort, judgeEnv));
515
544
  }
@@ -25,4 +25,18 @@ export interface KnownNames {
25
25
  * rule is skipped rather than guessed at. */
26
26
  outwardAgents?: string[];
27
27
  }
28
+ /** A `verify:` that is a claim in English rather than a command. Conservative
29
+ * on purpose, the same bargain as REFERS_BACK: it must have no assertion
30
+ * prefix, nothing shell-shaped anywhere in it, and enough words to be a
31
+ * sentence — so a bare `./check.sh` or a one-word command is left alone and
32
+ * the warning only fires where the author plainly wrote prose. */
33
+ /**
34
+ * Why a `matches:` pattern will not compile, or null when it does. The
35
+ * runtime tests it as a JavaScript RegExp with the `i` flag (step-exec.ts,
36
+ * evals.ts), and a pattern that throws fails its check on every run with
37
+ * "invalid regular expression". The usual cause is another dialect: on
38
+ * 2026-10-21 Python's inline `(?m)` broke five eval checks on vaza, and
39
+ * nothing said so before they ran.
40
+ */
41
+ export declare function regexProblem(pattern: string): string | null;
28
42
  export declare function lintFlow(flow: FlowInfo, known?: KnownNames): FlowWarning[];
@@ -54,6 +54,27 @@ const LOOKS_LIKE_SHELL = /&&|\|\||\s\|\s|\$\(|\$\{|`|\s[<>]\s|\d?>[>&]?\s|^\[|^[
54
54
  * prefix, nothing shell-shaped anywhere in it, and enough words to be a
55
55
  * sentence — so a bare `./check.sh` or a one-word command is left alone and
56
56
  * the warning only fires where the author plainly wrote prose. */
57
+ /**
58
+ * Why a `matches:` pattern will not compile, or null when it does. The
59
+ * runtime tests it as a JavaScript RegExp with the `i` flag (step-exec.ts,
60
+ * evals.ts), and a pattern that throws fails its check on every run with
61
+ * "invalid regular expression". The usual cause is another dialect: on
62
+ * 2026-10-21 Python's inline `(?m)` broke five eval checks on vaza, and
63
+ * nothing said so before they ran.
64
+ */
65
+ export function regexProblem(pattern) {
66
+ try {
67
+ new RegExp(pattern, "i");
68
+ return null;
69
+ }
70
+ catch (err) {
71
+ const why = err instanceof Error ? err.message.replace(/^Invalid regular expression: /, "") : String(err);
72
+ const inline = /\(\?[a-z]+\)/.test(pattern)
73
+ ? " — JavaScript has no inline flags like (?m); write (^|\\n) for the start of a line"
74
+ : "";
75
+ return `is not a valid JavaScript regular expression (${why})${inline}`;
76
+ }
77
+ }
57
78
  function looksLikeProse(verify) {
58
79
  const v = verify.trim();
59
80
  if (!v || VERIFY_ASSERTION.test(v))
@@ -377,6 +398,20 @@ export function lintFlow(flow, known) {
377
398
  "claim, or write it as a shell command that exits 0 when the claim holds.",
378
399
  });
379
400
  }
401
+ // A `verify: matches:` the runtime cannot compile fails the step on
402
+ // every run, whatever the agent did.
403
+ const verifyRe = step.verify ? /^matches:\s*(.*)$/.exec(step.verify.trim()) : null;
404
+ // The same normalisation checkVerify applies (step-exec.ts): one quote
405
+ // off each end, so the pattern checked is the pattern that runs.
406
+ const reProblem = verifyRe ? regexProblem(verifyRe[1].trim().replace(/^["']|["']$/g, "")) : null;
407
+ if (reProblem) {
408
+ warnings.push({
409
+ step: i,
410
+ line: step.line,
411
+ message: `verify: matches: ${reProblem}`,
412
+ detail: "The runtime tests matches: as a JavaScript regular expression, case-insensitive. One that does not compile fails this step on every run.",
413
+ });
414
+ }
380
415
  // A step's OTHER agent references — the ones no parser validates,
381
416
  // because they are options rather than the step's target. Both fail
382
417
  // silently and at the worst possible moment: a misspelled `delegate:`
@@ -24,6 +24,7 @@ import { joinGroup, splitToRail } from "./arrange.js";
24
24
  import { lintFlow } from "./flow-lint.js";
25
25
  import { toLF, asRaw } from "./eol.js";
26
26
  import { removalImpact, stepLabel } from "./step-removal.js";
27
+ import { keySpan, InlineBlockError } from "./frontmatter-text.js";
27
28
  // ---------- flow files ----------
28
29
  /** A flow's steps as groups of indices into its parsed steps — what
29
30
  * reorderFlowSteps and the arrange helpers take. */
@@ -437,14 +438,16 @@ function splitComment(value) {
437
438
  const m = value.match(/^(.*?)(\s+#.*)$/);
438
439
  return m ? { value: m[1], comment: m[2] } : { value, comment: "" };
439
440
  }
441
+ /** Where a key is written — at the top or inside its job block
442
+ * (frontmatter-text.ts) — with the text after its colon and its indent. */
440
443
  function keyLine(lines, close, key) {
441
- const re = new RegExp(`^${key.replace(/[-]/g, "\\-")}:(.*)$`);
442
- for (let i = 1; i < close; i++) {
443
- const m = lines[i].match(re);
444
- if (m)
445
- return { index: i, rest: m[1] };
446
- }
447
- return null;
444
+ const where = keySpan(lines.slice(1, close), key);
445
+ if (!where)
446
+ return null;
447
+ const index = where.at + 1;
448
+ // Quoted or not, as keySpan matched it.
449
+ const m = lines[index].slice(where.indent.length).match(new RegExp(`^["']?${key.replace(/[.*+?^${}()|[\]\\-]/g, "\\$&")}["']?\\s*:(.*)$`));
450
+ return m ? { index, rest: m[1], indent: where.indent } : null;
448
451
  }
449
452
  /** Block-style item lines under a key: ` - name`, until anything else. */
450
453
  function blockItems(lines, from, close) {
@@ -467,7 +470,17 @@ export function frontmatterList(raw, key) {
467
470
  const b = frontBounds(lines);
468
471
  if (!b)
469
472
  return [];
470
- const k = keyLine(lines, b.close, key);
473
+ let k;
474
+ try {
475
+ k = keyLine(lines, b.close, key);
476
+ }
477
+ catch (err) {
478
+ // A block these text edits do not handle (frontmatter-text.ts) is the
479
+ // writer's to refuse; a read of it is empty, not an error.
480
+ if (err instanceof InlineBlockError)
481
+ return [];
482
+ throw err;
483
+ }
471
484
  if (!k)
472
485
  return [];
473
486
  const v = splitComment(k.rest).value.trim();
@@ -491,10 +504,12 @@ export function editFrontmatterList(raw, key, name, action) {
491
504
  const b = frontBounds(lines);
492
505
  if (!b)
493
506
  return action === "remove" ? raw : ["---", `${key}: [${name}]`, "---", ...lines].join(eol);
507
+ // Located first, so a key in a block these edits do not handle is refused
508
+ // (InlineBlockError) rather than read as an empty list.
509
+ const k = keyLine(lines, b.close, key);
494
510
  const has = frontmatterList(raw, key).includes(name);
495
511
  if ((action === "add") === has)
496
512
  return raw;
497
- const k = keyLine(lines, b.close, key);
498
513
  if (!k) {
499
514
  lines.splice(b.close, 0, `${key}: [${name}]`);
500
515
  return lines.join(eol);
@@ -505,12 +520,12 @@ export function editFrontmatterList(raw, key, name, action) {
505
520
  // Flow style, or a lone scalar (read as a one-item list): that one line.
506
521
  const tokens = v.startsWith("[") ? v.replace(/^\[|\]$/g, "").split(",").map((t) => t.trim()).filter(Boolean) : [v];
507
522
  const next = action === "add" ? [...tokens, name] : tokens.filter((t) => itemName(t) !== name);
508
- lines[k.index] = `${key}: [${next.join(", ")}]${comment}`;
523
+ lines[k.index] = `${k.indent}${key}: [${next.join(", ")}]${comment}`;
509
524
  return lines.join(eol);
510
525
  }
511
526
  const items = blockItems(lines, k.index, b.close);
512
527
  if (action === "add") {
513
- const indent = items.length ? lines[items[0]].match(/^(\s*)-/)[1] : " ";
528
+ const indent = items.length ? lines[items[0]].match(/^(\s*)-/)[1] : `${k.indent} `;
514
529
  lines.splice(items.length ? items[items.length - 1] + 1 : k.index + 1, 0, `${indent}- ${name}`);
515
530
  return lines.join(eol);
516
531
  }
@@ -518,7 +533,7 @@ export function editFrontmatterList(raw, key, name, action) {
518
533
  for (const i of [...drop].reverse())
519
534
  lines.splice(i, 1);
520
535
  if (drop.length === items.length)
521
- lines[k.index] = `${key}: []${comment}`;
536
+ lines[k.index] = `${k.indent}${key}: []${comment}`;
522
537
  return lines.join(eol);
523
538
  }
524
539
  const YAML_WORDS = /^(true|false|yes|no|on|off|null|~|y|n)$/i;
@@ -546,10 +561,10 @@ export function setFrontmatterScalar(raw, key, value) {
546
561
  const { value: v, comment } = splitComment(k.rest);
547
562
  let end = k.index + 1;
548
563
  if (/^\s*[|>][-+]?\d*\s*$/.test(v) || !v.trim()) {
549
- while (end < b.close && /^\s+\S/.test(lines[end]))
564
+ while (end < b.close && (lines[end].match(/^\s*/)?.[0].length ?? 0) > k.indent.length && /\S/.test(lines[end]))
550
565
  end++;
551
566
  }
552
- lines.splice(k.index, end - k.index, `${line}${comment}`);
567
+ lines.splice(k.index, end - k.index, `${k.indent}${line}${comment}`);
553
568
  return lines.join(eol);
554
569
  }
555
570
  const name = keyLine(lines, b.close, "name");