@foldrun/core 2.5.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/agent-import.js +2 -2
- package/dist/src/agent-tools.js +2 -2
- package/dist/src/agents-md.d.ts +5 -1
- package/dist/src/agents-md.js +42 -18
- package/dist/src/completions.js +25 -3
- package/dist/src/deploy.js +72 -7
- package/dist/src/evals.d.ts +22 -1
- package/dist/src/evals.js +43 -14
- package/dist/src/flow-lint.d.ts +14 -0
- package/dist/src/flow-lint.js +35 -0
- package/dist/src/flow-patterns.js +29 -14
- package/dist/src/frontmatter-fields.d.ts +27 -0
- package/dist/src/frontmatter-fields.js +51 -3
- package/dist/src/frontmatter-guard.d.ts +19 -0
- package/dist/src/frontmatter-guard.js +67 -0
- package/dist/src/frontmatter-schema.d.ts +2 -2
- package/dist/src/frontmatter-schema.js +30 -2
- package/dist/src/frontmatter-text.d.ts +31 -0
- package/dist/src/frontmatter-text.js +102 -0
- package/dist/src/judge-verdict.d.ts +19 -0
- package/dist/src/judge-verdict.js +54 -0
- package/dist/src/matter.d.ts +34 -0
- package/dist/src/matter.js +104 -0
- package/dist/src/network.d.ts +0 -1
- package/dist/src/network.js +10 -7
- package/dist/src/package.js +9 -2
- package/dist/src/provider-health.js +2 -2
- package/dist/src/run-container.d.ts +9 -0
- package/dist/src/run-container.js +4 -1
- package/dist/src/runner.js +119 -36
- package/dist/src/scan.d.ts +11 -0
- package/dist/src/scan.js +86 -8
- package/dist/src/starter.js +23 -0
- package/dist/src/step-exec.d.ts +10 -2
- package/dist/src/step-exec.js +12 -9
- package/dist/src/store.d.ts +17 -3
- package/dist/src/store.js +14 -10
- package/package.json +1 -1
- package/src/agent-import.ts +2 -2
- package/src/agent-tools.ts +2 -2
- package/src/agents-md.ts +39 -14
- package/src/completions.ts +26 -3
- package/src/deploy.ts +59 -7
- package/src/evals.ts +45 -13
- package/src/flow-lint.ts +36 -0
- package/src/flow-patterns.ts +26 -13
- package/src/frontmatter-fields.ts +74 -3
- package/src/frontmatter-guard.ts +70 -0
- package/src/frontmatter-schema.ts +30 -4
- package/src/frontmatter-text.ts +109 -0
- package/src/judge-verdict.ts +52 -0
- package/src/matter.ts +112 -0
- package/src/network.ts +10 -6
- package/src/package.ts +8 -2
- package/src/provider-health.ts +2 -2
- package/src/run-container.ts +11 -1
- package/src/runner.ts +118 -37
- package/src/scan.ts +86 -8
- package/src/starter.ts +23 -0
- package/src/step-exec.ts +21 -8
- package/src/store.ts +23 -10
package/dist/src/agent-import.js
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
// were about somebody else's desk.
|
|
15
15
|
import fs from "node:fs";
|
|
16
16
|
import path from "node:path";
|
|
17
|
-
import
|
|
17
|
+
import { configMatter } from "./matter.js";
|
|
18
18
|
import { assertSafeName, listWorkspaceFiles, readWorkspaceFile, writeWorkspaceFile, workspaceDir, workspaceTools, listWorkspaces, } from "./store.js";
|
|
19
19
|
import { libraryDir, libraryTools } from "./library.js";
|
|
20
20
|
import { isRuntimeTool, toolEntryName } from "./tool-names.js";
|
|
@@ -64,7 +64,7 @@ export function importAgent(tenant, from, agent, to, opts = {}) {
|
|
|
64
64
|
let content = readWorkspaceFile(tenant, from, src);
|
|
65
65
|
const rel = src.slice(`agents/${agent}/`.length);
|
|
66
66
|
if (rel === "agent.md") {
|
|
67
|
-
front =
|
|
67
|
+
front = configMatter(content).data;
|
|
68
68
|
// The folder is the name flows address; a `name:` that disagrees with
|
|
69
69
|
// it is the first thing check would complain about.
|
|
70
70
|
if (name !== agent)
|
package/dist/src/agent-tools.js
CHANGED
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
import fs from "node:fs";
|
|
17
17
|
import os from "node:os";
|
|
18
18
|
import path from "node:path";
|
|
19
|
-
import
|
|
19
|
+
import { configMatter } from "./matter.js";
|
|
20
20
|
import { z } from "zod";
|
|
21
21
|
import { createSdkMcpServer, tool } from "@anthropic-ai/claude-agent-sdk";
|
|
22
22
|
import { resolveModel, resolveEffort } from "./store.js";
|
|
@@ -36,7 +36,7 @@ export function gatherConsults(workspaceRoot, names) {
|
|
|
36
36
|
missing.push(name);
|
|
37
37
|
continue;
|
|
38
38
|
}
|
|
39
|
-
const parsed =
|
|
39
|
+
const parsed = configMatter(fs.readFileSync(file, "utf8"));
|
|
40
40
|
consults.push({
|
|
41
41
|
name,
|
|
42
42
|
systemPrompt: `${parsed.content.trim()}\n\n` +
|
package/dist/src/agents-md.d.ts
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
|
+
import { keySpan } from "./frontmatter-text.ts";
|
|
2
|
+
export { keySpan };
|
|
1
3
|
/** The frontmatter of a directory's AGENTS.md (or project.md), or {}. */
|
|
2
4
|
export declare function readAgentsMdKeys(dir: string): Record<string, unknown>;
|
|
3
5
|
/** The text of `content` with each key in `patch` replaced (or removed for
|
|
4
6
|
* null), everything else byte for byte as it was. A file with no
|
|
5
|
-
* frontmatter gets one.
|
|
7
|
+
* frontmatter gets one. Read back before it is returned (frontmatter-guard
|
|
8
|
+
* .ts): an edit that would not parse, would change another key or would
|
|
9
|
+
* double one into its block throws UnsafeEditError (409). */
|
|
6
10
|
export declare function patchFrontmatter(content: string, patch: Record<string, unknown | null>): string;
|
|
7
11
|
/** Apply a frontmatter patch to a directory's AGENTS.md, creating the file
|
|
8
12
|
* when the directory has none. Returns the new content. */
|
package/dist/src/agents-md.js
CHANGED
|
@@ -8,7 +8,11 @@
|
|
|
8
8
|
// else in the file is untouched. Null removes the key.
|
|
9
9
|
import fs from "node:fs";
|
|
10
10
|
import path from "node:path";
|
|
11
|
-
import matter from "./matter.js";
|
|
11
|
+
import matter, { configMatter } from "./matter.js";
|
|
12
|
+
import { keySpan, dropEmptyBlock } from "./frontmatter-text.js";
|
|
13
|
+
import { guardEdit } from "./frontmatter-guard.js";
|
|
14
|
+
import { JOBS, FOLDABLE } from "./frontmatter-fields.js";
|
|
15
|
+
export { keySpan };
|
|
12
16
|
const FRONT = /^(---\r?\n)([\s\S]*?)(\r?\n---)(\r?\n|$)/;
|
|
13
17
|
/** The frontmatter of a directory's AGENTS.md (or project.md), or {}. */
|
|
14
18
|
export function readAgentsMdKeys(dir) {
|
|
@@ -17,7 +21,7 @@ export function readAgentsMdKeys(dir) {
|
|
|
17
21
|
if (!fs.existsSync(file))
|
|
18
22
|
continue;
|
|
19
23
|
try {
|
|
20
|
-
return (
|
|
24
|
+
return (configMatter(fs.readFileSync(file, "utf8")).data ?? {});
|
|
21
25
|
}
|
|
22
26
|
catch {
|
|
23
27
|
return {};
|
|
@@ -34,8 +38,33 @@ function dumpKey(key, value) {
|
|
|
34
38
|
}
|
|
35
39
|
/** The text of `content` with each key in `patch` replaced (or removed for
|
|
36
40
|
* null), everything else byte for byte as it was. A file with no
|
|
37
|
-
* frontmatter gets one.
|
|
41
|
+
* frontmatter gets one. Read back before it is returned (frontmatter-guard
|
|
42
|
+
* .ts): an edit that would not parse, would change another key or would
|
|
43
|
+
* double one into its block throws UnsafeEditError (409). */
|
|
38
44
|
export function patchFrontmatter(content, patch) {
|
|
45
|
+
// The guard reads the result folded (matter.ts configMatter), so a patch
|
|
46
|
+
// that writes a whole job block is checked as the keys it holds: each
|
|
47
|
+
// field the block may hold may change, and each one it sets reads back.
|
|
48
|
+
const changed = new Set();
|
|
49
|
+
const expect = {};
|
|
50
|
+
for (const [key, value] of Object.entries(patch)) {
|
|
51
|
+
changed.add(key);
|
|
52
|
+
const job = JOBS.find((j) => j.name === key)?.name;
|
|
53
|
+
if (!job) {
|
|
54
|
+
expect[key] = value;
|
|
55
|
+
continue;
|
|
56
|
+
}
|
|
57
|
+
for (const k of FOLDABLE[job])
|
|
58
|
+
changed.add(k);
|
|
59
|
+
if (value && typeof value === "object" && !Array.isArray(value)) {
|
|
60
|
+
for (const [k, v] of Object.entries(value))
|
|
61
|
+
if (FOLDABLE[job].has(k))
|
|
62
|
+
expect[k] = v;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
return guardEdit(content, patchFrontmatterText(content, patch), [...changed], expect);
|
|
66
|
+
}
|
|
67
|
+
function patchFrontmatterText(content, patch) {
|
|
39
68
|
const m = content.match(FRONT);
|
|
40
69
|
let open = "---\n";
|
|
41
70
|
let body = "";
|
|
@@ -50,22 +79,17 @@ export function patchFrontmatter(content, patch) {
|
|
|
50
79
|
}
|
|
51
80
|
let lines = body === "" ? [] : body.split("\n");
|
|
52
81
|
for (const [key, value] of Object.entries(patch)) {
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
while (end < lines.length && (/^[ \t]/.test(lines[end]) || lines[end].trim() === ""))
|
|
59
|
-
end++;
|
|
60
|
-
// A blank line at the block's foot belongs to what follows.
|
|
61
|
-
while (end > at + 1 && lines[end - 1].trim() === "")
|
|
62
|
-
end--;
|
|
63
|
-
}
|
|
64
|
-
const replacement = value === null || value === undefined ? [] : dumpKey(key, value);
|
|
65
|
-
if (at === -1)
|
|
82
|
+
// Where it is written now — at the top or inside its job block — is
|
|
83
|
+
// where it is edited; a key written nowhere is added at the top.
|
|
84
|
+
const where = keySpan(lines, key);
|
|
85
|
+
const replacement = value === null || value === undefined ? [] : dumpKey(key, value).map((l) => `${where?.indent ?? ""}${l}`);
|
|
86
|
+
if (!where)
|
|
66
87
|
lines = [...lines, ...replacement];
|
|
67
|
-
else
|
|
68
|
-
lines = [...lines.slice(0, at), ...replacement, ...lines.slice(end)];
|
|
88
|
+
else {
|
|
89
|
+
lines = [...lines.slice(0, where.at), ...replacement, ...lines.slice(where.end)];
|
|
90
|
+
if (!replacement.length && where.block !== null)
|
|
91
|
+
lines = dropEmptyBlock(lines, where.block);
|
|
92
|
+
}
|
|
69
93
|
}
|
|
70
94
|
const front = lines.length ? `${open}${lines.join("\n")}${close}` : "";
|
|
71
95
|
if (!front)
|
package/dist/src/completions.js
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
// go here. No React, no fetch — so the rules are testable and one component
|
|
10
10
|
// renders them.
|
|
11
11
|
import { PROVIDERS } from "./providers.js";
|
|
12
|
-
import { FRONTMATTER } from "./frontmatter-fields.js";
|
|
12
|
+
import { FRONTMATTER, JOBS, fieldsOfJob } from "./frontmatter-fields.js";
|
|
13
13
|
const MODELS = [
|
|
14
14
|
{ label: "fast", hint: "haiku — cheap, for checks and triage" },
|
|
15
15
|
{ label: "default", hint: "sonnet — the working default" },
|
|
@@ -587,8 +587,30 @@ export function completionsAt(path, text, cursor, vocab) {
|
|
|
587
587
|
};
|
|
588
588
|
}
|
|
589
589
|
}
|
|
590
|
-
//
|
|
591
|
-
|
|
590
|
+
// 5b. An indented line inside one of the four job blocks (governance,
|
|
591
|
+
// guardrail, observability, evaluation): that job's fields for this
|
|
592
|
+
// kind of file — the block takes nothing else.
|
|
593
|
+
const fmClose = text.indexOf("\n---", text.indexOf("---") + 3);
|
|
594
|
+
const insideFrontmatter = text.startsWith("---") && (fmClose === -1 || cursor <= fmClose);
|
|
595
|
+
const nested = line.match(/^(\s+)([a-zA-Z_]*)$/);
|
|
596
|
+
if (insideFrontmatter && nested) {
|
|
597
|
+
const header = before.slice(0, lineStart).split("\n").reverse().find((l) => /^\S/.test(l));
|
|
598
|
+
const job = JOBS.find((j) => new RegExp(`^${j.name}\\s*:\\s*$`).test(header ?? ""));
|
|
599
|
+
if (job) {
|
|
600
|
+
const schemaKind = /(^|\/)AGENTS\.md$/.test(path) || path === "project.md" ? "agents-md" : SCHEMA_OF[kind];
|
|
601
|
+
const keys = new Set(schemaKind ? fieldsOfJob(schemaKind, job.name) : []);
|
|
602
|
+
const partial = nested[2] ?? "";
|
|
603
|
+
return {
|
|
604
|
+
from: cursor - partial.length,
|
|
605
|
+
query: partial,
|
|
606
|
+
items: filter(fieldsFor(path, kind).filter((c) => keys.has(c.label.split(".")[0])), partial),
|
|
607
|
+
title: `${job.title} — ${job.what}`,
|
|
608
|
+
};
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
// 6. An indented option under a flow step — the body's steps, never a
|
|
612
|
+
// line inside the frontmatter.
|
|
613
|
+
if (kind === "flow" && !insideFrontmatter && /^\s+[a-z]*$/.test(line) && line.trim().length >= 0) {
|
|
592
614
|
const partial = line.trim();
|
|
593
615
|
return {
|
|
594
616
|
from: cursor - partial.length,
|
package/dist/src/deploy.js
CHANGED
|
@@ -23,21 +23,24 @@ import path from "node:path";
|
|
|
23
23
|
import { saveWorkspace, keptOnDeploy, workspaceDir, listRunSummaries, listWorkspaceFiles, parseFlow, parseToolDef, WORKSPACE_DIRS, assertSafeName, accountDir, frontmatterErrorText, } from "./store.js";
|
|
24
24
|
import { readAgentsMdKeys } from "./agents-md.js";
|
|
25
25
|
import { libraryTools } from "./library.js";
|
|
26
|
-
import { lintFlow } from "./flow-lint.js";
|
|
26
|
+
import { lintFlow, regexProblem } from "./flow-lint.js";
|
|
27
|
+
import { parseEval } from "./evals.js";
|
|
27
28
|
import { refNames } from "./refs.js";
|
|
28
|
-
import matter from "./matter.js";
|
|
29
|
+
import matter, { configMatter } from "./matter.js";
|
|
30
|
+
import { frontmatterProblems, schemaKindOf } from "./frontmatter-schema.js";
|
|
29
31
|
import { conformanceIssues } from "./okf.js";
|
|
30
32
|
import { timezoneProblem } from "./clock.js";
|
|
31
33
|
import { webProblems, webWarnings, runnerEngines } from "./providers.js";
|
|
32
34
|
import { readLimits } from "./limits.js";
|
|
33
35
|
import { parseAlerts } from "./alerts.js";
|
|
34
36
|
import { parseTelemetry } from "./otel-config.js";
|
|
37
|
+
import { parseJudge } from "./judge-verdict.js";
|
|
35
38
|
import { networkProblems } from "./network.js";
|
|
36
39
|
import { actionProblems } from "./web-actions.js";
|
|
37
40
|
import { languageProblem } from "./language.js";
|
|
38
41
|
import { localeProblems } from "./locale.js";
|
|
39
42
|
import { entriesNoFollow } from "./paths.js";
|
|
40
|
-
import { filterIgnored, findingFingerprint, ruleForWhere, scanFiles, scanRule } from "./scan.js";
|
|
43
|
+
import { filterIgnored, findingFingerprint, ruleForWhere, scanFiles, scanRule, STRICT_FRONTMATTER_PATH } from "./scan.js";
|
|
41
44
|
/** Directories a deploy never reads out of a source tree. */
|
|
42
45
|
// `.cache` is what a run's tools leave in an agent's folder (HOME is that
|
|
43
46
|
// folder): logs and font caches, never authored.
|
|
@@ -255,7 +258,7 @@ export function deployBlockMode(files, accountKeys) {
|
|
|
255
258
|
let fromWorkspace = null;
|
|
256
259
|
if (own) {
|
|
257
260
|
try {
|
|
258
|
-
fromWorkspace = blockMode(
|
|
261
|
+
fromWorkspace = blockMode(configMatter(own.content).data.deploy_block);
|
|
259
262
|
}
|
|
260
263
|
catch {
|
|
261
264
|
// an AGENTS.md that does not parse is reported elsewhere
|
|
@@ -394,6 +397,41 @@ function structuralWarnings(files, extraOutward = []) {
|
|
|
394
397
|
for (const w of webWarnings(data, { chrome, chromium }))
|
|
395
398
|
out.push({ where: f.path, message: w });
|
|
396
399
|
}
|
|
400
|
+
// An eval's `matches:` is the same JavaScript RegExp as a verify's; one
|
|
401
|
+
// that does not compile fails its case on every run. Read through
|
|
402
|
+
// parseEval, so a block scalar (`matches: |`) or a continued line is
|
|
403
|
+
// tested as the runtime will run it. The line is the item's own: the
|
|
404
|
+
// parsed assertions and the `- matches:` lines come in the same order.
|
|
405
|
+
for (const f of files) {
|
|
406
|
+
if (!/^evals\/[^/]+\.md$/.test(f.path))
|
|
407
|
+
continue;
|
|
408
|
+
let info;
|
|
409
|
+
try {
|
|
410
|
+
info = parseEval(f.path, f.content);
|
|
411
|
+
}
|
|
412
|
+
catch {
|
|
413
|
+
continue;
|
|
414
|
+
}
|
|
415
|
+
const lines = [];
|
|
416
|
+
f.content.split("\n").forEach((line, n) => { if (/^\s*-\s*matches:/.test(line))
|
|
417
|
+
lines.push(n + 1); });
|
|
418
|
+
// When a `- matches:` line sits outside a case (an example in the
|
|
419
|
+
// prose), the order no longer pairs up: name the file, not a wrong line.
|
|
420
|
+
const parsed = info.cases.reduce((n, c) => n + c.expect.filter((a) => a.type === "matches").length, 0);
|
|
421
|
+
const pairs = lines.length === parsed;
|
|
422
|
+
let k = 0;
|
|
423
|
+
for (const c of info.cases) {
|
|
424
|
+
for (const a of c.expect) {
|
|
425
|
+
if (a.type !== "matches")
|
|
426
|
+
continue;
|
|
427
|
+
const at = pairs ? lines[k] : undefined;
|
|
428
|
+
k++;
|
|
429
|
+
const problem = regexProblem(a.value);
|
|
430
|
+
if (problem)
|
|
431
|
+
out.push({ where: at ? `${f.path}:${at}` : f.path, message: `case "${c.name}": matches: ${problem}` });
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
}
|
|
397
435
|
if (!outwardAgents.length)
|
|
398
436
|
return out;
|
|
399
437
|
for (const f of files) {
|
|
@@ -446,10 +484,11 @@ export function deployIssues(files) {
|
|
|
446
484
|
// nothing said so — gray-matter's cache let a second parse pass — and one
|
|
447
485
|
// such agent file kept the worker from booting for every account.
|
|
448
486
|
// Definition files only: a skill is read leniently (an unquoted colon is
|
|
449
|
-
// quoted and retried,
|
|
487
|
+
// quoted and retried, matter.ts parseLenient), and the markdown beside a
|
|
450
488
|
// definition — references, a tool's README — is prose, not frontmatter.
|
|
489
|
+
// The scanner warns about skills and library files (config/invalid-frontmatter).
|
|
451
490
|
for (const f of files) {
|
|
452
|
-
if (
|
|
491
|
+
if (!STRICT_FRONTMATTER_PATH.test(f.path))
|
|
453
492
|
continue;
|
|
454
493
|
if (!/^\uFEFF?---\r?\n/.test(f.content))
|
|
455
494
|
continue;
|
|
@@ -460,6 +499,29 @@ export function deployIssues(files) {
|
|
|
460
499
|
at(f.path, frontmatterErrorText(err));
|
|
461
500
|
}
|
|
462
501
|
}
|
|
502
|
+
// The four job blocks (frontmatter-fields.ts JOB_FIELDS): a setting
|
|
503
|
+
// written both at the top and in its block, or in another job's block, is
|
|
504
|
+
// a setting that does not do what the file says — a guardrail under
|
|
505
|
+
// governance: guards nothing — so the deploy refuses it as check does
|
|
506
|
+
// (config/conflict, config/misplaced; Codex debate, 21 Oct 2026).
|
|
507
|
+
for (const f of files) {
|
|
508
|
+
const kind = schemaKindOf(f.path);
|
|
509
|
+
if (kind !== "agent" && kind !== "flow" && kind !== "agents-md")
|
|
510
|
+
continue;
|
|
511
|
+
if (!/^\uFEFF?---\r?\n/.test(f.content))
|
|
512
|
+
continue;
|
|
513
|
+
let front;
|
|
514
|
+
try {
|
|
515
|
+
front = configMatter(f.content).data;
|
|
516
|
+
}
|
|
517
|
+
catch {
|
|
518
|
+
continue; // unparseable frontmatter is its own report
|
|
519
|
+
}
|
|
520
|
+
for (const p of frontmatterProblems(kind, front, () => null)) {
|
|
521
|
+
if (p.level === "error" && (p.rule === "config/conflict" || p.rule === "config/misplaced"))
|
|
522
|
+
at(f.path, `${p.message} [${p.rule}]`);
|
|
523
|
+
}
|
|
524
|
+
}
|
|
463
525
|
// `timezone:` is read at four levels and a value nothing can parse falls
|
|
464
526
|
// silently through to the level above — so it is refused where it was
|
|
465
527
|
// written, not discovered as a date that is a day out.
|
|
@@ -471,7 +533,7 @@ export function deployIssues(files) {
|
|
|
471
533
|
at(f.path, unopened);
|
|
472
534
|
let front = {};
|
|
473
535
|
try {
|
|
474
|
-
front =
|
|
536
|
+
front = configMatter(f.content).data;
|
|
475
537
|
}
|
|
476
538
|
catch {
|
|
477
539
|
continue; // unparseable frontmatter is its own report
|
|
@@ -502,6 +564,9 @@ export function deployIssues(files) {
|
|
|
502
564
|
// written into it would ship in the file.
|
|
503
565
|
for (const w of parseTelemetry(front.telemetry).problems)
|
|
504
566
|
at(f.path, w);
|
|
567
|
+
// `judge:` that cannot be read would leave every judge on the default.
|
|
568
|
+
for (const w of parseJudge(front.judge).problems)
|
|
569
|
+
at(f.path, w);
|
|
505
570
|
const lang = languageProblem(front.language);
|
|
506
571
|
if (lang)
|
|
507
572
|
at(f.path, lang);
|
package/dist/src/evals.d.ts
CHANGED
|
@@ -49,8 +49,20 @@ export interface EvalInfo {
|
|
|
49
49
|
inputs: boolean;
|
|
50
50
|
cases: EvalCase[];
|
|
51
51
|
}
|
|
52
|
-
|
|
52
|
+
/** `defaults`: the workspace's `judge:` (AGENTS.md), used where the eval
|
|
53
|
+
* names no model or effort of its own. */
|
|
54
|
+
export declare function parseEval(file: string, raw: string, defaults?: {
|
|
55
|
+
model?: string;
|
|
56
|
+
effort?: string;
|
|
57
|
+
}): EvalInfo;
|
|
53
58
|
export declare function listEvals(tenant: string, workspace: string): EvalInfo[];
|
|
59
|
+
/** `judge:` from AGENTS.md, field by field: the workspace's model (or
|
|
60
|
+
* effort), else the account's — so a workspace that sets only an effort
|
|
61
|
+
* keeps the account's model (Codex debate: the form says so). */
|
|
62
|
+
export declare function judgeDefaults(tenant: string, workspace: string): {
|
|
63
|
+
model?: string;
|
|
64
|
+
effort?: string;
|
|
65
|
+
};
|
|
54
66
|
/**
|
|
55
67
|
* Turn a finished run into a regression case.
|
|
56
68
|
*
|
|
@@ -105,6 +117,15 @@ export interface EvalResult {
|
|
|
105
117
|
* answers. */
|
|
106
118
|
error?: string;
|
|
107
119
|
}
|
|
120
|
+
/**
|
|
121
|
+
* The judge's environment: this process's, with the platform's model
|
|
122
|
+
* credential read live. The process's own CLAUDE_CODE_OAUTH_TOKEN is the copy
|
|
123
|
+
* it was handed at boot; the box refreshes the login every four hours and the
|
|
124
|
+
* old token is then revoked, so a judge on the boot copy failed with "OAuth
|
|
125
|
+
* token revoked" from the first refresh until the pod restarted, while the
|
|
126
|
+
* agent steps (which read the live file) kept working (2026-10-21).
|
|
127
|
+
*/
|
|
128
|
+
export declare function judgeProcessEnv(): Record<string, string | undefined>;
|
|
108
129
|
export declare function runEval(tenant: string, workspace: string, info: EvalInfo): Promise<EvalResult>;
|
|
109
130
|
/**
|
|
110
131
|
* The latest result, and one line of history.
|
package/dist/src/evals.js
CHANGED
|
@@ -32,14 +32,17 @@
|
|
|
32
32
|
// wrong.
|
|
33
33
|
import { accountModelEnv, withModelEnv } from "./model-env.js";
|
|
34
34
|
import { sdkEnv } from "./step-exec.js";
|
|
35
|
+
import { platformModelEnv } from "./runner.js";
|
|
35
36
|
import fs from "node:fs";
|
|
36
37
|
import path from "node:path";
|
|
37
38
|
import { deployedCommit } from "./deploy.js";
|
|
38
39
|
import { latestRevisionId } from "./history.js";
|
|
39
40
|
import matter from "./matter.js";
|
|
40
41
|
import { spawn } from "node:child_process";
|
|
42
|
+
import { judgeVerdict, JUDGE_VERDICT_INSTRUCTION, parseJudge } from "./judge-verdict.js";
|
|
43
|
+
import { readAgentsMdKeys } from "./agents-md.js";
|
|
41
44
|
import { query } from "@anthropic-ai/claude-agent-sdk";
|
|
42
|
-
import { workspaceDir, readRun, runVerdict, writeWorkspaceFile, resolveModel, resolveEffort, assertSafeName, } from "./store.js";
|
|
45
|
+
import { workspaceDir, accountDir, readRun, runVerdict, writeWorkspaceFile, resolveModel, resolveEffort, assertSafeName, } from "./store.js";
|
|
43
46
|
import { startFlowRun, loadFlow } from "./runner.js";
|
|
44
47
|
import { platform, platformQueues } from "./platform.js";
|
|
45
48
|
const ASSERTION_TYPES = ["contains", "not-contains", "matches", "file", "run", "judge"];
|
|
@@ -85,7 +88,9 @@ function readScalar(lines, start, inline, indent) {
|
|
|
85
88
|
const first = inline.trim().replace(/^["']|["']$/g, "");
|
|
86
89
|
return { value: [first, ...parts].join(" ").trim(), end: i };
|
|
87
90
|
}
|
|
88
|
-
|
|
91
|
+
/** `defaults`: the workspace's `judge:` (AGENTS.md), used where the eval
|
|
92
|
+
* names no model or effort of its own. */
|
|
93
|
+
export function parseEval(file, raw, defaults = {}) {
|
|
89
94
|
const { data, content } = matter(raw);
|
|
90
95
|
const cases = [];
|
|
91
96
|
let current = null;
|
|
@@ -137,8 +142,8 @@ export function parseEval(file, raw) {
|
|
|
137
142
|
file,
|
|
138
143
|
agent: typeof data.agent === "string" ? data.agent : null,
|
|
139
144
|
flow: typeof data.flow === "string" ? data.flow : null,
|
|
140
|
-
model: resolveModel(data.model ?? "fast"),
|
|
141
|
-
effort: resolveEffort(data.effort ?? "low"),
|
|
145
|
+
model: resolveModel(data.model ?? defaults.model ?? "fast"),
|
|
146
|
+
effort: resolveEffort(data.effort ?? defaults.effort ?? "low"),
|
|
142
147
|
trigger: data.trigger === "manual" ? "manual" : "deploy",
|
|
143
148
|
live: data.live === true,
|
|
144
149
|
inputs: data.inputs === true,
|
|
@@ -153,7 +158,17 @@ export function listEvals(tenant, workspace) {
|
|
|
153
158
|
.readdirSync(dir)
|
|
154
159
|
.filter((f) => f.endsWith(".md"))
|
|
155
160
|
.sort()
|
|
156
|
-
.map((f) => parseEval(f, fs.readFileSync(path.join(dir, f), "utf8")));
|
|
161
|
+
.map((f) => parseEval(f, fs.readFileSync(path.join(dir, f), "utf8"), judgeDefaults(tenant, workspace)));
|
|
162
|
+
}
|
|
163
|
+
/** `judge:` from AGENTS.md, field by field: the workspace's model (or
|
|
164
|
+
* effort), else the account's — so a workspace that sets only an effort
|
|
165
|
+
* keeps the account's model (Codex debate: the form says so). */
|
|
166
|
+
export function judgeDefaults(tenant, workspace) {
|
|
167
|
+
const ws = parseJudge(readAgentsMdKeys(workspaceDir(tenant, workspace)).judge);
|
|
168
|
+
const account = parseJudge(readAgentsMdKeys(accountDir(tenant)).judge);
|
|
169
|
+
const model = ws.model ?? account.model;
|
|
170
|
+
const effort = ws.effort ?? account.effort;
|
|
171
|
+
return { ...(model ? { model } : {}), ...(effort ? { effort } : {}) };
|
|
157
172
|
}
|
|
158
173
|
// ---------- promoting a run ----------
|
|
159
174
|
/** An error a route can turn into the right status without knowing HTTP. */
|
|
@@ -365,12 +380,14 @@ function shell(command, cwd) {
|
|
|
365
380
|
child.on("error", (err) => resolve({ code: null, out: String(err) }));
|
|
366
381
|
});
|
|
367
382
|
}
|
|
368
|
-
// The judge gets the rubric and the output, and
|
|
383
|
+
// The judge gets the rubric and the output, and ends with one word.
|
|
369
384
|
// Constrained output beats asking for a score: "PASS" or "FAIL" can't be
|
|
370
|
-
// misread, where "7/10" needs a threshold nobody agreed on.
|
|
385
|
+
// misread, where "7/10" needs a threshold nobody agreed on. The word comes
|
|
386
|
+
// LAST, after the reasoning: asked for it first, the judge committed before
|
|
387
|
+
// it had looked, then argued the other way — "FAIL … which meets the
|
|
388
|
+
// criterion" (vaza evals, 21 Oct 2026).
|
|
371
389
|
async function judge(rubric, output, model, effort, env) {
|
|
372
|
-
const prompt = `You are grading one piece of output against one criterion.
|
|
373
|
-
`PASS or FAIL on the first line, then one short sentence of reason.\n\n` +
|
|
390
|
+
const prompt = `You are grading one piece of output against one criterion. ${JUDGE_VERDICT_INSTRUCTION}\n\n` +
|
|
374
391
|
`Criterion: ${rubric}\n\n<output>\n${output.slice(0, 20000)}\n</output>`;
|
|
375
392
|
let text = "";
|
|
376
393
|
const q = query({
|
|
@@ -396,7 +413,19 @@ async function judge(rubric, output, model, effort, env) {
|
|
|
396
413
|
}
|
|
397
414
|
return text.trim();
|
|
398
415
|
}
|
|
399
|
-
|
|
416
|
+
/**
|
|
417
|
+
* The judge's environment: this process's, with the platform's model
|
|
418
|
+
* credential read live. The process's own CLAUDE_CODE_OAUTH_TOKEN is the copy
|
|
419
|
+
* it was handed at boot; the box refreshes the login every four hours and the
|
|
420
|
+
* old token is then revoked, so a judge on the boot copy failed with "OAuth
|
|
421
|
+
* token revoked" from the first refresh until the pod restarted, while the
|
|
422
|
+
* agent steps (which read the live file) kept working (2026-10-21).
|
|
423
|
+
*/
|
|
424
|
+
export function judgeProcessEnv() {
|
|
425
|
+
const live = Object.fromEntries(Object.entries(platformModelEnv()).filter(([, v]) => v !== undefined));
|
|
426
|
+
return sdkEnv({ ...process.env, ...live });
|
|
427
|
+
}
|
|
428
|
+
async function checkAssertion(assertion, output, agentDir, model, effort, judgeEnv = () => ({ env: judgeProcessEnv() })) {
|
|
400
429
|
const hay = output.toLowerCase();
|
|
401
430
|
switch (assertion.type) {
|
|
402
431
|
case "contains": {
|
|
@@ -438,9 +467,9 @@ async function checkAssertion(assertion, output, agentDir, model, effort, judgeE
|
|
|
438
467
|
catch (err) {
|
|
439
468
|
return { assertion, passed: false, detail: `judge not run — ${err instanceof Error ? err.message : String(err)}` };
|
|
440
469
|
}
|
|
441
|
-
const
|
|
442
|
-
const passed =
|
|
443
|
-
return { assertion, passed, detail
|
|
470
|
+
const text = await withModelEnv(m, (env) => judge(assertion.value, output, model, effort, env));
|
|
471
|
+
const { passed, detail } = judgeVerdict(text);
|
|
472
|
+
return { assertion, passed, detail };
|
|
444
473
|
}
|
|
445
474
|
}
|
|
446
475
|
}
|
|
@@ -509,7 +538,7 @@ export async function runEval(tenant, workspace, info) {
|
|
|
509
538
|
for (const a of cheap)
|
|
510
539
|
assertions.push(await checkAssertion(a, output, agentDir, info.model, info.effort));
|
|
511
540
|
if (assertions.every((r) => r.passed)) {
|
|
512
|
-
const judgeEnv = () => accountModelEnv(tenant, workspace,
|
|
541
|
+
const judgeEnv = () => accountModelEnv(tenant, workspace, judgeProcessEnv);
|
|
513
542
|
for (const a of judges)
|
|
514
543
|
assertions.push(await checkAssertion(a, output, agentDir, info.model, info.effort, judgeEnv));
|
|
515
544
|
}
|
package/dist/src/flow-lint.d.ts
CHANGED
|
@@ -25,4 +25,18 @@ export interface KnownNames {
|
|
|
25
25
|
* rule is skipped rather than guessed at. */
|
|
26
26
|
outwardAgents?: string[];
|
|
27
27
|
}
|
|
28
|
+
/** A `verify:` that is a claim in English rather than a command. Conservative
|
|
29
|
+
* on purpose, the same bargain as REFERS_BACK: it must have no assertion
|
|
30
|
+
* prefix, nothing shell-shaped anywhere in it, and enough words to be a
|
|
31
|
+
* sentence — so a bare `./check.sh` or a one-word command is left alone and
|
|
32
|
+
* the warning only fires where the author plainly wrote prose. */
|
|
33
|
+
/**
|
|
34
|
+
* Why a `matches:` pattern will not compile, or null when it does. The
|
|
35
|
+
* runtime tests it as a JavaScript RegExp with the `i` flag (step-exec.ts,
|
|
36
|
+
* evals.ts), and a pattern that throws fails its check on every run with
|
|
37
|
+
* "invalid regular expression". The usual cause is another dialect: on
|
|
38
|
+
* 2026-10-21 Python's inline `(?m)` broke five eval checks on vaza, and
|
|
39
|
+
* nothing said so before they ran.
|
|
40
|
+
*/
|
|
41
|
+
export declare function regexProblem(pattern: string): string | null;
|
|
28
42
|
export declare function lintFlow(flow: FlowInfo, known?: KnownNames): FlowWarning[];
|
package/dist/src/flow-lint.js
CHANGED
|
@@ -54,6 +54,27 @@ const LOOKS_LIKE_SHELL = /&&|\|\||\s\|\s|\$\(|\$\{|`|\s[<>]\s|\d?>[>&]?\s|^\[|^[
|
|
|
54
54
|
* prefix, nothing shell-shaped anywhere in it, and enough words to be a
|
|
55
55
|
* sentence — so a bare `./check.sh` or a one-word command is left alone and
|
|
56
56
|
* the warning only fires where the author plainly wrote prose. */
|
|
57
|
+
/**
|
|
58
|
+
* Why a `matches:` pattern will not compile, or null when it does. The
|
|
59
|
+
* runtime tests it as a JavaScript RegExp with the `i` flag (step-exec.ts,
|
|
60
|
+
* evals.ts), and a pattern that throws fails its check on every run with
|
|
61
|
+
* "invalid regular expression". The usual cause is another dialect: on
|
|
62
|
+
* 2026-10-21 Python's inline `(?m)` broke five eval checks on vaza, and
|
|
63
|
+
* nothing said so before they ran.
|
|
64
|
+
*/
|
|
65
|
+
export function regexProblem(pattern) {
|
|
66
|
+
try {
|
|
67
|
+
new RegExp(pattern, "i");
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
catch (err) {
|
|
71
|
+
const why = err instanceof Error ? err.message.replace(/^Invalid regular expression: /, "") : String(err);
|
|
72
|
+
const inline = /\(\?[a-z]+\)/.test(pattern)
|
|
73
|
+
? " — JavaScript has no inline flags like (?m); write (^|\\n) for the start of a line"
|
|
74
|
+
: "";
|
|
75
|
+
return `is not a valid JavaScript regular expression (${why})${inline}`;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
57
78
|
function looksLikeProse(verify) {
|
|
58
79
|
const v = verify.trim();
|
|
59
80
|
if (!v || VERIFY_ASSERTION.test(v))
|
|
@@ -377,6 +398,20 @@ export function lintFlow(flow, known) {
|
|
|
377
398
|
"claim, or write it as a shell command that exits 0 when the claim holds.",
|
|
378
399
|
});
|
|
379
400
|
}
|
|
401
|
+
// A `verify: matches:` the runtime cannot compile fails the step on
|
|
402
|
+
// every run, whatever the agent did.
|
|
403
|
+
const verifyRe = step.verify ? /^matches:\s*(.*)$/.exec(step.verify.trim()) : null;
|
|
404
|
+
// The same normalisation checkVerify applies (step-exec.ts): one quote
|
|
405
|
+
// off each end, so the pattern checked is the pattern that runs.
|
|
406
|
+
const reProblem = verifyRe ? regexProblem(verifyRe[1].trim().replace(/^["']|["']$/g, "")) : null;
|
|
407
|
+
if (reProblem) {
|
|
408
|
+
warnings.push({
|
|
409
|
+
step: i,
|
|
410
|
+
line: step.line,
|
|
411
|
+
message: `verify: matches: ${reProblem}`,
|
|
412
|
+
detail: "The runtime tests matches: as a JavaScript regular expression, case-insensitive. One that does not compile fails this step on every run.",
|
|
413
|
+
});
|
|
414
|
+
}
|
|
380
415
|
// A step's OTHER agent references — the ones no parser validates,
|
|
381
416
|
// because they are options rather than the step's target. Both fail
|
|
382
417
|
// silently and at the worst possible moment: a misspelled `delegate:`
|
|
@@ -24,6 +24,7 @@ import { joinGroup, splitToRail } from "./arrange.js";
|
|
|
24
24
|
import { lintFlow } from "./flow-lint.js";
|
|
25
25
|
import { toLF, asRaw } from "./eol.js";
|
|
26
26
|
import { removalImpact, stepLabel } from "./step-removal.js";
|
|
27
|
+
import { keySpan, InlineBlockError } from "./frontmatter-text.js";
|
|
27
28
|
// ---------- flow files ----------
|
|
28
29
|
/** A flow's steps as groups of indices into its parsed steps — what
|
|
29
30
|
* reorderFlowSteps and the arrange helpers take. */
|
|
@@ -437,14 +438,16 @@ function splitComment(value) {
|
|
|
437
438
|
const m = value.match(/^(.*?)(\s+#.*)$/);
|
|
438
439
|
return m ? { value: m[1], comment: m[2] } : { value, comment: "" };
|
|
439
440
|
}
|
|
441
|
+
/** Where a key is written — at the top or inside its job block
|
|
442
|
+
* (frontmatter-text.ts) — with the text after its colon and its indent. */
|
|
440
443
|
function keyLine(lines, close, key) {
|
|
441
|
-
const
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
}
|
|
447
|
-
return null;
|
|
444
|
+
const where = keySpan(lines.slice(1, close), key);
|
|
445
|
+
if (!where)
|
|
446
|
+
return null;
|
|
447
|
+
const index = where.at + 1;
|
|
448
|
+
// Quoted or not, as keySpan matched it.
|
|
449
|
+
const m = lines[index].slice(where.indent.length).match(new RegExp(`^["']?${key.replace(/[.*+?^${}()|[\]\\-]/g, "\\$&")}["']?\\s*:(.*)$`));
|
|
450
|
+
return m ? { index, rest: m[1], indent: where.indent } : null;
|
|
448
451
|
}
|
|
449
452
|
/** Block-style item lines under a key: ` - name`, until anything else. */
|
|
450
453
|
function blockItems(lines, from, close) {
|
|
@@ -467,7 +470,17 @@ export function frontmatterList(raw, key) {
|
|
|
467
470
|
const b = frontBounds(lines);
|
|
468
471
|
if (!b)
|
|
469
472
|
return [];
|
|
470
|
-
|
|
473
|
+
let k;
|
|
474
|
+
try {
|
|
475
|
+
k = keyLine(lines, b.close, key);
|
|
476
|
+
}
|
|
477
|
+
catch (err) {
|
|
478
|
+
// A block these text edits do not handle (frontmatter-text.ts) is the
|
|
479
|
+
// writer's to refuse; a read of it is empty, not an error.
|
|
480
|
+
if (err instanceof InlineBlockError)
|
|
481
|
+
return [];
|
|
482
|
+
throw err;
|
|
483
|
+
}
|
|
471
484
|
if (!k)
|
|
472
485
|
return [];
|
|
473
486
|
const v = splitComment(k.rest).value.trim();
|
|
@@ -491,10 +504,12 @@ export function editFrontmatterList(raw, key, name, action) {
|
|
|
491
504
|
const b = frontBounds(lines);
|
|
492
505
|
if (!b)
|
|
493
506
|
return action === "remove" ? raw : ["---", `${key}: [${name}]`, "---", ...lines].join(eol);
|
|
507
|
+
// Located first, so a key in a block these edits do not handle is refused
|
|
508
|
+
// (InlineBlockError) rather than read as an empty list.
|
|
509
|
+
const k = keyLine(lines, b.close, key);
|
|
494
510
|
const has = frontmatterList(raw, key).includes(name);
|
|
495
511
|
if ((action === "add") === has)
|
|
496
512
|
return raw;
|
|
497
|
-
const k = keyLine(lines, b.close, key);
|
|
498
513
|
if (!k) {
|
|
499
514
|
lines.splice(b.close, 0, `${key}: [${name}]`);
|
|
500
515
|
return lines.join(eol);
|
|
@@ -505,12 +520,12 @@ export function editFrontmatterList(raw, key, name, action) {
|
|
|
505
520
|
// Flow style, or a lone scalar (read as a one-item list): that one line.
|
|
506
521
|
const tokens = v.startsWith("[") ? v.replace(/^\[|\]$/g, "").split(",").map((t) => t.trim()).filter(Boolean) : [v];
|
|
507
522
|
const next = action === "add" ? [...tokens, name] : tokens.filter((t) => itemName(t) !== name);
|
|
508
|
-
lines[k.index] = `${key}: [${next.join(", ")}]${comment}`;
|
|
523
|
+
lines[k.index] = `${k.indent}${key}: [${next.join(", ")}]${comment}`;
|
|
509
524
|
return lines.join(eol);
|
|
510
525
|
}
|
|
511
526
|
const items = blockItems(lines, k.index, b.close);
|
|
512
527
|
if (action === "add") {
|
|
513
|
-
const indent = items.length ? lines[items[0]].match(/^(\s*)-/)[1] :
|
|
528
|
+
const indent = items.length ? lines[items[0]].match(/^(\s*)-/)[1] : `${k.indent} `;
|
|
514
529
|
lines.splice(items.length ? items[items.length - 1] + 1 : k.index + 1, 0, `${indent}- ${name}`);
|
|
515
530
|
return lines.join(eol);
|
|
516
531
|
}
|
|
@@ -518,7 +533,7 @@ export function editFrontmatterList(raw, key, name, action) {
|
|
|
518
533
|
for (const i of [...drop].reverse())
|
|
519
534
|
lines.splice(i, 1);
|
|
520
535
|
if (drop.length === items.length)
|
|
521
|
-
lines[k.index] = `${key}: []${comment}`;
|
|
536
|
+
lines[k.index] = `${k.indent}${key}: []${comment}`;
|
|
522
537
|
return lines.join(eol);
|
|
523
538
|
}
|
|
524
539
|
const YAML_WORDS = /^(true|false|yes|no|on|off|null|~|y|n)$/i;
|
|
@@ -546,10 +561,10 @@ export function setFrontmatterScalar(raw, key, value) {
|
|
|
546
561
|
const { value: v, comment } = splitComment(k.rest);
|
|
547
562
|
let end = k.index + 1;
|
|
548
563
|
if (/^\s*[|>][-+]?\d*\s*$/.test(v) || !v.trim()) {
|
|
549
|
-
while (end < b.close && /^\s
|
|
564
|
+
while (end < b.close && (lines[end].match(/^\s*/)?.[0].length ?? 0) > k.indent.length && /\S/.test(lines[end]))
|
|
550
565
|
end++;
|
|
551
566
|
}
|
|
552
|
-
lines.splice(k.index, end - k.index, `${line}${comment}`);
|
|
567
|
+
lines.splice(k.index, end - k.index, `${k.indent}${line}${comment}`);
|
|
553
568
|
return lines.join(eol);
|
|
554
569
|
}
|
|
555
570
|
const name = keyLine(lines, b.close, "name");
|