@foldrun/core 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -10
- package/dist/src/approvals.d.ts +104 -1
- package/dist/src/approvals.js +192 -2
- package/dist/src/budget.d.ts +32 -0
- package/dist/src/budget.js +177 -0
- package/dist/src/catalog.js +2 -1
- package/dist/src/confine.js +6 -1
- package/dist/src/deploy.js +4 -2
- package/dist/src/egress.d.ts +19 -2
- package/dist/src/egress.js +2 -1
- package/dist/src/evals.d.ts +4 -0
- package/dist/src/evals.js +20 -11
- package/dist/src/flow-lint.d.ts +4 -0
- package/dist/src/flow-lint.js +108 -0
- package/dist/src/host-env.d.ts +8 -0
- package/dist/src/host-env.js +48 -0
- package/dist/src/json-schema.d.ts +12 -0
- package/dist/src/json-schema.js +202 -0
- package/dist/src/library.js +13 -1
- package/dist/src/notify.d.ts +79 -1
- package/dist/src/notify.js +243 -9
- package/dist/src/paths.d.ts +5 -0
- package/dist/src/paths.js +32 -0
- package/dist/src/platform.d.ts +112 -2
- package/dist/src/platform.js +20 -2
- package/dist/src/provider-health.d.ts +55 -0
- package/dist/src/provider-health.js +232 -0
- package/dist/src/run-container.d.ts +80 -4
- package/dist/src/run-container.js +176 -37
- package/dist/src/runner.d.ts +41 -5
- package/dist/src/runner.js +513 -81
- package/dist/src/script-tools.d.ts +12 -0
- package/dist/src/script-tools.js +10 -2
- package/dist/src/secret-health.d.ts +59 -0
- package/dist/src/secret-health.js +155 -0
- package/dist/src/secrets.d.ts +19 -1
- package/dist/src/secrets.js +44 -12
- package/dist/src/starter.js +4 -0
- package/dist/src/step-exec.d.ts +62 -6
- package/dist/src/step-exec.js +199 -68
- package/dist/src/store.d.ts +300 -3
- package/dist/src/store.js +574 -26
- package/dist/src/tar.js +2 -1
- package/dist/src/test-mode.d.ts +111 -0
- package/dist/src/test-mode.js +285 -0
- package/dist/src/tool-test.js +48 -21
- package/dist/src/translator.js +2 -1
- package/dist/src/trigger-log.d.ts +43 -0
- package/dist/src/trigger-log.js +122 -0
- package/dist/src/triggers.d.ts +20 -0
- package/dist/src/triggers.js +91 -10
- package/dist/src/webhook.d.ts +15 -3
- package/dist/src/webhook.js +11 -11
- package/package.json +6 -5
- package/src/approvals.ts +253 -2
- package/src/budget.ts +178 -0
- package/src/catalog.ts +2 -1
- package/src/confine.ts +6 -1
- package/src/deploy.ts +4 -1
- package/src/egress.ts +20 -4
- package/src/evals.ts +25 -9
- package/src/flow-lint.ts +118 -0
- package/src/host-env.ts +51 -0
- package/src/json-schema.ts +184 -0
- package/src/library.ts +13 -1
- package/src/notify.ts +272 -11
- package/src/paths.ts +29 -0
- package/src/platform.ts +119 -3
- package/src/provider-health.ts +274 -0
- package/src/run-container.ts +215 -35
- package/src/runner.ts +525 -73
- package/src/script-tools.ts +18 -2
- package/src/secret-health.ts +185 -0
- package/src/secrets.ts +44 -10
- package/src/starter.ts +4 -0
- package/src/step-exec.ts +176 -21
- package/src/storage.ts +8 -2
- package/src/store.ts +785 -25
- package/src/tar.ts +2 -1
- package/src/test-mode.ts +311 -0
- package/src/tool-test.ts +49 -20
- package/src/translator.ts +2 -1
- package/src/trigger-log.ts +159 -0
- package/src/triggers.ts +92 -10
- package/src/webhook.ts +17 -11
package/src/budget.ts
ADDED
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
// A budget is a number and a period — what may be spent, and over what.
|
|
2
|
+
//
|
|
3
|
+
// One grammar, read in four places: an agent's `budget:` (the most that
|
|
4
|
+
// agent may spend in one run), a flow's (the most one run may spend), a
|
|
5
|
+
// workspace's and the account's (a cap over a window of time). The value
|
|
6
|
+
// was a bare number meaning "per month" in a workspace or account file and
|
|
7
|
+
// "per run" in a flow; it still is, and it can now also say which:
|
|
8
|
+
//
|
|
9
|
+
// budget: 60 a number alone: per run in a flow or an agent,
|
|
10
|
+
// per month in a workspace or the account
|
|
11
|
+
// budget: 60/day or 60/week, 60/month
|
|
12
|
+
// budget: unlimited no cap — the same as leaving the key out
|
|
13
|
+
//
|
|
14
|
+
// Unset means no limit. That is the platform's rule, not an accident of
|
|
15
|
+
// parsing: a customer who has not written a cap has not asked for one, and
|
|
16
|
+
// nothing here invents a default they would then have to find and raise.
|
|
17
|
+
//
|
|
18
|
+
// Windows follow the account's calendar (`timezone:` in AGENTS.md), the
|
|
19
|
+
// same one the scheduler and "today" already use, because a daily cap that
|
|
20
|
+
// resets at 10am Sydney time is not a daily cap anyone meant.
|
|
21
|
+
|
|
22
|
+
export type BudgetPeriod = "run" | "day" | "week" | "month";
|
|
23
|
+
|
|
24
|
+
export interface Budget {
|
|
25
|
+
usd: number;
|
|
26
|
+
period: BudgetPeriod;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** Words that mean "no cap", case-insensitively. */
|
|
30
|
+
const UNLIMITED = new Set(["unlimited", "none", "no limit", "no-limit", "off", "-", "~"]);
|
|
31
|
+
|
|
32
|
+
const PERIODS: Record<string, BudgetPeriod> = {
|
|
33
|
+
run: "run",
|
|
34
|
+
day: "day",
|
|
35
|
+
daily: "day",
|
|
36
|
+
week: "week",
|
|
37
|
+
weekly: "week",
|
|
38
|
+
month: "month",
|
|
39
|
+
monthly: "month",
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
// `\d+(?:\.\d+)?` with an optional separator and one period word: every
|
|
43
|
+
// part has one way to match, so a hostile string cannot make it backtrack.
|
|
44
|
+
const FORM_RE = /^\$?(\d+(?:\.\d+)?)(?:\s*(?:\/|per|a|each)?\s*([a-z]+))?$/;
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* The budget a frontmatter value declares, or null for "no cap".
|
|
48
|
+
*
|
|
49
|
+
* Lenient on purpose: a value nobody can read is treated as no cap rather
|
|
50
|
+
* than as a cap of zero, because refusing every run over a typo is the worse
|
|
51
|
+
* failure. `budgetProblem` is the strict reading, for lint and the editor.
|
|
52
|
+
*/
|
|
53
|
+
export function parseBudget(raw: unknown, defaultPeriod: BudgetPeriod): Budget | null {
|
|
54
|
+
if (raw === undefined || raw === null) return null;
|
|
55
|
+
if (typeof raw === "number") return Number.isFinite(raw) && raw > 0 ? { usd: raw, period: defaultPeriod } : null;
|
|
56
|
+
if (typeof raw !== "string") return null;
|
|
57
|
+
const s = raw.trim().toLowerCase();
|
|
58
|
+
if (!s || UNLIMITED.has(s)) return null;
|
|
59
|
+
const m = FORM_RE.exec(s);
|
|
60
|
+
if (!m) return null;
|
|
61
|
+
const usd = Number(m[1]);
|
|
62
|
+
if (!Number.isFinite(usd) || usd <= 0) return null;
|
|
63
|
+
const period = m[2] ? PERIODS[m[2]] : defaultPeriod;
|
|
64
|
+
if (!period) return null;
|
|
65
|
+
return { usd, period };
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* What is wrong with a `budget:` value, in a sentence, or null when it is
|
|
70
|
+
* fine. `allowed` names the periods this file may use — an agent's or a
|
|
71
|
+
* flow's budget is per run and nothing else; a workspace's or the account's
|
|
72
|
+
* is over time and never per run.
|
|
73
|
+
*/
|
|
74
|
+
export function budgetProblem(raw: unknown, allowed: BudgetPeriod[]): string | null {
|
|
75
|
+
if (raw === undefined || raw === null) return null;
|
|
76
|
+
if (typeof raw === "number") {
|
|
77
|
+
if (!Number.isFinite(raw) || raw < 0) return `budget: ${String(raw)} is not an amount in USD`;
|
|
78
|
+
return null;
|
|
79
|
+
}
|
|
80
|
+
if (typeof raw !== "string") return `budget: must be a number, "<number>/<day|week|month>", or unlimited`;
|
|
81
|
+
const s = raw.trim().toLowerCase();
|
|
82
|
+
if (!s || UNLIMITED.has(s)) return null;
|
|
83
|
+
const m = FORM_RE.exec(s);
|
|
84
|
+
if (!m) return `budget: "${raw}" is not a number, "<number>/<day|week|month>", or unlimited`;
|
|
85
|
+
if (m[2]) {
|
|
86
|
+
const period = PERIODS[m[2]];
|
|
87
|
+
if (!period) return `budget: "${m[2]}" is not a period — day, week or month`;
|
|
88
|
+
if (!allowed.includes(period)) {
|
|
89
|
+
return period === "run"
|
|
90
|
+
? `budget: is per run here already — write the number alone`
|
|
91
|
+
: `budget: "${period}" does not apply here — this cap is per run; a cap over time belongs in the workspace's or the account's AGENTS.md`;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return null;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** "$60/day", "$500/month", "$6/run". */
|
|
98
|
+
export function formatBudget(b: Budget): string {
|
|
99
|
+
const usd = Number.isInteger(b.usd) ? String(b.usd) : b.usd.toFixed(2);
|
|
100
|
+
return `$${usd}/${b.period}`;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** "today", "this week", "this month" — the window a spend is measured in. */
|
|
104
|
+
export function windowName(period: BudgetPeriod): string {
|
|
105
|
+
return period === "day" ? "today" : period === "week" ? "this week" : period === "month" ? "this month" : "this run";
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** What to wait for when the cap is reached. */
|
|
109
|
+
export function windowTurnsHint(period: BudgetPeriod): string {
|
|
110
|
+
return period === "day" ? "wait for tomorrow" : period === "week" ? "wait for next week" : "wait for the new month";
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// ------------------------------------------------------------ the window
|
|
114
|
+
|
|
115
|
+
function partsIn(timeZone: string, at: Date): { y: number; m: number; d: number; h: number; min: number; s: number; weekday: number } {
|
|
116
|
+
const f = new Intl.DateTimeFormat("en-US", {
|
|
117
|
+
timeZone,
|
|
118
|
+
hourCycle: "h23",
|
|
119
|
+
year: "numeric",
|
|
120
|
+
month: "numeric",
|
|
121
|
+
day: "numeric",
|
|
122
|
+
hour: "numeric",
|
|
123
|
+
minute: "numeric",
|
|
124
|
+
second: "numeric",
|
|
125
|
+
weekday: "short",
|
|
126
|
+
});
|
|
127
|
+
const p: Record<string, string> = {};
|
|
128
|
+
for (const { type, value } of f.formatToParts(at)) p[type] = value;
|
|
129
|
+
const weekdays = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
|
|
130
|
+
return {
|
|
131
|
+
y: Number(p.year),
|
|
132
|
+
m: Number(p.month),
|
|
133
|
+
d: Number(p.day),
|
|
134
|
+
h: Number(p.hour),
|
|
135
|
+
min: Number(p.minute),
|
|
136
|
+
s: Number(p.second),
|
|
137
|
+
weekday: weekdays.indexOf(p.weekday),
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** The zone's offset from UTC at an instant, in ms (positive east). */
|
|
142
|
+
function offsetMs(timeZone: string, atMs: number): number {
|
|
143
|
+
const p = partsIn(timeZone, new Date(atMs));
|
|
144
|
+
return Date.UTC(p.y, p.m - 1, p.d, p.h, p.min, p.s) - Math.floor(atMs / 1000) * 1000;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** Midnight at the start of a local calendar date, as a UTC instant. */
|
|
148
|
+
function localMidnightMs(timeZone: string, y: number, m: number, d: number): number {
|
|
149
|
+
const naive = Date.UTC(y, m - 1, d);
|
|
150
|
+
let t = naive - offsetMs(timeZone, naive);
|
|
151
|
+
// Once more from the answer: across a DST change the first guess can be
|
|
152
|
+
// off by the shift, and the second pass lands on the real midnight.
|
|
153
|
+
const again = naive - offsetMs(timeZone, t);
|
|
154
|
+
if (again !== t) t = again;
|
|
155
|
+
return t;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* When the current window began, as a UTC instant — the moment spend is
|
|
160
|
+
* summed from. Today's midnight, Monday's midnight, or the first of the
|
|
161
|
+
* month, all in the given calendar. A per-run budget has no window.
|
|
162
|
+
*/
|
|
163
|
+
export function windowStartMs(period: BudgetPeriod, timeZone: string, now = new Date()): number | null {
|
|
164
|
+
if (period === "run") return null;
|
|
165
|
+
let tz = timeZone;
|
|
166
|
+
try {
|
|
167
|
+
new Intl.DateTimeFormat("en-US", { timeZone: tz });
|
|
168
|
+
} catch {
|
|
169
|
+
tz = "UTC";
|
|
170
|
+
}
|
|
171
|
+
const p = partsIn(tz, now);
|
|
172
|
+
if (period === "day") return localMidnightMs(tz, p.y, p.m, p.d);
|
|
173
|
+
if (period === "month") return localMidnightMs(tz, p.y, p.m, 1);
|
|
174
|
+
// Monday. Walk back on the naive calendar, then take that date's midnight.
|
|
175
|
+
const back = (p.weekday + 6) % 7;
|
|
176
|
+
const monday = new Date(Date.UTC(p.y, p.m - 1, p.d - back));
|
|
177
|
+
return localMidnightMs(tz, monday.getUTCFullYear(), monday.getUTCMonth() + 1, monday.getUTCDate());
|
|
178
|
+
}
|
package/src/catalog.ts
CHANGED
|
@@ -23,6 +23,7 @@ import path from "node:path";
|
|
|
23
23
|
import { dataRoot } from "./paths.ts";
|
|
24
24
|
import { resolveModel, resolveTier, MODEL_TIERS, EFFORT_LEVELS, type Effort, type Tier, type FlowInfo } from "./store.ts";
|
|
25
25
|
import type { FlowWarning } from "./flow-lint.ts";
|
|
26
|
+
import { trimSlashes } from "./paths.ts";
|
|
26
27
|
|
|
27
28
|
export interface CatalogModel {
|
|
28
29
|
id: string;
|
|
@@ -115,7 +116,7 @@ export async function loadCatalog(baseUrl: string): Promise<Catalog | null> {
|
|
|
115
116
|
if (cached && Date.now() - Date.parse(cached.fetchedAt) < TTL_MS) return cached;
|
|
116
117
|
|
|
117
118
|
try {
|
|
118
|
-
const res = await fetch(`${baseUrl
|
|
119
|
+
const res = await fetch(`${trimSlashes(baseUrl)}/v1/models`, {
|
|
119
120
|
signal: AbortSignal.timeout(10_000),
|
|
120
121
|
});
|
|
121
122
|
if (!res.ok) throw new Error(`${res.status}`);
|
package/src/confine.ts
CHANGED
|
@@ -148,7 +148,12 @@ export const isFilesystemTool = (toolName: string) => FS_TOOLS.has(toolName);
|
|
|
148
148
|
*/
|
|
149
149
|
export function suggestPath(raw: string): string {
|
|
150
150
|
const posix = raw.split(path.sep).join("/");
|
|
151
|
-
|
|
151
|
+
// By segments, not by regex: an unanchored `(.+)$` re-scans from every
|
|
152
|
+
// slash, and this runs on a path an agent chose.
|
|
153
|
+
const KNOWN = ["outputs", "storage", "state", "memory", "knowledge"];
|
|
154
|
+
const segs = posix.split("/");
|
|
155
|
+
const at = segs.findIndex((seg, i) => KNOWN.includes(seg) && segs.slice(i + 1).join("/") !== "");
|
|
156
|
+
const m = at === -1 ? null : ([null, segs[at], segs.slice(at + 1).join("/")] as const);
|
|
152
157
|
if (m) {
|
|
153
158
|
const [, dir, rest] = m;
|
|
154
159
|
return dir === "outputs"
|
package/src/deploy.ts
CHANGED
|
@@ -29,6 +29,7 @@ import {
|
|
|
29
29
|
WORKSPACE_DIRS,
|
|
30
30
|
assertSafeName,
|
|
31
31
|
type DeployFile,
|
|
32
|
+
notifyWorkspaceChanged,
|
|
32
33
|
} from "./store.ts";
|
|
33
34
|
import { conformanceIssues } from "./okf.ts";
|
|
34
35
|
import { repoDir } from "./gitrepo.ts";
|
|
@@ -373,6 +374,8 @@ export function writePushNote(tenant: string, workspace: string, note: PushNote)
|
|
|
373
374
|
} catch {
|
|
374
375
|
// The note is a record, not the deploy. Losing it must not fail a push.
|
|
375
376
|
}
|
|
377
|
+
// A push waiting on a run is the scheduler's to apply; tell it there is one.
|
|
378
|
+
notifyWorkspaceChanged(tenant, workspace, "push");
|
|
376
379
|
}
|
|
377
380
|
|
|
378
381
|
/**
|
|
@@ -428,7 +431,7 @@ export function deployWorkspace(
|
|
|
428
431
|
const { preserved } = saveWorkspace(tenant, workspace, files, { commit: opts.commit ?? null, by: opts.by ?? "deploy", message: opts.message });
|
|
429
432
|
const commit = opts.commit ?? null;
|
|
430
433
|
if (commit) writeDeployedCommit(tenant, workspace, commit);
|
|
431
|
-
return { ...plan, applied: true, commit, preserved };
|
|
434
|
+
return { ...plan, applied: true, commit, preserved }; // saveWorkspace told the platform
|
|
432
435
|
}
|
|
433
436
|
|
|
434
437
|
// Which commit a workspace is running.
|
package/src/egress.ts
CHANGED
|
@@ -35,6 +35,9 @@
|
|
|
35
35
|
// platform's (it needs a long-lived process and a Service); the pod side is
|
|
36
36
|
// here in core, because it is what the runner image runs.
|
|
37
37
|
|
|
38
|
+
import { trimSlashes } from "./paths.ts";
|
|
39
|
+
import type { TestEffect } from "./test-mode.ts";
|
|
40
|
+
|
|
38
41
|
/** `${NAME}` — the shape a secret reference takes everywhere in foldrun. */
|
|
39
42
|
export const PLACEHOLDER = /\$\{([A-Z][A-Z0-9_]*)\}/g;
|
|
40
43
|
|
|
@@ -51,7 +54,7 @@ export const MODEL_KEY_NAME = "FOLDRUN_MODEL_KEY";
|
|
|
51
54
|
*/
|
|
52
55
|
export function viaEgress(egress: string | undefined | null, target: string): string {
|
|
53
56
|
if (!egress) return target;
|
|
54
|
-
return `${egress
|
|
57
|
+
return `${trimSlashes(egress)}/${target}`;
|
|
55
58
|
}
|
|
56
59
|
|
|
57
60
|
/** The names referenced by placeholders in a string. */
|
|
@@ -86,6 +89,11 @@ export function hostOf(url: string): string | null {
|
|
|
86
89
|
export interface EgressGrant {
|
|
87
90
|
/** Secret name → the hosts that secret may be substituted for. */
|
|
88
91
|
secrets: Record<string, { value: string; hosts: string[] }>;
|
|
92
|
+
/** Secret name → the vault entry it came from, as secret-health keys it:
|
|
93
|
+
* "account:NAME" or "workspace:<ws>:NAME". Two workspaces each holding an
|
|
94
|
+
* API_KEY are two credentials, and one's refusal must not paint the
|
|
95
|
+
* other red. Absent for a secret whose origin the runner did not know. */
|
|
96
|
+
healthKeys?: Record<string, string>;
|
|
89
97
|
}
|
|
90
98
|
|
|
91
99
|
export interface EgressLease {
|
|
@@ -96,19 +104,27 @@ export interface EgressLease {
|
|
|
96
104
|
* lease URL first), so a store that is not this process's memory has
|
|
97
105
|
* to be told when the grant is complete. Call before every attempt. */
|
|
98
106
|
commit(): Promise<void>;
|
|
99
|
-
/** One line per request, for the run trace. Drained by the runner.
|
|
100
|
-
|
|
107
|
+
/** One line per request, for the run trace. Drained by the runner. A
|
|
108
|
+
* line may carry a test-mode effect beside its text — what the proxy
|
|
109
|
+
* refused or redirected on a test lease — for the run page to list. */
|
|
110
|
+
drainLog(): Promise<EgressLogLine[]>;
|
|
101
111
|
/** The step is over: the lease is void and its values are dropped. */
|
|
102
112
|
release(): Promise<void>;
|
|
103
113
|
}
|
|
104
114
|
|
|
115
|
+
export type EgressLogLine = string | { text: string; effect: TestEffect };
|
|
116
|
+
|
|
105
117
|
/**
|
|
106
118
|
* The seam the platform fills. `null` from `lease` (or no `egress` at all)
|
|
107
119
|
* means "no proxy here" and the runner falls back to materialising, exactly
|
|
108
120
|
* as it always did.
|
|
109
121
|
*/
|
|
110
122
|
export interface EgressHooks {
|
|
111
|
-
|
|
123
|
+
/** `test` — the run is a test run: the proxy applies the outward-write
|
|
124
|
+
* policy in test-mode.ts to every request on this lease. Carried on the
|
|
125
|
+
* lease, not the request, because the pod is the party not trusted to
|
|
126
|
+
* say so. */
|
|
127
|
+
lease(args: { tenant: string; runId: string; grant: EgressGrant; test?: boolean }): Promise<EgressLease | null>;
|
|
112
128
|
}
|
|
113
129
|
|
|
114
130
|
/** Strip a header set of the proxy's own and hop-by-hop headers. */
|
package/src/evals.ts
CHANGED
|
@@ -48,7 +48,7 @@ import {
|
|
|
48
48
|
type Effort,
|
|
49
49
|
} from "./store.ts";
|
|
50
50
|
import { startFlowRun, loadFlow } from "./runner.ts";
|
|
51
|
-
import { platform } from "./platform.ts";
|
|
51
|
+
import { platform, platformQueues } from "./platform.ts";
|
|
52
52
|
import type { FlowStep } from "./store.ts";
|
|
53
53
|
|
|
54
54
|
export type Assertion =
|
|
@@ -80,6 +80,10 @@ export interface EvalInfo {
|
|
|
80
80
|
* `manual`: only when a person presses Run or the CLI asks — for a flow
|
|
81
81
|
* eval that costs a whole run and touches real systems every time. */
|
|
82
82
|
trigger: "deploy" | "manual";
|
|
83
|
+
/** An eval's runs are test runs — nothing outward, state untouched —
|
|
84
|
+
* unless the file says `live: true`. A flow eval that must really send
|
|
85
|
+
* to prove itself opts in; nothing opts in by accident. */
|
|
86
|
+
live: boolean;
|
|
83
87
|
cases: EvalCase[];
|
|
84
88
|
}
|
|
85
89
|
|
|
@@ -187,6 +191,7 @@ export function parseEval(file: string, raw: string): EvalInfo {
|
|
|
187
191
|
model: resolveModel(data.model ?? "fast"),
|
|
188
192
|
effort: resolveEffort(data.effort ?? "low"),
|
|
189
193
|
trigger: data.trigger === "manual" ? "manual" : "deploy",
|
|
194
|
+
live: data.live === true,
|
|
190
195
|
cases,
|
|
191
196
|
};
|
|
192
197
|
}
|
|
@@ -286,7 +291,8 @@ export function promoteRunToEval(
|
|
|
286
291
|
let expect: string[];
|
|
287
292
|
if (opts.expect && opts.expect.length > 0) {
|
|
288
293
|
expect = opts.expect.map((line) => {
|
|
289
|
-
const m = oneLine(line).match(/^([a-z-]+)
|
|
294
|
+
const m = oneLine(line).match(/^([a-z-]+):(.*)$/);
|
|
295
|
+
if (m) m[2] = m[2].trim();
|
|
290
296
|
if (!m || !(ASSERTION_TYPES as readonly string[]).includes(m[1]) || !m[2]) {
|
|
291
297
|
throw refuse(
|
|
292
298
|
`"${line}" is not an assertion — expected one of ${ASSERTION_TYPES.map((t) => `${t}:`).join(", ")} followed by a value`,
|
|
@@ -325,7 +331,7 @@ export function promoteRunToEval(
|
|
|
325
331
|
409,
|
|
326
332
|
);
|
|
327
333
|
}
|
|
328
|
-
content = `${existing.
|
|
334
|
+
content = `${existing.trimEnd()}\n\n${caseText}`;
|
|
329
335
|
} else {
|
|
330
336
|
// A flow eval costs a whole run and touches real systems, so it waits
|
|
331
337
|
// for a person to press Run rather than following every push.
|
|
@@ -389,16 +395,20 @@ export interface EvalResult {
|
|
|
389
395
|
error?: string;
|
|
390
396
|
}
|
|
391
397
|
|
|
392
|
-
/** Wait for a run to finish, with a ceiling so a hung agent can't hang an eval. */
|
|
393
398
|
/**
|
|
394
399
|
* A web replica never drives a run: it has no route to the cluster API and
|
|
395
400
|
* no right to create a pod, by design — so an eval started there (a push,
|
|
396
401
|
* the Run button) queues its run for the worker like any other. From 29 Aug
|
|
397
402
|
* every eval on the split platform died at "pod create failed: connection
|
|
398
403
|
* refused" for exactly this reason. A single process — the CLI, a laptop,
|
|
399
|
-
*
|
|
404
|
+
* no platform registered — still drives it right here, as it always did.
|
|
405
|
+
*
|
|
406
|
+
* The test is whether a platform owns the queue, not which role this
|
|
407
|
+
* process was started as: on a worker the role read "worker", the eval
|
|
408
|
+
* started its run in-process, and the run never passed through the
|
|
409
|
+
* account cap or the lanes the platform enforces on every other run.
|
|
400
410
|
*/
|
|
401
|
-
const runsViaQueue = () =>
|
|
411
|
+
const runsViaQueue = () => platformQueues();
|
|
402
412
|
|
|
403
413
|
async function beginEvalRun(
|
|
404
414
|
tenant: string,
|
|
@@ -407,8 +417,9 @@ async function beginEvalRun(
|
|
|
407
417
|
flowName: string,
|
|
408
418
|
model?: string | null,
|
|
409
419
|
effort?: string | null,
|
|
420
|
+
test = true,
|
|
410
421
|
) {
|
|
411
|
-
if (!runsViaQueue()) return startFlowRun(tenant, workspace, steps, flowName, model, [], effort);
|
|
422
|
+
if (!runsViaQueue()) return startFlowRun(tenant, workspace, steps, flowName, model, [], effort, null, { test });
|
|
412
423
|
// The job carries a model override but no effort one, and driveRun cannot
|
|
413
424
|
// re-read a flow called `eval:<name>`. Put the flow's defaults onto the
|
|
414
425
|
// steps that have none — which is what nearest-wins resolves to anyway.
|
|
@@ -417,7 +428,7 @@ async function beginEvalRun(
|
|
|
417
428
|
...(s.model || !model ? {} : { model }),
|
|
418
429
|
...(s.effort || !effort ? {} : { effort }),
|
|
419
430
|
}));
|
|
420
|
-
return platform.enqueueFlowRun(tenant, workspace, withDefaults, flowName, model ?? null, []);
|
|
431
|
+
return platform.enqueueFlowRun(tenant, workspace, withDefaults, flowName, model ?? null, [], null, { test });
|
|
421
432
|
}
|
|
422
433
|
|
|
423
434
|
// Waits for as long as the run runs. There was a cap here (5 minutes, then
|
|
@@ -558,7 +569,9 @@ export async function runEval(
|
|
|
558
569
|
? { ...s, instruction: `${s.instruction}\n\n<run_task>\n${testCase.task}\n</run_task>` }
|
|
559
570
|
: s,
|
|
560
571
|
);
|
|
561
|
-
|
|
572
|
+
// Test unless the eval OR the flow says live: an eval that runs a
|
|
573
|
+
// flow written `live: true` was written by someone who knew.
|
|
574
|
+
run = await beginEvalRun(tenant, workspace, steps, `eval:${info.name}`, flow.model, flow.effort, !(info.live || flow.live));
|
|
562
575
|
} else {
|
|
563
576
|
if (!target) throw new Error("this eval names neither an agent nor a flow");
|
|
564
577
|
run = await beginEvalRun(
|
|
@@ -566,6 +579,9 @@ export async function runEval(
|
|
|
566
579
|
workspace,
|
|
567
580
|
[{ agent: target, instruction: testCase.task, group: 1, optional: false }],
|
|
568
581
|
`eval:${info.name}`,
|
|
582
|
+
undefined,
|
|
583
|
+
undefined,
|
|
584
|
+
!info.live,
|
|
569
585
|
);
|
|
570
586
|
}
|
|
571
587
|
runId = run.id;
|
package/src/flow-lint.ts
CHANGED
|
@@ -20,6 +20,10 @@ export interface FlowWarning {
|
|
|
20
20
|
line?: number;
|
|
21
21
|
message: string;
|
|
22
22
|
detail: string;
|
|
23
|
+
/** "error" for a value the parser could not read — `timeout: abc` — which
|
|
24
|
+
* `foldrun check` must fail on rather than note. Absent means advisory,
|
|
25
|
+
* which every structural warning in this file is. */
|
|
26
|
+
level?: "error" | "warn";
|
|
23
27
|
}
|
|
24
28
|
|
|
25
29
|
// Anaphora: phrases that only mean something if earlier output exists.
|
|
@@ -35,6 +39,18 @@ export interface KnownNames {
|
|
|
35
39
|
agents: string[];
|
|
36
40
|
}
|
|
37
41
|
|
|
42
|
+
/** One @, something either side, a dot in the domain, no whitespace. Written
|
|
43
|
+
* as string checks rather than a regex: the obvious pattern backtracks
|
|
44
|
+
* polynomially on a domain full of dots, and a lint that can be made slow
|
|
45
|
+
* by a flow file is a lint someone will turn off. */
|
|
46
|
+
function looksLikeAddress(a: string): boolean {
|
|
47
|
+
const at = a.indexOf("@");
|
|
48
|
+
if (at < 1 || at !== a.lastIndexOf("@") || /\s/.test(a)) return false;
|
|
49
|
+
const domain = a.slice(at + 1);
|
|
50
|
+
const dot = domain.indexOf(".");
|
|
51
|
+
return dot > 0 && dot < domain.length - 1;
|
|
52
|
+
}
|
|
53
|
+
|
|
38
54
|
export function lintFlow(flow: FlowInfo, known?: KnownNames): FlowWarning[] {
|
|
39
55
|
const warnings: FlowWarning[] = [];
|
|
40
56
|
const agentNames = known ? new Set(known.agents) : null;
|
|
@@ -47,6 +63,71 @@ export function lintFlow(flow: FlowInfo, known?: KnownNames): FlowWarning[] {
|
|
|
47
63
|
// eleven runs where one was meant. Nothing about the expression looks wrong,
|
|
48
64
|
// which is exactly why it needs saying here rather than being discovered on
|
|
49
65
|
// the bill.
|
|
66
|
+
// An approvers list only one word can satisfy. `approvers: [editors]` is
|
|
67
|
+
// a role that is not `admins` and not an address; mayApprove would then
|
|
68
|
+
// refuse every person, and the gate could only ever be answered by an
|
|
69
|
+
// emailed link. Said here rather than discovered at the gate.
|
|
70
|
+
const odd = (flow.approvers ?? []).filter((a) => a !== "admins" && !looksLikeAddress(a));
|
|
71
|
+
if (odd.length) {
|
|
72
|
+
warnings.push({
|
|
73
|
+
step: null,
|
|
74
|
+
message: `approvers: ${odd.join(", ")} — not an email address, and the only role word is admins`,
|
|
75
|
+
detail: "Each entry is an address that may answer this flow's gates, or the word `admins`. Anything else matches nobody, and a list nobody matches locks every gate.",
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
// A flow with an approvers list and no gate at all is a rule the author
|
|
79
|
+
// believes is in force and which does nothing — the worst kind of safety
|
|
80
|
+
// control.
|
|
81
|
+
if (flow.approvers?.length && !flow.steps.some((s) => s.approve || s.ask)) {
|
|
82
|
+
warnings.push({
|
|
83
|
+
step: null,
|
|
84
|
+
message: "approvers: is set but no step in this flow is a gate",
|
|
85
|
+
detail:
|
|
86
|
+
"Nothing here waits for a person, so the list has no effect. Mark the step that needs " +
|
|
87
|
+
"a second pair of eyes with `!` (or `approve: true`), or drop the approvers: line.",
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
// A deadline on a gate that does not exist reads as a safety net and is not
|
|
91
|
+
// one. Same reasoning, said separately because the fix is different.
|
|
92
|
+
if (flow.approveWithin && !flow.steps.some((s) => s.approve || s.ask)) {
|
|
93
|
+
warnings.push({
|
|
94
|
+
step: null,
|
|
95
|
+
message: "approve_within: is set but no step in this flow is a gate",
|
|
96
|
+
detail: "Nothing waits for a person, so nothing can expire. Mark a step with `!`, or drop the line.",
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
// A throttle no schedule can ever trip: the flow cannot run more often
|
|
100
|
+
// than its cron fires, so a floor below that interval never fires either.
|
|
101
|
+
// Cheap to write, silently pointless, and a reader will believe it.
|
|
102
|
+
if (flow.throttle && flow.trigger === "schedule" && flow.schedule) {
|
|
103
|
+
warnings.push({
|
|
104
|
+
step: null,
|
|
105
|
+
message: `throttle: on a scheduled flow — check it is shorter than the schedule, or it will drop runs you meant to happen`,
|
|
106
|
+
detail:
|
|
107
|
+
"A schedule already decides how often this runs. A throttle longer than the gap between " +
|
|
108
|
+
"fires silently skips them; if that is what you want, say it in the cron instead.",
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
// `idempotency:` only has a delivery to read on a trigger that receives
|
|
112
|
+
// one. On a schedule or a chained flow there is no request and no field,
|
|
113
|
+
// so the key is inert.
|
|
114
|
+
if (flow.idempotency && flow.trigger !== "webhook" && flow.trigger !== "email") {
|
|
115
|
+
warnings.push({
|
|
116
|
+
step: null,
|
|
117
|
+
message: `idempotency: has nothing to read on trigger: ${flow.trigger}`,
|
|
118
|
+
detail:
|
|
119
|
+
"A delivery id comes off an incoming request, so this only does something for " +
|
|
120
|
+
"trigger: webhook or trigger: email. On any other trigger it is ignored.",
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
// `catchup:` is about a fire that was missed, which only a clock can miss.
|
|
124
|
+
if (flow.catchup && flow.trigger !== "schedule") {
|
|
125
|
+
warnings.push({
|
|
126
|
+
step: null,
|
|
127
|
+
message: `catchup: has no effect on trigger: ${flow.trigger}`,
|
|
128
|
+
detail: "Only a schedule can miss a fire. Every other trigger happens when it happens.",
|
|
129
|
+
});
|
|
130
|
+
}
|
|
50
131
|
if (flow.schedule) {
|
|
51
132
|
const [, , dom, , dow] = flow.schedule.trim().split(/\s+/);
|
|
52
133
|
if (dom && dow && dom !== "*" && dow !== "*") {
|
|
@@ -63,6 +144,9 @@ export function lintFlow(flow: FlowInfo, known?: KnownNames): FlowWarning[] {
|
|
|
63
144
|
|
|
64
145
|
// Trigger shapes that can never fire, said at check time rather than
|
|
65
146
|
// discovered as a flow that "never runs".
|
|
147
|
+
if (flow.budgetProblem) {
|
|
148
|
+
warnings.push({ step: null, message: flow.budgetProblem, detail: "A flow's `budget:` is the most one run may spend — a number in USD, or `unlimited`. A cap over a day, week or month belongs in the workspace's or the account's AGENTS.md, where it means every run put together." });
|
|
149
|
+
}
|
|
66
150
|
if (flow.trigger === "once" && !flow.at) {
|
|
67
151
|
warnings.push({ step: null, message: "trigger: once needs an `at:` instant", detail: "Write `at: 2026-09-05T09:00:00+10:00` (ISO 8601). Without a readable instant this flow never fires." });
|
|
68
152
|
}
|
|
@@ -87,6 +171,22 @@ export function lintFlow(flow: FlowInfo, known?: KnownNames): FlowWarning[] {
|
|
|
87
171
|
flow.steps.forEach((step, i) => {
|
|
88
172
|
const inFirstGroup = step.group === firstGroup;
|
|
89
173
|
|
|
174
|
+
// A value the parser refused. Not advisory: the run would go ahead on
|
|
175
|
+
// a number the author never wrote — `timeout: 15m` ran as one second
|
|
176
|
+
// for as long as this was a coercion — so the check fails instead.
|
|
177
|
+
for (const problem of step.problems ?? []) {
|
|
178
|
+
warnings.push({
|
|
179
|
+
step: i,
|
|
180
|
+
line: step.line,
|
|
181
|
+
level: "error",
|
|
182
|
+
message: problem,
|
|
183
|
+
detail:
|
|
184
|
+
"The option is written but its value cannot be read, and a run would not do what the " +
|
|
185
|
+
"line says. Durations are seconds or 90s / 30m / 4h / 3d; counts are whole numbers " +
|
|
186
|
+
"within the key's cap (retry and loop at most 5, max at most 20).",
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
|
|
90
190
|
// The first group starts with context = null. An instruction that points
|
|
91
191
|
// at earlier output therefore points at nothing.
|
|
92
192
|
if (inFirstGroup && step.instruction && REFERS_BACK.test(step.instruction)) {
|
|
@@ -133,6 +233,24 @@ export function lintFlow(flow: FlowInfo, known?: KnownNames): FlowWarning[] {
|
|
|
133
233
|
}
|
|
134
234
|
}
|
|
135
235
|
|
|
236
|
+
// Two options that read another option to mean anything.
|
|
237
|
+
if (step.parallel && !step.each) {
|
|
238
|
+
warnings.push({
|
|
239
|
+
step: i,
|
|
240
|
+
line: step.line,
|
|
241
|
+
message: `"${step.subflow ?? step.agent}" has parallel: but no each:`,
|
|
242
|
+
detail: "parallel: caps how many instances of a fan-out run at once. Without each: there is one instance, and the line does nothing.",
|
|
243
|
+
});
|
|
244
|
+
}
|
|
245
|
+
if ((step.schema !== undefined || step.schemaPath) && step.output !== "json") {
|
|
246
|
+
warnings.push({
|
|
247
|
+
step: i,
|
|
248
|
+
line: step.line,
|
|
249
|
+
message: `"${step.subflow ?? step.agent}" has schema: but no output: json`,
|
|
250
|
+
detail: "schema: describes the JSON value an output: json step returns. Without output: json nothing is parsed, so nothing is checked. Add `output: json`.",
|
|
251
|
+
});
|
|
252
|
+
}
|
|
253
|
+
|
|
136
254
|
// A step's OTHER agent references — the ones no parser validates,
|
|
137
255
|
// because they are options rather than the step's target. Both fail
|
|
138
256
|
// silently and at the worst possible moment: a misspelled `delegate:`
|
package/src/host-env.ts
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
// The environment a step's children start from, on the in-process path.
|
|
2
|
+
//
|
|
3
|
+
// Scripts, `verify:` shells and the SDK's own Bash tool used to inherit the
|
|
4
|
+
// whole of process.env — which on a platform holds FOLDRUN_SECRET_KEY (the
|
|
5
|
+
// key every tenant's vault and hook token derives from), the database URL,
|
|
6
|
+
// the object-store and mail credentials, the Stripe key. `console.log(
|
|
7
|
+
// process.env)` in a script tool was cross-tenant compromise. tool-test.ts
|
|
8
|
+
// built a stripped base for the Test button; this is that base, made the one
|
|
9
|
+
// definition, used everywhere a child is spawned host-side.
|
|
10
|
+
//
|
|
11
|
+
// A run container needs none of this: its process.env IS the boundary,
|
|
12
|
+
// assembled from an allowlist on the host before the container started.
|
|
13
|
+
//
|
|
14
|
+
// What passes is what an interpreter needs to be found and to run, the
|
|
15
|
+
// locale, the clock, the network's proxy settings, and the identifiers the
|
|
16
|
+
// runner stamps on a step (FOLDRUN_RUN_*, FOLDRUN_STEP_*) so a script can
|
|
17
|
+
// say which run made its records. The secrets an agent declares are layered
|
|
18
|
+
// on top by the caller — by name, from the vault, never from the host.
|
|
19
|
+
|
|
20
|
+
const EXACT = new Set([
|
|
21
|
+
"PATH", "HOME", "TMPDIR", "TMP", "TEMP", "LANG", "LANGUAGE",
|
|
22
|
+
"TZ", "TERM", "SHELL", "USER", "LOGNAME",
|
|
23
|
+
// The host's way to the network is the child's way to the network.
|
|
24
|
+
"HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy",
|
|
25
|
+
"SSL_CERT_FILE", "SSL_CERT_DIR",
|
|
26
|
+
// Windows equivalents, harmless elsewhere.
|
|
27
|
+
"SYSTEMROOT", "COMSPEC", "PATHEXT", "WINDIR",
|
|
28
|
+
]);
|
|
29
|
+
|
|
30
|
+
const PREFIXES = ["LC_", "NODE_", "FOLDRUN_RUN_", "FOLDRUN_STEP_"];
|
|
31
|
+
|
|
32
|
+
/** Does this name pass the allowlist? Exported so a test can pin the rule
|
|
33
|
+
* rather than the list. */
|
|
34
|
+
export function hostEnvAllowed(name: string): boolean {
|
|
35
|
+
return EXACT.has(name) || PREFIXES.some((p) => name.startsWith(p));
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* process.env, reduced to the allowlist. A fresh object every call, so a
|
|
40
|
+
* caller that layers secrets on top never writes into the host's own env.
|
|
41
|
+
*/
|
|
42
|
+
export function hostSafeEnv(from: NodeJS.ProcessEnv = process.env): NodeJS.ProcessEnv {
|
|
43
|
+
const out: Record<string, string> = {};
|
|
44
|
+
for (const [k, v] of Object.entries(from)) {
|
|
45
|
+
if (typeof v === "string" && hostEnvAllowed(k)) out[k] = v;
|
|
46
|
+
}
|
|
47
|
+
// Typed as ProcessEnv, not a string map: a host whose type augmentation
|
|
48
|
+
// makes a key required (Next declares NODE_ENV so) refuses a plain map at
|
|
49
|
+
// every spawn. The value is the same either way.
|
|
50
|
+
return out as NodeJS.ProcessEnv;
|
|
51
|
+
}
|