@foldrun/core 2.10.1 → 2.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +3 -0
- package/dist/index.js +3 -0
- package/dist/src/catalog.d.ts +3 -0
- package/dist/src/catalog.js +4 -1
- package/dist/src/deploy.js +16 -0
- package/dist/src/eval-compare.d.ts +74 -0
- package/dist/src/eval-compare.js +187 -0
- package/dist/src/evals.d.ts +62 -3
- package/dist/src/evals.js +161 -36
- package/dist/src/flow-lint.js +4 -0
- package/dist/src/frontmatter-fields.js +4 -1
- package/dist/src/git-hook.d.ts +1 -0
- package/dist/src/git-hook.js +70 -0
- package/dist/src/git.d.ts +37 -0
- package/dist/src/git.js +101 -0
- package/dist/src/gitrepo.d.ts +113 -0
- package/dist/src/gitrepo.js +404 -0
- package/dist/src/grades.d.ts +118 -0
- package/dist/src/grades.js +481 -0
- package/dist/src/judge-verdict.d.ts +1 -0
- package/dist/src/judge-verdict.js +3 -1
- package/dist/src/online-checks.d.ts +4 -0
- package/dist/src/online-checks.js +5 -1
- package/dist/src/openapi.d.ts +4 -0
- package/dist/src/openapi.js +5 -1
- package/dist/src/platform.d.ts +12 -3
- package/dist/src/platform.js +9 -1
- package/dist/src/starter.js +5 -0
- package/dist/src/store.d.ts +8 -0
- package/dist/src/store.js +5 -1
- package/dist/src/tar.d.ts +30 -0
- package/dist/src/tar.js +112 -0
- package/dist/src/tool-test.js +19 -7
- package/dist/src/translator.d.ts +4 -0
- package/dist/src/translator.js +22 -3
- package/dist/src/trigger-filter.d.ts +28 -0
- package/dist/src/trigger-filter.js +272 -0
- package/dist/src/trigger-log.d.ts +1 -1
- package/dist/src/triggers.d.ts +2 -0
- package/dist/src/triggers.js +13 -2
- package/dist/src/untrusted-fetch.d.ts +16 -0
- package/dist/src/untrusted-fetch.js +60 -2
- package/index.ts +3 -0
- package/package.json +2 -2
- package/src/catalog.ts +5 -1
- package/src/deploy.ts +10 -0
- package/src/eval-compare.ts +243 -0
- package/src/evals.ts +195 -38
- package/src/flow-lint.ts +4 -0
- package/src/frontmatter-fields.ts +4 -1
- package/src/grades.ts +532 -0
- package/src/judge-verdict.ts +4 -2
- package/src/online-checks.ts +5 -1
- package/src/openapi.ts +6 -1
- package/src/platform.ts +16 -4
- package/src/starter.ts +5 -0
- package/src/store.ts +11 -1
- package/src/tool-test.ts +21 -7
- package/src/translator.ts +24 -3
- package/src/trigger-filter.ts +264 -0
- package/src/trigger-log.ts +1 -0
- package/src/triggers.ts +14 -2
- package/src/untrusted-fetch.ts +65 -2
package/dist/src/evals.js
CHANGED
|
@@ -34,6 +34,7 @@ import { accountModelEnv, withModelEnv } from "./model-env.js";
|
|
|
34
34
|
import { sdkEnv } from "./step-exec.js";
|
|
35
35
|
import { platformModelEnv } from "./runner.js";
|
|
36
36
|
import fs from "node:fs";
|
|
37
|
+
import crypto from "node:crypto";
|
|
37
38
|
import path from "node:path";
|
|
38
39
|
import { deployedCommit } from "./deploy.js";
|
|
39
40
|
import { latestRevisionId } from "./history.js";
|
|
@@ -137,6 +138,45 @@ export function parseEval(file, raw, defaults = {}) {
|
|
|
137
138
|
}
|
|
138
139
|
if (current)
|
|
139
140
|
cases.push(current);
|
|
141
|
+
const trigger = data.trigger === "manual" ? "manual" : data.trigger === "runs" ? "runs" : "deploy";
|
|
142
|
+
const problems = [];
|
|
143
|
+
const sample = parseSample(data.sample, problems);
|
|
144
|
+
const improve = parseImprove(data.improve, problems);
|
|
145
|
+
if (trigger !== "runs") {
|
|
146
|
+
if (data.sample !== undefined)
|
|
147
|
+
problems.push("sample: only applies to an eval with trigger: runs — this one runs its cases");
|
|
148
|
+
if (data.improve !== undefined)
|
|
149
|
+
problems.push("improve: only applies to an eval with trigger: runs");
|
|
150
|
+
}
|
|
151
|
+
else {
|
|
152
|
+
// A grader reads what a run said; it never starts one, so a task has
|
|
153
|
+
// nothing to go to, and file:/run: would execute or read on the
|
|
154
|
+
// platform's host rather than in the run's sandbox.
|
|
155
|
+
for (const c of cases) {
|
|
156
|
+
if (c.task)
|
|
157
|
+
problems.push(`## ${c.name}: task: does nothing in a trigger: runs eval — it grades real runs, so each check is only an expect: list`);
|
|
158
|
+
for (const a of c.expect) {
|
|
159
|
+
if (a.type === "file" || a.type === "run")
|
|
160
|
+
problems.push(`## ${c.name}: ${a.type}: cannot grade a finished run — use contains, not-contains or judge`);
|
|
161
|
+
// Graded on the platform's scheduler, over whatever a run said: a
|
|
162
|
+
// regular expression that backtracks would hold the scheduler with it.
|
|
163
|
+
if (a.type === "matches")
|
|
164
|
+
problems.push(`## ${c.name}: matches: is not used to grade live runs — a regular expression over any run's output can stall the platform; use contains, not-contains or judge`);
|
|
165
|
+
}
|
|
166
|
+
if (c.expect.length === 0)
|
|
167
|
+
problems.push(`## ${c.name}: has no expect: lines, so it grades nothing`);
|
|
168
|
+
}
|
|
169
|
+
if (cases.length === 0)
|
|
170
|
+
problems.push("a grader needs at least one check — a `## <name>` section with an expect: list; with none it would pass every run");
|
|
171
|
+
// The pass budgets graded runs; this bounds the judges each one costs.
|
|
172
|
+
const judges = cases.reduce((n, c) => n + c.expect.filter((x) => x.type === "judge").length, 0);
|
|
173
|
+
if (judges > MAX_GRADER_JUDGES)
|
|
174
|
+
problems.push(`${judges} judge: lines — a grader asks at most ${MAX_GRADER_JUDGES} judges per run; fold related claims into one`);
|
|
175
|
+
if (data.live === true)
|
|
176
|
+
problems.push("live: does nothing in a trigger: runs eval — it never starts a run");
|
|
177
|
+
if (data.inputs === true)
|
|
178
|
+
problems.push("inputs: and trigger: runs cannot go together");
|
|
179
|
+
}
|
|
140
180
|
return {
|
|
141
181
|
name: data.name ?? file.replace(/\.md$/, ""),
|
|
142
182
|
file,
|
|
@@ -144,12 +184,41 @@ export function parseEval(file, raw, defaults = {}) {
|
|
|
144
184
|
flow: typeof data.flow === "string" ? data.flow : null,
|
|
145
185
|
model: resolveModel(data.model ?? defaults.model ?? "fast"),
|
|
146
186
|
effort: resolveEffort(data.effort ?? defaults.effort ?? "low"),
|
|
147
|
-
trigger
|
|
187
|
+
trigger,
|
|
188
|
+
sample,
|
|
189
|
+
improve: trigger === "runs" ? improve : null,
|
|
190
|
+
problems,
|
|
148
191
|
live: data.live === true,
|
|
149
192
|
inputs: data.inputs === true,
|
|
150
193
|
cases,
|
|
151
194
|
};
|
|
152
195
|
}
|
|
196
|
+
/** Judges one graded run may cost — the grade pass counts runs, not calls. */
|
|
197
|
+
export const MAX_GRADER_JUDGES = 5;
|
|
198
|
+
/** `sample: 25`, `sample: "25%"` — percent of finished runs graded. */
|
|
199
|
+
function parseSample(v, problems) {
|
|
200
|
+
if (v === undefined || v === null)
|
|
201
|
+
return 25;
|
|
202
|
+
const n = typeof v === "number" ? v : typeof v === "string" ? Number(v.trim().replace(/%$/, "")) : NaN;
|
|
203
|
+
if (!Number.isFinite(n) || n < 0 || n > 100) {
|
|
204
|
+
problems.push(`sample: ${String(v)} — a percent from 0 to 100 (25 grades about one run in four; 0 grades none on its own)`);
|
|
205
|
+
return 25;
|
|
206
|
+
}
|
|
207
|
+
return Math.round(n * 100) / 100;
|
|
208
|
+
}
|
|
209
|
+
/** `improve: true` (5), `improve: 3`, `improve: false`. */
|
|
210
|
+
function parseImprove(v, problems) {
|
|
211
|
+
if (v === undefined || v === null || v === false)
|
|
212
|
+
return null;
|
|
213
|
+
if (v === true)
|
|
214
|
+
return 5;
|
|
215
|
+
const n = typeof v === "number" ? v : NaN;
|
|
216
|
+
if (!Number.isInteger(n) || n < 1 || n > 50) {
|
|
217
|
+
problems.push(`improve: ${String(v)} — true, or how many failed grades open a proposal (1 to 50)`);
|
|
218
|
+
return null;
|
|
219
|
+
}
|
|
220
|
+
return n;
|
|
221
|
+
}
|
|
153
222
|
export function listEvals(tenant, workspace) {
|
|
154
223
|
const dir = path.join(workspaceDir(tenant, workspace), "evals");
|
|
155
224
|
if (!fs.existsSync(dir))
|
|
@@ -158,7 +227,13 @@ export function listEvals(tenant, workspace) {
|
|
|
158
227
|
.readdirSync(dir)
|
|
159
228
|
.filter((f) => f.endsWith(".md"))
|
|
160
229
|
.sort()
|
|
161
|
-
.map((f) =>
|
|
230
|
+
.map((f) => {
|
|
231
|
+
const raw = fs.readFileSync(path.join(dir, f), "utf8");
|
|
232
|
+
// The version is taken from the same read the eval is parsed from, so
|
|
233
|
+
// a grade names the rubric that produced it even if the file changes
|
|
234
|
+
// while its judges answer.
|
|
235
|
+
return { ...parseEval(f, raw, judgeDefaults(tenant, workspace)), revision: crypto.createHash("sha256").update(raw).digest("hex").slice(0, 12) };
|
|
236
|
+
});
|
|
162
237
|
}
|
|
163
238
|
/** `judge:` from AGENTS.md, field by field: the workspace's model (or
|
|
164
239
|
* effort), else the account's — so a workspace that sets only an effort
|
|
@@ -386,32 +461,59 @@ function shell(command, cwd) {
|
|
|
386
461
|
// LAST, after the reasoning: asked for it first, the judge committed before
|
|
387
462
|
// it had looked, then argued the other way — "FAIL … which meets the
|
|
388
463
|
// criterion" (vaza evals, 21 Oct 2026).
|
|
389
|
-
|
|
464
|
+
/** How long one judge may take before it is cut off: a verdict on one
|
|
465
|
+
* sentence is seconds; a stalled model must not hold a grade pass. */
|
|
466
|
+
export const JUDGE_TIMEOUT_MS = 120_000;
|
|
467
|
+
/** One judge call: its words and what it cost (the SDK's result line). */
|
|
468
|
+
export async function judgeCall(rubric, output, model, effort, env,
|
|
469
|
+
/** What the output was an answer to — a live run's task, so the judge
|
|
470
|
+
* grades the reply against what was asked rather than in a vacuum. */
|
|
471
|
+
context) {
|
|
390
472
|
const prompt = `You are grading one piece of output against one criterion. ${JUDGE_VERDICT_INSTRUCTION}\n\n` +
|
|
391
|
-
`Criterion: ${rubric}\n\n
|
|
473
|
+
`Criterion: ${rubric}\n\n` +
|
|
474
|
+
(context ? `<task>\n${context.slice(0, 4000)}\n</task>\n\n` : "") +
|
|
475
|
+
`<output>\n${output.slice(0, 20000)}\n</output>`;
|
|
392
476
|
let text = "";
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
477
|
+
let costUsd = 0;
|
|
478
|
+
const abort = new AbortController();
|
|
479
|
+
const timer = setTimeout(() => abort.abort(), JUDGE_TIMEOUT_MS);
|
|
480
|
+
timer.unref?.();
|
|
481
|
+
try {
|
|
482
|
+
const q = query({
|
|
483
|
+
prompt,
|
|
484
|
+
options: {
|
|
485
|
+
model,
|
|
486
|
+
...(effort ? { effort } : {}),
|
|
487
|
+
tools: [],
|
|
488
|
+
allowedTools: [],
|
|
489
|
+
settingSources: [],
|
|
490
|
+
strictMcpConfig: true,
|
|
491
|
+
env,
|
|
492
|
+
maxTurns: 1,
|
|
493
|
+
abortController: abort,
|
|
494
|
+
},
|
|
495
|
+
});
|
|
496
|
+
for await (const message of q) {
|
|
497
|
+
if (message.type === "assistant") {
|
|
498
|
+
for (const block of message.message.content) {
|
|
499
|
+
if (block.type === "text")
|
|
500
|
+
text += block.text;
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
else if (message.type === "result") {
|
|
504
|
+
costUsd = Number(message.total_cost_usd) || 0;
|
|
411
505
|
}
|
|
412
506
|
}
|
|
413
507
|
}
|
|
414
|
-
|
|
508
|
+
catch (err) {
|
|
509
|
+
if (abort.signal.aborted)
|
|
510
|
+
throw new Error(`the judge took longer than ${JUDGE_TIMEOUT_MS / 1000}s and was stopped`);
|
|
511
|
+
throw err;
|
|
512
|
+
}
|
|
513
|
+
finally {
|
|
514
|
+
clearTimeout(timer);
|
|
515
|
+
}
|
|
516
|
+
return { text: text.trim(), costUsd };
|
|
415
517
|
}
|
|
416
518
|
/**
|
|
417
519
|
* The judge's environment: this process's, with the platform's model
|
|
@@ -425,7 +527,7 @@ export function judgeProcessEnv() {
|
|
|
425
527
|
const live = Object.fromEntries(Object.entries(platformModelEnv()).filter(([, v]) => v !== undefined));
|
|
426
528
|
return sdkEnv({ ...process.env, ...live });
|
|
427
529
|
}
|
|
428
|
-
async function checkAssertion(assertion, output, agentDir, model, effort, judgeEnv = () => ({ env: judgeProcessEnv() })) {
|
|
530
|
+
export async function checkAssertion(assertion, output, agentDir, model, effort, judgeEnv = () => ({ env: judgeProcessEnv() })) {
|
|
429
531
|
const hay = output.toLowerCase();
|
|
430
532
|
switch (assertion.type) {
|
|
431
533
|
case "contains": {
|
|
@@ -467,13 +569,16 @@ async function checkAssertion(assertion, output, agentDir, model, effort, judgeE
|
|
|
467
569
|
catch (err) {
|
|
468
570
|
return { assertion, passed: false, detail: `judge not run — ${err instanceof Error ? err.message : String(err)}` };
|
|
469
571
|
}
|
|
470
|
-
const
|
|
471
|
-
const { passed, detail } = judgeVerdict(text);
|
|
472
|
-
return { assertion, passed, detail };
|
|
572
|
+
const said = await withModelEnv(m, (env) => judgeCall(assertion.value, output, model, effort, env));
|
|
573
|
+
const { passed, detail } = judgeVerdict(said.text);
|
|
574
|
+
return { assertion, passed, detail, costUsd: said.costUsd };
|
|
473
575
|
}
|
|
474
576
|
}
|
|
475
577
|
}
|
|
476
|
-
export async function runEval(tenant, workspace, info) {
|
|
578
|
+
export async function runEval(tenant, workspace, info, opts = {}) {
|
|
579
|
+
if (info.trigger === "runs") {
|
|
580
|
+
throw refuse(`${info.name} grades real runs (trigger: runs) — it has no cases to run; grade a run with POST …/evals/${info.name}/grade { run }`, 400);
|
|
581
|
+
}
|
|
477
582
|
const startedAt = new Date().toISOString();
|
|
478
583
|
const results = [];
|
|
479
584
|
// Saved inputs carry no assertions: running them would start and bill a
|
|
@@ -485,6 +590,16 @@ export async function runEval(tenant, workspace, info) {
|
|
|
485
590
|
let output = "";
|
|
486
591
|
let error = null;
|
|
487
592
|
let costUsd = 0;
|
|
593
|
+
let secs = null;
|
|
594
|
+
const stop = opts.beforeCase?.() ?? null;
|
|
595
|
+
if (stop) {
|
|
596
|
+
results.push({ name: testCase.name, task: testCase.task, runId: null, error: `not run — ${stop}`, passed: false, assertions: [], costUsd: 0, judgeCostUsd: 0, secs: null });
|
|
597
|
+
continue;
|
|
598
|
+
}
|
|
599
|
+
// A step forced onto the comparison's model: step-level, so it wins.
|
|
600
|
+
const force = (st) => opts.model && (!opts.agent || st.agent === opts.agent)
|
|
601
|
+
? { ...st, model: opts.model, ...(opts.effort ? { effort: opts.effort } : {}) }
|
|
602
|
+
: st;
|
|
488
603
|
try {
|
|
489
604
|
let run;
|
|
490
605
|
if (info.flow) {
|
|
@@ -492,27 +607,33 @@ export async function runEval(tenant, workspace, info) {
|
|
|
492
607
|
if (!flow || flow.steps.length === 0)
|
|
493
608
|
throw new Error(`flow ${info.flow} not found or empty`);
|
|
494
609
|
// The case's task reaches the first step; output passing carries it on.
|
|
495
|
-
const steps = flow.steps.map((s, i) => i === 0
|
|
610
|
+
const steps = flow.steps.map((s, i) => force(i === 0
|
|
496
611
|
? { ...s, instruction: `${s.instruction}\n\n<run_task>\n${testCase.task}\n</run_task>` }
|
|
497
|
-
: s);
|
|
612
|
+
: s));
|
|
498
613
|
// Test unless the eval OR the flow says live: an eval that runs a
|
|
499
|
-
// flow written `live: true` was written by someone who knew.
|
|
500
|
-
|
|
614
|
+
// flow written `live: true` was written by someone who knew. A
|
|
615
|
+
// comparison is a test whatever they say.
|
|
616
|
+
run = await beginEvalRun(tenant, workspace, steps, `eval:${info.name}`, flow.model, flow.effort, opts.forceTest ? true : !(info.live || flow.live));
|
|
501
617
|
}
|
|
502
618
|
else {
|
|
503
619
|
if (!target)
|
|
504
620
|
throw new Error("this eval names neither an agent nor a flow");
|
|
505
|
-
run = await beginEvalRun(tenant, workspace, [{ agent: target, instruction: testCase.task, group: 1, optional: false }], `eval:${info.name}`, undefined, undefined, !info.live);
|
|
621
|
+
run = await beginEvalRun(tenant, workspace, [force({ agent: target, instruction: testCase.task, group: 1, optional: false })], `eval:${info.name}`, undefined, undefined, opts.forceTest ? true : !info.live);
|
|
506
622
|
}
|
|
507
623
|
runId = run.id;
|
|
508
624
|
const finished = await waitForRun(tenant, workspace, run.id);
|
|
509
625
|
if (!finished)
|
|
510
626
|
throw new Error("run disappeared");
|
|
627
|
+
// What the steps before a gate spent is spent: counted before the
|
|
628
|
+
// gate is refused, so a comparison's limit sees it.
|
|
629
|
+
costUsd = finished.steps.reduce((sum, s) => sum + (s.costUsd ?? 0), 0);
|
|
511
630
|
if (finished.status === "awaiting-approval") {
|
|
512
631
|
throw new Error("this run pauses for approval — an eval cannot answer a human gate");
|
|
513
632
|
}
|
|
514
633
|
output = finished.steps.map((s) => s.result ?? "").join("\n\n");
|
|
515
|
-
|
|
634
|
+
const from = Date.parse(finished.drivenAt ?? finished.startedAt);
|
|
635
|
+
const to = finished.finishedAt ? Date.parse(finished.finishedAt) : NaN;
|
|
636
|
+
secs = Number.isFinite(from) && Number.isFinite(to) ? Math.max(0, Math.round((to - from) / 100) / 10) : null;
|
|
516
637
|
if (finished.status === "failed" && !output.trim()) {
|
|
517
638
|
// Quote the run's own last error. "Produced nothing" on a fresh
|
|
518
639
|
// workspace is almost always "secret X is not set" or a tool that
|
|
@@ -556,7 +677,10 @@ export async function runEval(tenant, workspace, info) {
|
|
|
556
677
|
passed: !error && assertions.length > 0 && assertions.every((r) => r.passed),
|
|
557
678
|
assertions,
|
|
558
679
|
costUsd,
|
|
680
|
+
judgeCostUsd: Math.round(assertions.reduce((n, a) => n + (a.costUsd ?? 0), 0) * 1e6) / 1e6,
|
|
681
|
+
secs,
|
|
559
682
|
});
|
|
683
|
+
opts.afterCase?.(results[results.length - 1]);
|
|
560
684
|
}
|
|
561
685
|
return {
|
|
562
686
|
eval: info.name,
|
|
@@ -661,8 +785,9 @@ commit = null) {
|
|
|
661
785
|
if (!stamp)
|
|
662
786
|
return { ran: 0, skipped: "no revision to stamp the result with" };
|
|
663
787
|
// Only the evals that asked to run on deploy — a `trigger: manual` eval is
|
|
664
|
-
// one a person runs on purpose, because it costs a whole flow run
|
|
665
|
-
|
|
788
|
+
// one a person runs on purpose, because it costs a whole flow run, and a
|
|
789
|
+
// `trigger: runs` one grades real runs and has no cases to run.
|
|
790
|
+
const evals = listEvals(tenant, workspace).filter((e) => e.trigger === "deploy" && !e.inputs);
|
|
666
791
|
if (evals.length === 0)
|
|
667
792
|
return { ran: 0, skipped: "no evals" };
|
|
668
793
|
const dir = path.join(workspaceDir(tenant, workspace), "evals", ".results");
|
package/dist/src/flow-lint.js
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
// blocked — flows still run, warnings are advisory.
|
|
13
13
|
import { whenRowsPath } from "./store.js";
|
|
14
14
|
import { timezoneProblem } from "./clock.js";
|
|
15
|
+
import { FILTERABLE_TRIGGERS } from "./trigger-filter.js";
|
|
15
16
|
/** What a `[[flow:x]]` step's options are called in a flow file, for the
|
|
16
17
|
* ones expandSubflows does not carry into the nested steps. */
|
|
17
18
|
const SUBFLOW_DROPPED = [
|
|
@@ -209,6 +210,9 @@ export function lintFlow(flow, known) {
|
|
|
209
210
|
if (flow.signature && !flow.signingSecret) {
|
|
210
211
|
warnings.push({ step: null, message: `signature: ${flow.signature} needs \`signing_secret: \${NAME}\``, detail: "Every delivery will be refused until the secret is named and set in Settings → Secrets." });
|
|
211
212
|
}
|
|
213
|
+
if (flow.filter != null && !FILTERABLE_TRIGGERS.includes(flow.trigger)) {
|
|
214
|
+
warnings.push({ step: null, message: `filter: only applies to a webhook, email, flow or storage trigger`, detail: `trigger: ${flow.trigger} delivers no payload to match, so the filter does nothing here.` });
|
|
215
|
+
}
|
|
212
216
|
if (flow.signature && flow.trigger !== "webhook") {
|
|
213
217
|
warnings.push({ step: null, message: "signature: only applies to trigger: webhook", detail: "Inbox and other triggers do not carry a provider signature; the line does nothing here." });
|
|
214
218
|
}
|
|
@@ -113,6 +113,7 @@ const FLOW = [
|
|
|
113
113
|
{ key: "url", type: "string", help: "trigger: watch — the page watched" },
|
|
114
114
|
{ key: "every", type: ["string", "number"], help: "trigger: watch — how often" },
|
|
115
115
|
{ key: "path", type: "string", help: "trigger: storage — the prefix under storage/" },
|
|
116
|
+
{ key: "filter", type: "object", validated: true, help: "webhook, email, flow or storage trigger: the fields a delivery must have to start a run — e.g. {action: [opened], sender: {type: {not_in: [Bot]}}}" },
|
|
116
117
|
{ key: "idempotency", type: "string", help: "a header name, or body:<field>, that marks a repeat delivery" },
|
|
117
118
|
{ key: "debounce", type: ["string", "number"], help: "wait for a burst to end" },
|
|
118
119
|
{ key: "throttle", type: ["string", "number"], help: "at most one run per window" },
|
|
@@ -166,7 +167,9 @@ const EVAL = [
|
|
|
166
167
|
{ key: "flow", type: "string", help: "or the flow" },
|
|
167
168
|
{ key: "model", type: "string", help: "the judge's model" },
|
|
168
169
|
{ key: "effort", type: "string", values: EFFORT_LEVELS, help: "the judge's effort" },
|
|
169
|
-
{ key: "trigger", type: "string", values: ["deploy", "manual"], help: "deploy (default) | manual
|
|
170
|
+
{ key: "trigger", type: "string", values: ["deploy", "manual", "runs"], help: "deploy (default) | manual | runs — runs: grade real finished runs of the agent or flow instead of running cases" },
|
|
171
|
+
{ key: "sample", type: ["number", "string"], validated: true, help: "trigger: runs — the share of finished runs graded, 0 to 100 percent (default 25)" },
|
|
172
|
+
{ key: "improve", type: ["boolean", "number"], validated: true, help: "trigger: runs — true (or a count, default 5): propose a fix once that many graded runs failed" },
|
|
170
173
|
{ key: "live", type: "boolean", help: "true: real calls" },
|
|
171
174
|
{ key: "inputs", type: "boolean", help: "true: cases carry inputs" },
|
|
172
175
|
];
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
// The pre-receive check, run by git inside a push.
|
|
2
|
+
//
|
|
3
|
+
// git feeds "<old> <new> <ref>" lines on stdin. For a push to main, read the
|
|
4
|
+
// tree at the new commit straight from the object store and run the deploy
|
|
5
|
+
// checks on it. Any issue: print them and exit non-zero — the push is refused
|
|
6
|
+
// and the person sees exactly what a dashboard deploy would have told them,
|
|
7
|
+
// in their terminal, before anything changed.
|
|
8
|
+
//
|
|
9
|
+
// Runs as its own process under git's environment. GIT_DIR is set by git.
|
|
10
|
+
import fs from "node:fs";
|
|
11
|
+
import path from "node:path";
|
|
12
|
+
import { spawnSync } from "node:child_process";
|
|
13
|
+
import { deployIssues, runsInFlight } from "./deploy.js";
|
|
14
|
+
const dir = process.env.GIT_DIR;
|
|
15
|
+
if (!dir)
|
|
16
|
+
process.exit(0);
|
|
17
|
+
// `_library.git` is the account shelf, not a workspace: it holds tools,
|
|
18
|
+
// skills and knowledge and has no agents by definition. The workspace rules
|
|
19
|
+
// refused every push to it with "no agents". Its sync (syncLibraryFromTree)
|
|
20
|
+
// validates what it accepts on receive; nothing here to add yet.
|
|
21
|
+
// Inside a hook git sets GIT_DIR relative to the repo ("."), so resolve it
|
|
22
|
+
// against the cwd before asking for its name.
|
|
23
|
+
if (path.basename(path.resolve(process.cwd(), dir)) === "_library.git")
|
|
24
|
+
process.exit(0);
|
|
25
|
+
const input = fs.readFileSync(0, "utf8");
|
|
26
|
+
const git = (args) => {
|
|
27
|
+
const r = spawnSync("git", ["--git-dir", dir, ...args], { maxBuffer: 64 * 1024 * 1024 });
|
|
28
|
+
return r.status === 0 ? r.stdout.toString() : null;
|
|
29
|
+
};
|
|
30
|
+
// The client is gone by the time the deploy runs, so this is the only moment
|
|
31
|
+
// a person can be told their push will not take effect yet. A warning, not a
|
|
32
|
+
// refusal: the commit is fine and the scheduler applies it when the runs end.
|
|
33
|
+
const here = path.basename(path.resolve(process.cwd(), dir));
|
|
34
|
+
if (here.endsWith(".git")) {
|
|
35
|
+
const workspace = here.slice(0, -4);
|
|
36
|
+
const tenant = path.basename(path.resolve(process.cwd(), dir, "..", ".."));
|
|
37
|
+
try {
|
|
38
|
+
const live = runsInFlight(tenant, workspace);
|
|
39
|
+
if (live.length) {
|
|
40
|
+
process.stderr.write(`\nfoldrun: ${live.length} run${live.length === 1 ? " is" : "s are"} in flight, so this push is not live yet.\n` +
|
|
41
|
+
` It is on main and will be applied automatically when they finish.\n\n`);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
catch {
|
|
45
|
+
// A warning that cannot be computed must never fail a push.
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
let refused = false;
|
|
49
|
+
for (const line of input.split("\n").filter(Boolean)) {
|
|
50
|
+
const [, newSha, ref] = line.split(" ");
|
|
51
|
+
if (ref !== "refs/heads/main" || /^0+$/.test(newSha))
|
|
52
|
+
continue;
|
|
53
|
+
const tree = git(["ls-tree", "-r", "--name-only", newSha]) ?? "";
|
|
54
|
+
const files = tree
|
|
55
|
+
.split("\n")
|
|
56
|
+
.filter(Boolean)
|
|
57
|
+
.flatMap((p) => {
|
|
58
|
+
const content = git(["show", `${newSha}:${p}`]);
|
|
59
|
+
return content === null ? [] : [{ path: p, content }];
|
|
60
|
+
});
|
|
61
|
+
const issues = deployIssues(files);
|
|
62
|
+
if (issues.length) {
|
|
63
|
+
refused = true;
|
|
64
|
+
process.stderr.write(`\nfoldrun: refusing this push — ${issues.length} problem${issues.length === 1 ? "" : "s"} the checks would have stopped at deploy:\n`);
|
|
65
|
+
for (const i of issues)
|
|
66
|
+
process.stderr.write(` ${i.where}: ${i.message}\n`);
|
|
67
|
+
process.stderr.write(`\nFix and push again. (foldrun check runs the same rules locally.)\n\n`);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
process.exit(refused ? 1 : 0);
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The secret a repository signs its pushes with.
|
|
3
|
+
*
|
|
4
|
+
* Derived rather than stored, exactly like a flow's webhook token: HMAC of the
|
|
5
|
+
* workspace's identity under the install key. Nothing new to persist, unique
|
|
6
|
+
* per workspace, and rotating FOLDRUN_SECRET_KEY invalidates every connection
|
|
7
|
+
* at once.
|
|
8
|
+
*/
|
|
9
|
+
export declare function gitSecret(tenant: string, workspace: string): string;
|
|
10
|
+
export declare function gitHookPath(tenant: string, workspace: string): string;
|
|
11
|
+
/**
|
|
12
|
+
* Whether a request really came from the repository it claims to.
|
|
13
|
+
*
|
|
14
|
+
* Compared in constant time: a byte-by-byte early return leaks the expected
|
|
15
|
+
* signature to anyone willing to time enough requests. The raw body is
|
|
16
|
+
* required — re-serializing parsed JSON changes bytes GitHub signed.
|
|
17
|
+
*/
|
|
18
|
+
export declare function verifySignature(secret: string, rawBody: string, header: string | null): boolean;
|
|
19
|
+
export interface PushEvent {
|
|
20
|
+
repo: string;
|
|
21
|
+
/** Full ref, e.g. refs/heads/main. */
|
|
22
|
+
ref: string;
|
|
23
|
+
branch: string;
|
|
24
|
+
commit: string;
|
|
25
|
+
message: string | null;
|
|
26
|
+
pusher: string | null;
|
|
27
|
+
}
|
|
28
|
+
/** What we need out of a GitHub push payload, or null if it isn't one. */
|
|
29
|
+
export declare function parsePush(payload: unknown): PushEvent | null;
|
|
30
|
+
/**
|
|
31
|
+
* The repository at one commit, as a gzipped tarball.
|
|
32
|
+
*
|
|
33
|
+
* A token is only needed for a private repository. It is read from the
|
|
34
|
+
* environment rather than taken as an argument so a caller cannot be tricked
|
|
35
|
+
* into forwarding one somewhere else.
|
|
36
|
+
*/
|
|
37
|
+
export declare function fetchTarball(repo: string, commit: string): Promise<Buffer>;
|
package/dist/src/git.js
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
// git push → deployed.
|
|
2
|
+
//
|
|
3
|
+
// A workspace is markdown, so there is no build to run: a push is a fetch of
|
|
4
|
+
// the tree at that commit, a check, and a swap. This module is the GitHub half
|
|
5
|
+
// — the secret a repository signs with, the payload shape, and pulling the
|
|
6
|
+
// tarball. `deploy.ts` decides whether the result is allowed to go live.
|
|
7
|
+
//
|
|
8
|
+
// The tarball, not a clone: one HTTP call, no git binary, no working copy kept
|
|
9
|
+
// on the server. A markdown workspace is a few hundred kilobytes.
|
|
10
|
+
import crypto from "node:crypto";
|
|
11
|
+
import { installKey } from "./webhook.js";
|
|
12
|
+
/** GitHub refuses to fetch anything larger, and neither should we. */
|
|
13
|
+
const MAX_TARBALL = 25 * 1024 * 1024;
|
|
14
|
+
/**
|
|
15
|
+
* The secret a repository signs its pushes with.
|
|
16
|
+
*
|
|
17
|
+
* Derived rather than stored, exactly like a flow's webhook token: HMAC of the
|
|
18
|
+
* workspace's identity under the install key. Nothing new to persist, unique
|
|
19
|
+
* per workspace, and rotating FOLDRUN_SECRET_KEY invalidates every connection
|
|
20
|
+
* at once.
|
|
21
|
+
*/
|
|
22
|
+
export function gitSecret(tenant, workspace) {
|
|
23
|
+
return crypto
|
|
24
|
+
.createHmac("sha256", installKey())
|
|
25
|
+
.update(`git/${tenant}/${workspace}`)
|
|
26
|
+
.digest("hex");
|
|
27
|
+
}
|
|
28
|
+
export function gitHookPath(tenant, workspace) {
|
|
29
|
+
return `/api/git/${tenant}/${workspace}`;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Whether a request really came from the repository it claims to.
|
|
33
|
+
*
|
|
34
|
+
* Compared in constant time: a byte-by-byte early return leaks the expected
|
|
35
|
+
* signature to anyone willing to time enough requests. The raw body is
|
|
36
|
+
* required — re-serializing parsed JSON changes bytes GitHub signed.
|
|
37
|
+
*/
|
|
38
|
+
export function verifySignature(secret, rawBody, header) {
|
|
39
|
+
if (!header)
|
|
40
|
+
return false;
|
|
41
|
+
const expected = `sha256=${crypto.createHmac("sha256", secret).update(rawBody).digest("hex")}`;
|
|
42
|
+
const a = Buffer.from(expected);
|
|
43
|
+
const b = Buffer.from(header);
|
|
44
|
+
return a.length === b.length && crypto.timingSafeEqual(a, b);
|
|
45
|
+
}
|
|
46
|
+
/** What we need out of a GitHub push payload, or null if it isn't one. */
|
|
47
|
+
export function parsePush(payload) {
|
|
48
|
+
if (!payload || typeof payload !== "object")
|
|
49
|
+
return null;
|
|
50
|
+
const p = payload;
|
|
51
|
+
const repo = p.repository?.full_name;
|
|
52
|
+
const ref = p.ref;
|
|
53
|
+
const commit = p.after;
|
|
54
|
+
if (typeof repo !== "string" || typeof ref !== "string" || typeof commit !== "string") {
|
|
55
|
+
return null;
|
|
56
|
+
}
|
|
57
|
+
// A deleted branch reports all zeros and ships no tree.
|
|
58
|
+
if (/^0+$/.test(commit))
|
|
59
|
+
return null;
|
|
60
|
+
return {
|
|
61
|
+
repo,
|
|
62
|
+
ref,
|
|
63
|
+
branch: ref.replace(/^refs\/heads\//, ""),
|
|
64
|
+
commit,
|
|
65
|
+
message: typeof p.head_commit?.message === "string" ? p.head_commit.message : null,
|
|
66
|
+
pusher: typeof p.pusher?.name === "string" ? p.pusher.name : null,
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* The repository at one commit, as a gzipped tarball.
|
|
71
|
+
*
|
|
72
|
+
* A token is only needed for a private repository. It is read from the
|
|
73
|
+
* environment rather than taken as an argument so a caller cannot be tricked
|
|
74
|
+
* into forwarding one somewhere else.
|
|
75
|
+
*/
|
|
76
|
+
export async function fetchTarball(repo, commit) {
|
|
77
|
+
if (!/^[\w.-]+\/[\w.-]+$/.test(repo))
|
|
78
|
+
throw new Error(`refusing to fetch "${repo}"`);
|
|
79
|
+
if (!/^[0-9a-f]{7,40}$/i.test(commit))
|
|
80
|
+
throw new Error(`refusing to fetch commit "${commit}"`);
|
|
81
|
+
const headers = {
|
|
82
|
+
accept: "application/vnd.github+json",
|
|
83
|
+
"user-agent": "foldrun",
|
|
84
|
+
"x-github-api-version": "2022-11-28",
|
|
85
|
+
};
|
|
86
|
+
const token = process.env.GITHUB_TOKEN;
|
|
87
|
+
if (token)
|
|
88
|
+
headers.authorization = `Bearer ${token}`;
|
|
89
|
+
const res = await fetch(`https://api.github.com/repos/${repo}/tarball/${commit}`, { headers });
|
|
90
|
+
if (!res.ok) {
|
|
91
|
+
throw new Error(`GitHub returned ${res.status} for ${repo}@${commit.slice(0, 8)}` +
|
|
92
|
+
(res.status === 404 && !token
|
|
93
|
+
? " — a private repository needs GITHUB_TOKEN set on the server"
|
|
94
|
+
: ""));
|
|
95
|
+
}
|
|
96
|
+
const buf = Buffer.from(await res.arrayBuffer());
|
|
97
|
+
if (buf.length > MAX_TARBALL) {
|
|
98
|
+
throw new Error(`${repo} is ${Math.round(buf.length / 1e6)}MB — too large to deploy`);
|
|
99
|
+
}
|
|
100
|
+
return buf;
|
|
101
|
+
}
|