@foldrun/core 2.10.0 → 2.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/dist/index.d.ts +3 -0
  2. package/dist/index.js +3 -0
  3. package/dist/src/catalog.d.ts +3 -0
  4. package/dist/src/catalog.js +4 -1
  5. package/dist/src/deploy.js +16 -0
  6. package/dist/src/eval-compare.d.ts +74 -0
  7. package/dist/src/eval-compare.js +187 -0
  8. package/dist/src/evals.d.ts +62 -3
  9. package/dist/src/evals.js +161 -36
  10. package/dist/src/flow-lint.js +4 -0
  11. package/dist/src/frontmatter-fields.js +4 -1
  12. package/dist/src/git-hook.d.ts +1 -0
  13. package/dist/src/git-hook.js +70 -0
  14. package/dist/src/git.d.ts +37 -0
  15. package/dist/src/git.js +101 -0
  16. package/dist/src/gitrepo.d.ts +113 -0
  17. package/dist/src/gitrepo.js +404 -0
  18. package/dist/src/grades.d.ts +118 -0
  19. package/dist/src/grades.js +481 -0
  20. package/dist/src/judge-verdict.d.ts +1 -0
  21. package/dist/src/judge-verdict.js +3 -1
  22. package/dist/src/online-checks.d.ts +4 -0
  23. package/dist/src/online-checks.js +5 -1
  24. package/dist/src/openapi.d.ts +4 -0
  25. package/dist/src/openapi.js +5 -1
  26. package/dist/src/platform.d.ts +12 -3
  27. package/dist/src/platform.js +9 -1
  28. package/dist/src/providers.js +8 -1
  29. package/dist/src/starter.js +5 -0
  30. package/dist/src/store.d.ts +8 -0
  31. package/dist/src/store.js +5 -1
  32. package/dist/src/tar.d.ts +30 -0
  33. package/dist/src/tar.js +112 -0
  34. package/dist/src/tool-test.js +19 -7
  35. package/dist/src/translator.d.ts +4 -0
  36. package/dist/src/translator.js +22 -3
  37. package/dist/src/trigger-filter.d.ts +28 -0
  38. package/dist/src/trigger-filter.js +272 -0
  39. package/dist/src/trigger-log.d.ts +1 -1
  40. package/dist/src/triggers.d.ts +2 -0
  41. package/dist/src/triggers.js +13 -2
  42. package/dist/src/untrusted-fetch.d.ts +16 -0
  43. package/dist/src/untrusted-fetch.js +60 -2
  44. package/index.ts +3 -0
  45. package/package.json +2 -2
  46. package/src/catalog.ts +5 -1
  47. package/src/deploy.ts +10 -0
  48. package/src/eval-compare.ts +243 -0
  49. package/src/evals.ts +195 -38
  50. package/src/flow-lint.ts +4 -0
  51. package/src/frontmatter-fields.ts +4 -1
  52. package/src/grades.ts +532 -0
  53. package/src/judge-verdict.ts +4 -2
  54. package/src/online-checks.ts +5 -1
  55. package/src/openapi.ts +6 -1
  56. package/src/platform.ts +16 -4
  57. package/src/providers.ts +8 -1
  58. package/src/starter.ts +5 -0
  59. package/src/store.ts +11 -1
  60. package/src/tool-test.ts +21 -7
  61. package/src/translator.ts +24 -3
  62. package/src/trigger-filter.ts +264 -0
  63. package/src/trigger-log.ts +1 -0
  64. package/src/triggers.ts +14 -2
  65. package/src/untrusted-fetch.ts +65 -2
package/dist/src/evals.js CHANGED
@@ -34,6 +34,7 @@ import { accountModelEnv, withModelEnv } from "./model-env.js";
34
34
  import { sdkEnv } from "./step-exec.js";
35
35
  import { platformModelEnv } from "./runner.js";
36
36
  import fs from "node:fs";
37
+ import crypto from "node:crypto";
37
38
  import path from "node:path";
38
39
  import { deployedCommit } from "./deploy.js";
39
40
  import { latestRevisionId } from "./history.js";
@@ -137,6 +138,45 @@ export function parseEval(file, raw, defaults = {}) {
137
138
  }
138
139
  if (current)
139
140
  cases.push(current);
141
+ const trigger = data.trigger === "manual" ? "manual" : data.trigger === "runs" ? "runs" : "deploy";
142
+ const problems = [];
143
+ const sample = parseSample(data.sample, problems);
144
+ const improve = parseImprove(data.improve, problems);
145
+ if (trigger !== "runs") {
146
+ if (data.sample !== undefined)
147
+ problems.push("sample: only applies to an eval with trigger: runs — this one runs its cases");
148
+ if (data.improve !== undefined)
149
+ problems.push("improve: only applies to an eval with trigger: runs");
150
+ }
151
+ else {
152
+ // A grader reads what a run said; it never starts one, so a task has
153
+ // nothing to go to, and file:/run: would execute or read on the
154
+ // platform's host rather than in the run's sandbox.
155
+ for (const c of cases) {
156
+ if (c.task)
157
+ problems.push(`## ${c.name}: task: does nothing in a trigger: runs eval — it grades real runs, so each check is only an expect: list`);
158
+ for (const a of c.expect) {
159
+ if (a.type === "file" || a.type === "run")
160
+ problems.push(`## ${c.name}: ${a.type}: cannot grade a finished run — use contains, not-contains or judge`);
161
+ // Graded on the platform's scheduler, over whatever a run said: a
162
+ // regular expression that backtracks would hold the scheduler with it.
163
+ if (a.type === "matches")
164
+ problems.push(`## ${c.name}: matches: is not used to grade live runs — a regular expression over any run's output can stall the platform; use contains, not-contains or judge`);
165
+ }
166
+ if (c.expect.length === 0)
167
+ problems.push(`## ${c.name}: has no expect: lines, so it grades nothing`);
168
+ }
169
+ if (cases.length === 0)
170
+ problems.push("a grader needs at least one check — a `## <name>` section with an expect: list; with none it would pass every run");
171
+ // The pass budgets graded runs; this bounds the judges each one costs.
172
+ const judges = cases.reduce((n, c) => n + c.expect.filter((x) => x.type === "judge").length, 0);
173
+ if (judges > MAX_GRADER_JUDGES)
174
+ problems.push(`${judges} judge: lines — a grader asks at most ${MAX_GRADER_JUDGES} judges per run; fold related claims into one`);
175
+ if (data.live === true)
176
+ problems.push("live: does nothing in a trigger: runs eval — it never starts a run");
177
+ if (data.inputs === true)
178
+ problems.push("inputs: and trigger: runs cannot go together");
179
+ }
140
180
  return {
141
181
  name: data.name ?? file.replace(/\.md$/, ""),
142
182
  file,
@@ -144,12 +184,41 @@ export function parseEval(file, raw, defaults = {}) {
144
184
  flow: typeof data.flow === "string" ? data.flow : null,
145
185
  model: resolveModel(data.model ?? defaults.model ?? "fast"),
146
186
  effort: resolveEffort(data.effort ?? defaults.effort ?? "low"),
147
- trigger: data.trigger === "manual" ? "manual" : "deploy",
187
+ trigger,
188
+ sample,
189
+ improve: trigger === "runs" ? improve : null,
190
+ problems,
148
191
  live: data.live === true,
149
192
  inputs: data.inputs === true,
150
193
  cases,
151
194
  };
152
195
  }
196
+ /** Judges one graded run may cost — the grade pass counts runs, not calls. */
197
+ export const MAX_GRADER_JUDGES = 5;
198
+ /** `sample: 25`, `sample: "25%"` — percent of finished runs graded. */
199
+ function parseSample(v, problems) {
200
+ if (v === undefined || v === null)
201
+ return 25;
202
+ const n = typeof v === "number" ? v : typeof v === "string" ? Number(v.trim().replace(/%$/, "")) : NaN;
203
+ if (!Number.isFinite(n) || n < 0 || n > 100) {
204
+ problems.push(`sample: ${String(v)} — a percent from 0 to 100 (25 grades about one run in four; 0 grades none on its own)`);
205
+ return 25;
206
+ }
207
+ return Math.round(n * 100) / 100;
208
+ }
209
+ /** `improve: true` (5), `improve: 3`, `improve: false`. */
210
+ function parseImprove(v, problems) {
211
+ if (v === undefined || v === null || v === false)
212
+ return null;
213
+ if (v === true)
214
+ return 5;
215
+ const n = typeof v === "number" ? v : NaN;
216
+ if (!Number.isInteger(n) || n < 1 || n > 50) {
217
+ problems.push(`improve: ${String(v)} — true, or how many failed grades open a proposal (1 to 50)`);
218
+ return null;
219
+ }
220
+ return n;
221
+ }
153
222
  export function listEvals(tenant, workspace) {
154
223
  const dir = path.join(workspaceDir(tenant, workspace), "evals");
155
224
  if (!fs.existsSync(dir))
@@ -158,7 +227,13 @@ export function listEvals(tenant, workspace) {
158
227
  .readdirSync(dir)
159
228
  .filter((f) => f.endsWith(".md"))
160
229
  .sort()
161
- .map((f) => parseEval(f, fs.readFileSync(path.join(dir, f), "utf8"), judgeDefaults(tenant, workspace)));
230
+ .map((f) => {
231
+ const raw = fs.readFileSync(path.join(dir, f), "utf8");
232
+ // The version is taken from the same read the eval is parsed from, so
233
+ // a grade names the rubric that produced it even if the file changes
234
+ // while its judges answer.
235
+ return { ...parseEval(f, raw, judgeDefaults(tenant, workspace)), revision: crypto.createHash("sha256").update(raw).digest("hex").slice(0, 12) };
236
+ });
162
237
  }
163
238
  /** `judge:` from AGENTS.md, field by field: the workspace's model (or
164
239
  * effort), else the account's — so a workspace that sets only an effort
@@ -386,32 +461,59 @@ function shell(command, cwd) {
386
461
  // LAST, after the reasoning: asked for it first, the judge committed before
387
462
  // it had looked, then argued the other way — "FAIL … which meets the
388
463
  // criterion" (vaza evals, 21 Oct 2026).
389
- async function judge(rubric, output, model, effort, env) {
464
+ /** How long one judge may take before it is cut off: a verdict on one
465
+ * sentence is seconds; a stalled model must not hold a grade pass. */
466
+ export const JUDGE_TIMEOUT_MS = 120_000;
467
+ /** One judge call: its words and what it cost (the SDK's result line). */
468
+ export async function judgeCall(rubric, output, model, effort, env,
469
+ /** What the output was an answer to — a live run's task, so the judge
470
+ * grades the reply against what was asked rather than in a vacuum. */
471
+ context) {
390
472
  const prompt = `You are grading one piece of output against one criterion. ${JUDGE_VERDICT_INSTRUCTION}\n\n` +
391
- `Criterion: ${rubric}\n\n<output>\n${output.slice(0, 20000)}\n</output>`;
473
+ `Criterion: ${rubric}\n\n` +
474
+ (context ? `<task>\n${context.slice(0, 4000)}\n</task>\n\n` : "") +
475
+ `<output>\n${output.slice(0, 20000)}\n</output>`;
392
476
  let text = "";
393
- const q = query({
394
- prompt,
395
- options: {
396
- model,
397
- ...(effort ? { effort } : {}),
398
- tools: [],
399
- allowedTools: [],
400
- settingSources: [],
401
- strictMcpConfig: true,
402
- env,
403
- maxTurns: 1,
404
- },
405
- });
406
- for await (const message of q) {
407
- if (message.type === "assistant") {
408
- for (const block of message.message.content) {
409
- if (block.type === "text")
410
- text += block.text;
477
+ let costUsd = 0;
478
+ const abort = new AbortController();
479
+ const timer = setTimeout(() => abort.abort(), JUDGE_TIMEOUT_MS);
480
+ timer.unref?.();
481
+ try {
482
+ const q = query({
483
+ prompt,
484
+ options: {
485
+ model,
486
+ ...(effort ? { effort } : {}),
487
+ tools: [],
488
+ allowedTools: [],
489
+ settingSources: [],
490
+ strictMcpConfig: true,
491
+ env,
492
+ maxTurns: 1,
493
+ abortController: abort,
494
+ },
495
+ });
496
+ for await (const message of q) {
497
+ if (message.type === "assistant") {
498
+ for (const block of message.message.content) {
499
+ if (block.type === "text")
500
+ text += block.text;
501
+ }
502
+ }
503
+ else if (message.type === "result") {
504
+ costUsd = Number(message.total_cost_usd) || 0;
411
505
  }
412
506
  }
413
507
  }
414
- return text.trim();
508
+ catch (err) {
509
+ if (abort.signal.aborted)
510
+ throw new Error(`the judge took longer than ${JUDGE_TIMEOUT_MS / 1000}s and was stopped`);
511
+ throw err;
512
+ }
513
+ finally {
514
+ clearTimeout(timer);
515
+ }
516
+ return { text: text.trim(), costUsd };
415
517
  }
416
518
  /**
417
519
  * The judge's environment: this process's, with the platform's model
@@ -425,7 +527,7 @@ export function judgeProcessEnv() {
425
527
  const live = Object.fromEntries(Object.entries(platformModelEnv()).filter(([, v]) => v !== undefined));
426
528
  return sdkEnv({ ...process.env, ...live });
427
529
  }
428
- async function checkAssertion(assertion, output, agentDir, model, effort, judgeEnv = () => ({ env: judgeProcessEnv() })) {
530
+ export async function checkAssertion(assertion, output, agentDir, model, effort, judgeEnv = () => ({ env: judgeProcessEnv() })) {
429
531
  const hay = output.toLowerCase();
430
532
  switch (assertion.type) {
431
533
  case "contains": {
@@ -467,13 +569,16 @@ async function checkAssertion(assertion, output, agentDir, model, effort, judgeE
467
569
  catch (err) {
468
570
  return { assertion, passed: false, detail: `judge not run — ${err instanceof Error ? err.message : String(err)}` };
469
571
  }
470
- const text = await withModelEnv(m, (env) => judge(assertion.value, output, model, effort, env));
471
- const { passed, detail } = judgeVerdict(text);
472
- return { assertion, passed, detail };
572
+ const said = await withModelEnv(m, (env) => judgeCall(assertion.value, output, model, effort, env));
573
+ const { passed, detail } = judgeVerdict(said.text);
574
+ return { assertion, passed, detail, costUsd: said.costUsd };
473
575
  }
474
576
  }
475
577
  }
476
- export async function runEval(tenant, workspace, info) {
578
+ export async function runEval(tenant, workspace, info, opts = {}) {
579
+ if (info.trigger === "runs") {
580
+ throw refuse(`${info.name} grades real runs (trigger: runs) — it has no cases to run; grade a run with POST …/evals/${info.name}/grade { run }`, 400);
581
+ }
477
582
  const startedAt = new Date().toISOString();
478
583
  const results = [];
479
584
  // Saved inputs carry no assertions: running them would start and bill a
@@ -485,6 +590,16 @@ export async function runEval(tenant, workspace, info) {
485
590
  let output = "";
486
591
  let error = null;
487
592
  let costUsd = 0;
593
+ let secs = null;
594
+ const stop = opts.beforeCase?.() ?? null;
595
+ if (stop) {
596
+ results.push({ name: testCase.name, task: testCase.task, runId: null, error: `not run — ${stop}`, passed: false, assertions: [], costUsd: 0, judgeCostUsd: 0, secs: null });
597
+ continue;
598
+ }
599
+ // A step forced onto the comparison's model: step-level, so it wins.
600
+ const force = (st) => opts.model && (!opts.agent || st.agent === opts.agent)
601
+ ? { ...st, model: opts.model, ...(opts.effort ? { effort: opts.effort } : {}) }
602
+ : st;
488
603
  try {
489
604
  let run;
490
605
  if (info.flow) {
@@ -492,27 +607,33 @@ export async function runEval(tenant, workspace, info) {
492
607
  if (!flow || flow.steps.length === 0)
493
608
  throw new Error(`flow ${info.flow} not found or empty`);
494
609
  // The case's task reaches the first step; output passing carries it on.
495
- const steps = flow.steps.map((s, i) => i === 0
610
+ const steps = flow.steps.map((s, i) => force(i === 0
496
611
  ? { ...s, instruction: `${s.instruction}\n\n<run_task>\n${testCase.task}\n</run_task>` }
497
- : s);
612
+ : s));
498
613
  // Test unless the eval OR the flow says live: an eval that runs a
499
- // flow written `live: true` was written by someone who knew.
500
- run = await beginEvalRun(tenant, workspace, steps, `eval:${info.name}`, flow.model, flow.effort, !(info.live || flow.live));
614
+ // flow written `live: true` was written by someone who knew. A
615
+ // comparison is a test whatever they say.
616
+ run = await beginEvalRun(tenant, workspace, steps, `eval:${info.name}`, flow.model, flow.effort, opts.forceTest ? true : !(info.live || flow.live));
501
617
  }
502
618
  else {
503
619
  if (!target)
504
620
  throw new Error("this eval names neither an agent nor a flow");
505
- run = await beginEvalRun(tenant, workspace, [{ agent: target, instruction: testCase.task, group: 1, optional: false }], `eval:${info.name}`, undefined, undefined, !info.live);
621
+ run = await beginEvalRun(tenant, workspace, [force({ agent: target, instruction: testCase.task, group: 1, optional: false })], `eval:${info.name}`, undefined, undefined, opts.forceTest ? true : !info.live);
506
622
  }
507
623
  runId = run.id;
508
624
  const finished = await waitForRun(tenant, workspace, run.id);
509
625
  if (!finished)
510
626
  throw new Error("run disappeared");
627
+ // What the steps before a gate spent is spent: counted before the
628
+ // gate is refused, so a comparison's limit sees it.
629
+ costUsd = finished.steps.reduce((sum, s) => sum + (s.costUsd ?? 0), 0);
511
630
  if (finished.status === "awaiting-approval") {
512
631
  throw new Error("this run pauses for approval — an eval cannot answer a human gate");
513
632
  }
514
633
  output = finished.steps.map((s) => s.result ?? "").join("\n\n");
515
- costUsd = finished.steps.reduce((sum, s) => sum + (s.costUsd ?? 0), 0);
634
+ const from = Date.parse(finished.drivenAt ?? finished.startedAt);
635
+ const to = finished.finishedAt ? Date.parse(finished.finishedAt) : NaN;
636
+ secs = Number.isFinite(from) && Number.isFinite(to) ? Math.max(0, Math.round((to - from) / 100) / 10) : null;
516
637
  if (finished.status === "failed" && !output.trim()) {
517
638
  // Quote the run's own last error. "Produced nothing" on a fresh
518
639
  // workspace is almost always "secret X is not set" or a tool that
@@ -556,7 +677,10 @@ export async function runEval(tenant, workspace, info) {
556
677
  passed: !error && assertions.length > 0 && assertions.every((r) => r.passed),
557
678
  assertions,
558
679
  costUsd,
680
+ judgeCostUsd: Math.round(assertions.reduce((n, a) => n + (a.costUsd ?? 0), 0) * 1e6) / 1e6,
681
+ secs,
559
682
  });
683
+ opts.afterCase?.(results[results.length - 1]);
560
684
  }
561
685
  return {
562
686
  eval: info.name,
@@ -661,8 +785,9 @@ commit = null) {
661
785
  if (!stamp)
662
786
  return { ran: 0, skipped: "no revision to stamp the result with" };
663
787
  // Only the evals that asked to run on deploy — a `trigger: manual` eval is
664
- // one a person runs on purpose, because it costs a whole flow run.
665
- const evals = listEvals(tenant, workspace).filter((e) => e.trigger !== "manual" && !e.inputs);
788
+ // one a person runs on purpose, because it costs a whole flow run, and a
789
+ // `trigger: runs` one grades real runs and has no cases to run.
790
+ const evals = listEvals(tenant, workspace).filter((e) => e.trigger === "deploy" && !e.inputs);
666
791
  if (evals.length === 0)
667
792
  return { ran: 0, skipped: "no evals" };
668
793
  const dir = path.join(workspaceDir(tenant, workspace), "evals", ".results");
@@ -12,6 +12,7 @@
12
12
  // blocked — flows still run, warnings are advisory.
13
13
  import { whenRowsPath } from "./store.js";
14
14
  import { timezoneProblem } from "./clock.js";
15
+ import { FILTERABLE_TRIGGERS } from "./trigger-filter.js";
15
16
  /** What a `[[flow:x]]` step's options are called in a flow file, for the
16
17
  * ones expandSubflows does not carry into the nested steps. */
17
18
  const SUBFLOW_DROPPED = [
@@ -209,6 +210,9 @@ export function lintFlow(flow, known) {
209
210
  if (flow.signature && !flow.signingSecret) {
210
211
  warnings.push({ step: null, message: `signature: ${flow.signature} needs \`signing_secret: \${NAME}\``, detail: "Every delivery will be refused until the secret is named and set in Settings → Secrets." });
211
212
  }
213
+ if (flow.filter != null && !FILTERABLE_TRIGGERS.includes(flow.trigger)) {
214
+ warnings.push({ step: null, message: `filter: only applies to a webhook, email, flow or storage trigger`, detail: `trigger: ${flow.trigger} delivers no payload to match, so the filter does nothing here.` });
215
+ }
212
216
  if (flow.signature && flow.trigger !== "webhook") {
213
217
  warnings.push({ step: null, message: "signature: only applies to trigger: webhook", detail: "Inbox and other triggers do not carry a provider signature; the line does nothing here." });
214
218
  }
@@ -113,6 +113,7 @@ const FLOW = [
113
113
  { key: "url", type: "string", help: "trigger: watch — the page watched" },
114
114
  { key: "every", type: ["string", "number"], help: "trigger: watch — how often" },
115
115
  { key: "path", type: "string", help: "trigger: storage — the prefix under storage/" },
116
+ { key: "filter", type: "object", validated: true, help: "webhook, email, flow or storage trigger: the fields a delivery must have to start a run — e.g. {action: [opened], sender: {type: {not_in: [Bot]}}}" },
116
117
  { key: "idempotency", type: "string", help: "a header name, or body:<field>, that marks a repeat delivery" },
117
118
  { key: "debounce", type: ["string", "number"], help: "wait for a burst to end" },
118
119
  { key: "throttle", type: ["string", "number"], help: "at most one run per window" },
@@ -166,7 +167,9 @@ const EVAL = [
166
167
  { key: "flow", type: "string", help: "or the flow" },
167
168
  { key: "model", type: "string", help: "the judge's model" },
168
169
  { key: "effort", type: "string", values: EFFORT_LEVELS, help: "the judge's effort" },
169
- { key: "trigger", type: "string", values: ["deploy", "manual"], help: "deploy (default) | manual; anything else is deploy" },
170
+ { key: "trigger", type: "string", values: ["deploy", "manual", "runs"], help: "deploy (default) | manual | runs — runs: grade real finished runs of the agent or flow instead of running cases" },
171
+ { key: "sample", type: ["number", "string"], validated: true, help: "trigger: runs — the share of finished runs graded, 0 to 100 percent (default 25)" },
172
+ { key: "improve", type: ["boolean", "number"], validated: true, help: "trigger: runs — true (or a count, default 5): propose a fix once that many graded runs failed" },
170
173
  { key: "live", type: "boolean", help: "true: real calls" },
171
174
  { key: "inputs", type: "boolean", help: "true: cases carry inputs" },
172
175
  ];
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,70 @@
1
+ // The pre-receive check, run by git inside a push.
2
+ //
3
+ // git feeds "<old> <new> <ref>" lines on stdin. For a push to main, read the
4
+ // tree at the new commit straight from the object store and run the deploy
5
+ // checks on it. Any issue: print them and exit non-zero — the push is refused
6
+ // and the person sees exactly what a dashboard deploy would have told them,
7
+ // in their terminal, before anything changed.
8
+ //
9
+ // Runs as its own process under git's environment. GIT_DIR is set by git.
10
+ import fs from "node:fs";
11
+ import path from "node:path";
12
+ import { spawnSync } from "node:child_process";
13
+ import { deployIssues, runsInFlight } from "./deploy.js";
14
+ const dir = process.env.GIT_DIR;
15
+ if (!dir)
16
+ process.exit(0);
17
+ // `_library.git` is the account shelf, not a workspace: it holds tools,
18
+ // skills and knowledge and has no agents by definition. The workspace rules
19
+ // refused every push to it with "no agents". Its sync (syncLibraryFromTree)
20
+ // validates what it accepts on receive; nothing here to add yet.
21
+ // Inside a hook git sets GIT_DIR relative to the repo ("."), so resolve it
22
+ // against the cwd before asking for its name.
23
+ if (path.basename(path.resolve(process.cwd(), dir)) === "_library.git")
24
+ process.exit(0);
25
+ const input = fs.readFileSync(0, "utf8");
26
+ const git = (args) => {
27
+ const r = spawnSync("git", ["--git-dir", dir, ...args], { maxBuffer: 64 * 1024 * 1024 });
28
+ return r.status === 0 ? r.stdout.toString() : null;
29
+ };
30
+ // The client is gone by the time the deploy runs, so this is the only moment
31
+ // a person can be told their push will not take effect yet. A warning, not a
32
+ // refusal: the commit is fine and the scheduler applies it when the runs end.
33
+ const here = path.basename(path.resolve(process.cwd(), dir));
34
+ if (here.endsWith(".git")) {
35
+ const workspace = here.slice(0, -4);
36
+ const tenant = path.basename(path.resolve(process.cwd(), dir, "..", ".."));
37
+ try {
38
+ const live = runsInFlight(tenant, workspace);
39
+ if (live.length) {
40
+ process.stderr.write(`\nfoldrun: ${live.length} run${live.length === 1 ? " is" : "s are"} in flight, so this push is not live yet.\n` +
41
+ ` It is on main and will be applied automatically when they finish.\n\n`);
42
+ }
43
+ }
44
+ catch {
45
+ // A warning that cannot be computed must never fail a push.
46
+ }
47
+ }
48
+ let refused = false;
49
+ for (const line of input.split("\n").filter(Boolean)) {
50
+ const [, newSha, ref] = line.split(" ");
51
+ if (ref !== "refs/heads/main" || /^0+$/.test(newSha))
52
+ continue;
53
+ const tree = git(["ls-tree", "-r", "--name-only", newSha]) ?? "";
54
+ const files = tree
55
+ .split("\n")
56
+ .filter(Boolean)
57
+ .flatMap((p) => {
58
+ const content = git(["show", `${newSha}:${p}`]);
59
+ return content === null ? [] : [{ path: p, content }];
60
+ });
61
+ const issues = deployIssues(files);
62
+ if (issues.length) {
63
+ refused = true;
64
+ process.stderr.write(`\nfoldrun: refusing this push — ${issues.length} problem${issues.length === 1 ? "" : "s"} the checks would have stopped at deploy:\n`);
65
+ for (const i of issues)
66
+ process.stderr.write(` ${i.where}: ${i.message}\n`);
67
+ process.stderr.write(`\nFix and push again. (foldrun check runs the same rules locally.)\n\n`);
68
+ }
69
+ }
70
+ process.exit(refused ? 1 : 0);
@@ -0,0 +1,37 @@
1
+ /**
2
+ * The secret a repository signs its pushes with.
3
+ *
4
+ * Derived rather than stored, exactly like a flow's webhook token: HMAC of the
5
+ * workspace's identity under the install key. Nothing new to persist, unique
6
+ * per workspace, and rotating FOLDRUN_SECRET_KEY invalidates every connection
7
+ * at once.
8
+ */
9
+ export declare function gitSecret(tenant: string, workspace: string): string;
10
+ export declare function gitHookPath(tenant: string, workspace: string): string;
11
+ /**
12
+ * Whether a request really came from the repository it claims to.
13
+ *
14
+ * Compared in constant time: a byte-by-byte early return leaks the expected
15
+ * signature to anyone willing to time enough requests. The raw body is
16
+ * required — re-serializing parsed JSON changes bytes GitHub signed.
17
+ */
18
+ export declare function verifySignature(secret: string, rawBody: string, header: string | null): boolean;
19
+ export interface PushEvent {
20
+ repo: string;
21
+ /** Full ref, e.g. refs/heads/main. */
22
+ ref: string;
23
+ branch: string;
24
+ commit: string;
25
+ message: string | null;
26
+ pusher: string | null;
27
+ }
28
+ /** What we need out of a GitHub push payload, or null if it isn't one. */
29
+ export declare function parsePush(payload: unknown): PushEvent | null;
30
+ /**
31
+ * The repository at one commit, as a gzipped tarball.
32
+ *
33
+ * A token is only needed for a private repository. It is read from the
34
+ * environment rather than taken as an argument so a caller cannot be tricked
35
+ * into forwarding one somewhere else.
36
+ */
37
+ export declare function fetchTarball(repo: string, commit: string): Promise<Buffer>;
@@ -0,0 +1,101 @@
1
+ // git push → deployed.
2
+ //
3
+ // A workspace is markdown, so there is no build to run: a push is a fetch of
4
+ // the tree at that commit, a check, and a swap. This module is the GitHub half
5
+ // — the secret a repository signs with, the payload shape, and pulling the
6
+ // tarball. `deploy.ts` decides whether the result is allowed to go live.
7
+ //
8
+ // The tarball, not a clone: one HTTP call, no git binary, no working copy kept
9
+ // on the server. A markdown workspace is a few hundred kilobytes.
10
+ import crypto from "node:crypto";
11
+ import { installKey } from "./webhook.js";
12
+ /** GitHub refuses to fetch anything larger, and neither should we. */
13
+ const MAX_TARBALL = 25 * 1024 * 1024;
14
+ /**
15
+ * The secret a repository signs its pushes with.
16
+ *
17
+ * Derived rather than stored, exactly like a flow's webhook token: HMAC of the
18
+ * workspace's identity under the install key. Nothing new to persist, unique
19
+ * per workspace, and rotating FOLDRUN_SECRET_KEY invalidates every connection
20
+ * at once.
21
+ */
22
+ export function gitSecret(tenant, workspace) {
23
+ return crypto
24
+ .createHmac("sha256", installKey())
25
+ .update(`git/${tenant}/${workspace}`)
26
+ .digest("hex");
27
+ }
28
+ export function gitHookPath(tenant, workspace) {
29
+ return `/api/git/${tenant}/${workspace}`;
30
+ }
31
+ /**
32
+ * Whether a request really came from the repository it claims to.
33
+ *
34
+ * Compared in constant time: a byte-by-byte early return leaks the expected
35
+ * signature to anyone willing to time enough requests. The raw body is
36
+ * required — re-serializing parsed JSON changes bytes GitHub signed.
37
+ */
38
+ export function verifySignature(secret, rawBody, header) {
39
+ if (!header)
40
+ return false;
41
+ const expected = `sha256=${crypto.createHmac("sha256", secret).update(rawBody).digest("hex")}`;
42
+ const a = Buffer.from(expected);
43
+ const b = Buffer.from(header);
44
+ return a.length === b.length && crypto.timingSafeEqual(a, b);
45
+ }
46
+ /** What we need out of a GitHub push payload, or null if it isn't one. */
47
+ export function parsePush(payload) {
48
+ if (!payload || typeof payload !== "object")
49
+ return null;
50
+ const p = payload;
51
+ const repo = p.repository?.full_name;
52
+ const ref = p.ref;
53
+ const commit = p.after;
54
+ if (typeof repo !== "string" || typeof ref !== "string" || typeof commit !== "string") {
55
+ return null;
56
+ }
57
+ // A deleted branch reports all zeros and ships no tree.
58
+ if (/^0+$/.test(commit))
59
+ return null;
60
+ return {
61
+ repo,
62
+ ref,
63
+ branch: ref.replace(/^refs\/heads\//, ""),
64
+ commit,
65
+ message: typeof p.head_commit?.message === "string" ? p.head_commit.message : null,
66
+ pusher: typeof p.pusher?.name === "string" ? p.pusher.name : null,
67
+ };
68
+ }
69
+ /**
70
+ * The repository at one commit, as a gzipped tarball.
71
+ *
72
+ * A token is only needed for a private repository. It is read from the
73
+ * environment rather than taken as an argument so a caller cannot be tricked
74
+ * into forwarding one somewhere else.
75
+ */
76
+ export async function fetchTarball(repo, commit) {
77
+ if (!/^[\w.-]+\/[\w.-]+$/.test(repo))
78
+ throw new Error(`refusing to fetch "${repo}"`);
79
+ if (!/^[0-9a-f]{7,40}$/i.test(commit))
80
+ throw new Error(`refusing to fetch commit "${commit}"`);
81
+ const headers = {
82
+ accept: "application/vnd.github+json",
83
+ "user-agent": "foldrun",
84
+ "x-github-api-version": "2022-11-28",
85
+ };
86
+ const token = process.env.GITHUB_TOKEN;
87
+ if (token)
88
+ headers.authorization = `Bearer ${token}`;
89
+ const res = await fetch(`https://api.github.com/repos/${repo}/tarball/${commit}`, { headers });
90
+ if (!res.ok) {
91
+ throw new Error(`GitHub returned ${res.status} for ${repo}@${commit.slice(0, 8)}` +
92
+ (res.status === 404 && !token
93
+ ? " — a private repository needs GITHUB_TOKEN set on the server"
94
+ : ""));
95
+ }
96
+ const buf = Buffer.from(await res.arrayBuffer());
97
+ if (buf.length > MAX_TARBALL) {
98
+ throw new Error(`${repo} is ${Math.round(buf.length / 1e6)}MB — too large to deploy`);
99
+ }
100
+ return buf;
101
+ }