@foldrun/core 1.4.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +0 -3
- package/dist/index.js +0 -3
- package/dist/src/deploy.d.ts +0 -26
- package/dist/src/deploy.js +10 -53
- package/dist/src/history.d.ts +30 -10
- package/dist/src/history.js +232 -99
- package/dist/src/library.d.ts +0 -13
- package/dist/src/library.js +0 -50
- package/dist/src/platform.d.ts +1 -3
- package/dist/src/platform.js +0 -1
- package/dist/src/run-container.d.ts +29 -0
- package/dist/src/run-container.js +74 -2
- package/dist/src/runner.js +10 -2
- package/dist/src/secrets.js +1 -12
- package/dist/src/starter.js +2 -2
- package/dist/src/store.d.ts +15 -10
- package/dist/src/store.js +45 -21
- package/index.ts +0 -3
- package/package.json +1 -3
- package/src/deploy.ts +5 -71
- package/src/history.ts +230 -108
- package/src/library.ts +0 -44
- package/src/platform.ts +1 -4
- package/src/run-container.ts +78 -2
- package/src/runner.ts +10 -1
- package/src/secrets.ts +1 -11
- package/src/starter.ts +2 -2
- package/src/store.ts +55 -27
- package/dist/src/git-hook.d.ts +0 -1
- package/dist/src/git-hook.js +0 -70
- package/dist/src/git.d.ts +0 -37
- package/dist/src/git.js +0 -101
- package/dist/src/gitrepo.d.ts +0 -113
- package/dist/src/gitrepo.js +0 -404
- package/dist/src/tar.d.ts +0 -30
- package/dist/src/tar.js +0 -112
- package/src/git-hook.ts +0 -73
- package/src/git.ts +0 -117
- package/src/gitrepo.ts +0 -439
- package/src/tar.ts +0 -125
package/src/run-container.ts
CHANGED
|
@@ -121,6 +121,47 @@ export interface StepTiming {
|
|
|
121
121
|
sandboxMs: number;
|
|
122
122
|
firstOutputMs: number | null;
|
|
123
123
|
totalMs: number;
|
|
124
|
+
/** Where the start-up went, phase by phase — see STARTUP_PHASES. Absent
|
|
125
|
+
* from an executor that marks none, and from a resumed driver (the
|
|
126
|
+
* first driver's phases are on the record already). */
|
|
127
|
+
phases?: StartupPhases;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* The start-up of one step, in the order it happens. Each is the moment the
|
|
132
|
+
* phase ENDED, in ms since the executor began the step, so the gap between
|
|
133
|
+
* two neighbours is what the later one cost (FOL-29: "pod ready" was the
|
|
134
|
+
* only number, and the six seconds after it had no names).
|
|
135
|
+
*
|
|
136
|
+
* The executor marks its own: `services` (the account's search/browser pods
|
|
137
|
+
* up), `room` (room on the cluster to start — platform pod-room.ts),
|
|
138
|
+
* `applied` (pod created), `ready`, `files` (the workspace copied in),
|
|
139
|
+
* `go` (the step released). The sandbox marks the rest as `{"e":"mark"}`
|
|
140
|
+
* lines: `shim` (the shim saw go), `system` (system packages restored),
|
|
141
|
+
* `driver` (node up), `imports` (core loaded), `tools` (runtime and tools
|
|
142
|
+
* built, the SDK about to start) and `model` (the model's first text or tool
|
|
143
|
+
* call). A worker that predates a mark ignores the line — parseDriverLine
|
|
144
|
+
* answers null for an `e` it does not know.
|
|
145
|
+
*/
|
|
146
|
+
export const STARTUP_PHASES = [
|
|
147
|
+
"services", "room", "applied", "ready", "files", "go",
|
|
148
|
+
"shim", "system", "driver", "imports", "tools", "model",
|
|
149
|
+
] as const;
|
|
150
|
+
export type StartupPhase = (typeof STARTUP_PHASES)[number];
|
|
151
|
+
export type StartupPhases = Partial<Record<StartupPhase, number>>;
|
|
152
|
+
|
|
153
|
+
/** A sandbox-side mark: which phase ended, and when (epoch ms, the
|
|
154
|
+
* sandbox's clock — the same machine's as the worker's on one node). */
|
|
155
|
+
export function parseDriverMark(line: string): { name: StartupPhase; at: number } | null {
|
|
156
|
+
const trimmed = line.trim();
|
|
157
|
+
if (!trimmed.startsWith('{"e":"mark"')) return null;
|
|
158
|
+
try {
|
|
159
|
+
const parsed = JSON.parse(trimmed);
|
|
160
|
+
if (parsed.e !== "mark" || typeof parsed.at !== "number" || !Number.isFinite(parsed.at)) return null;
|
|
161
|
+
return (STARTUP_PHASES as readonly string[]).includes(parsed.name) ? { name: parsed.name, at: parsed.at } : null;
|
|
162
|
+
} catch {
|
|
163
|
+
return null;
|
|
164
|
+
}
|
|
124
165
|
}
|
|
125
166
|
|
|
126
167
|
export interface StepActuals {
|
|
@@ -168,7 +209,11 @@ export interface ContainerStepOutcome {
|
|
|
168
209
|
export function timingLine(t: StepTiming): string {
|
|
169
210
|
const s = (ms: number) => `${(ms / 1000).toFixed(2)}s`;
|
|
170
211
|
const first = t.firstOutputMs === null ? "no output" : `first output +${s(t.firstOutputMs)}`;
|
|
171
|
-
|
|
212
|
+
const phases = t.phases
|
|
213
|
+
? STARTUP_PHASES.filter((p) => typeof t.phases![p] === "number").map((p) => `${p} ${s(t.phases![p]!)}`)
|
|
214
|
+
: [];
|
|
215
|
+
return `timing: sandbox ready in ${s(t.sandboxMs)}, ${first}, ${s(t.totalMs)} total` +
|
|
216
|
+
(phases.length ? ` · start-up: ${phases.join(" → ")}` : "");
|
|
172
217
|
}
|
|
173
218
|
|
|
174
219
|
const cli = () => process.env.FOLDRUN_CONTAINER_CLI ?? "docker";
|
|
@@ -192,6 +237,10 @@ const DENY_BACK_GIT_RE = /(^|\/)\.git(\/|$)/;
|
|
|
192
237
|
const DENY_BACK_SECRET_FILES = /(^|\/)\.secret-files\//;
|
|
193
238
|
// And per-agent knowledge, at any depth under agents/<name>/.
|
|
194
239
|
const DENY_BACK_RE = /^agents\/[^/]+\/knowledge\//;
|
|
240
|
+
// A run's caches are not workspace content either: HOME is the agent's own
|
|
241
|
+
// folder, so a tool that ignores XDG_CACHE_HOME (set by the driver) still
|
|
242
|
+
// writes ~/.cache or ~/.npm there. Never copied back.
|
|
243
|
+
const DENY_BACK_CACHE = /^agents\/[^/]+\/\.(?:cache|npm)(?:\/|$)/;
|
|
195
244
|
// A dependency tree is not workspace content. Nothing authored lives in
|
|
196
245
|
// node_modules, it is enormous, and a runtime linked into a tool's directory
|
|
197
246
|
// so ESM can resolve it would otherwise be copied back file by file.
|
|
@@ -207,6 +256,7 @@ export function allowedBack(rel: string): boolean {
|
|
|
207
256
|
if (DENY_BACK_GIT_RE.test(norm)) return false;
|
|
208
257
|
if (DENY_BACK_SECRET_FILES.test(norm)) return false;
|
|
209
258
|
if (DENY_BACK_NODE_MODULES.test(norm)) return false;
|
|
259
|
+
if (DENY_BACK_CACHE.test(norm)) return false;
|
|
210
260
|
return true;
|
|
211
261
|
}
|
|
212
262
|
|
|
@@ -499,7 +549,17 @@ const DRIVER = `// The runner container's entrypoint. Reads one step, executes i
|
|
|
499
549
|
// same core the server uses, speaks JSONL on stdout, exits.
|
|
500
550
|
import fs from "node:fs";
|
|
501
551
|
const input = JSON.parse(fs.readFileSync("/opt/runner/job/input.json", "utf8"));
|
|
502
|
-
|
|
552
|
+
// Start-up marks (run-container.ts#STARTUP_PHASES): which phase just ended,
|
|
553
|
+
// on the sandbox's clock. A worker that does not know a mark skips the line.
|
|
554
|
+
const mark = (name) => process.stdout.write(JSON.stringify({ e: "mark", name, at: Date.now() }) + "\\n");
|
|
555
|
+
mark("driver");
|
|
556
|
+
let modelSpoke = false;
|
|
557
|
+
const emit = (type, text, extra) => {
|
|
558
|
+
// The model's first text or tool call is where start-up ends and the
|
|
559
|
+
// step's own work begins — runtime and tool lines before it are set-up.
|
|
560
|
+
if (!modelSpoke && (type === "text" || type === "tool")) { modelSpoke = true; mark("model"); }
|
|
561
|
+
process.stdout.write(JSON.stringify({ e: "event", type, text, ...extra }) + "\\n");
|
|
562
|
+
};
|
|
503
563
|
// What this sandbox actually touched, read at the last moment from the
|
|
504
564
|
// kernel's own books: cgroup v2 for CPU busy-time and peak memory,
|
|
505
565
|
// /proc/net/dev for bytes on the wire. Every read is best-effort — gVisor
|
|
@@ -555,6 +615,7 @@ try {
|
|
|
555
615
|
const { prepareRuntime } = await import("@foldrun/core/runtime");
|
|
556
616
|
const { materializeFileSecrets } = await import("@foldrun/core/secret-files");
|
|
557
617
|
const { paidWebActions } = await import("@foldrun/core/browser-pod");
|
|
618
|
+
mark("imports");
|
|
558
619
|
|
|
559
620
|
const agentDir = "/workspace/" + input.agentRel;
|
|
560
621
|
// HOME is where a bare relative path lands when anything resolves one
|
|
@@ -564,6 +625,11 @@ try {
|
|
|
564
625
|
// @file secrets crossed as content; materialise them to 0600 paths inside
|
|
565
626
|
// the container (a host path from the other side would not exist here).
|
|
566
627
|
const env = materializeFileSecrets(agentDir, { ...process.env, HOME: agentDir }).env;
|
|
628
|
+
// Caches go to the sandbox user's own home, not to HOME: the Claude CLI's
|
|
629
|
+
// logs, fontconfig, Mesa and gstreamer all follow XDG_CACHE_HOME, and under
|
|
630
|
+
// agents/<name>/.cache they were copied back after every step — 3,855 files
|
|
631
|
+
// across 179 agent folders on the box by 17 Oct 2026, read by nothing.
|
|
632
|
+
env.XDG_CACHE_HOME = "/home/agent/.cache";
|
|
567
633
|
|
|
568
634
|
// The declared runtime, built in here — pip and npm have the network, which
|
|
569
635
|
// the host side may not. It lands in .foldrun/runner/.runtimes, which the
|
|
@@ -632,6 +698,7 @@ try {
|
|
|
632
698
|
const ask = input.ask ? buildAskTool({ channel, timeoutSec: input.ask.timeoutSec, emit }) : { server: null };
|
|
633
699
|
const inbox = channel ? inboxReader(channel, emit) : undefined;
|
|
634
700
|
|
|
701
|
+
mark("tools");
|
|
635
702
|
const outcome = await executeStep({
|
|
636
703
|
agentDir,
|
|
637
704
|
workspaceRoot: "/workspace",
|
|
@@ -1460,6 +1527,8 @@ export async function runStepInContainer(args: RunInContainerArgs): Promise<Cont
|
|
|
1460
1527
|
if (create.status !== 0) throw new Error(`container create failed:\n${create.stderr}`);
|
|
1461
1528
|
containerId = create.stdout.trim();
|
|
1462
1529
|
const sandboxMs = Date.now() - t0;
|
|
1530
|
+
// Docker's phases are measured from create, its rent clock.
|
|
1531
|
+
const phases: StartupPhases = { ready: sandboxMs };
|
|
1463
1532
|
|
|
1464
1533
|
const cp = (from: string, to: string) => {
|
|
1465
1534
|
const out = spawnSync(cli(), ["cp", from, to], { encoding: "utf8" });
|
|
@@ -1468,6 +1537,7 @@ export async function runStepInContainer(args: RunInContainerArgs): Promise<Cont
|
|
|
1468
1537
|
cp(`${wsIn}/.`, `${containerId}:/workspace`);
|
|
1469
1538
|
cp(`${libIn}/.`, `${containerId}:/library`);
|
|
1470
1539
|
cp(`${jobIn}/input.json`, `${containerId}:/opt/runner/job/input.json`);
|
|
1540
|
+
phases.files = Date.now() - t0;
|
|
1471
1541
|
|
|
1472
1542
|
// Start attached and read the driver's JSONL. The step timeout gets a
|
|
1473
1543
|
// backstop out here — a wedged container is killed, not waited on.
|
|
@@ -1493,6 +1563,11 @@ export async function runStepInContainer(args: RunInContainerArgs): Promise<Cont
|
|
|
1493
1563
|
if (nl < 0) break;
|
|
1494
1564
|
const line = buffer.slice(0, nl);
|
|
1495
1565
|
buffer = buffer.slice(nl + 1);
|
|
1566
|
+
const marked = parseDriverMark(line);
|
|
1567
|
+
if (marked) {
|
|
1568
|
+
phases[marked.name] ??= Math.max(0, marked.at - t0);
|
|
1569
|
+
continue;
|
|
1570
|
+
}
|
|
1496
1571
|
const parsed = parseDriverLine(line);
|
|
1497
1572
|
if (!parsed) continue;
|
|
1498
1573
|
firstOutputAt ??= Date.now();
|
|
@@ -1536,6 +1611,7 @@ export async function runStepInContainer(args: RunInContainerArgs): Promise<Cont
|
|
|
1536
1611
|
sandboxMs,
|
|
1537
1612
|
firstOutputMs: firstOutputAt === null ? null : firstOutputAt - t0 - sandboxMs,
|
|
1538
1613
|
totalMs: Date.now() - t0,
|
|
1614
|
+
phases,
|
|
1539
1615
|
};
|
|
1540
1616
|
args.emit("info", timingLine(timing));
|
|
1541
1617
|
return { ...outcome, timing };
|
package/src/runner.ts
CHANGED
|
@@ -49,6 +49,7 @@ import {
|
|
|
49
49
|
listWorkspaces,
|
|
50
50
|
listTenants,
|
|
51
51
|
listRuns,
|
|
52
|
+
liveRuns,
|
|
52
53
|
syncBundleFor,
|
|
53
54
|
parseToolDef,
|
|
54
55
|
type ToolDef,
|
|
@@ -2890,6 +2891,12 @@ async function runStep(
|
|
|
2890
2891
|
const secs = (ms: number) => Math.round(ms) / 1000;
|
|
2891
2892
|
step.computeSecs = outcome.timing ? secs(outcome.timing.totalMs) : null;
|
|
2892
2893
|
step.startupSecs = outcome.timing ? secs(outcome.timing.sandboxMs) : null;
|
|
2894
|
+
// Where the start-up went, phase by phase (run-container.ts#STARTUP_PHASES),
|
|
2895
|
+
// in seconds — what the cold-start numbers are computed from.
|
|
2896
|
+
const phases = outcome.timing?.phases;
|
|
2897
|
+
step.startup = phases && Object.keys(phases).length
|
|
2898
|
+
? Object.fromEntries(Object.entries(phases).map(([k, v]) => [k, secs(v)]))
|
|
2899
|
+
: null;
|
|
2893
2900
|
// Which reservation those seconds held — as numbers, because the bill
|
|
2894
2901
|
// is per core-second and per GiB-second, and what was held must come
|
|
2895
2902
|
// from the record, never from what the config says at billing time.
|
|
@@ -5054,7 +5061,9 @@ export function reconcileRuns(tenant: string, now = Date.now(), live?: ReadonlyS
|
|
|
5054
5061
|
const closed: Reconciliation[] = [];
|
|
5055
5062
|
|
|
5056
5063
|
for (const workspace of listWorkspaces(tenant)) {
|
|
5057
|
-
|
|
5064
|
+
// Live runs only, read whole — every other status is passed over below,
|
|
5065
|
+
// and parsing the workspace's whole history every sweep was the cost.
|
|
5066
|
+
for (const summary of liveRuns(tenant, workspace.name)) {
|
|
5058
5067
|
if (drivingRuns.has(summary.id)) continue;
|
|
5059
5068
|
// Driven by another process — a second worker pod, on a platform with
|
|
5060
5069
|
// more than one. The idle window protects a quiet step from THIS
|
package/src/secrets.ts
CHANGED
|
@@ -252,11 +252,9 @@ export function setSecretSites(tenant: string, name: string, sites: string[], wo
|
|
|
252
252
|
export function secretSites(tenant: string, names: string[], workspace?: string): Record<string, string[]> {
|
|
253
253
|
const out: Record<string, string[]> = {};
|
|
254
254
|
const own = workspace ? read(tenant, workspace) : {};
|
|
255
|
-
const source = workspace ? platform.previewSourceOf(tenant, workspace) : null;
|
|
256
|
-
const inherited = source ? read(tenant, source) : {};
|
|
257
255
|
const account = read(tenant);
|
|
258
256
|
for (const name of names) {
|
|
259
|
-
const rec = own[name] ??
|
|
257
|
+
const rec = own[name] ?? account[name];
|
|
260
258
|
if (rec?.sites?.length) out[name] = [...rec.sites];
|
|
261
259
|
}
|
|
262
260
|
return out;
|
|
@@ -349,14 +347,6 @@ export function getSecret(
|
|
|
349
347
|
if (workspace) {
|
|
350
348
|
const own = decrypt(tenant, read(tenant, workspace)[name]);
|
|
351
349
|
if (own !== null) return { value: own, scope: "workspace" };
|
|
352
|
-
// A preview is a branch of its source workspace, and the branch's
|
|
353
|
-
// agents need the same credentials the source's do. Its own store is
|
|
354
|
-
// still consulted first, so a preview can override one on purpose.
|
|
355
|
-
const source = platform.previewSourceOf(tenant, workspace);
|
|
356
|
-
if (source) {
|
|
357
|
-
const inherited = decrypt(tenant, read(tenant, source)[name]);
|
|
358
|
-
if (inherited !== null) return { value: inherited, scope: "workspace" };
|
|
359
|
-
}
|
|
360
350
|
}
|
|
361
351
|
const shared = decrypt(tenant, read(tenant)[name]);
|
|
362
352
|
return shared === null ? null : { value: shared, scope: "account" };
|
package/src/starter.ts
CHANGED
|
@@ -389,8 +389,8 @@ Debugging a failed run is usually: \`runs\` to find it, \`runs/<id>\` to read wh
|
|
|
389
389
|
step failed and what it cost, fix the markdown, \`deploy\`, then \`rerun\`.
|
|
390
390
|
|
|
391
391
|
Every route, with request and response shapes, is in \`docs/api.md\` in the
|
|
392
|
-
foldrun repository — including secrets, keys, billing, OAuth, webhooks and
|
|
393
|
-
|
|
392
|
+
foldrun repository — including secrets, keys, billing, OAuth, webhooks and
|
|
393
|
+
restore.
|
|
394
394
|
|
|
395
395
|
## Writing a flow
|
|
396
396
|
|
package/src/store.ts
CHANGED
|
@@ -2095,7 +2095,7 @@ export function mayApprove(
|
|
|
2095
2095
|
|
|
2096
2096
|
/** A live run of this flow — the fact overlap: decisions are made on. */
|
|
2097
2097
|
export function flowHasLiveRun(tenant: string, workspace: string, flowName: string): boolean {
|
|
2098
|
-
return
|
|
2098
|
+
return listRunSummaries(tenant, workspace).some(
|
|
2099
2099
|
(r) => r.flow === flowName && (r.status === "running" || r.status === "queued" || r.status === "awaiting-approval"),
|
|
2100
2100
|
);
|
|
2101
2101
|
}
|
|
@@ -2396,10 +2396,6 @@ export interface WorkspaceSummary {
|
|
|
2396
2396
|
flows: number;
|
|
2397
2397
|
deployedAt: string;
|
|
2398
2398
|
runCount: number;
|
|
2399
|
-
/** Set when this workspace is a branch's preview of another (preview.ts):
|
|
2400
|
-
* which one, which branch, which commit. Read from the marker beside the
|
|
2401
|
-
* directory here rather than through preview.ts, which imports this file. */
|
|
2402
|
-
preview?: { source: string; branch: string; commit: string; at: string };
|
|
2403
2399
|
/** The workspace's picture, as a URL, when the page that lists it looked
|
|
2404
2400
|
* one up (avatars.ts) — the store itself does not. */
|
|
2405
2401
|
avatar?: string | null;
|
|
@@ -2460,12 +2456,6 @@ export function listWorkspaces(tenant: string): WorkspaceSummary[] {
|
|
|
2460
2456
|
}
|
|
2461
2457
|
}
|
|
2462
2458
|
const runsDir = path.join(pDir, "runs");
|
|
2463
|
-
let preview: WorkspaceSummary["preview"];
|
|
2464
|
-
try {
|
|
2465
|
-
preview = JSON.parse(fs.readFileSync(path.join(dir, `.${name}.preview.json`), "utf8"));
|
|
2466
|
-
} catch {
|
|
2467
|
-
// not a preview
|
|
2468
|
-
}
|
|
2469
2459
|
return {
|
|
2470
2460
|
name,
|
|
2471
2461
|
description,
|
|
@@ -2473,7 +2463,6 @@ export function listWorkspaces(tenant: string): WorkspaceSummary[] {
|
|
|
2473
2463
|
flows: listFlows(tenant, name).length,
|
|
2474
2464
|
deployedAt: fs.statSync(pDir).mtime.toISOString(),
|
|
2475
2465
|
runCount: fs.existsSync(runsDir) ? fs.readdirSync(runsDir).filter((f) => f.endsWith(".json") && !f.startsWith(".")).length : 0,
|
|
2476
|
-
...(preview ? { preview } : {}),
|
|
2477
2466
|
};
|
|
2478
2467
|
})
|
|
2479
2468
|
.sort((a, b) => b.deployedAt.localeCompare(a.deployedAt));
|
|
@@ -2709,7 +2698,7 @@ export function writeWorkspaceFile(
|
|
|
2709
2698
|
}
|
|
2710
2699
|
|
|
2711
2700
|
/** Tell the platform, and never let its listener fail a write. */
|
|
2712
|
-
function workspaceChanged(tenant: string, workspace: string, why: "write" | "deploy" | "
|
|
2701
|
+
function workspaceChanged(tenant: string, workspace: string, why: "write" | "deploy" | "delete") {
|
|
2713
2702
|
try {
|
|
2714
2703
|
platform.workspaceChanged(tenant, workspace, why);
|
|
2715
2704
|
} catch {
|
|
@@ -3291,6 +3280,12 @@ export interface StepRecord {
|
|
|
3291
3280
|
*/
|
|
3292
3281
|
computeSecs?: number | null;
|
|
3293
3282
|
startupSecs?: number | null;
|
|
3283
|
+
/** When each start-up phase ended, in seconds since the executor began
|
|
3284
|
+
* the step — `services`, `applied`, `ready`, `files`, `go`, `shim`,
|
|
3285
|
+
* `system`, `driver`, `imports`, `tools`, `model` (run-container.ts
|
|
3286
|
+
* #STARTUP_PHASES). Only the phases this step went through; null from
|
|
3287
|
+
* an executor that marks none, absent on steps from before 17 Oct 2026. */
|
|
3288
|
+
startup?: Record<string, number> | null;
|
|
3294
3289
|
/** Which reservation class the sandbox held (small | large), and the
|
|
3295
3290
|
* actual limits it held, as numbers. Facts of the step: a price is per
|
|
3296
3291
|
* core-second, so what was reserved must come from the record, never
|
|
@@ -3512,6 +3507,23 @@ export function writeRun(tenant: string, workspace: string, run: RunRecord) {
|
|
|
3512
3507
|
noteRunWritten(path.dirname(p), run, p);
|
|
3513
3508
|
}
|
|
3514
3509
|
|
|
3510
|
+
/**
|
|
3511
|
+
* The workspace's live runs — queued, running, or parked at a gate — read
|
|
3512
|
+
* whole, newest first. Off the run index: only the live records are opened,
|
|
3513
|
+
* where listRuns parses every record the workspace ever kept. What the
|
|
3514
|
+
* periodic sweeps want (reconcile, the watchdog), and on a box that keeps
|
|
3515
|
+
* ninety days of runs that is a handful of files instead of thousands.
|
|
3516
|
+
*/
|
|
3517
|
+
export function liveRuns(tenant: string, workspace: string): RunRecord[] {
|
|
3518
|
+
const out: RunRecord[] = [];
|
|
3519
|
+
for (const s of listRunSummaries(tenant, workspace)) {
|
|
3520
|
+
if (!LIVE_STATUSES.has(s.status)) continue;
|
|
3521
|
+
const run = readRun(tenant, workspace, s.id);
|
|
3522
|
+
if (run) out.push(run);
|
|
3523
|
+
}
|
|
3524
|
+
return out;
|
|
3525
|
+
}
|
|
3526
|
+
|
|
3515
3527
|
// Every run in an account, each tagged with the workspace it belongs to —
|
|
3516
3528
|
// the observability surface.
|
|
3517
3529
|
export interface RunWithWorkspace extends RunRecord {
|
|
@@ -3603,7 +3615,18 @@ export interface RunSummary {
|
|
|
3603
3615
|
interface RunIndexRow extends RunSummary {
|
|
3604
3616
|
/** mtime of the record this row was made from; a moved mtime means re-read. */
|
|
3605
3617
|
mtimeMs: number;
|
|
3618
|
+
/** mtime, inode and size together (fileStamp) — what freshness is decided
|
|
3619
|
+
* on now. A row from before it has none and is re-read once. */
|
|
3620
|
+
stamp?: string;
|
|
3606
3621
|
}
|
|
3622
|
+
|
|
3623
|
+
/**
|
|
3624
|
+
* Which version of a file this is. mtime alone missed a rewrite inside one
|
|
3625
|
+
* mtime tick (coarse on some NFS servers), leaving a stale row that the
|
|
3626
|
+
* sweeps act on; every write here is a rename to a new file, so the inode
|
|
3627
|
+
* moves on each write whatever the clock says, and the size usually does.
|
|
3628
|
+
*/
|
|
3629
|
+
const fileStamp = (st: fs.Stats) => `${st.mtimeMs}:${st.ino}:${st.size}`;
|
|
3607
3630
|
interface RunIndex {
|
|
3608
3631
|
v: 1;
|
|
3609
3632
|
runs: Record<string, RunIndexRow>;
|
|
@@ -3668,23 +3691,23 @@ export function runFromSummary(s: RunSummary): RunRecord {
|
|
|
3668
3691
|
};
|
|
3669
3692
|
}
|
|
3670
3693
|
|
|
3671
|
-
const runIndexCache = new Map<string, { index: RunIndex;
|
|
3694
|
+
const runIndexCache = new Map<string, { index: RunIndex; stamp: string }>();
|
|
3672
3695
|
|
|
3673
3696
|
function readRunIndex(dir: string): RunIndex | null {
|
|
3674
3697
|
const file = path.join(dir, RUN_INDEX);
|
|
3675
|
-
let
|
|
3698
|
+
let stamp: string;
|
|
3676
3699
|
try {
|
|
3677
|
-
|
|
3700
|
+
stamp = fileStamp(fs.statSync(file));
|
|
3678
3701
|
} catch {
|
|
3679
3702
|
runIndexCache.delete(dir);
|
|
3680
3703
|
return null;
|
|
3681
3704
|
}
|
|
3682
3705
|
const cached = runIndexCache.get(dir);
|
|
3683
|
-
if (cached && cached.
|
|
3706
|
+
if (cached && cached.stamp === stamp) return cached.index;
|
|
3684
3707
|
try {
|
|
3685
3708
|
const parsed = JSON.parse(fs.readFileSync(file, "utf8")) as RunIndex;
|
|
3686
3709
|
if (parsed?.v !== 1 || typeof parsed.runs !== "object" || !parsed.runs) return null;
|
|
3687
|
-
runIndexCache.set(dir, { index: parsed,
|
|
3710
|
+
runIndexCache.set(dir, { index: parsed, stamp });
|
|
3688
3711
|
return parsed;
|
|
3689
3712
|
} catch {
|
|
3690
3713
|
return null;
|
|
@@ -3697,14 +3720,17 @@ function writeRunIndex(dir: string, index: RunIndex) {
|
|
|
3697
3720
|
fs.writeFileSync(tmp, JSON.stringify(index));
|
|
3698
3721
|
fs.renameSync(tmp, file);
|
|
3699
3722
|
try {
|
|
3700
|
-
runIndexCache.set(dir, { index,
|
|
3723
|
+
runIndexCache.set(dir, { index, stamp: fileStamp(fs.statSync(file)) });
|
|
3701
3724
|
} catch {
|
|
3702
3725
|
runIndexCache.delete(dir);
|
|
3703
3726
|
}
|
|
3704
3727
|
}
|
|
3705
3728
|
|
|
3706
|
-
const sameRow = (a: RunIndexRow | undefined, b: RunSummary) =>
|
|
3707
|
-
|
|
3729
|
+
const sameRow = (a: RunIndexRow | undefined, b: RunSummary) => {
|
|
3730
|
+
if (!a) return false;
|
|
3731
|
+
const { mtimeMs: _m, stamp: _s, ...row } = a;
|
|
3732
|
+
return JSON.stringify(row) === JSON.stringify(b);
|
|
3733
|
+
};
|
|
3708
3734
|
|
|
3709
3735
|
/** writeRun just saved `run` at `file`: bring its row up to date. The file
|
|
3710
3736
|
* is rewritten only when the row's content moved — a live step appends an
|
|
@@ -3717,7 +3743,8 @@ function noteRunWritten(dir: string, run: RunRecord, file: string) {
|
|
|
3717
3743
|
if (!index) return; // no index yet — the first reader builds it whole
|
|
3718
3744
|
const row = summarizeRun(run);
|
|
3719
3745
|
if (sameRow(index.runs[run.id], row)) return;
|
|
3720
|
-
|
|
3746
|
+
const st = fs.statSync(file);
|
|
3747
|
+
index.runs[run.id] = { ...row, mtimeMs: st.mtimeMs, stamp: fileStamp(st) };
|
|
3721
3748
|
writeRunIndex(dir, index);
|
|
3722
3749
|
} catch {
|
|
3723
3750
|
// the index is a cache; the record is the truth and it is already on disk
|
|
@@ -3752,17 +3779,18 @@ export function listRunSummaries(tenant: string, workspace: string): RunSummary[
|
|
|
3752
3779
|
if (!f.endsWith(".json") || f.startsWith(".")) continue;
|
|
3753
3780
|
const id = f.slice(0, -".json".length);
|
|
3754
3781
|
present.add(id);
|
|
3755
|
-
let
|
|
3782
|
+
let st: fs.Stats;
|
|
3756
3783
|
try {
|
|
3757
|
-
|
|
3784
|
+
st = fs.statSync(path.join(dir, f));
|
|
3758
3785
|
} catch {
|
|
3759
3786
|
continue; // pruned between the listing and the stat
|
|
3760
3787
|
}
|
|
3788
|
+
const stamp = fileStamp(st);
|
|
3761
3789
|
const row = index.runs[id];
|
|
3762
|
-
if (row && row.
|
|
3790
|
+
if (row && row.stamp === stamp) continue;
|
|
3763
3791
|
try {
|
|
3764
3792
|
const run = JSON.parse(fs.readFileSync(path.join(dir, f), "utf8")) as RunRecord;
|
|
3765
|
-
index.runs[id] = { ...summarizeRun(run), mtimeMs };
|
|
3793
|
+
index.runs[id] = { ...summarizeRun(run), mtimeMs: st.mtimeMs, stamp };
|
|
3766
3794
|
changed = true;
|
|
3767
3795
|
} catch {
|
|
3768
3796
|
// a torn or foreign file is not a run; leave whatever row it had
|
|
@@ -3781,7 +3809,7 @@ export function listRunSummaries(tenant: string, workspace: string): RunSummary[
|
|
|
3781
3809
|
}
|
|
3782
3810
|
}
|
|
3783
3811
|
return Object.values(index.runs)
|
|
3784
|
-
.map(({ mtimeMs: _m, ...row }) => row)
|
|
3812
|
+
.map(({ mtimeMs: _m, stamp: _s, ...row }) => row)
|
|
3785
3813
|
.sort((a, b) => b.startedAt.localeCompare(a.startedAt));
|
|
3786
3814
|
}
|
|
3787
3815
|
|
package/dist/src/git-hook.d.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
package/dist/src/git-hook.js
DELETED
|
@@ -1,70 +0,0 @@
|
|
|
1
|
-
// The pre-receive check, run by git inside a push.
|
|
2
|
-
//
|
|
3
|
-
// git feeds "<old> <new> <ref>" lines on stdin. For a push to main, read the
|
|
4
|
-
// tree at the new commit straight from the object store and run the deploy
|
|
5
|
-
// checks on it. Any issue: print them and exit non-zero — the push is refused
|
|
6
|
-
// and the person sees exactly what a dashboard deploy would have told them,
|
|
7
|
-
// in their terminal, before anything changed.
|
|
8
|
-
//
|
|
9
|
-
// Runs as its own process under git's environment. GIT_DIR is set by git.
|
|
10
|
-
import fs from "node:fs";
|
|
11
|
-
import path from "node:path";
|
|
12
|
-
import { spawnSync } from "node:child_process";
|
|
13
|
-
import { deployIssues, runsInFlight } from "./deploy.js";
|
|
14
|
-
const dir = process.env.GIT_DIR;
|
|
15
|
-
if (!dir)
|
|
16
|
-
process.exit(0);
|
|
17
|
-
// `_library.git` is the account shelf, not a workspace: it holds tools,
|
|
18
|
-
// skills and knowledge and has no agents by definition. The workspace rules
|
|
19
|
-
// refused every push to it with "no agents". Its sync (syncLibraryFromTree)
|
|
20
|
-
// validates what it accepts on receive; nothing here to add yet.
|
|
21
|
-
// Inside a hook git sets GIT_DIR relative to the repo ("."), so resolve it
|
|
22
|
-
// against the cwd before asking for its name.
|
|
23
|
-
if (path.basename(path.resolve(process.cwd(), dir)) === "_library.git")
|
|
24
|
-
process.exit(0);
|
|
25
|
-
const input = fs.readFileSync(0, "utf8");
|
|
26
|
-
const git = (args) => {
|
|
27
|
-
const r = spawnSync("git", ["--git-dir", dir, ...args], { maxBuffer: 64 * 1024 * 1024 });
|
|
28
|
-
return r.status === 0 ? r.stdout.toString() : null;
|
|
29
|
-
};
|
|
30
|
-
// The client is gone by the time the deploy runs, so this is the only moment
|
|
31
|
-
// a person can be told their push will not take effect yet. A warning, not a
|
|
32
|
-
// refusal: the commit is fine and the scheduler applies it when the runs end.
|
|
33
|
-
const here = path.basename(path.resolve(process.cwd(), dir));
|
|
34
|
-
if (here.endsWith(".git")) {
|
|
35
|
-
const workspace = here.slice(0, -4);
|
|
36
|
-
const tenant = path.basename(path.resolve(process.cwd(), dir, "..", ".."));
|
|
37
|
-
try {
|
|
38
|
-
const live = runsInFlight(tenant, workspace);
|
|
39
|
-
if (live.length) {
|
|
40
|
-
process.stderr.write(`\nfoldrun: ${live.length} run${live.length === 1 ? " is" : "s are"} in flight, so this push is not live yet.\n` +
|
|
41
|
-
` It is on main and will be applied automatically when they finish.\n\n`);
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
catch {
|
|
45
|
-
// A warning that cannot be computed must never fail a push.
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
let refused = false;
|
|
49
|
-
for (const line of input.split("\n").filter(Boolean)) {
|
|
50
|
-
const [, newSha, ref] = line.split(" ");
|
|
51
|
-
if (ref !== "refs/heads/main" || /^0+$/.test(newSha))
|
|
52
|
-
continue;
|
|
53
|
-
const tree = git(["ls-tree", "-r", "--name-only", newSha]) ?? "";
|
|
54
|
-
const files = tree
|
|
55
|
-
.split("\n")
|
|
56
|
-
.filter(Boolean)
|
|
57
|
-
.flatMap((p) => {
|
|
58
|
-
const content = git(["show", `${newSha}:${p}`]);
|
|
59
|
-
return content === null ? [] : [{ path: p, content }];
|
|
60
|
-
});
|
|
61
|
-
const issues = deployIssues(files);
|
|
62
|
-
if (issues.length) {
|
|
63
|
-
refused = true;
|
|
64
|
-
process.stderr.write(`\nfoldrun: refusing this push — ${issues.length} problem${issues.length === 1 ? "" : "s"} the checks would have stopped at deploy:\n`);
|
|
65
|
-
for (const i of issues)
|
|
66
|
-
process.stderr.write(` ${i.where}: ${i.message}\n`);
|
|
67
|
-
process.stderr.write(`\nFix and push again. (foldrun check runs the same rules locally.)\n\n`);
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
process.exit(refused ? 1 : 0);
|
package/dist/src/git.d.ts
DELETED
|
@@ -1,37 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* The secret a repository signs its pushes with.
|
|
3
|
-
*
|
|
4
|
-
* Derived rather than stored, exactly like a flow's webhook token: HMAC of the
|
|
5
|
-
* workspace's identity under the install key. Nothing new to persist, unique
|
|
6
|
-
* per workspace, and rotating FOLDRUN_SECRET_KEY invalidates every connection
|
|
7
|
-
* at once.
|
|
8
|
-
*/
|
|
9
|
-
export declare function gitSecret(tenant: string, workspace: string): string;
|
|
10
|
-
export declare function gitHookPath(tenant: string, workspace: string): string;
|
|
11
|
-
/**
|
|
12
|
-
* Whether a request really came from the repository it claims to.
|
|
13
|
-
*
|
|
14
|
-
* Compared in constant time: a byte-by-byte early return leaks the expected
|
|
15
|
-
* signature to anyone willing to time enough requests. The raw body is
|
|
16
|
-
* required — re-serializing parsed JSON changes bytes GitHub signed.
|
|
17
|
-
*/
|
|
18
|
-
export declare function verifySignature(secret: string, rawBody: string, header: string | null): boolean;
|
|
19
|
-
export interface PushEvent {
|
|
20
|
-
repo: string;
|
|
21
|
-
/** Full ref, e.g. refs/heads/main. */
|
|
22
|
-
ref: string;
|
|
23
|
-
branch: string;
|
|
24
|
-
commit: string;
|
|
25
|
-
message: string | null;
|
|
26
|
-
pusher: string | null;
|
|
27
|
-
}
|
|
28
|
-
/** What we need out of a GitHub push payload, or null if it isn't one. */
|
|
29
|
-
export declare function parsePush(payload: unknown): PushEvent | null;
|
|
30
|
-
/**
|
|
31
|
-
* The repository at one commit, as a gzipped tarball.
|
|
32
|
-
*
|
|
33
|
-
* A token is only needed for a private repository. It is read from the
|
|
34
|
-
* environment rather than taken as an argument so a caller cannot be tricked
|
|
35
|
-
* into forwarding one somewhere else.
|
|
36
|
-
*/
|
|
37
|
-
export declare function fetchTarball(repo: string, commit: string): Promise<Buffer>;
|
package/dist/src/git.js
DELETED
|
@@ -1,101 +0,0 @@
|
|
|
1
|
-
// git push → deployed.
|
|
2
|
-
//
|
|
3
|
-
// A workspace is markdown, so there is no build to run: a push is a fetch of
|
|
4
|
-
// the tree at that commit, a check, and a swap. This module is the GitHub half
|
|
5
|
-
// — the secret a repository signs with, the payload shape, and pulling the
|
|
6
|
-
// tarball. `deploy.ts` decides whether the result is allowed to go live.
|
|
7
|
-
//
|
|
8
|
-
// The tarball, not a clone: one HTTP call, no git binary, no working copy kept
|
|
9
|
-
// on the server. A markdown workspace is a few hundred kilobytes.
|
|
10
|
-
import crypto from "node:crypto";
|
|
11
|
-
import { installKey } from "./webhook.js";
|
|
12
|
-
/** GitHub refuses to fetch anything larger, and neither should we. */
|
|
13
|
-
const MAX_TARBALL = 25 * 1024 * 1024;
|
|
14
|
-
/**
|
|
15
|
-
* The secret a repository signs its pushes with.
|
|
16
|
-
*
|
|
17
|
-
* Derived rather than stored, exactly like a flow's webhook token: HMAC of the
|
|
18
|
-
* workspace's identity under the install key. Nothing new to persist, unique
|
|
19
|
-
* per workspace, and rotating FOLDRUN_SECRET_KEY invalidates every connection
|
|
20
|
-
* at once.
|
|
21
|
-
*/
|
|
22
|
-
export function gitSecret(tenant, workspace) {
|
|
23
|
-
return crypto
|
|
24
|
-
.createHmac("sha256", installKey())
|
|
25
|
-
.update(`git/${tenant}/${workspace}`)
|
|
26
|
-
.digest("hex");
|
|
27
|
-
}
|
|
28
|
-
export function gitHookPath(tenant, workspace) {
|
|
29
|
-
return `/api/git/${tenant}/${workspace}`;
|
|
30
|
-
}
|
|
31
|
-
/**
|
|
32
|
-
* Whether a request really came from the repository it claims to.
|
|
33
|
-
*
|
|
34
|
-
* Compared in constant time: a byte-by-byte early return leaks the expected
|
|
35
|
-
* signature to anyone willing to time enough requests. The raw body is
|
|
36
|
-
* required — re-serializing parsed JSON changes bytes GitHub signed.
|
|
37
|
-
*/
|
|
38
|
-
export function verifySignature(secret, rawBody, header) {
|
|
39
|
-
if (!header)
|
|
40
|
-
return false;
|
|
41
|
-
const expected = `sha256=${crypto.createHmac("sha256", secret).update(rawBody).digest("hex")}`;
|
|
42
|
-
const a = Buffer.from(expected);
|
|
43
|
-
const b = Buffer.from(header);
|
|
44
|
-
return a.length === b.length && crypto.timingSafeEqual(a, b);
|
|
45
|
-
}
|
|
46
|
-
/** What we need out of a GitHub push payload, or null if it isn't one. */
|
|
47
|
-
export function parsePush(payload) {
|
|
48
|
-
if (!payload || typeof payload !== "object")
|
|
49
|
-
return null;
|
|
50
|
-
const p = payload;
|
|
51
|
-
const repo = p.repository?.full_name;
|
|
52
|
-
const ref = p.ref;
|
|
53
|
-
const commit = p.after;
|
|
54
|
-
if (typeof repo !== "string" || typeof ref !== "string" || typeof commit !== "string") {
|
|
55
|
-
return null;
|
|
56
|
-
}
|
|
57
|
-
// A deleted branch reports all zeros and ships no tree.
|
|
58
|
-
if (/^0+$/.test(commit))
|
|
59
|
-
return null;
|
|
60
|
-
return {
|
|
61
|
-
repo,
|
|
62
|
-
ref,
|
|
63
|
-
branch: ref.replace(/^refs\/heads\//, ""),
|
|
64
|
-
commit,
|
|
65
|
-
message: typeof p.head_commit?.message === "string" ? p.head_commit.message : null,
|
|
66
|
-
pusher: typeof p.pusher?.name === "string" ? p.pusher.name : null,
|
|
67
|
-
};
|
|
68
|
-
}
|
|
69
|
-
/**
|
|
70
|
-
* The repository at one commit, as a gzipped tarball.
|
|
71
|
-
*
|
|
72
|
-
* A token is only needed for a private repository. It is read from the
|
|
73
|
-
* environment rather than taken as an argument so a caller cannot be tricked
|
|
74
|
-
* into forwarding one somewhere else.
|
|
75
|
-
*/
|
|
76
|
-
export async function fetchTarball(repo, commit) {
|
|
77
|
-
if (!/^[\w.-]+\/[\w.-]+$/.test(repo))
|
|
78
|
-
throw new Error(`refusing to fetch "${repo}"`);
|
|
79
|
-
if (!/^[0-9a-f]{7,40}$/i.test(commit))
|
|
80
|
-
throw new Error(`refusing to fetch commit "${commit}"`);
|
|
81
|
-
const headers = {
|
|
82
|
-
accept: "application/vnd.github+json",
|
|
83
|
-
"user-agent": "foldrun",
|
|
84
|
-
"x-github-api-version": "2022-11-28",
|
|
85
|
-
};
|
|
86
|
-
const token = process.env.GITHUB_TOKEN;
|
|
87
|
-
if (token)
|
|
88
|
-
headers.authorization = `Bearer ${token}`;
|
|
89
|
-
const res = await fetch(`https://api.github.com/repos/${repo}/tarball/${commit}`, { headers });
|
|
90
|
-
if (!res.ok) {
|
|
91
|
-
throw new Error(`GitHub returned ${res.status} for ${repo}@${commit.slice(0, 8)}` +
|
|
92
|
-
(res.status === 404 && !token
|
|
93
|
-
? " — a private repository needs GITHUB_TOKEN set on the server"
|
|
94
|
-
: ""));
|
|
95
|
-
}
|
|
96
|
-
const buf = Buffer.from(await res.arrayBuffer());
|
|
97
|
-
if (buf.length > MAX_TARBALL) {
|
|
98
|
-
throw new Error(`${repo} is ${Math.round(buf.length / 1e6)}MB — too large to deploy`);
|
|
99
|
-
}
|
|
100
|
-
return buf;
|
|
101
|
-
}
|