@foldrun/core 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,5 +1,9 @@
1
1
  # foldrun
2
2
 
3
+ [![ci](https://github.com/foldrun-io/foldrun-core/actions/workflows/ci.yml/badge.svg)](https://github.com/foldrun-io/foldrun-core/actions/workflows/ci.yml)
4
+ [![npm](https://img.shields.io/npm/v/@foldrun/core)](https://www.npmjs.com/package/@foldrun/core)
5
+ [![licence](https://img.shields.io/badge/licence-Apache--2.0-blue)](LICENSE)
6
+
3
7
  **Agents are just folders. Write them, check them, run them.**
4
8
 
5
9
  An agent is a folder. Its instructions, skills, knowledge, memory, tools and
@@ -20,7 +20,7 @@
20
20
  import fs from "node:fs";
21
21
  import os from "node:os";
22
22
  import path from "node:path";
23
- import { saveWorkspace, workspaceDir, listRuns, parseFlow, WORKSPACE_DIRS, assertSafeName, } from "./store.js";
23
+ import { saveWorkspace, workspaceDir, listRuns, parseFlow, WORKSPACE_DIRS, assertSafeName, notifyWorkspaceChanged, } from "./store.js";
24
24
  import { conformanceIssues } from "./okf.js";
25
25
  import { repoDir } from "./gitrepo.js";
26
26
  /** Directories a deploy never reads out of a source tree. */
@@ -307,6 +307,8 @@ export function writePushNote(tenant, workspace, note) {
307
307
  catch {
308
308
  // The note is a record, not the deploy. Losing it must not fail a push.
309
309
  }
310
+ // A push waiting on a run is the scheduler's to apply; tell it there is one.
311
+ notifyWorkspaceChanged(tenant, workspace, "push");
310
312
  }
311
313
  /**
312
314
  * A push whose deploy was refused only because a run was in flight is not a
@@ -353,7 +355,7 @@ export function deployWorkspace(tenant, workspace, files, opts = {}) {
353
355
  const commit = opts.commit ?? null;
354
356
  if (commit)
355
357
  writeDeployedCommit(tenant, workspace, commit);
356
- return { ...plan, applied: true, commit, preserved };
358
+ return { ...plan, applied: true, commit, preserved }; // saveWorkspace told the platform
357
359
  }
358
360
  // Which commit a workspace is running.
359
361
  //
@@ -24,6 +24,16 @@ export interface PlatformHooks {
24
24
  /** The egress proxy — a per-step lease that keeps credentials out of the
25
25
  * sandbox. Default: none, and the runner materialises as it always did. */
26
26
  egress: EgressHooks;
27
+ /** A workspace's files changed — written, deployed, pushed, or the
28
+ * workspace deleted. What a scheduler that keeps flows in memory rather
29
+ * than re-reading every workspace each tick needs to hear. Default:
30
+ * nothing — locally the scheduler reads the disk. Never throws. */
31
+ workspaceChanged(tenant: string, workspace: string, why: "write" | "deploy" | "push" | "delete"): void;
32
+ /** Can a step in this isolation mode be re-attached to after the driver
33
+ * that started it is gone? True means an orphaned step with a `sandbox`
34
+ * on its record is resumed rather than destroyed and run again. Default:
35
+ * nothing is. */
36
+ sandboxResumable(kind: string): boolean;
27
37
  }
28
38
  export declare const platform: PlatformHooks;
29
39
  /** Install the platform's implementations. Partial: what is not given keeps its local default. */
@@ -24,6 +24,8 @@ const local = {
24
24
  previewSourceOf: () => null,
25
25
  tenantKey: () => null,
26
26
  egress: { lease: async () => null },
27
+ workspaceChanged() { },
28
+ sandboxResumable: () => false,
27
29
  };
28
30
  // One object per PROCESS, not per module instance. A bundler that compiles
29
31
  // the boot file and each route handler separately (Next does) gives every
@@ -78,6 +78,10 @@ export interface StepActuals {
78
78
  export interface ContainerStepOutcome {
79
79
  status: "completed" | "failed";
80
80
  result: string | null;
81
+ /** Why the sandbox ended when the step did not: what the cluster said
82
+ * (OOMKilled, Evicted, the pod gone). Absent when the driver finished
83
+ * and said so itself. A retry policy reads this. */
84
+ reason?: string | null;
81
85
  /** The model's final text block — see StepRecord.conclusion. Crosses the
82
86
  * boundary with the rest of the outcome; the driver spreads it. */
83
87
  conclusion?: string | null;
@@ -120,7 +124,14 @@ export declare function allowedBack(rel: string): boolean;
120
124
  * the step did not touch it, so it is not ours to write back. Concurrent
121
125
  * edits survive, and a step that genuinely changed a file still wins.
122
126
  */
123
- export declare function applyContainerChanges(hostWs: string, containerWs: string, baseline?: string): string[];
127
+ /**
128
+ * What a workspace copy looked like when it went in: the sha256 of every
129
+ * file the step could hand back. Small enough to ride into the sandbox
130
+ * beside the input, so a driver that inherits the step can tell what the
131
+ * step changed without the staging copy the first driver held in /tmp.
132
+ */
133
+ export declare function hashTree(dir: string): Record<string, string>;
134
+ export declare function applyContainerChanges(hostWs: string, containerWs: string, baseline?: string | Record<string, string>): string[];
124
135
  /**
125
136
  * Where the driver builds venvs and npm prefixes, as an absolute path *inside*
126
137
  * the runner. Fixed, and it has to be: a venv bakes its own absolute path into
@@ -185,6 +196,18 @@ export interface RunInContainerArgs {
185
196
  runId?: string;
186
197
  /** The reservation class — which limits this step's sandbox holds. */
187
198
  size?: "small" | "large" | "heavy";
199
+ /** Re-attach to a sandbox a previous driver started, instead of creating
200
+ * one: its handle, and how many output lines that driver had applied.
201
+ * The executor replays nothing before `consumed`, and reads the step's
202
+ * files back from the sandbox's own copy-in snapshot. */
203
+ resume?: {
204
+ ref: string;
205
+ consumed: number;
206
+ } | null;
207
+ /** Called by an executor that can be resumed: when its sandbox exists
208
+ * (consumed 0), and before each output line it applies. The runner
209
+ * records these on the step, so the next driver knows where to attach. */
210
+ checkpoint?: (ref: string, consumed: number) => void;
188
211
  }
189
212
  /** The limits a size class reserves. Large is the install's configured
190
213
  * limits unchanged, so existing installs bill and behave identically;
@@ -95,6 +95,34 @@ export function allowedBack(rel) {
95
95
  * the step did not touch it, so it is not ours to write back. Concurrent
96
96
  * edits survive, and a step that genuinely changed a file still wins.
97
97
  */
98
+ /**
99
+ * What a workspace copy looked like when it went in: the sha256 of every
100
+ * file the step could hand back. Small enough to ride into the sandbox
101
+ * beside the input, so a driver that inherits the step can tell what the
102
+ * step changed without the staging copy the first driver held in /tmp.
103
+ */
104
+ export function hashTree(dir) {
105
+ const out = {};
106
+ const walk = (d) => {
107
+ for (const entry of fs.readdirSync(d, { withFileTypes: true })) {
108
+ const abs = path.join(d, entry.name);
109
+ const rel = path.relative(dir, abs).replaceAll("\\", "/");
110
+ if (entry.isSymbolicLink())
111
+ continue;
112
+ if (entry.isDirectory()) {
113
+ if (allowedBack(rel + "/"))
114
+ walk(abs);
115
+ continue;
116
+ }
117
+ if (!allowedBack(rel))
118
+ continue;
119
+ out[rel] = crypto.createHash("sha256").update(fs.readFileSync(abs)).digest("hex");
120
+ }
121
+ };
122
+ if (fs.existsSync(dir))
123
+ walk(dir);
124
+ return out;
125
+ }
98
126
  export function applyContainerChanges(hostWs, containerWs, baseline) {
99
127
  const applied = [];
100
128
  const walk = (dir) => {
@@ -119,11 +147,16 @@ export function applyContainerChanges(hostWs, containerWs, baseline) {
119
147
  const next = fs.readFileSync(abs);
120
148
  // Untouched by this step: whatever the host says now is more current
121
149
  // than what we handed in, including a change made while it ran.
122
- if (baseline) {
150
+ if (typeof baseline === "string") {
123
151
  const was = path.join(baseline, rel);
124
152
  if (fs.existsSync(was) && fs.readFileSync(was).equals(next))
125
153
  continue;
126
154
  }
155
+ else if (baseline) {
156
+ const was = baseline[rel.replaceAll("\\", "/")];
157
+ if (was && was === crypto.createHash("sha256").update(next).digest("hex"))
158
+ continue;
159
+ }
127
160
  const target = path.join(hostWs, rel);
128
161
  if (fs.existsSync(target) && fs.readFileSync(target).equals(next))
129
162
  continue;
@@ -206,7 +206,7 @@ export interface Reconciliation {
206
206
  * outlives any process. A run whose person already answered is not waiting,
207
207
  * and is picked back up — see below.
208
208
  */
209
- export declare function reconcileRuns(tenant: string, now?: number): Reconciliation[];
209
+ export declare function reconcileRuns(tenant: string, now?: number, live?: ReadonlySet<string>): Reconciliation[];
210
210
  /**
211
211
  * The same, for every account this installation holds.
212
212
  *
@@ -214,7 +214,7 @@ export declare function reconcileRuns(tenant: string, now?: number): Reconciliat
214
214
  * one hard-coded account left every other account's interrupted runs marked
215
215
  * `running` forever, with no process left to finish them.
216
216
  */
217
- export declare function reconcileAllRuns(now?: number): Reconciliation[];
217
+ export declare function reconcileAllRuns(now?: number, live?: ReadonlySet<string>): Reconciliation[];
218
218
  /**
219
219
  * Wait for a run to stop being in progress.
220
220
  *
@@ -5,7 +5,7 @@
5
5
  import fs from "node:fs";
6
6
  import path from "node:path";
7
7
  import matter from "gray-matter";
8
- import { executeStep, extractJson, stepCeiling } from "./step-exec.js";
8
+ import { knownPrice, executeStep, extractJson, stepCeiling } from "./step-exec.js";
9
9
  import { eventUrl } from "./webhook.js";
10
10
  import { runStepInContainer, sizeLimits, killRunSandboxes } from "./run-container.js";
11
11
  import { EGRESS_ENV, MODEL_KEY_NAME, addGrant, hostOf, placeholderNames, proxyModelEnv, unsubstitute } from "./egress.js";
@@ -1158,7 +1158,10 @@ stepBudgetUsd = null) {
1158
1158
  // step: the generic HTTP tool still works without the typed ones.
1159
1159
  for (const w of await prefetchOpenApi(tenant, openApiSources(agentDir, tenant)))
1160
1160
  push("error", `openapi: ${w}`);
1161
- const { front, clockEnv, systemPrompt, allowed, disabled, apiTools, scriptTools, secretEnv, secretScopes, missingSecrets, missingTools, runtime, unknownTools, shadowed, legacyUse, mcpServers, mcpNames, apiSpecs, scriptSpecs, brokenTools, size, providerEnv, providerLabel, providerSecrets, providerWarnings, formatWarning, fallbackEnv, apiWarnings, searchRoots, historyDigest, deskDigest, searchTools, historyTools, deskTools, translator, fallbackTranslator, } = agentContext(agentDir, tenant, tags, { runId, agent: step.agent });
1161
+ const { front, clockEnv, systemPrompt, allowed, disabled, apiTools, scriptTools, secretEnv, secretScopes, missingSecrets, missingTools, runtime, unknownTools, shadowed, legacyUse, mcpServers, mcpNames, apiSpecs, scriptSpecs, brokenTools, size: agentSize, providerEnv, providerLabel, providerSecrets, providerWarnings, formatWarning, fallbackEnv, apiWarnings, searchRoots, historyDigest, deskDigest, searchTools, historyTools, deskTools, translator, fallbackTranslator, } = agentContext(agentDir, tenant, tags, { runId, agent: step.agent });
1162
+ // A retry that moved the step up a class (an evicted or OOM-killed
1163
+ // attempt) wins over the agent's own `size:` for the attempts after.
1164
+ const size = step.sizeUp ?? agentSize;
1162
1165
  // oauth2 secrets become live access tokens here — the one async moment
1163
1166
  // in secret resolution, at the last host-side point before anything is
1164
1167
  // injected or substituted. The refresh recipes never leave this process:
@@ -1289,6 +1292,11 @@ stepBudgetUsd = null) {
1289
1292
  push("info", fitted.note);
1290
1293
  effort = fitted.effort;
1291
1294
  }
1295
+ // No catalogue — the platform's own Anthropic key, or a gateway that
1296
+ // publishes no prices — but the model's NAME is often enough. Without
1297
+ // this the in-pod meter priced haiku and sonnet at Opus rates and a
1298
+ // step was stopped at up to fifteen times what it had spent.
1299
+ tokenPrice ??= knownPrice(wireModel);
1292
1300
  fs.mkdirSync(path.join(agentDir, "outputs"), { recursive: true });
1293
1301
  fs.mkdirSync(path.join(agentDir, "memory"), { recursive: true });
1294
1302
  // Expose the workspace's shared scripts/ as ./shared so every agent reaches
@@ -1530,6 +1538,19 @@ stepBudgetUsd = null) {
1530
1538
  // Stamps the sandbox, so stopping this run can destroy it.
1531
1539
  runId,
1532
1540
  size,
1541
+ // Resume: a sandbox a previous driver left running, recorded on the
1542
+ // step. Consumed once — a fresh attempt after this one (a retry, a
1543
+ // fallback) starts its own sandbox.
1544
+ resume: step.sandbox && step.sandbox.kind === isolation ? { ref: step.sandbox.ref, consumed: step.sandbox.consumed } : null,
1545
+ // Checkpoint: where the executor is, on the record, so a driver
1546
+ // that inherits this run can attach where this one left off. The
1547
+ // executor calls it before applying each line; the line's own emit
1548
+ // saves the record, so this only updates the fields.
1549
+ checkpoint: (ref, consumed) => {
1550
+ step.sandbox = { kind: isolation, ref, consumed, since: step.sandbox?.since ?? new Date().toISOString() };
1551
+ if (consumed === 0)
1552
+ save();
1553
+ },
1533
1554
  });
1534
1555
  // A busy provider gets another moment on the SAME supply before the
1535
1556
  // fallback is considered: under a fan-out, twenty steps against one
@@ -1539,6 +1560,7 @@ stepBudgetUsd = null) {
1539
1560
  const firstArgs = isolatedArgs(platformModelEnv());
1540
1561
  await lease?.commit();
1541
1562
  let outcome = await runIsolated(firstArgs);
1563
+ step.sandbox = null; // whatever happens next starts its own
1542
1564
  for (let n = 1; n <= OVERLOAD_RETRIES && outcome.status === "failed" && isTransientOverload(lastRefusal); n++) {
1543
1565
  const wait = backoffMs(n, lastRefusal);
1544
1566
  push("info", `provider busy (${lastRefusal.slice(0, 60)}) — waiting ${(wait / 1000).toFixed(1)}s, attempt ${n} of ${OVERLOAD_RETRIES}`);
@@ -1587,6 +1609,11 @@ stepBudgetUsd = null) {
1587
1609
  step.status = outcome.status;
1588
1610
  step.result = outcome.result;
1589
1611
  step.conclusion = outcome.conclusion;
1612
+ step.sandbox = null;
1613
+ // The cluster's word on why a sandbox ended without the driver saying
1614
+ // so — OOMKilled, Evicted — on the record, where the retry reads it.
1615
+ if (outcome.status === "failed" && outcome.reason)
1616
+ push("error", `sandbox ended: ${outcome.reason}`);
1590
1617
  if (outcome.data !== undefined)
1591
1618
  step.data = outcome.data;
1592
1619
  step.costUsd = repriced(catalog, wireModel, outcome.usage ?? null, outcome.costUsd, push);
@@ -2262,16 +2289,26 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
2262
2289
  // never happened, which reconcile can then never see. Reset it to
2263
2290
  // pending and it simply runs again; driveRun's whole contract is that
2264
2291
  // re-driving a partly-finished record is safe.
2292
+ // Unless the step's sandbox outlived its driver AND the executor can
2293
+ // re-attach to it: then the record carries the pod and how far the
2294
+ // last driver got, runStep passes that as `resume`, and the step
2295
+ // carries on from where it was — a deploy no longer costs the run.
2265
2296
  let orphaned = false;
2266
2297
  for (const step of run.steps) {
2267
2298
  if (step.status !== "running")
2268
2299
  continue;
2269
- orphaned = true;
2300
+ const resumable = !!step.sandbox && platform.sandboxResumable(step.sandbox.kind);
2301
+ if (!resumable) {
2302
+ orphaned = true;
2303
+ step.sandbox = null;
2304
+ }
2270
2305
  step.status = "pending";
2271
2306
  step.events.push({
2272
2307
  t: new Date().toISOString(),
2273
2308
  type: "info",
2274
- text: "interrupted mid-step (platform restart) — running again from the start of the step",
2309
+ text: resumable
2310
+ ? `driver changed mid-step — re-attaching to the running sandbox (${step.sandbox.ref}, ${step.sandbox.consumed} lines already applied)`
2311
+ : "interrupted mid-step (platform restart) — running again from the start of the step",
2275
2312
  });
2276
2313
  }
2277
2314
  if (orphaned) {
@@ -2710,7 +2747,12 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
2710
2747
  await Promise.all(launching
2711
2748
  .map(async (step) => {
2712
2749
  const attempts = (step.retry ?? 0) + 1;
2713
- for (let attempt = 1; attempt <= attempts; attempt++) {
2750
+ // A step re-attaching to its sandbox is still on the attempt
2751
+ // that started it; a step re-run after a backoff picks up the
2752
+ // count it left. Only a step nobody has touched starts at 1.
2753
+ const interruptedInBackoff = !step.sandbox && !!step.attempts && /retrying in \d+s/.test(step.events[step.events.length - 1]?.text ?? "");
2754
+ const from = step.sandbox ? Math.max(1, step.attempts ?? 1) : interruptedInBackoff ? Math.min(attempts, step.attempts + 1) : 1;
2755
+ for (let attempt = from; attempt <= attempts; attempt++) {
2714
2756
  step.attempts = attempt;
2715
2757
  step.status = "running";
2716
2758
  save();
@@ -2727,12 +2769,43 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
2727
2769
  save();
2728
2770
  if (run.stopRequested)
2729
2771
  break;
2772
+ // The retry policy. An attempt the cluster ended for want of
2773
+ // memory or disk is retried one size class up — the same
2774
+ // size would end the same way. Every retry waits: 15 s, 30 s,
2775
+ // 1 m, 2 m, 4 m, jittered, so a provider or a site that was
2776
+ // struggling gets a moment, and twenty fan-out steps do not
2777
+ // hammer it in lock-step. The wait holds a worker slot, not
2778
+ // a sandbox — nothing is rented while it waits — and a stop
2779
+ // ends it. The step is `pending` meanwhile, so a driver that
2780
+ // inherits the run mid-wait simply runs the attempt.
2781
+ const last = [...step.events].reverse().find((e) => e.type === "error")?.text ?? "";
2782
+ let sizeNote = "";
2783
+ if (/OOMKilled|Evicted|ephemeral-storage|out of memory/i.test(last)) {
2784
+ const up = step.sizeUp === "large" || (!step.sizeUp && step.size === "large") ? "heavy" : step.sizeUp === "heavy" || step.size === "heavy" ? null : "large";
2785
+ if (up) {
2786
+ step.sizeUp = up;
2787
+ sizeNote = ` at size: ${up}`;
2788
+ }
2789
+ }
2790
+ const waitMs = Math.round(Math.min(retryBaseMs() * 16, retryBaseMs() * 2 ** (attempt - 1)) * (0.75 + Math.random() * 0.5));
2791
+ step.status = "pending";
2730
2792
  step.events.push({
2731
2793
  t: new Date().toISOString(),
2732
2794
  type: "info",
2733
- text: `attempt ${attempt} failed — retrying (${attempts - attempt} left)`,
2795
+ text: `attempt ${attempt} failed — retrying in ${Math.round(waitMs / 1000)}s${sizeNote} (${attempts - attempt} left)`,
2734
2796
  });
2735
2797
  save();
2798
+ const until = Date.now() + waitMs;
2799
+ while (Date.now() < until) {
2800
+ await sleep(Math.min(1000, until - Date.now()));
2801
+ save();
2802
+ if (run.stopRequested)
2803
+ break;
2804
+ }
2805
+ if (run.stopRequested) {
2806
+ step.status = "failed";
2807
+ break;
2808
+ }
2736
2809
  }
2737
2810
  }));
2738
2811
  // on-fail: another agent takes the step over. Sequential and rare:
@@ -3051,6 +3124,9 @@ export function loadFlow(tenant, workspace, flowName) {
3051
3124
  // once it has been quiet for two minutes — the margin is for a record
3052
3125
  // written a beat before its driver registered, not a judgement about pace.
3053
3126
  const ABANDONED_AFTER_MS = 2 * 60 * 1000;
3127
+ /** The first retry's wait; each one after doubles it, to sixteen times
3128
+ * this (15 s → 4 min by default). An env, so a test can make it small. */
3129
+ const retryBaseMs = () => Number(process.env.FOLDRUN_RETRY_BASE_MS) || 15_000;
3054
3130
  /** The most recent moment a run showed any sign of life. */
3055
3131
  function lastActivity(run) {
3056
3132
  let latest = Date.parse(run.startedAt);
@@ -3082,12 +3158,19 @@ function lastActivity(run) {
3082
3158
  * outlives any process. A run whose person already answered is not waiting,
3083
3159
  * and is picked back up — see below.
3084
3160
  */
3085
- export function reconcileRuns(tenant, now = Date.now()) {
3161
+ export function reconcileRuns(tenant, now = Date.now(), live) {
3086
3162
  const closed = [];
3087
3163
  for (const workspace of listWorkspaces(tenant)) {
3088
3164
  for (const summary of listRuns(tenant, workspace.name)) {
3089
3165
  if (drivingRuns.has(summary.id))
3090
3166
  continue;
3167
+ // Driven by another process — a second worker pod, on a platform with
3168
+ // more than one. The idle window protects a quiet step from THIS
3169
+ // process's judgement only; a worker that boots beside a live one
3170
+ // would otherwise close every long step its neighbour is driving.
3171
+ // The caller says which runs those are (a heartbeated claim).
3172
+ if (live?.has(summary.id))
3173
+ continue;
3091
3174
  if (now - lastActivity(summary) < ABANDONED_AFTER_MS)
3092
3175
  continue;
3093
3176
  // Approved, then abandoned. Approval only writes to the run record; the
@@ -3168,8 +3251,8 @@ export function reconcileRuns(tenant, now = Date.now()) {
3168
3251
  * one hard-coded account left every other account's interrupted runs marked
3169
3252
  * `running` forever, with no process left to finish them.
3170
3253
  */
3171
- export function reconcileAllRuns(now = Date.now()) {
3172
- return listTenants().flatMap((tenant) => reconcileRuns(tenant, now));
3254
+ export function reconcileAllRuns(now = Date.now(), live) {
3255
+ return listTenants().flatMap((tenant) => reconcileRuns(tenant, now, live));
3173
3256
  }
3174
3257
  /**
3175
3258
  * Wait for a run to stop being in progress.
@@ -304,6 +304,7 @@ timezone: Australia/Sydney
304
304
  model: default # a default for every step here
305
305
  effort: low
306
306
  overlap: skip # skip | queue — what a second run does while one is live
307
+ priority: normal # high | normal | low — where its runs stand in the queue
307
308
  ---
308
309
 
309
310
  1. [[researcher]] — Find three sources on {{topic}}.
@@ -70,16 +70,26 @@ export type EventExtra = {
70
70
  ms?: number;
71
71
  err?: boolean;
72
72
  };
73
- /** Conservative per-token rates for a model the catalogue cannot price —
74
- * Opus-class — so a ceiling still means something on an unknown gateway. */
73
+ /** Conservative per-token rates for a model nothing else can price — an
74
+ * unknown id on a gateway with no catalogue. Opus-class, so a ceiling
75
+ * still means something. Only ever reached when `knownPrice` below does
76
+ * not recognise the model either. */
75
77
  export declare const FALLBACK_PRICE: {
76
78
  input: number;
77
79
  output: number;
78
80
  };
81
+ /** What a token costs on this model, when we can say without a catalogue. */
82
+ export declare function knownPrice(model: string | undefined): {
83
+ input: number;
84
+ output: number;
85
+ } | null;
79
86
  /**
80
- * What one assistant turn cost. Cache traffic is counted as input at the
81
- * input rate, which over-approximates on gateways with cheaper cache reads
82
- * — on the money side, the honest direction to be wrong in. Pure.
87
+ * What one assistant turn cost, from the usage the SDK reports. Pure.
88
+ *
89
+ * A gateway whose cache is priced differently is over- or under-counted
90
+ * here by that difference alone; the run record's own cost is repriced
91
+ * from the catalogue afterwards and stays the authority on the bill. This
92
+ * number exists to stop a runaway step mid-turn, and wants to be close.
83
93
  */
84
94
  export declare function priceTurn(u: Record<string, number | undefined>, price: {
85
95
  input: number;
@@ -12,19 +12,55 @@ import path from "node:path";
12
12
  import { query } from "@anthropic-ai/claude-agent-sdk";
13
13
  import { spawn } from "node:child_process";
14
14
  import { checkPaths, checkBash, isFilesystemTool } from "./confine.js";
15
- /** Conservative per-token rates for a model the catalogue cannot price —
16
- * Opus-class — so a ceiling still means something on an unknown gateway. */
15
+ /** Conservative per-token rates for a model nothing else can price — an
16
+ * unknown id on a gateway with no catalogue. Opus-class, so a ceiling
17
+ * still means something. Only ever reached when `knownPrice` below does
18
+ * not recognise the model either. */
17
19
  export const FALLBACK_PRICE = { input: 15e-6, output: 75e-6 };
20
+ /** The rates we know without asking anyone: the tiers this runtime resolves
21
+ * (`fast`/`default`/`max` → haiku/sonnet/opus) and the model ids that carry
22
+ * those words. A gateway with a catalogue still wins — that is the
23
+ * authority — but the platform's own Anthropic key has no catalogue, and
24
+ * pricing its steps at Opus rates made every mid-turn ceiling read up to
25
+ * fifteen times high: an indexing run was stopped at "$1.17" having
26
+ * actually spent $0.15 (2026-09-08). Per token, input/output. */
27
+ const KNOWN_PRICES = [
28
+ [/haiku/i, { input: 1e-6, output: 5e-6 }],
29
+ [/sonnet/i, { input: 2e-6, output: 10e-6 }],
30
+ [/\bopus\b/i, { input: 5e-6, output: 25e-6 }],
31
+ ];
32
+ /** What a token costs on this model, when we can say without a catalogue. */
33
+ export function knownPrice(model) {
34
+ if (!model)
35
+ return null;
36
+ return KNOWN_PRICES.find(([re]) => re.test(model))?.[1] ?? null;
37
+ }
38
+ /** Anthropic's cache multipliers, and the shape every gateway that bills
39
+ * for caching at all has copied: a write costs a quarter more than fresh
40
+ * input, a read a tenth of it. Counting both at the full input rate — as
41
+ * this did — makes a long cached prompt read about ten times its price,
42
+ * which on a ceiling is not "the honest direction to be wrong in", it is
43
+ * a step killed for spending money it never spent. */
44
+ const CACHE_WRITE_MULTIPLIER = 1.25;
45
+ const CACHE_READ_MULTIPLIER = 0.1;
18
46
  /**
19
- * What one assistant turn cost. Cache traffic is counted as input at the
20
- * input rate, which over-approximates on gateways with cheaper cache reads
21
- * — on the money side, the honest direction to be wrong in. Pure.
47
+ * What one assistant turn cost, from the usage the SDK reports. Pure.
48
+ *
49
+ * A gateway whose cache is priced differently is over- or under-counted
50
+ * here by that difference alone; the run record's own cost is repriced
51
+ * from the catalogue afterwards and stays the authority on the bill. This
52
+ * number exists to stop a runaway step mid-turn, and wants to be close.
22
53
  */
23
54
  export function priceTurn(u, price) {
24
- const inTok = (u.input_tokens ?? 0) + (u.cache_creation_input_tokens ?? 0) + (u.cache_read_input_tokens ?? 0);
25
- const outTok = u.output_tokens ?? 0;
26
55
  const p = price ?? FALLBACK_PRICE;
27
- return inTok * p.input + outTok * p.output;
56
+ const fresh = u.input_tokens ?? 0;
57
+ const written = u.cache_creation_input_tokens ?? 0;
58
+ const read = u.cache_read_input_tokens ?? 0;
59
+ const outTok = u.output_tokens ?? 0;
60
+ return (fresh * p.input +
61
+ written * p.input * CACHE_WRITE_MULTIPLIER +
62
+ read * p.input * CACHE_READ_MULTIPLIER +
63
+ outTok * p.output);
28
64
  }
29
65
  /**
30
66
  * Each step launched together gets an equal share of what is left of the
@@ -378,6 +378,12 @@ export interface FlowInfo {
378
378
  * overlap) — a default that changed underfoot would surprise every flow
379
379
  * that legitimately runs in parallel. */
380
380
  overlap: "skip" | "queue" | null;
381
+ /** Where this flow's runs stand in the queue against everyone else's:
382
+ * "high" is claimed before the round-robin, "low" after it. Adds to the
383
+ * account's own priority; the account's cap still bounds how many run
384
+ * at once, so a high lane never starves the rest outright. Unset is
385
+ * normal — first come, fair across accounts. */
386
+ priority: "high" | "normal" | "low" | null;
381
387
  /** The most one run of this flow may spend, in USD. A literal, never an
382
388
  * expression — the cap is readable off the file, which is the property
383
389
  * the whole grammar exists to keep. Checked between groups: the group
@@ -565,6 +571,9 @@ export interface ChangeMeta {
565
571
  message?: string;
566
572
  }
567
573
  export declare function writeWorkspaceFile(tenant: string, workspace: string, rel: string, content: string, meta?: ChangeMeta): void;
574
+ /** Tell the platform, and never let its listener fail a write. */
575
+ declare function workspaceChanged(tenant: string, workspace: string, why: "write" | "deploy" | "push" | "delete"): void;
576
+ export { workspaceChanged as notifyWorkspaceChanged };
568
577
  export declare function syncBundleFor(file: string, change?: "Creation" | "Update"): void;
569
578
  /**
570
579
  * Rename one editable file. Both ends pass the same gate as a write — a
@@ -653,6 +662,21 @@ export interface StepRecord {
653
662
  verify?: string;
654
663
  /** Attempts made so far, for the run trace. */
655
664
  attempts?: number;
665
+ /** The sandbox this step is executing in, while it is: which executor,
666
+ * its handle (a pod name), and how many of its output lines the driver
667
+ * has already applied to this record. What a driver that inherits the
668
+ * run — after a deploy rolled the last one, or it crashed — needs to
669
+ * re-attach and carry on rather than run the step again. Cleared when
670
+ * the step settles. */
671
+ sandbox?: {
672
+ kind: string;
673
+ ref: string;
674
+ consumed: number;
675
+ since: string;
676
+ } | null;
677
+ /** A size class a retry moved the step up to (an evicted or OOM-killed
678
+ * attempt), overriding the agent's own `size:` for the attempts after. */
679
+ sizeUp?: "large" | "heavy";
656
680
  /** When the step actually started and finished executing — not when the
657
681
  * run reached it. A step's events approximate this (first and last), but
658
682
  * a model can think for minutes before its first tool call, and a step
package/dist/src/store.js CHANGED
@@ -23,6 +23,7 @@ import fs from "node:fs";
23
23
  import path from "node:path";
24
24
  import { recordRevision, registerTreeReader } from "./history.js";
25
25
  import { dataRoot, singleWorkspace } from "./paths.js";
26
+ import { platform } from "./platform.js";
26
27
  import matter from "gray-matter";
27
28
  /**
28
29
  * The frontmatter of one markdown file, as the runtime reads it. Exported
@@ -413,6 +414,12 @@ meta = {}) {
413
414
  // them are not bundles until their index.md exists to carry okf_version. A
414
415
  // deploy writes straight to disk, so nothing else would generate them.
415
416
  syncWorkspaceBundles(dir);
417
+ // Here, not in deployWorkspace: a workspace CREATED with files (the API's
418
+ // create, the CLI's first deploy, a template) comes through this function
419
+ // without going through deployWorkspace, and the first live check found
420
+ // exactly that — a new desk's cron was seen on the five-minute rescan, not
421
+ // the next tick.
422
+ workspaceChanged(tenant, workspace, "deploy");
416
423
  return { preserved: preserved.length };
417
424
  }
418
425
  // The prompt-facing index for an OKF bundle (memory/ or knowledge/).
@@ -1052,6 +1059,7 @@ export function parseFlow(file, raw) {
1052
1059
  model: data.model ?? null,
1053
1060
  effort: data.effort ?? null,
1054
1061
  overlap: data.overlap === "skip" || data.overlap === "queue" ? data.overlap : null,
1062
+ priority: data.priority === "high" || data.priority === "low" || data.priority === "normal" ? data.priority : null,
1055
1063
  budget: Number(data.budget) > 0 ? Number(data.budget) : null,
1056
1064
  // `after: [[flow:publish]]` — a link or a bare name, read the same way
1057
1065
  // as every other file-naming field (refs.ts).
@@ -1606,7 +1614,18 @@ export function writeWorkspaceFile(tenant, workspace, rel, content, meta = {}) {
1606
1614
  // The revision: what it was, what it is now. Identical content records
1607
1615
  // nothing — see history.ts.
1608
1616
  recordRevision(tenant, workspace, [{ path: norm, before, after: fs.readFileSync(p, "utf8") }], meta);
1617
+ workspaceChanged(tenant, workspace, "write");
1609
1618
  }
1619
+ /** Tell the platform, and never let its listener fail a write. */
1620
+ function workspaceChanged(tenant, workspace, why) {
1621
+ try {
1622
+ platform.workspaceChanged(tenant, workspace, why);
1623
+ }
1624
+ catch {
1625
+ // a scheduler that could not be told will find out on its next rescan
1626
+ }
1627
+ }
1628
+ export { workspaceChanged as notifyWorkspaceChanged };
1610
1629
  // memory/ and knowledge/ are OKF bundles, so index.md has to stay current —
1611
1630
  // a generated index that drifts is the same class of bug as a memory index
1612
1631
  // naming files that don't exist.
@@ -1697,6 +1716,7 @@ export function renameWorkspaceFile(tenant, workspace, from, to, meta = {}) {
1697
1716
  ], { message: `renamed ${from} → ${to}`, ...meta });
1698
1717
  syncBundleFor(src);
1699
1718
  syncBundleFor(dst, "Update");
1719
+ workspaceChanged(tenant, workspace, "write");
1700
1720
  }
1701
1721
  /** Every text file under a path, as revision entries recording deletion. */
1702
1722
  function deletionsUnder(dir, norm) {
@@ -1759,12 +1779,14 @@ export function deleteWorkspacePath(tenant, workspace, rel, meta = {}) {
1759
1779
  recordRevision(tenant, workspace, gone, meta);
1760
1780
  // A bundle's index must stop naming what is gone.
1761
1781
  syncBundleFor(p);
1782
+ workspaceChanged(tenant, workspace, "write");
1762
1783
  }
1763
1784
  export function deleteWorkspace(tenant, workspace) {
1764
1785
  const dir = workspaceDir(tenant, workspace);
1765
1786
  if (!fs.existsSync(dir))
1766
1787
  throw new Error(`workspace ${workspace} not found`);
1767
1788
  fs.rmSync(dir, { recursive: true, force: true });
1789
+ workspaceChanged(tenant, workspace, "delete");
1768
1790
  }
1769
1791
  /**
1770
1792
  * What a rename would break, so the caller can say so before doing it.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@foldrun/core",
3
- "version": "0.2.0",
3
+ "version": "0.3.0",
4
4
  "description": "Agents are just folders. The format, the runner, the checks.",
5
5
  "license": "Apache-2.0",
6
6
  "type": "module",
@@ -29,7 +29,8 @@
29
29
  "prepare": "npm run build",
30
30
  "prepublishOnly": "npm run build",
31
31
  "test": "FOLDRUN_DATA=$(mktemp -d /tmp/foldrun-test-XXXXXX) node --test *.test.ts $(ls tests/*.test.ts | grep -v e2e)",
32
- "container": "FOLDRUN_CONTAINER_E2E=1 node scripts/ts-test.mjs tests/container-e2e.test.ts --test-timeout=600000"
32
+ "container": "FOLDRUN_CONTAINER_E2E=1 node scripts/ts-test.mjs tests/container-e2e.test.ts --test-timeout=600000",
33
+ "release": "node scripts/release.mjs"
33
34
  },
34
35
  "engines": {
35
36
  "node": ">=22"
package/src/deploy.ts CHANGED
@@ -29,6 +29,7 @@ import {
29
29
  WORKSPACE_DIRS,
30
30
  assertSafeName,
31
31
  type DeployFile,
32
+ notifyWorkspaceChanged,
32
33
  } from "./store.ts";
33
34
  import { conformanceIssues } from "./okf.ts";
34
35
  import { repoDir } from "./gitrepo.ts";
@@ -373,6 +374,8 @@ export function writePushNote(tenant: string, workspace: string, note: PushNote)
373
374
  } catch {
374
375
  // The note is a record, not the deploy. Losing it must not fail a push.
375
376
  }
377
+ // A push waiting on a run is the scheduler's to apply; tell it there is one.
378
+ notifyWorkspaceChanged(tenant, workspace, "push");
376
379
  }
377
380
 
378
381
  /**
@@ -428,7 +431,7 @@ export function deployWorkspace(
428
431
  const { preserved } = saveWorkspace(tenant, workspace, files, { commit: opts.commit ?? null, by: opts.by ?? "deploy", message: opts.message });
429
432
  const commit = opts.commit ?? null;
430
433
  if (commit) writeDeployedCommit(tenant, workspace, commit);
431
- return { ...plan, applied: true, commit, preserved };
434
+ return { ...plan, applied: true, commit, preserved }; // saveWorkspace told the platform
432
435
  }
433
436
 
434
437
  // Which commit a workspace is running.
package/src/platform.ts CHANGED
@@ -45,6 +45,16 @@ export interface PlatformHooks {
45
45
  /** The egress proxy — a per-step lease that keeps credentials out of the
46
46
  * sandbox. Default: none, and the runner materialises as it always did. */
47
47
  egress: EgressHooks;
48
+ /** A workspace's files changed — written, deployed, pushed, or the
49
+ * workspace deleted. What a scheduler that keeps flows in memory rather
50
+ * than re-reading every workspace each tick needs to hear. Default:
51
+ * nothing — locally the scheduler reads the disk. Never throws. */
52
+ workspaceChanged(tenant: string, workspace: string, why: "write" | "deploy" | "push" | "delete"): void;
53
+ /** Can a step in this isolation mode be re-attached to after the driver
54
+ * that started it is gone? True means an orphaned step with a `sandbox`
55
+ * on its record is resumed rather than destroyed and run again. Default:
56
+ * nothing is. */
57
+ sandboxResumable(kind: string): boolean;
48
58
  }
49
59
 
50
60
  const local: PlatformHooks = {
@@ -60,6 +70,8 @@ const local: PlatformHooks = {
60
70
  previewSourceOf: () => null,
61
71
  tenantKey: () => null,
62
72
  egress: { lease: async () => null },
73
+ workspaceChanged() {},
74
+ sandboxResumable: () => false,
63
75
  };
64
76
 
65
77
  // One object per PROCESS, not per module instance. A bundler that compiles
@@ -106,6 +106,10 @@ export interface StepActuals {
106
106
  export interface ContainerStepOutcome {
107
107
  status: "completed" | "failed";
108
108
  result: string | null;
109
+ /** Why the sandbox ended when the step did not: what the cluster said
110
+ * (OOMKilled, Evicted, the pod gone). Absent when the driver finished
111
+ * and said so itself. A retry policy reads this. */
112
+ reason?: string | null;
109
113
  /** The model's final text block — see StepRecord.conclusion. Crosses the
110
114
  * boundary with the rest of the outcome; the driver spreads it. */
111
115
  conclusion?: string | null;
@@ -186,10 +190,35 @@ export function allowedBack(rel: string): boolean {
186
190
  * the step did not touch it, so it is not ours to write back. Concurrent
187
191
  * edits survive, and a step that genuinely changed a file still wins.
188
192
  */
193
+ /**
194
+ * What a workspace copy looked like when it went in: the sha256 of every
195
+ * file the step could hand back. Small enough to ride into the sandbox
196
+ * beside the input, so a driver that inherits the step can tell what the
197
+ * step changed without the staging copy the first driver held in /tmp.
198
+ */
199
+ export function hashTree(dir: string): Record<string, string> {
200
+ const out: Record<string, string> = {};
201
+ const walk = (d: string) => {
202
+ for (const entry of fs.readdirSync(d, { withFileTypes: true })) {
203
+ const abs = path.join(d, entry.name);
204
+ const rel = path.relative(dir, abs).replaceAll("\\", "/");
205
+ if (entry.isSymbolicLink()) continue;
206
+ if (entry.isDirectory()) {
207
+ if (allowedBack(rel + "/")) walk(abs);
208
+ continue;
209
+ }
210
+ if (!allowedBack(rel)) continue;
211
+ out[rel] = crypto.createHash("sha256").update(fs.readFileSync(abs)).digest("hex");
212
+ }
213
+ };
214
+ if (fs.existsSync(dir)) walk(dir);
215
+ return out;
216
+ }
217
+
189
218
  export function applyContainerChanges(
190
219
  hostWs: string,
191
220
  containerWs: string,
192
- baseline?: string,
221
+ baseline?: string | Record<string, string>,
193
222
  ): string[] {
194
223
  const applied: string[] = [];
195
224
  const walk = (dir: string) => {
@@ -211,9 +240,12 @@ export function applyContainerChanges(
211
240
  const next = fs.readFileSync(abs);
212
241
  // Untouched by this step: whatever the host says now is more current
213
242
  // than what we handed in, including a change made while it ran.
214
- if (baseline) {
243
+ if (typeof baseline === "string") {
215
244
  const was = path.join(baseline, rel);
216
245
  if (fs.existsSync(was) && fs.readFileSync(was).equals(next)) continue;
246
+ } else if (baseline) {
247
+ const was = baseline[rel.replaceAll("\\", "/")];
248
+ if (was && was === crypto.createHash("sha256").update(next).digest("hex")) continue;
217
249
  }
218
250
  const target = path.join(hostWs, rel);
219
251
  if (fs.existsSync(target) && fs.readFileSync(target).equals(next)) continue;
@@ -625,6 +657,15 @@ export interface RunInContainerArgs {
625
657
  runId?: string;
626
658
  /** The reservation class — which limits this step's sandbox holds. */
627
659
  size?: "small" | "large" | "heavy";
660
+ /** Re-attach to a sandbox a previous driver started, instead of creating
661
+ * one: its handle, and how many output lines that driver had applied.
662
+ * The executor replays nothing before `consumed`, and reads the step's
663
+ * files back from the sandbox's own copy-in snapshot. */
664
+ resume?: { ref: string; consumed: number } | null;
665
+ /** Called by an executor that can be resumed: when its sandbox exists
666
+ * (consumed 0), and before each output line it applies. The runner
667
+ * records these on the step, so the next driver knows where to attach. */
668
+ checkpoint?: (ref: string, consumed: number) => void;
628
669
  }
629
670
 
630
671
  /** The limits a size class reserves. Large is the install's configured
package/src/runner.ts CHANGED
@@ -7,7 +7,8 @@ import fs from "node:fs";
7
7
  import path from "node:path";
8
8
  import matter from "gray-matter";
9
9
  import { spawn } from "node:child_process";
10
- import { executeStep, extractJson, stepCeiling, type EventExtra } from "./step-exec.ts";
10
+ import {
11
+ knownPrice, executeStep, extractJson, stepCeiling, type EventExtra } from "./step-exec.ts";
11
12
  import { eventUrl } from "./webhook.ts";
12
13
  import { runStepInContainer, sizeLimits, killRunSandboxes, type StepTiming } from "./run-container.ts";
13
14
  import { EGRESS_ENV, MODEL_KEY_NAME, addGrant, hostOf, placeholderNames, proxyModelEnv, unsubstitute, type EgressGrant } from "./egress.ts";
@@ -1297,11 +1298,14 @@ async function runStep(
1297
1298
  front, clockEnv, systemPrompt, allowed, disabled, apiTools, scriptTools,
1298
1299
  secretEnv, secretScopes, missingSecrets, missingTools, runtime,
1299
1300
  unknownTools, shadowed, legacyUse, mcpServers, mcpNames,
1300
- apiSpecs, scriptSpecs, brokenTools, size,
1301
+ apiSpecs, scriptSpecs, brokenTools, size: agentSize,
1301
1302
  providerEnv, providerLabel, providerSecrets, providerWarnings, formatWarning,
1302
1303
  fallbackEnv, apiWarnings, searchRoots, historyDigest, deskDigest, searchTools, historyTools, deskTools,
1303
1304
  translator, fallbackTranslator,
1304
1305
  } = agentContext(agentDir, tenant, tags, { runId, agent: step.agent });
1306
+ // A retry that moved the step up a class (an evicted or OOM-killed
1307
+ // attempt) wins over the agent's own `size:` for the attempts after.
1308
+ const size = step.sizeUp ?? agentSize;
1305
1309
 
1306
1310
  // oauth2 secrets become live access tokens here — the one async moment
1307
1311
  // in secret resolution, at the last host-side point before anything is
@@ -1439,6 +1443,11 @@ async function runStep(
1439
1443
  if (fitted.note) push("info", fitted.note);
1440
1444
  effort = fitted.effort;
1441
1445
  }
1446
+ // No catalogue — the platform's own Anthropic key, or a gateway that
1447
+ // publishes no prices — but the model's NAME is often enough. Without
1448
+ // this the in-pod meter priced haiku and sonnet at Opus rates and a
1449
+ // step was stopped at up to fifteen times what it had spent.
1450
+ tokenPrice ??= knownPrice(wireModel);
1442
1451
 
1443
1452
  fs.mkdirSync(path.join(agentDir, "outputs"), { recursive: true });
1444
1453
  fs.mkdirSync(path.join(agentDir, "memory"), { recursive: true });
@@ -1690,6 +1699,18 @@ async function runStep(
1690
1699
  // Stamps the sandbox, so stopping this run can destroy it.
1691
1700
  runId,
1692
1701
  size,
1702
+ // Resume: a sandbox a previous driver left running, recorded on the
1703
+ // step. Consumed once — a fresh attempt after this one (a retry, a
1704
+ // fallback) starts its own sandbox.
1705
+ resume: step.sandbox && step.sandbox.kind === isolation ? { ref: step.sandbox.ref, consumed: step.sandbox.consumed } : null,
1706
+ // Checkpoint: where the executor is, on the record, so a driver
1707
+ // that inherits this run can attach where this one left off. The
1708
+ // executor calls it before applying each line; the line's own emit
1709
+ // saves the record, so this only updates the fields.
1710
+ checkpoint: (ref: string, consumed: number) => {
1711
+ step.sandbox = { kind: isolation!, ref, consumed, since: step.sandbox?.since ?? new Date().toISOString() };
1712
+ if (consumed === 0) save();
1713
+ },
1693
1714
  });
1694
1715
 
1695
1716
  // A busy provider gets another moment on the SAME supply before the
@@ -1700,6 +1721,7 @@ async function runStep(
1700
1721
  const firstArgs = isolatedArgs(platformModelEnv());
1701
1722
  await lease?.commit();
1702
1723
  let outcome = await runIsolated(firstArgs);
1724
+ step.sandbox = null; // whatever happens next starts its own
1703
1725
  for (let n = 1; n <= OVERLOAD_RETRIES && outcome.status === "failed" && isTransientOverload(lastRefusal); n++) {
1704
1726
  const wait = backoffMs(n, lastRefusal);
1705
1727
  push("info", `provider busy (${lastRefusal.slice(0, 60)}) — waiting ${(wait / 1000).toFixed(1)}s, attempt ${n} of ${OVERLOAD_RETRIES}`);
@@ -1752,6 +1774,10 @@ async function runStep(
1752
1774
  step.status = outcome.status;
1753
1775
  step.result = outcome.result;
1754
1776
  step.conclusion = outcome.conclusion;
1777
+ step.sandbox = null;
1778
+ // The cluster's word on why a sandbox ended without the driver saying
1779
+ // so — OOMKilled, Evicted — on the record, where the retry reads it.
1780
+ if (outcome.status === "failed" && outcome.reason) push("error", `sandbox ended: ${outcome.reason}`);
1755
1781
  if (outcome.data !== undefined) step.data = outcome.data;
1756
1782
  step.costUsd = repriced(catalog, wireModel, outcome.usage ?? null, outcome.costUsd, push);
1757
1783
  step.tokens = outcome.usage
@@ -2450,15 +2476,25 @@ function driveRunInner(
2450
2476
  // never happened, which reconcile can then never see. Reset it to
2451
2477
  // pending and it simply runs again; driveRun's whole contract is that
2452
2478
  // re-driving a partly-finished record is safe.
2479
+ // Unless the step's sandbox outlived its driver AND the executor can
2480
+ // re-attach to it: then the record carries the pod and how far the
2481
+ // last driver got, runStep passes that as `resume`, and the step
2482
+ // carries on from where it was — a deploy no longer costs the run.
2453
2483
  let orphaned = false;
2454
2484
  for (const step of run.steps) {
2455
2485
  if (step.status !== "running") continue;
2456
- orphaned = true;
2486
+ const resumable = !!step.sandbox && platform.sandboxResumable(step.sandbox.kind);
2487
+ if (!resumable) {
2488
+ orphaned = true;
2489
+ step.sandbox = null;
2490
+ }
2457
2491
  step.status = "pending";
2458
2492
  step.events.push({
2459
2493
  t: new Date().toISOString(),
2460
2494
  type: "info",
2461
- text: "interrupted mid-step (platform restart) — running again from the start of the step",
2495
+ text: resumable
2496
+ ? `driver changed mid-step — re-attaching to the running sandbox (${step.sandbox!.ref}, ${step.sandbox!.consumed} lines already applied)`
2497
+ : "interrupted mid-step (platform restart) — running again from the start of the step",
2462
2498
  });
2463
2499
  }
2464
2500
  if (orphaned) {
@@ -2908,7 +2944,13 @@ function driveRunInner(
2908
2944
  launching
2909
2945
  .map(async (step) => {
2910
2946
  const attempts = (step.retry ?? 0) + 1;
2911
- for (let attempt = 1; attempt <= attempts; attempt++) {
2947
+ // A step re-attaching to its sandbox is still on the attempt
2948
+ // that started it; a step re-run after a backoff picks up the
2949
+ // count it left. Only a step nobody has touched starts at 1.
2950
+ const interruptedInBackoff =
2951
+ !step.sandbox && !!step.attempts && /retrying in \d+s/.test(step.events[step.events.length - 1]?.text ?? "");
2952
+ const from = step.sandbox ? Math.max(1, step.attempts ?? 1) : interruptedInBackoff ? Math.min(attempts, step.attempts! + 1) : 1;
2953
+ for (let attempt = from; attempt <= attempts; attempt++) {
2912
2954
  step.attempts = attempt;
2913
2955
  step.status = "running";
2914
2956
  save();
@@ -2935,12 +2977,42 @@ function driveRunInner(
2935
2977
  // more. save() merges the flag from the record.
2936
2978
  save();
2937
2979
  if (run.stopRequested) break;
2980
+ // The retry policy. An attempt the cluster ended for want of
2981
+ // memory or disk is retried one size class up — the same
2982
+ // size would end the same way. Every retry waits: 15 s, 30 s,
2983
+ // 1 m, 2 m, 4 m, jittered, so a provider or a site that was
2984
+ // struggling gets a moment, and twenty fan-out steps do not
2985
+ // hammer it in lock-step. The wait holds a worker slot, not
2986
+ // a sandbox — nothing is rented while it waits — and a stop
2987
+ // ends it. The step is `pending` meanwhile, so a driver that
2988
+ // inherits the run mid-wait simply runs the attempt.
2989
+ const last = [...step.events].reverse().find((e) => e.type === "error")?.text ?? "";
2990
+ let sizeNote = "";
2991
+ if (/OOMKilled|Evicted|ephemeral-storage|out of memory/i.test(last)) {
2992
+ const up = step.sizeUp === "large" || (!step.sizeUp && step.size === "large") ? "heavy" : step.sizeUp === "heavy" || step.size === "heavy" ? null : "large";
2993
+ if (up) {
2994
+ step.sizeUp = up;
2995
+ sizeNote = ` at size: ${up}`;
2996
+ }
2997
+ }
2998
+ const waitMs = Math.round(Math.min(retryBaseMs() * 16, retryBaseMs() * 2 ** (attempt - 1)) * (0.75 + Math.random() * 0.5));
2999
+ step.status = "pending";
2938
3000
  step.events.push({
2939
3001
  t: new Date().toISOString(),
2940
3002
  type: "info",
2941
- text: `attempt ${attempt} failed — retrying (${attempts - attempt} left)`,
3003
+ text: `attempt ${attempt} failed — retrying in ${Math.round(waitMs / 1000)}s${sizeNote} (${attempts - attempt} left)`,
2942
3004
  });
2943
3005
  save();
3006
+ const until = Date.now() + waitMs;
3007
+ while (Date.now() < until) {
3008
+ await sleep(Math.min(1000, until - Date.now()));
3009
+ save();
3010
+ if (run.stopRequested) break;
3011
+ }
3012
+ if (run.stopRequested) {
3013
+ step.status = "failed";
3014
+ break;
3015
+ }
2944
3016
  }
2945
3017
  }),
2946
3018
  );
@@ -3270,6 +3342,10 @@ export function loadFlow(tenant: string, workspace: string, flowName: string) {
3270
3342
  // written a beat before its driver registered, not a judgement about pace.
3271
3343
  const ABANDONED_AFTER_MS = 2 * 60 * 1000;
3272
3344
 
3345
+ /** The first retry's wait; each one after doubles it, to sixteen times
3346
+ * this (15 s → 4 min by default). An env, so a test can make it small. */
3347
+ const retryBaseMs = () => Number(process.env.FOLDRUN_RETRY_BASE_MS) || 15_000;
3348
+
3273
3349
  /** The most recent moment a run showed any sign of life. */
3274
3350
  function lastActivity(run: RunRecord): number {
3275
3351
  let latest = Date.parse(run.startedAt);
@@ -3311,12 +3387,18 @@ export interface Reconciliation {
3311
3387
  * outlives any process. A run whose person already answered is not waiting,
3312
3388
  * and is picked back up — see below.
3313
3389
  */
3314
- export function reconcileRuns(tenant: string, now = Date.now()): Reconciliation[] {
3390
+ export function reconcileRuns(tenant: string, now = Date.now(), live?: ReadonlySet<string>): Reconciliation[] {
3315
3391
  const closed: Reconciliation[] = [];
3316
3392
 
3317
3393
  for (const workspace of listWorkspaces(tenant)) {
3318
3394
  for (const summary of listRuns(tenant, workspace.name)) {
3319
3395
  if (drivingRuns.has(summary.id)) continue;
3396
+ // Driven by another process — a second worker pod, on a platform with
3397
+ // more than one. The idle window protects a quiet step from THIS
3398
+ // process's judgement only; a worker that boots beside a live one
3399
+ // would otherwise close every long step its neighbour is driving.
3400
+ // The caller says which runs those are (a heartbeated claim).
3401
+ if (live?.has(summary.id)) continue;
3320
3402
  if (now - lastActivity(summary) < ABANDONED_AFTER_MS) continue;
3321
3403
 
3322
3404
  // Approved, then abandoned. Approval only writes to the run record; the
@@ -3396,8 +3478,8 @@ export function reconcileRuns(tenant: string, now = Date.now()): Reconciliation[
3396
3478
  * one hard-coded account left every other account's interrupted runs marked
3397
3479
  * `running` forever, with no process left to finish them.
3398
3480
  */
3399
- export function reconcileAllRuns(now = Date.now()): Reconciliation[] {
3400
- return listTenants().flatMap((tenant) => reconcileRuns(tenant, now));
3481
+ export function reconcileAllRuns(now = Date.now(), live?: ReadonlySet<string>): Reconciliation[] {
3482
+ return listTenants().flatMap((tenant) => reconcileRuns(tenant, now, live));
3401
3483
  }
3402
3484
 
3403
3485
  /**
package/src/starter.ts CHANGED
@@ -319,6 +319,7 @@ timezone: Australia/Sydney
319
319
  model: default # a default for every step here
320
320
  effort: low
321
321
  overlap: skip # skip | queue — what a second run does while one is live
322
+ priority: normal # high | normal | low — where its runs stand in the queue
322
323
  ---
323
324
 
324
325
  1. [[researcher]] — Find three sources on {{topic}}.
package/src/step-exec.ts CHANGED
@@ -79,20 +79,60 @@ export interface ExecOptions {
79
79
  /** The pairing fields on a tool event — see RunEvent in store.ts. */
80
80
  export type EventExtra = { call?: string; ms?: number; err?: boolean };
81
81
 
82
- /** Conservative per-token rates for a model the catalogue cannot price —
83
- * Opus-class — so a ceiling still means something on an unknown gateway. */
82
+ /** Conservative per-token rates for a model nothing else can price — an
83
+ * unknown id on a gateway with no catalogue. Opus-class, so a ceiling
84
+ * still means something. Only ever reached when `knownPrice` below does
85
+ * not recognise the model either. */
84
86
  export const FALLBACK_PRICE = { input: 15e-6, output: 75e-6 };
85
87
 
88
+ /** The rates we know without asking anyone: the tiers this runtime resolves
89
+ * (`fast`/`default`/`max` → haiku/sonnet/opus) and the model ids that carry
90
+ * those words. A gateway with a catalogue still wins — that is the
91
+ * authority — but the platform's own Anthropic key has no catalogue, and
92
+ * pricing its steps at Opus rates made every mid-turn ceiling read up to
93
+ * fifteen times high: an indexing run was stopped at "$1.17" having
94
+ * actually spent $0.15 (2026-09-08). Per token, input/output. */
95
+ const KNOWN_PRICES: [RegExp, { input: number; output: number }][] = [
96
+ [/haiku/i, { input: 1e-6, output: 5e-6 }],
97
+ [/sonnet/i, { input: 2e-6, output: 10e-6 }],
98
+ [/\bopus\b/i, { input: 5e-6, output: 25e-6 }],
99
+ ];
100
+
101
+ /** What a token costs on this model, when we can say without a catalogue. */
102
+ export function knownPrice(model: string | undefined): { input: number; output: number } | null {
103
+ if (!model) return null;
104
+ return KNOWN_PRICES.find(([re]) => re.test(model))?.[1] ?? null;
105
+ }
106
+
107
+ /** Anthropic's cache multipliers, and the shape every gateway that bills
108
+ * for caching at all has copied: a write costs a quarter more than fresh
109
+ * input, a read a tenth of it. Counting both at the full input rate — as
110
+ * this did — makes a long cached prompt read about ten times its price,
111
+ * which on a ceiling is not "the honest direction to be wrong in", it is
112
+ * a step killed for spending money it never spent. */
113
+ const CACHE_WRITE_MULTIPLIER = 1.25;
114
+ const CACHE_READ_MULTIPLIER = 0.1;
115
+
86
116
  /**
87
- * What one assistant turn cost. Cache traffic is counted as input at the
88
- * input rate, which over-approximates on gateways with cheaper cache reads
89
- * — on the money side, the honest direction to be wrong in. Pure.
117
+ * What one assistant turn cost, from the usage the SDK reports. Pure.
118
+ *
119
+ * A gateway whose cache is priced differently is over- or under-counted
120
+ * here by that difference alone; the run record's own cost is repriced
121
+ * from the catalogue afterwards and stays the authority on the bill. This
122
+ * number exists to stop a runaway step mid-turn, and wants to be close.
90
123
  */
91
124
  export function priceTurn(u: Record<string, number | undefined>, price: { input: number; output: number } | null): number {
92
- const inTok = (u.input_tokens ?? 0) + (u.cache_creation_input_tokens ?? 0) + (u.cache_read_input_tokens ?? 0);
93
- const outTok = u.output_tokens ?? 0;
94
125
  const p = price ?? FALLBACK_PRICE;
95
- return inTok * p.input + outTok * p.output;
126
+ const fresh = u.input_tokens ?? 0;
127
+ const written = u.cache_creation_input_tokens ?? 0;
128
+ const read = u.cache_read_input_tokens ?? 0;
129
+ const outTok = u.output_tokens ?? 0;
130
+ return (
131
+ fresh * p.input +
132
+ written * p.input * CACHE_WRITE_MULTIPLIER +
133
+ read * p.input * CACHE_READ_MULTIPLIER +
134
+ outTok * p.output
135
+ );
96
136
  }
97
137
 
98
138
  /**
package/src/store.ts CHANGED
@@ -24,6 +24,7 @@ import fs from "node:fs";
24
24
  import path from "node:path";
25
25
  import { recordRevision, registerTreeReader, type RevisionFile } from "./history.ts";
26
26
  import { dataRoot, singleWorkspace } from "./paths.ts";
27
+ import { platform } from "./platform.ts";
27
28
  import matter from "gray-matter";
28
29
 
29
30
  /**
@@ -452,6 +453,13 @@ export function saveWorkspace(
452
453
  // deploy writes straight to disk, so nothing else would generate them.
453
454
  syncWorkspaceBundles(dir);
454
455
 
456
+ // Here, not in deployWorkspace: a workspace CREATED with files (the API's
457
+ // create, the CLI's first deploy, a template) comes through this function
458
+ // without going through deployWorkspace, and the first live check found
459
+ // exactly that — a new desk's cron was seen on the five-minute rescan, not
460
+ // the next tick.
461
+ workspaceChanged(tenant, workspace, "deploy");
462
+
455
463
  return { preserved: preserved.length };
456
464
  }
457
465
 
@@ -1083,6 +1091,12 @@ export interface FlowInfo {
1083
1091
  * overlap) — a default that changed underfoot would surprise every flow
1084
1092
  * that legitimately runs in parallel. */
1085
1093
  overlap: "skip" | "queue" | null;
1094
+ /** Where this flow's runs stand in the queue against everyone else's:
1095
+ * "high" is claimed before the round-robin, "low" after it. Adds to the
1096
+ * account's own priority; the account's cap still bounds how many run
1097
+ * at once, so a high lane never starves the rest outright. Unset is
1098
+ * normal — first come, fair across accounts. */
1099
+ priority: "high" | "normal" | "low" | null;
1086
1100
  /** The most one run of this flow may spend, in USD. A literal, never an
1087
1101
  * expression — the cap is readable off the file, which is the property
1088
1102
  * the whole grammar exists to keep. Checked between groups: the group
@@ -1313,6 +1327,7 @@ export function parseFlow(file: string, raw: string): FlowInfo {
1313
1327
  model: data.model ?? null,
1314
1328
  effort: data.effort ?? null,
1315
1329
  overlap: data.overlap === "skip" || data.overlap === "queue" ? data.overlap : null,
1330
+ priority: data.priority === "high" || data.priority === "low" || data.priority === "normal" ? data.priority : null,
1316
1331
  budget: Number(data.budget) > 0 ? Number(data.budget) : null,
1317
1332
  // `after: [[flow:publish]]` — a link or a bare name, read the same way
1318
1333
  // as every other file-naming field (refs.ts).
@@ -1895,7 +1910,18 @@ export function writeWorkspaceFile(
1895
1910
  // The revision: what it was, what it is now. Identical content records
1896
1911
  // nothing — see history.ts.
1897
1912
  recordRevision(tenant, workspace, [{ path: norm, before, after: fs.readFileSync(p, "utf8") }], meta);
1913
+ workspaceChanged(tenant, workspace, "write");
1914
+ }
1915
+
1916
+ /** Tell the platform, and never let its listener fail a write. */
1917
+ function workspaceChanged(tenant: string, workspace: string, why: "write" | "deploy" | "push" | "delete") {
1918
+ try {
1919
+ platform.workspaceChanged(tenant, workspace, why);
1920
+ } catch {
1921
+ // a scheduler that could not be told will find out on its next rescan
1922
+ }
1898
1923
  }
1924
+ export { workspaceChanged as notifyWorkspaceChanged };
1899
1925
 
1900
1926
  // memory/ and knowledge/ are OKF bundles, so index.md has to stay current —
1901
1927
  // a generated index that drifts is the same class of bug as a memory index
@@ -1990,6 +2016,7 @@ export function renameWorkspaceFile(tenant: string, workspace: string, from: str
1990
2016
  );
1991
2017
  syncBundleFor(src);
1992
2018
  syncBundleFor(dst, "Update");
2019
+ workspaceChanged(tenant, workspace, "write");
1993
2020
  }
1994
2021
 
1995
2022
  /** Every text file under a path, as revision entries recording deletion. */
@@ -2051,12 +2078,14 @@ export function deleteWorkspacePath(tenant: string, workspace: string, rel: stri
2051
2078
  recordRevision(tenant, workspace, gone, meta);
2052
2079
  // A bundle's index must stop naming what is gone.
2053
2080
  syncBundleFor(p);
2081
+ workspaceChanged(tenant, workspace, "write");
2054
2082
  }
2055
2083
 
2056
2084
  export function deleteWorkspace(tenant: string, workspace: string) {
2057
2085
  const dir = workspaceDir(tenant, workspace);
2058
2086
  if (!fs.existsSync(dir)) throw new Error(`workspace ${workspace} not found`);
2059
2087
  fs.rmSync(dir, { recursive: true, force: true });
2088
+ workspaceChanged(tenant, workspace, "delete");
2060
2089
  }
2061
2090
 
2062
2091
  /**
@@ -2235,6 +2264,16 @@ export interface StepRecord {
2235
2264
  verify?: string;
2236
2265
  /** Attempts made so far, for the run trace. */
2237
2266
  attempts?: number;
2267
+ /** The sandbox this step is executing in, while it is: which executor,
2268
+ * its handle (a pod name), and how many of its output lines the driver
2269
+ * has already applied to this record. What a driver that inherits the
2270
+ * run — after a deploy rolled the last one, or it crashed — needs to
2271
+ * re-attach and carry on rather than run the step again. Cleared when
2272
+ * the step settles. */
2273
+ sandbox?: { kind: string; ref: string; consumed: number; since: string } | null;
2274
+ /** A size class a retry moved the step up to (an evicted or OOM-killed
2275
+ * attempt), overriding the agent's own `size:` for the attempts after. */
2276
+ sizeUp?: "large" | "heavy";
2238
2277
  /** When the step actually started and finished executing — not when the
2239
2278
  * run reached it. A step's events approximate this (first and last), but
2240
2279
  * a model can think for minutes before its first tool call, and a step