@foldrun/core 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/dist/src/deploy.js +4 -2
- package/dist/src/platform.d.ts +10 -0
- package/dist/src/platform.js +2 -0
- package/dist/src/run-container.d.ts +24 -1
- package/dist/src/run-container.js +34 -1
- package/dist/src/runner.d.ts +2 -2
- package/dist/src/runner.js +92 -9
- package/dist/src/starter.js +1 -0
- package/dist/src/step-exec.d.ts +15 -5
- package/dist/src/step-exec.js +44 -8
- package/dist/src/store.d.ts +24 -0
- package/dist/src/store.js +22 -0
- package/package.json +3 -2
- package/src/deploy.ts +4 -1
- package/src/platform.ts +12 -0
- package/src/run-container.ts +43 -2
- package/src/runner.ts +91 -9
- package/src/starter.ts +1 -0
- package/src/step-exec.ts +48 -8
- package/src/store.ts +39 -0
package/README.md
CHANGED
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
# foldrun
|
|
2
2
|
|
|
3
|
+
[](https://github.com/foldrun-io/foldrun-core/actions/workflows/ci.yml)
|
|
4
|
+
[](https://www.npmjs.com/package/@foldrun/core)
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
|
|
3
7
|
**Agents are just folders. Write them, check them, run them.**
|
|
4
8
|
|
|
5
9
|
An agent is a folder. Its instructions, skills, knowledge, memory, tools and
|
package/dist/src/deploy.js
CHANGED
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
import fs from "node:fs";
|
|
21
21
|
import os from "node:os";
|
|
22
22
|
import path from "node:path";
|
|
23
|
-
import { saveWorkspace, workspaceDir, listRuns, parseFlow, WORKSPACE_DIRS, assertSafeName, } from "./store.js";
|
|
23
|
+
import { saveWorkspace, workspaceDir, listRuns, parseFlow, WORKSPACE_DIRS, assertSafeName, notifyWorkspaceChanged, } from "./store.js";
|
|
24
24
|
import { conformanceIssues } from "./okf.js";
|
|
25
25
|
import { repoDir } from "./gitrepo.js";
|
|
26
26
|
/** Directories a deploy never reads out of a source tree. */
|
|
@@ -307,6 +307,8 @@ export function writePushNote(tenant, workspace, note) {
|
|
|
307
307
|
catch {
|
|
308
308
|
// The note is a record, not the deploy. Losing it must not fail a push.
|
|
309
309
|
}
|
|
310
|
+
// A push waiting on a run is the scheduler's to apply; tell it there is one.
|
|
311
|
+
notifyWorkspaceChanged(tenant, workspace, "push");
|
|
310
312
|
}
|
|
311
313
|
/**
|
|
312
314
|
* A push whose deploy was refused only because a run was in flight is not a
|
|
@@ -353,7 +355,7 @@ export function deployWorkspace(tenant, workspace, files, opts = {}) {
|
|
|
353
355
|
const commit = opts.commit ?? null;
|
|
354
356
|
if (commit)
|
|
355
357
|
writeDeployedCommit(tenant, workspace, commit);
|
|
356
|
-
return { ...plan, applied: true, commit, preserved };
|
|
358
|
+
return { ...plan, applied: true, commit, preserved }; // saveWorkspace told the platform
|
|
357
359
|
}
|
|
358
360
|
// Which commit a workspace is running.
|
|
359
361
|
//
|
package/dist/src/platform.d.ts
CHANGED
|
@@ -24,6 +24,16 @@ export interface PlatformHooks {
|
|
|
24
24
|
/** The egress proxy — a per-step lease that keeps credentials out of the
|
|
25
25
|
* sandbox. Default: none, and the runner materialises as it always did. */
|
|
26
26
|
egress: EgressHooks;
|
|
27
|
+
/** A workspace's files changed — written, deployed, pushed, or the
|
|
28
|
+
* workspace deleted. What a scheduler that keeps flows in memory rather
|
|
29
|
+
* than re-reading every workspace each tick needs to hear. Default:
|
|
30
|
+
* nothing — locally the scheduler reads the disk. Never throws. */
|
|
31
|
+
workspaceChanged(tenant: string, workspace: string, why: "write" | "deploy" | "push" | "delete"): void;
|
|
32
|
+
/** Can a step in this isolation mode be re-attached to after the driver
|
|
33
|
+
* that started it is gone? True means an orphaned step with a `sandbox`
|
|
34
|
+
* on its record is resumed rather than destroyed and run again. Default:
|
|
35
|
+
* nothing is. */
|
|
36
|
+
sandboxResumable(kind: string): boolean;
|
|
27
37
|
}
|
|
28
38
|
export declare const platform: PlatformHooks;
|
|
29
39
|
/** Install the platform's implementations. Partial: what is not given keeps its local default. */
|
package/dist/src/platform.js
CHANGED
|
@@ -24,6 +24,8 @@ const local = {
|
|
|
24
24
|
previewSourceOf: () => null,
|
|
25
25
|
tenantKey: () => null,
|
|
26
26
|
egress: { lease: async () => null },
|
|
27
|
+
workspaceChanged() { },
|
|
28
|
+
sandboxResumable: () => false,
|
|
27
29
|
};
|
|
28
30
|
// One object per PROCESS, not per module instance. A bundler that compiles
|
|
29
31
|
// the boot file and each route handler separately (Next does) gives every
|
|
@@ -78,6 +78,10 @@ export interface StepActuals {
|
|
|
78
78
|
export interface ContainerStepOutcome {
|
|
79
79
|
status: "completed" | "failed";
|
|
80
80
|
result: string | null;
|
|
81
|
+
/** Why the sandbox ended when the step did not: what the cluster said
|
|
82
|
+
* (OOMKilled, Evicted, the pod gone). Absent when the driver finished
|
|
83
|
+
* and said so itself. A retry policy reads this. */
|
|
84
|
+
reason?: string | null;
|
|
81
85
|
/** The model's final text block — see StepRecord.conclusion. Crosses the
|
|
82
86
|
* boundary with the rest of the outcome; the driver spreads it. */
|
|
83
87
|
conclusion?: string | null;
|
|
@@ -120,7 +124,14 @@ export declare function allowedBack(rel: string): boolean;
|
|
|
120
124
|
* the step did not touch it, so it is not ours to write back. Concurrent
|
|
121
125
|
* edits survive, and a step that genuinely changed a file still wins.
|
|
122
126
|
*/
|
|
123
|
-
|
|
127
|
+
/**
|
|
128
|
+
* What a workspace copy looked like when it went in: the sha256 of every
|
|
129
|
+
* file the step could hand back. Small enough to ride into the sandbox
|
|
130
|
+
* beside the input, so a driver that inherits the step can tell what the
|
|
131
|
+
* step changed without the staging copy the first driver held in /tmp.
|
|
132
|
+
*/
|
|
133
|
+
export declare function hashTree(dir: string): Record<string, string>;
|
|
134
|
+
export declare function applyContainerChanges(hostWs: string, containerWs: string, baseline?: string | Record<string, string>): string[];
|
|
124
135
|
/**
|
|
125
136
|
* Where the driver builds venvs and npm prefixes, as an absolute path *inside*
|
|
126
137
|
* the runner. Fixed, and it has to be: a venv bakes its own absolute path into
|
|
@@ -185,6 +196,18 @@ export interface RunInContainerArgs {
|
|
|
185
196
|
runId?: string;
|
|
186
197
|
/** The reservation class — which limits this step's sandbox holds. */
|
|
187
198
|
size?: "small" | "large" | "heavy";
|
|
199
|
+
/** Re-attach to a sandbox a previous driver started, instead of creating
|
|
200
|
+
* one: its handle, and how many output lines that driver had applied.
|
|
201
|
+
* The executor replays nothing before `consumed`, and reads the step's
|
|
202
|
+
* files back from the sandbox's own copy-in snapshot. */
|
|
203
|
+
resume?: {
|
|
204
|
+
ref: string;
|
|
205
|
+
consumed: number;
|
|
206
|
+
} | null;
|
|
207
|
+
/** Called by an executor that can be resumed: when its sandbox exists
|
|
208
|
+
* (consumed 0), and before each output line it applies. The runner
|
|
209
|
+
* records these on the step, so the next driver knows where to attach. */
|
|
210
|
+
checkpoint?: (ref: string, consumed: number) => void;
|
|
188
211
|
}
|
|
189
212
|
/** The limits a size class reserves. Large is the install's configured
|
|
190
213
|
* limits unchanged, so existing installs bill and behave identically;
|
|
@@ -95,6 +95,34 @@ export function allowedBack(rel) {
|
|
|
95
95
|
* the step did not touch it, so it is not ours to write back. Concurrent
|
|
96
96
|
* edits survive, and a step that genuinely changed a file still wins.
|
|
97
97
|
*/
|
|
98
|
+
/**
|
|
99
|
+
* What a workspace copy looked like when it went in: the sha256 of every
|
|
100
|
+
* file the step could hand back. Small enough to ride into the sandbox
|
|
101
|
+
* beside the input, so a driver that inherits the step can tell what the
|
|
102
|
+
* step changed without the staging copy the first driver held in /tmp.
|
|
103
|
+
*/
|
|
104
|
+
export function hashTree(dir) {
|
|
105
|
+
const out = {};
|
|
106
|
+
const walk = (d) => {
|
|
107
|
+
for (const entry of fs.readdirSync(d, { withFileTypes: true })) {
|
|
108
|
+
const abs = path.join(d, entry.name);
|
|
109
|
+
const rel = path.relative(dir, abs).replaceAll("\\", "/");
|
|
110
|
+
if (entry.isSymbolicLink())
|
|
111
|
+
continue;
|
|
112
|
+
if (entry.isDirectory()) {
|
|
113
|
+
if (allowedBack(rel + "/"))
|
|
114
|
+
walk(abs);
|
|
115
|
+
continue;
|
|
116
|
+
}
|
|
117
|
+
if (!allowedBack(rel))
|
|
118
|
+
continue;
|
|
119
|
+
out[rel] = crypto.createHash("sha256").update(fs.readFileSync(abs)).digest("hex");
|
|
120
|
+
}
|
|
121
|
+
};
|
|
122
|
+
if (fs.existsSync(dir))
|
|
123
|
+
walk(dir);
|
|
124
|
+
return out;
|
|
125
|
+
}
|
|
98
126
|
export function applyContainerChanges(hostWs, containerWs, baseline) {
|
|
99
127
|
const applied = [];
|
|
100
128
|
const walk = (dir) => {
|
|
@@ -119,11 +147,16 @@ export function applyContainerChanges(hostWs, containerWs, baseline) {
|
|
|
119
147
|
const next = fs.readFileSync(abs);
|
|
120
148
|
// Untouched by this step: whatever the host says now is more current
|
|
121
149
|
// than what we handed in, including a change made while it ran.
|
|
122
|
-
if (baseline) {
|
|
150
|
+
if (typeof baseline === "string") {
|
|
123
151
|
const was = path.join(baseline, rel);
|
|
124
152
|
if (fs.existsSync(was) && fs.readFileSync(was).equals(next))
|
|
125
153
|
continue;
|
|
126
154
|
}
|
|
155
|
+
else if (baseline) {
|
|
156
|
+
const was = baseline[rel.replaceAll("\\", "/")];
|
|
157
|
+
if (was && was === crypto.createHash("sha256").update(next).digest("hex"))
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
127
160
|
const target = path.join(hostWs, rel);
|
|
128
161
|
if (fs.existsSync(target) && fs.readFileSync(target).equals(next))
|
|
129
162
|
continue;
|
package/dist/src/runner.d.ts
CHANGED
|
@@ -206,7 +206,7 @@ export interface Reconciliation {
|
|
|
206
206
|
* outlives any process. A run whose person already answered is not waiting,
|
|
207
207
|
* and is picked back up — see below.
|
|
208
208
|
*/
|
|
209
|
-
export declare function reconcileRuns(tenant: string, now?: number): Reconciliation[];
|
|
209
|
+
export declare function reconcileRuns(tenant: string, now?: number, live?: ReadonlySet<string>): Reconciliation[];
|
|
210
210
|
/**
|
|
211
211
|
* The same, for every account this installation holds.
|
|
212
212
|
*
|
|
@@ -214,7 +214,7 @@ export declare function reconcileRuns(tenant: string, now?: number): Reconciliat
|
|
|
214
214
|
* one hard-coded account left every other account's interrupted runs marked
|
|
215
215
|
* `running` forever, with no process left to finish them.
|
|
216
216
|
*/
|
|
217
|
-
export declare function reconcileAllRuns(now?: number): Reconciliation[];
|
|
217
|
+
export declare function reconcileAllRuns(now?: number, live?: ReadonlySet<string>): Reconciliation[];
|
|
218
218
|
/**
|
|
219
219
|
* Wait for a run to stop being in progress.
|
|
220
220
|
*
|
package/dist/src/runner.js
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
import fs from "node:fs";
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import matter from "gray-matter";
|
|
8
|
-
import { executeStep, extractJson, stepCeiling } from "./step-exec.js";
|
|
8
|
+
import { knownPrice, executeStep, extractJson, stepCeiling } from "./step-exec.js";
|
|
9
9
|
import { eventUrl } from "./webhook.js";
|
|
10
10
|
import { runStepInContainer, sizeLimits, killRunSandboxes } from "./run-container.js";
|
|
11
11
|
import { EGRESS_ENV, MODEL_KEY_NAME, addGrant, hostOf, placeholderNames, proxyModelEnv, unsubstitute } from "./egress.js";
|
|
@@ -1158,7 +1158,10 @@ stepBudgetUsd = null) {
|
|
|
1158
1158
|
// step: the generic HTTP tool still works without the typed ones.
|
|
1159
1159
|
for (const w of await prefetchOpenApi(tenant, openApiSources(agentDir, tenant)))
|
|
1160
1160
|
push("error", `openapi: ${w}`);
|
|
1161
|
-
const { front, clockEnv, systemPrompt, allowed, disabled, apiTools, scriptTools, secretEnv, secretScopes, missingSecrets, missingTools, runtime, unknownTools, shadowed, legacyUse, mcpServers, mcpNames, apiSpecs, scriptSpecs, brokenTools, size, providerEnv, providerLabel, providerSecrets, providerWarnings, formatWarning, fallbackEnv, apiWarnings, searchRoots, historyDigest, deskDigest, searchTools, historyTools, deskTools, translator, fallbackTranslator, } = agentContext(agentDir, tenant, tags, { runId, agent: step.agent });
|
|
1161
|
+
const { front, clockEnv, systemPrompt, allowed, disabled, apiTools, scriptTools, secretEnv, secretScopes, missingSecrets, missingTools, runtime, unknownTools, shadowed, legacyUse, mcpServers, mcpNames, apiSpecs, scriptSpecs, brokenTools, size: agentSize, providerEnv, providerLabel, providerSecrets, providerWarnings, formatWarning, fallbackEnv, apiWarnings, searchRoots, historyDigest, deskDigest, searchTools, historyTools, deskTools, translator, fallbackTranslator, } = agentContext(agentDir, tenant, tags, { runId, agent: step.agent });
|
|
1162
|
+
// A retry that moved the step up a class (an evicted or OOM-killed
|
|
1163
|
+
// attempt) wins over the agent's own `size:` for the attempts after.
|
|
1164
|
+
const size = step.sizeUp ?? agentSize;
|
|
1162
1165
|
// oauth2 secrets become live access tokens here — the one async moment
|
|
1163
1166
|
// in secret resolution, at the last host-side point before anything is
|
|
1164
1167
|
// injected or substituted. The refresh recipes never leave this process:
|
|
@@ -1289,6 +1292,11 @@ stepBudgetUsd = null) {
|
|
|
1289
1292
|
push("info", fitted.note);
|
|
1290
1293
|
effort = fitted.effort;
|
|
1291
1294
|
}
|
|
1295
|
+
// No catalogue — the platform's own Anthropic key, or a gateway that
|
|
1296
|
+
// publishes no prices — but the model's NAME is often enough. Without
|
|
1297
|
+
// this the in-pod meter priced haiku and sonnet at Opus rates and a
|
|
1298
|
+
// step was stopped at up to fifteen times what it had spent.
|
|
1299
|
+
tokenPrice ??= knownPrice(wireModel);
|
|
1292
1300
|
fs.mkdirSync(path.join(agentDir, "outputs"), { recursive: true });
|
|
1293
1301
|
fs.mkdirSync(path.join(agentDir, "memory"), { recursive: true });
|
|
1294
1302
|
// Expose the workspace's shared scripts/ as ./shared so every agent reaches
|
|
@@ -1530,6 +1538,19 @@ stepBudgetUsd = null) {
|
|
|
1530
1538
|
// Stamps the sandbox, so stopping this run can destroy it.
|
|
1531
1539
|
runId,
|
|
1532
1540
|
size,
|
|
1541
|
+
// Resume: a sandbox a previous driver left running, recorded on the
|
|
1542
|
+
// step. Consumed once — a fresh attempt after this one (a retry, a
|
|
1543
|
+
// fallback) starts its own sandbox.
|
|
1544
|
+
resume: step.sandbox && step.sandbox.kind === isolation ? { ref: step.sandbox.ref, consumed: step.sandbox.consumed } : null,
|
|
1545
|
+
// Checkpoint: where the executor is, on the record, so a driver
|
|
1546
|
+
// that inherits this run can attach where this one left off. The
|
|
1547
|
+
// executor calls it before applying each line; the line's own emit
|
|
1548
|
+
// saves the record, so this only updates the fields.
|
|
1549
|
+
checkpoint: (ref, consumed) => {
|
|
1550
|
+
step.sandbox = { kind: isolation, ref, consumed, since: step.sandbox?.since ?? new Date().toISOString() };
|
|
1551
|
+
if (consumed === 0)
|
|
1552
|
+
save();
|
|
1553
|
+
},
|
|
1533
1554
|
});
|
|
1534
1555
|
// A busy provider gets another moment on the SAME supply before the
|
|
1535
1556
|
// fallback is considered: under a fan-out, twenty steps against one
|
|
@@ -1539,6 +1560,7 @@ stepBudgetUsd = null) {
|
|
|
1539
1560
|
const firstArgs = isolatedArgs(platformModelEnv());
|
|
1540
1561
|
await lease?.commit();
|
|
1541
1562
|
let outcome = await runIsolated(firstArgs);
|
|
1563
|
+
step.sandbox = null; // whatever happens next starts its own
|
|
1542
1564
|
for (let n = 1; n <= OVERLOAD_RETRIES && outcome.status === "failed" && isTransientOverload(lastRefusal); n++) {
|
|
1543
1565
|
const wait = backoffMs(n, lastRefusal);
|
|
1544
1566
|
push("info", `provider busy (${lastRefusal.slice(0, 60)}) — waiting ${(wait / 1000).toFixed(1)}s, attempt ${n} of ${OVERLOAD_RETRIES}`);
|
|
@@ -1587,6 +1609,11 @@ stepBudgetUsd = null) {
|
|
|
1587
1609
|
step.status = outcome.status;
|
|
1588
1610
|
step.result = outcome.result;
|
|
1589
1611
|
step.conclusion = outcome.conclusion;
|
|
1612
|
+
step.sandbox = null;
|
|
1613
|
+
// The cluster's word on why a sandbox ended without the driver saying
|
|
1614
|
+
// so — OOMKilled, Evicted — on the record, where the retry reads it.
|
|
1615
|
+
if (outcome.status === "failed" && outcome.reason)
|
|
1616
|
+
push("error", `sandbox ended: ${outcome.reason}`);
|
|
1590
1617
|
if (outcome.data !== undefined)
|
|
1591
1618
|
step.data = outcome.data;
|
|
1592
1619
|
step.costUsd = repriced(catalog, wireModel, outcome.usage ?? null, outcome.costUsd, push);
|
|
@@ -2262,16 +2289,26 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
|
|
|
2262
2289
|
// never happened, which reconcile can then never see. Reset it to
|
|
2263
2290
|
// pending and it simply runs again; driveRun's whole contract is that
|
|
2264
2291
|
// re-driving a partly-finished record is safe.
|
|
2292
|
+
// Unless the step's sandbox outlived its driver AND the executor can
|
|
2293
|
+
// re-attach to it: then the record carries the pod and how far the
|
|
2294
|
+
// last driver got, runStep passes that as `resume`, and the step
|
|
2295
|
+
// carries on from where it was — a deploy no longer costs the run.
|
|
2265
2296
|
let orphaned = false;
|
|
2266
2297
|
for (const step of run.steps) {
|
|
2267
2298
|
if (step.status !== "running")
|
|
2268
2299
|
continue;
|
|
2269
|
-
|
|
2300
|
+
const resumable = !!step.sandbox && platform.sandboxResumable(step.sandbox.kind);
|
|
2301
|
+
if (!resumable) {
|
|
2302
|
+
orphaned = true;
|
|
2303
|
+
step.sandbox = null;
|
|
2304
|
+
}
|
|
2270
2305
|
step.status = "pending";
|
|
2271
2306
|
step.events.push({
|
|
2272
2307
|
t: new Date().toISOString(),
|
|
2273
2308
|
type: "info",
|
|
2274
|
-
text:
|
|
2309
|
+
text: resumable
|
|
2310
|
+
? `driver changed mid-step — re-attaching to the running sandbox (${step.sandbox.ref}, ${step.sandbox.consumed} lines already applied)`
|
|
2311
|
+
: "interrupted mid-step (platform restart) — running again from the start of the step",
|
|
2275
2312
|
});
|
|
2276
2313
|
}
|
|
2277
2314
|
if (orphaned) {
|
|
@@ -2710,7 +2747,12 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
|
|
|
2710
2747
|
await Promise.all(launching
|
|
2711
2748
|
.map(async (step) => {
|
|
2712
2749
|
const attempts = (step.retry ?? 0) + 1;
|
|
2713
|
-
|
|
2750
|
+
// A step re-attaching to its sandbox is still on the attempt
|
|
2751
|
+
// that started it; a step re-run after a backoff picks up the
|
|
2752
|
+
// count it left. Only a step nobody has touched starts at 1.
|
|
2753
|
+
const interruptedInBackoff = !step.sandbox && !!step.attempts && /retrying in \d+s/.test(step.events[step.events.length - 1]?.text ?? "");
|
|
2754
|
+
const from = step.sandbox ? Math.max(1, step.attempts ?? 1) : interruptedInBackoff ? Math.min(attempts, step.attempts + 1) : 1;
|
|
2755
|
+
for (let attempt = from; attempt <= attempts; attempt++) {
|
|
2714
2756
|
step.attempts = attempt;
|
|
2715
2757
|
step.status = "running";
|
|
2716
2758
|
save();
|
|
@@ -2727,12 +2769,43 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
|
|
|
2727
2769
|
save();
|
|
2728
2770
|
if (run.stopRequested)
|
|
2729
2771
|
break;
|
|
2772
|
+
// The retry policy. An attempt the cluster ended for want of
|
|
2773
|
+
// memory or disk is retried one size class up — the same
|
|
2774
|
+
// size would end the same way. Every retry waits: 15 s, 30 s,
|
|
2775
|
+
// 1 m, 2 m, 4 m, jittered, so a provider or a site that was
|
|
2776
|
+
// struggling gets a moment, and twenty fan-out steps do not
|
|
2777
|
+
// hammer it in lock-step. The wait holds a worker slot, not
|
|
2778
|
+
// a sandbox — nothing is rented while it waits — and a stop
|
|
2779
|
+
// ends it. The step is `pending` meanwhile, so a driver that
|
|
2780
|
+
// inherits the run mid-wait simply runs the attempt.
|
|
2781
|
+
const last = [...step.events].reverse().find((e) => e.type === "error")?.text ?? "";
|
|
2782
|
+
let sizeNote = "";
|
|
2783
|
+
if (/OOMKilled|Evicted|ephemeral-storage|out of memory/i.test(last)) {
|
|
2784
|
+
const up = step.sizeUp === "large" || (!step.sizeUp && step.size === "large") ? "heavy" : step.sizeUp === "heavy" || step.size === "heavy" ? null : "large";
|
|
2785
|
+
if (up) {
|
|
2786
|
+
step.sizeUp = up;
|
|
2787
|
+
sizeNote = ` at size: ${up}`;
|
|
2788
|
+
}
|
|
2789
|
+
}
|
|
2790
|
+
const waitMs = Math.round(Math.min(retryBaseMs() * 16, retryBaseMs() * 2 ** (attempt - 1)) * (0.75 + Math.random() * 0.5));
|
|
2791
|
+
step.status = "pending";
|
|
2730
2792
|
step.events.push({
|
|
2731
2793
|
t: new Date().toISOString(),
|
|
2732
2794
|
type: "info",
|
|
2733
|
-
text: `attempt ${attempt} failed — retrying (${attempts - attempt} left)`,
|
|
2795
|
+
text: `attempt ${attempt} failed — retrying in ${Math.round(waitMs / 1000)}s${sizeNote} (${attempts - attempt} left)`,
|
|
2734
2796
|
});
|
|
2735
2797
|
save();
|
|
2798
|
+
const until = Date.now() + waitMs;
|
|
2799
|
+
while (Date.now() < until) {
|
|
2800
|
+
await sleep(Math.min(1000, until - Date.now()));
|
|
2801
|
+
save();
|
|
2802
|
+
if (run.stopRequested)
|
|
2803
|
+
break;
|
|
2804
|
+
}
|
|
2805
|
+
if (run.stopRequested) {
|
|
2806
|
+
step.status = "failed";
|
|
2807
|
+
break;
|
|
2808
|
+
}
|
|
2736
2809
|
}
|
|
2737
2810
|
}));
|
|
2738
2811
|
// on-fail: another agent takes the step over. Sequential and rare:
|
|
@@ -3051,6 +3124,9 @@ export function loadFlow(tenant, workspace, flowName) {
|
|
|
3051
3124
|
// once it has been quiet for two minutes — the margin is for a record
|
|
3052
3125
|
// written a beat before its driver registered, not a judgement about pace.
|
|
3053
3126
|
const ABANDONED_AFTER_MS = 2 * 60 * 1000;
|
|
3127
|
+
/** The first retry's wait; each one after doubles it, to sixteen times
|
|
3128
|
+
* this (15 s → 4 min by default). An env, so a test can make it small. */
|
|
3129
|
+
const retryBaseMs = () => Number(process.env.FOLDRUN_RETRY_BASE_MS) || 15_000;
|
|
3054
3130
|
/** The most recent moment a run showed any sign of life. */
|
|
3055
3131
|
function lastActivity(run) {
|
|
3056
3132
|
let latest = Date.parse(run.startedAt);
|
|
@@ -3082,12 +3158,19 @@ function lastActivity(run) {
|
|
|
3082
3158
|
* outlives any process. A run whose person already answered is not waiting,
|
|
3083
3159
|
* and is picked back up — see below.
|
|
3084
3160
|
*/
|
|
3085
|
-
export function reconcileRuns(tenant, now = Date.now()) {
|
|
3161
|
+
export function reconcileRuns(tenant, now = Date.now(), live) {
|
|
3086
3162
|
const closed = [];
|
|
3087
3163
|
for (const workspace of listWorkspaces(tenant)) {
|
|
3088
3164
|
for (const summary of listRuns(tenant, workspace.name)) {
|
|
3089
3165
|
if (drivingRuns.has(summary.id))
|
|
3090
3166
|
continue;
|
|
3167
|
+
// Driven by another process — a second worker pod, on a platform with
|
|
3168
|
+
// more than one. The idle window protects a quiet step from THIS
|
|
3169
|
+
// process's judgement only; a worker that boots beside a live one
|
|
3170
|
+
// would otherwise close every long step its neighbour is driving.
|
|
3171
|
+
// The caller says which runs those are (a heartbeated claim).
|
|
3172
|
+
if (live?.has(summary.id))
|
|
3173
|
+
continue;
|
|
3091
3174
|
if (now - lastActivity(summary) < ABANDONED_AFTER_MS)
|
|
3092
3175
|
continue;
|
|
3093
3176
|
// Approved, then abandoned. Approval only writes to the run record; the
|
|
@@ -3168,8 +3251,8 @@ export function reconcileRuns(tenant, now = Date.now()) {
|
|
|
3168
3251
|
* one hard-coded account left every other account's interrupted runs marked
|
|
3169
3252
|
* `running` forever, with no process left to finish them.
|
|
3170
3253
|
*/
|
|
3171
|
-
export function reconcileAllRuns(now = Date.now()) {
|
|
3172
|
-
return listTenants().flatMap((tenant) => reconcileRuns(tenant, now));
|
|
3254
|
+
export function reconcileAllRuns(now = Date.now(), live) {
|
|
3255
|
+
return listTenants().flatMap((tenant) => reconcileRuns(tenant, now, live));
|
|
3173
3256
|
}
|
|
3174
3257
|
/**
|
|
3175
3258
|
* Wait for a run to stop being in progress.
|
package/dist/src/starter.js
CHANGED
|
@@ -304,6 +304,7 @@ timezone: Australia/Sydney
|
|
|
304
304
|
model: default # a default for every step here
|
|
305
305
|
effort: low
|
|
306
306
|
overlap: skip # skip | queue — what a second run does while one is live
|
|
307
|
+
priority: normal # high | normal | low — where its runs stand in the queue
|
|
307
308
|
---
|
|
308
309
|
|
|
309
310
|
1. [[researcher]] — Find three sources on {{topic}}.
|
package/dist/src/step-exec.d.ts
CHANGED
|
@@ -70,16 +70,26 @@ export type EventExtra = {
|
|
|
70
70
|
ms?: number;
|
|
71
71
|
err?: boolean;
|
|
72
72
|
};
|
|
73
|
-
/** Conservative per-token rates for a model
|
|
74
|
-
*
|
|
73
|
+
/** Conservative per-token rates for a model nothing else can price — an
|
|
74
|
+
* unknown id on a gateway with no catalogue. Opus-class, so a ceiling
|
|
75
|
+
* still means something. Only ever reached when `knownPrice` below does
|
|
76
|
+
* not recognise the model either. */
|
|
75
77
|
export declare const FALLBACK_PRICE: {
|
|
76
78
|
input: number;
|
|
77
79
|
output: number;
|
|
78
80
|
};
|
|
81
|
+
/** What a token costs on this model, when we can say without a catalogue. */
|
|
82
|
+
export declare function knownPrice(model: string | undefined): {
|
|
83
|
+
input: number;
|
|
84
|
+
output: number;
|
|
85
|
+
} | null;
|
|
79
86
|
/**
|
|
80
|
-
* What one assistant turn cost
|
|
81
|
-
*
|
|
82
|
-
*
|
|
87
|
+
* What one assistant turn cost, from the usage the SDK reports. Pure.
|
|
88
|
+
*
|
|
89
|
+
* A gateway whose cache is priced differently is over- or under-counted
|
|
90
|
+
* here by that difference alone; the run record's own cost is repriced
|
|
91
|
+
* from the catalogue afterwards and stays the authority on the bill. This
|
|
92
|
+
* number exists to stop a runaway step mid-turn, and wants to be close.
|
|
83
93
|
*/
|
|
84
94
|
export declare function priceTurn(u: Record<string, number | undefined>, price: {
|
|
85
95
|
input: number;
|
package/dist/src/step-exec.js
CHANGED
|
@@ -12,19 +12,55 @@ import path from "node:path";
|
|
|
12
12
|
import { query } from "@anthropic-ai/claude-agent-sdk";
|
|
13
13
|
import { spawn } from "node:child_process";
|
|
14
14
|
import { checkPaths, checkBash, isFilesystemTool } from "./confine.js";
|
|
15
|
-
/** Conservative per-token rates for a model
|
|
16
|
-
*
|
|
15
|
+
/** Conservative per-token rates for a model nothing else can price — an
|
|
16
|
+
* unknown id on a gateway with no catalogue. Opus-class, so a ceiling
|
|
17
|
+
* still means something. Only ever reached when `knownPrice` below does
|
|
18
|
+
* not recognise the model either. */
|
|
17
19
|
export const FALLBACK_PRICE = { input: 15e-6, output: 75e-6 };
|
|
20
|
+
/** The rates we know without asking anyone: the tiers this runtime resolves
|
|
21
|
+
* (`fast`/`default`/`max` → haiku/sonnet/opus) and the model ids that carry
|
|
22
|
+
* those words. A gateway with a catalogue still wins — that is the
|
|
23
|
+
* authority — but the platform's own Anthropic key has no catalogue, and
|
|
24
|
+
* pricing its steps at Opus rates made every mid-turn ceiling read up to
|
|
25
|
+
* fifteen times high: an indexing run was stopped at "$1.17" having
|
|
26
|
+
* actually spent $0.15 (2026-09-08). Per token, input/output. */
|
|
27
|
+
const KNOWN_PRICES = [
|
|
28
|
+
[/haiku/i, { input: 1e-6, output: 5e-6 }],
|
|
29
|
+
[/sonnet/i, { input: 2e-6, output: 10e-6 }],
|
|
30
|
+
[/\bopus\b/i, { input: 5e-6, output: 25e-6 }],
|
|
31
|
+
];
|
|
32
|
+
/** What a token costs on this model, when we can say without a catalogue. */
|
|
33
|
+
export function knownPrice(model) {
|
|
34
|
+
if (!model)
|
|
35
|
+
return null;
|
|
36
|
+
return KNOWN_PRICES.find(([re]) => re.test(model))?.[1] ?? null;
|
|
37
|
+
}
|
|
38
|
+
/** Anthropic's cache multipliers, and the shape every gateway that bills
|
|
39
|
+
* for caching at all has copied: a write costs a quarter more than fresh
|
|
40
|
+
* input, a read a tenth of it. Counting both at the full input rate — as
|
|
41
|
+
* this did — makes a long cached prompt read about ten times its price,
|
|
42
|
+
* which on a ceiling is not "the honest direction to be wrong in", it is
|
|
43
|
+
* a step killed for spending money it never spent. */
|
|
44
|
+
const CACHE_WRITE_MULTIPLIER = 1.25;
|
|
45
|
+
const CACHE_READ_MULTIPLIER = 0.1;
|
|
18
46
|
/**
|
|
19
|
-
* What one assistant turn cost
|
|
20
|
-
*
|
|
21
|
-
*
|
|
47
|
+
* What one assistant turn cost, from the usage the SDK reports. Pure.
|
|
48
|
+
*
|
|
49
|
+
* A gateway whose cache is priced differently is over- or under-counted
|
|
50
|
+
* here by that difference alone; the run record's own cost is repriced
|
|
51
|
+
* from the catalogue afterwards and stays the authority on the bill. This
|
|
52
|
+
* number exists to stop a runaway step mid-turn, and wants to be close.
|
|
22
53
|
*/
|
|
23
54
|
export function priceTurn(u, price) {
|
|
24
|
-
const inTok = (u.input_tokens ?? 0) + (u.cache_creation_input_tokens ?? 0) + (u.cache_read_input_tokens ?? 0);
|
|
25
|
-
const outTok = u.output_tokens ?? 0;
|
|
26
55
|
const p = price ?? FALLBACK_PRICE;
|
|
27
|
-
|
|
56
|
+
const fresh = u.input_tokens ?? 0;
|
|
57
|
+
const written = u.cache_creation_input_tokens ?? 0;
|
|
58
|
+
const read = u.cache_read_input_tokens ?? 0;
|
|
59
|
+
const outTok = u.output_tokens ?? 0;
|
|
60
|
+
return (fresh * p.input +
|
|
61
|
+
written * p.input * CACHE_WRITE_MULTIPLIER +
|
|
62
|
+
read * p.input * CACHE_READ_MULTIPLIER +
|
|
63
|
+
outTok * p.output);
|
|
28
64
|
}
|
|
29
65
|
/**
|
|
30
66
|
* Each step launched together gets an equal share of what is left of the
|
package/dist/src/store.d.ts
CHANGED
|
@@ -378,6 +378,12 @@ export interface FlowInfo {
|
|
|
378
378
|
* overlap) — a default that changed underfoot would surprise every flow
|
|
379
379
|
* that legitimately runs in parallel. */
|
|
380
380
|
overlap: "skip" | "queue" | null;
|
|
381
|
+
/** Where this flow's runs stand in the queue against everyone else's:
|
|
382
|
+
* "high" is claimed before the round-robin, "low" after it. Adds to the
|
|
383
|
+
* account's own priority; the account's cap still bounds how many run
|
|
384
|
+
* at once, so a high lane never starves the rest outright. Unset is
|
|
385
|
+
* normal — first come, fair across accounts. */
|
|
386
|
+
priority: "high" | "normal" | "low" | null;
|
|
381
387
|
/** The most one run of this flow may spend, in USD. A literal, never an
|
|
382
388
|
* expression — the cap is readable off the file, which is the property
|
|
383
389
|
* the whole grammar exists to keep. Checked between groups: the group
|
|
@@ -565,6 +571,9 @@ export interface ChangeMeta {
|
|
|
565
571
|
message?: string;
|
|
566
572
|
}
|
|
567
573
|
export declare function writeWorkspaceFile(tenant: string, workspace: string, rel: string, content: string, meta?: ChangeMeta): void;
|
|
574
|
+
/** Tell the platform, and never let its listener fail a write. */
|
|
575
|
+
declare function workspaceChanged(tenant: string, workspace: string, why: "write" | "deploy" | "push" | "delete"): void;
|
|
576
|
+
export { workspaceChanged as notifyWorkspaceChanged };
|
|
568
577
|
export declare function syncBundleFor(file: string, change?: "Creation" | "Update"): void;
|
|
569
578
|
/**
|
|
570
579
|
* Rename one editable file. Both ends pass the same gate as a write — a
|
|
@@ -653,6 +662,21 @@ export interface StepRecord {
|
|
|
653
662
|
verify?: string;
|
|
654
663
|
/** Attempts made so far, for the run trace. */
|
|
655
664
|
attempts?: number;
|
|
665
|
+
/** The sandbox this step is executing in, while it is: which executor,
|
|
666
|
+
* its handle (a pod name), and how many of its output lines the driver
|
|
667
|
+
* has already applied to this record. What a driver that inherits the
|
|
668
|
+
* run — after a deploy rolled the last one, or it crashed — needs to
|
|
669
|
+
* re-attach and carry on rather than run the step again. Cleared when
|
|
670
|
+
* the step settles. */
|
|
671
|
+
sandbox?: {
|
|
672
|
+
kind: string;
|
|
673
|
+
ref: string;
|
|
674
|
+
consumed: number;
|
|
675
|
+
since: string;
|
|
676
|
+
} | null;
|
|
677
|
+
/** A size class a retry moved the step up to (an evicted or OOM-killed
|
|
678
|
+
* attempt), overriding the agent's own `size:` for the attempts after. */
|
|
679
|
+
sizeUp?: "large" | "heavy";
|
|
656
680
|
/** When the step actually started and finished executing — not when the
|
|
657
681
|
* run reached it. A step's events approximate this (first and last), but
|
|
658
682
|
* a model can think for minutes before its first tool call, and a step
|
package/dist/src/store.js
CHANGED
|
@@ -23,6 +23,7 @@ import fs from "node:fs";
|
|
|
23
23
|
import path from "node:path";
|
|
24
24
|
import { recordRevision, registerTreeReader } from "./history.js";
|
|
25
25
|
import { dataRoot, singleWorkspace } from "./paths.js";
|
|
26
|
+
import { platform } from "./platform.js";
|
|
26
27
|
import matter from "gray-matter";
|
|
27
28
|
/**
|
|
28
29
|
* The frontmatter of one markdown file, as the runtime reads it. Exported
|
|
@@ -413,6 +414,12 @@ meta = {}) {
|
|
|
413
414
|
// them are not bundles until their index.md exists to carry okf_version. A
|
|
414
415
|
// deploy writes straight to disk, so nothing else would generate them.
|
|
415
416
|
syncWorkspaceBundles(dir);
|
|
417
|
+
// Here, not in deployWorkspace: a workspace CREATED with files (the API's
|
|
418
|
+
// create, the CLI's first deploy, a template) comes through this function
|
|
419
|
+
// without going through deployWorkspace, and the first live check found
|
|
420
|
+
// exactly that — a new desk's cron was seen on the five-minute rescan, not
|
|
421
|
+
// the next tick.
|
|
422
|
+
workspaceChanged(tenant, workspace, "deploy");
|
|
416
423
|
return { preserved: preserved.length };
|
|
417
424
|
}
|
|
418
425
|
// The prompt-facing index for an OKF bundle (memory/ or knowledge/).
|
|
@@ -1052,6 +1059,7 @@ export function parseFlow(file, raw) {
|
|
|
1052
1059
|
model: data.model ?? null,
|
|
1053
1060
|
effort: data.effort ?? null,
|
|
1054
1061
|
overlap: data.overlap === "skip" || data.overlap === "queue" ? data.overlap : null,
|
|
1062
|
+
priority: data.priority === "high" || data.priority === "low" || data.priority === "normal" ? data.priority : null,
|
|
1055
1063
|
budget: Number(data.budget) > 0 ? Number(data.budget) : null,
|
|
1056
1064
|
// `after: [[flow:publish]]` — a link or a bare name, read the same way
|
|
1057
1065
|
// as every other file-naming field (refs.ts).
|
|
@@ -1606,7 +1614,18 @@ export function writeWorkspaceFile(tenant, workspace, rel, content, meta = {}) {
|
|
|
1606
1614
|
// The revision: what it was, what it is now. Identical content records
|
|
1607
1615
|
// nothing — see history.ts.
|
|
1608
1616
|
recordRevision(tenant, workspace, [{ path: norm, before, after: fs.readFileSync(p, "utf8") }], meta);
|
|
1617
|
+
workspaceChanged(tenant, workspace, "write");
|
|
1609
1618
|
}
|
|
1619
|
+
/** Tell the platform, and never let its listener fail a write. */
|
|
1620
|
+
function workspaceChanged(tenant, workspace, why) {
|
|
1621
|
+
try {
|
|
1622
|
+
platform.workspaceChanged(tenant, workspace, why);
|
|
1623
|
+
}
|
|
1624
|
+
catch {
|
|
1625
|
+
// a scheduler that could not be told will find out on its next rescan
|
|
1626
|
+
}
|
|
1627
|
+
}
|
|
1628
|
+
export { workspaceChanged as notifyWorkspaceChanged };
|
|
1610
1629
|
// memory/ and knowledge/ are OKF bundles, so index.md has to stay current —
|
|
1611
1630
|
// a generated index that drifts is the same class of bug as a memory index
|
|
1612
1631
|
// naming files that don't exist.
|
|
@@ -1697,6 +1716,7 @@ export function renameWorkspaceFile(tenant, workspace, from, to, meta = {}) {
|
|
|
1697
1716
|
], { message: `renamed ${from} → ${to}`, ...meta });
|
|
1698
1717
|
syncBundleFor(src);
|
|
1699
1718
|
syncBundleFor(dst, "Update");
|
|
1719
|
+
workspaceChanged(tenant, workspace, "write");
|
|
1700
1720
|
}
|
|
1701
1721
|
/** Every text file under a path, as revision entries recording deletion. */
|
|
1702
1722
|
function deletionsUnder(dir, norm) {
|
|
@@ -1759,12 +1779,14 @@ export function deleteWorkspacePath(tenant, workspace, rel, meta = {}) {
|
|
|
1759
1779
|
recordRevision(tenant, workspace, gone, meta);
|
|
1760
1780
|
// A bundle's index must stop naming what is gone.
|
|
1761
1781
|
syncBundleFor(p);
|
|
1782
|
+
workspaceChanged(tenant, workspace, "write");
|
|
1762
1783
|
}
|
|
1763
1784
|
export function deleteWorkspace(tenant, workspace) {
|
|
1764
1785
|
const dir = workspaceDir(tenant, workspace);
|
|
1765
1786
|
if (!fs.existsSync(dir))
|
|
1766
1787
|
throw new Error(`workspace ${workspace} not found`);
|
|
1767
1788
|
fs.rmSync(dir, { recursive: true, force: true });
|
|
1789
|
+
workspaceChanged(tenant, workspace, "delete");
|
|
1768
1790
|
}
|
|
1769
1791
|
/**
|
|
1770
1792
|
* What a rename would break, so the caller can say so before doing it.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@foldrun/core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "Agents are just folders. The format, the runner, the checks.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"type": "module",
|
|
@@ -29,7 +29,8 @@
|
|
|
29
29
|
"prepare": "npm run build",
|
|
30
30
|
"prepublishOnly": "npm run build",
|
|
31
31
|
"test": "FOLDRUN_DATA=$(mktemp -d /tmp/foldrun-test-XXXXXX) node --test *.test.ts $(ls tests/*.test.ts | grep -v e2e)",
|
|
32
|
-
"container": "FOLDRUN_CONTAINER_E2E=1 node scripts/ts-test.mjs tests/container-e2e.test.ts --test-timeout=600000"
|
|
32
|
+
"container": "FOLDRUN_CONTAINER_E2E=1 node scripts/ts-test.mjs tests/container-e2e.test.ts --test-timeout=600000",
|
|
33
|
+
"release": "node scripts/release.mjs"
|
|
33
34
|
},
|
|
34
35
|
"engines": {
|
|
35
36
|
"node": ">=22"
|
package/src/deploy.ts
CHANGED
|
@@ -29,6 +29,7 @@ import {
|
|
|
29
29
|
WORKSPACE_DIRS,
|
|
30
30
|
assertSafeName,
|
|
31
31
|
type DeployFile,
|
|
32
|
+
notifyWorkspaceChanged,
|
|
32
33
|
} from "./store.ts";
|
|
33
34
|
import { conformanceIssues } from "./okf.ts";
|
|
34
35
|
import { repoDir } from "./gitrepo.ts";
|
|
@@ -373,6 +374,8 @@ export function writePushNote(tenant: string, workspace: string, note: PushNote)
|
|
|
373
374
|
} catch {
|
|
374
375
|
// The note is a record, not the deploy. Losing it must not fail a push.
|
|
375
376
|
}
|
|
377
|
+
// A push waiting on a run is the scheduler's to apply; tell it there is one.
|
|
378
|
+
notifyWorkspaceChanged(tenant, workspace, "push");
|
|
376
379
|
}
|
|
377
380
|
|
|
378
381
|
/**
|
|
@@ -428,7 +431,7 @@ export function deployWorkspace(
|
|
|
428
431
|
const { preserved } = saveWorkspace(tenant, workspace, files, { commit: opts.commit ?? null, by: opts.by ?? "deploy", message: opts.message });
|
|
429
432
|
const commit = opts.commit ?? null;
|
|
430
433
|
if (commit) writeDeployedCommit(tenant, workspace, commit);
|
|
431
|
-
return { ...plan, applied: true, commit, preserved };
|
|
434
|
+
return { ...plan, applied: true, commit, preserved }; // saveWorkspace told the platform
|
|
432
435
|
}
|
|
433
436
|
|
|
434
437
|
// Which commit a workspace is running.
|
package/src/platform.ts
CHANGED
|
@@ -45,6 +45,16 @@ export interface PlatformHooks {
|
|
|
45
45
|
/** The egress proxy — a per-step lease that keeps credentials out of the
|
|
46
46
|
* sandbox. Default: none, and the runner materialises as it always did. */
|
|
47
47
|
egress: EgressHooks;
|
|
48
|
+
/** A workspace's files changed — written, deployed, pushed, or the
|
|
49
|
+
* workspace deleted. What a scheduler that keeps flows in memory rather
|
|
50
|
+
* than re-reading every workspace each tick needs to hear. Default:
|
|
51
|
+
* nothing — locally the scheduler reads the disk. Never throws. */
|
|
52
|
+
workspaceChanged(tenant: string, workspace: string, why: "write" | "deploy" | "push" | "delete"): void;
|
|
53
|
+
/** Can a step in this isolation mode be re-attached to after the driver
|
|
54
|
+
* that started it is gone? True means an orphaned step with a `sandbox`
|
|
55
|
+
* on its record is resumed rather than destroyed and run again. Default:
|
|
56
|
+
* nothing is. */
|
|
57
|
+
sandboxResumable(kind: string): boolean;
|
|
48
58
|
}
|
|
49
59
|
|
|
50
60
|
const local: PlatformHooks = {
|
|
@@ -60,6 +70,8 @@ const local: PlatformHooks = {
|
|
|
60
70
|
previewSourceOf: () => null,
|
|
61
71
|
tenantKey: () => null,
|
|
62
72
|
egress: { lease: async () => null },
|
|
73
|
+
workspaceChanged() {},
|
|
74
|
+
sandboxResumable: () => false,
|
|
63
75
|
};
|
|
64
76
|
|
|
65
77
|
// One object per PROCESS, not per module instance. A bundler that compiles
|
package/src/run-container.ts
CHANGED
|
@@ -106,6 +106,10 @@ export interface StepActuals {
|
|
|
106
106
|
export interface ContainerStepOutcome {
|
|
107
107
|
status: "completed" | "failed";
|
|
108
108
|
result: string | null;
|
|
109
|
+
/** Why the sandbox ended when the step did not: what the cluster said
|
|
110
|
+
* (OOMKilled, Evicted, the pod gone). Absent when the driver finished
|
|
111
|
+
* and said so itself. A retry policy reads this. */
|
|
112
|
+
reason?: string | null;
|
|
109
113
|
/** The model's final text block — see StepRecord.conclusion. Crosses the
|
|
110
114
|
* boundary with the rest of the outcome; the driver spreads it. */
|
|
111
115
|
conclusion?: string | null;
|
|
@@ -186,10 +190,35 @@ export function allowedBack(rel: string): boolean {
|
|
|
186
190
|
* the step did not touch it, so it is not ours to write back. Concurrent
|
|
187
191
|
* edits survive, and a step that genuinely changed a file still wins.
|
|
188
192
|
*/
|
|
193
|
+
/**
|
|
194
|
+
* What a workspace copy looked like when it went in: the sha256 of every
|
|
195
|
+
* file the step could hand back. Small enough to ride into the sandbox
|
|
196
|
+
* beside the input, so a driver that inherits the step can tell what the
|
|
197
|
+
* step changed without the staging copy the first driver held in /tmp.
|
|
198
|
+
*/
|
|
199
|
+
export function hashTree(dir: string): Record<string, string> {
|
|
200
|
+
const out: Record<string, string> = {};
|
|
201
|
+
const walk = (d: string) => {
|
|
202
|
+
for (const entry of fs.readdirSync(d, { withFileTypes: true })) {
|
|
203
|
+
const abs = path.join(d, entry.name);
|
|
204
|
+
const rel = path.relative(dir, abs).replaceAll("\\", "/");
|
|
205
|
+
if (entry.isSymbolicLink()) continue;
|
|
206
|
+
if (entry.isDirectory()) {
|
|
207
|
+
if (allowedBack(rel + "/")) walk(abs);
|
|
208
|
+
continue;
|
|
209
|
+
}
|
|
210
|
+
if (!allowedBack(rel)) continue;
|
|
211
|
+
out[rel] = crypto.createHash("sha256").update(fs.readFileSync(abs)).digest("hex");
|
|
212
|
+
}
|
|
213
|
+
};
|
|
214
|
+
if (fs.existsSync(dir)) walk(dir);
|
|
215
|
+
return out;
|
|
216
|
+
}
|
|
217
|
+
|
|
189
218
|
export function applyContainerChanges(
|
|
190
219
|
hostWs: string,
|
|
191
220
|
containerWs: string,
|
|
192
|
-
baseline?: string,
|
|
221
|
+
baseline?: string | Record<string, string>,
|
|
193
222
|
): string[] {
|
|
194
223
|
const applied: string[] = [];
|
|
195
224
|
const walk = (dir: string) => {
|
|
@@ -211,9 +240,12 @@ export function applyContainerChanges(
|
|
|
211
240
|
const next = fs.readFileSync(abs);
|
|
212
241
|
// Untouched by this step: whatever the host says now is more current
|
|
213
242
|
// than what we handed in, including a change made while it ran.
|
|
214
|
-
if (baseline) {
|
|
243
|
+
if (typeof baseline === "string") {
|
|
215
244
|
const was = path.join(baseline, rel);
|
|
216
245
|
if (fs.existsSync(was) && fs.readFileSync(was).equals(next)) continue;
|
|
246
|
+
} else if (baseline) {
|
|
247
|
+
const was = baseline[rel.replaceAll("\\", "/")];
|
|
248
|
+
if (was && was === crypto.createHash("sha256").update(next).digest("hex")) continue;
|
|
217
249
|
}
|
|
218
250
|
const target = path.join(hostWs, rel);
|
|
219
251
|
if (fs.existsSync(target) && fs.readFileSync(target).equals(next)) continue;
|
|
@@ -625,6 +657,15 @@ export interface RunInContainerArgs {
|
|
|
625
657
|
runId?: string;
|
|
626
658
|
/** The reservation class — which limits this step's sandbox holds. */
|
|
627
659
|
size?: "small" | "large" | "heavy";
|
|
660
|
+
/** Re-attach to a sandbox a previous driver started, instead of creating
|
|
661
|
+
* one: its handle, and how many output lines that driver had applied.
|
|
662
|
+
* The executor replays nothing before `consumed`, and reads the step's
|
|
663
|
+
* files back from the sandbox's own copy-in snapshot. */
|
|
664
|
+
resume?: { ref: string; consumed: number } | null;
|
|
665
|
+
/** Called by an executor that can be resumed: when its sandbox exists
|
|
666
|
+
* (consumed 0), and before each output line it applies. The runner
|
|
667
|
+
* records these on the step, so the next driver knows where to attach. */
|
|
668
|
+
checkpoint?: (ref: string, consumed: number) => void;
|
|
628
669
|
}
|
|
629
670
|
|
|
630
671
|
/** The limits a size class reserves. Large is the install's configured
|
package/src/runner.ts
CHANGED
|
@@ -7,7 +7,8 @@ import fs from "node:fs";
|
|
|
7
7
|
import path from "node:path";
|
|
8
8
|
import matter from "gray-matter";
|
|
9
9
|
import { spawn } from "node:child_process";
|
|
10
|
-
import {
|
|
10
|
+
import {
|
|
11
|
+
knownPrice, executeStep, extractJson, stepCeiling, type EventExtra } from "./step-exec.ts";
|
|
11
12
|
import { eventUrl } from "./webhook.ts";
|
|
12
13
|
import { runStepInContainer, sizeLimits, killRunSandboxes, type StepTiming } from "./run-container.ts";
|
|
13
14
|
import { EGRESS_ENV, MODEL_KEY_NAME, addGrant, hostOf, placeholderNames, proxyModelEnv, unsubstitute, type EgressGrant } from "./egress.ts";
|
|
@@ -1297,11 +1298,14 @@ async function runStep(
|
|
|
1297
1298
|
front, clockEnv, systemPrompt, allowed, disabled, apiTools, scriptTools,
|
|
1298
1299
|
secretEnv, secretScopes, missingSecrets, missingTools, runtime,
|
|
1299
1300
|
unknownTools, shadowed, legacyUse, mcpServers, mcpNames,
|
|
1300
|
-
apiSpecs, scriptSpecs, brokenTools, size,
|
|
1301
|
+
apiSpecs, scriptSpecs, brokenTools, size: agentSize,
|
|
1301
1302
|
providerEnv, providerLabel, providerSecrets, providerWarnings, formatWarning,
|
|
1302
1303
|
fallbackEnv, apiWarnings, searchRoots, historyDigest, deskDigest, searchTools, historyTools, deskTools,
|
|
1303
1304
|
translator, fallbackTranslator,
|
|
1304
1305
|
} = agentContext(agentDir, tenant, tags, { runId, agent: step.agent });
|
|
1306
|
+
// A retry that moved the step up a class (an evicted or OOM-killed
|
|
1307
|
+
// attempt) wins over the agent's own `size:` for the attempts after.
|
|
1308
|
+
const size = step.sizeUp ?? agentSize;
|
|
1305
1309
|
|
|
1306
1310
|
// oauth2 secrets become live access tokens here — the one async moment
|
|
1307
1311
|
// in secret resolution, at the last host-side point before anything is
|
|
@@ -1439,6 +1443,11 @@ async function runStep(
|
|
|
1439
1443
|
if (fitted.note) push("info", fitted.note);
|
|
1440
1444
|
effort = fitted.effort;
|
|
1441
1445
|
}
|
|
1446
|
+
// No catalogue — the platform's own Anthropic key, or a gateway that
|
|
1447
|
+
// publishes no prices — but the model's NAME is often enough. Without
|
|
1448
|
+
// this the in-pod meter priced haiku and sonnet at Opus rates and a
|
|
1449
|
+
// step was stopped at up to fifteen times what it had spent.
|
|
1450
|
+
tokenPrice ??= knownPrice(wireModel);
|
|
1442
1451
|
|
|
1443
1452
|
fs.mkdirSync(path.join(agentDir, "outputs"), { recursive: true });
|
|
1444
1453
|
fs.mkdirSync(path.join(agentDir, "memory"), { recursive: true });
|
|
@@ -1690,6 +1699,18 @@ async function runStep(
|
|
|
1690
1699
|
// Stamps the sandbox, so stopping this run can destroy it.
|
|
1691
1700
|
runId,
|
|
1692
1701
|
size,
|
|
1702
|
+
// Resume: a sandbox a previous driver left running, recorded on the
|
|
1703
|
+
// step. Consumed once — a fresh attempt after this one (a retry, a
|
|
1704
|
+
// fallback) starts its own sandbox.
|
|
1705
|
+
resume: step.sandbox && step.sandbox.kind === isolation ? { ref: step.sandbox.ref, consumed: step.sandbox.consumed } : null,
|
|
1706
|
+
// Checkpoint: where the executor is, on the record, so a driver
|
|
1707
|
+
// that inherits this run can attach where this one left off. The
|
|
1708
|
+
// executor calls it before applying each line; the line's own emit
|
|
1709
|
+
// saves the record, so this only updates the fields.
|
|
1710
|
+
checkpoint: (ref: string, consumed: number) => {
|
|
1711
|
+
step.sandbox = { kind: isolation!, ref, consumed, since: step.sandbox?.since ?? new Date().toISOString() };
|
|
1712
|
+
if (consumed === 0) save();
|
|
1713
|
+
},
|
|
1693
1714
|
});
|
|
1694
1715
|
|
|
1695
1716
|
// A busy provider gets another moment on the SAME supply before the
|
|
@@ -1700,6 +1721,7 @@ async function runStep(
|
|
|
1700
1721
|
const firstArgs = isolatedArgs(platformModelEnv());
|
|
1701
1722
|
await lease?.commit();
|
|
1702
1723
|
let outcome = await runIsolated(firstArgs);
|
|
1724
|
+
step.sandbox = null; // whatever happens next starts its own
|
|
1703
1725
|
for (let n = 1; n <= OVERLOAD_RETRIES && outcome.status === "failed" && isTransientOverload(lastRefusal); n++) {
|
|
1704
1726
|
const wait = backoffMs(n, lastRefusal);
|
|
1705
1727
|
push("info", `provider busy (${lastRefusal.slice(0, 60)}) — waiting ${(wait / 1000).toFixed(1)}s, attempt ${n} of ${OVERLOAD_RETRIES}`);
|
|
@@ -1752,6 +1774,10 @@ async function runStep(
|
|
|
1752
1774
|
step.status = outcome.status;
|
|
1753
1775
|
step.result = outcome.result;
|
|
1754
1776
|
step.conclusion = outcome.conclusion;
|
|
1777
|
+
step.sandbox = null;
|
|
1778
|
+
// The cluster's word on why a sandbox ended without the driver saying
|
|
1779
|
+
// so — OOMKilled, Evicted — on the record, where the retry reads it.
|
|
1780
|
+
if (outcome.status === "failed" && outcome.reason) push("error", `sandbox ended: ${outcome.reason}`);
|
|
1755
1781
|
if (outcome.data !== undefined) step.data = outcome.data;
|
|
1756
1782
|
step.costUsd = repriced(catalog, wireModel, outcome.usage ?? null, outcome.costUsd, push);
|
|
1757
1783
|
step.tokens = outcome.usage
|
|
@@ -2450,15 +2476,25 @@ function driveRunInner(
|
|
|
2450
2476
|
// never happened, which reconcile can then never see. Reset it to
|
|
2451
2477
|
// pending and it simply runs again; driveRun's whole contract is that
|
|
2452
2478
|
// re-driving a partly-finished record is safe.
|
|
2479
|
+
// Unless the step's sandbox outlived its driver AND the executor can
|
|
2480
|
+
// re-attach to it: then the record carries the pod and how far the
|
|
2481
|
+
// last driver got, runStep passes that as `resume`, and the step
|
|
2482
|
+
// carries on from where it was — a deploy no longer costs the run.
|
|
2453
2483
|
let orphaned = false;
|
|
2454
2484
|
for (const step of run.steps) {
|
|
2455
2485
|
if (step.status !== "running") continue;
|
|
2456
|
-
|
|
2486
|
+
const resumable = !!step.sandbox && platform.sandboxResumable(step.sandbox.kind);
|
|
2487
|
+
if (!resumable) {
|
|
2488
|
+
orphaned = true;
|
|
2489
|
+
step.sandbox = null;
|
|
2490
|
+
}
|
|
2457
2491
|
step.status = "pending";
|
|
2458
2492
|
step.events.push({
|
|
2459
2493
|
t: new Date().toISOString(),
|
|
2460
2494
|
type: "info",
|
|
2461
|
-
text:
|
|
2495
|
+
text: resumable
|
|
2496
|
+
? `driver changed mid-step — re-attaching to the running sandbox (${step.sandbox!.ref}, ${step.sandbox!.consumed} lines already applied)`
|
|
2497
|
+
: "interrupted mid-step (platform restart) — running again from the start of the step",
|
|
2462
2498
|
});
|
|
2463
2499
|
}
|
|
2464
2500
|
if (orphaned) {
|
|
@@ -2908,7 +2944,13 @@ function driveRunInner(
|
|
|
2908
2944
|
launching
|
|
2909
2945
|
.map(async (step) => {
|
|
2910
2946
|
const attempts = (step.retry ?? 0) + 1;
|
|
2911
|
-
|
|
2947
|
+
// A step re-attaching to its sandbox is still on the attempt
|
|
2948
|
+
// that started it; a step re-run after a backoff picks up the
|
|
2949
|
+
// count it left. Only a step nobody has touched starts at 1.
|
|
2950
|
+
const interruptedInBackoff =
|
|
2951
|
+
!step.sandbox && !!step.attempts && /retrying in \d+s/.test(step.events[step.events.length - 1]?.text ?? "");
|
|
2952
|
+
const from = step.sandbox ? Math.max(1, step.attempts ?? 1) : interruptedInBackoff ? Math.min(attempts, step.attempts! + 1) : 1;
|
|
2953
|
+
for (let attempt = from; attempt <= attempts; attempt++) {
|
|
2912
2954
|
step.attempts = attempt;
|
|
2913
2955
|
step.status = "running";
|
|
2914
2956
|
save();
|
|
@@ -2935,12 +2977,42 @@ function driveRunInner(
|
|
|
2935
2977
|
// more. save() merges the flag from the record.
|
|
2936
2978
|
save();
|
|
2937
2979
|
if (run.stopRequested) break;
|
|
2980
|
+
// The retry policy. An attempt the cluster ended for want of
|
|
2981
|
+
// memory or disk is retried one size class up — the same
|
|
2982
|
+
// size would end the same way. Every retry waits: 15 s, 30 s,
|
|
2983
|
+
// 1 m, 2 m, 4 m, jittered, so a provider or a site that was
|
|
2984
|
+
// struggling gets a moment, and twenty fan-out steps do not
|
|
2985
|
+
// hammer it in lock-step. The wait holds a worker slot, not
|
|
2986
|
+
// a sandbox — nothing is rented while it waits — and a stop
|
|
2987
|
+
// ends it. The step is `pending` meanwhile, so a driver that
|
|
2988
|
+
// inherits the run mid-wait simply runs the attempt.
|
|
2989
|
+
const last = [...step.events].reverse().find((e) => e.type === "error")?.text ?? "";
|
|
2990
|
+
let sizeNote = "";
|
|
2991
|
+
if (/OOMKilled|Evicted|ephemeral-storage|out of memory/i.test(last)) {
|
|
2992
|
+
const up = step.sizeUp === "large" || (!step.sizeUp && step.size === "large") ? "heavy" : step.sizeUp === "heavy" || step.size === "heavy" ? null : "large";
|
|
2993
|
+
if (up) {
|
|
2994
|
+
step.sizeUp = up;
|
|
2995
|
+
sizeNote = ` at size: ${up}`;
|
|
2996
|
+
}
|
|
2997
|
+
}
|
|
2998
|
+
const waitMs = Math.round(Math.min(retryBaseMs() * 16, retryBaseMs() * 2 ** (attempt - 1)) * (0.75 + Math.random() * 0.5));
|
|
2999
|
+
step.status = "pending";
|
|
2938
3000
|
step.events.push({
|
|
2939
3001
|
t: new Date().toISOString(),
|
|
2940
3002
|
type: "info",
|
|
2941
|
-
text: `attempt ${attempt} failed — retrying (${attempts - attempt} left)`,
|
|
3003
|
+
text: `attempt ${attempt} failed — retrying in ${Math.round(waitMs / 1000)}s${sizeNote} (${attempts - attempt} left)`,
|
|
2942
3004
|
});
|
|
2943
3005
|
save();
|
|
3006
|
+
const until = Date.now() + waitMs;
|
|
3007
|
+
while (Date.now() < until) {
|
|
3008
|
+
await sleep(Math.min(1000, until - Date.now()));
|
|
3009
|
+
save();
|
|
3010
|
+
if (run.stopRequested) break;
|
|
3011
|
+
}
|
|
3012
|
+
if (run.stopRequested) {
|
|
3013
|
+
step.status = "failed";
|
|
3014
|
+
break;
|
|
3015
|
+
}
|
|
2944
3016
|
}
|
|
2945
3017
|
}),
|
|
2946
3018
|
);
|
|
@@ -3270,6 +3342,10 @@ export function loadFlow(tenant: string, workspace: string, flowName: string) {
|
|
|
3270
3342
|
// written a beat before its driver registered, not a judgement about pace.
|
|
3271
3343
|
const ABANDONED_AFTER_MS = 2 * 60 * 1000;
|
|
3272
3344
|
|
|
3345
|
+
/** The first retry's wait; each one after doubles it, to sixteen times
|
|
3346
|
+
* this (15 s → 4 min by default). An env, so a test can make it small. */
|
|
3347
|
+
const retryBaseMs = () => Number(process.env.FOLDRUN_RETRY_BASE_MS) || 15_000;
|
|
3348
|
+
|
|
3273
3349
|
/** The most recent moment a run showed any sign of life. */
|
|
3274
3350
|
function lastActivity(run: RunRecord): number {
|
|
3275
3351
|
let latest = Date.parse(run.startedAt);
|
|
@@ -3311,12 +3387,18 @@ export interface Reconciliation {
|
|
|
3311
3387
|
* outlives any process. A run whose person already answered is not waiting,
|
|
3312
3388
|
* and is picked back up — see below.
|
|
3313
3389
|
*/
|
|
3314
|
-
export function reconcileRuns(tenant: string, now = Date.now()): Reconciliation[] {
|
|
3390
|
+
export function reconcileRuns(tenant: string, now = Date.now(), live?: ReadonlySet<string>): Reconciliation[] {
|
|
3315
3391
|
const closed: Reconciliation[] = [];
|
|
3316
3392
|
|
|
3317
3393
|
for (const workspace of listWorkspaces(tenant)) {
|
|
3318
3394
|
for (const summary of listRuns(tenant, workspace.name)) {
|
|
3319
3395
|
if (drivingRuns.has(summary.id)) continue;
|
|
3396
|
+
// Driven by another process — a second worker pod, on a platform with
|
|
3397
|
+
// more than one. The idle window protects a quiet step from THIS
|
|
3398
|
+
// process's judgement only; a worker that boots beside a live one
|
|
3399
|
+
// would otherwise close every long step its neighbour is driving.
|
|
3400
|
+
// The caller says which runs those are (a heartbeated claim).
|
|
3401
|
+
if (live?.has(summary.id)) continue;
|
|
3320
3402
|
if (now - lastActivity(summary) < ABANDONED_AFTER_MS) continue;
|
|
3321
3403
|
|
|
3322
3404
|
// Approved, then abandoned. Approval only writes to the run record; the
|
|
@@ -3396,8 +3478,8 @@ export function reconcileRuns(tenant: string, now = Date.now()): Reconciliation[
|
|
|
3396
3478
|
* one hard-coded account left every other account's interrupted runs marked
|
|
3397
3479
|
* `running` forever, with no process left to finish them.
|
|
3398
3480
|
*/
|
|
3399
|
-
export function reconcileAllRuns(now = Date.now()): Reconciliation[] {
|
|
3400
|
-
return listTenants().flatMap((tenant) => reconcileRuns(tenant, now));
|
|
3481
|
+
export function reconcileAllRuns(now = Date.now(), live?: ReadonlySet<string>): Reconciliation[] {
|
|
3482
|
+
return listTenants().flatMap((tenant) => reconcileRuns(tenant, now, live));
|
|
3401
3483
|
}
|
|
3402
3484
|
|
|
3403
3485
|
/**
|
package/src/starter.ts
CHANGED
|
@@ -319,6 +319,7 @@ timezone: Australia/Sydney
|
|
|
319
319
|
model: default # a default for every step here
|
|
320
320
|
effort: low
|
|
321
321
|
overlap: skip # skip | queue — what a second run does while one is live
|
|
322
|
+
priority: normal # high | normal | low — where its runs stand in the queue
|
|
322
323
|
---
|
|
323
324
|
|
|
324
325
|
1. [[researcher]] — Find three sources on {{topic}}.
|
package/src/step-exec.ts
CHANGED
|
@@ -79,20 +79,60 @@ export interface ExecOptions {
|
|
|
79
79
|
/** The pairing fields on a tool event — see RunEvent in store.ts. */
|
|
80
80
|
export type EventExtra = { call?: string; ms?: number; err?: boolean };
|
|
81
81
|
|
|
82
|
-
/** Conservative per-token rates for a model
|
|
83
|
-
*
|
|
82
|
+
/** Conservative per-token rates for a model nothing else can price — an
|
|
83
|
+
* unknown id on a gateway with no catalogue. Opus-class, so a ceiling
|
|
84
|
+
* still means something. Only ever reached when `knownPrice` below does
|
|
85
|
+
* not recognise the model either. */
|
|
84
86
|
export const FALLBACK_PRICE = { input: 15e-6, output: 75e-6 };
|
|
85
87
|
|
|
88
|
+
/** The rates we know without asking anyone: the tiers this runtime resolves
|
|
89
|
+
* (`fast`/`default`/`max` → haiku/sonnet/opus) and the model ids that carry
|
|
90
|
+
* those words. A gateway with a catalogue still wins — that is the
|
|
91
|
+
* authority — but the platform's own Anthropic key has no catalogue, and
|
|
92
|
+
* pricing its steps at Opus rates made every mid-turn ceiling read up to
|
|
93
|
+
* fifteen times high: an indexing run was stopped at "$1.17" having
|
|
94
|
+
* actually spent $0.15 (2026-09-08). Per token, input/output. */
|
|
95
|
+
const KNOWN_PRICES: [RegExp, { input: number; output: number }][] = [
|
|
96
|
+
[/haiku/i, { input: 1e-6, output: 5e-6 }],
|
|
97
|
+
[/sonnet/i, { input: 2e-6, output: 10e-6 }],
|
|
98
|
+
[/\bopus\b/i, { input: 5e-6, output: 25e-6 }],
|
|
99
|
+
];
|
|
100
|
+
|
|
101
|
+
/** What a token costs on this model, when we can say without a catalogue. */
|
|
102
|
+
export function knownPrice(model: string | undefined): { input: number; output: number } | null {
|
|
103
|
+
if (!model) return null;
|
|
104
|
+
return KNOWN_PRICES.find(([re]) => re.test(model))?.[1] ?? null;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Anthropic's cache multipliers, and the shape every gateway that bills
|
|
108
|
+
* for caching at all has copied: a write costs a quarter more than fresh
|
|
109
|
+
* input, a read a tenth of it. Counting both at the full input rate — as
|
|
110
|
+
* this did — makes a long cached prompt read about ten times its price,
|
|
111
|
+
* which on a ceiling is not "the honest direction to be wrong in", it is
|
|
112
|
+
* a step killed for spending money it never spent. */
|
|
113
|
+
const CACHE_WRITE_MULTIPLIER = 1.25;
|
|
114
|
+
const CACHE_READ_MULTIPLIER = 0.1;
|
|
115
|
+
|
|
86
116
|
/**
|
|
87
|
-
* What one assistant turn cost
|
|
88
|
-
*
|
|
89
|
-
*
|
|
117
|
+
* What one assistant turn cost, from the usage the SDK reports. Pure.
|
|
118
|
+
*
|
|
119
|
+
* A gateway whose cache is priced differently is over- or under-counted
|
|
120
|
+
* here by that difference alone; the run record's own cost is repriced
|
|
121
|
+
* from the catalogue afterwards and stays the authority on the bill. This
|
|
122
|
+
* number exists to stop a runaway step mid-turn, and wants to be close.
|
|
90
123
|
*/
|
|
91
124
|
export function priceTurn(u: Record<string, number | undefined>, price: { input: number; output: number } | null): number {
|
|
92
|
-
const inTok = (u.input_tokens ?? 0) + (u.cache_creation_input_tokens ?? 0) + (u.cache_read_input_tokens ?? 0);
|
|
93
|
-
const outTok = u.output_tokens ?? 0;
|
|
94
125
|
const p = price ?? FALLBACK_PRICE;
|
|
95
|
-
|
|
126
|
+
const fresh = u.input_tokens ?? 0;
|
|
127
|
+
const written = u.cache_creation_input_tokens ?? 0;
|
|
128
|
+
const read = u.cache_read_input_tokens ?? 0;
|
|
129
|
+
const outTok = u.output_tokens ?? 0;
|
|
130
|
+
return (
|
|
131
|
+
fresh * p.input +
|
|
132
|
+
written * p.input * CACHE_WRITE_MULTIPLIER +
|
|
133
|
+
read * p.input * CACHE_READ_MULTIPLIER +
|
|
134
|
+
outTok * p.output
|
|
135
|
+
);
|
|
96
136
|
}
|
|
97
137
|
|
|
98
138
|
/**
|
package/src/store.ts
CHANGED
|
@@ -24,6 +24,7 @@ import fs from "node:fs";
|
|
|
24
24
|
import path from "node:path";
|
|
25
25
|
import { recordRevision, registerTreeReader, type RevisionFile } from "./history.ts";
|
|
26
26
|
import { dataRoot, singleWorkspace } from "./paths.ts";
|
|
27
|
+
import { platform } from "./platform.ts";
|
|
27
28
|
import matter from "gray-matter";
|
|
28
29
|
|
|
29
30
|
/**
|
|
@@ -452,6 +453,13 @@ export function saveWorkspace(
|
|
|
452
453
|
// deploy writes straight to disk, so nothing else would generate them.
|
|
453
454
|
syncWorkspaceBundles(dir);
|
|
454
455
|
|
|
456
|
+
// Here, not in deployWorkspace: a workspace CREATED with files (the API's
|
|
457
|
+
// create, the CLI's first deploy, a template) comes through this function
|
|
458
|
+
// without going through deployWorkspace, and the first live check found
|
|
459
|
+
// exactly that — a new desk's cron was seen on the five-minute rescan, not
|
|
460
|
+
// the next tick.
|
|
461
|
+
workspaceChanged(tenant, workspace, "deploy");
|
|
462
|
+
|
|
455
463
|
return { preserved: preserved.length };
|
|
456
464
|
}
|
|
457
465
|
|
|
@@ -1083,6 +1091,12 @@ export interface FlowInfo {
|
|
|
1083
1091
|
* overlap) — a default that changed underfoot would surprise every flow
|
|
1084
1092
|
* that legitimately runs in parallel. */
|
|
1085
1093
|
overlap: "skip" | "queue" | null;
|
|
1094
|
+
/** Where this flow's runs stand in the queue against everyone else's:
|
|
1095
|
+
* "high" is claimed before the round-robin, "low" after it. Adds to the
|
|
1096
|
+
* account's own priority; the account's cap still bounds how many run
|
|
1097
|
+
* at once, so a high lane never starves the rest outright. Unset is
|
|
1098
|
+
* normal — first come, fair across accounts. */
|
|
1099
|
+
priority: "high" | "normal" | "low" | null;
|
|
1086
1100
|
/** The most one run of this flow may spend, in USD. A literal, never an
|
|
1087
1101
|
* expression — the cap is readable off the file, which is the property
|
|
1088
1102
|
* the whole grammar exists to keep. Checked between groups: the group
|
|
@@ -1313,6 +1327,7 @@ export function parseFlow(file: string, raw: string): FlowInfo {
|
|
|
1313
1327
|
model: data.model ?? null,
|
|
1314
1328
|
effort: data.effort ?? null,
|
|
1315
1329
|
overlap: data.overlap === "skip" || data.overlap === "queue" ? data.overlap : null,
|
|
1330
|
+
priority: data.priority === "high" || data.priority === "low" || data.priority === "normal" ? data.priority : null,
|
|
1316
1331
|
budget: Number(data.budget) > 0 ? Number(data.budget) : null,
|
|
1317
1332
|
// `after: [[flow:publish]]` — a link or a bare name, read the same way
|
|
1318
1333
|
// as every other file-naming field (refs.ts).
|
|
@@ -1895,7 +1910,18 @@ export function writeWorkspaceFile(
|
|
|
1895
1910
|
// The revision: what it was, what it is now. Identical content records
|
|
1896
1911
|
// nothing — see history.ts.
|
|
1897
1912
|
recordRevision(tenant, workspace, [{ path: norm, before, after: fs.readFileSync(p, "utf8") }], meta);
|
|
1913
|
+
workspaceChanged(tenant, workspace, "write");
|
|
1914
|
+
}
|
|
1915
|
+
|
|
1916
|
+
/** Tell the platform, and never let its listener fail a write. */
|
|
1917
|
+
function workspaceChanged(tenant: string, workspace: string, why: "write" | "deploy" | "push" | "delete") {
|
|
1918
|
+
try {
|
|
1919
|
+
platform.workspaceChanged(tenant, workspace, why);
|
|
1920
|
+
} catch {
|
|
1921
|
+
// a scheduler that could not be told will find out on its next rescan
|
|
1922
|
+
}
|
|
1898
1923
|
}
|
|
1924
|
+
export { workspaceChanged as notifyWorkspaceChanged };
|
|
1899
1925
|
|
|
1900
1926
|
// memory/ and knowledge/ are OKF bundles, so index.md has to stay current —
|
|
1901
1927
|
// a generated index that drifts is the same class of bug as a memory index
|
|
@@ -1990,6 +2016,7 @@ export function renameWorkspaceFile(tenant: string, workspace: string, from: str
|
|
|
1990
2016
|
);
|
|
1991
2017
|
syncBundleFor(src);
|
|
1992
2018
|
syncBundleFor(dst, "Update");
|
|
2019
|
+
workspaceChanged(tenant, workspace, "write");
|
|
1993
2020
|
}
|
|
1994
2021
|
|
|
1995
2022
|
/** Every text file under a path, as revision entries recording deletion. */
|
|
@@ -2051,12 +2078,14 @@ export function deleteWorkspacePath(tenant: string, workspace: string, rel: stri
|
|
|
2051
2078
|
recordRevision(tenant, workspace, gone, meta);
|
|
2052
2079
|
// A bundle's index must stop naming what is gone.
|
|
2053
2080
|
syncBundleFor(p);
|
|
2081
|
+
workspaceChanged(tenant, workspace, "write");
|
|
2054
2082
|
}
|
|
2055
2083
|
|
|
2056
2084
|
export function deleteWorkspace(tenant: string, workspace: string) {
|
|
2057
2085
|
const dir = workspaceDir(tenant, workspace);
|
|
2058
2086
|
if (!fs.existsSync(dir)) throw new Error(`workspace ${workspace} not found`);
|
|
2059
2087
|
fs.rmSync(dir, { recursive: true, force: true });
|
|
2088
|
+
workspaceChanged(tenant, workspace, "delete");
|
|
2060
2089
|
}
|
|
2061
2090
|
|
|
2062
2091
|
/**
|
|
@@ -2235,6 +2264,16 @@ export interface StepRecord {
|
|
|
2235
2264
|
verify?: string;
|
|
2236
2265
|
/** Attempts made so far, for the run trace. */
|
|
2237
2266
|
attempts?: number;
|
|
2267
|
+
/** The sandbox this step is executing in, while it is: which executor,
|
|
2268
|
+
* its handle (a pod name), and how many of its output lines the driver
|
|
2269
|
+
* has already applied to this record. What a driver that inherits the
|
|
2270
|
+
* run — after a deploy rolled the last one, or it crashed — needs to
|
|
2271
|
+
* re-attach and carry on rather than run the step again. Cleared when
|
|
2272
|
+
* the step settles. */
|
|
2273
|
+
sandbox?: { kind: string; ref: string; consumed: number; since: string } | null;
|
|
2274
|
+
/** A size class a retry moved the step up to (an evicted or OOM-killed
|
|
2275
|
+
* attempt), overriding the agent's own `size:` for the attempts after. */
|
|
2276
|
+
sizeUp?: "large" | "heavy";
|
|
2238
2277
|
/** When the step actually started and finished executing — not when the
|
|
2239
2278
|
* run reached it. A step's events approximate this (first and last), but
|
|
2240
2279
|
* a model can think for minutes before its first tool call, and a step
|