@foldrun/core 2.13.1 → 2.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/frontmatter-fields.js +2 -1
- package/dist/src/frontmatter-schema.js +1 -0
- package/dist/src/platform.d.ts +19 -0
- package/dist/src/resources.d.ts +8 -4
- package/dist/src/resources.js +10 -3
- package/dist/src/run-container.d.ts +17 -0
- package/dist/src/runner.js +143 -9
- package/dist/src/starter.js +1 -0
- package/dist/src/store.d.ts +37 -0
- package/dist/src/store.js +8 -2
- package/package.json +1 -1
- package/src/frontmatter-fields.ts +2 -1
- package/src/frontmatter-schema.ts +1 -0
- package/src/platform.ts +11 -0
- package/src/resources.ts +15 -6
- package/src/run-container.ts +17 -0
- package/src/runner.ts +135 -7
- package/src/starter.ts +1 -0
- package/src/store.ts +31 -4
|
@@ -85,7 +85,7 @@ const AGENTS_MD = [
|
|
|
85
85
|
{ key: "budget", type: ["number", "string"], help: "USD per month, or N/day | N/week | N/month" },
|
|
86
86
|
{ key: "priority", type: "string", values: ["high", "normal", "low"], help: "account: where its runs stand in the queue" },
|
|
87
87
|
{ key: "concurrency", type: ["number", "string"], help: "account: runs at once" },
|
|
88
|
-
{ key: "compute", type: "object", help: "
|
|
88
|
+
{ key: "compute", type: "object", help: "{vcpu, memory, disk} the running steps here may hold together — the account's pool, or this workspace's; only narrows the level above" },
|
|
89
89
|
{ key: "alerts", type: "object", validated: true, help: "thresholds that tell notify: — failure_rate (30%), min_runs, credits_day, run_minutes, stale_days, cost_spike, runs_per_hour, new_host" },
|
|
90
90
|
{ key: "capture", type: "string", values: ["full", "metadata"], help: "what runs record of each tool call: full (its input and output, secrets masked) or metadata (name, size and time only)" },
|
|
91
91
|
{ key: "network_enforce", type: "boolean", help: "true: a site off a step's network: list is refused, not only named in the trace. The account's or the workspace's true is enough" },
|
|
@@ -125,6 +125,7 @@ const FLOW = [
|
|
|
125
125
|
{ key: "pause_when", type: "string", help: "a marker that pauses the flow when a run's result says it" },
|
|
126
126
|
{ key: "approvers", type: ["list", "string"], help: "who may approve its gates" },
|
|
127
127
|
{ key: "budget", type: ["number", "string"], help: "USD per run" },
|
|
128
|
+
{ key: "compute", type: "object", help: "{vcpu, memory, disk} one run's steps may hold together at once — only narrows the workspace's and account's" },
|
|
128
129
|
{ key: "live", type: "boolean", help: "true: steps may act for real in a test run" },
|
|
129
130
|
];
|
|
130
131
|
const TOOL = [
|
|
@@ -53,6 +53,7 @@ const CHECKS = {
|
|
|
53
53
|
disable_after: (v) => (Number(v) > 0 && Number(v) <= 100 ? null : `${String(v)} is not a count from 1 to 100; it is ignored`),
|
|
54
54
|
pause_when: (v) => (String(v).length <= 60 ? null : "is longer than 60 characters"),
|
|
55
55
|
budget: (v) => budgetProblem(v, ["run"]),
|
|
56
|
+
compute: computeProblem,
|
|
56
57
|
},
|
|
57
58
|
"tool": {
|
|
58
59
|
transport: (v) => (["http", "script", "mcp"].includes(String(v).toLowerCase()) ? null : `"${String(v)}" is not a transport — http, script or mcp`),
|
package/dist/src/platform.d.ts
CHANGED
|
@@ -99,6 +99,25 @@ export interface PlatformHooks {
|
|
|
99
99
|
kind: string;
|
|
100
100
|
ref: string;
|
|
101
101
|
}): void;
|
|
102
|
+
/** A restarted driver's sweep of the run's orphans: destroy its
|
|
103
|
+
* sandboxes except `keep` (the refs its steps re-attach to). Not a stop —
|
|
104
|
+
* the run goes on, and steps of it are about to start. Optional: without
|
|
105
|
+
* it the whole run's are killed (killRunSandboxes). */
|
|
106
|
+
killOrphanSandboxes?(runId: string, keep: readonly string[]): void;
|
|
107
|
+
/** Is there room for a step waiting in the queue to start now — by the
|
|
108
|
+
* same rules as the executor's room gate — and if not, why. Asked before
|
|
109
|
+
* such a step is started again, so a run claimed only to find no room
|
|
110
|
+
* yet adds nothing to its trail. Optional: without it the step goes to
|
|
111
|
+
* its executor, which looks there. */
|
|
112
|
+
roomCheck?(ask: {
|
|
113
|
+
tenant: string;
|
|
114
|
+
workspace: string;
|
|
115
|
+
runId: string;
|
|
116
|
+
since: string;
|
|
117
|
+
cpus: number;
|
|
118
|
+
memGiB: number;
|
|
119
|
+
diskGiB: number;
|
|
120
|
+
}): Promise<string | null>;
|
|
102
121
|
/** Publish storage/public/ as share links after a run. Default: no links. */
|
|
103
122
|
syncPublicShares(tenant: string, workspace: string): Promise<{
|
|
104
123
|
added: string[];
|
package/dist/src/resources.d.ts
CHANGED
|
@@ -42,15 +42,19 @@ export declare function readResources(raw: unknown): {
|
|
|
42
42
|
export declare function resourcesProblem(v: unknown): string | null;
|
|
43
43
|
/** Whether an override changes anything. */
|
|
44
44
|
export declare function hasOverride(o: ResourceOverride | null | undefined): o is ResourceOverride;
|
|
45
|
-
/**
|
|
46
|
-
*
|
|
47
|
-
*
|
|
45
|
+
/** `compute:` — the most the steps running inside one scope may hold
|
|
46
|
+
* together, in cores, memory and disk (27 Oct 2026: disk, and the scopes
|
|
47
|
+
* past the account). Written in the account's AGENTS.md (its pool), a
|
|
48
|
+
* workspace's AGENTS.md (that workspace's pool) and a flow (each run's
|
|
49
|
+
* pool). Each level only narrows the one above it — the plan or install
|
|
50
|
+
* caps the account (platform limits.ts). Here it is only read:
|
|
48
51
|
*
|
|
49
|
-
* compute: { vcpu: 4, memory: 8Gi }
|
|
52
|
+
* compute: { vcpu: 4, memory: 8Gi, disk: 40Gi }
|
|
50
53
|
*/
|
|
51
54
|
export interface ComputeAsk {
|
|
52
55
|
vcpu?: number;
|
|
53
56
|
memGiB?: number;
|
|
57
|
+
diskGiB?: number;
|
|
54
58
|
}
|
|
55
59
|
export declare function readCompute(raw: unknown): {
|
|
56
60
|
value: ComputeAsk;
|
package/dist/src/resources.js
CHANGED
|
@@ -112,12 +112,12 @@ export function readCompute(raw) {
|
|
|
112
112
|
if (raw === undefined || raw === null)
|
|
113
113
|
return { value, problems };
|
|
114
114
|
if (typeof raw !== "object" || Array.isArray(raw)) {
|
|
115
|
-
return { value, problems: [`is ${shown(raw)}, not a map — write {vcpu: 4, memory: 8Gi}`] };
|
|
115
|
+
return { value, problems: [`is ${shown(raw)}, not a map — write {vcpu: 4, memory: 8Gi, disk: 40Gi}`] };
|
|
116
116
|
}
|
|
117
117
|
const r = raw;
|
|
118
118
|
for (const k of Object.keys(r)) {
|
|
119
|
-
if (k !== "vcpu" && k !== "memory")
|
|
120
|
-
problems.push(`\`${k}\` is not vcpu or
|
|
119
|
+
if (k !== "vcpu" && k !== "memory" && k !== "disk")
|
|
120
|
+
problems.push(`\`${k}\` is not vcpu, memory or disk — it is ignored`);
|
|
121
121
|
}
|
|
122
122
|
if (r.vcpu !== undefined && r.vcpu !== null) {
|
|
123
123
|
const numeric = typeof r.vcpu === "number" || (typeof r.vcpu === "string" && /^\s*\d+(\.\d+)?\s*$/.test(r.vcpu));
|
|
@@ -134,6 +134,13 @@ export function readCompute(raw) {
|
|
|
134
134
|
else
|
|
135
135
|
value.memGiB = mi / 1024;
|
|
136
136
|
}
|
|
137
|
+
if (r.disk !== undefined && r.disk !== null) {
|
|
138
|
+
const mi = quantityMi(r.disk);
|
|
139
|
+
if (!Number.isFinite(mi) || mi < 1024)
|
|
140
|
+
problems.push(`disk: ${shown(r.disk)} is not a size of 1Gi or more — write 40Gi`);
|
|
141
|
+
else
|
|
142
|
+
value.diskGiB = mi / 1024;
|
|
143
|
+
}
|
|
137
144
|
return { value, problems };
|
|
138
145
|
}
|
|
139
146
|
export function computeProblem(v) {
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { RoomLevel } from "./store.ts";
|
|
1
2
|
import type { PeerInfo } from "./operator.ts";
|
|
2
3
|
import { type HookRecord } from "./hooks.ts";
|
|
3
4
|
import type { McpServerConfig } from "@anthropic-ai/claude-agent-sdk";
|
|
@@ -152,6 +153,10 @@ export interface StepActuals {
|
|
|
152
153
|
}
|
|
153
154
|
export interface ContainerStepOutcome {
|
|
154
155
|
status: "completed" | "failed";
|
|
156
|
+
/** No room to start, and the step may wait in the queue rather than here
|
|
157
|
+
* (`room.mayDefer`): why. Nothing ran and nothing was rented — not a
|
|
158
|
+
* failure, not an attempt. `status` is not read when this is set. */
|
|
159
|
+
deferred?: string;
|
|
155
160
|
result: string | null;
|
|
156
161
|
/** Why the sandbox ended when the step did not: what the cluster said
|
|
157
162
|
* (OOMKilled, Evicted, the pod gone). Absent when the driver finished
|
|
@@ -475,6 +480,18 @@ export interface RunInContainerArgs {
|
|
|
475
480
|
* (consumed 0), and before each output line it applies. The runner
|
|
476
481
|
* records these on the step, so the next driver knows where to attach. */
|
|
477
482
|
checkpoint?: (ref: string, consumed: number) => void;
|
|
483
|
+
/** The wait for room (27 Oct 2026), for an executor with a room gate.
|
|
484
|
+
* `since`: when this step first found none, carried across re-drives —
|
|
485
|
+
* its place in line. `waiting` is called when it finds none (and again
|
|
486
|
+
* when the reason changes); `mayDefer` says whether, instead of waiting
|
|
487
|
+
* on, it may answer `deferred` so its run goes back to the queue. */
|
|
488
|
+
room?: {
|
|
489
|
+
since: string | null;
|
|
490
|
+
/** `${runId}#${step}` — breaks a tie between two equal ages. */
|
|
491
|
+
key?: string;
|
|
492
|
+
waiting: (why: string, level?: RoomLevel) => void;
|
|
493
|
+
mayDefer: () => boolean;
|
|
494
|
+
};
|
|
478
495
|
/** A test run: where state/ and storage/ changes go instead of the
|
|
479
496
|
* workspace, and who to tell about each. See test-mode.ts. */
|
|
480
497
|
divert?: WriteDivert;
|
package/dist/src/runner.js
CHANGED
|
@@ -16,7 +16,7 @@ import { cascadeLimits, toolOwners } from "./limits.js";
|
|
|
16
16
|
import { resolveAgentPath } from "./confine.js";
|
|
17
17
|
import { browserPodLine, podLossDecision, RERAN_ON_FULL } from "./browser-pod.js";
|
|
18
18
|
import { eventUrl } from "./webhook.js";
|
|
19
|
-
import { runStepInContainer, resolveResources, memoryGiB, heldCores, killRunSandboxes } from "./run-container.js";
|
|
19
|
+
import { runStepInContainer, resolveResources, memoryGiB, heldCores, killRunSandboxes, classDisk } from "./run-container.js";
|
|
20
20
|
import { readResources, hasOverride } from "./resources.js";
|
|
21
21
|
import { hostSafeEnv, platformApiUrl } from "./host-env.js";
|
|
22
22
|
import { resolveModelCredential, credentialLine, accountModelBlock, isClaudeLoginToken, isModelEnvName, DECLARED_ENV_PREFIX } from "./model-credential.js";
|
|
@@ -1909,7 +1909,12 @@ peerBus,
|
|
|
1909
1909
|
/** Runs one job for a `sandbox: own` sub-agent as a step of its own and
|
|
1910
1910
|
* answers with its reply (the drive loop's delegateFor). Absent for a
|
|
1911
1911
|
* step that is itself a sub-agent's: depth one. */
|
|
1912
|
-
delegateJob
|
|
1912
|
+
delegateJob,
|
|
1913
|
+
/** Whether this step, finding no room to start, may go back to the queue
|
|
1914
|
+
* with its run instead of waiting in this driver (the drive loop's
|
|
1915
|
+
* mayDefer). Absent: it waits here, however long — a rescue, a step
|
|
1916
|
+
* outside a flow, a foreground run. */
|
|
1917
|
+
mayDefer) {
|
|
1913
1918
|
// Secret values are injected into scripts as environment variables and
|
|
1914
1919
|
// substituted into API headers, so a model that reads one back — from a
|
|
1915
1920
|
// script that prints it, an API error that echoes it, or its own `env` —
|
|
@@ -2129,7 +2134,7 @@ delegateJob) {
|
|
|
2129
2134
|
const limits = resolveResources(size, asked.value, { atLeastClass: !!step.sizeUp });
|
|
2130
2135
|
// What the sandbox holds while it exists — `step.holding`, what the
|
|
2131
2136
|
// account's compute in use sums (Settings, `foldrun usage compute`).
|
|
2132
|
-
const held = { cpus: heldCores(limits.cpus, process.env.FOLDRUN_RUN_ISOLATION), memGiB: memoryGiB(limits.memory) };
|
|
2137
|
+
const held = { cpus: heldCores(limits.cpus, process.env.FOLDRUN_RUN_ISOLATION), memGiB: memoryGiB(limits.memory), diskGiB: memoryGiB(limits.disk ?? classDisk(size)) };
|
|
2133
2138
|
// oauth2 secrets become live access tokens here — the one async moment
|
|
2134
2139
|
// in secret resolution, at the last host-side point before anything is
|
|
2135
2140
|
// injected or substituted. The refresh recipes never leave this process:
|
|
@@ -2807,6 +2812,8 @@ delegateJob) {
|
|
|
2807
2812
|
};
|
|
2808
2813
|
const primaryTranslator = proxiedTranslator(translator, MODEL_KEY_NAME);
|
|
2809
2814
|
const secondTranslator = proxiedTranslator(fallbackTranslator, `${MODEL_KEY_NAME}_FALLBACK`);
|
|
2815
|
+
// Set once the first sandbox call has run (see room.mayDefer below).
|
|
2816
|
+
let executed = false;
|
|
2810
2817
|
const runIsolated = isolation === "container" ? runStepInContainer : platform.isolation[isolation];
|
|
2811
2818
|
const isolatedArgs = (modelEnv, keyName = MODEL_KEY_NAME) => ({
|
|
2812
2819
|
workspaceRoot,
|
|
@@ -2921,7 +2928,23 @@ delegateJob) {
|
|
|
2921
2928
|
if (consumed === 0)
|
|
2922
2929
|
step.holding = held;
|
|
2923
2930
|
if (consumed === 0)
|
|
2931
|
+
step.roomWait = null;
|
|
2932
|
+
if (consumed === 0)
|
|
2933
|
+
save();
|
|
2934
|
+
},
|
|
2935
|
+
// The wait for room: its age is the step's place in line, so it is
|
|
2936
|
+
// on the record from the first "no" and survives a re-drive.
|
|
2937
|
+
room: {
|
|
2938
|
+
since: step.roomWait?.since ?? null,
|
|
2939
|
+
...(runId && stepIndex !== undefined ? { key: `${runId}#${stepIndex}` } : {}),
|
|
2940
|
+
waiting: (why, level) => {
|
|
2941
|
+
step.roomWait = { since: step.roomWait?.since ?? new Date().toISOString(), why, cpus: held.cpus, memGiB: held.memGiB, diskGiB: held.diskGiB, ...(level ? { level } : {}) };
|
|
2924
2942
|
save();
|
|
2943
|
+
},
|
|
2944
|
+
// Only before anything ran: a later call of this attempt (a
|
|
2945
|
+
// rotated credential, a busy provider, a re-run on the full
|
|
2946
|
+
// image) waits for room where it is — its outcome is the step's.
|
|
2947
|
+
mayDefer: () => !executed && (mayDefer?.() ?? false),
|
|
2925
2948
|
},
|
|
2926
2949
|
});
|
|
2927
2950
|
// A busy provider gets another moment on the SAME supply before the
|
|
@@ -2942,21 +2965,46 @@ delegateJob) {
|
|
|
2942
2965
|
// its first checkpoint, after any wait for room.
|
|
2943
2966
|
if (!platform.sandboxResumable(isolation)) {
|
|
2944
2967
|
step.holding = held;
|
|
2968
|
+
// Started: no longer waiting for room (an executor that can
|
|
2969
|
+
// report its sandbox clears this at its first checkpoint).
|
|
2970
|
+
step.roomWait = null;
|
|
2945
2971
|
save();
|
|
2946
2972
|
}
|
|
2973
|
+
let out;
|
|
2947
2974
|
try {
|
|
2948
|
-
|
|
2975
|
+
out = await runIsolated(a);
|
|
2976
|
+
return out;
|
|
2949
2977
|
}
|
|
2950
2978
|
finally {
|
|
2951
2979
|
// Saved now: the clean-up after an attempt (the log drain, the
|
|
2952
2980
|
// lease, publishing) can take a while, and the sandbox is gone.
|
|
2953
2981
|
step.holding = null;
|
|
2982
|
+
// A call that ran, or failed, is past waiting; only a step sent
|
|
2983
|
+
// back to the queue keeps its place in line.
|
|
2984
|
+
if (!out?.deferred)
|
|
2985
|
+
step.roomWait = null;
|
|
2954
2986
|
save();
|
|
2955
2987
|
}
|
|
2956
2988
|
};
|
|
2957
2989
|
let attemptFrom = step.events.length;
|
|
2958
2990
|
let outcome = await runTracked(firstArgs);
|
|
2991
|
+
if (!outcome.deferred)
|
|
2992
|
+
executed = true;
|
|
2959
2993
|
step.sandbox = null; // whatever happens next starts its own
|
|
2994
|
+
// No room, and the run may wait for it in the queue: nothing ran and
|
|
2995
|
+
// nothing was rented, so this is neither a failure nor a try. The
|
|
2996
|
+
// step goes back to pending with its place in line (roomWait).
|
|
2997
|
+
if (outcome.deferred) {
|
|
2998
|
+
if (lease) {
|
|
2999
|
+
await serving?.close();
|
|
3000
|
+
serving = null;
|
|
3001
|
+
await lease.release();
|
|
3002
|
+
}
|
|
3003
|
+
step.status = "pending";
|
|
3004
|
+
push("info", `room: none yet — ${outcome.deferred}; the run waits in the queue and starts this step when there is`);
|
|
3005
|
+
save();
|
|
3006
|
+
return { actedThenFailedCheck: false, deferred: true };
|
|
3007
|
+
}
|
|
2960
3008
|
// A 401 from a token that rotated mid-step is not a broken key: the
|
|
2961
3009
|
// replacement is already on its way to the mounted file. Wait for it
|
|
2962
3010
|
// once, then fall into the same retry the busy case uses.
|
|
@@ -3168,6 +3216,7 @@ delegateJob) {
|
|
|
3168
3216
|
step.size = size;
|
|
3169
3217
|
if (hasOverride(asked.value))
|
|
3170
3218
|
step.resources = asked.value;
|
|
3219
|
+
// Cores, GiB and disk (27 Oct 2026): all three are billed as held.
|
|
3171
3220
|
step.reserved = held;
|
|
3172
3221
|
}
|
|
3173
3222
|
else {
|
|
@@ -4255,6 +4304,10 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
|
|
|
4255
4304
|
if (entry && entry.status === "running")
|
|
4256
4305
|
entry.status = "failed";
|
|
4257
4306
|
}
|
|
4307
|
+
// The sandboxes the steps below re-attach to — the orphan sweep must
|
|
4308
|
+
// leave them standing (27 Oct 2026: it deleted every pod of the run,
|
|
4309
|
+
// and each sibling that was about to re-attach found its pod gone).
|
|
4310
|
+
const reattaching = [];
|
|
4258
4311
|
for (const step of run.steps) {
|
|
4259
4312
|
if (step.status !== "running")
|
|
4260
4313
|
continue;
|
|
@@ -4263,6 +4316,9 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
|
|
|
4263
4316
|
orphaned = true;
|
|
4264
4317
|
step.sandbox = null;
|
|
4265
4318
|
}
|
|
4319
|
+
else {
|
|
4320
|
+
reattaching.push(step.sandbox.ref);
|
|
4321
|
+
}
|
|
4266
4322
|
step.status = "pending";
|
|
4267
4323
|
step.events.push({
|
|
4268
4324
|
t: new Date().toISOString(),
|
|
@@ -4277,10 +4333,14 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
|
|
|
4277
4333
|
// `timeout:` carries no pod deadline at all — so the orphaned step
|
|
4278
4334
|
// may still be executing, non-idempotent work included, while the
|
|
4279
4335
|
// new attempt starts. Destroy it first; re-driving is only safe
|
|
4280
|
-
// once nothing else is driving.
|
|
4336
|
+
// once nothing else is driving. Not a stop: the run goes on, so the
|
|
4337
|
+
// platform's sweep keeps the re-attaching sandboxes and leaves no
|
|
4338
|
+
// stop word for the steps about to start.
|
|
4281
4339
|
try {
|
|
4282
4340
|
if (process.env.FOLDRUN_RUN_ISOLATION === "container")
|
|
4283
4341
|
killRunSandboxes(run.id);
|
|
4342
|
+
else if (platform.killOrphanSandboxes)
|
|
4343
|
+
platform.killOrphanSandboxes(run.id, reattaching);
|
|
4284
4344
|
else
|
|
4285
4345
|
platform.killRunSandboxes(run.id);
|
|
4286
4346
|
}
|
|
@@ -4447,6 +4507,12 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
|
|
|
4447
4507
|
run.budgetUsd = readFlow(tenant, workspace, run.flow)?.budget ?? null;
|
|
4448
4508
|
save();
|
|
4449
4509
|
}
|
|
4510
|
+
// compute: — the flow's run pool, stamped the same way: the cap a run
|
|
4511
|
+
// is held to is on its record (27 Oct 2026).
|
|
4512
|
+
if (run.compute === undefined) {
|
|
4513
|
+
run.compute = readFlow(tenant, workspace, run.flow)?.compute ?? null;
|
|
4514
|
+
save();
|
|
4515
|
+
}
|
|
4450
4516
|
let gi = 0;
|
|
4451
4517
|
let context = null;
|
|
4452
4518
|
let ctxData = undefined;
|
|
@@ -4968,8 +5034,43 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
|
|
|
4968
5034
|
// prompt cache it wrote — see warmGate. The claim is the callback's
|
|
4969
5035
|
// first statement, before any await; the wait comes before the
|
|
4970
5036
|
// pool, so nothing holds a slot while it waits.
|
|
4971
|
-
|
|
4972
|
-
|
|
5037
|
+
// Room (27 Oct 2026): a step that finds no room may go back to the
|
|
5038
|
+
// queue with its run — freeing the worker — only when nothing else
|
|
5039
|
+
// of its group is under way (a sibling still running keeps the run
|
|
5040
|
+
// here anyway), and only once the group has waited a while, so a
|
|
5041
|
+
// short wait stays a short wait. Never on a foreground run.
|
|
5042
|
+
const mayDefer = (step) => () => {
|
|
5043
|
+
if (!opts.parkOnApproval)
|
|
5044
|
+
return false;
|
|
5045
|
+
if (launching.some((s) => s !== step && s.status === "running" && !s.roomWait))
|
|
5046
|
+
return false;
|
|
5047
|
+
const since = Math.min(...launching.map((s) => (s.roomWait ? Date.parse(s.roomWait.since) : Infinity)));
|
|
5048
|
+
return Number.isFinite(since) && Date.now() - since >= roomParkAfterMs();
|
|
5049
|
+
};
|
|
5050
|
+
// Steps waiting in the queue for room, the run claimed again: look
|
|
5051
|
+
// first, quietly, before anything (the prompt-cache gate's line
|
|
5052
|
+
// included) writes to their trails. Still none — they stay pending
|
|
5053
|
+
// with their place, the run parks again below, and the trail gains
|
|
5054
|
+
// nothing on each look.
|
|
5055
|
+
const roomStill = new Set();
|
|
5056
|
+
if (platform.roomCheck) {
|
|
5057
|
+
for (const step of launching) {
|
|
5058
|
+
if (!step.roomWait || !mayDefer(step)())
|
|
5059
|
+
continue;
|
|
5060
|
+
const w = step.roomWait;
|
|
5061
|
+
const why = await platform.roomCheck({ tenant, workspace, runId: run.id, since: w.since, cpus: w.cpus, memGiB: w.memGiB, diskGiB: w.diskGiB ?? 0 }).catch(() => null);
|
|
5062
|
+
if (!why)
|
|
5063
|
+
continue;
|
|
5064
|
+
roomStill.add(step);
|
|
5065
|
+
if (why !== w.why) {
|
|
5066
|
+
step.roomWait = { ...w, why };
|
|
5067
|
+
save();
|
|
5068
|
+
}
|
|
5069
|
+
}
|
|
5070
|
+
}
|
|
5071
|
+
const starting = launching.filter((s) => !roomStill.has(s));
|
|
5072
|
+
const gate = warmGate(starting);
|
|
5073
|
+
await Promise.all(starting
|
|
4973
5074
|
.map(async (step) => {
|
|
4974
5075
|
const lead = gate.claim(step);
|
|
4975
5076
|
if (lead) {
|
|
@@ -5015,7 +5116,14 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
|
|
|
5015
5116
|
// that started it; a step re-run after a backoff picks up the
|
|
5016
5117
|
// count it left. Only a step nobody has touched starts at 1.
|
|
5017
5118
|
const interruptedInBackoff = !step.sandbox && !!step.attempts && /retrying in \d+s/.test(step.events[step.events.length - 1]?.text ?? "");
|
|
5018
|
-
|
|
5119
|
+
// A step that went back to the queue for room resumes the attempt it was on.
|
|
5120
|
+
const from = step.sandbox
|
|
5121
|
+
? Math.max(1, step.attempts ?? 1)
|
|
5122
|
+
: step.resumeAttempt
|
|
5123
|
+
? Math.min(attempts, step.resumeAttempt)
|
|
5124
|
+
: interruptedInBackoff
|
|
5125
|
+
? Math.min(attempts, step.attempts + 1)
|
|
5126
|
+
: 1;
|
|
5019
5127
|
for (let attempt = from; attempt <= attempts; attempt++) {
|
|
5020
5128
|
const ceiling = ceilingFor(step);
|
|
5021
5129
|
// Nothing left is a refusal, not "no ceiling": a zero passed
|
|
@@ -5042,6 +5150,7 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
|
|
|
5042
5150
|
const attemptStartedAt = new Date().toISOString();
|
|
5043
5151
|
const eventsBefore = step.events.length;
|
|
5044
5152
|
step.attempts = attempt;
|
|
5153
|
+
step.resumeAttempt = null;
|
|
5045
5154
|
step.status = "running";
|
|
5046
5155
|
if (ceiling.ceilingUsd !== null)
|
|
5047
5156
|
reserved.set(step, ceiling.ceilingUsd);
|
|
@@ -5052,7 +5161,14 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
|
|
|
5052
5161
|
() => {
|
|
5053
5162
|
save();
|
|
5054
5163
|
return run.stopRequested === true;
|
|
5055
|
-
}, flowTimezone, run.steps.indexOf(step), siblingsOf(step), peerBus, delegateFor(step)).finally(() => reserved.delete(step));
|
|
5164
|
+
}, flowTimezone, run.steps.indexOf(step), siblingsOf(step), peerBus, delegateFor(step), mayDefer(step)).finally(() => reserved.delete(step));
|
|
5165
|
+
// Sent back for room: not a try. The run parks once the
|
|
5166
|
+
// group is done (below) and this attempt starts again later.
|
|
5167
|
+
if (verdict?.deferred) {
|
|
5168
|
+
step.resumeAttempt = attempt;
|
|
5169
|
+
save();
|
|
5170
|
+
return;
|
|
5171
|
+
}
|
|
5056
5172
|
// runStep mutates step.status; read it through a widened local
|
|
5057
5173
|
// so TS doesn't keep the "running" narrowing from above.
|
|
5058
5174
|
const outcome = step.status;
|
|
@@ -5149,6 +5265,17 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
|
|
|
5149
5265
|
}
|
|
5150
5266
|
}
|
|
5151
5267
|
}
|
|
5268
|
+
// Steps sent back for want of room: the run waits in the queue — its
|
|
5269
|
+
// row keeps its place — and comes back to this group when it is
|
|
5270
|
+
// claimed again. Nothing after the group runs meanwhile.
|
|
5271
|
+
if (opts.parkOnApproval && launching.some((s) => s.status === "pending" && s.roomWait)) {
|
|
5272
|
+
await harvestQuietly(tenant, workspace, run);
|
|
5273
|
+
run.status = "queued";
|
|
5274
|
+
run.parkedAt = new Date().toISOString();
|
|
5275
|
+
parked = true;
|
|
5276
|
+
save();
|
|
5277
|
+
return;
|
|
5278
|
+
}
|
|
5152
5279
|
// on-fail: another agent takes the step over. Sequential and rare:
|
|
5153
5280
|
// the rescue is a second full attempt at the step's job with the
|
|
5154
5281
|
// failure in front of it, and its success is the step's success —
|
|
@@ -5585,6 +5712,13 @@ const ABANDONED_AFTER_MS = 2 * 60 * 1000;
|
|
|
5585
5712
|
/** The first retry's wait; each one after doubles it, to sixteen times
|
|
5586
5713
|
* this (15 s → 4 min by default). An env, so a test can make it small. */
|
|
5587
5714
|
const retryBaseMs = () => Number(process.env.FOLDRUN_RETRY_BASE_MS) || 15_000;
|
|
5715
|
+
/** How long a group waits for room in its driver before its run goes back
|
|
5716
|
+
* to the queue to wait there: FOLDRUN_ROOM_PARK_AFTER_SEC, default 60 s. */
|
|
5717
|
+
const roomParkAfterMs = () => {
|
|
5718
|
+
const v = process.env.FOLDRUN_ROOM_PARK_AFTER_SEC;
|
|
5719
|
+
const n = v ? Number(v) : NaN;
|
|
5720
|
+
return Number.isFinite(n) && n >= 0 ? n * 1000 : 60_000;
|
|
5721
|
+
};
|
|
5588
5722
|
/** The most recent moment a run showed any sign of life. */
|
|
5589
5723
|
function lastActivity(run) {
|
|
5590
5724
|
let latest = Date.parse(run.startedAt);
|
package/dist/src/starter.js
CHANGED
|
@@ -417,6 +417,7 @@ model: default # a default for every step here
|
|
|
417
417
|
effort: low
|
|
418
418
|
overlap: skip # skip | queue — what a second run does while one is live
|
|
419
419
|
priority: normal # high | normal | low — where its runs stand in the queue
|
|
420
|
+
compute: { vcpu: 4, memory: 8Gi, disk: 40Gi } # optional: what one run may hold at once
|
|
420
421
|
---
|
|
421
422
|
|
|
422
423
|
1. [[researcher]] — Find three sources on {{topic}}.
|
package/dist/src/store.d.ts
CHANGED
|
@@ -542,6 +542,13 @@ export interface FlowInfo {
|
|
|
542
542
|
budget: number | null;
|
|
543
543
|
/** What is wrong with the `budget:` line, or null. */
|
|
544
544
|
budgetProblem: string | null;
|
|
545
|
+
/** `compute:` — what one run's steps may hold together at once (27 Oct
|
|
546
|
+
* 2026); null is no cap of its own. Copied onto each run as it starts. */
|
|
547
|
+
compute?: {
|
|
548
|
+
vcpu?: number;
|
|
549
|
+
memGiB?: number;
|
|
550
|
+
diskGiB?: number;
|
|
551
|
+
} | null;
|
|
545
552
|
/** `trigger: flow` — start when this other flow of the workspace finishes,
|
|
546
553
|
* with `on:` saying which endings count. The finished run's summary and
|
|
547
554
|
* result become this run's task. */
|
|
@@ -1105,6 +1112,9 @@ export declare function isSplicedRecord(step: {
|
|
|
1105
1112
|
subagentOf?: string;
|
|
1106
1113
|
instruction?: string | null;
|
|
1107
1114
|
}): boolean;
|
|
1115
|
+
/** Which cap holds a step back from starting (27 Oct 2026): its run's,
|
|
1116
|
+
* its workspace's, its account's, or the node's budget for runs. */
|
|
1117
|
+
export type RoomLevel = "run" | "workspace" | "account" | "node";
|
|
1108
1118
|
export interface StepRecord {
|
|
1109
1119
|
agent: string;
|
|
1110
1120
|
instruction: string;
|
|
@@ -1311,6 +1321,7 @@ export interface StepRecord {
|
|
|
1311
1321
|
reserved?: {
|
|
1312
1322
|
cpus: number;
|
|
1313
1323
|
memGiB: number;
|
|
1324
|
+
diskGiB?: number;
|
|
1314
1325
|
} | null;
|
|
1315
1326
|
/** The agent's `resources:` as applied (resources.ts), when it set any:
|
|
1316
1327
|
* why `reserved` differs from the class's figures. Absent before 26 Oct
|
|
@@ -1326,7 +1337,24 @@ export interface StepRecord {
|
|
|
1326
1337
|
holding?: {
|
|
1327
1338
|
cpus: number;
|
|
1328
1339
|
memGiB: number;
|
|
1340
|
+
diskGiB?: number;
|
|
1341
|
+
} | null;
|
|
1342
|
+
/** Waiting for room to start (27 Oct 2026): since when — the first time
|
|
1343
|
+
* the room gate said no, kept across re-drives so its place in line is
|
|
1344
|
+
* its age — what it asked for, and why there was none. Set while it
|
|
1345
|
+
* waits in its driver and while its run waits in the queue; cleared the
|
|
1346
|
+
* moment its sandbox exists. A wait for room never fails the step. */
|
|
1347
|
+
roomWait?: {
|
|
1348
|
+
since: string;
|
|
1349
|
+
why: string;
|
|
1350
|
+
cpus: number;
|
|
1351
|
+
memGiB: number;
|
|
1352
|
+
diskGiB?: number;
|
|
1353
|
+
level?: RoomLevel;
|
|
1329
1354
|
} | null;
|
|
1355
|
+
/** The attempt a step sent back to the queue for room resumes on — a
|
|
1356
|
+
* wait is not a try, so it spends none of its `retry:`. */
|
|
1357
|
+
resumeAttempt?: number | null;
|
|
1330
1358
|
/** Token counts behind costUsd, kept so usage reports can say "how many"
|
|
1331
1359
|
* and not only "how much" — the price of a token changes, the count is
|
|
1332
1360
|
* the fact. Absent on steps recorded before this existed. */
|
|
@@ -1407,6 +1435,13 @@ export interface RunRecord {
|
|
|
1407
1435
|
* a run was held to is on its record rather than read from a file that
|
|
1408
1436
|
* may since have changed. Null: none. */
|
|
1409
1437
|
budgetUsd?: number | null;
|
|
1438
|
+
/** The flow's `compute:` when this run was first driven — what its steps
|
|
1439
|
+
* may hold together at once, its own pool (27 Oct 2026). Null: none. */
|
|
1440
|
+
compute?: {
|
|
1441
|
+
vcpu?: number;
|
|
1442
|
+
memGiB?: number;
|
|
1443
|
+
diskGiB?: number;
|
|
1444
|
+
} | null;
|
|
1410
1445
|
/** Memory files this run created or changed, workspace-relative — what
|
|
1411
1446
|
* the agents decided was worth remembering, listed so a person can review
|
|
1412
1447
|
* the lesson without diffing the folder. */
|
|
@@ -1628,6 +1663,8 @@ export declare function runMeter(run: RunRecord): {
|
|
|
1628
1663
|
export declare function runComputeMeter(run: RunRecord): {
|
|
1629
1664
|
coreSecs: number;
|
|
1630
1665
|
gibSecs: number;
|
|
1666
|
+
/** GiB of disk allowed × seconds (27 Oct 2026): the third hold. */
|
|
1667
|
+
diskGibSecs: number;
|
|
1631
1668
|
busyCpuSecs: number;
|
|
1632
1669
|
flatSecs: number;
|
|
1633
1670
|
};
|
package/dist/src/store.js
CHANGED
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
// or an agent, and belongs in git.
|
|
20
20
|
//
|
|
21
21
|
// An account is a named directory (per OKF: tenancy is a hosting concern).
|
|
22
|
+
import { readCompute } from "./resources.js";
|
|
22
23
|
import { expandPlatformVars } from "./host-env.js";
|
|
23
24
|
import { FLOW_TRIGGERS } from "./frontmatter-fields.js";
|
|
24
25
|
import fs from "node:fs";
|
|
@@ -1635,6 +1636,10 @@ export function parseFlow(file, raw) {
|
|
|
1635
1636
|
priority: data.priority === "high" || data.priority === "low" || data.priority === "normal" ? data.priority : null,
|
|
1636
1637
|
budget: runBudget(data.budget),
|
|
1637
1638
|
budgetProblem: budgetProblem(data.budget, ["run"]),
|
|
1639
|
+
compute: (() => {
|
|
1640
|
+
const c = readCompute(data.compute).value;
|
|
1641
|
+
return Object.keys(c).length ? c : null;
|
|
1642
|
+
})(),
|
|
1638
1643
|
// `after: [[flow:publish]]` — a link or a bare name, read the same way
|
|
1639
1644
|
// as every other file-naming field (refs.ts).
|
|
1640
1645
|
after: (refNames(data.after)[0] ?? "").replace(/^flow:/, "").trim() || null,
|
|
@@ -3156,7 +3161,7 @@ export function runMeter(run) {
|
|
|
3156
3161
|
* flat-rate fallback prices those.
|
|
3157
3162
|
*/
|
|
3158
3163
|
export function runComputeMeter(run) {
|
|
3159
|
-
let coreSecs = 0, gibSecs = 0, busyCpuSecs = 0, flatSecs = 0;
|
|
3164
|
+
let coreSecs = 0, gibSecs = 0, diskGibSecs = 0, busyCpuSecs = 0, flatSecs = 0;
|
|
3160
3165
|
for (const s of run.steps) {
|
|
3161
3166
|
if (s.status !== "completed" && s.status !== "failed")
|
|
3162
3167
|
continue;
|
|
@@ -3168,13 +3173,14 @@ export function runComputeMeter(run) {
|
|
|
3168
3173
|
if (s.reserved) {
|
|
3169
3174
|
coreSecs += secs * s.reserved.cpus;
|
|
3170
3175
|
gibSecs += secs * s.reserved.memGiB;
|
|
3176
|
+
diskGibSecs += secs * (s.reserved.diskGiB ?? 0);
|
|
3171
3177
|
}
|
|
3172
3178
|
else {
|
|
3173
3179
|
flatSecs += secs;
|
|
3174
3180
|
}
|
|
3175
3181
|
busyCpuSecs += s.actual?.busyCpuSecs ?? 0;
|
|
3176
3182
|
}
|
|
3177
|
-
return { coreSecs, gibSecs, busyCpuSecs, flatSecs };
|
|
3183
|
+
return { coreSecs, gibSecs, diskGibSecs, busyCpuSecs, flatSecs };
|
|
3178
3184
|
}
|
|
3179
3185
|
export function runDurationSecs(run) {
|
|
3180
3186
|
if (!run.finishedAt)
|
package/package.json
CHANGED
|
@@ -131,7 +131,7 @@ const AGENTS_MD: FieldSpec[] = [
|
|
|
131
131
|
{ key: "budget", type: ["number", "string"], help: "USD per month, or N/day | N/week | N/month" },
|
|
132
132
|
{ key: "priority", type: "string", values: ["high", "normal", "low"], help: "account: where its runs stand in the queue" },
|
|
133
133
|
{ key: "concurrency", type: ["number", "string"], help: "account: runs at once" },
|
|
134
|
-
{ key: "compute", type: "object", help: "
|
|
134
|
+
{ key: "compute", type: "object", help: "{vcpu, memory, disk} the running steps here may hold together — the account's pool, or this workspace's; only narrows the level above" },
|
|
135
135
|
{ key: "alerts", type: "object", validated: true, help: "thresholds that tell notify: — failure_rate (30%), min_runs, credits_day, run_minutes, stale_days, cost_spike, runs_per_hour, new_host" },
|
|
136
136
|
{ key: "capture", type: "string", values: ["full", "metadata"], help: "what runs record of each tool call: full (its input and output, secrets masked) or metadata (name, size and time only)" },
|
|
137
137
|
{ key: "network_enforce", type: "boolean", help: "true: a site off a step's network: list is refused, not only named in the trace. The account's or the workspace's true is enough" },
|
|
@@ -172,6 +172,7 @@ const FLOW: FieldSpec[] = [
|
|
|
172
172
|
{ key: "pause_when", type: "string", help: "a marker that pauses the flow when a run's result says it" },
|
|
173
173
|
{ key: "approvers", type: ["list", "string"], help: "who may approve its gates" },
|
|
174
174
|
{ key: "budget", type: ["number", "string"], help: "USD per run" },
|
|
175
|
+
{ key: "compute", type: "object", help: "{vcpu, memory, disk} one run's steps may hold together at once — only narrows the workspace's and account's" },
|
|
175
176
|
{ key: "live", type: "boolean", help: "true: steps may act for real in a test run" },
|
|
176
177
|
];
|
|
177
178
|
|
|
@@ -55,6 +55,7 @@ const CHECKS: Partial<Record<SchemaKind, Record<string, (value: unknown) => stri
|
|
|
55
55
|
disable_after: (v) => (Number(v) > 0 && Number(v) <= 100 ? null : `${String(v)} is not a count from 1 to 100; it is ignored`),
|
|
56
56
|
pause_when: (v) => (String(v).length <= 60 ? null : "is longer than 60 characters"),
|
|
57
57
|
budget: (v) => budgetProblem(v, ["run"]),
|
|
58
|
+
compute: computeProblem,
|
|
58
59
|
},
|
|
59
60
|
"tool": {
|
|
60
61
|
transport: (v) => (["http", "script", "mcp"].includes(String(v).toLowerCase()) ? null : `"${String(v)}" is not a transport — http, script or mcp`),
|
package/src/platform.ts
CHANGED
|
@@ -116,6 +116,17 @@ export interface PlatformHooks {
|
|
|
116
116
|
* the whole run's would take the parent too. Optional: an executor that
|
|
117
117
|
* cannot re-attach never needs it (the whole run's are killed instead). */
|
|
118
118
|
killSandbox?(sandbox: { kind: string; ref: string }): void;
|
|
119
|
+
/** A restarted driver's sweep of the run's orphans: destroy its
|
|
120
|
+
* sandboxes except `keep` (the refs its steps re-attach to). Not a stop —
|
|
121
|
+
* the run goes on, and steps of it are about to start. Optional: without
|
|
122
|
+
* it the whole run's are killed (killRunSandboxes). */
|
|
123
|
+
killOrphanSandboxes?(runId: string, keep: readonly string[]): void;
|
|
124
|
+
/** Is there room for a step waiting in the queue to start now — by the
|
|
125
|
+
* same rules as the executor's room gate — and if not, why. Asked before
|
|
126
|
+
* such a step is started again, so a run claimed only to find no room
|
|
127
|
+
* yet adds nothing to its trail. Optional: without it the step goes to
|
|
128
|
+
* its executor, which looks there. */
|
|
129
|
+
roomCheck?(ask: { tenant: string; workspace: string; runId: string; since: string; cpus: number; memGiB: number; diskGiB: number }): Promise<string | null>;
|
|
119
130
|
/** Publish storage/public/ as share links after a run. Default: no links. */
|
|
120
131
|
syncPublicShares(tenant: string, workspace: string): Promise<{ added: string[] }>;
|
|
121
132
|
/** This account's own data key, or null to use the install key. */
|
package/src/resources.ts
CHANGED
|
@@ -109,15 +109,19 @@ export function hasOverride(o: ResourceOverride | null | undefined): o is Resour
|
|
|
109
109
|
return !!o && (o.cpu !== undefined || o.memory !== undefined || o.disk !== undefined);
|
|
110
110
|
}
|
|
111
111
|
|
|
112
|
-
/**
|
|
113
|
-
*
|
|
114
|
-
*
|
|
112
|
+
/** `compute:` — the most the steps running inside one scope may hold
|
|
113
|
+
* together, in cores, memory and disk (27 Oct 2026: disk, and the scopes
|
|
114
|
+
* past the account). Written in the account's AGENTS.md (its pool), a
|
|
115
|
+
* workspace's AGENTS.md (that workspace's pool) and a flow (each run's
|
|
116
|
+
* pool). Each level only narrows the one above it — the plan or install
|
|
117
|
+
* caps the account (platform limits.ts). Here it is only read:
|
|
115
118
|
*
|
|
116
|
-
* compute: { vcpu: 4, memory: 8Gi }
|
|
119
|
+
* compute: { vcpu: 4, memory: 8Gi, disk: 40Gi }
|
|
117
120
|
*/
|
|
118
121
|
export interface ComputeAsk {
|
|
119
122
|
vcpu?: number;
|
|
120
123
|
memGiB?: number;
|
|
124
|
+
diskGiB?: number;
|
|
121
125
|
}
|
|
122
126
|
|
|
123
127
|
export function readCompute(raw: unknown): { value: ComputeAsk; problems: string[] } {
|
|
@@ -125,11 +129,11 @@ export function readCompute(raw: unknown): { value: ComputeAsk; problems: string
|
|
|
125
129
|
const problems: string[] = [];
|
|
126
130
|
if (raw === undefined || raw === null) return { value, problems };
|
|
127
131
|
if (typeof raw !== "object" || Array.isArray(raw)) {
|
|
128
|
-
return { value, problems: [`is ${shown(raw)}, not a map — write {vcpu: 4, memory: 8Gi}`] };
|
|
132
|
+
return { value, problems: [`is ${shown(raw)}, not a map — write {vcpu: 4, memory: 8Gi, disk: 40Gi}`] };
|
|
129
133
|
}
|
|
130
134
|
const r = raw as Record<string, unknown>;
|
|
131
135
|
for (const k of Object.keys(r)) {
|
|
132
|
-
if (k !== "vcpu" && k !== "memory") problems.push(`\`${k}\` is not vcpu or
|
|
136
|
+
if (k !== "vcpu" && k !== "memory" && k !== "disk") problems.push(`\`${k}\` is not vcpu, memory or disk — it is ignored`);
|
|
133
137
|
}
|
|
134
138
|
if (r.vcpu !== undefined && r.vcpu !== null) {
|
|
135
139
|
const numeric = typeof r.vcpu === "number" || (typeof r.vcpu === "string" && /^\s*\d+(\.\d+)?\s*$/.test(r.vcpu));
|
|
@@ -142,6 +146,11 @@ export function readCompute(raw: unknown): { value: ComputeAsk; problems: string
|
|
|
142
146
|
if (!Number.isFinite(mi) || mi < 256) problems.push(`memory: ${shown(r.memory)} is not a size of 256Mi or more — write 8Gi`);
|
|
143
147
|
else value.memGiB = mi / 1024;
|
|
144
148
|
}
|
|
149
|
+
if (r.disk !== undefined && r.disk !== null) {
|
|
150
|
+
const mi = quantityMi(r.disk);
|
|
151
|
+
if (!Number.isFinite(mi) || mi < 1024) problems.push(`disk: ${shown(r.disk)} is not a size of 1Gi or more — write 40Gi`);
|
|
152
|
+
else value.diskGiB = mi / 1024;
|
|
153
|
+
}
|
|
145
154
|
return { value, problems };
|
|
146
155
|
}
|
|
147
156
|
|
package/src/run-container.ts
CHANGED
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
// The runner image bundles @foldrun/core itself, so the driver runs the same
|
|
19
19
|
// executeStep the server would — one implementation, two homes.
|
|
20
20
|
|
|
21
|
+
import type { RoomLevel } from "./store.ts";
|
|
21
22
|
import type { PeerInfo } from "./operator.ts";
|
|
22
23
|
import { isHookRecord, type HookRecord } from "./hooks.ts";
|
|
23
24
|
import fs from "node:fs";
|
|
@@ -190,6 +191,10 @@ export interface StepActuals {
|
|
|
190
191
|
|
|
191
192
|
export interface ContainerStepOutcome {
|
|
192
193
|
status: "completed" | "failed";
|
|
194
|
+
/** No room to start, and the step may wait in the queue rather than here
|
|
195
|
+
* (`room.mayDefer`): why. Nothing ran and nothing was rented — not a
|
|
196
|
+
* failure, not an attempt. `status` is not read when this is set. */
|
|
197
|
+
deferred?: string;
|
|
193
198
|
result: string | null;
|
|
194
199
|
/** Why the sandbox ended when the step did not: what the cluster said
|
|
195
200
|
* (OOMKilled, Evicted, the pod gone). Absent when the driver finished
|
|
@@ -1523,6 +1528,18 @@ export interface RunInContainerArgs {
|
|
|
1523
1528
|
* (consumed 0), and before each output line it applies. The runner
|
|
1524
1529
|
* records these on the step, so the next driver knows where to attach. */
|
|
1525
1530
|
checkpoint?: (ref: string, consumed: number) => void;
|
|
1531
|
+
/** The wait for room (27 Oct 2026), for an executor with a room gate.
|
|
1532
|
+
* `since`: when this step first found none, carried across re-drives —
|
|
1533
|
+
* its place in line. `waiting` is called when it finds none (and again
|
|
1534
|
+
* when the reason changes); `mayDefer` says whether, instead of waiting
|
|
1535
|
+
* on, it may answer `deferred` so its run goes back to the queue. */
|
|
1536
|
+
room?: {
|
|
1537
|
+
since: string | null;
|
|
1538
|
+
/** `${runId}#${step}` — breaks a tie between two equal ages. */
|
|
1539
|
+
key?: string;
|
|
1540
|
+
waiting: (why: string, level?: RoomLevel) => void;
|
|
1541
|
+
mayDefer: () => boolean;
|
|
1542
|
+
};
|
|
1526
1543
|
/** A test run: where state/ and storage/ changes go instead of the
|
|
1527
1544
|
* workspace, and who to tell about each. See test-mode.ts. */
|
|
1528
1545
|
divert?: WriteDivert;
|
package/src/runner.ts
CHANGED
|
@@ -19,7 +19,7 @@ import { cascadeLimits, toolOwners } from "./limits.ts";
|
|
|
19
19
|
import { resolveAgentPath } from "./confine.ts";
|
|
20
20
|
import { browserPodLine, podLossDecision, RERAN_ON_FULL } from "./browser-pod.ts";
|
|
21
21
|
import { eventUrl } from "./webhook.ts";
|
|
22
|
-
import { runStepInContainer, resolveResources, memoryGiB, heldCores, killRunSandboxes, type StepTiming } from "./run-container.ts";
|
|
22
|
+
import { runStepInContainer, resolveResources, memoryGiB, heldCores, killRunSandboxes, classDisk, type StepTiming } from "./run-container.ts";
|
|
23
23
|
import { readResources, hasOverride } from "./resources.ts";
|
|
24
24
|
import { hostSafeEnv, platformApiUrl } from "./host-env.ts";
|
|
25
25
|
import { resolveModelCredential, credentialLine, accountModelBlock, isClaudeLoginToken, isModelEnvName, DECLARED_ENV_PREFIX } from "./model-credential.ts";
|
|
@@ -47,6 +47,7 @@ import { sendPlainNotification } from "./notify.ts";
|
|
|
47
47
|
import { TOOL_MAP, BUILTIN_TOOLS, ownToolNames, toolRefs, legacyUseNames, legacyUseError, retiredToolNames, retiredToolError } from "./tool-names.ts";
|
|
48
48
|
import { refNames } from "./refs.ts";
|
|
49
49
|
import {
|
|
50
|
+
type RoomLevel,
|
|
50
51
|
accountDir,
|
|
51
52
|
workspaceDir,
|
|
52
53
|
buildMemoryIndex,
|
|
@@ -2093,6 +2094,11 @@ async function runStep(
|
|
|
2093
2094
|
* answers with its reply (the drive loop's delegateFor). Absent for a
|
|
2094
2095
|
* step that is itself a sub-agent's: depth one. */
|
|
2095
2096
|
delegateJob?: (agent: string, job: string, cancelled: () => boolean) => Promise<DelegationResult>,
|
|
2097
|
+
/** Whether this step, finding no room to start, may go back to the queue
|
|
2098
|
+
* with its run instead of waiting in this driver (the drive loop's
|
|
2099
|
+
* mayDefer). Absent: it waits here, however long — a rescue, a step
|
|
2100
|
+
* outside a flow, a foreground run. */
|
|
2101
|
+
mayDefer?: () => boolean,
|
|
2096
2102
|
) {
|
|
2097
2103
|
// Secret values are injected into scripts as environment variables and
|
|
2098
2104
|
// substituted into API headers, so a model that reads one back — from a
|
|
@@ -2322,7 +2328,7 @@ async function runStep(
|
|
|
2322
2328
|
const limits = resolveResources(size, asked.value, { atLeastClass: !!step.sizeUp });
|
|
2323
2329
|
// What the sandbox holds while it exists — `step.holding`, what the
|
|
2324
2330
|
// account's compute in use sums (Settings, `foldrun usage compute`).
|
|
2325
|
-
const held = { cpus: heldCores(limits.cpus, process.env.FOLDRUN_RUN_ISOLATION), memGiB: memoryGiB(limits.memory) };
|
|
2331
|
+
const held = { cpus: heldCores(limits.cpus, process.env.FOLDRUN_RUN_ISOLATION), memGiB: memoryGiB(limits.memory), diskGiB: memoryGiB(limits.disk ?? classDisk(size)) };
|
|
2326
2332
|
|
|
2327
2333
|
// oauth2 secrets become live access tokens here — the one async moment
|
|
2328
2334
|
// in secret resolution, at the last host-side point before anything is
|
|
@@ -3018,6 +3024,8 @@ async function runStep(
|
|
|
3018
3024
|
};
|
|
3019
3025
|
const primaryTranslator = proxiedTranslator(translator, MODEL_KEY_NAME);
|
|
3020
3026
|
const secondTranslator = proxiedTranslator(fallbackTranslator, `${MODEL_KEY_NAME}_FALLBACK`);
|
|
3027
|
+
// Set once the first sandbox call has run (see room.mayDefer below).
|
|
3028
|
+
let executed = false;
|
|
3021
3029
|
const runIsolated = isolation === "container" ? runStepInContainer : platform.isolation[isolation!]!;
|
|
3022
3030
|
const isolatedArgs = (modelEnv: Record<string, string | undefined>, keyName = MODEL_KEY_NAME) => ({
|
|
3023
3031
|
workspaceRoot,
|
|
@@ -3132,8 +3140,23 @@ async function runStep(
|
|
|
3132
3140
|
step.sandbox = { kind: isolation!, ref, consumed, since: step.sandbox?.since ?? new Date().toISOString() };
|
|
3133
3141
|
// The sandbox exists: past the room gate, it now holds its figures.
|
|
3134
3142
|
if (consumed === 0) step.holding = held;
|
|
3143
|
+
if (consumed === 0) step.roomWait = null;
|
|
3135
3144
|
if (consumed === 0) save();
|
|
3136
3145
|
},
|
|
3146
|
+
// The wait for room: its age is the step's place in line, so it is
|
|
3147
|
+
// on the record from the first "no" and survives a re-drive.
|
|
3148
|
+
room: {
|
|
3149
|
+
since: step.roomWait?.since ?? null,
|
|
3150
|
+
...(runId && stepIndex !== undefined ? { key: `${runId}#${stepIndex}` } : {}),
|
|
3151
|
+
waiting: (why: string, level?: RoomLevel) => {
|
|
3152
|
+
step.roomWait = { since: step.roomWait?.since ?? new Date().toISOString(), why, cpus: held.cpus, memGiB: held.memGiB, diskGiB: held.diskGiB, ...(level ? { level } : {}) };
|
|
3153
|
+
save();
|
|
3154
|
+
},
|
|
3155
|
+
// Only before anything ran: a later call of this attempt (a
|
|
3156
|
+
// rotated credential, a busy provider, a re-run on the full
|
|
3157
|
+
// image) waits for room where it is — its outcome is the step's.
|
|
3158
|
+
mayDefer: () => !executed && (mayDefer?.() ?? false),
|
|
3159
|
+
},
|
|
3137
3160
|
});
|
|
3138
3161
|
|
|
3139
3162
|
// A busy provider gets another moment on the SAME supply before the
|
|
@@ -3154,20 +3177,43 @@ async function runStep(
|
|
|
3154
3177
|
// its first checkpoint, after any wait for room.
|
|
3155
3178
|
if (!platform.sandboxResumable(isolation!)) {
|
|
3156
3179
|
step.holding = held;
|
|
3180
|
+
// Started: no longer waiting for room (an executor that can
|
|
3181
|
+
// report its sandbox clears this at its first checkpoint).
|
|
3182
|
+
step.roomWait = null;
|
|
3157
3183
|
save();
|
|
3158
3184
|
}
|
|
3185
|
+
let out: Awaited<ReturnType<typeof runIsolated>> | undefined;
|
|
3159
3186
|
try {
|
|
3160
|
-
|
|
3187
|
+
out = await runIsolated(a);
|
|
3188
|
+
return out;
|
|
3161
3189
|
} finally {
|
|
3162
3190
|
// Saved now: the clean-up after an attempt (the log drain, the
|
|
3163
3191
|
// lease, publishing) can take a while, and the sandbox is gone.
|
|
3164
3192
|
step.holding = null;
|
|
3193
|
+
// A call that ran, or failed, is past waiting; only a step sent
|
|
3194
|
+
// back to the queue keeps its place in line.
|
|
3195
|
+
if (!out?.deferred) step.roomWait = null;
|
|
3165
3196
|
save();
|
|
3166
3197
|
}
|
|
3167
3198
|
};
|
|
3168
3199
|
let attemptFrom = step.events.length;
|
|
3169
3200
|
let outcome = await runTracked(firstArgs);
|
|
3201
|
+
if (!outcome.deferred) executed = true;
|
|
3170
3202
|
step.sandbox = null; // whatever happens next starts its own
|
|
3203
|
+
// No room, and the run may wait for it in the queue: nothing ran and
|
|
3204
|
+
// nothing was rented, so this is neither a failure nor a try. The
|
|
3205
|
+
// step goes back to pending with its place in line (roomWait).
|
|
3206
|
+
if (outcome.deferred) {
|
|
3207
|
+
if (lease) {
|
|
3208
|
+
await serving?.close();
|
|
3209
|
+
serving = null;
|
|
3210
|
+
await lease.release();
|
|
3211
|
+
}
|
|
3212
|
+
step.status = "pending";
|
|
3213
|
+
push("info", `room: none yet — ${outcome.deferred}; the run waits in the queue and starts this step when there is`);
|
|
3214
|
+
save();
|
|
3215
|
+
return { actedThenFailedCheck: false, deferred: true };
|
|
3216
|
+
}
|
|
3171
3217
|
// A 401 from a token that rotated mid-step is not a broken key: the
|
|
3172
3218
|
// replacement is already on its way to the mounted file. Wait for it
|
|
3173
3219
|
// once, then fall into the same retry the busy case uses.
|
|
@@ -3378,6 +3424,7 @@ async function runStep(
|
|
|
3378
3424
|
// from the record, never from what the config says at billing time.
|
|
3379
3425
|
step.size = size;
|
|
3380
3426
|
if (hasOverride(asked.value)) step.resources = asked.value;
|
|
3427
|
+
// Cores, GiB and disk (27 Oct 2026): all three are billed as held.
|
|
3381
3428
|
step.reserved = held;
|
|
3382
3429
|
} else {
|
|
3383
3430
|
// Same withholding as the sandbox path — the in-process step is what
|
|
@@ -4497,12 +4544,18 @@ function driveRunInner(
|
|
|
4497
4544
|
const entry = parent?.delegations?.find((d) => d.id === step.delegation);
|
|
4498
4545
|
if (entry && entry.status === "running") entry.status = "failed";
|
|
4499
4546
|
}
|
|
4547
|
+
// The sandboxes the steps below re-attach to — the orphan sweep must
|
|
4548
|
+
// leave them standing (27 Oct 2026: it deleted every pod of the run,
|
|
4549
|
+
// and each sibling that was about to re-attach found its pod gone).
|
|
4550
|
+
const reattaching: string[] = [];
|
|
4500
4551
|
for (const step of run.steps) {
|
|
4501
4552
|
if (step.status !== "running") continue;
|
|
4502
4553
|
const resumable = !!step.sandbox && platform.sandboxResumable(step.sandbox.kind);
|
|
4503
4554
|
if (!resumable) {
|
|
4504
4555
|
orphaned = true;
|
|
4505
4556
|
step.sandbox = null;
|
|
4557
|
+
} else {
|
|
4558
|
+
reattaching.push(step.sandbox!.ref);
|
|
4506
4559
|
}
|
|
4507
4560
|
step.status = "pending";
|
|
4508
4561
|
step.events.push({
|
|
@@ -4518,9 +4571,12 @@ function driveRunInner(
|
|
|
4518
4571
|
// `timeout:` carries no pod deadline at all — so the orphaned step
|
|
4519
4572
|
// may still be executing, non-idempotent work included, while the
|
|
4520
4573
|
// new attempt starts. Destroy it first; re-driving is only safe
|
|
4521
|
-
// once nothing else is driving.
|
|
4574
|
+
// once nothing else is driving. Not a stop: the run goes on, so the
|
|
4575
|
+
// platform's sweep keeps the re-attaching sandboxes and leaves no
|
|
4576
|
+
// stop word for the steps about to start.
|
|
4522
4577
|
try {
|
|
4523
4578
|
if (process.env.FOLDRUN_RUN_ISOLATION === "container") killRunSandboxes(run.id);
|
|
4579
|
+
else if (platform.killOrphanSandboxes) platform.killOrphanSandboxes(run.id, reattaching);
|
|
4524
4580
|
else platform.killRunSandboxes(run.id);
|
|
4525
4581
|
} catch {
|
|
4526
4582
|
// a sandbox we cannot reach must not stop the run from resuming
|
|
@@ -4683,6 +4739,12 @@ function driveRunInner(
|
|
|
4683
4739
|
run.budgetUsd = readFlow(tenant, workspace, run.flow)?.budget ?? null;
|
|
4684
4740
|
save();
|
|
4685
4741
|
}
|
|
4742
|
+
// compute: — the flow's run pool, stamped the same way: the cap a run
|
|
4743
|
+
// is held to is on its record (27 Oct 2026).
|
|
4744
|
+
if (run.compute === undefined) {
|
|
4745
|
+
run.compute = readFlow(tenant, workspace, run.flow)?.compute ?? null;
|
|
4746
|
+
save();
|
|
4747
|
+
}
|
|
4686
4748
|
|
|
4687
4749
|
let gi = 0;
|
|
4688
4750
|
let context: string | null = null;
|
|
@@ -5227,9 +5289,40 @@ function driveRunInner(
|
|
|
5227
5289
|
// prompt cache it wrote — see warmGate. The claim is the callback's
|
|
5228
5290
|
// first statement, before any await; the wait comes before the
|
|
5229
5291
|
// pool, so nothing holds a slot while it waits.
|
|
5230
|
-
|
|
5292
|
+
// Room (27 Oct 2026): a step that finds no room may go back to the
|
|
5293
|
+
// queue with its run — freeing the worker — only when nothing else
|
|
5294
|
+
// of its group is under way (a sibling still running keeps the run
|
|
5295
|
+
// here anyway), and only once the group has waited a while, so a
|
|
5296
|
+
// short wait stays a short wait. Never on a foreground run.
|
|
5297
|
+
const mayDefer = (step: StepRecord) => () => {
|
|
5298
|
+
if (!opts.parkOnApproval) return false;
|
|
5299
|
+
if (launching.some((s) => s !== step && s.status === "running" && !s.roomWait)) return false;
|
|
5300
|
+
const since = Math.min(...launching.map((s) => (s.roomWait ? Date.parse(s.roomWait.since) : Infinity)));
|
|
5301
|
+
return Number.isFinite(since) && Date.now() - since >= roomParkAfterMs();
|
|
5302
|
+
};
|
|
5303
|
+
// Steps waiting in the queue for room, the run claimed again: look
|
|
5304
|
+
// first, quietly, before anything (the prompt-cache gate's line
|
|
5305
|
+
// included) writes to their trails. Still none — they stay pending
|
|
5306
|
+
// with their place, the run parks again below, and the trail gains
|
|
5307
|
+
// nothing on each look.
|
|
5308
|
+
const roomStill = new Set<StepRecord>();
|
|
5309
|
+
if (platform.roomCheck) {
|
|
5310
|
+
for (const step of launching) {
|
|
5311
|
+
if (!step.roomWait || !mayDefer(step)()) continue;
|
|
5312
|
+
const w = step.roomWait;
|
|
5313
|
+
const why = await platform.roomCheck({ tenant, workspace, runId: run.id, since: w.since, cpus: w.cpus, memGiB: w.memGiB, diskGiB: w.diskGiB ?? 0 }).catch(() => null);
|
|
5314
|
+
if (!why) continue;
|
|
5315
|
+
roomStill.add(step);
|
|
5316
|
+
if (why !== w.why) {
|
|
5317
|
+
step.roomWait = { ...w, why };
|
|
5318
|
+
save();
|
|
5319
|
+
}
|
|
5320
|
+
}
|
|
5321
|
+
}
|
|
5322
|
+
const starting = launching.filter((s) => !roomStill.has(s));
|
|
5323
|
+
const gate = warmGate(starting);
|
|
5231
5324
|
await Promise.all(
|
|
5232
|
-
|
|
5325
|
+
starting
|
|
5233
5326
|
.map(async (step) => {
|
|
5234
5327
|
const lead = gate.claim(step);
|
|
5235
5328
|
if (lead) {
|
|
@@ -5275,7 +5368,14 @@ function driveRunInner(
|
|
|
5275
5368
|
// count it left. Only a step nobody has touched starts at 1.
|
|
5276
5369
|
const interruptedInBackoff =
|
|
5277
5370
|
!step.sandbox && !!step.attempts && /retrying in \d+s/.test(step.events[step.events.length - 1]?.text ?? "");
|
|
5278
|
-
|
|
5371
|
+
// A step that went back to the queue for room resumes the attempt it was on.
|
|
5372
|
+
const from = step.sandbox
|
|
5373
|
+
? Math.max(1, step.attempts ?? 1)
|
|
5374
|
+
: step.resumeAttempt
|
|
5375
|
+
? Math.min(attempts, step.resumeAttempt)
|
|
5376
|
+
: interruptedInBackoff
|
|
5377
|
+
? Math.min(attempts, step.attempts! + 1)
|
|
5378
|
+
: 1;
|
|
5279
5379
|
for (let attempt = from; attempt <= attempts; attempt++) {
|
|
5280
5380
|
const ceiling = ceilingFor(step);
|
|
5281
5381
|
// Nothing left is a refusal, not "no ceiling": a zero passed
|
|
@@ -5303,6 +5403,7 @@ function driveRunInner(
|
|
|
5303
5403
|
const attemptStartedAt = new Date().toISOString();
|
|
5304
5404
|
const eventsBefore = step.events.length;
|
|
5305
5405
|
step.attempts = attempt;
|
|
5406
|
+
step.resumeAttempt = null;
|
|
5306
5407
|
step.status = "running";
|
|
5307
5408
|
if (ceiling.ceilingUsd !== null) reserved.set(step, ceiling.ceilingUsd);
|
|
5308
5409
|
save();
|
|
@@ -5331,7 +5432,15 @@ function driveRunInner(
|
|
|
5331
5432
|
siblingsOf(step),
|
|
5332
5433
|
peerBus,
|
|
5333
5434
|
delegateFor(step),
|
|
5435
|
+
mayDefer(step),
|
|
5334
5436
|
).finally(() => reserved.delete(step));
|
|
5437
|
+
// Sent back for room: not a try. The run parks once the
|
|
5438
|
+
// group is done (below) and this attempt starts again later.
|
|
5439
|
+
if (verdict?.deferred) {
|
|
5440
|
+
step.resumeAttempt = attempt;
|
|
5441
|
+
save();
|
|
5442
|
+
return;
|
|
5443
|
+
}
|
|
5335
5444
|
// runStep mutates step.status; read it through a widened local
|
|
5336
5445
|
// so TS doesn't keep the "running" narrowing from above.
|
|
5337
5446
|
const outcome: string = step.status;
|
|
@@ -5426,6 +5535,18 @@ function driveRunInner(
|
|
|
5426
5535
|
}
|
|
5427
5536
|
}
|
|
5428
5537
|
|
|
5538
|
+
// Steps sent back for want of room: the run waits in the queue — its
|
|
5539
|
+
// row keeps its place — and comes back to this group when it is
|
|
5540
|
+
// claimed again. Nothing after the group runs meanwhile.
|
|
5541
|
+
if (opts.parkOnApproval && launching.some((s) => s.status === "pending" && s.roomWait)) {
|
|
5542
|
+
await harvestQuietly(tenant, workspace, run);
|
|
5543
|
+
run.status = "queued";
|
|
5544
|
+
run.parkedAt = new Date().toISOString();
|
|
5545
|
+
parked = true;
|
|
5546
|
+
save();
|
|
5547
|
+
return;
|
|
5548
|
+
}
|
|
5549
|
+
|
|
5429
5550
|
// on-fail: another agent takes the step over. Sequential and rare:
|
|
5430
5551
|
// the rescue is a second full attempt at the step's job with the
|
|
5431
5552
|
// failure in front of it, and its success is the step's success —
|
|
@@ -5868,6 +5989,13 @@ const ABANDONED_AFTER_MS = 2 * 60 * 1000;
|
|
|
5868
5989
|
/** The first retry's wait; each one after doubles it, to sixteen times
|
|
5869
5990
|
* this (15 s → 4 min by default). An env, so a test can make it small. */
|
|
5870
5991
|
const retryBaseMs = () => Number(process.env.FOLDRUN_RETRY_BASE_MS) || 15_000;
|
|
5992
|
+
/** How long a group waits for room in its driver before its run goes back
|
|
5993
|
+
* to the queue to wait there: FOLDRUN_ROOM_PARK_AFTER_SEC, default 60 s. */
|
|
5994
|
+
const roomParkAfterMs = () => {
|
|
5995
|
+
const v = process.env.FOLDRUN_ROOM_PARK_AFTER_SEC;
|
|
5996
|
+
const n = v ? Number(v) : NaN;
|
|
5997
|
+
return Number.isFinite(n) && n >= 0 ? n * 1000 : 60_000;
|
|
5998
|
+
};
|
|
5871
5999
|
|
|
5872
6000
|
/** The most recent moment a run showed any sign of life. */
|
|
5873
6001
|
function lastActivity(run: RunRecord): number {
|
package/src/starter.ts
CHANGED
|
@@ -433,6 +433,7 @@ model: default # a default for every step here
|
|
|
433
433
|
effort: low
|
|
434
434
|
overlap: skip # skip | queue — what a second run does while one is live
|
|
435
435
|
priority: normal # high | normal | low — where its runs stand in the queue
|
|
436
|
+
compute: { vcpu: 4, memory: 8Gi, disk: 40Gi } # optional: what one run may hold at once
|
|
436
437
|
---
|
|
437
438
|
|
|
438
439
|
1. [[researcher]] — Find three sources on {{topic}}.
|
package/src/store.ts
CHANGED
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
//
|
|
21
21
|
// An account is a named directory (per OKF: tenancy is a hosting concern).
|
|
22
22
|
|
|
23
|
+
import { readCompute } from "./resources.ts";
|
|
23
24
|
import { expandPlatformVars } from "./host-env.ts";
|
|
24
25
|
import { EFFORT_LEVELS, FLOW_TRIGGERS } from "./frontmatter-fields.ts";
|
|
25
26
|
import type { BrowserPodOutcome } from "./browser-pod.ts";
|
|
@@ -1386,6 +1387,9 @@ export interface FlowInfo {
|
|
|
1386
1387
|
budget: number | null;
|
|
1387
1388
|
/** What is wrong with the `budget:` line, or null. */
|
|
1388
1389
|
budgetProblem: string | null;
|
|
1390
|
+
/** `compute:` — what one run's steps may hold together at once (27 Oct
|
|
1391
|
+
* 2026); null is no cap of its own. Copied onto each run as it starts. */
|
|
1392
|
+
compute?: { vcpu?: number; memGiB?: number; diskGiB?: number } | null;
|
|
1389
1393
|
/** `trigger: flow` — start when this other flow of the workspace finishes,
|
|
1390
1394
|
* with `on:` saying which endings count. The finished run's summary and
|
|
1391
1395
|
* result become this run's task. */
|
|
@@ -2080,6 +2084,10 @@ export function parseFlow(file: string, raw: string): FlowInfo {
|
|
|
2080
2084
|
priority: data.priority === "high" || data.priority === "low" || data.priority === "normal" ? data.priority : null,
|
|
2081
2085
|
budget: runBudget(data.budget),
|
|
2082
2086
|
budgetProblem: budgetProblem(data.budget, ["run"]),
|
|
2087
|
+
compute: (() => {
|
|
2088
|
+
const c = readCompute(data.compute).value;
|
|
2089
|
+
return Object.keys(c).length ? c : null;
|
|
2090
|
+
})(),
|
|
2083
2091
|
// `after: [[flow:publish]]` — a link or a bare name, read the same way
|
|
2084
2092
|
// as every other file-naming field (refs.ts).
|
|
2085
2093
|
after: (refNames(data.after)[0] ?? "").replace(/^flow:/, "").trim() || null,
|
|
@@ -3335,6 +3343,10 @@ export function isSplicedRecord(step: { rescueOf?: string; subagentOf?: string;
|
|
|
3335
3343
|
return !!step.subagentOf || isRescueRecord(step);
|
|
3336
3344
|
}
|
|
3337
3345
|
|
|
3346
|
+
/** Which cap holds a step back from starting (27 Oct 2026): its run's,
|
|
3347
|
+
* its workspace's, its account's, or the node's budget for runs. */
|
|
3348
|
+
export type RoomLevel = "run" | "workspace" | "account" | "node";
|
|
3349
|
+
|
|
3338
3350
|
export interface StepRecord {
|
|
3339
3351
|
agent: string;
|
|
3340
3352
|
instruction: string;
|
|
@@ -3532,7 +3544,7 @@ export interface StepRecord {
|
|
|
3532
3544
|
* from what the config says when the bill is computed. Absent on steps
|
|
3533
3545
|
* from before sizes existed — billing falls back to the flat rate. */
|
|
3534
3546
|
size?: "small" | "large" | "heavy";
|
|
3535
|
-
reserved?: { cpus: number; memGiB: number } | null;
|
|
3547
|
+
reserved?: { cpus: number; memGiB: number; diskGiB?: number } | null;
|
|
3536
3548
|
/** The agent's `resources:` as applied (resources.ts), when it set any:
|
|
3537
3549
|
* why `reserved` differs from the class's figures. Absent before 26 Oct
|
|
3538
3550
|
* 2026 and on steps of an agent that wrote none. */
|
|
@@ -3540,7 +3552,16 @@ export interface StepRecord {
|
|
|
3540
3552
|
/** What the step's sandbox holds while it exists — set once it is past
|
|
3541
3553
|
* the room gate, cleared when the attempt ends. What an account's
|
|
3542
3554
|
* compute in use sums; `reserved` is the bill's record of the same. */
|
|
3543
|
-
holding?: { cpus: number; memGiB: number } | null;
|
|
3555
|
+
holding?: { cpus: number; memGiB: number; diskGiB?: number } | null;
|
|
3556
|
+
/** Waiting for room to start (27 Oct 2026): since when — the first time
|
|
3557
|
+
* the room gate said no, kept across re-drives so its place in line is
|
|
3558
|
+
* its age — what it asked for, and why there was none. Set while it
|
|
3559
|
+
* waits in its driver and while its run waits in the queue; cleared the
|
|
3560
|
+
* moment its sandbox exists. A wait for room never fails the step. */
|
|
3561
|
+
roomWait?: { since: string; why: string; cpus: number; memGiB: number; diskGiB?: number; level?: RoomLevel } | null;
|
|
3562
|
+
/** The attempt a step sent back to the queue for room resumes on — a
|
|
3563
|
+
* wait is not a try, so it spends none of its `retry:`. */
|
|
3564
|
+
resumeAttempt?: number | null;
|
|
3544
3565
|
/** Token counts behind costUsd, kept so usage reports can say "how many"
|
|
3545
3566
|
* and not only "how much" — the price of a token changes, the count is
|
|
3546
3567
|
* the fact. Absent on steps recorded before this existed. */
|
|
@@ -3622,6 +3643,9 @@ export interface RunRecord {
|
|
|
3622
3643
|
* a run was held to is on its record rather than read from a file that
|
|
3623
3644
|
* may since have changed. Null: none. */
|
|
3624
3645
|
budgetUsd?: number | null;
|
|
3646
|
+
/** The flow's `compute:` when this run was first driven — what its steps
|
|
3647
|
+
* may hold together at once, its own pool (27 Oct 2026). Null: none. */
|
|
3648
|
+
compute?: { vcpu?: number; memGiB?: number; diskGiB?: number } | null;
|
|
3625
3649
|
/** Memory files this run created or changed, workspace-relative — what
|
|
3626
3650
|
* the agents decided was worth remembering, listed so a person can review
|
|
3627
3651
|
* the lesson without diffing the folder. */
|
|
@@ -4234,10 +4258,12 @@ export function runMeter(run: RunRecord): {
|
|
|
4234
4258
|
export function runComputeMeter(run: RunRecord): {
|
|
4235
4259
|
coreSecs: number;
|
|
4236
4260
|
gibSecs: number;
|
|
4261
|
+
/** GiB of disk allowed × seconds (27 Oct 2026): the third hold. */
|
|
4262
|
+
diskGibSecs: number;
|
|
4237
4263
|
busyCpuSecs: number;
|
|
4238
4264
|
flatSecs: number;
|
|
4239
4265
|
} {
|
|
4240
|
-
let coreSecs = 0, gibSecs = 0, busyCpuSecs = 0, flatSecs = 0;
|
|
4266
|
+
let coreSecs = 0, gibSecs = 0, diskGibSecs = 0, busyCpuSecs = 0, flatSecs = 0;
|
|
4241
4267
|
for (const s of run.steps) {
|
|
4242
4268
|
if (s.status !== "completed" && s.status !== "failed") continue;
|
|
4243
4269
|
if (s.carriedFrom) continue;
|
|
@@ -4246,12 +4272,13 @@ export function runComputeMeter(run: RunRecord): {
|
|
|
4246
4272
|
if (s.reserved) {
|
|
4247
4273
|
coreSecs += secs * s.reserved.cpus;
|
|
4248
4274
|
gibSecs += secs * s.reserved.memGiB;
|
|
4275
|
+
diskGibSecs += secs * (s.reserved.diskGiB ?? 0);
|
|
4249
4276
|
} else {
|
|
4250
4277
|
flatSecs += secs;
|
|
4251
4278
|
}
|
|
4252
4279
|
busyCpuSecs += s.actual?.busyCpuSecs ?? 0;
|
|
4253
4280
|
}
|
|
4254
|
-
return { coreSecs, gibSecs, busyCpuSecs, flatSecs };
|
|
4281
|
+
return { coreSecs, gibSecs, diskGibSecs, busyCpuSecs, flatSecs };
|
|
4255
4282
|
}
|
|
4256
4283
|
|
|
4257
4284
|
export function runDurationSecs(run: RunRecord): number | null {
|