@foldrun/core 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. package/README.md +27 -10
  2. package/dist/index.d.ts +3 -0
  3. package/dist/index.js +3 -0
  4. package/dist/src/account.d.ts +45 -0
  5. package/dist/src/account.js +130 -0
  6. package/dist/src/api-tools.d.ts +8 -0
  7. package/dist/src/api-tools.js +22 -4
  8. package/dist/src/approvals.d.ts +104 -1
  9. package/dist/src/approvals.js +192 -2
  10. package/dist/src/browser-pod.d.ts +129 -0
  11. package/dist/src/browser-pod.js +284 -0
  12. package/dist/src/budget.d.ts +32 -0
  13. package/dist/src/budget.js +177 -0
  14. package/dist/src/catalog.js +3 -1
  15. package/dist/src/clock.d.ts +49 -0
  16. package/dist/src/clock.js +121 -0
  17. package/dist/src/completions.js +17 -8
  18. package/dist/src/confine.d.ts +22 -0
  19. package/dist/src/confine.js +224 -4
  20. package/dist/src/deploy.d.ts +33 -0
  21. package/dist/src/deploy.js +145 -6
  22. package/dist/src/egress.d.ts +26 -2
  23. package/dist/src/egress.js +8 -1
  24. package/dist/src/eol.d.ts +2 -0
  25. package/dist/src/eol.js +7 -0
  26. package/dist/src/evals.d.ts +9 -0
  27. package/dist/src/evals.js +26 -13
  28. package/dist/src/flow-lint.d.ts +12 -0
  29. package/dist/src/flow-lint.js +331 -1
  30. package/dist/src/flow-patterns.d.ts +141 -0
  31. package/dist/src/flow-patterns.js +568 -0
  32. package/dist/src/gitrepo.d.ts +18 -0
  33. package/dist/src/gitrepo.js +49 -0
  34. package/dist/src/host-env.d.ts +8 -0
  35. package/dist/src/host-env.js +48 -0
  36. package/dist/src/json-schema.d.ts +12 -0
  37. package/dist/src/json-schema.js +202 -0
  38. package/dist/src/kinds.js +1 -1
  39. package/dist/src/language.d.ts +17 -0
  40. package/dist/src/language.js +55 -0
  41. package/dist/src/layout.d.ts +69 -0
  42. package/dist/src/layout.js +200 -0
  43. package/dist/src/library.d.ts +9 -0
  44. package/dist/src/library.js +43 -4
  45. package/dist/src/limits.d.ts +84 -0
  46. package/dist/src/limits.js +287 -0
  47. package/dist/src/locale.d.ts +47 -0
  48. package/dist/src/locale.js +177 -0
  49. package/dist/src/mail.d.ts +88 -0
  50. package/dist/src/mail.js +245 -0
  51. package/dist/src/model-probe.d.ts +3 -0
  52. package/dist/src/model-probe.js +1 -0
  53. package/dist/src/notify.d.ts +132 -8
  54. package/dist/src/notify.js +360 -58
  55. package/dist/src/openapi.js +5 -3
  56. package/dist/src/operator.d.ts +201 -0
  57. package/dist/src/operator.js +251 -0
  58. package/dist/src/paths.d.ts +33 -0
  59. package/dist/src/paths.js +87 -0
  60. package/dist/src/platform.d.ts +144 -4
  61. package/dist/src/platform.js +27 -3
  62. package/dist/src/provider-health.d.ts +55 -0
  63. package/dist/src/provider-health.js +233 -0
  64. package/dist/src/providers.d.ts +481 -0
  65. package/dist/src/providers.js +1105 -6
  66. package/dist/src/refusal.d.ts +83 -0
  67. package/dist/src/refusal.js +281 -0
  68. package/dist/src/run-container.d.ts +206 -17
  69. package/dist/src/run-container.js +547 -80
  70. package/dist/src/runner.d.ts +179 -10
  71. package/dist/src/runner.js +1423 -169
  72. package/dist/src/runtime.d.ts +52 -0
  73. package/dist/src/runtime.js +493 -58
  74. package/dist/src/script-tools.d.ts +15 -2
  75. package/dist/src/script-tools.js +44 -4
  76. package/dist/src/secret-files.d.ts +6 -5
  77. package/dist/src/secret-files.js +24 -11
  78. package/dist/src/secret-health.d.ts +83 -0
  79. package/dist/src/secret-health.js +177 -0
  80. package/dist/src/secrets.d.ts +38 -1
  81. package/dist/src/secrets.js +197 -96
  82. package/dist/src/starter.d.ts +9 -0
  83. package/dist/src/starter.js +95 -11
  84. package/dist/src/step-exec.d.ts +106 -1
  85. package/dist/src/step-exec.js +558 -85
  86. package/dist/src/step-removal.d.ts +15 -0
  87. package/dist/src/step-removal.js +49 -0
  88. package/dist/src/storage.d.ts +23 -4
  89. package/dist/src/storage.js +125 -23
  90. package/dist/src/store.d.ts +492 -38
  91. package/dist/src/store.js +882 -69
  92. package/dist/src/subagents.d.ts +75 -0
  93. package/dist/src/subagents.js +149 -0
  94. package/dist/src/tar.js +2 -1
  95. package/dist/src/test-mode.d.ts +111 -0
  96. package/dist/src/test-mode.js +285 -0
  97. package/dist/src/tool-names.d.ts +13 -1
  98. package/dist/src/tool-names.js +38 -11
  99. package/dist/src/tool-programs.d.ts +26 -0
  100. package/dist/src/tool-programs.js +172 -0
  101. package/dist/src/tool-test.js +152 -33
  102. package/dist/src/translator.d.ts +8 -2
  103. package/dist/src/translator.js +71 -24
  104. package/dist/src/trigger-log.d.ts +43 -0
  105. package/dist/src/trigger-log.js +122 -0
  106. package/dist/src/triggers.d.ts +23 -0
  107. package/dist/src/triggers.js +101 -12
  108. package/dist/src/untrusted-fetch.d.ts +11 -0
  109. package/dist/src/untrusted-fetch.js +77 -0
  110. package/dist/src/web-actions.d.ts +91 -0
  111. package/dist/src/web-actions.js +199 -0
  112. package/dist/src/webhook.d.ts +19 -3
  113. package/dist/src/webhook.js +50 -11
  114. package/index.ts +3 -0
  115. package/package.json +5 -5
  116. package/src/account.ts +149 -0
  117. package/src/api-tools.ts +32 -4
  118. package/src/approvals.ts +253 -2
  119. package/src/browser-pod.ts +313 -0
  120. package/src/budget.ts +178 -0
  121. package/src/catalog.ts +3 -1
  122. package/src/clock.ts +149 -0
  123. package/src/completions.ts +17 -8
  124. package/src/confine.ts +224 -4
  125. package/src/deploy.ts +137 -6
  126. package/src/egress.ts +32 -4
  127. package/src/eol.ts +7 -0
  128. package/src/evals.ts +36 -11
  129. package/src/flow-lint.ts +371 -1
  130. package/src/flow-patterns.ts +597 -0
  131. package/src/gitrepo.ts +48 -0
  132. package/src/host-env.ts +51 -0
  133. package/src/json-schema.ts +184 -0
  134. package/src/kinds.ts +1 -1
  135. package/src/language.ts +62 -0
  136. package/src/layout.ts +228 -0
  137. package/src/library.ts +50 -4
  138. package/src/limits.ts +290 -0
  139. package/src/locale.ts +217 -0
  140. package/src/mail.ts +316 -0
  141. package/src/model-probe.ts +4 -0
  142. package/src/notify.ts +391 -44
  143. package/src/openapi.ts +5 -3
  144. package/src/operator.ts +300 -0
  145. package/src/paths.ts +83 -0
  146. package/src/platform.ts +158 -5
  147. package/src/provider-health.ts +275 -0
  148. package/src/providers.ts +1368 -6
  149. package/src/refusal.ts +320 -0
  150. package/src/run-container.ts +611 -76
  151. package/src/runner.ts +1463 -165
  152. package/src/runtime.ts +487 -55
  153. package/src/script-tools.ts +55 -6
  154. package/src/secret-files.ts +22 -10
  155. package/src/secret-health.ts +230 -0
  156. package/src/secrets.ts +191 -95
  157. package/src/starter.ts +96 -11
  158. package/src/step-exec.ts +548 -41
  159. package/src/step-removal.ts +59 -0
  160. package/src/storage.ts +149 -24
  161. package/src/store.ts +1227 -74
  162. package/src/subagents.ts +189 -0
  163. package/src/tar.ts +2 -1
  164. package/src/test-mode.ts +311 -0
  165. package/src/tool-names.ts +40 -11
  166. package/src/tool-programs.ts +179 -0
  167. package/src/tool-test.ts +159 -32
  168. package/src/translator.ts +74 -23
  169. package/src/trigger-log.ts +160 -0
  170. package/src/triggers.ts +102 -12
  171. package/src/untrusted-fetch.ts +88 -0
  172. package/src/web-actions.ts +237 -0
  173. package/src/webhook.ts +58 -11
  174. package/templates/hello/agents/curator/agent.md +1 -1
  175. package/templates/hello/agents/notetaker/agent.md +1 -1
package/src/runner.ts CHANGED
@@ -8,11 +8,19 @@ import path from "node:path";
8
8
  import matter from "gray-matter";
9
9
  import { spawn } from "node:child_process";
10
10
  import {
11
- knownPrice, executeStep, extractJson, stepCeiling, type EventExtra } from "./step-exec.ts";
11
+ knownPrice, executeStep, extractJson, stepCeiling, stepCeilingFor, type EventExtra } from "./step-exec.ts";
12
+ import { cascadeLimits, toolOwners } from "./limits.ts";
13
+ import { resolveAgentPath } from "./confine.ts";
14
+ import { browserPodLine, podLossDecision, RERAN_ON_FULL } from "./browser-pod.ts";
12
15
  import { eventUrl } from "./webhook.ts";
13
16
  import { runStepInContainer, sizeLimits, killRunSandboxes, type StepTiming } from "./run-container.ts";
17
+ import { hostSafeEnv } from "./host-env.ts";
18
+ import { validateSchema, describeSchemaErrors, looksLikeSchema } from "./json-schema.ts";
14
19
  import { EGRESS_ENV, MODEL_KEY_NAME, addGrant, hostOf, placeholderNames, proxyModelEnv, unsubstitute, type EgressGrant } from "./egress.ts";
20
+ import { explainRefusal, isRefusalStatus, refusalFromLine, supplyNote, type SupplyState } from "./refusal.ts";
15
21
  import { platform } from "./platform.ts";
22
+ import { healthKey } from "./secret-health.ts";
23
+ import { TEST_WRITES_DIR, restoreDivertedDirs, snapshotDivertedDirs, testHeadline, testModeEnv, withholdSecrets } from "./test-mode.ts";
16
24
 
17
25
  /** Does this process run steps in a sandbox — the container core ships, or
18
26
  * one the platform registered (a pod)? */
@@ -21,7 +29,10 @@ const isolatedRun = () => {
21
29
  return mode === "container" || (!!mode && mode in platform.isolation);
22
30
  };
23
31
  import { gatherConsults, buildConsultTools } from "./agent-tools.ts";
24
- import { TOOL_MAP, BUILTIN_TOOLS, ownToolNames, toolRefs, legacyUseNames, legacyUseError } from "./tool-names.ts";
32
+ import { DELEGATE_TOOL, gatherSubagents, subagentPrompt, subagentTools, type SubagentSpec } from "./subagents.ts";
33
+ import { ASK_TOOL, applyOperatorEvent, askPromptLine, askTimeoutSec, buildAskTool } from "./operator.ts";
34
+ import { sendPlainNotification } from "./notify.ts";
35
+ import { TOOL_MAP, BUILTIN_TOOLS, ownToolNames, toolRefs, legacyUseNames, legacyUseError, retiredToolNames, retiredToolError } from "./tool-names.ts";
25
36
  import { refNames } from "./refs.ts";
26
37
  import {
27
38
  accountDir,
@@ -37,7 +48,11 @@ import {
37
48
  listRuns,
38
49
  syncBundleFor,
39
50
  parseToolDef,
51
+ type ToolDef,
40
52
  parseFlow,
53
+ readFlow,
54
+ markerPresent,
55
+ whenRowsPath,
41
56
  parseApis,
42
57
  resolveModel,
43
58
  parseProvider,
@@ -49,12 +64,17 @@ import {
49
64
  readRun,
50
65
  writeRun,
51
66
  runSummary,
67
+ verdictOf,
52
68
  type FlowStep,
53
69
  type McpSpec,
54
70
  type RunRecord,
55
71
  type StepRecord,
72
+ type StepAttempt,
73
+ sumAttempts,
74
+ parseSchemaText,
56
75
  STORAGE_DIR,
57
76
  adoptLegacyVersionKey,
77
+ listAgents,
58
78
  runCost,
59
79
  } from "./store.ts";
60
80
  import { loadCatalog, checkModel, clampEffort, catalogCost, findModel, type Catalog } from "./catalog.ts";
@@ -67,11 +87,19 @@ import { startTranslator, translatorSpecFor, type TranslatorSpec } from "./trans
67
87
  import { providerPreset } from "./providers.ts";
68
88
  import { buildScriptTools, parseScripts, type ExecutionContext } from "./script-tools.ts";
69
89
  import { libraryDir, libraryTools, libraryMemoryIndex } from "./library.ts";
70
- import { mergeRuntimes, parseRuntime, prepareRuntime, type RuntimeSpec } from "./runtime.ts";
90
+ import { fingerprint, mergeRuntimes, parseRuntime, prepareRuntime, type PreparedRuntime, type RuntimeSpec } from "./runtime.ts";
71
91
  import { materializeFiles, harvestFiles } from "./storage.ts";
72
92
  import { chooseExecutor, ensureImage } from "./container.ts";
73
93
  import { stampBundle } from "./okf.ts";
74
94
  import type { McpServerConfig } from "@anthropic-ai/claude-agent-sdk";
95
+ import { readBrowseSettings, readSearchSettings, resolveSearch, searchSettingsEnv, webConfig, browseSessionProblems } from "./providers.ts";
96
+ import { NEWER_ACTIONS, resolveActionApi } from "./web-actions.ts";
97
+ import { resolveLanguage, languageName, type LanguageChoice } from "./language.ts";
98
+ import { resolveRegion, deriveLocale, localeProse, regionName, type RegionChoice, type LocaleFacts } from "./locale.ts";
99
+ import { trimChars } from "./paths.ts";
100
+ import { resolveClock, localDate, type ClockChoice } from "./clock.ts";
101
+
102
+ export { localDate } from "./clock.ts";
75
103
 
76
104
  // An MCP tool definition becomes an SDK server config. ${SECRET} placeholders
77
105
  // in env and headers resolve server-side, so a credential reaches the server
@@ -123,6 +151,32 @@ export function joinEarlierGroups(groupResults: (string | null)[], gi: number, c
123
151
  return parts.join("\n\n");
124
152
  }
125
153
 
154
+ /**
155
+ * `parallel: N` — a pool of N slots the instances of one fan-out share.
156
+ * Fan-out always ran every instance at once, and still does unless the
157
+ * step says otherwise: twenty browser steps against one site, or one
158
+ * vendor's rate limit, is what the cap is for. Nothing is rented while an
159
+ * instance waits for a slot.
160
+ */
161
+ export function slotPool(size: number): { acquire: () => Promise<void>; release: () => void } {
162
+ let free = Math.max(1, size);
163
+ const waiting: (() => void)[] = [];
164
+ return {
165
+ acquire: () =>
166
+ new Promise<void>((resolve) => {
167
+ if (free > 0) {
168
+ free -= 1;
169
+ resolve();
170
+ } else waiting.push(resolve);
171
+ }),
172
+ release: () => {
173
+ const next = waiting.shift();
174
+ if (next) next();
175
+ else free += 1;
176
+ },
177
+ };
178
+ }
179
+
126
180
  export interface DiscoveredSkill {
127
181
  name: string;
128
182
  description: string;
@@ -237,6 +291,29 @@ export function discoverSkills(agentDir: string, subdir = "skills"): DiscoveredS
237
291
  * Returns null rather than throwing on unparseable YAML: a broken shared file
238
292
  * must not take down every agent under it.
239
293
  */
294
+ /**
295
+ * AGENTS.md without the block a coding tool reads.
296
+ *
297
+ * AGENTS.md is two things in a foldrun folder: the account's (or
298
+ * workspace's) own context, which every running agent is given, and — the
299
+ * convention Next.js set — the file a coding agent (Claude Code, Cursor,
300
+ * Codex) reads before it edits the folder. The CLI writes a short managed
301
+ * block for the second reader between these markers; it is about editing
302
+ * the files, not about doing the work, so no running agent is shown it.
303
+ */
304
+ export const AGENT_RULES_START = "<!-- BEGIN:foldrun-agent-rules -->";
305
+ export const AGENT_RULES_END = "<!-- END:foldrun-agent-rules -->";
306
+ export function withoutAgentRules(body: string): string {
307
+ let out = body;
308
+ for (;;) {
309
+ const start = out.indexOf(AGENT_RULES_START);
310
+ if (start === -1) return out;
311
+ const end = out.indexOf(AGENT_RULES_END, start);
312
+ if (end === -1) return out;
313
+ out = out.slice(0, start) + out.slice(end + AGENT_RULES_END.length);
314
+ }
315
+ }
316
+
240
317
  export function readAgentsMd(dir: string): { data: Record<string, unknown>; body: string } | null {
241
318
  // Convert a pre-rename `mdagent_version:` before parsing. One name in the
242
319
  // code, one on disk — the reader no longer knows two spellings, so a file
@@ -247,7 +324,7 @@ export function readAgentsMd(dir: string): { data: Record<string, unknown>; body
247
324
  if (!fs.existsSync(file)) continue;
248
325
  try {
249
326
  const parsed = matter(fs.readFileSync(file, "utf8"));
250
- return { data: parsed.data as Record<string, unknown>, body: parsed.content.trim() };
327
+ return { data: parsed.data as Record<string, unknown>, body: withoutAgentRules(parsed.content).trim() };
251
328
  } catch {
252
329
  return null;
253
330
  }
@@ -266,6 +343,57 @@ const workspaceRootOf = (agentDir: string) => path.join(agentDir, "..", "..");
266
343
  * writing instructions means *as well as*, because an account rule nobody can
267
344
  * silently drop is the point of having one.
268
345
  */
346
+ /** An agent's own `runtime:` block, else its workspace's (AGENTS.md, the
347
+ * Linux Foundation standard name; project.md stays accepted so nothing
348
+ * that already exists breaks). */
349
+ function ownRuntime(front: Record<string, unknown>, agentDir: string, tenant: string): RuntimeSpec | null {
350
+ return parseRuntime(front.runtime) ?? parseRuntime(workspaceFrontmatter(agentDir, tenant).runtime);
351
+ }
352
+
353
+ /** What one granted script tool's program needs. */
354
+ function toolRuntime(def: ToolDef): RuntimeSpec | null {
355
+ return def.kind === "script" ? parseRuntime((def.spec as { runtime?: unknown }).runtime) : null;
356
+ }
357
+
358
+ /**
359
+ * The runtime one agent's steps are built with: its own declaration (else its
360
+ * workspace's), merged with every script tool it grants. The runner builds
361
+ * exactly this for a step; the platform builds it ahead of time when the
362
+ * workspace is deployed, so a package that will not install is reported at
363
+ * deploy and a scheduled step finds its environment ready. One function, so
364
+ * the two can never disagree about what an agent needs.
365
+ */
366
+ export function agentRuntimeSpec(tenant: string, workspace: string, agentDir: string): RuntimeSpec | null {
367
+ const file = path.join(agentDir, "agent.md");
368
+ if (!fs.existsSync(file)) return null;
369
+ const front = safeMatter(fs.readFileSync(file, "utf8")).data;
370
+ const available = { ...libraryTools(tenant), ...workspaceTools(tenant, workspace) };
371
+ const tools = ownToolNames(front)
372
+ .map((name) => available[name])
373
+ .filter((def): def is ToolDef => Boolean(def))
374
+ .map(toolRuntime);
375
+ return mergeRuntimes(ownRuntime(front, agentDir, tenant), ...tools);
376
+ }
377
+
378
+ /** Every distinct runtime a workspace's agents need, with who needs it. */
379
+ export function workspaceRuntimePlan(
380
+ tenant: string,
381
+ workspace: string,
382
+ ): { fingerprint: string; spec: RuntimeSpec; agents: string[] }[] {
383
+ const byFp = new Map<string, { fingerprint: string; spec: RuntimeSpec; agents: string[] }>();
384
+ const agentsRoot = path.join(workspaceDir(tenant, workspace), "agents");
385
+ if (!fs.existsSync(agentsRoot)) return [];
386
+ for (const name of fs.readdirSync(agentsRoot).sort()) {
387
+ const spec = agentRuntimeSpec(tenant, workspace, path.join(agentsRoot, name));
388
+ if (!spec) continue;
389
+ const fp = fingerprint(spec);
390
+ const entry = byFp.get(fp) ?? { fingerprint: fp, spec, agents: [] };
391
+ entry.agents.push(name);
392
+ byFp.set(fp, entry);
393
+ }
394
+ return [...byFp.values()];
395
+ }
396
+
269
397
  function workspaceFrontmatter(agentDir: string, tenant: string): Record<string, unknown> {
270
398
  return {
271
399
  ...(readAgentsMd(accountDir(tenant))?.data ?? {}),
@@ -274,31 +402,69 @@ function workspaceFrontmatter(agentDir: string, tenant: string): Record<string,
274
402
  }
275
403
 
276
404
  /**
277
- * The calendar an agent works to. `timezone:` in AGENTS.md, nearest-wins like
278
- * every other key there; a flow's cron already means its own timezone, this
279
- * makes "today" mean it too. Unset is UTC — what the pods run — which for a
280
- * Sydney desk made an article dated today "one day in the future" until 10am.
281
- * An unknown name falls back to UTC rather than failing every step.
405
+ * The calendar a set of frontmatter asks for, with no cascade around it —
406
+ * one merged record in, one zone out. The full cascade lives in
407
+ * `agentClock` below and in clock.ts; this is what the account-level parts
408
+ * of the platform (a wallet's month, a budget window) call.
282
409
  */
283
410
  export function resolveTimezone(front: Record<string, unknown>): string {
284
- const tz = typeof front.timezone === "string" ? front.timezone.trim() : "";
285
- const chosen = tz || process.env.FOLDRUN_TIMEZONE?.trim() || "UTC";
286
- try {
287
- new Intl.DateTimeFormat("en-CA", { timeZone: chosen });
288
- return chosen;
289
- } catch {
290
- return "UTC";
291
- }
411
+ return resolveClock([{ level: "workspace", value: front.timezone }]).timezone;
412
+ }
413
+
414
+ /**
415
+ * The clock this step works to: agent, then flow, then workspace, then
416
+ * account, then FOLDRUN_TIMEZONE, then UTC. Nearest wins, and a value that
417
+ * is not a zone falls through to the next level with a line to say so —
418
+ * see clock.ts for why this exists at all.
419
+ */
420
+ export function agentClock(
421
+ agentDir: string,
422
+ tenant: string,
423
+ agentFront: Record<string, unknown>,
424
+ flowTimezone?: string | null,
425
+ ): ClockChoice {
426
+ return resolveClock([
427
+ { level: "agent", value: agentFront.timezone },
428
+ { level: "flow", value: flowTimezone },
429
+ { level: "workspace", value: readAgentsMd(workspaceRootOf(agentDir))?.data?.timezone },
430
+ { level: "account", value: readAgentsMd(accountDir(tenant))?.data?.timezone },
431
+ ]);
432
+ }
433
+
434
+ /**
435
+ * The language this step works in: agent, then workspace, then account, then
436
+ * FOLDRUN_LANGUAGE, then English. The flow level is not read yet — a flow's
437
+ * timezone: travels through three signatures to get here, and this will
438
+ * take the same road when a flow needs it.
439
+ */
440
+ export function agentLanguage(agentDir: string, tenant: string, agentFront: Record<string, unknown>): LanguageChoice {
441
+ return resolveLanguage([
442
+ { level: "agent", value: agentFront.language },
443
+ { level: "workspace", value: readAgentsMd(workspaceRootOf(agentDir))?.data?.language },
444
+ { level: "account", value: readAgentsMd(accountDir(tenant))?.data?.language },
445
+ ]);
292
446
  }
293
447
 
294
- /** YYYY-MM-DD in a timezone — the date the workspace believes it is. */
295
- export function localDate(timeZone: string, now = new Date()): string {
296
- return new Intl.DateTimeFormat("en-CA", {
297
- timeZone,
298
- year: "numeric",
299
- month: "2-digit",
300
- day: "2-digit",
301
- }).format(now);
448
+ /**
449
+ * The country this step works for, on the same road as the language, then
450
+ * FOLDRUN_REGION, then whatever the language tag carried (`en-AU` says AU).
451
+ * Everything a country changes — currency, units, calendar, weekend, text
452
+ * direction — is derived from it in locale.ts; the three overrides
453
+ * (`currency:`, `calendar:`, `units:`) are read at the nearest level that
454
+ * wrote them.
455
+ */
456
+ export function agentRegion(agentDir: string, tenant: string, agentFront: Record<string, unknown>, languageTag: string): RegionChoice {
457
+ return resolveRegion([
458
+ { level: "agent", value: agentFront.region },
459
+ { level: "workspace", value: readAgentsMd(workspaceRootOf(agentDir))?.data?.region },
460
+ { level: "account", value: readAgentsMd(accountDir(tenant))?.data?.region },
461
+ ], languageTag);
462
+ }
463
+
464
+ export function agentLocaleOverrides(agentDir: string, tenant: string, agentFront: Record<string, unknown>): { currency?: unknown; calendar?: unknown; units?: unknown } {
465
+ const levels = [agentFront, readAgentsMd(workspaceRootOf(agentDir))?.data ?? {}, readAgentsMd(accountDir(tenant))?.data ?? {}];
466
+ const pick = (k: string) => levels.map((l) => l[k]).find((v) => v !== undefined && v !== null && v !== "");
467
+ return { currency: pick("currency"), calendar: pick("calendar"), units: pick("units") };
302
468
  }
303
469
 
304
470
  /**
@@ -308,6 +474,65 @@ export function localDate(timeZone: string, now = new Date()): string {
308
474
  * template as "context every agent here shares" — and never read. Only the
309
475
  * frontmatter was, so the instructions people actually wrote reached no model.
310
476
  */
477
+ /**
478
+ * The sub-agents a step may delegate to (`subagents:`), as the SDK will get
479
+ * them: each resolved through agentContext exactly as its own step would be
480
+ * (grants only — no runtime; it runs inside its parent's), then cut to what
481
+ * the parent holds. Problems are lines on the step, never a failed step.
482
+ */
483
+ export function buildSubagentSpecs(o: {
484
+ names: unknown;
485
+ self: string;
486
+ workspaceRoot: string;
487
+ tenant: string;
488
+ tags?: string[];
489
+ runId?: string;
490
+ flowTimezone?: string | null;
491
+ parentAllowed: string[];
492
+ push: (type: "info" | "error", text: string) => void;
493
+ }): SubagentSpec[] {
494
+ const out: SubagentSpec[] = [];
495
+ const { found, missing } = gatherSubagents(o.workspaceRoot, o.names);
496
+ for (const m of missing) o.push("error", `subagents: "${m}" is not an agent in this workspace — not delegated to`);
497
+ for (const sub of found) {
498
+ if (sub.name === o.self) {
499
+ o.push("error", `subagents: "${sub.name}" is this agent itself — an agent does not delegate to itself`);
500
+ continue;
501
+ }
502
+ try {
503
+ const tags = o.tags ?? [];
504
+ const own = agentContext(sub.dir, o.tenant, tags, { runId: o.runId, agent: sub.name }, o.flowTimezone, true);
505
+ const description = typeof own.front.description === "string" ? own.front.description.trim() : "";
506
+ if (!description) {
507
+ o.push("error", `subagents: "${sub.name}" has no description — the model picks a sub-agent by it; not delegated to`);
508
+ continue;
509
+ }
510
+ const tools = subagentTools(own.allowed, o.parentAllowed);
511
+ if (!tools.length) {
512
+ o.push("error", `subagents: "${sub.name}" holds none of its tools inside this step — it can only answer from its prompt (use agents: for that)`);
513
+ }
514
+ const body = withoutAgentRules(matter(fs.readFileSync(path.join(sub.dir, "agent.md"), "utf8")).content);
515
+ out.push({
516
+ name: sub.name,
517
+ description,
518
+ prompt: subagentPrompt({
519
+ body: resolveDocLinks(body, o.workspaceRoot),
520
+ name: sub.name,
521
+ parent: o.self,
522
+ workspace: path.basename(o.workspaceRoot),
523
+ shared: sharedInstructions(sub.dir, o.tenant),
524
+ }),
525
+ tools,
526
+ disallowedTools: refNames(own.front.disallowedTools),
527
+ model: own.front.model ? resolveModel(own.front.model) : "inherit",
528
+ });
529
+ } catch (err) {
530
+ o.push("error", `subagents: "${sub.name}" could not be read — ${err instanceof Error ? err.message : String(err)}`);
531
+ }
532
+ }
533
+ return out;
534
+ }
535
+
311
536
  export function sharedInstructions(agentDir: string, tenant: string): string | null {
312
537
  const sections = [
313
538
  ["Everyone in this account", readAgentsMd(accountDir(tenant))?.body],
@@ -378,10 +603,9 @@ export function saysUntilMarker(result: string | null, until: string): boolean {
378
603
  (line) =>
379
604
  // Strip markdown emphasis and punctuation from BOTH ends: "**APPROVED.**"
380
605
  // is a decision written by someone using bold, not a sentence.
381
- line
382
- .trim()
383
- .replace(/^[#>\-*_`\s]+/, "")
384
- .replace(/[.!:,*_`\s]+$/, "")
606
+ // trimChars, not a `[…]+$` regex: that one is quadratic on a
607
+ // line of tabs, and this is a line an agent wrote.
608
+ trimChars(trimChars(line.trim(), "#>-*_`"), ".!:,*_`").trim()
385
609
  .toLowerCase() === marker,
386
610
  );
387
611
  }
@@ -397,7 +621,9 @@ export function resolveDocLinks(text: string, workspaceRoot: string): string {
397
621
  // instruction names needs to say WHERE without naming a file. Added
398
622
  // first, so a document that happens to share a folder's name still wins.
399
623
  for (const dir of [STORAGE_DIR, "state", "knowledge", "memory", "skills", "outputs"]) {
400
- if (fs.existsSync(path.join(workspaceRoot, dir))) map.set(norm(dir), `../../${dir}/`);
624
+ // storage/ and state/ by the one spelling that works in every tool,
625
+ // the shell included; the rest as they always were.
626
+ if (fs.existsSync(path.join(workspaceRoot, dir))) map.set(norm(dir), dir === STORAGE_DIR || dir === "state" ? `workspace/${dir}/` : `../../${dir}/`);
401
627
  }
402
628
  for (const kind of ["knowledge", "memory"] as const) {
403
629
  const dir = path.join(workspaceRoot, kind);
@@ -436,7 +662,7 @@ export function resolveDocLinks(text: string, workspaceRoot: string): string {
436
662
  continue;
437
663
  }
438
664
  const fwd = entry.split(path.sep).join("/");
439
- const rel = `../../state/${fwd}`;
665
+ const rel = `workspace/state/${fwd}`;
440
666
  map.set(norm(`state/${fwd}`), rel);
441
667
  map.set(norm(fwd), rel);
442
668
  }
@@ -452,7 +678,7 @@ export function resolveDocLinks(text: string, workspaceRoot: string): string {
452
678
  "utf8",
453
679
  ),
454
680
  ) as { files?: { path: string }[] };
455
- for (const f of index.files ?? []) map.set(norm(`${STORAGE_DIR}/${f.path}`), `../../${STORAGE_DIR}/${f.path}`);
681
+ for (const f of index.files ?? []) map.set(norm(`${STORAGE_DIR}/${f.path}`), `workspace/${STORAGE_DIR}/${f.path}`);
456
682
  } catch {
457
683
  // no file store index is normal
458
684
  }
@@ -467,11 +693,14 @@ export function resolveDocLinks(text: string, workspaceRoot: string): string {
467
693
  }
468
694
  const fwd = entry.split(path.sep).join("/");
469
695
  if (fwd.startsWith(".store/")) continue;
470
- map.set(norm(`${STORAGE_DIR}/${fwd}`), `../../${STORAGE_DIR}/${fwd}`);
471
- map.set(norm(fwd), `../../${STORAGE_DIR}/${fwd}`);
696
+ map.set(norm(`${STORAGE_DIR}/${fwd}`), `workspace/${STORAGE_DIR}/${fwd}`);
697
+ map.set(norm(fwd), `workspace/${STORAGE_DIR}/${fwd}`);
472
698
  }
473
699
  }
474
- return text.replace(/\[\[([^\]\n]+)\]\]/g, (whole, name: string) => {
700
+ // `[` excluded from the name: with it allowed, a run of `[[` makes the
701
+ // engine re-scan to the end from every opening — quadratic on a result
702
+ // an agent wrote.
703
+ return text.replace(/\[\[([^\]\[\n]+)\]\]/g, (whole, name: string) => {
475
704
  const hit = map.get(norm(name.trim()));
476
705
  return hit ? `\`${hit}\`` : whole;
477
706
  });
@@ -483,6 +712,12 @@ function agentContext(
483
712
  tags: string[] = [],
484
713
  /** Who is running, so scripts can stamp what they write with its provenance. */
485
714
  identity: { runId?: string; agent?: string } = {},
715
+ /** The flow's own `timezone:`, when this step belongs to one. Between the
716
+ * agent's frontmatter and the workspace's AGENTS.md in the cascade. */
717
+ flowTimezone?: string | null,
718
+ /** Only what the agent is granted — no runtime prepared. For a sub-agent,
719
+ * whose tools are cut to its parent's and run in the parent's runtime. */
720
+ grantsOnly = false,
486
721
  ) {
487
722
  // agentDir is <data>/<tenant>/projects/<workspace>/agents/<agent>, so the
488
723
  // workspace a secret should resolve against is two levels up.
@@ -497,13 +732,37 @@ function agentContext(
497
732
  // than implied: told to read `../../knowledge/index.md`, the model asked for
498
733
  // `/knowledge/index.md` twice before working it out. Two denials and a turn,
499
734
  // for want of one sentence.
735
+ // The language this step works in, resolved before the prompt is built:
736
+ // its sentence goes in here, the env the tools read is set below, and the
737
+ // step prints any level that wrote something unreadable.
738
+ const language = agentLanguage(agentDir, tenant, front);
739
+ const region = agentRegion(agentDir, tenant, front, language.language);
740
+ // The clock is resolved further down for the env; the locale needs its
741
+ // zone now so "today" in a Jalali or Hijri calendar is today where the
742
+ // step works, not in UTC.
743
+ const localeZone = agentClock(agentDir, tenant, front, flowTimezone).timezone;
744
+ const locale: LocaleFacts = deriveLocale(language.language, region.region, agentLocaleOverrides(agentDir, tenant, front), localeZone);
745
+ // Said only when someone set it: an agent that never mentions a language
746
+ // or a region reads exactly the prompt it always did.
747
+ const languageSet = language.from !== "default" && language.from !== "env";
748
+ const regionSet = region.from !== "none" && region.from !== "env" && region.from !== "language";
749
+ if (languageSet || regionSet) {
750
+ parts.push(
751
+ `# Locale\n\n${localeProse(locale, languageName(language.language), regionName(region.region))} ` +
752
+ `Write in ${languageName(language.language)} unless the step you are given says otherwise; ` +
753
+ `use these currency, units and calendar conventions in anything you produce. ` +
754
+ `Your tools already know: a search asks in it and for this country, the browser reports it, a fetch requests it.`,
755
+ );
756
+ }
500
757
  parts.push(
501
758
  `# Where you are\n\n` +
502
759
  `Your working directory is \`agents/${path.basename(agentDir)}/\` inside the ` +
503
760
  `\`${workspace}\` workspace. Every path in this prompt is relative to it — ` +
504
- `\`outputs/\` is yours, and \`../../\` is the workspace root, so the workspace's ` +
505
- `own knowledge is at \`../../knowledge/\`. Absolute paths are outside the ` +
506
- `workspace and will be refused.\n\n` +
761
+ `\`outputs/\` is yours, and \`workspace/\` is the workspace root, in every tool ` +
762
+ `and in the shell: the shared file store is \`workspace/storage/\`, what runs ` +
763
+ `carry forward is \`workspace/state/\`. (\`../../\` reaches the same root and ` +
764
+ `still works, so a path like \`../../knowledge/\` below is \`workspace/knowledge/\`.) ` +
765
+ `Absolute paths are outside the workspace and will be refused.\n\n` +
507
766
  // A flow's whole point is that a later step works on what an earlier one
508
767
  // produced, and each agent writes to its own outputs/. A run showed what
509
768
  // omitting this costs: the checker looked in its own empty outputs/ and
@@ -524,10 +783,18 @@ function agentContext(
524
783
  // adoptLegacyFilesDir's "move files/ to storage/" is a no-op by the time
525
784
  // the run ends — a deliverable written where this text pointed was
526
785
  // harvested from nowhere and silently never appeared.
527
- `\`../../storage/\` is the workspace file store: anything you leave there is kept ` +
786
+ `\`workspace/storage/\` is the workspace file store: anything you leave there is kept ` +
528
787
  `after the run and shown on the Storage page for people to download. Write ` +
529
- `deliverables people asked for — CSVs, reports, PDFs, images — to \`../../storage/\`; ` +
530
- `use \`outputs/\` for working text the next step reads.`,
788
+ `deliverables people asked for — CSVs, reports, PDFs, images — to \`workspace/storage/\`; ` +
789
+ `use \`outputs/\` for working text the next step reads. ` +
790
+ // A Glob PATTERN that climbs (`../../storage/draft/*.md`) always answered
791
+ // "No files found", even when the file was there. On blog-desk that made
792
+ // every loop pass look like a first pass: the illustrator never saw the
793
+ // fact-check report, re-picked the cover, and 2 of 3 publish runs failed
794
+ // (run-munef2am-439a, 2026-09-30). checkPaths now splits such a pattern
795
+ // into Glob's `path` and the rest; the advice stays, as the plain form.
796
+ `To look for files there, open a known file with Read by its exact path, or ` +
797
+ `Glob with \`path: "workspace/storage/..."\` and a pattern like \`*.md\`.`,
531
798
  );
532
799
 
533
800
  // Shared context before anything derived — an account or workspace rule is
@@ -681,7 +948,7 @@ function agentContext(
681
948
  // for it. The difference is that state is small enough that inlining would
682
949
  // usually be free — that is a threshold worth adding once there is a real
683
950
  // file to size it against, not before.
684
- const state = listDir(path.join(agentDir, "..", "..", "state"), "../../state/");
951
+ const state = listDir(path.join(agentDir, "..", "..", "state"), "workspace/state/");
685
952
  if (state.length) {
686
953
  parts.push(
687
954
  `# State — what you carry between runs\n\n` +
@@ -719,16 +986,29 @@ function agentContext(
719
986
  // below. The tool is the unit of code, so its dependencies live in tool.md
720
987
  // and travel with it; the agent should not have to repeat them.
721
988
  const toolRuntimes: RuntimeSpec[] = [];
989
+ // The secrets a granted script tool's file lists (`secrets: [RESEND_API_KEY,
990
+ // EMAIL_FROM]`): granting the tool grants them, the way an API tool's
991
+ // referenced credentials always travelled with it. Before, only the
992
+ // agent's own `secrets:` counted, so an agent that granted desk_email and
993
+ // declared nothing failed with "RESEND_API_KEY is not set" (strata-desk,
994
+ // 1 Oct). `proxied` / `materialised` say how, not what, and add nothing.
995
+ const toolSecrets: string[] = [];
996
+ // Does any granted tool act outside the workspace (`outward: true`)? The
997
+ // retry policy reads it: see launchStep.
998
+ let outward = false;
722
999
  const grantOwnTool = (name: string) => {
723
1000
  const def = available[name];
724
1001
  if (!def) {
725
1002
  missingTools.push(name);
726
1003
  return;
727
1004
  }
1005
+ if (def.outward) outward = true;
728
1006
  if (def.kind === "http") apis.push(def.spec);
729
1007
  else if (def.kind === "script") {
730
1008
  scriptSpecs.push(...parseScripts([def.spec]));
731
- const rt = parseRuntime((def.spec as { runtime?: unknown }).runtime);
1009
+ const listed = (def.spec as { secrets?: unknown }).secrets;
1010
+ if (Array.isArray(listed)) toolSecrets.push(...listed.map(String).filter((n) => /^[A-Z][A-Z0-9_]*$/.test(n)));
1011
+ const rt = toolRuntime(def);
732
1012
  if (rt) toolRuntimes.push(rt);
733
1013
  }
734
1014
  else {
@@ -773,7 +1053,65 @@ function agentContext(
773
1053
  //
774
1054
  // Declared-and-absent is still an error; referenced-and-absent is the API
775
1055
  // tool's own missingSecrets, reported where the tool is described.
776
- const declared: string[] = Array.isArray(front.secrets) ? front.secrets.map(String) : [];
1056
+ // `web: {search: …, fetch: …}` name who answers; resolved here, early,
1057
+ // because three later things hang off the answer — the secret the step
1058
+ // must hold, the env the tool reads, and the egress grant for the host.
1059
+ // `web.search` carries the account engine's settings (engines, categories,
1060
+ // safesearch, …) beside the provider name, read off first the way
1061
+ // browse's are, so the provider resolver sees only its own part.
1062
+ const web = webConfig(front as Record<string, unknown>, workspaceFrontmatter(agentDir, tenant));
1063
+ const searchRead = readSearchSettings(web.raw.search);
1064
+ const searchChoice = resolveSearch(searchRead.rest, "search");
1065
+ if (searchRead.error) searchChoice.error = searchChoice.error ?? searchRead.error;
1066
+ const fetchChoice = resolveSearch(web.raw.fetch, "fetch");
1067
+ // A remote browser's key is materialised, not proxied: CDP is a websocket
1068
+ // and the wrapper opens the session itself, the way it seeds a cookie.
1069
+ // `web.browse` carries two things in one key: how the browser presents
1070
+ // itself (engine, user agent, device, locale, timezone) and, optionally,
1071
+ // which remote vendor renders the page. Settings are read off first so the
1072
+ // vendor resolver sees only the vendor part, and a block of settings alone
1073
+ // means the account's own browser with those defaults. Nearest wins: the
1074
+ // agent's block, else the workspace's, else the account's — the shape
1075
+ // `provider:` already uses.
1076
+ const browseRead = readBrowseSettings(web.raw.browse);
1077
+ const browseSettings = browseRead.settings;
1078
+ const browseChoice = resolveSearch(browseRead.rest, "browse");
1079
+ if (browseRead.error) browseChoice.error = browseChoice.error ?? browseRead.error;
1080
+ // What the vendor's own session is asked for, checked against what that
1081
+ // vendor offers; a problem is on the record and the session is not sent.
1082
+ const sessionProblems = browseChoice.error ? [] : browseSessionProblems(browseSettings, browseChoice.provider);
1083
+ const browseSession = browseChoice.shape === "direct" && browseChoice.provider && !sessionProblems.length ? browseSettings.vendor_session : undefined;
1084
+ // crawl, map, extract, answer, monitor: a provider each, or foldrun's own.
1085
+ // Same three consequences as a search API — the secret the step holds, the
1086
+ // env the web tool reads, the one host the key may reach.
1087
+ const actionChoices = NEWER_ACTIONS.map((action) => ({ action, ...resolveActionApi(action, web.raw[action]) }));
1088
+ const builtins = [
1089
+ ...(searchChoice.provider && searchChoice.shape !== "direct" && !searchChoice.error ? ["search=WebSearch"] : []),
1090
+ ...(fetchChoice.provider && fetchChoice.shape !== "direct" && !fetchChoice.error ? ["fetch=WebFetch"] : []),
1091
+ ];
1092
+ const webBuiltin: Record<string, string> = builtins.length ? { FOLDRUN_WEB_BUILTIN: builtins.join(",") } : {};
1093
+ // A direct search API is paid for with the customer's own key, stored
1094
+ // under a fixed vault name (EXA_API_KEY, BRAVE_SEARCH_API_KEY, …). Naming
1095
+ // the API in frontmatter is declaring that secret — nobody should have to
1096
+ // write it twice, and a run that forgot the second line would search
1097
+ // nothing and say so only at the provider.
1098
+ const declared: string[] = [
1099
+ ...(Array.isArray(front.secrets) ? front.secrets.map(String) : []),
1100
+ ...toolSecrets,
1101
+ ...actionChoices.flatMap((c) => (c.provider ? [c.secret] : [])),
1102
+ // Your own proxy for a vendor's session: its URL is a secret, declared by
1103
+ // naming it, like a provider's key.
1104
+ ...(browseSession && typeof browseSession.proxy === "object" && browseSession.proxy.own ? [browseSession.proxy.own] : []),
1105
+ ...[searchChoice, fetchChoice, browseChoice].flatMap((c) => {
1106
+ if (c.shape !== "direct" || !c.secret) return [];
1107
+ // Jina's reader answers without a key. Declaring an absent optional
1108
+ // key would fail the step for a credential it never needed.
1109
+ if (c.secretOptional && !getSecret(tenant, c.secret, workspace)) return [];
1110
+ return [c.secret];
1111
+ }),
1112
+ ];
1113
+ // One name once, however many places asked for it.
1114
+ declared.splice(0, declared.length, ...new Set(declared));
777
1115
  const referenced = [...new Set(apis.flatMap(secretsUsedByApi))];
778
1116
  const {
779
1117
  env: secretEnv,
@@ -888,24 +1226,17 @@ function agentContext(
888
1226
  // Runtime: an agent's own `runtime:` block, else its workspace's. Built once
889
1227
  // per declaration and cached, so scripts get their dependencies without
890
1228
  // polluting the host.
891
- let runtimeSpec = parseRuntime(front.runtime);
892
- if (!runtimeSpec) {
893
- // AGENTS.md is the Linux Foundation standard name (60k+ repos, same
894
- // nearest-wins cascade we already use); project.md stays accepted so
895
- // nothing that already exists breaks.
896
- runtimeSpec = parseRuntime(workspaceFrontmatter(agentDir, tenant).runtime);
897
- }
898
1229
  // Plus whatever the tools this agent granted declared for themselves.
899
- runtimeSpec = mergeRuntimes(runtimeSpec, ...toolRuntimes);
1230
+ let runtimeSpec = mergeRuntimes(ownRuntime(front, agentDir, tenant), ...toolRuntimes);
900
1231
  // Execution: a container per run when Docker is available (real isolation),
901
1232
  // otherwise the host venv path so local development still works.
902
1233
  const executor = chooseExecutor();
903
1234
  const runtimeLog: string[] = [];
904
1235
  let runtimeError: string | null = null;
905
1236
  let exec: ExecutionContext | null = null;
906
- let runtime = { interpreters: {} as Record<string, string>, env: {} as Record<string, string>, log: [] as string[], error: null as string | null };
1237
+ let runtime: PreparedRuntime = { interpreters: {}, env: {}, log: [], error: null };
907
1238
 
908
- if (executor === "docker") {
1239
+ if (executor === "docker" && !grantsOnly) {
909
1240
  const image = ensureImage(runtimeSpec);
910
1241
  runtimeLog.push(...image.log);
911
1242
  runtimeError = image.error;
@@ -922,23 +1253,103 @@ function agentContext(
922
1253
  network: parseApis(front.apis).length > 0 || ownToolNames(front).length > 0,
923
1254
  };
924
1255
  }
1256
+ } else if (isolatedRun() || grantsOnly) {
1257
+ // (A sub-agent's grants are all that is asked of it: it runs inside its
1258
+ // parent's step and runtime.)
1259
+ // The sandbox builds it (run-container.ts, beside pip's network and the
1260
+ // mounted cache); building it here as well was pure waste, and on a
1261
+ // worker image without python it was worse than waste — it logged
1262
+ // "python3 is not available on this host" as an error on every Python
1263
+ // step, above a sandbox build that had succeeded, and sent every reader
1264
+ // of the trace after the wrong machine.
925
1265
  } else {
926
1266
  runtime = prepareRuntime(tenant, runtimeSpec);
927
1267
  runtimeLog.push(...runtime.log);
928
1268
  runtimeError = runtime.error;
929
1269
  }
930
1270
 
931
- // What day it is, in the workspace's own timezone. Not a secret, so it
932
- // travels beside secretEnv rather than inside it — a value in there is
933
- // redacted from every log line that quotes it.
934
- const timezone = resolveTimezone(workspaceFrontmatter(agentDir, tenant));
1271
+ // What day it is, in the calendar this step works to — the agent's own
1272
+ // `timezone:`, else its flow's, else the workspace's, else the account's.
1273
+ // Not a secret, so it travels beside secretEnv rather than inside it — a
1274
+ // value in there is redacted from every log line that quotes it.
1275
+ const clock = agentClock(agentDir, tenant, front, flowTimezone);
1276
+ const timezone = clock.timezone;
935
1277
  const clockEnv = {
936
- TZ: timezone,
1278
+ // The language the agent works in — what the search asks in, the locale
1279
+ // the browser reports, the Accept-Language a fetch sends. Not a secret.
1280
+ FOLDRUN_LANGUAGE: language.language,
1281
+ // The country and what follows from it — web search's gl, a script's
1282
+ // currency. Unset where nobody said, so nothing is invented.
1283
+ ...(region.region ? { FOLDRUN_REGION: region.region } : {}),
1284
+ ...(locale.currency ? { FOLDRUN_CURRENCY: locale.currency } : {}),
1285
+ FOLDRUN_UNITS: locale.units,
1286
+ FOLDRUN_CALENDAR: locale.calendar,
1287
+ ...(locale.dateLocal ? { FOLDRUN_DATE_LOCAL: locale.dateLocal } : {}),
1288
+ FOLDRUN_RTL: locale.direction === "rtl" ? "1" : "0",
1289
+ // TZ is the form the sandbox's shell and Node both read; for a fixed
1290
+ // offset that is not the same string Intl was given (see clock.ts).
1291
+ TZ: clock.tz,
937
1292
  FOLDRUN_DATE: localDate(timezone),
938
1293
  // Where the account's own search engine answers, when the install runs
939
1294
  // one — the `websearch` library tool reads it. Not a secret; rides with
940
1295
  // the clock so it reaches scripts, the sandbox and verify alike.
941
1296
  ...(process.env.FOLDRUN_SEARCH_URL ? { FOLDRUN_SEARCH_URL: process.env.FOLDRUN_SEARCH_URL } : {}),
1297
+ // Whose index our web search asks when the agent named a search API.
1298
+ // Unset means the account's own engine. The name, never the key.
1299
+ ...(searchChoice.shape === "direct" && searchChoice.provider
1300
+ ? { FOLDRUN_WEB_SEARCH_VIA: searchChoice.provider, FOLDRUN_WEB_SEARCH_SECRET: searchChoice.secret ?? "" }
1301
+ : {}),
1302
+ // The `web.search` block's SearXNG settings, as defaults the tool reads.
1303
+ ...searchSettingsEnv(searchRead.settings),
1304
+ ...(fetchChoice.shape === "direct" && fetchChoice.provider
1305
+ ? { FOLDRUN_WEB_FETCH_VIA: fetchChoice.provider, FOLDRUN_WEB_FETCH_SECRET: fetchChoice.secret ?? "" }
1306
+ : {}),
1307
+ ...Object.fromEntries(actionChoices.flatMap((c) => (c.provider
1308
+ ? [[`FOLDRUN_WEB_${c.action.toUpperCase()}_VIA`, c.provider], [`FOLDRUN_WEB_${c.action.toUpperCase()}_SECRET`, c.secret]]
1309
+ : []))),
1310
+ // Which actions the web tool may take (`web: {actions: [...]}`); unset is all.
1311
+ ...(web.actions ? { FOLDRUN_WEB_ACTIONS: web.actions.join(",") } : {}),
1312
+ // A model provider's own search or fetch, which the web tool hands over to.
1313
+ ...webBuiltin,
1314
+ // A remote browser: the wrapper connects over CDP to this vendor with
1315
+ // the named key, and the platform leaves the account's pod alone.
1316
+ ...(browseChoice.shape === "direct" && browseChoice.provider
1317
+ ? { FOLDRUN_BROWSER_VENDOR: browseChoice.provider, FOLDRUN_BROWSER_SECRET: browseChoice.secret ?? "" }
1318
+ : {}),
1319
+ // The vendor session's settings (`web.browse.vendor_session:`), as JSON
1320
+ // the tool maps onto that vendor's own option names. Under both names
1321
+ // for one release: a web tool from before the rename reads the old one.
1322
+ ...(browseSession ? { FOLDRUN_BROWSER_VENDOR_SESSION: JSON.stringify(browseSession), FOLDRUN_BROWSER_SESSION: JSON.stringify(browseSession) } : {}),
1323
+ // The `web.browse` block, as defaults the tool reads when the call did
1324
+ // not say otherwise. Env rather than arguments because the tool is a
1325
+ // script the model calls: a default nobody typed should not have to be
1326
+ // typed by the model either.
1327
+ ...(browseSettings.engine ? { FOLDRUN_BROWSER_ENGINE: browseSettings.engine } : {}),
1328
+ ...(browseSettings.user_agent ? { FOLDRUN_BROWSER_USER_AGENT: browseSettings.user_agent } : {}),
1329
+ ...(browseSettings.device ? { FOLDRUN_BROWSER_DEVICE: browseSettings.device } : {}),
1330
+ ...(browseSettings.locale ? { FOLDRUN_BROWSER_LOCALE: browseSettings.locale } : {}),
1331
+ ...(browseSettings.timezone ? { FOLDRUN_BROWSER_TIMEZONE: browseSettings.timezone } : {}),
1332
+ ...(browseSettings.cookies ? { FOLDRUN_BROWSER_COOKIES: browseSettings.cookies } : {}),
1333
+ ...(browseSettings.cookie_domain ? { FOLDRUN_BROWSER_COOKIE_DOMAIN: browseSettings.cookie_domain } : {}),
1334
+ ...(browseSettings.storage ? { FOLDRUN_BROWSER_STORAGE: browseSettings.storage } : {}),
1335
+ ...(browseSettings.storage_origin ? { FOLDRUN_BROWSER_STORAGE_ORIGIN: browseSettings.storage_origin } : {}),
1336
+ // The named identities, as one JSON map the tool picks from by name.
1337
+ ...(browseSettings.identities ? { FOLDRUN_BROWSER_IDENTITIES: JSON.stringify(browseSettings.identities) } : {}),
1338
+ // Only the non-default travels: a window, on a virtual screen if need be.
1339
+ ...(browseSettings.headless === false ? { FOLDRUN_BROWSER_HEADLESS: "0" } : {}),
1340
+ ...(browseSettings.version ? { FOLDRUN_BROWSER_VERSION: browseSettings.version } : {}),
1341
+ ...(browseSettings.live ? { FOLDRUN_BROWSER_LIVE: "1" } : {}),
1342
+ // What the agent may do, from its file: a lock the call cannot widen.
1343
+ ...(browseSettings.allowed_domains ? { FOLDRUN_BROWSER_ALLOWED_DOMAINS: browseSettings.allowed_domains.join(",") } : {}),
1344
+ ...(browseSettings.deny ? { FOLDRUN_BROWSER_DENY: browseSettings.deny.join(",") } : {}),
1345
+ ...(browseSettings.boundaries ? { FOLDRUN_BROWSER_BOUNDARIES: "1" } : {}),
1346
+ ...(browseSettings.init ? { FOLDRUN_BROWSER_INIT: browseSettings.init.join(",") } : {}),
1347
+ ...(browseSettings.extensions ? { FOLDRUN_BROWSER_EXTENSIONS: browseSettings.extensions.join(",") } : {}),
1348
+ ...(browseSettings.webgpu ? { FOLDRUN_BROWSER_WEBGPU: "1" } : {}),
1349
+ ...(browseSettings.ignore_https_errors ? { FOLDRUN_BROWSER_IGNORE_HTTPS_ERRORS: "1" } : {}),
1350
+ ...(browseSettings.state_key ? { FOLDRUN_BROWSER_STATE_KEY: browseSettings.state_key } : {}),
1351
+ ...(browseSettings.video ? { FOLDRUN_BROWSER_VIDEO: "1" } : {}),
1352
+ ...(browseSettings.live_view === false ? { FOLDRUN_BROWSER_LIVE_VIEW: "0" } : {}),
942
1353
  };
943
1354
 
944
1355
  // Scripts declared as tools — callable by name, no bash required.
@@ -983,7 +1394,7 @@ function agentContext(
983
1394
  parts.push(
984
1395
  "Write deliverables for people to workspace/storage/ (kept and downloadable); write working text for later steps to outputs/; " +
985
1396
  "what the next run needs goes to workspace/state/. Paths are relative to your agent directory, or start with workspace/ " +
986
- "for the workspace root (../../storage/ and workspace/storage/ are the same place). There is no /tmp for you: a " +
1397
+ "for the workspace root (workspace/storage/ and the older ../../storage/ are the same place). There is no /tmp for you: a " +
987
1398
  "directory a tool made outside the workspace, such as a checkout, is reached only through that tool.",
988
1399
  );
989
1400
 
@@ -1010,9 +1421,37 @@ function agentContext(
1010
1421
  const disabled: string[] = [];
1011
1422
  const unknownTools: string[] = [];
1012
1423
  const shadowed: string[] = [];
1424
+ // `web: {search: …, fetch: …}` name who answers. Unset means ours — the
1425
+ // account's own engine, and our own fetch, both inside the run's sandbox
1426
+ // and both on the run record. A model provider's name adds the SDK's
1427
+ // server-side tool beside ours, which the run's endpoint executes: point
1428
+ // the run at z.ai and z.ai's index answers the search. The web tool hands
1429
+ // the action over to it (FOLDRUN_WEB_BUILTIN).
1430
+ const providerWebTools: string[] = [];
1431
+ if (browseChoice.error) providerWarnings.push(browseChoice.error);
1432
+ providerWarnings.push(
1433
+ ...sessionProblems,
1434
+ ...web.problems,
1435
+ ...actionChoices.flatMap((c) => (c.error ? [c.error] : [])),
1436
+ );
1437
+ for (const [choice, builtin] of [
1438
+ [searchChoice, "WebSearch"],
1439
+ [fetchChoice, "WebFetch"],
1440
+ ] as const) {
1441
+ if (choice.error) {
1442
+ providerWarnings.push(choice.error);
1443
+ } else if (choice.provider && choice.shape !== "direct") {
1444
+ // A provider's server-side tool replaces ours. A direct search API
1445
+ // does not: our tool stays, and reads FOLDRUN_WEB_SEARCH_VIA to know
1446
+ // whose index to ask — through the egress proxy, with the customer's
1447
+ // own key, on the run record.
1448
+ providerWebTools.push(builtin);
1449
+ }
1450
+ }
1013
1451
  let wantSearch = false;
1014
1452
  let wantHistory = false;
1015
1453
  let wantDesks = false;
1454
+ let wantAsk = false;
1016
1455
 
1017
1456
  for (const ref of toolRefs(front)) {
1018
1457
  const toolName = ref.name;
@@ -1020,6 +1459,14 @@ function agentContext(
1020
1459
  disabled.push(toolName);
1021
1460
  continue;
1022
1461
  }
1462
+ // `web` is our own tool, granted with the rest of them above. A model
1463
+ // provider answering search or fetch on its own side (`web: {search:
1464
+ // zai}`) adds that provider's tool beside it; the web tool's search then
1465
+ // names it (FOLDRUN_WEB_BUILTIN) rather than asking our engine.
1466
+ if (toolName === "web" && !ref.linked) {
1467
+ allowed.push(...providerWebTools);
1468
+ continue;
1469
+ }
1023
1470
 
1024
1471
  // Resolution order: built-in group, exact SDK name, then your own tools.
1025
1472
  // Built-ins win a bare name, because their names are a small fixed set —
@@ -1031,11 +1478,12 @@ function agentContext(
1031
1478
  } else if (TOOL_MAP[toolName]) {
1032
1479
  allowed.push(...TOOL_MAP[toolName]);
1033
1480
  if (available[toolName]) shadowed.push(toolName);
1034
- } else if (toolName === "search" || toolName === "history" || toolName === "desks") {
1481
+ } else if (toolName === "search" || toolName === "history" || toolName === "desks" || toolName === "ask") {
1035
1482
  // The platform's own groups, served in-process — built below, once
1036
1483
  // the roots and the records are known.
1037
1484
  if (toolName === "search") wantSearch = true;
1038
1485
  else if (toolName === "history") wantHistory = true;
1486
+ else if (toolName === "ask") wantAsk = true;
1039
1487
  else wantDesks = true;
1040
1488
  if (available[toolName]) shadowed.push(toolName);
1041
1489
  } else if (BUILTIN_TOOLS.has(toolName)) {
@@ -1060,8 +1508,8 @@ function agentContext(
1060
1508
  { label: "memory/", dir: path.join(agentDir, "memory") },
1061
1509
  { label: "../../knowledge/", dir: path.join(wsRoot, "knowledge") },
1062
1510
  { label: "../../memory/", dir: path.join(wsRoot, "memory") },
1063
- { label: "../../state/", dir: path.join(wsRoot, "state") },
1064
- { label: `../../${STORAGE_DIR}/`, dir: path.join(wsRoot, STORAGE_DIR) },
1511
+ { label: "workspace/state/", dir: path.join(wsRoot, "state") },
1512
+ { label: `workspace/${STORAGE_DIR}/`, dir: path.join(wsRoot, STORAGE_DIR) },
1065
1513
  { label: "outputs/", dir: path.join(agentDir, "outputs") },
1066
1514
  { label: "[library] knowledge/", dir: libraryDir(tenant, "knowledge") },
1067
1515
  { label: "[library] memory/", dir: libraryDir(tenant, "memory") },
@@ -1083,6 +1531,14 @@ function agentContext(
1083
1531
  const historyTools = wantHistory ? buildHistoryTools(historyDigest) : { server: null, toolNames: [], promptLines: [] };
1084
1532
  const deskTools = wantDesks ? buildDeskTools(deskDigest) : { server: null, toolNames: [], promptLines: [] };
1085
1533
  allowed.push(...searchTools.toolNames, ...historyTools.toolNames, ...deskTools.toolNames);
1534
+ // tools: [ask] — ask_person. Its server is built where the step runs (it
1535
+ // needs the step's emit, and in a pod the proxy's address); here only the
1536
+ // grant, the wait and the prompt's line.
1537
+ const ask = wantAsk ? { timeoutSec: askTimeoutSec(front) } : null;
1538
+ if (ask) {
1539
+ allowed.push(ASK_TOOL);
1540
+ parts.push(`# Asking a person\n\n${askPromptLine(ask.timeoutSec)}`);
1541
+ }
1086
1542
  if (searchTools.promptLines.length || historyTools.promptLines.length || deskTools.promptLines.length) {
1087
1543
  parts.push(`# Finding things\n\n${[...searchTools.promptLines, ...historyTools.promptLines, ...deskTools.promptLines].join("\n")}`);
1088
1544
  }
@@ -1103,12 +1559,16 @@ function agentContext(
1103
1559
  return {
1104
1560
  front,
1105
1561
  clockEnv,
1562
+ /** The zone, where it came from, and anything unreadable on the way —
1563
+ * reported by the step, not here, so it lands in the run's trace. */
1564
+ clock,
1106
1565
  systemPrompt: parts.join("\n\n"),
1107
1566
  allowed: finalAllowed,
1108
1567
  disabled,
1109
1568
  unknownTools,
1110
1569
  shadowed,
1111
1570
  legacyUse,
1571
+ retiredTools: retiredToolNames(front),
1112
1572
  mcpServers,
1113
1573
  mcpNames,
1114
1574
  apiTools,
@@ -1118,19 +1578,21 @@ function agentContext(
1118
1578
  // than using the servers built here.
1119
1579
  apiSpecs: apis,
1120
1580
  scriptSpecs,
1581
+ outward: outward || scriptSpecs.some((sc) => sc.outward),
1121
1582
  searchRoots,
1122
1583
  historyDigest,
1123
1584
  deskDigest,
1124
1585
  searchTools,
1125
1586
  historyTools,
1126
1587
  deskTools,
1588
+ ask,
1127
1589
  // `spec` is the MERGED declaration — the agent's own block or its
1128
1590
  // workspace's, plus every granted tool's. The isolated path builds its
1129
1591
  // runtime from this; passing the agent's frontmatter alone silently drops
1130
1592
  // what the tools declared, and a tool whose program needs a package it
1131
1593
  // asked for by name fails inside the container with "cannot find" while
1132
1594
  // the host path, which uses this same merged spec, works fine.
1133
- runtime: { log: runtimeLog, error: runtimeError, executor, spec: runtimeSpec },
1595
+ runtime: { log: runtimeLog, error: runtimeError, executor, spec: runtimeSpec, dispose: runtime.dispose },
1134
1596
  secretEnv: { ...runtime.env, ...secretEnv },
1135
1597
  formatWarning: checkFormatVersion(workspaceFrontmatter(agentDir, tenant).foldrun_version).warning,
1136
1598
  providerEnv,
@@ -1138,8 +1600,19 @@ function agentContext(
1138
1600
  providerSecrets,
1139
1601
  providerWarnings,
1140
1602
  fallbackEnv,
1141
- translator,
1142
- fallbackTranslator,
1603
+ region,
1604
+ // Who answers web search and fetch, resolved once up top. The step
1605
+ // needs them again to grant a direct API's key to its host.
1606
+ searchChoice,
1607
+ fetchChoice,
1608
+ actionChoices,
1609
+ web,
1610
+ language,
1611
+ // Stamped with the step's own zone: when one of these refuses, the
1612
+ // reset time it reports is told in the calendar the step works to,
1613
+ // not in the pod's UTC.
1614
+ translator: translator && { ...translator, timezone },
1615
+ fallbackTranslator: fallbackTranslator && { ...fallbackTranslator, timezone },
1143
1616
  apiWarnings,
1144
1617
  secretScopes,
1145
1618
  missingTools,
@@ -1169,34 +1642,136 @@ function agentContext(
1169
1642
  * Never fails a step: a link is bookkeeping, and a run that did its work must
1170
1643
  * not be marked failed because a URL could not be minted.
1171
1644
  */
1172
- function publishPublicDir(
1645
+ async function publishPublicDir(
1173
1646
  tenant: string,
1174
1647
  workspace: string,
1175
1648
  push: (type: "info" | "error", text: string) => void,
1176
- ) {
1649
+ ): Promise<void> {
1177
1650
  try {
1178
- const { added } = platform.syncPublicShares(tenant, workspace);
1651
+ const { added } = await platform.syncPublicShares(tenant, workspace);
1179
1652
  if (added.length) push("info", `shared: ${added.join(", ")}`);
1180
1653
  } catch (err) {
1181
1654
  push("error", `shares: ${err instanceof Error ? err.message : String(err)}`);
1182
1655
  }
1183
1656
  }
1184
1657
 
1185
- function repriced(
1658
+ export function repriced(
1186
1659
  catalog: Catalog | null,
1187
1660
  model: string,
1188
1661
  usage: { inputTokens: number; outputTokens: number } | null,
1189
1662
  sdkCost: number | null,
1190
1663
  push: (type: "info" | "error", text: string) => void,
1664
+ /** The step's turns each priced at its own model — set when a sub-agent
1665
+ * ran on a different model (ExecOutcome.turnsCostUsd). The lump `usage`
1666
+ * priced at the step's model would bill those turns at the wrong rate. */
1667
+ turnsCostUsd?: number | null,
1191
1668
  ): number | null {
1192
1669
  const known = usage ? catalogCost(catalog, model, usage) : null;
1193
1670
  if (known === null) return sdkCost;
1671
+ if (typeof turnsCostUsd === "number") {
1672
+ push("info", `cost from each model turn at its own model's rate: $${turnsCostUsd.toFixed(6)} (sub-agents ran on other models)`);
1673
+ return turnsCostUsd;
1674
+ }
1194
1675
  if (sdkCost !== null && Math.abs(known - sdkCost) > 0.000001) {
1195
1676
  push("info", `cost repriced from the gateway's catalogue: $${known.toFixed(6)} (sdk said $${sdkCost.toFixed(6)})`);
1196
1677
  }
1197
1678
  return known;
1198
1679
  }
1199
1680
 
1681
+ /**
1682
+ * Write one attempt onto the step's history and re-total the step.
1683
+ *
1684
+ * runStep reports the attempt it just ran — its cost, tokens and sandbox
1685
+ * seconds — by assigning the step's fields. Those become this attempt's row;
1686
+ * the step's fields then become the sum of every row, which is what the
1687
+ * run's spend, the flow's budget: and the platform's bill read. A retried
1688
+ * step used to carry only its last attempt's cost, so a `retry: 2` step
1689
+ * could spend three shares while reporting one.
1690
+ */
1691
+ export function recordAttempt(
1692
+ step: StepRecord,
1693
+ /** The step's attempt (`attempts`) these rows belong to. */
1694
+ attempt: number,
1695
+ status: "completed" | "failed",
1696
+ startedAt: string,
1697
+ eventsBefore: number,
1698
+ /** The attempt re-attached to a sandbox a previous driver left: same
1699
+ * attempt, so its row is replaced rather than repeated. */
1700
+ reattach = false,
1701
+ ) {
1702
+ const tries = step.tries ?? [];
1703
+ const lastError = step.events
1704
+ .slice(eventsBefore)
1705
+ .filter((e) => e.type === "error")
1706
+ .at(-1)?.text;
1707
+ // Re-attached: the same attempt, so its rows — the last one, and the lost
1708
+ // slim go before it — are replaced, not repeated.
1709
+ const kept = [...tries];
1710
+ const attemptOf = (t: StepAttempt) => t.attempt ?? t.n;
1711
+ if (reattach && kept.length && attemptOf(kept.at(-1)!) === attempt) {
1712
+ kept.pop();
1713
+ if (kept.length && attemptOf(kept.at(-1)!) === attempt && kept.at(-1)!.status === "lost") kept.pop();
1714
+ }
1715
+ // Tries are numbered in the order they ran: a full re-run after a lost
1716
+ // slim go is the next try, not the same number again (live, 2026-10-01,
1717
+ // both rows read n: 1). `attempt` says which attempt a row belongs to
1718
+ // when its number no longer does.
1719
+ let next = (kept.at(-1)?.n ?? 0) + 1;
1720
+ const numbered = () => {
1721
+ const n = next++;
1722
+ return n === attempt ? { n } : { n, attempt };
1723
+ };
1724
+ // A slim go the browser pod was lost under, before this attempt re-ran
1725
+ // on the full image: its own row, first (runStep set it aside).
1726
+ const lost: StepAttempt[] = step.podLostTry ? [{ ...numbered(), ...step.podLostTry, startedAt }] : [];
1727
+ const row: StepAttempt = {
1728
+ ...numbered(),
1729
+ status,
1730
+ costUsd: step.costUsd,
1731
+ tokens: step.tokens ?? null,
1732
+ computeSecs: step.computeSecs ?? null,
1733
+ startedAt: step.podLostTry?.finishedAt ?? startedAt,
1734
+ finishedAt: step.finishedAt ?? new Date().toISOString(),
1735
+ ...(status === "failed" && lastError ? { error: lastError.slice(0, 500) } : {}),
1736
+ ...(step.image?.variant ? { image: step.image.variant } : {}),
1737
+ ...(step.browserPod ? { browserPod: browserPodLine(step.browserPod) } : {}),
1738
+ };
1739
+ delete step.podLostTry;
1740
+ step.tries = [...kept, ...lost, row];
1741
+ const totals = sumAttempts(step.tries);
1742
+ step.costUsd = totals.costUsd;
1743
+ step.tokens = totals.tokens;
1744
+ step.computeSecs = totals.computeSecs;
1745
+ }
1746
+
1747
+ /** A question an agent asked mid-step goes where approvals go (the
1748
+ * workspace's notify:, `awaiting-approval` events), with the way to answer. */
1749
+ async function notifyQuestion(tenant: string, workspace: string, runId: string, agent: string, question: string, options?: string[]): Promise<void> {
1750
+ const origin = (process.env.FOLDRUN_PUBLIC_URL ?? "").replace(/\/+$/, "");
1751
+ const where = origin ? `${origin}/dashboard/${encodeURIComponent(workspace)}/runs/${encodeURIComponent(runId)}${tenant !== "default" ? `?tenant=${encodeURIComponent(tenant)}` : ""}` : `run ${runId}`;
1752
+ await sendPlainNotification(tenant, workspace, {
1753
+ event: "awaiting-approval",
1754
+ headline: `${agent} is asking you something`,
1755
+ detail: `${question}${options?.length ? `\n\nOptions: ${options.join(" · ")}` : ""}\n\nAnswer it on ${where}, or \`foldrun answer ${runId} "…"\`.`,
1756
+ runId,
1757
+ }).catch(() => false);
1758
+ }
1759
+
1760
+ /** The local stand-in for a person: the terminal, when there is one. */
1761
+ async function terminalAsk(question: string, options?: string[]): Promise<string | null> {
1762
+ if (!process.stdin.isTTY || !process.stdout.isTTY) return null;
1763
+ const { createInterface } = await import("node:readline/promises");
1764
+ const rl = createInterface({ input: process.stdin, output: process.stdout });
1765
+ try {
1766
+ const list = options?.length ? `\n${options.map((o, i) => ` ${i + 1}. ${o}`).join("\n")}\n` : "";
1767
+ const raw = (await rl.question(`\n the agent asks: ${question}${list}\n your answer: `)).trim();
1768
+ const n = Number(raw);
1769
+ return options?.length && Number.isInteger(n) && n >= 1 && n <= options.length ? options[n - 1] : raw || null;
1770
+ } finally {
1771
+ rl.close();
1772
+ }
1773
+ }
1774
+
1200
1775
  async function runStep(
1201
1776
  agentDir: string,
1202
1777
  tenant: string,
@@ -1214,6 +1789,22 @@ async function runStep(
1214
1789
  * remaining `budget:`, shared across the steps launched with it. Null
1215
1790
  * means the flow set no budget. */
1216
1791
  stepBudgetUsd: number | null = null,
1792
+ /** Which line set that ceiling, for the error that names it. */
1793
+ stepBudgetNote?: string,
1794
+ /** The run is a test run — see test-mode.ts. Read off the run record by
1795
+ * the caller, never from anything the sandbox could have written. */
1796
+ testRun = false,
1797
+ /** Has a person asked for the run to stop? Read on a clock by the
1798
+ * in-process model loop, so a stop lands mid-step and not at the next
1799
+ * group. The isolated path needs none: stopRun destroys the sandbox. */
1800
+ stopRequested?: () => boolean,
1801
+ /** The flow's `timezone:`, for the clock cascade. Undefined for a step
1802
+ * run outside any flow, which still gets agent → workspace → account. */
1803
+ flowTimezone?: string | null,
1804
+ /** The step's index in run.steps, carried on its egress lease so a
1805
+ * person's message to this step reaches this step's own inbox and not a
1806
+ * sibling's running in the same group. Undefined outside a flow. */
1807
+ stepIndex?: number,
1217
1808
  ) {
1218
1809
  // Secret values are injected into scripts as environment variables and
1219
1810
  // substituted into API headers, so a model that reads one back — from a
@@ -1232,14 +1823,44 @@ async function runStep(
1232
1823
  // cover a script's own stdout.
1233
1824
  let redactions: [string, string][] = [];
1234
1825
  let fileDir: string | null = null;
1826
+ let disposeRuntime: (() => void) | undefined;
1827
+ // Can this step's tools act outside the workspace? Set once its context is
1828
+ // read; the verdict at the end of this function reads it.
1829
+ let canActOutward = false;
1830
+ const eventsAtStart = step.events.length;
1235
1831
  const redact = (text: string) => {
1236
1832
  let out = text;
1237
1833
  for (const [value, name] of redactions) out = out.split(value).join(`[redacted:${name}]`);
1238
1834
  return out;
1239
1835
  };
1240
1836
 
1837
+ // The reply, the conclusion and an `output: json` value land on the same
1838
+ // record the events do, and a model that quotes a script's stdout puts
1839
+ // the credential in its answer rather than in a tool event. Same scrub,
1840
+ // same place. The JSON value goes through its text form: a secret inside
1841
+ // a nested field is still a secret, and the key name survives either way.
1842
+ const redactText = <T extends string | null | undefined>(text: T): T =>
1843
+ (typeof text === "string" ? redact(text) : text) as T;
1844
+ const redactData = (value: unknown): unknown => {
1845
+ if (value === undefined || redactions.length === 0) return value;
1846
+ try {
1847
+ return JSON.parse(redact(JSON.stringify(value)));
1848
+ } catch {
1849
+ return value;
1850
+ }
1851
+ };
1852
+
1241
1853
  const push = (type: StepRecord["events"][number]["type"], text: string, extra?: EventExtra) => {
1242
- step.events.push({ t: new Date().toISOString(), type, text: redact(text), ...extra });
1854
+ const t = new Date().toISOString();
1855
+ step.events.push({ t, type, text: redact(text), ...extra });
1856
+ // A person in the loop: the question, its answer, a message delivered —
1857
+ // onto the step, so the run page, the API and history show them.
1858
+ if (extra?.operator) {
1859
+ applyOperatorEvent(step, extra.operator, t);
1860
+ if (extra.operator.kind === "asked" && runId && !testRun) {
1861
+ void notifyQuestion(tenant, path.basename(path.resolve(agentDir, "..", "..")), runId, step.agent, extra.operator.question, extra.operator.options);
1862
+ }
1863
+ }
1243
1864
  save();
1244
1865
  };
1245
1866
 
@@ -1259,7 +1880,9 @@ async function runStep(
1259
1880
  ? fs.readFileSync(stubFile, "utf8").split(/\n---\n/)
1260
1881
  : [`stub result from ${step.agent}`];
1261
1882
  const call = step.events.filter((e) => e.text.startsWith("stub call")).length;
1262
- push("info", `stub call ${call + 1}`);
1883
+ // The ceiling it was given rides on the line, so the budget arithmetic
1884
+ // around fan-out is testable without a model to stop.
1885
+ push("info", `stub call ${call + 1}${stepBudgetUsd !== null ? ` (ceiling $${stepBudgetUsd.toFixed(2)})` : ""}`);
1263
1886
  step.status = "completed";
1264
1887
  // A stub answer may open with `cost: 0.25` — a pretend price, so the
1265
1888
  // parts of orchestration that watch money (budget:) are testable too.
@@ -1277,8 +1900,15 @@ async function runStep(
1277
1900
  // the failure of a step that returned no JSON are testable at no cost.
1278
1901
  if (step.output === "json") {
1279
1902
  const extracted = extractJson(step.result);
1280
- if (extracted.ok) step.data = extracted.value;
1281
- else {
1903
+ if (extracted.ok) {
1904
+ step.data = extracted.value;
1905
+ const schema = step.schema ?? (step.schemaPath ? readSchemaFile(path.resolve(agentDir, "..", ".."), step.schemaPath) : null);
1906
+ const errors = schema && looksLikeSchema(schema) ? validateSchema(step.data, schema) : [];
1907
+ if (errors.length) {
1908
+ push("error", `schema: the value does not fit — ${describeSchemaErrors(errors)}`);
1909
+ step.status = "failed";
1910
+ }
1911
+ } else {
1282
1912
  push("error", `output: json — ${extracted.reason}`);
1283
1913
  step.status = "failed";
1284
1914
  }
@@ -1295,14 +1925,19 @@ async function runStep(
1295
1925
  // step: the generic HTTP tool still works without the typed ones.
1296
1926
  for (const w of await prefetchOpenApi(tenant, openApiSources(agentDir, tenant))) push("error", `openapi: ${w}`);
1297
1927
  const {
1298
- front, clockEnv, systemPrompt, allowed, disabled, apiTools, scriptTools,
1928
+ front, clockEnv, clock, systemPrompt, allowed, disabled, apiTools, scriptTools,
1299
1929
  secretEnv, secretScopes, missingSecrets, missingTools, runtime,
1300
- unknownTools, shadowed, legacyUse, mcpServers, mcpNames,
1930
+ unknownTools, shadowed, legacyUse, retiredTools, mcpServers, mcpNames,
1301
1931
  apiSpecs, scriptSpecs, brokenTools, size: agentSize,
1302
1932
  providerEnv, providerLabel, providerSecrets, providerWarnings, formatWarning,
1933
+ searchChoice, fetchChoice, actionChoices, web, language, region,
1303
1934
  fallbackEnv, apiWarnings, searchRoots, historyDigest, deskDigest, searchTools, historyTools, deskTools,
1935
+ ask,
1304
1936
  translator, fallbackTranslator,
1305
- } = agentContext(agentDir, tenant, tags, { runId, agent: step.agent });
1937
+ outward,
1938
+ } = agentContext(agentDir, tenant, tags, { runId, agent: step.agent }, flowTimezone);
1939
+ disposeRuntime = runtime.dispose;
1940
+ canActOutward = outward;
1306
1941
  // A retry that moved the step up a class (an evicted or OOM-killed
1307
1942
  // attempt) wins over the agent's own `size:` for the attempts after.
1308
1943
  const size = step.sizeUp ?? agentSize;
@@ -1313,7 +1948,7 @@ async function runStep(
1313
1948
  // what crosses into containers, headers and env files is the token. A
1314
1949
  // failed refresh fails the step naming the secret and the provider's
1315
1950
  // reason, which beats a 401 three layers later.
1316
- const liveSecrets = await materializeSecrets(secretEnv);
1951
+ const liveSecrets = await materializeSecrets(secretEnv, { tenant, workspace: path.basename(workspaceRootOf(agentDir)) });
1317
1952
  // @file values (SSH keys, certs) still carry their content at this point.
1318
1953
  // The in-process branch materialises them to 0600 paths on this host
1319
1954
  // below; the container branch passes the content across and the driver
@@ -1328,15 +1963,24 @@ async function runStep(
1328
1963
  // Longest first, so a value containing another is replaced whole.
1329
1964
  .sort((a, b) => b[0].length - a[0].length);
1330
1965
 
1966
+ // One line for the clock: which zone this step works to, and which level
1967
+ // set it. "Why is this dated yesterday" should be answerable from the run.
1968
+ push("info", `clock: ${clock.timezone} (${clock.source}) — today is ${clockEnv.FOLDRUN_DATE}`);
1969
+ // And one more only when something written could not be read as a zone.
1970
+ for (const problem of clock.problems) push("error", problem);
1331
1971
  if (providerLabel) push("info", `provider: ${providerLabel}`);
1332
1972
  for (const w of providerWarnings) push("error", w);
1973
+ // A level that wrote a language nobody can read was skipped, not obeyed.
1974
+ for (const l of language.lines) push("info", l);
1975
+ for (const l of region.lines) push("info", l);
1333
1976
  for (const w of apiWarnings) push("error", `openapi: ${w}`);
1334
1977
  if (formatWarning) push("error", formatWarning);
1335
1978
  if (legacyUse.length) push("error", legacyUseError(legacyUse));
1979
+ for (const t of retiredTools) push("error", retiredToolError(t));
1336
1980
  for (const t of unknownTools) {
1337
1981
  push(
1338
1982
  "error",
1339
- `tools: "${t}" is not a tool group (web, fetch, read, files, bash, search, history), an SDK tool name, or a tool in this workspace's library — nothing was granted for it`,
1983
+ `tools: "${t}" is not a tool group (read, write, code, search, history, desks), an SDK tool name, or a tool in this workspace's library — nothing was granted for it`,
1340
1984
  );
1341
1985
  }
1342
1986
  for (const name of mcpNames) push("info", `mcp server connected: ${name}`);
@@ -1454,13 +2098,22 @@ async function runStep(
1454
2098
 
1455
2099
  // Expose the workspace's shared scripts/ as ./shared so every agent reaches
1456
2100
  // them by the same path, without each one carrying a copy.
1457
- // No symlinks into the agent directory — they would recurse into the
1458
- // workspace's own file listing. Shared paths are resolved by the script
1459
- // tool layer and described to the agent below.
2101
+ // No permanent symlinks into the agent directory — they would recurse
2102
+ // into the workspace's own file listing. Shared paths are resolved by the
2103
+ // script tool layer and described to the agent below; the one link there
2104
+ // is, `workspace` → the root for the shell, lives only for the length of
2105
+ // a step (executeStep, confine.ts#linkWorkspace).
1460
2106
 
1461
2107
  // The confinement boundary: agents in a workspace are one team, but the
1462
2108
  // workspace is where isolation is enforced.
1463
2109
  const workspaceRoot = path.resolve(agentDir, "..", "..");
2110
+ // Where a test run's state/ and storage/ writes go, and the line each
2111
+ // one leaves: under the run, beside its archived outputs.
2112
+ const divertFor = (id: string) => ({
2113
+ to: path.join(workspaceRoot, "runs", id, TEST_WRITES_DIR),
2114
+ note: (rel: string, summary: string) =>
2115
+ push("info", `test: ${summary}`, { effect: { kind: "diverted" as const, path: rel, summary } }),
2116
+ });
1464
2117
 
1465
2118
  let prompt = step.instruction
1466
2119
  ? resolveDocLinks(step.instruction, workspaceRoot)
@@ -1477,11 +2130,18 @@ async function runStep(
1477
2130
  if (step.output === "json") {
1478
2131
  // The contract the runner parses back out — stated in the prompt so
1479
2132
  // the instruction and the parser agree by construction, like delegate:.
2133
+ // The schema, when there is one, is quoted whole: the model that has
2134
+ // to satisfy it should not have to guess it from the field names.
2135
+ const schema = step.schema ?? (step.schemaPath ? readSchemaFile(path.resolve(agentDir, "..", ".."), step.schemaPath) : null);
1480
2136
  prompt +=
1481
2137
  `\n\n<output>\nThis step returns data. End your reply with exactly one JSON value inside a ` +
1482
2138
  "```json fenced block — the value the next step will compute on. Prose before it is " +
1483
2139
  `welcome (lead with a one-line headline); nothing may follow the block. If the instruction ` +
1484
- `names fields, use exactly those names.\n</output>`;
2140
+ `names fields, use exactly those names.` +
2141
+ (schema && looksLikeSchema(schema)
2142
+ ? ` The value must satisfy this JSON Schema, and the step fails if it does not:\n${JSON.stringify(schema, null, 2).slice(0, 20_000)}`
2143
+ : "") +
2144
+ `\n</output>`;
1485
2145
  }
1486
2146
  if (step.waitFor === "event" && step.eventPayload !== undefined) {
1487
2147
  // Not resolved for links, like the previous step's results: it is
@@ -1532,14 +2192,39 @@ async function runStep(
1532
2192
  for (const m of missingConsults) {
1533
2193
  push("error", `agents: "${m}" is not an agent in this workspace — no consult tool granted`);
1534
2194
  }
1535
- const consultNames = consults.length ? [...mcpNames, "foldrun_agents"] : mcpNames;
2195
+ const consultNames = [
2196
+ ...mcpNames,
2197
+ ...(consults.length ? ["foldrun_agents"] : []),
2198
+ ...(ask ? ["foldrun_ask"] : []),
2199
+ ];
2200
+
2201
+ // Colleagues this agent may DELEGATE to: each with its own context and
2202
+ // its own tools — resolved exactly as its own step's would be, then cut
2203
+ // to what THIS step holds (subagents.ts). Built host-side, like consults.
2204
+ const subagents = buildSubagentSpecs({
2205
+ names: front.subagents, self: step.agent, workspaceRoot, tenant, tags, runId, flowTimezone,
2206
+ parentAllowed: [...allowed, ...consultNames.map((n) => `mcp__${n}`)], push,
2207
+ });
2208
+ if (subagents.length) {
2209
+ // The delegation tool itself, granted only when there is someone to
2210
+ // delegate to.
2211
+ allowed.push(DELEGATE_TOOL);
2212
+ for (const s of subagents) push("info", `subagent: ${s.name} — ${s.tools.length ? s.tools.join(", ") : "no tools"}`);
2213
+ }
1536
2214
 
1537
2215
  // The last provider refusal this step saw, for the fallback decision —
1538
2216
  // watched on both paths, because a refusal reads the same from a pod
1539
2217
  // and from this process.
1540
2218
  let lastRefusal = "";
2219
+ // And WHY it refused, in words, when anything on the way saw the
2220
+ // headers that say so: the translator here, the egress proxy over
2221
+ // there. Kept apart from `lastRefusal` — that one decides whether to
2222
+ // fall back, this one is what a person reads afterwards.
2223
+ let refusalWhy = "";
1541
2224
  const pushWatching: typeof push = (type, text, extra) => {
1542
2225
  if (type === "error" && isProviderRefusal(text)) lastRefusal = text;
2226
+ const why = refusalFromLine(text);
2227
+ if (why) refusalWhy = why;
1543
2228
  push(type, text, extra);
1544
2229
  };
1545
2230
  // The second supply: the block's own fallback when it declared one (a
@@ -1547,6 +2232,50 @@ async function runStep(
1547
2232
  // fallback), else the platform's, and only for a step riding the
1548
2233
  // platform credential.
1549
2234
  const secondSupply = fallbackEnv ?? (Object.keys(providerEnv).length === 0 ? platformFallbackEnv() : null);
2235
+ // Where this step stands on a second supply, for the refusal sentence.
2236
+ // It starts as the truth before anything has run and becomes
2237
+ // "exhausted" the moment the fallback has had its turn — the owner is
2238
+ // about to remove the OpenRouter fallback, so "none" becomes the normal
2239
+ // reading and has to look like a decision on the page.
2240
+ let supplyState: SupplyState = secondSupply ? "trying" : "none";
2241
+ // The line a person reads when this step ends failed: the reason, if
2242
+ // anything on the way learned one, else the status put into words, and
2243
+ // in every case where the step could go next. Pushed last so it is the
2244
+ // last error event — which is what runFailure, the desks' headlines and
2245
+ // `foldrun report` read.
2246
+ const sayWhy = (status: StepRecord["status"]) => {
2247
+ if (status !== "failed" || !lastRefusal) return;
2248
+ const said = Number(lastRefusal.match(/API Error: (\d{3})/)?.[1] ?? NaN);
2249
+ const why =
2250
+ refusalWhy ||
2251
+ explainRefusal({
2252
+ status: Number.isFinite(said) && isRefusalStatus(said) ? said : 402,
2253
+ timezone: clock.timezone,
2254
+ });
2255
+ push("error", `${why}${supplyNote(supplyState)}`);
2256
+ };
2257
+
2258
+ // `schema:` — inline on the step, or a file under the workspace read
2259
+ // now, host-side, so both executors are handed the same document. A
2260
+ // file that cannot be read is a failed step before any model runs:
2261
+ // the contract the author wrote down cannot be checked.
2262
+ const resolvedSchema: Record<string, unknown> | boolean | null = step.schema ?? (step.schemaPath ? readSchemaFile(workspaceRoot, step.schemaPath) : null);
2263
+ if (step.schemaPath && resolvedSchema === null) {
2264
+ throw new Error(`schema: ${step.schemaPath} — not a readable JSON or YAML schema under the workspace`);
2265
+ }
2266
+ if (step.maxTurns) push("info", `max_turns: ${step.maxTurns}`);
2267
+ // `limits:` — account, workspace, agent, then this step's option;
2268
+ // nearest wins per key. Enforced in the model loop's hook, on either
2269
+ // path (step-exec.ts), so it is resolved here as plain values.
2270
+ const stepLimits = cascadeLimits([
2271
+ readAgentsMd(accountDir(tenant))?.data?.limits,
2272
+ readAgentsMd(workspaceRootOf(agentDir))?.data?.limits,
2273
+ front.limits,
2274
+ step.limits,
2275
+ ]);
2276
+ const limitOwners = toolOwners(apiTools.owners, scriptSpecs);
2277
+ const limitOpts = Object.keys(stepLimits).length ? { limits: stepLimits, toolOwners: limitOwners } : {};
2278
+ if (limitOpts.limits) push("info", `limits: ${Object.entries(stepLimits).map(([k, n]) => `${k} ${n}`).join(", ")} — per step; a call past one is refused`);
1550
2279
 
1551
2280
  const isolation = process.env.FOLDRUN_RUN_ISOLATION;
1552
2281
  if (isolatedRun()) {
@@ -1570,21 +2299,45 @@ async function runStep(
1570
2299
  // before anything is copied in. Null where no proxy is configured
1571
2300
  // (the CLI, a compose install without FOLDRUN_EGRESS_URL), and then
1572
2301
  // everything below materialises exactly as it always did.
1573
- const grant: EgressGrant = { secrets: {} };
2302
+ const grant: EgressGrant = { secrets: {}, timezone: clock.timezone };
2303
+ // A direct search API's key: granted to that API's host and no other,
2304
+ // exactly as an http tool's key is granted to its base URL's host. The
2305
+ // script sends the placeholder; the worker fills it on the way out.
2306
+ for (const choice of [searchChoice, fetchChoice]) {
2307
+ if (choice.shape === "direct" && choice.secret && choice.host && choice.secret in liveSecrets) {
2308
+ addGrant(grant, choice.secret, liveSecrets[choice.secret], choice.host);
2309
+ }
2310
+ }
2311
+ for (const c of actionChoices) {
2312
+ if (c.provider && c.secret in liveSecrets) addGrant(grant, c.secret, liveSecrets[c.secret], c.host);
2313
+ }
1574
2314
  for (const api of apiSpecs) {
1575
2315
  const host = hostOf(api.base);
1576
2316
  if (!host) continue;
1577
2317
  for (const v of [...Object.values(api.headers), ...Object.values(api.query)]) {
1578
- for (const name of placeholderNames(v)) if (name in liveSecrets) addGrant(grant, name, liveSecrets[name], host);
2318
+ for (const name of placeholderNames(v)) {
2319
+ if (!(name in liveSecrets)) continue;
2320
+ addGrant(grant, name, liveSecrets[name], host);
2321
+ // Which vault entry this is, so the proxy's record of what the
2322
+ // far end said lands on the right credential.
2323
+ (grant.healthKeys ??= {})[name] = healthKey(
2324
+ name,
2325
+ secretScopes[name] === "workspace" ? "workspace" : "account",
2326
+ path.basename(path.resolve(agentDir, "..", "..")),
2327
+ );
2328
+ }
1579
2329
  }
1580
2330
  }
1581
- const lease = await platform.egress.lease({ tenant, runId: runId ?? "adhoc", grant });
2331
+ const lease = await platform.egress.lease({ tenant, runId: runId ?? "adhoc", grant, test: testRun, ...(stepIndex !== undefined && stepIndex >= 0 ? { step: stepIndex } : {}) });
1582
2332
  // Which secrets must still be real inside the sandbox: a script tool
1583
2333
  // reads them from its environment, and unless it says `secrets:
1584
2334
  // proxied` it needs the value, not the name. bash is a script by
1585
2335
  // another route. The record says which way it went, and why.
1586
2336
  const materialisers = [
1587
2337
  ...scriptSpecs.filter((sc) => sc.secrets !== "proxied").map((sc) => sc.name),
2338
+ // web proxies its keys — unless it may browse: a remote browser's key
2339
+ // goes into a websocket, and cookie and login secrets into the page.
2340
+ ...(scriptSpecs.some((sc) => sc.name === "web") && (!web.actions || web.actions.includes("browse")) ? ["web (browse)"] : []),
1588
2341
  ...(allowed.includes("Bash") ? ["bash"] : []),
1589
2342
  ];
1590
2343
  const materialise = !lease || materialisers.length > 0;
@@ -1601,15 +2354,36 @@ async function runStep(
1601
2354
  }
1602
2355
  // Sandbox env for the declared secrets: the value, or its own name as
1603
2356
  // a placeholder that the proxy fills on the way out.
1604
- const sandboxSecrets: Record<string, string> = materialise
1605
- ? liveSecrets
1606
- : Object.fromEntries(declared.map((name) => [name, `\${${name}}`]));
2357
+ const placeholders = Object.fromEntries(declared.map((name) => [name, `\${${name}}`]));
2358
+ let sandboxSecrets: Record<string, string> = materialise ? liveSecrets : placeholders;
2359
+ // A test run hands the sandbox no send-capable value. The env var is
2360
+ // still there, reading TEST_MODE_WITHHELD, so a script that never
2361
+ // checked FOLDRUN_TEST_MODE fails at the provider's door with an auth
2362
+ // error — and sends nothing. The script tools say whether the step
2363
+ // has a declared sender (everything withheld) or a declared reader
2364
+ // (nothing withheld); see test-mode.ts for the order.
2365
+ if (testRun && materialise) {
2366
+ const held = withholdSecrets(liveSecrets, {
2367
+ outward: scriptSpecs.some((sc) => sc.outward),
2368
+ allow: scriptSpecs.some((sc) => sc.testMode === "allow"),
2369
+ });
2370
+ sandboxSecrets = held.env;
2371
+ for (const name of held.withheld) {
2372
+ push("info", `test: ${name} withheld from the sandbox — TEST_MODE_WITHHELD`, { effect: { kind: "withheld", summary: `${name} withheld from the sandbox` } });
2373
+ }
2374
+ }
2375
+ // On a test run the http tools always go through the proxy as
2376
+ // placeholders, even when a script made the step materialise: the
2377
+ // proxy is where the outward-write policy lives, and a header
2378
+ // carrying TEST_MODE_WITHHELD would only fail where it should have
2379
+ // been sunk or refused.
2380
+ const apiValues = testRun && lease ? placeholders : sandboxSecrets;
1607
2381
  const substitutedApis = apiSpecs.map((api) => ({
1608
2382
  ...api,
1609
2383
  headers: Object.fromEntries(
1610
2384
  Object.entries(api.headers).map(([k, v]) => [
1611
2385
  k,
1612
- v.replace(/\$\{([A-Z][A-Z0-9_]*)\}/g, (whole, name) => sandboxSecrets[name] ?? whole),
2386
+ v.replace(/\$\{([A-Z][A-Z0-9_]*)\}/g, (whole, name) => apiValues[name] ?? whole),
1613
2387
  ]),
1614
2388
  ),
1615
2389
  }));
@@ -1639,8 +2413,9 @@ async function runStep(
1639
2413
  const isolatedArgs = (modelEnv: Record<string, string | undefined>, keyName = MODEL_KEY_NAME) => ({
1640
2414
  workspaceRoot,
1641
2415
  libraryRoot: libraryDir(tenant),
2416
+ galleryRoot: platform.galleryDir() ?? undefined,
1642
2417
  // Keys this step's dependency cache. Per-account by design — see
1643
- // run-container.ts#runtimeCacheDir for why it is not shared.
2418
+ // run-container.ts#runtimeCacheMount for why it is not shared.
1644
2419
  tenant,
1645
2420
  input: {
1646
2421
  agentRel: path.relative(workspaceRoot, agentDir).replaceAll("\\", "/"),
@@ -1657,12 +2432,21 @@ async function runStep(
1657
2432
  // field. A tool's own `runtime:` is part of what this step needs.
1658
2433
  runtime: runtime.spec,
1659
2434
  consults,
2435
+ ...(subagents.length ? { subagents } : {}),
1660
2436
  timeoutSec: step.timeout,
1661
2437
  budgetUsd: stepBudgetUsd,
2438
+ budgetNote: stepBudgetNote,
1662
2439
  price: tokenPrice,
1663
2440
  verify: step.verify,
1664
2441
  output: step.output,
2442
+ schema: resolvedSchema ?? undefined,
2443
+ maxTurns: step.maxTurns,
2444
+ ...limitOpts,
1665
2445
  translator: keyName === MODEL_KEY_NAME ? primaryTranslator : secondTranslator,
2446
+ // Outward steps keep a browser of their own: an executor that
2447
+ // could browse from the slim image through the account's pod
2448
+ // leaves them on the full one (browser-pod.ts).
2449
+ ...(outward ? { outward: true } : {}),
1666
2450
  // The container sees the workspace at /workspace and the library
1667
2451
  // at /library; the roots are named host-side and moved here.
1668
2452
  search: searchRoots.map((r) => ({
@@ -1673,6 +2457,7 @@ async function runStep(
1673
2457
  })),
1674
2458
  history: historyDigest,
1675
2459
  desks: deskDigest,
2460
+ ...(ask ? { ask } : {}),
1676
2461
  },
1677
2462
  env: Object.fromEntries(
1678
2463
  Object.entries({
@@ -1692,6 +2477,7 @@ async function runStep(
1692
2477
  FOLDRUN_RUN_ID: runId,
1693
2478
  FOLDRUN_AGENT: step.agent,
1694
2479
  FOLDRUN_WORKSPACE: path.basename(workspaceRoot),
2480
+ ...(testRun ? testModeEnv() : {}),
1695
2481
  ...sandboxSecrets,
1696
2482
  }).filter((entry): entry is [string, string] => typeof entry[1] === "string"),
1697
2483
  ),
@@ -1699,6 +2485,9 @@ async function runStep(
1699
2485
  // Stamps the sandbox, so stopping this run can destroy it.
1700
2486
  runId,
1701
2487
  size,
2488
+ // A test run's state/ and storage/ changes come out under the run.
2489
+ divert: testRun && runId ? divertFor(runId) : undefined,
2490
+ test: testRun,
1702
2491
  // Resume: a sandbox a previous driver left running, recorded on the
1703
2492
  // step. Consumed once — a fresh attempt after this one (a retry, a
1704
2493
  // fallback) starts its own sandbox.
@@ -1720,8 +2509,38 @@ async function runStep(
1720
2509
  // attempt holds its own sandbox, so the meter owes the sum.
1721
2510
  const firstArgs = isolatedArgs(platformModelEnv());
1722
2511
  await lease?.commit();
1723
- let outcome = await runIsolated(firstArgs);
2512
+ let credentialUsed = platformModelCredential();
2513
+ // The arguments of the attempt whose outcome is current — what a
2514
+ // re-run on the full image repeats, on the same model supply.
2515
+ let lastArgs: typeof firstArgs = firstArgs;
2516
+ const runTracked = (a: typeof firstArgs) => {
2517
+ lastArgs = a;
2518
+ return runIsolated(a);
2519
+ };
2520
+ let outcome = await runTracked(firstArgs);
1724
2521
  step.sandbox = null; // whatever happens next starts its own
2522
+ // A 401 from a token that rotated mid-step is not a broken key: the
2523
+ // replacement is already on its way to the mounted file. Wait for it
2524
+ // once, then fall into the same retry the busy case uses.
2525
+ if (outcome.status === "failed" && isAuthRefusal(lastRefusal)) {
2526
+ push("info", `the model credential was refused mid-step — waiting for the refreshed one`);
2527
+ const rotated = await awaitRotatedCredential(credentialUsed);
2528
+ if (rotated) {
2529
+ push("info", "the credential rotated under this step; retrying on the new one");
2530
+ credentialUsed = rotated;
2531
+ lastRefusal = "";
2532
+ const previous = outcome.timing;
2533
+ // Build first: isolatedArgs is what re-grants the model key, and
2534
+ // commit() is what publishes the grant to Redis, where the proxy
2535
+ // reads it. Committing first sent the revoked token again — every
2536
+ // rotation retry on 2026-09-22/23 401'd this way.
2537
+ const retry = isolatedArgs(platformModelEnv());
2538
+ await lease?.commit();
2539
+ outcome = withEarlierTiming(await runTracked(retry), previous);
2540
+ } else {
2541
+ push("info", "the credential did not change — this is the key itself, not a rotation");
2542
+ }
2543
+ }
1725
2544
  for (let n = 1; n <= OVERLOAD_RETRIES && outcome.status === "failed" && isTransientOverload(lastRefusal); n++) {
1726
2545
  const wait = backoffMs(n, lastRefusal);
1727
2546
  push("info", `provider busy (${lastRefusal.slice(0, 60)}) — waiting ${(wait / 1000).toFixed(1)}s, attempt ${n} of ${OVERLOAD_RETRIES}`);
@@ -1732,7 +2551,7 @@ async function runStep(
1732
2551
  const previous = outcome.timing;
1733
2552
  const again = isolatedArgs(platformModelEnv());
1734
2553
  await lease?.commit();
1735
- outcome = await runIsolated(again);
2554
+ outcome = await runTracked(again);
1736
2555
  outcome = withEarlierTiming(outcome, previous);
1737
2556
  }
1738
2557
  // The second supply, tried exactly once, and only when the primary
@@ -1760,26 +2579,75 @@ async function runStep(
1760
2579
  ),
1761
2580
  };
1762
2581
  await lease?.commit();
1763
- outcome = await runIsolated(fallbackArgs);
2582
+ supplyState = "exhausted";
2583
+ outcome = await runTracked(fallbackArgs);
1764
2584
  // Both attempts held sandboxes; the meter owes the sum. The first
1765
2585
  // try's pod ran, was billed for by the platform, and must not
1766
2586
  // vanish from the record because a second try replaced its outcome.
1767
2587
  outcome = withEarlierTiming(outcome, first);
1768
2588
  }
2589
+ // Slim browsing (browser-pod.ts): the step ran on the slim image and
2590
+ // the account's browser pod went away under it, past the web tool's
2591
+ // reconnects. Reads only — run it again from the start on the full
2592
+ // image, where a browser in the step is always there. Anything
2593
+ // written — fail it, naming what, and leave retry:/on-fail:/a person
2594
+ // to decide; a second go could send or charge twice.
2595
+ step.image = outcome.image ?? null;
2596
+ step.browserPod = outcome.browserPod ? { ...outcome.browserPod } : null;
2597
+ if (outcome.status === "failed" && outcome.browserPod?.lost) {
2598
+ const decision = podLossDecision(outcome.browserPod);
2599
+ if (decision.rerun) {
2600
+ push("info", decision.note);
2601
+ const first = outcome;
2602
+ // The slim go is a try of its own on the record — image slim,
2603
+ // status lost, its own cost, tokens and sandbox seconds — so the
2604
+ // run page and `foldrun report` show two tries, slim (lost) then
2605
+ // full, and the step's totals are the sum of both rows
2606
+ // (recordAttempt). Live, 2026-10-01, the step showed only the
2607
+ // full re-run.
2608
+ step.podLostTry = {
2609
+ status: "lost",
2610
+ costUsd: repriced(catalog, wireModel, first.usage ?? null, first.costUsd, push, first.turnsCostUsd),
2611
+ tokens: first.usage ? { input: first.usage.inputTokens, output: first.usage.outputTokens } : null,
2612
+ computeSecs: first.timing ? Math.round(first.timing.totalMs) / 1000 : null,
2613
+ finishedAt: new Date().toISOString(),
2614
+ error: decision.note.slice(0, 500),
2615
+ image: "slim",
2616
+ browserPod: browserPodLine(first.browserPod!),
2617
+ };
2618
+ const again = { ...lastArgs, input: { ...lastArgs.input, image: "full" as const }, resume: null };
2619
+ await lease?.commit();
2620
+ outcome = await runTracked(again);
2621
+ step.image = outcome.image ?? { variant: "full", why: "re-run after the browser pod was lost" };
2622
+ step.browserPod = { ...first.browserPod!, fallback: RERAN_ON_FULL };
2623
+ } else {
2624
+ push("error", decision.message);
2625
+ step.browserPod = { ...outcome.browserPod, failure: decision.message };
2626
+ }
2627
+ }
1769
2628
  if (lease) {
1770
- for (const line of await lease.drainLog()) push("info", line);
2629
+ for (const line of await lease.drainLog()) {
2630
+ // pushWatching, not push: a line the proxy wrote may carry the
2631
+ // refusal mark, and that sentence is the whole reason the proxy
2632
+ // reads response headers at all.
2633
+ if (typeof line === "string") pushWatching("info", line);
2634
+ else pushWatching("info", line.text, { effect: line.effect });
2635
+ }
1771
2636
  await lease.release();
1772
2637
  }
1773
- publishPublicDir(tenant, path.basename(workspaceRoot), push);
2638
+ // Nothing a test run left in storage/public/ reached storage/, so
2639
+ // there is nothing to publish — and a share link is an outward act.
2640
+ if (!testRun) await publishPublicDir(tenant, path.basename(workspaceRoot), push);
2641
+ sayWhy(outcome.status);
1774
2642
  step.status = outcome.status;
1775
- step.result = outcome.result;
1776
- step.conclusion = outcome.conclusion;
2643
+ step.result = redactText(outcome.result);
2644
+ step.conclusion = redactText(outcome.conclusion);
1777
2645
  step.sandbox = null;
1778
2646
  // The cluster's word on why a sandbox ended without the driver saying
1779
2647
  // so — OOMKilled, Evicted — on the record, where the retry reads it.
1780
2648
  if (outcome.status === "failed" && outcome.reason) push("error", `sandbox ended: ${outcome.reason}`);
1781
- if (outcome.data !== undefined) step.data = outcome.data;
1782
- step.costUsd = repriced(catalog, wireModel, outcome.usage ?? null, outcome.costUsd, push);
2649
+ if (outcome.data !== undefined) step.data = redactData(outcome.data);
2650
+ step.costUsd = repriced(catalog, wireModel, outcome.usage ?? null, outcome.costUsd, push, outcome.turnsCostUsd);
1783
2651
  step.tokens = outcome.usage
1784
2652
  ? { input: outcome.usage.inputTokens, output: outcome.usage.outputTokens }
1785
2653
  : null;
@@ -1811,19 +2679,36 @@ async function runStep(
1811
2679
  })();
1812
2680
  step.reserved = { cpus: Number(lim.cpus) || 2, memGiB: gib };
1813
2681
  } else {
2682
+ // Same withholding as the sandbox path — the in-process step is what
2683
+ // `foldrun run --test` drives on a laptop, where there is no proxy
2684
+ // and this is the only guard there is.
2685
+ const held = testRun
2686
+ ? withholdSecrets(liveSecrets, {
2687
+ outward: scriptSpecs.some((sc) => sc.outward),
2688
+ allow: scriptSpecs.some((sc) => sc.testMode === "allow"),
2689
+ })
2690
+ : { env: liveSecrets, withheld: [] as string[] };
2691
+ for (const name of held.withheld) {
2692
+ push("info", `test: ${name} withheld from scripts — TEST_MODE_WITHHELD`, { effect: { kind: "withheld", summary: `${name} withheld from scripts` } });
2693
+ }
2694
+ const stepSecrets = held.env;
1814
2695
  const consultTools = buildConsultTools(
1815
2696
  consults,
1816
- { ...process.env, ...liveSecrets, ...providerEnv },
2697
+ { ...hostSafeEnv(), ...stepSecrets, ...providerEnv },
1817
2698
  (type, text) => push(type, text),
1818
2699
  );
2700
+ // On this machine nobody can write in and there is no proxy: the
2701
+ // question goes to the terminal when there is one, else unanswered.
2702
+ const askTools = ask ? buildAskTool({ channel: null, timeoutSec: ask.timeoutSec, emit: push, local: terminalAsk }) : null;
1819
2703
  // @file secrets become 0600 paths here (host run), cleaned up in the
1820
2704
  // finally — once, however many attempts follow.
1821
- const mat = materializeFileSecrets(agentDir, liveSecrets);
2705
+ const mat = materializeFileSecrets(agentDir, stepSecrets);
1822
2706
  fileDir = mat.dir;
1823
2707
  const attemptInProcess = (modelEnv: Record<string, string | undefined>) => executeStep({
1824
2708
  agentDir,
1825
2709
  workspaceRoot,
1826
2710
  libraryRoot: libraryDir(tenant),
2711
+ ...(subagents.length ? { subagents } : {}),
1827
2712
  prompt,
1828
2713
  model,
1829
2714
  effort,
@@ -1837,16 +2722,25 @@ async function runStep(
1837
2722
  ...(searchTools.server ? { foldrun_search: searchTools.server } : {}),
1838
2723
  ...(historyTools.server ? { foldrun_history: historyTools.server } : {}),
1839
2724
  ...(deskTools.server ? { foldrun_desks: deskTools.server } : {}),
2725
+ ...(askTools ? { foldrun_ask: askTools.server } : {}),
1840
2726
  ...mcpServers,
1841
2727
  },
1842
2728
  // Declared secrets reach the agent's scripts as env vars; the model
1843
- // only ever sees the variable names, not the values.
1844
- env: { ...process.env, ...clockEnv, ...mat.env, ...modelEnv },
2729
+ // only ever sees the variable names, not the values. The base is
2730
+ // the allowlisted host environment, not process.env: the SDK's
2731
+ // Bash tool runs with this, and on a platform process.env holds
2732
+ // every tenant's root key and the datastore credentials.
2733
+ env: { ...hostSafeEnv(), ...clockEnv, ...(testRun ? testModeEnv() : {}), ...mat.env, ...modelEnv },
1845
2734
  timeoutSec: step.timeout,
1846
2735
  budgetUsd: stepBudgetUsd,
2736
+ budgetNote: stepBudgetNote,
1847
2737
  price: tokenPrice,
1848
2738
  verify: step.verify,
1849
2739
  output: step.output,
2740
+ schema: resolvedSchema ?? undefined,
2741
+ maxTurns: step.maxTurns,
2742
+ ...limitOpts,
2743
+ stopRequested,
1850
2744
  // What the scripts saw, the verify sees: a flow can then check that
1851
2745
  // the proof a step left names THIS run, not one that came before —
1852
2746
  // copy-back never propagates a deletion, so a marker from an earlier
@@ -1856,23 +2750,49 @@ async function runStep(
1856
2750
  ...(runId ? { FOLDRUN_RUN_ID: runId } : {}),
1857
2751
  FOLDRUN_AGENT: step.agent,
1858
2752
  FOLDRUN_WORKSPACE: path.basename(path.resolve(agentDir, "..", "..")),
1859
- ...liveSecrets,
2753
+ ...(testRun ? testModeEnv() : {}),
2754
+ ...stepSecrets,
1860
2755
  },
1861
2756
  emit: pushWatching,
1862
2757
  });
1863
2758
  // With a translator the step's endpoint is the translator: started
1864
2759
  // here, pointed at by the SDK's env, closed when the attempt ends.
2760
+ // A test run brackets each attempt with a snapshot of state/ and
2761
+ // storage/: the step writes them for real, because there is no
2762
+ // sandbox to write instead, and afterwards what changed is moved
2763
+ // under the run and the originals come back.
1865
2764
  const attemptThrough = async (modelEnv: Record<string, string | undefined>, spec: TranslatorSpec | null) => {
1866
- if (!spec) return attemptInProcess(modelEnv);
1867
- const t = await startTranslator(spec);
2765
+ const before = testRun ? snapshotDivertedDirs(workspaceRoot) : null;
2766
+ const t = spec ? await startTranslator(spec) : null;
1868
2767
  try {
1869
- return await attemptInProcess({ ...modelEnv, ...t.env });
2768
+ return await attemptInProcess(t ? { ...modelEnv, ...t.env } : modelEnv);
1870
2769
  } finally {
1871
- for (const line of t.drainLog()) push("info", line.startsWith("translator") ? line : `translator: ${line}`);
1872
- await t.close();
2770
+ if (t) {
2771
+ for (const line of t.drainLog()) pushWatching("info", line.startsWith("translator") ? line : `translator: ${line}`);
2772
+ await t.close();
2773
+ }
2774
+ if (before && runId) {
2775
+ const d = divertFor(runId);
2776
+ restoreDivertedDirs(workspaceRoot, before, d.to, d.note);
2777
+ }
1873
2778
  }
1874
2779
  };
2780
+ let credentialUsed = platformModelCredential();
1875
2781
  let outcome = await attemptThrough(providerEnv, translator);
2782
+ // Same rule as the isolated path: a credential that rotated under the
2783
+ // step is waited out and retried before anything else is considered.
2784
+ if (outcome.status === "failed" && isAuthRefusal(lastRefusal)) {
2785
+ push("info", `the model credential was refused mid-step — waiting for the refreshed one`);
2786
+ const rotated = await awaitRotatedCredential(credentialUsed);
2787
+ if (rotated) {
2788
+ push("info", "the credential rotated under this step; retrying on the new one");
2789
+ credentialUsed = rotated;
2790
+ lastRefusal = "";
2791
+ outcome = await attemptThrough(providerEnv, translator);
2792
+ } else {
2793
+ push("info", "the credential did not change — this is the key itself, not a rotation");
2794
+ }
2795
+ }
1876
2796
  // Same rule as the isolated path: a busy provider is waited out on
1877
2797
  // the supply it refused from, before any fallback is considered.
1878
2798
  for (let n = 1; n <= OVERLOAD_RETRIES && outcome.status === "failed" && isTransientOverload(lastRefusal); n++) {
@@ -1889,16 +2809,18 @@ async function runStep(
1889
2809
  "info",
1890
2810
  `primary model supply refused (${lastRefusal.slice(0, 80)}) — retrying this step on the fallback provider`,
1891
2811
  );
2812
+ supplyState = "exhausted";
1892
2813
  outcome = await attemptThrough(secondSupply, fallbackEnv && secondSupply === fallbackEnv ? fallbackTranslator : null);
1893
2814
  }
1894
- publishPublicDir(tenant, path.basename(workspaceRoot), push);
2815
+ if (!testRun) await publishPublicDir(tenant, path.basename(workspaceRoot), push);
2816
+ sayWhy(outcome.status);
1895
2817
  step.status = outcome.status;
1896
- step.result = outcome.result;
1897
- step.conclusion = outcome.conclusion;
1898
- if (outcome.data !== undefined) step.data = outcome.data;
2818
+ step.result = redactText(outcome.result);
2819
+ step.conclusion = redactText(outcome.conclusion);
2820
+ if (outcome.data !== undefined) step.data = redactData(outcome.data);
1899
2821
  // A consult's spend belongs to the step that asked.
1900
2822
  const consultCost = consultTools.drainCost();
1901
- const stepCost = repriced(catalog, wireModel, outcome.usage, outcome.costUsd, push);
2823
+ const stepCost = repriced(catalog, wireModel, outcome.usage, outcome.costUsd, push, outcome.turnsCostUsd);
1902
2824
  step.costUsd = stepCost === null && consultCost === 0 ? null : (stepCost ?? 0) + consultCost;
1903
2825
  step.tokens = outcome.usage
1904
2826
  ? { input: outcome.usage.inputTokens, output: outcome.usage.outputTokens }
@@ -1915,9 +2837,33 @@ async function runStep(
1915
2837
  } catch {
1916
2838
  // best-effort — a leftover scratch file is swept with the run's outputs
1917
2839
  }
2840
+ try {
2841
+ // A private runtime build (see prepareRuntime) is this step's alone.
2842
+ disposeRuntime?.();
2843
+ } catch {
2844
+ // best-effort — it is in the tmpdir
2845
+ }
1918
2846
  }
1919
2847
  step.finishedAt = new Date().toISOString();
1920
2848
  save();
2849
+ return { actedThenFailedCheck: canActOutward && actedThenFailedCheck(step.events.slice(eventsAtStart)) };
2850
+ }
2851
+
2852
+ /**
2853
+ * Did this attempt call its tools and then fail only on the check after —
2854
+ * `verify:`, `output: json`, a schema? For a step whose tools act outward
2855
+ * that is the one failure a retry must not answer: the email went, the post
2856
+ * is up, and the check failing says nothing about whether they happened. A
2857
+ * second attempt sends them again. Read off the attempt's own events, which
2858
+ * both executors write the same way (step-exec.ts): every event a failed
2859
+ * check writes carries `check: true`. It used to read only the LAST error's
2860
+ * wording, and a failed shell verify or judge writes its detail after its
2861
+ * headline — so the step was retried and the email went twice. The wording
2862
+ * is still honoured, for a driver image older than the tag.
2863
+ */
2864
+ export function actedThenFailedCheck(events: { type: string; text: string; check?: boolean }[]): boolean {
2865
+ if (!events.some((e) => e.type === "tool")) return false;
2866
+ return events.some((e) => e.type === "error" && (e.check === true || /^(verify `|output: json — |schema: the value does not fit)/.test(e.text)));
1921
2867
  }
1922
2868
 
1923
2869
  // Block until a human approves or rejects the pending steps. Polls the run
@@ -2018,6 +2964,59 @@ function platformFallbackEnv(): Record<string, string> | null {
2018
2964
  return out;
2019
2965
  }
2020
2966
 
2967
+ /**
2968
+ * A 401 the token refresh caused, rather than a key that is simply wrong.
2969
+ *
2970
+ * The refresher on the box turns the long-lived grant into an 8h access
2971
+ * token every four hours, and Anthropic revokes the old one the instant the
2972
+ * new one is minted. The platform reads the credential per step from the
2973
+ * mounted file, which fixed the boot-time copy — but not this: a step
2974
+ * already talking to the API when the rotation lands gets a hard 401
2975
+ * mid-turn, and the replacement does not appear in the mounted file until
2976
+ * the kubelet syncs it, about a minute later.
2977
+ *
2978
+ * That window is narrow and it is not rare. Every 401 on this account in
2979
+ * three weeks of runs fell inside it: 02:07:22 and 02:07:24 against a 02:07
2980
+ * refresh, 22:08:12, 22:08:17, 22:08:30 and 22:09:03 against 22:08. The
2981
+ * desks that lost a run nightly were simply the ones scheduled on the hour
2982
+ * the refresher shares.
2983
+ *
2984
+ * So: on a 401, wait for the file to hold a DIFFERENT credential, then let
2985
+ * the caller retry with it. A token that never changes inside the window is
2986
+ * a key that is actually invalid — no waiting fixes that, and the caller
2987
+ * falls through to the second supply exactly as before. Moving the timer
2988
+ * cannot fix this; a run that takes forty minutes spans any minute chosen.
2989
+ */
2990
+ const CREDENTIAL_ROTATION_WAIT_MS = 90_000;
2991
+ const CREDENTIAL_POLL_MS = 2_000;
2992
+
2993
+ function isAuthRefusal(text: string): boolean {
2994
+ return /API Error: 401|access token has (been revoked|expired)/i.test(text);
2995
+ }
2996
+
2997
+ /**
2998
+ * Wait, briefly, for the mounted credential to become one we have not
2999
+ * already been refused on. Returns the new value, or null when the window
3000
+ * passes without a change — which is the answer that means "this key is
3001
+ * wrong", not "this key is stale".
3002
+ */
3003
+ export async function awaitRotatedCredential(
3004
+ before: string | undefined,
3005
+ now: () => number = Date.now,
3006
+ read: () => string | undefined = () => platformModelCredential(),
3007
+ waitMs = CREDENTIAL_ROTATION_WAIT_MS,
3008
+ pollMs = CREDENTIAL_POLL_MS,
3009
+ ): Promise<string | null> {
3010
+ if (!before) return null;
3011
+ const until = now() + waitMs;
3012
+ for (;;) {
3013
+ const current = read();
3014
+ if (current && current !== before) return current;
3015
+ if (now() >= until) return null;
3016
+ await sleep(pollMs);
3017
+ }
3018
+ }
3019
+
2021
3020
  /** A model-call failure the fallback can answer: the primary refusing over
2022
3021
  * credit, quota or auth — never a failure of the work itself, which would
2023
3022
  * fail identically anywhere. */
@@ -2028,7 +3027,19 @@ function isProviderRefusal(text: string): boolean {
2028
3027
  // fallback it could only time out on again. A refusal the fallback can
2029
3028
  // answer is about the ACCOUNT; anything about the work or the sandbox is
2030
3029
  // not, however similar the vocabulary.
2031
- return /API Error: (401|402|403|429|5\d\d)|credits?\b|quota|billing|insufficient|overloaded/i.test(text);
3030
+ // A subscription's own ceiling is the same kind of refusal, worded
3031
+ // differently: Claude Code on a Max plan answers "You've hit your weekly
3032
+ // limit · resets 5am" with no status code and none of the money words.
3033
+ // On 2026-09-21 every opus-tier step on the account failed on exactly that
3034
+ // sentence while sonnet steps beside them ran fine — the plan has a
3035
+ // separate weekly cap for Opus — and nothing here recognised it, so a
3036
+ // second supply could not have answered even if one had been configured.
3037
+ // It is not transient (isTransientOverload is right to ignore it: the
3038
+ // window is a week), it is the account's supply being exhausted, which is
3039
+ // precisely what the fallback provider is for.
3040
+ return /API Error: (401|402|403|429|5\d\d)|credits?\b|quota|billing|insufficient|overloaded|(weekly|daily|monthly|usage) limit|rate.?limit(ed)?\b.*resets?\b|limit\b.*\bresets?\b/i.test(
3041
+ text,
3042
+ );
2032
3043
  }
2033
3044
 
2034
3045
  /** How many times a busy provider is given another moment, and the longest
@@ -2084,7 +3095,47 @@ export function backoffMs(attempt: number, text = "", random: () => number = Mat
2084
3095
  return Math.round(base * (0.5 + random() * 0.5));
2085
3096
  }
2086
3097
 
2087
- function platformModelEnv(): Record<string, string | undefined> {
3098
+ /**
3099
+ * The platform's model credential, read at the moment a step needs it.
3100
+ *
3101
+ * It used to come only from this process's environment, which meant the
3102
+ * worker held whatever token it was started with. Refreshing an OAuth token
3103
+ * REVOKES the old one, so a refresh without a restart left every run
3104
+ * answering 401 — on 2026-09-18 that took every desk down for two hours, and
3105
+ * nothing said so until a person read a failed run. The refresher's own
3106
+ * comment had named the fix years of incidents earlier: stop shipping the
3107
+ * token as an env var.
3108
+ *
3109
+ * So: when `FOLDRUN_MODEL_KEY_FILE` names a file (the Kubernetes secret,
3110
+ * mounted rather than injected), it is read fresh every time. The kubelet
3111
+ * updates a mounted secret in place within about a minute of it changing, so
3112
+ * a refreshed token reaches the next step on its own, with no restart, no
3113
+ * timing window, and nothing to skip while work is in flight.
3114
+ *
3115
+ * A missing or empty file falls back to the environment, which is what a
3116
+ * local install, the CLI and every compose setup use.
3117
+ */
3118
+ export function platformModelCredential(env: NodeJS.ProcessEnv = process.env): string | undefined {
3119
+ const file = env.FOLDRUN_MODEL_KEY_FILE;
3120
+ if (file) {
3121
+ try {
3122
+ const value = fs.readFileSync(file, "utf8").trim();
3123
+ if (value) return value;
3124
+ } catch {
3125
+ // Unreadable is not the same as absent: say so once, then fall back,
3126
+ // because a running platform that cannot read its credential file is a
3127
+ // thing an operator needs to know about before the next step fails.
3128
+ if (!credentialFileWarned) {
3129
+ credentialFileWarned = true;
3130
+ console.warn(`[model] FOLDRUN_MODEL_KEY_FILE=${file} could not be read; falling back to the environment`);
3131
+ }
3132
+ }
3133
+ }
3134
+ return env.CLAUDE_CODE_OAUTH_TOKEN || undefined;
3135
+ }
3136
+ let credentialFileWarned = false;
3137
+
3138
+ export function platformModelEnv(): Record<string, string | undefined> {
2088
3139
  const {
2089
3140
  ANTHROPIC_API_KEY,
2090
3141
  ANTHROPIC_AUTH_TOKEN,
@@ -2103,7 +3154,9 @@ function platformModelEnv(): Record<string, string | undefined> {
2103
3154
  ANTHROPIC_API_KEY,
2104
3155
  ANTHROPIC_AUTH_TOKEN,
2105
3156
  ANTHROPIC_BASE_URL,
2106
- CLAUDE_CODE_OAUTH_TOKEN,
3157
+ // The file wins when there is one: it is the live secret, and this
3158
+ // process's copy is whatever it was handed at boot.
3159
+ CLAUDE_CODE_OAUTH_TOKEN: platformModelCredential() ?? CLAUDE_CODE_OAUTH_TOKEN,
2107
3160
  ANTHROPIC_DEFAULT_HAIKU_MODEL,
2108
3161
  ANTHROPIC_DEFAULT_SONNET_MODEL,
2109
3162
  ANTHROPIC_DEFAULT_OPUS_MODEL,
@@ -2152,20 +3205,38 @@ function jsonItems(data: unknown, step: StepRecord, take: number): string[] {
2152
3205
  function csvItems(workspaceDir: string, step: StepRecord, take: number): string[] {
2153
3206
  const raw = step.eachPath ?? "";
2154
3207
  if (!raw) {
2155
- step.skipReason = "each: rows needs a path — `each: rows of ../../storage/x.csv`";
3208
+ step.skipReason = "each: rows needs a path — `each: rows of workspace/storage/x.csv`";
2156
3209
  return [];
2157
3210
  }
2158
- const resolved = path.resolve(workspaceDir, "agents", step.agent, raw);
2159
- if (!resolved.startsWith(path.resolve(workspaceDir) + path.sep)) {
2160
- step.skipReason = `each: rows — ${raw} is outside this workspace`;
3211
+ const read = csvDataRows(workspaceDir, step.agent, raw);
3212
+ if ("problem" in read) {
3213
+ step.skipReason = `each: rows — ${read.problem}`;
2161
3214
  return [];
2162
3215
  }
3216
+ return read.data.slice(0, take).map((r) => `${read.header}\n${r}`);
3217
+ }
3218
+
3219
+ /**
3220
+ * The data rows of a CSV, path agent-relative like every path in a flow and
3221
+ * confined to the workspace. Shared by `each: rows of`, which fans out over
3222
+ * them, and `when: rows of`, which asks only whether there are any.
3223
+ */
3224
+ function csvDataRows(
3225
+ workspaceDir: string,
3226
+ agent: string,
3227
+ raw: string,
3228
+ ): { header: string; data: string[] } | { problem: string } {
3229
+ // `workspace/storage/x.csv` from the workspace root, anything else from
3230
+ // the agent's folder (`../../storage/x.csv` is the same file).
3231
+ const resolved = resolveAgentPath(workspaceDir, path.join(workspaceDir, "agents", agent), raw);
3232
+ if (!resolved.startsWith(path.resolve(workspaceDir) + path.sep)) {
3233
+ return { problem: `${raw} is outside this workspace` };
3234
+ }
2163
3235
  let text: string;
2164
3236
  try {
2165
3237
  text = fs.readFileSync(resolved, "utf8");
2166
3238
  } catch {
2167
- step.skipReason = `each: rows — ${raw} does not exist (yet?)`;
2168
- return [];
3239
+ return { problem: `${raw} does not exist (yet?)` };
2169
3240
  }
2170
3241
  // Minimal CSV: quoted fields with embedded commas/newlines survive.
2171
3242
  const rows: string[] = [];
@@ -2182,11 +3253,28 @@ function csvItems(workspaceDir: string, step: StepRecord, take: number): string[
2182
3253
  }
2183
3254
  if (row + field) rows.push(row + field);
2184
3255
  const [header, ...data] = rows.filter((r) => r.trim() !== "");
2185
- if (!header || data.length === 0) {
2186
- step.skipReason = `each: rows — ${raw} has no data rows`;
2187
- return [];
3256
+ if (!header || data.length === 0) return { problem: `${raw} has no data rows` };
3257
+ return { header, data };
3258
+ }
3259
+
3260
+ /**
3261
+ * `schema: <path>` — the file, read and parsed, or null. Agent-relative like
3262
+ * every other path in a flow (`workspace/schemas/lead.json`), confined to the
3263
+ * workspace: a schema is the author's document, never something a step
3264
+ * can point outside.
3265
+ */
3266
+ export function readSchemaFile(workspaceRoot: string, rel: string): Record<string, unknown> | boolean | null {
3267
+ const root = path.resolve(workspaceRoot);
3268
+ // Written `workspace/schemas/x.json`, agent-relative (`../../schemas/x.json`)
3269
+ // or workspace-relative (`schemas/x.json`); all land on the same file.
3270
+ const candidates = [path.resolve(root, rel.replace(/^((\.\.\/){2}|workspace\/)/, "")), path.resolve(root, "agents", "x", rel)];
3271
+ for (const abs of candidates) {
3272
+ if (abs !== root && !abs.startsWith(root + path.sep)) continue;
3273
+ if (!fs.existsSync(abs) || !fs.statSync(abs).isFile()) continue;
3274
+ const parsed = parseSchemaText(fs.readFileSync(abs, "utf8"));
3275
+ return "problem" in parsed ? null : parsed.schema;
2188
3276
  }
2189
- return data.slice(0, take).map((r) => `${header}\n${r}`);
3277
+ return null;
2190
3278
  }
2191
3279
 
2192
3280
  export function createFlowRun(
@@ -2196,12 +3284,16 @@ export function createFlowRun(
2196
3284
  flowName: string,
2197
3285
  status: "queued" | "running",
2198
3286
  tags: string[] = [],
3287
+ startedBy: string | null = null,
3288
+ opts: { test?: boolean } = {},
2199
3289
  ): RunRecord {
2200
3290
  const run: RunRecord = {
2201
3291
  id: `run-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`,
2202
3292
  flow: flowName,
2203
3293
  tags,
2204
3294
  status,
3295
+ ...(startedBy ? { startedBy: startedBy.trim().toLowerCase() } : {}),
3296
+ ...(opts.test ? { test: true } : {}),
2205
3297
  startedAt: new Date().toISOString(),
2206
3298
  finishedAt: null,
2207
3299
  steps: steps.map((s) => ({
@@ -2231,6 +3323,11 @@ export function createFlowRun(
2231
3323
  preview: s.preview,
2232
3324
  delegate: s.delegate,
2233
3325
  output: s.output,
3326
+ schema: s.schema,
3327
+ schemaPath: s.schemaPath,
3328
+ parallel: s.parallel,
3329
+ maxTurns: s.maxTurns,
3330
+ limits: s.limits,
2234
3331
  attempts: 0,
2235
3332
  status: "pending",
2236
3333
  events: [],
@@ -2267,8 +3364,10 @@ export function startFlowRun(
2267
3364
  modelOverride?: string | null,
2268
3365
  tags: string[] = [],
2269
3366
  effortOverride?: string | null,
3367
+ startedBy: string | null = null,
3368
+ opts: { test?: boolean } = {},
2270
3369
  ): RunRecord {
2271
- const run = createFlowRun(tenant, workspace, steps, flowName, "running", tags);
3370
+ const run = createFlowRun(tenant, workspace, steps, flowName, "running", tags, startedBy, opts);
2272
3371
  void driveRun(tenant, workspace, run, modelOverride, tags, { effortOverride });
2273
3372
  return run;
2274
3373
  }
@@ -2384,6 +3483,18 @@ function driveRunInner(
2384
3483
  opts: { parkOnApproval?: boolean; effortOverride?: string | null },
2385
3484
  ): Promise<void> {
2386
3485
  const effortOverride = opts.effortOverride ?? null;
3486
+ // The flow's own calendar. Read from the flow file rather than passed in,
3487
+ // so every door into a run — the queue, an eval, the CLI, a rerun, a
3488
+ // reconcile that picks an abandoned run back up — gets the same clock
3489
+ // without each caller having to remember it. A run that belongs to no
3490
+ // flow (an ad-hoc agent run) finds nothing here and cascades from the
3491
+ // agent to the workspace, which is what it should do.
3492
+ let flowTimezone: string | null = null;
3493
+ try {
3494
+ flowTimezone = readFlow(tenant, workspace, run.flow)?.timezone ?? null;
3495
+ } catch {
3496
+ // an unreadable flow file is reported where it is parsed for real
3497
+ }
2387
3498
  const pDir = workspaceDir(tenant, workspace);
2388
3499
  const runRoot = pDir;
2389
3500
  /**
@@ -2566,6 +3677,17 @@ function driveRunInner(
2566
3677
  // is assembled) keeps the newest: when the total is over it, the
2567
3678
  // oldest groups are dropped whole before anything is cut mid-text.
2568
3679
  const contextBefore = (gi: number) => joinEarlierGroups(groupResults, gi);
3680
+ // What `when:` and `case:` read: the PREVIOUS group's result alone —
3681
+ // the nearest earlier group that produced one, so a routed-past group
3682
+ // between is looked through rather than read as silence. The prompt
3683
+ // gets every earlier group joined (contextBefore); routing must not,
3684
+ // because a verdict two groups back would still open a gate a later
3685
+ // group had overruled: a classifier's BUG line matched a `case: BUG`
3686
+ // three groups on, after the debugger had already answered it.
3687
+ const routingBefore = (gi: number): string | null => {
3688
+ for (let i = gi - 1; i >= 0; i--) if (groupResults[i]) return groupResults[i];
3689
+ return null;
3690
+ };
2569
3691
  // The data half of the same handoff: what the nearest earlier group
2570
3692
  // returned through `output: json`. Rebuilt from the record rather than
2571
3693
  // kept only in memory, so a resumed run hands step 3 the value step 1
@@ -2630,8 +3752,11 @@ function driveRunInner(
2630
3752
  context = contextBefore(gi);
2631
3753
  ctxData = dataBefore(gi);
2632
3754
 
2633
- if (run.budgetUsd && runCost(run) >= run.budgetUsd && group.some((s) => s.status === "pending")) {
2634
- const spent = runCost(run);
3755
+ // What the run has spent, not just what its tokens cost: on an
3756
+ // install that prices sandbox seconds, a step that called no model
3757
+ // still spent money, and a cap that ignored it capped nothing.
3758
+ if (run.budgetUsd && platform.runSpend(run) >= run.budgetUsd && group.some((s) => s.status === "pending")) {
3759
+ const spent = platform.runSpend(run);
2635
3760
  const last = run.steps.findLast((s) => s.status === "completed" || s.status === "failed") ?? run.steps[0];
2636
3761
  last?.events.push({
2637
3762
  t: new Date().toISOString(),
@@ -2720,6 +3845,7 @@ function driveRunInner(
2720
3845
  type: "info",
2721
3846
  text:
2722
3847
  `fan-out: ${used.length} instance${used.length === 1 ? "" : "s"}` +
3848
+ (step.parallel && step.parallel < used.length ? `, ${step.parallel} at a time (parallel:)` : "") +
2723
3849
  (dropped ? ` (more items existed — capped at ${cap})` : ""),
2724
3850
  });
2725
3851
  const at = run.steps.indexOf(step);
@@ -2734,6 +3860,7 @@ function driveRunInner(
2734
3860
  result: null,
2735
3861
  costUsd: null,
2736
3862
  attempts: 0,
3863
+ tries: undefined,
2737
3864
  }));
2738
3865
  run.steps.splice(at + 1, 0, ...instances);
2739
3866
  ordered = orderedGroups();
@@ -2747,12 +3874,25 @@ function driveRunInner(
2747
3874
  // this is what keeps the *next* group from starting.
2748
3875
  const stopCheck = readRun(tenant, workspace, run.id);
2749
3876
  if (stopCheck?.stopRequested) {
3877
+ let skipped = 0;
2750
3878
  for (const s of run.steps) {
2751
3879
  if (s.status === "pending" || s.status === "running") {
2752
3880
  s.status = "skipped";
2753
3881
  s.skipReason = "run stopped";
3882
+ skipped += 1;
2754
3883
  }
2755
3884
  }
3885
+ // Say so on the trail. stopRun wrote "run stopped" onto the
3886
+ // record's first step, but the save() below writes this process's
3887
+ // copy of the record over it, so the only witness was the
3888
+ // stopRequested flag — and a run that "failed" with no error line
3889
+ // anywhere took a code read to explain (rank-desk, 2026-09-13).
3890
+ const witness = run.steps.findLast((s) => s.status === "completed" || s.status === "failed") ?? run.steps[0];
3891
+ witness?.events.push({
3892
+ t: new Date().toISOString(),
3893
+ type: "info",
3894
+ text: `run stopped by a person between groups — ${skipped} remaining step${skipped === 1 ? "" : "s"} skipped`,
3895
+ });
2756
3896
  run.status = "failed";
2757
3897
  run.stopRequested = true;
2758
3898
  save();
@@ -2788,40 +3928,67 @@ function driveRunInner(
2788
3928
  }
2789
3929
 
2790
3930
  const ctx = context;
3931
+ // The previous group's result — see routingBefore.
3932
+ const routing = routingBefore(gi);
2791
3933
 
2792
- // `when:` — skip a step whose condition isn't met by prior results.
3934
+ // `when:` — skip a step whose condition isn't met by the previous
3935
+ // group's result.
2793
3936
  for (const step of freshGroup) {
2794
3937
  if (step.status !== "pending" || !step.when) continue;
2795
- const met = (ctx ?? "").toLowerCase().includes(step.when.toLowerCase());
3938
+ // `when: rows of <csv>` asks the data, not the prose: run only if
3939
+ // the file has a data row. It is the condition a fan-out's
3940
+ // followers need — an `each:` over an empty file skips itself but
3941
+ // leaves nothing for a marker to be read from, so the steps after
3942
+ // it used to run on nothing.
3943
+ const rowsOf = whenRowsPath(step.when);
3944
+ if (rowsOf !== null) {
3945
+ const read = csvDataRows(pDir, step.agent, rowsOf);
3946
+ if ("problem" in read) {
3947
+ step.status = "skipped";
3948
+ step.skipReason = `condition not met: when rows — ${read.problem}`;
3949
+ step.events.push({ t: new Date().toISOString(), type: "info", text: `skipped — ${read.problem}` });
3950
+ }
3951
+ continue;
3952
+ }
3953
+ // A marker at the start of a line, not a word anywhere in the
3954
+ // prose — see markerPresent. "There are no BLOCKED items" must
3955
+ // not open a gate keyed on BLOCKED.
3956
+ const met = markerPresent(routing, step.when);
2796
3957
  if (!met) {
2797
3958
  step.status = "skipped";
2798
3959
  step.skipReason = `condition not met: when "${step.when}"`;
2799
3960
  step.events.push({
2800
3961
  t: new Date().toISOString(),
2801
3962
  type: "info",
2802
- text: `skipped — previous results do not mention "${step.when}"`,
3963
+ text: `skipped — the previous group's result has no line beginning "${step.when}" (when: reads a marker at the start of a line of the previous result, not a word in a sentence)`,
2803
3964
  });
2804
3965
  }
2805
3966
  }
2806
3967
  save();
2807
3968
 
2808
3969
  // `case:`/`else:` — exclusive routing. Of this group's case steps,
2809
- // the FIRST whose text appears in the previous results runs; the
2810
- // rest are routed past. `else:` runs only when no case matched.
3970
+ // the FIRST whose marker leads a line of the previous results runs;
3971
+ // the rest are routed past. `else:` runs only when no case matched.
2811
3972
  // (`when:` above stays independent — every matching when runs —
2812
3973
  // which is why routing is its own vocabulary instead of a mode.)
3974
+ //
3975
+ // Same marker rule as `when:`, and it matters more here: routing is
3976
+ // exclusive, so a label matched out of a sentence does not merely
3977
+ // run an extra step, it sends the flow down the wrong branch and
3978
+ // skips the right one. "This is not a COMPLAINT, it is a QUESTION"
3979
+ // used to route to the complaints handler.
2813
3980
  const caseSteps = freshGroup.filter((s) => s.status === "pending" && s.case);
2814
3981
  if (caseSteps.length || freshGroup.some((s) => s.status === "pending" && s.else)) {
2815
3982
  let matchedCase: StepRecord | null = null;
2816
3983
  for (const step of caseSteps) {
2817
- if (!matchedCase && (ctx ?? "").toLowerCase().includes(step.case!.toLowerCase())) {
3984
+ if (!matchedCase && markerPresent(routing, step.case!)) {
2818
3985
  matchedCase = step;
2819
3986
  continue;
2820
3987
  }
2821
3988
  step.status = "skipped";
2822
3989
  step.skipReason = matchedCase
2823
3990
  ? `routed past — "${matchedCase.case}" matched first`
2824
- : `case not matched: "${step.case}"`;
3991
+ : `case not matched: the previous group's result has no line beginning "${step.case}"`;
2825
3992
  step.events.push({
2826
3993
  t: new Date().toISOString(),
2827
3994
  type: "info",
@@ -2865,6 +4032,20 @@ function driveRunInner(
2865
4032
  });
2866
4033
  }
2867
4034
  run.status = "awaiting-approval";
4035
+ // approve_within: — the deadline is stamped when the gate opens,
4036
+ // not counted from the run's start, so "two days to answer" means
4037
+ // two days from being asked however long the work before it took.
4038
+ // Only a gate that asks a PERSON gets one: an external event has
4039
+ // no one to chase.
4040
+ const within = readFlow(tenant, workspace, run.flow)?.approveWithin ?? null;
4041
+ // Always assigned, never only set: a run with two gates would
4042
+ // otherwise carry the FIRST gate's deadline into the second, and
4043
+ // a deadline already in the past expires a gate the moment it
4044
+ // opens. Null when this gate has none.
4045
+ run.approveBy =
4046
+ within && needsApproval.some((s) => s.waitFor !== "event")
4047
+ ? new Date(Date.now() + within * 1000).toISOString()
4048
+ : null;
2868
4049
  save();
2869
4050
 
2870
4051
  // The gate's declared preview, resolved now against what the run
@@ -2939,10 +4120,55 @@ function driveRunInner(
2939
4120
  // twenty steps over — the shares sum to what was left. Between
2940
4121
  // groups the check above still refuses to start the next one.
2941
4122
  const launching = freshGroup.filter((s) => s.status === "pending");
2942
- const ceilingUsd = stepCeiling(run.budgetUsd, runCost(run), launching.length);
4123
+ // An agent's own `budget:` — the most it may spend in one run — is
4124
+ // read at launch, so a cap raised mid-run applies to the next step.
4125
+ const agentBudgets = new Map(listAgents(tenant, workspace).map((a) => [a.name, a.budget]));
4126
+ // The ceiling each attempt now running was given. A step records its
4127
+ // cost only when it ends, so while it runs its ceiling stands in for
4128
+ // what it may yet spend — otherwise every fan-out copy of one agent
4129
+ // saw none of its siblings' spend and got the agent's whole budget.
4130
+ const reserved = new Map<StepRecord, number>();
4131
+ // The ceiling for one attempt of one step, read NOW: what is left of
4132
+ // the run's budget shared across the group, or the agent's own cap
4133
+ // less what its steps have cost or hold in flight, shared with its
4134
+ // steps in this group still waiting to start. Computed before every
4135
+ // attempt, not once before the loop — a retry's share is what is left
4136
+ // after the failed attempt spent, and the step's own figure already
4137
+ // carries that spend (it is the sum of its tries).
4138
+ const ceilingFor = (step: StepRecord) =>
4139
+ stepCeilingFor(
4140
+ stepCeiling(run.budgetUsd, platform.runSpend(run), launching.length),
4141
+ agentBudgets.get(step.agent),
4142
+ run.steps
4143
+ .filter((s) => s.agent === step.agent)
4144
+ .reduce((sum, s) => sum + (s.costUsd ?? 0) + (s !== step && s.status === "running" ? (reserved.get(s) ?? 0) : 0), 0),
4145
+ step.agent,
4146
+ 1 + launching.filter((s) => s !== step && s.agent === step.agent && s.status === "pending" && !reserved.has(s)).length,
4147
+ );
4148
+ // parallel: — the instances of one fan-out template share a pool.
4149
+ // Keyed by what the instances share (agent + instruction), since
4150
+ // the template itself is behind them in the record as "expanded".
4151
+ const pools = new Map<string, ReturnType<typeof slotPool>>();
4152
+ const poolFor = (step: StepRecord) => {
4153
+ if (!step.parallel || !step.item) return null;
4154
+ const key = `${step.agent}\u0000${step.instruction}`;
4155
+ let pool = pools.get(key);
4156
+ if (!pool) pools.set(key, (pool = slotPool(step.parallel)));
4157
+ return pool;
4158
+ };
2943
4159
  await Promise.all(
2944
4160
  launching
2945
4161
  .map(async (step) => {
4162
+ const pool = poolFor(step);
4163
+ if (pool) await pool.acquire();
4164
+ try {
4165
+ await launchStep(step);
4166
+ } finally {
4167
+ pool?.release();
4168
+ }
4169
+ }),
4170
+ );
4171
+ async function launchStep(step: StepRecord) {
2946
4172
  const attempts = (step.retry ?? 0) + 1;
2947
4173
  // A step re-attaching to its sandbox is still on the attempt
2948
4174
  // that started it; a step re-run after a backoff picks up the
@@ -2951,10 +4177,36 @@ function driveRunInner(
2951
4177
  !step.sandbox && !!step.attempts && /retrying in \d+s/.test(step.events[step.events.length - 1]?.text ?? "");
2952
4178
  const from = step.sandbox ? Math.max(1, step.attempts ?? 1) : interruptedInBackoff ? Math.min(attempts, step.attempts! + 1) : 1;
2953
4179
  for (let attempt = from; attempt <= attempts; attempt++) {
4180
+ const ceiling = ceilingFor(step);
4181
+ // Nothing left is a refusal, not "no ceiling": a zero passed
4182
+ // down would read as unset, and the one step that must not
4183
+ // run would run with no cap at all. On a retry this is what
4184
+ // stops the second attempt spending a share the first one
4185
+ // already spent.
4186
+ if (ceiling.ceilingUsd === 0) {
4187
+ step.status = "failed";
4188
+ step.events.push({
4189
+ t: new Date().toISOString(),
4190
+ type: "error",
4191
+ text:
4192
+ attempt === from
4193
+ ? `over budget — nothing left to spend before this step (${ceiling.note}); it was not started`
4194
+ : `over budget — nothing left to spend for attempt ${attempt} (${ceiling.note}); not retried`,
4195
+ });
4196
+ save();
4197
+ return;
4198
+ }
4199
+ // Re-attaching to a sandbox is the same attempt, so its row
4200
+ // is replaced; anything else is a new row, a loop cycle's
4201
+ // re-run included.
4202
+ const reattach = !!step.sandbox;
4203
+ const attemptStartedAt = new Date().toISOString();
4204
+ const eventsBefore = step.events.length;
2954
4205
  step.attempts = attempt;
2955
4206
  step.status = "running";
4207
+ if (ceiling.ceilingUsd !== null) reserved.set(step, ceiling.ceilingUsd);
2956
4208
  save();
2957
- await runStep(
4209
+ const verdict = await runStep(
2958
4210
  path.join(pDir, "agents", step.agent),
2959
4211
  tenant,
2960
4212
  step,
@@ -2965,16 +4217,58 @@ function driveRunInner(
2965
4217
  effortOverride,
2966
4218
  run.id,
2967
4219
  ctxData,
2968
- ceilingUsd,
2969
- );
4220
+ ceiling.ceilingUsd,
4221
+ ceiling.note,
4222
+ run.test === true,
4223
+ // save() merges a stop written by another process into
4224
+ // this record; the model loop polls this on a clock.
4225
+ () => {
4226
+ save();
4227
+ return run.stopRequested === true;
4228
+ },
4229
+ flowTimezone,
4230
+ run.steps.indexOf(step),
4231
+ ).finally(() => reserved.delete(step));
2970
4232
  // runStep mutates step.status; read it through a widened local
2971
4233
  // so TS doesn't keep the "running" narrowing from above.
2972
4234
  const outcome: string = step.status;
2973
- if (outcome !== "failed" || attempt === attempts) break;
4235
+ recordAttempt(step, attempt, outcome === "completed" ? "completed" : "failed", attemptStartedAt, eventsBefore, reattach);
4236
+ save();
2974
4237
  // A stop destroys the step's sandbox, which reads here as a
2975
- // failed attempt — and a failed attempt used to be retried in
2976
- // a fresh sandbox, so a stopped `retry: 2` step ran twice
2977
- // more. save() merges the flag from the record.
4238
+ // failed attempt. stopRun writes "stopped by a person" onto
4239
+ // this step in the record, and the save() above writes this
4240
+ // process's copy over it — so the trail ended on the
4241
+ // sandbox's own last words, "sandbox ended: Error (exit 143)",
4242
+ // and a person's stop read as the platform killing a healthy
4243
+ // step (abandon-desk, 2026-09-29: stopped from the CLI after
4244
+ // ten minutes, then chased as an unexplained kill). Say so,
4245
+ // whether or not a retry was left.
4246
+ if (outcome === "failed" && run.stopRequested) {
4247
+ step.events.push({
4248
+ t: new Date().toISOString(),
4249
+ type: "info",
4250
+ text: "stopped by a person mid-step — its sandbox was destroyed, which is what ended this attempt; not retried",
4251
+ });
4252
+ save();
4253
+ break;
4254
+ }
4255
+ if (outcome !== "failed" || attempt === attempts) break;
4256
+ // An outward step whose tools ran and whose check then failed
4257
+ // is not retried: the check cannot say the send did not
4258
+ // happen, and a retry would do it twice.
4259
+ if (verdict?.actedThenFailedCheck) {
4260
+ step.events.push({
4261
+ t: new Date().toISOString(),
4262
+ type: "info",
4263
+ text: `not retried — this step's tools can act outside the workspace and ran before the check failed; another attempt could repeat what they did (${attempts - attempt} left unused)`,
4264
+ });
4265
+ save();
4266
+ break;
4267
+ }
4268
+ // A failed attempt used to be retried in a fresh sandbox even
4269
+ // after a stop, so a stopped `retry: 2` step ran twice more.
4270
+ // The check above ends the loop on a stop; this one covers a
4271
+ // stop that lands while the outward check was being written.
2978
4272
  save();
2979
4273
  if (run.stopRequested) break;
2980
4274
  // The retry policy. An attempt the cluster ended for want of
@@ -3014,8 +4308,7 @@ function driveRunInner(
3014
4308
  break;
3015
4309
  }
3016
4310
  }
3017
- }),
3018
- );
4311
+ }
3019
4312
 
3020
4313
  // on-fail: another agent takes the step over. Sequential and rare:
3021
4314
  // the rescue is a second full attempt at the step's job with the
@@ -3067,7 +4360,7 @@ function driveRunInner(
3067
4360
  rescue.attempts = 1;
3068
4361
  rescue.status = "running";
3069
4362
  save();
3070
- await runStep(rescuerDir, tenant, rescue, ctx, save, modelOverride, tags, effortOverride, run.id, ctxData);
4363
+ await runStep(rescuerDir, tenant, rescue, ctx, save, modelOverride, tags, effortOverride, run.id, ctxData, null, undefined, run.test === true, undefined, flowTimezone, run.steps.indexOf(rescue));
3071
4364
  // runStep mutates rescue.status; widen past TS's "running" narrowing.
3072
4365
  const rescueOutcome: string = rescue.status;
3073
4366
  if (rescueOutcome === "completed") {
@@ -3197,16 +4490,25 @@ function driveRunInner(
3197
4490
  for (const s of run.steps) if (s.status === "pending") s.status = "skipped";
3198
4491
  } catch (err) {
3199
4492
  run.status = "failed";
4493
+ const text = err instanceof Error ? err.message : String(err);
4494
+ let told = false;
3200
4495
  run.steps.forEach((s) => {
3201
4496
  if (s.status === "running") {
3202
4497
  s.status = "failed";
3203
- s.events.push({
3204
- t: new Date().toISOString(),
3205
- type: "error",
3206
- text: err instanceof Error ? err.message : String(err),
3207
- });
4498
+ s.events.push({ t: new Date().toISOString(), type: "error", text });
4499
+ told = true;
3208
4500
  }
3209
4501
  });
4502
+ // Thrown between groups, with nothing running: the error still has to
4503
+ // land on some step's trail, or the run reads as failed for no reason.
4504
+ if (!told) {
4505
+ const witness = run.steps.findLast((s) => s.status !== "pending") ?? run.steps[0];
4506
+ witness?.events.push({
4507
+ t: new Date().toISOString(),
4508
+ type: "error",
4509
+ text: `run failed between steps: ${text}`,
4510
+ });
4511
+ }
3210
4512
  } finally {
3211
4513
  if (!parked) {
3212
4514
  run.finishedAt = new Date().toISOString();
@@ -3216,6 +4518,12 @@ function driveRunInner(
3216
4518
  // learn, and the runs list and the notification stop having to say
3217
4519
  // only "completed".
3218
4520
  run.summary = runSummary(run);
4521
+ // Read before a test run's prefix goes on the summary, which would
4522
+ // otherwise hide the word it leads with.
4523
+ run.verdict = verdictOf(run.summary);
4524
+ // A test run's headline says so wherever the headline goes — the
4525
+ // runs list, the history tool, a later run's recall.
4526
+ if (run.test && run.summary) run.summary = testHeadline(run.summary);
3219
4527
  // Archive before the next run resets outputs/ — a failed run's
3220
4528
  // artifacts are usually the most interesting ones, so this runs on
3221
4529
  // failure too.
@@ -3256,16 +4564,6 @@ function driveRunInner(
3256
4564
  })();
3257
4565
  }
3258
4566
 
3259
- function readFlow(tenant: string, workspace: string, flowName: string) {
3260
- const dir = path.join(workspaceDir(tenant, workspace), "flows");
3261
- if (!fs.existsSync(dir)) return null;
3262
- for (const f of fs.readdirSync(dir).filter((f) => f.endsWith(".md"))) {
3263
- const flow = parseFlow(f, fs.readFileSync(path.join(dir, f), "utf8"));
3264
- if (flow.name === flowName) return flow;
3265
- }
3266
- return null;
3267
- }
3268
-
3269
4567
  // Splice `[[flow:other]]` steps into the parent's step list, renumbering
3270
4568
  // groups so the nested flow's own parallelism survives. Depth-limited, and
3271
4569
  // a cycle raises rather than expanding forever.