@foldrun/core 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (166) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +239 -0
  3. package/dist/index.d.ts +32 -0
  4. package/dist/index.js +43 -0
  5. package/dist/src/agent-tools.d.ts +31 -0
  6. package/dist/src/agent-tools.js +99 -0
  7. package/dist/src/agents-md.d.ts +9 -0
  8. package/dist/src/agents-md.js +83 -0
  9. package/dist/src/api-tools.d.ts +38 -0
  10. package/dist/src/api-tools.js +313 -0
  11. package/dist/src/approvals.d.ts +39 -0
  12. package/dist/src/approvals.js +103 -0
  13. package/dist/src/arrange.d.ts +8 -0
  14. package/dist/src/arrange.js +40 -0
  15. package/dist/src/catalog.d.ts +108 -0
  16. package/dist/src/catalog.js +255 -0
  17. package/dist/src/completions.d.ts +61 -0
  18. package/dist/src/completions.js +578 -0
  19. package/dist/src/confine.d.ts +31 -0
  20. package/dist/src/confine.js +298 -0
  21. package/dist/src/container.d.ts +30 -0
  22. package/dist/src/container.js +172 -0
  23. package/dist/src/context-tools.d.ts +81 -0
  24. package/dist/src/context-tools.js +347 -0
  25. package/dist/src/deploy.d.ts +100 -0
  26. package/dist/src/deploy.js +377 -0
  27. package/dist/src/egress.d.ts +75 -0
  28. package/dist/src/egress.js +133 -0
  29. package/dist/src/evals.d.ts +150 -0
  30. package/dist/src/evals.js +651 -0
  31. package/dist/src/flow-lint.d.ts +16 -0
  32. package/dist/src/flow-lint.js +177 -0
  33. package/dist/src/git-hook.d.ts +1 -0
  34. package/dist/src/git-hook.js +70 -0
  35. package/dist/src/git.d.ts +37 -0
  36. package/dist/src/git.js +101 -0
  37. package/dist/src/gitrepo.d.ts +95 -0
  38. package/dist/src/gitrepo.js +355 -0
  39. package/dist/src/history.d.ts +82 -0
  40. package/dist/src/history.js +259 -0
  41. package/dist/src/kinds.d.ts +95 -0
  42. package/dist/src/kinds.js +421 -0
  43. package/dist/src/library.d.ts +94 -0
  44. package/dist/src/library.js +341 -0
  45. package/dist/src/linkable.d.ts +20 -0
  46. package/dist/src/linkable.js +113 -0
  47. package/dist/src/model-probe.d.ts +21 -0
  48. package/dist/src/model-probe.js +81 -0
  49. package/dist/src/notify.d.ts +61 -0
  50. package/dist/src/notify.js +252 -0
  51. package/dist/src/oauth-presets.d.ts +13 -0
  52. package/dist/src/oauth-presets.js +39 -0
  53. package/dist/src/okf.d.ts +220 -0
  54. package/dist/src/okf.js +656 -0
  55. package/dist/src/openapi.d.ts +66 -0
  56. package/dist/src/openapi.js +342 -0
  57. package/dist/src/paths.d.ts +33 -0
  58. package/dist/src/paths.js +78 -0
  59. package/dist/src/platform.d.ts +32 -0
  60. package/dist/src/platform.js +48 -0
  61. package/dist/src/providers.d.ts +47 -0
  62. package/dist/src/providers.js +114 -0
  63. package/dist/src/refs.d.ts +20 -0
  64. package/dist/src/refs.js +78 -0
  65. package/dist/src/run-container.d.ts +227 -0
  66. package/dist/src/run-container.js +760 -0
  67. package/dist/src/runner.d.ts +241 -0
  68. package/dist/src/runner.js +3215 -0
  69. package/dist/src/runtime.d.ts +41 -0
  70. package/dist/src/runtime.js +357 -0
  71. package/dist/src/script-tools.d.ts +62 -0
  72. package/dist/src/script-tools.js +296 -0
  73. package/dist/src/secret-files.d.ts +12 -0
  74. package/dist/src/secret-files.js +92 -0
  75. package/dist/src/secrets.d.ts +147 -0
  76. package/dist/src/secrets.js +626 -0
  77. package/dist/src/starter.d.ts +21 -0
  78. package/dist/src/starter.js +399 -0
  79. package/dist/src/step-exec.d.ts +136 -0
  80. package/dist/src/step-exec.js +391 -0
  81. package/dist/src/storage.d.ts +123 -0
  82. package/dist/src/storage.js +640 -0
  83. package/dist/src/store.d.ts +1052 -0
  84. package/dist/src/store.js +2378 -0
  85. package/dist/src/tar.d.ts +30 -0
  86. package/dist/src/tar.js +111 -0
  87. package/dist/src/tool-names.d.ts +39 -0
  88. package/dist/src/tool-names.js +79 -0
  89. package/dist/src/tool-programs.d.ts +18 -0
  90. package/dist/src/tool-programs.js +52 -0
  91. package/dist/src/tool-test.d.ts +19 -0
  92. package/dist/src/tool-test.js +335 -0
  93. package/dist/src/translator.d.ts +159 -0
  94. package/dist/src/translator.js +986 -0
  95. package/dist/src/triggers.d.ts +60 -0
  96. package/dist/src/triggers.js +270 -0
  97. package/dist/src/webhook.d.ts +47 -0
  98. package/dist/src/webhook.js +148 -0
  99. package/index.ts +44 -0
  100. package/package.json +64 -0
  101. package/src/agent-tools.ts +138 -0
  102. package/src/agents-md.ts +81 -0
  103. package/src/api-tools.ts +389 -0
  104. package/src/approvals.ts +129 -0
  105. package/src/arrange.ts +42 -0
  106. package/src/catalog.ts +321 -0
  107. package/src/completions.ts +652 -0
  108. package/src/confine.ts +350 -0
  109. package/src/container.ts +220 -0
  110. package/src/context-tools.ts +435 -0
  111. package/src/deploy.ts +458 -0
  112. package/src/egress.ts +176 -0
  113. package/src/evals.ts +812 -0
  114. package/src/flow-lint.ts +215 -0
  115. package/src/git-hook.ts +73 -0
  116. package/src/git.ts +117 -0
  117. package/src/gitrepo.ts +391 -0
  118. package/src/history.ts +329 -0
  119. package/src/kinds.ts +507 -0
  120. package/src/library.ts +419 -0
  121. package/src/linkable.ts +123 -0
  122. package/src/model-probe.ts +112 -0
  123. package/src/notify.ts +281 -0
  124. package/src/oauth-presets.ts +54 -0
  125. package/src/okf.ts +768 -0
  126. package/src/openapi.ts +383 -0
  127. package/src/paths.ts +79 -0
  128. package/src/platform.ts +87 -0
  129. package/src/providers.ts +147 -0
  130. package/src/refs.ts +83 -0
  131. package/src/run-container.ts +883 -0
  132. package/src/runner.ts +3446 -0
  133. package/src/runtime.ts +404 -0
  134. package/src/script-tools.ts +383 -0
  135. package/src/secret-files.ts +101 -0
  136. package/src/secrets.ts +799 -0
  137. package/src/starter.ts +415 -0
  138. package/src/step-exec.ts +498 -0
  139. package/src/storage.ts +771 -0
  140. package/src/store.ts +2985 -0
  141. package/src/tar.ts +124 -0
  142. package/src/tool-names.ts +91 -0
  143. package/src/tool-programs.ts +65 -0
  144. package/src/tool-test.ts +401 -0
  145. package/src/translator.ts +1029 -0
  146. package/src/triggers.ts +287 -0
  147. package/src/webhook.ts +183 -0
  148. package/templates/blog-desk/AGENTS.md +13 -0
  149. package/templates/blog-desk/agents/researcher/agent.md +11 -0
  150. package/templates/blog-desk/agents/writer/agent.md +8 -0
  151. package/templates/blog-desk/evals/writer-quality.md +10 -0
  152. package/templates/blog-desk/flows/publish.md +10 -0
  153. package/templates/blog-desk/knowledge/house-style.md +9 -0
  154. package/templates/blog-desk/knowledge/index.md +9 -0
  155. package/templates/hello/AGENTS.md +18 -0
  156. package/templates/hello/agents/curator/agent.md +32 -0
  157. package/templates/hello/agents/notetaker/agent.md +24 -0
  158. package/templates/hello/evals/house-style.md +16 -0
  159. package/templates/hello/flows/memory-review.md +7 -0
  160. package/templates/hello/flows/note.md +7 -0
  161. package/templates/hello/knowledge/house-style.md +15 -0
  162. package/templates/hello/knowledge/index.md +9 -0
  163. package/templates/hello/skills/plain-english/SKILL.md +11 -0
  164. package/templates/hello/state/publishing.md +7 -0
  165. package/templates/hello/tools/wordcount/run.py +13 -0
  166. package/templates/hello/tools/wordcount/tool.md +22 -0
package/src/runner.ts ADDED
@@ -0,0 +1,3446 @@
1
+ // Flow orchestration. Steps sharing a group number run in parallel; the next
2
+ // group receives every prior result, labeled per agent. Optional steps ("2?")
3
+ // fail without failing the flow. A direct agent run is a one-step ad-hoc
4
+ // flow, so there is a single execution path.
5
+
6
+ import fs from "node:fs";
7
+ import path from "node:path";
8
+ import matter from "gray-matter";
9
+ import { spawn } from "node:child_process";
10
+ import { executeStep, extractJson, stepCeiling, type EventExtra } from "./step-exec.ts";
11
+ import { eventUrl } from "./webhook.ts";
12
+ import { runStepInContainer, sizeLimits, killRunSandboxes, type StepTiming } from "./run-container.ts";
13
+ import { EGRESS_ENV, MODEL_KEY_NAME, addGrant, hostOf, placeholderNames, proxyModelEnv, unsubstitute, type EgressGrant } from "./egress.ts";
14
+ import { platform } from "./platform.ts";
15
+
16
+ /** Does this process run steps in a sandbox — the container core ships, or
17
+ * one the platform registered (a pod)? */
18
+ const isolatedRun = () => {
19
+ const mode = process.env.FOLDRUN_RUN_ISOLATION;
20
+ return mode === "container" || (!!mode && mode in platform.isolation);
21
+ };
22
+ import { gatherConsults, buildConsultTools } from "./agent-tools.ts";
23
+ import { TOOL_MAP, BUILTIN_TOOLS, ownToolNames, toolRefs, legacyUseNames, legacyUseError } from "./tool-names.ts";
24
+ import { refNames } from "./refs.ts";
25
+ import {
26
+ accountDir,
27
+ workspaceDir,
28
+ buildMemoryIndex,
29
+ workspaceMemoryIndex,
30
+ knowledgeIndex,
31
+ workspaceKnowledgeIndex,
32
+ workspaceTools,
33
+ brokenToolReport,
34
+ listWorkspaces,
35
+ listTenants,
36
+ listRuns,
37
+ syncBundleFor,
38
+ parseToolDef,
39
+ parseFlow,
40
+ parseApis,
41
+ resolveModel,
42
+ parseProvider,
43
+ providerEnvFor,
44
+ resolveEffort,
45
+ isEffortWord,
46
+ EFFORT_LEVELS,
47
+ checkFormatVersion,
48
+ readRun,
49
+ writeRun,
50
+ runSummary,
51
+ type FlowStep,
52
+ type McpSpec,
53
+ type RunRecord,
54
+ type StepRecord,
55
+ STORAGE_DIR,
56
+ adoptLegacyVersionKey,
57
+ runCost,
58
+ } from "./store.ts";
59
+ import { loadCatalog, checkModel, clampEffort, catalogCost, findModel, type Catalog } from "./catalog.ts";
60
+ import { resolveSecrets, getSecret, materializeSecrets } from "./secrets.ts";
61
+ import { materializeFileSecrets, cleanupFileSecrets } from "./secret-files.ts";
62
+ import { buildApiTools, secretsUsedByApi } from "./api-tools.ts";
63
+ import { attachOperations, prefetchOpenApi } from "./openapi.ts";
64
+ import { buildSearchTools, buildHistoryTools, buildDeskTools, digestRuns, type SearchRoot, type RunDigest } from "./context-tools.ts";
65
+ import { startTranslator, translatorSpecFor, type TranslatorSpec } from "./translator.ts";
66
+ import { providerPreset } from "./providers.ts";
67
+ import { buildScriptTools, parseScripts, type ExecutionContext } from "./script-tools.ts";
68
+ import { libraryDir, libraryTools, libraryMemoryIndex } from "./library.ts";
69
+ import { mergeRuntimes, parseRuntime, prepareRuntime, type RuntimeSpec } from "./runtime.ts";
70
+ import { materializeFiles, harvestFiles } from "./storage.ts";
71
+ import { chooseExecutor, ensureImage } from "./container.ts";
72
+ import { stampBundle } from "./okf.ts";
73
+ import type { McpServerConfig } from "@anthropic-ai/claude-agent-sdk";
74
+
75
+ // An MCP tool definition becomes an SDK server config. ${SECRET} placeholders
76
+ // in env and headers resolve server-side, so a credential reaches the server
77
+ // process without ever entering the model's context.
78
+ function mcpConfig(spec: McpSpec, tenant: string, workspace: string): McpServerConfig {
79
+ const fill = (values: Record<string, string>) => {
80
+ const out: Record<string, string> = {};
81
+ for (const [k, v] of Object.entries(values)) {
82
+ out[k] = v.replace(/\$\{([A-Z][A-Z0-9_]*)\}/g, (whole, secretName) => {
83
+ const hit = getSecret(tenant, secretName, workspace);
84
+ return hit ? hit.value : whole;
85
+ });
86
+ }
87
+ return out;
88
+ };
89
+ // alwaysLoad: the SDK defers MCP tools behind tool search by default, which
90
+ // hides them from an agent whose toolset we've already narrowed. A server
91
+ // the author explicitly granted should be present from turn one.
92
+ return spec.url
93
+ ? { type: "http", url: spec.url, headers: fill(spec.headers), alwaysLoad: true }
94
+ : {
95
+ type: "stdio",
96
+ command: spec.command!,
97
+ args: spec.args,
98
+ env: fill(spec.env),
99
+ alwaysLoad: true,
100
+ };
101
+ }
102
+
103
+
104
+ /** How much of the earlier groups' output a step is handed, in characters.
105
+ * Newest groups are kept when the total is over it. */
106
+ export const CONTEXT_CHARS = 30_000;
107
+ /** tools: [desks] — how many other-workspace runs cross into a step, in all. */
108
+ export const DESK_RUNS_CAP = 100;
109
+
110
+ /**
111
+ * What a group is handed from the groups before it: all of them, oldest
112
+ * first, so a group-3 reporter sees what group 1 measured and not only what
113
+ * group 2 diagnosed. When the total is over the cap, the oldest groups are
114
+ * dropped whole before anything is cut mid-text — the newest is the one the
115
+ * step is most likely to be about.
116
+ */
117
+ export function joinEarlierGroups(groupResults: (string | null)[], gi: number, cap = CONTEXT_CHARS): string | null {
118
+ const parts: string[] = [];
119
+ for (let i = 0; i < gi; i++) if (groupResults[i]) parts.push(groupResults[i]!);
120
+ if (parts.length === 0) return null;
121
+ while (parts.length > 1 && parts.reduce((n, t) => n + t.length + 2, 0) > cap) parts.shift();
122
+ return parts.join("\n\n");
123
+ }
124
+
125
+ export interface DiscoveredSkill {
126
+ name: string;
127
+ description: string;
128
+ /** Tags that must be present on the run for this skill to load. A skill
129
+ * with none always loads. Mirrors "apply when this check failed". */
130
+ when: string[];
131
+ /** Path to SKILL.md (or the flat .md), relative to the agent directory. */
132
+ path: string;
133
+ /** Skill folder relative to the agent directory, or null for flat skills. */
134
+ dir: string | null;
135
+ hasScripts: boolean;
136
+ /** The rest of the Agent Skills layout: references/ and assets/. */
137
+ hasReferences: boolean;
138
+ hasAssets: boolean;
139
+ }
140
+
141
+ const toTags = (v: unknown): string[] =>
142
+ Array.isArray(v) ? v.map(String) : typeof v === "string" && v.trim() ? [v.trim()] : [];
143
+
144
+ // gray-matter throws on invalid YAML. Skills authored for another client
145
+ // sometimes ship YAML that that client's parser happened to accept — most
146
+ // often an unquoted value containing a colon (`description: Use when: ...`).
147
+ // The Agent Skills client guide recommends quoting such values and retrying
148
+ // rather than dropping the skill, so a cross-client skill still loads here.
149
+ function safeMatter(raw: string): { data: Record<string, unknown> } {
150
+ // gray-matter caches by input string AND caches a *failed* parse: the first
151
+ // call on invalid YAML throws, but every later call on the same bytes returns
152
+ // a hollow object with the fields silently missing. Clearing the cache makes
153
+ // the throw deterministic, so the fallback below fires every time rather than
154
+ // once — otherwise the same skill loads on one run and vanishes on the next.
155
+ (matter as unknown as { clearCache: () => void }).clearCache();
156
+ try {
157
+ return matter(raw) as { data: Record<string, unknown> };
158
+ } catch {
159
+ const fm = raw.match(/^---\r?\n([\s\S]*?)\r?\n---/);
160
+ if (fm) {
161
+ const fixed = fm[1].replace(
162
+ /^([A-Za-z0-9_-]+):[ \t]+(?!["'>|])(.*:.*)$/gm,
163
+ (_m, k: string, v: string) => `${k}: ${JSON.stringify(v)}`,
164
+ );
165
+ try {
166
+ // A function replacer, not a string: a fixed block containing `$&` or
167
+ // `$$` (a value with those in it) would otherwise be treated as a
168
+ // replacement pattern and splice the whole frontmatter into itself.
169
+ return matter(raw.replace(fm[1], () => fixed)) as { data: Record<string, unknown> };
170
+ } catch {
171
+ /* fall through */
172
+ }
173
+ }
174
+ return { data: {} };
175
+ }
176
+ }
177
+
178
+ // Both layouts: <root>/<name>/SKILL.md (Agent Skills standard) and the flat
179
+ // <root>/<name>.md convenience form. `subdir` is the skills root under the
180
+ // scanned directory — "skills" (foldrun-native) or ".agents/skills" (the
181
+ // cross-client convention other skills-compatible tools read and write).
182
+ export function discoverSkills(agentDir: string, subdir = "skills"): DiscoveredSkill[] {
183
+ const skillsDir = path.join(agentDir, subdir);
184
+ if (!fs.existsSync(skillsDir)) return [];
185
+ const out: DiscoveredSkill[] = [];
186
+
187
+ for (const entry of fs.readdirSync(skillsDir).sort()) {
188
+ const full = path.join(skillsDir, entry);
189
+ const stat = fs.statSync(full);
190
+
191
+ if (stat.isDirectory()) {
192
+ const skillMd = path.join(full, "SKILL.md");
193
+ if (!fs.existsSync(skillMd)) continue;
194
+ const { data } = safeMatter(fs.readFileSync(skillMd, "utf8"));
195
+ const description = typeof data.description === "string" ? data.description.trim() : "";
196
+ // The spec makes description essential — it is the only thing the model
197
+ // sees before activating the skill. Without one there is nothing to
198
+ // disclose, so the skill is skipped rather than offered blind. `check`
199
+ // reports it; here it is simply not loaded.
200
+ if (!description) continue;
201
+ out.push({
202
+ name: typeof data.name === "string" && data.name ? data.name : entry,
203
+ description,
204
+ when: toTags(data.when),
205
+ path: `${subdir}/${entry}/SKILL.md`,
206
+ dir: `${subdir}/${entry}`,
207
+ hasScripts: fs.existsSync(path.join(full, "scripts")),
208
+ hasReferences: fs.existsSync(path.join(full, "references")),
209
+ hasAssets: fs.existsSync(path.join(full, "assets")),
210
+ });
211
+ } else if (entry.endsWith(".md")) {
212
+ const { data } = safeMatter(fs.readFileSync(full, "utf8"));
213
+ const description = typeof data.description === "string" ? data.description.trim() : "";
214
+ if (!description) continue;
215
+ out.push({
216
+ name: typeof data.name === "string" && data.name ? data.name : entry.replace(/\.md$/, ""),
217
+ description,
218
+ when: toTags(data.when),
219
+ path: `${subdir}/${entry}`,
220
+ dir: null,
221
+ hasScripts: false,
222
+ hasReferences: false,
223
+ hasAssets: false,
224
+ });
225
+ }
226
+ }
227
+ return out;
228
+ }
229
+
230
+ /**
231
+ * AGENTS.md, at both scopes it can live at.
232
+ *
233
+ * `AGENTS.md` is the Linux Foundation name; `project.md` is still read so
234
+ * nothing written before the rename breaks.
235
+ *
236
+ * Returns null rather than throwing on unparseable YAML: a broken shared file
237
+ * must not take down every agent under it.
238
+ */
239
+ export function readAgentsMd(dir: string): { data: Record<string, unknown>; body: string } | null {
240
+ // Convert a pre-rename `mdagent_version:` before parsing. One name in the
241
+ // code, one on disk — the reader no longer knows two spellings, so a file
242
+ // written before the rename is migrated here rather than tolerated forever.
243
+ adoptLegacyVersionKey(dir);
244
+ for (const name of ["AGENTS.md", "project.md"]) {
245
+ const file = path.join(dir, name);
246
+ if (!fs.existsSync(file)) continue;
247
+ try {
248
+ const parsed = matter(fs.readFileSync(file, "utf8"));
249
+ return { data: parsed.data as Record<string, unknown>, body: parsed.content.trim() };
250
+ } catch {
251
+ return null;
252
+ }
253
+ }
254
+ return null;
255
+ }
256
+
257
+ const workspaceRootOf = (agentDir: string) => path.join(agentDir, "..", "..");
258
+
259
+ /**
260
+ * Configuration from AGENTS.md, nearest-wins: the workspace's overrides the
261
+ * account's, key by key.
262
+ *
263
+ * Config overrides and prose accumulates — see sharedInstructions. A workspace
264
+ * naming its own `provider:` means *instead of* the account's; a workspace
265
+ * writing instructions means *as well as*, because an account rule nobody can
266
+ * silently drop is the point of having one.
267
+ */
268
+ function workspaceFrontmatter(agentDir: string, tenant: string): Record<string, unknown> {
269
+ return {
270
+ ...(readAgentsMd(accountDir(tenant))?.data ?? {}),
271
+ ...(readAgentsMd(workspaceRootOf(agentDir))?.data ?? {}),
272
+ };
273
+ }
274
+
275
+ /**
276
+ * The calendar an agent works to. `timezone:` in AGENTS.md, nearest-wins like
277
+ * every other key there; a flow's cron already means its own timezone, this
278
+ * makes "today" mean it too. Unset is UTC — what the pods run — which for a
279
+ * Sydney desk made an article dated today "one day in the future" until 10am.
280
+ * An unknown name falls back to UTC rather than failing every step.
281
+ */
282
+ export function resolveTimezone(front: Record<string, unknown>): string {
283
+ const tz = typeof front.timezone === "string" ? front.timezone.trim() : "";
284
+ const chosen = tz || process.env.FOLDRUN_TIMEZONE?.trim() || "UTC";
285
+ try {
286
+ new Intl.DateTimeFormat("en-CA", { timeZone: chosen });
287
+ return chosen;
288
+ } catch {
289
+ return "UTC";
290
+ }
291
+ }
292
+
293
+ /** YYYY-MM-DD in a timezone — the date the workspace believes it is. */
294
+ export function localDate(timeZone: string, now = new Date()): string {
295
+ return new Intl.DateTimeFormat("en-CA", {
296
+ timeZone,
297
+ year: "numeric",
298
+ month: "2-digit",
299
+ day: "2-digit",
300
+ }).format(now);
301
+ }
302
+
303
+ /**
304
+ * The prose from AGENTS.md, outermost first.
305
+ *
306
+ * This was written by every scaffold, shown in the dashboard, described in the
307
+ * template as "context every agent here shares" — and never read. Only the
308
+ * frontmatter was, so the instructions people actually wrote reached no model.
309
+ */
310
+ export function sharedInstructions(agentDir: string, tenant: string): string | null {
311
+ const sections = [
312
+ ["Everyone in this account", readAgentsMd(accountDir(tenant))?.body],
313
+ ["Everyone in this workspace", readAgentsMd(workspaceRootOf(agentDir))?.body],
314
+ ].filter(([, body]) => body) as [string, string][];
315
+ if (!sections.length) return null;
316
+
317
+ return (
318
+ `# Shared context\n\nThis applies to you in addition to your own instructions ` +
319
+ `above. Where it is more specific than they are, it is not optional.\n\n` +
320
+ sections.map(([label, body]) => `## ${label}\n\n${body}`).join("\n\n")
321
+ );
322
+ }
323
+
324
+ /**
325
+ * Which of an agent's in-scope skills load for this run.
326
+ *
327
+ * Two independent gates: `skills:` in the agent's frontmatter is an allowlist
328
+ * (absent means everything in scope), and a skill's own `when:` names the run
329
+ * tags it is for (absent means every run).
330
+ *
331
+ * Note what an empty `tags` means — a skill with `when:` matches nothing, so
332
+ * an untagged run gets none of them. That is the intent, and it is also why
333
+ * failing to thread the run's tags this far made every `when:` skill
334
+ * permanently invisible instead of conditionally loaded.
335
+ */
336
+ export function applicableSkills<T extends { name: string; when: string[] }>(
337
+ skills: T[],
338
+ only: string[],
339
+ tags: string[],
340
+ ): T[] {
341
+ return skills
342
+ .filter((s) => only.length === 0 || only.includes(s.name))
343
+ .filter((s) => s.when.length === 0 || s.when.some((t) => tags.includes(t)));
344
+ }
345
+
346
+ /**
347
+ * [[name]] as a reference to a document or file — the Obsidian instinct,
348
+ * honoured where it is safe. A prompt is text the model acts on, and a
349
+ * model cannot open "[[sources-of-conveyancers]]"; it can open
350
+ * `../../knowledge/sources-of-conveyancers.md`. So links resolve to real
351
+ * agent-relative paths BEFORE the prompt is built, matching by filename or
352
+ * OKF title, hyphen/underscore/case-blind. Anything unmatched — agent
353
+ * names, flow links, prose in brackets — passes through untouched: this is
354
+ * sugar over paths, never a gate in front of them.
355
+ */
356
+ /**
357
+ * Whether an evaluator's reply actually carries its `until:` marker.
358
+ *
359
+ * Not a substring test. A reviewer that writes "APPROVED once the tribunal
360
+ * sentence is corrected" contains the word, and a substring match reads that
361
+ * conditional as a pass — the loop ends, the correction is never made, and the
362
+ * step after it inherits the problem the loop existed to catch. That happened:
363
+ * a factual error rode a conditional approval past the gate.
364
+ *
365
+ * So the marker has to stand alone on its own line, which is a thing a model
366
+ * does deliberately and cannot do by accident mid-sentence. Trailing
367
+ * punctuation is allowed ("APPROVED." is still a decision), and the line may
368
+ * sit anywhere in the reply, so an approval followed by notes still passes —
369
+ * the point is that the word is a switch, not a clause.
370
+ */
371
+ export function saysUntilMarker(result: string | null, until: string): boolean {
372
+ const marker = until.trim().toLowerCase();
373
+ if (!marker) return true;
374
+ return (result ?? "")
375
+ .split(/\r?\n/)
376
+ .some(
377
+ (line) =>
378
+ // Strip markdown emphasis and punctuation from BOTH ends: "**APPROVED.**"
379
+ // is a decision written by someone using bold, not a sentence.
380
+ line
381
+ .trim()
382
+ .replace(/^[#>\-*_`\s]+/, "")
383
+ .replace(/[.!:,*_`\s]+$/, "")
384
+ .toLowerCase() === marker,
385
+ );
386
+ }
387
+
388
+ export function resolveDocLinks(text: string, workspaceRoot: string): string {
389
+ if (!text.includes("[[")) return text;
390
+ const map = new Map<string, string>();
391
+ const norm = (v: string) =>
392
+ v.toLowerCase().replace(/\.md$/, "").replace(/\/+$/, "").replace(/[^a-z0-9/]+/g, "-");
393
+
394
+ // Folders resolve too: a reference to "the file store" is as real as a
395
+ // reference to one file in it, and an agent told to read whatever its
396
+ // instruction names needs to say WHERE without naming a file. Added
397
+ // first, so a document that happens to share a folder's name still wins.
398
+ for (const dir of [STORAGE_DIR, "state", "knowledge", "memory", "skills", "outputs"]) {
399
+ if (fs.existsSync(path.join(workspaceRoot, dir))) map.set(norm(dir), `../../${dir}/`);
400
+ }
401
+ for (const kind of ["knowledge", "memory"] as const) {
402
+ const dir = path.join(workspaceRoot, kind);
403
+ if (!fs.existsSync(dir)) continue;
404
+ for (const entry of fs.readdirSync(dir, { recursive: true }).map(String)) {
405
+ if (!entry.endsWith(".md")) continue;
406
+ const rel = `../../${kind}/${entry.split(path.sep).join("/")}`;
407
+ map.set(norm(entry), rel);
408
+ map.set(norm(`${kind}/${entry}`), rel);
409
+ try {
410
+ const data = matter(fs.readFileSync(path.join(dir, entry), "utf8")).data as Record<string, unknown>;
411
+ // `title:` is OKF v0.2's word; `name:` is the older spelling still
412
+ // read, never written. A link should match what the file says it
413
+ // is, in either era's vocabulary.
414
+ for (const key of ["title", "name"] as const) {
415
+ const v = data?.[key];
416
+ if (typeof v === "string" && v) map.set(norm(v), rel);
417
+ }
418
+ } catch {
419
+ // unparseable frontmatter still resolves by filename
420
+ }
421
+ }
422
+ }
423
+ // state/ — the cursors, snapshots and histories a scheduled desk reads from
424
+ // itself every run. Individual files here were not linkable for a long time
425
+ // while the folder was, so the one directory a recurring flow reads MOST was
426
+ // the one that could only be reached by typing `../../state/x.md` by hand.
427
+ // Indexed by path like storage/ rather than by frontmatter like knowledge/:
428
+ // these are data files, often .json or .csv, and many carry no frontmatter.
429
+ const stateDir = path.join(workspaceRoot, "state");
430
+ if (fs.existsSync(stateDir)) {
431
+ for (const entry of fs.readdirSync(stateDir, { recursive: true }).map(String)) {
432
+ try {
433
+ if (!fs.statSync(path.join(stateDir, entry)).isFile()) continue;
434
+ } catch {
435
+ continue;
436
+ }
437
+ const fwd = entry.split(path.sep).join("/");
438
+ const rel = `../../state/${fwd}`;
439
+ map.set(norm(`state/${fwd}`), rel);
440
+ map.set(norm(fwd), rel);
441
+ }
442
+ }
443
+ try {
444
+ // The hosted store's index lives at <tenant>/storage/<workspace>/ — beside
445
+ // the workspaces, not inside one. workspaceRoot is <tenant>/…/<ws>, so
446
+ // up two and across. Single-workspace installs have no index and fall
447
+ // through to the plain-directory walk below.
448
+ const index = JSON.parse(
449
+ fs.readFileSync(
450
+ path.join(workspaceRoot, "..", "..", STORAGE_DIR, path.basename(workspaceRoot), "index.json"),
451
+ "utf8",
452
+ ),
453
+ ) as { files?: { path: string }[] };
454
+ for (const f of index.files ?? []) map.set(norm(`${STORAGE_DIR}/${f.path}`), `../../${STORAGE_DIR}/${f.path}`);
455
+ } catch {
456
+ // no file store index is normal
457
+ }
458
+ // Plain files on disk under storage/ too — local installs write there directly.
459
+ const filesDir = path.join(workspaceRoot, STORAGE_DIR);
460
+ if (fs.existsSync(filesDir)) {
461
+ for (const entry of fs.readdirSync(filesDir, { recursive: true }).map(String)) {
462
+ try {
463
+ if (!fs.statSync(path.join(filesDir, entry)).isFile()) continue;
464
+ } catch {
465
+ continue;
466
+ }
467
+ const fwd = entry.split(path.sep).join("/");
468
+ if (fwd.startsWith(".store/")) continue;
469
+ map.set(norm(`${STORAGE_DIR}/${fwd}`), `../../${STORAGE_DIR}/${fwd}`);
470
+ map.set(norm(fwd), `../../${STORAGE_DIR}/${fwd}`);
471
+ }
472
+ }
473
+ return text.replace(/\[\[([^\]\n]+)\]\]/g, (whole, name: string) => {
474
+ const hit = map.get(norm(name.trim()));
475
+ return hit ? `\`${hit}\`` : whole;
476
+ });
477
+ }
478
+
479
+ function agentContext(
480
+ agentDir: string,
481
+ tenant: string,
482
+ tags: string[] = [],
483
+ /** Who is running, so scripts can stamp what they write with its provenance. */
484
+ identity: { runId?: string; agent?: string } = {},
485
+ ) {
486
+ // agentDir is <data>/<tenant>/projects/<workspace>/agents/<agent>, so the
487
+ // workspace a secret should resolve against is two levels up.
488
+ const workspace = path.basename(path.resolve(agentDir, "..", ".."));
489
+ const { data: front, content: body } = matter(
490
+ fs.readFileSync(path.join(agentDir, "agent.md"), "utf8"),
491
+ );
492
+ const parts = [resolveDocLinks(body.trim(), path.resolve(agentDir, "..", ".."))];
493
+
494
+ // Where the agent is standing. Every path below this line is relative to the
495
+ // agent's own directory, and a run showed why that has to be said rather
496
+ // than implied: told to read `../../knowledge/index.md`, the model asked for
497
+ // `/knowledge/index.md` twice before working it out. Two denials and a turn,
498
+ // for want of one sentence.
499
+ parts.push(
500
+ `# Where you are\n\n` +
501
+ `Your working directory is \`agents/${path.basename(agentDir)}/\` inside the ` +
502
+ `\`${workspace}\` workspace. Every path in this prompt is relative to it — ` +
503
+ `\`outputs/\` is yours, and \`../../\` is the workspace root, so the workspace's ` +
504
+ `own knowledge is at \`../../knowledge/\`. Absolute paths are outside the ` +
505
+ `workspace and will be refused.\n\n` +
506
+ // A flow's whole point is that a later step works on what an earlier one
507
+ // produced, and each agent writes to its own outputs/. A run showed what
508
+ // omitting this costs: the checker looked in its own empty outputs/ and
509
+ // reported the draft missing — reviewing nothing, and saying so
510
+ // confidently — while the publisher only found it by guessing at a glob.
511
+ `Each agent writes to its own \`outputs/\`, so a file an earlier step produced is ` +
512
+ `at \`../<that-agent>/outputs/\`, not in yours. If a previous step says it wrote ` +
513
+ `something, look there before concluding it does not exist.\n\n` +
514
+ // The distinction agents kept getting wrong, said where they can read
515
+ // it: outputs/ is working text between steps and never leaves the
516
+ // source tree; storage/ is the store the dashboard's Storage page lists.
517
+ // A 100-row CSV written to outputs/ is delivered nowhere — a person
518
+ // looking for it on the Storage page finds it empty and concludes the run
519
+ // produced nothing.
520
+ // The path here MUST be the one harvestFiles reads. It said
521
+ // `../../files/` after the rename, and nothing recovered from that:
522
+ // materializeFiles always creates `storage/` before the step, so
523
+ // adoptLegacyFilesDir's "move files/ to storage/" is a no-op by the time
524
+ // the run ends — a deliverable written where this text pointed was
525
+ // harvested from nowhere and silently never appeared.
526
+ `\`../../storage/\` is the workspace file store: anything you leave there is kept ` +
527
+ `after the run and shown on the Storage page for people to download. Write ` +
528
+ `deliverables people asked for — CSVs, reports, PDFs, images — to \`../../storage/\`; ` +
529
+ `use \`outputs/\` for working text the next step reads.`,
530
+ );
531
+
532
+ // Shared context before anything derived — an account or workspace rule is
533
+ // background the rest of the prompt is read against, not an afterthought.
534
+ const sharedContext = sharedInstructions(agentDir, tenant);
535
+ if (sharedContext) parts.push(sharedContext);
536
+
537
+ // Skills follow the open Agent Skills format (agentskills.io): a folder
538
+ // with SKILL.md, optionally bundling its own scripts/ and references/. The
539
+ // flat `skills/<name>.md` form is also accepted for small skills.
540
+ //
541
+ // Progressive disclosure, as the standard prescribes: only each skill's
542
+ // name and description go into context up front. The agent reads the full
543
+ // SKILL.md when a task actually calls for it — so an agent can carry many
544
+ // skills for a few tokens each.
545
+ // Own skills, then the workspace's, then the workspace library — nearest
546
+ // definition of a name wins, so a team default can be overridden locally.
547
+ const skills: DiscoveredSkill[] = [];
548
+ const seenSkills = new Set<string>();
549
+ const addSkills = (dir: string, prefix: string, subdir = "skills") => {
550
+ for (const skill of discoverSkills(dir, subdir)) {
551
+ if (seenSkills.has(skill.name)) continue;
552
+ seenSkills.add(skill.name);
553
+ skills.push({ ...skill, path: `${prefix}${skill.path}`, dir: skill.dir ? `${prefix}${skill.dir}` : null });
554
+ }
555
+ };
556
+ const workspaceRoot = path.join(agentDir, "..", "..");
557
+ addSkills(agentDir, "");
558
+ addSkills(workspaceRoot, "../../"); // the workspace's skills/
559
+ // The cross-client convention: skills any skills-compatible tool installs
560
+ // under .agents/skills/ are visible here too, and vice versa. Scanned after
561
+ // the native location so a foldrun-native skill wins a name clash, but both
562
+ // are discovered. project scope only — foldrun workspaces are self-contained.
563
+ addSkills(workspaceRoot, "../../", ".agents/skills");
564
+ addSkills(path.join(libraryDir(tenant)), "../../../../library/"); // the account library
565
+
566
+ // A skill with `when:` only loads when the run carries a matching tag —
567
+ // so an agent with 119 skills costs 119 lines of context only when every
568
+ // one is genuinely relevant.
569
+ // `skills:` is Claude Code's field for naming which skills an agent gets.
570
+ // Absent, an agent inherits every skill in scope — knowledge cascades. Named,
571
+ // it's an allowlist, which is how you keep a focused agent focused.
572
+ const only: string[] = refNames(front.skills);
573
+ const applicable = applicableSkills(skills, only, tags);
574
+ const withheld = skills.length - applicable.length;
575
+
576
+ if (applicable.length) {
577
+ const lines = applicable.map((s) => {
578
+ // The rest of the Agent Skills layout, named so the agent knows to look
579
+ // rather than guessing that a skill folder holds only SKILL.md.
580
+ const bundles = [
581
+ s.hasScripts && `scripts in \`${s.dir}/scripts/\``,
582
+ s.hasReferences && `reference docs in \`${s.dir}/references/\``,
583
+ s.hasAssets && `assets in \`${s.dir}/assets/\``,
584
+ ].filter(Boolean);
585
+ return (
586
+ `- **${s.name}** — ${s.description || "(no description)"} ` +
587
+ `Full instructions: \`${s.path}\`` +
588
+ (bundles.length ? ` (also bundles ${bundles.join(", ")})` : "")
589
+ );
590
+ });
591
+ parts.push(
592
+ `# Skills\n\nCapabilities available to you. Each line is a summary — when a task matches one, ` +
593
+ `read its file for the full instructions before proceeding:\n\n${lines.join("\n")}` +
594
+ (withheld ? `\n\n(${withheld} further skills exist but do not apply to this run.)` : ""),
595
+ );
596
+ }
597
+
598
+ // Memory: the index is loaded every run, and the agent is told how to add
599
+ // to it. Writing is only possible if it has the `files` tool.
600
+ const canWrite = refNames(front.tools).includes("files");
601
+ const memoryDir = path.join(agentDir, "memory");
602
+ const memoryIndex = buildMemoryIndex(memoryDir);
603
+ if (memoryIndex || canWrite) {
604
+ const howTo = canWrite
605
+ ? `\n\nTo remember something durable, write one fact per file to memory/<slug>.md with ` +
606
+ `frontmatter: type (e.g. Fact, Decision, Preference), name and description — these ` +
607
+ `bundles follow the Open Knowledge Format. The index above maintains itself — you do not ` +
608
+ `need to edit it. Read a memory file before relying on it, update an existing file ` +
609
+ `rather than duplicating it, and don't record what this workspace's files already say.`
610
+ : "";
611
+ const projectMem = workspaceMemoryIndex(tenant, workspace);
612
+ const shared = libraryMemoryIndex(tenant);
613
+ parts.push(
614
+ `# Memory — what you have learned\n\n${memoryIndex ? `Your own (files under memory/):\n\n${memoryIndex}` : "You have no memories of your own yet."}` +
615
+ (projectMem ? `\n\nShared across this workspace:\n\n${projectMem}` : "") +
616
+ (shared ? `\n\nShared across every workspace:\n\n${shared}` : "") +
617
+ howTo,
618
+ );
619
+ }
620
+
621
+ // Knowledge — reference material the agent was given. Same three scopes and
622
+ // the same nearest-wins reading order as memory, but never written back:
623
+ // an agent that edits the price list it was handed has corrupted its own
624
+ // source of truth.
625
+ const ownKnowledge = knowledgeIndex(path.join(agentDir, "knowledge"), "knowledge/");
626
+ const projectKnowledge = workspaceKnowledgeIndex(tenant, workspace);
627
+ const sharedKnowledge = knowledgeIndex(libraryDir(tenant, "knowledge"), "../../../../library/knowledge/");
628
+ if (ownKnowledge || projectKnowledge || sharedKnowledge) {
629
+ const sections = [
630
+ ownKnowledge && `Your own (files under knowledge/):\n\n${ownKnowledge}`,
631
+ projectKnowledge && `Shared across this workspace:\n\n${projectKnowledge}`,
632
+ sharedKnowledge && `Shared across every workspace:\n\n${sharedKnowledge}`,
633
+ ].filter(Boolean);
634
+ parts.push(
635
+ `# Knowledge — what you were given\n\nReference material. Open a file when a task depends ` +
636
+ `on it rather than guessing at its contents. This is read-only: it is maintained by ` +
637
+ `people, not by you — if you learn something new, write it to memory/ instead.\n\n` +
638
+ sections.join("\n\n"),
639
+ );
640
+ }
641
+ // Scripts an agent can run with bash: its own scripts/, plus the
642
+ // workspace-level shared scripts/ exposed at shared/ in its working dir.
643
+ const listDir = (dir: string, prefix: string) => {
644
+ if (!fs.existsSync(dir)) return [];
645
+ const out: string[] = [];
646
+ for (const entry of fs.readdirSync(dir, { recursive: true })) {
647
+ const rel = String(entry);
648
+ if (fs.statSync(path.join(dir, rel)).isFile()) out.push(`- ${prefix}${rel}`);
649
+ }
650
+ return out.sort();
651
+ };
652
+
653
+ const own = listDir(path.join(agentDir, "scripts"), "scripts/");
654
+ const shared = listDir(path.join(agentDir, "..", "..", "scripts"), "../../scripts/");
655
+ // The library's path depends on where this run executes: isolated runs see
656
+ // it at /library, host runs at its real location. Printing the host path
657
+ // into a container's prompt handed the model files it could never open.
658
+ const libraryScriptsRoot = isolatedRun() ? "/library/scripts" : libraryDir(tenant, "scripts");
659
+ const global = listDir(libraryDir(tenant, "scripts"), `${libraryScriptsRoot}/`);
660
+
661
+ if (own.length || shared.length || global.length) {
662
+ const sections = [
663
+ own.length ? `Your own scripts:\n${own.join("\n")}` : null,
664
+ shared.length ? `Shared across this workspace:\n${shared.join("\n")}` : null,
665
+ global.length ? `From the account library (read-only):\n${global.join("\n")}` : null,
666
+ ].filter(Boolean);
667
+ parts.push(
668
+ `# Scripts\n\nTooling you can run with bash from your working directory:\n\n${sections.join("\n\n")}\n\n` +
669
+ `Read a script before running it if you are unsure what it does. Credentials your ` +
670
+ `scripts need are already in the environment as the secrets listed below.`,
671
+ );
672
+ }
673
+
674
+ // State — data a run carries to the next one. It was a directory in the
675
+ // workspace list, preserved across deploys, and nothing else: never named to
676
+ // an agent, never loaded, never written. A promise with no implementation.
677
+ //
678
+ // Listed rather than inlined, for the same reason knowledge and memory are:
679
+ // an agent that needs a cursor opens it, and one that doesn't shouldn't pay
680
+ // for it. The difference is that state is small enough that inlining would
681
+ // usually be free — that is a threshold worth adding once there is a real
682
+ // file to size it against, not before.
683
+ const state = listDir(path.join(agentDir, "..", "..", "state"), "../../state/");
684
+ if (state.length) {
685
+ parts.push(
686
+ `# State — what you carry between runs\n\n` +
687
+ `What this workspace keeps across runs: a count, a cursor, where you got to. ` +
688
+ `Read and write these directly, and update them before you finish so the next ` +
689
+ `run starts where you left off.\n\n${state.join("\n")}\n\n` +
690
+ `Keep a file in the format it is already in, and write markdown for anything new. ` +
691
+ `A count reads as well in a sentence as in JSON, and a person edits these in the ` +
692
+ `same editor as everything else — data formats are allowed here, not preferred.\n\n` +
693
+ `This is not memory. A fact you learned goes in memory/, where it is indexed and ` +
694
+ `its provenance recorded — state is the bookmark, not the lesson.`,
695
+ );
696
+ }
697
+
698
+ // APIs — the agent's own definitions, plus workspace tools it granted by
699
+ // name. Library tools are defined once and reused.
700
+ const apis = parseApis(front.apis);
701
+ // Scripts declared inline on the agent, plus any script-type tool it
702
+ // granted — one list grants both kinds.
703
+ const scriptSpecs = parseScripts(front.scripts);
704
+ // Workspace tools shadow account tools of the same name (nearest wins).
705
+ const available = { ...libraryTools(tenant), ...workspaceTools(tenant, workspace) };
706
+ // Resolved in one place (tool-names.ts) so nothing downstream can disagree
707
+ // about which names are the author's.
708
+ const requested = ownToolNames(front);
709
+ const legacyUse = legacyUseNames(front);
710
+ const missingTools: string[] = [];
711
+ // MCP servers an agent connects to. `mcpServers:` is Claude Code's spelling
712
+ // for the inline form; a `type: mcp` tool file is the shared form. Both end
713
+ // up here, and both reach the model as ordinary MCP tools.
714
+ const mcpServers: Record<string, McpServerConfig> = {};
715
+ const mcpNames: string[] = [];
716
+
717
+ // What the granted tools' programs need — merged into this step's runtime
718
+ // below. The tool is the unit of code, so its dependencies live in tool.md
719
+ // and travel with it; the agent should not have to repeat them.
720
+ const toolRuntimes: RuntimeSpec[] = [];
721
+ const grantOwnTool = (name: string) => {
722
+ const def = available[name];
723
+ if (!def) {
724
+ missingTools.push(name);
725
+ return;
726
+ }
727
+ if (def.kind === "http") apis.push(def.spec);
728
+ else if (def.kind === "script") {
729
+ scriptSpecs.push(...parseScripts([def.spec]));
730
+ const rt = parseRuntime((def.spec as { runtime?: unknown }).runtime);
731
+ if (rt) toolRuntimes.push(rt);
732
+ }
733
+ else {
734
+ mcpServers[def.name] = mcpConfig(def.spec, tenant, workspace);
735
+ mcpNames.push(def.name);
736
+ }
737
+ };
738
+
739
+ for (const name of requested) grantOwnTool(name);
740
+
741
+ // Inline servers, Claude Code's field name and shape.
742
+ if (front.mcpServers && typeof front.mcpServers === "object") {
743
+ for (const [name, raw] of Object.entries(front.mcpServers as Record<string, unknown>)) {
744
+ const def = parseToolDef({ ...(raw as object), type: "mcp" }, name);
745
+ if (def?.kind === "mcp") {
746
+ mcpServers[name] = mcpConfig(def.spec, tenant, workspace);
747
+ mcpNames.push(name);
748
+ }
749
+ }
750
+ }
751
+ // An `openapi:` document turns one generic HTTP tool into typed ones, one
752
+ // per operation. Resolved here, host-side, so the operations cross into a
753
+ // container as values like the rest of the spec.
754
+ const attached = attachOperations(apis, tenant, path.resolve(agentDir, "..", ".."));
755
+ const apiWarnings = attached.warnings;
756
+ apis.splice(0, apis.length, ...attached.specs);
757
+ const apiTools = buildApiTools(tenant, apis, workspace);
758
+ if (apiTools.promptLines.length) {
759
+ parts.push(`# APIs you can call\n\n${apiTools.promptLines.join("\n")}`);
760
+ }
761
+
762
+ // Secrets an agent declared become env vars for its scripts and bash.
763
+ //
764
+ // Plus the credentials its API tools reference. An agent that opted into
765
+ // `tools: [email]` has said nothing about RESEND_API_KEY and should not have
766
+ // to: the whole point of a tool definition is that the capability travels
767
+ // with its credential and the agent never learns the name. The in-process
768
+ // path always worked this way — buildApiTools resolves what the specs
769
+ // reference — so without this the isolated path substituted nothing and
770
+ // sent `Bearer ${RESEND_API_KEY}` literally, which the provider answers
771
+ // with a 4xx the agent then reports as "the credential is missing".
772
+ //
773
+ // Declared-and-absent is still an error; referenced-and-absent is the API
774
+ // tool's own missingSecrets, reported where the tool is described.
775
+ const declared: string[] = Array.isArray(front.secrets) ? front.secrets.map(String) : [];
776
+ const referenced = [...new Set(apis.flatMap(secretsUsedByApi))];
777
+ const {
778
+ env: secretEnv,
779
+ from: secretScopes,
780
+ } = resolveSecrets(tenant, [...new Set([...declared, ...referenced])], workspace);
781
+ const { missing: missingDeclared } = resolveSecrets(tenant, declared, workspace);
782
+
783
+ // The step's pod size. Three classes, not a dial: a size is a price and a
784
+ // promise, and three of each is a menu while ten is a support queue. Default
785
+ // is large — the install's configured limits — so existing agents change
786
+ // nothing. `size: small` is the opt-in for agents that wait on APIs more
787
+ // than they work; `size: heavy` for a rendered browser or real data in
788
+ // memory. Memory is a hard ceiling per class; CPU bursts uncapped and bills
789
+ // at the class rate, so heavy both may use more and costs more. The actuals
790
+ // line in any trace is the evidence for right-sizing.
791
+ const size: "small" | "large" | "heavy" =
792
+ front.size === "small" ? "small" : front.size === "heavy" ? "heavy" : "large";
793
+
794
+ // Provider: which endpoint the model calls run against. Anthropic by
795
+ // default; an Anthropic-compatible gateway (z.ai, LiteLLM, OpenRouter) by
796
+ // declaring one. The format is portable — this is where the runtime stops
797
+ // being tied to one vendor's endpoint.
798
+ //
799
+ // provider:
800
+ // base_url: https://api.z.ai/api/anthropic
801
+ // token: ${ZAI_TOKEN}
802
+ //
803
+ // The URL sits in git; the token is a ${SECRET} resolved server-side. An
804
+ // agent's own block wins over the workspace's, like everything else here.
805
+ const providerBlock =
806
+ (front.provider as Record<string, unknown> | undefined) ?? workspaceFrontmatter(agentDir, tenant).provider;
807
+ let providerEnv: Record<string, string> = {};
808
+ let providerLabel: string | null = null;
809
+ // Secret values pulled into the provider block, name → value. Kept apart
810
+ // from providerEnv because a header blob is a poor redaction key: what
811
+ // must never reach the journal is the credential inside it, not the blob.
812
+ const providerSecrets: Record<string, string> = {};
813
+ const providerWarnings: string[] = [];
814
+ // The declared second supply, resolved the same way. Null when the block
815
+ // names none; the platform's own fallback still applies to a step that
816
+ // rides the platform credential.
817
+ let fallbackEnv: Record<string, string> | null = null;
818
+ // A Chat-Completions endpoint is reached through the runtime's own
819
+ // translator; these are its instructions, as values, or null when the
820
+ // block speaks Anthropic. One for the primary, one for the fallback.
821
+ let translator: TranslatorSpec | null = null;
822
+ let fallbackTranslator: TranslatorSpec | null = null;
823
+ const spec = parseProvider(providerBlock);
824
+ if (spec) {
825
+ providerWarnings.push(...spec.warnings);
826
+ const substitute = (text: string) =>
827
+ text.replace(/\$\{([A-Z][A-Z0-9_]*)\}/g, (whole, secretName) => {
828
+ const hit = getSecret(tenant, secretName, workspace);
829
+ if (!hit) return whole;
830
+ providerSecrets[secretName] = hit.value;
831
+ return hit.value;
832
+ });
833
+ const resolved = substitute(spec.token);
834
+ if (spec.baseUrl) {
835
+ const headers = Object.fromEntries(
836
+ Object.entries(spec.headers).map(([k, v]) => [k, substitute(v)]),
837
+ );
838
+ providerEnv = providerEnvFor({
839
+ baseUrl: spec.baseUrl,
840
+ token: resolved,
841
+ auth: spec.auth,
842
+ models: spec.models,
843
+ headers,
844
+ });
845
+ const preset = providerPreset(spec.name);
846
+ translator = translatorSpecFor({
847
+ format: spec.format,
848
+ baseUrl: spec.baseUrl,
849
+ token: resolved,
850
+ headers,
851
+ params: spec.params,
852
+ name: spec.name,
853
+ maxTokensParam: preset?.maxTokensParam,
854
+ reasoningEffort: preset?.reasoningEffort,
855
+ });
856
+ // What the run is actually pointed at, in one line: the endpoint, the
857
+ // tiers this gateway renames, and whether the credential landed.
858
+ const remapped = Object.entries(spec.models).map(([tier, id]) => `${tier}→${id}`);
859
+ providerLabel = (spec.name ? `${spec.name} ` : "") + spec.baseUrl + (translator ? " via translator" : "");
860
+ if (remapped.length) providerLabel += ` (${remapped.join(", ")})`;
861
+ // An unresolved ${SECRET} would silently fall back to Anthropic's
862
+ // endpoint with the host's key — say so instead.
863
+ if (spec.token && resolved === spec.token && spec.token.includes("${")) {
864
+ providerLabel += " — token secret not set";
865
+ }
866
+ }
867
+ if (spec.fallback?.baseUrl) {
868
+ const fb = spec.fallback;
869
+ const headers = Object.fromEntries(Object.entries(fb.headers).map(([k, v]) => [k, substitute(v)]));
870
+ const fbToken = substitute(fb.token);
871
+ fallbackEnv = providerEnvFor({ baseUrl: fb.baseUrl, token: fbToken, auth: fb.auth, models: fb.models, headers });
872
+ const fbPreset = providerPreset(fb.name);
873
+ fallbackTranslator = translatorSpecFor({
874
+ format: fb.format,
875
+ baseUrl: fb.baseUrl,
876
+ token: fbToken,
877
+ headers,
878
+ params: fb.params,
879
+ name: fb.name,
880
+ maxTokensParam: fbPreset?.maxTokensParam,
881
+ reasoningEffort: fbPreset?.reasoningEffort,
882
+ });
883
+ providerLabel = `${providerLabel ?? "platform"} (fallback: ${fb.name ?? fb.baseUrl})`;
884
+ }
885
+ }
886
+
887
+ // Runtime: an agent's own `runtime:` block, else its workspace's. Built once
888
+ // per declaration and cached, so scripts get their dependencies without
889
+ // polluting the host.
890
+ let runtimeSpec = parseRuntime(front.runtime);
891
+ if (!runtimeSpec) {
892
+ // AGENTS.md is the Linux Foundation standard name (60k+ repos, same
893
+ // nearest-wins cascade we already use); project.md stays accepted so
894
+ // nothing that already exists breaks.
895
+ runtimeSpec = parseRuntime(workspaceFrontmatter(agentDir, tenant).runtime);
896
+ }
897
+ // Plus whatever the tools this agent granted declared for themselves.
898
+ runtimeSpec = mergeRuntimes(runtimeSpec, ...toolRuntimes);
899
+ // Execution: a container per run when Docker is available (real isolation),
900
+ // otherwise the host venv path so local development still works.
901
+ const executor = chooseExecutor();
902
+ const runtimeLog: string[] = [];
903
+ let runtimeError: string | null = null;
904
+ let exec: ExecutionContext | null = null;
905
+ let runtime = { interpreters: {} as Record<string, string>, env: {} as Record<string, string>, log: [] as string[], error: null as string | null };
906
+
907
+ if (executor === "docker") {
908
+ const image = ensureImage(runtimeSpec);
909
+ runtimeLog.push(...image.log);
910
+ runtimeError = image.error;
911
+ if (!image.error) {
912
+ exec = {
913
+ executor: "docker",
914
+ image: image.tag,
915
+ mounts: {
916
+ [path.resolve(agentDir, "..", "..", "scripts")]: "/workspace-scripts",
917
+ [libraryDir(tenant, "scripts")]: "/library-scripts",
918
+ },
919
+ // Only agents that granted something reaching out — an `apis:` entry
920
+ // or one of their own tools — get egress.
921
+ network: parseApis(front.apis).length > 0 || ownToolNames(front).length > 0,
922
+ };
923
+ }
924
+ } else {
925
+ runtime = prepareRuntime(tenant, runtimeSpec);
926
+ runtimeLog.push(...runtime.log);
927
+ runtimeError = runtime.error;
928
+ }
929
+
930
+ // What day it is, in the workspace's own timezone. Not a secret, so it
931
+ // travels beside secretEnv rather than inside it — a value in there is
932
+ // redacted from every log line that quotes it.
933
+ const timezone = resolveTimezone(workspaceFrontmatter(agentDir, tenant));
934
+ const clockEnv = {
935
+ TZ: timezone,
936
+ FOLDRUN_DATE: localDate(timezone),
937
+ // Where the account's own search engine answers, when the install runs
938
+ // one — the `websearch` library tool reads it. Not a secret; rides with
939
+ // the clock so it reaches scripts, the sandbox and verify alike.
940
+ ...(process.env.FOLDRUN_SEARCH_URL ? { FOLDRUN_SEARCH_URL: process.env.FOLDRUN_SEARCH_URL } : {}),
941
+ };
942
+
943
+ // Scripts declared as tools — callable by name, no bash required.
944
+ const scriptTools = buildScriptTools(
945
+ agentDir,
946
+ scriptSpecs,
947
+ // A script can say where its work came from. Without this, anything a
948
+ // tool writes into another system lands anonymous: you can see 100 new
949
+ // contacts and have no way back to the run that made them. The run id,
950
+ // the agent, and the date the platform believes it is.
951
+ {
952
+ ...runtime.env,
953
+ ...secretEnv,
954
+ ...(identity.runId ? { FOLDRUN_RUN_ID: identity.runId } : {}),
955
+ ...(identity.agent ? { FOLDRUN_AGENT: identity.agent } : {}),
956
+ // And the workspace: a commit or a record made by a shared library
957
+ // tool should say which desk made it, and the pod's own paths
958
+ // (/workspace/agents/<agent>) do not carry the name.
959
+ FOLDRUN_WORKSPACE: workspace,
960
+ ...clockEnv,
961
+ },
962
+ libraryDir(tenant, "scripts"),
963
+ runtime.interpreters,
964
+ exec,
965
+ );
966
+ parts.push(
967
+ `# Clock\n\nToday is ${clockEnv.FOLDRUN_DATE} (${timezone}). Use this date for anything you ` +
968
+ `write or check — it is the workspace's calendar, not the container's clock.`,
969
+ );
970
+ if (scriptTools.promptLines.length) {
971
+ parts.push(
972
+ `# Tools from scripts\n\nCall these directly — they run workspace scripts for you:\n\n${scriptTools.promptLines.join("\n")}`,
973
+ );
974
+ }
975
+ if (declared.length) {
976
+ parts.push(
977
+ `# Secrets\n\nAvailable to your scripts as environment variables: ${declared.join(", ")}. ` +
978
+ `Never print, echo, or write them to a file.`,
979
+ );
980
+ }
981
+
982
+ parts.push(
983
+ "Write deliverables for people to workspace/storage/ (kept and downloadable); write working text for later steps to outputs/; " +
984
+ "what the next run needs goes to workspace/state/. Paths are relative to your agent directory, or start with workspace/ " +
985
+ "for the workspace root (../../storage/ and workspace/storage/ are the same place). There is no /tmp for you: a " +
986
+ "directory a tool made outside the workspace, such as a checkout, is reached only through that tool.",
987
+ );
988
+
989
+ // The run's one-line summary is the first meaningful line of the last step
990
+ // that produced a result — read, never declared (runSummary in store.ts).
991
+ // That convention was load-bearing and unstated: it decides the runs-list
992
+ // row, the notification subject, and what `recall_runs()` shows a later
993
+ // run, and an agent that opened with "I'll start by cloning the repo" made
994
+ // all three useless. Asking here rather than in each agent.md means every
995
+ // agent obeys it, including the ones written before the summary existed.
996
+ parts.push(
997
+ "Open your reply with one sentence saying what you concluded or produced — the outcome, " +
998
+ "not your plan. Everything else follows it. That sentence is what a person sees in the " +
999
+ "run list and in the notification, without opening the run.",
1000
+ );
1001
+
1002
+ // One list. `tools:` grants anything: a built-in group, an exact SDK tool
1003
+ // name, or a tool defined in this workspace or account.
1004
+ //
1005
+ // The split used to be ours, not the author's: they don't care whether a
1006
+ // capability comes from the SDK or a file, only what the agent may do. Two
1007
+ // lists meant two places to look for the blast radius.
1008
+ const allowed: string[] = [];
1009
+ const disabled: string[] = [];
1010
+ const unknownTools: string[] = [];
1011
+ const shadowed: string[] = [];
1012
+ let wantSearch = false;
1013
+ let wantHistory = false;
1014
+ let wantDesks = false;
1015
+
1016
+ for (const ref of toolRefs(front)) {
1017
+ const toolName = ref.name;
1018
+ if (ref.mode === "ask") {
1019
+ disabled.push(toolName);
1020
+ continue;
1021
+ }
1022
+
1023
+ // Resolution order: built-in group, exact SDK name, then your own tools.
1024
+ // Built-ins win a bare name, because their names are a small fixed set —
1025
+ // but a tool hidden by one is worth saying out loud rather than silently
1026
+ // ignoring. A `[[link]]` skips the built-ins entirely: the brackets mean
1027
+ // "my file", which is how a tool named `search` is granted at all.
1028
+ if (ref.linked) {
1029
+ // Granted above through ownToolNames, or reported missing there.
1030
+ } else if (TOOL_MAP[toolName]) {
1031
+ allowed.push(...TOOL_MAP[toolName]);
1032
+ if (available[toolName]) shadowed.push(toolName);
1033
+ } else if (toolName === "search" || toolName === "history" || toolName === "desks") {
1034
+ // The platform's own groups, served in-process — built below, once
1035
+ // the roots and the records are known.
1036
+ if (toolName === "search") wantSearch = true;
1037
+ else if (toolName === "history") wantHistory = true;
1038
+ else wantDesks = true;
1039
+ if (available[toolName]) shadowed.push(toolName);
1040
+ } else if (BUILTIN_TOOLS.has(toolName)) {
1041
+ allowed.push(toolName);
1042
+ if (available[toolName]) shadowed.push(toolName);
1043
+ } else if (available[toolName]) {
1044
+ // Already granted above — nothing to do here.
1045
+ } else if (!missingTools.includes(toolName)) {
1046
+ unknownTools.push(toolName);
1047
+ }
1048
+ }
1049
+ allowed.push(...apiTools.toolNames, ...scriptTools.toolNames);
1050
+
1051
+ // tools: [search] — every place a fact of this agent's could be, in the
1052
+ // same nearest-wins order the prompt lists them. tools: [history] — the
1053
+ // workspace's finished runs, minus the one being driven. Both are values
1054
+ // by the time they leave here, so the container can rebuild the servers.
1055
+ const wsRoot = path.resolve(agentDir, "..", "..");
1056
+ const searchRoots: SearchRoot[] = wantSearch
1057
+ ? [
1058
+ { label: "knowledge/", dir: path.join(agentDir, "knowledge") },
1059
+ { label: "memory/", dir: path.join(agentDir, "memory") },
1060
+ { label: "../../knowledge/", dir: path.join(wsRoot, "knowledge") },
1061
+ { label: "../../memory/", dir: path.join(wsRoot, "memory") },
1062
+ { label: "../../state/", dir: path.join(wsRoot, "state") },
1063
+ { label: `../../${STORAGE_DIR}/`, dir: path.join(wsRoot, STORAGE_DIR) },
1064
+ { label: "outputs/", dir: path.join(agentDir, "outputs") },
1065
+ { label: "[library] knowledge/", dir: libraryDir(tenant, "knowledge") },
1066
+ { label: "[library] memory/", dir: libraryDir(tenant, "memory") },
1067
+ ]
1068
+ : [];
1069
+ const historyDigest: RunDigest[] = wantHistory ? digestRuns(listRuns(tenant, workspace), 30, identity.runId) : [];
1070
+ // tools: [desks] — the account's OTHER workspaces, ten runs each. A run
1071
+ // pod cannot reach the platform's API and the library is read-only, so
1072
+ // this is the only way one desk sees what another concluded — and it
1073
+ // stays a value the host chose to send, not a door the sandbox opened.
1074
+ const deskDigest: RunDigest[] = wantDesks
1075
+ ? listWorkspaces(tenant)
1076
+ .filter((w) => w.name !== workspace)
1077
+ .flatMap((w) => digestRuns(listRuns(tenant, w.name), 10).map((d) => ({ ...d, workspace: w.name })))
1078
+ .sort((a, b) => (b.startedAt > a.startedAt ? 1 : -1))
1079
+ .slice(0, DESK_RUNS_CAP)
1080
+ : [];
1081
+ const searchTools = buildSearchTools(searchRoots);
1082
+ const historyTools = wantHistory ? buildHistoryTools(historyDigest) : { server: null, toolNames: [], promptLines: [] };
1083
+ const deskTools = wantDesks ? buildDeskTools(deskDigest) : { server: null, toolNames: [], promptLines: [] };
1084
+ allowed.push(...searchTools.toolNames, ...historyTools.toolNames, ...deskTools.toolNames);
1085
+ if (searchTools.promptLines.length || historyTools.promptLines.length || deskTools.promptLines.length) {
1086
+ parts.push(`# Finding things\n\n${[...searchTools.promptLines, ...historyTools.promptLines, ...deskTools.promptLines].join("\n")}`);
1087
+ }
1088
+ // An MCP server declares its tools when it connects, so they can't be listed
1089
+ // ahead of time. Server-level specs cover them: granting the server grants
1090
+ // what it exposes, which is the unit a person actually reasons about anyway.
1091
+ for (const name of mcpNames) allowed.push(`mcp__${name}`, `mcp__${name}__*`);
1092
+
1093
+ // Claude Code's spellings, so an agent written for it mostly runs here.
1094
+ // `permissionMode: plan` is its read-only mode; ours is the `read` group.
1095
+ if (front.permissionMode === "plan") {
1096
+ allowed.length = 0;
1097
+ allowed.push(...TOOL_MAP.read, ...apiTools.toolNames, ...scriptTools.toolNames);
1098
+ }
1099
+ const denied: string[] = refNames(front.disallowedTools);
1100
+ const finalAllowed = allowed.filter((t) => !denied.includes(t));
1101
+
1102
+ return {
1103
+ front,
1104
+ clockEnv,
1105
+ systemPrompt: parts.join("\n\n"),
1106
+ allowed: finalAllowed,
1107
+ disabled,
1108
+ unknownTools,
1109
+ shadowed,
1110
+ legacyUse,
1111
+ mcpServers,
1112
+ mcpNames,
1113
+ apiTools,
1114
+ scriptTools,
1115
+ // The merged spec lists — inline plus tools:-granted — for the isolated
1116
+ // path, which serialises specs across the container boundary rather
1117
+ // than using the servers built here.
1118
+ apiSpecs: apis,
1119
+ scriptSpecs,
1120
+ searchRoots,
1121
+ historyDigest,
1122
+ deskDigest,
1123
+ searchTools,
1124
+ historyTools,
1125
+ deskTools,
1126
+ // `spec` is the MERGED declaration — the agent's own block or its
1127
+ // workspace's, plus every granted tool's. The isolated path builds its
1128
+ // runtime from this; passing the agent's frontmatter alone silently drops
1129
+ // what the tools declared, and a tool whose program needs a package it
1130
+ // asked for by name fails inside the container with "cannot find" while
1131
+ // the host path, which uses this same merged spec, works fine.
1132
+ runtime: { log: runtimeLog, error: runtimeError, executor, spec: runtimeSpec },
1133
+ secretEnv: { ...runtime.env, ...secretEnv },
1134
+ formatWarning: checkFormatVersion(workspaceFrontmatter(agentDir, tenant).foldrun_version).warning,
1135
+ providerEnv,
1136
+ providerLabel,
1137
+ providerSecrets,
1138
+ providerWarnings,
1139
+ fallbackEnv,
1140
+ translator,
1141
+ fallbackTranslator,
1142
+ apiWarnings,
1143
+ secretScopes,
1144
+ missingTools,
1145
+ size,
1146
+ // Definitions that exist but could not be read. Carried alongside the
1147
+ // missing ones so the step can tell "you never made this" apart from
1148
+ // "it's right there with a typo in it".
1149
+ brokenTools: brokenToolReport(tenant, workspace),
1150
+ missingSecrets: [...new Set([...missingDeclared, ...apiTools.missingSecrets])],
1151
+ };
1152
+ }
1153
+
1154
+ /**
1155
+ * The SDK prices a step off Anthropic's table; through a gateway to a
1156
+ * routed model that number is fiction. When the catalogue can price the
1157
+ * tokens itself, its number replaces the SDK's, and the trace says so —
1158
+ * a silently different bill is the one kind nobody forgives.
1159
+ */
1160
+ /**
1161
+ * Publish whatever this step left in `storage/public/` and record the URLs.
1162
+ *
1163
+ * Runs after every step, on both execution paths, because the file has to be
1164
+ * on the host (copy-back has happened) and the manifest has to be in place
1165
+ * before the next step's sandbox is filled from that same workspace. A step
1166
+ * generates an image; the step after it reads the URL from a plain file.
1167
+ *
1168
+ * Never fails a step: a link is bookkeeping, and a run that did its work must
1169
+ * not be marked failed because a URL could not be minted.
1170
+ */
1171
+ function publishPublicDir(
1172
+ tenant: string,
1173
+ workspace: string,
1174
+ push: (type: "info" | "error", text: string) => void,
1175
+ ) {
1176
+ try {
1177
+ const { added } = platform.syncPublicShares(tenant, workspace);
1178
+ if (added.length) push("info", `shared: ${added.join(", ")}`);
1179
+ } catch (err) {
1180
+ push("error", `shares: ${err instanceof Error ? err.message : String(err)}`);
1181
+ }
1182
+ }
1183
+
1184
+ function repriced(
1185
+ catalog: Catalog | null,
1186
+ model: string,
1187
+ usage: { inputTokens: number; outputTokens: number } | null,
1188
+ sdkCost: number | null,
1189
+ push: (type: "info" | "error", text: string) => void,
1190
+ ): number | null {
1191
+ const known = usage ? catalogCost(catalog, model, usage) : null;
1192
+ if (known === null) return sdkCost;
1193
+ if (sdkCost !== null && Math.abs(known - sdkCost) > 0.000001) {
1194
+ push("info", `cost repriced from the gateway's catalogue: $${known.toFixed(6)} (sdk said $${sdkCost.toFixed(6)})`);
1195
+ }
1196
+ return known;
1197
+ }
1198
+
1199
+ async function runStep(
1200
+ agentDir: string,
1201
+ tenant: string,
1202
+ step: StepRecord,
1203
+ context: string | null,
1204
+ save: () => void,
1205
+ modelOverride?: string | null,
1206
+ tags: string[] = [],
1207
+ effortOverride?: string | null,
1208
+ runId?: string,
1209
+ /** What the previous group returned as DATA (an `output: json` step's
1210
+ * value), beside the prose in `context`. Undefined when nothing did. */
1211
+ priorData?: unknown,
1212
+ /** The most this step may spend before it is stopped mid-turn: the run's
1213
+ * remaining `budget:`, shared across the steps launched with it. Null
1214
+ * means the flow set no budget. */
1215
+ stepBudgetUsd: number | null = null,
1216
+ ) {
1217
+ // Secret values are injected into scripts as environment variables and
1218
+ // substituted into API headers, so a model that reads one back — from a
1219
+ // script that prints it, an API error that echoes it, or its own `env` —
1220
+ // can put a live credential into its reply. Everything a step says is
1221
+ // written to runs/<id>.json and rendered in the dashboard, which would make
1222
+ // that credential permanent and visible to anyone who can read a run.
1223
+ //
1224
+ // So values are scrubbed on the way into the journal, at the one place every
1225
+ // event passes through. The name survives, because "the token was here" is
1226
+ // the part worth keeping.
1227
+ //
1228
+ // This is the last line of defence, not the only one: the model is never
1229
+ // given the values in the first place, and checkBash already refuses the
1230
+ // obvious `echo $SOMETHING_TOKEN`. It exists because neither of those can
1231
+ // cover a script's own stdout.
1232
+ let redactions: [string, string][] = [];
1233
+ let fileDir: string | null = null;
1234
+ const redact = (text: string) => {
1235
+ let out = text;
1236
+ for (const [value, name] of redactions) out = out.split(value).join(`[redacted:${name}]`);
1237
+ return out;
1238
+ };
1239
+
1240
+ const push = (type: StepRecord["events"][number]["type"], text: string, extra?: EventExtra) => {
1241
+ step.events.push({ t: new Date().toISOString(), type, text: redact(text), ...extra });
1242
+ save();
1243
+ };
1244
+
1245
+ // The clock on the step itself. `??=` so a retry extends the first
1246
+ // attempt's span rather than starting a new one; finishedAt is stamped on
1247
+ // every exit below and the last attempt's stands.
1248
+ step.startedAt ??= new Date().toISOString();
1249
+
1250
+ // The test seam: FOLDRUN_STUB_STEP=1 completes the step from a canned
1251
+ // script instead of a model, so the orchestration around steps — loops,
1252
+ // fan-out, gates — is testable at zero cost. An agent's `stub.md` holds
1253
+ // the answers, `\n---\n`-separated, consumed one per call; the last one
1254
+ // repeats. Never set outside a test.
1255
+ if (process.env.FOLDRUN_STUB_STEP === "1") {
1256
+ const stubFile = path.join(agentDir, "stub.md");
1257
+ const answers = fs.existsSync(stubFile)
1258
+ ? fs.readFileSync(stubFile, "utf8").split(/\n---\n/)
1259
+ : [`stub result from ${step.agent}`];
1260
+ const call = step.events.filter((e) => e.text.startsWith("stub call")).length;
1261
+ push("info", `stub call ${call + 1}`);
1262
+ step.status = "completed";
1263
+ // A stub answer may open with `cost: 0.25` — a pretend price, so the
1264
+ // parts of orchestration that watch money (budget:) are testable too.
1265
+ const answer = (answers[Math.min(call, answers.length - 1)] ?? "").trim();
1266
+ const priced = answer.match(/^cost:\s*([\d.]+)\n?/);
1267
+ const body = (priced ? answer.slice(priced[0].length) : answer).trim();
1268
+ // A `===` line inside an answer separates earlier turns from the final
1269
+ // one, so the handoffs that read a conclusion rather than a transcript
1270
+ // are testable: result is everything joined, conclusion the last part.
1271
+ const turns = body.split(/\n===\n/);
1272
+ step.result = turns.join("\n").trim() + (step.item ? ` [item: ${step.item}]` : "");
1273
+ step.conclusion = turns.length > 1 ? turns.at(-1)!.trim() : null;
1274
+ step.costUsd = priced ? Number(priced[1]) : 0;
1275
+ // The stub honours the output contract too, so fan-out over data and
1276
+ // the failure of a step that returned no JSON are testable at no cost.
1277
+ if (step.output === "json") {
1278
+ const extracted = extractJson(step.result);
1279
+ if (extracted.ok) step.data = extracted.value;
1280
+ else {
1281
+ push("error", `output: json — ${extracted.reason}`);
1282
+ step.status = "failed";
1283
+ }
1284
+ }
1285
+ step.finishedAt = new Date().toISOString();
1286
+ save();
1287
+ return;
1288
+ }
1289
+
1290
+ try {
1291
+ // An `openapi:` URL is fetched (and cached a day) before the context is
1292
+ // assembled, because assembling it is synchronous and the document has
1293
+ // to be on disk by then. Failures are lines in the trace, not a failed
1294
+ // step: the generic HTTP tool still works without the typed ones.
1295
+ for (const w of await prefetchOpenApi(tenant, openApiSources(agentDir, tenant))) push("error", `openapi: ${w}`);
1296
+ const {
1297
+ front, clockEnv, systemPrompt, allowed, disabled, apiTools, scriptTools,
1298
+ secretEnv, secretScopes, missingSecrets, missingTools, runtime,
1299
+ unknownTools, shadowed, legacyUse, mcpServers, mcpNames,
1300
+ apiSpecs, scriptSpecs, brokenTools, size,
1301
+ providerEnv, providerLabel, providerSecrets, providerWarnings, formatWarning,
1302
+ fallbackEnv, apiWarnings, searchRoots, historyDigest, deskDigest, searchTools, historyTools, deskTools,
1303
+ translator, fallbackTranslator,
1304
+ } = agentContext(agentDir, tenant, tags, { runId, agent: step.agent });
1305
+
1306
+ // oauth2 secrets become live access tokens here — the one async moment
1307
+ // in secret resolution, at the last host-side point before anything is
1308
+ // injected or substituted. The refresh recipes never leave this process:
1309
+ // what crosses into containers, headers and env files is the token. A
1310
+ // failed refresh fails the step naming the secret and the provider's
1311
+ // reason, which beats a 401 three layers later.
1312
+ const liveSecrets = await materializeSecrets(secretEnv);
1313
+ // @file values (SSH keys, certs) still carry their content at this point.
1314
+ // The in-process branch materialises them to 0600 paths on this host
1315
+ // below; the container branch passes the content across and the driver
1316
+ // materialises inside, because a host path is meaningless in there.
1317
+
1318
+ // Populate before the first push: everything after this point may quote a
1319
+ // credential. Short values are skipped — a two-character secret would
1320
+ // redact half the English language, and anything that short is not one.
1321
+ redactions = Object.entries({ ...liveSecrets, ...providerSecrets, ...providerEnv })
1322
+ .filter(([, value]) => typeof value === "string" && value.length >= 8)
1323
+ .map(([name, value]) => [value, name] as [string, string])
1324
+ // Longest first, so a value containing another is replaced whole.
1325
+ .sort((a, b) => b[0].length - a[0].length);
1326
+
1327
+ if (providerLabel) push("info", `provider: ${providerLabel}`);
1328
+ for (const w of providerWarnings) push("error", w);
1329
+ for (const w of apiWarnings) push("error", `openapi: ${w}`);
1330
+ if (formatWarning) push("error", formatWarning);
1331
+ if (legacyUse.length) push("error", legacyUseError(legacyUse));
1332
+ for (const t of unknownTools) {
1333
+ push(
1334
+ "error",
1335
+ `tools: "${t}" is not a tool group (web, fetch, read, files, bash, search, history), an SDK tool name, or a tool in this workspace's library — nothing was granted for it`,
1336
+ );
1337
+ }
1338
+ for (const name of mcpNames) push("info", `mcp server connected: ${name}`);
1339
+ for (const name of shadowed) {
1340
+ push("error", `tools: "${name}" is a built-in, so your tool of the same name was not granted — write it as [[${name}]] to mean yours`);
1341
+ }
1342
+ for (const t of disabled) push("info", `tool ${t}: ask-mode is disabled for platform runs`);
1343
+ for (const s of missingSecrets) {
1344
+ push("error", `secret ${s} is not set in this workspace — add it in Settings → Secrets`);
1345
+ }
1346
+ // Say which store each secret came from: a workspace quietly falling back
1347
+ // to an account credential is the kind of thing you want to see, not infer.
1348
+ for (const [name, scope] of Object.entries(secretScopes)) {
1349
+ push("info", `secret ${name} ← ${scope} scope`);
1350
+ }
1351
+ for (const t of missingTools) {
1352
+ // "Not found" is the wrong sentence when the file is right there with a
1353
+ // typo in it, and it sends the author to the wrong place. Say which.
1354
+ const broken = brokenTools.filter((b) => b.startsWith(`${t}:`));
1355
+ push(
1356
+ "error",
1357
+ broken.length
1358
+ ? `tool "${t}" exists but could not be read — ${broken[0].slice(t.length + 2)}`
1359
+ : `tool "${t}" is not in the workspace library — add it under Library → Tools`,
1360
+ );
1361
+ }
1362
+ for (const line of runtime.log) push("info", line);
1363
+ push("info", `script executor: ${runtime.executor}`);
1364
+ if (runtime.error) push("error", `runtime: ${runtime.error}`);
1365
+ // Nearest wins, for both halves of model selection: the step names one,
1366
+ // else the flow does, else the agent's own frontmatter. Which level won
1367
+ // goes in the trace — "why did this run on haiku" should be answerable
1368
+ // from the run, not by opening three files.
1369
+ const modelSource = step.model ? "step" : modelOverride ? "flow" : "agent";
1370
+ const model = resolveModel(step.model ?? modelOverride ?? front.model);
1371
+ const rawEffort = step.effort ?? effortOverride ?? front.effort;
1372
+ const effortSource = step.effort ? "step" : effortOverride ? "flow" : "agent";
1373
+ let effort = resolveEffort(rawEffort);
1374
+ if (isEffortWord(rawEffort)) {
1375
+ push(
1376
+ "error",
1377
+ `effort: "${String(rawEffort).trim()}" is not a level — use ${EFFORT_LEVELS.join(", ")}. ` +
1378
+ `Running at the model's default instead.`,
1379
+ );
1380
+ }
1381
+ push(
1382
+ "info",
1383
+ `step started (agent: ${step.agent}, model: ${model} — ${modelSource}` +
1384
+ `${effort ? `, effort: ${effort} — ${effortSource}` : ""}` +
1385
+ // Say it every time, so a limit is never a surprise and its absence
1386
+ // is a choice on the record, not an accident.
1387
+ `${step.timeout ? `, timeout: ${step.timeout}s` : ", no timeout"})`,
1388
+ );
1389
+
1390
+ // The run-start gate, when a gateway is in play. "Any model they want"
1391
+ // is only an offer if a wrong pick fails here, in one line naming the
1392
+ // fix — not minutes in, as an agent narrating tool calls into prose
1393
+ // because its model cannot make them. The catalogue only ever fails a
1394
+ // step on knowledge; unknown ids, presets and offline all pass through
1395
+ // to the model call, which stays the authority of last resort.
1396
+ let catalog: Catalog | null = null;
1397
+ let tokenPrice: { input: number; output: number } | null = null;
1398
+ // The env this run's model calls actually ride: the workspace's own
1399
+ // provider block when one is declared, otherwise the platform's — the
1400
+ // "models included" path, where the host env points every run at the
1401
+ // platform's gateway and users only ever pick a model. The gate, the
1402
+ // effort fit and the repricing follow the credential, not the file:
1403
+ // keying them off the provider block alone left the platform-provided
1404
+ // path — the main one — without any of them.
1405
+ const rideEnv: Record<string, string | undefined> =
1406
+ Object.keys(providerEnv).length > 0 ? providerEnv : process.env;
1407
+ // Through the translator the endpoint is localhost and the model id is
1408
+ // the provider's own, sent as written; there is no catalogue to ask.
1409
+ if (translator) push("info", `translator: ${translator.label} — Anthropic Messages in, Chat Completions out`);
1410
+ // What actually goes on the wire: a tier the gateway remapped is sent
1411
+ // as the gateway's id, and that id — not our tier word — is what the
1412
+ // catalogue can have an opinion about.
1413
+ const wireModel =
1414
+ { haiku: rideEnv.ANTHROPIC_DEFAULT_HAIKU_MODEL,
1415
+ sonnet: rideEnv.ANTHROPIC_DEFAULT_SONNET_MODEL,
1416
+ opus: rideEnv.ANTHROPIC_DEFAULT_OPUS_MODEL }[model] ?? model;
1417
+ if (rideEnv.ANTHROPIC_BASE_URL && !translator) {
1418
+ catalog = await loadCatalog(rideEnv.ANTHROPIC_BASE_URL);
1419
+ const needsTools = allowed.length > 0 || mcpNames.length > 0;
1420
+ const verdict = checkModel(catalog, wireModel, { tools: needsTools });
1421
+ // Also what a token costs here, so a step can count its own spend
1422
+ // turn by turn against its ceiling instead of learning at the end.
1423
+ const priced = findModel(catalog, wireModel);
1424
+ if (priced && priced.promptPrice != null && priced.completionPrice != null) {
1425
+ tokenPrice = { input: priced.promptPrice, output: priced.completionPrice };
1426
+ }
1427
+ if (!verdict.ok) {
1428
+ push("error", verdict.reason!);
1429
+ step.status = "failed";
1430
+ save();
1431
+ return;
1432
+ }
1433
+ // Effort, fitted to what this model actually takes: unchanged when the
1434
+ // level is supported, the nearest supported level when it isn't, gone
1435
+ // when the model doesn't reason — each departure said aloud. This is
1436
+ // what makes `effort:` portable rather than Anthropic-only: the word
1437
+ // in the file stays the intent, the wire carries what the model has.
1438
+ const fitted = clampEffort(catalog, wireModel, effort);
1439
+ if (fitted.note) push("info", fitted.note);
1440
+ effort = fitted.effort;
1441
+ }
1442
+
1443
+ fs.mkdirSync(path.join(agentDir, "outputs"), { recursive: true });
1444
+ fs.mkdirSync(path.join(agentDir, "memory"), { recursive: true });
1445
+
1446
+ // Expose the workspace's shared scripts/ as ./shared so every agent reaches
1447
+ // them by the same path, without each one carrying a copy.
1448
+ // No symlinks into the agent directory — they would recurse into the
1449
+ // workspace's own file listing. Shared paths are resolved by the script
1450
+ // tool layer and described to the agent below.
1451
+
1452
+ // The confinement boundary: agents in a workspace are one team, but the
1453
+ // workspace is where isolation is enforced.
1454
+ const workspaceRoot = path.resolve(agentDir, "..", "..");
1455
+
1456
+ let prompt = step.instruction
1457
+ ? resolveDocLinks(step.instruction, workspaceRoot)
1458
+ : "Begin your run now, following your instructions.";
1459
+ if (context) prompt += `\n\n<previous_step_results>\n${context.slice(0, CONTEXT_CHARS)}\n</previous_step_results>`;
1460
+ if (priorData !== undefined) {
1461
+ // The value itself, beside the prose that mentioned it. This is the
1462
+ // lossless half of a handoff: a URL step 1 returned reaches step 3 as
1463
+ // the URL, not as whatever a re-reading of step 1's paragraph yields.
1464
+ const json = JSON.stringify(priorData, null, 2);
1465
+ prompt +=
1466
+ `\n\n<previous_step_data>\n${json.length > 60000 ? `${json.slice(0, 60000)}\n…[truncated]` : json}\n</previous_step_data>`;
1467
+ }
1468
+ if (step.output === "json") {
1469
+ // The contract the runner parses back out — stated in the prompt so
1470
+ // the instruction and the parser agree by construction, like delegate:.
1471
+ prompt +=
1472
+ `\n\n<output>\nThis step returns data. End your reply with exactly one JSON value inside a ` +
1473
+ "```json fenced block — the value the next step will compute on. Prose before it is " +
1474
+ `welcome (lead with a one-line headline); nothing may follow the block. If the instruction ` +
1475
+ `names fields, use exactly those names.\n</output>`;
1476
+ }
1477
+ if (step.waitFor === "event" && step.eventPayload !== undefined) {
1478
+ // Not resolved for links, like the previous step's results: it is
1479
+ // whatever the outside world sent, and must not name a file by
1480
+ // happening to contain brackets.
1481
+ prompt += `\n\n<event>\nThis step waited for an external event. What arrived:\n${step.eventPayload}\n</event>`;
1482
+ }
1483
+ if (step.approvalNote) {
1484
+ // The person at the gate said something while letting the run through.
1485
+ // That is the freshest instruction the step has — later than the flow
1486
+ // file, aimed at this exact run. When the gate asked a question
1487
+ // (ask:), the note IS the answer and is framed as one.
1488
+ //
1489
+ // Both halves go through resolveDocLinks for the same reason the
1490
+ // instruction above does: this is authored text on its way to becoming
1491
+ // a prompt, and the rule is that ALL of it expands. A person answering
1492
+ // "which list?" with [[dead-ends]] was sending the model two literal
1493
+ // brackets — the one reference syntax the workspace teaches, and the
1494
+ // one place it silently did nothing.
1495
+ //
1496
+ // What is NOT resolved, deliberately: the previous step's results
1497
+ // above. That is model output, not something a person wrote, and a
1498
+ // model that happens to emit [[brackets]] must not thereby name a file.
1499
+ const ask = step.ask ? resolveDocLinks(step.ask, workspaceRoot) : "";
1500
+ const note = resolveDocLinks(step.approvalNote, workspaceRoot);
1501
+ prompt += step.ask
1502
+ ? `\n\n<operator_answer>\nThis step asked a human: ${ask}\nTheir answer:\n${note}\n</operator_answer>`
1503
+ : `\n\n<operator_guidance>\nThe human who approved this step added:\n${note}\n</operator_guidance>`;
1504
+ }
1505
+ if (step.delegate?.length) {
1506
+ // The chooser's contract. The allowed set comes from the flow file;
1507
+ // the runner parses exactly this shape back out of the result, so the
1508
+ // instruction and the parser agree by construction.
1509
+ prompt +=
1510
+ `\n\n<delegation>\nAfter your analysis, end your reply with one line per agent you are delegating to, ` +
1511
+ `chosen ONLY from: ${step.delegate.join(", ")}.\nFormat, one per line (at most 5):\n` +
1512
+ `agent-name: the instruction for that agent\n` +
1513
+ `Lines naming any other agent are ignored.\n</delegation>`;
1514
+ }
1515
+ if (step.item) {
1516
+ // A fan-out instance: same instruction as its siblings, one item each.
1517
+ prompt += `\n\n<item>\nOf everything above, this instance of the step handles exactly one item — this one:\n${step.item}\n</item>`;
1518
+ }
1519
+
1520
+ // Colleagues this agent may consult mid-turn — declared, like every
1521
+ // other capability.
1522
+ const { consults, missing: missingConsults } = gatherConsults(workspaceRoot, front.agents);
1523
+ for (const m of missingConsults) {
1524
+ push("error", `agents: "${m}" is not an agent in this workspace — no consult tool granted`);
1525
+ }
1526
+ const consultNames = consults.length ? [...mcpNames, "foldrun_agents"] : mcpNames;
1527
+
1528
+ // The last provider refusal this step saw, for the fallback decision —
1529
+ // watched on both paths, because a refusal reads the same from a pod
1530
+ // and from this process.
1531
+ let lastRefusal = "";
1532
+ const pushWatching: typeof push = (type, text, extra) => {
1533
+ if (type === "error" && isProviderRefusal(text)) lastRefusal = text;
1534
+ push(type, text, extra);
1535
+ };
1536
+ // The second supply: the block's own fallback when it declared one (a
1537
+ // BYOK step's provider is the customer's arrangement, and so is its
1538
+ // fallback), else the platform's, and only for a step riding the
1539
+ // platform credential.
1540
+ const secondSupply = fallbackEnv ?? (Object.keys(providerEnv).length === 0 ? platformFallbackEnv() : null);
1541
+
1542
+ const isolation = process.env.FOLDRUN_RUN_ISOLATION;
1543
+ if (isolatedRun()) {
1544
+ // The isolated path: the whole loop — model, built-in tools, scripts —
1545
+ // runs inside a throwaway container (a docker sibling, or a pod), and
1546
+ // only filtered file changes come back. The vault stays out here:
1547
+ // secrets are substituted into API headers before the specs cross,
1548
+ // and reach scripts as env.
1549
+ push("info", `isolation: ${isolation}`);
1550
+ // The *merged* lists — inline declarations plus everything `tools:`
1551
+ // granted — not a re-parse of the frontmatter. Rebuilding from `front`
1552
+ // here silently dropped every tools:-granted http and script tool on
1553
+ // the isolated path (mcp grants survived only because mcpServers is
1554
+ // passed as the merged object): the agent's prompt promised a tool
1555
+ // the function list didn't have, and the step failed as the model
1556
+ // politely reporting the gap.
1557
+ // The egress lease: the sandbox sends through the worker, which holds
1558
+ // the values and fills each ${NAME} for the host that secret was
1559
+ // granted to. Built from what this step declared — every http tool's
1560
+ // secrets for that tool's host, the model key for the provider's —
1561
+ // before anything is copied in. Null where no proxy is configured
1562
+ // (the CLI, a compose install without FOLDRUN_EGRESS_URL), and then
1563
+ // everything below materialises exactly as it always did.
1564
+ const grant: EgressGrant = { secrets: {} };
1565
+ for (const api of apiSpecs) {
1566
+ const host = hostOf(api.base);
1567
+ if (!host) continue;
1568
+ for (const v of [...Object.values(api.headers), ...Object.values(api.query)]) {
1569
+ for (const name of placeholderNames(v)) if (name in liveSecrets) addGrant(grant, name, liveSecrets[name], host);
1570
+ }
1571
+ }
1572
+ const lease = await platform.egress.lease({ tenant, runId: runId ?? "adhoc", grant });
1573
+ // Which secrets must still be real inside the sandbox: a script tool
1574
+ // reads them from its environment, and unless it says `secrets:
1575
+ // proxied` it needs the value, not the name. bash is a script by
1576
+ // another route. The record says which way it went, and why.
1577
+ const materialisers = [
1578
+ ...scriptSpecs.filter((sc) => sc.secrets !== "proxied").map((sc) => sc.name),
1579
+ ...(allowed.includes("Bash") ? ["bash"] : []),
1580
+ ];
1581
+ const materialise = !lease || materialisers.length > 0;
1582
+ const declared = Object.keys(liveSecrets);
1583
+ if (lease) {
1584
+ push(
1585
+ "info",
1586
+ declared.length
1587
+ ? materialise
1588
+ ? `secrets: ${declared.join(", ")} materialised in the sandbox for ${materialisers.join(", ")}; api tools and the model key go through the egress proxy`
1589
+ : `secrets: ${declared.join(", ")} proxied — nothing in the sandbox but the names`
1590
+ : "secrets: none declared; the model key goes through the egress proxy",
1591
+ );
1592
+ }
1593
+ // Sandbox env for the declared secrets: the value, or its own name as
1594
+ // a placeholder that the proxy fills on the way out.
1595
+ const sandboxSecrets: Record<string, string> = materialise
1596
+ ? liveSecrets
1597
+ : Object.fromEntries(declared.map((name) => [name, `\${${name}}`]));
1598
+ const substitutedApis = apiSpecs.map((api) => ({
1599
+ ...api,
1600
+ headers: Object.fromEntries(
1601
+ Object.entries(api.headers).map(([k, v]) => [
1602
+ k,
1603
+ v.replace(/\$\{([A-Z][A-Z0-9_]*)\}/g, (whole, name) => sandboxSecrets[name] ?? whole),
1604
+ ]),
1605
+ ),
1606
+ }));
1607
+ // The provider's own key and headers, when the step has a lease: the
1608
+ // key becomes ${FOLDRUN_MODEL_KEY}, the base URL goes via the proxy,
1609
+ // and a translator's upstream request does the same. The fallback
1610
+ // supply gets its own placeholder name — two keys, two names.
1611
+ const proxied = <T extends Record<string, string | undefined>>(env: T, name: string): T =>
1612
+ lease ? (proxyModelEnv(env, lease.url, grant, providerSecrets, name) as T) : env;
1613
+ const proxiedTranslator = (spec: TranslatorSpec | null, name: string): TranslatorSpec | null => {
1614
+ if (!spec || !lease) return spec;
1615
+ const host = hostOf(spec.upstreamBase);
1616
+ if (!host) return spec;
1617
+ addGrant(grant, name, spec.upstreamKey, host);
1618
+ return {
1619
+ ...spec,
1620
+ egress: lease.url,
1621
+ upstreamKey: `\${${name}}`,
1622
+ headers: Object.fromEntries(
1623
+ Object.entries(spec.headers ?? {}).map(([k, v]) => [k, unsubstitute(v, providerSecrets, host, grant)]),
1624
+ ),
1625
+ };
1626
+ };
1627
+ const primaryTranslator = proxiedTranslator(translator, MODEL_KEY_NAME);
1628
+ const secondTranslator = proxiedTranslator(fallbackTranslator, `${MODEL_KEY_NAME}_FALLBACK`);
1629
+ const runIsolated = isolation === "container" ? runStepInContainer : platform.isolation[isolation!]!;
1630
+ const isolatedArgs = (modelEnv: Record<string, string | undefined>, keyName = MODEL_KEY_NAME) => ({
1631
+ workspaceRoot,
1632
+ libraryRoot: libraryDir(tenant),
1633
+ // Keys this step's dependency cache. Per-account by design — see
1634
+ // run-container.ts#runtimeCacheDir for why it is not shared.
1635
+ tenant,
1636
+ input: {
1637
+ agentRel: path.relative(workspaceRoot, agentDir).replaceAll("\\", "/"),
1638
+ prompt,
1639
+ model,
1640
+ effort,
1641
+ systemPrompt,
1642
+ allowed,
1643
+ mcpNames: consultNames,
1644
+ mcpServers,
1645
+ apis: substitutedApis,
1646
+ scripts: scriptSpecs,
1647
+ // The merged spec, not `front.runtime` — see the plan's `runtime`
1648
+ // field. A tool's own `runtime:` is part of what this step needs.
1649
+ runtime: runtime.spec,
1650
+ consults,
1651
+ timeoutSec: step.timeout,
1652
+ budgetUsd: stepBudgetUsd,
1653
+ price: tokenPrice,
1654
+ verify: step.verify,
1655
+ output: step.output,
1656
+ translator: keyName === MODEL_KEY_NAME ? primaryTranslator : secondTranslator,
1657
+ // The container sees the workspace at /workspace and the library
1658
+ // at /library; the roots are named host-side and moved here.
1659
+ search: searchRoots.map((r) => ({
1660
+ label: r.label,
1661
+ dir: r.dir.startsWith(libraryDir(tenant))
1662
+ ? path.posix.join("/library", path.relative(libraryDir(tenant), r.dir).replaceAll("\\", "/"))
1663
+ : path.posix.join("/workspace", path.relative(workspaceRoot, r.dir).replaceAll("\\", "/")),
1664
+ })),
1665
+ history: historyDigest,
1666
+ desks: deskDigest,
1667
+ },
1668
+ env: Object.fromEntries(
1669
+ Object.entries({
1670
+ // The platform's own model credential, when the agent names no
1671
+ // provider of its own — this is "models included": the host
1672
+ // process holds one key, and every isolated run borrows it.
1673
+ // A provider: block still wins, because providerEnv is spread
1674
+ // after and carries the agent's chosen endpoint + token.
1675
+ ...proxied(Object.keys(providerEnv).length === 0 ? modelEnv : providerEnv, keyName),
1676
+ ...(lease ? { [EGRESS_ENV]: lease.url } : {}),
1677
+ ...clockEnv,
1678
+ // Who is running: the in-process path stamps scripts with these
1679
+ // host-side, but a container rebuilds its tools from its own
1680
+ // environment, so without them here every script and verify in
1681
+ // the pod ran anonymous — and a verify comparing a marker to
1682
+ // $FOLDRUN_RUN_ID compared it to nothing.
1683
+ FOLDRUN_RUN_ID: runId,
1684
+ FOLDRUN_AGENT: step.agent,
1685
+ FOLDRUN_WORKSPACE: path.basename(workspaceRoot),
1686
+ ...sandboxSecrets,
1687
+ }).filter((entry): entry is [string, string] => typeof entry[1] === "string"),
1688
+ ),
1689
+ emit: pushWatching,
1690
+ // Stamps the sandbox, so stopping this run can destroy it.
1691
+ runId,
1692
+ size,
1693
+ });
1694
+
1695
+ // A busy provider gets another moment on the SAME supply before the
1696
+ // fallback is considered: under a fan-out, twenty steps against one
1697
+ // key are told "too many requests" together, and failing them all is
1698
+ // a broken run where a few seconds would have been a slow one. Each
1699
+ // attempt holds its own sandbox, so the meter owes the sum.
1700
+ const firstArgs = isolatedArgs(platformModelEnv());
1701
+ await lease?.commit();
1702
+ let outcome = await runIsolated(firstArgs);
1703
+ for (let n = 1; n <= OVERLOAD_RETRIES && outcome.status === "failed" && isTransientOverload(lastRefusal); n++) {
1704
+ const wait = backoffMs(n, lastRefusal);
1705
+ push("info", `provider busy (${lastRefusal.slice(0, 60)}) — waiting ${(wait / 1000).toFixed(1)}s, attempt ${n} of ${OVERLOAD_RETRIES}`);
1706
+ await sleep(wait);
1707
+ // Judged fresh: the next attempt's own failure decides what happens
1708
+ // next, not the one that sent us here.
1709
+ lastRefusal = "";
1710
+ const previous = outcome.timing;
1711
+ const again = isolatedArgs(platformModelEnv());
1712
+ await lease?.commit();
1713
+ outcome = await runIsolated(again);
1714
+ outcome = withEarlierTiming(outcome, previous);
1715
+ }
1716
+ // The second supply, tried exactly once, and only when the primary
1717
+ // refused over money/auth/limits rather than the work failing. The
1718
+ // retry is a fresh sandbox — the failed one is gone, and driveRun's
1719
+ // whole contract is that re-driving is safe. A declared fallback
1720
+ // replaces the provider block's env wholesale, so the spread order in
1721
+ // isolatedArgs (providerEnv last) has to be overridden here.
1722
+ if (outcome.status === "failed" && lastRefusal && secondSupply) {
1723
+ push(
1724
+ "info",
1725
+ `primary model supply refused (${lastRefusal.slice(0, 80)}) — retrying this step on the fallback provider`,
1726
+ );
1727
+ const first = outcome.timing;
1728
+ const fallbackName = `${MODEL_KEY_NAME}_FALLBACK`;
1729
+ const retryArgs = isolatedArgs(secondSupply, fallbackName);
1730
+ // The fallback's own translator, or none: a fallback that speaks
1731
+ // Anthropic must not inherit the primary's Chat-Completions door.
1732
+ const fallbackArgs = {
1733
+ ...retryArgs,
1734
+ input: { ...retryArgs.input, translator: fallbackEnv && secondSupply === fallbackEnv ? secondTranslator : null },
1735
+ env: Object.fromEntries(
1736
+ Object.entries({ ...retryArgs.env, ...proxied(secondSupply, fallbackName) })
1737
+ .filter((entry): entry is [string, string] => typeof entry[1] === "string"),
1738
+ ),
1739
+ };
1740
+ await lease?.commit();
1741
+ outcome = await runIsolated(fallbackArgs);
1742
+ // Both attempts held sandboxes; the meter owes the sum. The first
1743
+ // try's pod ran, was billed for by the platform, and must not
1744
+ // vanish from the record because a second try replaced its outcome.
1745
+ outcome = withEarlierTiming(outcome, first);
1746
+ }
1747
+ if (lease) {
1748
+ for (const line of await lease.drainLog()) push("info", line);
1749
+ await lease.release();
1750
+ }
1751
+ publishPublicDir(tenant, path.basename(workspaceRoot), push);
1752
+ step.status = outcome.status;
1753
+ step.result = outcome.result;
1754
+ step.conclusion = outcome.conclusion;
1755
+ if (outcome.data !== undefined) step.data = outcome.data;
1756
+ step.costUsd = repriced(catalog, wireModel, outcome.usage ?? null, outcome.costUsd, push);
1757
+ step.tokens = outcome.usage
1758
+ ? { input: outcome.usage.inputTokens, output: outcome.usage.outputTokens }
1759
+ : null;
1760
+ step.actual = outcome.res ?? null;
1761
+ if (outcome.res?.peakMemBytes || outcome.res?.rxBytes) {
1762
+ const mb = (n: number | null) => (n === null ? "?" : (n / 1024 / 1024).toFixed(0));
1763
+ push(
1764
+ "info",
1765
+ `resources: cpu busy ${outcome.res.busyCpuSecs?.toFixed(1) ?? "?"}s · ` +
1766
+ `peak mem ${mb(outcome.res.peakMemBytes)}MB · ` +
1767
+ `net ${mb(outcome.res.rxBytes)}MB in / ${mb(outcome.res.txBytes)}MB out`,
1768
+ );
1769
+ }
1770
+ // The compute meter, recorded on the step rather than derived later:
1771
+ // the sandbox is gone by the time the run settles, and its lifetime
1772
+ // is not recoverable from the run's own timestamps.
1773
+ const secs = (ms: number) => Math.round(ms) / 1000;
1774
+ step.computeSecs = outcome.timing ? secs(outcome.timing.totalMs) : null;
1775
+ step.startupSecs = outcome.timing ? secs(outcome.timing.sandboxMs) : null;
1776
+ // Which reservation those seconds held — as numbers, because the bill
1777
+ // is per core-second and per GiB-second, and what was held must come
1778
+ // from the record, never from what the config says at billing time.
1779
+ step.size = size;
1780
+ const lim = sizeLimits(size);
1781
+ const gib = (() => {
1782
+ const m = lim.memory.match(/^(\d+(?:\.\d+)?)\s*(Gi?|Mi?|g|m)?/i);
1783
+ const n = m ? Number(m[1]) : 2;
1784
+ return (m?.[2] ?? "Gi").toLowerCase().startsWith("m") ? n / 1024 : n;
1785
+ })();
1786
+ step.reserved = { cpus: Number(lim.cpus) || 2, memGiB: gib };
1787
+ } else {
1788
+ const consultTools = buildConsultTools(
1789
+ consults,
1790
+ { ...process.env, ...liveSecrets, ...providerEnv },
1791
+ (type, text) => push(type, text),
1792
+ );
1793
+ // @file secrets become 0600 paths here (host run), cleaned up in the
1794
+ // finally — once, however many attempts follow.
1795
+ const mat = materializeFileSecrets(agentDir, liveSecrets);
1796
+ fileDir = mat.dir;
1797
+ const attemptInProcess = (modelEnv: Record<string, string | undefined>) => executeStep({
1798
+ agentDir,
1799
+ workspaceRoot,
1800
+ libraryRoot: libraryDir(tenant),
1801
+ prompt,
1802
+ model,
1803
+ effort,
1804
+ systemPrompt,
1805
+ allowed,
1806
+ mcpNames: consultNames,
1807
+ mcpServers: {
1808
+ ...(apiTools.server ? { foldrun_apis: apiTools.server } : {}),
1809
+ ...(scriptTools.server ? { foldrun_scripts: scriptTools.server } : {}),
1810
+ ...(consultTools.server ? { foldrun_agents: consultTools.server } : {}),
1811
+ ...(searchTools.server ? { foldrun_search: searchTools.server } : {}),
1812
+ ...(historyTools.server ? { foldrun_history: historyTools.server } : {}),
1813
+ ...(deskTools.server ? { foldrun_desks: deskTools.server } : {}),
1814
+ ...mcpServers,
1815
+ },
1816
+ // Declared secrets reach the agent's scripts as env vars; the model
1817
+ // only ever sees the variable names, not the values.
1818
+ env: { ...process.env, ...clockEnv, ...mat.env, ...modelEnv },
1819
+ timeoutSec: step.timeout,
1820
+ budgetUsd: stepBudgetUsd,
1821
+ price: tokenPrice,
1822
+ verify: step.verify,
1823
+ output: step.output,
1824
+ // What the scripts saw, the verify sees: a flow can then check that
1825
+ // the proof a step left names THIS run, not one that came before —
1826
+ // copy-back never propagates a deletion, so a marker from an earlier
1827
+ // run rides into every later sandbox.
1828
+ verifyEnv: {
1829
+ ...clockEnv,
1830
+ ...(runId ? { FOLDRUN_RUN_ID: runId } : {}),
1831
+ FOLDRUN_AGENT: step.agent,
1832
+ FOLDRUN_WORKSPACE: path.basename(path.resolve(agentDir, "..", "..")),
1833
+ ...liveSecrets,
1834
+ },
1835
+ emit: pushWatching,
1836
+ });
1837
+ // With a translator the step's endpoint is the translator: started
1838
+ // here, pointed at by the SDK's env, closed when the attempt ends.
1839
+ const attemptThrough = async (modelEnv: Record<string, string | undefined>, spec: TranslatorSpec | null) => {
1840
+ if (!spec) return attemptInProcess(modelEnv);
1841
+ const t = await startTranslator(spec);
1842
+ try {
1843
+ return await attemptInProcess({ ...modelEnv, ...t.env });
1844
+ } finally {
1845
+ for (const line of t.drainLog()) push("info", line.startsWith("translator") ? line : `translator: ${line}`);
1846
+ await t.close();
1847
+ }
1848
+ };
1849
+ let outcome = await attemptThrough(providerEnv, translator);
1850
+ // Same rule as the isolated path: a busy provider is waited out on
1851
+ // the supply it refused from, before any fallback is considered.
1852
+ for (let n = 1; n <= OVERLOAD_RETRIES && outcome.status === "failed" && isTransientOverload(lastRefusal); n++) {
1853
+ const wait = backoffMs(n, lastRefusal);
1854
+ push("info", `provider busy (${lastRefusal.slice(0, 60)}) — waiting ${(wait / 1000).toFixed(1)}s, attempt ${n} of ${OVERLOAD_RETRIES}`);
1855
+ await sleep(wait);
1856
+ lastRefusal = "";
1857
+ outcome = await attemptThrough(providerEnv, translator);
1858
+ }
1859
+ // Same rule as the isolated path: one retry, on a refusal, on the
1860
+ // declared or platform second supply.
1861
+ if (outcome.status === "failed" && lastRefusal && secondSupply) {
1862
+ push(
1863
+ "info",
1864
+ `primary model supply refused (${lastRefusal.slice(0, 80)}) — retrying this step on the fallback provider`,
1865
+ );
1866
+ outcome = await attemptThrough(secondSupply, fallbackEnv && secondSupply === fallbackEnv ? fallbackTranslator : null);
1867
+ }
1868
+ publishPublicDir(tenant, path.basename(workspaceRoot), push);
1869
+ step.status = outcome.status;
1870
+ step.result = outcome.result;
1871
+ step.conclusion = outcome.conclusion;
1872
+ if (outcome.data !== undefined) step.data = outcome.data;
1873
+ // A consult's spend belongs to the step that asked.
1874
+ const consultCost = consultTools.drainCost();
1875
+ const stepCost = repriced(catalog, wireModel, outcome.usage, outcome.costUsd, push);
1876
+ step.costUsd = stepCost === null && consultCost === 0 ? null : (stepCost ?? 0) + consultCost;
1877
+ step.tokens = outcome.usage
1878
+ ? { input: outcome.usage.inputTokens, output: outcome.usage.outputTokens }
1879
+ : null;
1880
+ for (const line of apiTools.drainLog()) push("info", `api: ${line}`);
1881
+ for (const line of scriptTools.drainLog()) push("info", `script: ${line}`);
1882
+ }
1883
+ } catch (err) {
1884
+ step.status = "failed";
1885
+ push("error", err instanceof Error ? err.message : String(err));
1886
+ } finally {
1887
+ try {
1888
+ cleanupFileSecrets(fileDir);
1889
+ } catch {
1890
+ // best-effort — a leftover scratch file is swept with the run's outputs
1891
+ }
1892
+ }
1893
+ step.finishedAt = new Date().toISOString();
1894
+ save();
1895
+ }
1896
+
1897
+ // Block until a human approves or rejects the pending steps. Polls the run
1898
+ // record — the same file the dashboard writes through the approval API — so
1899
+ // no in-memory queue has to survive a restart. For as long as it takes: a
1900
+ // person answering on the 25th hour used to find the run had given up on
1901
+ // them at the 24th, and no clock of the platform's gets to decide that.
1902
+ async function waitForDecision(
1903
+ tenant: string,
1904
+ workspace: string,
1905
+ runId: string,
1906
+ indexes: number[],
1907
+ ): Promise<boolean> {
1908
+ for (;;) {
1909
+ await new Promise((r) => setTimeout(r, 1000));
1910
+ const latest = readRun(tenant, workspace, runId);
1911
+ if (!latest) return false;
1912
+ const stillWaiting = indexes.some((i) => latest.steps[i]?.status === "awaiting-approval");
1913
+ if (!stillWaiting) return true;
1914
+ }
1915
+ }
1916
+
1917
+ /**
1918
+ * Copy a directory tree by reading and writing the bytes.
1919
+ *
1920
+ * Not fs.cpSync: that copies through the kernel's clone/copy_file_range
1921
+ * path, which on a bind-mounted or network volume — exactly where a hosted
1922
+ * install keeps its data — can report success and leave a zero-length file
1923
+ * behind. Every archived run output was empty, and nothing failed to say
1924
+ * so, which is the worst shape a bug can take. This works on every
1925
+ * filesystem, and normalises the mode while it is here: what an agent
1926
+ * produced belongs to the platform, not to the container umask that
1927
+ * happened to create it.
1928
+ */
1929
+ export function copyTreeBytes(from: string, to: string) {
1930
+ fs.mkdirSync(to, { recursive: true });
1931
+ for (const entry of fs.readdirSync(from, { withFileTypes: true })) {
1932
+ const src = path.join(from, entry.name);
1933
+ const dst = path.join(to, entry.name);
1934
+ if (entry.isDirectory()) copyTreeBytes(src, dst);
1935
+ else if (entry.isFile()) fs.writeFileSync(dst, fs.readFileSync(src), { mode: 0o644 });
1936
+ }
1937
+ }
1938
+
1939
+ // Which of the host's own credentials an isolated run may borrow. Only the
1940
+ // model credential — a run gets the ability to think on the platform's
1941
+ // account, never the platform's other secrets, which is the entire reason
1942
+ // the container env is allowlisted instead of inheriting process.env.
1943
+ /**
1944
+ * The platform's second model supply, tried when the first refuses over
1945
+ * money or auth. Order is a cost decision: a flat-rate subscription as the
1946
+ * engine, pay-per-token only when it runs dry —
1947
+ *
1948
+ * FOLDRUN_FALLBACK_BASE_URL e.g. https://openrouter.ai/api
1949
+ * FOLDRUN_FALLBACK_TOKEN bearer for that endpoint
1950
+ * FOLDRUN_FALLBACK_HAIKU/SONNET/OPUS_MODEL tier ids in ITS catalogue
1951
+ *
1952
+ * Empty ANTHROPIC_API_KEY on purpose: set, it goes out as x-api-key and a
1953
+ * gateway rejects it; the fallback's whole credential is the bearer.
1954
+ */
1955
+ /**
1956
+ * Every `openapi:` URL the agent's API tools name — its own `apis:` and the
1957
+ * http tools it opted into — so they can be fetched before the context is
1958
+ * assembled. File paths need no fetching and are left out.
1959
+ */
1960
+ function openApiSources(agentDir: string, tenant: string): string[] {
1961
+ const out = new Set<string>();
1962
+ try {
1963
+ const { data } = matter(fs.readFileSync(path.join(agentDir, "agent.md"), "utf8"));
1964
+ const front = data as { apis?: unknown; tools?: unknown };
1965
+ for (const api of parseApis(front.apis)) if (api.openapi?.startsWith("https://")) out.add(api.openapi);
1966
+ const workspace = path.basename(path.resolve(agentDir, "..", ".."));
1967
+ const available = { ...libraryTools(tenant), ...workspaceTools(tenant, workspace) };
1968
+ for (const name of ownToolNames(front)) {
1969
+ const def = available[name];
1970
+ if (def?.kind === "http" && def.spec.openapi?.startsWith("https://")) out.add(def.spec.openapi);
1971
+ }
1972
+ } catch {
1973
+ // an unreadable agent.md fails the step properly a moment later
1974
+ }
1975
+ return [...out];
1976
+ }
1977
+
1978
+ function platformFallbackEnv(): Record<string, string> | null {
1979
+ const base = process.env.FOLDRUN_FALLBACK_BASE_URL;
1980
+ const token = process.env.FOLDRUN_FALLBACK_TOKEN;
1981
+ if (!base || !token) return null;
1982
+ const out: Record<string, string> = {
1983
+ ANTHROPIC_BASE_URL: base,
1984
+ ANTHROPIC_AUTH_TOKEN: token,
1985
+ ANTHROPIC_API_KEY: "",
1986
+ CLAUDE_CODE_OAUTH_TOKEN: "",
1987
+ };
1988
+ for (const tier of ["HAIKU", "SONNET", "OPUS"] as const) {
1989
+ const id = process.env[`FOLDRUN_FALLBACK_${tier}_MODEL`];
1990
+ if (id) out[`ANTHROPIC_DEFAULT_${tier}_MODEL`] = id;
1991
+ }
1992
+ return out;
1993
+ }
1994
+
1995
+ /** A model-call failure the fallback can answer: the primary refusing over
1996
+ * credit, quota or auth — never a failure of the work itself, which would
1997
+ * fail identically anywhere. */
1998
+ function isProviderRefusal(text: string): boolean {
1999
+ // Money and auth words only, plus the API's own status codes. The first
2000
+ // version matched bare "exceed", which is also how the pod backstop
2001
+ // phrases a timeout — and a timed-out step got retried on the paid
2002
+ // fallback it could only time out on again. A refusal the fallback can
2003
+ // answer is about the ACCOUNT; anything about the work or the sandbox is
2004
+ // not, however similar the vocabulary.
2005
+ return /API Error: (401|402|403|429|5\d\d)|credits?\b|quota|billing|insufficient|overloaded/i.test(text);
2006
+ }
2007
+
2008
+ /** How many times a busy provider is given another moment, and the longest
2009
+ * any one wait may be. Three is the industry's habit and the point where
2010
+ * waiting longer stops being kinder than failing. */
2011
+ export const OVERLOAD_RETRIES = 3;
2012
+ const MAX_BACKOFF_MS = 30_000;
2013
+
2014
+ /**
2015
+ * A refusal the SAME supply may answer in a moment — the provider is busy,
2016
+ * not the account broken.
2017
+ *
2018
+ * The distinction is the whole point. 429 means "too many requests", and a
2019
+ * provider says it for two unrelated reasons: the endpoint is momentarily
2020
+ * saturated (wait, and it works), or the account is out of credit or over
2021
+ * its quota (wait forever, it never works). Money words decide it: a 429
2022
+ * that mentions credit, quota or billing is the account, and belongs to the
2023
+ * fallback provider, not to a retry that can only fail again.
2024
+ */
2025
+ export function isTransientOverload(text: string): boolean {
2026
+ if (/credits?\b|quota|billing|insufficient|payment|401|402|403/i.test(text)) return false;
2027
+ return /\b(429|503|529)\b|overloaded|rate.?limit|too many requests|try again/i.test(text);
2028
+ }
2029
+
2030
+ /**
2031
+ * How long to wait before attempt `n` (1-based). A `retry-after` the
2032
+ * provider named wins — it knows when it will be ready. Otherwise
2033
+ * exponential, with jitter: twenty steps of a fan-out that all failed
2034
+ * together must not all come back together, or the second wave is the first
2035
+ * wave again. Capped, because a step waiting half a minute is already the
2036
+ * limit of what a slow run should cost a person watching it.
2037
+ */
2038
+ const sleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
2039
+
2040
+ /**
2041
+ * Carry an earlier attempt's sandbox time into the outcome that replaced
2042
+ * it. Every attempt rented a pod; the meter owes all of them, and a record
2043
+ * that shows only the last one under-bills the platform and lies to the
2044
+ * customer about how long their step actually took.
2045
+ */
2046
+ function withEarlierTiming<T extends { timing?: StepTiming | null }>(outcome: T, earlier: StepTiming | null | undefined): T {
2047
+ if (!earlier) return outcome;
2048
+ if (!outcome.timing) return { ...outcome, timing: earlier };
2049
+ return { ...outcome, timing: { ...outcome.timing, totalMs: outcome.timing.totalMs + earlier.totalMs } };
2050
+ }
2051
+
2052
+ export function backoffMs(attempt: number, text = "", random: () => number = Math.random): number {
2053
+ // retry-after, retry_after, "retryAfter": — every spelling a provider or
2054
+ // its SDK has used for the same header.
2055
+ const named = text.match(/retry[-_\s]?after["':\s]+(\d+)/i);
2056
+ if (named) return Math.min(Math.max(Number(named[1]), 1) * 1000, MAX_BACKOFF_MS);
2057
+ const base = Math.min(2 ** attempt * 1000, MAX_BACKOFF_MS);
2058
+ return Math.round(base * (0.5 + random() * 0.5));
2059
+ }
2060
+
2061
+ function platformModelEnv(): Record<string, string | undefined> {
2062
+ const {
2063
+ ANTHROPIC_API_KEY,
2064
+ ANTHROPIC_AUTH_TOKEN,
2065
+ ANTHROPIC_BASE_URL,
2066
+ CLAUDE_CODE_OAUTH_TOKEN,
2067
+ // What the tiers mean when the platform's own provider is not Anthropic.
2068
+ // These belong to the credential: a base URL pointing at OpenRouter with
2069
+ // Anthropic's model ids still attached is not a working provider, it is
2070
+ // a 404 per step. Forwarded for the same reason the base URL is — an
2071
+ // agent that names no provider gets the platform's, whole.
2072
+ ANTHROPIC_DEFAULT_HAIKU_MODEL,
2073
+ ANTHROPIC_DEFAULT_SONNET_MODEL,
2074
+ ANTHROPIC_DEFAULT_OPUS_MODEL,
2075
+ } = process.env;
2076
+ return {
2077
+ ANTHROPIC_API_KEY,
2078
+ ANTHROPIC_AUTH_TOKEN,
2079
+ ANTHROPIC_BASE_URL,
2080
+ CLAUDE_CODE_OAUTH_TOKEN,
2081
+ ANTHROPIC_DEFAULT_HAIKU_MODEL,
2082
+ ANTHROPIC_DEFAULT_SONNET_MODEL,
2083
+ ANTHROPIC_DEFAULT_OPUS_MODEL,
2084
+ };
2085
+ }
2086
+
2087
+ /**
2088
+ * Write a fresh run record and reset outputs — everything starting a run
2089
+ * means *except* driving it. Split out so the queue can create a `queued`
2090
+ * record for a worker to pick up, while `startFlowRun` keeps its promise of
2091
+ * a run that is already going when it returns.
2092
+ */
2093
+ /**
2094
+ * `each: rows of <path>` — one item per CSV data row, each carrying the
2095
+ * header so an instance reads as a one-row CSV. The path resolves the way
2096
+ * the agent's own instructions do (relative to its folder) and is confined
2097
+ * to the workspace: fan-out must not become a way to read past it.
2098
+ */
2099
+ /**
2100
+ * `each: items` — one instance per element of the previous group's data.
2101
+ * An array is the items; an object with one array-valued field is that
2102
+ * field (the common `{ "leads": [...] }` shape); anything else is nothing,
2103
+ * said out loud. Each item crosses as JSON text, because `item` is what the
2104
+ * instance's prompt quotes and what its result is labelled with.
2105
+ */
2106
+ function jsonItems(data: unknown, step: StepRecord, take: number): string[] {
2107
+ let list: unknown[] | null = Array.isArray(data) ? data : null;
2108
+ if (!list && data && typeof data === "object") {
2109
+ const arrays = Object.entries(data as Record<string, unknown>).filter(([, v]) => Array.isArray(v));
2110
+ if (arrays.length === 1) list = arrays[0][1] as unknown[];
2111
+ else if (arrays.length > 1) {
2112
+ step.skipReason = `each: items — the previous data has ${arrays.length} array fields; return the list itself`;
2113
+ return [];
2114
+ }
2115
+ }
2116
+ if (!list) {
2117
+ step.skipReason =
2118
+ data === undefined
2119
+ ? "each: items needs an `output: json` step before it that returned an array"
2120
+ : "each: items — the previous data is not an array";
2121
+ return [];
2122
+ }
2123
+ return list.slice(0, take).map((v) => (typeof v === "string" ? v : JSON.stringify(v)));
2124
+ }
2125
+
2126
+ function csvItems(workspaceDir: string, step: StepRecord, take: number): string[] {
2127
+ const raw = step.eachPath ?? "";
2128
+ if (!raw) {
2129
+ step.skipReason = "each: rows needs a path — `each: rows of ../../storage/x.csv`";
2130
+ return [];
2131
+ }
2132
+ const resolved = path.resolve(workspaceDir, "agents", step.agent, raw);
2133
+ if (!resolved.startsWith(path.resolve(workspaceDir) + path.sep)) {
2134
+ step.skipReason = `each: rows — ${raw} is outside this workspace`;
2135
+ return [];
2136
+ }
2137
+ let text: string;
2138
+ try {
2139
+ text = fs.readFileSync(resolved, "utf8");
2140
+ } catch {
2141
+ step.skipReason = `each: rows — ${raw} does not exist (yet?)`;
2142
+ return [];
2143
+ }
2144
+ // Minimal CSV: quoted fields with embedded commas/newlines survive.
2145
+ const rows: string[] = [];
2146
+ let field = "", row = "", quoted = false;
2147
+ for (let i = 0; i < text.length; i++) {
2148
+ const c = text[i];
2149
+ if (quoted) {
2150
+ if (c === '"' && text[i + 1] === '"') { field += '""'; i++; }
2151
+ else if (c === '"') { field += c; quoted = false; }
2152
+ else field += c;
2153
+ } else if (c === '"') { field += c; quoted = true; }
2154
+ else if (c === "\n") { rows.push(row + field); row = ""; field = ""; }
2155
+ else if (c !== "\r") field += c;
2156
+ }
2157
+ if (row + field) rows.push(row + field);
2158
+ const [header, ...data] = rows.filter((r) => r.trim() !== "");
2159
+ if (!header || data.length === 0) {
2160
+ step.skipReason = `each: rows — ${raw} has no data rows`;
2161
+ return [];
2162
+ }
2163
+ return data.slice(0, take).map((r) => `${header}\n${r}`);
2164
+ }
2165
+
2166
+ export function createFlowRun(
2167
+ tenant: string,
2168
+ workspace: string,
2169
+ steps: FlowStep[],
2170
+ flowName: string,
2171
+ status: "queued" | "running",
2172
+ tags: string[] = [],
2173
+ ): RunRecord {
2174
+ const run: RunRecord = {
2175
+ id: `run-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`,
2176
+ flow: flowName,
2177
+ tags,
2178
+ status,
2179
+ startedAt: new Date().toISOString(),
2180
+ finishedAt: null,
2181
+ steps: steps.map((s) => ({
2182
+ agent: s.agent,
2183
+ instruction: s.instruction,
2184
+ group: s.group ?? 1,
2185
+ optional: s.optional ?? false,
2186
+ model: s.model,
2187
+ effort: s.effort,
2188
+ approve: s.approve,
2189
+ when: s.when,
2190
+ case: s.case,
2191
+ else: s.else,
2192
+ retry: s.retry,
2193
+ timeout: s.timeout,
2194
+ verify: s.verify,
2195
+ loop: s.loop,
2196
+ until: s.until,
2197
+ loopRemaining: s.loop,
2198
+ each: s.each,
2199
+ eachPath: s.eachPath,
2200
+ max: s.max,
2201
+ onFail: s.onFail,
2202
+ waitSecs: s.waitSecs,
2203
+ waitFor: s.waitFor,
2204
+ ask: s.ask,
2205
+ preview: s.preview,
2206
+ delegate: s.delegate,
2207
+ output: s.output,
2208
+ attempts: 0,
2209
+ status: "pending",
2210
+ events: [],
2211
+ result: null,
2212
+ costUsd: null,
2213
+ })),
2214
+ };
2215
+ writeRun(tenant, workspace, run);
2216
+
2217
+ // Start every run from an empty outputs/ for each agent it will touch.
2218
+ //
2219
+ // Without this, `verify: test -s outputs/report.md` passes on run 2 because
2220
+ // run 1 left the file behind — the one check that exists to catch a model
2221
+ // claiming work it didn't do can be satisfied by a stale artifact. The
2222
+ // previous contents aren't lost: they were archived when that run finished.
2223
+ //
2224
+ // This is the one thing resuming must not do: a resumed run's earlier steps
2225
+ // already wrote there, and clearing it would delete the work being resumed.
2226
+ const runRoot = workspaceDir(tenant, workspace);
2227
+ for (const agent of new Set(run.steps.map((s) => s.agent))) {
2228
+ const outputs = path.join(runRoot, "agents", agent, "outputs");
2229
+ fs.rmSync(outputs, { recursive: true, force: true });
2230
+ fs.mkdirSync(outputs, { recursive: true });
2231
+ }
2232
+
2233
+ return run;
2234
+ }
2235
+
2236
+ export function startFlowRun(
2237
+ tenant: string,
2238
+ workspace: string,
2239
+ steps: FlowStep[],
2240
+ flowName: string,
2241
+ modelOverride?: string | null,
2242
+ tags: string[] = [],
2243
+ effortOverride?: string | null,
2244
+ ): RunRecord {
2245
+ const run = createFlowRun(tenant, workspace, steps, flowName, "running", tags);
2246
+ void driveRun(tenant, workspace, run, modelOverride, tags, { effortOverride });
2247
+ return run;
2248
+ }
2249
+
2250
+ /**
2251
+ * Drive a run to completion: groups in order, approval gates, retries.
2252
+ *
2253
+ * Separate from starting one so a run can be picked up again. The loop reads
2254
+ * every decision off the run record — `pending` steps are the work left, and
2255
+ * a completed group's results are already on it — so re-entering it on a
2256
+ * partly-finished record continues rather than restarts.
2257
+ *
2258
+ * `parkOnApproval` — a queue worker's slot is too expensive to spend polling
2259
+ * a file for a person who may answer tomorrow. With it set, an approval gate
2260
+ * saves the run as parked and returns; the approval API re-enqueues it and a
2261
+ * worker re-enters this loop, which continues where the record says it
2262
+ * stopped. Without it (the CLI, whose process belongs to the person waiting)
2263
+ * the gate blocks in place exactly as before.
2264
+ */
2265
+ /**
2266
+ * Turn a gate's `preview:` patterns into the files under storage/ that
2267
+ * exist right now, relative to storage/. `*` matches within a segment,
2268
+ * `**` across segments; a pattern with no wildcard is kept only if the file
2269
+ * is there. Sorted, deduplicated, capped — a preview is a handful of files,
2270
+ * not a directory dump.
2271
+ */
2272
+ export function resolvePreview(storageDir: string, patterns: string[], cap = 12): string[] {
2273
+ const out = new Set<string>();
2274
+ for (const pattern of patterns) {
2275
+ if (!/[*?]/.test(pattern)) {
2276
+ if (fs.existsSync(path.join(storageDir, pattern))) out.add(pattern);
2277
+ continue;
2278
+ }
2279
+ const re = new RegExp(
2280
+ "^" + pattern
2281
+ .split("**").map((part) => part.split("*").map((x) => x.replace(/[.+^${}()|[\]\\?]/g, "\\$&")).join("[^/]*"))
2282
+ .join(".*") + "$",
2283
+ );
2284
+ const walk = (dir: string, rel: string) => {
2285
+ let entries: fs.Dirent[];
2286
+ try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }
2287
+ for (const e of entries) {
2288
+ if (e.name.startsWith(".")) continue;
2289
+ const r = rel ? `${rel}/${e.name}` : e.name;
2290
+ if (e.isDirectory()) walk(path.join(dir, e.name), r);
2291
+ else if (re.test(r)) out.add(r);
2292
+ }
2293
+ };
2294
+ walk(storageDir, "");
2295
+ }
2296
+ return [...out].sort().slice(0, cap);
2297
+ }
2298
+
2299
+ /** Store what a run has left in storage/ so far. Never fails the run: a
2300
+ * store that is down costs the gate its thumbnail, not the run its step. */
2301
+ /**
2302
+ * Copy onto the in-memory run any gate decision that reached the record on
2303
+ * disk while this driver was busy — status, stamps, note, the event line —
2304
+ * so the driver's next save keeps it. Returns whether anything landed. Only
2305
+ * steps this copy still shows as asking are touched: everything else on the
2306
+ * record is this driver's own, and newer in memory than on disk.
2307
+ */
2308
+ export function adoptDecisions(run: RunRecord, disk: RunRecord | null): boolean {
2309
+ if (!disk) return false;
2310
+ let any = false;
2311
+ run.steps.forEach((s, i) => {
2312
+ const d = disk.steps[i];
2313
+ if (!d || s.status !== "awaiting-approval" || d.status === "awaiting-approval") return;
2314
+ s.status = d.status;
2315
+ s.events = d.events;
2316
+ if (d.approvedAt) s.approvedAt = d.approvedAt;
2317
+ if (d.approvalNote) s.approvalNote = d.approvalNote;
2318
+ any = true;
2319
+ });
2320
+ return any;
2321
+ }
2322
+
2323
+ async function harvestQuietly(tenant: string, workspace: string, run: RunRecord): Promise<void> {
2324
+ try {
2325
+ await harvestFiles(tenant, workspace, `run:${run.id}`);
2326
+ } catch {
2327
+ // bookkeeping only
2328
+ }
2329
+ }
2330
+
2331
+ /**
2332
+ * The runs this process is driving right now. The one fact reconcile needs:
2333
+ * a run in here is alive however quiet its step is, and a `running` run not
2334
+ * in here has no driver — this is the only process that drives.
2335
+ */
2336
+ export const drivingRuns = new Set<string>();
2337
+
2338
+ export function driveRun(
2339
+ tenant: string,
2340
+ workspace: string,
2341
+ run: RunRecord,
2342
+ modelOverride?: string | null,
2343
+ tags: string[] = [],
2344
+ opts: { parkOnApproval?: boolean; effortOverride?: string | null } = {},
2345
+ ): Promise<void> {
2346
+ drivingRuns.add(run.id);
2347
+ return driveRunInner(tenant, workspace, run, modelOverride, tags, opts).finally(() => {
2348
+ drivingRuns.delete(run.id);
2349
+ });
2350
+ }
2351
+
2352
+ function driveRunInner(
2353
+ tenant: string,
2354
+ workspace: string,
2355
+ run: RunRecord,
2356
+ modelOverride: string | null | undefined,
2357
+ tags: string[],
2358
+ opts: { parkOnApproval?: boolean; effortOverride?: string | null },
2359
+ ): Promise<void> {
2360
+ const effortOverride = opts.effortOverride ?? null;
2361
+ const pDir = workspaceDir(tenant, workspace);
2362
+ const runRoot = pDir;
2363
+ /**
2364
+ * Persist the run — carrying forward a stop that arrived while we were busy.
2365
+ *
2366
+ * A stop is written to the record by another process (stopRun), and this
2367
+ * loop holds the run in memory for as long as a group takes. Without the
2368
+ * merge below, the next save silently overwrote `stopRequested`, and the
2369
+ * between-groups check then read back the value it had just erased. A run
2370
+ * stopped mid-group carried on to the end of the flow as if nothing had
2371
+ * happened — observed on 2026-09-03, when a run stopped at 04:52 published
2372
+ * to five live Google Business Profiles at 06:05.
2373
+ *
2374
+ * The flag is one-way: nothing here clears it, so the read-modify-write can
2375
+ * only ever turn a stop on, never off.
2376
+ */
2377
+ const save = () => {
2378
+ if (!run.stopRequested && readRun(tenant, workspace, run.id)?.stopRequested) {
2379
+ run.stopRequested = true;
2380
+ }
2381
+ writeRun(tenant, workspace, run);
2382
+ };
2383
+
2384
+ // Provenance: a memory an agent wrote gets stamped with who wrote it, per
2385
+ // OKF v0.2. Without this a fact the model invented and a fact a person
2386
+ // verified are indistinguishable on disk — and the reader has no way to
2387
+ // know which one they're trusting.
2388
+ const stampMemories = () => {
2389
+ const agents = new Set(run.steps.map((s) => s.agent));
2390
+
2391
+ // Every memory bundle a run can write to, not just the agent's own. The
2392
+ // workspace bundle is writable from any step — confine only denies
2393
+ // knowledge/ — so a fact left there was going unstamped and untyped, and
2394
+ // the bundle stopped conforming until someone ran `foldrun check`.
2395
+ // The account library is not here: it is read-only from a run.
2396
+ const dirs: { dir: string; agent: string | null }[] = [
2397
+ // Attributable only when one agent could have written it.
2398
+ { dir: path.join(runRoot, "memory"), agent: agents.size === 1 ? [...agents][0] : null },
2399
+ ...[...agents].map((agent) => ({
2400
+ dir: path.join(runRoot, "agents", agent, "memory"),
2401
+ agent,
2402
+ })),
2403
+ ];
2404
+
2405
+ const written: string[] = [];
2406
+ for (const { dir, agent } of dirs) {
2407
+ // Each stamped file is a Creation in the bundle's log — that is what the
2408
+ // log is for, and an agent's writes are exactly the changes a reader
2409
+ // most wants dated. syncBundleFor logs the file it is given, so passing
2410
+ // index.md logged nothing about the concept that actually appeared.
2411
+ for (const rel of stampBundle(dir, agent)) {
2412
+ syncBundleFor(path.join(dir, rel), "Creation");
2413
+ written.push(path.relative(runRoot, path.join(dir, rel)).replaceAll("\\", "/"));
2414
+ }
2415
+ }
2416
+ // On the record, so the run page can say "this run learned two things"
2417
+ // and a reviewer can open exactly those files.
2418
+ if (written.length) run.memoryWrites = [...(run.memoryWrites ?? []), ...written];
2419
+ };
2420
+
2421
+ // Archive what a run produced, so history survives the next run's reset.
2422
+ const archive = () => {
2423
+ for (const agent of new Set(run.steps.map((s) => s.agent))) {
2424
+ const from = path.join(runRoot, "agents", agent, "outputs");
2425
+ if (!fs.existsSync(from) || fs.readdirSync(from).length === 0) continue;
2426
+ copyTreeBytes(from, path.join(runRoot, "runs", run.id, "outputs", agent));
2427
+ }
2428
+ };
2429
+
2430
+ // A parked run is paused, not finished — the completion bookkeeping in
2431
+ // `finally` (finishedAt, archive, memory stamps) must not touch it.
2432
+ let parked = false;
2433
+
2434
+ return (async () => {
2435
+ try {
2436
+ // Entering the loop is what makes a run live, whatever state it was
2437
+ // saved in: `queued` from the queue, `awaiting-approval` from a park.
2438
+ if (run.status !== "running") {
2439
+ run.status = "running";
2440
+ run.parkedAt = null;
2441
+ save();
2442
+ }
2443
+
2444
+ // A step still marked `running` here is an orphan: exactly one driver
2445
+ // holds a run's job, so whatever process set that status is gone —
2446
+ // killed mid-step by a deploy or a crash. Left alone it is worse than
2447
+ // failed: the group loop only runs `pending` steps, so the driver
2448
+ // would step over the unfinished work, run everything after it, and
2449
+ // mark the run completed — a run that reports success on work that
2450
+ // never happened, which reconcile can then never see. Reset it to
2451
+ // pending and it simply runs again; driveRun's whole contract is that
2452
+ // re-driving a partly-finished record is safe.
2453
+ let orphaned = false;
2454
+ for (const step of run.steps) {
2455
+ if (step.status !== "running") continue;
2456
+ orphaned = true;
2457
+ step.status = "pending";
2458
+ step.events.push({
2459
+ t: new Date().toISOString(),
2460
+ type: "info",
2461
+ text: "interrupted mid-step (platform restart) — running again from the start of the step",
2462
+ });
2463
+ }
2464
+ if (orphaned) {
2465
+ // The sandbox outlives the process that created it — a step with no
2466
+ // `timeout:` carries no pod deadline at all — so the orphaned step
2467
+ // may still be executing, non-idempotent work included, while the
2468
+ // new attempt starts. Destroy it first; re-driving is only safe
2469
+ // once nothing else is driving.
2470
+ try {
2471
+ if (process.env.FOLDRUN_RUN_ISOLATION === "container") killRunSandboxes(run.id);
2472
+ else platform.killRunSandboxes(run.id);
2473
+ } catch {
2474
+ // a sandbox we cannot reach must not stop the run from resuming
2475
+ }
2476
+ }
2477
+ save();
2478
+
2479
+ // Put the workspace's files on disk before the first step, so they are
2480
+ // simply *there* when the step's container copy is taken. This is the
2481
+ // whole of the agent-facing file API: a directory. Nothing here hands
2482
+ // the model a bucket, a URL or a credential, and a run pod that has
2483
+ // never heard of S3 reads a 200MB PDF the same way it reads AGENTS.md.
2484
+ //
2485
+ // Runs on resume too — a parked run's container is long gone, and the
2486
+ // one it comes back in needs the same files the first one had.
2487
+ try {
2488
+ const brought = await materializeFiles(tenant, workspace);
2489
+ if (brought.length) {
2490
+ run.steps[0]?.events.push({
2491
+ t: new Date().toISOString(),
2492
+ type: "info",
2493
+ text: `files: ${brought.length} brought into the workspace`,
2494
+ });
2495
+ }
2496
+ } catch (err) {
2497
+ // A file store that is down must not take the run with it: most flows
2498
+ // never touch storage/, and the ones that do will fail their own verify.
2499
+ run.steps[0]?.events.push({
2500
+ t: new Date().toISOString(),
2501
+ type: "error",
2502
+ text: `files: could not be brought in — ${err instanceof Error ? err.message : String(err)}`,
2503
+ });
2504
+ }
2505
+
2506
+ // Group steps by their group number, ascending. Rebuilt on demand
2507
+ // because two patterns mutate the step list mid-run: fan-out expands a
2508
+ // step into instances, and an evaluator loop winds groups back.
2509
+ const orderedGroups = () => {
2510
+ const groups = new Map<number, StepRecord[]>();
2511
+ for (const s of run.steps) {
2512
+ const list = groups.get(s.group) ?? [];
2513
+ list.push(s);
2514
+ groups.set(s.group, list);
2515
+ }
2516
+ return [...groups.entries()].sort(([a], [b]) => a - b).map(([, g]) => g);
2517
+ };
2518
+ let ordered = orderedGroups();
2519
+
2520
+ // What each finished group produced, by position — so a loop that
2521
+ // winds back can hand the re-run the same upstream context the first
2522
+ // pass saw, instead of its own stale output.
2523
+ const groupResults: (string | null)[] = [];
2524
+ const groupConclusions: (string | null)[] = [];
2525
+ // Every earlier group, oldest first, so a group-3 reporter sees what
2526
+ // group 1 measured and not only what group 2 diagnosed. This used to
2527
+ // hand back the nearest earlier group alone, and a reporter two groups
2528
+ // after the measurement went looking on disk for a feed that was
2529
+ // already in the run. The cap (CONTEXT_CHARS, applied where the prompt
2530
+ // is assembled) keeps the newest: when the total is over it, the
2531
+ // oldest groups are dropped whole before anything is cut mid-text.
2532
+ const contextBefore = (gi: number) => joinEarlierGroups(groupResults, gi);
2533
+ // The data half of the same handoff: what the nearest earlier group
2534
+ // returned through `output: json`. Rebuilt from the record rather than
2535
+ // kept only in memory, so a resumed run hands step 3 the value step 1
2536
+ // returned before the park.
2537
+ const groupData: unknown[] = [];
2538
+ const dataBefore = (gi: number): unknown => {
2539
+ for (let i = gi - 1; i >= 0; i--) if (groupData[i] !== undefined) return groupData[i];
2540
+ return undefined;
2541
+ };
2542
+ const dataOf = (group: StepRecord[]): unknown => {
2543
+ const carrying = group.filter((s) => s.status === "completed" && s.data !== undefined);
2544
+ if (carrying.length === 0) return undefined;
2545
+ if (carrying.length === 1 && !carrying[0].item) return carrying[0].data;
2546
+ return carrying.map((s) => ({ agent: s.agent, ...(s.item ? { item: s.item } : {}), data: s.data }));
2547
+ };
2548
+ const resultsOf = (group: StepRecord[]): string | null => {
2549
+ const results = group
2550
+ .filter((s) => s.status === "completed" && s.result)
2551
+ .map((s) =>
2552
+ s.item
2553
+ ? `## Result for item: ${s.item.slice(0, 80)} (${s.agent})\n\n${s.result}`
2554
+ : group.length > 1
2555
+ ? `## Result from ${s.agent}\n\n${s.result}`
2556
+ : s.result!,
2557
+ );
2558
+ return results.length ? results.join("\n\n") : null;
2559
+ };
2560
+ // What each step concluded — its final turn — for the handoffs that
2561
+ // should read a conclusion rather than a transcript (`each: lines`).
2562
+ const conclusionsOf = (group: StepRecord[]): string | null => {
2563
+ const out = group
2564
+ .filter((s) => s.status === "completed" && (s.conclusion || s.result))
2565
+ .map((s) => s.conclusion || s.result!);
2566
+ return out.length ? out.join("\n") : null;
2567
+ };
2568
+ // Groups already finished before this drive (a resume, or a run picked
2569
+ // up after a gate) still carry their results and data on the record.
2570
+ // Rebuilding all three here is what lets the step after an approval
2571
+ // read what came before it, not only the gate's question.
2572
+ for (const [i, g] of ordered.entries()) {
2573
+ if (g.every((s) => s.status !== "pending" && s.status !== "running" && s.status !== "awaiting-approval")) {
2574
+ groupResults[i] = resultsOf(g);
2575
+ groupConclusions[i] = conclusionsOf(g);
2576
+ groupData[i] = dataOf(g);
2577
+ }
2578
+ }
2579
+
2580
+ // budget: — the flow's own per-run cap, stamped on the record the first
2581
+ // time it is driven. Between groups, not mid-step: a step in flight is
2582
+ // money already committed, and stopping it half-way keeps the cost
2583
+ // and loses the work.
2584
+ if (run.budgetUsd === undefined) {
2585
+ run.budgetUsd = readFlow(tenant, workspace, run.flow)?.budget ?? null;
2586
+ save();
2587
+ }
2588
+
2589
+ let gi = 0;
2590
+ let context: string | null = null;
2591
+ let ctxData: unknown = undefined;
2592
+ while (gi < ordered.length) {
2593
+ const group = ordered[gi];
2594
+ context = contextBefore(gi);
2595
+ ctxData = dataBefore(gi);
2596
+
2597
+ if (run.budgetUsd && runCost(run) >= run.budgetUsd && group.some((s) => s.status === "pending")) {
2598
+ const spent = runCost(run);
2599
+ const last = run.steps.findLast((s) => s.status === "completed" || s.status === "failed") ?? run.steps[0];
2600
+ last?.events.push({
2601
+ t: new Date().toISOString(),
2602
+ type: "error",
2603
+ text: `over budget — $${spent.toFixed(4)} spent of the flow's $${run.budgetUsd.toFixed(2)} (budget: in the flow file); the remaining steps were not run`,
2604
+ });
2605
+ for (const s of run.steps) {
2606
+ if (s.status === "pending") {
2607
+ s.status = "skipped";
2608
+ s.skipReason = "over budget";
2609
+ }
2610
+ }
2611
+ run.status = "failed";
2612
+ save();
2613
+ break;
2614
+ }
2615
+
2616
+ // wait: — the clock starts when the run REACHES the step, so
2617
+ // "wait: 3d" means three days after the previous group finished,
2618
+ // surviving restarts because the deadline is stamped on the record.
2619
+ for (const s of group) {
2620
+ if (s.status === "pending" && s.waitSecs && !s.waitUntil) {
2621
+ s.waitUntil = new Date(Date.now() + s.waitSecs * 1000).toISOString();
2622
+ s.events.push({
2623
+ t: new Date().toISOString(),
2624
+ type: "info",
2625
+ text: `waiting until ${s.waitUntil} (wait: ${s.waitSecs}s)`,
2626
+ });
2627
+ }
2628
+ }
2629
+ const stillWaiting = group
2630
+ .filter((s) => s.status === "pending" && s.waitUntil)
2631
+ .map((s) => new Date(s.waitUntil!).getTime())
2632
+ .filter((t) => t > Date.now());
2633
+ if (stillWaiting.length) {
2634
+ save();
2635
+ if (opts.parkOnApproval) {
2636
+ // Back to the queue with a not-before — a three-day wait must
2637
+ // not hold a worker slot, a process, or survive on either. The
2638
+ // files so far are stored first, as at an approval park.
2639
+ await harvestQuietly(tenant, workspace, run);
2640
+ run.status = "queued";
2641
+ run.parkedAt = new Date().toISOString();
2642
+ parked = true;
2643
+ save();
2644
+ return;
2645
+ }
2646
+ // Foreground (CLI) runs sleep in slices so Ctrl-C stays prompt.
2647
+ await new Promise((r) => setTimeout(r, Math.min(Math.min(...stillWaiting) - Date.now(), 30_000)));
2648
+ continue;
2649
+ }
2650
+
2651
+ // Fan-out: a step with `each:` becomes one instance per item of the
2652
+ // previous group's result, capped, all in this same group — the
2653
+ // declared shape stays readable in the file, the width comes from
2654
+ // the data. The template survives as a skipped step for the trace.
2655
+ // `each: lines` splits what the previous group CONCLUDED — each step's
2656
+ // final turn — not every turn joined. A translator that narrated one
2657
+ // line per file write put the same sentence in front of eight
2658
+ // validators; its final turn was the list. Falls back to the joined
2659
+ // result for records made before `conclusion` existed.
2660
+ const eachSource = gi > 0 ? (groupConclusions[gi - 1] ?? groupResults[gi - 1]) : null;
2661
+ for (const step of [...group]) {
2662
+ if (!step.each || step.status !== "pending" || step.item) continue;
2663
+ const cap = Math.min(step.max ?? 10, 20);
2664
+ const items =
2665
+ step.each === "rows"
2666
+ ? csvItems(pDir, step, cap + 1)
2667
+ : step.each === "items"
2668
+ ? jsonItems(ctxData, step, cap + 1)
2669
+ : (eachSource ?? "")
2670
+ .split("\n")
2671
+ .map((l) => l.replace(/^[-*\d.)\s]+/, "").trim())
2672
+ .filter(Boolean)
2673
+ .slice(0, cap + 1);
2674
+ const dropped = items.length > cap ? "at least one" : "";
2675
+ const used = items.slice(0, cap);
2676
+ step.status = "skipped";
2677
+ if (used.length === 0) {
2678
+ step.skipReason ||= "each: the previous group produced no items";
2679
+ continue;
2680
+ }
2681
+ step.skipReason = `expanded into ${used.length} item${used.length === 1 ? "" : "s"}`;
2682
+ step.events.push({
2683
+ t: new Date().toISOString(),
2684
+ type: "info",
2685
+ text:
2686
+ `fan-out: ${used.length} instance${used.length === 1 ? "" : "s"}` +
2687
+ (dropped ? ` (more items existed — capped at ${cap})` : ""),
2688
+ });
2689
+ const at = run.steps.indexOf(step);
2690
+ const instances: StepRecord[] = used.map((item) => ({
2691
+ ...step,
2692
+ each: undefined,
2693
+ max: undefined,
2694
+ skipReason: undefined,
2695
+ item,
2696
+ status: "pending",
2697
+ events: [],
2698
+ result: null,
2699
+ costUsd: null,
2700
+ attempts: 0,
2701
+ }));
2702
+ run.steps.splice(at + 1, 0, ...instances);
2703
+ ordered = orderedGroups();
2704
+ }
2705
+ save();
2706
+
2707
+ // A person asked for this to stop. Checked between groups, off the
2708
+ // record rather than in memory, because the request arrives in
2709
+ // another process — the same way an approval decision does. The
2710
+ // sandbox for any in-flight step was already destroyed by stopRun;
2711
+ // this is what keeps the *next* group from starting.
2712
+ const stopCheck = readRun(tenant, workspace, run.id);
2713
+ if (stopCheck?.stopRequested) {
2714
+ for (const s of run.steps) {
2715
+ if (s.status === "pending" || s.status === "running") {
2716
+ s.status = "skipped";
2717
+ s.skipReason = "run stopped";
2718
+ }
2719
+ }
2720
+ run.status = "failed";
2721
+ run.stopRequested = true;
2722
+ save();
2723
+ break;
2724
+ }
2725
+
2726
+ const freshGroup = ordered[gi];
2727
+
2728
+ // Validate agents exist before launching the group.
2729
+ let missing = false;
2730
+ for (const step of freshGroup) {
2731
+ // Only what still has work to do. This walked every step in the
2732
+ // group regardless of status, which never showed on a fresh run
2733
+ // because they all start pending — but on a resumed one it rewrote
2734
+ // finished steps to `failed`, destroying the history it was
2735
+ // resuming around.
2736
+ if (step.status !== "pending") continue;
2737
+ const aDir = path.join(pDir, "agents", step.agent);
2738
+ if (!fs.existsSync(path.join(aDir, "agent.md"))) {
2739
+ step.status = "failed";
2740
+ step.events.push({
2741
+ t: new Date().toISOString(),
2742
+ type: "error",
2743
+ text: `agent "${step.agent}" not found in workspace`,
2744
+ });
2745
+ if (!step.optional) missing = true;
2746
+ }
2747
+ }
2748
+ save();
2749
+ if (missing) {
2750
+ run.status = "failed";
2751
+ break;
2752
+ }
2753
+
2754
+ const ctx = context;
2755
+
2756
+ // `when:` — skip a step whose condition isn't met by prior results.
2757
+ for (const step of freshGroup) {
2758
+ if (step.status !== "pending" || !step.when) continue;
2759
+ const met = (ctx ?? "").toLowerCase().includes(step.when.toLowerCase());
2760
+ if (!met) {
2761
+ step.status = "skipped";
2762
+ step.skipReason = `condition not met: when "${step.when}"`;
2763
+ step.events.push({
2764
+ t: new Date().toISOString(),
2765
+ type: "info",
2766
+ text: `skipped — previous results do not mention "${step.when}"`,
2767
+ });
2768
+ }
2769
+ }
2770
+ save();
2771
+
2772
+ // `case:`/`else:` — exclusive routing. Of this group's case steps,
2773
+ // the FIRST whose text appears in the previous results runs; the
2774
+ // rest are routed past. `else:` runs only when no case matched.
2775
+ // (`when:` above stays independent — every matching when runs —
2776
+ // which is why routing is its own vocabulary instead of a mode.)
2777
+ const caseSteps = freshGroup.filter((s) => s.status === "pending" && s.case);
2778
+ if (caseSteps.length || freshGroup.some((s) => s.status === "pending" && s.else)) {
2779
+ let matchedCase: StepRecord | null = null;
2780
+ for (const step of caseSteps) {
2781
+ if (!matchedCase && (ctx ?? "").toLowerCase().includes(step.case!.toLowerCase())) {
2782
+ matchedCase = step;
2783
+ continue;
2784
+ }
2785
+ step.status = "skipped";
2786
+ step.skipReason = matchedCase
2787
+ ? `routed past — "${matchedCase.case}" matched first`
2788
+ : `case not matched: "${step.case}"`;
2789
+ step.events.push({
2790
+ t: new Date().toISOString(),
2791
+ type: "info",
2792
+ text: step.skipReason,
2793
+ });
2794
+ }
2795
+ for (const step of freshGroup) {
2796
+ if (step.status !== "pending" || !step.else) continue;
2797
+ if (matchedCase) {
2798
+ step.status = "skipped";
2799
+ step.skipReason = `routed past — "${matchedCase.case}" matched`;
2800
+ step.events.push({ t: new Date().toISOString(), type: "info", text: step.skipReason });
2801
+ }
2802
+ }
2803
+ save();
2804
+ }
2805
+
2806
+ // Approval gate — pause the whole run until a human decides. The run
2807
+ // record is the queue: it sits in awaiting-approval until the API
2808
+ // flips the step back to pending.
2809
+ // `!s.approvedAt` — a step approved before the process went away is
2810
+ // already answered. Filtering on status alone asked again.
2811
+ // `wait: event` parks in the same place: the gate is the same, the
2812
+ // key is held by a machine. A person can still release it by hand.
2813
+ const needsApproval = freshGroup.filter(
2814
+ (s) => s.status === "pending" && (s.approve || s.ask || s.waitFor === "event") && !s.approvedAt,
2815
+ );
2816
+ if (needsApproval.length) {
2817
+ for (const step of needsApproval) {
2818
+ step.status = "awaiting-approval";
2819
+ step.events.push({
2820
+ t: new Date().toISOString(),
2821
+ type: "info",
2822
+ // ask: is the same gate carrying a question — the answer
2823
+ // arrives as the approval note and rides into the prompt.
2824
+ text: step.ask
2825
+ ? `waiting for an answer: ${step.ask}`
2826
+ : step.waitFor === "event"
2827
+ ? `waiting for an external event — release it with POST ${eventUrl(tenant, workspace, run.id)}`
2828
+ : "waiting for approval before running",
2829
+ });
2830
+ }
2831
+ run.status = "awaiting-approval";
2832
+ save();
2833
+
2834
+ // The gate's declared preview, resolved now against what the run
2835
+ // has actually produced — a glob at park time, a list of files on
2836
+ // the record for as long as the run waits.
2837
+ for (const s of needsApproval) {
2838
+ if (s.preview?.length) s.previewFiles = resolvePreview(path.join(workspaceDir(tenant, workspace), "storage"), s.preview);
2839
+ }
2840
+ save();
2841
+ if (opts.parkOnApproval) {
2842
+ // What the run has produced so far becomes stored files NOW, not
2843
+ // at the end: the gate shows the thumbnail a step is about to
2844
+ // publish, and the Storage page answers "what did it make" while
2845
+ // the person decides. Unchanged files are skipped by hash, and
2846
+ // the local copies stay for the resumed step to read.
2847
+ await harvestQuietly(tenant, workspace, run);
2848
+ // The gate has been on the record — and in an inbox — since the
2849
+ // save above, so a decision can land while the harvest runs.
2850
+ // The API wrote it to disk; it did not enqueue a resume, because
2851
+ // parkedAt was not set yet; and the save below, from memory,
2852
+ // would erase it. That was a lost approval and a run that waited
2853
+ // forever (2026-09-06, approved two seconds after it parked).
2854
+ // Take what landed, then do the enqueue the API would have done.
2855
+ const decided = adoptDecisions(run, readRun(tenant, workspace, run.id));
2856
+ // Hand the slot back. The approval API sees parkedAt and
2857
+ // re-enqueues; re-entering this loop skips finished groups and
2858
+ // lands back here with the decision already on the record.
2859
+ run.parkedAt = new Date().toISOString();
2860
+ parked = true;
2861
+ save();
2862
+ if (decided && run.steps.every((s) => s.status !== "awaiting-approval")) {
2863
+ await platform.enqueueResume(tenant, workspace, run.id);
2864
+ }
2865
+ return;
2866
+ }
2867
+
2868
+ const decided = await waitForDecision(tenant, workspace, run.id, needsApproval.map((s) => run.steps.indexOf(s)));
2869
+ if (!decided) {
2870
+ run.status = "failed";
2871
+ break;
2872
+ }
2873
+ // Re-read: the API rewrote the file, so refresh statuses in place.
2874
+ const latest = readRun(tenant, workspace, run.id);
2875
+ if (latest) {
2876
+ latest.steps.forEach((s, i) => {
2877
+ run.steps[i].status = s.status;
2878
+ run.steps[i].events = s.events;
2879
+ // What the decision carried, not just that it happened: the
2880
+ // note and the event payload are what the step's prompt reads,
2881
+ // and copying status alone dropped them on the in-process path.
2882
+ run.steps[i].approvedAt = s.approvedAt;
2883
+ run.steps[i].approvalNote = s.approvalNote;
2884
+ run.steps[i].eventPayload = s.eventPayload;
2885
+ });
2886
+ }
2887
+ run.status = "running";
2888
+ save();
2889
+ if (freshGroup.every((s) => s.status !== "pending")) {
2890
+ // everything in this group was rejected
2891
+ if (freshGroup.some((s) => s.status === "failed" && !s.optional)) {
2892
+ run.status = "failed";
2893
+ break;
2894
+ }
2895
+ gi += 1;
2896
+ continue;
2897
+ }
2898
+ }
2899
+
2900
+ // The hard cap: what is left of the flow's budget, shared evenly
2901
+ // across the steps this group launches together. Each step stops
2902
+ // itself mid-turn at its share, so a fan-out of twenty cannot end
2903
+ // twenty steps over — the shares sum to what was left. Between
2904
+ // groups the check above still refuses to start the next one.
2905
+ const launching = freshGroup.filter((s) => s.status === "pending");
2906
+ const ceilingUsd = stepCeiling(run.budgetUsd, runCost(run), launching.length);
2907
+ await Promise.all(
2908
+ launching
2909
+ .map(async (step) => {
2910
+ const attempts = (step.retry ?? 0) + 1;
2911
+ for (let attempt = 1; attempt <= attempts; attempt++) {
2912
+ step.attempts = attempt;
2913
+ step.status = "running";
2914
+ save();
2915
+ await runStep(
2916
+ path.join(pDir, "agents", step.agent),
2917
+ tenant,
2918
+ step,
2919
+ ctx,
2920
+ save,
2921
+ modelOverride,
2922
+ tags,
2923
+ effortOverride,
2924
+ run.id,
2925
+ ctxData,
2926
+ ceilingUsd,
2927
+ );
2928
+ // runStep mutates step.status; read it through a widened local
2929
+ // so TS doesn't keep the "running" narrowing from above.
2930
+ const outcome: string = step.status;
2931
+ if (outcome !== "failed" || attempt === attempts) break;
2932
+ // A stop destroys the step's sandbox, which reads here as a
2933
+ // failed attempt — and a failed attempt used to be retried in
2934
+ // a fresh sandbox, so a stopped `retry: 2` step ran twice
2935
+ // more. save() merges the flag from the record.
2936
+ save();
2937
+ if (run.stopRequested) break;
2938
+ step.events.push({
2939
+ t: new Date().toISOString(),
2940
+ type: "info",
2941
+ text: `attempt ${attempt} failed — retrying (${attempts - attempt} left)`,
2942
+ });
2943
+ save();
2944
+ }
2945
+ }),
2946
+ );
2947
+
2948
+ // on-fail: another agent takes the step over. Sequential and rare:
2949
+ // the rescue is a second full attempt at the step's job with the
2950
+ // failure in front of it, and its success is the step's success —
2951
+ // the trace keeps both records, so "who actually did this" stays
2952
+ // answerable.
2953
+ // Not after a stop: the failure a stop leaves behind is not one a
2954
+ // rescuer should take over. The merge in save() surfaces the flag.
2955
+ save();
2956
+ for (const step of freshGroup) {
2957
+ if (run.stopRequested) break;
2958
+ if (step.status !== "failed" || !step.onFail) continue;
2959
+ const rescuerDir = path.join(pDir, "agents", step.onFail);
2960
+ if (!fs.existsSync(path.join(rescuerDir, "agent.md"))) {
2961
+ step.events.push({
2962
+ t: new Date().toISOString(),
2963
+ type: "error",
2964
+ text: `on-fail: "${step.onFail}" is not an agent in this workspace — no rescue`,
2965
+ });
2966
+ continue;
2967
+ }
2968
+ step.events.push({
2969
+ t: new Date().toISOString(),
2970
+ type: "info",
2971
+ text: `failed — handing to [[${step.onFail}]] (on-fail)`,
2972
+ });
2973
+ const failure = (step.result ?? step.events.filter((e) => e.type === "error").map((e) => e.text).join("\n")).slice(0, 4000);
2974
+ const rescue: StepRecord = {
2975
+ agent: step.onFail,
2976
+ instruction:
2977
+ `${step.instruction}\n\n<rescue>\nYou are taking this step over: [[${step.agent}]] attempted it and failed. ` +
2978
+ `Its failure, for whatever it is worth:\n${failure}\n</rescue>`,
2979
+ group: step.group,
2980
+ optional: false,
2981
+ model: step.model,
2982
+ effort: step.effort,
2983
+ timeout: step.timeout,
2984
+ // The rescuer inherits the contract, not just the job: a step
2985
+ // that promised data still owes data when someone else does it.
2986
+ output: step.output,
2987
+ attempts: 0,
2988
+ status: "pending",
2989
+ events: [],
2990
+ result: null,
2991
+ costUsd: null,
2992
+ };
2993
+ run.steps.splice(run.steps.indexOf(step) + 1, 0, rescue);
2994
+ save();
2995
+ rescue.attempts = 1;
2996
+ rescue.status = "running";
2997
+ save();
2998
+ await runStep(rescuerDir, tenant, rescue, ctx, save, modelOverride, tags, effortOverride, run.id, ctxData);
2999
+ // runStep mutates rescue.status; widen past TS's "running" narrowing.
3000
+ const rescueOutcome: string = rescue.status;
3001
+ if (rescueOutcome === "completed") {
3002
+ step.status = "completed";
3003
+ step.result = rescue.result;
3004
+ step.conclusion = rescue.conclusion;
3005
+ if (rescue.data !== undefined) step.data = rescue.data;
3006
+ step.events.push({
3007
+ t: new Date().toISOString(),
3008
+ type: "info",
3009
+ text: `rescued by [[${step.onFail}]] — its result stands as this step's`,
3010
+ });
3011
+ } else {
3012
+ step.events.push({
3013
+ t: new Date().toISOString(),
3014
+ type: "error",
3015
+ text: `on-fail: [[${step.onFail}]] also failed`,
3016
+ });
3017
+ }
3018
+ save();
3019
+ }
3020
+
3021
+ const requiredFailed = freshGroup.some((s) => s.status === "failed" && !s.optional);
3022
+ if (requiredFailed) {
3023
+ run.status = "failed";
3024
+ break;
3025
+ }
3026
+
3027
+ groupResults[gi] = resultsOf(freshGroup);
3028
+ groupConclusions[gi] = conclusionsOf(freshGroup);
3029
+ groupData[gi] = dataOf(freshGroup);
3030
+
3031
+ // delegate: — the completed chooser's picks become a fresh group
3032
+ // right after this one. The choice SET is the file's; only the
3033
+ // picks and their instructions are the model's, and both are on
3034
+ // the record. Once per chooser, loop rewinds included.
3035
+ for (const chooser of freshGroup) {
3036
+ if (!chooser.delegate?.length || chooser.status !== "completed" || chooser.delegated) continue;
3037
+ chooser.delegated = true;
3038
+ const allowed = new Set(chooser.delegate);
3039
+ const picks: StepRecord[] = [];
3040
+ const seenAgents = new Set<string>();
3041
+ for (const line of (chooser.result ?? "").split("\n")) {
3042
+ const m = line.match(/^\s*(?:[-*\d.)\s]*)\[?\[?([a-z0-9-]+)\]?\]?\s*:\s*(.+)$/);
3043
+ if (!m || !allowed.has(m[1]) || seenAgents.has(m[1])) continue;
3044
+ if (!fs.existsSync(path.join(pDir, "agents", m[1], "agent.md"))) continue;
3045
+ seenAgents.add(m[1]);
3046
+ picks.push({
3047
+ agent: m[1],
3048
+ instruction: m[2].trim(),
3049
+ group: chooser.group,
3050
+ optional: false,
3051
+ attempts: 0,
3052
+ status: "pending",
3053
+ events: [],
3054
+ result: null,
3055
+ costUsd: null,
3056
+ });
3057
+ if (picks.length >= 5) break;
3058
+ }
3059
+ // Delegating to nobody is a legitimate answer, not a fault: a
3060
+ // `delegate:` set is who the step MAY call on, and most runs of a
3061
+ // healthy flow need none of them. Logging that as an error put a
3062
+ // red line on a step that did exactly the right thing, which is
3063
+ // how a person learns to stop reading red lines.
3064
+ chooser.events.push({
3065
+ t: new Date().toISOString(),
3066
+ type: "info",
3067
+ text: picks.length
3068
+ ? `delegated to ${picks.map((p) => `[[${p.agent}]]`).join(", ")}`
3069
+ : `delegate: chose nobody (could have called on ${chooser.delegate.join(", ")})`,
3070
+ });
3071
+ if (picks.length) {
3072
+ run.steps.splice(run.steps.indexOf(chooser) + 1, 0, ...picks);
3073
+ ordered.splice(gi + 1, 0, picks);
3074
+ }
3075
+ save();
3076
+ }
3077
+
3078
+ // Evaluator loop: a completed step with `loop:` whose result lacks
3079
+ // its `until:` marker sends this group and the one before it around
3080
+ // again — at most `loop:` times, so the worst case is still readable
3081
+ // off the flow file. Approval marks survive the rewind: a person
3082
+ // answered, and the question has not changed.
3083
+ const looper = freshGroup.find((s) => s.loop && s.until && s.status === "completed");
3084
+ if (
3085
+ looper?.until &&
3086
+ !saysUntilMarker(looper.result ?? null, looper.until)
3087
+ ) {
3088
+ if ((looper.loopRemaining ?? 0) > 0 && gi > 0) {
3089
+ looper.loopRemaining = (looper.loopRemaining ?? 0) - 1;
3090
+ looper.events.push({
3091
+ t: new Date().toISOString(),
3092
+ type: "info",
3093
+ text: `loop: result does not say "${looper.until}" on a line of its own — winding back one group (${looper.loopRemaining} cycle${looper.loopRemaining === 1 ? "" : "s"} left)`,
3094
+ });
3095
+ for (const s of [...ordered[gi - 1], ...freshGroup]) {
3096
+ if (s.status === "skipped" && s.each) continue; // an expanded template stays expanded
3097
+ s.status = "pending";
3098
+ s.result = null;
3099
+ }
3100
+ groupResults[gi] = null;
3101
+ groupResults[gi - 1] = null;
3102
+ groupConclusions[gi] = null;
3103
+ groupConclusions[gi - 1] = null;
3104
+ save();
3105
+ gi -= 1;
3106
+ continue;
3107
+ }
3108
+ looper.status = "failed";
3109
+ looper.events.push({
3110
+ t: new Date().toISOString(),
3111
+ type: "error",
3112
+ text: `loop exhausted: after ${looper.loop} cycle${looper.loop === 1 ? "" : "s"} the result still does not say "${looper.until}"`,
3113
+ });
3114
+ save();
3115
+ if (!looper.optional) {
3116
+ run.status = "failed";
3117
+ break;
3118
+ }
3119
+ }
3120
+
3121
+ gi += 1;
3122
+ }
3123
+
3124
+ if (run.status === "running") run.status = "completed";
3125
+ for (const s of run.steps) if (s.status === "pending") s.status = "skipped";
3126
+ } catch (err) {
3127
+ run.status = "failed";
3128
+ run.steps.forEach((s) => {
3129
+ if (s.status === "running") {
3130
+ s.status = "failed";
3131
+ s.events.push({
3132
+ t: new Date().toISOString(),
3133
+ type: "error",
3134
+ text: err instanceof Error ? err.message : String(err),
3135
+ });
3136
+ }
3137
+ });
3138
+ } finally {
3139
+ if (!parked) {
3140
+ run.finishedAt = new Date().toISOString();
3141
+ // What this run concluded, in one line, captured before the record is
3142
+ // saved for the last time. Read from the reply that already says it
3143
+ // rather than asked for separately: the folder gains nothing to
3144
+ // learn, and the runs list and the notification stop having to say
3145
+ // only "completed".
3146
+ run.summary = runSummary(run);
3147
+ // Archive before the next run resets outputs/ — a failed run's
3148
+ // artifacts are usually the most interesting ones, so this runs on
3149
+ // failure too.
3150
+ try {
3151
+ stampMemories();
3152
+ archive();
3153
+ } catch {
3154
+ // never let bookkeeping fail a run that already finished
3155
+ }
3156
+ // Whatever the run left in storage/ becomes a stored file, stamped with
3157
+ // the run and the agent that wrote it — which is what lets the
3158
+ // dashboard answer "where did this PDF come from?" without a join
3159
+ // table. On failure too: a failed run's half-written artifact is
3160
+ // usually the one worth looking at.
3161
+ try {
3162
+ const { saved, errors } = await harvestFiles(
3163
+ tenant,
3164
+ workspace,
3165
+ `run:${run.id}`,
3166
+ );
3167
+ const last = run.steps.findLast((s) => s.status !== "pending") ?? run.steps[0];
3168
+ if (saved.length) {
3169
+ last?.events.push({
3170
+ t: new Date().toISOString(),
3171
+ type: "info",
3172
+ text: `files: saved ${saved.join(", ")}`,
3173
+ });
3174
+ }
3175
+ for (const e of errors) {
3176
+ last?.events.push({ t: new Date().toISOString(), type: "error", text: `files: ${e}` });
3177
+ }
3178
+ } catch {
3179
+ // same rule as archive(): bookkeeping never fails a finished run
3180
+ }
3181
+ save();
3182
+ }
3183
+ }
3184
+ })();
3185
+ }
3186
+
3187
+ function readFlow(tenant: string, workspace: string, flowName: string) {
3188
+ const dir = path.join(workspaceDir(tenant, workspace), "flows");
3189
+ if (!fs.existsSync(dir)) return null;
3190
+ for (const f of fs.readdirSync(dir).filter((f) => f.endsWith(".md"))) {
3191
+ const flow = parseFlow(f, fs.readFileSync(path.join(dir, f), "utf8"));
3192
+ if (flow.name === flowName) return flow;
3193
+ }
3194
+ return null;
3195
+ }
3196
+
3197
+ // Splice `[[flow:other]]` steps into the parent's step list, renumbering
3198
+ // groups so the nested flow's own parallelism survives. Depth-limited, and
3199
+ // a cycle raises rather than expanding forever.
3200
+ function expandSubflows(
3201
+ tenant: string,
3202
+ workspace: string,
3203
+ steps: FlowStep[],
3204
+ seen: string[] = [],
3205
+ depth = 0,
3206
+ ): FlowStep[] {
3207
+ if (depth > 3) throw new Error("flow nesting too deep (max 3)");
3208
+ const out: FlowStep[] = [];
3209
+ let group = 0;
3210
+ for (const [i, step] of steps.entries()) {
3211
+ const startsNewGroup = i === 0 || step.group !== steps[i - 1].group;
3212
+ if (startsNewGroup) group += 1;
3213
+
3214
+ if (!step.subflow) {
3215
+ out.push({ ...step, group });
3216
+ continue;
3217
+ }
3218
+ if (seen.includes(step.subflow)) {
3219
+ throw new Error(`flow cycle: ${[...seen, step.subflow].join(" → ")}`);
3220
+ }
3221
+ const nested = readFlow(tenant, workspace, step.subflow);
3222
+ if (!nested) throw new Error(`flow "${step.subflow}" not found`);
3223
+ const nestedSteps = expandSubflows(
3224
+ tenant,
3225
+ workspace,
3226
+ nested.steps,
3227
+ [...seen, step.subflow],
3228
+ depth + 1,
3229
+ );
3230
+ // Nested groups continue from here; the parent's next group follows them.
3231
+ const base = group - 1;
3232
+ let maxGroup = group;
3233
+ for (const ns of nestedSteps) {
3234
+ const g = base + ns.group;
3235
+ maxGroup = Math.max(maxGroup, g);
3236
+ out.push({
3237
+ ...ns,
3238
+ group: g,
3239
+ optional: ns.optional || step.optional,
3240
+ instruction: ns.instruction || step.instruction,
3241
+ });
3242
+ }
3243
+ group = maxGroup;
3244
+ }
3245
+ return out;
3246
+ }
3247
+
3248
+ export function loadFlow(tenant: string, workspace: string, flowName: string) {
3249
+ const flow = readFlow(tenant, workspace, flowName);
3250
+ if (!flow) return null;
3251
+ if (!flow.steps.some((s) => s.subflow)) return flow;
3252
+ return { ...flow, steps: expandSubflows(tenant, workspace, flow.steps, [flowName]) };
3253
+ }
3254
+
3255
+ // ---------------------------------------------------------------- recovery
3256
+
3257
+ /**
3258
+ * How long a run may sit in `running` with nobody touching it before it is
3259
+ * considered abandoned.
3260
+ *
3261
+ * Long enough that a slow model call is never mistaken for a dead process —
3262
+ * a single step can legitimately take minutes — and short enough that a
3263
+ * restarted server does not leave yesterday's run looking live.
3264
+ */
3265
+ // A run this process is driving is never abandoned, however long its step
3266
+ // is quiet — a 40-minute script is work, not a fault, and the platform sets
3267
+ // no clock on work. A `running` run nobody here is driving has lost its
3268
+ // driver (a restart; this is the only process that drives) and is closed
3269
+ // once it has been quiet for two minutes — the margin is for a record
3270
+ // written a beat before its driver registered, not a judgement about pace.
3271
+ const ABANDONED_AFTER_MS = 2 * 60 * 1000;
3272
+
3273
+ /** The most recent moment a run showed any sign of life. */
3274
+ function lastActivity(run: RunRecord): number {
3275
+ let latest = Date.parse(run.startedAt);
3276
+ for (const step of run.steps) {
3277
+ for (const event of step.events) {
3278
+ const t = Date.parse(event.t);
3279
+ if (t > latest) latest = t;
3280
+ }
3281
+ }
3282
+ return latest;
3283
+ }
3284
+
3285
+ export interface Reconciliation {
3286
+ runId: string;
3287
+ tenant: string;
3288
+ workspace: string;
3289
+ /** What was done about it: closed as dead, or picked back up. */
3290
+ action: "closed" | "resumed";
3291
+ /** Steps that were mid-flight when the process went away. */
3292
+ interrupted: string[];
3293
+ }
3294
+
3295
+ /**
3296
+ * Close out runs abandoned by a process that is no longer here.
3297
+ *
3298
+ * Runs execute in the server process, and nothing writes a terminal status
3299
+ * when that process goes away — so a deploy, a crash or a Ctrl-C during a run
3300
+ * leaves `status: "running"` on disk forever. The run history then contains
3301
+ * entries that are not merely stale but false: they claim to be in progress,
3302
+ * the dashboard renders them as live, and no one can tell them apart from a
3303
+ * step that is genuinely still thinking.
3304
+ *
3305
+ * Called at startup, before anything reads run history. A run still being
3306
+ * worked on by a live process is protected by the idle window, not by a lock —
3307
+ * so this is safe to call while runs are in flight, and deliberately
3308
+ * conservative about what it declares dead.
3309
+ *
3310
+ * A run genuinely waiting for a person is left alone: that state legitimately
3311
+ * outlives any process. A run whose person already answered is not waiting,
3312
+ * and is picked back up — see below.
3313
+ */
3314
+ export function reconcileRuns(tenant: string, now = Date.now()): Reconciliation[] {
3315
+ const closed: Reconciliation[] = [];
3316
+
3317
+ for (const workspace of listWorkspaces(tenant)) {
3318
+ for (const summary of listRuns(tenant, workspace.name)) {
3319
+ if (drivingRuns.has(summary.id)) continue;
3320
+ if (now - lastActivity(summary) < ABANDONED_AFTER_MS) continue;
3321
+
3322
+ // Approved, then abandoned. Approval only writes to the run record; the
3323
+ // loop that acts on it polls in memory, inside the process that started
3324
+ // the run. If that process goes away between the click and the next
3325
+ // poll, the decision is on disk with nobody left to read it — and the
3326
+ // run sits in `awaiting-approval` with no step awaiting approval, so
3327
+ // the dashboard cannot even offer the button again. Nothing else here
3328
+ // rescues it: this status is exempt from being closed, by design.
3329
+ if (summary.status === "awaiting-approval") {
3330
+ const stillAsking = summary.steps.some((s) => s.status === "awaiting-approval");
3331
+ const answered = summary.steps.some((s) => s.approvedAt && s.status === "pending");
3332
+ if (stillAsking || !answered) continue;
3333
+
3334
+ const run = readRun(tenant, workspace.name, summary.id);
3335
+ if (!run || run.status !== "awaiting-approval") continue;
3336
+ run.status = "running";
3337
+ writeRun(tenant, workspace.name, run);
3338
+ // Park at any *later* gate rather than poll: reconcile runs inside
3339
+ // long-lived server processes, where a 24h file-poll is exactly the
3340
+ // slot-burning the queue exists to avoid.
3341
+ void driveRun(tenant, workspace.name, run, null, run.tags ?? [], { parkOnApproval: true });
3342
+ closed.push({
3343
+ runId: run.id,
3344
+ tenant,
3345
+ workspace: workspace.name,
3346
+ action: "resumed",
3347
+ interrupted: [],
3348
+ });
3349
+ continue;
3350
+ }
3351
+
3352
+ if (summary.status !== "running") continue;
3353
+
3354
+ const run = readRun(tenant, workspace.name, summary.id);
3355
+ if (!run || run.status !== "running") continue;
3356
+
3357
+ const interrupted: string[] = [];
3358
+ for (const step of run.steps) {
3359
+ if (step.status === "running") {
3360
+ interrupted.push(step.agent);
3361
+ step.status = "failed";
3362
+ step.events.push({
3363
+ t: new Date(now).toISOString(),
3364
+ type: "error",
3365
+ text: "interrupted — the process running this step went away before it finished",
3366
+ });
3367
+ } else if (step.status === "pending") {
3368
+ step.status = "skipped";
3369
+ step.skipReason = "the run was interrupted before this step started";
3370
+ }
3371
+ }
3372
+
3373
+ run.status = "failed";
3374
+ run.finishedAt = new Date(now).toISOString();
3375
+ writeRun(tenant, workspace.name, run);
3376
+ // The record is closed; make the sandbox match. A step with no
3377
+ // `timeout:` has no pod deadline, so an orphan from a mid-step restart
3378
+ // is reaped here, by the run that owned it, not by a clock.
3379
+ try {
3380
+ if (process.env.FOLDRUN_RUN_ISOLATION === "container") killRunSandboxes(run.id);
3381
+ else platform.killRunSandboxes(run.id);
3382
+ } catch {
3383
+ // a sandbox we cannot reach must not stop the reconcile
3384
+ }
3385
+ closed.push({ runId: run.id, tenant, workspace: workspace.name, action: "closed", interrupted });
3386
+ }
3387
+ }
3388
+
3389
+ return closed;
3390
+ }
3391
+
3392
+ /**
3393
+ * The same, for every account this installation holds.
3394
+ *
3395
+ * A server reconciles on boot, and it boots for everyone — reconciling only
3396
+ * one hard-coded account left every other account's interrupted runs marked
3397
+ * `running` forever, with no process left to finish them.
3398
+ */
3399
+ export function reconcileAllRuns(now = Date.now()): Reconciliation[] {
3400
+ return listTenants().flatMap((tenant) => reconcileRuns(tenant, now));
3401
+ }
3402
+
3403
+ /**
3404
+ * Wait for a run to stop being in progress.
3405
+ *
3406
+ * Runs start and return immediately, which is right for a schedule and wrong
3407
+ * for a caller putting an agent behind their own request: they want the
3408
+ * answer, not a receipt. Polls the run record rather than holding a callback,
3409
+ * so it works from a different process than the one doing the work.
3410
+ *
3411
+ * `awaiting-approval` counts as stopping. It is not finished, but it is not
3412
+ * going to progress without a person, and a caller blocked on it would hang
3413
+ * for as long as the approval window allows.
3414
+ */
3415
+ export async function waitForRun(
3416
+ tenant: string,
3417
+ workspace: string,
3418
+ runId: string,
3419
+ timeoutMs = 5 * 60_000,
3420
+ ): Promise<{ run: RunRecord | null; timedOut: boolean }> {
3421
+ const deadline = Date.now() + timeoutMs;
3422
+ for (;;) {
3423
+ const run = readRun(tenant, workspace, runId);
3424
+ if (!run) return { run: null, timedOut: false };
3425
+ // Queued counts as in flight — a caller asking to wait wants the result,
3426
+ // not a report that the worker hadn't picked the job up yet.
3427
+ if (run.status !== "running" && run.status !== "queued") return { run, timedOut: false };
3428
+ if (Date.now() >= deadline) return { run, timedOut: true };
3429
+ await new Promise((r) => setTimeout(r, 250));
3430
+ }
3431
+ }
3432
+
3433
+ /**
3434
+ * What a run produced, for a caller that wanted an answer.
3435
+ *
3436
+ * The last step that actually returned something: a flow's final step is the
3437
+ * conclusion, and skipping back past steps that were skipped or produced
3438
+ * nothing is what makes `?wait=true` useful rather than merely blocking.
3439
+ */
3440
+ export function runResult(run: RunRecord): string | null {
3441
+ for (let i = run.steps.length - 1; i >= 0; i--) {
3442
+ const r = run.steps[i].result;
3443
+ if (r && r.trim()) return r;
3444
+ }
3445
+ return null;
3446
+ }