@foldrun/core 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -10
- package/dist/index.d.ts +3 -0
- package/dist/index.js +3 -0
- package/dist/src/account.d.ts +45 -0
- package/dist/src/account.js +130 -0
- package/dist/src/api-tools.d.ts +8 -0
- package/dist/src/api-tools.js +22 -4
- package/dist/src/approvals.d.ts +104 -1
- package/dist/src/approvals.js +192 -2
- package/dist/src/browser-pod.d.ts +129 -0
- package/dist/src/browser-pod.js +284 -0
- package/dist/src/budget.d.ts +32 -0
- package/dist/src/budget.js +177 -0
- package/dist/src/catalog.js +3 -1
- package/dist/src/clock.d.ts +49 -0
- package/dist/src/clock.js +121 -0
- package/dist/src/completions.js +17 -8
- package/dist/src/confine.d.ts +22 -0
- package/dist/src/confine.js +224 -4
- package/dist/src/deploy.d.ts +33 -0
- package/dist/src/deploy.js +145 -6
- package/dist/src/egress.d.ts +26 -2
- package/dist/src/egress.js +8 -1
- package/dist/src/eol.d.ts +2 -0
- package/dist/src/eol.js +7 -0
- package/dist/src/evals.d.ts +9 -0
- package/dist/src/evals.js +26 -13
- package/dist/src/flow-lint.d.ts +12 -0
- package/dist/src/flow-lint.js +331 -1
- package/dist/src/flow-patterns.d.ts +141 -0
- package/dist/src/flow-patterns.js +568 -0
- package/dist/src/gitrepo.d.ts +18 -0
- package/dist/src/gitrepo.js +49 -0
- package/dist/src/host-env.d.ts +8 -0
- package/dist/src/host-env.js +48 -0
- package/dist/src/json-schema.d.ts +12 -0
- package/dist/src/json-schema.js +202 -0
- package/dist/src/kinds.js +1 -1
- package/dist/src/language.d.ts +17 -0
- package/dist/src/language.js +55 -0
- package/dist/src/layout.d.ts +69 -0
- package/dist/src/layout.js +200 -0
- package/dist/src/library.d.ts +9 -0
- package/dist/src/library.js +43 -4
- package/dist/src/limits.d.ts +84 -0
- package/dist/src/limits.js +287 -0
- package/dist/src/locale.d.ts +47 -0
- package/dist/src/locale.js +177 -0
- package/dist/src/mail.d.ts +88 -0
- package/dist/src/mail.js +245 -0
- package/dist/src/model-probe.d.ts +3 -0
- package/dist/src/model-probe.js +1 -0
- package/dist/src/notify.d.ts +132 -8
- package/dist/src/notify.js +360 -58
- package/dist/src/openapi.js +5 -3
- package/dist/src/operator.d.ts +201 -0
- package/dist/src/operator.js +251 -0
- package/dist/src/paths.d.ts +33 -0
- package/dist/src/paths.js +87 -0
- package/dist/src/platform.d.ts +144 -4
- package/dist/src/platform.js +27 -3
- package/dist/src/provider-health.d.ts +55 -0
- package/dist/src/provider-health.js +233 -0
- package/dist/src/providers.d.ts +481 -0
- package/dist/src/providers.js +1105 -6
- package/dist/src/refusal.d.ts +83 -0
- package/dist/src/refusal.js +281 -0
- package/dist/src/run-container.d.ts +206 -17
- package/dist/src/run-container.js +547 -80
- package/dist/src/runner.d.ts +179 -10
- package/dist/src/runner.js +1423 -169
- package/dist/src/runtime.d.ts +52 -0
- package/dist/src/runtime.js +493 -58
- package/dist/src/script-tools.d.ts +15 -2
- package/dist/src/script-tools.js +44 -4
- package/dist/src/secret-files.d.ts +6 -5
- package/dist/src/secret-files.js +24 -11
- package/dist/src/secret-health.d.ts +83 -0
- package/dist/src/secret-health.js +177 -0
- package/dist/src/secrets.d.ts +38 -1
- package/dist/src/secrets.js +197 -96
- package/dist/src/starter.d.ts +9 -0
- package/dist/src/starter.js +95 -11
- package/dist/src/step-exec.d.ts +106 -1
- package/dist/src/step-exec.js +558 -85
- package/dist/src/step-removal.d.ts +15 -0
- package/dist/src/step-removal.js +49 -0
- package/dist/src/storage.d.ts +23 -4
- package/dist/src/storage.js +125 -23
- package/dist/src/store.d.ts +492 -38
- package/dist/src/store.js +882 -69
- package/dist/src/subagents.d.ts +75 -0
- package/dist/src/subagents.js +149 -0
- package/dist/src/tar.js +2 -1
- package/dist/src/test-mode.d.ts +111 -0
- package/dist/src/test-mode.js +285 -0
- package/dist/src/tool-names.d.ts +13 -1
- package/dist/src/tool-names.js +38 -11
- package/dist/src/tool-programs.d.ts +26 -0
- package/dist/src/tool-programs.js +172 -0
- package/dist/src/tool-test.js +152 -33
- package/dist/src/translator.d.ts +8 -2
- package/dist/src/translator.js +71 -24
- package/dist/src/trigger-log.d.ts +43 -0
- package/dist/src/trigger-log.js +122 -0
- package/dist/src/triggers.d.ts +23 -0
- package/dist/src/triggers.js +101 -12
- package/dist/src/untrusted-fetch.d.ts +11 -0
- package/dist/src/untrusted-fetch.js +77 -0
- package/dist/src/web-actions.d.ts +91 -0
- package/dist/src/web-actions.js +199 -0
- package/dist/src/webhook.d.ts +19 -3
- package/dist/src/webhook.js +50 -11
- package/index.ts +3 -0
- package/package.json +5 -5
- package/src/account.ts +149 -0
- package/src/api-tools.ts +32 -4
- package/src/approvals.ts +253 -2
- package/src/browser-pod.ts +313 -0
- package/src/budget.ts +178 -0
- package/src/catalog.ts +3 -1
- package/src/clock.ts +149 -0
- package/src/completions.ts +17 -8
- package/src/confine.ts +224 -4
- package/src/deploy.ts +137 -6
- package/src/egress.ts +32 -4
- package/src/eol.ts +7 -0
- package/src/evals.ts +36 -11
- package/src/flow-lint.ts +371 -1
- package/src/flow-patterns.ts +597 -0
- package/src/gitrepo.ts +48 -0
- package/src/host-env.ts +51 -0
- package/src/json-schema.ts +184 -0
- package/src/kinds.ts +1 -1
- package/src/language.ts +62 -0
- package/src/layout.ts +228 -0
- package/src/library.ts +50 -4
- package/src/limits.ts +290 -0
- package/src/locale.ts +217 -0
- package/src/mail.ts +316 -0
- package/src/model-probe.ts +4 -0
- package/src/notify.ts +391 -44
- package/src/openapi.ts +5 -3
- package/src/operator.ts +300 -0
- package/src/paths.ts +83 -0
- package/src/platform.ts +158 -5
- package/src/provider-health.ts +275 -0
- package/src/providers.ts +1368 -6
- package/src/refusal.ts +320 -0
- package/src/run-container.ts +611 -76
- package/src/runner.ts +1463 -165
- package/src/runtime.ts +487 -55
- package/src/script-tools.ts +55 -6
- package/src/secret-files.ts +22 -10
- package/src/secret-health.ts +230 -0
- package/src/secrets.ts +191 -95
- package/src/starter.ts +96 -11
- package/src/step-exec.ts +548 -41
- package/src/step-removal.ts +59 -0
- package/src/storage.ts +149 -24
- package/src/store.ts +1227 -74
- package/src/subagents.ts +189 -0
- package/src/tar.ts +2 -1
- package/src/test-mode.ts +311 -0
- package/src/tool-names.ts +40 -11
- package/src/tool-programs.ts +179 -0
- package/src/tool-test.ts +159 -32
- package/src/translator.ts +74 -23
- package/src/trigger-log.ts +160 -0
- package/src/triggers.ts +102 -12
- package/src/untrusted-fetch.ts +88 -0
- package/src/web-actions.ts +237 -0
- package/src/webhook.ts +58 -11
- package/templates/hello/agents/curator/agent.md +1 -1
- package/templates/hello/agents/notetaker/agent.md +1 -1
package/dist/src/runner.js
CHANGED
|
@@ -5,11 +5,19 @@
|
|
|
5
5
|
import fs from "node:fs";
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import matter from "gray-matter";
|
|
8
|
-
import { knownPrice, executeStep, extractJson, stepCeiling } from "./step-exec.js";
|
|
8
|
+
import { knownPrice, executeStep, extractJson, stepCeiling, stepCeilingFor } from "./step-exec.js";
|
|
9
|
+
import { cascadeLimits, toolOwners } from "./limits.js";
|
|
10
|
+
import { resolveAgentPath } from "./confine.js";
|
|
11
|
+
import { browserPodLine, podLossDecision, RERAN_ON_FULL } from "./browser-pod.js";
|
|
9
12
|
import { eventUrl } from "./webhook.js";
|
|
10
13
|
import { runStepInContainer, sizeLimits, killRunSandboxes } from "./run-container.js";
|
|
14
|
+
import { hostSafeEnv } from "./host-env.js";
|
|
15
|
+
import { validateSchema, describeSchemaErrors, looksLikeSchema } from "./json-schema.js";
|
|
11
16
|
import { EGRESS_ENV, MODEL_KEY_NAME, addGrant, hostOf, placeholderNames, proxyModelEnv, unsubstitute } from "./egress.js";
|
|
17
|
+
import { explainRefusal, isRefusalStatus, refusalFromLine, supplyNote } from "./refusal.js";
|
|
12
18
|
import { platform } from "./platform.js";
|
|
19
|
+
import { healthKey } from "./secret-health.js";
|
|
20
|
+
import { TEST_WRITES_DIR, restoreDivertedDirs, snapshotDivertedDirs, testHeadline, testModeEnv, withholdSecrets } from "./test-mode.js";
|
|
13
21
|
/** Does this process run steps in a sandbox — the container core ships, or
|
|
14
22
|
* one the platform registered (a pod)? */
|
|
15
23
|
const isolatedRun = () => {
|
|
@@ -17,9 +25,12 @@ const isolatedRun = () => {
|
|
|
17
25
|
return mode === "container" || (!!mode && mode in platform.isolation);
|
|
18
26
|
};
|
|
19
27
|
import { gatherConsults, buildConsultTools } from "./agent-tools.js";
|
|
20
|
-
import {
|
|
28
|
+
import { DELEGATE_TOOL, gatherSubagents, subagentPrompt, subagentTools } from "./subagents.js";
|
|
29
|
+
import { ASK_TOOL, applyOperatorEvent, askPromptLine, askTimeoutSec, buildAskTool } from "./operator.js";
|
|
30
|
+
import { sendPlainNotification } from "./notify.js";
|
|
31
|
+
import { TOOL_MAP, BUILTIN_TOOLS, ownToolNames, toolRefs, legacyUseNames, legacyUseError, retiredToolNames, retiredToolError } from "./tool-names.js";
|
|
21
32
|
import { refNames } from "./refs.js";
|
|
22
|
-
import { accountDir, workspaceDir, buildMemoryIndex, workspaceMemoryIndex, knowledgeIndex, workspaceKnowledgeIndex, workspaceTools, brokenToolReport, listWorkspaces, listTenants, listRuns, syncBundleFor, parseToolDef,
|
|
33
|
+
import { accountDir, workspaceDir, buildMemoryIndex, workspaceMemoryIndex, knowledgeIndex, workspaceKnowledgeIndex, workspaceTools, brokenToolReport, listWorkspaces, listTenants, listRuns, syncBundleFor, parseToolDef, readFlow, markerPresent, whenRowsPath, parseApis, resolveModel, parseProvider, providerEnvFor, resolveEffort, isEffortWord, EFFORT_LEVELS, checkFormatVersion, readRun, writeRun, runSummary, verdictOf, sumAttempts, parseSchemaText, STORAGE_DIR, adoptLegacyVersionKey, listAgents, } from "./store.js";
|
|
23
34
|
import { loadCatalog, checkModel, clampEffort, catalogCost, findModel } from "./catalog.js";
|
|
24
35
|
import { resolveSecrets, getSecret, materializeSecrets } from "./secrets.js";
|
|
25
36
|
import { materializeFileSecrets, cleanupFileSecrets } from "./secret-files.js";
|
|
@@ -30,10 +41,17 @@ import { startTranslator, translatorSpecFor } from "./translator.js";
|
|
|
30
41
|
import { providerPreset } from "./providers.js";
|
|
31
42
|
import { buildScriptTools, parseScripts } from "./script-tools.js";
|
|
32
43
|
import { libraryDir, libraryTools, libraryMemoryIndex } from "./library.js";
|
|
33
|
-
import { mergeRuntimes, parseRuntime, prepareRuntime } from "./runtime.js";
|
|
44
|
+
import { fingerprint, mergeRuntimes, parseRuntime, prepareRuntime } from "./runtime.js";
|
|
34
45
|
import { materializeFiles, harvestFiles } from "./storage.js";
|
|
35
46
|
import { chooseExecutor, ensureImage } from "./container.js";
|
|
36
47
|
import { stampBundle } from "./okf.js";
|
|
48
|
+
import { readBrowseSettings, readSearchSettings, resolveSearch, searchSettingsEnv, webConfig, browseSessionProblems } from "./providers.js";
|
|
49
|
+
import { NEWER_ACTIONS, resolveActionApi } from "./web-actions.js";
|
|
50
|
+
import { resolveLanguage, languageName } from "./language.js";
|
|
51
|
+
import { resolveRegion, deriveLocale, localeProse, regionName } from "./locale.js";
|
|
52
|
+
import { trimChars } from "./paths.js";
|
|
53
|
+
import { resolveClock, localDate } from "./clock.js";
|
|
54
|
+
export { localDate } from "./clock.js";
|
|
37
55
|
// An MCP tool definition becomes an SDK server config. ${SECRET} placeholders
|
|
38
56
|
// in env and headers resolve server-side, so a credential reaches the server
|
|
39
57
|
// process without ever entering the model's context.
|
|
@@ -84,6 +102,34 @@ export function joinEarlierGroups(groupResults, gi, cap = CONTEXT_CHARS) {
|
|
|
84
102
|
parts.shift();
|
|
85
103
|
return parts.join("\n\n");
|
|
86
104
|
}
|
|
105
|
+
/**
|
|
106
|
+
* `parallel: N` — a pool of N slots the instances of one fan-out share.
|
|
107
|
+
* Fan-out always ran every instance at once, and still does unless the
|
|
108
|
+
* step says otherwise: twenty browser steps against one site, or one
|
|
109
|
+
* vendor's rate limit, is what the cap is for. Nothing is rented while an
|
|
110
|
+
* instance waits for a slot.
|
|
111
|
+
*/
|
|
112
|
+
export function slotPool(size) {
|
|
113
|
+
let free = Math.max(1, size);
|
|
114
|
+
const waiting = [];
|
|
115
|
+
return {
|
|
116
|
+
acquire: () => new Promise((resolve) => {
|
|
117
|
+
if (free > 0) {
|
|
118
|
+
free -= 1;
|
|
119
|
+
resolve();
|
|
120
|
+
}
|
|
121
|
+
else
|
|
122
|
+
waiting.push(resolve);
|
|
123
|
+
}),
|
|
124
|
+
release: () => {
|
|
125
|
+
const next = waiting.shift();
|
|
126
|
+
if (next)
|
|
127
|
+
next();
|
|
128
|
+
else
|
|
129
|
+
free += 1;
|
|
130
|
+
},
|
|
131
|
+
};
|
|
132
|
+
}
|
|
87
133
|
const toTags = (v) => Array.isArray(v) ? v.map(String) : typeof v === "string" && v.trim() ? [v.trim()] : [];
|
|
88
134
|
// gray-matter throws on invalid YAML. Skills authored for another client
|
|
89
135
|
// sometimes ship YAML that that client's parser happened to accept — most
|
|
@@ -180,6 +226,30 @@ export function discoverSkills(agentDir, subdir = "skills") {
|
|
|
180
226
|
* Returns null rather than throwing on unparseable YAML: a broken shared file
|
|
181
227
|
* must not take down every agent under it.
|
|
182
228
|
*/
|
|
229
|
+
/**
|
|
230
|
+
* AGENTS.md without the block a coding tool reads.
|
|
231
|
+
*
|
|
232
|
+
* AGENTS.md is two things in a foldrun folder: the account's (or
|
|
233
|
+
* workspace's) own context, which every running agent is given, and — the
|
|
234
|
+
* convention Next.js set — the file a coding agent (Claude Code, Cursor,
|
|
235
|
+
* Codex) reads before it edits the folder. The CLI writes a short managed
|
|
236
|
+
* block for the second reader between these markers; it is about editing
|
|
237
|
+
* the files, not about doing the work, so no running agent is shown it.
|
|
238
|
+
*/
|
|
239
|
+
export const AGENT_RULES_START = "<!-- BEGIN:foldrun-agent-rules -->";
|
|
240
|
+
export const AGENT_RULES_END = "<!-- END:foldrun-agent-rules -->";
|
|
241
|
+
export function withoutAgentRules(body) {
|
|
242
|
+
let out = body;
|
|
243
|
+
for (;;) {
|
|
244
|
+
const start = out.indexOf(AGENT_RULES_START);
|
|
245
|
+
if (start === -1)
|
|
246
|
+
return out;
|
|
247
|
+
const end = out.indexOf(AGENT_RULES_END, start);
|
|
248
|
+
if (end === -1)
|
|
249
|
+
return out;
|
|
250
|
+
out = out.slice(0, start) + out.slice(end + AGENT_RULES_END.length);
|
|
251
|
+
}
|
|
252
|
+
}
|
|
183
253
|
export function readAgentsMd(dir) {
|
|
184
254
|
// Convert a pre-rename `mdagent_version:` before parsing. One name in the
|
|
185
255
|
// code, one on disk — the reader no longer knows two spellings, so a file
|
|
@@ -191,7 +261,7 @@ export function readAgentsMd(dir) {
|
|
|
191
261
|
continue;
|
|
192
262
|
try {
|
|
193
263
|
const parsed = matter(fs.readFileSync(file, "utf8"));
|
|
194
|
-
return { data: parsed.data, body: parsed.content.trim() };
|
|
264
|
+
return { data: parsed.data, body: withoutAgentRules(parsed.content).trim() };
|
|
195
265
|
}
|
|
196
266
|
catch {
|
|
197
267
|
return null;
|
|
@@ -209,6 +279,53 @@ const workspaceRootOf = (agentDir) => path.join(agentDir, "..", "..");
|
|
|
209
279
|
* writing instructions means *as well as*, because an account rule nobody can
|
|
210
280
|
* silently drop is the point of having one.
|
|
211
281
|
*/
|
|
282
|
+
/** An agent's own `runtime:` block, else its workspace's (AGENTS.md, the
|
|
283
|
+
* Linux Foundation standard name; project.md stays accepted so nothing
|
|
284
|
+
* that already exists breaks). */
|
|
285
|
+
function ownRuntime(front, agentDir, tenant) {
|
|
286
|
+
return parseRuntime(front.runtime) ?? parseRuntime(workspaceFrontmatter(agentDir, tenant).runtime);
|
|
287
|
+
}
|
|
288
|
+
/** What one granted script tool's program needs. */
|
|
289
|
+
function toolRuntime(def) {
|
|
290
|
+
return def.kind === "script" ? parseRuntime(def.spec.runtime) : null;
|
|
291
|
+
}
|
|
292
|
+
/**
|
|
293
|
+
* The runtime one agent's steps are built with: its own declaration (else its
|
|
294
|
+
* workspace's), merged with every script tool it grants. The runner builds
|
|
295
|
+
* exactly this for a step; the platform builds it ahead of time when the
|
|
296
|
+
* workspace is deployed, so a package that will not install is reported at
|
|
297
|
+
* deploy and a scheduled step finds its environment ready. One function, so
|
|
298
|
+
* the two can never disagree about what an agent needs.
|
|
299
|
+
*/
|
|
300
|
+
export function agentRuntimeSpec(tenant, workspace, agentDir) {
|
|
301
|
+
const file = path.join(agentDir, "agent.md");
|
|
302
|
+
if (!fs.existsSync(file))
|
|
303
|
+
return null;
|
|
304
|
+
const front = safeMatter(fs.readFileSync(file, "utf8")).data;
|
|
305
|
+
const available = { ...libraryTools(tenant), ...workspaceTools(tenant, workspace) };
|
|
306
|
+
const tools = ownToolNames(front)
|
|
307
|
+
.map((name) => available[name])
|
|
308
|
+
.filter((def) => Boolean(def))
|
|
309
|
+
.map(toolRuntime);
|
|
310
|
+
return mergeRuntimes(ownRuntime(front, agentDir, tenant), ...tools);
|
|
311
|
+
}
|
|
312
|
+
/** Every distinct runtime a workspace's agents need, with who needs it. */
|
|
313
|
+
export function workspaceRuntimePlan(tenant, workspace) {
|
|
314
|
+
const byFp = new Map();
|
|
315
|
+
const agentsRoot = path.join(workspaceDir(tenant, workspace), "agents");
|
|
316
|
+
if (!fs.existsSync(agentsRoot))
|
|
317
|
+
return [];
|
|
318
|
+
for (const name of fs.readdirSync(agentsRoot).sort()) {
|
|
319
|
+
const spec = agentRuntimeSpec(tenant, workspace, path.join(agentsRoot, name));
|
|
320
|
+
if (!spec)
|
|
321
|
+
continue;
|
|
322
|
+
const fp = fingerprint(spec);
|
|
323
|
+
const entry = byFp.get(fp) ?? { fingerprint: fp, spec, agents: [] };
|
|
324
|
+
entry.agents.push(name);
|
|
325
|
+
byFp.set(fp, entry);
|
|
326
|
+
}
|
|
327
|
+
return [...byFp.values()];
|
|
328
|
+
}
|
|
212
329
|
function workspaceFrontmatter(agentDir, tenant) {
|
|
213
330
|
return {
|
|
214
331
|
...(readAgentsMd(accountDir(tenant))?.data ?? {}),
|
|
@@ -216,31 +333,60 @@ function workspaceFrontmatter(agentDir, tenant) {
|
|
|
216
333
|
};
|
|
217
334
|
}
|
|
218
335
|
/**
|
|
219
|
-
* The calendar
|
|
220
|
-
*
|
|
221
|
-
*
|
|
222
|
-
*
|
|
223
|
-
* An unknown name falls back to UTC rather than failing every step.
|
|
336
|
+
* The calendar a set of frontmatter asks for, with no cascade around it —
|
|
337
|
+
* one merged record in, one zone out. The full cascade lives in
|
|
338
|
+
* `agentClock` below and in clock.ts; this is what the account-level parts
|
|
339
|
+
* of the platform (a wallet's month, a budget window) call.
|
|
224
340
|
*/
|
|
225
341
|
export function resolveTimezone(front) {
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
342
|
+
return resolveClock([{ level: "workspace", value: front.timezone }]).timezone;
|
|
343
|
+
}
|
|
344
|
+
/**
|
|
345
|
+
* The clock this step works to: agent, then flow, then workspace, then
|
|
346
|
+
* account, then FOLDRUN_TIMEZONE, then UTC. Nearest wins, and a value that
|
|
347
|
+
* is not a zone falls through to the next level with a line to say so —
|
|
348
|
+
* see clock.ts for why this exists at all.
|
|
349
|
+
*/
|
|
350
|
+
export function agentClock(agentDir, tenant, agentFront, flowTimezone) {
|
|
351
|
+
return resolveClock([
|
|
352
|
+
{ level: "agent", value: agentFront.timezone },
|
|
353
|
+
{ level: "flow", value: flowTimezone },
|
|
354
|
+
{ level: "workspace", value: readAgentsMd(workspaceRootOf(agentDir))?.data?.timezone },
|
|
355
|
+
{ level: "account", value: readAgentsMd(accountDir(tenant))?.data?.timezone },
|
|
356
|
+
]);
|
|
357
|
+
}
|
|
358
|
+
/**
|
|
359
|
+
* The language this step works in: agent, then workspace, then account, then
|
|
360
|
+
* FOLDRUN_LANGUAGE, then English. The flow level is not read yet — a flow's
|
|
361
|
+
* timezone: travels through three signatures to get here, and this will
|
|
362
|
+
* take the same road when a flow needs it.
|
|
363
|
+
*/
|
|
364
|
+
export function agentLanguage(agentDir, tenant, agentFront) {
|
|
365
|
+
return resolveLanguage([
|
|
366
|
+
{ level: "agent", value: agentFront.language },
|
|
367
|
+
{ level: "workspace", value: readAgentsMd(workspaceRootOf(agentDir))?.data?.language },
|
|
368
|
+
{ level: "account", value: readAgentsMd(accountDir(tenant))?.data?.language },
|
|
369
|
+
]);
|
|
370
|
+
}
|
|
371
|
+
/**
|
|
372
|
+
* The country this step works for, on the same road as the language, then
|
|
373
|
+
* FOLDRUN_REGION, then whatever the language tag carried (`en-AU` says AU).
|
|
374
|
+
* Everything a country changes — currency, units, calendar, weekend, text
|
|
375
|
+
* direction — is derived from it in locale.ts; the three overrides
|
|
376
|
+
* (`currency:`, `calendar:`, `units:`) are read at the nearest level that
|
|
377
|
+
* wrote them.
|
|
378
|
+
*/
|
|
379
|
+
export function agentRegion(agentDir, tenant, agentFront, languageTag) {
|
|
380
|
+
return resolveRegion([
|
|
381
|
+
{ level: "agent", value: agentFront.region },
|
|
382
|
+
{ level: "workspace", value: readAgentsMd(workspaceRootOf(agentDir))?.data?.region },
|
|
383
|
+
{ level: "account", value: readAgentsMd(accountDir(tenant))?.data?.region },
|
|
384
|
+
], languageTag);
|
|
235
385
|
}
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
year: "numeric",
|
|
241
|
-
month: "2-digit",
|
|
242
|
-
day: "2-digit",
|
|
243
|
-
}).format(now);
|
|
386
|
+
export function agentLocaleOverrides(agentDir, tenant, agentFront) {
|
|
387
|
+
const levels = [agentFront, readAgentsMd(workspaceRootOf(agentDir))?.data ?? {}, readAgentsMd(accountDir(tenant))?.data ?? {}];
|
|
388
|
+
const pick = (k) => levels.map((l) => l[k]).find((v) => v !== undefined && v !== null && v !== "");
|
|
389
|
+
return { currency: pick("currency"), calendar: pick("calendar"), units: pick("units") };
|
|
244
390
|
}
|
|
245
391
|
/**
|
|
246
392
|
* The prose from AGENTS.md, outermost first.
|
|
@@ -249,6 +395,56 @@ export function localDate(timeZone, now = new Date()) {
|
|
|
249
395
|
* template as "context every agent here shares" — and never read. Only the
|
|
250
396
|
* frontmatter was, so the instructions people actually wrote reached no model.
|
|
251
397
|
*/
|
|
398
|
+
/**
|
|
399
|
+
* The sub-agents a step may delegate to (`subagents:`), as the SDK will get
|
|
400
|
+
* them: each resolved through agentContext exactly as its own step would be
|
|
401
|
+
* (grants only — no runtime; it runs inside its parent's), then cut to what
|
|
402
|
+
* the parent holds. Problems are lines on the step, never a failed step.
|
|
403
|
+
*/
|
|
404
|
+
export function buildSubagentSpecs(o) {
|
|
405
|
+
const out = [];
|
|
406
|
+
const { found, missing } = gatherSubagents(o.workspaceRoot, o.names);
|
|
407
|
+
for (const m of missing)
|
|
408
|
+
o.push("error", `subagents: "${m}" is not an agent in this workspace — not delegated to`);
|
|
409
|
+
for (const sub of found) {
|
|
410
|
+
if (sub.name === o.self) {
|
|
411
|
+
o.push("error", `subagents: "${sub.name}" is this agent itself — an agent does not delegate to itself`);
|
|
412
|
+
continue;
|
|
413
|
+
}
|
|
414
|
+
try {
|
|
415
|
+
const tags = o.tags ?? [];
|
|
416
|
+
const own = agentContext(sub.dir, o.tenant, tags, { runId: o.runId, agent: sub.name }, o.flowTimezone, true);
|
|
417
|
+
const description = typeof own.front.description === "string" ? own.front.description.trim() : "";
|
|
418
|
+
if (!description) {
|
|
419
|
+
o.push("error", `subagents: "${sub.name}" has no description — the model picks a sub-agent by it; not delegated to`);
|
|
420
|
+
continue;
|
|
421
|
+
}
|
|
422
|
+
const tools = subagentTools(own.allowed, o.parentAllowed);
|
|
423
|
+
if (!tools.length) {
|
|
424
|
+
o.push("error", `subagents: "${sub.name}" holds none of its tools inside this step — it can only answer from its prompt (use agents: for that)`);
|
|
425
|
+
}
|
|
426
|
+
const body = withoutAgentRules(matter(fs.readFileSync(path.join(sub.dir, "agent.md"), "utf8")).content);
|
|
427
|
+
out.push({
|
|
428
|
+
name: sub.name,
|
|
429
|
+
description,
|
|
430
|
+
prompt: subagentPrompt({
|
|
431
|
+
body: resolveDocLinks(body, o.workspaceRoot),
|
|
432
|
+
name: sub.name,
|
|
433
|
+
parent: o.self,
|
|
434
|
+
workspace: path.basename(o.workspaceRoot),
|
|
435
|
+
shared: sharedInstructions(sub.dir, o.tenant),
|
|
436
|
+
}),
|
|
437
|
+
tools,
|
|
438
|
+
disallowedTools: refNames(own.front.disallowedTools),
|
|
439
|
+
model: own.front.model ? resolveModel(own.front.model) : "inherit",
|
|
440
|
+
});
|
|
441
|
+
}
|
|
442
|
+
catch (err) {
|
|
443
|
+
o.push("error", `subagents: "${sub.name}" could not be read — ${err instanceof Error ? err.message : String(err)}`);
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
return out;
|
|
447
|
+
}
|
|
252
448
|
export function sharedInstructions(agentDir, tenant) {
|
|
253
449
|
const sections = [
|
|
254
450
|
["Everyone in this account", readAgentsMd(accountDir(tenant))?.body],
|
|
@@ -311,10 +507,9 @@ export function saysUntilMarker(result, until) {
|
|
|
311
507
|
.some((line) =>
|
|
312
508
|
// Strip markdown emphasis and punctuation from BOTH ends: "**APPROVED.**"
|
|
313
509
|
// is a decision written by someone using bold, not a sentence.
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
.replace(/[.!:,*_`\s]+$/, "")
|
|
510
|
+
// trimChars, not a `[…]+$` regex: that one is quadratic on a
|
|
511
|
+
// line of tabs, and this is a line an agent wrote.
|
|
512
|
+
trimChars(trimChars(line.trim(), "#>-*_`"), ".!:,*_`").trim()
|
|
318
513
|
.toLowerCase() === marker);
|
|
319
514
|
}
|
|
320
515
|
export function resolveDocLinks(text, workspaceRoot) {
|
|
@@ -327,8 +522,10 @@ export function resolveDocLinks(text, workspaceRoot) {
|
|
|
327
522
|
// instruction names needs to say WHERE without naming a file. Added
|
|
328
523
|
// first, so a document that happens to share a folder's name still wins.
|
|
329
524
|
for (const dir of [STORAGE_DIR, "state", "knowledge", "memory", "skills", "outputs"]) {
|
|
525
|
+
// storage/ and state/ by the one spelling that works in every tool,
|
|
526
|
+
// the shell included; the rest as they always were.
|
|
330
527
|
if (fs.existsSync(path.join(workspaceRoot, dir)))
|
|
331
|
-
map.set(norm(dir), `../../${dir}/`);
|
|
528
|
+
map.set(norm(dir), dir === STORAGE_DIR || dir === "state" ? `workspace/${dir}/` : `../../${dir}/`);
|
|
332
529
|
}
|
|
333
530
|
for (const kind of ["knowledge", "memory"]) {
|
|
334
531
|
const dir = path.join(workspaceRoot, kind);
|
|
@@ -373,7 +570,7 @@ export function resolveDocLinks(text, workspaceRoot) {
|
|
|
373
570
|
continue;
|
|
374
571
|
}
|
|
375
572
|
const fwd = entry.split(path.sep).join("/");
|
|
376
|
-
const rel =
|
|
573
|
+
const rel = `workspace/state/${fwd}`;
|
|
377
574
|
map.set(norm(`state/${fwd}`), rel);
|
|
378
575
|
map.set(norm(fwd), rel);
|
|
379
576
|
}
|
|
@@ -385,7 +582,7 @@ export function resolveDocLinks(text, workspaceRoot) {
|
|
|
385
582
|
// through to the plain-directory walk below.
|
|
386
583
|
const index = JSON.parse(fs.readFileSync(path.join(workspaceRoot, "..", "..", STORAGE_DIR, path.basename(workspaceRoot), "index.json"), "utf8"));
|
|
387
584
|
for (const f of index.files ?? [])
|
|
388
|
-
map.set(norm(`${STORAGE_DIR}/${f.path}`),
|
|
585
|
+
map.set(norm(`${STORAGE_DIR}/${f.path}`), `workspace/${STORAGE_DIR}/${f.path}`);
|
|
389
586
|
}
|
|
390
587
|
catch {
|
|
391
588
|
// no file store index is normal
|
|
@@ -404,18 +601,27 @@ export function resolveDocLinks(text, workspaceRoot) {
|
|
|
404
601
|
const fwd = entry.split(path.sep).join("/");
|
|
405
602
|
if (fwd.startsWith(".store/"))
|
|
406
603
|
continue;
|
|
407
|
-
map.set(norm(`${STORAGE_DIR}/${fwd}`),
|
|
408
|
-
map.set(norm(fwd),
|
|
604
|
+
map.set(norm(`${STORAGE_DIR}/${fwd}`), `workspace/${STORAGE_DIR}/${fwd}`);
|
|
605
|
+
map.set(norm(fwd), `workspace/${STORAGE_DIR}/${fwd}`);
|
|
409
606
|
}
|
|
410
607
|
}
|
|
411
|
-
|
|
608
|
+
// `[` excluded from the name: with it allowed, a run of `[[` makes the
|
|
609
|
+
// engine re-scan to the end from every opening — quadratic on a result
|
|
610
|
+
// an agent wrote.
|
|
611
|
+
return text.replace(/\[\[([^\]\[\n]+)\]\]/g, (whole, name) => {
|
|
412
612
|
const hit = map.get(norm(name.trim()));
|
|
413
613
|
return hit ? `\`${hit}\`` : whole;
|
|
414
614
|
});
|
|
415
615
|
}
|
|
416
616
|
function agentContext(agentDir, tenant, tags = [],
|
|
417
617
|
/** Who is running, so scripts can stamp what they write with its provenance. */
|
|
418
|
-
identity = {}
|
|
618
|
+
identity = {},
|
|
619
|
+
/** The flow's own `timezone:`, when this step belongs to one. Between the
|
|
620
|
+
* agent's frontmatter and the workspace's AGENTS.md in the cascade. */
|
|
621
|
+
flowTimezone,
|
|
622
|
+
/** Only what the agent is granted — no runtime prepared. For a sub-agent,
|
|
623
|
+
* whose tools are cut to its parent's and run in the parent's runtime. */
|
|
624
|
+
grantsOnly = false) {
|
|
419
625
|
// agentDir is <data>/<tenant>/projects/<workspace>/agents/<agent>, so the
|
|
420
626
|
// workspace a secret should resolve against is two levels up.
|
|
421
627
|
const workspace = path.basename(path.resolve(agentDir, "..", ".."));
|
|
@@ -426,12 +632,34 @@ identity = {}) {
|
|
|
426
632
|
// than implied: told to read `../../knowledge/index.md`, the model asked for
|
|
427
633
|
// `/knowledge/index.md` twice before working it out. Two denials and a turn,
|
|
428
634
|
// for want of one sentence.
|
|
635
|
+
// The language this step works in, resolved before the prompt is built:
|
|
636
|
+
// its sentence goes in here, the env the tools read is set below, and the
|
|
637
|
+
// step prints any level that wrote something unreadable.
|
|
638
|
+
const language = agentLanguage(agentDir, tenant, front);
|
|
639
|
+
const region = agentRegion(agentDir, tenant, front, language.language);
|
|
640
|
+
// The clock is resolved further down for the env; the locale needs its
|
|
641
|
+
// zone now so "today" in a Jalali or Hijri calendar is today where the
|
|
642
|
+
// step works, not in UTC.
|
|
643
|
+
const localeZone = agentClock(agentDir, tenant, front, flowTimezone).timezone;
|
|
644
|
+
const locale = deriveLocale(language.language, region.region, agentLocaleOverrides(agentDir, tenant, front), localeZone);
|
|
645
|
+
// Said only when someone set it: an agent that never mentions a language
|
|
646
|
+
// or a region reads exactly the prompt it always did.
|
|
647
|
+
const languageSet = language.from !== "default" && language.from !== "env";
|
|
648
|
+
const regionSet = region.from !== "none" && region.from !== "env" && region.from !== "language";
|
|
649
|
+
if (languageSet || regionSet) {
|
|
650
|
+
parts.push(`# Locale\n\n${localeProse(locale, languageName(language.language), regionName(region.region))} ` +
|
|
651
|
+
`Write in ${languageName(language.language)} unless the step you are given says otherwise; ` +
|
|
652
|
+
`use these currency, units and calendar conventions in anything you produce. ` +
|
|
653
|
+
`Your tools already know: a search asks in it and for this country, the browser reports it, a fetch requests it.`);
|
|
654
|
+
}
|
|
429
655
|
parts.push(`# Where you are\n\n` +
|
|
430
656
|
`Your working directory is \`agents/${path.basename(agentDir)}/\` inside the ` +
|
|
431
657
|
`\`${workspace}\` workspace. Every path in this prompt is relative to it — ` +
|
|
432
|
-
`\`outputs/\` is yours, and
|
|
433
|
-
`
|
|
434
|
-
`workspace
|
|
658
|
+
`\`outputs/\` is yours, and \`workspace/\` is the workspace root, in every tool ` +
|
|
659
|
+
`and in the shell: the shared file store is \`workspace/storage/\`, what runs ` +
|
|
660
|
+
`carry forward is \`workspace/state/\`. (\`../../\` reaches the same root and ` +
|
|
661
|
+
`still works, so a path like \`../../knowledge/\` below is \`workspace/knowledge/\`.) ` +
|
|
662
|
+
`Absolute paths are outside the workspace and will be refused.\n\n` +
|
|
435
663
|
// A flow's whole point is that a later step works on what an earlier one
|
|
436
664
|
// produced, and each agent writes to its own outputs/. A run showed what
|
|
437
665
|
// omitting this costs: the checker looked in its own empty outputs/ and
|
|
@@ -452,10 +680,18 @@ identity = {}) {
|
|
|
452
680
|
// adoptLegacyFilesDir's "move files/ to storage/" is a no-op by the time
|
|
453
681
|
// the run ends — a deliverable written where this text pointed was
|
|
454
682
|
// harvested from nowhere and silently never appeared.
|
|
455
|
-
|
|
683
|
+
`\`workspace/storage/\` is the workspace file store: anything you leave there is kept ` +
|
|
456
684
|
`after the run and shown on the Storage page for people to download. Write ` +
|
|
457
|
-
`deliverables people asked for — CSVs, reports, PDFs, images — to
|
|
458
|
-
`use \`outputs/\` for working text the next step reads
|
|
685
|
+
`deliverables people asked for — CSVs, reports, PDFs, images — to \`workspace/storage/\`; ` +
|
|
686
|
+
`use \`outputs/\` for working text the next step reads. ` +
|
|
687
|
+
// A Glob PATTERN that climbs (`../../storage/draft/*.md`) always answered
|
|
688
|
+
// "No files found", even when the file was there. On blog-desk that made
|
|
689
|
+
// every loop pass look like a first pass: the illustrator never saw the
|
|
690
|
+
// fact-check report, re-picked the cover, and 2 of 3 publish runs failed
|
|
691
|
+
// (run-munef2am-439a, 2026-09-30). checkPaths now splits such a pattern
|
|
692
|
+
// into Glob's `path` and the rest; the advice stays, as the plain form.
|
|
693
|
+
`To look for files there, open a known file with Read by its exact path, or ` +
|
|
694
|
+
`Glob with \`path: "workspace/storage/..."\` and a pattern like \`*.md\`.`);
|
|
459
695
|
// Shared context before anything derived — an account or workspace rule is
|
|
460
696
|
// background the rest of the prompt is read against, not an afterthought.
|
|
461
697
|
const sharedContext = sharedInstructions(agentDir, tenant);
|
|
@@ -593,7 +829,7 @@ identity = {}) {
|
|
|
593
829
|
// for it. The difference is that state is small enough that inlining would
|
|
594
830
|
// usually be free — that is a threshold worth adding once there is a real
|
|
595
831
|
// file to size it against, not before.
|
|
596
|
-
const state = listDir(path.join(agentDir, "..", "..", "state"), "
|
|
832
|
+
const state = listDir(path.join(agentDir, "..", "..", "state"), "workspace/state/");
|
|
597
833
|
if (state.length) {
|
|
598
834
|
parts.push(`# State — what you carry between runs\n\n` +
|
|
599
835
|
`What this workspace keeps across runs: a count, a cursor, where you got to. ` +
|
|
@@ -627,17 +863,32 @@ identity = {}) {
|
|
|
627
863
|
// below. The tool is the unit of code, so its dependencies live in tool.md
|
|
628
864
|
// and travel with it; the agent should not have to repeat them.
|
|
629
865
|
const toolRuntimes = [];
|
|
866
|
+
// The secrets a granted script tool's file lists (`secrets: [RESEND_API_KEY,
|
|
867
|
+
// EMAIL_FROM]`): granting the tool grants them, the way an API tool's
|
|
868
|
+
// referenced credentials always travelled with it. Before, only the
|
|
869
|
+
// agent's own `secrets:` counted, so an agent that granted desk_email and
|
|
870
|
+
// declared nothing failed with "RESEND_API_KEY is not set" (strata-desk,
|
|
871
|
+
// 1 Oct). `proxied` / `materialised` say how, not what, and add nothing.
|
|
872
|
+
const toolSecrets = [];
|
|
873
|
+
// Does any granted tool act outside the workspace (`outward: true`)? The
|
|
874
|
+
// retry policy reads it: see launchStep.
|
|
875
|
+
let outward = false;
|
|
630
876
|
const grantOwnTool = (name) => {
|
|
631
877
|
const def = available[name];
|
|
632
878
|
if (!def) {
|
|
633
879
|
missingTools.push(name);
|
|
634
880
|
return;
|
|
635
881
|
}
|
|
882
|
+
if (def.outward)
|
|
883
|
+
outward = true;
|
|
636
884
|
if (def.kind === "http")
|
|
637
885
|
apis.push(def.spec);
|
|
638
886
|
else if (def.kind === "script") {
|
|
639
887
|
scriptSpecs.push(...parseScripts([def.spec]));
|
|
640
|
-
const
|
|
888
|
+
const listed = def.spec.secrets;
|
|
889
|
+
if (Array.isArray(listed))
|
|
890
|
+
toolSecrets.push(...listed.map(String).filter((n) => /^[A-Z][A-Z0-9_]*$/.test(n)));
|
|
891
|
+
const rt = toolRuntime(def);
|
|
641
892
|
if (rt)
|
|
642
893
|
toolRuntimes.push(rt);
|
|
643
894
|
}
|
|
@@ -681,7 +932,69 @@ identity = {}) {
|
|
|
681
932
|
//
|
|
682
933
|
// Declared-and-absent is still an error; referenced-and-absent is the API
|
|
683
934
|
// tool's own missingSecrets, reported where the tool is described.
|
|
684
|
-
|
|
935
|
+
// `web: {search: …, fetch: …}` name who answers; resolved here, early,
|
|
936
|
+
// because three later things hang off the answer — the secret the step
|
|
937
|
+
// must hold, the env the tool reads, and the egress grant for the host.
|
|
938
|
+
// `web.search` carries the account engine's settings (engines, categories,
|
|
939
|
+
// safesearch, …) beside the provider name, read off first the way
|
|
940
|
+
// browse's are, so the provider resolver sees only its own part.
|
|
941
|
+
const web = webConfig(front, workspaceFrontmatter(agentDir, tenant));
|
|
942
|
+
const searchRead = readSearchSettings(web.raw.search);
|
|
943
|
+
const searchChoice = resolveSearch(searchRead.rest, "search");
|
|
944
|
+
if (searchRead.error)
|
|
945
|
+
searchChoice.error = searchChoice.error ?? searchRead.error;
|
|
946
|
+
const fetchChoice = resolveSearch(web.raw.fetch, "fetch");
|
|
947
|
+
// A remote browser's key is materialised, not proxied: CDP is a websocket
|
|
948
|
+
// and the wrapper opens the session itself, the way it seeds a cookie.
|
|
949
|
+
// `web.browse` carries two things in one key: how the browser presents
|
|
950
|
+
// itself (engine, user agent, device, locale, timezone) and, optionally,
|
|
951
|
+
// which remote vendor renders the page. Settings are read off first so the
|
|
952
|
+
// vendor resolver sees only the vendor part, and a block of settings alone
|
|
953
|
+
// means the account's own browser with those defaults. Nearest wins: the
|
|
954
|
+
// agent's block, else the workspace's, else the account's — the shape
|
|
955
|
+
// `provider:` already uses.
|
|
956
|
+
const browseRead = readBrowseSettings(web.raw.browse);
|
|
957
|
+
const browseSettings = browseRead.settings;
|
|
958
|
+
const browseChoice = resolveSearch(browseRead.rest, "browse");
|
|
959
|
+
if (browseRead.error)
|
|
960
|
+
browseChoice.error = browseChoice.error ?? browseRead.error;
|
|
961
|
+
// What the vendor's own session is asked for, checked against what that
|
|
962
|
+
// vendor offers; a problem is on the record and the session is not sent.
|
|
963
|
+
const sessionProblems = browseChoice.error ? [] : browseSessionProblems(browseSettings, browseChoice.provider);
|
|
964
|
+
const browseSession = browseChoice.shape === "direct" && browseChoice.provider && !sessionProblems.length ? browseSettings.vendor_session : undefined;
|
|
965
|
+
// crawl, map, extract, answer, monitor: a provider each, or foldrun's own.
|
|
966
|
+
// Same three consequences as a search API — the secret the step holds, the
|
|
967
|
+
// env the web tool reads, the one host the key may reach.
|
|
968
|
+
const actionChoices = NEWER_ACTIONS.map((action) => ({ action, ...resolveActionApi(action, web.raw[action]) }));
|
|
969
|
+
const builtins = [
|
|
970
|
+
...(searchChoice.provider && searchChoice.shape !== "direct" && !searchChoice.error ? ["search=WebSearch"] : []),
|
|
971
|
+
...(fetchChoice.provider && fetchChoice.shape !== "direct" && !fetchChoice.error ? ["fetch=WebFetch"] : []),
|
|
972
|
+
];
|
|
973
|
+
const webBuiltin = builtins.length ? { FOLDRUN_WEB_BUILTIN: builtins.join(",") } : {};
|
|
974
|
+
// A direct search API is paid for with the customer's own key, stored
|
|
975
|
+
// under a fixed vault name (EXA_API_KEY, BRAVE_SEARCH_API_KEY, …). Naming
|
|
976
|
+
// the API in frontmatter is declaring that secret — nobody should have to
|
|
977
|
+
// write it twice, and a run that forgot the second line would search
|
|
978
|
+
// nothing and say so only at the provider.
|
|
979
|
+
const declared = [
|
|
980
|
+
...(Array.isArray(front.secrets) ? front.secrets.map(String) : []),
|
|
981
|
+
...toolSecrets,
|
|
982
|
+
...actionChoices.flatMap((c) => (c.provider ? [c.secret] : [])),
|
|
983
|
+
// Your own proxy for a vendor's session: its URL is a secret, declared by
|
|
984
|
+
// naming it, like a provider's key.
|
|
985
|
+
...(browseSession && typeof browseSession.proxy === "object" && browseSession.proxy.own ? [browseSession.proxy.own] : []),
|
|
986
|
+
...[searchChoice, fetchChoice, browseChoice].flatMap((c) => {
|
|
987
|
+
if (c.shape !== "direct" || !c.secret)
|
|
988
|
+
return [];
|
|
989
|
+
// Jina's reader answers without a key. Declaring an absent optional
|
|
990
|
+
// key would fail the step for a credential it never needed.
|
|
991
|
+
if (c.secretOptional && !getSecret(tenant, c.secret, workspace))
|
|
992
|
+
return [];
|
|
993
|
+
return [c.secret];
|
|
994
|
+
}),
|
|
995
|
+
];
|
|
996
|
+
// One name once, however many places asked for it.
|
|
997
|
+
declared.splice(0, declared.length, ...new Set(declared));
|
|
685
998
|
const referenced = [...new Set(apis.flatMap(secretsUsedByApi))];
|
|
686
999
|
const { env: secretEnv, from: secretScopes, } = resolveSecrets(tenant, [...new Set([...declared, ...referenced])], workspace);
|
|
687
1000
|
const { missing: missingDeclared } = resolveSecrets(tenant, declared, workspace);
|
|
@@ -787,15 +1100,8 @@ identity = {}) {
|
|
|
787
1100
|
// Runtime: an agent's own `runtime:` block, else its workspace's. Built once
|
|
788
1101
|
// per declaration and cached, so scripts get their dependencies without
|
|
789
1102
|
// polluting the host.
|
|
790
|
-
let runtimeSpec = parseRuntime(front.runtime);
|
|
791
|
-
if (!runtimeSpec) {
|
|
792
|
-
// AGENTS.md is the Linux Foundation standard name (60k+ repos, same
|
|
793
|
-
// nearest-wins cascade we already use); project.md stays accepted so
|
|
794
|
-
// nothing that already exists breaks.
|
|
795
|
-
runtimeSpec = parseRuntime(workspaceFrontmatter(agentDir, tenant).runtime);
|
|
796
|
-
}
|
|
797
1103
|
// Plus whatever the tools this agent granted declared for themselves.
|
|
798
|
-
runtimeSpec = mergeRuntimes(
|
|
1104
|
+
let runtimeSpec = mergeRuntimes(ownRuntime(front, agentDir, tenant), ...toolRuntimes);
|
|
799
1105
|
// Execution: a container per run when Docker is available (real isolation),
|
|
800
1106
|
// otherwise the host venv path so local development still works.
|
|
801
1107
|
const executor = chooseExecutor();
|
|
@@ -803,7 +1109,7 @@ identity = {}) {
|
|
|
803
1109
|
let runtimeError = null;
|
|
804
1110
|
let exec = null;
|
|
805
1111
|
let runtime = { interpreters: {}, env: {}, log: [], error: null };
|
|
806
|
-
if (executor === "docker") {
|
|
1112
|
+
if (executor === "docker" && !grantsOnly) {
|
|
807
1113
|
const image = ensureImage(runtimeSpec);
|
|
808
1114
|
runtimeLog.push(...image.log);
|
|
809
1115
|
runtimeError = image.error;
|
|
@@ -821,22 +1127,103 @@ identity = {}) {
|
|
|
821
1127
|
};
|
|
822
1128
|
}
|
|
823
1129
|
}
|
|
1130
|
+
else if (isolatedRun() || grantsOnly) {
|
|
1131
|
+
// (A sub-agent's grants are all that is asked of it: it runs inside its
|
|
1132
|
+
// parent's step and runtime.)
|
|
1133
|
+
// The sandbox builds it (run-container.ts, beside pip's network and the
|
|
1134
|
+
// mounted cache); building it here as well was pure waste, and on a
|
|
1135
|
+
// worker image without python it was worse than waste — it logged
|
|
1136
|
+
// "python3 is not available on this host" as an error on every Python
|
|
1137
|
+
// step, above a sandbox build that had succeeded, and sent every reader
|
|
1138
|
+
// of the trace after the wrong machine.
|
|
1139
|
+
}
|
|
824
1140
|
else {
|
|
825
1141
|
runtime = prepareRuntime(tenant, runtimeSpec);
|
|
826
1142
|
runtimeLog.push(...runtime.log);
|
|
827
1143
|
runtimeError = runtime.error;
|
|
828
1144
|
}
|
|
829
|
-
// What day it is, in the
|
|
830
|
-
//
|
|
831
|
-
//
|
|
832
|
-
|
|
1145
|
+
// What day it is, in the calendar this step works to — the agent's own
|
|
1146
|
+
// `timezone:`, else its flow's, else the workspace's, else the account's.
|
|
1147
|
+
// Not a secret, so it travels beside secretEnv rather than inside it — a
|
|
1148
|
+
// value in there is redacted from every log line that quotes it.
|
|
1149
|
+
const clock = agentClock(agentDir, tenant, front, flowTimezone);
|
|
1150
|
+
const timezone = clock.timezone;
|
|
833
1151
|
const clockEnv = {
|
|
834
|
-
|
|
1152
|
+
// The language the agent works in — what the search asks in, the locale
|
|
1153
|
+
// the browser reports, the Accept-Language a fetch sends. Not a secret.
|
|
1154
|
+
FOLDRUN_LANGUAGE: language.language,
|
|
1155
|
+
// The country and what follows from it — web search's gl, a script's
|
|
1156
|
+
// currency. Unset where nobody said, so nothing is invented.
|
|
1157
|
+
...(region.region ? { FOLDRUN_REGION: region.region } : {}),
|
|
1158
|
+
...(locale.currency ? { FOLDRUN_CURRENCY: locale.currency } : {}),
|
|
1159
|
+
FOLDRUN_UNITS: locale.units,
|
|
1160
|
+
FOLDRUN_CALENDAR: locale.calendar,
|
|
1161
|
+
...(locale.dateLocal ? { FOLDRUN_DATE_LOCAL: locale.dateLocal } : {}),
|
|
1162
|
+
FOLDRUN_RTL: locale.direction === "rtl" ? "1" : "0",
|
|
1163
|
+
// TZ is the form the sandbox's shell and Node both read; for a fixed
|
|
1164
|
+
// offset that is not the same string Intl was given (see clock.ts).
|
|
1165
|
+
TZ: clock.tz,
|
|
835
1166
|
FOLDRUN_DATE: localDate(timezone),
|
|
836
1167
|
// Where the account's own search engine answers, when the install runs
|
|
837
1168
|
// one — the `websearch` library tool reads it. Not a secret; rides with
|
|
838
1169
|
// the clock so it reaches scripts, the sandbox and verify alike.
|
|
839
1170
|
...(process.env.FOLDRUN_SEARCH_URL ? { FOLDRUN_SEARCH_URL: process.env.FOLDRUN_SEARCH_URL } : {}),
|
|
1171
|
+
// Whose index our web search asks when the agent named a search API.
|
|
1172
|
+
// Unset means the account's own engine. The name, never the key.
|
|
1173
|
+
...(searchChoice.shape === "direct" && searchChoice.provider
|
|
1174
|
+
? { FOLDRUN_WEB_SEARCH_VIA: searchChoice.provider, FOLDRUN_WEB_SEARCH_SECRET: searchChoice.secret ?? "" }
|
|
1175
|
+
: {}),
|
|
1176
|
+
// The `web.search` block's SearXNG settings, as defaults the tool reads.
|
|
1177
|
+
...searchSettingsEnv(searchRead.settings),
|
|
1178
|
+
...(fetchChoice.shape === "direct" && fetchChoice.provider
|
|
1179
|
+
? { FOLDRUN_WEB_FETCH_VIA: fetchChoice.provider, FOLDRUN_WEB_FETCH_SECRET: fetchChoice.secret ?? "" }
|
|
1180
|
+
: {}),
|
|
1181
|
+
...Object.fromEntries(actionChoices.flatMap((c) => (c.provider
|
|
1182
|
+
? [[`FOLDRUN_WEB_${c.action.toUpperCase()}_VIA`, c.provider], [`FOLDRUN_WEB_${c.action.toUpperCase()}_SECRET`, c.secret]]
|
|
1183
|
+
: []))),
|
|
1184
|
+
// Which actions the web tool may take (`web: {actions: [...]}`); unset is all.
|
|
1185
|
+
...(web.actions ? { FOLDRUN_WEB_ACTIONS: web.actions.join(",") } : {}),
|
|
1186
|
+
// A model provider's own search or fetch, which the web tool hands over to.
|
|
1187
|
+
...webBuiltin,
|
|
1188
|
+
// A remote browser: the wrapper connects over CDP to this vendor with
|
|
1189
|
+
// the named key, and the platform leaves the account's pod alone.
|
|
1190
|
+
...(browseChoice.shape === "direct" && browseChoice.provider
|
|
1191
|
+
? { FOLDRUN_BROWSER_VENDOR: browseChoice.provider, FOLDRUN_BROWSER_SECRET: browseChoice.secret ?? "" }
|
|
1192
|
+
: {}),
|
|
1193
|
+
// The vendor session's settings (`web.browse.vendor_session:`), as JSON
|
|
1194
|
+
// the tool maps onto that vendor's own option names. Under both names
|
|
1195
|
+
// for one release: a web tool from before the rename reads the old one.
|
|
1196
|
+
...(browseSession ? { FOLDRUN_BROWSER_VENDOR_SESSION: JSON.stringify(browseSession), FOLDRUN_BROWSER_SESSION: JSON.stringify(browseSession) } : {}),
|
|
1197
|
+
// The `web.browse` block, as defaults the tool reads when the call did
|
|
1198
|
+
// not say otherwise. Env rather than arguments because the tool is a
|
|
1199
|
+
// script the model calls: a default nobody typed should not have to be
|
|
1200
|
+
// typed by the model either.
|
|
1201
|
+
...(browseSettings.engine ? { FOLDRUN_BROWSER_ENGINE: browseSettings.engine } : {}),
|
|
1202
|
+
...(browseSettings.user_agent ? { FOLDRUN_BROWSER_USER_AGENT: browseSettings.user_agent } : {}),
|
|
1203
|
+
...(browseSettings.device ? { FOLDRUN_BROWSER_DEVICE: browseSettings.device } : {}),
|
|
1204
|
+
...(browseSettings.locale ? { FOLDRUN_BROWSER_LOCALE: browseSettings.locale } : {}),
|
|
1205
|
+
...(browseSettings.timezone ? { FOLDRUN_BROWSER_TIMEZONE: browseSettings.timezone } : {}),
|
|
1206
|
+
...(browseSettings.cookies ? { FOLDRUN_BROWSER_COOKIES: browseSettings.cookies } : {}),
|
|
1207
|
+
...(browseSettings.cookie_domain ? { FOLDRUN_BROWSER_COOKIE_DOMAIN: browseSettings.cookie_domain } : {}),
|
|
1208
|
+
...(browseSettings.storage ? { FOLDRUN_BROWSER_STORAGE: browseSettings.storage } : {}),
|
|
1209
|
+
...(browseSettings.storage_origin ? { FOLDRUN_BROWSER_STORAGE_ORIGIN: browseSettings.storage_origin } : {}),
|
|
1210
|
+
// The named identities, as one JSON map the tool picks from by name.
|
|
1211
|
+
...(browseSettings.identities ? { FOLDRUN_BROWSER_IDENTITIES: JSON.stringify(browseSettings.identities) } : {}),
|
|
1212
|
+
// Only the non-default travels: a window, on a virtual screen if need be.
|
|
1213
|
+
...(browseSettings.headless === false ? { FOLDRUN_BROWSER_HEADLESS: "0" } : {}),
|
|
1214
|
+
...(browseSettings.version ? { FOLDRUN_BROWSER_VERSION: browseSettings.version } : {}),
|
|
1215
|
+
...(browseSettings.live ? { FOLDRUN_BROWSER_LIVE: "1" } : {}),
|
|
1216
|
+
// What the agent may do, from its file: a lock the call cannot widen.
|
|
1217
|
+
...(browseSettings.allowed_domains ? { FOLDRUN_BROWSER_ALLOWED_DOMAINS: browseSettings.allowed_domains.join(",") } : {}),
|
|
1218
|
+
...(browseSettings.deny ? { FOLDRUN_BROWSER_DENY: browseSettings.deny.join(",") } : {}),
|
|
1219
|
+
...(browseSettings.boundaries ? { FOLDRUN_BROWSER_BOUNDARIES: "1" } : {}),
|
|
1220
|
+
...(browseSettings.init ? { FOLDRUN_BROWSER_INIT: browseSettings.init.join(",") } : {}),
|
|
1221
|
+
...(browseSettings.extensions ? { FOLDRUN_BROWSER_EXTENSIONS: browseSettings.extensions.join(",") } : {}),
|
|
1222
|
+
...(browseSettings.webgpu ? { FOLDRUN_BROWSER_WEBGPU: "1" } : {}),
|
|
1223
|
+
...(browseSettings.ignore_https_errors ? { FOLDRUN_BROWSER_IGNORE_HTTPS_ERRORS: "1" } : {}),
|
|
1224
|
+
...(browseSettings.state_key ? { FOLDRUN_BROWSER_STATE_KEY: browseSettings.state_key } : {}),
|
|
1225
|
+
...(browseSettings.video ? { FOLDRUN_BROWSER_VIDEO: "1" } : {}),
|
|
1226
|
+
...(browseSettings.live_view === false ? { FOLDRUN_BROWSER_LIVE_VIEW: "0" } : {}),
|
|
840
1227
|
};
|
|
841
1228
|
// Scripts declared as tools — callable by name, no bash required.
|
|
842
1229
|
const scriptTools = buildScriptTools(agentDir, scriptSpecs,
|
|
@@ -866,7 +1253,7 @@ identity = {}) {
|
|
|
866
1253
|
}
|
|
867
1254
|
parts.push("Write deliverables for people to workspace/storage/ (kept and downloadable); write working text for later steps to outputs/; " +
|
|
868
1255
|
"what the next run needs goes to workspace/state/. Paths are relative to your agent directory, or start with workspace/ " +
|
|
869
|
-
"for the workspace root (
|
|
1256
|
+
"for the workspace root (workspace/storage/ and the older ../../storage/ are the same place). There is no /tmp for you: a " +
|
|
870
1257
|
"directory a tool made outside the workspace, such as a checkout, is reached only through that tool.");
|
|
871
1258
|
// The run's one-line summary is the first meaningful line of the last step
|
|
872
1259
|
// that produced a result — read, never declared (runSummary in store.ts).
|
|
@@ -888,15 +1275,49 @@ identity = {}) {
|
|
|
888
1275
|
const disabled = [];
|
|
889
1276
|
const unknownTools = [];
|
|
890
1277
|
const shadowed = [];
|
|
1278
|
+
// `web: {search: …, fetch: …}` name who answers. Unset means ours — the
|
|
1279
|
+
// account's own engine, and our own fetch, both inside the run's sandbox
|
|
1280
|
+
// and both on the run record. A model provider's name adds the SDK's
|
|
1281
|
+
// server-side tool beside ours, which the run's endpoint executes: point
|
|
1282
|
+
// the run at z.ai and z.ai's index answers the search. The web tool hands
|
|
1283
|
+
// the action over to it (FOLDRUN_WEB_BUILTIN).
|
|
1284
|
+
const providerWebTools = [];
|
|
1285
|
+
if (browseChoice.error)
|
|
1286
|
+
providerWarnings.push(browseChoice.error);
|
|
1287
|
+
providerWarnings.push(...sessionProblems, ...web.problems, ...actionChoices.flatMap((c) => (c.error ? [c.error] : [])));
|
|
1288
|
+
for (const [choice, builtin] of [
|
|
1289
|
+
[searchChoice, "WebSearch"],
|
|
1290
|
+
[fetchChoice, "WebFetch"],
|
|
1291
|
+
]) {
|
|
1292
|
+
if (choice.error) {
|
|
1293
|
+
providerWarnings.push(choice.error);
|
|
1294
|
+
}
|
|
1295
|
+
else if (choice.provider && choice.shape !== "direct") {
|
|
1296
|
+
// A provider's server-side tool replaces ours. A direct search API
|
|
1297
|
+
// does not: our tool stays, and reads FOLDRUN_WEB_SEARCH_VIA to know
|
|
1298
|
+
// whose index to ask — through the egress proxy, with the customer's
|
|
1299
|
+
// own key, on the run record.
|
|
1300
|
+
providerWebTools.push(builtin);
|
|
1301
|
+
}
|
|
1302
|
+
}
|
|
891
1303
|
let wantSearch = false;
|
|
892
1304
|
let wantHistory = false;
|
|
893
1305
|
let wantDesks = false;
|
|
1306
|
+
let wantAsk = false;
|
|
894
1307
|
for (const ref of toolRefs(front)) {
|
|
895
1308
|
const toolName = ref.name;
|
|
896
1309
|
if (ref.mode === "ask") {
|
|
897
1310
|
disabled.push(toolName);
|
|
898
1311
|
continue;
|
|
899
1312
|
}
|
|
1313
|
+
// `web` is our own tool, granted with the rest of them above. A model
|
|
1314
|
+
// provider answering search or fetch on its own side (`web: {search:
|
|
1315
|
+
// zai}`) adds that provider's tool beside it; the web tool's search then
|
|
1316
|
+
// names it (FOLDRUN_WEB_BUILTIN) rather than asking our engine.
|
|
1317
|
+
if (toolName === "web" && !ref.linked) {
|
|
1318
|
+
allowed.push(...providerWebTools);
|
|
1319
|
+
continue;
|
|
1320
|
+
}
|
|
900
1321
|
// Resolution order: built-in group, exact SDK name, then your own tools.
|
|
901
1322
|
// Built-ins win a bare name, because their names are a small fixed set —
|
|
902
1323
|
// but a tool hidden by one is worth saying out loud rather than silently
|
|
@@ -910,13 +1331,15 @@ identity = {}) {
|
|
|
910
1331
|
if (available[toolName])
|
|
911
1332
|
shadowed.push(toolName);
|
|
912
1333
|
}
|
|
913
|
-
else if (toolName === "search" || toolName === "history" || toolName === "desks") {
|
|
1334
|
+
else if (toolName === "search" || toolName === "history" || toolName === "desks" || toolName === "ask") {
|
|
914
1335
|
// The platform's own groups, served in-process — built below, once
|
|
915
1336
|
// the roots and the records are known.
|
|
916
1337
|
if (toolName === "search")
|
|
917
1338
|
wantSearch = true;
|
|
918
1339
|
else if (toolName === "history")
|
|
919
1340
|
wantHistory = true;
|
|
1341
|
+
else if (toolName === "ask")
|
|
1342
|
+
wantAsk = true;
|
|
920
1343
|
else
|
|
921
1344
|
wantDesks = true;
|
|
922
1345
|
if (available[toolName])
|
|
@@ -946,8 +1369,8 @@ identity = {}) {
|
|
|
946
1369
|
{ label: "memory/", dir: path.join(agentDir, "memory") },
|
|
947
1370
|
{ label: "../../knowledge/", dir: path.join(wsRoot, "knowledge") },
|
|
948
1371
|
{ label: "../../memory/", dir: path.join(wsRoot, "memory") },
|
|
949
|
-
{ label: "
|
|
950
|
-
{ label:
|
|
1372
|
+
{ label: "workspace/state/", dir: path.join(wsRoot, "state") },
|
|
1373
|
+
{ label: `workspace/${STORAGE_DIR}/`, dir: path.join(wsRoot, STORAGE_DIR) },
|
|
951
1374
|
{ label: "outputs/", dir: path.join(agentDir, "outputs") },
|
|
952
1375
|
{ label: "[library] knowledge/", dir: libraryDir(tenant, "knowledge") },
|
|
953
1376
|
{ label: "[library] memory/", dir: libraryDir(tenant, "memory") },
|
|
@@ -969,6 +1392,14 @@ identity = {}) {
|
|
|
969
1392
|
const historyTools = wantHistory ? buildHistoryTools(historyDigest) : { server: null, toolNames: [], promptLines: [] };
|
|
970
1393
|
const deskTools = wantDesks ? buildDeskTools(deskDigest) : { server: null, toolNames: [], promptLines: [] };
|
|
971
1394
|
allowed.push(...searchTools.toolNames, ...historyTools.toolNames, ...deskTools.toolNames);
|
|
1395
|
+
// tools: [ask] — ask_person. Its server is built where the step runs (it
|
|
1396
|
+
// needs the step's emit, and in a pod the proxy's address); here only the
|
|
1397
|
+
// grant, the wait and the prompt's line.
|
|
1398
|
+
const ask = wantAsk ? { timeoutSec: askTimeoutSec(front) } : null;
|
|
1399
|
+
if (ask) {
|
|
1400
|
+
allowed.push(ASK_TOOL);
|
|
1401
|
+
parts.push(`# Asking a person\n\n${askPromptLine(ask.timeoutSec)}`);
|
|
1402
|
+
}
|
|
972
1403
|
if (searchTools.promptLines.length || historyTools.promptLines.length || deskTools.promptLines.length) {
|
|
973
1404
|
parts.push(`# Finding things\n\n${[...searchTools.promptLines, ...historyTools.promptLines, ...deskTools.promptLines].join("\n")}`);
|
|
974
1405
|
}
|
|
@@ -988,12 +1419,16 @@ identity = {}) {
|
|
|
988
1419
|
return {
|
|
989
1420
|
front,
|
|
990
1421
|
clockEnv,
|
|
1422
|
+
/** The zone, where it came from, and anything unreadable on the way —
|
|
1423
|
+
* reported by the step, not here, so it lands in the run's trace. */
|
|
1424
|
+
clock,
|
|
991
1425
|
systemPrompt: parts.join("\n\n"),
|
|
992
1426
|
allowed: finalAllowed,
|
|
993
1427
|
disabled,
|
|
994
1428
|
unknownTools,
|
|
995
1429
|
shadowed,
|
|
996
1430
|
legacyUse,
|
|
1431
|
+
retiredTools: retiredToolNames(front),
|
|
997
1432
|
mcpServers,
|
|
998
1433
|
mcpNames,
|
|
999
1434
|
apiTools,
|
|
@@ -1003,19 +1438,21 @@ identity = {}) {
|
|
|
1003
1438
|
// than using the servers built here.
|
|
1004
1439
|
apiSpecs: apis,
|
|
1005
1440
|
scriptSpecs,
|
|
1441
|
+
outward: outward || scriptSpecs.some((sc) => sc.outward),
|
|
1006
1442
|
searchRoots,
|
|
1007
1443
|
historyDigest,
|
|
1008
1444
|
deskDigest,
|
|
1009
1445
|
searchTools,
|
|
1010
1446
|
historyTools,
|
|
1011
1447
|
deskTools,
|
|
1448
|
+
ask,
|
|
1012
1449
|
// `spec` is the MERGED declaration — the agent's own block or its
|
|
1013
1450
|
// workspace's, plus every granted tool's. The isolated path builds its
|
|
1014
1451
|
// runtime from this; passing the agent's frontmatter alone silently drops
|
|
1015
1452
|
// what the tools declared, and a tool whose program needs a package it
|
|
1016
1453
|
// asked for by name fails inside the container with "cannot find" while
|
|
1017
1454
|
// the host path, which uses this same merged spec, works fine.
|
|
1018
|
-
runtime: { log: runtimeLog, error: runtimeError, executor, spec: runtimeSpec },
|
|
1455
|
+
runtime: { log: runtimeLog, error: runtimeError, executor, spec: runtimeSpec, dispose: runtime.dispose },
|
|
1019
1456
|
secretEnv: { ...runtime.env, ...secretEnv },
|
|
1020
1457
|
formatWarning: checkFormatVersion(workspaceFrontmatter(agentDir, tenant).foldrun_version).warning,
|
|
1021
1458
|
providerEnv,
|
|
@@ -1023,8 +1460,19 @@ identity = {}) {
|
|
|
1023
1460
|
providerSecrets,
|
|
1024
1461
|
providerWarnings,
|
|
1025
1462
|
fallbackEnv,
|
|
1026
|
-
|
|
1027
|
-
|
|
1463
|
+
region,
|
|
1464
|
+
// Who answers web search and fetch, resolved once up top. The step
|
|
1465
|
+
// needs them again to grant a direct API's key to its host.
|
|
1466
|
+
searchChoice,
|
|
1467
|
+
fetchChoice,
|
|
1468
|
+
actionChoices,
|
|
1469
|
+
web,
|
|
1470
|
+
language,
|
|
1471
|
+
// Stamped with the step's own zone: when one of these refuses, the
|
|
1472
|
+
// reset time it reports is told in the calendar the step works to,
|
|
1473
|
+
// not in the pod's UTC.
|
|
1474
|
+
translator: translator && { ...translator, timezone },
|
|
1475
|
+
fallbackTranslator: fallbackTranslator && { ...fallbackTranslator, timezone },
|
|
1028
1476
|
apiWarnings,
|
|
1029
1477
|
secretScopes,
|
|
1030
1478
|
missingTools,
|
|
@@ -1053,9 +1501,9 @@ identity = {}) {
|
|
|
1053
1501
|
* Never fails a step: a link is bookkeeping, and a run that did its work must
|
|
1054
1502
|
* not be marked failed because a URL could not be minted.
|
|
1055
1503
|
*/
|
|
1056
|
-
function publishPublicDir(tenant, workspace, push) {
|
|
1504
|
+
async function publishPublicDir(tenant, workspace, push) {
|
|
1057
1505
|
try {
|
|
1058
|
-
const { added } = platform.syncPublicShares(tenant, workspace);
|
|
1506
|
+
const { added } = await platform.syncPublicShares(tenant, workspace);
|
|
1059
1507
|
if (added.length)
|
|
1060
1508
|
push("info", `shared: ${added.join(", ")}`);
|
|
1061
1509
|
}
|
|
@@ -1063,15 +1511,112 @@ function publishPublicDir(tenant, workspace, push) {
|
|
|
1063
1511
|
push("error", `shares: ${err instanceof Error ? err.message : String(err)}`);
|
|
1064
1512
|
}
|
|
1065
1513
|
}
|
|
1066
|
-
function repriced(catalog, model, usage, sdkCost, push
|
|
1514
|
+
export function repriced(catalog, model, usage, sdkCost, push,
|
|
1515
|
+
/** The step's turns each priced at its own model — set when a sub-agent
|
|
1516
|
+
* ran on a different model (ExecOutcome.turnsCostUsd). The lump `usage`
|
|
1517
|
+
* priced at the step's model would bill those turns at the wrong rate. */
|
|
1518
|
+
turnsCostUsd) {
|
|
1067
1519
|
const known = usage ? catalogCost(catalog, model, usage) : null;
|
|
1068
1520
|
if (known === null)
|
|
1069
1521
|
return sdkCost;
|
|
1522
|
+
if (typeof turnsCostUsd === "number") {
|
|
1523
|
+
push("info", `cost from each model turn at its own model's rate: $${turnsCostUsd.toFixed(6)} (sub-agents ran on other models)`);
|
|
1524
|
+
return turnsCostUsd;
|
|
1525
|
+
}
|
|
1070
1526
|
if (sdkCost !== null && Math.abs(known - sdkCost) > 0.000001) {
|
|
1071
1527
|
push("info", `cost repriced from the gateway's catalogue: $${known.toFixed(6)} (sdk said $${sdkCost.toFixed(6)})`);
|
|
1072
1528
|
}
|
|
1073
1529
|
return known;
|
|
1074
1530
|
}
|
|
1531
|
+
/**
|
|
1532
|
+
* Write one attempt onto the step's history and re-total the step.
|
|
1533
|
+
*
|
|
1534
|
+
* runStep reports the attempt it just ran — its cost, tokens and sandbox
|
|
1535
|
+
* seconds — by assigning the step's fields. Those become this attempt's row;
|
|
1536
|
+
* the step's fields then become the sum of every row, which is what the
|
|
1537
|
+
* run's spend, the flow's budget: and the platform's bill read. A retried
|
|
1538
|
+
* step used to carry only its last attempt's cost, so a `retry: 2` step
|
|
1539
|
+
* could spend three shares while reporting one.
|
|
1540
|
+
*/
|
|
1541
|
+
export function recordAttempt(step,
|
|
1542
|
+
/** The step's attempt (`attempts`) these rows belong to. */
|
|
1543
|
+
attempt, status, startedAt, eventsBefore,
|
|
1544
|
+
/** The attempt re-attached to a sandbox a previous driver left: same
|
|
1545
|
+
* attempt, so its row is replaced rather than repeated. */
|
|
1546
|
+
reattach = false) {
|
|
1547
|
+
const tries = step.tries ?? [];
|
|
1548
|
+
const lastError = step.events
|
|
1549
|
+
.slice(eventsBefore)
|
|
1550
|
+
.filter((e) => e.type === "error")
|
|
1551
|
+
.at(-1)?.text;
|
|
1552
|
+
// Re-attached: the same attempt, so its rows — the last one, and the lost
|
|
1553
|
+
// slim go before it — are replaced, not repeated.
|
|
1554
|
+
const kept = [...tries];
|
|
1555
|
+
const attemptOf = (t) => t.attempt ?? t.n;
|
|
1556
|
+
if (reattach && kept.length && attemptOf(kept.at(-1)) === attempt) {
|
|
1557
|
+
kept.pop();
|
|
1558
|
+
if (kept.length && attemptOf(kept.at(-1)) === attempt && kept.at(-1).status === "lost")
|
|
1559
|
+
kept.pop();
|
|
1560
|
+
}
|
|
1561
|
+
// Tries are numbered in the order they ran: a full re-run after a lost
|
|
1562
|
+
// slim go is the next try, not the same number again (live, 2026-10-01,
|
|
1563
|
+
// both rows read n: 1). `attempt` says which attempt a row belongs to
|
|
1564
|
+
// when its number no longer does.
|
|
1565
|
+
let next = (kept.at(-1)?.n ?? 0) + 1;
|
|
1566
|
+
const numbered = () => {
|
|
1567
|
+
const n = next++;
|
|
1568
|
+
return n === attempt ? { n } : { n, attempt };
|
|
1569
|
+
};
|
|
1570
|
+
// A slim go the browser pod was lost under, before this attempt re-ran
|
|
1571
|
+
// on the full image: its own row, first (runStep set it aside).
|
|
1572
|
+
const lost = step.podLostTry ? [{ ...numbered(), ...step.podLostTry, startedAt }] : [];
|
|
1573
|
+
const row = {
|
|
1574
|
+
...numbered(),
|
|
1575
|
+
status,
|
|
1576
|
+
costUsd: step.costUsd,
|
|
1577
|
+
tokens: step.tokens ?? null,
|
|
1578
|
+
computeSecs: step.computeSecs ?? null,
|
|
1579
|
+
startedAt: step.podLostTry?.finishedAt ?? startedAt,
|
|
1580
|
+
finishedAt: step.finishedAt ?? new Date().toISOString(),
|
|
1581
|
+
...(status === "failed" && lastError ? { error: lastError.slice(0, 500) } : {}),
|
|
1582
|
+
...(step.image?.variant ? { image: step.image.variant } : {}),
|
|
1583
|
+
...(step.browserPod ? { browserPod: browserPodLine(step.browserPod) } : {}),
|
|
1584
|
+
};
|
|
1585
|
+
delete step.podLostTry;
|
|
1586
|
+
step.tries = [...kept, ...lost, row];
|
|
1587
|
+
const totals = sumAttempts(step.tries);
|
|
1588
|
+
step.costUsd = totals.costUsd;
|
|
1589
|
+
step.tokens = totals.tokens;
|
|
1590
|
+
step.computeSecs = totals.computeSecs;
|
|
1591
|
+
}
|
|
1592
|
+
/** A question an agent asked mid-step goes where approvals go (the
|
|
1593
|
+
* workspace's notify:, `awaiting-approval` events), with the way to answer. */
|
|
1594
|
+
async function notifyQuestion(tenant, workspace, runId, agent, question, options) {
|
|
1595
|
+
const origin = (process.env.FOLDRUN_PUBLIC_URL ?? "").replace(/\/+$/, "");
|
|
1596
|
+
const where = origin ? `${origin}/dashboard/${encodeURIComponent(workspace)}/runs/${encodeURIComponent(runId)}${tenant !== "default" ? `?tenant=${encodeURIComponent(tenant)}` : ""}` : `run ${runId}`;
|
|
1597
|
+
await sendPlainNotification(tenant, workspace, {
|
|
1598
|
+
event: "awaiting-approval",
|
|
1599
|
+
headline: `${agent} is asking you something`,
|
|
1600
|
+
detail: `${question}${options?.length ? `\n\nOptions: ${options.join(" · ")}` : ""}\n\nAnswer it on ${where}, or \`foldrun answer ${runId} "…"\`.`,
|
|
1601
|
+
runId,
|
|
1602
|
+
}).catch(() => false);
|
|
1603
|
+
}
|
|
1604
|
+
/** The local stand-in for a person: the terminal, when there is one. */
|
|
1605
|
+
async function terminalAsk(question, options) {
|
|
1606
|
+
if (!process.stdin.isTTY || !process.stdout.isTTY)
|
|
1607
|
+
return null;
|
|
1608
|
+
const { createInterface } = await import("node:readline/promises");
|
|
1609
|
+
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
|
1610
|
+
try {
|
|
1611
|
+
const list = options?.length ? `\n${options.map((o, i) => ` ${i + 1}. ${o}`).join("\n")}\n` : "";
|
|
1612
|
+
const raw = (await rl.question(`\n the agent asks: ${question}${list}\n your answer: `)).trim();
|
|
1613
|
+
const n = Number(raw);
|
|
1614
|
+
return options?.length && Number.isInteger(n) && n >= 1 && n <= options.length ? options[n - 1] : raw || null;
|
|
1615
|
+
}
|
|
1616
|
+
finally {
|
|
1617
|
+
rl.close();
|
|
1618
|
+
}
|
|
1619
|
+
}
|
|
1075
1620
|
async function runStep(agentDir, tenant, step, context, save, modelOverride, tags = [], effortOverride, runId,
|
|
1076
1621
|
/** What the previous group returned as DATA (an `output: json` step's
|
|
1077
1622
|
* value), beside the prose in `context`. Undefined when nothing did. */
|
|
@@ -1079,7 +1624,23 @@ priorData,
|
|
|
1079
1624
|
/** The most this step may spend before it is stopped mid-turn: the run's
|
|
1080
1625
|
* remaining `budget:`, shared across the steps launched with it. Null
|
|
1081
1626
|
* means the flow set no budget. */
|
|
1082
|
-
stepBudgetUsd = null
|
|
1627
|
+
stepBudgetUsd = null,
|
|
1628
|
+
/** Which line set that ceiling, for the error that names it. */
|
|
1629
|
+
stepBudgetNote,
|
|
1630
|
+
/** The run is a test run — see test-mode.ts. Read off the run record by
|
|
1631
|
+
* the caller, never from anything the sandbox could have written. */
|
|
1632
|
+
testRun = false,
|
|
1633
|
+
/** Has a person asked for the run to stop? Read on a clock by the
|
|
1634
|
+
* in-process model loop, so a stop lands mid-step and not at the next
|
|
1635
|
+
* group. The isolated path needs none: stopRun destroys the sandbox. */
|
|
1636
|
+
stopRequested,
|
|
1637
|
+
/** The flow's `timezone:`, for the clock cascade. Undefined for a step
|
|
1638
|
+
* run outside any flow, which still gets agent → workspace → account. */
|
|
1639
|
+
flowTimezone,
|
|
1640
|
+
/** The step's index in run.steps, carried on its egress lease so a
|
|
1641
|
+
* person's message to this step reaches this step's own inbox and not a
|
|
1642
|
+
* sibling's running in the same group. Undefined outside a flow. */
|
|
1643
|
+
stepIndex) {
|
|
1083
1644
|
// Secret values are injected into scripts as environment variables and
|
|
1084
1645
|
// substituted into API headers, so a model that reads one back — from a
|
|
1085
1646
|
// script that prints it, an API error that echoes it, or its own `env` —
|
|
@@ -1097,14 +1658,44 @@ stepBudgetUsd = null) {
|
|
|
1097
1658
|
// cover a script's own stdout.
|
|
1098
1659
|
let redactions = [];
|
|
1099
1660
|
let fileDir = null;
|
|
1661
|
+
let disposeRuntime;
|
|
1662
|
+
// Can this step's tools act outside the workspace? Set once its context is
|
|
1663
|
+
// read; the verdict at the end of this function reads it.
|
|
1664
|
+
let canActOutward = false;
|
|
1665
|
+
const eventsAtStart = step.events.length;
|
|
1100
1666
|
const redact = (text) => {
|
|
1101
1667
|
let out = text;
|
|
1102
1668
|
for (const [value, name] of redactions)
|
|
1103
1669
|
out = out.split(value).join(`[redacted:${name}]`);
|
|
1104
1670
|
return out;
|
|
1105
1671
|
};
|
|
1672
|
+
// The reply, the conclusion and an `output: json` value land on the same
|
|
1673
|
+
// record the events do, and a model that quotes a script's stdout puts
|
|
1674
|
+
// the credential in its answer rather than in a tool event. Same scrub,
|
|
1675
|
+
// same place. The JSON value goes through its text form: a secret inside
|
|
1676
|
+
// a nested field is still a secret, and the key name survives either way.
|
|
1677
|
+
const redactText = (text) => (typeof text === "string" ? redact(text) : text);
|
|
1678
|
+
const redactData = (value) => {
|
|
1679
|
+
if (value === undefined || redactions.length === 0)
|
|
1680
|
+
return value;
|
|
1681
|
+
try {
|
|
1682
|
+
return JSON.parse(redact(JSON.stringify(value)));
|
|
1683
|
+
}
|
|
1684
|
+
catch {
|
|
1685
|
+
return value;
|
|
1686
|
+
}
|
|
1687
|
+
};
|
|
1106
1688
|
const push = (type, text, extra) => {
|
|
1107
|
-
|
|
1689
|
+
const t = new Date().toISOString();
|
|
1690
|
+
step.events.push({ t, type, text: redact(text), ...extra });
|
|
1691
|
+
// A person in the loop: the question, its answer, a message delivered —
|
|
1692
|
+
// onto the step, so the run page, the API and history show them.
|
|
1693
|
+
if (extra?.operator) {
|
|
1694
|
+
applyOperatorEvent(step, extra.operator, t);
|
|
1695
|
+
if (extra.operator.kind === "asked" && runId && !testRun) {
|
|
1696
|
+
void notifyQuestion(tenant, path.basename(path.resolve(agentDir, "..", "..")), runId, step.agent, extra.operator.question, extra.operator.options);
|
|
1697
|
+
}
|
|
1698
|
+
}
|
|
1108
1699
|
save();
|
|
1109
1700
|
};
|
|
1110
1701
|
// The clock on the step itself. `??=` so a retry extends the first
|
|
@@ -1122,7 +1713,9 @@ stepBudgetUsd = null) {
|
|
|
1122
1713
|
? fs.readFileSync(stubFile, "utf8").split(/\n---\n/)
|
|
1123
1714
|
: [`stub result from ${step.agent}`];
|
|
1124
1715
|
const call = step.events.filter((e) => e.text.startsWith("stub call")).length;
|
|
1125
|
-
|
|
1716
|
+
// The ceiling it was given rides on the line, so the budget arithmetic
|
|
1717
|
+
// around fan-out is testable without a model to stop.
|
|
1718
|
+
push("info", `stub call ${call + 1}${stepBudgetUsd !== null ? ` (ceiling $${stepBudgetUsd.toFixed(2)})` : ""}`);
|
|
1126
1719
|
step.status = "completed";
|
|
1127
1720
|
// A stub answer may open with `cost: 0.25` — a pretend price, so the
|
|
1128
1721
|
// parts of orchestration that watch money (budget:) are testable too.
|
|
@@ -1140,8 +1733,15 @@ stepBudgetUsd = null) {
|
|
|
1140
1733
|
// the failure of a step that returned no JSON are testable at no cost.
|
|
1141
1734
|
if (step.output === "json") {
|
|
1142
1735
|
const extracted = extractJson(step.result);
|
|
1143
|
-
if (extracted.ok)
|
|
1736
|
+
if (extracted.ok) {
|
|
1144
1737
|
step.data = extracted.value;
|
|
1738
|
+
const schema = step.schema ?? (step.schemaPath ? readSchemaFile(path.resolve(agentDir, "..", ".."), step.schemaPath) : null);
|
|
1739
|
+
const errors = schema && looksLikeSchema(schema) ? validateSchema(step.data, schema) : [];
|
|
1740
|
+
if (errors.length) {
|
|
1741
|
+
push("error", `schema: the value does not fit — ${describeSchemaErrors(errors)}`);
|
|
1742
|
+
step.status = "failed";
|
|
1743
|
+
}
|
|
1744
|
+
}
|
|
1145
1745
|
else {
|
|
1146
1746
|
push("error", `output: json — ${extracted.reason}`);
|
|
1147
1747
|
step.status = "failed";
|
|
@@ -1158,7 +1758,9 @@ stepBudgetUsd = null) {
|
|
|
1158
1758
|
// step: the generic HTTP tool still works without the typed ones.
|
|
1159
1759
|
for (const w of await prefetchOpenApi(tenant, openApiSources(agentDir, tenant)))
|
|
1160
1760
|
push("error", `openapi: ${w}`);
|
|
1161
|
-
const { front, clockEnv, systemPrompt, allowed, disabled, apiTools, scriptTools, secretEnv, secretScopes, missingSecrets, missingTools, runtime, unknownTools, shadowed, legacyUse, mcpServers, mcpNames, apiSpecs, scriptSpecs, brokenTools, size: agentSize, providerEnv, providerLabel, providerSecrets, providerWarnings, formatWarning, fallbackEnv, apiWarnings, searchRoots, historyDigest, deskDigest, searchTools, historyTools, deskTools, translator, fallbackTranslator, } = agentContext(agentDir, tenant, tags, { runId, agent: step.agent });
|
|
1761
|
+
const { front, clockEnv, clock, systemPrompt, allowed, disabled, apiTools, scriptTools, secretEnv, secretScopes, missingSecrets, missingTools, runtime, unknownTools, shadowed, legacyUse, retiredTools, mcpServers, mcpNames, apiSpecs, scriptSpecs, brokenTools, size: agentSize, providerEnv, providerLabel, providerSecrets, providerWarnings, formatWarning, searchChoice, fetchChoice, actionChoices, web, language, region, fallbackEnv, apiWarnings, searchRoots, historyDigest, deskDigest, searchTools, historyTools, deskTools, ask, translator, fallbackTranslator, outward, } = agentContext(agentDir, tenant, tags, { runId, agent: step.agent }, flowTimezone);
|
|
1762
|
+
disposeRuntime = runtime.dispose;
|
|
1763
|
+
canActOutward = outward;
|
|
1162
1764
|
// A retry that moved the step up a class (an evicted or OOM-killed
|
|
1163
1765
|
// attempt) wins over the agent's own `size:` for the attempts after.
|
|
1164
1766
|
const size = step.sizeUp ?? agentSize;
|
|
@@ -1168,7 +1770,7 @@ stepBudgetUsd = null) {
|
|
|
1168
1770
|
// what crosses into containers, headers and env files is the token. A
|
|
1169
1771
|
// failed refresh fails the step naming the secret and the provider's
|
|
1170
1772
|
// reason, which beats a 401 three layers later.
|
|
1171
|
-
const liveSecrets = await materializeSecrets(secretEnv);
|
|
1773
|
+
const liveSecrets = await materializeSecrets(secretEnv, { tenant, workspace: path.basename(workspaceRootOf(agentDir)) });
|
|
1172
1774
|
// @file values (SSH keys, certs) still carry their content at this point.
|
|
1173
1775
|
// The in-process branch materialises them to 0600 paths on this host
|
|
1174
1776
|
// below; the container branch passes the content across and the driver
|
|
@@ -1181,18 +1783,31 @@ stepBudgetUsd = null) {
|
|
|
1181
1783
|
.map(([name, value]) => [value, name])
|
|
1182
1784
|
// Longest first, so a value containing another is replaced whole.
|
|
1183
1785
|
.sort((a, b) => b[0].length - a[0].length);
|
|
1786
|
+
// One line for the clock: which zone this step works to, and which level
|
|
1787
|
+
// set it. "Why is this dated yesterday" should be answerable from the run.
|
|
1788
|
+
push("info", `clock: ${clock.timezone} (${clock.source}) — today is ${clockEnv.FOLDRUN_DATE}`);
|
|
1789
|
+
// And one more only when something written could not be read as a zone.
|
|
1790
|
+
for (const problem of clock.problems)
|
|
1791
|
+
push("error", problem);
|
|
1184
1792
|
if (providerLabel)
|
|
1185
1793
|
push("info", `provider: ${providerLabel}`);
|
|
1186
1794
|
for (const w of providerWarnings)
|
|
1187
1795
|
push("error", w);
|
|
1796
|
+
// A level that wrote a language nobody can read was skipped, not obeyed.
|
|
1797
|
+
for (const l of language.lines)
|
|
1798
|
+
push("info", l);
|
|
1799
|
+
for (const l of region.lines)
|
|
1800
|
+
push("info", l);
|
|
1188
1801
|
for (const w of apiWarnings)
|
|
1189
1802
|
push("error", `openapi: ${w}`);
|
|
1190
1803
|
if (formatWarning)
|
|
1191
1804
|
push("error", formatWarning);
|
|
1192
1805
|
if (legacyUse.length)
|
|
1193
1806
|
push("error", legacyUseError(legacyUse));
|
|
1807
|
+
for (const t of retiredTools)
|
|
1808
|
+
push("error", retiredToolError(t));
|
|
1194
1809
|
for (const t of unknownTools) {
|
|
1195
|
-
push("error", `tools: "${t}" is not a tool group (
|
|
1810
|
+
push("error", `tools: "${t}" is not a tool group (read, write, code, search, history, desks), an SDK tool name, or a tool in this workspace's library — nothing was granted for it`);
|
|
1196
1811
|
}
|
|
1197
1812
|
for (const name of mcpNames)
|
|
1198
1813
|
push("info", `mcp server connected: ${name}`);
|
|
@@ -1301,12 +1916,20 @@ stepBudgetUsd = null) {
|
|
|
1301
1916
|
fs.mkdirSync(path.join(agentDir, "memory"), { recursive: true });
|
|
1302
1917
|
// Expose the workspace's shared scripts/ as ./shared so every agent reaches
|
|
1303
1918
|
// them by the same path, without each one carrying a copy.
|
|
1304
|
-
// No symlinks into the agent directory — they would recurse
|
|
1305
|
-
// workspace's own file listing. Shared paths are resolved by the
|
|
1306
|
-
// tool layer and described to the agent below
|
|
1919
|
+
// No permanent symlinks into the agent directory — they would recurse
|
|
1920
|
+
// into the workspace's own file listing. Shared paths are resolved by the
|
|
1921
|
+
// script tool layer and described to the agent below; the one link there
|
|
1922
|
+
// is, `workspace` → the root for the shell, lives only for the length of
|
|
1923
|
+
// a step (executeStep, confine.ts#linkWorkspace).
|
|
1307
1924
|
// The confinement boundary: agents in a workspace are one team, but the
|
|
1308
1925
|
// workspace is where isolation is enforced.
|
|
1309
1926
|
const workspaceRoot = path.resolve(agentDir, "..", "..");
|
|
1927
|
+
// Where a test run's state/ and storage/ writes go, and the line each
|
|
1928
|
+
// one leaves: under the run, beside its archived outputs.
|
|
1929
|
+
const divertFor = (id) => ({
|
|
1930
|
+
to: path.join(workspaceRoot, "runs", id, TEST_WRITES_DIR),
|
|
1931
|
+
note: (rel, summary) => push("info", `test: ${summary}`, { effect: { kind: "diverted", path: rel, summary } }),
|
|
1932
|
+
});
|
|
1310
1933
|
let prompt = step.instruction
|
|
1311
1934
|
? resolveDocLinks(step.instruction, workspaceRoot)
|
|
1312
1935
|
: "Begin your run now, following your instructions.";
|
|
@@ -1323,11 +1946,18 @@ stepBudgetUsd = null) {
|
|
|
1323
1946
|
if (step.output === "json") {
|
|
1324
1947
|
// The contract the runner parses back out — stated in the prompt so
|
|
1325
1948
|
// the instruction and the parser agree by construction, like delegate:.
|
|
1949
|
+
// The schema, when there is one, is quoted whole: the model that has
|
|
1950
|
+
// to satisfy it should not have to guess it from the field names.
|
|
1951
|
+
const schema = step.schema ?? (step.schemaPath ? readSchemaFile(path.resolve(agentDir, "..", ".."), step.schemaPath) : null);
|
|
1326
1952
|
prompt +=
|
|
1327
1953
|
`\n\n<output>\nThis step returns data. End your reply with exactly one JSON value inside a ` +
|
|
1328
1954
|
"```json fenced block — the value the next step will compute on. Prose before it is " +
|
|
1329
1955
|
`welcome (lead with a one-line headline); nothing may follow the block. If the instruction ` +
|
|
1330
|
-
`names fields, use exactly those names
|
|
1956
|
+
`names fields, use exactly those names.` +
|
|
1957
|
+
(schema && looksLikeSchema(schema)
|
|
1958
|
+
? ` The value must satisfy this JSON Schema, and the step fails if it does not:\n${JSON.stringify(schema, null, 2).slice(0, 20_000)}`
|
|
1959
|
+
: "") +
|
|
1960
|
+
`\n</output>`;
|
|
1331
1961
|
}
|
|
1332
1962
|
if (step.waitFor === "event" && step.eventPayload !== undefined) {
|
|
1333
1963
|
// Not resolved for links, like the previous step's results: it is
|
|
@@ -1377,14 +2007,40 @@ stepBudgetUsd = null) {
|
|
|
1377
2007
|
for (const m of missingConsults) {
|
|
1378
2008
|
push("error", `agents: "${m}" is not an agent in this workspace — no consult tool granted`);
|
|
1379
2009
|
}
|
|
1380
|
-
const consultNames =
|
|
2010
|
+
const consultNames = [
|
|
2011
|
+
...mcpNames,
|
|
2012
|
+
...(consults.length ? ["foldrun_agents"] : []),
|
|
2013
|
+
...(ask ? ["foldrun_ask"] : []),
|
|
2014
|
+
];
|
|
2015
|
+
// Colleagues this agent may DELEGATE to: each with its own context and
|
|
2016
|
+
// its own tools — resolved exactly as its own step's would be, then cut
|
|
2017
|
+
// to what THIS step holds (subagents.ts). Built host-side, like consults.
|
|
2018
|
+
const subagents = buildSubagentSpecs({
|
|
2019
|
+
names: front.subagents, self: step.agent, workspaceRoot, tenant, tags, runId, flowTimezone,
|
|
2020
|
+
parentAllowed: [...allowed, ...consultNames.map((n) => `mcp__${n}`)], push,
|
|
2021
|
+
});
|
|
2022
|
+
if (subagents.length) {
|
|
2023
|
+
// The delegation tool itself, granted only when there is someone to
|
|
2024
|
+
// delegate to.
|
|
2025
|
+
allowed.push(DELEGATE_TOOL);
|
|
2026
|
+
for (const s of subagents)
|
|
2027
|
+
push("info", `subagent: ${s.name} — ${s.tools.length ? s.tools.join(", ") : "no tools"}`);
|
|
2028
|
+
}
|
|
1381
2029
|
// The last provider refusal this step saw, for the fallback decision —
|
|
1382
2030
|
// watched on both paths, because a refusal reads the same from a pod
|
|
1383
2031
|
// and from this process.
|
|
1384
2032
|
let lastRefusal = "";
|
|
2033
|
+
// And WHY it refused, in words, when anything on the way saw the
|
|
2034
|
+
// headers that say so: the translator here, the egress proxy over
|
|
2035
|
+
// there. Kept apart from `lastRefusal` — that one decides whether to
|
|
2036
|
+
// fall back, this one is what a person reads afterwards.
|
|
2037
|
+
let refusalWhy = "";
|
|
1385
2038
|
const pushWatching = (type, text, extra) => {
|
|
1386
2039
|
if (type === "error" && isProviderRefusal(text))
|
|
1387
2040
|
lastRefusal = text;
|
|
2041
|
+
const why = refusalFromLine(text);
|
|
2042
|
+
if (why)
|
|
2043
|
+
refusalWhy = why;
|
|
1388
2044
|
push(type, text, extra);
|
|
1389
2045
|
};
|
|
1390
2046
|
// The second supply: the block's own fallback when it declared one (a
|
|
@@ -1392,6 +2048,51 @@ stepBudgetUsd = null) {
|
|
|
1392
2048
|
// fallback), else the platform's, and only for a step riding the
|
|
1393
2049
|
// platform credential.
|
|
1394
2050
|
const secondSupply = fallbackEnv ?? (Object.keys(providerEnv).length === 0 ? platformFallbackEnv() : null);
|
|
2051
|
+
// Where this step stands on a second supply, for the refusal sentence.
|
|
2052
|
+
// It starts as the truth before anything has run and becomes
|
|
2053
|
+
// "exhausted" the moment the fallback has had its turn — the owner is
|
|
2054
|
+
// about to remove the OpenRouter fallback, so "none" becomes the normal
|
|
2055
|
+
// reading and has to look like a decision on the page.
|
|
2056
|
+
let supplyState = secondSupply ? "trying" : "none";
|
|
2057
|
+
// The line a person reads when this step ends failed: the reason, if
|
|
2058
|
+
// anything on the way learned one, else the status put into words, and
|
|
2059
|
+
// in every case where the step could go next. Pushed last so it is the
|
|
2060
|
+
// last error event — which is what runFailure, the desks' headlines and
|
|
2061
|
+
// `foldrun report` read.
|
|
2062
|
+
const sayWhy = (status) => {
|
|
2063
|
+
if (status !== "failed" || !lastRefusal)
|
|
2064
|
+
return;
|
|
2065
|
+
const said = Number(lastRefusal.match(/API Error: (\d{3})/)?.[1] ?? NaN);
|
|
2066
|
+
const why = refusalWhy ||
|
|
2067
|
+
explainRefusal({
|
|
2068
|
+
status: Number.isFinite(said) && isRefusalStatus(said) ? said : 402,
|
|
2069
|
+
timezone: clock.timezone,
|
|
2070
|
+
});
|
|
2071
|
+
push("error", `${why}${supplyNote(supplyState)}`);
|
|
2072
|
+
};
|
|
2073
|
+
// `schema:` — inline on the step, or a file under the workspace read
|
|
2074
|
+
// now, host-side, so both executors are handed the same document. A
|
|
2075
|
+
// file that cannot be read is a failed step before any model runs:
|
|
2076
|
+
// the contract the author wrote down cannot be checked.
|
|
2077
|
+
const resolvedSchema = step.schema ?? (step.schemaPath ? readSchemaFile(workspaceRoot, step.schemaPath) : null);
|
|
2078
|
+
if (step.schemaPath && resolvedSchema === null) {
|
|
2079
|
+
throw new Error(`schema: ${step.schemaPath} — not a readable JSON or YAML schema under the workspace`);
|
|
2080
|
+
}
|
|
2081
|
+
if (step.maxTurns)
|
|
2082
|
+
push("info", `max_turns: ${step.maxTurns}`);
|
|
2083
|
+
// `limits:` — account, workspace, agent, then this step's option;
|
|
2084
|
+
// nearest wins per key. Enforced in the model loop's hook, on either
|
|
2085
|
+
// path (step-exec.ts), so it is resolved here as plain values.
|
|
2086
|
+
const stepLimits = cascadeLimits([
|
|
2087
|
+
readAgentsMd(accountDir(tenant))?.data?.limits,
|
|
2088
|
+
readAgentsMd(workspaceRootOf(agentDir))?.data?.limits,
|
|
2089
|
+
front.limits,
|
|
2090
|
+
step.limits,
|
|
2091
|
+
]);
|
|
2092
|
+
const limitOwners = toolOwners(apiTools.owners, scriptSpecs);
|
|
2093
|
+
const limitOpts = Object.keys(stepLimits).length ? { limits: stepLimits, toolOwners: limitOwners } : {};
|
|
2094
|
+
if (limitOpts.limits)
|
|
2095
|
+
push("info", `limits: ${Object.entries(stepLimits).map(([k, n]) => `${k} ${n}`).join(", ")} — per step; a call past one is refused`);
|
|
1395
2096
|
const isolation = process.env.FOLDRUN_RUN_ISOLATION;
|
|
1396
2097
|
if (isolatedRun()) {
|
|
1397
2098
|
// The isolated path: the whole loop — model, built-in tools, scripts —
|
|
@@ -1414,24 +2115,44 @@ stepBudgetUsd = null) {
|
|
|
1414
2115
|
// before anything is copied in. Null where no proxy is configured
|
|
1415
2116
|
// (the CLI, a compose install without FOLDRUN_EGRESS_URL), and then
|
|
1416
2117
|
// everything below materialises exactly as it always did.
|
|
1417
|
-
const grant = { secrets: {} };
|
|
2118
|
+
const grant = { secrets: {}, timezone: clock.timezone };
|
|
2119
|
+
// A direct search API's key: granted to that API's host and no other,
|
|
2120
|
+
// exactly as an http tool's key is granted to its base URL's host. The
|
|
2121
|
+
// script sends the placeholder; the worker fills it on the way out.
|
|
2122
|
+
for (const choice of [searchChoice, fetchChoice]) {
|
|
2123
|
+
if (choice.shape === "direct" && choice.secret && choice.host && choice.secret in liveSecrets) {
|
|
2124
|
+
addGrant(grant, choice.secret, liveSecrets[choice.secret], choice.host);
|
|
2125
|
+
}
|
|
2126
|
+
}
|
|
2127
|
+
for (const c of actionChoices) {
|
|
2128
|
+
if (c.provider && c.secret in liveSecrets)
|
|
2129
|
+
addGrant(grant, c.secret, liveSecrets[c.secret], c.host);
|
|
2130
|
+
}
|
|
1418
2131
|
for (const api of apiSpecs) {
|
|
1419
2132
|
const host = hostOf(api.base);
|
|
1420
2133
|
if (!host)
|
|
1421
2134
|
continue;
|
|
1422
2135
|
for (const v of [...Object.values(api.headers), ...Object.values(api.query)]) {
|
|
1423
|
-
for (const name of placeholderNames(v))
|
|
1424
|
-
if (name in liveSecrets)
|
|
1425
|
-
|
|
2136
|
+
for (const name of placeholderNames(v)) {
|
|
2137
|
+
if (!(name in liveSecrets))
|
|
2138
|
+
continue;
|
|
2139
|
+
addGrant(grant, name, liveSecrets[name], host);
|
|
2140
|
+
// Which vault entry this is, so the proxy's record of what the
|
|
2141
|
+
// far end said lands on the right credential.
|
|
2142
|
+
(grant.healthKeys ??= {})[name] = healthKey(name, secretScopes[name] === "workspace" ? "workspace" : "account", path.basename(path.resolve(agentDir, "..", "..")));
|
|
2143
|
+
}
|
|
1426
2144
|
}
|
|
1427
2145
|
}
|
|
1428
|
-
const lease = await platform.egress.lease({ tenant, runId: runId ?? "adhoc", grant });
|
|
2146
|
+
const lease = await platform.egress.lease({ tenant, runId: runId ?? "adhoc", grant, test: testRun, ...(stepIndex !== undefined && stepIndex >= 0 ? { step: stepIndex } : {}) });
|
|
1429
2147
|
// Which secrets must still be real inside the sandbox: a script tool
|
|
1430
2148
|
// reads them from its environment, and unless it says `secrets:
|
|
1431
2149
|
// proxied` it needs the value, not the name. bash is a script by
|
|
1432
2150
|
// another route. The record says which way it went, and why.
|
|
1433
2151
|
const materialisers = [
|
|
1434
2152
|
...scriptSpecs.filter((sc) => sc.secrets !== "proxied").map((sc) => sc.name),
|
|
2153
|
+
// web proxies its keys — unless it may browse: a remote browser's key
|
|
2154
|
+
// goes into a websocket, and cookie and login secrets into the page.
|
|
2155
|
+
...(scriptSpecs.some((sc) => sc.name === "web") && (!web.actions || web.actions.includes("browse")) ? ["web (browse)"] : []),
|
|
1435
2156
|
...(allowed.includes("Bash") ? ["bash"] : []),
|
|
1436
2157
|
];
|
|
1437
2158
|
const materialise = !lease || materialisers.length > 0;
|
|
@@ -1445,14 +2166,35 @@ stepBudgetUsd = null) {
|
|
|
1445
2166
|
}
|
|
1446
2167
|
// Sandbox env for the declared secrets: the value, or its own name as
|
|
1447
2168
|
// a placeholder that the proxy fills on the way out.
|
|
1448
|
-
const
|
|
1449
|
-
|
|
1450
|
-
|
|
2169
|
+
const placeholders = Object.fromEntries(declared.map((name) => [name, `\${${name}}`]));
|
|
2170
|
+
let sandboxSecrets = materialise ? liveSecrets : placeholders;
|
|
2171
|
+
// A test run hands the sandbox no send-capable value. The env var is
|
|
2172
|
+
// still there, reading TEST_MODE_WITHHELD, so a script that never
|
|
2173
|
+
// checked FOLDRUN_TEST_MODE fails at the provider's door with an auth
|
|
2174
|
+
// error — and sends nothing. The script tools say whether the step
|
|
2175
|
+
// has a declared sender (everything withheld) or a declared reader
|
|
2176
|
+
// (nothing withheld); see test-mode.ts for the order.
|
|
2177
|
+
if (testRun && materialise) {
|
|
2178
|
+
const held = withholdSecrets(liveSecrets, {
|
|
2179
|
+
outward: scriptSpecs.some((sc) => sc.outward),
|
|
2180
|
+
allow: scriptSpecs.some((sc) => sc.testMode === "allow"),
|
|
2181
|
+
});
|
|
2182
|
+
sandboxSecrets = held.env;
|
|
2183
|
+
for (const name of held.withheld) {
|
|
2184
|
+
push("info", `test: ${name} withheld from the sandbox — TEST_MODE_WITHHELD`, { effect: { kind: "withheld", summary: `${name} withheld from the sandbox` } });
|
|
2185
|
+
}
|
|
2186
|
+
}
|
|
2187
|
+
// On a test run the http tools always go through the proxy as
|
|
2188
|
+
// placeholders, even when a script made the step materialise: the
|
|
2189
|
+
// proxy is where the outward-write policy lives, and a header
|
|
2190
|
+
// carrying TEST_MODE_WITHHELD would only fail where it should have
|
|
2191
|
+
// been sunk or refused.
|
|
2192
|
+
const apiValues = testRun && lease ? placeholders : sandboxSecrets;
|
|
1451
2193
|
const substitutedApis = apiSpecs.map((api) => ({
|
|
1452
2194
|
...api,
|
|
1453
2195
|
headers: Object.fromEntries(Object.entries(api.headers).map(([k, v]) => [
|
|
1454
2196
|
k,
|
|
1455
|
-
v.replace(/\$\{([A-Z][A-Z0-9_]*)\}/g, (whole, name) =>
|
|
2197
|
+
v.replace(/\$\{([A-Z][A-Z0-9_]*)\}/g, (whole, name) => apiValues[name] ?? whole),
|
|
1456
2198
|
])),
|
|
1457
2199
|
}));
|
|
1458
2200
|
// The provider's own key and headers, when the step has a lease: the
|
|
@@ -1480,8 +2222,9 @@ stepBudgetUsd = null) {
|
|
|
1480
2222
|
const isolatedArgs = (modelEnv, keyName = MODEL_KEY_NAME) => ({
|
|
1481
2223
|
workspaceRoot,
|
|
1482
2224
|
libraryRoot: libraryDir(tenant),
|
|
2225
|
+
galleryRoot: platform.galleryDir() ?? undefined,
|
|
1483
2226
|
// Keys this step's dependency cache. Per-account by design — see
|
|
1484
|
-
// run-container.ts#
|
|
2227
|
+
// run-container.ts#runtimeCacheMount for why it is not shared.
|
|
1485
2228
|
tenant,
|
|
1486
2229
|
input: {
|
|
1487
2230
|
agentRel: path.relative(workspaceRoot, agentDir).replaceAll("\\", "/"),
|
|
@@ -1498,12 +2241,21 @@ stepBudgetUsd = null) {
|
|
|
1498
2241
|
// field. A tool's own `runtime:` is part of what this step needs.
|
|
1499
2242
|
runtime: runtime.spec,
|
|
1500
2243
|
consults,
|
|
2244
|
+
...(subagents.length ? { subagents } : {}),
|
|
1501
2245
|
timeoutSec: step.timeout,
|
|
1502
2246
|
budgetUsd: stepBudgetUsd,
|
|
2247
|
+
budgetNote: stepBudgetNote,
|
|
1503
2248
|
price: tokenPrice,
|
|
1504
2249
|
verify: step.verify,
|
|
1505
2250
|
output: step.output,
|
|
2251
|
+
schema: resolvedSchema ?? undefined,
|
|
2252
|
+
maxTurns: step.maxTurns,
|
|
2253
|
+
...limitOpts,
|
|
1506
2254
|
translator: keyName === MODEL_KEY_NAME ? primaryTranslator : secondTranslator,
|
|
2255
|
+
// Outward steps keep a browser of their own: an executor that
|
|
2256
|
+
// could browse from the slim image through the account's pod
|
|
2257
|
+
// leaves them on the full one (browser-pod.ts).
|
|
2258
|
+
...(outward ? { outward: true } : {}),
|
|
1507
2259
|
// The container sees the workspace at /workspace and the library
|
|
1508
2260
|
// at /library; the roots are named host-side and moved here.
|
|
1509
2261
|
search: searchRoots.map((r) => ({
|
|
@@ -1514,6 +2266,7 @@ stepBudgetUsd = null) {
|
|
|
1514
2266
|
})),
|
|
1515
2267
|
history: historyDigest,
|
|
1516
2268
|
desks: deskDigest,
|
|
2269
|
+
...(ask ? { ask } : {}),
|
|
1517
2270
|
},
|
|
1518
2271
|
env: Object.fromEntries(Object.entries({
|
|
1519
2272
|
// The platform's own model credential, when the agent names no
|
|
@@ -1532,12 +2285,16 @@ stepBudgetUsd = null) {
|
|
|
1532
2285
|
FOLDRUN_RUN_ID: runId,
|
|
1533
2286
|
FOLDRUN_AGENT: step.agent,
|
|
1534
2287
|
FOLDRUN_WORKSPACE: path.basename(workspaceRoot),
|
|
2288
|
+
...(testRun ? testModeEnv() : {}),
|
|
1535
2289
|
...sandboxSecrets,
|
|
1536
2290
|
}).filter((entry) => typeof entry[1] === "string")),
|
|
1537
2291
|
emit: pushWatching,
|
|
1538
2292
|
// Stamps the sandbox, so stopping this run can destroy it.
|
|
1539
2293
|
runId,
|
|
1540
2294
|
size,
|
|
2295
|
+
// A test run's state/ and storage/ changes come out under the run.
|
|
2296
|
+
divert: testRun && runId ? divertFor(runId) : undefined,
|
|
2297
|
+
test: testRun,
|
|
1541
2298
|
// Resume: a sandbox a previous driver left running, recorded on the
|
|
1542
2299
|
// step. Consumed once — a fresh attempt after this one (a retry, a
|
|
1543
2300
|
// fallback) starts its own sandbox.
|
|
@@ -1559,8 +2316,39 @@ stepBudgetUsd = null) {
|
|
|
1559
2316
|
// attempt holds its own sandbox, so the meter owes the sum.
|
|
1560
2317
|
const firstArgs = isolatedArgs(platformModelEnv());
|
|
1561
2318
|
await lease?.commit();
|
|
1562
|
-
let
|
|
2319
|
+
let credentialUsed = platformModelCredential();
|
|
2320
|
+
// The arguments of the attempt whose outcome is current — what a
|
|
2321
|
+
// re-run on the full image repeats, on the same model supply.
|
|
2322
|
+
let lastArgs = firstArgs;
|
|
2323
|
+
const runTracked = (a) => {
|
|
2324
|
+
lastArgs = a;
|
|
2325
|
+
return runIsolated(a);
|
|
2326
|
+
};
|
|
2327
|
+
let outcome = await runTracked(firstArgs);
|
|
1563
2328
|
step.sandbox = null; // whatever happens next starts its own
|
|
2329
|
+
// A 401 from a token that rotated mid-step is not a broken key: the
|
|
2330
|
+
// replacement is already on its way to the mounted file. Wait for it
|
|
2331
|
+
// once, then fall into the same retry the busy case uses.
|
|
2332
|
+
if (outcome.status === "failed" && isAuthRefusal(lastRefusal)) {
|
|
2333
|
+
push("info", `the model credential was refused mid-step — waiting for the refreshed one`);
|
|
2334
|
+
const rotated = await awaitRotatedCredential(credentialUsed);
|
|
2335
|
+
if (rotated) {
|
|
2336
|
+
push("info", "the credential rotated under this step; retrying on the new one");
|
|
2337
|
+
credentialUsed = rotated;
|
|
2338
|
+
lastRefusal = "";
|
|
2339
|
+
const previous = outcome.timing;
|
|
2340
|
+
// Build first: isolatedArgs is what re-grants the model key, and
|
|
2341
|
+
// commit() is what publishes the grant to Redis, where the proxy
|
|
2342
|
+
// reads it. Committing first sent the revoked token again — every
|
|
2343
|
+
// rotation retry on 2026-09-22/23 401'd this way.
|
|
2344
|
+
const retry = isolatedArgs(platformModelEnv());
|
|
2345
|
+
await lease?.commit();
|
|
2346
|
+
outcome = withEarlierTiming(await runTracked(retry), previous);
|
|
2347
|
+
}
|
|
2348
|
+
else {
|
|
2349
|
+
push("info", "the credential did not change — this is the key itself, not a rotation");
|
|
2350
|
+
}
|
|
2351
|
+
}
|
|
1564
2352
|
for (let n = 1; n <= OVERLOAD_RETRIES && outcome.status === "failed" && isTransientOverload(lastRefusal); n++) {
|
|
1565
2353
|
const wait = backoffMs(n, lastRefusal);
|
|
1566
2354
|
push("info", `provider busy (${lastRefusal.slice(0, 60)}) — waiting ${(wait / 1000).toFixed(1)}s, attempt ${n} of ${OVERLOAD_RETRIES}`);
|
|
@@ -1571,7 +2359,7 @@ stepBudgetUsd = null) {
|
|
|
1571
2359
|
const previous = outcome.timing;
|
|
1572
2360
|
const again = isolatedArgs(platformModelEnv());
|
|
1573
2361
|
await lease?.commit();
|
|
1574
|
-
outcome = await
|
|
2362
|
+
outcome = await runTracked(again);
|
|
1575
2363
|
outcome = withEarlierTiming(outcome, previous);
|
|
1576
2364
|
}
|
|
1577
2365
|
// The second supply, tried exactly once, and only when the primary
|
|
@@ -1594,29 +2382,81 @@ stepBudgetUsd = null) {
|
|
|
1594
2382
|
.filter((entry) => typeof entry[1] === "string")),
|
|
1595
2383
|
};
|
|
1596
2384
|
await lease?.commit();
|
|
1597
|
-
|
|
2385
|
+
supplyState = "exhausted";
|
|
2386
|
+
outcome = await runTracked(fallbackArgs);
|
|
1598
2387
|
// Both attempts held sandboxes; the meter owes the sum. The first
|
|
1599
2388
|
// try's pod ran, was billed for by the platform, and must not
|
|
1600
2389
|
// vanish from the record because a second try replaced its outcome.
|
|
1601
2390
|
outcome = withEarlierTiming(outcome, first);
|
|
1602
2391
|
}
|
|
2392
|
+
// Slim browsing (browser-pod.ts): the step ran on the slim image and
|
|
2393
|
+
// the account's browser pod went away under it, past the web tool's
|
|
2394
|
+
// reconnects. Reads only — run it again from the start on the full
|
|
2395
|
+
// image, where a browser in the step is always there. Anything
|
|
2396
|
+
// written — fail it, naming what, and leave retry:/on-fail:/a person
|
|
2397
|
+
// to decide; a second go could send or charge twice.
|
|
2398
|
+
step.image = outcome.image ?? null;
|
|
2399
|
+
step.browserPod = outcome.browserPod ? { ...outcome.browserPod } : null;
|
|
2400
|
+
if (outcome.status === "failed" && outcome.browserPod?.lost) {
|
|
2401
|
+
const decision = podLossDecision(outcome.browserPod);
|
|
2402
|
+
if (decision.rerun) {
|
|
2403
|
+
push("info", decision.note);
|
|
2404
|
+
const first = outcome;
|
|
2405
|
+
// The slim go is a try of its own on the record — image slim,
|
|
2406
|
+
// status lost, its own cost, tokens and sandbox seconds — so the
|
|
2407
|
+
// run page and `foldrun report` show two tries, slim (lost) then
|
|
2408
|
+
// full, and the step's totals are the sum of both rows
|
|
2409
|
+
// (recordAttempt). Live, 2026-10-01, the step showed only the
|
|
2410
|
+
// full re-run.
|
|
2411
|
+
step.podLostTry = {
|
|
2412
|
+
status: "lost",
|
|
2413
|
+
costUsd: repriced(catalog, wireModel, first.usage ?? null, first.costUsd, push, first.turnsCostUsd),
|
|
2414
|
+
tokens: first.usage ? { input: first.usage.inputTokens, output: first.usage.outputTokens } : null,
|
|
2415
|
+
computeSecs: first.timing ? Math.round(first.timing.totalMs) / 1000 : null,
|
|
2416
|
+
finishedAt: new Date().toISOString(),
|
|
2417
|
+
error: decision.note.slice(0, 500),
|
|
2418
|
+
image: "slim",
|
|
2419
|
+
browserPod: browserPodLine(first.browserPod),
|
|
2420
|
+
};
|
|
2421
|
+
const again = { ...lastArgs, input: { ...lastArgs.input, image: "full" }, resume: null };
|
|
2422
|
+
await lease?.commit();
|
|
2423
|
+
outcome = await runTracked(again);
|
|
2424
|
+
step.image = outcome.image ?? { variant: "full", why: "re-run after the browser pod was lost" };
|
|
2425
|
+
step.browserPod = { ...first.browserPod, fallback: RERAN_ON_FULL };
|
|
2426
|
+
}
|
|
2427
|
+
else {
|
|
2428
|
+
push("error", decision.message);
|
|
2429
|
+
step.browserPod = { ...outcome.browserPod, failure: decision.message };
|
|
2430
|
+
}
|
|
2431
|
+
}
|
|
1603
2432
|
if (lease) {
|
|
1604
|
-
for (const line of await lease.drainLog())
|
|
1605
|
-
|
|
2433
|
+
for (const line of await lease.drainLog()) {
|
|
2434
|
+
// pushWatching, not push: a line the proxy wrote may carry the
|
|
2435
|
+
// refusal mark, and that sentence is the whole reason the proxy
|
|
2436
|
+
// reads response headers at all.
|
|
2437
|
+
if (typeof line === "string")
|
|
2438
|
+
pushWatching("info", line);
|
|
2439
|
+
else
|
|
2440
|
+
pushWatching("info", line.text, { effect: line.effect });
|
|
2441
|
+
}
|
|
1606
2442
|
await lease.release();
|
|
1607
2443
|
}
|
|
1608
|
-
|
|
2444
|
+
// Nothing a test run left in storage/public/ reached storage/, so
|
|
2445
|
+
// there is nothing to publish — and a share link is an outward act.
|
|
2446
|
+
if (!testRun)
|
|
2447
|
+
await publishPublicDir(tenant, path.basename(workspaceRoot), push);
|
|
2448
|
+
sayWhy(outcome.status);
|
|
1609
2449
|
step.status = outcome.status;
|
|
1610
|
-
step.result = outcome.result;
|
|
1611
|
-
step.conclusion = outcome.conclusion;
|
|
2450
|
+
step.result = redactText(outcome.result);
|
|
2451
|
+
step.conclusion = redactText(outcome.conclusion);
|
|
1612
2452
|
step.sandbox = null;
|
|
1613
2453
|
// The cluster's word on why a sandbox ended without the driver saying
|
|
1614
2454
|
// so — OOMKilled, Evicted — on the record, where the retry reads it.
|
|
1615
2455
|
if (outcome.status === "failed" && outcome.reason)
|
|
1616
2456
|
push("error", `sandbox ended: ${outcome.reason}`);
|
|
1617
2457
|
if (outcome.data !== undefined)
|
|
1618
|
-
step.data = outcome.data;
|
|
1619
|
-
step.costUsd = repriced(catalog, wireModel, outcome.usage ?? null, outcome.costUsd, push);
|
|
2458
|
+
step.data = redactData(outcome.data);
|
|
2459
|
+
step.costUsd = repriced(catalog, wireModel, outcome.usage ?? null, outcome.costUsd, push, outcome.turnsCostUsd);
|
|
1620
2460
|
step.tokens = outcome.usage
|
|
1621
2461
|
? { input: outcome.usage.inputTokens, output: outcome.usage.outputTokens }
|
|
1622
2462
|
: null;
|
|
@@ -1646,15 +2486,32 @@ stepBudgetUsd = null) {
|
|
|
1646
2486
|
step.reserved = { cpus: Number(lim.cpus) || 2, memGiB: gib };
|
|
1647
2487
|
}
|
|
1648
2488
|
else {
|
|
1649
|
-
|
|
2489
|
+
// Same withholding as the sandbox path — the in-process step is what
|
|
2490
|
+
// `foldrun run --test` drives on a laptop, where there is no proxy
|
|
2491
|
+
// and this is the only guard there is.
|
|
2492
|
+
const held = testRun
|
|
2493
|
+
? withholdSecrets(liveSecrets, {
|
|
2494
|
+
outward: scriptSpecs.some((sc) => sc.outward),
|
|
2495
|
+
allow: scriptSpecs.some((sc) => sc.testMode === "allow"),
|
|
2496
|
+
})
|
|
2497
|
+
: { env: liveSecrets, withheld: [] };
|
|
2498
|
+
for (const name of held.withheld) {
|
|
2499
|
+
push("info", `test: ${name} withheld from scripts — TEST_MODE_WITHHELD`, { effect: { kind: "withheld", summary: `${name} withheld from scripts` } });
|
|
2500
|
+
}
|
|
2501
|
+
const stepSecrets = held.env;
|
|
2502
|
+
const consultTools = buildConsultTools(consults, { ...hostSafeEnv(), ...stepSecrets, ...providerEnv }, (type, text) => push(type, text));
|
|
2503
|
+
// On this machine nobody can write in and there is no proxy: the
|
|
2504
|
+
// question goes to the terminal when there is one, else unanswered.
|
|
2505
|
+
const askTools = ask ? buildAskTool({ channel: null, timeoutSec: ask.timeoutSec, emit: push, local: terminalAsk }) : null;
|
|
1650
2506
|
// @file secrets become 0600 paths here (host run), cleaned up in the
|
|
1651
2507
|
// finally — once, however many attempts follow.
|
|
1652
|
-
const mat = materializeFileSecrets(agentDir,
|
|
2508
|
+
const mat = materializeFileSecrets(agentDir, stepSecrets);
|
|
1653
2509
|
fileDir = mat.dir;
|
|
1654
2510
|
const attemptInProcess = (modelEnv) => executeStep({
|
|
1655
2511
|
agentDir,
|
|
1656
2512
|
workspaceRoot,
|
|
1657
2513
|
libraryRoot: libraryDir(tenant),
|
|
2514
|
+
...(subagents.length ? { subagents } : {}),
|
|
1658
2515
|
prompt,
|
|
1659
2516
|
model,
|
|
1660
2517
|
effort,
|
|
@@ -1668,16 +2525,25 @@ stepBudgetUsd = null) {
|
|
|
1668
2525
|
...(searchTools.server ? { foldrun_search: searchTools.server } : {}),
|
|
1669
2526
|
...(historyTools.server ? { foldrun_history: historyTools.server } : {}),
|
|
1670
2527
|
...(deskTools.server ? { foldrun_desks: deskTools.server } : {}),
|
|
2528
|
+
...(askTools ? { foldrun_ask: askTools.server } : {}),
|
|
1671
2529
|
...mcpServers,
|
|
1672
2530
|
},
|
|
1673
2531
|
// Declared secrets reach the agent's scripts as env vars; the model
|
|
1674
|
-
// only ever sees the variable names, not the values.
|
|
1675
|
-
|
|
2532
|
+
// only ever sees the variable names, not the values. The base is
|
|
2533
|
+
// the allowlisted host environment, not process.env: the SDK's
|
|
2534
|
+
// Bash tool runs with this, and on a platform process.env holds
|
|
2535
|
+
// every tenant's root key and the datastore credentials.
|
|
2536
|
+
env: { ...hostSafeEnv(), ...clockEnv, ...(testRun ? testModeEnv() : {}), ...mat.env, ...modelEnv },
|
|
1676
2537
|
timeoutSec: step.timeout,
|
|
1677
2538
|
budgetUsd: stepBudgetUsd,
|
|
2539
|
+
budgetNote: stepBudgetNote,
|
|
1678
2540
|
price: tokenPrice,
|
|
1679
2541
|
verify: step.verify,
|
|
1680
2542
|
output: step.output,
|
|
2543
|
+
schema: resolvedSchema ?? undefined,
|
|
2544
|
+
maxTurns: step.maxTurns,
|
|
2545
|
+
...limitOpts,
|
|
2546
|
+
stopRequested,
|
|
1681
2547
|
// What the scripts saw, the verify sees: a flow can then check that
|
|
1682
2548
|
// the proof a step left names THIS run, not one that came before —
|
|
1683
2549
|
// copy-back never propagates a deletion, so a marker from an earlier
|
|
@@ -1687,26 +2553,52 @@ stepBudgetUsd = null) {
|
|
|
1687
2553
|
...(runId ? { FOLDRUN_RUN_ID: runId } : {}),
|
|
1688
2554
|
FOLDRUN_AGENT: step.agent,
|
|
1689
2555
|
FOLDRUN_WORKSPACE: path.basename(path.resolve(agentDir, "..", "..")),
|
|
1690
|
-
...
|
|
2556
|
+
...(testRun ? testModeEnv() : {}),
|
|
2557
|
+
...stepSecrets,
|
|
1691
2558
|
},
|
|
1692
2559
|
emit: pushWatching,
|
|
1693
2560
|
});
|
|
1694
2561
|
// With a translator the step's endpoint is the translator: started
|
|
1695
2562
|
// here, pointed at by the SDK's env, closed when the attempt ends.
|
|
2563
|
+
// A test run brackets each attempt with a snapshot of state/ and
|
|
2564
|
+
// storage/: the step writes them for real, because there is no
|
|
2565
|
+
// sandbox to write instead, and afterwards what changed is moved
|
|
2566
|
+
// under the run and the originals come back.
|
|
1696
2567
|
const attemptThrough = async (modelEnv, spec) => {
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
const t = await startTranslator(spec);
|
|
2568
|
+
const before = testRun ? snapshotDivertedDirs(workspaceRoot) : null;
|
|
2569
|
+
const t = spec ? await startTranslator(spec) : null;
|
|
1700
2570
|
try {
|
|
1701
|
-
return await attemptInProcess({ ...modelEnv, ...t.env });
|
|
2571
|
+
return await attemptInProcess(t ? { ...modelEnv, ...t.env } : modelEnv);
|
|
1702
2572
|
}
|
|
1703
2573
|
finally {
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
2574
|
+
if (t) {
|
|
2575
|
+
for (const line of t.drainLog())
|
|
2576
|
+
pushWatching("info", line.startsWith("translator") ? line : `translator: ${line}`);
|
|
2577
|
+
await t.close();
|
|
2578
|
+
}
|
|
2579
|
+
if (before && runId) {
|
|
2580
|
+
const d = divertFor(runId);
|
|
2581
|
+
restoreDivertedDirs(workspaceRoot, before, d.to, d.note);
|
|
2582
|
+
}
|
|
1707
2583
|
}
|
|
1708
2584
|
};
|
|
2585
|
+
let credentialUsed = platformModelCredential();
|
|
1709
2586
|
let outcome = await attemptThrough(providerEnv, translator);
|
|
2587
|
+
// Same rule as the isolated path: a credential that rotated under the
|
|
2588
|
+
// step is waited out and retried before anything else is considered.
|
|
2589
|
+
if (outcome.status === "failed" && isAuthRefusal(lastRefusal)) {
|
|
2590
|
+
push("info", `the model credential was refused mid-step — waiting for the refreshed one`);
|
|
2591
|
+
const rotated = await awaitRotatedCredential(credentialUsed);
|
|
2592
|
+
if (rotated) {
|
|
2593
|
+
push("info", "the credential rotated under this step; retrying on the new one");
|
|
2594
|
+
credentialUsed = rotated;
|
|
2595
|
+
lastRefusal = "";
|
|
2596
|
+
outcome = await attemptThrough(providerEnv, translator);
|
|
2597
|
+
}
|
|
2598
|
+
else {
|
|
2599
|
+
push("info", "the credential did not change — this is the key itself, not a rotation");
|
|
2600
|
+
}
|
|
2601
|
+
}
|
|
1710
2602
|
// Same rule as the isolated path: a busy provider is waited out on
|
|
1711
2603
|
// the supply it refused from, before any fallback is considered.
|
|
1712
2604
|
for (let n = 1; n <= OVERLOAD_RETRIES && outcome.status === "failed" && isTransientOverload(lastRefusal); n++) {
|
|
@@ -1720,17 +2612,20 @@ stepBudgetUsd = null) {
|
|
|
1720
2612
|
// declared or platform second supply.
|
|
1721
2613
|
if (outcome.status === "failed" && lastRefusal && secondSupply) {
|
|
1722
2614
|
push("info", `primary model supply refused (${lastRefusal.slice(0, 80)}) — retrying this step on the fallback provider`);
|
|
2615
|
+
supplyState = "exhausted";
|
|
1723
2616
|
outcome = await attemptThrough(secondSupply, fallbackEnv && secondSupply === fallbackEnv ? fallbackTranslator : null);
|
|
1724
2617
|
}
|
|
1725
|
-
|
|
2618
|
+
if (!testRun)
|
|
2619
|
+
await publishPublicDir(tenant, path.basename(workspaceRoot), push);
|
|
2620
|
+
sayWhy(outcome.status);
|
|
1726
2621
|
step.status = outcome.status;
|
|
1727
|
-
step.result = outcome.result;
|
|
1728
|
-
step.conclusion = outcome.conclusion;
|
|
2622
|
+
step.result = redactText(outcome.result);
|
|
2623
|
+
step.conclusion = redactText(outcome.conclusion);
|
|
1729
2624
|
if (outcome.data !== undefined)
|
|
1730
|
-
step.data = outcome.data;
|
|
2625
|
+
step.data = redactData(outcome.data);
|
|
1731
2626
|
// A consult's spend belongs to the step that asked.
|
|
1732
2627
|
const consultCost = consultTools.drainCost();
|
|
1733
|
-
const stepCost = repriced(catalog, wireModel, outcome.usage, outcome.costUsd, push);
|
|
2628
|
+
const stepCost = repriced(catalog, wireModel, outcome.usage, outcome.costUsd, push, outcome.turnsCostUsd);
|
|
1734
2629
|
step.costUsd = stepCost === null && consultCost === 0 ? null : (stepCost ?? 0) + consultCost;
|
|
1735
2630
|
step.tokens = outcome.usage
|
|
1736
2631
|
? { input: outcome.usage.inputTokens, output: outcome.usage.outputTokens }
|
|
@@ -1752,9 +2647,34 @@ stepBudgetUsd = null) {
|
|
|
1752
2647
|
catch {
|
|
1753
2648
|
// best-effort — a leftover scratch file is swept with the run's outputs
|
|
1754
2649
|
}
|
|
2650
|
+
try {
|
|
2651
|
+
// A private runtime build (see prepareRuntime) is this step's alone.
|
|
2652
|
+
disposeRuntime?.();
|
|
2653
|
+
}
|
|
2654
|
+
catch {
|
|
2655
|
+
// best-effort — it is in the tmpdir
|
|
2656
|
+
}
|
|
1755
2657
|
}
|
|
1756
2658
|
step.finishedAt = new Date().toISOString();
|
|
1757
2659
|
save();
|
|
2660
|
+
return { actedThenFailedCheck: canActOutward && actedThenFailedCheck(step.events.slice(eventsAtStart)) };
|
|
2661
|
+
}
|
|
2662
|
+
/**
|
|
2663
|
+
* Did this attempt call its tools and then fail only on the check after —
|
|
2664
|
+
* `verify:`, `output: json`, a schema? For a step whose tools act outward
|
|
2665
|
+
* that is the one failure a retry must not answer: the email went, the post
|
|
2666
|
+
* is up, and the check failing says nothing about whether they happened. A
|
|
2667
|
+
* second attempt sends them again. Read off the attempt's own events, which
|
|
2668
|
+
* both executors write the same way (step-exec.ts): every event a failed
|
|
2669
|
+
* check writes carries `check: true`. It used to read only the LAST error's
|
|
2670
|
+
* wording, and a failed shell verify or judge writes its detail after its
|
|
2671
|
+
* headline — so the step was retried and the email went twice. The wording
|
|
2672
|
+
* is still honoured, for a driver image older than the tag.
|
|
2673
|
+
*/
|
|
2674
|
+
export function actedThenFailedCheck(events) {
|
|
2675
|
+
if (!events.some((e) => e.type === "tool"))
|
|
2676
|
+
return false;
|
|
2677
|
+
return events.some((e) => e.type === "error" && (e.check === true || /^(verify `|output: json — |schema: the value does not fit)/.test(e.text)));
|
|
1758
2678
|
}
|
|
1759
2679
|
// Block until a human approves or rejects the pending steps. Polls the run
|
|
1760
2680
|
// record — the same file the dashboard writes through the approval API — so
|
|
@@ -1855,6 +2775,53 @@ function platformFallbackEnv() {
|
|
|
1855
2775
|
}
|
|
1856
2776
|
return out;
|
|
1857
2777
|
}
|
|
2778
|
+
/**
|
|
2779
|
+
* A 401 the token refresh caused, rather than a key that is simply wrong.
|
|
2780
|
+
*
|
|
2781
|
+
* The refresher on the box turns the long-lived grant into an 8h access
|
|
2782
|
+
* token every four hours, and Anthropic revokes the old one the instant the
|
|
2783
|
+
* new one is minted. The platform reads the credential per step from the
|
|
2784
|
+
* mounted file, which fixed the boot-time copy — but not this: a step
|
|
2785
|
+
* already talking to the API when the rotation lands gets a hard 401
|
|
2786
|
+
* mid-turn, and the replacement does not appear in the mounted file until
|
|
2787
|
+
* the kubelet syncs it, about a minute later.
|
|
2788
|
+
*
|
|
2789
|
+
* That window is narrow and it is not rare. Every 401 on this account in
|
|
2790
|
+
* three weeks of runs fell inside it: 02:07:22 and 02:07:24 against a 02:07
|
|
2791
|
+
* refresh, 22:08:12, 22:08:17, 22:08:30 and 22:09:03 against 22:08. The
|
|
2792
|
+
* desks that lost a run nightly were simply the ones scheduled on the hour
|
|
2793
|
+
* the refresher shares.
|
|
2794
|
+
*
|
|
2795
|
+
* So: on a 401, wait for the file to hold a DIFFERENT credential, then let
|
|
2796
|
+
* the caller retry with it. A token that never changes inside the window is
|
|
2797
|
+
* a key that is actually invalid — no waiting fixes that, and the caller
|
|
2798
|
+
* falls through to the second supply exactly as before. Moving the timer
|
|
2799
|
+
* cannot fix this; a run that takes forty minutes spans any minute chosen.
|
|
2800
|
+
*/
|
|
2801
|
+
const CREDENTIAL_ROTATION_WAIT_MS = 90_000;
|
|
2802
|
+
const CREDENTIAL_POLL_MS = 2_000;
|
|
2803
|
+
function isAuthRefusal(text) {
|
|
2804
|
+
return /API Error: 401|access token has (been revoked|expired)/i.test(text);
|
|
2805
|
+
}
|
|
2806
|
+
/**
|
|
2807
|
+
* Wait, briefly, for the mounted credential to become one we have not
|
|
2808
|
+
* already been refused on. Returns the new value, or null when the window
|
|
2809
|
+
* passes without a change — which is the answer that means "this key is
|
|
2810
|
+
* wrong", not "this key is stale".
|
|
2811
|
+
*/
|
|
2812
|
+
export async function awaitRotatedCredential(before, now = Date.now, read = () => platformModelCredential(), waitMs = CREDENTIAL_ROTATION_WAIT_MS, pollMs = CREDENTIAL_POLL_MS) {
|
|
2813
|
+
if (!before)
|
|
2814
|
+
return null;
|
|
2815
|
+
const until = now() + waitMs;
|
|
2816
|
+
for (;;) {
|
|
2817
|
+
const current = read();
|
|
2818
|
+
if (current && current !== before)
|
|
2819
|
+
return current;
|
|
2820
|
+
if (now() >= until)
|
|
2821
|
+
return null;
|
|
2822
|
+
await sleep(pollMs);
|
|
2823
|
+
}
|
|
2824
|
+
}
|
|
1858
2825
|
/** A model-call failure the fallback can answer: the primary refusing over
|
|
1859
2826
|
* credit, quota or auth — never a failure of the work itself, which would
|
|
1860
2827
|
* fail identically anywhere. */
|
|
@@ -1865,7 +2832,17 @@ function isProviderRefusal(text) {
|
|
|
1865
2832
|
// fallback it could only time out on again. A refusal the fallback can
|
|
1866
2833
|
// answer is about the ACCOUNT; anything about the work or the sandbox is
|
|
1867
2834
|
// not, however similar the vocabulary.
|
|
1868
|
-
|
|
2835
|
+
// A subscription's own ceiling is the same kind of refusal, worded
|
|
2836
|
+
// differently: Claude Code on a Max plan answers "You've hit your weekly
|
|
2837
|
+
// limit · resets 5am" with no status code and none of the money words.
|
|
2838
|
+
// On 2026-09-21 every opus-tier step on the account failed on exactly that
|
|
2839
|
+
// sentence while sonnet steps beside them ran fine — the plan has a
|
|
2840
|
+
// separate weekly cap for Opus — and nothing here recognised it, so a
|
|
2841
|
+
// second supply could not have answered even if one had been configured.
|
|
2842
|
+
// It is not transient (isTransientOverload is right to ignore it: the
|
|
2843
|
+
// window is a week), it is the account's supply being exhausted, which is
|
|
2844
|
+
// precisely what the fallback provider is for.
|
|
2845
|
+
return /API Error: (401|402|403|429|5\d\d)|credits?\b|quota|billing|insufficient|overloaded|(weekly|daily|monthly|usage) limit|rate.?limit(ed)?\b.*resets?\b|limit\b.*\bresets?\b/i.test(text);
|
|
1869
2846
|
}
|
|
1870
2847
|
/** How many times a busy provider is given another moment, and the longest
|
|
1871
2848
|
* any one wait may be. Three is the industry's habit and the point where
|
|
@@ -1919,7 +2896,48 @@ export function backoffMs(attempt, text = "", random = Math.random) {
|
|
|
1919
2896
|
const base = Math.min(2 ** attempt * 1000, MAX_BACKOFF_MS);
|
|
1920
2897
|
return Math.round(base * (0.5 + random() * 0.5));
|
|
1921
2898
|
}
|
|
1922
|
-
|
|
2899
|
+
/**
|
|
2900
|
+
* The platform's model credential, read at the moment a step needs it.
|
|
2901
|
+
*
|
|
2902
|
+
* It used to come only from this process's environment, which meant the
|
|
2903
|
+
* worker held whatever token it was started with. Refreshing an OAuth token
|
|
2904
|
+
* REVOKES the old one, so a refresh without a restart left every run
|
|
2905
|
+
* answering 401 — on 2026-09-18 that took every desk down for two hours, and
|
|
2906
|
+
* nothing said so until a person read a failed run. The refresher's own
|
|
2907
|
+
* comment had named the fix years of incidents earlier: stop shipping the
|
|
2908
|
+
* token as an env var.
|
|
2909
|
+
*
|
|
2910
|
+
* So: when `FOLDRUN_MODEL_KEY_FILE` names a file (the Kubernetes secret,
|
|
2911
|
+
* mounted rather than injected), it is read fresh every time. The kubelet
|
|
2912
|
+
* updates a mounted secret in place within about a minute of it changing, so
|
|
2913
|
+
* a refreshed token reaches the next step on its own, with no restart, no
|
|
2914
|
+
* timing window, and nothing to skip while work is in flight.
|
|
2915
|
+
*
|
|
2916
|
+
* A missing or empty file falls back to the environment, which is what a
|
|
2917
|
+
* local install, the CLI and every compose setup use.
|
|
2918
|
+
*/
|
|
2919
|
+
export function platformModelCredential(env = process.env) {
|
|
2920
|
+
const file = env.FOLDRUN_MODEL_KEY_FILE;
|
|
2921
|
+
if (file) {
|
|
2922
|
+
try {
|
|
2923
|
+
const value = fs.readFileSync(file, "utf8").trim();
|
|
2924
|
+
if (value)
|
|
2925
|
+
return value;
|
|
2926
|
+
}
|
|
2927
|
+
catch {
|
|
2928
|
+
// Unreadable is not the same as absent: say so once, then fall back,
|
|
2929
|
+
// because a running platform that cannot read its credential file is a
|
|
2930
|
+
// thing an operator needs to know about before the next step fails.
|
|
2931
|
+
if (!credentialFileWarned) {
|
|
2932
|
+
credentialFileWarned = true;
|
|
2933
|
+
console.warn(`[model] FOLDRUN_MODEL_KEY_FILE=${file} could not be read; falling back to the environment`);
|
|
2934
|
+
}
|
|
2935
|
+
}
|
|
2936
|
+
}
|
|
2937
|
+
return env.CLAUDE_CODE_OAUTH_TOKEN || undefined;
|
|
2938
|
+
}
|
|
2939
|
+
let credentialFileWarned = false;
|
|
2940
|
+
export function platformModelEnv() {
|
|
1923
2941
|
const { ANTHROPIC_API_KEY, ANTHROPIC_AUTH_TOKEN, ANTHROPIC_BASE_URL, CLAUDE_CODE_OAUTH_TOKEN,
|
|
1924
2942
|
// What the tiers mean when the platform's own provider is not Anthropic.
|
|
1925
2943
|
// These belong to the credential: a base URL pointing at OpenRouter with
|
|
@@ -1931,7 +2949,9 @@ function platformModelEnv() {
|
|
|
1931
2949
|
ANTHROPIC_API_KEY,
|
|
1932
2950
|
ANTHROPIC_AUTH_TOKEN,
|
|
1933
2951
|
ANTHROPIC_BASE_URL,
|
|
1934
|
-
|
|
2952
|
+
// The file wins when there is one: it is the live secret, and this
|
|
2953
|
+
// process's copy is whatever it was handed at boot.
|
|
2954
|
+
CLAUDE_CODE_OAUTH_TOKEN: platformModelCredential() ?? CLAUDE_CODE_OAUTH_TOKEN,
|
|
1935
2955
|
ANTHROPIC_DEFAULT_HAIKU_MODEL,
|
|
1936
2956
|
ANTHROPIC_DEFAULT_SONNET_MODEL,
|
|
1937
2957
|
ANTHROPIC_DEFAULT_OPUS_MODEL,
|
|
@@ -1979,21 +2999,34 @@ function jsonItems(data, step, take) {
|
|
|
1979
2999
|
function csvItems(workspaceDir, step, take) {
|
|
1980
3000
|
const raw = step.eachPath ?? "";
|
|
1981
3001
|
if (!raw) {
|
|
1982
|
-
step.skipReason = "each: rows needs a path — `each: rows of
|
|
3002
|
+
step.skipReason = "each: rows needs a path — `each: rows of workspace/storage/x.csv`";
|
|
1983
3003
|
return [];
|
|
1984
3004
|
}
|
|
1985
|
-
const
|
|
1986
|
-
if (
|
|
1987
|
-
step.skipReason = `each: rows — ${
|
|
3005
|
+
const read = csvDataRows(workspaceDir, step.agent, raw);
|
|
3006
|
+
if ("problem" in read) {
|
|
3007
|
+
step.skipReason = `each: rows — ${read.problem}`;
|
|
1988
3008
|
return [];
|
|
1989
3009
|
}
|
|
3010
|
+
return read.data.slice(0, take).map((r) => `${read.header}\n${r}`);
|
|
3011
|
+
}
|
|
3012
|
+
/**
|
|
3013
|
+
* The data rows of a CSV, path agent-relative like every path in a flow and
|
|
3014
|
+
* confined to the workspace. Shared by `each: rows of`, which fans out over
|
|
3015
|
+
* them, and `when: rows of`, which asks only whether there are any.
|
|
3016
|
+
*/
|
|
3017
|
+
function csvDataRows(workspaceDir, agent, raw) {
|
|
3018
|
+
// `workspace/storage/x.csv` from the workspace root, anything else from
|
|
3019
|
+
// the agent's folder (`../../storage/x.csv` is the same file).
|
|
3020
|
+
const resolved = resolveAgentPath(workspaceDir, path.join(workspaceDir, "agents", agent), raw);
|
|
3021
|
+
if (!resolved.startsWith(path.resolve(workspaceDir) + path.sep)) {
|
|
3022
|
+
return { problem: `${raw} is outside this workspace` };
|
|
3023
|
+
}
|
|
1990
3024
|
let text;
|
|
1991
3025
|
try {
|
|
1992
3026
|
text = fs.readFileSync(resolved, "utf8");
|
|
1993
3027
|
}
|
|
1994
3028
|
catch {
|
|
1995
|
-
|
|
1996
|
-
return [];
|
|
3029
|
+
return { problem: `${raw} does not exist (yet?)` };
|
|
1997
3030
|
}
|
|
1998
3031
|
// Minimal CSV: quoted fields with embedded commas/newlines survive.
|
|
1999
3032
|
const rows = [];
|
|
@@ -2027,18 +3060,39 @@ function csvItems(workspaceDir, step, take) {
|
|
|
2027
3060
|
if (row + field)
|
|
2028
3061
|
rows.push(row + field);
|
|
2029
3062
|
const [header, ...data] = rows.filter((r) => r.trim() !== "");
|
|
2030
|
-
if (!header || data.length === 0)
|
|
2031
|
-
|
|
2032
|
-
|
|
3063
|
+
if (!header || data.length === 0)
|
|
3064
|
+
return { problem: `${raw} has no data rows` };
|
|
3065
|
+
return { header, data };
|
|
3066
|
+
}
|
|
3067
|
+
/**
|
|
3068
|
+
* `schema: <path>` — the file, read and parsed, or null. Agent-relative like
|
|
3069
|
+
* every other path in a flow (`workspace/schemas/lead.json`), confined to the
|
|
3070
|
+
* workspace: a schema is the author's document, never something a step
|
|
3071
|
+
* can point outside.
|
|
3072
|
+
*/
|
|
3073
|
+
export function readSchemaFile(workspaceRoot, rel) {
|
|
3074
|
+
const root = path.resolve(workspaceRoot);
|
|
3075
|
+
// Written `workspace/schemas/x.json`, agent-relative (`../../schemas/x.json`)
|
|
3076
|
+
// or workspace-relative (`schemas/x.json`); all land on the same file.
|
|
3077
|
+
const candidates = [path.resolve(root, rel.replace(/^((\.\.\/){2}|workspace\/)/, "")), path.resolve(root, "agents", "x", rel)];
|
|
3078
|
+
for (const abs of candidates) {
|
|
3079
|
+
if (abs !== root && !abs.startsWith(root + path.sep))
|
|
3080
|
+
continue;
|
|
3081
|
+
if (!fs.existsSync(abs) || !fs.statSync(abs).isFile())
|
|
3082
|
+
continue;
|
|
3083
|
+
const parsed = parseSchemaText(fs.readFileSync(abs, "utf8"));
|
|
3084
|
+
return "problem" in parsed ? null : parsed.schema;
|
|
2033
3085
|
}
|
|
2034
|
-
return
|
|
3086
|
+
return null;
|
|
2035
3087
|
}
|
|
2036
|
-
export function createFlowRun(tenant, workspace, steps, flowName, status, tags = []) {
|
|
3088
|
+
export function createFlowRun(tenant, workspace, steps, flowName, status, tags = [], startedBy = null, opts = {}) {
|
|
2037
3089
|
const run = {
|
|
2038
3090
|
id: `run-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`,
|
|
2039
3091
|
flow: flowName,
|
|
2040
3092
|
tags,
|
|
2041
3093
|
status,
|
|
3094
|
+
...(startedBy ? { startedBy: startedBy.trim().toLowerCase() } : {}),
|
|
3095
|
+
...(opts.test ? { test: true } : {}),
|
|
2042
3096
|
startedAt: new Date().toISOString(),
|
|
2043
3097
|
finishedAt: null,
|
|
2044
3098
|
steps: steps.map((s) => ({
|
|
@@ -2068,6 +3122,11 @@ export function createFlowRun(tenant, workspace, steps, flowName, status, tags =
|
|
|
2068
3122
|
preview: s.preview,
|
|
2069
3123
|
delegate: s.delegate,
|
|
2070
3124
|
output: s.output,
|
|
3125
|
+
schema: s.schema,
|
|
3126
|
+
schemaPath: s.schemaPath,
|
|
3127
|
+
parallel: s.parallel,
|
|
3128
|
+
maxTurns: s.maxTurns,
|
|
3129
|
+
limits: s.limits,
|
|
2071
3130
|
attempts: 0,
|
|
2072
3131
|
status: "pending",
|
|
2073
3132
|
events: [],
|
|
@@ -2093,8 +3152,8 @@ export function createFlowRun(tenant, workspace, steps, flowName, status, tags =
|
|
|
2093
3152
|
}
|
|
2094
3153
|
return run;
|
|
2095
3154
|
}
|
|
2096
|
-
export function startFlowRun(tenant, workspace, steps, flowName, modelOverride, tags = [], effortOverride) {
|
|
2097
|
-
const run = createFlowRun(tenant, workspace, steps, flowName, "running", tags);
|
|
3155
|
+
export function startFlowRun(tenant, workspace, steps, flowName, modelOverride, tags = [], effortOverride, startedBy = null, opts = {}) {
|
|
3156
|
+
const run = createFlowRun(tenant, workspace, steps, flowName, "running", tags, startedBy, opts);
|
|
2098
3157
|
void driveRun(tenant, workspace, run, modelOverride, tags, { effortOverride });
|
|
2099
3158
|
return run;
|
|
2100
3159
|
}
|
|
@@ -2202,6 +3261,19 @@ export function driveRun(tenant, workspace, run, modelOverride, tags = [], opts
|
|
|
2202
3261
|
}
|
|
2203
3262
|
function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
|
|
2204
3263
|
const effortOverride = opts.effortOverride ?? null;
|
|
3264
|
+
// The flow's own calendar. Read from the flow file rather than passed in,
|
|
3265
|
+
// so every door into a run — the queue, an eval, the CLI, a rerun, a
|
|
3266
|
+
// reconcile that picks an abandoned run back up — gets the same clock
|
|
3267
|
+
// without each caller having to remember it. A run that belongs to no
|
|
3268
|
+
// flow (an ad-hoc agent run) finds nothing here and cascades from the
|
|
3269
|
+
// agent to the workspace, which is what it should do.
|
|
3270
|
+
let flowTimezone = null;
|
|
3271
|
+
try {
|
|
3272
|
+
flowTimezone = readFlow(tenant, workspace, run.flow)?.timezone ?? null;
|
|
3273
|
+
}
|
|
3274
|
+
catch {
|
|
3275
|
+
// an unreadable flow file is reported where it is parsed for real
|
|
3276
|
+
}
|
|
2205
3277
|
const pDir = workspaceDir(tenant, workspace);
|
|
2206
3278
|
const runRoot = pDir;
|
|
2207
3279
|
/**
|
|
@@ -2381,6 +3453,19 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
|
|
|
2381
3453
|
// is assembled) keeps the newest: when the total is over it, the
|
|
2382
3454
|
// oldest groups are dropped whole before anything is cut mid-text.
|
|
2383
3455
|
const contextBefore = (gi) => joinEarlierGroups(groupResults, gi);
|
|
3456
|
+
// What `when:` and `case:` read: the PREVIOUS group's result alone —
|
|
3457
|
+
// the nearest earlier group that produced one, so a routed-past group
|
|
3458
|
+
// between is looked through rather than read as silence. The prompt
|
|
3459
|
+
// gets every earlier group joined (contextBefore); routing must not,
|
|
3460
|
+
// because a verdict two groups back would still open a gate a later
|
|
3461
|
+
// group had overruled: a classifier's BUG line matched a `case: BUG`
|
|
3462
|
+
// three groups on, after the debugger had already answered it.
|
|
3463
|
+
const routingBefore = (gi) => {
|
|
3464
|
+
for (let i = gi - 1; i >= 0; i--)
|
|
3465
|
+
if (groupResults[i])
|
|
3466
|
+
return groupResults[i];
|
|
3467
|
+
return null;
|
|
3468
|
+
};
|
|
2384
3469
|
// The data half of the same handoff: what the nearest earlier group
|
|
2385
3470
|
// returned through `output: json`. Rebuilt from the record rather than
|
|
2386
3471
|
// kept only in memory, so a resumed run hands step 3 the value step 1
|
|
@@ -2444,8 +3529,11 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
|
|
|
2444
3529
|
const group = ordered[gi];
|
|
2445
3530
|
context = contextBefore(gi);
|
|
2446
3531
|
ctxData = dataBefore(gi);
|
|
2447
|
-
|
|
2448
|
-
|
|
3532
|
+
// What the run has spent, not just what its tokens cost: on an
|
|
3533
|
+
// install that prices sandbox seconds, a step that called no model
|
|
3534
|
+
// still spent money, and a cap that ignored it capped nothing.
|
|
3535
|
+
if (run.budgetUsd && platform.runSpend(run) >= run.budgetUsd && group.some((s) => s.status === "pending")) {
|
|
3536
|
+
const spent = platform.runSpend(run);
|
|
2449
3537
|
const last = run.steps.findLast((s) => s.status === "completed" || s.status === "failed") ?? run.steps[0];
|
|
2450
3538
|
last?.events.push({
|
|
2451
3539
|
t: new Date().toISOString(),
|
|
@@ -2531,6 +3619,7 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
|
|
|
2531
3619
|
t: new Date().toISOString(),
|
|
2532
3620
|
type: "info",
|
|
2533
3621
|
text: `fan-out: ${used.length} instance${used.length === 1 ? "" : "s"}` +
|
|
3622
|
+
(step.parallel && step.parallel < used.length ? `, ${step.parallel} at a time (parallel:)` : "") +
|
|
2534
3623
|
(dropped ? ` (more items existed — capped at ${cap})` : ""),
|
|
2535
3624
|
});
|
|
2536
3625
|
const at = run.steps.indexOf(step);
|
|
@@ -2545,6 +3634,7 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
|
|
|
2545
3634
|
result: null,
|
|
2546
3635
|
costUsd: null,
|
|
2547
3636
|
attempts: 0,
|
|
3637
|
+
tries: undefined,
|
|
2548
3638
|
}));
|
|
2549
3639
|
run.steps.splice(at + 1, 0, ...instances);
|
|
2550
3640
|
ordered = orderedGroups();
|
|
@@ -2557,12 +3647,25 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
|
|
|
2557
3647
|
// this is what keeps the *next* group from starting.
|
|
2558
3648
|
const stopCheck = readRun(tenant, workspace, run.id);
|
|
2559
3649
|
if (stopCheck?.stopRequested) {
|
|
3650
|
+
let skipped = 0;
|
|
2560
3651
|
for (const s of run.steps) {
|
|
2561
3652
|
if (s.status === "pending" || s.status === "running") {
|
|
2562
3653
|
s.status = "skipped";
|
|
2563
3654
|
s.skipReason = "run stopped";
|
|
3655
|
+
skipped += 1;
|
|
2564
3656
|
}
|
|
2565
3657
|
}
|
|
3658
|
+
// Say so on the trail. stopRun wrote "run stopped" onto the
|
|
3659
|
+
// record's first step, but the save() below writes this process's
|
|
3660
|
+
// copy of the record over it, so the only witness was the
|
|
3661
|
+
// stopRequested flag — and a run that "failed" with no error line
|
|
3662
|
+
// anywhere took a code read to explain (rank-desk, 2026-09-13).
|
|
3663
|
+
const witness = run.steps.findLast((s) => s.status === "completed" || s.status === "failed") ?? run.steps[0];
|
|
3664
|
+
witness?.events.push({
|
|
3665
|
+
t: new Date().toISOString(),
|
|
3666
|
+
type: "info",
|
|
3667
|
+
text: `run stopped by a person between groups — ${skipped} remaining step${skipped === 1 ? "" : "s"} skipped`,
|
|
3668
|
+
});
|
|
2566
3669
|
run.status = "failed";
|
|
2567
3670
|
run.stopRequested = true;
|
|
2568
3671
|
save();
|
|
@@ -2597,39 +3700,66 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
|
|
|
2597
3700
|
break;
|
|
2598
3701
|
}
|
|
2599
3702
|
const ctx = context;
|
|
2600
|
-
//
|
|
3703
|
+
// The previous group's result — see routingBefore.
|
|
3704
|
+
const routing = routingBefore(gi);
|
|
3705
|
+
// `when:` — skip a step whose condition isn't met by the previous
|
|
3706
|
+
// group's result.
|
|
2601
3707
|
for (const step of freshGroup) {
|
|
2602
3708
|
if (step.status !== "pending" || !step.when)
|
|
2603
3709
|
continue;
|
|
2604
|
-
|
|
3710
|
+
// `when: rows of <csv>` asks the data, not the prose: run only if
|
|
3711
|
+
// the file has a data row. It is the condition a fan-out's
|
|
3712
|
+
// followers need — an `each:` over an empty file skips itself but
|
|
3713
|
+
// leaves nothing for a marker to be read from, so the steps after
|
|
3714
|
+
// it used to run on nothing.
|
|
3715
|
+
const rowsOf = whenRowsPath(step.when);
|
|
3716
|
+
if (rowsOf !== null) {
|
|
3717
|
+
const read = csvDataRows(pDir, step.agent, rowsOf);
|
|
3718
|
+
if ("problem" in read) {
|
|
3719
|
+
step.status = "skipped";
|
|
3720
|
+
step.skipReason = `condition not met: when rows — ${read.problem}`;
|
|
3721
|
+
step.events.push({ t: new Date().toISOString(), type: "info", text: `skipped — ${read.problem}` });
|
|
3722
|
+
}
|
|
3723
|
+
continue;
|
|
3724
|
+
}
|
|
3725
|
+
// A marker at the start of a line, not a word anywhere in the
|
|
3726
|
+
// prose — see markerPresent. "There are no BLOCKED items" must
|
|
3727
|
+
// not open a gate keyed on BLOCKED.
|
|
3728
|
+
const met = markerPresent(routing, step.when);
|
|
2605
3729
|
if (!met) {
|
|
2606
3730
|
step.status = "skipped";
|
|
2607
3731
|
step.skipReason = `condition not met: when "${step.when}"`;
|
|
2608
3732
|
step.events.push({
|
|
2609
3733
|
t: new Date().toISOString(),
|
|
2610
3734
|
type: "info",
|
|
2611
|
-
text: `skipped — previous
|
|
3735
|
+
text: `skipped — the previous group's result has no line beginning "${step.when}" (when: reads a marker at the start of a line of the previous result, not a word in a sentence)`,
|
|
2612
3736
|
});
|
|
2613
3737
|
}
|
|
2614
3738
|
}
|
|
2615
3739
|
save();
|
|
2616
3740
|
// `case:`/`else:` — exclusive routing. Of this group's case steps,
|
|
2617
|
-
// the FIRST whose
|
|
2618
|
-
// rest are routed past. `else:` runs only when no case matched.
|
|
3741
|
+
// the FIRST whose marker leads a line of the previous results runs;
|
|
3742
|
+
// the rest are routed past. `else:` runs only when no case matched.
|
|
2619
3743
|
// (`when:` above stays independent — every matching when runs —
|
|
2620
3744
|
// which is why routing is its own vocabulary instead of a mode.)
|
|
3745
|
+
//
|
|
3746
|
+
// Same marker rule as `when:`, and it matters more here: routing is
|
|
3747
|
+
// exclusive, so a label matched out of a sentence does not merely
|
|
3748
|
+
// run an extra step, it sends the flow down the wrong branch and
|
|
3749
|
+
// skips the right one. "This is not a COMPLAINT, it is a QUESTION"
|
|
3750
|
+
// used to route to the complaints handler.
|
|
2621
3751
|
const caseSteps = freshGroup.filter((s) => s.status === "pending" && s.case);
|
|
2622
3752
|
if (caseSteps.length || freshGroup.some((s) => s.status === "pending" && s.else)) {
|
|
2623
3753
|
let matchedCase = null;
|
|
2624
3754
|
for (const step of caseSteps) {
|
|
2625
|
-
if (!matchedCase && (
|
|
3755
|
+
if (!matchedCase && markerPresent(routing, step.case)) {
|
|
2626
3756
|
matchedCase = step;
|
|
2627
3757
|
continue;
|
|
2628
3758
|
}
|
|
2629
3759
|
step.status = "skipped";
|
|
2630
3760
|
step.skipReason = matchedCase
|
|
2631
3761
|
? `routed past — "${matchedCase.case}" matched first`
|
|
2632
|
-
: `case not matched: "${step.case}"`;
|
|
3762
|
+
: `case not matched: the previous group's result has no line beginning "${step.case}"`;
|
|
2633
3763
|
step.events.push({
|
|
2634
3764
|
t: new Date().toISOString(),
|
|
2635
3765
|
type: "info",
|
|
@@ -2671,6 +3801,20 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
|
|
|
2671
3801
|
});
|
|
2672
3802
|
}
|
|
2673
3803
|
run.status = "awaiting-approval";
|
|
3804
|
+
// approve_within: — the deadline is stamped when the gate opens,
|
|
3805
|
+
// not counted from the run's start, so "two days to answer" means
|
|
3806
|
+
// two days from being asked however long the work before it took.
|
|
3807
|
+
// Only a gate that asks a PERSON gets one: an external event has
|
|
3808
|
+
// no one to chase.
|
|
3809
|
+
const within = readFlow(tenant, workspace, run.flow)?.approveWithin ?? null;
|
|
3810
|
+
// Always assigned, never only set: a run with two gates would
|
|
3811
|
+
// otherwise carry the FIRST gate's deadline into the second, and
|
|
3812
|
+
// a deadline already in the past expires a gate the moment it
|
|
3813
|
+
// opens. Null when this gate has none.
|
|
3814
|
+
run.approveBy =
|
|
3815
|
+
within && needsApproval.some((s) => s.waitFor !== "event")
|
|
3816
|
+
? new Date(Date.now() + within * 1000).toISOString()
|
|
3817
|
+
: null;
|
|
2674
3818
|
save();
|
|
2675
3819
|
// The gate's declared preview, resolved now against what the run
|
|
2676
3820
|
// has actually produced — a glob at park time, a list of files on
|
|
@@ -2743,9 +3887,50 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
|
|
|
2743
3887
|
// twenty steps over — the shares sum to what was left. Between
|
|
2744
3888
|
// groups the check above still refuses to start the next one.
|
|
2745
3889
|
const launching = freshGroup.filter((s) => s.status === "pending");
|
|
2746
|
-
|
|
3890
|
+
// An agent's own `budget:` — the most it may spend in one run — is
|
|
3891
|
+
// read at launch, so a cap raised mid-run applies to the next step.
|
|
3892
|
+
const agentBudgets = new Map(listAgents(tenant, workspace).map((a) => [a.name, a.budget]));
|
|
3893
|
+
// The ceiling each attempt now running was given. A step records its
|
|
3894
|
+
// cost only when it ends, so while it runs its ceiling stands in for
|
|
3895
|
+
// what it may yet spend — otherwise every fan-out copy of one agent
|
|
3896
|
+
// saw none of its siblings' spend and got the agent's whole budget.
|
|
3897
|
+
const reserved = new Map();
|
|
3898
|
+
// The ceiling for one attempt of one step, read NOW: what is left of
|
|
3899
|
+
// the run's budget shared across the group, or the agent's own cap
|
|
3900
|
+
// less what its steps have cost or hold in flight, shared with its
|
|
3901
|
+
// steps in this group still waiting to start. Computed before every
|
|
3902
|
+
// attempt, not once before the loop — a retry's share is what is left
|
|
3903
|
+
// after the failed attempt spent, and the step's own figure already
|
|
3904
|
+
// carries that spend (it is the sum of its tries).
|
|
3905
|
+
const ceilingFor = (step) => stepCeilingFor(stepCeiling(run.budgetUsd, platform.runSpend(run), launching.length), agentBudgets.get(step.agent), run.steps
|
|
3906
|
+
.filter((s) => s.agent === step.agent)
|
|
3907
|
+
.reduce((sum, s) => sum + (s.costUsd ?? 0) + (s !== step && s.status === "running" ? (reserved.get(s) ?? 0) : 0), 0), step.agent, 1 + launching.filter((s) => s !== step && s.agent === step.agent && s.status === "pending" && !reserved.has(s)).length);
|
|
3908
|
+
// parallel: — the instances of one fan-out template share a pool.
|
|
3909
|
+
// Keyed by what the instances share (agent + instruction), since
|
|
3910
|
+
// the template itself is behind them in the record as "expanded".
|
|
3911
|
+
const pools = new Map();
|
|
3912
|
+
const poolFor = (step) => {
|
|
3913
|
+
if (!step.parallel || !step.item)
|
|
3914
|
+
return null;
|
|
3915
|
+
const key = `${step.agent}\u0000${step.instruction}`;
|
|
3916
|
+
let pool = pools.get(key);
|
|
3917
|
+
if (!pool)
|
|
3918
|
+
pools.set(key, (pool = slotPool(step.parallel)));
|
|
3919
|
+
return pool;
|
|
3920
|
+
};
|
|
2747
3921
|
await Promise.all(launching
|
|
2748
3922
|
.map(async (step) => {
|
|
3923
|
+
const pool = poolFor(step);
|
|
3924
|
+
if (pool)
|
|
3925
|
+
await pool.acquire();
|
|
3926
|
+
try {
|
|
3927
|
+
await launchStep(step);
|
|
3928
|
+
}
|
|
3929
|
+
finally {
|
|
3930
|
+
pool?.release();
|
|
3931
|
+
}
|
|
3932
|
+
}));
|
|
3933
|
+
async function launchStep(step) {
|
|
2749
3934
|
const attempts = (step.retry ?? 0) + 1;
|
|
2750
3935
|
// A step re-attaching to its sandbox is still on the attempt
|
|
2751
3936
|
// that started it; a step re-run after a backoff picks up the
|
|
@@ -2753,19 +3938,83 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
|
|
|
2753
3938
|
const interruptedInBackoff = !step.sandbox && !!step.attempts && /retrying in \d+s/.test(step.events[step.events.length - 1]?.text ?? "");
|
|
2754
3939
|
const from = step.sandbox ? Math.max(1, step.attempts ?? 1) : interruptedInBackoff ? Math.min(attempts, step.attempts + 1) : 1;
|
|
2755
3940
|
for (let attempt = from; attempt <= attempts; attempt++) {
|
|
3941
|
+
const ceiling = ceilingFor(step);
|
|
3942
|
+
// Nothing left is a refusal, not "no ceiling": a zero passed
|
|
3943
|
+
// down would read as unset, and the one step that must not
|
|
3944
|
+
// run would run with no cap at all. On a retry this is what
|
|
3945
|
+
// stops the second attempt spending a share the first one
|
|
3946
|
+
// already spent.
|
|
3947
|
+
if (ceiling.ceilingUsd === 0) {
|
|
3948
|
+
step.status = "failed";
|
|
3949
|
+
step.events.push({
|
|
3950
|
+
t: new Date().toISOString(),
|
|
3951
|
+
type: "error",
|
|
3952
|
+
text: attempt === from
|
|
3953
|
+
? `over budget — nothing left to spend before this step (${ceiling.note}); it was not started`
|
|
3954
|
+
: `over budget — nothing left to spend for attempt ${attempt} (${ceiling.note}); not retried`,
|
|
3955
|
+
});
|
|
3956
|
+
save();
|
|
3957
|
+
return;
|
|
3958
|
+
}
|
|
3959
|
+
// Re-attaching to a sandbox is the same attempt, so its row
|
|
3960
|
+
// is replaced; anything else is a new row, a loop cycle's
|
|
3961
|
+
// re-run included.
|
|
3962
|
+
const reattach = !!step.sandbox;
|
|
3963
|
+
const attemptStartedAt = new Date().toISOString();
|
|
3964
|
+
const eventsBefore = step.events.length;
|
|
2756
3965
|
step.attempts = attempt;
|
|
2757
3966
|
step.status = "running";
|
|
3967
|
+
if (ceiling.ceilingUsd !== null)
|
|
3968
|
+
reserved.set(step, ceiling.ceilingUsd);
|
|
2758
3969
|
save();
|
|
2759
|
-
await runStep(path.join(pDir, "agents", step.agent), tenant, step, ctx, save, modelOverride, tags, effortOverride, run.id, ctxData, ceilingUsd
|
|
3970
|
+
const verdict = await runStep(path.join(pDir, "agents", step.agent), tenant, step, ctx, save, modelOverride, tags, effortOverride, run.id, ctxData, ceiling.ceilingUsd, ceiling.note, run.test === true,
|
|
3971
|
+
// save() merges a stop written by another process into
|
|
3972
|
+
// this record; the model loop polls this on a clock.
|
|
3973
|
+
() => {
|
|
3974
|
+
save();
|
|
3975
|
+
return run.stopRequested === true;
|
|
3976
|
+
}, flowTimezone, run.steps.indexOf(step)).finally(() => reserved.delete(step));
|
|
2760
3977
|
// runStep mutates step.status; read it through a widened local
|
|
2761
3978
|
// so TS doesn't keep the "running" narrowing from above.
|
|
2762
3979
|
const outcome = step.status;
|
|
3980
|
+
recordAttempt(step, attempt, outcome === "completed" ? "completed" : "failed", attemptStartedAt, eventsBefore, reattach);
|
|
3981
|
+
save();
|
|
3982
|
+
// A stop destroys the step's sandbox, which reads here as a
|
|
3983
|
+
// failed attempt. stopRun writes "stopped by a person" onto
|
|
3984
|
+
// this step in the record, and the save() above writes this
|
|
3985
|
+
// process's copy over it — so the trail ended on the
|
|
3986
|
+
// sandbox's own last words, "sandbox ended: Error (exit 143)",
|
|
3987
|
+
// and a person's stop read as the platform killing a healthy
|
|
3988
|
+
// step (abandon-desk, 2026-09-29: stopped from the CLI after
|
|
3989
|
+
// ten minutes, then chased as an unexplained kill). Say so,
|
|
3990
|
+
// whether or not a retry was left.
|
|
3991
|
+
if (outcome === "failed" && run.stopRequested) {
|
|
3992
|
+
step.events.push({
|
|
3993
|
+
t: new Date().toISOString(),
|
|
3994
|
+
type: "info",
|
|
3995
|
+
text: "stopped by a person mid-step — its sandbox was destroyed, which is what ended this attempt; not retried",
|
|
3996
|
+
});
|
|
3997
|
+
save();
|
|
3998
|
+
break;
|
|
3999
|
+
}
|
|
2763
4000
|
if (outcome !== "failed" || attempt === attempts)
|
|
2764
4001
|
break;
|
|
2765
|
-
//
|
|
2766
|
-
//
|
|
2767
|
-
//
|
|
2768
|
-
|
|
4002
|
+
// An outward step whose tools ran and whose check then failed
|
|
4003
|
+
// is not retried: the check cannot say the send did not
|
|
4004
|
+
// happen, and a retry would do it twice.
|
|
4005
|
+
if (verdict?.actedThenFailedCheck) {
|
|
4006
|
+
step.events.push({
|
|
4007
|
+
t: new Date().toISOString(),
|
|
4008
|
+
type: "info",
|
|
4009
|
+
text: `not retried — this step's tools can act outside the workspace and ran before the check failed; another attempt could repeat what they did (${attempts - attempt} left unused)`,
|
|
4010
|
+
});
|
|
4011
|
+
save();
|
|
4012
|
+
break;
|
|
4013
|
+
}
|
|
4014
|
+
// A failed attempt used to be retried in a fresh sandbox even
|
|
4015
|
+
// after a stop, so a stopped `retry: 2` step ran twice more.
|
|
4016
|
+
// The check above ends the loop on a stop; this one covers a
|
|
4017
|
+
// stop that lands while the outward check was being written.
|
|
2769
4018
|
save();
|
|
2770
4019
|
if (run.stopRequested)
|
|
2771
4020
|
break;
|
|
@@ -2807,7 +4056,7 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
|
|
|
2807
4056
|
break;
|
|
2808
4057
|
}
|
|
2809
4058
|
}
|
|
2810
|
-
}
|
|
4059
|
+
}
|
|
2811
4060
|
// on-fail: another agent takes the step over. Sequential and rare:
|
|
2812
4061
|
// the rescue is a second full attempt at the step's job with the
|
|
2813
4062
|
// failure in front of it, and its success is the step's success —
|
|
@@ -2859,7 +4108,7 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
|
|
|
2859
4108
|
rescue.attempts = 1;
|
|
2860
4109
|
rescue.status = "running";
|
|
2861
4110
|
save();
|
|
2862
|
-
await runStep(rescuerDir, tenant, rescue, ctx, save, modelOverride, tags, effortOverride, run.id, ctxData);
|
|
4111
|
+
await runStep(rescuerDir, tenant, rescue, ctx, save, modelOverride, tags, effortOverride, run.id, ctxData, null, undefined, run.test === true, undefined, flowTimezone, run.steps.indexOf(rescue));
|
|
2863
4112
|
// runStep mutates rescue.status; widen past TS's "running" narrowing.
|
|
2864
4113
|
const rescueOutcome = rescue.status;
|
|
2865
4114
|
if (rescueOutcome === "completed") {
|
|
@@ -2992,16 +4241,25 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
|
|
|
2992
4241
|
}
|
|
2993
4242
|
catch (err) {
|
|
2994
4243
|
run.status = "failed";
|
|
4244
|
+
const text = err instanceof Error ? err.message : String(err);
|
|
4245
|
+
let told = false;
|
|
2995
4246
|
run.steps.forEach((s) => {
|
|
2996
4247
|
if (s.status === "running") {
|
|
2997
4248
|
s.status = "failed";
|
|
2998
|
-
s.events.push({
|
|
2999
|
-
|
|
3000
|
-
type: "error",
|
|
3001
|
-
text: err instanceof Error ? err.message : String(err),
|
|
3002
|
-
});
|
|
4249
|
+
s.events.push({ t: new Date().toISOString(), type: "error", text });
|
|
4250
|
+
told = true;
|
|
3003
4251
|
}
|
|
3004
4252
|
});
|
|
4253
|
+
// Thrown between groups, with nothing running: the error still has to
|
|
4254
|
+
// land on some step's trail, or the run reads as failed for no reason.
|
|
4255
|
+
if (!told) {
|
|
4256
|
+
const witness = run.steps.findLast((s) => s.status !== "pending") ?? run.steps[0];
|
|
4257
|
+
witness?.events.push({
|
|
4258
|
+
t: new Date().toISOString(),
|
|
4259
|
+
type: "error",
|
|
4260
|
+
text: `run failed between steps: ${text}`,
|
|
4261
|
+
});
|
|
4262
|
+
}
|
|
3005
4263
|
}
|
|
3006
4264
|
finally {
|
|
3007
4265
|
if (!parked) {
|
|
@@ -3012,6 +4270,13 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
|
|
|
3012
4270
|
// learn, and the runs list and the notification stop having to say
|
|
3013
4271
|
// only "completed".
|
|
3014
4272
|
run.summary = runSummary(run);
|
|
4273
|
+
// Read before a test run's prefix goes on the summary, which would
|
|
4274
|
+
// otherwise hide the word it leads with.
|
|
4275
|
+
run.verdict = verdictOf(run.summary);
|
|
4276
|
+
// A test run's headline says so wherever the headline goes — the
|
|
4277
|
+
// runs list, the history tool, a later run's recall.
|
|
4278
|
+
if (run.test && run.summary)
|
|
4279
|
+
run.summary = testHeadline(run.summary);
|
|
3015
4280
|
// Archive before the next run resets outputs/ — a failed run's
|
|
3016
4281
|
// artifacts are usually the most interesting ones, so this runs on
|
|
3017
4282
|
// failure too.
|
|
@@ -3049,17 +4314,6 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
|
|
|
3049
4314
|
}
|
|
3050
4315
|
})();
|
|
3051
4316
|
}
|
|
3052
|
-
function readFlow(tenant, workspace, flowName) {
|
|
3053
|
-
const dir = path.join(workspaceDir(tenant, workspace), "flows");
|
|
3054
|
-
if (!fs.existsSync(dir))
|
|
3055
|
-
return null;
|
|
3056
|
-
for (const f of fs.readdirSync(dir).filter((f) => f.endsWith(".md"))) {
|
|
3057
|
-
const flow = parseFlow(f, fs.readFileSync(path.join(dir, f), "utf8"));
|
|
3058
|
-
if (flow.name === flowName)
|
|
3059
|
-
return flow;
|
|
3060
|
-
}
|
|
3061
|
-
return null;
|
|
3062
|
-
}
|
|
3063
4317
|
// Splice `[[flow:other]]` steps into the parent's step list, renumbering
|
|
3064
4318
|
// groups so the nested flow's own parallelism survives. Depth-limited, and
|
|
3065
4319
|
// a cycle raises rather than expanding forever.
|