@foldrun/core 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -10
- package/dist/index.d.ts +3 -0
- package/dist/index.js +3 -0
- package/dist/src/account.d.ts +45 -0
- package/dist/src/account.js +130 -0
- package/dist/src/api-tools.d.ts +8 -0
- package/dist/src/api-tools.js +22 -4
- package/dist/src/approvals.d.ts +104 -1
- package/dist/src/approvals.js +192 -2
- package/dist/src/browser-pod.d.ts +129 -0
- package/dist/src/browser-pod.js +284 -0
- package/dist/src/budget.d.ts +32 -0
- package/dist/src/budget.js +177 -0
- package/dist/src/catalog.js +3 -1
- package/dist/src/clock.d.ts +49 -0
- package/dist/src/clock.js +121 -0
- package/dist/src/completions.js +17 -8
- package/dist/src/confine.d.ts +22 -0
- package/dist/src/confine.js +224 -4
- package/dist/src/deploy.d.ts +33 -0
- package/dist/src/deploy.js +145 -6
- package/dist/src/egress.d.ts +26 -2
- package/dist/src/egress.js +8 -1
- package/dist/src/eol.d.ts +2 -0
- package/dist/src/eol.js +7 -0
- package/dist/src/evals.d.ts +9 -0
- package/dist/src/evals.js +26 -13
- package/dist/src/flow-lint.d.ts +12 -0
- package/dist/src/flow-lint.js +331 -1
- package/dist/src/flow-patterns.d.ts +141 -0
- package/dist/src/flow-patterns.js +568 -0
- package/dist/src/gitrepo.d.ts +18 -0
- package/dist/src/gitrepo.js +49 -0
- package/dist/src/host-env.d.ts +8 -0
- package/dist/src/host-env.js +48 -0
- package/dist/src/json-schema.d.ts +12 -0
- package/dist/src/json-schema.js +202 -0
- package/dist/src/kinds.js +1 -1
- package/dist/src/language.d.ts +17 -0
- package/dist/src/language.js +55 -0
- package/dist/src/layout.d.ts +69 -0
- package/dist/src/layout.js +200 -0
- package/dist/src/library.d.ts +9 -0
- package/dist/src/library.js +43 -4
- package/dist/src/limits.d.ts +84 -0
- package/dist/src/limits.js +287 -0
- package/dist/src/locale.d.ts +47 -0
- package/dist/src/locale.js +177 -0
- package/dist/src/mail.d.ts +88 -0
- package/dist/src/mail.js +245 -0
- package/dist/src/model-probe.d.ts +3 -0
- package/dist/src/model-probe.js +1 -0
- package/dist/src/notify.d.ts +132 -8
- package/dist/src/notify.js +360 -58
- package/dist/src/openapi.js +5 -3
- package/dist/src/operator.d.ts +201 -0
- package/dist/src/operator.js +251 -0
- package/dist/src/paths.d.ts +33 -0
- package/dist/src/paths.js +87 -0
- package/dist/src/platform.d.ts +144 -4
- package/dist/src/platform.js +27 -3
- package/dist/src/provider-health.d.ts +55 -0
- package/dist/src/provider-health.js +233 -0
- package/dist/src/providers.d.ts +481 -0
- package/dist/src/providers.js +1105 -6
- package/dist/src/refusal.d.ts +83 -0
- package/dist/src/refusal.js +281 -0
- package/dist/src/run-container.d.ts +206 -17
- package/dist/src/run-container.js +547 -80
- package/dist/src/runner.d.ts +179 -10
- package/dist/src/runner.js +1423 -169
- package/dist/src/runtime.d.ts +52 -0
- package/dist/src/runtime.js +493 -58
- package/dist/src/script-tools.d.ts +15 -2
- package/dist/src/script-tools.js +44 -4
- package/dist/src/secret-files.d.ts +6 -5
- package/dist/src/secret-files.js +24 -11
- package/dist/src/secret-health.d.ts +83 -0
- package/dist/src/secret-health.js +177 -0
- package/dist/src/secrets.d.ts +38 -1
- package/dist/src/secrets.js +197 -96
- package/dist/src/starter.d.ts +9 -0
- package/dist/src/starter.js +95 -11
- package/dist/src/step-exec.d.ts +106 -1
- package/dist/src/step-exec.js +558 -85
- package/dist/src/step-removal.d.ts +15 -0
- package/dist/src/step-removal.js +49 -0
- package/dist/src/storage.d.ts +23 -4
- package/dist/src/storage.js +125 -23
- package/dist/src/store.d.ts +492 -38
- package/dist/src/store.js +882 -69
- package/dist/src/subagents.d.ts +75 -0
- package/dist/src/subagents.js +149 -0
- package/dist/src/tar.js +2 -1
- package/dist/src/test-mode.d.ts +111 -0
- package/dist/src/test-mode.js +285 -0
- package/dist/src/tool-names.d.ts +13 -1
- package/dist/src/tool-names.js +38 -11
- package/dist/src/tool-programs.d.ts +26 -0
- package/dist/src/tool-programs.js +172 -0
- package/dist/src/tool-test.js +152 -33
- package/dist/src/translator.d.ts +8 -2
- package/dist/src/translator.js +71 -24
- package/dist/src/trigger-log.d.ts +43 -0
- package/dist/src/trigger-log.js +122 -0
- package/dist/src/triggers.d.ts +23 -0
- package/dist/src/triggers.js +101 -12
- package/dist/src/untrusted-fetch.d.ts +11 -0
- package/dist/src/untrusted-fetch.js +77 -0
- package/dist/src/web-actions.d.ts +91 -0
- package/dist/src/web-actions.js +199 -0
- package/dist/src/webhook.d.ts +19 -3
- package/dist/src/webhook.js +50 -11
- package/index.ts +3 -0
- package/package.json +5 -5
- package/src/account.ts +149 -0
- package/src/api-tools.ts +32 -4
- package/src/approvals.ts +253 -2
- package/src/browser-pod.ts +313 -0
- package/src/budget.ts +178 -0
- package/src/catalog.ts +3 -1
- package/src/clock.ts +149 -0
- package/src/completions.ts +17 -8
- package/src/confine.ts +224 -4
- package/src/deploy.ts +137 -6
- package/src/egress.ts +32 -4
- package/src/eol.ts +7 -0
- package/src/evals.ts +36 -11
- package/src/flow-lint.ts +371 -1
- package/src/flow-patterns.ts +597 -0
- package/src/gitrepo.ts +48 -0
- package/src/host-env.ts +51 -0
- package/src/json-schema.ts +184 -0
- package/src/kinds.ts +1 -1
- package/src/language.ts +62 -0
- package/src/layout.ts +228 -0
- package/src/library.ts +50 -4
- package/src/limits.ts +290 -0
- package/src/locale.ts +217 -0
- package/src/mail.ts +316 -0
- package/src/model-probe.ts +4 -0
- package/src/notify.ts +391 -44
- package/src/openapi.ts +5 -3
- package/src/operator.ts +300 -0
- package/src/paths.ts +83 -0
- package/src/platform.ts +158 -5
- package/src/provider-health.ts +275 -0
- package/src/providers.ts +1368 -6
- package/src/refusal.ts +320 -0
- package/src/run-container.ts +611 -76
- package/src/runner.ts +1463 -165
- package/src/runtime.ts +487 -55
- package/src/script-tools.ts +55 -6
- package/src/secret-files.ts +22 -10
- package/src/secret-health.ts +230 -0
- package/src/secrets.ts +191 -95
- package/src/starter.ts +96 -11
- package/src/step-exec.ts +548 -41
- package/src/step-removal.ts +59 -0
- package/src/storage.ts +149 -24
- package/src/store.ts +1227 -74
- package/src/subagents.ts +189 -0
- package/src/tar.ts +2 -1
- package/src/test-mode.ts +311 -0
- package/src/tool-names.ts +40 -11
- package/src/tool-programs.ts +179 -0
- package/src/tool-test.ts +159 -32
- package/src/translator.ts +74 -23
- package/src/trigger-log.ts +160 -0
- package/src/triggers.ts +102 -12
- package/src/untrusted-fetch.ts +88 -0
- package/src/web-actions.ts +237 -0
- package/src/webhook.ts +58 -11
- package/templates/hello/agents/curator/agent.md +1 -1
- package/templates/hello/agents/notetaker/agent.md +1 -1
package/src/runner.ts
CHANGED
|
@@ -8,11 +8,19 @@ import path from "node:path";
|
|
|
8
8
|
import matter from "gray-matter";
|
|
9
9
|
import { spawn } from "node:child_process";
|
|
10
10
|
import {
|
|
11
|
-
knownPrice, executeStep, extractJson, stepCeiling, type EventExtra } from "./step-exec.ts";
|
|
11
|
+
knownPrice, executeStep, extractJson, stepCeiling, stepCeilingFor, type EventExtra } from "./step-exec.ts";
|
|
12
|
+
import { cascadeLimits, toolOwners } from "./limits.ts";
|
|
13
|
+
import { resolveAgentPath } from "./confine.ts";
|
|
14
|
+
import { browserPodLine, podLossDecision, RERAN_ON_FULL } from "./browser-pod.ts";
|
|
12
15
|
import { eventUrl } from "./webhook.ts";
|
|
13
16
|
import { runStepInContainer, sizeLimits, killRunSandboxes, type StepTiming } from "./run-container.ts";
|
|
17
|
+
import { hostSafeEnv } from "./host-env.ts";
|
|
18
|
+
import { validateSchema, describeSchemaErrors, looksLikeSchema } from "./json-schema.ts";
|
|
14
19
|
import { EGRESS_ENV, MODEL_KEY_NAME, addGrant, hostOf, placeholderNames, proxyModelEnv, unsubstitute, type EgressGrant } from "./egress.ts";
|
|
20
|
+
import { explainRefusal, isRefusalStatus, refusalFromLine, supplyNote, type SupplyState } from "./refusal.ts";
|
|
15
21
|
import { platform } from "./platform.ts";
|
|
22
|
+
import { healthKey } from "./secret-health.ts";
|
|
23
|
+
import { TEST_WRITES_DIR, restoreDivertedDirs, snapshotDivertedDirs, testHeadline, testModeEnv, withholdSecrets } from "./test-mode.ts";
|
|
16
24
|
|
|
17
25
|
/** Does this process run steps in a sandbox — the container core ships, or
|
|
18
26
|
* one the platform registered (a pod)? */
|
|
@@ -21,7 +29,10 @@ const isolatedRun = () => {
|
|
|
21
29
|
return mode === "container" || (!!mode && mode in platform.isolation);
|
|
22
30
|
};
|
|
23
31
|
import { gatherConsults, buildConsultTools } from "./agent-tools.ts";
|
|
24
|
-
import {
|
|
32
|
+
import { DELEGATE_TOOL, gatherSubagents, subagentPrompt, subagentTools, type SubagentSpec } from "./subagents.ts";
|
|
33
|
+
import { ASK_TOOL, applyOperatorEvent, askPromptLine, askTimeoutSec, buildAskTool } from "./operator.ts";
|
|
34
|
+
import { sendPlainNotification } from "./notify.ts";
|
|
35
|
+
import { TOOL_MAP, BUILTIN_TOOLS, ownToolNames, toolRefs, legacyUseNames, legacyUseError, retiredToolNames, retiredToolError } from "./tool-names.ts";
|
|
25
36
|
import { refNames } from "./refs.ts";
|
|
26
37
|
import {
|
|
27
38
|
accountDir,
|
|
@@ -37,7 +48,11 @@ import {
|
|
|
37
48
|
listRuns,
|
|
38
49
|
syncBundleFor,
|
|
39
50
|
parseToolDef,
|
|
51
|
+
type ToolDef,
|
|
40
52
|
parseFlow,
|
|
53
|
+
readFlow,
|
|
54
|
+
markerPresent,
|
|
55
|
+
whenRowsPath,
|
|
41
56
|
parseApis,
|
|
42
57
|
resolveModel,
|
|
43
58
|
parseProvider,
|
|
@@ -49,12 +64,17 @@ import {
|
|
|
49
64
|
readRun,
|
|
50
65
|
writeRun,
|
|
51
66
|
runSummary,
|
|
67
|
+
verdictOf,
|
|
52
68
|
type FlowStep,
|
|
53
69
|
type McpSpec,
|
|
54
70
|
type RunRecord,
|
|
55
71
|
type StepRecord,
|
|
72
|
+
type StepAttempt,
|
|
73
|
+
sumAttempts,
|
|
74
|
+
parseSchemaText,
|
|
56
75
|
STORAGE_DIR,
|
|
57
76
|
adoptLegacyVersionKey,
|
|
77
|
+
listAgents,
|
|
58
78
|
runCost,
|
|
59
79
|
} from "./store.ts";
|
|
60
80
|
import { loadCatalog, checkModel, clampEffort, catalogCost, findModel, type Catalog } from "./catalog.ts";
|
|
@@ -67,11 +87,19 @@ import { startTranslator, translatorSpecFor, type TranslatorSpec } from "./trans
|
|
|
67
87
|
import { providerPreset } from "./providers.ts";
|
|
68
88
|
import { buildScriptTools, parseScripts, type ExecutionContext } from "./script-tools.ts";
|
|
69
89
|
import { libraryDir, libraryTools, libraryMemoryIndex } from "./library.ts";
|
|
70
|
-
import { mergeRuntimes, parseRuntime, prepareRuntime, type RuntimeSpec } from "./runtime.ts";
|
|
90
|
+
import { fingerprint, mergeRuntimes, parseRuntime, prepareRuntime, type PreparedRuntime, type RuntimeSpec } from "./runtime.ts";
|
|
71
91
|
import { materializeFiles, harvestFiles } from "./storage.ts";
|
|
72
92
|
import { chooseExecutor, ensureImage } from "./container.ts";
|
|
73
93
|
import { stampBundle } from "./okf.ts";
|
|
74
94
|
import type { McpServerConfig } from "@anthropic-ai/claude-agent-sdk";
|
|
95
|
+
import { readBrowseSettings, readSearchSettings, resolveSearch, searchSettingsEnv, webConfig, browseSessionProblems } from "./providers.ts";
|
|
96
|
+
import { NEWER_ACTIONS, resolveActionApi } from "./web-actions.ts";
|
|
97
|
+
import { resolveLanguage, languageName, type LanguageChoice } from "./language.ts";
|
|
98
|
+
import { resolveRegion, deriveLocale, localeProse, regionName, type RegionChoice, type LocaleFacts } from "./locale.ts";
|
|
99
|
+
import { trimChars } from "./paths.ts";
|
|
100
|
+
import { resolveClock, localDate, type ClockChoice } from "./clock.ts";
|
|
101
|
+
|
|
102
|
+
export { localDate } from "./clock.ts";
|
|
75
103
|
|
|
76
104
|
// An MCP tool definition becomes an SDK server config. ${SECRET} placeholders
|
|
77
105
|
// in env and headers resolve server-side, so a credential reaches the server
|
|
@@ -123,6 +151,32 @@ export function joinEarlierGroups(groupResults: (string | null)[], gi: number, c
|
|
|
123
151
|
return parts.join("\n\n");
|
|
124
152
|
}
|
|
125
153
|
|
|
154
|
+
/**
|
|
155
|
+
* `parallel: N` — a pool of N slots the instances of one fan-out share.
|
|
156
|
+
* Fan-out always ran every instance at once, and still does unless the
|
|
157
|
+
* step says otherwise: twenty browser steps against one site, or one
|
|
158
|
+
* vendor's rate limit, is what the cap is for. Nothing is rented while an
|
|
159
|
+
* instance waits for a slot.
|
|
160
|
+
*/
|
|
161
|
+
export function slotPool(size: number): { acquire: () => Promise<void>; release: () => void } {
|
|
162
|
+
let free = Math.max(1, size);
|
|
163
|
+
const waiting: (() => void)[] = [];
|
|
164
|
+
return {
|
|
165
|
+
acquire: () =>
|
|
166
|
+
new Promise<void>((resolve) => {
|
|
167
|
+
if (free > 0) {
|
|
168
|
+
free -= 1;
|
|
169
|
+
resolve();
|
|
170
|
+
} else waiting.push(resolve);
|
|
171
|
+
}),
|
|
172
|
+
release: () => {
|
|
173
|
+
const next = waiting.shift();
|
|
174
|
+
if (next) next();
|
|
175
|
+
else free += 1;
|
|
176
|
+
},
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
|
|
126
180
|
export interface DiscoveredSkill {
|
|
127
181
|
name: string;
|
|
128
182
|
description: string;
|
|
@@ -237,6 +291,29 @@ export function discoverSkills(agentDir: string, subdir = "skills"): DiscoveredS
|
|
|
237
291
|
* Returns null rather than throwing on unparseable YAML: a broken shared file
|
|
238
292
|
* must not take down every agent under it.
|
|
239
293
|
*/
|
|
294
|
+
/**
|
|
295
|
+
* AGENTS.md without the block a coding tool reads.
|
|
296
|
+
*
|
|
297
|
+
* AGENTS.md is two things in a foldrun folder: the account's (or
|
|
298
|
+
* workspace's) own context, which every running agent is given, and — the
|
|
299
|
+
* convention Next.js set — the file a coding agent (Claude Code, Cursor,
|
|
300
|
+
* Codex) reads before it edits the folder. The CLI writes a short managed
|
|
301
|
+
* block for the second reader between these markers; it is about editing
|
|
302
|
+
* the files, not about doing the work, so no running agent is shown it.
|
|
303
|
+
*/
|
|
304
|
+
export const AGENT_RULES_START = "<!-- BEGIN:foldrun-agent-rules -->";
|
|
305
|
+
export const AGENT_RULES_END = "<!-- END:foldrun-agent-rules -->";
|
|
306
|
+
export function withoutAgentRules(body: string): string {
|
|
307
|
+
let out = body;
|
|
308
|
+
for (;;) {
|
|
309
|
+
const start = out.indexOf(AGENT_RULES_START);
|
|
310
|
+
if (start === -1) return out;
|
|
311
|
+
const end = out.indexOf(AGENT_RULES_END, start);
|
|
312
|
+
if (end === -1) return out;
|
|
313
|
+
out = out.slice(0, start) + out.slice(end + AGENT_RULES_END.length);
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
|
|
240
317
|
export function readAgentsMd(dir: string): { data: Record<string, unknown>; body: string } | null {
|
|
241
318
|
// Convert a pre-rename `mdagent_version:` before parsing. One name in the
|
|
242
319
|
// code, one on disk — the reader no longer knows two spellings, so a file
|
|
@@ -247,7 +324,7 @@ export function readAgentsMd(dir: string): { data: Record<string, unknown>; body
|
|
|
247
324
|
if (!fs.existsSync(file)) continue;
|
|
248
325
|
try {
|
|
249
326
|
const parsed = matter(fs.readFileSync(file, "utf8"));
|
|
250
|
-
return { data: parsed.data as Record<string, unknown>, body: parsed.content.trim() };
|
|
327
|
+
return { data: parsed.data as Record<string, unknown>, body: withoutAgentRules(parsed.content).trim() };
|
|
251
328
|
} catch {
|
|
252
329
|
return null;
|
|
253
330
|
}
|
|
@@ -266,6 +343,57 @@ const workspaceRootOf = (agentDir: string) => path.join(agentDir, "..", "..");
|
|
|
266
343
|
* writing instructions means *as well as*, because an account rule nobody can
|
|
267
344
|
* silently drop is the point of having one.
|
|
268
345
|
*/
|
|
346
|
+
/** An agent's own `runtime:` block, else its workspace's (AGENTS.md, the
|
|
347
|
+
* Linux Foundation standard name; project.md stays accepted so nothing
|
|
348
|
+
* that already exists breaks). */
|
|
349
|
+
function ownRuntime(front: Record<string, unknown>, agentDir: string, tenant: string): RuntimeSpec | null {
|
|
350
|
+
return parseRuntime(front.runtime) ?? parseRuntime(workspaceFrontmatter(agentDir, tenant).runtime);
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
/** What one granted script tool's program needs. */
|
|
354
|
+
function toolRuntime(def: ToolDef): RuntimeSpec | null {
|
|
355
|
+
return def.kind === "script" ? parseRuntime((def.spec as { runtime?: unknown }).runtime) : null;
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
/**
|
|
359
|
+
* The runtime one agent's steps are built with: its own declaration (else its
|
|
360
|
+
* workspace's), merged with every script tool it grants. The runner builds
|
|
361
|
+
* exactly this for a step; the platform builds it ahead of time when the
|
|
362
|
+
* workspace is deployed, so a package that will not install is reported at
|
|
363
|
+
* deploy and a scheduled step finds its environment ready. One function, so
|
|
364
|
+
* the two can never disagree about what an agent needs.
|
|
365
|
+
*/
|
|
366
|
+
export function agentRuntimeSpec(tenant: string, workspace: string, agentDir: string): RuntimeSpec | null {
|
|
367
|
+
const file = path.join(agentDir, "agent.md");
|
|
368
|
+
if (!fs.existsSync(file)) return null;
|
|
369
|
+
const front = safeMatter(fs.readFileSync(file, "utf8")).data;
|
|
370
|
+
const available = { ...libraryTools(tenant), ...workspaceTools(tenant, workspace) };
|
|
371
|
+
const tools = ownToolNames(front)
|
|
372
|
+
.map((name) => available[name])
|
|
373
|
+
.filter((def): def is ToolDef => Boolean(def))
|
|
374
|
+
.map(toolRuntime);
|
|
375
|
+
return mergeRuntimes(ownRuntime(front, agentDir, tenant), ...tools);
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
/** Every distinct runtime a workspace's agents need, with who needs it. */
|
|
379
|
+
export function workspaceRuntimePlan(
|
|
380
|
+
tenant: string,
|
|
381
|
+
workspace: string,
|
|
382
|
+
): { fingerprint: string; spec: RuntimeSpec; agents: string[] }[] {
|
|
383
|
+
const byFp = new Map<string, { fingerprint: string; spec: RuntimeSpec; agents: string[] }>();
|
|
384
|
+
const agentsRoot = path.join(workspaceDir(tenant, workspace), "agents");
|
|
385
|
+
if (!fs.existsSync(agentsRoot)) return [];
|
|
386
|
+
for (const name of fs.readdirSync(agentsRoot).sort()) {
|
|
387
|
+
const spec = agentRuntimeSpec(tenant, workspace, path.join(agentsRoot, name));
|
|
388
|
+
if (!spec) continue;
|
|
389
|
+
const fp = fingerprint(spec);
|
|
390
|
+
const entry = byFp.get(fp) ?? { fingerprint: fp, spec, agents: [] };
|
|
391
|
+
entry.agents.push(name);
|
|
392
|
+
byFp.set(fp, entry);
|
|
393
|
+
}
|
|
394
|
+
return [...byFp.values()];
|
|
395
|
+
}
|
|
396
|
+
|
|
269
397
|
function workspaceFrontmatter(agentDir: string, tenant: string): Record<string, unknown> {
|
|
270
398
|
return {
|
|
271
399
|
...(readAgentsMd(accountDir(tenant))?.data ?? {}),
|
|
@@ -274,31 +402,69 @@ function workspaceFrontmatter(agentDir: string, tenant: string): Record<string,
|
|
|
274
402
|
}
|
|
275
403
|
|
|
276
404
|
/**
|
|
277
|
-
* The calendar
|
|
278
|
-
*
|
|
279
|
-
*
|
|
280
|
-
*
|
|
281
|
-
* An unknown name falls back to UTC rather than failing every step.
|
|
405
|
+
* The calendar a set of frontmatter asks for, with no cascade around it —
|
|
406
|
+
* one merged record in, one zone out. The full cascade lives in
|
|
407
|
+
* `agentClock` below and in clock.ts; this is what the account-level parts
|
|
408
|
+
* of the platform (a wallet's month, a budget window) call.
|
|
282
409
|
*/
|
|
283
410
|
export function resolveTimezone(front: Record<string, unknown>): string {
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
411
|
+
return resolveClock([{ level: "workspace", value: front.timezone }]).timezone;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
/**
|
|
415
|
+
* The clock this step works to: agent, then flow, then workspace, then
|
|
416
|
+
* account, then FOLDRUN_TIMEZONE, then UTC. Nearest wins, and a value that
|
|
417
|
+
* is not a zone falls through to the next level with a line to say so —
|
|
418
|
+
* see clock.ts for why this exists at all.
|
|
419
|
+
*/
|
|
420
|
+
export function agentClock(
|
|
421
|
+
agentDir: string,
|
|
422
|
+
tenant: string,
|
|
423
|
+
agentFront: Record<string, unknown>,
|
|
424
|
+
flowTimezone?: string | null,
|
|
425
|
+
): ClockChoice {
|
|
426
|
+
return resolveClock([
|
|
427
|
+
{ level: "agent", value: agentFront.timezone },
|
|
428
|
+
{ level: "flow", value: flowTimezone },
|
|
429
|
+
{ level: "workspace", value: readAgentsMd(workspaceRootOf(agentDir))?.data?.timezone },
|
|
430
|
+
{ level: "account", value: readAgentsMd(accountDir(tenant))?.data?.timezone },
|
|
431
|
+
]);
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
/**
|
|
435
|
+
* The language this step works in: agent, then workspace, then account, then
|
|
436
|
+
* FOLDRUN_LANGUAGE, then English. The flow level is not read yet — a flow's
|
|
437
|
+
* timezone: travels through three signatures to get here, and this will
|
|
438
|
+
* take the same road when a flow needs it.
|
|
439
|
+
*/
|
|
440
|
+
export function agentLanguage(agentDir: string, tenant: string, agentFront: Record<string, unknown>): LanguageChoice {
|
|
441
|
+
return resolveLanguage([
|
|
442
|
+
{ level: "agent", value: agentFront.language },
|
|
443
|
+
{ level: "workspace", value: readAgentsMd(workspaceRootOf(agentDir))?.data?.language },
|
|
444
|
+
{ level: "account", value: readAgentsMd(accountDir(tenant))?.data?.language },
|
|
445
|
+
]);
|
|
292
446
|
}
|
|
293
447
|
|
|
294
|
-
/**
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
448
|
+
/**
|
|
449
|
+
* The country this step works for, on the same road as the language, then
|
|
450
|
+
* FOLDRUN_REGION, then whatever the language tag carried (`en-AU` says AU).
|
|
451
|
+
* Everything a country changes — currency, units, calendar, weekend, text
|
|
452
|
+
* direction — is derived from it in locale.ts; the three overrides
|
|
453
|
+
* (`currency:`, `calendar:`, `units:`) are read at the nearest level that
|
|
454
|
+
* wrote them.
|
|
455
|
+
*/
|
|
456
|
+
export function agentRegion(agentDir: string, tenant: string, agentFront: Record<string, unknown>, languageTag: string): RegionChoice {
|
|
457
|
+
return resolveRegion([
|
|
458
|
+
{ level: "agent", value: agentFront.region },
|
|
459
|
+
{ level: "workspace", value: readAgentsMd(workspaceRootOf(agentDir))?.data?.region },
|
|
460
|
+
{ level: "account", value: readAgentsMd(accountDir(tenant))?.data?.region },
|
|
461
|
+
], languageTag);
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
export function agentLocaleOverrides(agentDir: string, tenant: string, agentFront: Record<string, unknown>): { currency?: unknown; calendar?: unknown; units?: unknown } {
|
|
465
|
+
const levels = [agentFront, readAgentsMd(workspaceRootOf(agentDir))?.data ?? {}, readAgentsMd(accountDir(tenant))?.data ?? {}];
|
|
466
|
+
const pick = (k: string) => levels.map((l) => l[k]).find((v) => v !== undefined && v !== null && v !== "");
|
|
467
|
+
return { currency: pick("currency"), calendar: pick("calendar"), units: pick("units") };
|
|
302
468
|
}
|
|
303
469
|
|
|
304
470
|
/**
|
|
@@ -308,6 +474,65 @@ export function localDate(timeZone: string, now = new Date()): string {
|
|
|
308
474
|
* template as "context every agent here shares" — and never read. Only the
|
|
309
475
|
* frontmatter was, so the instructions people actually wrote reached no model.
|
|
310
476
|
*/
|
|
477
|
+
/**
|
|
478
|
+
* The sub-agents a step may delegate to (`subagents:`), as the SDK will get
|
|
479
|
+
* them: each resolved through agentContext exactly as its own step would be
|
|
480
|
+
* (grants only — no runtime; it runs inside its parent's), then cut to what
|
|
481
|
+
* the parent holds. Problems are lines on the step, never a failed step.
|
|
482
|
+
*/
|
|
483
|
+
export function buildSubagentSpecs(o: {
|
|
484
|
+
names: unknown;
|
|
485
|
+
self: string;
|
|
486
|
+
workspaceRoot: string;
|
|
487
|
+
tenant: string;
|
|
488
|
+
tags?: string[];
|
|
489
|
+
runId?: string;
|
|
490
|
+
flowTimezone?: string | null;
|
|
491
|
+
parentAllowed: string[];
|
|
492
|
+
push: (type: "info" | "error", text: string) => void;
|
|
493
|
+
}): SubagentSpec[] {
|
|
494
|
+
const out: SubagentSpec[] = [];
|
|
495
|
+
const { found, missing } = gatherSubagents(o.workspaceRoot, o.names);
|
|
496
|
+
for (const m of missing) o.push("error", `subagents: "${m}" is not an agent in this workspace — not delegated to`);
|
|
497
|
+
for (const sub of found) {
|
|
498
|
+
if (sub.name === o.self) {
|
|
499
|
+
o.push("error", `subagents: "${sub.name}" is this agent itself — an agent does not delegate to itself`);
|
|
500
|
+
continue;
|
|
501
|
+
}
|
|
502
|
+
try {
|
|
503
|
+
const tags = o.tags ?? [];
|
|
504
|
+
const own = agentContext(sub.dir, o.tenant, tags, { runId: o.runId, agent: sub.name }, o.flowTimezone, true);
|
|
505
|
+
const description = typeof own.front.description === "string" ? own.front.description.trim() : "";
|
|
506
|
+
if (!description) {
|
|
507
|
+
o.push("error", `subagents: "${sub.name}" has no description — the model picks a sub-agent by it; not delegated to`);
|
|
508
|
+
continue;
|
|
509
|
+
}
|
|
510
|
+
const tools = subagentTools(own.allowed, o.parentAllowed);
|
|
511
|
+
if (!tools.length) {
|
|
512
|
+
o.push("error", `subagents: "${sub.name}" holds none of its tools inside this step — it can only answer from its prompt (use agents: for that)`);
|
|
513
|
+
}
|
|
514
|
+
const body = withoutAgentRules(matter(fs.readFileSync(path.join(sub.dir, "agent.md"), "utf8")).content);
|
|
515
|
+
out.push({
|
|
516
|
+
name: sub.name,
|
|
517
|
+
description,
|
|
518
|
+
prompt: subagentPrompt({
|
|
519
|
+
body: resolveDocLinks(body, o.workspaceRoot),
|
|
520
|
+
name: sub.name,
|
|
521
|
+
parent: o.self,
|
|
522
|
+
workspace: path.basename(o.workspaceRoot),
|
|
523
|
+
shared: sharedInstructions(sub.dir, o.tenant),
|
|
524
|
+
}),
|
|
525
|
+
tools,
|
|
526
|
+
disallowedTools: refNames(own.front.disallowedTools),
|
|
527
|
+
model: own.front.model ? resolveModel(own.front.model) : "inherit",
|
|
528
|
+
});
|
|
529
|
+
} catch (err) {
|
|
530
|
+
o.push("error", `subagents: "${sub.name}" could not be read — ${err instanceof Error ? err.message : String(err)}`);
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
return out;
|
|
534
|
+
}
|
|
535
|
+
|
|
311
536
|
export function sharedInstructions(agentDir: string, tenant: string): string | null {
|
|
312
537
|
const sections = [
|
|
313
538
|
["Everyone in this account", readAgentsMd(accountDir(tenant))?.body],
|
|
@@ -378,10 +603,9 @@ export function saysUntilMarker(result: string | null, until: string): boolean {
|
|
|
378
603
|
(line) =>
|
|
379
604
|
// Strip markdown emphasis and punctuation from BOTH ends: "**APPROVED.**"
|
|
380
605
|
// is a decision written by someone using bold, not a sentence.
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
.replace(/[.!:,*_`\s]+$/, "")
|
|
606
|
+
// trimChars, not a `[…]+$` regex: that one is quadratic on a
|
|
607
|
+
// line of tabs, and this is a line an agent wrote.
|
|
608
|
+
trimChars(trimChars(line.trim(), "#>-*_`"), ".!:,*_`").trim()
|
|
385
609
|
.toLowerCase() === marker,
|
|
386
610
|
);
|
|
387
611
|
}
|
|
@@ -397,7 +621,9 @@ export function resolveDocLinks(text: string, workspaceRoot: string): string {
|
|
|
397
621
|
// instruction names needs to say WHERE without naming a file. Added
|
|
398
622
|
// first, so a document that happens to share a folder's name still wins.
|
|
399
623
|
for (const dir of [STORAGE_DIR, "state", "knowledge", "memory", "skills", "outputs"]) {
|
|
400
|
-
|
|
624
|
+
// storage/ and state/ by the one spelling that works in every tool,
|
|
625
|
+
// the shell included; the rest as they always were.
|
|
626
|
+
if (fs.existsSync(path.join(workspaceRoot, dir))) map.set(norm(dir), dir === STORAGE_DIR || dir === "state" ? `workspace/${dir}/` : `../../${dir}/`);
|
|
401
627
|
}
|
|
402
628
|
for (const kind of ["knowledge", "memory"] as const) {
|
|
403
629
|
const dir = path.join(workspaceRoot, kind);
|
|
@@ -436,7 +662,7 @@ export function resolveDocLinks(text: string, workspaceRoot: string): string {
|
|
|
436
662
|
continue;
|
|
437
663
|
}
|
|
438
664
|
const fwd = entry.split(path.sep).join("/");
|
|
439
|
-
const rel =
|
|
665
|
+
const rel = `workspace/state/${fwd}`;
|
|
440
666
|
map.set(norm(`state/${fwd}`), rel);
|
|
441
667
|
map.set(norm(fwd), rel);
|
|
442
668
|
}
|
|
@@ -452,7 +678,7 @@ export function resolveDocLinks(text: string, workspaceRoot: string): string {
|
|
|
452
678
|
"utf8",
|
|
453
679
|
),
|
|
454
680
|
) as { files?: { path: string }[] };
|
|
455
|
-
for (const f of index.files ?? []) map.set(norm(`${STORAGE_DIR}/${f.path}`),
|
|
681
|
+
for (const f of index.files ?? []) map.set(norm(`${STORAGE_DIR}/${f.path}`), `workspace/${STORAGE_DIR}/${f.path}`);
|
|
456
682
|
} catch {
|
|
457
683
|
// no file store index is normal
|
|
458
684
|
}
|
|
@@ -467,11 +693,14 @@ export function resolveDocLinks(text: string, workspaceRoot: string): string {
|
|
|
467
693
|
}
|
|
468
694
|
const fwd = entry.split(path.sep).join("/");
|
|
469
695
|
if (fwd.startsWith(".store/")) continue;
|
|
470
|
-
map.set(norm(`${STORAGE_DIR}/${fwd}`),
|
|
471
|
-
map.set(norm(fwd),
|
|
696
|
+
map.set(norm(`${STORAGE_DIR}/${fwd}`), `workspace/${STORAGE_DIR}/${fwd}`);
|
|
697
|
+
map.set(norm(fwd), `workspace/${STORAGE_DIR}/${fwd}`);
|
|
472
698
|
}
|
|
473
699
|
}
|
|
474
|
-
|
|
700
|
+
// `[` excluded from the name: with it allowed, a run of `[[` makes the
|
|
701
|
+
// engine re-scan to the end from every opening — quadratic on a result
|
|
702
|
+
// an agent wrote.
|
|
703
|
+
return text.replace(/\[\[([^\]\[\n]+)\]\]/g, (whole, name: string) => {
|
|
475
704
|
const hit = map.get(norm(name.trim()));
|
|
476
705
|
return hit ? `\`${hit}\`` : whole;
|
|
477
706
|
});
|
|
@@ -483,6 +712,12 @@ function agentContext(
|
|
|
483
712
|
tags: string[] = [],
|
|
484
713
|
/** Who is running, so scripts can stamp what they write with its provenance. */
|
|
485
714
|
identity: { runId?: string; agent?: string } = {},
|
|
715
|
+
/** The flow's own `timezone:`, when this step belongs to one. Between the
|
|
716
|
+
* agent's frontmatter and the workspace's AGENTS.md in the cascade. */
|
|
717
|
+
flowTimezone?: string | null,
|
|
718
|
+
/** Only what the agent is granted — no runtime prepared. For a sub-agent,
|
|
719
|
+
* whose tools are cut to its parent's and run in the parent's runtime. */
|
|
720
|
+
grantsOnly = false,
|
|
486
721
|
) {
|
|
487
722
|
// agentDir is <data>/<tenant>/projects/<workspace>/agents/<agent>, so the
|
|
488
723
|
// workspace a secret should resolve against is two levels up.
|
|
@@ -497,13 +732,37 @@ function agentContext(
|
|
|
497
732
|
// than implied: told to read `../../knowledge/index.md`, the model asked for
|
|
498
733
|
// `/knowledge/index.md` twice before working it out. Two denials and a turn,
|
|
499
734
|
// for want of one sentence.
|
|
735
|
+
// The language this step works in, resolved before the prompt is built:
|
|
736
|
+
// its sentence goes in here, the env the tools read is set below, and the
|
|
737
|
+
// step prints any level that wrote something unreadable.
|
|
738
|
+
const language = agentLanguage(agentDir, tenant, front);
|
|
739
|
+
const region = agentRegion(agentDir, tenant, front, language.language);
|
|
740
|
+
// The clock is resolved further down for the env; the locale needs its
|
|
741
|
+
// zone now so "today" in a Jalali or Hijri calendar is today where the
|
|
742
|
+
// step works, not in UTC.
|
|
743
|
+
const localeZone = agentClock(agentDir, tenant, front, flowTimezone).timezone;
|
|
744
|
+
const locale: LocaleFacts = deriveLocale(language.language, region.region, agentLocaleOverrides(agentDir, tenant, front), localeZone);
|
|
745
|
+
// Said only when someone set it: an agent that never mentions a language
|
|
746
|
+
// or a region reads exactly the prompt it always did.
|
|
747
|
+
const languageSet = language.from !== "default" && language.from !== "env";
|
|
748
|
+
const regionSet = region.from !== "none" && region.from !== "env" && region.from !== "language";
|
|
749
|
+
if (languageSet || regionSet) {
|
|
750
|
+
parts.push(
|
|
751
|
+
`# Locale\n\n${localeProse(locale, languageName(language.language), regionName(region.region))} ` +
|
|
752
|
+
`Write in ${languageName(language.language)} unless the step you are given says otherwise; ` +
|
|
753
|
+
`use these currency, units and calendar conventions in anything you produce. ` +
|
|
754
|
+
`Your tools already know: a search asks in it and for this country, the browser reports it, a fetch requests it.`,
|
|
755
|
+
);
|
|
756
|
+
}
|
|
500
757
|
parts.push(
|
|
501
758
|
`# Where you are\n\n` +
|
|
502
759
|
`Your working directory is \`agents/${path.basename(agentDir)}/\` inside the ` +
|
|
503
760
|
`\`${workspace}\` workspace. Every path in this prompt is relative to it — ` +
|
|
504
|
-
`\`outputs/\` is yours, and
|
|
505
|
-
`
|
|
506
|
-
`workspace
|
|
761
|
+
`\`outputs/\` is yours, and \`workspace/\` is the workspace root, in every tool ` +
|
|
762
|
+
`and in the shell: the shared file store is \`workspace/storage/\`, what runs ` +
|
|
763
|
+
`carry forward is \`workspace/state/\`. (\`../../\` reaches the same root and ` +
|
|
764
|
+
`still works, so a path like \`../../knowledge/\` below is \`workspace/knowledge/\`.) ` +
|
|
765
|
+
`Absolute paths are outside the workspace and will be refused.\n\n` +
|
|
507
766
|
// A flow's whole point is that a later step works on what an earlier one
|
|
508
767
|
// produced, and each agent writes to its own outputs/. A run showed what
|
|
509
768
|
// omitting this costs: the checker looked in its own empty outputs/ and
|
|
@@ -524,10 +783,18 @@ function agentContext(
|
|
|
524
783
|
// adoptLegacyFilesDir's "move files/ to storage/" is a no-op by the time
|
|
525
784
|
// the run ends — a deliverable written where this text pointed was
|
|
526
785
|
// harvested from nowhere and silently never appeared.
|
|
527
|
-
|
|
786
|
+
`\`workspace/storage/\` is the workspace file store: anything you leave there is kept ` +
|
|
528
787
|
`after the run and shown on the Storage page for people to download. Write ` +
|
|
529
|
-
`deliverables people asked for — CSVs, reports, PDFs, images — to
|
|
530
|
-
`use \`outputs/\` for working text the next step reads
|
|
788
|
+
`deliverables people asked for — CSVs, reports, PDFs, images — to \`workspace/storage/\`; ` +
|
|
789
|
+
`use \`outputs/\` for working text the next step reads. ` +
|
|
790
|
+
// A Glob PATTERN that climbs (`../../storage/draft/*.md`) always answered
|
|
791
|
+
// "No files found", even when the file was there. On blog-desk that made
|
|
792
|
+
// every loop pass look like a first pass: the illustrator never saw the
|
|
793
|
+
// fact-check report, re-picked the cover, and 2 of 3 publish runs failed
|
|
794
|
+
// (run-munef2am-439a, 2026-09-30). checkPaths now splits such a pattern
|
|
795
|
+
// into Glob's `path` and the rest; the advice stays, as the plain form.
|
|
796
|
+
`To look for files there, open a known file with Read by its exact path, or ` +
|
|
797
|
+
`Glob with \`path: "workspace/storage/..."\` and a pattern like \`*.md\`.`,
|
|
531
798
|
);
|
|
532
799
|
|
|
533
800
|
// Shared context before anything derived — an account or workspace rule is
|
|
@@ -681,7 +948,7 @@ function agentContext(
|
|
|
681
948
|
// for it. The difference is that state is small enough that inlining would
|
|
682
949
|
// usually be free — that is a threshold worth adding once there is a real
|
|
683
950
|
// file to size it against, not before.
|
|
684
|
-
const state = listDir(path.join(agentDir, "..", "..", "state"), "
|
|
951
|
+
const state = listDir(path.join(agentDir, "..", "..", "state"), "workspace/state/");
|
|
685
952
|
if (state.length) {
|
|
686
953
|
parts.push(
|
|
687
954
|
`# State — what you carry between runs\n\n` +
|
|
@@ -719,16 +986,29 @@ function agentContext(
|
|
|
719
986
|
// below. The tool is the unit of code, so its dependencies live in tool.md
|
|
720
987
|
// and travel with it; the agent should not have to repeat them.
|
|
721
988
|
const toolRuntimes: RuntimeSpec[] = [];
|
|
989
|
+
// The secrets a granted script tool's file lists (`secrets: [RESEND_API_KEY,
|
|
990
|
+
// EMAIL_FROM]`): granting the tool grants them, the way an API tool's
|
|
991
|
+
// referenced credentials always travelled with it. Before, only the
|
|
992
|
+
// agent's own `secrets:` counted, so an agent that granted desk_email and
|
|
993
|
+
// declared nothing failed with "RESEND_API_KEY is not set" (strata-desk,
|
|
994
|
+
// 1 Oct). `proxied` / `materialised` say how, not what, and add nothing.
|
|
995
|
+
const toolSecrets: string[] = [];
|
|
996
|
+
// Does any granted tool act outside the workspace (`outward: true`)? The
|
|
997
|
+
// retry policy reads it: see launchStep.
|
|
998
|
+
let outward = false;
|
|
722
999
|
const grantOwnTool = (name: string) => {
|
|
723
1000
|
const def = available[name];
|
|
724
1001
|
if (!def) {
|
|
725
1002
|
missingTools.push(name);
|
|
726
1003
|
return;
|
|
727
1004
|
}
|
|
1005
|
+
if (def.outward) outward = true;
|
|
728
1006
|
if (def.kind === "http") apis.push(def.spec);
|
|
729
1007
|
else if (def.kind === "script") {
|
|
730
1008
|
scriptSpecs.push(...parseScripts([def.spec]));
|
|
731
|
-
const
|
|
1009
|
+
const listed = (def.spec as { secrets?: unknown }).secrets;
|
|
1010
|
+
if (Array.isArray(listed)) toolSecrets.push(...listed.map(String).filter((n) => /^[A-Z][A-Z0-9_]*$/.test(n)));
|
|
1011
|
+
const rt = toolRuntime(def);
|
|
732
1012
|
if (rt) toolRuntimes.push(rt);
|
|
733
1013
|
}
|
|
734
1014
|
else {
|
|
@@ -773,7 +1053,65 @@ function agentContext(
|
|
|
773
1053
|
//
|
|
774
1054
|
// Declared-and-absent is still an error; referenced-and-absent is the API
|
|
775
1055
|
// tool's own missingSecrets, reported where the tool is described.
|
|
776
|
-
|
|
1056
|
+
// `web: {search: …, fetch: …}` name who answers; resolved here, early,
|
|
1057
|
+
// because three later things hang off the answer — the secret the step
|
|
1058
|
+
// must hold, the env the tool reads, and the egress grant for the host.
|
|
1059
|
+
// `web.search` carries the account engine's settings (engines, categories,
|
|
1060
|
+
// safesearch, …) beside the provider name, read off first the way
|
|
1061
|
+
// browse's are, so the provider resolver sees only its own part.
|
|
1062
|
+
const web = webConfig(front as Record<string, unknown>, workspaceFrontmatter(agentDir, tenant));
|
|
1063
|
+
const searchRead = readSearchSettings(web.raw.search);
|
|
1064
|
+
const searchChoice = resolveSearch(searchRead.rest, "search");
|
|
1065
|
+
if (searchRead.error) searchChoice.error = searchChoice.error ?? searchRead.error;
|
|
1066
|
+
const fetchChoice = resolveSearch(web.raw.fetch, "fetch");
|
|
1067
|
+
// A remote browser's key is materialised, not proxied: CDP is a websocket
|
|
1068
|
+
// and the wrapper opens the session itself, the way it seeds a cookie.
|
|
1069
|
+
// `web.browse` carries two things in one key: how the browser presents
|
|
1070
|
+
// itself (engine, user agent, device, locale, timezone) and, optionally,
|
|
1071
|
+
// which remote vendor renders the page. Settings are read off first so the
|
|
1072
|
+
// vendor resolver sees only the vendor part, and a block of settings alone
|
|
1073
|
+
// means the account's own browser with those defaults. Nearest wins: the
|
|
1074
|
+
// agent's block, else the workspace's, else the account's — the shape
|
|
1075
|
+
// `provider:` already uses.
|
|
1076
|
+
const browseRead = readBrowseSettings(web.raw.browse);
|
|
1077
|
+
const browseSettings = browseRead.settings;
|
|
1078
|
+
const browseChoice = resolveSearch(browseRead.rest, "browse");
|
|
1079
|
+
if (browseRead.error) browseChoice.error = browseChoice.error ?? browseRead.error;
|
|
1080
|
+
// What the vendor's own session is asked for, checked against what that
|
|
1081
|
+
// vendor offers; a problem is on the record and the session is not sent.
|
|
1082
|
+
const sessionProblems = browseChoice.error ? [] : browseSessionProblems(browseSettings, browseChoice.provider);
|
|
1083
|
+
const browseSession = browseChoice.shape === "direct" && browseChoice.provider && !sessionProblems.length ? browseSettings.vendor_session : undefined;
|
|
1084
|
+
// crawl, map, extract, answer, monitor: a provider each, or foldrun's own.
|
|
1085
|
+
// Same three consequences as a search API — the secret the step holds, the
|
|
1086
|
+
// env the web tool reads, the one host the key may reach.
|
|
1087
|
+
const actionChoices = NEWER_ACTIONS.map((action) => ({ action, ...resolveActionApi(action, web.raw[action]) }));
|
|
1088
|
+
const builtins = [
|
|
1089
|
+
...(searchChoice.provider && searchChoice.shape !== "direct" && !searchChoice.error ? ["search=WebSearch"] : []),
|
|
1090
|
+
...(fetchChoice.provider && fetchChoice.shape !== "direct" && !fetchChoice.error ? ["fetch=WebFetch"] : []),
|
|
1091
|
+
];
|
|
1092
|
+
const webBuiltin: Record<string, string> = builtins.length ? { FOLDRUN_WEB_BUILTIN: builtins.join(",") } : {};
|
|
1093
|
+
// A direct search API is paid for with the customer's own key, stored
|
|
1094
|
+
// under a fixed vault name (EXA_API_KEY, BRAVE_SEARCH_API_KEY, …). Naming
|
|
1095
|
+
// the API in frontmatter is declaring that secret — nobody should have to
|
|
1096
|
+
// write it twice, and a run that forgot the second line would search
|
|
1097
|
+
// nothing and say so only at the provider.
|
|
1098
|
+
const declared: string[] = [
|
|
1099
|
+
...(Array.isArray(front.secrets) ? front.secrets.map(String) : []),
|
|
1100
|
+
...toolSecrets,
|
|
1101
|
+
...actionChoices.flatMap((c) => (c.provider ? [c.secret] : [])),
|
|
1102
|
+
// Your own proxy for a vendor's session: its URL is a secret, declared by
|
|
1103
|
+
// naming it, like a provider's key.
|
|
1104
|
+
...(browseSession && typeof browseSession.proxy === "object" && browseSession.proxy.own ? [browseSession.proxy.own] : []),
|
|
1105
|
+
...[searchChoice, fetchChoice, browseChoice].flatMap((c) => {
|
|
1106
|
+
if (c.shape !== "direct" || !c.secret) return [];
|
|
1107
|
+
// Jina's reader answers without a key. Declaring an absent optional
|
|
1108
|
+
// key would fail the step for a credential it never needed.
|
|
1109
|
+
if (c.secretOptional && !getSecret(tenant, c.secret, workspace)) return [];
|
|
1110
|
+
return [c.secret];
|
|
1111
|
+
}),
|
|
1112
|
+
];
|
|
1113
|
+
// One name once, however many places asked for it.
|
|
1114
|
+
declared.splice(0, declared.length, ...new Set(declared));
|
|
777
1115
|
const referenced = [...new Set(apis.flatMap(secretsUsedByApi))];
|
|
778
1116
|
const {
|
|
779
1117
|
env: secretEnv,
|
|
@@ -888,24 +1226,17 @@ function agentContext(
|
|
|
888
1226
|
// Runtime: an agent's own `runtime:` block, else its workspace's. Built once
|
|
889
1227
|
// per declaration and cached, so scripts get their dependencies without
|
|
890
1228
|
// polluting the host.
|
|
891
|
-
let runtimeSpec = parseRuntime(front.runtime);
|
|
892
|
-
if (!runtimeSpec) {
|
|
893
|
-
// AGENTS.md is the Linux Foundation standard name (60k+ repos, same
|
|
894
|
-
// nearest-wins cascade we already use); project.md stays accepted so
|
|
895
|
-
// nothing that already exists breaks.
|
|
896
|
-
runtimeSpec = parseRuntime(workspaceFrontmatter(agentDir, tenant).runtime);
|
|
897
|
-
}
|
|
898
1229
|
// Plus whatever the tools this agent granted declared for themselves.
|
|
899
|
-
runtimeSpec = mergeRuntimes(
|
|
1230
|
+
let runtimeSpec = mergeRuntimes(ownRuntime(front, agentDir, tenant), ...toolRuntimes);
|
|
900
1231
|
// Execution: a container per run when Docker is available (real isolation),
|
|
901
1232
|
// otherwise the host venv path so local development still works.
|
|
902
1233
|
const executor = chooseExecutor();
|
|
903
1234
|
const runtimeLog: string[] = [];
|
|
904
1235
|
let runtimeError: string | null = null;
|
|
905
1236
|
let exec: ExecutionContext | null = null;
|
|
906
|
-
let runtime = { interpreters: {}
|
|
1237
|
+
let runtime: PreparedRuntime = { interpreters: {}, env: {}, log: [], error: null };
|
|
907
1238
|
|
|
908
|
-
if (executor === "docker") {
|
|
1239
|
+
if (executor === "docker" && !grantsOnly) {
|
|
909
1240
|
const image = ensureImage(runtimeSpec);
|
|
910
1241
|
runtimeLog.push(...image.log);
|
|
911
1242
|
runtimeError = image.error;
|
|
@@ -922,23 +1253,103 @@ function agentContext(
|
|
|
922
1253
|
network: parseApis(front.apis).length > 0 || ownToolNames(front).length > 0,
|
|
923
1254
|
};
|
|
924
1255
|
}
|
|
1256
|
+
} else if (isolatedRun() || grantsOnly) {
|
|
1257
|
+
// (A sub-agent's grants are all that is asked of it: it runs inside its
|
|
1258
|
+
// parent's step and runtime.)
|
|
1259
|
+
// The sandbox builds it (run-container.ts, beside pip's network and the
|
|
1260
|
+
// mounted cache); building it here as well was pure waste, and on a
|
|
1261
|
+
// worker image without python it was worse than waste — it logged
|
|
1262
|
+
// "python3 is not available on this host" as an error on every Python
|
|
1263
|
+
// step, above a sandbox build that had succeeded, and sent every reader
|
|
1264
|
+
// of the trace after the wrong machine.
|
|
925
1265
|
} else {
|
|
926
1266
|
runtime = prepareRuntime(tenant, runtimeSpec);
|
|
927
1267
|
runtimeLog.push(...runtime.log);
|
|
928
1268
|
runtimeError = runtime.error;
|
|
929
1269
|
}
|
|
930
1270
|
|
|
931
|
-
// What day it is, in the
|
|
932
|
-
//
|
|
933
|
-
//
|
|
934
|
-
|
|
1271
|
+
// What day it is, in the calendar this step works to — the agent's own
|
|
1272
|
+
// `timezone:`, else its flow's, else the workspace's, else the account's.
|
|
1273
|
+
// Not a secret, so it travels beside secretEnv rather than inside it — a
|
|
1274
|
+
// value in there is redacted from every log line that quotes it.
|
|
1275
|
+
const clock = agentClock(agentDir, tenant, front, flowTimezone);
|
|
1276
|
+
const timezone = clock.timezone;
|
|
935
1277
|
const clockEnv = {
|
|
936
|
-
|
|
1278
|
+
// The language the agent works in — what the search asks in, the locale
|
|
1279
|
+
// the browser reports, the Accept-Language a fetch sends. Not a secret.
|
|
1280
|
+
FOLDRUN_LANGUAGE: language.language,
|
|
1281
|
+
// The country and what follows from it — web search's gl, a script's
|
|
1282
|
+
// currency. Unset where nobody said, so nothing is invented.
|
|
1283
|
+
...(region.region ? { FOLDRUN_REGION: region.region } : {}),
|
|
1284
|
+
...(locale.currency ? { FOLDRUN_CURRENCY: locale.currency } : {}),
|
|
1285
|
+
FOLDRUN_UNITS: locale.units,
|
|
1286
|
+
FOLDRUN_CALENDAR: locale.calendar,
|
|
1287
|
+
...(locale.dateLocal ? { FOLDRUN_DATE_LOCAL: locale.dateLocal } : {}),
|
|
1288
|
+
FOLDRUN_RTL: locale.direction === "rtl" ? "1" : "0",
|
|
1289
|
+
// TZ is the form the sandbox's shell and Node both read; for a fixed
|
|
1290
|
+
// offset that is not the same string Intl was given (see clock.ts).
|
|
1291
|
+
TZ: clock.tz,
|
|
937
1292
|
FOLDRUN_DATE: localDate(timezone),
|
|
938
1293
|
// Where the account's own search engine answers, when the install runs
|
|
939
1294
|
// one — the `websearch` library tool reads it. Not a secret; rides with
|
|
940
1295
|
// the clock so it reaches scripts, the sandbox and verify alike.
|
|
941
1296
|
...(process.env.FOLDRUN_SEARCH_URL ? { FOLDRUN_SEARCH_URL: process.env.FOLDRUN_SEARCH_URL } : {}),
|
|
1297
|
+
// Whose index our web search asks when the agent named a search API.
|
|
1298
|
+
// Unset means the account's own engine. The name, never the key.
|
|
1299
|
+
...(searchChoice.shape === "direct" && searchChoice.provider
|
|
1300
|
+
? { FOLDRUN_WEB_SEARCH_VIA: searchChoice.provider, FOLDRUN_WEB_SEARCH_SECRET: searchChoice.secret ?? "" }
|
|
1301
|
+
: {}),
|
|
1302
|
+
// The `web.search` block's SearXNG settings, as defaults the tool reads.
|
|
1303
|
+
...searchSettingsEnv(searchRead.settings),
|
|
1304
|
+
...(fetchChoice.shape === "direct" && fetchChoice.provider
|
|
1305
|
+
? { FOLDRUN_WEB_FETCH_VIA: fetchChoice.provider, FOLDRUN_WEB_FETCH_SECRET: fetchChoice.secret ?? "" }
|
|
1306
|
+
: {}),
|
|
1307
|
+
...Object.fromEntries(actionChoices.flatMap((c) => (c.provider
|
|
1308
|
+
? [[`FOLDRUN_WEB_${c.action.toUpperCase()}_VIA`, c.provider], [`FOLDRUN_WEB_${c.action.toUpperCase()}_SECRET`, c.secret]]
|
|
1309
|
+
: []))),
|
|
1310
|
+
// Which actions the web tool may take (`web: {actions: [...]}`); unset is all.
|
|
1311
|
+
...(web.actions ? { FOLDRUN_WEB_ACTIONS: web.actions.join(",") } : {}),
|
|
1312
|
+
// A model provider's own search or fetch, which the web tool hands over to.
|
|
1313
|
+
...webBuiltin,
|
|
1314
|
+
// A remote browser: the wrapper connects over CDP to this vendor with
|
|
1315
|
+
// the named key, and the platform leaves the account's pod alone.
|
|
1316
|
+
...(browseChoice.shape === "direct" && browseChoice.provider
|
|
1317
|
+
? { FOLDRUN_BROWSER_VENDOR: browseChoice.provider, FOLDRUN_BROWSER_SECRET: browseChoice.secret ?? "" }
|
|
1318
|
+
: {}),
|
|
1319
|
+
// The vendor session's settings (`web.browse.vendor_session:`), as JSON
|
|
1320
|
+
// the tool maps onto that vendor's own option names. Under both names
|
|
1321
|
+
// for one release: a web tool from before the rename reads the old one.
|
|
1322
|
+
...(browseSession ? { FOLDRUN_BROWSER_VENDOR_SESSION: JSON.stringify(browseSession), FOLDRUN_BROWSER_SESSION: JSON.stringify(browseSession) } : {}),
|
|
1323
|
+
// The `web.browse` block, as defaults the tool reads when the call did
|
|
1324
|
+
// not say otherwise. Env rather than arguments because the tool is a
|
|
1325
|
+
// script the model calls: a default nobody typed should not have to be
|
|
1326
|
+
// typed by the model either.
|
|
1327
|
+
...(browseSettings.engine ? { FOLDRUN_BROWSER_ENGINE: browseSettings.engine } : {}),
|
|
1328
|
+
...(browseSettings.user_agent ? { FOLDRUN_BROWSER_USER_AGENT: browseSettings.user_agent } : {}),
|
|
1329
|
+
...(browseSettings.device ? { FOLDRUN_BROWSER_DEVICE: browseSettings.device } : {}),
|
|
1330
|
+
...(browseSettings.locale ? { FOLDRUN_BROWSER_LOCALE: browseSettings.locale } : {}),
|
|
1331
|
+
...(browseSettings.timezone ? { FOLDRUN_BROWSER_TIMEZONE: browseSettings.timezone } : {}),
|
|
1332
|
+
...(browseSettings.cookies ? { FOLDRUN_BROWSER_COOKIES: browseSettings.cookies } : {}),
|
|
1333
|
+
...(browseSettings.cookie_domain ? { FOLDRUN_BROWSER_COOKIE_DOMAIN: browseSettings.cookie_domain } : {}),
|
|
1334
|
+
...(browseSettings.storage ? { FOLDRUN_BROWSER_STORAGE: browseSettings.storage } : {}),
|
|
1335
|
+
...(browseSettings.storage_origin ? { FOLDRUN_BROWSER_STORAGE_ORIGIN: browseSettings.storage_origin } : {}),
|
|
1336
|
+
// The named identities, as one JSON map the tool picks from by name.
|
|
1337
|
+
...(browseSettings.identities ? { FOLDRUN_BROWSER_IDENTITIES: JSON.stringify(browseSettings.identities) } : {}),
|
|
1338
|
+
// Only the non-default travels: a window, on a virtual screen if need be.
|
|
1339
|
+
...(browseSettings.headless === false ? { FOLDRUN_BROWSER_HEADLESS: "0" } : {}),
|
|
1340
|
+
...(browseSettings.version ? { FOLDRUN_BROWSER_VERSION: browseSettings.version } : {}),
|
|
1341
|
+
...(browseSettings.live ? { FOLDRUN_BROWSER_LIVE: "1" } : {}),
|
|
1342
|
+
// What the agent may do, from its file: a lock the call cannot widen.
|
|
1343
|
+
...(browseSettings.allowed_domains ? { FOLDRUN_BROWSER_ALLOWED_DOMAINS: browseSettings.allowed_domains.join(",") } : {}),
|
|
1344
|
+
...(browseSettings.deny ? { FOLDRUN_BROWSER_DENY: browseSettings.deny.join(",") } : {}),
|
|
1345
|
+
...(browseSettings.boundaries ? { FOLDRUN_BROWSER_BOUNDARIES: "1" } : {}),
|
|
1346
|
+
...(browseSettings.init ? { FOLDRUN_BROWSER_INIT: browseSettings.init.join(",") } : {}),
|
|
1347
|
+
...(browseSettings.extensions ? { FOLDRUN_BROWSER_EXTENSIONS: browseSettings.extensions.join(",") } : {}),
|
|
1348
|
+
...(browseSettings.webgpu ? { FOLDRUN_BROWSER_WEBGPU: "1" } : {}),
|
|
1349
|
+
...(browseSettings.ignore_https_errors ? { FOLDRUN_BROWSER_IGNORE_HTTPS_ERRORS: "1" } : {}),
|
|
1350
|
+
...(browseSettings.state_key ? { FOLDRUN_BROWSER_STATE_KEY: browseSettings.state_key } : {}),
|
|
1351
|
+
...(browseSettings.video ? { FOLDRUN_BROWSER_VIDEO: "1" } : {}),
|
|
1352
|
+
...(browseSettings.live_view === false ? { FOLDRUN_BROWSER_LIVE_VIEW: "0" } : {}),
|
|
942
1353
|
};
|
|
943
1354
|
|
|
944
1355
|
// Scripts declared as tools — callable by name, no bash required.
|
|
@@ -983,7 +1394,7 @@ function agentContext(
|
|
|
983
1394
|
parts.push(
|
|
984
1395
|
"Write deliverables for people to workspace/storage/ (kept and downloadable); write working text for later steps to outputs/; " +
|
|
985
1396
|
"what the next run needs goes to workspace/state/. Paths are relative to your agent directory, or start with workspace/ " +
|
|
986
|
-
"for the workspace root (
|
|
1397
|
+
"for the workspace root (workspace/storage/ and the older ../../storage/ are the same place). There is no /tmp for you: a " +
|
|
987
1398
|
"directory a tool made outside the workspace, such as a checkout, is reached only through that tool.",
|
|
988
1399
|
);
|
|
989
1400
|
|
|
@@ -1010,9 +1421,37 @@ function agentContext(
|
|
|
1010
1421
|
const disabled: string[] = [];
|
|
1011
1422
|
const unknownTools: string[] = [];
|
|
1012
1423
|
const shadowed: string[] = [];
|
|
1424
|
+
// `web: {search: …, fetch: …}` name who answers. Unset means ours — the
|
|
1425
|
+
// account's own engine, and our own fetch, both inside the run's sandbox
|
|
1426
|
+
// and both on the run record. A model provider's name adds the SDK's
|
|
1427
|
+
// server-side tool beside ours, which the run's endpoint executes: point
|
|
1428
|
+
// the run at z.ai and z.ai's index answers the search. The web tool hands
|
|
1429
|
+
// the action over to it (FOLDRUN_WEB_BUILTIN).
|
|
1430
|
+
const providerWebTools: string[] = [];
|
|
1431
|
+
if (browseChoice.error) providerWarnings.push(browseChoice.error);
|
|
1432
|
+
providerWarnings.push(
|
|
1433
|
+
...sessionProblems,
|
|
1434
|
+
...web.problems,
|
|
1435
|
+
...actionChoices.flatMap((c) => (c.error ? [c.error] : [])),
|
|
1436
|
+
);
|
|
1437
|
+
for (const [choice, builtin] of [
|
|
1438
|
+
[searchChoice, "WebSearch"],
|
|
1439
|
+
[fetchChoice, "WebFetch"],
|
|
1440
|
+
] as const) {
|
|
1441
|
+
if (choice.error) {
|
|
1442
|
+
providerWarnings.push(choice.error);
|
|
1443
|
+
} else if (choice.provider && choice.shape !== "direct") {
|
|
1444
|
+
// A provider's server-side tool replaces ours. A direct search API
|
|
1445
|
+
// does not: our tool stays, and reads FOLDRUN_WEB_SEARCH_VIA to know
|
|
1446
|
+
// whose index to ask — through the egress proxy, with the customer's
|
|
1447
|
+
// own key, on the run record.
|
|
1448
|
+
providerWebTools.push(builtin);
|
|
1449
|
+
}
|
|
1450
|
+
}
|
|
1013
1451
|
let wantSearch = false;
|
|
1014
1452
|
let wantHistory = false;
|
|
1015
1453
|
let wantDesks = false;
|
|
1454
|
+
let wantAsk = false;
|
|
1016
1455
|
|
|
1017
1456
|
for (const ref of toolRefs(front)) {
|
|
1018
1457
|
const toolName = ref.name;
|
|
@@ -1020,6 +1459,14 @@ function agentContext(
|
|
|
1020
1459
|
disabled.push(toolName);
|
|
1021
1460
|
continue;
|
|
1022
1461
|
}
|
|
1462
|
+
// `web` is our own tool, granted with the rest of them above. A model
|
|
1463
|
+
// provider answering search or fetch on its own side (`web: {search:
|
|
1464
|
+
// zai}`) adds that provider's tool beside it; the web tool's search then
|
|
1465
|
+
// names it (FOLDRUN_WEB_BUILTIN) rather than asking our engine.
|
|
1466
|
+
if (toolName === "web" && !ref.linked) {
|
|
1467
|
+
allowed.push(...providerWebTools);
|
|
1468
|
+
continue;
|
|
1469
|
+
}
|
|
1023
1470
|
|
|
1024
1471
|
// Resolution order: built-in group, exact SDK name, then your own tools.
|
|
1025
1472
|
// Built-ins win a bare name, because their names are a small fixed set —
|
|
@@ -1031,11 +1478,12 @@ function agentContext(
|
|
|
1031
1478
|
} else if (TOOL_MAP[toolName]) {
|
|
1032
1479
|
allowed.push(...TOOL_MAP[toolName]);
|
|
1033
1480
|
if (available[toolName]) shadowed.push(toolName);
|
|
1034
|
-
} else if (toolName === "search" || toolName === "history" || toolName === "desks") {
|
|
1481
|
+
} else if (toolName === "search" || toolName === "history" || toolName === "desks" || toolName === "ask") {
|
|
1035
1482
|
// The platform's own groups, served in-process — built below, once
|
|
1036
1483
|
// the roots and the records are known.
|
|
1037
1484
|
if (toolName === "search") wantSearch = true;
|
|
1038
1485
|
else if (toolName === "history") wantHistory = true;
|
|
1486
|
+
else if (toolName === "ask") wantAsk = true;
|
|
1039
1487
|
else wantDesks = true;
|
|
1040
1488
|
if (available[toolName]) shadowed.push(toolName);
|
|
1041
1489
|
} else if (BUILTIN_TOOLS.has(toolName)) {
|
|
@@ -1060,8 +1508,8 @@ function agentContext(
|
|
|
1060
1508
|
{ label: "memory/", dir: path.join(agentDir, "memory") },
|
|
1061
1509
|
{ label: "../../knowledge/", dir: path.join(wsRoot, "knowledge") },
|
|
1062
1510
|
{ label: "../../memory/", dir: path.join(wsRoot, "memory") },
|
|
1063
|
-
{ label: "
|
|
1064
|
-
{ label:
|
|
1511
|
+
{ label: "workspace/state/", dir: path.join(wsRoot, "state") },
|
|
1512
|
+
{ label: `workspace/${STORAGE_DIR}/`, dir: path.join(wsRoot, STORAGE_DIR) },
|
|
1065
1513
|
{ label: "outputs/", dir: path.join(agentDir, "outputs") },
|
|
1066
1514
|
{ label: "[library] knowledge/", dir: libraryDir(tenant, "knowledge") },
|
|
1067
1515
|
{ label: "[library] memory/", dir: libraryDir(tenant, "memory") },
|
|
@@ -1083,6 +1531,14 @@ function agentContext(
|
|
|
1083
1531
|
const historyTools = wantHistory ? buildHistoryTools(historyDigest) : { server: null, toolNames: [], promptLines: [] };
|
|
1084
1532
|
const deskTools = wantDesks ? buildDeskTools(deskDigest) : { server: null, toolNames: [], promptLines: [] };
|
|
1085
1533
|
allowed.push(...searchTools.toolNames, ...historyTools.toolNames, ...deskTools.toolNames);
|
|
1534
|
+
// tools: [ask] — ask_person. Its server is built where the step runs (it
|
|
1535
|
+
// needs the step's emit, and in a pod the proxy's address); here only the
|
|
1536
|
+
// grant, the wait and the prompt's line.
|
|
1537
|
+
const ask = wantAsk ? { timeoutSec: askTimeoutSec(front) } : null;
|
|
1538
|
+
if (ask) {
|
|
1539
|
+
allowed.push(ASK_TOOL);
|
|
1540
|
+
parts.push(`# Asking a person\n\n${askPromptLine(ask.timeoutSec)}`);
|
|
1541
|
+
}
|
|
1086
1542
|
if (searchTools.promptLines.length || historyTools.promptLines.length || deskTools.promptLines.length) {
|
|
1087
1543
|
parts.push(`# Finding things\n\n${[...searchTools.promptLines, ...historyTools.promptLines, ...deskTools.promptLines].join("\n")}`);
|
|
1088
1544
|
}
|
|
@@ -1103,12 +1559,16 @@ function agentContext(
|
|
|
1103
1559
|
return {
|
|
1104
1560
|
front,
|
|
1105
1561
|
clockEnv,
|
|
1562
|
+
/** The zone, where it came from, and anything unreadable on the way —
|
|
1563
|
+
* reported by the step, not here, so it lands in the run's trace. */
|
|
1564
|
+
clock,
|
|
1106
1565
|
systemPrompt: parts.join("\n\n"),
|
|
1107
1566
|
allowed: finalAllowed,
|
|
1108
1567
|
disabled,
|
|
1109
1568
|
unknownTools,
|
|
1110
1569
|
shadowed,
|
|
1111
1570
|
legacyUse,
|
|
1571
|
+
retiredTools: retiredToolNames(front),
|
|
1112
1572
|
mcpServers,
|
|
1113
1573
|
mcpNames,
|
|
1114
1574
|
apiTools,
|
|
@@ -1118,19 +1578,21 @@ function agentContext(
|
|
|
1118
1578
|
// than using the servers built here.
|
|
1119
1579
|
apiSpecs: apis,
|
|
1120
1580
|
scriptSpecs,
|
|
1581
|
+
outward: outward || scriptSpecs.some((sc) => sc.outward),
|
|
1121
1582
|
searchRoots,
|
|
1122
1583
|
historyDigest,
|
|
1123
1584
|
deskDigest,
|
|
1124
1585
|
searchTools,
|
|
1125
1586
|
historyTools,
|
|
1126
1587
|
deskTools,
|
|
1588
|
+
ask,
|
|
1127
1589
|
// `spec` is the MERGED declaration — the agent's own block or its
|
|
1128
1590
|
// workspace's, plus every granted tool's. The isolated path builds its
|
|
1129
1591
|
// runtime from this; passing the agent's frontmatter alone silently drops
|
|
1130
1592
|
// what the tools declared, and a tool whose program needs a package it
|
|
1131
1593
|
// asked for by name fails inside the container with "cannot find" while
|
|
1132
1594
|
// the host path, which uses this same merged spec, works fine.
|
|
1133
|
-
runtime: { log: runtimeLog, error: runtimeError, executor, spec: runtimeSpec },
|
|
1595
|
+
runtime: { log: runtimeLog, error: runtimeError, executor, spec: runtimeSpec, dispose: runtime.dispose },
|
|
1134
1596
|
secretEnv: { ...runtime.env, ...secretEnv },
|
|
1135
1597
|
formatWarning: checkFormatVersion(workspaceFrontmatter(agentDir, tenant).foldrun_version).warning,
|
|
1136
1598
|
providerEnv,
|
|
@@ -1138,8 +1600,19 @@ function agentContext(
|
|
|
1138
1600
|
providerSecrets,
|
|
1139
1601
|
providerWarnings,
|
|
1140
1602
|
fallbackEnv,
|
|
1141
|
-
|
|
1142
|
-
|
|
1603
|
+
region,
|
|
1604
|
+
// Who answers web search and fetch, resolved once up top. The step
|
|
1605
|
+
// needs them again to grant a direct API's key to its host.
|
|
1606
|
+
searchChoice,
|
|
1607
|
+
fetchChoice,
|
|
1608
|
+
actionChoices,
|
|
1609
|
+
web,
|
|
1610
|
+
language,
|
|
1611
|
+
// Stamped with the step's own zone: when one of these refuses, the
|
|
1612
|
+
// reset time it reports is told in the calendar the step works to,
|
|
1613
|
+
// not in the pod's UTC.
|
|
1614
|
+
translator: translator && { ...translator, timezone },
|
|
1615
|
+
fallbackTranslator: fallbackTranslator && { ...fallbackTranslator, timezone },
|
|
1143
1616
|
apiWarnings,
|
|
1144
1617
|
secretScopes,
|
|
1145
1618
|
missingTools,
|
|
@@ -1169,34 +1642,136 @@ function agentContext(
|
|
|
1169
1642
|
* Never fails a step: a link is bookkeeping, and a run that did its work must
|
|
1170
1643
|
* not be marked failed because a URL could not be minted.
|
|
1171
1644
|
*/
|
|
1172
|
-
function publishPublicDir(
|
|
1645
|
+
async function publishPublicDir(
|
|
1173
1646
|
tenant: string,
|
|
1174
1647
|
workspace: string,
|
|
1175
1648
|
push: (type: "info" | "error", text: string) => void,
|
|
1176
|
-
) {
|
|
1649
|
+
): Promise<void> {
|
|
1177
1650
|
try {
|
|
1178
|
-
const { added } = platform.syncPublicShares(tenant, workspace);
|
|
1651
|
+
const { added } = await platform.syncPublicShares(tenant, workspace);
|
|
1179
1652
|
if (added.length) push("info", `shared: ${added.join(", ")}`);
|
|
1180
1653
|
} catch (err) {
|
|
1181
1654
|
push("error", `shares: ${err instanceof Error ? err.message : String(err)}`);
|
|
1182
1655
|
}
|
|
1183
1656
|
}
|
|
1184
1657
|
|
|
1185
|
-
function repriced(
|
|
1658
|
+
export function repriced(
|
|
1186
1659
|
catalog: Catalog | null,
|
|
1187
1660
|
model: string,
|
|
1188
1661
|
usage: { inputTokens: number; outputTokens: number } | null,
|
|
1189
1662
|
sdkCost: number | null,
|
|
1190
1663
|
push: (type: "info" | "error", text: string) => void,
|
|
1664
|
+
/** The step's turns each priced at its own model — set when a sub-agent
|
|
1665
|
+
* ran on a different model (ExecOutcome.turnsCostUsd). The lump `usage`
|
|
1666
|
+
* priced at the step's model would bill those turns at the wrong rate. */
|
|
1667
|
+
turnsCostUsd?: number | null,
|
|
1191
1668
|
): number | null {
|
|
1192
1669
|
const known = usage ? catalogCost(catalog, model, usage) : null;
|
|
1193
1670
|
if (known === null) return sdkCost;
|
|
1671
|
+
if (typeof turnsCostUsd === "number") {
|
|
1672
|
+
push("info", `cost from each model turn at its own model's rate: $${turnsCostUsd.toFixed(6)} (sub-agents ran on other models)`);
|
|
1673
|
+
return turnsCostUsd;
|
|
1674
|
+
}
|
|
1194
1675
|
if (sdkCost !== null && Math.abs(known - sdkCost) > 0.000001) {
|
|
1195
1676
|
push("info", `cost repriced from the gateway's catalogue: $${known.toFixed(6)} (sdk said $${sdkCost.toFixed(6)})`);
|
|
1196
1677
|
}
|
|
1197
1678
|
return known;
|
|
1198
1679
|
}
|
|
1199
1680
|
|
|
1681
|
+
/**
|
|
1682
|
+
* Write one attempt onto the step's history and re-total the step.
|
|
1683
|
+
*
|
|
1684
|
+
* runStep reports the attempt it just ran — its cost, tokens and sandbox
|
|
1685
|
+
* seconds — by assigning the step's fields. Those become this attempt's row;
|
|
1686
|
+
* the step's fields then become the sum of every row, which is what the
|
|
1687
|
+
* run's spend, the flow's budget: and the platform's bill read. A retried
|
|
1688
|
+
* step used to carry only its last attempt's cost, so a `retry: 2` step
|
|
1689
|
+
* could spend three shares while reporting one.
|
|
1690
|
+
*/
|
|
1691
|
+
export function recordAttempt(
|
|
1692
|
+
step: StepRecord,
|
|
1693
|
+
/** The step's attempt (`attempts`) these rows belong to. */
|
|
1694
|
+
attempt: number,
|
|
1695
|
+
status: "completed" | "failed",
|
|
1696
|
+
startedAt: string,
|
|
1697
|
+
eventsBefore: number,
|
|
1698
|
+
/** The attempt re-attached to a sandbox a previous driver left: same
|
|
1699
|
+
* attempt, so its row is replaced rather than repeated. */
|
|
1700
|
+
reattach = false,
|
|
1701
|
+
) {
|
|
1702
|
+
const tries = step.tries ?? [];
|
|
1703
|
+
const lastError = step.events
|
|
1704
|
+
.slice(eventsBefore)
|
|
1705
|
+
.filter((e) => e.type === "error")
|
|
1706
|
+
.at(-1)?.text;
|
|
1707
|
+
// Re-attached: the same attempt, so its rows — the last one, and the lost
|
|
1708
|
+
// slim go before it — are replaced, not repeated.
|
|
1709
|
+
const kept = [...tries];
|
|
1710
|
+
const attemptOf = (t: StepAttempt) => t.attempt ?? t.n;
|
|
1711
|
+
if (reattach && kept.length && attemptOf(kept.at(-1)!) === attempt) {
|
|
1712
|
+
kept.pop();
|
|
1713
|
+
if (kept.length && attemptOf(kept.at(-1)!) === attempt && kept.at(-1)!.status === "lost") kept.pop();
|
|
1714
|
+
}
|
|
1715
|
+
// Tries are numbered in the order they ran: a full re-run after a lost
|
|
1716
|
+
// slim go is the next try, not the same number again (live, 2026-10-01,
|
|
1717
|
+
// both rows read n: 1). `attempt` says which attempt a row belongs to
|
|
1718
|
+
// when its number no longer does.
|
|
1719
|
+
let next = (kept.at(-1)?.n ?? 0) + 1;
|
|
1720
|
+
const numbered = () => {
|
|
1721
|
+
const n = next++;
|
|
1722
|
+
return n === attempt ? { n } : { n, attempt };
|
|
1723
|
+
};
|
|
1724
|
+
// A slim go the browser pod was lost under, before this attempt re-ran
|
|
1725
|
+
// on the full image: its own row, first (runStep set it aside).
|
|
1726
|
+
const lost: StepAttempt[] = step.podLostTry ? [{ ...numbered(), ...step.podLostTry, startedAt }] : [];
|
|
1727
|
+
const row: StepAttempt = {
|
|
1728
|
+
...numbered(),
|
|
1729
|
+
status,
|
|
1730
|
+
costUsd: step.costUsd,
|
|
1731
|
+
tokens: step.tokens ?? null,
|
|
1732
|
+
computeSecs: step.computeSecs ?? null,
|
|
1733
|
+
startedAt: step.podLostTry?.finishedAt ?? startedAt,
|
|
1734
|
+
finishedAt: step.finishedAt ?? new Date().toISOString(),
|
|
1735
|
+
...(status === "failed" && lastError ? { error: lastError.slice(0, 500) } : {}),
|
|
1736
|
+
...(step.image?.variant ? { image: step.image.variant } : {}),
|
|
1737
|
+
...(step.browserPod ? { browserPod: browserPodLine(step.browserPod) } : {}),
|
|
1738
|
+
};
|
|
1739
|
+
delete step.podLostTry;
|
|
1740
|
+
step.tries = [...kept, ...lost, row];
|
|
1741
|
+
const totals = sumAttempts(step.tries);
|
|
1742
|
+
step.costUsd = totals.costUsd;
|
|
1743
|
+
step.tokens = totals.tokens;
|
|
1744
|
+
step.computeSecs = totals.computeSecs;
|
|
1745
|
+
}
|
|
1746
|
+
|
|
1747
|
+
/** A question an agent asked mid-step goes where approvals go (the
|
|
1748
|
+
* workspace's notify:, `awaiting-approval` events), with the way to answer. */
|
|
1749
|
+
async function notifyQuestion(tenant: string, workspace: string, runId: string, agent: string, question: string, options?: string[]): Promise<void> {
|
|
1750
|
+
const origin = (process.env.FOLDRUN_PUBLIC_URL ?? "").replace(/\/+$/, "");
|
|
1751
|
+
const where = origin ? `${origin}/dashboard/${encodeURIComponent(workspace)}/runs/${encodeURIComponent(runId)}${tenant !== "default" ? `?tenant=${encodeURIComponent(tenant)}` : ""}` : `run ${runId}`;
|
|
1752
|
+
await sendPlainNotification(tenant, workspace, {
|
|
1753
|
+
event: "awaiting-approval",
|
|
1754
|
+
headline: `${agent} is asking you something`,
|
|
1755
|
+
detail: `${question}${options?.length ? `\n\nOptions: ${options.join(" · ")}` : ""}\n\nAnswer it on ${where}, or \`foldrun answer ${runId} "…"\`.`,
|
|
1756
|
+
runId,
|
|
1757
|
+
}).catch(() => false);
|
|
1758
|
+
}
|
|
1759
|
+
|
|
1760
|
+
/** The local stand-in for a person: the terminal, when there is one. */
|
|
1761
|
+
async function terminalAsk(question: string, options?: string[]): Promise<string | null> {
|
|
1762
|
+
if (!process.stdin.isTTY || !process.stdout.isTTY) return null;
|
|
1763
|
+
const { createInterface } = await import("node:readline/promises");
|
|
1764
|
+
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
|
1765
|
+
try {
|
|
1766
|
+
const list = options?.length ? `\n${options.map((o, i) => ` ${i + 1}. ${o}`).join("\n")}\n` : "";
|
|
1767
|
+
const raw = (await rl.question(`\n the agent asks: ${question}${list}\n your answer: `)).trim();
|
|
1768
|
+
const n = Number(raw);
|
|
1769
|
+
return options?.length && Number.isInteger(n) && n >= 1 && n <= options.length ? options[n - 1] : raw || null;
|
|
1770
|
+
} finally {
|
|
1771
|
+
rl.close();
|
|
1772
|
+
}
|
|
1773
|
+
}
|
|
1774
|
+
|
|
1200
1775
|
async function runStep(
|
|
1201
1776
|
agentDir: string,
|
|
1202
1777
|
tenant: string,
|
|
@@ -1214,6 +1789,22 @@ async function runStep(
|
|
|
1214
1789
|
* remaining `budget:`, shared across the steps launched with it. Null
|
|
1215
1790
|
* means the flow set no budget. */
|
|
1216
1791
|
stepBudgetUsd: number | null = null,
|
|
1792
|
+
/** Which line set that ceiling, for the error that names it. */
|
|
1793
|
+
stepBudgetNote?: string,
|
|
1794
|
+
/** The run is a test run — see test-mode.ts. Read off the run record by
|
|
1795
|
+
* the caller, never from anything the sandbox could have written. */
|
|
1796
|
+
testRun = false,
|
|
1797
|
+
/** Has a person asked for the run to stop? Read on a clock by the
|
|
1798
|
+
* in-process model loop, so a stop lands mid-step and not at the next
|
|
1799
|
+
* group. The isolated path needs none: stopRun destroys the sandbox. */
|
|
1800
|
+
stopRequested?: () => boolean,
|
|
1801
|
+
/** The flow's `timezone:`, for the clock cascade. Undefined for a step
|
|
1802
|
+
* run outside any flow, which still gets agent → workspace → account. */
|
|
1803
|
+
flowTimezone?: string | null,
|
|
1804
|
+
/** The step's index in run.steps, carried on its egress lease so a
|
|
1805
|
+
* person's message to this step reaches this step's own inbox and not a
|
|
1806
|
+
* sibling's running in the same group. Undefined outside a flow. */
|
|
1807
|
+
stepIndex?: number,
|
|
1217
1808
|
) {
|
|
1218
1809
|
// Secret values are injected into scripts as environment variables and
|
|
1219
1810
|
// substituted into API headers, so a model that reads one back — from a
|
|
@@ -1232,14 +1823,44 @@ async function runStep(
|
|
|
1232
1823
|
// cover a script's own stdout.
|
|
1233
1824
|
let redactions: [string, string][] = [];
|
|
1234
1825
|
let fileDir: string | null = null;
|
|
1826
|
+
let disposeRuntime: (() => void) | undefined;
|
|
1827
|
+
// Can this step's tools act outside the workspace? Set once its context is
|
|
1828
|
+
// read; the verdict at the end of this function reads it.
|
|
1829
|
+
let canActOutward = false;
|
|
1830
|
+
const eventsAtStart = step.events.length;
|
|
1235
1831
|
const redact = (text: string) => {
|
|
1236
1832
|
let out = text;
|
|
1237
1833
|
for (const [value, name] of redactions) out = out.split(value).join(`[redacted:${name}]`);
|
|
1238
1834
|
return out;
|
|
1239
1835
|
};
|
|
1240
1836
|
|
|
1837
|
+
// The reply, the conclusion and an `output: json` value land on the same
|
|
1838
|
+
// record the events do, and a model that quotes a script's stdout puts
|
|
1839
|
+
// the credential in its answer rather than in a tool event. Same scrub,
|
|
1840
|
+
// same place. The JSON value goes through its text form: a secret inside
|
|
1841
|
+
// a nested field is still a secret, and the key name survives either way.
|
|
1842
|
+
const redactText = <T extends string | null | undefined>(text: T): T =>
|
|
1843
|
+
(typeof text === "string" ? redact(text) : text) as T;
|
|
1844
|
+
const redactData = (value: unknown): unknown => {
|
|
1845
|
+
if (value === undefined || redactions.length === 0) return value;
|
|
1846
|
+
try {
|
|
1847
|
+
return JSON.parse(redact(JSON.stringify(value)));
|
|
1848
|
+
} catch {
|
|
1849
|
+
return value;
|
|
1850
|
+
}
|
|
1851
|
+
};
|
|
1852
|
+
|
|
1241
1853
|
const push = (type: StepRecord["events"][number]["type"], text: string, extra?: EventExtra) => {
|
|
1242
|
-
|
|
1854
|
+
const t = new Date().toISOString();
|
|
1855
|
+
step.events.push({ t, type, text: redact(text), ...extra });
|
|
1856
|
+
// A person in the loop: the question, its answer, a message delivered —
|
|
1857
|
+
// onto the step, so the run page, the API and history show them.
|
|
1858
|
+
if (extra?.operator) {
|
|
1859
|
+
applyOperatorEvent(step, extra.operator, t);
|
|
1860
|
+
if (extra.operator.kind === "asked" && runId && !testRun) {
|
|
1861
|
+
void notifyQuestion(tenant, path.basename(path.resolve(agentDir, "..", "..")), runId, step.agent, extra.operator.question, extra.operator.options);
|
|
1862
|
+
}
|
|
1863
|
+
}
|
|
1243
1864
|
save();
|
|
1244
1865
|
};
|
|
1245
1866
|
|
|
@@ -1259,7 +1880,9 @@ async function runStep(
|
|
|
1259
1880
|
? fs.readFileSync(stubFile, "utf8").split(/\n---\n/)
|
|
1260
1881
|
: [`stub result from ${step.agent}`];
|
|
1261
1882
|
const call = step.events.filter((e) => e.text.startsWith("stub call")).length;
|
|
1262
|
-
|
|
1883
|
+
// The ceiling it was given rides on the line, so the budget arithmetic
|
|
1884
|
+
// around fan-out is testable without a model to stop.
|
|
1885
|
+
push("info", `stub call ${call + 1}${stepBudgetUsd !== null ? ` (ceiling $${stepBudgetUsd.toFixed(2)})` : ""}`);
|
|
1263
1886
|
step.status = "completed";
|
|
1264
1887
|
// A stub answer may open with `cost: 0.25` — a pretend price, so the
|
|
1265
1888
|
// parts of orchestration that watch money (budget:) are testable too.
|
|
@@ -1277,8 +1900,15 @@ async function runStep(
|
|
|
1277
1900
|
// the failure of a step that returned no JSON are testable at no cost.
|
|
1278
1901
|
if (step.output === "json") {
|
|
1279
1902
|
const extracted = extractJson(step.result);
|
|
1280
|
-
if (extracted.ok)
|
|
1281
|
-
|
|
1903
|
+
if (extracted.ok) {
|
|
1904
|
+
step.data = extracted.value;
|
|
1905
|
+
const schema = step.schema ?? (step.schemaPath ? readSchemaFile(path.resolve(agentDir, "..", ".."), step.schemaPath) : null);
|
|
1906
|
+
const errors = schema && looksLikeSchema(schema) ? validateSchema(step.data, schema) : [];
|
|
1907
|
+
if (errors.length) {
|
|
1908
|
+
push("error", `schema: the value does not fit — ${describeSchemaErrors(errors)}`);
|
|
1909
|
+
step.status = "failed";
|
|
1910
|
+
}
|
|
1911
|
+
} else {
|
|
1282
1912
|
push("error", `output: json — ${extracted.reason}`);
|
|
1283
1913
|
step.status = "failed";
|
|
1284
1914
|
}
|
|
@@ -1295,14 +1925,19 @@ async function runStep(
|
|
|
1295
1925
|
// step: the generic HTTP tool still works without the typed ones.
|
|
1296
1926
|
for (const w of await prefetchOpenApi(tenant, openApiSources(agentDir, tenant))) push("error", `openapi: ${w}`);
|
|
1297
1927
|
const {
|
|
1298
|
-
front, clockEnv, systemPrompt, allowed, disabled, apiTools, scriptTools,
|
|
1928
|
+
front, clockEnv, clock, systemPrompt, allowed, disabled, apiTools, scriptTools,
|
|
1299
1929
|
secretEnv, secretScopes, missingSecrets, missingTools, runtime,
|
|
1300
|
-
unknownTools, shadowed, legacyUse, mcpServers, mcpNames,
|
|
1930
|
+
unknownTools, shadowed, legacyUse, retiredTools, mcpServers, mcpNames,
|
|
1301
1931
|
apiSpecs, scriptSpecs, brokenTools, size: agentSize,
|
|
1302
1932
|
providerEnv, providerLabel, providerSecrets, providerWarnings, formatWarning,
|
|
1933
|
+
searchChoice, fetchChoice, actionChoices, web, language, region,
|
|
1303
1934
|
fallbackEnv, apiWarnings, searchRoots, historyDigest, deskDigest, searchTools, historyTools, deskTools,
|
|
1935
|
+
ask,
|
|
1304
1936
|
translator, fallbackTranslator,
|
|
1305
|
-
|
|
1937
|
+
outward,
|
|
1938
|
+
} = agentContext(agentDir, tenant, tags, { runId, agent: step.agent }, flowTimezone);
|
|
1939
|
+
disposeRuntime = runtime.dispose;
|
|
1940
|
+
canActOutward = outward;
|
|
1306
1941
|
// A retry that moved the step up a class (an evicted or OOM-killed
|
|
1307
1942
|
// attempt) wins over the agent's own `size:` for the attempts after.
|
|
1308
1943
|
const size = step.sizeUp ?? agentSize;
|
|
@@ -1313,7 +1948,7 @@ async function runStep(
|
|
|
1313
1948
|
// what crosses into containers, headers and env files is the token. A
|
|
1314
1949
|
// failed refresh fails the step naming the secret and the provider's
|
|
1315
1950
|
// reason, which beats a 401 three layers later.
|
|
1316
|
-
const liveSecrets = await materializeSecrets(secretEnv);
|
|
1951
|
+
const liveSecrets = await materializeSecrets(secretEnv, { tenant, workspace: path.basename(workspaceRootOf(agentDir)) });
|
|
1317
1952
|
// @file values (SSH keys, certs) still carry their content at this point.
|
|
1318
1953
|
// The in-process branch materialises them to 0600 paths on this host
|
|
1319
1954
|
// below; the container branch passes the content across and the driver
|
|
@@ -1328,15 +1963,24 @@ async function runStep(
|
|
|
1328
1963
|
// Longest first, so a value containing another is replaced whole.
|
|
1329
1964
|
.sort((a, b) => b[0].length - a[0].length);
|
|
1330
1965
|
|
|
1966
|
+
// One line for the clock: which zone this step works to, and which level
|
|
1967
|
+
// set it. "Why is this dated yesterday" should be answerable from the run.
|
|
1968
|
+
push("info", `clock: ${clock.timezone} (${clock.source}) — today is ${clockEnv.FOLDRUN_DATE}`);
|
|
1969
|
+
// And one more only when something written could not be read as a zone.
|
|
1970
|
+
for (const problem of clock.problems) push("error", problem);
|
|
1331
1971
|
if (providerLabel) push("info", `provider: ${providerLabel}`);
|
|
1332
1972
|
for (const w of providerWarnings) push("error", w);
|
|
1973
|
+
// A level that wrote a language nobody can read was skipped, not obeyed.
|
|
1974
|
+
for (const l of language.lines) push("info", l);
|
|
1975
|
+
for (const l of region.lines) push("info", l);
|
|
1333
1976
|
for (const w of apiWarnings) push("error", `openapi: ${w}`);
|
|
1334
1977
|
if (formatWarning) push("error", formatWarning);
|
|
1335
1978
|
if (legacyUse.length) push("error", legacyUseError(legacyUse));
|
|
1979
|
+
for (const t of retiredTools) push("error", retiredToolError(t));
|
|
1336
1980
|
for (const t of unknownTools) {
|
|
1337
1981
|
push(
|
|
1338
1982
|
"error",
|
|
1339
|
-
`tools: "${t}" is not a tool group (
|
|
1983
|
+
`tools: "${t}" is not a tool group (read, write, code, search, history, desks), an SDK tool name, or a tool in this workspace's library — nothing was granted for it`,
|
|
1340
1984
|
);
|
|
1341
1985
|
}
|
|
1342
1986
|
for (const name of mcpNames) push("info", `mcp server connected: ${name}`);
|
|
@@ -1454,13 +2098,22 @@ async function runStep(
|
|
|
1454
2098
|
|
|
1455
2099
|
// Expose the workspace's shared scripts/ as ./shared so every agent reaches
|
|
1456
2100
|
// them by the same path, without each one carrying a copy.
|
|
1457
|
-
// No symlinks into the agent directory — they would recurse
|
|
1458
|
-
// workspace's own file listing. Shared paths are resolved by the
|
|
1459
|
-
// tool layer and described to the agent below
|
|
2101
|
+
// No permanent symlinks into the agent directory — they would recurse
|
|
2102
|
+
// into the workspace's own file listing. Shared paths are resolved by the
|
|
2103
|
+
// script tool layer and described to the agent below; the one link there
|
|
2104
|
+
// is, `workspace` → the root for the shell, lives only for the length of
|
|
2105
|
+
// a step (executeStep, confine.ts#linkWorkspace).
|
|
1460
2106
|
|
|
1461
2107
|
// The confinement boundary: agents in a workspace are one team, but the
|
|
1462
2108
|
// workspace is where isolation is enforced.
|
|
1463
2109
|
const workspaceRoot = path.resolve(agentDir, "..", "..");
|
|
2110
|
+
// Where a test run's state/ and storage/ writes go, and the line each
|
|
2111
|
+
// one leaves: under the run, beside its archived outputs.
|
|
2112
|
+
const divertFor = (id: string) => ({
|
|
2113
|
+
to: path.join(workspaceRoot, "runs", id, TEST_WRITES_DIR),
|
|
2114
|
+
note: (rel: string, summary: string) =>
|
|
2115
|
+
push("info", `test: ${summary}`, { effect: { kind: "diverted" as const, path: rel, summary } }),
|
|
2116
|
+
});
|
|
1464
2117
|
|
|
1465
2118
|
let prompt = step.instruction
|
|
1466
2119
|
? resolveDocLinks(step.instruction, workspaceRoot)
|
|
@@ -1477,11 +2130,18 @@ async function runStep(
|
|
|
1477
2130
|
if (step.output === "json") {
|
|
1478
2131
|
// The contract the runner parses back out — stated in the prompt so
|
|
1479
2132
|
// the instruction and the parser agree by construction, like delegate:.
|
|
2133
|
+
// The schema, when there is one, is quoted whole: the model that has
|
|
2134
|
+
// to satisfy it should not have to guess it from the field names.
|
|
2135
|
+
const schema = step.schema ?? (step.schemaPath ? readSchemaFile(path.resolve(agentDir, "..", ".."), step.schemaPath) : null);
|
|
1480
2136
|
prompt +=
|
|
1481
2137
|
`\n\n<output>\nThis step returns data. End your reply with exactly one JSON value inside a ` +
|
|
1482
2138
|
"```json fenced block — the value the next step will compute on. Prose before it is " +
|
|
1483
2139
|
`welcome (lead with a one-line headline); nothing may follow the block. If the instruction ` +
|
|
1484
|
-
`names fields, use exactly those names
|
|
2140
|
+
`names fields, use exactly those names.` +
|
|
2141
|
+
(schema && looksLikeSchema(schema)
|
|
2142
|
+
? ` The value must satisfy this JSON Schema, and the step fails if it does not:\n${JSON.stringify(schema, null, 2).slice(0, 20_000)}`
|
|
2143
|
+
: "") +
|
|
2144
|
+
`\n</output>`;
|
|
1485
2145
|
}
|
|
1486
2146
|
if (step.waitFor === "event" && step.eventPayload !== undefined) {
|
|
1487
2147
|
// Not resolved for links, like the previous step's results: it is
|
|
@@ -1532,14 +2192,39 @@ async function runStep(
|
|
|
1532
2192
|
for (const m of missingConsults) {
|
|
1533
2193
|
push("error", `agents: "${m}" is not an agent in this workspace — no consult tool granted`);
|
|
1534
2194
|
}
|
|
1535
|
-
const consultNames =
|
|
2195
|
+
const consultNames = [
|
|
2196
|
+
...mcpNames,
|
|
2197
|
+
...(consults.length ? ["foldrun_agents"] : []),
|
|
2198
|
+
...(ask ? ["foldrun_ask"] : []),
|
|
2199
|
+
];
|
|
2200
|
+
|
|
2201
|
+
// Colleagues this agent may DELEGATE to: each with its own context and
|
|
2202
|
+
// its own tools — resolved exactly as its own step's would be, then cut
|
|
2203
|
+
// to what THIS step holds (subagents.ts). Built host-side, like consults.
|
|
2204
|
+
const subagents = buildSubagentSpecs({
|
|
2205
|
+
names: front.subagents, self: step.agent, workspaceRoot, tenant, tags, runId, flowTimezone,
|
|
2206
|
+
parentAllowed: [...allowed, ...consultNames.map((n) => `mcp__${n}`)], push,
|
|
2207
|
+
});
|
|
2208
|
+
if (subagents.length) {
|
|
2209
|
+
// The delegation tool itself, granted only when there is someone to
|
|
2210
|
+
// delegate to.
|
|
2211
|
+
allowed.push(DELEGATE_TOOL);
|
|
2212
|
+
for (const s of subagents) push("info", `subagent: ${s.name} — ${s.tools.length ? s.tools.join(", ") : "no tools"}`);
|
|
2213
|
+
}
|
|
1536
2214
|
|
|
1537
2215
|
// The last provider refusal this step saw, for the fallback decision —
|
|
1538
2216
|
// watched on both paths, because a refusal reads the same from a pod
|
|
1539
2217
|
// and from this process.
|
|
1540
2218
|
let lastRefusal = "";
|
|
2219
|
+
// And WHY it refused, in words, when anything on the way saw the
|
|
2220
|
+
// headers that say so: the translator here, the egress proxy over
|
|
2221
|
+
// there. Kept apart from `lastRefusal` — that one decides whether to
|
|
2222
|
+
// fall back, this one is what a person reads afterwards.
|
|
2223
|
+
let refusalWhy = "";
|
|
1541
2224
|
const pushWatching: typeof push = (type, text, extra) => {
|
|
1542
2225
|
if (type === "error" && isProviderRefusal(text)) lastRefusal = text;
|
|
2226
|
+
const why = refusalFromLine(text);
|
|
2227
|
+
if (why) refusalWhy = why;
|
|
1543
2228
|
push(type, text, extra);
|
|
1544
2229
|
};
|
|
1545
2230
|
// The second supply: the block's own fallback when it declared one (a
|
|
@@ -1547,6 +2232,50 @@ async function runStep(
|
|
|
1547
2232
|
// fallback), else the platform's, and only for a step riding the
|
|
1548
2233
|
// platform credential.
|
|
1549
2234
|
const secondSupply = fallbackEnv ?? (Object.keys(providerEnv).length === 0 ? platformFallbackEnv() : null);
|
|
2235
|
+
// Where this step stands on a second supply, for the refusal sentence.
|
|
2236
|
+
// It starts as the truth before anything has run and becomes
|
|
2237
|
+
// "exhausted" the moment the fallback has had its turn — the owner is
|
|
2238
|
+
// about to remove the OpenRouter fallback, so "none" becomes the normal
|
|
2239
|
+
// reading and has to look like a decision on the page.
|
|
2240
|
+
let supplyState: SupplyState = secondSupply ? "trying" : "none";
|
|
2241
|
+
// The line a person reads when this step ends failed: the reason, if
|
|
2242
|
+
// anything on the way learned one, else the status put into words, and
|
|
2243
|
+
// in every case where the step could go next. Pushed last so it is the
|
|
2244
|
+
// last error event — which is what runFailure, the desks' headlines and
|
|
2245
|
+
// `foldrun report` read.
|
|
2246
|
+
const sayWhy = (status: StepRecord["status"]) => {
|
|
2247
|
+
if (status !== "failed" || !lastRefusal) return;
|
|
2248
|
+
const said = Number(lastRefusal.match(/API Error: (\d{3})/)?.[1] ?? NaN);
|
|
2249
|
+
const why =
|
|
2250
|
+
refusalWhy ||
|
|
2251
|
+
explainRefusal({
|
|
2252
|
+
status: Number.isFinite(said) && isRefusalStatus(said) ? said : 402,
|
|
2253
|
+
timezone: clock.timezone,
|
|
2254
|
+
});
|
|
2255
|
+
push("error", `${why}${supplyNote(supplyState)}`);
|
|
2256
|
+
};
|
|
2257
|
+
|
|
2258
|
+
// `schema:` — inline on the step, or a file under the workspace read
|
|
2259
|
+
// now, host-side, so both executors are handed the same document. A
|
|
2260
|
+
// file that cannot be read is a failed step before any model runs:
|
|
2261
|
+
// the contract the author wrote down cannot be checked.
|
|
2262
|
+
const resolvedSchema: Record<string, unknown> | boolean | null = step.schema ?? (step.schemaPath ? readSchemaFile(workspaceRoot, step.schemaPath) : null);
|
|
2263
|
+
if (step.schemaPath && resolvedSchema === null) {
|
|
2264
|
+
throw new Error(`schema: ${step.schemaPath} — not a readable JSON or YAML schema under the workspace`);
|
|
2265
|
+
}
|
|
2266
|
+
if (step.maxTurns) push("info", `max_turns: ${step.maxTurns}`);
|
|
2267
|
+
// `limits:` — account, workspace, agent, then this step's option;
|
|
2268
|
+
// nearest wins per key. Enforced in the model loop's hook, on either
|
|
2269
|
+
// path (step-exec.ts), so it is resolved here as plain values.
|
|
2270
|
+
const stepLimits = cascadeLimits([
|
|
2271
|
+
readAgentsMd(accountDir(tenant))?.data?.limits,
|
|
2272
|
+
readAgentsMd(workspaceRootOf(agentDir))?.data?.limits,
|
|
2273
|
+
front.limits,
|
|
2274
|
+
step.limits,
|
|
2275
|
+
]);
|
|
2276
|
+
const limitOwners = toolOwners(apiTools.owners, scriptSpecs);
|
|
2277
|
+
const limitOpts = Object.keys(stepLimits).length ? { limits: stepLimits, toolOwners: limitOwners } : {};
|
|
2278
|
+
if (limitOpts.limits) push("info", `limits: ${Object.entries(stepLimits).map(([k, n]) => `${k} ${n}`).join(", ")} — per step; a call past one is refused`);
|
|
1550
2279
|
|
|
1551
2280
|
const isolation = process.env.FOLDRUN_RUN_ISOLATION;
|
|
1552
2281
|
if (isolatedRun()) {
|
|
@@ -1570,21 +2299,45 @@ async function runStep(
|
|
|
1570
2299
|
// before anything is copied in. Null where no proxy is configured
|
|
1571
2300
|
// (the CLI, a compose install without FOLDRUN_EGRESS_URL), and then
|
|
1572
2301
|
// everything below materialises exactly as it always did.
|
|
1573
|
-
const grant: EgressGrant = { secrets: {} };
|
|
2302
|
+
const grant: EgressGrant = { secrets: {}, timezone: clock.timezone };
|
|
2303
|
+
// A direct search API's key: granted to that API's host and no other,
|
|
2304
|
+
// exactly as an http tool's key is granted to its base URL's host. The
|
|
2305
|
+
// script sends the placeholder; the worker fills it on the way out.
|
|
2306
|
+
for (const choice of [searchChoice, fetchChoice]) {
|
|
2307
|
+
if (choice.shape === "direct" && choice.secret && choice.host && choice.secret in liveSecrets) {
|
|
2308
|
+
addGrant(grant, choice.secret, liveSecrets[choice.secret], choice.host);
|
|
2309
|
+
}
|
|
2310
|
+
}
|
|
2311
|
+
for (const c of actionChoices) {
|
|
2312
|
+
if (c.provider && c.secret in liveSecrets) addGrant(grant, c.secret, liveSecrets[c.secret], c.host);
|
|
2313
|
+
}
|
|
1574
2314
|
for (const api of apiSpecs) {
|
|
1575
2315
|
const host = hostOf(api.base);
|
|
1576
2316
|
if (!host) continue;
|
|
1577
2317
|
for (const v of [...Object.values(api.headers), ...Object.values(api.query)]) {
|
|
1578
|
-
for (const name of placeholderNames(v))
|
|
2318
|
+
for (const name of placeholderNames(v)) {
|
|
2319
|
+
if (!(name in liveSecrets)) continue;
|
|
2320
|
+
addGrant(grant, name, liveSecrets[name], host);
|
|
2321
|
+
// Which vault entry this is, so the proxy's record of what the
|
|
2322
|
+
// far end said lands on the right credential.
|
|
2323
|
+
(grant.healthKeys ??= {})[name] = healthKey(
|
|
2324
|
+
name,
|
|
2325
|
+
secretScopes[name] === "workspace" ? "workspace" : "account",
|
|
2326
|
+
path.basename(path.resolve(agentDir, "..", "..")),
|
|
2327
|
+
);
|
|
2328
|
+
}
|
|
1579
2329
|
}
|
|
1580
2330
|
}
|
|
1581
|
-
const lease = await platform.egress.lease({ tenant, runId: runId ?? "adhoc", grant });
|
|
2331
|
+
const lease = await platform.egress.lease({ tenant, runId: runId ?? "adhoc", grant, test: testRun, ...(stepIndex !== undefined && stepIndex >= 0 ? { step: stepIndex } : {}) });
|
|
1582
2332
|
// Which secrets must still be real inside the sandbox: a script tool
|
|
1583
2333
|
// reads them from its environment, and unless it says `secrets:
|
|
1584
2334
|
// proxied` it needs the value, not the name. bash is a script by
|
|
1585
2335
|
// another route. The record says which way it went, and why.
|
|
1586
2336
|
const materialisers = [
|
|
1587
2337
|
...scriptSpecs.filter((sc) => sc.secrets !== "proxied").map((sc) => sc.name),
|
|
2338
|
+
// web proxies its keys — unless it may browse: a remote browser's key
|
|
2339
|
+
// goes into a websocket, and cookie and login secrets into the page.
|
|
2340
|
+
...(scriptSpecs.some((sc) => sc.name === "web") && (!web.actions || web.actions.includes("browse")) ? ["web (browse)"] : []),
|
|
1588
2341
|
...(allowed.includes("Bash") ? ["bash"] : []),
|
|
1589
2342
|
];
|
|
1590
2343
|
const materialise = !lease || materialisers.length > 0;
|
|
@@ -1601,15 +2354,36 @@ async function runStep(
|
|
|
1601
2354
|
}
|
|
1602
2355
|
// Sandbox env for the declared secrets: the value, or its own name as
|
|
1603
2356
|
// a placeholder that the proxy fills on the way out.
|
|
1604
|
-
const
|
|
1605
|
-
|
|
1606
|
-
|
|
2357
|
+
const placeholders = Object.fromEntries(declared.map((name) => [name, `\${${name}}`]));
|
|
2358
|
+
let sandboxSecrets: Record<string, string> = materialise ? liveSecrets : placeholders;
|
|
2359
|
+
// A test run hands the sandbox no send-capable value. The env var is
|
|
2360
|
+
// still there, reading TEST_MODE_WITHHELD, so a script that never
|
|
2361
|
+
// checked FOLDRUN_TEST_MODE fails at the provider's door with an auth
|
|
2362
|
+
// error — and sends nothing. The script tools say whether the step
|
|
2363
|
+
// has a declared sender (everything withheld) or a declared reader
|
|
2364
|
+
// (nothing withheld); see test-mode.ts for the order.
|
|
2365
|
+
if (testRun && materialise) {
|
|
2366
|
+
const held = withholdSecrets(liveSecrets, {
|
|
2367
|
+
outward: scriptSpecs.some((sc) => sc.outward),
|
|
2368
|
+
allow: scriptSpecs.some((sc) => sc.testMode === "allow"),
|
|
2369
|
+
});
|
|
2370
|
+
sandboxSecrets = held.env;
|
|
2371
|
+
for (const name of held.withheld) {
|
|
2372
|
+
push("info", `test: ${name} withheld from the sandbox — TEST_MODE_WITHHELD`, { effect: { kind: "withheld", summary: `${name} withheld from the sandbox` } });
|
|
2373
|
+
}
|
|
2374
|
+
}
|
|
2375
|
+
// On a test run the http tools always go through the proxy as
|
|
2376
|
+
// placeholders, even when a script made the step materialise: the
|
|
2377
|
+
// proxy is where the outward-write policy lives, and a header
|
|
2378
|
+
// carrying TEST_MODE_WITHHELD would only fail where it should have
|
|
2379
|
+
// been sunk or refused.
|
|
2380
|
+
const apiValues = testRun && lease ? placeholders : sandboxSecrets;
|
|
1607
2381
|
const substitutedApis = apiSpecs.map((api) => ({
|
|
1608
2382
|
...api,
|
|
1609
2383
|
headers: Object.fromEntries(
|
|
1610
2384
|
Object.entries(api.headers).map(([k, v]) => [
|
|
1611
2385
|
k,
|
|
1612
|
-
v.replace(/\$\{([A-Z][A-Z0-9_]*)\}/g, (whole, name) =>
|
|
2386
|
+
v.replace(/\$\{([A-Z][A-Z0-9_]*)\}/g, (whole, name) => apiValues[name] ?? whole),
|
|
1613
2387
|
]),
|
|
1614
2388
|
),
|
|
1615
2389
|
}));
|
|
@@ -1639,8 +2413,9 @@ async function runStep(
|
|
|
1639
2413
|
const isolatedArgs = (modelEnv: Record<string, string | undefined>, keyName = MODEL_KEY_NAME) => ({
|
|
1640
2414
|
workspaceRoot,
|
|
1641
2415
|
libraryRoot: libraryDir(tenant),
|
|
2416
|
+
galleryRoot: platform.galleryDir() ?? undefined,
|
|
1642
2417
|
// Keys this step's dependency cache. Per-account by design — see
|
|
1643
|
-
// run-container.ts#
|
|
2418
|
+
// run-container.ts#runtimeCacheMount for why it is not shared.
|
|
1644
2419
|
tenant,
|
|
1645
2420
|
input: {
|
|
1646
2421
|
agentRel: path.relative(workspaceRoot, agentDir).replaceAll("\\", "/"),
|
|
@@ -1657,12 +2432,21 @@ async function runStep(
|
|
|
1657
2432
|
// field. A tool's own `runtime:` is part of what this step needs.
|
|
1658
2433
|
runtime: runtime.spec,
|
|
1659
2434
|
consults,
|
|
2435
|
+
...(subagents.length ? { subagents } : {}),
|
|
1660
2436
|
timeoutSec: step.timeout,
|
|
1661
2437
|
budgetUsd: stepBudgetUsd,
|
|
2438
|
+
budgetNote: stepBudgetNote,
|
|
1662
2439
|
price: tokenPrice,
|
|
1663
2440
|
verify: step.verify,
|
|
1664
2441
|
output: step.output,
|
|
2442
|
+
schema: resolvedSchema ?? undefined,
|
|
2443
|
+
maxTurns: step.maxTurns,
|
|
2444
|
+
...limitOpts,
|
|
1665
2445
|
translator: keyName === MODEL_KEY_NAME ? primaryTranslator : secondTranslator,
|
|
2446
|
+
// Outward steps keep a browser of their own: an executor that
|
|
2447
|
+
// could browse from the slim image through the account's pod
|
|
2448
|
+
// leaves them on the full one (browser-pod.ts).
|
|
2449
|
+
...(outward ? { outward: true } : {}),
|
|
1666
2450
|
// The container sees the workspace at /workspace and the library
|
|
1667
2451
|
// at /library; the roots are named host-side and moved here.
|
|
1668
2452
|
search: searchRoots.map((r) => ({
|
|
@@ -1673,6 +2457,7 @@ async function runStep(
|
|
|
1673
2457
|
})),
|
|
1674
2458
|
history: historyDigest,
|
|
1675
2459
|
desks: deskDigest,
|
|
2460
|
+
...(ask ? { ask } : {}),
|
|
1676
2461
|
},
|
|
1677
2462
|
env: Object.fromEntries(
|
|
1678
2463
|
Object.entries({
|
|
@@ -1692,6 +2477,7 @@ async function runStep(
|
|
|
1692
2477
|
FOLDRUN_RUN_ID: runId,
|
|
1693
2478
|
FOLDRUN_AGENT: step.agent,
|
|
1694
2479
|
FOLDRUN_WORKSPACE: path.basename(workspaceRoot),
|
|
2480
|
+
...(testRun ? testModeEnv() : {}),
|
|
1695
2481
|
...sandboxSecrets,
|
|
1696
2482
|
}).filter((entry): entry is [string, string] => typeof entry[1] === "string"),
|
|
1697
2483
|
),
|
|
@@ -1699,6 +2485,9 @@ async function runStep(
|
|
|
1699
2485
|
// Stamps the sandbox, so stopping this run can destroy it.
|
|
1700
2486
|
runId,
|
|
1701
2487
|
size,
|
|
2488
|
+
// A test run's state/ and storage/ changes come out under the run.
|
|
2489
|
+
divert: testRun && runId ? divertFor(runId) : undefined,
|
|
2490
|
+
test: testRun,
|
|
1702
2491
|
// Resume: a sandbox a previous driver left running, recorded on the
|
|
1703
2492
|
// step. Consumed once — a fresh attempt after this one (a retry, a
|
|
1704
2493
|
// fallback) starts its own sandbox.
|
|
@@ -1720,8 +2509,38 @@ async function runStep(
|
|
|
1720
2509
|
// attempt holds its own sandbox, so the meter owes the sum.
|
|
1721
2510
|
const firstArgs = isolatedArgs(platformModelEnv());
|
|
1722
2511
|
await lease?.commit();
|
|
1723
|
-
let
|
|
2512
|
+
let credentialUsed = platformModelCredential();
|
|
2513
|
+
// The arguments of the attempt whose outcome is current — what a
|
|
2514
|
+
// re-run on the full image repeats, on the same model supply.
|
|
2515
|
+
let lastArgs: typeof firstArgs = firstArgs;
|
|
2516
|
+
const runTracked = (a: typeof firstArgs) => {
|
|
2517
|
+
lastArgs = a;
|
|
2518
|
+
return runIsolated(a);
|
|
2519
|
+
};
|
|
2520
|
+
let outcome = await runTracked(firstArgs);
|
|
1724
2521
|
step.sandbox = null; // whatever happens next starts its own
|
|
2522
|
+
// A 401 from a token that rotated mid-step is not a broken key: the
|
|
2523
|
+
// replacement is already on its way to the mounted file. Wait for it
|
|
2524
|
+
// once, then fall into the same retry the busy case uses.
|
|
2525
|
+
if (outcome.status === "failed" && isAuthRefusal(lastRefusal)) {
|
|
2526
|
+
push("info", `the model credential was refused mid-step — waiting for the refreshed one`);
|
|
2527
|
+
const rotated = await awaitRotatedCredential(credentialUsed);
|
|
2528
|
+
if (rotated) {
|
|
2529
|
+
push("info", "the credential rotated under this step; retrying on the new one");
|
|
2530
|
+
credentialUsed = rotated;
|
|
2531
|
+
lastRefusal = "";
|
|
2532
|
+
const previous = outcome.timing;
|
|
2533
|
+
// Build first: isolatedArgs is what re-grants the model key, and
|
|
2534
|
+
// commit() is what publishes the grant to Redis, where the proxy
|
|
2535
|
+
// reads it. Committing first sent the revoked token again — every
|
|
2536
|
+
// rotation retry on 2026-09-22/23 401'd this way.
|
|
2537
|
+
const retry = isolatedArgs(platformModelEnv());
|
|
2538
|
+
await lease?.commit();
|
|
2539
|
+
outcome = withEarlierTiming(await runTracked(retry), previous);
|
|
2540
|
+
} else {
|
|
2541
|
+
push("info", "the credential did not change — this is the key itself, not a rotation");
|
|
2542
|
+
}
|
|
2543
|
+
}
|
|
1725
2544
|
for (let n = 1; n <= OVERLOAD_RETRIES && outcome.status === "failed" && isTransientOverload(lastRefusal); n++) {
|
|
1726
2545
|
const wait = backoffMs(n, lastRefusal);
|
|
1727
2546
|
push("info", `provider busy (${lastRefusal.slice(0, 60)}) — waiting ${(wait / 1000).toFixed(1)}s, attempt ${n} of ${OVERLOAD_RETRIES}`);
|
|
@@ -1732,7 +2551,7 @@ async function runStep(
|
|
|
1732
2551
|
const previous = outcome.timing;
|
|
1733
2552
|
const again = isolatedArgs(platformModelEnv());
|
|
1734
2553
|
await lease?.commit();
|
|
1735
|
-
outcome = await
|
|
2554
|
+
outcome = await runTracked(again);
|
|
1736
2555
|
outcome = withEarlierTiming(outcome, previous);
|
|
1737
2556
|
}
|
|
1738
2557
|
// The second supply, tried exactly once, and only when the primary
|
|
@@ -1760,26 +2579,75 @@ async function runStep(
|
|
|
1760
2579
|
),
|
|
1761
2580
|
};
|
|
1762
2581
|
await lease?.commit();
|
|
1763
|
-
|
|
2582
|
+
supplyState = "exhausted";
|
|
2583
|
+
outcome = await runTracked(fallbackArgs);
|
|
1764
2584
|
// Both attempts held sandboxes; the meter owes the sum. The first
|
|
1765
2585
|
// try's pod ran, was billed for by the platform, and must not
|
|
1766
2586
|
// vanish from the record because a second try replaced its outcome.
|
|
1767
2587
|
outcome = withEarlierTiming(outcome, first);
|
|
1768
2588
|
}
|
|
2589
|
+
// Slim browsing (browser-pod.ts): the step ran on the slim image and
|
|
2590
|
+
// the account's browser pod went away under it, past the web tool's
|
|
2591
|
+
// reconnects. Reads only — run it again from the start on the full
|
|
2592
|
+
// image, where a browser in the step is always there. Anything
|
|
2593
|
+
// written — fail it, naming what, and leave retry:/on-fail:/a person
|
|
2594
|
+
// to decide; a second go could send or charge twice.
|
|
2595
|
+
step.image = outcome.image ?? null;
|
|
2596
|
+
step.browserPod = outcome.browserPod ? { ...outcome.browserPod } : null;
|
|
2597
|
+
if (outcome.status === "failed" && outcome.browserPod?.lost) {
|
|
2598
|
+
const decision = podLossDecision(outcome.browserPod);
|
|
2599
|
+
if (decision.rerun) {
|
|
2600
|
+
push("info", decision.note);
|
|
2601
|
+
const first = outcome;
|
|
2602
|
+
// The slim go is a try of its own on the record — image slim,
|
|
2603
|
+
// status lost, its own cost, tokens and sandbox seconds — so the
|
|
2604
|
+
// run page and `foldrun report` show two tries, slim (lost) then
|
|
2605
|
+
// full, and the step's totals are the sum of both rows
|
|
2606
|
+
// (recordAttempt). Live, 2026-10-01, the step showed only the
|
|
2607
|
+
// full re-run.
|
|
2608
|
+
step.podLostTry = {
|
|
2609
|
+
status: "lost",
|
|
2610
|
+
costUsd: repriced(catalog, wireModel, first.usage ?? null, first.costUsd, push, first.turnsCostUsd),
|
|
2611
|
+
tokens: first.usage ? { input: first.usage.inputTokens, output: first.usage.outputTokens } : null,
|
|
2612
|
+
computeSecs: first.timing ? Math.round(first.timing.totalMs) / 1000 : null,
|
|
2613
|
+
finishedAt: new Date().toISOString(),
|
|
2614
|
+
error: decision.note.slice(0, 500),
|
|
2615
|
+
image: "slim",
|
|
2616
|
+
browserPod: browserPodLine(first.browserPod!),
|
|
2617
|
+
};
|
|
2618
|
+
const again = { ...lastArgs, input: { ...lastArgs.input, image: "full" as const }, resume: null };
|
|
2619
|
+
await lease?.commit();
|
|
2620
|
+
outcome = await runTracked(again);
|
|
2621
|
+
step.image = outcome.image ?? { variant: "full", why: "re-run after the browser pod was lost" };
|
|
2622
|
+
step.browserPod = { ...first.browserPod!, fallback: RERAN_ON_FULL };
|
|
2623
|
+
} else {
|
|
2624
|
+
push("error", decision.message);
|
|
2625
|
+
step.browserPod = { ...outcome.browserPod, failure: decision.message };
|
|
2626
|
+
}
|
|
2627
|
+
}
|
|
1769
2628
|
if (lease) {
|
|
1770
|
-
for (const line of await lease.drainLog())
|
|
2629
|
+
for (const line of await lease.drainLog()) {
|
|
2630
|
+
// pushWatching, not push: a line the proxy wrote may carry the
|
|
2631
|
+
// refusal mark, and that sentence is the whole reason the proxy
|
|
2632
|
+
// reads response headers at all.
|
|
2633
|
+
if (typeof line === "string") pushWatching("info", line);
|
|
2634
|
+
else pushWatching("info", line.text, { effect: line.effect });
|
|
2635
|
+
}
|
|
1771
2636
|
await lease.release();
|
|
1772
2637
|
}
|
|
1773
|
-
|
|
2638
|
+
// Nothing a test run left in storage/public/ reached storage/, so
|
|
2639
|
+
// there is nothing to publish — and a share link is an outward act.
|
|
2640
|
+
if (!testRun) await publishPublicDir(tenant, path.basename(workspaceRoot), push);
|
|
2641
|
+
sayWhy(outcome.status);
|
|
1774
2642
|
step.status = outcome.status;
|
|
1775
|
-
step.result = outcome.result;
|
|
1776
|
-
step.conclusion = outcome.conclusion;
|
|
2643
|
+
step.result = redactText(outcome.result);
|
|
2644
|
+
step.conclusion = redactText(outcome.conclusion);
|
|
1777
2645
|
step.sandbox = null;
|
|
1778
2646
|
// The cluster's word on why a sandbox ended without the driver saying
|
|
1779
2647
|
// so — OOMKilled, Evicted — on the record, where the retry reads it.
|
|
1780
2648
|
if (outcome.status === "failed" && outcome.reason) push("error", `sandbox ended: ${outcome.reason}`);
|
|
1781
|
-
if (outcome.data !== undefined) step.data = outcome.data;
|
|
1782
|
-
step.costUsd = repriced(catalog, wireModel, outcome.usage ?? null, outcome.costUsd, push);
|
|
2649
|
+
if (outcome.data !== undefined) step.data = redactData(outcome.data);
|
|
2650
|
+
step.costUsd = repriced(catalog, wireModel, outcome.usage ?? null, outcome.costUsd, push, outcome.turnsCostUsd);
|
|
1783
2651
|
step.tokens = outcome.usage
|
|
1784
2652
|
? { input: outcome.usage.inputTokens, output: outcome.usage.outputTokens }
|
|
1785
2653
|
: null;
|
|
@@ -1811,19 +2679,36 @@ async function runStep(
|
|
|
1811
2679
|
})();
|
|
1812
2680
|
step.reserved = { cpus: Number(lim.cpus) || 2, memGiB: gib };
|
|
1813
2681
|
} else {
|
|
2682
|
+
// Same withholding as the sandbox path — the in-process step is what
|
|
2683
|
+
// `foldrun run --test` drives on a laptop, where there is no proxy
|
|
2684
|
+
// and this is the only guard there is.
|
|
2685
|
+
const held = testRun
|
|
2686
|
+
? withholdSecrets(liveSecrets, {
|
|
2687
|
+
outward: scriptSpecs.some((sc) => sc.outward),
|
|
2688
|
+
allow: scriptSpecs.some((sc) => sc.testMode === "allow"),
|
|
2689
|
+
})
|
|
2690
|
+
: { env: liveSecrets, withheld: [] as string[] };
|
|
2691
|
+
for (const name of held.withheld) {
|
|
2692
|
+
push("info", `test: ${name} withheld from scripts — TEST_MODE_WITHHELD`, { effect: { kind: "withheld", summary: `${name} withheld from scripts` } });
|
|
2693
|
+
}
|
|
2694
|
+
const stepSecrets = held.env;
|
|
1814
2695
|
const consultTools = buildConsultTools(
|
|
1815
2696
|
consults,
|
|
1816
|
-
{ ...
|
|
2697
|
+
{ ...hostSafeEnv(), ...stepSecrets, ...providerEnv },
|
|
1817
2698
|
(type, text) => push(type, text),
|
|
1818
2699
|
);
|
|
2700
|
+
// On this machine nobody can write in and there is no proxy: the
|
|
2701
|
+
// question goes to the terminal when there is one, else unanswered.
|
|
2702
|
+
const askTools = ask ? buildAskTool({ channel: null, timeoutSec: ask.timeoutSec, emit: push, local: terminalAsk }) : null;
|
|
1819
2703
|
// @file secrets become 0600 paths here (host run), cleaned up in the
|
|
1820
2704
|
// finally — once, however many attempts follow.
|
|
1821
|
-
const mat = materializeFileSecrets(agentDir,
|
|
2705
|
+
const mat = materializeFileSecrets(agentDir, stepSecrets);
|
|
1822
2706
|
fileDir = mat.dir;
|
|
1823
2707
|
const attemptInProcess = (modelEnv: Record<string, string | undefined>) => executeStep({
|
|
1824
2708
|
agentDir,
|
|
1825
2709
|
workspaceRoot,
|
|
1826
2710
|
libraryRoot: libraryDir(tenant),
|
|
2711
|
+
...(subagents.length ? { subagents } : {}),
|
|
1827
2712
|
prompt,
|
|
1828
2713
|
model,
|
|
1829
2714
|
effort,
|
|
@@ -1837,16 +2722,25 @@ async function runStep(
|
|
|
1837
2722
|
...(searchTools.server ? { foldrun_search: searchTools.server } : {}),
|
|
1838
2723
|
...(historyTools.server ? { foldrun_history: historyTools.server } : {}),
|
|
1839
2724
|
...(deskTools.server ? { foldrun_desks: deskTools.server } : {}),
|
|
2725
|
+
...(askTools ? { foldrun_ask: askTools.server } : {}),
|
|
1840
2726
|
...mcpServers,
|
|
1841
2727
|
},
|
|
1842
2728
|
// Declared secrets reach the agent's scripts as env vars; the model
|
|
1843
|
-
// only ever sees the variable names, not the values.
|
|
1844
|
-
|
|
2729
|
+
// only ever sees the variable names, not the values. The base is
|
|
2730
|
+
// the allowlisted host environment, not process.env: the SDK's
|
|
2731
|
+
// Bash tool runs with this, and on a platform process.env holds
|
|
2732
|
+
// every tenant's root key and the datastore credentials.
|
|
2733
|
+
env: { ...hostSafeEnv(), ...clockEnv, ...(testRun ? testModeEnv() : {}), ...mat.env, ...modelEnv },
|
|
1845
2734
|
timeoutSec: step.timeout,
|
|
1846
2735
|
budgetUsd: stepBudgetUsd,
|
|
2736
|
+
budgetNote: stepBudgetNote,
|
|
1847
2737
|
price: tokenPrice,
|
|
1848
2738
|
verify: step.verify,
|
|
1849
2739
|
output: step.output,
|
|
2740
|
+
schema: resolvedSchema ?? undefined,
|
|
2741
|
+
maxTurns: step.maxTurns,
|
|
2742
|
+
...limitOpts,
|
|
2743
|
+
stopRequested,
|
|
1850
2744
|
// What the scripts saw, the verify sees: a flow can then check that
|
|
1851
2745
|
// the proof a step left names THIS run, not one that came before —
|
|
1852
2746
|
// copy-back never propagates a deletion, so a marker from an earlier
|
|
@@ -1856,23 +2750,49 @@ async function runStep(
|
|
|
1856
2750
|
...(runId ? { FOLDRUN_RUN_ID: runId } : {}),
|
|
1857
2751
|
FOLDRUN_AGENT: step.agent,
|
|
1858
2752
|
FOLDRUN_WORKSPACE: path.basename(path.resolve(agentDir, "..", "..")),
|
|
1859
|
-
...
|
|
2753
|
+
...(testRun ? testModeEnv() : {}),
|
|
2754
|
+
...stepSecrets,
|
|
1860
2755
|
},
|
|
1861
2756
|
emit: pushWatching,
|
|
1862
2757
|
});
|
|
1863
2758
|
// With a translator the step's endpoint is the translator: started
|
|
1864
2759
|
// here, pointed at by the SDK's env, closed when the attempt ends.
|
|
2760
|
+
// A test run brackets each attempt with a snapshot of state/ and
|
|
2761
|
+
// storage/: the step writes them for real, because there is no
|
|
2762
|
+
// sandbox to write instead, and afterwards what changed is moved
|
|
2763
|
+
// under the run and the originals come back.
|
|
1865
2764
|
const attemptThrough = async (modelEnv: Record<string, string | undefined>, spec: TranslatorSpec | null) => {
|
|
1866
|
-
|
|
1867
|
-
const t = await startTranslator(spec);
|
|
2765
|
+
const before = testRun ? snapshotDivertedDirs(workspaceRoot) : null;
|
|
2766
|
+
const t = spec ? await startTranslator(spec) : null;
|
|
1868
2767
|
try {
|
|
1869
|
-
return await attemptInProcess({ ...modelEnv, ...t.env });
|
|
2768
|
+
return await attemptInProcess(t ? { ...modelEnv, ...t.env } : modelEnv);
|
|
1870
2769
|
} finally {
|
|
1871
|
-
|
|
1872
|
-
|
|
2770
|
+
if (t) {
|
|
2771
|
+
for (const line of t.drainLog()) pushWatching("info", line.startsWith("translator") ? line : `translator: ${line}`);
|
|
2772
|
+
await t.close();
|
|
2773
|
+
}
|
|
2774
|
+
if (before && runId) {
|
|
2775
|
+
const d = divertFor(runId);
|
|
2776
|
+
restoreDivertedDirs(workspaceRoot, before, d.to, d.note);
|
|
2777
|
+
}
|
|
1873
2778
|
}
|
|
1874
2779
|
};
|
|
2780
|
+
let credentialUsed = platformModelCredential();
|
|
1875
2781
|
let outcome = await attemptThrough(providerEnv, translator);
|
|
2782
|
+
// Same rule as the isolated path: a credential that rotated under the
|
|
2783
|
+
// step is waited out and retried before anything else is considered.
|
|
2784
|
+
if (outcome.status === "failed" && isAuthRefusal(lastRefusal)) {
|
|
2785
|
+
push("info", `the model credential was refused mid-step — waiting for the refreshed one`);
|
|
2786
|
+
const rotated = await awaitRotatedCredential(credentialUsed);
|
|
2787
|
+
if (rotated) {
|
|
2788
|
+
push("info", "the credential rotated under this step; retrying on the new one");
|
|
2789
|
+
credentialUsed = rotated;
|
|
2790
|
+
lastRefusal = "";
|
|
2791
|
+
outcome = await attemptThrough(providerEnv, translator);
|
|
2792
|
+
} else {
|
|
2793
|
+
push("info", "the credential did not change — this is the key itself, not a rotation");
|
|
2794
|
+
}
|
|
2795
|
+
}
|
|
1876
2796
|
// Same rule as the isolated path: a busy provider is waited out on
|
|
1877
2797
|
// the supply it refused from, before any fallback is considered.
|
|
1878
2798
|
for (let n = 1; n <= OVERLOAD_RETRIES && outcome.status === "failed" && isTransientOverload(lastRefusal); n++) {
|
|
@@ -1889,16 +2809,18 @@ async function runStep(
|
|
|
1889
2809
|
"info",
|
|
1890
2810
|
`primary model supply refused (${lastRefusal.slice(0, 80)}) — retrying this step on the fallback provider`,
|
|
1891
2811
|
);
|
|
2812
|
+
supplyState = "exhausted";
|
|
1892
2813
|
outcome = await attemptThrough(secondSupply, fallbackEnv && secondSupply === fallbackEnv ? fallbackTranslator : null);
|
|
1893
2814
|
}
|
|
1894
|
-
publishPublicDir(tenant, path.basename(workspaceRoot), push);
|
|
2815
|
+
if (!testRun) await publishPublicDir(tenant, path.basename(workspaceRoot), push);
|
|
2816
|
+
sayWhy(outcome.status);
|
|
1895
2817
|
step.status = outcome.status;
|
|
1896
|
-
step.result = outcome.result;
|
|
1897
|
-
step.conclusion = outcome.conclusion;
|
|
1898
|
-
if (outcome.data !== undefined) step.data = outcome.data;
|
|
2818
|
+
step.result = redactText(outcome.result);
|
|
2819
|
+
step.conclusion = redactText(outcome.conclusion);
|
|
2820
|
+
if (outcome.data !== undefined) step.data = redactData(outcome.data);
|
|
1899
2821
|
// A consult's spend belongs to the step that asked.
|
|
1900
2822
|
const consultCost = consultTools.drainCost();
|
|
1901
|
-
const stepCost = repriced(catalog, wireModel, outcome.usage, outcome.costUsd, push);
|
|
2823
|
+
const stepCost = repriced(catalog, wireModel, outcome.usage, outcome.costUsd, push, outcome.turnsCostUsd);
|
|
1902
2824
|
step.costUsd = stepCost === null && consultCost === 0 ? null : (stepCost ?? 0) + consultCost;
|
|
1903
2825
|
step.tokens = outcome.usage
|
|
1904
2826
|
? { input: outcome.usage.inputTokens, output: outcome.usage.outputTokens }
|
|
@@ -1915,9 +2837,33 @@ async function runStep(
|
|
|
1915
2837
|
} catch {
|
|
1916
2838
|
// best-effort — a leftover scratch file is swept with the run's outputs
|
|
1917
2839
|
}
|
|
2840
|
+
try {
|
|
2841
|
+
// A private runtime build (see prepareRuntime) is this step's alone.
|
|
2842
|
+
disposeRuntime?.();
|
|
2843
|
+
} catch {
|
|
2844
|
+
// best-effort — it is in the tmpdir
|
|
2845
|
+
}
|
|
1918
2846
|
}
|
|
1919
2847
|
step.finishedAt = new Date().toISOString();
|
|
1920
2848
|
save();
|
|
2849
|
+
return { actedThenFailedCheck: canActOutward && actedThenFailedCheck(step.events.slice(eventsAtStart)) };
|
|
2850
|
+
}
|
|
2851
|
+
|
|
2852
|
+
/**
|
|
2853
|
+
* Did this attempt call its tools and then fail only on the check after —
|
|
2854
|
+
* `verify:`, `output: json`, a schema? For a step whose tools act outward
|
|
2855
|
+
* that is the one failure a retry must not answer: the email went, the post
|
|
2856
|
+
* is up, and the check failing says nothing about whether they happened. A
|
|
2857
|
+
* second attempt sends them again. Read off the attempt's own events, which
|
|
2858
|
+
* both executors write the same way (step-exec.ts): every event a failed
|
|
2859
|
+
* check writes carries `check: true`. It used to read only the LAST error's
|
|
2860
|
+
* wording, and a failed shell verify or judge writes its detail after its
|
|
2861
|
+
* headline — so the step was retried and the email went twice. The wording
|
|
2862
|
+
* is still honoured, for a driver image older than the tag.
|
|
2863
|
+
*/
|
|
2864
|
+
export function actedThenFailedCheck(events: { type: string; text: string; check?: boolean }[]): boolean {
|
|
2865
|
+
if (!events.some((e) => e.type === "tool")) return false;
|
|
2866
|
+
return events.some((e) => e.type === "error" && (e.check === true || /^(verify `|output: json — |schema: the value does not fit)/.test(e.text)));
|
|
1921
2867
|
}
|
|
1922
2868
|
|
|
1923
2869
|
// Block until a human approves or rejects the pending steps. Polls the run
|
|
@@ -2018,6 +2964,59 @@ function platformFallbackEnv(): Record<string, string> | null {
|
|
|
2018
2964
|
return out;
|
|
2019
2965
|
}
|
|
2020
2966
|
|
|
2967
|
+
/**
|
|
2968
|
+
* A 401 the token refresh caused, rather than a key that is simply wrong.
|
|
2969
|
+
*
|
|
2970
|
+
* The refresher on the box turns the long-lived grant into an 8h access
|
|
2971
|
+
* token every four hours, and Anthropic revokes the old one the instant the
|
|
2972
|
+
* new one is minted. The platform reads the credential per step from the
|
|
2973
|
+
* mounted file, which fixed the boot-time copy — but not this: a step
|
|
2974
|
+
* already talking to the API when the rotation lands gets a hard 401
|
|
2975
|
+
* mid-turn, and the replacement does not appear in the mounted file until
|
|
2976
|
+
* the kubelet syncs it, about a minute later.
|
|
2977
|
+
*
|
|
2978
|
+
* That window is narrow and it is not rare. Every 401 on this account in
|
|
2979
|
+
* three weeks of runs fell inside it: 02:07:22 and 02:07:24 against a 02:07
|
|
2980
|
+
* refresh, 22:08:12, 22:08:17, 22:08:30 and 22:09:03 against 22:08. The
|
|
2981
|
+
* desks that lost a run nightly were simply the ones scheduled on the hour
|
|
2982
|
+
* the refresher shares.
|
|
2983
|
+
*
|
|
2984
|
+
* So: on a 401, wait for the file to hold a DIFFERENT credential, then let
|
|
2985
|
+
* the caller retry with it. A token that never changes inside the window is
|
|
2986
|
+
* a key that is actually invalid — no waiting fixes that, and the caller
|
|
2987
|
+
* falls through to the second supply exactly as before. Moving the timer
|
|
2988
|
+
* cannot fix this; a run that takes forty minutes spans any minute chosen.
|
|
2989
|
+
*/
|
|
2990
|
+
const CREDENTIAL_ROTATION_WAIT_MS = 90_000;
|
|
2991
|
+
const CREDENTIAL_POLL_MS = 2_000;
|
|
2992
|
+
|
|
2993
|
+
function isAuthRefusal(text: string): boolean {
|
|
2994
|
+
return /API Error: 401|access token has (been revoked|expired)/i.test(text);
|
|
2995
|
+
}
|
|
2996
|
+
|
|
2997
|
+
/**
|
|
2998
|
+
* Wait, briefly, for the mounted credential to become one we have not
|
|
2999
|
+
* already been refused on. Returns the new value, or null when the window
|
|
3000
|
+
* passes without a change — which is the answer that means "this key is
|
|
3001
|
+
* wrong", not "this key is stale".
|
|
3002
|
+
*/
|
|
3003
|
+
export async function awaitRotatedCredential(
|
|
3004
|
+
before: string | undefined,
|
|
3005
|
+
now: () => number = Date.now,
|
|
3006
|
+
read: () => string | undefined = () => platformModelCredential(),
|
|
3007
|
+
waitMs = CREDENTIAL_ROTATION_WAIT_MS,
|
|
3008
|
+
pollMs = CREDENTIAL_POLL_MS,
|
|
3009
|
+
): Promise<string | null> {
|
|
3010
|
+
if (!before) return null;
|
|
3011
|
+
const until = now() + waitMs;
|
|
3012
|
+
for (;;) {
|
|
3013
|
+
const current = read();
|
|
3014
|
+
if (current && current !== before) return current;
|
|
3015
|
+
if (now() >= until) return null;
|
|
3016
|
+
await sleep(pollMs);
|
|
3017
|
+
}
|
|
3018
|
+
}
|
|
3019
|
+
|
|
2021
3020
|
/** A model-call failure the fallback can answer: the primary refusing over
|
|
2022
3021
|
* credit, quota or auth — never a failure of the work itself, which would
|
|
2023
3022
|
* fail identically anywhere. */
|
|
@@ -2028,7 +3027,19 @@ function isProviderRefusal(text: string): boolean {
|
|
|
2028
3027
|
// fallback it could only time out on again. A refusal the fallback can
|
|
2029
3028
|
// answer is about the ACCOUNT; anything about the work or the sandbox is
|
|
2030
3029
|
// not, however similar the vocabulary.
|
|
2031
|
-
|
|
3030
|
+
// A subscription's own ceiling is the same kind of refusal, worded
|
|
3031
|
+
// differently: Claude Code on a Max plan answers "You've hit your weekly
|
|
3032
|
+
// limit · resets 5am" with no status code and none of the money words.
|
|
3033
|
+
// On 2026-09-21 every opus-tier step on the account failed on exactly that
|
|
3034
|
+
// sentence while sonnet steps beside them ran fine — the plan has a
|
|
3035
|
+
// separate weekly cap for Opus — and nothing here recognised it, so a
|
|
3036
|
+
// second supply could not have answered even if one had been configured.
|
|
3037
|
+
// It is not transient (isTransientOverload is right to ignore it: the
|
|
3038
|
+
// window is a week), it is the account's supply being exhausted, which is
|
|
3039
|
+
// precisely what the fallback provider is for.
|
|
3040
|
+
return /API Error: (401|402|403|429|5\d\d)|credits?\b|quota|billing|insufficient|overloaded|(weekly|daily|monthly|usage) limit|rate.?limit(ed)?\b.*resets?\b|limit\b.*\bresets?\b/i.test(
|
|
3041
|
+
text,
|
|
3042
|
+
);
|
|
2032
3043
|
}
|
|
2033
3044
|
|
|
2034
3045
|
/** How many times a busy provider is given another moment, and the longest
|
|
@@ -2084,7 +3095,47 @@ export function backoffMs(attempt: number, text = "", random: () => number = Mat
|
|
|
2084
3095
|
return Math.round(base * (0.5 + random() * 0.5));
|
|
2085
3096
|
}
|
|
2086
3097
|
|
|
2087
|
-
|
|
3098
|
+
/**
|
|
3099
|
+
* The platform's model credential, read at the moment a step needs it.
|
|
3100
|
+
*
|
|
3101
|
+
* It used to come only from this process's environment, which meant the
|
|
3102
|
+
* worker held whatever token it was started with. Refreshing an OAuth token
|
|
3103
|
+
* REVOKES the old one, so a refresh without a restart left every run
|
|
3104
|
+
* answering 401 — on 2026-09-18 that took every desk down for two hours, and
|
|
3105
|
+
* nothing said so until a person read a failed run. The refresher's own
|
|
3106
|
+
* comment had named the fix years of incidents earlier: stop shipping the
|
|
3107
|
+
* token as an env var.
|
|
3108
|
+
*
|
|
3109
|
+
* So: when `FOLDRUN_MODEL_KEY_FILE` names a file (the Kubernetes secret,
|
|
3110
|
+
* mounted rather than injected), it is read fresh every time. The kubelet
|
|
3111
|
+
* updates a mounted secret in place within about a minute of it changing, so
|
|
3112
|
+
* a refreshed token reaches the next step on its own, with no restart, no
|
|
3113
|
+
* timing window, and nothing to skip while work is in flight.
|
|
3114
|
+
*
|
|
3115
|
+
* A missing or empty file falls back to the environment, which is what a
|
|
3116
|
+
* local install, the CLI and every compose setup use.
|
|
3117
|
+
*/
|
|
3118
|
+
export function platformModelCredential(env: NodeJS.ProcessEnv = process.env): string | undefined {
|
|
3119
|
+
const file = env.FOLDRUN_MODEL_KEY_FILE;
|
|
3120
|
+
if (file) {
|
|
3121
|
+
try {
|
|
3122
|
+
const value = fs.readFileSync(file, "utf8").trim();
|
|
3123
|
+
if (value) return value;
|
|
3124
|
+
} catch {
|
|
3125
|
+
// Unreadable is not the same as absent: say so once, then fall back,
|
|
3126
|
+
// because a running platform that cannot read its credential file is a
|
|
3127
|
+
// thing an operator needs to know about before the next step fails.
|
|
3128
|
+
if (!credentialFileWarned) {
|
|
3129
|
+
credentialFileWarned = true;
|
|
3130
|
+
console.warn(`[model] FOLDRUN_MODEL_KEY_FILE=${file} could not be read; falling back to the environment`);
|
|
3131
|
+
}
|
|
3132
|
+
}
|
|
3133
|
+
}
|
|
3134
|
+
return env.CLAUDE_CODE_OAUTH_TOKEN || undefined;
|
|
3135
|
+
}
|
|
3136
|
+
let credentialFileWarned = false;
|
|
3137
|
+
|
|
3138
|
+
export function platformModelEnv(): Record<string, string | undefined> {
|
|
2088
3139
|
const {
|
|
2089
3140
|
ANTHROPIC_API_KEY,
|
|
2090
3141
|
ANTHROPIC_AUTH_TOKEN,
|
|
@@ -2103,7 +3154,9 @@ function platformModelEnv(): Record<string, string | undefined> {
|
|
|
2103
3154
|
ANTHROPIC_API_KEY,
|
|
2104
3155
|
ANTHROPIC_AUTH_TOKEN,
|
|
2105
3156
|
ANTHROPIC_BASE_URL,
|
|
2106
|
-
|
|
3157
|
+
// The file wins when there is one: it is the live secret, and this
|
|
3158
|
+
// process's copy is whatever it was handed at boot.
|
|
3159
|
+
CLAUDE_CODE_OAUTH_TOKEN: platformModelCredential() ?? CLAUDE_CODE_OAUTH_TOKEN,
|
|
2107
3160
|
ANTHROPIC_DEFAULT_HAIKU_MODEL,
|
|
2108
3161
|
ANTHROPIC_DEFAULT_SONNET_MODEL,
|
|
2109
3162
|
ANTHROPIC_DEFAULT_OPUS_MODEL,
|
|
@@ -2152,20 +3205,38 @@ function jsonItems(data: unknown, step: StepRecord, take: number): string[] {
|
|
|
2152
3205
|
function csvItems(workspaceDir: string, step: StepRecord, take: number): string[] {
|
|
2153
3206
|
const raw = step.eachPath ?? "";
|
|
2154
3207
|
if (!raw) {
|
|
2155
|
-
step.skipReason = "each: rows needs a path — `each: rows of
|
|
3208
|
+
step.skipReason = "each: rows needs a path — `each: rows of workspace/storage/x.csv`";
|
|
2156
3209
|
return [];
|
|
2157
3210
|
}
|
|
2158
|
-
const
|
|
2159
|
-
if (
|
|
2160
|
-
step.skipReason = `each: rows — ${
|
|
3211
|
+
const read = csvDataRows(workspaceDir, step.agent, raw);
|
|
3212
|
+
if ("problem" in read) {
|
|
3213
|
+
step.skipReason = `each: rows — ${read.problem}`;
|
|
2161
3214
|
return [];
|
|
2162
3215
|
}
|
|
3216
|
+
return read.data.slice(0, take).map((r) => `${read.header}\n${r}`);
|
|
3217
|
+
}
|
|
3218
|
+
|
|
3219
|
+
/**
|
|
3220
|
+
* The data rows of a CSV, path agent-relative like every path in a flow and
|
|
3221
|
+
* confined to the workspace. Shared by `each: rows of`, which fans out over
|
|
3222
|
+
* them, and `when: rows of`, which asks only whether there are any.
|
|
3223
|
+
*/
|
|
3224
|
+
function csvDataRows(
|
|
3225
|
+
workspaceDir: string,
|
|
3226
|
+
agent: string,
|
|
3227
|
+
raw: string,
|
|
3228
|
+
): { header: string; data: string[] } | { problem: string } {
|
|
3229
|
+
// `workspace/storage/x.csv` from the workspace root, anything else from
|
|
3230
|
+
// the agent's folder (`../../storage/x.csv` is the same file).
|
|
3231
|
+
const resolved = resolveAgentPath(workspaceDir, path.join(workspaceDir, "agents", agent), raw);
|
|
3232
|
+
if (!resolved.startsWith(path.resolve(workspaceDir) + path.sep)) {
|
|
3233
|
+
return { problem: `${raw} is outside this workspace` };
|
|
3234
|
+
}
|
|
2163
3235
|
let text: string;
|
|
2164
3236
|
try {
|
|
2165
3237
|
text = fs.readFileSync(resolved, "utf8");
|
|
2166
3238
|
} catch {
|
|
2167
|
-
|
|
2168
|
-
return [];
|
|
3239
|
+
return { problem: `${raw} does not exist (yet?)` };
|
|
2169
3240
|
}
|
|
2170
3241
|
// Minimal CSV: quoted fields with embedded commas/newlines survive.
|
|
2171
3242
|
const rows: string[] = [];
|
|
@@ -2182,11 +3253,28 @@ function csvItems(workspaceDir: string, step: StepRecord, take: number): string[
|
|
|
2182
3253
|
}
|
|
2183
3254
|
if (row + field) rows.push(row + field);
|
|
2184
3255
|
const [header, ...data] = rows.filter((r) => r.trim() !== "");
|
|
2185
|
-
if (!header || data.length === 0) {
|
|
2186
|
-
|
|
2187
|
-
|
|
3256
|
+
if (!header || data.length === 0) return { problem: `${raw} has no data rows` };
|
|
3257
|
+
return { header, data };
|
|
3258
|
+
}
|
|
3259
|
+
|
|
3260
|
+
/**
|
|
3261
|
+
* `schema: <path>` — the file, read and parsed, or null. Agent-relative like
|
|
3262
|
+
* every other path in a flow (`workspace/schemas/lead.json`), confined to the
|
|
3263
|
+
* workspace: a schema is the author's document, never something a step
|
|
3264
|
+
* can point outside.
|
|
3265
|
+
*/
|
|
3266
|
+
export function readSchemaFile(workspaceRoot: string, rel: string): Record<string, unknown> | boolean | null {
|
|
3267
|
+
const root = path.resolve(workspaceRoot);
|
|
3268
|
+
// Written `workspace/schemas/x.json`, agent-relative (`../../schemas/x.json`)
|
|
3269
|
+
// or workspace-relative (`schemas/x.json`); all land on the same file.
|
|
3270
|
+
const candidates = [path.resolve(root, rel.replace(/^((\.\.\/){2}|workspace\/)/, "")), path.resolve(root, "agents", "x", rel)];
|
|
3271
|
+
for (const abs of candidates) {
|
|
3272
|
+
if (abs !== root && !abs.startsWith(root + path.sep)) continue;
|
|
3273
|
+
if (!fs.existsSync(abs) || !fs.statSync(abs).isFile()) continue;
|
|
3274
|
+
const parsed = parseSchemaText(fs.readFileSync(abs, "utf8"));
|
|
3275
|
+
return "problem" in parsed ? null : parsed.schema;
|
|
2188
3276
|
}
|
|
2189
|
-
return
|
|
3277
|
+
return null;
|
|
2190
3278
|
}
|
|
2191
3279
|
|
|
2192
3280
|
export function createFlowRun(
|
|
@@ -2196,12 +3284,16 @@ export function createFlowRun(
|
|
|
2196
3284
|
flowName: string,
|
|
2197
3285
|
status: "queued" | "running",
|
|
2198
3286
|
tags: string[] = [],
|
|
3287
|
+
startedBy: string | null = null,
|
|
3288
|
+
opts: { test?: boolean } = {},
|
|
2199
3289
|
): RunRecord {
|
|
2200
3290
|
const run: RunRecord = {
|
|
2201
3291
|
id: `run-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`,
|
|
2202
3292
|
flow: flowName,
|
|
2203
3293
|
tags,
|
|
2204
3294
|
status,
|
|
3295
|
+
...(startedBy ? { startedBy: startedBy.trim().toLowerCase() } : {}),
|
|
3296
|
+
...(opts.test ? { test: true } : {}),
|
|
2205
3297
|
startedAt: new Date().toISOString(),
|
|
2206
3298
|
finishedAt: null,
|
|
2207
3299
|
steps: steps.map((s) => ({
|
|
@@ -2231,6 +3323,11 @@ export function createFlowRun(
|
|
|
2231
3323
|
preview: s.preview,
|
|
2232
3324
|
delegate: s.delegate,
|
|
2233
3325
|
output: s.output,
|
|
3326
|
+
schema: s.schema,
|
|
3327
|
+
schemaPath: s.schemaPath,
|
|
3328
|
+
parallel: s.parallel,
|
|
3329
|
+
maxTurns: s.maxTurns,
|
|
3330
|
+
limits: s.limits,
|
|
2234
3331
|
attempts: 0,
|
|
2235
3332
|
status: "pending",
|
|
2236
3333
|
events: [],
|
|
@@ -2267,8 +3364,10 @@ export function startFlowRun(
|
|
|
2267
3364
|
modelOverride?: string | null,
|
|
2268
3365
|
tags: string[] = [],
|
|
2269
3366
|
effortOverride?: string | null,
|
|
3367
|
+
startedBy: string | null = null,
|
|
3368
|
+
opts: { test?: boolean } = {},
|
|
2270
3369
|
): RunRecord {
|
|
2271
|
-
const run = createFlowRun(tenant, workspace, steps, flowName, "running", tags);
|
|
3370
|
+
const run = createFlowRun(tenant, workspace, steps, flowName, "running", tags, startedBy, opts);
|
|
2272
3371
|
void driveRun(tenant, workspace, run, modelOverride, tags, { effortOverride });
|
|
2273
3372
|
return run;
|
|
2274
3373
|
}
|
|
@@ -2384,6 +3483,18 @@ function driveRunInner(
|
|
|
2384
3483
|
opts: { parkOnApproval?: boolean; effortOverride?: string | null },
|
|
2385
3484
|
): Promise<void> {
|
|
2386
3485
|
const effortOverride = opts.effortOverride ?? null;
|
|
3486
|
+
// The flow's own calendar. Read from the flow file rather than passed in,
|
|
3487
|
+
// so every door into a run — the queue, an eval, the CLI, a rerun, a
|
|
3488
|
+
// reconcile that picks an abandoned run back up — gets the same clock
|
|
3489
|
+
// without each caller having to remember it. A run that belongs to no
|
|
3490
|
+
// flow (an ad-hoc agent run) finds nothing here and cascades from the
|
|
3491
|
+
// agent to the workspace, which is what it should do.
|
|
3492
|
+
let flowTimezone: string | null = null;
|
|
3493
|
+
try {
|
|
3494
|
+
flowTimezone = readFlow(tenant, workspace, run.flow)?.timezone ?? null;
|
|
3495
|
+
} catch {
|
|
3496
|
+
// an unreadable flow file is reported where it is parsed for real
|
|
3497
|
+
}
|
|
2387
3498
|
const pDir = workspaceDir(tenant, workspace);
|
|
2388
3499
|
const runRoot = pDir;
|
|
2389
3500
|
/**
|
|
@@ -2566,6 +3677,17 @@ function driveRunInner(
|
|
|
2566
3677
|
// is assembled) keeps the newest: when the total is over it, the
|
|
2567
3678
|
// oldest groups are dropped whole before anything is cut mid-text.
|
|
2568
3679
|
const contextBefore = (gi: number) => joinEarlierGroups(groupResults, gi);
|
|
3680
|
+
// What `when:` and `case:` read: the PREVIOUS group's result alone —
|
|
3681
|
+
// the nearest earlier group that produced one, so a routed-past group
|
|
3682
|
+
// between is looked through rather than read as silence. The prompt
|
|
3683
|
+
// gets every earlier group joined (contextBefore); routing must not,
|
|
3684
|
+
// because a verdict two groups back would still open a gate a later
|
|
3685
|
+
// group had overruled: a classifier's BUG line matched a `case: BUG`
|
|
3686
|
+
// three groups on, after the debugger had already answered it.
|
|
3687
|
+
const routingBefore = (gi: number): string | null => {
|
|
3688
|
+
for (let i = gi - 1; i >= 0; i--) if (groupResults[i]) return groupResults[i];
|
|
3689
|
+
return null;
|
|
3690
|
+
};
|
|
2569
3691
|
// The data half of the same handoff: what the nearest earlier group
|
|
2570
3692
|
// returned through `output: json`. Rebuilt from the record rather than
|
|
2571
3693
|
// kept only in memory, so a resumed run hands step 3 the value step 1
|
|
@@ -2630,8 +3752,11 @@ function driveRunInner(
|
|
|
2630
3752
|
context = contextBefore(gi);
|
|
2631
3753
|
ctxData = dataBefore(gi);
|
|
2632
3754
|
|
|
2633
|
-
|
|
2634
|
-
|
|
3755
|
+
// What the run has spent, not just what its tokens cost: on an
|
|
3756
|
+
// install that prices sandbox seconds, a step that called no model
|
|
3757
|
+
// still spent money, and a cap that ignored it capped nothing.
|
|
3758
|
+
if (run.budgetUsd && platform.runSpend(run) >= run.budgetUsd && group.some((s) => s.status === "pending")) {
|
|
3759
|
+
const spent = platform.runSpend(run);
|
|
2635
3760
|
const last = run.steps.findLast((s) => s.status === "completed" || s.status === "failed") ?? run.steps[0];
|
|
2636
3761
|
last?.events.push({
|
|
2637
3762
|
t: new Date().toISOString(),
|
|
@@ -2720,6 +3845,7 @@ function driveRunInner(
|
|
|
2720
3845
|
type: "info",
|
|
2721
3846
|
text:
|
|
2722
3847
|
`fan-out: ${used.length} instance${used.length === 1 ? "" : "s"}` +
|
|
3848
|
+
(step.parallel && step.parallel < used.length ? `, ${step.parallel} at a time (parallel:)` : "") +
|
|
2723
3849
|
(dropped ? ` (more items existed — capped at ${cap})` : ""),
|
|
2724
3850
|
});
|
|
2725
3851
|
const at = run.steps.indexOf(step);
|
|
@@ -2734,6 +3860,7 @@ function driveRunInner(
|
|
|
2734
3860
|
result: null,
|
|
2735
3861
|
costUsd: null,
|
|
2736
3862
|
attempts: 0,
|
|
3863
|
+
tries: undefined,
|
|
2737
3864
|
}));
|
|
2738
3865
|
run.steps.splice(at + 1, 0, ...instances);
|
|
2739
3866
|
ordered = orderedGroups();
|
|
@@ -2747,12 +3874,25 @@ function driveRunInner(
|
|
|
2747
3874
|
// this is what keeps the *next* group from starting.
|
|
2748
3875
|
const stopCheck = readRun(tenant, workspace, run.id);
|
|
2749
3876
|
if (stopCheck?.stopRequested) {
|
|
3877
|
+
let skipped = 0;
|
|
2750
3878
|
for (const s of run.steps) {
|
|
2751
3879
|
if (s.status === "pending" || s.status === "running") {
|
|
2752
3880
|
s.status = "skipped";
|
|
2753
3881
|
s.skipReason = "run stopped";
|
|
3882
|
+
skipped += 1;
|
|
2754
3883
|
}
|
|
2755
3884
|
}
|
|
3885
|
+
// Say so on the trail. stopRun wrote "run stopped" onto the
|
|
3886
|
+
// record's first step, but the save() below writes this process's
|
|
3887
|
+
// copy of the record over it, so the only witness was the
|
|
3888
|
+
// stopRequested flag — and a run that "failed" with no error line
|
|
3889
|
+
// anywhere took a code read to explain (rank-desk, 2026-09-13).
|
|
3890
|
+
const witness = run.steps.findLast((s) => s.status === "completed" || s.status === "failed") ?? run.steps[0];
|
|
3891
|
+
witness?.events.push({
|
|
3892
|
+
t: new Date().toISOString(),
|
|
3893
|
+
type: "info",
|
|
3894
|
+
text: `run stopped by a person between groups — ${skipped} remaining step${skipped === 1 ? "" : "s"} skipped`,
|
|
3895
|
+
});
|
|
2756
3896
|
run.status = "failed";
|
|
2757
3897
|
run.stopRequested = true;
|
|
2758
3898
|
save();
|
|
@@ -2788,40 +3928,67 @@ function driveRunInner(
|
|
|
2788
3928
|
}
|
|
2789
3929
|
|
|
2790
3930
|
const ctx = context;
|
|
3931
|
+
// The previous group's result — see routingBefore.
|
|
3932
|
+
const routing = routingBefore(gi);
|
|
2791
3933
|
|
|
2792
|
-
// `when:` — skip a step whose condition isn't met by
|
|
3934
|
+
// `when:` — skip a step whose condition isn't met by the previous
|
|
3935
|
+
// group's result.
|
|
2793
3936
|
for (const step of freshGroup) {
|
|
2794
3937
|
if (step.status !== "pending" || !step.when) continue;
|
|
2795
|
-
|
|
3938
|
+
// `when: rows of <csv>` asks the data, not the prose: run only if
|
|
3939
|
+
// the file has a data row. It is the condition a fan-out's
|
|
3940
|
+
// followers need — an `each:` over an empty file skips itself but
|
|
3941
|
+
// leaves nothing for a marker to be read from, so the steps after
|
|
3942
|
+
// it used to run on nothing.
|
|
3943
|
+
const rowsOf = whenRowsPath(step.when);
|
|
3944
|
+
if (rowsOf !== null) {
|
|
3945
|
+
const read = csvDataRows(pDir, step.agent, rowsOf);
|
|
3946
|
+
if ("problem" in read) {
|
|
3947
|
+
step.status = "skipped";
|
|
3948
|
+
step.skipReason = `condition not met: when rows — ${read.problem}`;
|
|
3949
|
+
step.events.push({ t: new Date().toISOString(), type: "info", text: `skipped — ${read.problem}` });
|
|
3950
|
+
}
|
|
3951
|
+
continue;
|
|
3952
|
+
}
|
|
3953
|
+
// A marker at the start of a line, not a word anywhere in the
|
|
3954
|
+
// prose — see markerPresent. "There are no BLOCKED items" must
|
|
3955
|
+
// not open a gate keyed on BLOCKED.
|
|
3956
|
+
const met = markerPresent(routing, step.when);
|
|
2796
3957
|
if (!met) {
|
|
2797
3958
|
step.status = "skipped";
|
|
2798
3959
|
step.skipReason = `condition not met: when "${step.when}"`;
|
|
2799
3960
|
step.events.push({
|
|
2800
3961
|
t: new Date().toISOString(),
|
|
2801
3962
|
type: "info",
|
|
2802
|
-
text: `skipped — previous
|
|
3963
|
+
text: `skipped — the previous group's result has no line beginning "${step.when}" (when: reads a marker at the start of a line of the previous result, not a word in a sentence)`,
|
|
2803
3964
|
});
|
|
2804
3965
|
}
|
|
2805
3966
|
}
|
|
2806
3967
|
save();
|
|
2807
3968
|
|
|
2808
3969
|
// `case:`/`else:` — exclusive routing. Of this group's case steps,
|
|
2809
|
-
// the FIRST whose
|
|
2810
|
-
// rest are routed past. `else:` runs only when no case matched.
|
|
3970
|
+
// the FIRST whose marker leads a line of the previous results runs;
|
|
3971
|
+
// the rest are routed past. `else:` runs only when no case matched.
|
|
2811
3972
|
// (`when:` above stays independent — every matching when runs —
|
|
2812
3973
|
// which is why routing is its own vocabulary instead of a mode.)
|
|
3974
|
+
//
|
|
3975
|
+
// Same marker rule as `when:`, and it matters more here: routing is
|
|
3976
|
+
// exclusive, so a label matched out of a sentence does not merely
|
|
3977
|
+
// run an extra step, it sends the flow down the wrong branch and
|
|
3978
|
+
// skips the right one. "This is not a COMPLAINT, it is a QUESTION"
|
|
3979
|
+
// used to route to the complaints handler.
|
|
2813
3980
|
const caseSteps = freshGroup.filter((s) => s.status === "pending" && s.case);
|
|
2814
3981
|
if (caseSteps.length || freshGroup.some((s) => s.status === "pending" && s.else)) {
|
|
2815
3982
|
let matchedCase: StepRecord | null = null;
|
|
2816
3983
|
for (const step of caseSteps) {
|
|
2817
|
-
if (!matchedCase && (
|
|
3984
|
+
if (!matchedCase && markerPresent(routing, step.case!)) {
|
|
2818
3985
|
matchedCase = step;
|
|
2819
3986
|
continue;
|
|
2820
3987
|
}
|
|
2821
3988
|
step.status = "skipped";
|
|
2822
3989
|
step.skipReason = matchedCase
|
|
2823
3990
|
? `routed past — "${matchedCase.case}" matched first`
|
|
2824
|
-
: `case not matched: "${step.case}"`;
|
|
3991
|
+
: `case not matched: the previous group's result has no line beginning "${step.case}"`;
|
|
2825
3992
|
step.events.push({
|
|
2826
3993
|
t: new Date().toISOString(),
|
|
2827
3994
|
type: "info",
|
|
@@ -2865,6 +4032,20 @@ function driveRunInner(
|
|
|
2865
4032
|
});
|
|
2866
4033
|
}
|
|
2867
4034
|
run.status = "awaiting-approval";
|
|
4035
|
+
// approve_within: — the deadline is stamped when the gate opens,
|
|
4036
|
+
// not counted from the run's start, so "two days to answer" means
|
|
4037
|
+
// two days from being asked however long the work before it took.
|
|
4038
|
+
// Only a gate that asks a PERSON gets one: an external event has
|
|
4039
|
+
// no one to chase.
|
|
4040
|
+
const within = readFlow(tenant, workspace, run.flow)?.approveWithin ?? null;
|
|
4041
|
+
// Always assigned, never only set: a run with two gates would
|
|
4042
|
+
// otherwise carry the FIRST gate's deadline into the second, and
|
|
4043
|
+
// a deadline already in the past expires a gate the moment it
|
|
4044
|
+
// opens. Null when this gate has none.
|
|
4045
|
+
run.approveBy =
|
|
4046
|
+
within && needsApproval.some((s) => s.waitFor !== "event")
|
|
4047
|
+
? new Date(Date.now() + within * 1000).toISOString()
|
|
4048
|
+
: null;
|
|
2868
4049
|
save();
|
|
2869
4050
|
|
|
2870
4051
|
// The gate's declared preview, resolved now against what the run
|
|
@@ -2939,10 +4120,55 @@ function driveRunInner(
|
|
|
2939
4120
|
// twenty steps over — the shares sum to what was left. Between
|
|
2940
4121
|
// groups the check above still refuses to start the next one.
|
|
2941
4122
|
const launching = freshGroup.filter((s) => s.status === "pending");
|
|
2942
|
-
|
|
4123
|
+
// An agent's own `budget:` — the most it may spend in one run — is
|
|
4124
|
+
// read at launch, so a cap raised mid-run applies to the next step.
|
|
4125
|
+
const agentBudgets = new Map(listAgents(tenant, workspace).map((a) => [a.name, a.budget]));
|
|
4126
|
+
// The ceiling each attempt now running was given. A step records its
|
|
4127
|
+
// cost only when it ends, so while it runs its ceiling stands in for
|
|
4128
|
+
// what it may yet spend — otherwise every fan-out copy of one agent
|
|
4129
|
+
// saw none of its siblings' spend and got the agent's whole budget.
|
|
4130
|
+
const reserved = new Map<StepRecord, number>();
|
|
4131
|
+
// The ceiling for one attempt of one step, read NOW: what is left of
|
|
4132
|
+
// the run's budget shared across the group, or the agent's own cap
|
|
4133
|
+
// less what its steps have cost or hold in flight, shared with its
|
|
4134
|
+
// steps in this group still waiting to start. Computed before every
|
|
4135
|
+
// attempt, not once before the loop — a retry's share is what is left
|
|
4136
|
+
// after the failed attempt spent, and the step's own figure already
|
|
4137
|
+
// carries that spend (it is the sum of its tries).
|
|
4138
|
+
const ceilingFor = (step: StepRecord) =>
|
|
4139
|
+
stepCeilingFor(
|
|
4140
|
+
stepCeiling(run.budgetUsd, platform.runSpend(run), launching.length),
|
|
4141
|
+
agentBudgets.get(step.agent),
|
|
4142
|
+
run.steps
|
|
4143
|
+
.filter((s) => s.agent === step.agent)
|
|
4144
|
+
.reduce((sum, s) => sum + (s.costUsd ?? 0) + (s !== step && s.status === "running" ? (reserved.get(s) ?? 0) : 0), 0),
|
|
4145
|
+
step.agent,
|
|
4146
|
+
1 + launching.filter((s) => s !== step && s.agent === step.agent && s.status === "pending" && !reserved.has(s)).length,
|
|
4147
|
+
);
|
|
4148
|
+
// parallel: — the instances of one fan-out template share a pool.
|
|
4149
|
+
// Keyed by what the instances share (agent + instruction), since
|
|
4150
|
+
// the template itself is behind them in the record as "expanded".
|
|
4151
|
+
const pools = new Map<string, ReturnType<typeof slotPool>>();
|
|
4152
|
+
const poolFor = (step: StepRecord) => {
|
|
4153
|
+
if (!step.parallel || !step.item) return null;
|
|
4154
|
+
const key = `${step.agent}\u0000${step.instruction}`;
|
|
4155
|
+
let pool = pools.get(key);
|
|
4156
|
+
if (!pool) pools.set(key, (pool = slotPool(step.parallel)));
|
|
4157
|
+
return pool;
|
|
4158
|
+
};
|
|
2943
4159
|
await Promise.all(
|
|
2944
4160
|
launching
|
|
2945
4161
|
.map(async (step) => {
|
|
4162
|
+
const pool = poolFor(step);
|
|
4163
|
+
if (pool) await pool.acquire();
|
|
4164
|
+
try {
|
|
4165
|
+
await launchStep(step);
|
|
4166
|
+
} finally {
|
|
4167
|
+
pool?.release();
|
|
4168
|
+
}
|
|
4169
|
+
}),
|
|
4170
|
+
);
|
|
4171
|
+
async function launchStep(step: StepRecord) {
|
|
2946
4172
|
const attempts = (step.retry ?? 0) + 1;
|
|
2947
4173
|
// A step re-attaching to its sandbox is still on the attempt
|
|
2948
4174
|
// that started it; a step re-run after a backoff picks up the
|
|
@@ -2951,10 +4177,36 @@ function driveRunInner(
|
|
|
2951
4177
|
!step.sandbox && !!step.attempts && /retrying in \d+s/.test(step.events[step.events.length - 1]?.text ?? "");
|
|
2952
4178
|
const from = step.sandbox ? Math.max(1, step.attempts ?? 1) : interruptedInBackoff ? Math.min(attempts, step.attempts! + 1) : 1;
|
|
2953
4179
|
for (let attempt = from; attempt <= attempts; attempt++) {
|
|
4180
|
+
const ceiling = ceilingFor(step);
|
|
4181
|
+
// Nothing left is a refusal, not "no ceiling": a zero passed
|
|
4182
|
+
// down would read as unset, and the one step that must not
|
|
4183
|
+
// run would run with no cap at all. On a retry this is what
|
|
4184
|
+
// stops the second attempt spending a share the first one
|
|
4185
|
+
// already spent.
|
|
4186
|
+
if (ceiling.ceilingUsd === 0) {
|
|
4187
|
+
step.status = "failed";
|
|
4188
|
+
step.events.push({
|
|
4189
|
+
t: new Date().toISOString(),
|
|
4190
|
+
type: "error",
|
|
4191
|
+
text:
|
|
4192
|
+
attempt === from
|
|
4193
|
+
? `over budget — nothing left to spend before this step (${ceiling.note}); it was not started`
|
|
4194
|
+
: `over budget — nothing left to spend for attempt ${attempt} (${ceiling.note}); not retried`,
|
|
4195
|
+
});
|
|
4196
|
+
save();
|
|
4197
|
+
return;
|
|
4198
|
+
}
|
|
4199
|
+
// Re-attaching to a sandbox is the same attempt, so its row
|
|
4200
|
+
// is replaced; anything else is a new row, a loop cycle's
|
|
4201
|
+
// re-run included.
|
|
4202
|
+
const reattach = !!step.sandbox;
|
|
4203
|
+
const attemptStartedAt = new Date().toISOString();
|
|
4204
|
+
const eventsBefore = step.events.length;
|
|
2954
4205
|
step.attempts = attempt;
|
|
2955
4206
|
step.status = "running";
|
|
4207
|
+
if (ceiling.ceilingUsd !== null) reserved.set(step, ceiling.ceilingUsd);
|
|
2956
4208
|
save();
|
|
2957
|
-
await runStep(
|
|
4209
|
+
const verdict = await runStep(
|
|
2958
4210
|
path.join(pDir, "agents", step.agent),
|
|
2959
4211
|
tenant,
|
|
2960
4212
|
step,
|
|
@@ -2965,16 +4217,58 @@ function driveRunInner(
|
|
|
2965
4217
|
effortOverride,
|
|
2966
4218
|
run.id,
|
|
2967
4219
|
ctxData,
|
|
2968
|
-
ceilingUsd,
|
|
2969
|
-
|
|
4220
|
+
ceiling.ceilingUsd,
|
|
4221
|
+
ceiling.note,
|
|
4222
|
+
run.test === true,
|
|
4223
|
+
// save() merges a stop written by another process into
|
|
4224
|
+
// this record; the model loop polls this on a clock.
|
|
4225
|
+
() => {
|
|
4226
|
+
save();
|
|
4227
|
+
return run.stopRequested === true;
|
|
4228
|
+
},
|
|
4229
|
+
flowTimezone,
|
|
4230
|
+
run.steps.indexOf(step),
|
|
4231
|
+
).finally(() => reserved.delete(step));
|
|
2970
4232
|
// runStep mutates step.status; read it through a widened local
|
|
2971
4233
|
// so TS doesn't keep the "running" narrowing from above.
|
|
2972
4234
|
const outcome: string = step.status;
|
|
2973
|
-
|
|
4235
|
+
recordAttempt(step, attempt, outcome === "completed" ? "completed" : "failed", attemptStartedAt, eventsBefore, reattach);
|
|
4236
|
+
save();
|
|
2974
4237
|
// A stop destroys the step's sandbox, which reads here as a
|
|
2975
|
-
// failed attempt
|
|
2976
|
-
//
|
|
2977
|
-
//
|
|
4238
|
+
// failed attempt. stopRun writes "stopped by a person" onto
|
|
4239
|
+
// this step in the record, and the save() above writes this
|
|
4240
|
+
// process's copy over it — so the trail ended on the
|
|
4241
|
+
// sandbox's own last words, "sandbox ended: Error (exit 143)",
|
|
4242
|
+
// and a person's stop read as the platform killing a healthy
|
|
4243
|
+
// step (abandon-desk, 2026-09-29: stopped from the CLI after
|
|
4244
|
+
// ten minutes, then chased as an unexplained kill). Say so,
|
|
4245
|
+
// whether or not a retry was left.
|
|
4246
|
+
if (outcome === "failed" && run.stopRequested) {
|
|
4247
|
+
step.events.push({
|
|
4248
|
+
t: new Date().toISOString(),
|
|
4249
|
+
type: "info",
|
|
4250
|
+
text: "stopped by a person mid-step — its sandbox was destroyed, which is what ended this attempt; not retried",
|
|
4251
|
+
});
|
|
4252
|
+
save();
|
|
4253
|
+
break;
|
|
4254
|
+
}
|
|
4255
|
+
if (outcome !== "failed" || attempt === attempts) break;
|
|
4256
|
+
// An outward step whose tools ran and whose check then failed
|
|
4257
|
+
// is not retried: the check cannot say the send did not
|
|
4258
|
+
// happen, and a retry would do it twice.
|
|
4259
|
+
if (verdict?.actedThenFailedCheck) {
|
|
4260
|
+
step.events.push({
|
|
4261
|
+
t: new Date().toISOString(),
|
|
4262
|
+
type: "info",
|
|
4263
|
+
text: `not retried — this step's tools can act outside the workspace and ran before the check failed; another attempt could repeat what they did (${attempts - attempt} left unused)`,
|
|
4264
|
+
});
|
|
4265
|
+
save();
|
|
4266
|
+
break;
|
|
4267
|
+
}
|
|
4268
|
+
// A failed attempt used to be retried in a fresh sandbox even
|
|
4269
|
+
// after a stop, so a stopped `retry: 2` step ran twice more.
|
|
4270
|
+
// The check above ends the loop on a stop; this one covers a
|
|
4271
|
+
// stop that lands while the outward check was being written.
|
|
2978
4272
|
save();
|
|
2979
4273
|
if (run.stopRequested) break;
|
|
2980
4274
|
// The retry policy. An attempt the cluster ended for want of
|
|
@@ -3014,8 +4308,7 @@ function driveRunInner(
|
|
|
3014
4308
|
break;
|
|
3015
4309
|
}
|
|
3016
4310
|
}
|
|
3017
|
-
|
|
3018
|
-
);
|
|
4311
|
+
}
|
|
3019
4312
|
|
|
3020
4313
|
// on-fail: another agent takes the step over. Sequential and rare:
|
|
3021
4314
|
// the rescue is a second full attempt at the step's job with the
|
|
@@ -3067,7 +4360,7 @@ function driveRunInner(
|
|
|
3067
4360
|
rescue.attempts = 1;
|
|
3068
4361
|
rescue.status = "running";
|
|
3069
4362
|
save();
|
|
3070
|
-
await runStep(rescuerDir, tenant, rescue, ctx, save, modelOverride, tags, effortOverride, run.id, ctxData);
|
|
4363
|
+
await runStep(rescuerDir, tenant, rescue, ctx, save, modelOverride, tags, effortOverride, run.id, ctxData, null, undefined, run.test === true, undefined, flowTimezone, run.steps.indexOf(rescue));
|
|
3071
4364
|
// runStep mutates rescue.status; widen past TS's "running" narrowing.
|
|
3072
4365
|
const rescueOutcome: string = rescue.status;
|
|
3073
4366
|
if (rescueOutcome === "completed") {
|
|
@@ -3197,16 +4490,25 @@ function driveRunInner(
|
|
|
3197
4490
|
for (const s of run.steps) if (s.status === "pending") s.status = "skipped";
|
|
3198
4491
|
} catch (err) {
|
|
3199
4492
|
run.status = "failed";
|
|
4493
|
+
const text = err instanceof Error ? err.message : String(err);
|
|
4494
|
+
let told = false;
|
|
3200
4495
|
run.steps.forEach((s) => {
|
|
3201
4496
|
if (s.status === "running") {
|
|
3202
4497
|
s.status = "failed";
|
|
3203
|
-
s.events.push({
|
|
3204
|
-
|
|
3205
|
-
type: "error",
|
|
3206
|
-
text: err instanceof Error ? err.message : String(err),
|
|
3207
|
-
});
|
|
4498
|
+
s.events.push({ t: new Date().toISOString(), type: "error", text });
|
|
4499
|
+
told = true;
|
|
3208
4500
|
}
|
|
3209
4501
|
});
|
|
4502
|
+
// Thrown between groups, with nothing running: the error still has to
|
|
4503
|
+
// land on some step's trail, or the run reads as failed for no reason.
|
|
4504
|
+
if (!told) {
|
|
4505
|
+
const witness = run.steps.findLast((s) => s.status !== "pending") ?? run.steps[0];
|
|
4506
|
+
witness?.events.push({
|
|
4507
|
+
t: new Date().toISOString(),
|
|
4508
|
+
type: "error",
|
|
4509
|
+
text: `run failed between steps: ${text}`,
|
|
4510
|
+
});
|
|
4511
|
+
}
|
|
3210
4512
|
} finally {
|
|
3211
4513
|
if (!parked) {
|
|
3212
4514
|
run.finishedAt = new Date().toISOString();
|
|
@@ -3216,6 +4518,12 @@ function driveRunInner(
|
|
|
3216
4518
|
// learn, and the runs list and the notification stop having to say
|
|
3217
4519
|
// only "completed".
|
|
3218
4520
|
run.summary = runSummary(run);
|
|
4521
|
+
// Read before a test run's prefix goes on the summary, which would
|
|
4522
|
+
// otherwise hide the word it leads with.
|
|
4523
|
+
run.verdict = verdictOf(run.summary);
|
|
4524
|
+
// A test run's headline says so wherever the headline goes — the
|
|
4525
|
+
// runs list, the history tool, a later run's recall.
|
|
4526
|
+
if (run.test && run.summary) run.summary = testHeadline(run.summary);
|
|
3219
4527
|
// Archive before the next run resets outputs/ — a failed run's
|
|
3220
4528
|
// artifacts are usually the most interesting ones, so this runs on
|
|
3221
4529
|
// failure too.
|
|
@@ -3256,16 +4564,6 @@ function driveRunInner(
|
|
|
3256
4564
|
})();
|
|
3257
4565
|
}
|
|
3258
4566
|
|
|
3259
|
-
function readFlow(tenant: string, workspace: string, flowName: string) {
|
|
3260
|
-
const dir = path.join(workspaceDir(tenant, workspace), "flows");
|
|
3261
|
-
if (!fs.existsSync(dir)) return null;
|
|
3262
|
-
for (const f of fs.readdirSync(dir).filter((f) => f.endsWith(".md"))) {
|
|
3263
|
-
const flow = parseFlow(f, fs.readFileSync(path.join(dir, f), "utf8"));
|
|
3264
|
-
if (flow.name === flowName) return flow;
|
|
3265
|
-
}
|
|
3266
|
-
return null;
|
|
3267
|
-
}
|
|
3268
|
-
|
|
3269
4567
|
// Splice `[[flow:other]]` steps into the parent's step list, renumbering
|
|
3270
4568
|
// groups so the nested flow's own parallelism survives. Depth-limited, and
|
|
3271
4569
|
// a cycle raises rather than expanding forever.
|