@foldrun/core 2.2.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/api-tools.d.ts +5 -1
- package/dist/src/api-tools.js +19 -10
- package/dist/src/evals.js +6 -12
- package/dist/src/model-env.d.ts +14 -2
- package/dist/src/model-env.js +34 -9
- package/dist/src/run-container.d.ts +4 -0
- package/dist/src/run-container.js +9 -1
- package/dist/src/runner.js +20 -2
- package/dist/src/script-tools.d.ts +4 -1
- package/dist/src/script-tools.js +9 -3
- package/dist/src/step-exec.d.ts +4 -0
- package/dist/src/step-exec.js +7 -0
- package/dist/src/store.d.ts +17 -39
- package/dist/src/store.js +4 -52
- package/dist/src/tokens.d.ts +38 -0
- package/dist/src/tokens.js +58 -0
- package/dist/src/tool-output.d.ts +29 -0
- package/dist/src/tool-output.js +64 -0
- package/package.json +1 -1
- package/src/api-tools.ts +17 -7
- package/src/evals.ts +7 -12
- package/src/model-env.ts +42 -10
- package/src/run-container.ts +13 -2
- package/src/runner.ts +19 -1
- package/src/script-tools.ts +8 -2
- package/src/step-exec.ts +10 -1
- package/src/store.ts +19 -69
- package/src/tokens.ts +75 -0
- package/src/tool-output.ts +82 -0
package/dist/src/api-tools.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createSdkMcpServer } from "@anthropic-ai/claude-agent-sdk";
|
|
2
2
|
import type { ApiSpec } from "./store.ts";
|
|
3
|
+
import { type KeepIn } from "./tool-output.ts";
|
|
3
4
|
export declare function secretsUsedByApi(api: ApiSpec): string[];
|
|
4
5
|
/**
|
|
5
6
|
* A token bucket: `count` tokens, refilled at `perSec`. `take()` resolves
|
|
@@ -43,4 +44,7 @@ export interface ApiToolResult {
|
|
|
43
44
|
export declare function buildApiTools(tenant: string, apis: ApiSpec[], workspace?: string, resolved?: {
|
|
44
45
|
env: Record<string, string | undefined>;
|
|
45
46
|
missing: string[];
|
|
46
|
-
}
|
|
47
|
+
},
|
|
48
|
+
/** Where a body too long to send whole is kept and previewed, when the
|
|
49
|
+
* agent can read it back (tool-output.ts). Without it, the old 20k cut. */
|
|
50
|
+
keepIn?: KeepIn): ApiToolResult;
|
package/dist/src/api-tools.js
CHANGED
|
@@ -16,8 +16,11 @@ import crypto from "node:crypto";
|
|
|
16
16
|
import { EGRESS_ENV, viaEgress } from "./egress.js";
|
|
17
17
|
import { z } from "zod";
|
|
18
18
|
import { createSdkMcpServer, tool } from "@anthropic-ai/claude-agent-sdk";
|
|
19
|
+
import { previewOrKeep, RESULT_KEEP_CHARS } from "./tool-output.js";
|
|
19
20
|
import { resolveSecrets, materializeSecrets } from "./secrets.js";
|
|
20
|
-
|
|
21
|
+
// Collected whole up to this; the model sees a preview and where the rest is
|
|
22
|
+
// kept (tool-output.ts), not a cut with the rest gone.
|
|
23
|
+
const MAX_BODY_CHARS = RESULT_KEEP_CHARS;
|
|
21
24
|
/** MCP tool names are capped; ours also carry the server prefix on the wire. */
|
|
22
25
|
const MAX_TOOL_NAME = 64;
|
|
23
26
|
/** How many typed tools the prompt lists by name before summarising. */
|
|
@@ -92,7 +95,9 @@ export function typedToolName(apiName, opId) {
|
|
|
92
95
|
* the truncation, the log line. Typed and generic tools differ only in how
|
|
93
96
|
* they arrive at a Request.
|
|
94
97
|
*/
|
|
95
|
-
async function performRequest(api, env, bucket, log, req
|
|
98
|
+
async function performRequest(api, env, bucket, log, req,
|
|
99
|
+
/** Where a long body is kept, when the agent can read it back (tool-output.ts). */
|
|
100
|
+
keepIn) {
|
|
96
101
|
const started = Date.now();
|
|
97
102
|
try {
|
|
98
103
|
const waited = bucket ? await bucket.take() : 0;
|
|
@@ -133,9 +138,10 @@ async function performRequest(api, env, bucket, log, req) {
|
|
|
133
138
|
...(api.timeout ? { signal: AbortSignal.timeout(api.timeout * 1000) } : {}),
|
|
134
139
|
});
|
|
135
140
|
const text = await res.text();
|
|
136
|
-
const
|
|
141
|
+
const body = text.length > MAX_BODY_CHARS
|
|
137
142
|
? `${text.slice(0, MAX_BODY_CHARS)}\n…[truncated ${text.length - MAX_BODY_CHARS} chars]`
|
|
138
143
|
: text;
|
|
144
|
+
const truncated = previewOrKeep(body, keepIn, `${api.name}-${req.method.toLowerCase()}`);
|
|
139
145
|
log.push(`${req.method} ${url.pathname}${url.search ? "?…" : ""} → ${res.status} (${Date.now() - started}ms)${waitNote}`);
|
|
140
146
|
return {
|
|
141
147
|
content: [{ type: "text", text: `HTTP ${res.status} ${res.statusText}\n\n${truncated}` }],
|
|
@@ -177,7 +183,7 @@ function argKeys(op) {
|
|
|
177
183
|
}
|
|
178
184
|
return out;
|
|
179
185
|
}
|
|
180
|
-
function typedTool(api, op, env, bucket, log) {
|
|
186
|
+
function typedTool(api, op, env, bucket, log, keepIn) {
|
|
181
187
|
const keys = argKeys(op);
|
|
182
188
|
const shape = {};
|
|
183
189
|
for (const [key, p] of keys)
|
|
@@ -229,10 +235,10 @@ function typedTool(api, op, env, bucket, log) {
|
|
|
229
235
|
query,
|
|
230
236
|
headers,
|
|
231
237
|
body: typeof values.body === "string" ? values.body : undefined,
|
|
232
|
-
});
|
|
238
|
+
}, keepIn);
|
|
233
239
|
});
|
|
234
240
|
}
|
|
235
|
-
function genericTool(api, env, bucket, log) {
|
|
241
|
+
function genericTool(api, env, bucket, log, keepIn) {
|
|
236
242
|
const typed = api.resolvedOperations?.length
|
|
237
243
|
? ` Prefer the typed ${api.name}_* tools; this one is the escape hatch for operations they do not cover.`
|
|
238
244
|
: "";
|
|
@@ -252,12 +258,15 @@ function genericTool(api, env, bucket, log) {
|
|
|
252
258
|
query: args.query ?? {},
|
|
253
259
|
headers: {},
|
|
254
260
|
body: args.body,
|
|
255
|
-
}));
|
|
261
|
+
}, keepIn));
|
|
256
262
|
}
|
|
257
263
|
export function buildApiTools(tenant, apis, workspace,
|
|
258
264
|
// The runner container passes this: secrets were resolved before the specs
|
|
259
265
|
// crossed the boundary, and there is no vault in there to ask.
|
|
260
|
-
resolved
|
|
266
|
+
resolved,
|
|
267
|
+
/** Where a body too long to send whole is kept and previewed, when the
|
|
268
|
+
* agent can read it back (tool-output.ts). Without it, the old 20k cut. */
|
|
269
|
+
keepIn) {
|
|
261
270
|
if (apis.length === 0) {
|
|
262
271
|
return { server: null, toolNames: [], owners: {}, methods: {}, missingSecrets: [], promptLines: [], drainLog: () => [] };
|
|
263
272
|
}
|
|
@@ -302,12 +311,12 @@ resolved) {
|
|
|
302
311
|
}
|
|
303
312
|
const typedNames = [];
|
|
304
313
|
for (const op of ops) {
|
|
305
|
-
tools.push(typedTool(api, op, env, bucket, log));
|
|
314
|
+
tools.push(typedTool(api, op, env, bucket, log, keepIn));
|
|
306
315
|
typedNames.push(typedToolName(api.name, op.id));
|
|
307
316
|
methods[`mcp__foldrun_apis__${typedToolName(api.name, op.id)}`] = String(op.method ?? "").toUpperCase();
|
|
308
317
|
}
|
|
309
318
|
if (withGeneric)
|
|
310
|
-
tools.push(genericTool(api, env, bucket, log));
|
|
319
|
+
tools.push(genericTool(api, env, bucket, log, keepIn));
|
|
311
320
|
const names = [...typedNames, ...(withGeneric ? [`call_${api.name}`] : [])];
|
|
312
321
|
for (const n of names) {
|
|
313
322
|
toolNames.push(`mcp__foldrun_apis__${n}`);
|
package/dist/src/evals.js
CHANGED
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
// Deterministic checks run first and cost nothing. The judge only runs if
|
|
31
31
|
// they pass — no point paying a model to grade output already known to be
|
|
32
32
|
// wrong.
|
|
33
|
-
import { accountModelEnv } from "./model-env.js";
|
|
33
|
+
import { accountModelEnv, withModelEnv } from "./model-env.js";
|
|
34
34
|
import { sdkEnv } from "./step-exec.js";
|
|
35
35
|
import fs from "node:fs";
|
|
36
36
|
import path from "node:path";
|
|
@@ -382,7 +382,7 @@ async function judge(rubric, output, model, effort, env) {
|
|
|
382
382
|
}
|
|
383
383
|
return text.trim();
|
|
384
384
|
}
|
|
385
|
-
async function checkAssertion(assertion, output, agentDir, model, effort, judgeEnv = () => sdkEnv(undefined)) {
|
|
385
|
+
async function checkAssertion(assertion, output, agentDir, model, effort, judgeEnv = () => ({ env: sdkEnv(undefined) })) {
|
|
386
386
|
const hay = output.toLowerCase();
|
|
387
387
|
switch (assertion.type) {
|
|
388
388
|
case "contains": {
|
|
@@ -417,14 +417,14 @@ async function checkAssertion(assertion, output, agentDir, model, effort, judgeE
|
|
|
417
417
|
case "judge": {
|
|
418
418
|
// The judge runs on the account's own model key, never quietly on the
|
|
419
419
|
// platform's: a customer's eval is not the operator's bill.
|
|
420
|
-
let
|
|
420
|
+
let m;
|
|
421
421
|
try {
|
|
422
|
-
|
|
422
|
+
m = judgeEnv();
|
|
423
423
|
}
|
|
424
424
|
catch (err) {
|
|
425
425
|
return { assertion, passed: false, detail: `judge not run — ${err instanceof Error ? err.message : String(err)}` };
|
|
426
426
|
}
|
|
427
|
-
const verdict = await judge(assertion.value, output, model, effort, env);
|
|
427
|
+
const verdict = await withModelEnv(m, (env) => judge(assertion.value, output, model, effort, env));
|
|
428
428
|
const passed = /^\s*PASS\b/i.test(verdict);
|
|
429
429
|
return { assertion, passed, detail: verdict.slice(0, 300) || "judge returned nothing" };
|
|
430
430
|
}
|
|
@@ -495,13 +495,7 @@ export async function runEval(tenant, workspace, info) {
|
|
|
495
495
|
for (const a of cheap)
|
|
496
496
|
assertions.push(await checkAssertion(a, output, agentDir, info.model, info.effort));
|
|
497
497
|
if (assertions.every((r) => r.passed)) {
|
|
498
|
-
const judgeEnv = () =>
|
|
499
|
-
const m = accountModelEnv(tenant, workspace, () => sdkEnv(undefined));
|
|
500
|
-
if (m.format !== "anthropic") {
|
|
501
|
-
throw new Error(`this account's provider speaks ${m.format}; a judge needs an Anthropic-format provider`);
|
|
502
|
-
}
|
|
503
|
-
return m.env;
|
|
504
|
-
};
|
|
498
|
+
const judgeEnv = () => accountModelEnv(tenant, workspace, () => sdkEnv(undefined));
|
|
505
499
|
for (const a of judges)
|
|
506
500
|
assertions.push(await checkAssertion(a, output, agentDir, info.model, info.effort, judgeEnv));
|
|
507
501
|
}
|
package/dist/src/model-env.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type TranslatorSpec } from "./translator.ts";
|
|
1
2
|
/** Refused for want of a usable key; `status` is what an HTTP layer answers. */
|
|
2
3
|
export declare class ModelKeyError extends Error {
|
|
3
4
|
readonly status: number;
|
|
@@ -7,11 +8,22 @@ export declare const NO_MODEL_KEY: string;
|
|
|
7
8
|
/**
|
|
8
9
|
* `platform` is a function: the caller's own way to build the platform's env
|
|
9
10
|
* (a step's, or the SDK's host env), called only when this account may use
|
|
10
|
-
* it. `format` says what the endpoint speaks
|
|
11
|
-
*
|
|
11
|
+
* it. `format` says what the endpoint speaks; `translator` is set when it is
|
|
12
|
+
* not Anthropic's, and `withModelEnv` puts the step's translator in front of
|
|
13
|
+
* it, so a judge or a draft works on any key a run works on (FOL-16).
|
|
12
14
|
*/
|
|
13
15
|
export declare function accountModelEnv(tenant: string, workspace: string, platformEnv: () => Record<string, string | undefined>): {
|
|
14
16
|
env: Record<string, string | undefined>;
|
|
15
17
|
supply: "provider" | "platform";
|
|
16
18
|
format: string;
|
|
19
|
+
translator: TranslatorSpec | null;
|
|
17
20
|
};
|
|
21
|
+
/**
|
|
22
|
+
* Run `fn` with the model env, through a translator on this host when the
|
|
23
|
+
* provider speaks Chat Completions or Responses — the same one a step uses
|
|
24
|
+
* (runner.ts), started for this call and closed after it.
|
|
25
|
+
*/
|
|
26
|
+
export declare function withModelEnv<T>(m: {
|
|
27
|
+
env: Record<string, string | undefined>;
|
|
28
|
+
translator?: TranslatorSpec | null;
|
|
29
|
+
}, fn: (env: Record<string, string | undefined>) => Promise<T>): Promise<T>;
|
package/dist/src/model-env.js
CHANGED
|
@@ -10,6 +10,8 @@ import { readAgentsMd } from "./runner.js";
|
|
|
10
10
|
import { getSecret } from "./secrets.js";
|
|
11
11
|
import { platform } from "./platform.js";
|
|
12
12
|
import { accountModelBlock, isClaudeLoginToken } from "./model-credential.js";
|
|
13
|
+
import { providerPreset } from "./providers.js";
|
|
14
|
+
import { startTranslator, translatorSpecFor } from "./translator.js";
|
|
13
15
|
/** Refused for want of a usable key; `status` is what an HTTP layer answers. */
|
|
14
16
|
export class ModelKeyError extends Error {
|
|
15
17
|
status;
|
|
@@ -23,8 +25,9 @@ export const NO_MODEL_KEY = "no model key: this account runs on its own API key,
|
|
|
23
25
|
/**
|
|
24
26
|
* `platform` is a function: the caller's own way to build the platform's env
|
|
25
27
|
* (a step's, or the SDK's host env), called only when this account may use
|
|
26
|
-
* it. `format` says what the endpoint speaks
|
|
27
|
-
*
|
|
28
|
+
* it. `format` says what the endpoint speaks; `translator` is set when it is
|
|
29
|
+
* not Anthropic's, and `withModelEnv` puts the step's translator in front of
|
|
30
|
+
* it, so a judge or a draft works on any key a run works on (FOL-16).
|
|
28
31
|
*/
|
|
29
32
|
export function accountModelEnv(tenant, workspace, platformEnv) {
|
|
30
33
|
const raw = readAgentsMd(workspaceDir(tenant, workspace))?.data?.provider ??
|
|
@@ -34,7 +37,7 @@ export function accountModelEnv(tenant, workspace, platformEnv) {
|
|
|
34
37
|
if (!spec?.baseUrl) {
|
|
35
38
|
if (!platform.platformKeyAllowed(tenant))
|
|
36
39
|
throw new ModelKeyError(NO_MODEL_KEY);
|
|
37
|
-
return { env: platformEnv(), supply: "platform", format: "anthropic" };
|
|
40
|
+
return { env: platformEnv(), supply: "platform", format: "anthropic", translator: null };
|
|
38
41
|
}
|
|
39
42
|
const sub = (text) => text.replace(/\$\{([A-Z][A-Z0-9_]*)\}/g, (whole, name) => getSecret(tenant, name, workspace)?.value ?? whole);
|
|
40
43
|
const token = sub(spec.token);
|
|
@@ -43,15 +46,37 @@ export function accountModelEnv(tenant, workspace, platformEnv) {
|
|
|
43
46
|
if (!platform.platformKeyAllowed(tenant) && isClaudeLoginToken(token)) {
|
|
44
47
|
throw new ModelKeyError("model key: a Claude login token (sk-ant-oat…) cannot be used here — use an API key from your provider", 422);
|
|
45
48
|
}
|
|
49
|
+
const headers = Object.fromEntries(Object.entries(spec.headers).map(([k, v]) => [k, sub(v)]));
|
|
50
|
+
const preset = providerPreset(spec.name);
|
|
46
51
|
return {
|
|
47
|
-
env: providerEnvFor({
|
|
52
|
+
env: providerEnvFor({ baseUrl: spec.baseUrl, token, auth: spec.auth, models: spec.models, headers }),
|
|
53
|
+
supply: "provider",
|
|
54
|
+
format: spec.format,
|
|
55
|
+
translator: translatorSpecFor({
|
|
56
|
+
format: spec.format,
|
|
48
57
|
baseUrl: spec.baseUrl,
|
|
49
58
|
token,
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
59
|
+
headers,
|
|
60
|
+
params: spec.params,
|
|
61
|
+
name: spec.name,
|
|
62
|
+
maxTokensParam: preset?.maxTokensParam,
|
|
63
|
+
reasoningEffort: preset?.reasoningEffort,
|
|
53
64
|
}),
|
|
54
|
-
supply: "provider",
|
|
55
|
-
format: spec.format,
|
|
56
65
|
};
|
|
57
66
|
}
|
|
67
|
+
/**
|
|
68
|
+
* Run `fn` with the model env, through a translator on this host when the
|
|
69
|
+
* provider speaks Chat Completions or Responses — the same one a step uses
|
|
70
|
+
* (runner.ts), started for this call and closed after it.
|
|
71
|
+
*/
|
|
72
|
+
export async function withModelEnv(m, fn) {
|
|
73
|
+
if (!m.translator)
|
|
74
|
+
return fn(m.env);
|
|
75
|
+
const t = await startTranslator(m.translator);
|
|
76
|
+
try {
|
|
77
|
+
return await fn({ ...m.env, ...t.env });
|
|
78
|
+
}
|
|
79
|
+
finally {
|
|
80
|
+
await t.close();
|
|
81
|
+
}
|
|
82
|
+
}
|
|
@@ -145,6 +145,9 @@ export interface ContainerStepOutcome {
|
|
|
145
145
|
data?: unknown;
|
|
146
146
|
costUsd: number | null;
|
|
147
147
|
usage?: ExecUsage | null;
|
|
148
|
+
/** Model turns, and why the last one ended — see ExecOutcome. */
|
|
149
|
+
turns?: number;
|
|
150
|
+
stopReason?: string | null;
|
|
148
151
|
/** Every turn priced at its own model, when sub-agents ran on others —
|
|
149
152
|
* see ExecOutcome.turnsCostUsd. */
|
|
150
153
|
turnsCostUsd?: number;
|
|
@@ -366,6 +369,7 @@ export declare function parseDriverLine(line: string): {
|
|
|
366
369
|
call?: string;
|
|
367
370
|
ms?: number;
|
|
368
371
|
err?: boolean;
|
|
372
|
+
bytes?: number;
|
|
369
373
|
operator?: OperatorEvent;
|
|
370
374
|
subagent?: string;
|
|
371
375
|
check?: boolean;
|
|
@@ -536,13 +536,18 @@ try {
|
|
|
536
536
|
// buildApiTools gets an already-resolved environment. verify: gets the
|
|
537
537
|
// container's own environment, whole: it is the boundary, assembled on
|
|
538
538
|
// the host from an allowlist, and the declared secrets are in it.
|
|
539
|
-
|
|
539
|
+
// A long result is kept in a file only when the agent can read it back.
|
|
540
|
+
// (tool-output.ts canReadBack, inlined: the driver imports core by subpath.)
|
|
541
|
+
const keepIn = { dir: agentDir, readable: () => (input.allowed ?? []).some((t) => t === "Read" || t === "Grep") };
|
|
542
|
+
const api = buildApiTools("", input.apis, undefined, { env, missing: [] }, keepIn);
|
|
540
543
|
const script = buildScriptTools(
|
|
541
544
|
agentDir,
|
|
542
545
|
input.scripts,
|
|
543
546
|
{ ...env, ...runtime.env },
|
|
544
547
|
"/library/scripts",
|
|
545
548
|
runtime.interpreters,
|
|
549
|
+
null,
|
|
550
|
+
keepIn.readable,
|
|
546
551
|
);
|
|
547
552
|
// The translator, when the provider speaks Chat Completions: a loopback
|
|
548
553
|
// server for the length of this step. Its env replaces the endpoint the
|
|
@@ -1049,6 +1054,7 @@ export function parseDriverLine(line) {
|
|
|
1049
1054
|
...(typeof parsed.call === "string" ? { call: parsed.call } : {}),
|
|
1050
1055
|
...(typeof parsed.ms === "number" ? { ms: parsed.ms } : {}),
|
|
1051
1056
|
...(parsed.err === true ? { err: true } : {}),
|
|
1057
|
+
...(typeof parsed.bytes === "number" && parsed.bytes >= 0 ? { bytes: parsed.bytes } : {}),
|
|
1052
1058
|
...(isOperatorEvent(parsed.operator) ? { operator: parsed.operator } : {}),
|
|
1053
1059
|
...(typeof parsed.subagent === "string" && /^[a-z0-9-]{1,64}$/.test(parsed.subagent) ? { subagent: parsed.subagent } : {}),
|
|
1054
1060
|
...(parsed.check === true ? { check: true } : {}),
|
|
@@ -1064,6 +1070,8 @@ export function parseDriverLine(line) {
|
|
|
1064
1070
|
...("data" in parsed ? { data: parsed.data } : {}),
|
|
1065
1071
|
costUsd: typeof parsed.costUsd === "number" ? parsed.costUsd : null,
|
|
1066
1072
|
...(typeof parsed.turnsCostUsd === "number" ? { turnsCostUsd: parsed.turnsCostUsd } : {}),
|
|
1073
|
+
...(typeof parsed.turns === "number" && parsed.turns >= 0 ? { turns: parsed.turns } : {}),
|
|
1074
|
+
...(typeof parsed.stopReason === "string" ? { stopReason: parsed.stopReason.slice(0, 64) } : {}),
|
|
1067
1075
|
...(parseBrowserPod(parsed.browserPod) ? { browserPod: parseBrowserPod(parsed.browserPod) } : {}),
|
|
1068
1076
|
usage: parsed.usage &&
|
|
1069
1077
|
typeof parsed.usage.inputTokens === "number" &&
|
package/dist/src/runner.js
CHANGED
|
@@ -34,6 +34,7 @@ import { sendPlainNotification } from "./notify.js";
|
|
|
34
34
|
import { TOOL_MAP, BUILTIN_TOOLS, ownToolNames, toolRefs, legacyUseNames, legacyUseError, retiredToolNames, retiredToolError } from "./tool-names.js";
|
|
35
35
|
import { refNames } from "./refs.js";
|
|
36
36
|
import { accountDir, workspaceDir, buildMemoryIndex, workspaceMemoryIndex, knowledgeIndex, workspaceKnowledgeIndex, workspaceTools, brokenToolReport, listWorkspaces, listTenants, listRuns, liveRuns, syncBundleFor, parseToolDef, readFlow, markerPresent, whenRowsPath, parseApis, resolveModel, parseProvider, providerEnvFor, resolveEffort, isEffortWord, EFFORT_LEVELS, checkFormatVersion, readRun, writeRun, runSummary, verdictOf, sumAttempts, tokensOf, parseSchemaText, STORAGE_DIR, adoptLegacyVersionKey, listAgents, } from "./store.js";
|
|
37
|
+
import { canReadBack } from "./tool-output.js";
|
|
37
38
|
import { loadCatalog, checkModel, clampEffort, catalogCost, findModel, tokenPriceOf } from "./catalog.js";
|
|
38
39
|
import { resolveSecrets, getSecret, materializeSecrets, secretSites } from "./secrets.js";
|
|
39
40
|
import { resolveNetwork, networkLine, proxyEnv, EGRESS_CA_ENV } from "./network.js";
|
|
@@ -920,7 +921,12 @@ grantsOnly = false) {
|
|
|
920
921
|
const attached = attachOperations(apis, tenant, path.resolve(agentDir, "..", ".."));
|
|
921
922
|
const apiWarnings = attached.warnings;
|
|
922
923
|
apis.splice(0, apis.length, ...attached.specs);
|
|
923
|
-
|
|
924
|
+
// A long tool result is kept in a file only for an agent that can read it
|
|
925
|
+
// back (tool-output.ts). The grants are settled further down, after the
|
|
926
|
+
// tools are built, so the tools ask when they are called.
|
|
927
|
+
let readsFiles = false;
|
|
928
|
+
const keepIn = { dir: agentDir, readable: () => readsFiles };
|
|
929
|
+
const apiTools = buildApiTools(tenant, apis, workspace, undefined, keepIn);
|
|
924
930
|
if (apiTools.promptLines.length) {
|
|
925
931
|
parts.push(`# APIs you can call\n\n${apiTools.promptLines.join("\n")}`);
|
|
926
932
|
}
|
|
@@ -1254,7 +1260,7 @@ grantsOnly = false) {
|
|
|
1254
1260
|
// (/workspace/agents/<agent>) do not carry the name.
|
|
1255
1261
|
FOLDRUN_WORKSPACE: workspace,
|
|
1256
1262
|
...clockEnv,
|
|
1257
|
-
}, libraryDir(tenant, "scripts"), runtime.interpreters, exec);
|
|
1263
|
+
}, libraryDir(tenant, "scripts"), runtime.interpreters, exec, keepIn.readable);
|
|
1258
1264
|
parts.push(`# Clock\n\nToday is ${clockEnv.FOLDRUN_DATE} (${timezone}). Use this date for anything you ` +
|
|
1259
1265
|
`write or check — it is the workspace's calendar, not the container's clock.`);
|
|
1260
1266
|
if (scriptTools.promptLines.length) {
|
|
@@ -1462,6 +1468,7 @@ grantsOnly = false) {
|
|
|
1462
1468
|
if (denied.includes("install"))
|
|
1463
1469
|
denied.push(INSTALL_TOOL);
|
|
1464
1470
|
const finalAllowed = allowed.filter((t) => !denied.includes(t));
|
|
1471
|
+
readsFiles = canReadBack(finalAllowed);
|
|
1465
1472
|
return {
|
|
1466
1473
|
front,
|
|
1467
1474
|
clockEnv,
|
|
@@ -1621,6 +1628,7 @@ reattach = false) {
|
|
|
1621
1628
|
status,
|
|
1622
1629
|
costUsd: step.costUsd,
|
|
1623
1630
|
tokens: step.tokens ?? null,
|
|
1631
|
+
...(typeof step.turns === "number" ? { turns: step.turns } : {}),
|
|
1624
1632
|
computeSecs: step.computeSecs ?? null,
|
|
1625
1633
|
startedAt: step.podLostTry?.finishedAt ?? startedAt,
|
|
1626
1634
|
finishedAt: step.finishedAt ?? new Date().toISOString(),
|
|
@@ -1634,6 +1642,8 @@ reattach = false) {
|
|
|
1634
1642
|
step.costUsd = totals.costUsd;
|
|
1635
1643
|
step.tokens = totals.tokens;
|
|
1636
1644
|
step.computeSecs = totals.computeSecs;
|
|
1645
|
+
if (totals.turns !== undefined)
|
|
1646
|
+
step.turns = totals.turns;
|
|
1637
1647
|
}
|
|
1638
1648
|
/** A question an agent asked mid-step goes where approvals go (the
|
|
1639
1649
|
* workspace's notify:, `awaiting-approval` events), with the way to answer. */
|
|
@@ -1842,6 +1852,9 @@ stepIndex) {
|
|
|
1842
1852
|
const { front, clockEnv, clock, systemPrompt, allowed, disabled, apiTools, scriptTools, secretEnv, secretScopes, missingSecrets, missingTools, runtime, unknownTools, shadowed, legacyUse, retiredTools, mcpServers, mcpNames, apiSpecs, scriptSpecs, brokenTools, size: agentSize, providerEnv, providerLabel, providerSecrets, providerWarnings, formatWarning, searchChoice, fetchChoice, actionChoices, web, language, region, fallbackEnv, apiWarnings, searchRoots, historyDigest, deskDigest, searchTools, historyTools, deskTools, ask, translator, fallbackTranslator, outward, } = agentContext(agentDir, tenant, tags, { runId, agent: step.agent }, flowTimezone);
|
|
1843
1853
|
disposeRuntime = runtime.dispose;
|
|
1844
1854
|
canActOutward = outward;
|
|
1855
|
+
// Sent again on every turn; with turns and the tools' result sizes, what
|
|
1856
|
+
// a token report reads to say where a step's input went.
|
|
1857
|
+
step.systemPromptChars = systemPrompt.length;
|
|
1845
1858
|
// A retry that moved the step up a class (an evicted or OOM-killed
|
|
1846
1859
|
// attempt) wins over the agent's own `size:` for the attempts after.
|
|
1847
1860
|
const size = step.sizeUp ?? agentSize;
|
|
@@ -2623,6 +2636,7 @@ stepIndex) {
|
|
|
2623
2636
|
status: "lost",
|
|
2624
2637
|
costUsd: repriced(catalog, wireModel, first.usage ?? null, first.costUsd, push, first.turnsCostUsd),
|
|
2625
2638
|
tokens: tokensOf(first.usage),
|
|
2639
|
+
...(typeof first.turns === "number" ? { turns: first.turns } : {}),
|
|
2626
2640
|
computeSecs: first.timing ? Math.round(first.timing.totalMs) / 1000 : null,
|
|
2627
2641
|
finishedAt: new Date().toISOString(),
|
|
2628
2642
|
error: decision.note.slice(0, 500),
|
|
@@ -2673,6 +2687,8 @@ stepIndex) {
|
|
|
2673
2687
|
step.data = redactData(outcome.data);
|
|
2674
2688
|
step.costUsd = repriced(catalog, wireModel, outcome.usage ?? null, outcome.costUsd, push, outcome.turnsCostUsd);
|
|
2675
2689
|
step.tokens = tokensOf(outcome.usage);
|
|
2690
|
+
step.turns = typeof outcome.turns === "number" ? outcome.turns : null;
|
|
2691
|
+
step.stopReason = outcome.stopReason ?? null;
|
|
2676
2692
|
step.actual = outcome.res ?? null;
|
|
2677
2693
|
if (outcome.res?.peakMemBytes || outcome.res?.rxBytes) {
|
|
2678
2694
|
const mb = (n) => (n === null ? "?" : (n / 1024 / 1024).toFixed(0));
|
|
@@ -2870,6 +2886,8 @@ stepIndex) {
|
|
|
2870
2886
|
const stepCost = repriced(catalog, wireModel, outcome.usage, outcome.costUsd, push, outcome.turnsCostUsd);
|
|
2871
2887
|
step.costUsd = stepCost === null && consultCost === 0 ? null : (stepCost ?? 0) + consultCost;
|
|
2872
2888
|
step.tokens = tokensOf(outcome.usage);
|
|
2889
|
+
step.turns = typeof outcome.turns === "number" ? outcome.turns : null;
|
|
2890
|
+
step.stopReason = outcome.stopReason ?? null;
|
|
2873
2891
|
for (const line of apiTools.drainLog())
|
|
2874
2892
|
push("info", `api: ${line}`);
|
|
2875
2893
|
for (const line of scriptTools.drainLog())
|
|
@@ -88,4 +88,7 @@ export declare function resolveRunPath(agentDir: string, run: string, libraryScr
|
|
|
88
88
|
* have been.
|
|
89
89
|
*/
|
|
90
90
|
export declare function scriptArgShape(args: Record<string, string>): Record<string, z.ZodTypeAny>;
|
|
91
|
-
export declare function buildScriptTools(agentDir: string, scripts: ScriptSpec[], env: Record<string, string>, libraryScripts?: string, interpreters?: Record<string, string>, exec?: ExecutionContext | null
|
|
91
|
+
export declare function buildScriptTools(agentDir: string, scripts: ScriptSpec[], env: Record<string, string>, libraryScripts?: string, interpreters?: Record<string, string>, exec?: ExecutionContext | null,
|
|
92
|
+
/** Whether the agent can read files back (Read or Grep): only then is a
|
|
93
|
+
* long output kept in a file and previewed (tool-output.ts). */
|
|
94
|
+
readable?: () => boolean): ScriptToolResult;
|
package/dist/src/script-tools.js
CHANGED
|
@@ -22,13 +22,16 @@ import { runInContainer } from "./container.js";
|
|
|
22
22
|
import { hostSafeEnv } from "./host-env.js";
|
|
23
23
|
import { workspaceLinkEnv } from "./confine.js";
|
|
24
24
|
import { platform } from "./platform.js";
|
|
25
|
+
import { previewOrKeep, RESULT_KEEP_CHARS } from "./tool-output.js";
|
|
25
26
|
// A script's only clock is the `timeout:` its own tool.md declares. There
|
|
26
27
|
// was a two-minute default and a ten-minute cap here; both were the
|
|
27
28
|
// platform's opinion about how long a person's script may take, and the
|
|
28
29
|
// deploy watcher had to be taught to call its tool four times to outlast the
|
|
29
30
|
// cap. The platform orchestrates; it does not decide that.
|
|
30
31
|
const timeoutFor = (spec) => (spec.timeout ? spec.timeout * 1000 : undefined);
|
|
31
|
-
|
|
32
|
+
// Collected up to this, then previewed and kept (tool-output.ts): the model
|
|
33
|
+
// sees the head and where the rest is, instead of a cut with the rest gone.
|
|
34
|
+
const MAX_OUTPUT = RESULT_KEEP_CHARS;
|
|
32
35
|
export function parseScripts(raw) {
|
|
33
36
|
if (!Array.isArray(raw))
|
|
34
37
|
return [];
|
|
@@ -339,7 +342,10 @@ export function scriptArgShape(args) {
|
|
|
339
342
|
}
|
|
340
343
|
return shape;
|
|
341
344
|
}
|
|
342
|
-
export function buildScriptTools(agentDir, scripts, env, libraryScripts = "", interpreters = {}, exec = null
|
|
345
|
+
export function buildScriptTools(agentDir, scripts, env, libraryScripts = "", interpreters = {}, exec = null,
|
|
346
|
+
/** Whether the agent can read files back (Read or Grep): only then is a
|
|
347
|
+
* long output kept in a file and previewed (tool-output.ts). */
|
|
348
|
+
readable = () => false) {
|
|
343
349
|
if (scripts.length === 0) {
|
|
344
350
|
return { server: null, toolNames: [], promptLines: [], drainLog: () => [] };
|
|
345
351
|
}
|
|
@@ -364,7 +370,7 @@ export function buildScriptTools(agentDir, scripts, env, libraryScripts = "", in
|
|
|
364
370
|
log.push(`${spec.name}(${shown}) → exit ${code ?? "error"} ` +
|
|
365
371
|
`(${Date.now() - started}ms, ${exec?.executor ?? "sandbox"})`);
|
|
366
372
|
return {
|
|
367
|
-
content: [{ type: "text", text: `exit ${code ?? "error"}\n\n${out}` }],
|
|
373
|
+
content: [{ type: "text", text: `exit ${code ?? "error"}\n\n${previewOrKeep(out, { dir: agentDir, readable }, spec.name)}` }],
|
|
368
374
|
isError: code !== 0,
|
|
369
375
|
};
|
|
370
376
|
});
|
package/dist/src/step-exec.d.ts
CHANGED
|
@@ -48,6 +48,9 @@ export interface ExecOutcome {
|
|
|
48
48
|
* end_turn, max_tokens, stop_sequence, tool_use, pause_turn, refusal or
|
|
49
49
|
* model_context_window_exceeded. Null when no result arrived. */
|
|
50
50
|
stopReason?: string | null;
|
|
51
|
+
/** Model turns the step took (the SDK's num_turns; counted from the
|
|
52
|
+
* turns seen when no closing result arrived). */
|
|
53
|
+
turns?: number;
|
|
51
54
|
/** Set when the step browsed through the account's browser pod from the
|
|
52
55
|
* slim image (ExecOptions.browserPod): its reconnects, and — when the pod
|
|
53
56
|
* was lost — why, and what the step had written by then. */
|
|
@@ -166,6 +169,7 @@ export type EventExtra = {
|
|
|
166
169
|
call?: string;
|
|
167
170
|
ms?: number;
|
|
168
171
|
err?: boolean;
|
|
172
|
+
bytes?: number;
|
|
169
173
|
effect?: TestEffect;
|
|
170
174
|
operator?: OperatorEvent;
|
|
171
175
|
subagent?: string;
|
package/dist/src/step-exec.js
CHANGED
|
@@ -157,6 +157,7 @@ async function executeStepInner(opts, runQuery) {
|
|
|
157
157
|
let costUsd = null;
|
|
158
158
|
let usage = null;
|
|
159
159
|
let stopReason = null;
|
|
160
|
+
let numTurns = null;
|
|
160
161
|
const texts = [];
|
|
161
162
|
fs.mkdirSync(path.join(agentDir, "outputs"), { recursive: true });
|
|
162
163
|
fs.mkdirSync(path.join(agentDir, "memory"), { recursive: true });
|
|
@@ -615,6 +616,9 @@ async function executeStepInner(opts, runQuery) {
|
|
|
615
616
|
emit("tool", open.name, {
|
|
616
617
|
call: block.tool_use_id,
|
|
617
618
|
ms: Date.now() - open.at,
|
|
619
|
+
// How much the result put into the conversation — sent again on
|
|
620
|
+
// every later turn, so the size is what a token report needs.
|
|
621
|
+
bytes: toolResultText(block.content).length,
|
|
618
622
|
...(block.is_error ? { err: true } : {}),
|
|
619
623
|
...(open.subagent ? { subagent: open.subagent } : {}),
|
|
620
624
|
});
|
|
@@ -642,6 +646,8 @@ async function executeStepInner(opts, runQuery) {
|
|
|
642
646
|
}
|
|
643
647
|
costUsd = "total_cost_usd" in message ? (message.total_cost_usd ?? null) : null;
|
|
644
648
|
stopReason = "stop_reason" in message && typeof message.stop_reason === "string" ? message.stop_reason : null;
|
|
649
|
+
if ("num_turns" in message && typeof message.num_turns === "number")
|
|
650
|
+
numTurns = message.num_turns;
|
|
645
651
|
if ("usage" in message && message.usage) {
|
|
646
652
|
// Cache reads and writes counted beside the input total: with only
|
|
647
653
|
// the total (until 18 Oct 2026) the hit rate could not be read and
|
|
@@ -821,6 +827,7 @@ async function executeStepInner(opts, runQuery) {
|
|
|
821
827
|
status, result, conclusion, ...(opts.output ? { data } : {}), costUsd, usage,
|
|
822
828
|
...(mixedModels() ? { turnsCostUsd: turnsTotal() } : {}),
|
|
823
829
|
stopReason,
|
|
830
|
+
turns: numTurns ?? turns.size,
|
|
824
831
|
...(pod ? { browserPod: podState } : {}),
|
|
825
832
|
};
|
|
826
833
|
}
|
package/dist/src/store.d.ts
CHANGED
|
@@ -893,6 +893,10 @@ export interface RunEvent {
|
|
|
893
893
|
call?: string;
|
|
894
894
|
ms?: number;
|
|
895
895
|
err?: boolean;
|
|
896
|
+
/** On a tool completion: the result's length in characters — what it put
|
|
897
|
+
* into the conversation, which every later turn sends again. Absent before
|
|
898
|
+
* 18 Oct 2026. */
|
|
899
|
+
bytes?: number;
|
|
896
900
|
/** Set on a test run's event that reports what the platform did instead
|
|
897
901
|
* of the real thing — a refused send, a redirected email, a diverted
|
|
898
902
|
* state write, a withheld secret. The run page lists these; the text is
|
|
@@ -954,6 +958,8 @@ export interface StepAttempt {
|
|
|
954
958
|
status: "completed" | "failed" | "lost";
|
|
955
959
|
costUsd: number | null;
|
|
956
960
|
tokens: TokenUsage | null;
|
|
961
|
+
/** Model turns this attempt took; absent before 18 Oct 2026. */
|
|
962
|
+
turns?: number | null;
|
|
957
963
|
computeSecs: number | null;
|
|
958
964
|
startedAt: string;
|
|
959
965
|
finishedAt: string;
|
|
@@ -968,45 +974,9 @@ export interface StepAttempt {
|
|
|
968
974
|
/** The step's totals across its attempts, from the rows. Null where no
|
|
969
975
|
* attempt reported the figure — an in-process step rents no sandbox, and
|
|
970
976
|
* billing reads null as zero, not as "unknown, charge something". */
|
|
971
|
-
export declare function sumAttempts(tries: StepAttempt[]): Pick<StepAttempt, "costUsd" | "tokens" | "computeSecs">;
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
* model saw; `cacheRead` and `cacheWrite` are the part of it the prompt
|
|
975
|
-
* cache served or stored — a read costs a tenth of fresh input, and a usage
|
|
976
|
-
* page wants to say how much of a step's context was re-read rather than
|
|
977
|
-
* re-sent. Steps recorded before 18 Oct 2026 have no cache fields: their
|
|
978
|
-
* `input` is the same total, but how much of it was cached is unknown.
|
|
979
|
-
*/
|
|
980
|
-
export interface TokenUsage {
|
|
981
|
-
input: number;
|
|
982
|
-
output: number;
|
|
983
|
-
cacheRead?: number;
|
|
984
|
-
cacheWrite?: number;
|
|
985
|
-
}
|
|
986
|
-
/** The record's shape from what a step reports (ExecOutcome.usage). */
|
|
987
|
-
export declare function tokensOf(usage: {
|
|
988
|
-
inputTokens: number;
|
|
989
|
-
outputTokens: number;
|
|
990
|
-
cacheReadTokens?: number;
|
|
991
|
-
cacheWriteTokens?: number;
|
|
992
|
-
} | null | undefined): TokenUsage | null;
|
|
993
|
-
/** Field-by-field sum; the cache fields are present in the result whenever
|
|
994
|
-
* any input had them, so a run of old and new steps still reads whole. */
|
|
995
|
-
export declare function sumTokens(list: readonly TokenUsage[]): TokenUsage;
|
|
996
|
-
/** The share of all input that the prompt cache served, 0–1; null when
|
|
997
|
-
* there was no input, or the record predates the split and cannot say. */
|
|
998
|
-
export declare function cacheHitRate(t: TokenUsage | null | undefined): number | null;
|
|
999
|
-
/** The cache share over several counts, taking only those that report the
|
|
1000
|
-
* split: a step recorded before it holds its cache traffic inside `input`,
|
|
1001
|
-
* and counting that as uncached would understate the rate. The rule the
|
|
1002
|
-
* Usage page and the CLI's totals follow too. */
|
|
1003
|
-
export declare function measuredHitRate(list: readonly (TokenUsage | null | undefined)[]): number | null;
|
|
1004
|
-
/** A run's token counts, summed over its steps; null when no step reported any. */
|
|
1005
|
-
export declare function runTokens(run: {
|
|
1006
|
-
steps: readonly {
|
|
1007
|
-
tokens?: TokenUsage | null;
|
|
1008
|
-
}[];
|
|
1009
|
-
}): TokenUsage | null;
|
|
977
|
+
export declare function sumAttempts(tries: StepAttempt[]): Pick<StepAttempt, "costUsd" | "tokens" | "computeSecs" | "turns">;
|
|
978
|
+
export { type TokenUsage, tokensOf, sumTokens, cacheHitRate, measuredHitRate, runTokens } from "./tokens.ts";
|
|
979
|
+
import { type TokenUsage } from "./tokens.ts";
|
|
1010
980
|
export interface StepRecord {
|
|
1011
981
|
agent: string;
|
|
1012
982
|
instruction: string;
|
|
@@ -1195,6 +1165,14 @@ export interface StepRecord {
|
|
|
1195
1165
|
* and not only "how much" — the price of a token changes, the count is
|
|
1196
1166
|
* the fact. Absent on steps recorded before this existed. */
|
|
1197
1167
|
tokens?: TokenUsage | null;
|
|
1168
|
+
/** Model turns, summed across attempts, and the system prompt's length in
|
|
1169
|
+
* characters — every turn sends it again. With the tool events' `bytes`,
|
|
1170
|
+
* what says where a step's input tokens went. Absent before 18 Oct 2026. */
|
|
1171
|
+
turns?: number | null;
|
|
1172
|
+
systemPromptChars?: number;
|
|
1173
|
+
/** Why the model's last turn ended (end_turn, max_tokens, tool_use…), as
|
|
1174
|
+
* the SDK's closing result says. */
|
|
1175
|
+
stopReason?: string | null;
|
|
1198
1176
|
/** What the sandbox actually touched, against what it reserved: CPU
|
|
1199
1177
|
* busy-seconds and peak memory from the pod's own cgroup, bytes on the
|
|
1200
1178
|
* wire from /proc/net/dev. Null per-field where the kernel interface
|