@poncho-ai/harness 0.59.13 → 0.59.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +5 -5
- package/CHANGELOG.md +46 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.js +203 -19
- package/package.json +1 -1
- package/src/ask-user-tool.ts +95 -0
- package/src/default-agent.ts +1 -1
- package/src/harness.ts +116 -8
- package/src/tool-result-guard.ts +133 -0
- package/test/tool-result-guard.test.ts +123 -0
package/.turbo/turbo-build.log
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
|
|
2
|
-
> @poncho-ai/harness@0.59.
|
|
2
|
+
> @poncho-ai/harness@0.59.15 build /home/runner/work/poncho-ai/poncho-ai/packages/harness
|
|
3
3
|
> node scripts/embed-docs.js && tsup src/index.ts --format esm --dts
|
|
4
4
|
|
|
5
5
|
[embed-docs] Generated poncho-docs.ts with 4 topics
|
|
@@ -9,8 +9,8 @@
|
|
|
9
9
|
[34mCLI[39m Target: es2022
|
|
10
10
|
[34mESM[39m Build start
|
|
11
11
|
[32mESM[39m [1mdist/isolate-F2PPSUL6.js [22m[32m53.82 KB[39m
|
|
12
|
-
[32mESM[39m [1mdist/index.js [22m[
|
|
13
|
-
[32mESM[39m ⚡️ Build success in
|
|
12
|
+
[32mESM[39m [1mdist/index.js [22m[32m570.16 KB[39m
|
|
13
|
+
[32mESM[39m ⚡️ Build success in 242ms
|
|
14
14
|
[34mDTS[39m Build start
|
|
15
|
-
[32mDTS[39m ⚡️ Build success in
|
|
16
|
-
[32mDTS[39m [1mdist/index.d.ts [22m[
|
|
15
|
+
[32mDTS[39m ⚡️ Build success in 6639ms
|
|
16
|
+
[32mDTS[39m [1mdist/index.d.ts [22m[32m103.12 KB[39m
|
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,51 @@
|
|
|
1
1
|
# @poncho-ai/harness
|
|
2
2
|
|
|
3
|
+
## 0.59.15
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- [`c5f6d7f`](https://github.com/cesr/poncho-ai/commit/c5f6d7f999bc3d4699295c376b23fc3894302e3a) Thanks [@cesr](https://github.com/cesr)! - Guard a single oversized FRESH tool result from overflowing the context
|
|
8
|
+
window. `truncateHistoricalToolResults` only shrinks results from prior runs
|
|
9
|
+
(it deliberately preserves the latest run's results so the model can read what
|
|
10
|
+
it just fetched), so a tool that returns a payload larger than the window in
|
|
11
|
+
one shot — e.g. an MCP email-fetch returning 1.6M–3.3M tokens of full bodies —
|
|
12
|
+
was never truncated and failed the very next step with "prompt is too long".
|
|
13
|
+
|
|
14
|
+
The result is now guarded at the moment it's produced, before the
|
|
15
|
+
`tool:completed` event is emitted (so the WS stream never carries the megabytes
|
|
16
|
+
either) and before it reaches the model:
|
|
17
|
+
- **Spill mode** (opt-in via the `__toolResultSpill` run parameter
|
|
18
|
+
`{ enabled, dir?, thresholdChars?, keepLast? }`): the full payload is written
|
|
19
|
+
to a VFS file (JSONL for arrays, pretty JSON otherwise) and the model gets a
|
|
20
|
+
small handle + preview pointing at it, to read back in bounded chunks with
|
|
21
|
+
bash/jq. Durable and lossless; the spill dir is pruned to `keepLast` files.
|
|
22
|
+
- **Inline-truncate mode** (default when spill isn't enabled, and the fallback
|
|
23
|
+
if a spill write fails): the payload is replaced with a preview + "re-run
|
|
24
|
+
narrower" notice.
|
|
25
|
+
|
|
26
|
+
Both modes apply to every tool uniformly (MCP, bash, run_code, subagent
|
|
27
|
+
results) and only trigger above `thresholdChars` (default 500k ≈ 125k tokens),
|
|
28
|
+
so normal-sized results are untouched. Inline media (images/PDFs) is separated
|
|
29
|
+
out first and unaffected.
|
|
30
|
+
|
|
31
|
+
## 0.59.14
|
|
32
|
+
|
|
33
|
+
### Patch Changes
|
|
34
|
+
|
|
35
|
+
- [#172](https://github.com/cesr/poncho-ai/pull/172) [`dc61836`](https://github.com/cesr/poncho-ai/commit/dc61836aa99e3cb6e1339dc6f82ffdab522918ba) Thanks [@cesr](https://github.com/cesr)! - Add the `ask_user` built-in tool: the agent can pause the run to ask the
|
|
36
|
+
user a structured, multiple-choice question (the in-app analog of Claude
|
|
37
|
+
Code's AskUserQuestion) instead of asking in plain prose. Each call
|
|
38
|
+
carries 1–4 questions, each with a short header, a `multiSelect` flag,
|
|
39
|
+
and pre-made options; a free-text "Other" escape is rendered by the
|
|
40
|
+
client.
|
|
41
|
+
|
|
42
|
+
The tool is forced to client (`device`) dispatch, so the harness pauses
|
|
43
|
+
the run on a checkpoint carrying the questions and the consumer resumes
|
|
44
|
+
by injecting the user's selections as the tool result — no server-side
|
|
45
|
+
execution (the handler is a defensive stub). The default agent prompt
|
|
46
|
+
now steers the model to reach for `ask_user` whenever it would otherwise
|
|
47
|
+
stop to ask the user to choose between options.
|
|
48
|
+
|
|
3
49
|
## 0.59.13
|
|
4
50
|
|
|
5
51
|
### Patch Changes
|
package/dist/index.d.ts
CHANGED
|
@@ -1494,6 +1494,17 @@ declare class AgentHarness {
|
|
|
1494
1494
|
get frontmatter(): AgentFrontmatter | undefined;
|
|
1495
1495
|
getToolResultArchive(conversationId: string): Record<string, ArchivedToolResult>;
|
|
1496
1496
|
private seedToolResultArchive;
|
|
1497
|
+
/**
|
|
1498
|
+
* Guard a single FRESH tool result against overflowing the context window.
|
|
1499
|
+
* `serialized` is the JSON-serialized (media-stripped) output. Returns a
|
|
1500
|
+
* replacement value to put in front of the model (a handle or a truncated
|
|
1501
|
+
* preview) when the result is oversized, or `null` to leave it untouched.
|
|
1502
|
+
* Spilling writes the full payload to a VFS file; on any write failure it
|
|
1503
|
+
* falls back to inline truncation, so this never throws into the run loop.
|
|
1504
|
+
*/
|
|
1505
|
+
private guardOversizedToolResult;
|
|
1506
|
+
/** Best-effort: keep only the newest `keepLast` files in the spill dir. */
|
|
1507
|
+
private pruneSpillDir;
|
|
1497
1508
|
private truncateHistoricalToolResults;
|
|
1498
1509
|
private shouldPreserveSkillToolResult;
|
|
1499
1510
|
getTodos(conversationId: string): Promise<TodoItem[]>;
|
package/dist/index.js
CHANGED
|
@@ -710,7 +710,7 @@ Environment: {{runtime.environment}}
|
|
|
710
710
|
|
|
711
711
|
- Use tools when needed
|
|
712
712
|
- Explain your reasoning clearly
|
|
713
|
-
-
|
|
713
|
+
- When requirements are ambiguous or you need the user to choose between options, use the \`ask_user\` tool to ask a structured multiple-choice question instead of writing the question as plain text. Reserve plain-text questions for genuinely open-ended asks that have no sensible pre-made options.
|
|
714
714
|
- Never claim a file/tool change unless the corresponding tool call actually succeeded
|
|
715
715
|
`;
|
|
716
716
|
};
|
|
@@ -2445,7 +2445,7 @@ var ponchoDocsTool = defineTool({
|
|
|
2445
2445
|
import { randomUUID as randomUUID5 } from "crypto";
|
|
2446
2446
|
import { readFile as readFile9 } from "fs/promises";
|
|
2447
2447
|
import { resolve as resolve11 } from "path";
|
|
2448
|
-
import { defineTool as
|
|
2448
|
+
import { defineTool as defineTool13, getTextContent as getTextContent2, createLogger as createLogger7, formatError as fmtErr, url as urlColor } from "@poncho-ai/sdk";
|
|
2449
2449
|
|
|
2450
2450
|
// src/upload-store.ts
|
|
2451
2451
|
import { createHash as createHash2 } from "crypto";
|
|
@@ -5649,6 +5649,61 @@ var BashEnvironmentManager = class {
|
|
|
5649
5649
|
}
|
|
5650
5650
|
};
|
|
5651
5651
|
|
|
5652
|
+
// src/tool-result-guard.ts
|
|
5653
|
+
var SPILL_POLICY_PARAM = "__toolResultSpill";
|
|
5654
|
+
var DEFAULT_OVERSIZED_TOOL_RESULT_CHARS = 5e5;
|
|
5655
|
+
var SPILLED_PREVIEW_CHARS = 6e3;
|
|
5656
|
+
var INLINE_TRUNCATE_PREVIEW_CHARS = 6e4;
|
|
5657
|
+
var DEFAULT_SPILL_DIR = "/tmp/tool-results";
|
|
5658
|
+
var DEFAULT_SPILL_KEEP_LAST = 20;
|
|
5659
|
+
var readSpillPolicy = (parameters) => {
|
|
5660
|
+
const raw = parameters?.[SPILL_POLICY_PARAM];
|
|
5661
|
+
const obj = typeof raw === "object" && raw !== null ? raw : {};
|
|
5662
|
+
const thresholdChars = typeof obj.thresholdChars === "number" && obj.thresholdChars > 0 ? obj.thresholdChars : DEFAULT_OVERSIZED_TOOL_RESULT_CHARS;
|
|
5663
|
+
const dir = typeof obj.dir === "string" && obj.dir.startsWith("/") ? obj.dir : DEFAULT_SPILL_DIR;
|
|
5664
|
+
const keepLast = typeof obj.keepLast === "number" && obj.keepLast > 0 ? Math.floor(obj.keepLast) : DEFAULT_SPILL_KEEP_LAST;
|
|
5665
|
+
return { enabled: obj.enabled === true, dir, thresholdChars, keepLast };
|
|
5666
|
+
};
|
|
5667
|
+
var isOversizedToolResult = (serialized, policy) => serialized.length > policy.thresholdChars;
|
|
5668
|
+
var formatSpillPayload = (output) => {
|
|
5669
|
+
if (Array.isArray(output)) {
|
|
5670
|
+
return {
|
|
5671
|
+
content: output.map((row) => JSON.stringify(row)).join("\n"),
|
|
5672
|
+
format: "jsonl",
|
|
5673
|
+
records: output.length
|
|
5674
|
+
};
|
|
5675
|
+
}
|
|
5676
|
+
return { content: JSON.stringify(output ?? null, null, 2), format: "json" };
|
|
5677
|
+
};
|
|
5678
|
+
var buildSpillHandle = (opts) => {
|
|
5679
|
+
const { toolName, path, format, serialized, records } = opts;
|
|
5680
|
+
const approxTokens = Math.ceil(serialized.length / 4);
|
|
5681
|
+
const isArray = format === "jsonl";
|
|
5682
|
+
return {
|
|
5683
|
+
__toolResultSpilled: true,
|
|
5684
|
+
tool: toolName,
|
|
5685
|
+
path,
|
|
5686
|
+
format,
|
|
5687
|
+
...records !== void 0 ? { records } : {},
|
|
5688
|
+
totalChars: serialized.length,
|
|
5689
|
+
approxTokens,
|
|
5690
|
+
preview: serialized.slice(0, SPILLED_PREVIEW_CHARS),
|
|
5691
|
+
note: `This result was too large to return inline (~${approxTokens.toLocaleString()} tokens) and was saved to ${path}. Inspect it with bash \u2014 e.g. \`wc -l ${path}\`, \`head -c 4000 ${path}\`, ` + (isArray ? `\`sed -n '1,5p' ${path}\`, \`grep -i <term> ${path}\`, or \`jq\` per line` : `\`grep -i <term> ${path}\``) + `. Do NOT cat or read_file the whole file (it re-overflows). Or re-run the tool with a narrower request (fewer items / a filter / metadata only).`
|
|
5692
|
+
};
|
|
5693
|
+
};
|
|
5694
|
+
var buildInlineTruncation = (toolName, serialized) => {
|
|
5695
|
+
const approxTokens = Math.ceil(serialized.length / 4);
|
|
5696
|
+
const omittedChars = Math.max(0, serialized.length - INLINE_TRUNCATE_PREVIEW_CHARS);
|
|
5697
|
+
return {
|
|
5698
|
+
__toolResultTruncated: true,
|
|
5699
|
+
tool: toolName,
|
|
5700
|
+
approxTokens,
|
|
5701
|
+
omittedChars,
|
|
5702
|
+
preview: serialized.slice(0, INLINE_TRUNCATE_PREVIEW_CHARS),
|
|
5703
|
+
note: `This result was too large (~${approxTokens.toLocaleString()} tokens) and was truncated to fit the context window \u2014 only the first ${INLINE_TRUNCATE_PREVIEW_CHARS.toLocaleString()} characters are shown. Re-run with a narrower request (fewer items / a filter / metadata only). Do NOT retry the same call unchanged \u2014 it will overflow again.`
|
|
5704
|
+
};
|
|
5705
|
+
};
|
|
5706
|
+
|
|
5652
5707
|
// src/vfs/bash-tool.ts
|
|
5653
5708
|
import { defineTool as defineTool2 } from "@poncho-ai/sdk";
|
|
5654
5709
|
var createBashTool = (bashManager) => defineTool2({
|
|
@@ -8591,10 +8646,71 @@ var createSearchTools = () => [
|
|
|
8591
8646
|
})
|
|
8592
8647
|
];
|
|
8593
8648
|
|
|
8594
|
-
// src/
|
|
8649
|
+
// src/ask-user-tool.ts
|
|
8595
8650
|
import { defineTool as defineTool11 } from "@poncho-ai/sdk";
|
|
8651
|
+
var createAskUserTool = () => defineTool11({
|
|
8652
|
+
name: "ask_user",
|
|
8653
|
+
description: "Ask the user one or more structured multiple-choice questions and wait for their answer. Use this INSTEAD of writing a question as plain text whenever you would otherwise pause to let the user choose between options \u2014 clarifying ambiguous requirements, picking an approach, confirming a direction. The user sees tappable option chips and answers with a tap. Prefer this over a prose question: it is faster for the user and gives you a clean answer. Guidelines: ask 1\u20134 questions at once, each with 2\u20134 concrete options; keep `header` very short (a few words); write each option `label` short and its `description` to one line. The user can always type a custom 'Other' answer, so you do not need to add one. Do NOT call any other tool in the same turn as ask_user, and call it at most once per turn. After the user answers you will receive their selections and may continue.",
|
|
8654
|
+
inputSchema: {
|
|
8655
|
+
type: "object",
|
|
8656
|
+
properties: {
|
|
8657
|
+
questions: {
|
|
8658
|
+
type: "array",
|
|
8659
|
+
description: "1\u20134 questions to ask the user at once.",
|
|
8660
|
+
items: {
|
|
8661
|
+
type: "object",
|
|
8662
|
+
properties: {
|
|
8663
|
+
question: {
|
|
8664
|
+
type: "string",
|
|
8665
|
+
description: "The full question text shown to the user."
|
|
8666
|
+
},
|
|
8667
|
+
header: {
|
|
8668
|
+
type: "string",
|
|
8669
|
+
description: "A very short label for this question (a few words, ~12 chars), shown as a chip."
|
|
8670
|
+
},
|
|
8671
|
+
multiSelect: {
|
|
8672
|
+
type: "boolean",
|
|
8673
|
+
description: "If true, the user may select multiple options. Defaults to false (single choice)."
|
|
8674
|
+
},
|
|
8675
|
+
options: {
|
|
8676
|
+
type: "array",
|
|
8677
|
+
description: "The pre-made options. A free-text 'Other' option is added automatically by the client.",
|
|
8678
|
+
items: {
|
|
8679
|
+
type: "object",
|
|
8680
|
+
properties: {
|
|
8681
|
+
label: {
|
|
8682
|
+
type: "string",
|
|
8683
|
+
description: "Short, selectable label for this option."
|
|
8684
|
+
},
|
|
8685
|
+
description: {
|
|
8686
|
+
type: "string",
|
|
8687
|
+
description: "A one-line explanation of what this option means."
|
|
8688
|
+
}
|
|
8689
|
+
},
|
|
8690
|
+
required: ["label"],
|
|
8691
|
+
additionalProperties: false
|
|
8692
|
+
}
|
|
8693
|
+
}
|
|
8694
|
+
},
|
|
8695
|
+
required: ["question", "header", "options"],
|
|
8696
|
+
additionalProperties: false
|
|
8697
|
+
}
|
|
8698
|
+
}
|
|
8699
|
+
},
|
|
8700
|
+
required: ["questions"],
|
|
8701
|
+
additionalProperties: false
|
|
8702
|
+
},
|
|
8703
|
+
handler: async () => {
|
|
8704
|
+
return {
|
|
8705
|
+
error: "ask_user must be answered by the user on the client; it cannot run server-side. This indicates a dispatch misconfiguration."
|
|
8706
|
+
};
|
|
8707
|
+
}
|
|
8708
|
+
});
|
|
8709
|
+
|
|
8710
|
+
// src/subagent-tools.ts
|
|
8711
|
+
import { defineTool as defineTool12 } from "@poncho-ai/sdk";
|
|
8596
8712
|
var createSubagentTools = (manager) => [
|
|
8597
|
-
|
|
8713
|
+
defineTool12({
|
|
8598
8714
|
name: "spawn_subagent",
|
|
8599
8715
|
description: "Spawn a subagent to work on a task in the background. Returns immediately with a subagent ID. The subagent runs independently and its result will be delivered to you as a message in the conversation when it completes.\n\nGuidelines:\n- Spawn all needed subagents in a SINGLE response (they run concurrently), then end your turn with a brief message to the user.\n- Do NOT spawn more subagents in follow-up steps. Wait for results to be delivered before deciding if more work is needed.\n- Prefer doing work yourself for simple or quick tasks. Spawn subagents for substantial, self-contained work.\n- The subagent has no memory of your conversation -- write thorough, self-contained instructions in the task.",
|
|
8600
8716
|
inputSchema: {
|
|
@@ -8629,7 +8745,7 @@ var createSubagentTools = (manager) => [
|
|
|
8629
8745
|
return { subagentId, status: "running" };
|
|
8630
8746
|
}
|
|
8631
8747
|
}),
|
|
8632
|
-
|
|
8748
|
+
defineTool12({
|
|
8633
8749
|
name: "message_subagent",
|
|
8634
8750
|
description: "Send a follow-up message to a completed or stopped subagent. The subagent restarts in the background and its result will be delivered to you as a message when it completes. Only works when the subagent is not currently running.",
|
|
8635
8751
|
inputSchema: {
|
|
@@ -8657,7 +8773,7 @@ var createSubagentTools = (manager) => [
|
|
|
8657
8773
|
return { subagentId: id, status: "running" };
|
|
8658
8774
|
}
|
|
8659
8775
|
}),
|
|
8660
|
-
|
|
8776
|
+
defineTool12({
|
|
8661
8777
|
name: "stop_subagent",
|
|
8662
8778
|
description: "Stop a running subagent. The subagent's conversation is preserved but it will stop processing. Use this to cancel work that is no longer needed.",
|
|
8663
8779
|
inputSchema: {
|
|
@@ -8680,7 +8796,7 @@ var createSubagentTools = (manager) => [
|
|
|
8680
8796
|
return { message: `Subagent "${subagentId}" has been stopped.` };
|
|
8681
8797
|
}
|
|
8682
8798
|
}),
|
|
8683
|
-
|
|
8799
|
+
defineTool12({
|
|
8684
8800
|
name: "list_subagents",
|
|
8685
8801
|
description: "List all subagents that have been spawned in this conversation. Returns each subagent's ID, original task, current status, and message count. Use this to look up subagent IDs before calling message_subagent or stop_subagent.",
|
|
8686
8802
|
inputSchema: {
|
|
@@ -8700,7 +8816,7 @@ var createSubagentTools = (manager) => [
|
|
|
8700
8816
|
return { subagents };
|
|
8701
8817
|
}
|
|
8702
8818
|
}),
|
|
8703
|
-
|
|
8819
|
+
defineTool12({
|
|
8704
8820
|
name: "read_subagent",
|
|
8705
8821
|
description: "Fetch the conversation transcript of a subagent you spawned. Use this to inspect a subagent's intermediate reasoning, tool calls, or full output -- instead of asking it to repeat its work via message_subagent.\n\nModes:\n- 'final' (default): just the last assistant message. Cheap.\n- 'assistant': all assistant messages, no tool calls/results.\n- 'full': every message including tool calls and results. Can be large.\n\nUse since_index / max_messages to page through long transcripts. Only works on subagents directly spawned by this conversation.",
|
|
8706
8822
|
inputSchema: {
|
|
@@ -9578,6 +9694,7 @@ var AgentHarness = class _AgentHarness {
|
|
|
9578
9694
|
}
|
|
9579
9695
|
/** Returns the normalized {access, dispatch} mode for the tool. */
|
|
9580
9696
|
resolveToolMode(toolName) {
|
|
9697
|
+
if (toolName === "ask_user") return { dispatch: "device" };
|
|
9581
9698
|
return normalizeToolAccess(this.resolveToolAccess(toolName));
|
|
9582
9699
|
}
|
|
9583
9700
|
isToolEnabled(name) {
|
|
@@ -9619,6 +9736,9 @@ var AgentHarness = class _AgentHarness {
|
|
|
9619
9736
|
this.registerIfMissing(tool);
|
|
9620
9737
|
}
|
|
9621
9738
|
}
|
|
9739
|
+
if (this.isToolEnabled("ask_user")) {
|
|
9740
|
+
this.registerIfMissing(createAskUserTool());
|
|
9741
|
+
}
|
|
9622
9742
|
if (this.environment === "development" && this.isToolEnabled("poncho_docs")) {
|
|
9623
9743
|
this.registerIfMissing(ponchoDocsTool);
|
|
9624
9744
|
}
|
|
@@ -9627,7 +9747,7 @@ var AgentHarness = class _AgentHarness {
|
|
|
9627
9747
|
}
|
|
9628
9748
|
}
|
|
9629
9749
|
createGetToolResultByIdTool() {
|
|
9630
|
-
return
|
|
9750
|
+
return defineTool13({
|
|
9631
9751
|
name: "get_tool_result_by_id",
|
|
9632
9752
|
description: "Retrieve a previously archived full tool result by id for the current conversation. Use this when older tool outputs were truncated in prompt history.",
|
|
9633
9753
|
inputSchema: {
|
|
@@ -9760,6 +9880,59 @@ var AgentHarness = class _AgentHarness {
|
|
|
9760
9880
|
this.archivedToolResultsByConversation.set(conversationId, merged);
|
|
9761
9881
|
return merged;
|
|
9762
9882
|
}
|
|
9883
|
+
/**
|
|
9884
|
+
* Guard a single FRESH tool result against overflowing the context window.
|
|
9885
|
+
* `serialized` is the JSON-serialized (media-stripped) output. Returns a
|
|
9886
|
+
* replacement value to put in front of the model (a handle or a truncated
|
|
9887
|
+
* preview) when the result is oversized, or `null` to leave it untouched.
|
|
9888
|
+
* Spilling writes the full payload to a VFS file; on any write failure it
|
|
9889
|
+
* falls back to inline truncation, so this never throws into the run loop.
|
|
9890
|
+
*/
|
|
9891
|
+
async guardOversizedToolResult(opts) {
|
|
9892
|
+
const { tenantId, toolName, toolCallId, output, serialized, policy } = opts;
|
|
9893
|
+
if (!isOversizedToolResult(serialized, policy)) return null;
|
|
9894
|
+
if (policy.enabled && this.bashManager) {
|
|
9895
|
+
try {
|
|
9896
|
+
const vfs = this.createVfsAccess(tenantId);
|
|
9897
|
+
const { content, format, records } = formatSpillPayload(output);
|
|
9898
|
+
const safeTool = toolName.replace(/[^A-Za-z0-9_-]/g, "_");
|
|
9899
|
+
const path = `${policy.dir}/${safeTool}_${toolCallId}.${format}`;
|
|
9900
|
+
await vfs.mkdir(policy.dir, { recursive: true });
|
|
9901
|
+
await vfs.writeText(path, content);
|
|
9902
|
+
await this.pruneSpillDir(vfs, policy.dir, policy.keepLast);
|
|
9903
|
+
return buildSpillHandle({ toolName, path, format, serialized, records });
|
|
9904
|
+
} catch {
|
|
9905
|
+
}
|
|
9906
|
+
}
|
|
9907
|
+
return buildInlineTruncation(toolName, serialized);
|
|
9908
|
+
}
|
|
9909
|
+
/** Best-effort: keep only the newest `keepLast` files in the spill dir. */
|
|
9910
|
+
async pruneSpillDir(vfs, dir, keepLast) {
|
|
9911
|
+
try {
|
|
9912
|
+
const entries = await vfs.readdir(dir);
|
|
9913
|
+
const names = (Array.isArray(entries) ? entries : []).map((e) => typeof e === "string" ? e : e.name).filter((n) => typeof n === "string");
|
|
9914
|
+
if (names.length <= keepLast) return;
|
|
9915
|
+
const stated = await Promise.all(
|
|
9916
|
+
names.map(async (name) => {
|
|
9917
|
+
try {
|
|
9918
|
+
const s = await vfs.stat(`${dir}/${name}`);
|
|
9919
|
+
return { name, mtime: s.updatedAt ?? 0 };
|
|
9920
|
+
} catch {
|
|
9921
|
+
return { name, mtime: 0 };
|
|
9922
|
+
}
|
|
9923
|
+
})
|
|
9924
|
+
);
|
|
9925
|
+
stated.sort((a, b) => a.mtime - b.mtime);
|
|
9926
|
+
const doomed = stated.slice(0, stated.length - keepLast);
|
|
9927
|
+
for (const d of doomed) {
|
|
9928
|
+
try {
|
|
9929
|
+
await vfs.rm(`${dir}/${d.name}`);
|
|
9930
|
+
} catch {
|
|
9931
|
+
}
|
|
9932
|
+
}
|
|
9933
|
+
} catch {
|
|
9934
|
+
}
|
|
9935
|
+
}
|
|
9763
9936
|
truncateHistoricalToolResults(messages, conversationId) {
|
|
9764
9937
|
let latestRunId;
|
|
9765
9938
|
let latestToolMessageIndex = -1;
|
|
@@ -10611,6 +10784,7 @@ var AgentHarness = class _AgentHarness {
|
|
|
10611
10784
|
const messages = [...input.messages ?? []];
|
|
10612
10785
|
const conversationId = input.conversationId ?? "__default__";
|
|
10613
10786
|
this.seedToolResultArchive(conversationId, input.parameters);
|
|
10787
|
+
const spillPolicy = readSpillPolicy(input.parameters);
|
|
10614
10788
|
const truncationSummary = this.truncateHistoricalToolResults(messages, conversationId);
|
|
10615
10789
|
if (truncationSummary.changed) {
|
|
10616
10790
|
costLog.debug(
|
|
@@ -11752,29 +11926,39 @@ ${textContent}` };
|
|
|
11752
11926
|
span.setStatus({ code: SpanStatusCode.OK });
|
|
11753
11927
|
span.end();
|
|
11754
11928
|
}
|
|
11755
|
-
const
|
|
11756
|
-
const
|
|
11929
|
+
const { mediaItems, strippedOutput } = extractMediaFromToolOutput(result2.output);
|
|
11930
|
+
const strippedSerialized = JSON.stringify(strippedOutput ?? null);
|
|
11931
|
+
const guarded = await this.guardOversizedToolResult({
|
|
11932
|
+
tenantId: input.tenantId ?? "__default__",
|
|
11933
|
+
toolName: result2.tool,
|
|
11934
|
+
toolCallId: result2.callId,
|
|
11935
|
+
output: strippedOutput,
|
|
11936
|
+
serialized: strippedSerialized,
|
|
11937
|
+
policy: spillPolicy
|
|
11938
|
+
});
|
|
11939
|
+
const effectiveOutput = guarded ?? strippedOutput ?? null;
|
|
11940
|
+
const effectiveSerialized = guarded ? JSON.stringify(guarded) : strippedSerialized;
|
|
11941
|
+
const outputTokenEstimate = Math.ceil(effectiveSerialized.length / 4);
|
|
11757
11942
|
toolOutputEstimateSinceModel += outputTokenEstimate;
|
|
11758
11943
|
yield pushEvent({
|
|
11759
11944
|
type: "tool:completed",
|
|
11760
11945
|
tool: result2.tool,
|
|
11761
11946
|
toolCallId: result2.callId,
|
|
11762
11947
|
input: callInputMap.get(result2.callId),
|
|
11763
|
-
output:
|
|
11948
|
+
output: effectiveOutput,
|
|
11764
11949
|
duration: now() - batchStart,
|
|
11765
11950
|
outputTokenEstimate
|
|
11766
11951
|
});
|
|
11767
|
-
const { mediaItems, strippedOutput } = extractMediaFromToolOutput(result2.output);
|
|
11768
11952
|
toolResultsForModel.push({
|
|
11769
11953
|
type: "tool_result",
|
|
11770
11954
|
tool_use_id: result2.callId,
|
|
11771
11955
|
tool_name: result2.tool,
|
|
11772
|
-
content:
|
|
11956
|
+
content: effectiveSerialized
|
|
11773
11957
|
});
|
|
11774
11958
|
{
|
|
11775
11959
|
const archive = this.archivedToolResultsByConversation.get(conversationId);
|
|
11776
11960
|
if (archive && !NON_ARCHIVABLE_TOOL_NAMES.has(result2.tool)) {
|
|
11777
|
-
const payload =
|
|
11961
|
+
const payload = effectiveSerialized;
|
|
11778
11962
|
archive[result2.callId] = {
|
|
11779
11963
|
toolResultId: result2.callId,
|
|
11780
11964
|
conversationId,
|
|
@@ -11795,7 +11979,7 @@ ${textContent}` };
|
|
|
11795
11979
|
output: {
|
|
11796
11980
|
type: "content",
|
|
11797
11981
|
value: [
|
|
11798
|
-
{ type: "text", text:
|
|
11982
|
+
{ type: "text", text: effectiveSerialized },
|
|
11799
11983
|
...mediaItems
|
|
11800
11984
|
]
|
|
11801
11985
|
}
|
|
@@ -11805,7 +11989,7 @@ ${textContent}` };
|
|
|
11805
11989
|
type: "tool-result",
|
|
11806
11990
|
toolCallId: result2.callId,
|
|
11807
11991
|
toolName: result2.tool,
|
|
11808
|
-
output: { type: "json", value:
|
|
11992
|
+
output: { type: "json", value: effectiveOutput }
|
|
11809
11993
|
});
|
|
11810
11994
|
}
|
|
11811
11995
|
}
|
|
@@ -14654,7 +14838,7 @@ ${draft.toolTimeline.join("\n")}`);
|
|
|
14654
14838
|
};
|
|
14655
14839
|
|
|
14656
14840
|
// src/index.ts
|
|
14657
|
-
import { defineTool as
|
|
14841
|
+
import { defineTool as defineTool14 } from "@poncho-ai/sdk";
|
|
14658
14842
|
export {
|
|
14659
14843
|
AgentHarness,
|
|
14660
14844
|
AgentOrchestrator,
|
|
@@ -14728,7 +14912,7 @@ export {
|
|
|
14728
14912
|
createWriteTool,
|
|
14729
14913
|
decodeFileInputData,
|
|
14730
14914
|
defaultAgentDefinition,
|
|
14731
|
-
|
|
14915
|
+
defineTool14 as defineTool,
|
|
14732
14916
|
deleteOpenAICodexSession,
|
|
14733
14917
|
deriveUploadKey,
|
|
14734
14918
|
ensureAgentIdentity,
|
package/package.json
CHANGED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { defineTool, type ToolDefinition } from "@poncho-ai/sdk";
|
|
2
|
+
|
|
3
|
+
// ---------------------------------------------------------------------------
|
|
4
|
+
// ask_user — pause the run to ask the user a structured, multiple-choice
|
|
5
|
+
// question with pre-made options (the in-app analog of Claude Code's
|
|
6
|
+
// AskUserQuestion). The client renders tappable option chips so the user
|
|
7
|
+
// answers with a tap instead of typing prose.
|
|
8
|
+
//
|
|
9
|
+
// This tool is dispatched to the client ("device" dispatch, forced in
|
|
10
|
+
// AgentHarness.resolveToolMode): the harness pauses the run, emits a
|
|
11
|
+
// checkpoint carrying the questions payload, and the consumer (PonchOS)
|
|
12
|
+
// resumes the run by POSTing the user's selections back as this tool's
|
|
13
|
+
// result. The handler below is a defensive stub — device dispatch
|
|
14
|
+
// intercepts the call before any server-side execution, so it must never
|
|
15
|
+
// actually run.
|
|
16
|
+
// ---------------------------------------------------------------------------
|
|
17
|
+
|
|
18
|
+
export const createAskUserTool = (): ToolDefinition =>
|
|
19
|
+
defineTool({
|
|
20
|
+
name: "ask_user",
|
|
21
|
+
description:
|
|
22
|
+
"Ask the user one or more structured multiple-choice questions and wait for their answer. " +
|
|
23
|
+
"Use this INSTEAD of writing a question as plain text whenever you would otherwise pause to " +
|
|
24
|
+
"let the user choose between options — clarifying ambiguous requirements, picking an approach, " +
|
|
25
|
+
"confirming a direction. The user sees tappable option chips and answers with a tap. " +
|
|
26
|
+
"Prefer this over a prose question: it is faster for the user and gives you a clean answer. " +
|
|
27
|
+
"Guidelines: ask 1–4 questions at once, each with 2–4 concrete options; keep `header` very short " +
|
|
28
|
+
"(a few words); write each option `label` short and its `description` to one line. The user can " +
|
|
29
|
+
"always type a custom 'Other' answer, so you do not need to add one. Do NOT call any other tool " +
|
|
30
|
+
"in the same turn as ask_user, and call it at most once per turn. After the user answers you will " +
|
|
31
|
+
"receive their selections and may continue.",
|
|
32
|
+
inputSchema: {
|
|
33
|
+
type: "object",
|
|
34
|
+
properties: {
|
|
35
|
+
questions: {
|
|
36
|
+
type: "array",
|
|
37
|
+
description: "1–4 questions to ask the user at once.",
|
|
38
|
+
items: {
|
|
39
|
+
type: "object",
|
|
40
|
+
properties: {
|
|
41
|
+
question: {
|
|
42
|
+
type: "string",
|
|
43
|
+
description: "The full question text shown to the user.",
|
|
44
|
+
},
|
|
45
|
+
header: {
|
|
46
|
+
type: "string",
|
|
47
|
+
description:
|
|
48
|
+
"A very short label for this question (a few words, ~12 chars), shown as a chip.",
|
|
49
|
+
},
|
|
50
|
+
multiSelect: {
|
|
51
|
+
type: "boolean",
|
|
52
|
+
description:
|
|
53
|
+
"If true, the user may select multiple options. Defaults to false (single choice).",
|
|
54
|
+
},
|
|
55
|
+
options: {
|
|
56
|
+
type: "array",
|
|
57
|
+
description:
|
|
58
|
+
"The pre-made options. A free-text 'Other' option is added automatically by the client.",
|
|
59
|
+
items: {
|
|
60
|
+
type: "object",
|
|
61
|
+
properties: {
|
|
62
|
+
label: {
|
|
63
|
+
type: "string",
|
|
64
|
+
description: "Short, selectable label for this option.",
|
|
65
|
+
},
|
|
66
|
+
description: {
|
|
67
|
+
type: "string",
|
|
68
|
+
description: "A one-line explanation of what this option means.",
|
|
69
|
+
},
|
|
70
|
+
},
|
|
71
|
+
required: ["label"],
|
|
72
|
+
additionalProperties: false,
|
|
73
|
+
},
|
|
74
|
+
},
|
|
75
|
+
},
|
|
76
|
+
required: ["question", "header", "options"],
|
|
77
|
+
additionalProperties: false,
|
|
78
|
+
},
|
|
79
|
+
},
|
|
80
|
+
},
|
|
81
|
+
required: ["questions"],
|
|
82
|
+
additionalProperties: false,
|
|
83
|
+
},
|
|
84
|
+
handler: async () => {
|
|
85
|
+
// Unreachable in normal operation: ask_user is forced to client/device
|
|
86
|
+
// dispatch, so the harness checkpoints before this handler is invoked.
|
|
87
|
+
// If it ever runs, the tool was misconfigured (dispatch not forced) —
|
|
88
|
+
// surface an error rather than silently resolving with no user input.
|
|
89
|
+
return {
|
|
90
|
+
error:
|
|
91
|
+
"ask_user must be answered by the user on the client; it cannot run server-side. " +
|
|
92
|
+
"This indicates a dispatch misconfiguration.",
|
|
93
|
+
};
|
|
94
|
+
},
|
|
95
|
+
});
|
package/src/default-agent.ts
CHANGED
|
@@ -100,7 +100,7 @@ Environment: {{runtime.environment}}
|
|
|
100
100
|
|
|
101
101
|
- Use tools when needed
|
|
102
102
|
- Explain your reasoning clearly
|
|
103
|
-
-
|
|
103
|
+
- When requirements are ambiguous or you need the user to choose between options, use the \`ask_user\` tool to ask a structured multiple-choice question instead of writing the question as plain text. Reserve plain-text questions for genuinely open-ended asks that have no sensible pre-made options.
|
|
104
104
|
- Never claim a file/tool change unless the corresponding tool call actually succeeded
|
|
105
105
|
`;
|
|
106
106
|
};
|
package/src/harness.ts
CHANGED
|
@@ -30,6 +30,14 @@ import {
|
|
|
30
30
|
createReminderStoreFromEngine,
|
|
31
31
|
} from "./storage/store-adapters.js";
|
|
32
32
|
import { BashEnvironmentManager } from "./vfs/bash-manager.js";
|
|
33
|
+
import {
|
|
34
|
+
type ToolResultSpillPolicy,
|
|
35
|
+
readSpillPolicy,
|
|
36
|
+
isOversizedToolResult,
|
|
37
|
+
formatSpillPayload,
|
|
38
|
+
buildSpillHandle,
|
|
39
|
+
buildInlineTruncation,
|
|
40
|
+
} from "./tool-result-guard.js";
|
|
33
41
|
import type { VirtualMount } from "./vfs/poncho-fs-adapter.js";
|
|
34
42
|
export type { VirtualMount } from "./vfs/poncho-fs-adapter.js";
|
|
35
43
|
import { createBashTool } from "./vfs/bash-tool.js";
|
|
@@ -65,6 +73,7 @@ import { jsonSchemaToZod } from "./schema-converter.js";
|
|
|
65
73
|
import type { SkillMetadata } from "./skill-context.js";
|
|
66
74
|
import { createSkillTools, normalizeScriptPolicyPath } from "./skill-tools.js";
|
|
67
75
|
import { createSearchTools } from "./search-tools.js";
|
|
76
|
+
import { createAskUserTool } from "./ask-user-tool.js";
|
|
68
77
|
import { createSubagentTools } from "./subagent-tools.js";
|
|
69
78
|
import type { SubagentManager } from "./subagent-manager.js";
|
|
70
79
|
import { trace, context as otelContext, createContextKey, type Context as OtelContextType, SpanStatusCode, SpanKind, diag, DiagConsoleLogger, DiagLogLevel } from "@opentelemetry/api";
|
|
@@ -965,6 +974,11 @@ export class AgentHarness {
|
|
|
965
974
|
|
|
966
975
|
/** Returns the normalized {access, dispatch} mode for the tool. */
|
|
967
976
|
private resolveToolMode(toolName: string): { access?: "approval"; dispatch?: "device" } {
|
|
977
|
+
// ask_user is always answered by the user on the client. Force device
|
|
978
|
+
// dispatch unconditionally so it pauses the run and checkpoints rather
|
|
979
|
+
// than running its (defensive, error-returning) server-side handler —
|
|
980
|
+
// even if no `poncho.config.js` entry exists for it.
|
|
981
|
+
if (toolName === "ask_user") return { dispatch: "device" };
|
|
968
982
|
return normalizeToolAccess(this.resolveToolAccess(toolName));
|
|
969
983
|
}
|
|
970
984
|
|
|
@@ -1014,6 +1028,9 @@ export class AgentHarness {
|
|
|
1014
1028
|
this.registerIfMissing(tool);
|
|
1015
1029
|
}
|
|
1016
1030
|
}
|
|
1031
|
+
if (this.isToolEnabled("ask_user")) {
|
|
1032
|
+
this.registerIfMissing(createAskUserTool());
|
|
1033
|
+
}
|
|
1017
1034
|
if (this.environment === "development" && this.isToolEnabled("poncho_docs")) {
|
|
1018
1035
|
this.registerIfMissing(ponchoDocsTool);
|
|
1019
1036
|
}
|
|
@@ -1172,6 +1189,77 @@ export class AgentHarness {
|
|
|
1172
1189
|
return merged;
|
|
1173
1190
|
}
|
|
1174
1191
|
|
|
1192
|
+
/**
|
|
1193
|
+
* Guard a single FRESH tool result against overflowing the context window.
|
|
1194
|
+
* `serialized` is the JSON-serialized (media-stripped) output. Returns a
|
|
1195
|
+
* replacement value to put in front of the model (a handle or a truncated
|
|
1196
|
+
* preview) when the result is oversized, or `null` to leave it untouched.
|
|
1197
|
+
* Spilling writes the full payload to a VFS file; on any write failure it
|
|
1198
|
+
* falls back to inline truncation, so this never throws into the run loop.
|
|
1199
|
+
*/
|
|
1200
|
+
private async guardOversizedToolResult(opts: {
|
|
1201
|
+
tenantId: string;
|
|
1202
|
+
toolName: string;
|
|
1203
|
+
toolCallId: string;
|
|
1204
|
+
output: unknown;
|
|
1205
|
+
serialized: string;
|
|
1206
|
+
policy: ToolResultSpillPolicy;
|
|
1207
|
+
}): Promise<Record<string, unknown> | null> {
|
|
1208
|
+
const { tenantId, toolName, toolCallId, output, serialized, policy } = opts;
|
|
1209
|
+
if (!isOversizedToolResult(serialized, policy)) return null;
|
|
1210
|
+
|
|
1211
|
+
if (policy.enabled && this.bashManager) {
|
|
1212
|
+
try {
|
|
1213
|
+
const vfs = this.createVfsAccess(tenantId);
|
|
1214
|
+
const { content, format, records } = formatSpillPayload(output);
|
|
1215
|
+
// toolCallId is unique per call and stable across a checkpoint→resume,
|
|
1216
|
+
// so the path is idempotent (a resumed run rewrites the same file).
|
|
1217
|
+
const safeTool = toolName.replace(/[^A-Za-z0-9_-]/g, "_");
|
|
1218
|
+
const path = `${policy.dir}/${safeTool}_${toolCallId}.${format}`;
|
|
1219
|
+
await vfs.mkdir(policy.dir, { recursive: true });
|
|
1220
|
+
await vfs.writeText(path, content);
|
|
1221
|
+
await this.pruneSpillDir(vfs, policy.dir, policy.keepLast);
|
|
1222
|
+
return buildSpillHandle({ toolName, path, format, serialized, records });
|
|
1223
|
+
} catch {
|
|
1224
|
+
// Fall through to inline truncation below.
|
|
1225
|
+
}
|
|
1226
|
+
}
|
|
1227
|
+
|
|
1228
|
+
return buildInlineTruncation(toolName, serialized);
|
|
1229
|
+
}
|
|
1230
|
+
|
|
1231
|
+
/** Best-effort: keep only the newest `keepLast` files in the spill dir. */
|
|
1232
|
+
private async pruneSpillDir(
|
|
1233
|
+
vfs: NonNullable<ToolContext["vfs"]>,
|
|
1234
|
+
dir: string,
|
|
1235
|
+
keepLast: number,
|
|
1236
|
+
): Promise<void> {
|
|
1237
|
+
try {
|
|
1238
|
+
const entries = await vfs.readdir(dir);
|
|
1239
|
+
const names = (Array.isArray(entries) ? entries : [])
|
|
1240
|
+
.map((e) => (typeof e === "string" ? e : (e as { name?: string }).name))
|
|
1241
|
+
.filter((n): n is string => typeof n === "string");
|
|
1242
|
+
if (names.length <= keepLast) return;
|
|
1243
|
+
const stated = await Promise.all(
|
|
1244
|
+
names.map(async (name) => {
|
|
1245
|
+
try {
|
|
1246
|
+
const s = await vfs.stat(`${dir}/${name}`);
|
|
1247
|
+
return { name, mtime: s.updatedAt ?? 0 };
|
|
1248
|
+
} catch {
|
|
1249
|
+
return { name, mtime: 0 };
|
|
1250
|
+
}
|
|
1251
|
+
}),
|
|
1252
|
+
);
|
|
1253
|
+
stated.sort((a, b) => a.mtime - b.mtime); // oldest first
|
|
1254
|
+
const doomed = stated.slice(0, stated.length - keepLast);
|
|
1255
|
+
for (const d of doomed) {
|
|
1256
|
+
try { await vfs.rm(`${dir}/${d.name}`); } catch { /* best-effort */ }
|
|
1257
|
+
}
|
|
1258
|
+
} catch {
|
|
1259
|
+
// Dir unreadable — nothing to prune.
|
|
1260
|
+
}
|
|
1261
|
+
}
|
|
1262
|
+
|
|
1175
1263
|
private truncateHistoricalToolResults(
|
|
1176
1264
|
messages: Message[],
|
|
1177
1265
|
conversationId: string,
|
|
@@ -2203,6 +2291,7 @@ export class AgentHarness {
|
|
|
2203
2291
|
const messages: Message[] = [...(input.messages ?? [])];
|
|
2204
2292
|
const conversationId = input.conversationId ?? "__default__";
|
|
2205
2293
|
this.seedToolResultArchive(conversationId, input.parameters);
|
|
2294
|
+
const spillPolicy = readSpillPolicy(input.parameters);
|
|
2206
2295
|
const truncationSummary = this.truncateHistoricalToolResults(messages, conversationId);
|
|
2207
2296
|
if (truncationSummary.changed) {
|
|
2208
2297
|
costLog.debug(
|
|
@@ -3683,30 +3772,49 @@ Code is wrapped in an async IIFE — use \`return\` to return a value to the too
|
|
|
3683
3772
|
span.setStatus({ code: SpanStatusCode.OK });
|
|
3684
3773
|
span.end();
|
|
3685
3774
|
}
|
|
3686
|
-
|
|
3687
|
-
|
|
3775
|
+
// Separate inline media (images/PDFs) from the textual output, then
|
|
3776
|
+
// guard the textual part against overflowing the window. A single
|
|
3777
|
+
// oversized fresh result is replaced with a handle (spill) or a
|
|
3778
|
+
// preview (inline) BEFORE we emit the event or feed the model — so
|
|
3779
|
+
// neither the WS stream nor the next step ever carries the full
|
|
3780
|
+
// multi-MB payload.
|
|
3781
|
+
const { mediaItems, strippedOutput } = extractMediaFromToolOutput(result.output);
|
|
3782
|
+
const strippedSerialized = JSON.stringify(strippedOutput ?? null);
|
|
3783
|
+
const guarded = await this.guardOversizedToolResult({
|
|
3784
|
+
tenantId: input.tenantId ?? "__default__",
|
|
3785
|
+
toolName: result.tool,
|
|
3786
|
+
toolCallId: result.callId,
|
|
3787
|
+
output: strippedOutput,
|
|
3788
|
+
serialized: strippedSerialized,
|
|
3789
|
+
policy: spillPolicy,
|
|
3790
|
+
});
|
|
3791
|
+
const effectiveOutput: unknown = guarded ?? strippedOutput ?? null;
|
|
3792
|
+
const effectiveSerialized = guarded ? JSON.stringify(guarded) : strippedSerialized;
|
|
3793
|
+
const outputTokenEstimate = Math.ceil(effectiveSerialized.length / 4);
|
|
3688
3794
|
toolOutputEstimateSinceModel += outputTokenEstimate;
|
|
3689
3795
|
yield pushEvent({
|
|
3690
3796
|
type: "tool:completed",
|
|
3691
3797
|
tool: result.tool,
|
|
3692
3798
|
toolCallId: result.callId,
|
|
3693
3799
|
input: callInputMap.get(result.callId),
|
|
3694
|
-
output:
|
|
3800
|
+
output: effectiveOutput,
|
|
3695
3801
|
duration: now() - batchStart,
|
|
3696
3802
|
outputTokenEstimate,
|
|
3697
3803
|
});
|
|
3698
3804
|
|
|
3699
|
-
const { mediaItems, strippedOutput } = extractMediaFromToolOutput(result.output);
|
|
3700
3805
|
toolResultsForModel.push({
|
|
3701
3806
|
type: "tool_result",
|
|
3702
3807
|
tool_use_id: result.callId,
|
|
3703
3808
|
tool_name: result.tool,
|
|
3704
|
-
content:
|
|
3809
|
+
content: effectiveSerialized,
|
|
3705
3810
|
});
|
|
3706
3811
|
{
|
|
3707
3812
|
const archive = this.archivedToolResultsByConversation.get(conversationId);
|
|
3708
3813
|
if (archive && !NON_ARCHIVABLE_TOOL_NAMES.has(result.tool)) {
|
|
3709
|
-
|
|
3814
|
+
// Archive the model-visible payload (handle/preview when guarded).
|
|
3815
|
+
// When spilled, the VFS file is the durable full copy; keeping the
|
|
3816
|
+
// in-memory archive small avoids re-introducing the same bloat.
|
|
3817
|
+
const payload = effectiveSerialized;
|
|
3710
3818
|
archive[result.callId] = {
|
|
3711
3819
|
toolResultId: result.callId,
|
|
3712
3820
|
conversationId,
|
|
@@ -3728,7 +3836,7 @@ Code is wrapped in an async IIFE — use \`return\` to return a value to the too
|
|
|
3728
3836
|
output: {
|
|
3729
3837
|
type: "content",
|
|
3730
3838
|
value: [
|
|
3731
|
-
{ type: "text", text:
|
|
3839
|
+
{ type: "text", text: effectiveSerialized },
|
|
3732
3840
|
...mediaItems,
|
|
3733
3841
|
],
|
|
3734
3842
|
},
|
|
@@ -3738,7 +3846,7 @@ Code is wrapped in an async IIFE — use \`return\` to return a value to the too
|
|
|
3738
3846
|
type: "tool-result",
|
|
3739
3847
|
toolCallId: result.callId,
|
|
3740
3848
|
toolName: result.tool,
|
|
3741
|
-
output: { type: "json", value:
|
|
3849
|
+
output: { type: "json", value: effectiveOutput },
|
|
3742
3850
|
});
|
|
3743
3851
|
}
|
|
3744
3852
|
}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
// Oversized FRESH tool-result guard — pure logic.
|
|
2
|
+
//
|
|
3
|
+
// `truncateHistoricalToolResults` (in harness.ts) only shrinks results from
|
|
4
|
+
// PRIOR runs — it deliberately preserves the latest run's results so the model
|
|
5
|
+
// can read what it just fetched. So a single fresh result that is itself larger
|
|
6
|
+
// than the context window is never truncated, and the next step fails with
|
|
7
|
+
// "prompt is too long" (seen with MCP email-fetch tools returning 1.6M–3.3M
|
|
8
|
+
// token payloads). This module decides what to put in front of the model when
|
|
9
|
+
// that happens; the I/O (the VFS write + prune) lives on the Harness class,
|
|
10
|
+
// which has the per-tenant filesystem. Splitting the pure decision out keeps it
|
|
11
|
+
// unit-testable without a Harness instance.
|
|
12
|
+
|
|
13
|
+
/** Run parameter key carrying the spill policy (a freeform-params bag entry,
|
|
14
|
+
* same mechanism as `__toolResultArchive`). */
|
|
15
|
+
export const SPILL_POLICY_PARAM = "__toolResultSpill";
|
|
16
|
+
|
|
17
|
+
/** Results whose serialized form exceeds this many chars are guarded. ~125k
|
|
18
|
+
* tokens — well under any current window, well above any normal result, so
|
|
19
|
+
* normal-sized results are never touched. */
|
|
20
|
+
export const DEFAULT_OVERSIZED_TOOL_RESULT_CHARS = 500_000;
|
|
21
|
+
/** Preview kept inline when SPILLING — small, because the full payload is in
|
|
22
|
+
* the file. */
|
|
23
|
+
export const SPILLED_PREVIEW_CHARS = 6_000;
|
|
24
|
+
/** Preview kept inline when TRUNCATING — larger, because there's no file to
|
|
25
|
+
* fall back to. */
|
|
26
|
+
export const INLINE_TRUNCATE_PREVIEW_CHARS = 60_000;
|
|
27
|
+
export const DEFAULT_SPILL_DIR = "/tmp/tool-results";
|
|
28
|
+
export const DEFAULT_SPILL_KEEP_LAST = 20;
|
|
29
|
+
|
|
30
|
+
export interface ToolResultSpillPolicy {
|
|
31
|
+
enabled: boolean;
|
|
32
|
+
dir: string;
|
|
33
|
+
thresholdChars: number;
|
|
34
|
+
keepLast: number;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** Parse the spill policy out of the run parameters, applying defaults. A
|
|
38
|
+
* missing/malformed bag yields a disabled policy with safe defaults. */
|
|
39
|
+
export const readSpillPolicy = (
|
|
40
|
+
parameters: Record<string, unknown> | undefined,
|
|
41
|
+
): ToolResultSpillPolicy => {
|
|
42
|
+
const raw = parameters?.[SPILL_POLICY_PARAM];
|
|
43
|
+
const obj = (typeof raw === "object" && raw !== null ? raw : {}) as Record<string, unknown>;
|
|
44
|
+
const thresholdChars =
|
|
45
|
+
typeof obj.thresholdChars === "number" && obj.thresholdChars > 0
|
|
46
|
+
? obj.thresholdChars
|
|
47
|
+
: DEFAULT_OVERSIZED_TOOL_RESULT_CHARS;
|
|
48
|
+
const dir =
|
|
49
|
+
typeof obj.dir === "string" && obj.dir.startsWith("/") ? obj.dir : DEFAULT_SPILL_DIR;
|
|
50
|
+
const keepLast =
|
|
51
|
+
typeof obj.keepLast === "number" && obj.keepLast > 0
|
|
52
|
+
? Math.floor(obj.keepLast)
|
|
53
|
+
: DEFAULT_SPILL_KEEP_LAST;
|
|
54
|
+
return { enabled: obj.enabled === true, dir, thresholdChars, keepLast };
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
/** True when this serialized result is large enough to need guarding. */
|
|
58
|
+
export const isOversizedToolResult = (
|
|
59
|
+
serialized: string,
|
|
60
|
+
policy: ToolResultSpillPolicy,
|
|
61
|
+
): boolean => serialized.length > policy.thresholdChars;
|
|
62
|
+
|
|
63
|
+
/** Serialize an oversized payload for the spill file. Arrays become JSONL (one
|
|
64
|
+
* record per line — bash/jq/sed friendly, line ranges map to records); other
|
|
65
|
+
* shapes become pretty JSON. */
|
|
66
|
+
export const formatSpillPayload = (
|
|
67
|
+
output: unknown,
|
|
68
|
+
): { content: string; format: "jsonl" | "json"; records?: number } => {
|
|
69
|
+
if (Array.isArray(output)) {
|
|
70
|
+
return {
|
|
71
|
+
content: output.map((row) => JSON.stringify(row)).join("\n"),
|
|
72
|
+
format: "jsonl",
|
|
73
|
+
records: output.length,
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
return { content: JSON.stringify(output ?? null, null, 2), format: "json" };
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
/** The handle object placed in front of the model when a result is spilled to
|
|
80
|
+
* a file: a small preview plus the path and bash-readback instructions. */
|
|
81
|
+
export const buildSpillHandle = (opts: {
|
|
82
|
+
toolName: string;
|
|
83
|
+
path: string;
|
|
84
|
+
format: "jsonl" | "json";
|
|
85
|
+
serialized: string;
|
|
86
|
+
records?: number;
|
|
87
|
+
}): Record<string, unknown> => {
|
|
88
|
+
const { toolName, path, format, serialized, records } = opts;
|
|
89
|
+
const approxTokens = Math.ceil(serialized.length / 4);
|
|
90
|
+
const isArray = format === "jsonl";
|
|
91
|
+
return {
|
|
92
|
+
__toolResultSpilled: true,
|
|
93
|
+
tool: toolName,
|
|
94
|
+
path,
|
|
95
|
+
format,
|
|
96
|
+
...(records !== undefined ? { records } : {}),
|
|
97
|
+
totalChars: serialized.length,
|
|
98
|
+
approxTokens,
|
|
99
|
+
preview: serialized.slice(0, SPILLED_PREVIEW_CHARS),
|
|
100
|
+
note:
|
|
101
|
+
`This result was too large to return inline (~${approxTokens.toLocaleString()} ` +
|
|
102
|
+
`tokens) and was saved to ${path}. Inspect it with bash — e.g. ` +
|
|
103
|
+
`\`wc -l ${path}\`, \`head -c 4000 ${path}\`, ` +
|
|
104
|
+
(isArray
|
|
105
|
+
? `\`sed -n '1,5p' ${path}\`, \`grep -i <term> ${path}\`, or \`jq\` per line`
|
|
106
|
+
: `\`grep -i <term> ${path}\``) +
|
|
107
|
+
`. Do NOT cat or read_file the whole file (it re-overflows). Or re-run the ` +
|
|
108
|
+
`tool with a narrower request (fewer items / a filter / metadata only).`,
|
|
109
|
+
};
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
/** The replacement placed in front of the model when a result is too large and
|
|
113
|
+
* spill is unavailable: a preview plus a "re-run narrower" instruction. */
|
|
114
|
+
export const buildInlineTruncation = (
|
|
115
|
+
toolName: string,
|
|
116
|
+
serialized: string,
|
|
117
|
+
): Record<string, unknown> => {
|
|
118
|
+
const approxTokens = Math.ceil(serialized.length / 4);
|
|
119
|
+
const omittedChars = Math.max(0, serialized.length - INLINE_TRUNCATE_PREVIEW_CHARS);
|
|
120
|
+
return {
|
|
121
|
+
__toolResultTruncated: true,
|
|
122
|
+
tool: toolName,
|
|
123
|
+
approxTokens,
|
|
124
|
+
omittedChars,
|
|
125
|
+
preview: serialized.slice(0, INLINE_TRUNCATE_PREVIEW_CHARS),
|
|
126
|
+
note:
|
|
127
|
+
`This result was too large (~${approxTokens.toLocaleString()} tokens) and was ` +
|
|
128
|
+
`truncated to fit the context window — only the first ` +
|
|
129
|
+
`${INLINE_TRUNCATE_PREVIEW_CHARS.toLocaleString()} characters are shown. Re-run ` +
|
|
130
|
+
`with a narrower request (fewer items / a filter / metadata only). Do NOT retry ` +
|
|
131
|
+
`the same call unchanged — it will overflow again.`,
|
|
132
|
+
};
|
|
133
|
+
};
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
import {
|
|
3
|
+
DEFAULT_OVERSIZED_TOOL_RESULT_CHARS,
|
|
4
|
+
DEFAULT_SPILL_DIR,
|
|
5
|
+
DEFAULT_SPILL_KEEP_LAST,
|
|
6
|
+
INLINE_TRUNCATE_PREVIEW_CHARS,
|
|
7
|
+
SPILLED_PREVIEW_CHARS,
|
|
8
|
+
buildInlineTruncation,
|
|
9
|
+
buildSpillHandle,
|
|
10
|
+
formatSpillPayload,
|
|
11
|
+
isOversizedToolResult,
|
|
12
|
+
readSpillPolicy,
|
|
13
|
+
} from "../src/tool-result-guard.js";
|
|
14
|
+
|
|
15
|
+
describe("readSpillPolicy", () => {
|
|
16
|
+
it("defaults to disabled with safe defaults when the param is absent", () => {
|
|
17
|
+
expect(readSpillPolicy(undefined)).toEqual({
|
|
18
|
+
enabled: false,
|
|
19
|
+
dir: DEFAULT_SPILL_DIR,
|
|
20
|
+
thresholdChars: DEFAULT_OVERSIZED_TOOL_RESULT_CHARS,
|
|
21
|
+
keepLast: DEFAULT_SPILL_KEEP_LAST,
|
|
22
|
+
});
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
it("defaults when the param is malformed", () => {
|
|
26
|
+
expect(readSpillPolicy({ __toolResultSpill: "nope" }).enabled).toBe(false);
|
|
27
|
+
expect(readSpillPolicy({ __toolResultSpill: 42 }).dir).toBe(DEFAULT_SPILL_DIR);
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
it("reads enabled + dir + threshold + keepLast", () => {
|
|
31
|
+
const p = readSpillPolicy({
|
|
32
|
+
__toolResultSpill: { enabled: true, dir: "/tmp/x", thresholdChars: 1000, keepLast: 5 },
|
|
33
|
+
});
|
|
34
|
+
expect(p).toEqual({ enabled: true, dir: "/tmp/x", thresholdChars: 1000, keepLast: 5 });
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
it("rejects a non-absolute dir, a non-positive threshold, and a bad keepLast", () => {
|
|
38
|
+
const p = readSpillPolicy({
|
|
39
|
+
__toolResultSpill: { enabled: true, dir: "relative", thresholdChars: 0, keepLast: -3 },
|
|
40
|
+
});
|
|
41
|
+
expect(p.dir).toBe(DEFAULT_SPILL_DIR);
|
|
42
|
+
expect(p.thresholdChars).toBe(DEFAULT_OVERSIZED_TOOL_RESULT_CHARS);
|
|
43
|
+
expect(p.keepLast).toBe(DEFAULT_SPILL_KEEP_LAST);
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
it("only treats enabled === true as enabled (not truthy strings)", () => {
|
|
47
|
+
expect(readSpillPolicy({ __toolResultSpill: { enabled: "yes" } }).enabled).toBe(false);
|
|
48
|
+
expect(readSpillPolicy({ __toolResultSpill: { enabled: 1 } }).enabled).toBe(false);
|
|
49
|
+
});
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
describe("isOversizedToolResult", () => {
|
|
53
|
+
const policy = readSpillPolicy({ __toolResultSpill: { enabled: true, thresholdChars: 100 } });
|
|
54
|
+
it("is false at or below the threshold (boundary)", () => {
|
|
55
|
+
expect(isOversizedToolResult("a".repeat(100), policy)).toBe(false);
|
|
56
|
+
expect(isOversizedToolResult("a".repeat(99), policy)).toBe(false);
|
|
57
|
+
});
|
|
58
|
+
it("is true above the threshold", () => {
|
|
59
|
+
expect(isOversizedToolResult("a".repeat(101), policy)).toBe(true);
|
|
60
|
+
});
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
describe("formatSpillPayload", () => {
|
|
64
|
+
it("formats an array as JSONL with a record count", () => {
|
|
65
|
+
const r = formatSpillPayload([{ a: 1 }, { b: 2 }, { c: 3 }]);
|
|
66
|
+
expect(r.format).toBe("jsonl");
|
|
67
|
+
expect(r.records).toBe(3);
|
|
68
|
+
expect(r.content.split("\n")).toEqual(['{"a":1}', '{"b":2}', '{"c":3}']);
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
it("formats a non-array as pretty JSON with no record count", () => {
|
|
72
|
+
const r = formatSpillPayload({ hello: "world" });
|
|
73
|
+
expect(r.format).toBe("json");
|
|
74
|
+
expect(r.records).toBeUndefined();
|
|
75
|
+
expect(r.content).toBe('{\n "hello": "world"\n}');
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
it("handles null", () => {
|
|
79
|
+
expect(formatSpillPayload(null)).toEqual({ content: "null", format: "json" });
|
|
80
|
+
});
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
describe("buildSpillHandle", () => {
|
|
84
|
+
it("returns a handle with path, format, records, preview, and bash instructions", () => {
|
|
85
|
+
const serialized = "x".repeat(SPILLED_PREVIEW_CHARS + 50_000);
|
|
86
|
+
const h = buildSpillHandle({
|
|
87
|
+
toolName: "mcp_gmail_GMAIL_FETCH_EMAILS",
|
|
88
|
+
path: "/tmp/tool-results/mcp_gmail_GMAIL_FETCH_EMAILS_tool_1.jsonl",
|
|
89
|
+
format: "jsonl",
|
|
90
|
+
serialized,
|
|
91
|
+
records: 312,
|
|
92
|
+
});
|
|
93
|
+
expect(h.__toolResultSpilled).toBe(true);
|
|
94
|
+
expect(h.records).toBe(312);
|
|
95
|
+
expect(h.totalChars).toBe(serialized.length);
|
|
96
|
+
expect((h.preview as string).length).toBe(SPILLED_PREVIEW_CHARS);
|
|
97
|
+
expect(String(h.note)).toContain("/tmp/tool-results/");
|
|
98
|
+
expect(String(h.note)).toContain("jq"); // array → jsonl hint
|
|
99
|
+
expect(String(h.note)).toContain("Do NOT cat");
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
it("omits the array-only jq hint and records for json format", () => {
|
|
103
|
+
const h = buildSpillHandle({
|
|
104
|
+
toolName: "t",
|
|
105
|
+
path: "/tmp/tool-results/t_x.json",
|
|
106
|
+
format: "json",
|
|
107
|
+
serialized: "y".repeat(10),
|
|
108
|
+
});
|
|
109
|
+
expect(h.records).toBeUndefined();
|
|
110
|
+
expect(String(h.note)).not.toContain("jq");
|
|
111
|
+
});
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
describe("buildInlineTruncation", () => {
|
|
115
|
+
it("returns a truncation marker with a capped preview and omitted-char count", () => {
|
|
116
|
+
const serialized = "z".repeat(INLINE_TRUNCATE_PREVIEW_CHARS + 1234);
|
|
117
|
+
const t = buildInlineTruncation("some_tool", serialized);
|
|
118
|
+
expect(t.__toolResultTruncated).toBe(true);
|
|
119
|
+
expect((t.preview as string).length).toBe(INLINE_TRUNCATE_PREVIEW_CHARS);
|
|
120
|
+
expect(t.omittedChars).toBe(1234);
|
|
121
|
+
expect(String(t.note)).toContain("Do NOT retry");
|
|
122
|
+
});
|
|
123
|
+
});
|