@vellumai/assistant 0.10.5-staging.2 → 0.10.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/__tests__/compactor-media-strip.test.ts +177 -0
- package/src/__tests__/conversation-history-web-search.test.ts +7 -0
- package/src/__tests__/guardian-question-mode.test.ts +57 -0
- package/src/__tests__/notification-decision-fallback.test.ts +94 -0
- package/src/__tests__/pre-model-call-sanitize.test.ts +1 -1
- package/src/__tests__/slack-notification-approval-card.test.ts +89 -2
- package/src/agent/ax-tree-compaction.test.ts +4 -1
- package/src/agent/loop.ts +1 -176
- package/src/context/compactor.ts +29 -15
- package/src/context/outbound-sanitize.ts +202 -0
- package/src/daemon/handlers/shared.ts +1 -23
- package/src/daemon/host-cu-proxy.ts +1 -1
- package/src/messaging/providers/slack/send.test.ts +79 -0
- package/src/messaging/providers/slack/send.ts +48 -12
- package/src/notifications/README.md +1 -1
- package/src/notifications/adapters/slack.ts +44 -13
- package/src/notifications/broadcaster.ts +21 -15
- package/src/notifications/decision-engine.ts +121 -31
- package/src/notifications/guardian-question-mode.ts +58 -10
- package/src/prompts/templates/SOUL.md +40 -14
- package/src/runtime/slack-reply-session.test.ts +81 -0
- package/src/runtime/slack-reply-session.ts +13 -0
- package/src/util/__tests__/text-spacing.test.ts +53 -0
- package/src/util/text-spacing.ts +53 -0
package/src/agent/loop.ts
CHANGED
|
@@ -4,6 +4,7 @@ import { getConfig } from "../config/loader.js";
|
|
|
4
4
|
import { isMemoryV3Live } from "../config/memory-v3-gate.js";
|
|
5
5
|
import type { LLMCallSite } from "../config/schemas/llm.js";
|
|
6
6
|
import { recordEstimate } from "../context/estimator-calibration.js";
|
|
7
|
+
import { preModelCallSanitize } from "../context/outbound-sanitize.js";
|
|
7
8
|
import {
|
|
8
9
|
estimatePromptTokensRaw,
|
|
9
10
|
estimatePromptTokensWithTools,
|
|
@@ -14,7 +15,6 @@ import { spoolAndStubOversizedToolResults } from "../context/tool-result-spool.j
|
|
|
14
15
|
import type { ToolActivityMetadata } from "../daemon/message-types/web-activity.js";
|
|
15
16
|
import { parseActualTokensFromError } from "../daemon/parse-actual-tokens-from-error.js";
|
|
16
17
|
import type { TrustContext } from "../daemon/trust-context.js";
|
|
17
|
-
import { stripHistoricalWebSearchResults } from "../daemon/web-search-history.js";
|
|
18
18
|
import {
|
|
19
19
|
timeSyncSection,
|
|
20
20
|
traceAsyncSection,
|
|
@@ -2560,178 +2560,3 @@ export class AgentLoop {
|
|
|
2560
2560
|
};
|
|
2561
2561
|
}
|
|
2562
2562
|
}
|
|
2563
|
-
|
|
2564
|
-
/** Number of most-recent AX tree snapshots to keep in conversation history. */
|
|
2565
|
-
const MAX_AX_TREES_IN_HISTORY = 2;
|
|
2566
|
-
|
|
2567
|
-
/** Regex that matches the `<ax-tree>...</ax-tree>` markers. */
|
|
2568
|
-
const AX_TREE_PATTERN = /<ax-tree>[\s\S]*?<\/ax-tree>/g;
|
|
2569
|
-
const AX_TREE_PLACEHOLDER = "<ax_tree_omitted />";
|
|
2570
|
-
|
|
2571
|
-
/**
|
|
2572
|
-
* Escapes any literal `</ax-tree>` occurrences inside AX tree content so
|
|
2573
|
-
* that the non-greedy compaction regex (`AX_TREE_PATTERN`) does not stop
|
|
2574
|
-
* prematurely when the user happens to be viewing XML/HTML source that
|
|
2575
|
-
* contains the closing tag. The escaped content does not need to be
|
|
2576
|
-
* unescaped because compaction replaces the entire block with a placeholder.
|
|
2577
|
-
*/
|
|
2578
|
-
export function escapeAxTreeContent(content: string): string {
|
|
2579
|
-
return content.replace(/<\/ax-tree>/gi, "</ax-tree>");
|
|
2580
|
-
}
|
|
2581
|
-
|
|
2582
|
-
/**
|
|
2583
|
-
* Returns a shallow copy of `messages` where all but the most recent
|
|
2584
|
-
* `MAX_AX_TREES_IN_HISTORY` `<ax-tree>` blocks have been replaced with a
|
|
2585
|
-
* short placeholder. This keeps the conversation context small so that
|
|
2586
|
-
* TTFT does not grow linearly with step count in computer-use sessions.
|
|
2587
|
-
*
|
|
2588
|
-
* Counting is per-block, not per-message — a single user message can
|
|
2589
|
-
* contain multiple tool_result blocks each with their own AX tree snapshot.
|
|
2590
|
-
*/
|
|
2591
|
-
export function compactAxTreeHistory(messages: Message[]): Message[] {
|
|
2592
|
-
// Collect (messageIndex, blockIndex) for every tool_result block with <ax-tree>
|
|
2593
|
-
const axBlocks: Array<{ msgIdx: number; blockIdx: number }> = [];
|
|
2594
|
-
for (let i = 0; i < messages.length; i++) {
|
|
2595
|
-
const msg = messages[i];
|
|
2596
|
-
if (msg.role !== "user") continue;
|
|
2597
|
-
for (let j = 0; j < msg.content.length; j++) {
|
|
2598
|
-
const block = msg.content[j];
|
|
2599
|
-
if (
|
|
2600
|
-
block.type === "tool_result" &&
|
|
2601
|
-
typeof block.content === "string" &&
|
|
2602
|
-
block.content.includes("<ax-tree>")
|
|
2603
|
-
) {
|
|
2604
|
-
axBlocks.push({ msgIdx: i, blockIdx: j });
|
|
2605
|
-
}
|
|
2606
|
-
}
|
|
2607
|
-
}
|
|
2608
|
-
|
|
2609
|
-
if (axBlocks.length <= MAX_AX_TREES_IN_HISTORY) {
|
|
2610
|
-
return messages;
|
|
2611
|
-
}
|
|
2612
|
-
|
|
2613
|
-
// Build a set of "msgIdx:blockIdx" keys for blocks that should be stripped
|
|
2614
|
-
const toStrip = new Set(
|
|
2615
|
-
axBlocks
|
|
2616
|
-
.slice(0, -MAX_AX_TREES_IN_HISTORY)
|
|
2617
|
-
.map((b) => `${b.msgIdx}:${b.blockIdx}`),
|
|
2618
|
-
);
|
|
2619
|
-
|
|
2620
|
-
return messages.map((msg, idx) => {
|
|
2621
|
-
// Quick check: does this message have any blocks to strip?
|
|
2622
|
-
const hasStripTarget = msg.content.some((_, j) =>
|
|
2623
|
-
toStrip.has(`${idx}:${j}`),
|
|
2624
|
-
);
|
|
2625
|
-
if (!hasStripTarget) return msg;
|
|
2626
|
-
|
|
2627
|
-
return {
|
|
2628
|
-
...msg,
|
|
2629
|
-
content: msg.content.map((block, j) => {
|
|
2630
|
-
if (
|
|
2631
|
-
toStrip.has(`${idx}:${j}`) &&
|
|
2632
|
-
block.type === "tool_result" &&
|
|
2633
|
-
typeof block.content === "string"
|
|
2634
|
-
) {
|
|
2635
|
-
return {
|
|
2636
|
-
...block,
|
|
2637
|
-
content: block.content.replace(
|
|
2638
|
-
AX_TREE_PATTERN,
|
|
2639
|
-
AX_TREE_PLACEHOLDER,
|
|
2640
|
-
),
|
|
2641
|
-
};
|
|
2642
|
-
}
|
|
2643
|
-
return block;
|
|
2644
|
-
}),
|
|
2645
|
-
};
|
|
2646
|
-
});
|
|
2647
|
-
}
|
|
2648
|
-
|
|
2649
|
-
/**
|
|
2650
|
-
* Strip image contentBlocks from all tool_result blocks except those in the
|
|
2651
|
-
* most recent user message that contains tool_result blocks. This prevents
|
|
2652
|
-
* screenshots from accumulating in the context window — each image is seen
|
|
2653
|
-
* once by the LLM on the turn it was captured, then replaced with a text
|
|
2654
|
-
* placeholder on subsequent turns.
|
|
2655
|
-
*
|
|
2656
|
-
* We target the last user message with tool_results (not just the last user
|
|
2657
|
-
* message) because a plain-text user message may follow the tool-result
|
|
2658
|
-
* turn. Using the last user message unconditionally would leave the most
|
|
2659
|
-
* recent tool screenshots unprotected from stripping.
|
|
2660
|
-
*/
|
|
2661
|
-
function stripOldMediaBlocks(history: Message[]): Message[] {
|
|
2662
|
-
// Find the last user message that contains tool_result blocks.
|
|
2663
|
-
let lastToolResultUserIdx = -1;
|
|
2664
|
-
for (let i = history.length - 1; i >= 0; i--) {
|
|
2665
|
-
if (
|
|
2666
|
-
history[i].role === "user" &&
|
|
2667
|
-
history[i].content.some((b) => b.type === "tool_result")
|
|
2668
|
-
) {
|
|
2669
|
-
lastToolResultUserIdx = i;
|
|
2670
|
-
break;
|
|
2671
|
-
}
|
|
2672
|
-
}
|
|
2673
|
-
|
|
2674
|
-
return history.map((msg, idx) => {
|
|
2675
|
-
// Keep the most recent tool-result user message intact (current turn)
|
|
2676
|
-
if (idx === lastToolResultUserIdx || msg.role !== "user") return msg;
|
|
2677
|
-
|
|
2678
|
-
// Check if any tool_result blocks carry embedded media (image or audio).
|
|
2679
|
-
const isMedia = (cb: ContentBlock) =>
|
|
2680
|
-
cb.type === "image" || cb.type === "file";
|
|
2681
|
-
const hasMedia = msg.content.some(
|
|
2682
|
-
(b) =>
|
|
2683
|
-
b.type === "tool_result" &&
|
|
2684
|
-
(b as ToolResultContent).contentBlocks?.some(isMedia),
|
|
2685
|
-
);
|
|
2686
|
-
if (!hasMedia) return msg;
|
|
2687
|
-
|
|
2688
|
-
// Strip media from tool_result blocks, replacing with a text marker. The
|
|
2689
|
-
// model already saw/heard the media in the turn it was captured; resending
|
|
2690
|
-
// the bytes every turn (a 12 MB audio clip isn't optimized like images)
|
|
2691
|
-
// bloats the request until compaction.
|
|
2692
|
-
return {
|
|
2693
|
-
...msg,
|
|
2694
|
-
content: msg.content.map((b) => {
|
|
2695
|
-
if (b.type !== "tool_result") return b;
|
|
2696
|
-
const tr = b as ToolResultContent;
|
|
2697
|
-
if (!tr.contentBlocks?.some(isMedia)) return b;
|
|
2698
|
-
return {
|
|
2699
|
-
...tr,
|
|
2700
|
-
contentBlocks: undefined,
|
|
2701
|
-
content:
|
|
2702
|
-
(tr.content || "") +
|
|
2703
|
-
"\n[Media (image/audio) was captured and shown previously — binary data removed to save context.]",
|
|
2704
|
-
};
|
|
2705
|
-
}),
|
|
2706
|
-
};
|
|
2707
|
-
});
|
|
2708
|
-
}
|
|
2709
|
-
|
|
2710
|
-
/**
|
|
2711
|
-
* Sanitize the outbound history immediately before a provider call, bundling
|
|
2712
|
-
* the pre-send transforms the loop applies to every request:
|
|
2713
|
-
* - {@link stripOldMediaBlocks} drops accumulated screenshot/audio bytes from
|
|
2714
|
-
* older tool results — the model saw the media on the turn it was captured.
|
|
2715
|
-
* - {@link compactAxTreeHistory} collapses all but the most recent few
|
|
2716
|
-
* `<ax-tree>` snapshots so TTFT does not grow linearly with step count.
|
|
2717
|
-
* - {@link stripHistoricalWebSearchResults} converts historical
|
|
2718
|
-
* `web_search_tool_result` blocks to text summaries; Anthropic's opaque
|
|
2719
|
-
* `encrypted_content` tokens expire / are route-scoped, and replaying a stale
|
|
2720
|
-
* one is rejected with `Invalid encrypted_content in search_result block`.
|
|
2721
|
-
*
|
|
2722
|
-
* Transforms the outbound copy only — the durable history keeps the rich
|
|
2723
|
-
* originals and each send re-derives the sanitized projection (every transform
|
|
2724
|
-
* is idempotent). Because it runs unconditionally before every provider call,
|
|
2725
|
-
* it is the single place where oversized media and expired web-search tokens
|
|
2726
|
-
* are guaranteed to be removed from a request.
|
|
2727
|
-
*
|
|
2728
|
-
* This is outbound-request preparation and should eventually move to a default
|
|
2729
|
-
* `pre-model-call` plugin hook ({@link HOOKS.PRE_MODEL_CALL}) once that hook's
|
|
2730
|
-
* context carries the outbound message list; for now it lives inline next to
|
|
2731
|
-
* the provider call it guards.
|
|
2732
|
-
*/
|
|
2733
|
-
export function preModelCallSanitize(history: Message[]): Message[] {
|
|
2734
|
-
const mediaStripped = stripOldMediaBlocks(history);
|
|
2735
|
-
const axCompacted = compactAxTreeHistory(mediaStripped);
|
|
2736
|
-
return stripHistoricalWebSearchResults(axCompacted).messages;
|
|
2737
|
-
}
|
package/src/context/compactor.ts
CHANGED
|
@@ -22,7 +22,6 @@ import { optimizeImageForTransport } from "../agent/image-optimize.js";
|
|
|
22
22
|
import type { CompactionConfig } from "../config/schemas/compaction.js";
|
|
23
23
|
import type { LLMCallSite } from "../config/schemas/llm.js";
|
|
24
24
|
import { filterMessagesForUntrustedActor } from "../daemon/message-provenance.js";
|
|
25
|
-
import { stripHistoricalWebSearchResults } from "../daemon/web-search-history.js";
|
|
26
25
|
import {
|
|
27
26
|
getAttachmentContent,
|
|
28
27
|
getAttachmentMetadataForMessage,
|
|
@@ -40,6 +39,7 @@ import type {
|
|
|
40
39
|
import { type TrustClass } from "../runtime/actor-trust-resolver.js";
|
|
41
40
|
import { resolveCapabilities } from "../runtime/capabilities.js";
|
|
42
41
|
import { getLogger } from "../util/logger.js";
|
|
42
|
+
import { preModelCallSanitize } from "./outbound-sanitize.js";
|
|
43
43
|
import { stripInjectionsForCompaction } from "./strip-injections.js";
|
|
44
44
|
import {
|
|
45
45
|
estimatePromptTokens,
|
|
@@ -826,14 +826,20 @@ function extractTextFromResponse(content: ContentBlock[]): string {
|
|
|
826
826
|
.join("\n");
|
|
827
827
|
}
|
|
828
828
|
|
|
829
|
-
// Build the outbound message list for a compaction provider call:
|
|
830
|
-
//
|
|
831
|
-
//
|
|
829
|
+
// Build the outbound message list for a compaction provider call: apply the
|
|
830
|
+
// same pre-send sanitization bundle as the agent loop's model calls
|
|
831
|
+
// (`preModelCallSanitize` — old tool-result media stripped, AX trees
|
|
832
|
+
// collapsed, historical web-search results converted to text), then append
|
|
833
|
+
// the summarization instruction at the tail.
|
|
832
834
|
//
|
|
833
|
-
//
|
|
834
|
-
//
|
|
835
|
-
//
|
|
836
|
-
//
|
|
835
|
+
// Matching the loop's projection matters for two reasons. First, the summary
|
|
836
|
+
// call's prefix stays byte-aligned with the agent's warm prompt cache — an
|
|
837
|
+
// unsanitized history diverges from what the loop actually sent at the first
|
|
838
|
+
// stripped block. Second, an unsanitized history carries every screenshot in
|
|
839
|
+
// the conversation; enough images cross Anthropic's many-image threshold,
|
|
840
|
+
// where a stricter per-image dimension cap applies and a single large
|
|
841
|
+
// screenshot rejects the whole summary call. Sanitizing here — the single
|
|
842
|
+
// seam every compaction provider call funnels through — covers both the
|
|
837
843
|
// assistant-driven and emergency summarization calls. Only this outbound copy
|
|
838
844
|
// is sanitized; tail resolution and the persisted compaction result read the
|
|
839
845
|
// caller's original messages, so durable history keeps the rich blocks. The
|
|
@@ -842,7 +848,7 @@ function buildCompactionRequest(
|
|
|
842
848
|
history: Message[],
|
|
843
849
|
instruction: Message,
|
|
844
850
|
): Message[] {
|
|
845
|
-
return [...
|
|
851
|
+
return [...preModelCallSanitize(history), instruction];
|
|
846
852
|
}
|
|
847
853
|
|
|
848
854
|
// Token headroom a compaction summary call reserves on top of its history: room
|
|
@@ -945,11 +951,14 @@ export async function runAssistantDrivenCompaction(
|
|
|
945
951
|
// Bound the summary call's own input to the context window. With no tool
|
|
946
952
|
// pair to anchor an emergency split, an overflow recovery routes the full
|
|
947
953
|
// history straight here, so the summary call must front-truncate itself or
|
|
948
|
-
// it overflows in turn.
|
|
949
|
-
//
|
|
950
|
-
//
|
|
954
|
+
// it overflows in turn. Truncation operates on the sanitized projection
|
|
955
|
+
// (what the request actually carries) so the budget estimate is honest —
|
|
956
|
+
// estimating on raw history would count media bytes the request strips.
|
|
957
|
+
// `args.messages` stays intact for tail resolution below — only the
|
|
958
|
+
// outbound request is truncated. A below-budget history is returned
|
|
959
|
+
// untouched, keeping the prefix aligned with the agent's warm cache.
|
|
951
960
|
const summaryHistory = truncateHistoryToBudget({
|
|
952
|
-
messages: args.messages,
|
|
961
|
+
messages: preModelCallSanitize(args.messages),
|
|
953
962
|
systemPrompt: args.systemPrompt,
|
|
954
963
|
budgetTokens: compactionPrefixBudget(args.maxInputTokens),
|
|
955
964
|
providerName: args.provider.tokenEstimationProvider ?? args.provider.name,
|
|
@@ -1371,9 +1380,14 @@ export async function runEmergencyCompaction(
|
|
|
1371
1380
|
);
|
|
1372
1381
|
// Bound the prefix to the context window so the summary call fits, reserving
|
|
1373
1382
|
// budget for the instruction message and the emitted summary. Truncates from
|
|
1374
|
-
// the front, keeping the recent portion the summary most needs.
|
|
1383
|
+
// the front, keeping the recent portion the summary most needs. The prefix
|
|
1384
|
+
// is sliced from the sanitized projection (sanitize-then-slice, matching the
|
|
1385
|
+
// agent's own sends byte-for-byte for cache alignment) so the budget
|
|
1386
|
+
// estimate counts what the request actually carries. All sanitize
|
|
1387
|
+
// transforms are 1:1 per message, so `splitIndex` maps onto the sanitized
|
|
1388
|
+
// array unchanged.
|
|
1375
1389
|
const prefix = truncateHistoryToBudget({
|
|
1376
|
-
messages: args.messages.slice(0, splitIndex),
|
|
1390
|
+
messages: preModelCallSanitize(args.messages).slice(0, splitIndex),
|
|
1377
1391
|
systemPrompt: args.systemPrompt,
|
|
1378
1392
|
budgetTokens: compactionPrefixBudget(args.maxInputTokens),
|
|
1379
1393
|
providerName: args.provider.tokenEstimationProvider ?? args.provider.name,
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Outbound-request history sanitization shared by every provider call that
|
|
3
|
+
* sends conversation history: the agent loop's model calls and the
|
|
4
|
+
* compactor's summary calls. Each transform derives a sanitized projection of
|
|
5
|
+
* the outbound copy only — durable history keeps the rich originals, and every
|
|
6
|
+
* transform is idempotent so each send re-derives the same projection.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { stripHistoricalWebSearchResults } from "../daemon/web-search-history.js";
|
|
10
|
+
import type {
|
|
11
|
+
ContentBlock,
|
|
12
|
+
Message,
|
|
13
|
+
ToolResultContent,
|
|
14
|
+
} from "../providers/types.js";
|
|
15
|
+
|
|
16
|
+
/** Number of most-recent AX tree snapshots to keep in conversation history. */
|
|
17
|
+
const MAX_AX_TREES_IN_HISTORY = 2;
|
|
18
|
+
|
|
19
|
+
/** Regex that matches the `<ax-tree>...</ax-tree>` markers. */
|
|
20
|
+
const AX_TREE_PATTERN = /<ax-tree>[\s\S]*?<\/ax-tree>/g;
|
|
21
|
+
const AX_TREE_PLACEHOLDER = "<ax_tree_omitted />";
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Escapes any literal `</ax-tree>` occurrences inside AX tree content so
|
|
25
|
+
* that the non-greedy compaction regex (`AX_TREE_PATTERN`) does not stop
|
|
26
|
+
* prematurely when the user happens to be viewing XML/HTML source that
|
|
27
|
+
* contains the closing tag. The escaped content does not need to be
|
|
28
|
+
* unescaped because compaction replaces the entire block with a placeholder.
|
|
29
|
+
*/
|
|
30
|
+
export function escapeAxTreeContent(content: string): string {
|
|
31
|
+
return content.replace(/<\/ax-tree>/gi, "</ax-tree>");
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Returns a shallow copy of `messages` where all but the most recent
|
|
36
|
+
* `MAX_AX_TREES_IN_HISTORY` `<ax-tree>` blocks have been replaced with a
|
|
37
|
+
* short placeholder. This keeps the conversation context small so that
|
|
38
|
+
* TTFT does not grow linearly with step count in computer-use sessions.
|
|
39
|
+
*
|
|
40
|
+
* Counting is per-block, not per-message — a single user message can
|
|
41
|
+
* contain multiple tool_result blocks each with their own AX tree snapshot.
|
|
42
|
+
*/
|
|
43
|
+
export function compactAxTreeHistory(messages: Message[]): Message[] {
|
|
44
|
+
// Collect (messageIndex, blockIndex) for every tool_result block with <ax-tree>
|
|
45
|
+
const axBlocks: Array<{ msgIdx: number; blockIdx: number }> = [];
|
|
46
|
+
for (let i = 0; i < messages.length; i++) {
|
|
47
|
+
const msg = messages[i];
|
|
48
|
+
if (msg.role !== "user") {
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
for (let j = 0; j < msg.content.length; j++) {
|
|
52
|
+
const block = msg.content[j];
|
|
53
|
+
if (
|
|
54
|
+
block.type === "tool_result" &&
|
|
55
|
+
typeof block.content === "string" &&
|
|
56
|
+
block.content.includes("<ax-tree>")
|
|
57
|
+
) {
|
|
58
|
+
axBlocks.push({ msgIdx: i, blockIdx: j });
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
if (axBlocks.length <= MAX_AX_TREES_IN_HISTORY) {
|
|
64
|
+
return messages;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// Build a set of "msgIdx:blockIdx" keys for blocks that should be stripped
|
|
68
|
+
const toStrip = new Set(
|
|
69
|
+
axBlocks
|
|
70
|
+
.slice(0, -MAX_AX_TREES_IN_HISTORY)
|
|
71
|
+
.map((b) => `${b.msgIdx}:${b.blockIdx}`),
|
|
72
|
+
);
|
|
73
|
+
|
|
74
|
+
return messages.map((msg, idx) => {
|
|
75
|
+
// Quick check: does this message have any blocks to strip?
|
|
76
|
+
const hasStripTarget = msg.content.some((_, j) =>
|
|
77
|
+
toStrip.has(`${idx}:${j}`),
|
|
78
|
+
);
|
|
79
|
+
if (!hasStripTarget) {
|
|
80
|
+
return msg;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
return {
|
|
84
|
+
...msg,
|
|
85
|
+
content: msg.content.map((block, j) => {
|
|
86
|
+
if (
|
|
87
|
+
toStrip.has(`${idx}:${j}`) &&
|
|
88
|
+
block.type === "tool_result" &&
|
|
89
|
+
typeof block.content === "string"
|
|
90
|
+
) {
|
|
91
|
+
return {
|
|
92
|
+
...block,
|
|
93
|
+
content: block.content.replace(
|
|
94
|
+
AX_TREE_PATTERN,
|
|
95
|
+
AX_TREE_PLACEHOLDER,
|
|
96
|
+
),
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
return block;
|
|
100
|
+
}),
|
|
101
|
+
};
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Strip image contentBlocks from all tool_result blocks except those in the
|
|
107
|
+
* most recent user message that contains tool_result blocks. This prevents
|
|
108
|
+
* screenshots from accumulating in the context window — each image is seen
|
|
109
|
+
* once by the LLM on the turn it was captured, then replaced with a text
|
|
110
|
+
* placeholder on subsequent turns.
|
|
111
|
+
*
|
|
112
|
+
* We target the last user message with tool_results (not just the last user
|
|
113
|
+
* message) because a plain-text user message may follow the tool-result
|
|
114
|
+
* turn. Using the last user message unconditionally would leave the most
|
|
115
|
+
* recent tool screenshots unprotected from stripping.
|
|
116
|
+
*/
|
|
117
|
+
function stripOldMediaBlocks(history: Message[]): Message[] {
|
|
118
|
+
// Find the last user message that contains tool_result blocks.
|
|
119
|
+
let lastToolResultUserIdx = -1;
|
|
120
|
+
for (let i = history.length - 1; i >= 0; i--) {
|
|
121
|
+
if (
|
|
122
|
+
history[i].role === "user" &&
|
|
123
|
+
history[i].content.some((b) => b.type === "tool_result")
|
|
124
|
+
) {
|
|
125
|
+
lastToolResultUserIdx = i;
|
|
126
|
+
break;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
return history.map((msg, idx) => {
|
|
131
|
+
// Keep the most recent tool-result user message intact (current turn)
|
|
132
|
+
if (idx === lastToolResultUserIdx || msg.role !== "user") {
|
|
133
|
+
return msg;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// Check if any tool_result blocks carry embedded media (image or audio).
|
|
137
|
+
const isMedia = (cb: ContentBlock) =>
|
|
138
|
+
cb.type === "image" || cb.type === "file";
|
|
139
|
+
const hasMedia = msg.content.some(
|
|
140
|
+
(b) =>
|
|
141
|
+
b.type === "tool_result" &&
|
|
142
|
+
(b as ToolResultContent).contentBlocks?.some(isMedia),
|
|
143
|
+
);
|
|
144
|
+
if (!hasMedia) {
|
|
145
|
+
return msg;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// Strip media from tool_result blocks, replacing with a text marker. The
|
|
149
|
+
// model already saw/heard the media in the turn it was captured; resending
|
|
150
|
+
// the bytes every turn (a 12 MB audio clip isn't optimized like images)
|
|
151
|
+
// bloats the request until compaction.
|
|
152
|
+
return {
|
|
153
|
+
...msg,
|
|
154
|
+
content: msg.content.map((b) => {
|
|
155
|
+
if (b.type !== "tool_result") {
|
|
156
|
+
return b;
|
|
157
|
+
}
|
|
158
|
+
const tr = b as ToolResultContent;
|
|
159
|
+
if (!tr.contentBlocks?.some(isMedia)) {
|
|
160
|
+
return b;
|
|
161
|
+
}
|
|
162
|
+
return {
|
|
163
|
+
...tr,
|
|
164
|
+
contentBlocks: undefined,
|
|
165
|
+
content:
|
|
166
|
+
(tr.content || "") +
|
|
167
|
+
"\n[Media (image/audio) was captured and shown previously — binary data removed to save context.]",
|
|
168
|
+
};
|
|
169
|
+
}),
|
|
170
|
+
};
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Sanitize the outbound history immediately before a provider call, bundling
|
|
176
|
+
* the pre-send transforms applied to every request that carries conversation
|
|
177
|
+
* history:
|
|
178
|
+
* - {@link stripOldMediaBlocks} drops accumulated screenshot/audio bytes from
|
|
179
|
+
* older tool results — the model saw the media on the turn it was captured.
|
|
180
|
+
* Beyond context bloat, unstripped history can carry enough images to cross
|
|
181
|
+
* Anthropic's many-image threshold, where a stricter per-image dimension
|
|
182
|
+
* cap applies and a single large screenshot rejects the whole request.
|
|
183
|
+
* - {@link compactAxTreeHistory} collapses all but the most recent few
|
|
184
|
+
* `<ax-tree>` snapshots so TTFT does not grow linearly with step count.
|
|
185
|
+
* - {@link stripHistoricalWebSearchResults} converts historical
|
|
186
|
+
* `web_search_tool_result` blocks to text summaries; Anthropic's opaque
|
|
187
|
+
* `encrypted_content` tokens expire / are route-scoped, and replaying a stale
|
|
188
|
+
* one is rejected with `Invalid encrypted_content in search_result block`.
|
|
189
|
+
*
|
|
190
|
+
* Transforms the outbound copy only — the durable history keeps the rich
|
|
191
|
+
* originals and each send re-derives the sanitized projection (every transform
|
|
192
|
+
* is idempotent). Both the agent loop's model calls and the compactor's
|
|
193
|
+
* summary calls funnel through this bundle, so their request prefixes stay
|
|
194
|
+
* byte-aligned (the summary call reuses the agent's warm prompt cache) and
|
|
195
|
+
* oversized media and expired web-search tokens are guaranteed to be removed
|
|
196
|
+
* from every request.
|
|
197
|
+
*/
|
|
198
|
+
export function preModelCallSanitize(history: Message[]): Message[] {
|
|
199
|
+
const mediaStripped = stripOldMediaBlocks(history);
|
|
200
|
+
const axCompacted = compactAxTreeHistory(mediaStripped);
|
|
201
|
+
return stripHistoricalWebSearchResults(axCompacted).messages;
|
|
202
|
+
}
|
|
@@ -16,6 +16,7 @@ import type { AuthContext } from "../../runtime/auth/types.js";
|
|
|
16
16
|
import * as pendingInteractions from "../../runtime/pending-interactions.js";
|
|
17
17
|
import { unwrapExternalContentForDisplay } from "../../security/untrusted-content.js";
|
|
18
18
|
import { getLogger } from "../../util/logger.js";
|
|
19
|
+
import { joinWithSpacing } from "../../util/text-spacing.js";
|
|
19
20
|
import { estimateBase64Bytes } from "../assistant-attachments.js";
|
|
20
21
|
import type { ConversationTransportMetadata } from "../message-protocol.js";
|
|
21
22
|
import type { TrustContext } from "../trust-context.js";
|
|
@@ -330,29 +331,6 @@ export function renderHistoryContent(
|
|
|
330
331
|
const contentBlocks: ConversationContentBlock[] = [];
|
|
331
332
|
let currentTextBlock: { type: "text"; text: string } | null = null;
|
|
332
333
|
|
|
333
|
-
function joinWithSpacing(parts: string[]): string {
|
|
334
|
-
let result = parts[0] ?? "";
|
|
335
|
-
for (let i = 1; i < parts.length; i++) {
|
|
336
|
-
const prev = result[result.length - 1];
|
|
337
|
-
const next = parts[i][0];
|
|
338
|
-
// Only insert a space when neither side already has whitespace
|
|
339
|
-
if (
|
|
340
|
-
prev &&
|
|
341
|
-
next &&
|
|
342
|
-
prev !== " " &&
|
|
343
|
-
prev !== "\n" &&
|
|
344
|
-
prev !== "\t" &&
|
|
345
|
-
next !== " " &&
|
|
346
|
-
next !== "\n" &&
|
|
347
|
-
next !== "\t"
|
|
348
|
-
) {
|
|
349
|
-
result += " ";
|
|
350
|
-
}
|
|
351
|
-
result += parts[i];
|
|
352
|
-
}
|
|
353
|
-
return result;
|
|
354
|
-
}
|
|
355
|
-
|
|
356
334
|
function finalizeSegment(): void {
|
|
357
335
|
if (hasOpenSegment) {
|
|
358
336
|
const joined = joinWithSpacing(currentSegmentParts);
|
|
@@ -16,8 +16,8 @@
|
|
|
16
16
|
|
|
17
17
|
import { v4 as uuid } from "uuid";
|
|
18
18
|
|
|
19
|
-
import { escapeAxTreeContent } from "../agent/loop.js";
|
|
20
19
|
import { loadConfig } from "../config/loader.js";
|
|
20
|
+
import { escapeAxTreeContent } from "../context/outbound-sanitize.js";
|
|
21
21
|
import type { ContentBlock } from "../providers/types.js";
|
|
22
22
|
import {
|
|
23
23
|
assistantEventHub,
|
|
@@ -280,3 +280,82 @@ describe("sendSlackReply post path", () => {
|
|
|
280
280
|
expect(callSlackApiMock).toHaveBeenCalledTimes(1);
|
|
281
281
|
});
|
|
282
282
|
});
|
|
283
|
+
|
|
284
|
+
describe("sendSlackReply approval fallback", () => {
|
|
285
|
+
const approval = {
|
|
286
|
+
requestId: "req-123",
|
|
287
|
+
actions: [
|
|
288
|
+
{ id: "approve_once", label: "Approve once" },
|
|
289
|
+
{ id: "reject", label: "Reject" },
|
|
290
|
+
],
|
|
291
|
+
plainTextFallback: 'Reply "ABC123 approve" or "ABC123 reject"',
|
|
292
|
+
};
|
|
293
|
+
const blocks: KnownBlock[] = [
|
|
294
|
+
{ type: "section", text: { type: "mrkdwn", text: "Approve tool: bash" } },
|
|
295
|
+
];
|
|
296
|
+
|
|
297
|
+
beforeEach(() => {
|
|
298
|
+
callSlackApiMock.mockReset();
|
|
299
|
+
callSlackApiMock.mockImplementation(async () => ({ ok: true }));
|
|
300
|
+
});
|
|
301
|
+
|
|
302
|
+
test("block-free retry re-attaches plain-text reply instructions", async () => {
|
|
303
|
+
// Dropping an approval's blocks drops its buttons — the retry text must
|
|
304
|
+
// carry the reply instructions so the recipient can still act.
|
|
305
|
+
callSlackApiMock
|
|
306
|
+
.mockImplementationOnce(async () => {
|
|
307
|
+
throw new SlackApiError("invalid_blocks");
|
|
308
|
+
})
|
|
309
|
+
.mockImplementationOnce(async () => ({
|
|
310
|
+
ok: true,
|
|
311
|
+
ts: "1700000000.000400",
|
|
312
|
+
}));
|
|
313
|
+
|
|
314
|
+
const result = await sendSlackReply("C123", "Approve tool: bash", {
|
|
315
|
+
blocks,
|
|
316
|
+
approval,
|
|
317
|
+
});
|
|
318
|
+
|
|
319
|
+
expect(result).toEqual({ ok: true, ts: "1700000000.000400" });
|
|
320
|
+
expect(callSlackApiMock).toHaveBeenCalledTimes(2);
|
|
321
|
+
expect(callSlackApiMock).toHaveBeenNthCalledWith(2, "chat.postMessage", {
|
|
322
|
+
channel: "C123",
|
|
323
|
+
text: 'Approve tool: bash\n\nReply "ABC123 approve" or "ABC123 reject"',
|
|
324
|
+
});
|
|
325
|
+
});
|
|
326
|
+
|
|
327
|
+
test("retry text is unchanged when it already contains the instructions", async () => {
|
|
328
|
+
callSlackApiMock
|
|
329
|
+
.mockImplementationOnce(async () => {
|
|
330
|
+
throw new SlackApiError("msg_blocks_too_long");
|
|
331
|
+
})
|
|
332
|
+
.mockImplementationOnce(async () => ({
|
|
333
|
+
ok: true,
|
|
334
|
+
ts: "1700000000.000500",
|
|
335
|
+
}));
|
|
336
|
+
|
|
337
|
+
const text = `Approve tool: bash\n\n${approval.plainTextFallback}`;
|
|
338
|
+
await sendSlackReply("C123", text, { blocks, approval });
|
|
339
|
+
|
|
340
|
+
expect(callSlackApiMock).toHaveBeenNthCalledWith(2, "chat.postMessage", {
|
|
341
|
+
channel: "C123",
|
|
342
|
+
text,
|
|
343
|
+
});
|
|
344
|
+
});
|
|
345
|
+
|
|
346
|
+
test("approval without usable instructions is never retried bare", async () => {
|
|
347
|
+
// A block-free approval with no reply instructions gives the recipient no
|
|
348
|
+
// way to respond — fail the delivery instead so it surfaces as an error.
|
|
349
|
+
callSlackApiMock.mockImplementationOnce(async () => {
|
|
350
|
+
throw new SlackApiError("invalid_blocks");
|
|
351
|
+
});
|
|
352
|
+
|
|
353
|
+
await expect(
|
|
354
|
+
sendSlackReply("C123", "Approve tool: bash", {
|
|
355
|
+
blocks,
|
|
356
|
+
approval: { ...approval, plainTextFallback: " " },
|
|
357
|
+
}),
|
|
358
|
+
).rejects.toThrow();
|
|
359
|
+
expect(callSlackApiMock).toHaveBeenCalledTimes(1);
|
|
360
|
+
});
|
|
361
|
+
});
|