@oh-my-pi/pi-agent-core 18.5.1 → 18.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -2,6 +2,18 @@
2
2
 
3
3
  ## [Unreleased]
4
4
 
5
+ ## [18.6.1] - 2026-10-04
6
+
7
+ ### Fixed
8
+
9
+ - Fixed native OpenAI context compaction for sessions containing many screenshots, preventing image-size estimates from incorrectly causing compaction requests to be rejected.
10
+
11
+ ## [18.6.0] - 2026-10-03
12
+
13
+ ### Fixed
14
+
15
+ - When a DeepSeek turn ends with raw `<|DSML|…>` tool-call text that could not be parsed into a real call, and no tool call was made, the broken markup is now removed from the message before it is saved to history. The agent then tells the model the tool call failed, shows it the correct format, and asks it again, at most twice in a row. Previously the markup stayed in history, and the agent stopped as if the model had finished ([#14202](https://github.com/can1357/oh-my-pi/pull/14202) by [@H4vC](https://github.com/H4vC)).
16
+
5
17
  ## [18.5.1] - 2026-10-03
6
18
 
7
19
  ### Added
@@ -33,14 +33,20 @@ export interface TrimRemoteCompactionInputResult {
33
33
  rewrittenOutputs: number;
34
34
  estimatedTokensBefore: number;
35
35
  estimatedTokensAfter: number;
36
- /** Whether `input` fits the model window; false means it must not be sent. */
36
+ /**
37
+ * Whether `input` may be sent; false means it must not be. True when the
38
+ * full estimate fits, or when only the image estimate pushes it over (image
39
+ * pricing is a guess; the provider's own accounting decides).
40
+ */
37
41
  fits: boolean;
38
42
  }
39
43
  /**
40
44
  * Preserve the full native transcript unless trailing tool outputs alone push a
41
45
  * remote compaction request beyond the model window. Replacing only those
42
46
  * outputs keeps call/result pairing and all earlier assistant/reasoning history,
43
- * matching Codex's recovery path for oversized tool turns.
47
+ * matching Codex's recovery path for oversized tool turns. A request whose text
48
+ * fits but whose image estimate does not is still sendable: refusing it on an
49
+ * image guess would skip native compaction the provider would have accepted.
44
50
  */
45
51
  export declare function trimRemoteCompactionInputToContextWindow(input: Array<Record<string, unknown>>, tokenizer: Tokenizer, contextWindow: number | null | undefined, instructions: string, tools?: unknown[]): TrimRemoteCompactionInputResult;
46
52
  /**
@@ -0,0 +1,30 @@
1
+ import type { ImageContent } from "@oh-my-pi/pi-ai";
2
+ /**
3
+ * Dimension-based image token estimates shared by the local context counter
4
+ * ({@link Tokenizer.countMessage}) and the native remote-compaction fit probe.
5
+ * Both sides must price the same image the same way: when they disagreed
6
+ * (1.2k flat locally vs a 12k flat worst case remotely) an image-heavy session
7
+ * stayed under the local compaction trigger yet was refused as over-window by
8
+ * the remote probe.
9
+ *
10
+ * Follows OpenAI's patch-based rule (32px patches, a per-detail pixel limit
11
+ * and patch budget, x1.2 multiplier), which is also a close upper estimate for
12
+ * other providers once omp has downscaled the image (≤1568px by default).
13
+ */
14
+ /** OpenAI image `detail` values; `undefined` means the provider default (`auto`). */
15
+ export type ImageDetail = ImageContent["detail"];
16
+ export interface ImageSize {
17
+ width: number;
18
+ height: number;
19
+ }
20
+ /**
21
+ * Estimated input tokens for one image. Unknown dimensions (undecodable data,
22
+ * remote URLs, provider file ids) charge the detail level's full patch budget.
23
+ */
24
+ export declare function estimateImageTokens(size: ImageSize | null | undefined, detail?: ImageDetail): number;
25
+ /** Pixel size read from the header of base64-encoded image data, or null. */
26
+ export declare function base64ImageSize(base64: string): ImageSize | null;
27
+ /** Pixel size of a `data:<mime>;base64,` URL, or null for any other URL. */
28
+ export declare function dataUrlImageSize(url: string): ImageSize | null;
29
+ /** {@link estimateImageTokens} for an in-memory image block. */
30
+ export declare function estimateImageContentTokens(image: Pick<ImageContent, "data" | "detail">): number;
@@ -70,7 +70,8 @@ export declare class Tokenizer {
70
70
  * Settled historical messages are counted once and reused until an owner
71
71
  * (prune/shake/strip-images) calls `invalidateMessageCache`; streaming
72
72
  * assistants bypass the memo entirely (see the message-cache settle-gate
73
- * invariant). Image blocks charge a fixed per-image estimate.
73
+ * invariant). Image blocks charge {@link estimateImageContentTokens}, the
74
+ * same dimension-based estimate the native remote-compaction probe applies.
74
75
  */
75
76
  countMessage(message: AgentMessage, options?: MessageCountOptions): number;
76
77
  /** Sum of {@link countMessage} over `messages`. */
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "type": "module",
3
3
  "name": "@oh-my-pi/pi-agent-core",
4
- "version": "18.5.1",
4
+ "version": "18.6.1",
5
5
  "description": "General-purpose agent with transport abstraction, state management, and attachment support",
6
6
  "homepage": "https://omp.sh",
7
7
  "author": {
@@ -38,16 +38,16 @@
38
38
  "fmt": "oxfmt --no-error-on-unmatched-pattern 'src/**/*.{ts,tsx}' '{test,bench,examples,scripts}/**/*.ts' '*.ts'"
39
39
  },
40
40
  "dependencies": {
41
- "@oh-my-pi/pi-ai": "18.5.1",
42
- "@oh-my-pi/pi-catalog": "18.5.1",
43
- "@oh-my-pi/pi-natives": "18.5.1",
44
- "@oh-my-pi/pi-utils": "18.5.1",
45
- "@oh-my-pi/pi-wire": "18.5.1",
46
- "@oh-my-pi/snapcompact": "18.5.1",
41
+ "@oh-my-pi/pi-ai": "18.6.1",
42
+ "@oh-my-pi/pi-catalog": "18.6.1",
43
+ "@oh-my-pi/pi-natives": "18.6.1",
44
+ "@oh-my-pi/pi-utils": "18.6.1",
45
+ "@oh-my-pi/pi-wire": "18.6.1",
46
+ "@oh-my-pi/snapcompact": "18.6.1",
47
47
  "@opentelemetry/api": "^1.9.1"
48
48
  },
49
49
  "devDependencies": {
50
- "@oh-my-pi/omptype": "18.5.1",
50
+ "@oh-my-pi/omptype": "18.6.1",
51
51
  "@opentelemetry/context-async-hooks": "^2.9.0",
52
52
  "@opentelemetry/sdk-trace-base": "^2.9.0",
53
53
  "@types/bun": "^1.3.14"
package/src/agent-loop.ts CHANGED
@@ -53,6 +53,7 @@ import {
53
53
  recoverHarmonyToolCall,
54
54
  signalListLabel,
55
55
  } from "@oh-my-pi/pi-ai/utils/harmony-leak";
56
+ import { isDsmlLeakRecoveryTarget, removeDsmlToolMarkupLeak } from "@oh-my-pi/pi-ai/utils/dsml-leak";
56
57
  import { cloneJsonTree, logger, sanitizeText, structuredCloneJSON } from "@oh-my-pi/pi-utils";
57
58
  import { INTENT_FIELD } from "@oh-my-pi/pi-wire";
58
59
  import { LiveSteeringChannel } from "./live-steering";
@@ -76,6 +77,7 @@ import {
76
77
  startInvokeAgentSpan,
77
78
  } from "./telemetry";
78
79
  import { createAdditionalContextMessage, isNonBlankContext, joinAdditionalContext } from "./tool-context";
80
+ import dsmlToolCallLeakPrompt from "./prompts/dsml-tool-call-leak.md" with { type: "text" };
79
81
  import type {
80
82
  AgentContext,
81
83
  AgentEvent,
@@ -116,6 +118,14 @@ const ABORTED: unique symbol = Symbol("agent-loop-aborted");
116
118
  */
117
119
  const MAX_PAUSED_TURN_CONTINUATIONS = 8;
118
120
 
121
+ /**
122
+ * Cap on consecutive "tool call failed" nudges for a turn that ended with raw
123
+ * DeepSeek DSML tool-call markup in its text and no structured call. A model
124
+ * that keeps leaking after this many corrections is left to yield. Resets
125
+ * whenever a turn carries tool calls.
126
+ */
127
+ const MAX_DSML_LEAK_NUDGES = 2;
128
+
119
129
  /**
120
130
  * Cap on consecutive forced escalations for a single soft tool requirement.
121
131
  * A forced `toolChoice` guarantees the call, so this is purely defensive: if a
@@ -1222,6 +1232,7 @@ async function runLoopBody(
1222
1232
  let harmonyRetryAttempt = 0;
1223
1233
  let harmonyTruncateResumeCount = 0;
1224
1234
  let pausedTurnContinuations = 0;
1235
+ let dsmlLeakNudges = 0;
1225
1236
 
1226
1237
  // Soft tool requirement lifecycle (reminder then escalation; see SoftToolRequirement).
1227
1238
  // The host-owned state survives only a gate stop between Agent.prompt calls.
@@ -1685,6 +1696,7 @@ async function runLoopBody(
1685
1696
 
1686
1697
  if (toolCalls.length > 0) {
1687
1698
  pausedTurnContinuations = 0;
1699
+ dsmlLeakNudges = 0;
1688
1700
  } else if (
1689
1701
  !hasMoreToolCalls &&
1690
1702
  message.stopReason === "stop" &&
@@ -1698,6 +1710,25 @@ async function runLoopBody(
1698
1710
  // mid-work turn.
1699
1711
  pausedTurnContinuations++;
1700
1712
  hasMoreToolCalls = true;
1713
+ } else if (
1714
+ !hasMoreToolCalls &&
1715
+ message.stopReason === "stop" &&
1716
+ dsmlLeakMessages.has(message) &&
1717
+ dsmlLeakNudges < MAX_DSML_LEAK_NUDGES
1718
+ ) {
1719
+ // DeepSeek wrote its tool call as raw DSML text the healer could not
1720
+ // recover (often missing the opening envelope). The markup was
1721
+ // stripped before commit; without a nudge the loop would yield as if
1722
+ // the model were done, so tell it the call failed and re-sample.
1723
+ const nudge = injectExecutionAdditionalContext(
1724
+ currentContext,
1725
+ newMessages,
1726
+ stream,
1727
+ dsmlToolCallLeakPrompt,
1728
+ );
1729
+ if (nudge) additionalMessages.push(nudge);
1730
+ dsmlLeakNudges++;
1731
+ hasMoreToolCalls = true;
1701
1732
  }
1702
1733
 
1703
1734
  await emitTurnEnd(stream, currentContext, message, toolResults, config, signal, {
@@ -1967,6 +1998,7 @@ async function streamAssistantResponse(
1967
1998
  // per model without touching the shared session `serviceTier`.
1968
1999
  const effectiveServiceTier = config.getServiceTier ? config.getServiceTier(model) : config.serviceTier;
1969
2000
  const harmonyMitigationEnabled = isHarmonyLeakMitigationTarget(model);
2001
+ const dsmlLeakRecoveryEnabled = isDsmlLeakRecoveryTarget(model);
1970
2002
  const harmonyAbortController = harmonyMitigationEnabled ? new AbortController() : undefined;
1971
2003
  const requestSignal = harmonyAbortController
1972
2004
  ? signal
@@ -2196,6 +2228,18 @@ async function streamAssistantResponse(
2196
2228
  }
2197
2229
  }
2198
2230
  finalMessage = snapshotAssistantMessage(finalMessage);
2231
+ // Unhealed DSML tool-call markup must never reach history: replaying
2232
+ // it teaches the model to keep writing calls as text (#10556). Strip
2233
+ // it before the context, the UI, and the session see the message.
2234
+ // Only DSML-speaking models qualify; from others it is prose.
2235
+ if (
2236
+ dsmlLeakRecoveryEnabled &&
2237
+ finalMessage.stopReason === "stop" &&
2238
+ !finalMessage.content.some(block => block.type === "toolCall") &&
2239
+ removeDsmlToolMarkupLeak(finalMessage)
2240
+ ) {
2241
+ dsmlLeakMessages.add(finalMessage);
2242
+ }
2199
2243
  // Expand inline macros (and any other registered rewrite) on the
2200
2244
  // finalized message before it reaches the context, the UI, or tool
2201
2245
  // dispatch — so a single mutation is the source of truth for all three.
@@ -2752,6 +2796,9 @@ interface PreparedToolCall {
2752
2796
  prepareError?: unknown;
2753
2797
  }
2754
2798
 
2799
+ /** Committed assistant messages whose unhealed DSML tool-call markup was stripped; the loop answers them with a failure nudge. */
2800
+ const dsmlLeakMessages = new WeakSet<AssistantMessage>();
2801
+
2755
2802
  /**
2756
2803
  * Prepare results computed in the stream-done branch (before `message_start`/
2757
2804
  * `message_end`) so a `beforeToolCall` args revision is baked into the message
@@ -53,6 +53,7 @@ import {
53
53
  OPENAI_HEADERS,
54
54
  } from "@oh-my-pi/pi-catalog/wire/codex";
55
55
  import { $env, isRecord, logger, prompt, ptree, stringifyJson, structuredCloneJSON } from "@oh-my-pi/pi-utils";
56
+ import { dataUrlImageSize, estimateImageTokens } from "../image-tokens";
56
57
  import { Tokenizer } from "../tokenizer";
57
58
  import { appendAzureApiVersion, resolveAzureOpenAiBaseUrl } from "./azure-openai-endpoint";
58
59
  import { prepareBedrockCompactionRequest } from "./bedrock";
@@ -80,7 +81,8 @@ export const REMOTE_COMPACTION_TIMEOUT_MS = 300_000;
80
81
  export const CONTEXT_WINDOW_TRUNCATED_OUTPUT_MESSAGE = prompt.render(contextWindowTruncatedOutputPrompt);
81
82
 
82
83
  const REMOTE_COMPACTION_REQUEST_OVERHEAD_TOKENS = 256;
83
- const REMOTE_COMPACTION_IMAGE_TOKEN_ESTIMATE = 12_000;
84
+ /** Per-`input_image` record estimates; trim re-probes share untouched items by reference. */
85
+ const remoteImageTokenCache = new WeakMap<Record<string, unknown>, number>();
84
86
  const TOOL_RESULT_IMAGE_ATTACHMENT_TEXT = "Attached image(s) from tool result:";
85
87
 
86
88
  interface NormalizedEstimateValue {
@@ -103,10 +105,16 @@ function normalizeRemoteCompactionEstimateValue(value: unknown): NormalizedEstim
103
105
 
104
106
  const record = value as Record<string, unknown>;
105
107
  if (record.type === "input_image") {
106
- return {
107
- value: { ...record, image_url: "<image>" },
108
- imageTokens: REMOTE_COMPACTION_IMAGE_TOKEN_ESTIMATE,
109
- };
108
+ let imageTokens = remoteImageTokenCache.get(record);
109
+ if (imageTokens === undefined) {
110
+ const detail = record.detail;
111
+ imageTokens = estimateImageTokens(
112
+ typeof record.image_url === "string" ? dataUrlImageSize(record.image_url) : null,
113
+ detail === "low" || detail === "high" || detail === "original" || detail === "auto" ? detail : undefined,
114
+ );
115
+ remoteImageTokenCache.set(record, imageTokens);
116
+ }
117
+ return { value: { ...record, image_url: "<image>" }, imageTokens };
110
118
  }
111
119
 
112
120
  const normalized: Record<string, unknown> = {};
@@ -128,7 +136,11 @@ export interface TrimRemoteCompactionInputResult {
128
136
  rewrittenOutputs: number;
129
137
  estimatedTokensBefore: number;
130
138
  estimatedTokensAfter: number;
131
- /** Whether `input` fits the model window; false means it must not be sent. */
139
+ /**
140
+ * Whether `input` may be sent; false means it must not be. True when the
141
+ * full estimate fits, or when only the image estimate pushes it over (image
142
+ * pricing is a guess; the provider's own accounting decides).
143
+ */
132
144
  fits: boolean;
133
145
  }
134
146
 
@@ -138,15 +150,17 @@ interface RemoteCompactionBudgetProbe {
138
150
  tokens: number;
139
151
  /** Whether the request fits the window. Always true when no window is known. */
140
152
  fits: boolean;
153
+ /** Whether the request minus its image estimate fits the window. */
154
+ textFits: boolean;
141
155
  }
142
156
 
143
157
  /**
144
- * Cheap-first sizing of a remote-compaction request. Images and the request
145
- * frame are charged flat, so they come off the budget rather than through the
146
- * tokenizer; opaque `encrypted_content` payloads are excluded. The serialized
147
- * transcript is then probed with {@link Tokenizer.checkTokenBudget}, which only
148
- * pays for an exact count when the byte bound cannot already prove the request
149
- * fits.
158
+ * Cheap-first sizing of a remote-compaction request. Images (priced from their
159
+ * dimensions) and the request frame come off the budget rather than through
160
+ * the tokenizer; opaque `encrypted_content` payloads are excluded. The
161
+ * serialized transcript is then probed with {@link Tokenizer.checkTokenBudget},
162
+ * which only pays for an exact count when the byte bound cannot already prove
163
+ * the request fits.
150
164
  */
151
165
  function probeRemoteCompactionInputBudget(
152
166
  input: Array<Record<string, unknown>>,
@@ -159,10 +173,15 @@ function probeRemoteCompactionInputBudget(
159
173
  const serialized = stringifyJson(normalized.value) ?? "";
160
174
  const flatTokens = normalized.imageTokens + REMOTE_COMPACTION_REQUEST_OVERHEAD_TOKENS;
161
175
  if (!contextWindow || contextWindow <= 0) {
162
- return { tokens: tokenizer.countTokens(serialized, "upperbound") + flatTokens, fits: true };
176
+ return { tokens: tokenizer.countTokens(serialized, "upperbound") + flatTokens, fits: true, textFits: true };
163
177
  }
164
178
  const budget = tokenizer.checkTokenBudget(serialized, Math.max(0, contextWindow - flatTokens));
165
- return { tokens: budget.tokens + flatTokens, fits: budget.fits };
179
+ return {
180
+ tokens: budget.tokens + flatTokens,
181
+ fits: budget.fits,
182
+ // A failed check carries the exact text count (or its byte upper bound).
183
+ textFits: budget.fits || budget.tokens + REMOTE_COMPACTION_REQUEST_OVERHEAD_TOKENS <= contextWindow,
184
+ };
166
185
  }
167
186
 
168
187
  function rewriteToolOutputForContextWindow(item: Record<string, unknown>): Record<string, unknown> | undefined {
@@ -192,7 +211,9 @@ function isToolResultImageAttachment(item: Record<string, unknown>): boolean {
192
211
  * Preserve the full native transcript unless trailing tool outputs alone push a
193
212
  * remote compaction request beyond the model window. Replacing only those
194
213
  * outputs keeps call/result pairing and all earlier assistant/reasoning history,
195
- * matching Codex's recovery path for oversized tool turns.
214
+ * matching Codex's recovery path for oversized tool turns. A request whose text
215
+ * fits but whose image estimate does not is still sendable: refusing it on an
216
+ * image guess would skip native compaction the provider would have accepted.
196
217
  */
197
218
  export function trimRemoteCompactionInputToContextWindow(
198
219
  input: Array<Record<string, unknown>>,
@@ -226,22 +247,25 @@ export function trimRemoteCompactionInputToContextWindow(
226
247
  after = probeRemoteCompactionInputBudget(rewrittenInput, tokenizer, instructions, tools, contextWindow);
227
248
  }
228
249
 
229
- if (!rewrittenInput || !after.fits) {
250
+ // Rewrites that reach a full fit are kept. When they cannot, and the text
251
+ // already fit, the overflow is image-only: rewriting cannot shrink the image
252
+ // estimate, so send the untouched input rather than discard tool output.
253
+ if (rewrittenInput && (after.fits || (!before.textFits && after.textFits))) {
230
254
  return {
231
- input,
232
- rewrittenOutputs: 0,
255
+ input: rewrittenInput,
256
+ rewrittenOutputs,
233
257
  estimatedTokensBefore: before.tokens,
234
- estimatedTokensAfter: before.tokens,
235
- fits: false,
258
+ estimatedTokensAfter: after.tokens,
259
+ fits: true,
236
260
  };
237
261
  }
238
262
 
239
263
  return {
240
- input: rewrittenInput,
241
- rewrittenOutputs,
264
+ input,
265
+ rewrittenOutputs: 0,
242
266
  estimatedTokensBefore: before.tokens,
243
- estimatedTokensAfter: after.tokens,
244
- fits: true,
267
+ estimatedTokensAfter: before.tokens,
268
+ fits: before.textFits,
245
269
  };
246
270
  }
247
271
 
@@ -0,0 +1,106 @@
1
+ import type { ImageContent } from "@oh-my-pi/pi-ai";
2
+ import { parseImageMetadata } from "@oh-my-pi/pi-utils";
3
+
4
+ /**
5
+ * Dimension-based image token estimates shared by the local context counter
6
+ * ({@link Tokenizer.countMessage}) and the native remote-compaction fit probe.
7
+ * Both sides must price the same image the same way: when they disagreed
8
+ * (1.2k flat locally vs a 12k flat worst case remotely) an image-heavy session
9
+ * stayed under the local compaction trigger yet was refused as over-window by
10
+ * the remote probe.
11
+ *
12
+ * Follows OpenAI's patch-based rule (32px patches, a per-detail pixel limit
13
+ * and patch budget, x1.2 multiplier), which is also a close upper estimate for
14
+ * other providers once omp has downscaled the image (≤1568px by default).
15
+ */
16
+
17
+ /** OpenAI image `detail` values; `undefined` means the provider default (`auto`). */
18
+ export type ImageDetail = ImageContent["detail"];
19
+
20
+ export interface ImageSize {
21
+ width: number;
22
+ height: number;
23
+ }
24
+
25
+ const PATCH_PX = 32;
26
+ const PATCH_TOKEN_MULTIPLIER = 1.2;
27
+
28
+ interface DetailSizing {
29
+ /** Longest side after the pixel-dimension fit. */
30
+ maxDimension: number;
31
+ /** Resizing patch budget, when the detail level defines one. */
32
+ patchBudget?: number;
33
+ /** Patches charged when the image size is unknown. */
34
+ unknownPatches: number;
35
+ }
36
+
37
+ const LOW_SIZING: DetailSizing = { maxDimension: 512, unknownPatches: 16 * 16 };
38
+ const HIGH_SIZING: DetailSizing = { maxDimension: 2048, patchBudget: 2_500, unknownPatches: 2_500 };
39
+ // `auto` resolves to `original` sizing on current models, so it shares the
40
+ // larger budget rather than risk undercounting.
41
+ const ORIGINAL_SIZING: DetailSizing = { maxDimension: 6_000, patchBudget: 10_000, unknownPatches: 10_000 };
42
+
43
+ function sizingFor(detail: ImageDetail): DetailSizing {
44
+ if (detail === "low") return LOW_SIZING;
45
+ if (detail === "high") return HIGH_SIZING;
46
+ return ORIGINAL_SIZING;
47
+ }
48
+
49
+ function countPatches(size: ImageSize, sizing: DetailSizing): number {
50
+ let { width, height } = size;
51
+ const longest = Math.max(width, height);
52
+ if (longest > sizing.maxDimension) {
53
+ const scale = sizing.maxDimension / longest;
54
+ width = Math.max(1, Math.round(width * scale));
55
+ height = Math.max(1, Math.round(height * scale));
56
+ }
57
+ const patches = Math.ceil(width / PATCH_PX) * Math.ceil(height / PATCH_PX);
58
+ const budget = sizing.patchBudget;
59
+ if (budget === undefined || patches <= budget) return patches;
60
+
61
+ const shrink = Math.sqrt((PATCH_PX * PATCH_PX * budget) / (width * height));
62
+ const scaledW = (width * shrink) / PATCH_PX;
63
+ const scaledH = (height * shrink) / PATCH_PX;
64
+ const adjusted = shrink * Math.min(Math.floor(scaledW) / scaledW, Math.floor(scaledH) / scaledH);
65
+ const resizedW = Math.floor(width * adjusted);
66
+ const resizedH = Math.floor(height * adjusted);
67
+ if (resizedW <= 0 || resizedH <= 0) return budget;
68
+ return Math.min(budget, Math.ceil(resizedW / PATCH_PX) * Math.ceil(resizedH / PATCH_PX));
69
+ }
70
+
71
+ /**
72
+ * Estimated input tokens for one image. Unknown dimensions (undecodable data,
73
+ * remote URLs, provider file ids) charge the detail level's full patch budget.
74
+ */
75
+ export function estimateImageTokens(size: ImageSize | null | undefined, detail?: ImageDetail): number {
76
+ const sizing = sizingFor(detail);
77
+ const patches = size && size.width > 0 && size.height > 0 ? countPatches(size, sizing) : sizing.unknownPatches;
78
+ return Math.ceil(patches * PATCH_TOKEN_MULTIPLIER);
79
+ }
80
+
81
+ // Enough decoded bytes to reach a JPEG SOF marker behind typical EXIF/ICC
82
+ // segments; PNG/GIF/WebP need only the first 30 bytes. Multiple of 4 so the
83
+ // base64 slice decodes cleanly.
84
+ const HEADER_BASE64_CHARS = 4 * Math.ceil((64 * 1024) / 3);
85
+
86
+ /** Pixel size read from the header of base64-encoded image data, or null. */
87
+ export function base64ImageSize(base64: string): ImageSize | null {
88
+ if (base64.length === 0) return null;
89
+ const header = Buffer.from(base64.slice(0, HEADER_BASE64_CHARS), "base64");
90
+ const metadata = parseImageMetadata(header);
91
+ if (!metadata?.width || !metadata.height) return null;
92
+ return { width: metadata.width, height: metadata.height };
93
+ }
94
+
95
+ /** Pixel size of a `data:<mime>;base64,` URL, or null for any other URL. */
96
+ export function dataUrlImageSize(url: string): ImageSize | null {
97
+ if (!url.startsWith("data:")) return null;
98
+ const comma = url.indexOf(",");
99
+ if (comma < 0 || !url.slice(0, comma).endsWith(";base64")) return null;
100
+ return base64ImageSize(url.slice(comma + 1));
101
+ }
102
+
103
+ /** {@link estimateImageTokens} for an in-memory image block. */
104
+ export function estimateImageContentTokens(image: Pick<ImageContent, "data" | "detail">): number {
105
+ return estimateImageTokens(base64ImageSize(image.data), image.detail);
106
+ }
@@ -0,0 +1,19 @@
1
+ Tool call failed: your previous response tried to invoke a tool, but the DSML markup was malformed, so it arrived as plain text, no tool was executed, and the broken markup was removed from your response. Usually the opening `<|DSML|tool_calls>` and/or `<|DSML|invoke name="…">` tags are missing, or the tool name was written as a plain line.
2
+
3
+ A tool call must be one complete envelope:
4
+
5
+ ```
6
+ <|DSML|tool_calls>
7
+ <|DSML|invoke name="TOOL_NAME">
8
+ <|DSML|parameter name="ARG_NAME" string="true">text value</|DSML|parameter>
9
+ <|DSML|parameter name="OTHER_ARG" string="false">{"json": "value"}</|DSML|parameter>
10
+ </|DSML|invoke>
11
+ </|DSML|tool_calls>
12
+ ```
13
+
14
+ - `<|DSML|tool_calls>` wraps every call; put one `<|DSML|invoke>` per call inside it.
15
+ - The tool name goes in the `name` attribute of `<|DSML|invoke>`, never on its own line.
16
+ - Use `string="true"` for raw text values and `string="false"` for numbers, booleans, arrays, and objects (JSON).
17
+ - Close every tag.
18
+
19
+ Re-issue the intended call now in this shape, or reply in plain prose if no tool is needed.
package/src/tokenizer.ts CHANGED
@@ -5,6 +5,7 @@ import { materializeString, stringifyJson } from "@oh-my-pi/pi-utils";
5
5
  import { LRUCache } from "@oh-my-pi/pi-utils/lru";
6
6
  import * as snapcompact from "@oh-my-pi/snapcompact";
7
7
  import { isEstimateCacheable, messageEstimateVersion } from "./compaction/message-cache";
8
+ import { estimateImageContentTokens } from "./image-tokens";
8
9
  import type { AgentMessage } from "./types";
9
10
 
10
11
  const testEnv = Bun.env.NODE_ENV === "test";
@@ -131,12 +132,6 @@ export interface TokenBudgetCheck {
131
132
  exact: boolean;
132
133
  }
133
134
 
134
- /**
135
- * Image content has no tokenizer representation; charge a fixed estimate
136
- * matching what providers typically bill for inline images.
137
- */
138
- const IMAGE_TOKEN_ESTIMATE = 1200;
139
-
140
135
  /**
141
136
  * Memoized estimates for one message under this tokenizer's encoding, split by
142
137
  * the {@link MessageCountOptions.excludeEncryptedReasoning} option so the two
@@ -219,7 +214,8 @@ export class Tokenizer {
219
214
  * Settled historical messages are counted once and reused until an owner
220
215
  * (prune/shake/strip-images) calls `invalidateMessageCache`; streaming
221
216
  * assistants bypass the memo entirely (see the message-cache settle-gate
222
- * invariant). Image blocks charge a fixed per-image estimate.
217
+ * invariant). Image blocks charge {@link estimateImageContentTokens}, the
218
+ * same dimension-based estimate the native remote-compaction probe applies.
223
219
  */
224
220
  countMessage(message: AgentMessage, options?: MessageCountOptions): number {
225
221
  const floored = options?.excludeEncryptedReasoning === true;
@@ -273,7 +269,7 @@ export class Tokenizer {
273
269
  if (block.type === "text" && block.text) {
274
270
  fragments.push(block.text);
275
271
  } else if (block.type === "image") {
276
- extra += IMAGE_TOKEN_ESTIMATE;
272
+ extra += estimateImageContentTokens(block);
277
273
  }
278
274
  }
279
275
  }
@@ -323,7 +319,7 @@ export class Tokenizer {
323
319
  if (block.type === "text" && block.text) {
324
320
  fragments.push(block.text);
325
321
  } else if (block.type === "image") {
326
- extra += IMAGE_TOKEN_ESTIMATE;
322
+ extra += estimateImageContentTokens(block);
327
323
  }
328
324
  }
329
325
  }