@oh-my-pi/pi-agent-core 18.5.1 → 18.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/types/compaction/openai.d.ts +8 -2
- package/dist/types/image-tokens.d.ts +30 -0
- package/dist/types/tokenizer.d.ts +2 -1
- package/package.json +8 -8
- package/src/agent-loop.ts +47 -0
- package/src/compaction/openai.ts +48 -24
- package/src/image-tokens.ts +106 -0
- package/src/prompts/dsml-tool-call-leak.md +19 -0
- package/src/tokenizer.ts +5 -9
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,18 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [18.6.1] - 2026-10-04
|
|
6
|
+
|
|
7
|
+
### Fixed
|
|
8
|
+
|
|
9
|
+
- Fixed native OpenAI context compaction for sessions containing many screenshots, preventing image-size estimates from incorrectly causing compaction requests to be rejected.
|
|
10
|
+
|
|
11
|
+
## [18.6.0] - 2026-10-03
|
|
12
|
+
|
|
13
|
+
### Fixed
|
|
14
|
+
|
|
15
|
+
- When a DeepSeek turn ends with raw `<|DSML|…>` tool-call text that could not be parsed into a real call, and no tool call was made, the broken markup is now removed from the message before it is saved to history. The agent then tells the model the tool call failed, shows it the correct format, and asks it again, at most twice in a row. Previously the markup stayed in history, and the agent stopped as if the model had finished ([#14202](https://github.com/can1357/oh-my-pi/pull/14202) by [@H4vC](https://github.com/H4vC)).
|
|
16
|
+
|
|
5
17
|
## [18.5.1] - 2026-10-03
|
|
6
18
|
|
|
7
19
|
### Added
|
|
@@ -33,14 +33,20 @@ export interface TrimRemoteCompactionInputResult {
|
|
|
33
33
|
rewrittenOutputs: number;
|
|
34
34
|
estimatedTokensBefore: number;
|
|
35
35
|
estimatedTokensAfter: number;
|
|
36
|
-
/**
|
|
36
|
+
/**
|
|
37
|
+
* Whether `input` may be sent; false means it must not be. True when the
|
|
38
|
+
* full estimate fits, or when only the image estimate pushes it over (image
|
|
39
|
+
* pricing is a guess; the provider's own accounting decides).
|
|
40
|
+
*/
|
|
37
41
|
fits: boolean;
|
|
38
42
|
}
|
|
39
43
|
/**
|
|
40
44
|
* Preserve the full native transcript unless trailing tool outputs alone push a
|
|
41
45
|
* remote compaction request beyond the model window. Replacing only those
|
|
42
46
|
* outputs keeps call/result pairing and all earlier assistant/reasoning history,
|
|
43
|
-
* matching Codex's recovery path for oversized tool turns.
|
|
47
|
+
* matching Codex's recovery path for oversized tool turns. A request whose text
|
|
48
|
+
* fits but whose image estimate does not is still sendable: refusing it on an
|
|
49
|
+
* image guess would skip native compaction the provider would have accepted.
|
|
44
50
|
*/
|
|
45
51
|
export declare function trimRemoteCompactionInputToContextWindow(input: Array<Record<string, unknown>>, tokenizer: Tokenizer, contextWindow: number | null | undefined, instructions: string, tools?: unknown[]): TrimRemoteCompactionInputResult;
|
|
46
52
|
/**
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { ImageContent } from "@oh-my-pi/pi-ai";
|
|
2
|
+
/**
|
|
3
|
+
* Dimension-based image token estimates shared by the local context counter
|
|
4
|
+
* ({@link Tokenizer.countMessage}) and the native remote-compaction fit probe.
|
|
5
|
+
* Both sides must price the same image the same way: when they disagreed
|
|
6
|
+
* (1.2k flat locally vs a 12k flat worst case remotely) an image-heavy session
|
|
7
|
+
* stayed under the local compaction trigger yet was refused as over-window by
|
|
8
|
+
* the remote probe.
|
|
9
|
+
*
|
|
10
|
+
* Follows OpenAI's patch-based rule (32px patches, a per-detail pixel limit
|
|
11
|
+
* and patch budget, x1.2 multiplier), which is also a close upper estimate for
|
|
12
|
+
* other providers once omp has downscaled the image (≤1568px by default).
|
|
13
|
+
*/
|
|
14
|
+
/** OpenAI image `detail` values; `undefined` means the provider default (`auto`). */
|
|
15
|
+
export type ImageDetail = ImageContent["detail"];
|
|
16
|
+
export interface ImageSize {
|
|
17
|
+
width: number;
|
|
18
|
+
height: number;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Estimated input tokens for one image. Unknown dimensions (undecodable data,
|
|
22
|
+
* remote URLs, provider file ids) charge the detail level's full patch budget.
|
|
23
|
+
*/
|
|
24
|
+
export declare function estimateImageTokens(size: ImageSize | null | undefined, detail?: ImageDetail): number;
|
|
25
|
+
/** Pixel size read from the header of base64-encoded image data, or null. */
|
|
26
|
+
export declare function base64ImageSize(base64: string): ImageSize | null;
|
|
27
|
+
/** Pixel size of a `data:<mime>;base64,` URL, or null for any other URL. */
|
|
28
|
+
export declare function dataUrlImageSize(url: string): ImageSize | null;
|
|
29
|
+
/** {@link estimateImageTokens} for an in-memory image block. */
|
|
30
|
+
export declare function estimateImageContentTokens(image: Pick<ImageContent, "data" | "detail">): number;
|
|
@@ -70,7 +70,8 @@ export declare class Tokenizer {
|
|
|
70
70
|
* Settled historical messages are counted once and reused until an owner
|
|
71
71
|
* (prune/shake/strip-images) calls `invalidateMessageCache`; streaming
|
|
72
72
|
* assistants bypass the memo entirely (see the message-cache settle-gate
|
|
73
|
-
* invariant). Image blocks charge
|
|
73
|
+
* invariant). Image blocks charge {@link estimateImageContentTokens}, the
|
|
74
|
+
* same dimension-based estimate the native remote-compaction probe applies.
|
|
74
75
|
*/
|
|
75
76
|
countMessage(message: AgentMessage, options?: MessageCountOptions): number;
|
|
76
77
|
/** Sum of {@link countMessage} over `messages`. */
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"type": "module",
|
|
3
3
|
"name": "@oh-my-pi/pi-agent-core",
|
|
4
|
-
"version": "18.
|
|
4
|
+
"version": "18.6.1",
|
|
5
5
|
"description": "General-purpose agent with transport abstraction, state management, and attachment support",
|
|
6
6
|
"homepage": "https://omp.sh",
|
|
7
7
|
"author": {
|
|
@@ -38,16 +38,16 @@
|
|
|
38
38
|
"fmt": "oxfmt --no-error-on-unmatched-pattern 'src/**/*.{ts,tsx}' '{test,bench,examples,scripts}/**/*.ts' '*.ts'"
|
|
39
39
|
},
|
|
40
40
|
"dependencies": {
|
|
41
|
-
"@oh-my-pi/pi-ai": "18.
|
|
42
|
-
"@oh-my-pi/pi-catalog": "18.
|
|
43
|
-
"@oh-my-pi/pi-natives": "18.
|
|
44
|
-
"@oh-my-pi/pi-utils": "18.
|
|
45
|
-
"@oh-my-pi/pi-wire": "18.
|
|
46
|
-
"@oh-my-pi/snapcompact": "18.
|
|
41
|
+
"@oh-my-pi/pi-ai": "18.6.1",
|
|
42
|
+
"@oh-my-pi/pi-catalog": "18.6.1",
|
|
43
|
+
"@oh-my-pi/pi-natives": "18.6.1",
|
|
44
|
+
"@oh-my-pi/pi-utils": "18.6.1",
|
|
45
|
+
"@oh-my-pi/pi-wire": "18.6.1",
|
|
46
|
+
"@oh-my-pi/snapcompact": "18.6.1",
|
|
47
47
|
"@opentelemetry/api": "^1.9.1"
|
|
48
48
|
},
|
|
49
49
|
"devDependencies": {
|
|
50
|
-
"@oh-my-pi/omptype": "18.
|
|
50
|
+
"@oh-my-pi/omptype": "18.6.1",
|
|
51
51
|
"@opentelemetry/context-async-hooks": "^2.9.0",
|
|
52
52
|
"@opentelemetry/sdk-trace-base": "^2.9.0",
|
|
53
53
|
"@types/bun": "^1.3.14"
|
package/src/agent-loop.ts
CHANGED
|
@@ -53,6 +53,7 @@ import {
|
|
|
53
53
|
recoverHarmonyToolCall,
|
|
54
54
|
signalListLabel,
|
|
55
55
|
} from "@oh-my-pi/pi-ai/utils/harmony-leak";
|
|
56
|
+
import { isDsmlLeakRecoveryTarget, removeDsmlToolMarkupLeak } from "@oh-my-pi/pi-ai/utils/dsml-leak";
|
|
56
57
|
import { cloneJsonTree, logger, sanitizeText, structuredCloneJSON } from "@oh-my-pi/pi-utils";
|
|
57
58
|
import { INTENT_FIELD } from "@oh-my-pi/pi-wire";
|
|
58
59
|
import { LiveSteeringChannel } from "./live-steering";
|
|
@@ -76,6 +77,7 @@ import {
|
|
|
76
77
|
startInvokeAgentSpan,
|
|
77
78
|
} from "./telemetry";
|
|
78
79
|
import { createAdditionalContextMessage, isNonBlankContext, joinAdditionalContext } from "./tool-context";
|
|
80
|
+
import dsmlToolCallLeakPrompt from "./prompts/dsml-tool-call-leak.md" with { type: "text" };
|
|
79
81
|
import type {
|
|
80
82
|
AgentContext,
|
|
81
83
|
AgentEvent,
|
|
@@ -116,6 +118,14 @@ const ABORTED: unique symbol = Symbol("agent-loop-aborted");
|
|
|
116
118
|
*/
|
|
117
119
|
const MAX_PAUSED_TURN_CONTINUATIONS = 8;
|
|
118
120
|
|
|
121
|
+
/**
|
|
122
|
+
* Cap on consecutive "tool call failed" nudges for a turn that ended with raw
|
|
123
|
+
* DeepSeek DSML tool-call markup in its text and no structured call. A model
|
|
124
|
+
* that keeps leaking after this many corrections is left to yield. Resets
|
|
125
|
+
* whenever a turn carries tool calls.
|
|
126
|
+
*/
|
|
127
|
+
const MAX_DSML_LEAK_NUDGES = 2;
|
|
128
|
+
|
|
119
129
|
/**
|
|
120
130
|
* Cap on consecutive forced escalations for a single soft tool requirement.
|
|
121
131
|
* A forced `toolChoice` guarantees the call, so this is purely defensive: if a
|
|
@@ -1222,6 +1232,7 @@ async function runLoopBody(
|
|
|
1222
1232
|
let harmonyRetryAttempt = 0;
|
|
1223
1233
|
let harmonyTruncateResumeCount = 0;
|
|
1224
1234
|
let pausedTurnContinuations = 0;
|
|
1235
|
+
let dsmlLeakNudges = 0;
|
|
1225
1236
|
|
|
1226
1237
|
// Soft tool requirement lifecycle (reminder then escalation; see SoftToolRequirement).
|
|
1227
1238
|
// The host-owned state survives only a gate stop between Agent.prompt calls.
|
|
@@ -1685,6 +1696,7 @@ async function runLoopBody(
|
|
|
1685
1696
|
|
|
1686
1697
|
if (toolCalls.length > 0) {
|
|
1687
1698
|
pausedTurnContinuations = 0;
|
|
1699
|
+
dsmlLeakNudges = 0;
|
|
1688
1700
|
} else if (
|
|
1689
1701
|
!hasMoreToolCalls &&
|
|
1690
1702
|
message.stopReason === "stop" &&
|
|
@@ -1698,6 +1710,25 @@ async function runLoopBody(
|
|
|
1698
1710
|
// mid-work turn.
|
|
1699
1711
|
pausedTurnContinuations++;
|
|
1700
1712
|
hasMoreToolCalls = true;
|
|
1713
|
+
} else if (
|
|
1714
|
+
!hasMoreToolCalls &&
|
|
1715
|
+
message.stopReason === "stop" &&
|
|
1716
|
+
dsmlLeakMessages.has(message) &&
|
|
1717
|
+
dsmlLeakNudges < MAX_DSML_LEAK_NUDGES
|
|
1718
|
+
) {
|
|
1719
|
+
// DeepSeek wrote its tool call as raw DSML text the healer could not
|
|
1720
|
+
// recover (often missing the opening envelope). The markup was
|
|
1721
|
+
// stripped before commit; without a nudge the loop would yield as if
|
|
1722
|
+
// the model were done, so tell it the call failed and re-sample.
|
|
1723
|
+
const nudge = injectExecutionAdditionalContext(
|
|
1724
|
+
currentContext,
|
|
1725
|
+
newMessages,
|
|
1726
|
+
stream,
|
|
1727
|
+
dsmlToolCallLeakPrompt,
|
|
1728
|
+
);
|
|
1729
|
+
if (nudge) additionalMessages.push(nudge);
|
|
1730
|
+
dsmlLeakNudges++;
|
|
1731
|
+
hasMoreToolCalls = true;
|
|
1701
1732
|
}
|
|
1702
1733
|
|
|
1703
1734
|
await emitTurnEnd(stream, currentContext, message, toolResults, config, signal, {
|
|
@@ -1967,6 +1998,7 @@ async function streamAssistantResponse(
|
|
|
1967
1998
|
// per model without touching the shared session `serviceTier`.
|
|
1968
1999
|
const effectiveServiceTier = config.getServiceTier ? config.getServiceTier(model) : config.serviceTier;
|
|
1969
2000
|
const harmonyMitigationEnabled = isHarmonyLeakMitigationTarget(model);
|
|
2001
|
+
const dsmlLeakRecoveryEnabled = isDsmlLeakRecoveryTarget(model);
|
|
1970
2002
|
const harmonyAbortController = harmonyMitigationEnabled ? new AbortController() : undefined;
|
|
1971
2003
|
const requestSignal = harmonyAbortController
|
|
1972
2004
|
? signal
|
|
@@ -2196,6 +2228,18 @@ async function streamAssistantResponse(
|
|
|
2196
2228
|
}
|
|
2197
2229
|
}
|
|
2198
2230
|
finalMessage = snapshotAssistantMessage(finalMessage);
|
|
2231
|
+
// Unhealed DSML tool-call markup must never reach history: replaying
|
|
2232
|
+
// it teaches the model to keep writing calls as text (#10556). Strip
|
|
2233
|
+
// it before the context, the UI, and the session see the message.
|
|
2234
|
+
// Only DSML-speaking models qualify; from others it is prose.
|
|
2235
|
+
if (
|
|
2236
|
+
dsmlLeakRecoveryEnabled &&
|
|
2237
|
+
finalMessage.stopReason === "stop" &&
|
|
2238
|
+
!finalMessage.content.some(block => block.type === "toolCall") &&
|
|
2239
|
+
removeDsmlToolMarkupLeak(finalMessage)
|
|
2240
|
+
) {
|
|
2241
|
+
dsmlLeakMessages.add(finalMessage);
|
|
2242
|
+
}
|
|
2199
2243
|
// Expand inline macros (and any other registered rewrite) on the
|
|
2200
2244
|
// finalized message before it reaches the context, the UI, or tool
|
|
2201
2245
|
// dispatch — so a single mutation is the source of truth for all three.
|
|
@@ -2752,6 +2796,9 @@ interface PreparedToolCall {
|
|
|
2752
2796
|
prepareError?: unknown;
|
|
2753
2797
|
}
|
|
2754
2798
|
|
|
2799
|
+
/** Committed assistant messages whose unhealed DSML tool-call markup was stripped; the loop answers them with a failure nudge. */
|
|
2800
|
+
const dsmlLeakMessages = new WeakSet<AssistantMessage>();
|
|
2801
|
+
|
|
2755
2802
|
/**
|
|
2756
2803
|
* Prepare results computed in the stream-done branch (before `message_start`/
|
|
2757
2804
|
* `message_end`) so a `beforeToolCall` args revision is baked into the message
|
package/src/compaction/openai.ts
CHANGED
|
@@ -53,6 +53,7 @@ import {
|
|
|
53
53
|
OPENAI_HEADERS,
|
|
54
54
|
} from "@oh-my-pi/pi-catalog/wire/codex";
|
|
55
55
|
import { $env, isRecord, logger, prompt, ptree, stringifyJson, structuredCloneJSON } from "@oh-my-pi/pi-utils";
|
|
56
|
+
import { dataUrlImageSize, estimateImageTokens } from "../image-tokens";
|
|
56
57
|
import { Tokenizer } from "../tokenizer";
|
|
57
58
|
import { appendAzureApiVersion, resolveAzureOpenAiBaseUrl } from "./azure-openai-endpoint";
|
|
58
59
|
import { prepareBedrockCompactionRequest } from "./bedrock";
|
|
@@ -80,7 +81,8 @@ export const REMOTE_COMPACTION_TIMEOUT_MS = 300_000;
|
|
|
80
81
|
export const CONTEXT_WINDOW_TRUNCATED_OUTPUT_MESSAGE = prompt.render(contextWindowTruncatedOutputPrompt);
|
|
81
82
|
|
|
82
83
|
const REMOTE_COMPACTION_REQUEST_OVERHEAD_TOKENS = 256;
|
|
83
|
-
|
|
84
|
+
/** Per-`input_image` record estimates; trim re-probes share untouched items by reference. */
|
|
85
|
+
const remoteImageTokenCache = new WeakMap<Record<string, unknown>, number>();
|
|
84
86
|
const TOOL_RESULT_IMAGE_ATTACHMENT_TEXT = "Attached image(s) from tool result:";
|
|
85
87
|
|
|
86
88
|
interface NormalizedEstimateValue {
|
|
@@ -103,10 +105,16 @@ function normalizeRemoteCompactionEstimateValue(value: unknown): NormalizedEstim
|
|
|
103
105
|
|
|
104
106
|
const record = value as Record<string, unknown>;
|
|
105
107
|
if (record.type === "input_image") {
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
108
|
+
let imageTokens = remoteImageTokenCache.get(record);
|
|
109
|
+
if (imageTokens === undefined) {
|
|
110
|
+
const detail = record.detail;
|
|
111
|
+
imageTokens = estimateImageTokens(
|
|
112
|
+
typeof record.image_url === "string" ? dataUrlImageSize(record.image_url) : null,
|
|
113
|
+
detail === "low" || detail === "high" || detail === "original" || detail === "auto" ? detail : undefined,
|
|
114
|
+
);
|
|
115
|
+
remoteImageTokenCache.set(record, imageTokens);
|
|
116
|
+
}
|
|
117
|
+
return { value: { ...record, image_url: "<image>" }, imageTokens };
|
|
110
118
|
}
|
|
111
119
|
|
|
112
120
|
const normalized: Record<string, unknown> = {};
|
|
@@ -128,7 +136,11 @@ export interface TrimRemoteCompactionInputResult {
|
|
|
128
136
|
rewrittenOutputs: number;
|
|
129
137
|
estimatedTokensBefore: number;
|
|
130
138
|
estimatedTokensAfter: number;
|
|
131
|
-
/**
|
|
139
|
+
/**
|
|
140
|
+
* Whether `input` may be sent; false means it must not be. True when the
|
|
141
|
+
* full estimate fits, or when only the image estimate pushes it over (image
|
|
142
|
+
* pricing is a guess; the provider's own accounting decides).
|
|
143
|
+
*/
|
|
132
144
|
fits: boolean;
|
|
133
145
|
}
|
|
134
146
|
|
|
@@ -138,15 +150,17 @@ interface RemoteCompactionBudgetProbe {
|
|
|
138
150
|
tokens: number;
|
|
139
151
|
/** Whether the request fits the window. Always true when no window is known. */
|
|
140
152
|
fits: boolean;
|
|
153
|
+
/** Whether the request minus its image estimate fits the window. */
|
|
154
|
+
textFits: boolean;
|
|
141
155
|
}
|
|
142
156
|
|
|
143
157
|
/**
|
|
144
|
-
* Cheap-first sizing of a remote-compaction request. Images
|
|
145
|
-
*
|
|
146
|
-
* tokenizer; opaque `encrypted_content` payloads are excluded. The
|
|
147
|
-
* transcript is then probed with {@link Tokenizer.checkTokenBudget},
|
|
148
|
-
* pays for an exact count when the byte bound cannot already prove
|
|
149
|
-
* fits.
|
|
158
|
+
* Cheap-first sizing of a remote-compaction request. Images (priced from their
|
|
159
|
+
* dimensions) and the request frame come off the budget rather than through
|
|
160
|
+
* the tokenizer; opaque `encrypted_content` payloads are excluded. The
|
|
161
|
+
* serialized transcript is then probed with {@link Tokenizer.checkTokenBudget},
|
|
162
|
+
* which only pays for an exact count when the byte bound cannot already prove
|
|
163
|
+
* the request fits.
|
|
150
164
|
*/
|
|
151
165
|
function probeRemoteCompactionInputBudget(
|
|
152
166
|
input: Array<Record<string, unknown>>,
|
|
@@ -159,10 +173,15 @@ function probeRemoteCompactionInputBudget(
|
|
|
159
173
|
const serialized = stringifyJson(normalized.value) ?? "";
|
|
160
174
|
const flatTokens = normalized.imageTokens + REMOTE_COMPACTION_REQUEST_OVERHEAD_TOKENS;
|
|
161
175
|
if (!contextWindow || contextWindow <= 0) {
|
|
162
|
-
return { tokens: tokenizer.countTokens(serialized, "upperbound") + flatTokens, fits: true };
|
|
176
|
+
return { tokens: tokenizer.countTokens(serialized, "upperbound") + flatTokens, fits: true, textFits: true };
|
|
163
177
|
}
|
|
164
178
|
const budget = tokenizer.checkTokenBudget(serialized, Math.max(0, contextWindow - flatTokens));
|
|
165
|
-
return {
|
|
179
|
+
return {
|
|
180
|
+
tokens: budget.tokens + flatTokens,
|
|
181
|
+
fits: budget.fits,
|
|
182
|
+
// A failed check carries the exact text count (or its byte upper bound).
|
|
183
|
+
textFits: budget.fits || budget.tokens + REMOTE_COMPACTION_REQUEST_OVERHEAD_TOKENS <= contextWindow,
|
|
184
|
+
};
|
|
166
185
|
}
|
|
167
186
|
|
|
168
187
|
function rewriteToolOutputForContextWindow(item: Record<string, unknown>): Record<string, unknown> | undefined {
|
|
@@ -192,7 +211,9 @@ function isToolResultImageAttachment(item: Record<string, unknown>): boolean {
|
|
|
192
211
|
* Preserve the full native transcript unless trailing tool outputs alone push a
|
|
193
212
|
* remote compaction request beyond the model window. Replacing only those
|
|
194
213
|
* outputs keeps call/result pairing and all earlier assistant/reasoning history,
|
|
195
|
-
* matching Codex's recovery path for oversized tool turns.
|
|
214
|
+
* matching Codex's recovery path for oversized tool turns. A request whose text
|
|
215
|
+
* fits but whose image estimate does not is still sendable: refusing it on an
|
|
216
|
+
* image guess would skip native compaction the provider would have accepted.
|
|
196
217
|
*/
|
|
197
218
|
export function trimRemoteCompactionInputToContextWindow(
|
|
198
219
|
input: Array<Record<string, unknown>>,
|
|
@@ -226,22 +247,25 @@ export function trimRemoteCompactionInputToContextWindow(
|
|
|
226
247
|
after = probeRemoteCompactionInputBudget(rewrittenInput, tokenizer, instructions, tools, contextWindow);
|
|
227
248
|
}
|
|
228
249
|
|
|
229
|
-
|
|
250
|
+
// Rewrites that reach a full fit are kept. When they cannot, and the text
|
|
251
|
+
// already fit, the overflow is image-only: rewriting cannot shrink the image
|
|
252
|
+
// estimate, so send the untouched input rather than discard tool output.
|
|
253
|
+
if (rewrittenInput && (after.fits || (!before.textFits && after.textFits))) {
|
|
230
254
|
return {
|
|
231
|
-
input,
|
|
232
|
-
rewrittenOutputs
|
|
255
|
+
input: rewrittenInput,
|
|
256
|
+
rewrittenOutputs,
|
|
233
257
|
estimatedTokensBefore: before.tokens,
|
|
234
|
-
estimatedTokensAfter:
|
|
235
|
-
fits:
|
|
258
|
+
estimatedTokensAfter: after.tokens,
|
|
259
|
+
fits: true,
|
|
236
260
|
};
|
|
237
261
|
}
|
|
238
262
|
|
|
239
263
|
return {
|
|
240
|
-
input
|
|
241
|
-
rewrittenOutputs,
|
|
264
|
+
input,
|
|
265
|
+
rewrittenOutputs: 0,
|
|
242
266
|
estimatedTokensBefore: before.tokens,
|
|
243
|
-
estimatedTokensAfter:
|
|
244
|
-
fits:
|
|
267
|
+
estimatedTokensAfter: before.tokens,
|
|
268
|
+
fits: before.textFits,
|
|
245
269
|
};
|
|
246
270
|
}
|
|
247
271
|
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import type { ImageContent } from "@oh-my-pi/pi-ai";
|
|
2
|
+
import { parseImageMetadata } from "@oh-my-pi/pi-utils";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Dimension-based image token estimates shared by the local context counter
|
|
6
|
+
* ({@link Tokenizer.countMessage}) and the native remote-compaction fit probe.
|
|
7
|
+
* Both sides must price the same image the same way: when they disagreed
|
|
8
|
+
* (1.2k flat locally vs a 12k flat worst case remotely) an image-heavy session
|
|
9
|
+
* stayed under the local compaction trigger yet was refused as over-window by
|
|
10
|
+
* the remote probe.
|
|
11
|
+
*
|
|
12
|
+
* Follows OpenAI's patch-based rule (32px patches, a per-detail pixel limit
|
|
13
|
+
* and patch budget, x1.2 multiplier), which is also a close upper estimate for
|
|
14
|
+
* other providers once omp has downscaled the image (≤1568px by default).
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
/** OpenAI image `detail` values; `undefined` means the provider default (`auto`). */
|
|
18
|
+
export type ImageDetail = ImageContent["detail"];
|
|
19
|
+
|
|
20
|
+
export interface ImageSize {
|
|
21
|
+
width: number;
|
|
22
|
+
height: number;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const PATCH_PX = 32;
|
|
26
|
+
const PATCH_TOKEN_MULTIPLIER = 1.2;
|
|
27
|
+
|
|
28
|
+
interface DetailSizing {
|
|
29
|
+
/** Longest side after the pixel-dimension fit. */
|
|
30
|
+
maxDimension: number;
|
|
31
|
+
/** Resizing patch budget, when the detail level defines one. */
|
|
32
|
+
patchBudget?: number;
|
|
33
|
+
/** Patches charged when the image size is unknown. */
|
|
34
|
+
unknownPatches: number;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const LOW_SIZING: DetailSizing = { maxDimension: 512, unknownPatches: 16 * 16 };
|
|
38
|
+
const HIGH_SIZING: DetailSizing = { maxDimension: 2048, patchBudget: 2_500, unknownPatches: 2_500 };
|
|
39
|
+
// `auto` resolves to `original` sizing on current models, so it shares the
|
|
40
|
+
// larger budget rather than risk undercounting.
|
|
41
|
+
const ORIGINAL_SIZING: DetailSizing = { maxDimension: 6_000, patchBudget: 10_000, unknownPatches: 10_000 };
|
|
42
|
+
|
|
43
|
+
function sizingFor(detail: ImageDetail): DetailSizing {
|
|
44
|
+
if (detail === "low") return LOW_SIZING;
|
|
45
|
+
if (detail === "high") return HIGH_SIZING;
|
|
46
|
+
return ORIGINAL_SIZING;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function countPatches(size: ImageSize, sizing: DetailSizing): number {
|
|
50
|
+
let { width, height } = size;
|
|
51
|
+
const longest = Math.max(width, height);
|
|
52
|
+
if (longest > sizing.maxDimension) {
|
|
53
|
+
const scale = sizing.maxDimension / longest;
|
|
54
|
+
width = Math.max(1, Math.round(width * scale));
|
|
55
|
+
height = Math.max(1, Math.round(height * scale));
|
|
56
|
+
}
|
|
57
|
+
const patches = Math.ceil(width / PATCH_PX) * Math.ceil(height / PATCH_PX);
|
|
58
|
+
const budget = sizing.patchBudget;
|
|
59
|
+
if (budget === undefined || patches <= budget) return patches;
|
|
60
|
+
|
|
61
|
+
const shrink = Math.sqrt((PATCH_PX * PATCH_PX * budget) / (width * height));
|
|
62
|
+
const scaledW = (width * shrink) / PATCH_PX;
|
|
63
|
+
const scaledH = (height * shrink) / PATCH_PX;
|
|
64
|
+
const adjusted = shrink * Math.min(Math.floor(scaledW) / scaledW, Math.floor(scaledH) / scaledH);
|
|
65
|
+
const resizedW = Math.floor(width * adjusted);
|
|
66
|
+
const resizedH = Math.floor(height * adjusted);
|
|
67
|
+
if (resizedW <= 0 || resizedH <= 0) return budget;
|
|
68
|
+
return Math.min(budget, Math.ceil(resizedW / PATCH_PX) * Math.ceil(resizedH / PATCH_PX));
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Estimated input tokens for one image. Unknown dimensions (undecodable data,
|
|
73
|
+
* remote URLs, provider file ids) charge the detail level's full patch budget.
|
|
74
|
+
*/
|
|
75
|
+
export function estimateImageTokens(size: ImageSize | null | undefined, detail?: ImageDetail): number {
|
|
76
|
+
const sizing = sizingFor(detail);
|
|
77
|
+
const patches = size && size.width > 0 && size.height > 0 ? countPatches(size, sizing) : sizing.unknownPatches;
|
|
78
|
+
return Math.ceil(patches * PATCH_TOKEN_MULTIPLIER);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// Enough decoded bytes to reach a JPEG SOF marker behind typical EXIF/ICC
|
|
82
|
+
// segments; PNG/GIF/WebP need only the first 30 bytes. Multiple of 4 so the
|
|
83
|
+
// base64 slice decodes cleanly.
|
|
84
|
+
const HEADER_BASE64_CHARS = 4 * Math.ceil((64 * 1024) / 3);
|
|
85
|
+
|
|
86
|
+
/** Pixel size read from the header of base64-encoded image data, or null. */
|
|
87
|
+
export function base64ImageSize(base64: string): ImageSize | null {
|
|
88
|
+
if (base64.length === 0) return null;
|
|
89
|
+
const header = Buffer.from(base64.slice(0, HEADER_BASE64_CHARS), "base64");
|
|
90
|
+
const metadata = parseImageMetadata(header);
|
|
91
|
+
if (!metadata?.width || !metadata.height) return null;
|
|
92
|
+
return { width: metadata.width, height: metadata.height };
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** Pixel size of a `data:<mime>;base64,` URL, or null for any other URL. */
|
|
96
|
+
export function dataUrlImageSize(url: string): ImageSize | null {
|
|
97
|
+
if (!url.startsWith("data:")) return null;
|
|
98
|
+
const comma = url.indexOf(",");
|
|
99
|
+
if (comma < 0 || !url.slice(0, comma).endsWith(";base64")) return null;
|
|
100
|
+
return base64ImageSize(url.slice(comma + 1));
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** {@link estimateImageTokens} for an in-memory image block. */
|
|
104
|
+
export function estimateImageContentTokens(image: Pick<ImageContent, "data" | "detail">): number {
|
|
105
|
+
return estimateImageTokens(base64ImageSize(image.data), image.detail);
|
|
106
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
Tool call failed: your previous response tried to invoke a tool, but the DSML markup was malformed, so it arrived as plain text, no tool was executed, and the broken markup was removed from your response. Usually the opening `<|DSML|tool_calls>` and/or `<|DSML|invoke name="…">` tags are missing, or the tool name was written as a plain line.
|
|
2
|
+
|
|
3
|
+
A tool call must be one complete envelope:
|
|
4
|
+
|
|
5
|
+
```
|
|
6
|
+
<|DSML|tool_calls>
|
|
7
|
+
<|DSML|invoke name="TOOL_NAME">
|
|
8
|
+
<|DSML|parameter name="ARG_NAME" string="true">text value</|DSML|parameter>
|
|
9
|
+
<|DSML|parameter name="OTHER_ARG" string="false">{"json": "value"}</|DSML|parameter>
|
|
10
|
+
</|DSML|invoke>
|
|
11
|
+
</|DSML|tool_calls>
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
- `<|DSML|tool_calls>` wraps every call; put one `<|DSML|invoke>` per call inside it.
|
|
15
|
+
- The tool name goes in the `name` attribute of `<|DSML|invoke>`, never on its own line.
|
|
16
|
+
- Use `string="true"` for raw text values and `string="false"` for numbers, booleans, arrays, and objects (JSON).
|
|
17
|
+
- Close every tag.
|
|
18
|
+
|
|
19
|
+
Re-issue the intended call now in this shape, or reply in plain prose if no tool is needed.
|
package/src/tokenizer.ts
CHANGED
|
@@ -5,6 +5,7 @@ import { materializeString, stringifyJson } from "@oh-my-pi/pi-utils";
|
|
|
5
5
|
import { LRUCache } from "@oh-my-pi/pi-utils/lru";
|
|
6
6
|
import * as snapcompact from "@oh-my-pi/snapcompact";
|
|
7
7
|
import { isEstimateCacheable, messageEstimateVersion } from "./compaction/message-cache";
|
|
8
|
+
import { estimateImageContentTokens } from "./image-tokens";
|
|
8
9
|
import type { AgentMessage } from "./types";
|
|
9
10
|
|
|
10
11
|
const testEnv = Bun.env.NODE_ENV === "test";
|
|
@@ -131,12 +132,6 @@ export interface TokenBudgetCheck {
|
|
|
131
132
|
exact: boolean;
|
|
132
133
|
}
|
|
133
134
|
|
|
134
|
-
/**
|
|
135
|
-
* Image content has no tokenizer representation; charge a fixed estimate
|
|
136
|
-
* matching what providers typically bill for inline images.
|
|
137
|
-
*/
|
|
138
|
-
const IMAGE_TOKEN_ESTIMATE = 1200;
|
|
139
|
-
|
|
140
135
|
/**
|
|
141
136
|
* Memoized estimates for one message under this tokenizer's encoding, split by
|
|
142
137
|
* the {@link MessageCountOptions.excludeEncryptedReasoning} option so the two
|
|
@@ -219,7 +214,8 @@ export class Tokenizer {
|
|
|
219
214
|
* Settled historical messages are counted once and reused until an owner
|
|
220
215
|
* (prune/shake/strip-images) calls `invalidateMessageCache`; streaming
|
|
221
216
|
* assistants bypass the memo entirely (see the message-cache settle-gate
|
|
222
|
-
* invariant). Image blocks charge
|
|
217
|
+
* invariant). Image blocks charge {@link estimateImageContentTokens}, the
|
|
218
|
+
* same dimension-based estimate the native remote-compaction probe applies.
|
|
223
219
|
*/
|
|
224
220
|
countMessage(message: AgentMessage, options?: MessageCountOptions): number {
|
|
225
221
|
const floored = options?.excludeEncryptedReasoning === true;
|
|
@@ -273,7 +269,7 @@ export class Tokenizer {
|
|
|
273
269
|
if (block.type === "text" && block.text) {
|
|
274
270
|
fragments.push(block.text);
|
|
275
271
|
} else if (block.type === "image") {
|
|
276
|
-
extra +=
|
|
272
|
+
extra += estimateImageContentTokens(block);
|
|
277
273
|
}
|
|
278
274
|
}
|
|
279
275
|
}
|
|
@@ -323,7 +319,7 @@ export class Tokenizer {
|
|
|
323
319
|
if (block.type === "text" && block.text) {
|
|
324
320
|
fragments.push(block.text);
|
|
325
321
|
} else if (block.type === "image") {
|
|
326
|
-
extra +=
|
|
322
|
+
extra += estimateImageContentTokens(block);
|
|
327
323
|
}
|
|
328
324
|
}
|
|
329
325
|
}
|