@oh-my-pi/pi-agent-core 18.6.0 → 18.6.1

package/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## [Unreleased]
 
+## [18.6.1] - 2026-10-04
+
+### Fixed
+
+- Fixed native OpenAI context compaction for sessions containing many screenshots, preventing image-size estimates from incorrectly causing compaction requests to be rejected.
+
 ## [18.6.0] - 2026-10-03
 
 ### Fixed

package/dist/types/compaction/openai.d.ts

@@ -33,14 +33,20 @@ export interface TrimRemoteCompactionInputResult {
     rewrittenOutputs: number;
     estimatedTokensBefore: number;
     estimatedTokensAfter: number;
-    /** Whether `input` fits the model window; false means it must not be sent. */
+    /**
+     * Whether `input` may be sent; false means it must not be. True when the
+     * full estimate fits, or when only the image estimate pushes it over (image
+     * pricing is a guess; the provider's own accounting decides).
+     */
     fits: boolean;
 }
 /**
  * Preserve the full native transcript unless trailing tool outputs alone push a
  * remote compaction request beyond the model window. Replacing only those
  * outputs keeps call/result pairing and all earlier assistant/reasoning history,
- * matching Codex's recovery path for oversized tool turns.
+ * matching Codex's recovery path for oversized tool turns. A request whose text
+ * fits but whose image estimate does not is still sendable: refusing it on an
+ * image guess would skip native compaction the provider would have accepted.
  */
 export declare function trimRemoteCompactionInputToContextWindow(input: Array<Record<string, unknown>>, tokenizer: Tokenizer, contextWindow: number | null | undefined, instructions: string, tools?: unknown[]): TrimRemoteCompactionInputResult;
 /**

package/dist/types/image-tokens.d.ts

@@ -0,0 +1,30 @@
+import type { ImageContent } from "@oh-my-pi/pi-ai";
+/**
+ * Dimension-based image token estimates shared by the local context counter
+ * ({@link Tokenizer.countMessage}) and the native remote-compaction fit probe.
+ * Both sides must price the same image the same way: when they disagreed
+ * (1.2k flat locally vs a 12k flat worst case remotely) an image-heavy session
+ * stayed under the local compaction trigger yet was refused as over-window by
+ * the remote probe.
+ *
+ * Follows OpenAI's patch-based rule (32px patches, a per-detail pixel limit
+ * and patch budget, x1.2 multiplier), which is also a close upper estimate for
+ * other providers once omp has downscaled the image (≤1568px by default).
+ */
+/** OpenAI image `detail` values; `undefined` means the provider default (`auto`). */
+export type ImageDetail = ImageContent["detail"];
+export interface ImageSize {
+    width: number;
+    height: number;
+}
+/**
+ * Estimated input tokens for one image. Unknown dimensions (undecodable data,
+ * remote URLs, provider file ids) charge the detail level's full patch budget.
+ */
+export declare function estimateImageTokens(size: ImageSize | null | undefined, detail?: ImageDetail): number;
+/** Pixel size read from the header of base64-encoded image data, or null. */
+export declare function base64ImageSize(base64: string): ImageSize | null;
+/** Pixel size of a `data:<mime>;base64,` URL, or null for any other URL. */
+export declare function dataUrlImageSize(url: string): ImageSize | null;
+/** {@link estimateImageTokens} for an in-memory image block. */
+export declare function estimateImageContentTokens(image: Pick<ImageContent, "data" | "detail">): number;

package/dist/types/tokenizer.d.ts

@@ -70,7 +70,8 @@ export declare class Tokenizer {
      * Settled historical messages are counted once and reused until an owner
      * (prune/shake/strip-images) calls `invalidateMessageCache`; streaming
      * assistants bypass the memo entirely (see the message-cache settle-gate
-     * invariant). Image blocks charge a fixed per-image estimate.
+     * invariant). Image blocks charge {@link estimateImageContentTokens}, the
+     * same dimension-based estimate the native remote-compaction probe applies.
      */
     countMessage(message: AgentMessage, options?: MessageCountOptions): number;
     /** Sum of {@link countMessage} over `messages`. */

package/package.json

@@ -1,7 +1,7 @@
 {
 	"type": "module",
 	"name": "@oh-my-pi/pi-agent-core",
-	"version": "18.6.0",
+	"version": "18.6.1",
 	"description": "General-purpose agent with transport abstraction, state management, and attachment support",
 	"homepage": "https://omp.sh",
 	"author": {
@@ -38,16 +38,16 @@
 		"fmt": "oxfmt --no-error-on-unmatched-pattern 'src/**/*.{ts,tsx}' '{test,bench,examples,scripts}/**/*.ts' '*.ts'"
 	},
 	"dependencies": {
-		"@oh-my-pi/pi-ai": "18.6.0",
-		"@oh-my-pi/pi-catalog": "18.6.0",
-		"@oh-my-pi/pi-natives": "18.6.0",
-		"@oh-my-pi/pi-utils": "18.6.0",
-		"@oh-my-pi/pi-wire": "18.6.0",
-		"@oh-my-pi/snapcompact": "18.6.0",
+		"@oh-my-pi/pi-ai": "18.6.1",
+		"@oh-my-pi/pi-catalog": "18.6.1",
+		"@oh-my-pi/pi-natives": "18.6.1",
+		"@oh-my-pi/pi-utils": "18.6.1",
+		"@oh-my-pi/pi-wire": "18.6.1",
+		"@oh-my-pi/snapcompact": "18.6.1",
 		"@opentelemetry/api": "^1.9.1"
 	},
 	"devDependencies": {
-		"@oh-my-pi/omptype": "18.6.0",
+		"@oh-my-pi/omptype": "18.6.1",
 		"@opentelemetry/context-async-hooks": "^2.9.0",
 		"@opentelemetry/sdk-trace-base": "^2.9.0",
 		"@types/bun": "^1.3.14"

package/src/compaction/openai.ts

@@ -53,6 +53,7 @@ import {
 	OPENAI_HEADERS,
 } from "@oh-my-pi/pi-catalog/wire/codex";
 import { $env, isRecord, logger, prompt, ptree, stringifyJson, structuredCloneJSON } from "@oh-my-pi/pi-utils";
+import { dataUrlImageSize, estimateImageTokens } from "../image-tokens";
 import { Tokenizer } from "../tokenizer";
 import { appendAzureApiVersion, resolveAzureOpenAiBaseUrl } from "./azure-openai-endpoint";
 import { prepareBedrockCompactionRequest } from "./bedrock";
@@ -80,7 +81,8 @@ export const REMOTE_COMPACTION_TIMEOUT_MS = 300_000;
 export const CONTEXT_WINDOW_TRUNCATED_OUTPUT_MESSAGE = prompt.render(contextWindowTruncatedOutputPrompt);
 
 const REMOTE_COMPACTION_REQUEST_OVERHEAD_TOKENS = 256;
-const REMOTE_COMPACTION_IMAGE_TOKEN_ESTIMATE = 12_000;
+/** Per-`input_image` record estimates; trim re-probes share untouched items by reference. */
+const remoteImageTokenCache = new WeakMap<Record<string, unknown>, number>();
 const TOOL_RESULT_IMAGE_ATTACHMENT_TEXT = "Attached image(s) from tool result:";
 
 interface NormalizedEstimateValue {
@@ -103,10 +105,16 @@ function normalizeRemoteCompactionEstimateValue(value: unknown): NormalizedEstim
 
 	const record = value as Record<string, unknown>;
 	if (record.type === "input_image") {
-		return {
-			value: { ...record, image_url: "<image>" },
-			imageTokens: REMOTE_COMPACTION_IMAGE_TOKEN_ESTIMATE,
-		};
+		let imageTokens = remoteImageTokenCache.get(record);
+		if (imageTokens === undefined) {
+			const detail = record.detail;
+			imageTokens = estimateImageTokens(
+				typeof record.image_url === "string" ? dataUrlImageSize(record.image_url) : null,
+				detail === "low" || detail === "high" || detail === "original" || detail === "auto" ? detail : undefined,
+			);
+			remoteImageTokenCache.set(record, imageTokens);
+		}
+		return { value: { ...record, image_url: "<image>" }, imageTokens };
 	}
 
 	const normalized: Record<string, unknown> = {};
@@ -128,7 +136,11 @@ export interface TrimRemoteCompactionInputResult {
 	rewrittenOutputs: number;
 	estimatedTokensBefore: number;
 	estimatedTokensAfter: number;
-	/** Whether `input` fits the model window; false means it must not be sent. */
+	/**
+	 * Whether `input` may be sent; false means it must not be. True when the
+	 * full estimate fits, or when only the image estimate pushes it over (image
+	 * pricing is a guess; the provider's own accounting decides).
+	 */
 	fits: boolean;
 }
 
@@ -138,15 +150,17 @@ interface RemoteCompactionBudgetProbe {
 	tokens: number;
 	/** Whether the request fits the window. Always true when no window is known. */
 	fits: boolean;
+	/** Whether the request minus its image estimate fits the window. */
+	textFits: boolean;
 }
 
 /**
- * Cheap-first sizing of a remote-compaction request. Images and the request
- * frame are charged flat, so they come off the budget rather than through the
- * tokenizer; opaque `encrypted_content` payloads are excluded. The serialized
- * transcript is then probed with {@link Tokenizer.checkTokenBudget}, which only
- * pays for an exact count when the byte bound cannot already prove the request
- * fits.
+ * Cheap-first sizing of a remote-compaction request. Images (priced from their
+ * dimensions) and the request frame come off the budget rather than through
+ * the tokenizer; opaque `encrypted_content` payloads are excluded. The
+ * serialized transcript is then probed with {@link Tokenizer.checkTokenBudget},
+ * which only pays for an exact count when the byte bound cannot already prove
+ * the request fits.
  */
 function probeRemoteCompactionInputBudget(
 	input: Array<Record<string, unknown>>,
@@ -159,10 +173,15 @@ function probeRemoteCompactionInputBudget(
 	const serialized = stringifyJson(normalized.value) ?? "";
 	const flatTokens = normalized.imageTokens + REMOTE_COMPACTION_REQUEST_OVERHEAD_TOKENS;
 	if (!contextWindow || contextWindow <= 0) {
-		return { tokens: tokenizer.countTokens(serialized, "upperbound") + flatTokens, fits: true };
+		return { tokens: tokenizer.countTokens(serialized, "upperbound") + flatTokens, fits: true, textFits: true };
 	}
 	const budget = tokenizer.checkTokenBudget(serialized, Math.max(0, contextWindow - flatTokens));
-	return { tokens: budget.tokens + flatTokens, fits: budget.fits };
+	return {
+		tokens: budget.tokens + flatTokens,
+		fits: budget.fits,
+		// A failed check carries the exact text count (or its byte upper bound).
+		textFits: budget.fits || budget.tokens + REMOTE_COMPACTION_REQUEST_OVERHEAD_TOKENS <= contextWindow,
+	};
 }
 
 function rewriteToolOutputForContextWindow(item: Record<string, unknown>): Record<string, unknown> | undefined {
@@ -192,7 +211,9 @@ function isToolResultImageAttachment(item: Record<string, unknown>): boolean {
  * Preserve the full native transcript unless trailing tool outputs alone push a
  * remote compaction request beyond the model window. Replacing only those
  * outputs keeps call/result pairing and all earlier assistant/reasoning history,
- * matching Codex's recovery path for oversized tool turns.
+ * matching Codex's recovery path for oversized tool turns. A request whose text
+ * fits but whose image estimate does not is still sendable: refusing it on an
+ * image guess would skip native compaction the provider would have accepted.
  */
 export function trimRemoteCompactionInputToContextWindow(
 	input: Array<Record<string, unknown>>,
@@ -226,22 +247,25 @@ export function trimRemoteCompactionInputToContextWindow(
 		after = probeRemoteCompactionInputBudget(rewrittenInput, tokenizer, instructions, tools, contextWindow);
 	}
 
-	if (!rewrittenInput || !after.fits) {
+	// Rewrites that reach a full fit are kept. When they cannot, and the text
+	// already fit, the overflow is image-only: rewriting cannot shrink the image
+	// estimate, so send the untouched input rather than discard tool output.
+	if (rewrittenInput && (after.fits || (!before.textFits && after.textFits))) {
 		return {
-			input,
-			rewrittenOutputs: 0,
+			input: rewrittenInput,
+			rewrittenOutputs,
 			estimatedTokensBefore: before.tokens,
-			estimatedTokensAfter: before.tokens,
-			fits: false,
+			estimatedTokensAfter: after.tokens,
+			fits: true,
 		};
 	}
 
 	return {
-		input: rewrittenInput,
-		rewrittenOutputs,
+		input,
+		rewrittenOutputs: 0,
 		estimatedTokensBefore: before.tokens,
-		estimatedTokensAfter: after.tokens,
-		fits: true,
+		estimatedTokensAfter: before.tokens,
+		fits: before.textFits,
 	};
 }
 

package/src/image-tokens.ts

@@ -0,0 +1,106 @@
+import type { ImageContent } from "@oh-my-pi/pi-ai";
+import { parseImageMetadata } from "@oh-my-pi/pi-utils";
+
+/**
+ * Dimension-based image token estimates shared by the local context counter
+ * ({@link Tokenizer.countMessage}) and the native remote-compaction fit probe.
+ * Both sides must price the same image the same way: when they disagreed
+ * (1.2k flat locally vs a 12k flat worst case remotely) an image-heavy session
+ * stayed under the local compaction trigger yet was refused as over-window by
+ * the remote probe.
+ *
+ * Follows OpenAI's patch-based rule (32px patches, a per-detail pixel limit
+ * and patch budget, x1.2 multiplier), which is also a close upper estimate for
+ * other providers once omp has downscaled the image (≤1568px by default).
+ */
+
+/** OpenAI image `detail` values; `undefined` means the provider default (`auto`). */
+export type ImageDetail = ImageContent["detail"];
+
+export interface ImageSize {
+	width: number;
+	height: number;
+}
+
+const PATCH_PX = 32;
+const PATCH_TOKEN_MULTIPLIER = 1.2;
+
+interface DetailSizing {
+	/** Longest side after the pixel-dimension fit. */
+	maxDimension: number;
+	/** Resizing patch budget, when the detail level defines one. */
+	patchBudget?: number;
+	/** Patches charged when the image size is unknown. */
+	unknownPatches: number;
+}
+
+const LOW_SIZING: DetailSizing = { maxDimension: 512, unknownPatches: 16 * 16 };
+const HIGH_SIZING: DetailSizing = { maxDimension: 2048, patchBudget: 2_500, unknownPatches: 2_500 };
+// `auto` resolves to `original` sizing on current models, so it shares the
+// larger budget rather than risk undercounting.
+const ORIGINAL_SIZING: DetailSizing = { maxDimension: 6_000, patchBudget: 10_000, unknownPatches: 10_000 };
+
+function sizingFor(detail: ImageDetail): DetailSizing {
+	if (detail === "low") return LOW_SIZING;
+	if (detail === "high") return HIGH_SIZING;
+	return ORIGINAL_SIZING;
+}
+
+function countPatches(size: ImageSize, sizing: DetailSizing): number {
+	let { width, height } = size;
+	const longest = Math.max(width, height);
+	if (longest > sizing.maxDimension) {
+		const scale = sizing.maxDimension / longest;
+		width = Math.max(1, Math.round(width * scale));
+		height = Math.max(1, Math.round(height * scale));
+	}
+	const patches = Math.ceil(width / PATCH_PX) * Math.ceil(height / PATCH_PX);
+	const budget = sizing.patchBudget;
+	if (budget === undefined || patches <= budget) return patches;
+
+	const shrink = Math.sqrt((PATCH_PX * PATCH_PX * budget) / (width * height));
+	const scaledW = (width * shrink) / PATCH_PX;
+	const scaledH = (height * shrink) / PATCH_PX;
+	const adjusted = shrink * Math.min(Math.floor(scaledW) / scaledW, Math.floor(scaledH) / scaledH);
+	const resizedW = Math.floor(width * adjusted);
+	const resizedH = Math.floor(height * adjusted);
+	if (resizedW <= 0 || resizedH <= 0) return budget;
+	return Math.min(budget, Math.ceil(resizedW / PATCH_PX) * Math.ceil(resizedH / PATCH_PX));
+}
+
+/**
+ * Estimated input tokens for one image. Unknown dimensions (undecodable data,
+ * remote URLs, provider file ids) charge the detail level's full patch budget.
+ */
+export function estimateImageTokens(size: ImageSize | null | undefined, detail?: ImageDetail): number {
+	const sizing = sizingFor(detail);
+	const patches = size && size.width > 0 && size.height > 0 ? countPatches(size, sizing) : sizing.unknownPatches;
+	return Math.ceil(patches * PATCH_TOKEN_MULTIPLIER);
+}
+
+// Enough decoded bytes to reach a JPEG SOF marker behind typical EXIF/ICC
+// segments; PNG/GIF/WebP need only the first 30 bytes. Multiple of 4 so the
+// base64 slice decodes cleanly.
+const HEADER_BASE64_CHARS = 4 * Math.ceil((64 * 1024) / 3);
+
+/** Pixel size read from the header of base64-encoded image data, or null. */
+export function base64ImageSize(base64: string): ImageSize | null {
+	if (base64.length === 0) return null;
+	const header = Buffer.from(base64.slice(0, HEADER_BASE64_CHARS), "base64");
+	const metadata = parseImageMetadata(header);
+	if (!metadata?.width || !metadata.height) return null;
+	return { width: metadata.width, height: metadata.height };
+}
+
+/** Pixel size of a `data:<mime>;base64,` URL, or null for any other URL. */
+export function dataUrlImageSize(url: string): ImageSize | null {
+	if (!url.startsWith("data:")) return null;
+	const comma = url.indexOf(",");
+	if (comma < 0 || !url.slice(0, comma).endsWith(";base64")) return null;
+	return base64ImageSize(url.slice(comma + 1));
+}
+
+/** {@link estimateImageTokens} for an in-memory image block. */
+export function estimateImageContentTokens(image: Pick<ImageContent, "data" | "detail">): number {
+	return estimateImageTokens(base64ImageSize(image.data), image.detail);
+}

package/src/tokenizer.ts

@@ -5,6 +5,7 @@ import { materializeString, stringifyJson } from "@oh-my-pi/pi-utils";
 import { LRUCache } from "@oh-my-pi/pi-utils/lru";
 import * as snapcompact from "@oh-my-pi/snapcompact";
 import { isEstimateCacheable, messageEstimateVersion } from "./compaction/message-cache";
+import { estimateImageContentTokens } from "./image-tokens";
 import type { AgentMessage } from "./types";
 
 const testEnv = Bun.env.NODE_ENV === "test";
@@ -131,12 +132,6 @@ export interface TokenBudgetCheck {
 	exact: boolean;
 }
 
-/**
- * Image content has no tokenizer representation; charge a fixed estimate
- * matching what providers typically bill for inline images.
- */
-const IMAGE_TOKEN_ESTIMATE = 1200;
-
 /**
  * Memoized estimates for one message under this tokenizer's encoding, split by
  * the {@link MessageCountOptions.excludeEncryptedReasoning} option so the two
@@ -219,7 +214,8 @@ export class Tokenizer {
 	 * Settled historical messages are counted once and reused until an owner
 	 * (prune/shake/strip-images) calls `invalidateMessageCache`; streaming
 	 * assistants bypass the memo entirely (see the message-cache settle-gate
-	 * invariant). Image blocks charge a fixed per-image estimate.
+	 * invariant). Image blocks charge {@link estimateImageContentTokens}, the
+	 * same dimension-based estimate the native remote-compaction probe applies.
 	 */
 	countMessage(message: AgentMessage, options?: MessageCountOptions): number {
 		const floored = options?.excludeEncryptedReasoning === true;
@@ -273,7 +269,7 @@ export class Tokenizer {
 						if (block.type === "text" && block.text) {
 							fragments.push(block.text);
 						} else if (block.type === "image") {
-							extra += IMAGE_TOKEN_ESTIMATE;
+							extra += estimateImageContentTokens(block);
 						}
 					}
 				}
@@ -323,7 +319,7 @@ export class Tokenizer {
 						if (block.type === "text" && block.text) {
 							fragments.push(block.text);
 						} else if (block.type === "image") {
-							extra += IMAGE_TOKEN_ESTIMATE;
+							extra += estimateImageContentTokens(block);
 						}
 					}
 				}