@oh-my-pi/pi-agent-core 18.0.10 → 18.1.0

package/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## [Unreleased]
 
+## [18.0.11] - 2026-08-29
+
+### Fixed
+
+- Fixed agent startup and context compaction failures for models with unrecognized tokenizer encodings.
+
 ## [18.0.10] - 2026-08-28
 
 ### Added

package/dist/types/tokenizer.d.ts

@@ -1,11 +1,12 @@
 import type { Model } from "@oh-my-pi/pi-ai";
-import { Encoding } from "@oh-my-pi/pi-natives";
+import * as natives from "@oh-my-pi/pi-natives";
 import type { AgentMessage } from "./types.js";
 /** Maps the catalog-resolved tokenizer family to its native implementation. */
-export declare function tokenizerEncodingForModel(model: Pick<Model, "tokenizer"> | null | undefined): Encoding | null;
+export declare function tokenizerEncodingForModel(model: Pick<Model, "tokenizer"> | null | undefined): natives.Encoding | null;
 /**
- * `strict` always pays for an exact native count (the catalog-resolved
- * tokenizer when known, o200k_base otherwise). `approximate` and
+ * `strict` always tries an exact native count (the catalog-resolved tokenizer
+ * when known, o200k_base otherwise), falling back to the byte upper bound when
+ * the loaded addon does not recognize that encoding. `approximate` and
  * `upperbound` prefer the same exact count for known tokenizer families or
  * when `PI_TOKENIZER_ACCURATE=1` is set; otherwise they use a cheap heuristic:
  * `approximate` a bytes/4 guess, `upperbound` the raw byte length (never
@@ -31,11 +32,10 @@ export interface TokenBudgetCheck {
     fits: boolean;
     /**
      * Token count behind the verdict: the exact native count when `exact` is
-     * set, otherwise the cheap byte upper bound (which already fit, so it is
-     * only an over-estimate of a count known to be under budget).
+     * set, otherwise the conservative byte upper bound.
      */
     tokens: number;
-    /** Whether the exact tokenizer had to run because the cheap bound busted. */
+    /** Whether `tokens` came from the exact native tokenizer. */
     exact: boolean;
 }
 /**
@@ -50,7 +50,7 @@ export interface TokenBudgetCheck {
 export declare class Tokenizer {
     #private;
     constructor(model?: Pick<Model, "tokenizer"> | null);
-    get encoding(): Encoding | null;
+    get encoding(): natives.Encoding | null;
     countTokens(text: string | string[], mode?: TokenCountMode): number;
     /**
      * Cheap-first budget probe — the way to ask "does this fit in `budget`

package/package.json

@@ -1,7 +1,7 @@
 {
 	"type": "module",
 	"name": "@oh-my-pi/pi-agent-core",
-	"version": "18.0.10",
+	"version": "18.1.0",
 	"description": "General-purpose agent with transport abstraction, state management, and attachment support",
 	"homepage": "https://omp.sh",
 	"author": "Stencil Labs, Inc.",
@@ -27,24 +27,24 @@
 	"main": "./src/index.ts",
 	"types": "./dist/types/index.d.ts",
 	"scripts": {
-		"check": "biome check . && bun run check:types",
+		"check": "oxlint . && oxfmt --check --no-error-on-unmatched-pattern 'src/**/*.{ts,tsx}' '{test,bench,examples,scripts}/**/*.ts' '*.ts' && bun run check:types",
 		"check:types": "tsgo -p tsconfig.json --noEmit",
-		"lint": "biome lint .",
+		"lint": "oxlint .",
 		"test": "bun test --parallel",
-		"fix": "biome check --write --unsafe .",
-		"fmt": "biome format --write ."
+		"fix": "oxlint --fix --fix-suggestions . && bun run fmt",
+		"fmt": "oxfmt --no-error-on-unmatched-pattern 'src/**/*.{ts,tsx}' '{test,bench,examples,scripts}/**/*.ts' '*.ts'"
 	},
 	"dependencies": {
-		"@oh-my-pi/pi-ai": "18.0.10",
-		"@oh-my-pi/pi-catalog": "18.0.10",
-		"@oh-my-pi/pi-natives": "18.0.10",
-		"@oh-my-pi/pi-utils": "18.0.10",
-		"@oh-my-pi/pi-wire": "18.0.10",
-		"@oh-my-pi/snapcompact": "18.0.10",
+		"@oh-my-pi/pi-ai": "18.1.0",
+		"@oh-my-pi/pi-catalog": "18.1.0",
+		"@oh-my-pi/pi-natives": "18.1.0",
+		"@oh-my-pi/pi-utils": "18.1.0",
+		"@oh-my-pi/pi-wire": "18.1.0",
+		"@oh-my-pi/snapcompact": "18.1.0",
 		"@opentelemetry/api": "^1.9.1"
 	},
 	"devDependencies": {
-		"@oh-my-pi/omptype": "18.0.10",
+		"@oh-my-pi/omptype": "18.1.0",
 		"@opentelemetry/context-async-hooks": "^2.9.0",
 		"@opentelemetry/sdk-trace-base": "^2.9.0",
 		"@types/bun": "^1.3.14"

package/src/agent-loop.ts

@@ -768,7 +768,7 @@ function createDetailedCapture(config: AgentLoopConfig): {
 	const wired: AgentLoopConfig = {
 		...config,
 		telemetry: {
-			...(config.telemetry ?? {}),
+			...config.telemetry,
 			onRunEnd: (summary, coverage) => {
 				captured = { summary, coverage };
 				userHook?.(summary, coverage);

package/src/compaction/compaction-v2-streaming.ts

@@ -324,9 +324,7 @@ async function attemptCompactionV2Streaming(
 		...(request.reasoning || model.useResponsesLite
 			? {
 					// Lite implies gpt-5.4+, where codex-rs sends `all_turns` replay.
-					reasoning: model.useResponsesLite
-						? { ...(request.reasoning ?? {}), context: "all_turns" }
-						: request.reasoning,
+					reasoning: model.useResponsesLite ? { ...request.reasoning, context: "all_turns" } : request.reasoning,
 					include: ["reasoning.encrypted_content"],
 				}
 			: {}),
@@ -403,7 +401,7 @@ function buildCompactionV2Headers(
 			? {
 					"content-type": "application/json",
 					"api-key": apiKey,
-					...(model.headers ?? {}),
+					...model.headers,
 				}
 			: {
 					"content-type": "application/json",

package/src/compaction/compaction.ts

@@ -1644,7 +1644,7 @@ export async function compact(
 						}),
 					{ signal },
 				);
-				preserveData = { ...(preserveData ?? {}), ...storeCompactionV2PreserveData(remote, model) };
+				preserveData = { ...preserveData, ...storeCompactionV2PreserveData(remote, model) };
 				usedRemoteCompaction = true;
 			} catch (err) {
 				// A user/session abort is a cancellation, not a remote failure —

package/src/compaction/openai.ts

@@ -390,7 +390,7 @@ export function withOpenAiRemoteCompactionPreserveData(
 ): Record<string, unknown> | undefined {
 	if (remoteCompaction) {
 		return {
-			...(preserveData ?? {}),
+			...preserveData,
 			[OPENAI_REMOTE_COMPACTION_PRESERVE_KEY]: remoteCompaction,
 		};
 	}
@@ -803,12 +803,12 @@ export async function requestOpenAiRemoteCompaction(
 		? {
 				"content-type": "application/json",
 				"api-key": apiKey,
-				...(model.headers ?? {}),
+				...model.headers,
 			}
 		: {
 				"content-type": "application/json",
 				Authorization: `Bearer ${apiKey}`,
-				...(model.headers ?? {}),
+				...model.headers,
 			};
 
 	// Codex endpoints require additional auth headers

package/src/compaction/pruning.ts

@@ -141,6 +141,7 @@ function estimatePrunedSavings(tokens: number, notice: string): number {
  * mutations inside the cheap-to-recache tail.
  */
 function computeMessageSuffixTokens(entries: readonly SessionEntry[], tokenizer: Tokenizer): number[] {
+	// oxlint-disable-next-line unicorn/no-new-array -- length preallocation
 	const suffix = new Array<number>(entries.length);
 	let accumulated = 0;
 	for (let i = entries.length - 1; i >= 0; i--) {

package/src/compaction/shake.ts

@@ -318,6 +318,7 @@ export function collectShakeRegions(entries: SessionEntry[], tokenizer: Tokenize
 	if (n === 0) return [];
 
 	// Tokens of all entries strictly more recent than index i.
+	// oxlint-disable-next-line unicorn/no-new-array -- length preallocation
 	const accumulatedAfter = new Array<number>(n);
 	let acc = 0;
 	for (let i = n - 1; i >= 0; i--) {

package/src/tokenizer.ts

@@ -1,6 +1,6 @@
 import type { Model } from "@oh-my-pi/pi-ai";
 import type { ModelTokenizer } from "@oh-my-pi/pi-catalog/types";
-import { countTokens as countTokensNat, Encoding } from "@oh-my-pi/pi-natives";
+import * as natives from "@oh-my-pi/pi-natives";
 import { stringifyJson } from "@oh-my-pi/pi-utils";
 import * as snapcompact from "@oh-my-pi/snapcompact";
 import { isEstimateCacheable, messageEstimateVersion } from "./compaction/message-cache";
@@ -9,25 +9,26 @@ import type { AgentMessage } from "./types";
 const testEnv = Bun.env.NODE_ENV === "test";
 const accurate = process.env.PI_TOKENIZER_ACCURATE === "1" && !testEnv;
 
-const NATIVE_ENCODING: Record<ModelTokenizer, Encoding> = {
-	"claude-v3": Encoding.ClaudeV3,
-	"claude-v47": Encoding.ClaudeV47,
-	"claude-v5": Encoding.ClaudeV5,
-	"claude-v5-sonnet": Encoding.ClaudeV5Sonnet,
-	qwen3: Encoding.Qwen3,
-	"deepseek-v3": Encoding.DeepSeekV3,
-	"kimi-k2": Encoding.KimiK2,
-	glm5: Encoding.Glm5,
+const NATIVE_ENCODING: Record<ModelTokenizer, natives.Encoding> = {
+	"claude-v3": natives.Encoding.ClaudeV3,
+	"claude-v47": natives.Encoding.ClaudeV47,
+	"claude-v5": natives.Encoding.ClaudeV5,
+	"claude-v5-sonnet": natives.Encoding.ClaudeV5Sonnet,
+	qwen3: natives.Encoding.Qwen3,
+	"deepseek-v3": natives.Encoding.DeepSeekV3,
+	"kimi-k2": natives.Encoding.KimiK2,
+	glm5: natives.Encoding.Glm5,
 };
 
 /** Maps the catalog-resolved tokenizer family to its native implementation. */
-export function tokenizerEncodingForModel(model: Pick<Model, "tokenizer"> | null | undefined): Encoding | null {
+export function tokenizerEncodingForModel(model: Pick<Model, "tokenizer"> | null | undefined): natives.Encoding | null {
 	return model?.tokenizer ? NATIVE_ENCODING[model.tokenizer] : null;
 }
 
 /**
- * `strict` always pays for an exact native count (the catalog-resolved
- * tokenizer when known, o200k_base otherwise). `approximate` and
+ * `strict` always tries an exact native count (the catalog-resolved tokenizer
+ * when known, o200k_base otherwise), falling back to the byte upper bound when
+ * the loaded addon does not recognize that encoding. `approximate` and
  * `upperbound` prefer the same exact count for known tokenizer families or
  * when `PI_TOKENIZER_ACCURATE=1` is set; otherwise they use a cheap heuristic:
  * `approximate` a bytes/4 guess, `upperbound` the raw byte length (never
@@ -61,17 +62,41 @@ function sumFragments(text: string | string[], perFragment: (t: string) => numbe
 	return Array.isArray(text) ? text.reduce((sum, t) => sum + perFragment(t), 0) : perFragment(text);
 }
 
+interface NativeTokenCount {
+	tokens: number;
+	exact: boolean;
+}
+
+function countTokensNat(
+	text: string | string[],
+	encoding: natives.Encoding | null | undefined,
+	mode: TokenCountMode,
+): NativeTokenCount {
+	try {
+		return { tokens: natives.countTokens(text, encoding), exact: true };
+	} catch (error) {
+		if (
+			!(error instanceof Error) ||
+			(!error.message.includes("does not match any variant of enum") &&
+				!error.message.includes("unknown enum variant"))
+		) {
+			throw error;
+		}
+		const tokens = sumFragments(text, mode === "approximate" ? byteEstimate : byteLength);
+		return { tokens, exact: false };
+	}
+}
+
 /** Verdict from {@link Tokenizer.checkTokenBudget}. */
 export interface TokenBudgetCheck {
 	/** Whether the text fits the budget. */
 	fits: boolean;
 	/**
 	 * Token count behind the verdict: the exact native count when `exact` is
-	 * set, otherwise the cheap byte upper bound (which already fit, so it is
-	 * only an over-estimate of a count known to be under budget).
+	 * set, otherwise the conservative byte upper bound.
 	 */
 	tokens: number;
-	/** Whether the exact tokenizer had to run because the cheap bound busted. */
+	/** Whether `tokens` came from the exact native tokenizer. */
 	exact: boolean;
 }
 
@@ -104,7 +129,7 @@ interface MessageEstimate {
  * `PI_TOKENIZER_ACCURATE=1`).
  */
 export class Tokenizer {
-	readonly #encoding: Encoding | null;
+	readonly #encoding: natives.Encoding | null;
 
 	/**
 	 * Per-message estimate memo. Keyed by message identity, deliberately not a
@@ -120,14 +145,14 @@ export class Tokenizer {
 		this.#encoding = tokenizerEncodingForModel(model);
 	}
 
-	get encoding(): Encoding | null {
+	get encoding(): natives.Encoding | null {
 		return this.#encoding;
 	}
 
 	countTokens(text: string | string[], mode: TokenCountMode = "approximate"): number {
-		if (mode === "strict") return countTokensNat(text, this.#encoding);
-		if (!testEnv && this.#encoding !== null) return countTokensNat(text, this.#encoding);
-		if (accurate) return countTokensNat(text);
+		if (mode === "strict") return countTokensNat(text, this.#encoding, mode).tokens;
+		if (!testEnv && this.#encoding !== null) return countTokensNat(text, this.#encoding, mode).tokens;
+		if (accurate) return countTokensNat(text, undefined, mode).tokens;
 		return sumFragments(text, mode === "upperbound" ? byteLength : byteEstimate);
 	}
 
@@ -145,8 +170,8 @@ export class Tokenizer {
 	checkTokenBudget(text: string | string[], budget: number): TokenBudgetCheck {
 		const bound = sumFragments(text, byteLength);
 		if (bound <= budget) return { fits: true, tokens: bound, exact: false };
-		const tokens = this.countTokens(text, "strict");
-		return { fits: tokens <= budget, tokens, exact: true };
+		const result = countTokensNat(text, this.#encoding, "strict");
+		return { fits: result.tokens <= budget, tokens: result.tokens, exact: result.exact };
 	}
 
 	/**

package/src/types.ts

@@ -759,8 +759,11 @@ export type AgentToolExecFn<TParameters extends TSchema = TSchema, TDetails = an
 ) => Promise<AgentToolResult<TDetails, TParameters>>;
 
 // AgentTool extends Tool but adds the execute function
-export interface AgentTool<TParameters extends TSchema = TSchema, TDetails = any, TTheme = unknown>
-	extends Tool<TParameters> {
+export interface AgentTool<
+	TParameters extends TSchema = TSchema,
+	TDetails = any,
+	TTheme = unknown,
+> extends Tool<TParameters> {
 	// A human-readable label for the tool to be displayed in UI
 	label: string;
 	/** If true, tool is excluded unless explicitly listed in --tools or agent's tools field */