@oh-my-pi/pi-agent-core 18.0.10 → 18.0.11

package/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## [Unreleased]
 
+## [18.0.11] - 2026-08-29
+
+### Fixed
+
+- Fixed agent startup and context compaction failures for models with unrecognized tokenizer encodings.
+
 ## [18.0.10] - 2026-08-28
 
 ### Added

package/dist/types/tokenizer.d.ts

@@ -1,11 +1,12 @@
 import type { Model } from "@oh-my-pi/pi-ai";
-import { Encoding } from "@oh-my-pi/pi-natives";
+import * as natives from "@oh-my-pi/pi-natives";
 import type { AgentMessage } from "./types.js";
 /** Maps the catalog-resolved tokenizer family to its native implementation. */
-export declare function tokenizerEncodingForModel(model: Pick<Model, "tokenizer"> | null | undefined): Encoding | null;
+export declare function tokenizerEncodingForModel(model: Pick<Model, "tokenizer"> | null | undefined): natives.Encoding | null;
 /**
- * `strict` always pays for an exact native count (the catalog-resolved
- * tokenizer when known, o200k_base otherwise). `approximate` and
+ * `strict` always tries an exact native count (the catalog-resolved tokenizer
+ * when known, o200k_base otherwise), falling back to the byte upper bound when
+ * the loaded addon does not recognize that encoding. `approximate` and
  * `upperbound` prefer the same exact count for known tokenizer families or
  * when `PI_TOKENIZER_ACCURATE=1` is set; otherwise they use a cheap heuristic:
  * `approximate` a bytes/4 guess, `upperbound` the raw byte length (never
@@ -31,11 +32,10 @@ export interface TokenBudgetCheck {
     fits: boolean;
     /**
      * Token count behind the verdict: the exact native count when `exact` is
-     * set, otherwise the cheap byte upper bound (which already fit, so it is
-     * only an over-estimate of a count known to be under budget).
+     * set, otherwise the conservative byte upper bound.
      */
     tokens: number;
-    /** Whether the exact tokenizer had to run because the cheap bound busted. */
+    /** Whether `tokens` came from the exact native tokenizer. */
     exact: boolean;
 }
 /**
@@ -50,7 +50,7 @@ export interface TokenBudgetCheck {
 export declare class Tokenizer {
     #private;
     constructor(model?: Pick<Model, "tokenizer"> | null);
-    get encoding(): Encoding | null;
+    get encoding(): natives.Encoding | null;
     countTokens(text: string | string[], mode?: TokenCountMode): number;
     /**
      * Cheap-first budget probe — the way to ask "does this fit in `budget`

package/package.json

@@ -1,7 +1,7 @@
 {
 	"type": "module",
 	"name": "@oh-my-pi/pi-agent-core",
-	"version": "18.0.10",
+	"version": "18.0.11",
 	"description": "General-purpose agent with transport abstraction, state management, and attachment support",
 	"homepage": "https://omp.sh",
 	"author": "Stencil Labs, Inc.",
@@ -35,16 +35,16 @@
 		"fmt": "biome format --write ."
 	},
 	"dependencies": {
-		"@oh-my-pi/pi-ai": "18.0.10",
-		"@oh-my-pi/pi-catalog": "18.0.10",
-		"@oh-my-pi/pi-natives": "18.0.10",
-		"@oh-my-pi/pi-utils": "18.0.10",
-		"@oh-my-pi/pi-wire": "18.0.10",
-		"@oh-my-pi/snapcompact": "18.0.10",
+		"@oh-my-pi/pi-ai": "18.0.11",
+		"@oh-my-pi/pi-catalog": "18.0.11",
+		"@oh-my-pi/pi-natives": "18.0.11",
+		"@oh-my-pi/pi-utils": "18.0.11",
+		"@oh-my-pi/pi-wire": "18.0.11",
+		"@oh-my-pi/snapcompact": "18.0.11",
 		"@opentelemetry/api": "^1.9.1"
 	},
 	"devDependencies": {
-		"@oh-my-pi/omptype": "18.0.10",
+		"@oh-my-pi/omptype": "18.0.11",
 		"@opentelemetry/context-async-hooks": "^2.9.0",
 		"@opentelemetry/sdk-trace-base": "^2.9.0",
 		"@types/bun": "^1.3.14"

package/src/tokenizer.ts

@@ -1,6 +1,6 @@
 import type { Model } from "@oh-my-pi/pi-ai";
 import type { ModelTokenizer } from "@oh-my-pi/pi-catalog/types";
-import { countTokens as countTokensNat, Encoding } from "@oh-my-pi/pi-natives";
+import * as natives from "@oh-my-pi/pi-natives";
 import { stringifyJson } from "@oh-my-pi/pi-utils";
 import * as snapcompact from "@oh-my-pi/snapcompact";
 import { isEstimateCacheable, messageEstimateVersion } from "./compaction/message-cache";
@@ -9,25 +9,26 @@ import type { AgentMessage } from "./types";
 const testEnv = Bun.env.NODE_ENV === "test";
 const accurate = process.env.PI_TOKENIZER_ACCURATE === "1" && !testEnv;
 
-const NATIVE_ENCODING: Record<ModelTokenizer, Encoding> = {
-	"claude-v3": Encoding.ClaudeV3,
-	"claude-v47": Encoding.ClaudeV47,
-	"claude-v5": Encoding.ClaudeV5,
-	"claude-v5-sonnet": Encoding.ClaudeV5Sonnet,
-	qwen3: Encoding.Qwen3,
-	"deepseek-v3": Encoding.DeepSeekV3,
-	"kimi-k2": Encoding.KimiK2,
-	glm5: Encoding.Glm5,
+const NATIVE_ENCODING: Record<ModelTokenizer, natives.Encoding> = {
+	"claude-v3": natives.Encoding.ClaudeV3,
+	"claude-v47": natives.Encoding.ClaudeV47,
+	"claude-v5": natives.Encoding.ClaudeV5,
+	"claude-v5-sonnet": natives.Encoding.ClaudeV5Sonnet,
+	qwen3: natives.Encoding.Qwen3,
+	"deepseek-v3": natives.Encoding.DeepSeekV3,
+	"kimi-k2": natives.Encoding.KimiK2,
+	glm5: natives.Encoding.Glm5,
 };
 
 /** Maps the catalog-resolved tokenizer family to its native implementation. */
-export function tokenizerEncodingForModel(model: Pick<Model, "tokenizer"> | null | undefined): Encoding | null {
+export function tokenizerEncodingForModel(model: Pick<Model, "tokenizer"> | null | undefined): natives.Encoding | null {
 	return model?.tokenizer ? NATIVE_ENCODING[model.tokenizer] : null;
 }
 
 /**
- * `strict` always pays for an exact native count (the catalog-resolved
- * tokenizer when known, o200k_base otherwise). `approximate` and
+ * `strict` always tries an exact native count (the catalog-resolved tokenizer
+ * when known, o200k_base otherwise), falling back to the byte upper bound when
+ * the loaded addon does not recognize that encoding. `approximate` and
  * `upperbound` prefer the same exact count for known tokenizer families or
  * when `PI_TOKENIZER_ACCURATE=1` is set; otherwise they use a cheap heuristic:
  * `approximate` a bytes/4 guess, `upperbound` the raw byte length (never
@@ -61,17 +62,41 @@ function sumFragments(text: string | string[], perFragment: (t: string) => numbe
 	return Array.isArray(text) ? text.reduce((sum, t) => sum + perFragment(t), 0) : perFragment(text);
 }
 
+interface NativeTokenCount {
+	tokens: number;
+	exact: boolean;
+}
+
+function countTokensNat(
+	text: string | string[],
+	encoding: natives.Encoding | null | undefined,
+	mode: TokenCountMode,
+): NativeTokenCount {
+	try {
+		return { tokens: natives.countTokens(text, encoding), exact: true };
+	} catch (error) {
+		if (
+			!(error instanceof Error) ||
+			(!error.message.includes("does not match any variant of enum") &&
+				!error.message.includes("unknown enum variant"))
+		) {
+			throw error;
+		}
+		const tokens = sumFragments(text, mode === "approximate" ? byteEstimate : byteLength);
+		return { tokens, exact: false };
+	}
+}
+
 /** Verdict from {@link Tokenizer.checkTokenBudget}. */
 export interface TokenBudgetCheck {
 	/** Whether the text fits the budget. */
 	fits: boolean;
 	/**
 	 * Token count behind the verdict: the exact native count when `exact` is
-	 * set, otherwise the cheap byte upper bound (which already fit, so it is
-	 * only an over-estimate of a count known to be under budget).
+	 * set, otherwise the conservative byte upper bound.
 	 */
 	tokens: number;
-	/** Whether the exact tokenizer had to run because the cheap bound busted. */
+	/** Whether `tokens` came from the exact native tokenizer. */
 	exact: boolean;
 }
 
@@ -104,7 +129,7 @@ interface MessageEstimate {
  * `PI_TOKENIZER_ACCURATE=1`).
  */
 export class Tokenizer {
-	readonly #encoding: Encoding | null;
+	readonly #encoding: natives.Encoding | null;
 
 	/**
 	 * Per-message estimate memo. Keyed by message identity, deliberately not a
@@ -120,14 +145,14 @@ export class Tokenizer {
 		this.#encoding = tokenizerEncodingForModel(model);
 	}
 
-	get encoding(): Encoding | null {
+	get encoding(): natives.Encoding | null {
 		return this.#encoding;
 	}
 
 	countTokens(text: string | string[], mode: TokenCountMode = "approximate"): number {
-		if (mode === "strict") return countTokensNat(text, this.#encoding);
-		if (!testEnv && this.#encoding !== null) return countTokensNat(text, this.#encoding);
-		if (accurate) return countTokensNat(text);
+		if (mode === "strict") return countTokensNat(text, this.#encoding, mode).tokens;
+		if (!testEnv && this.#encoding !== null) return countTokensNat(text, this.#encoding, mode).tokens;
+		if (accurate) return countTokensNat(text, undefined, mode).tokens;
 		return sumFragments(text, mode === "upperbound" ? byteLength : byteEstimate);
 	}
 
@@ -145,8 +170,8 @@ export class Tokenizer {
 	checkTokenBudget(text: string | string[], budget: number): TokenBudgetCheck {
 		const bound = sumFragments(text, byteLength);
 		if (bound <= budget) return { fits: true, tokens: bound, exact: false };
-		const tokens = this.countTokens(text, "strict");
-		return { fits: tokens <= budget, tokens, exact: true };
+		const result = countTokensNat(text, this.#encoding, "strict");
+		return { fits: result.tokens <= budget, tokens: result.tokens, exact: result.exact };
 	}
 
 	/**