@sayknow-cli/agent-core 0.3.12 → 0.3.15

package/CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ## [Unreleased]
 
+## [0.10.0] - 2026-07-12
+
+### Fixed
+
+- The native-free token heuristic is now script-aware: common-BMP CJK characters (Hangul, unified/compat Han, Kana, CJK punctuation, full-width forms) are charged at 1 token each (measured o200k_base upper bound 0.96 tokens/char) and supplementary code points (surrogate pairs: rare Han extensions, emoji) at 1 token per code point, instead of chars/4 for everything. The old estimate undercounted Korean/CJK-heavy unsent context by 2–4x and could delay threshold compaction past the provider window; ASCII estimates are unchanged. `boundConversationTextForSummary` now derives its truncation cut from the text's own estimated token density, validates the complete assembled excerpt (elision marker included) against the estimator, and fails closed — bare marker only when the marker itself fits the budget, otherwise an empty excerpt, including when the computed input budget is non-positive — instead of assuming 4 chars/token and returning over-budget or unbounded text.
+
+- A tool call for a name absent from the active tool set now appends a recovery hint pointing at `search_tool_bm25` (gated on a callable `search_tool_bm25`, matched by internal name or `customWireName`), so a model no longer abandons a discoverable tool such as `task` after a bare "Tool <name> not found"; the base error wording stays byte-for-byte stable when discovery is unavailable (#2042).
+
 ## [0.9.2] - 2026-07-09
 
 ### Fixed

package/dist/types/agent.d.ts

@@ -5,6 +5,12 @@ import { type AssistantMessage, type AssistantMessageEvent, type CursorExecHandl
 import type { AppendOnlyContextManager } from "./append-only-context";
 import type { HarmonyAuditEvent } from "./harmony-leak";
 import type { AgentEvent, AgentLoopConfig, AgentMessage, AgentState, AgentTool, AgentToolContext, StreamFn, ToolCallContext } from "./types";
+/**
+ * Whether persisted history ends at a point where a new model turn can resume.
+ * Assistant-ended histories require an in-memory queued message and are handled
+ * separately by `Agent.continue()`.
+ */
+export declare function canContinuePersistedHistory(messages: readonly AgentMessage[]): boolean;
 export declare class AgentBusyError extends Error {
     constructor(message?: string);
 }
@@ -292,6 +298,7 @@ export declare class Agent {
     get streamMaxRetries(): number | undefined;
     set streamMaxRetries(value: number | undefined);
     get state(): AgentState;
+    get contextRevision(): number;
     get appendOnlyContext(): AppendOnlyContextManager | undefined;
     setAppendOnlyContext(manager?: AppendOnlyContextManager): void;
     subscribe(fn: (e: AgentEvent) => void): () => void;
@@ -315,6 +322,11 @@ export declare class Agent {
     replaceMessages(ms: AgentMessage[]): void;
     appendMessage(m: AgentMessage): void;
     popMessage(): AgentMessage | undefined;
+    /**
+     * For callers that mutate committed messages or the system prompt in place
+     * outside Agent-owned mutators.
+     */
+    touchContext(): void;
     /**
      * Queue a steering message to interrupt the agent mid-run.
      * Delivered after current tool execution, skips remaining tools.

package/dist/types/compaction/compaction.d.ts

@@ -118,7 +118,7 @@ export declare function resolveThresholdTokens(contextWindow: number, settings:
  */
 export declare const IMAGE_TOKEN_ESTIMATE = 1200;
 /**
- * Native-free chars/4 token estimate for a message. This is the only message
+ * Native-free token estimate for a message. This is the only message
  * token estimator: provider usage (see {@link calculatePromptTokens}) anchors
  * the already-sent context, and this covers unsent/trailing deltas, per-entry
  * budgeting, and display surfaces. Callers add a conservative inflation factor
@@ -126,8 +126,8 @@ export declare const IMAGE_TOKEN_ESTIMATE = 1200;
  */
 export declare function estimateMessageTokensHeuristic(message: AgentMessage): number;
 /**
- * Native-free chars/4 token estimate for plain string fragments. Fragment-level
- * counterpart of {@link estimateMessageTokensHeuristic}.
+ * Script-aware native-free token estimate for plain string fragments.
+ * Fragment-level counterpart of {@link estimateMessageTokensHeuristic}.
  */
 export declare function estimateTextTokensHeuristic(fragments: string | readonly string[]): number;
 export declare function estimateEntryTokens(entry: SessionEntry): number;
@@ -207,8 +207,8 @@ export interface SummaryOptions {
  * on the very overflow the recovery was meant to absorb.
  *
  * The budget reserves the summary's own output tokens plus prompt/system/template
- * overhead, and applies a conservative safety factor because the chars/4 heuristic
- * undercounts dense or CJK text (the reason the original overflow was missed).
+ * overhead, and applies a conservative safety factor for estimator error on
+ * dense text (the reason the original overflow was missed).
  * Truncation keeps the head (origin/goals) and the tail (most recent state) and
  * elides the middle; it is a last resort that only triggers when the input would
  * otherwise not fit.

package/package.json

@@ -1,7 +1,7 @@
 {
 	"type": "module",
 	"name": "@sayknow-cli/agent-core",
-	"version": "0.3.12",
+	"version": "0.3.15",
 	"description": "General-purpose agent with transport abstraction, state management, and attachment support",
 	"homepage": "https://sayknow-cli.com",
 	"author": "jaybeyond",
@@ -35,9 +35,9 @@
 		"fmt": "biome format --write ."
 	},
 	"dependencies": {
-		"@sayknow-cli/ai": "0.3.12",
-		"@sayknow-cli/natives": "0.3.12",
-		"@sayknow-cli/utils": "0.3.12",
+		"@sayknow-cli/ai": "0.3.15",
+		"@sayknow-cli/natives": "0.3.15",
+		"@sayknow-cli/utils": "0.3.15",
 		"@opentelemetry/api": "^1.9.0"
 	},
 	"devDependencies": {

package/src/agent-loop.ts

@@ -1092,6 +1092,17 @@ function emitAbortedAssistantMessage(
 	return abortedMessage;
 }
 
+/**
+ * Match a tool against the model-visible call name. Tools emitted via OpenAI's
+ * custom-tool path (e.g. `apply_patch` on GPT-5) arrive under their wire-level
+ * name, which may differ from the harness-internal `name`, so dispatch and any
+ * "is this tool callable" check must consider both. Internal `name` takes
+ * precedence when a caller needs a single match.
+ */
+function toolMatchesCallName(tool: { name: string; customWireName?: string }, callName: string): boolean {
+	return tool.name === callName || (tool.customWireName !== undefined && tool.customWireName === callName);
+}
+
 /**
  * Execute tool calls from an assistant message.
  */
@@ -1273,7 +1284,22 @@ async function executeToolCalls(
 							`Re-issue the call with complete arguments, splitting the work into smaller steps if needed.`,
 					);
 				}
-				if (!tool) throw new Error(`Tool ${toolCall.name} not found`);
+				if (!tool) {
+					// A discoverable tool that hasn't been activated yet resolves to
+					// undefined here. The model often "remembers" such a tool (e.g.
+					// `task`) from earlier context and calls it by name without first
+					// re-discovering it. Point it at tool discovery so it can activate
+					// the tool and retry instead of giving up on the capability. The
+					// base wording stays byte-for-byte stable for downstream consumers;
+					// the period and hint are appended only when discovery is callable.
+					const base = `Tool ${toolCall.name} not found`;
+					const hasToolDiscovery = tools?.some(t => toolMatchesCallName(t, "search_tool_bm25")) ?? false;
+					throw new Error(
+						hasToolDiscovery
+							? `${base}. If you are unsure whether this tool exists or how to use it, call \`search_tool_bm25\` to discover and activate the matching tool, then retry.`
+							: base,
+					);
+				}
 
 				let effectiveArgs: Record<string, unknown>;
 				try {

package/src/agent.ts

@@ -53,6 +53,16 @@ function assertUserImagePlaceholdersHavePayload(messages: readonly AgentMessage[
 	}
 }
 
+/**
+ * Whether persisted history ends at a point where a new model turn can resume.
+ * Assistant-ended histories require an in-memory queued message and are handled
+ * separately by `Agent.continue()`.
+ */
+export function canContinuePersistedHistory(messages: readonly AgentMessage[]): boolean {
+	const lastMessage = messages.at(-1);
+	return lastMessage !== undefined && lastMessage.role !== "assistant";
+}
+
 /**
  * Default convertToLlm: Keep only LLM-compatible messages, convert attachments.
  */
@@ -286,6 +296,7 @@ export class Agent {
 		pendingToolCalls: new Set<string>(),
 		error: undefined,
 	};
+	#contextRevision = 0;
 
 	#listeners = new Set<(e: AgentEvent) => void>();
 	#abortController?: AbortController;
@@ -630,6 +641,10 @@ export class Agent {
 		return this.#state;
 	}
 
+	get contextRevision(): number {
+		return this.#contextRevision;
+	}
+
 	get appendOnlyContext(): AppendOnlyContextManager | undefined {
 		return this.#appendOnlyContext;
 	}
@@ -812,10 +827,12 @@ export class Agent {
 	// State mutators
 	setSystemPrompt(v: string[]) {
 		this.#state.systemPrompt = v;
+		this.#contextRevision++;
 	}
 
 	setModel(m: Model) {
 		this.#state.model = m;
+		this.#contextRevision++;
 	}
 
 	setThinkingLevel(l: Effort | undefined) {
@@ -848,10 +865,12 @@ export class Agent {
 
 	setTools(t: AgentTool<any>[]) {
 		this.#state.tools = t;
+		this.#contextRevision++;
 	}
 
 	replaceMessages(ms: AgentMessage[]) {
 		this.#state.messages = ms.slice();
+		this.#contextRevision++;
 	}
 
 	appendMessage(m: AgentMessage) {
@@ -859,12 +878,14 @@ export class Agent {
 		// N is O(N+M), not O(M*N). Consumers read state.messages fresh; run() snapshots
 		// via slice() at the API boundary, so no caller relies on per-append array identity.
 		this.#state.messages.push(m);
+		this.#contextRevision++;
 	}
 
 	popMessage(): AgentMessage | undefined {
 		const messages = this.#state.messages.slice(0, -1);
 		const removed = this.#state.messages.at(-1);
 		this.#state.messages = messages;
+		this.#contextRevision++;
 
 		if (removed && this.#state.streamMessage === removed) {
 			this.#state.streamMessage = null;
@@ -873,6 +894,14 @@ export class Agent {
 		return removed;
 	}
 
+	/**
+	 * For callers that mutate committed messages or the system prompt in place
+	 * outside Agent-owned mutators.
+	 */
+	touchContext(): void {
+		this.#contextRevision++;
+	}
+
 	/**
 	 * Queue a steering message to interrupt the agent mid-run.
 	 * Delivered after current tool execution, skips remaining tools.
@@ -1046,6 +1075,7 @@ export class Agent {
 
 	clearMessages() {
 		this.#state.messages = [];
+		this.#contextRevision++;
 	}
 
 	abort() {
@@ -1084,6 +1114,7 @@ export class Agent {
 
 	reset() {
 		this.#state.messages = [];
+		this.#contextRevision++;
 		this.#state.isStreaming = false;
 		this.#state.streamMessage = null;
 		this.#state.pendingToolCalls = new Set<string>();
@@ -1171,6 +1202,10 @@ export class Agent {
 			throw new Error("Cannot continue from message role: assistant");
 		}
 
+		if (!canContinuePersistedHistory(messages)) {
+			throw new Error("No messages to continue from");
+		}
+
 		await this.#runLoop(undefined);
 	}
 

package/src/compaction/compaction.ts

@@ -405,7 +405,61 @@ function countCollectedMessageFragments(collected: { fragments: string[]; extra:
 const HEURISTIC_BYTES_PER_TOKEN = 4;
 
 /**
- * Native-free chars/4 token estimate for a message. This is the only message
+ * Token-dense character weight for the script-aware heuristic.
+ *
+ * Common-BMP CJK blocks (Hangul, unified/compat Han, Kana, CJK punctuation,
+ * full-width forms) tokenize at ~0.6–1.0 tokens per character under
+ * o200k-class BPE vocabularies (measured o200k_base: Hangul prose 0.604,
+ * spaceless Hangul 0.964, Han 0.793, Kana 0.740 tokens/char — versus the
+ * 0.25 the chars/4 heuristic assumes). Each such character is charged 1
+ * token: an upper bound for these measured blocks whose only failure mode is
+ * compacting slightly early, while undercounting risks overflowing the
+ * provider window.
+ *
+ * Surrogate code units are charged 0.5 each, i.e. 1 token per supplementary
+ * code point (supplementary Han extensions, emoji, and other astral chars).
+ * That is a floor rather than an upper bound — rare ideographs and emoji can
+ * cost several tokens — but it is strictly safer than the 0.5-per-pair the
+ * plain chars/4 rule produced.
+ */
+function tokenDenseCharWeight(text: string): { weight: number; units: number } {
+	let weight = 0;
+	let units = 0;
+	for (let i = 0; i < text.length; i++) {
+		const c = text.charCodeAt(i);
+		if (
+			(c >= 0x1100 && c <= 0x11ff) || // Hangul Jamo
+			(c >= 0x3000 && c <= 0x303f) || // CJK symbols & punctuation
+			(c >= 0x3040 && c <= 0x30ff) || // Hiragana & Katakana
+			(c >= 0x3130 && c <= 0x318f) || // Hangul compatibility Jamo
+			(c >= 0x3400 && c <= 0x4dbf) || // CJK ideographs extension A
+			(c >= 0x4e00 && c <= 0x9fff) || // CJK unified ideographs
+			(c >= 0xac00 && c <= 0xd7af) || // Hangul syllables
+			(c >= 0xf900 && c <= 0xfaff) || // CJK compatibility ideographs
+			(c >= 0xff00 && c <= 0xffef) // Half/full-width forms
+		) {
+			weight += 1;
+			units += 1;
+		} else if (c >= 0xd800 && c <= 0xdfff) {
+			// Surrogate half: a supplementary code point contributes two units.
+			weight += 0.5;
+			units += 1;
+		}
+	}
+	return { weight, units };
+}
+
+/**
+ * Script-aware native-free token estimate for a plain string fragment:
+ * token-dense characters cost ~1 token each, everything else chars/4.
+ */
+function estimateFragmentTokensHeuristic(fragment: string): { dense: number; otherChars: number } {
+	const { weight, units } = tokenDenseCharWeight(fragment);
+	return { dense: weight, otherChars: fragment.length - units };
+}
+
+/**
+ * Native-free token estimate for a message. This is the only message
  * token estimator: provider usage (see {@link calculatePromptTokens}) anchors
  * the already-sent context, and this covers unsent/trailing deltas, per-entry
  * budgeting, and display surfaces. Callers add a conservative inflation factor
@@ -413,24 +467,23 @@ const HEURISTIC_BYTES_PER_TOKEN = 4;
  */
 export function estimateMessageTokensHeuristic(message: AgentMessage): number {
 	const { fragments, extra } = collectMessageFragments(message);
-	let bytes = 0;
-	for (const fragment of fragments) {
-		bytes += fragment.length;
-	}
-	return extra + Math.ceil(bytes / HEURISTIC_BYTES_PER_TOKEN);
+	return extra + estimateTextTokensHeuristic(fragments);
 }
 
 /**
- * Native-free chars/4 token estimate for plain string fragments. Fragment-level
- * counterpart of {@link estimateMessageTokensHeuristic}.
+ * Script-aware native-free token estimate for plain string fragments.
+ * Fragment-level counterpart of {@link estimateMessageTokensHeuristic}.
  */
 export function estimateTextTokensHeuristic(fragments: string | readonly string[]): number {
-	if (typeof fragments === "string") return Math.ceil(fragments.length / HEURISTIC_BYTES_PER_TOKEN);
-	let bytes = 0;
-	for (const fragment of fragments) {
-		bytes += fragment.length;
+	const list = typeof fragments === "string" ? [fragments] : fragments;
+	let dense = 0;
+	let otherChars = 0;
+	for (const fragment of list) {
+		const counts = estimateFragmentTokensHeuristic(fragment);
+		dense += counts.dense;
+		otherChars += counts.otherChars;
 	}
-	return Math.ceil(bytes / HEURISTIC_BYTES_PER_TOKEN);
+	return Math.ceil(dense + Math.max(0, otherChars) / HEURISTIC_BYTES_PER_TOKEN);
 }
 
 /** Shared content walk for both the native and heuristic estimators. */
@@ -760,8 +813,8 @@ export interface SummaryOptions {
  * on the very overflow the recovery was meant to absorb.
  *
  * The budget reserves the summary's own output tokens plus prompt/system/template
- * overhead, and applies a conservative safety factor because the chars/4 heuristic
- * undercounts dense or CJK text (the reason the original overflow was missed).
+ * overhead, and applies a conservative safety factor for estimator error on
+ * dense text (the reason the original overflow was missed).
  * Truncation keeps the head (origin/goals) and the tail (most recent state) and
  * elides the middle; it is a last resort that only triggers when the input would
  * otherwise not fit.
@@ -779,16 +832,43 @@ export function boundConversationTextForSummary(
 	const inputBudgetTokens = Math.floor(
 		(contextWindow - Math.max(0, outputMaxTokens) - OVERHEAD_TOKENS) * SAFETY_FACTOR,
 	);
-	if (inputBudgetTokens <= 0) return conversationText;
-	if (estimateTextTokensHeuristic(conversationText) <= inputBudgetTokens) return conversationText;
-
-	const budgetChars = inputBudgetTokens * HEURISTIC_BYTES_PER_TOKEN;
-	const headChars = Math.floor(budgetChars * 0.35);
-	const tailChars = Math.max(0, budgetChars - headChars);
-	const head = conversationText.slice(0, headChars);
-	const tail = tailChars > 0 ? conversationText.slice(conversationText.length - tailChars) : "";
-	const elided = conversationText.length - head.length - tail.length;
-	return `${head}\n\n[... ${elided} characters of older conversation elided so this summarization request fits within the model context window ...]\n\n${tail}`;
+	const totalEstimatedTokens = estimateTextTokensHeuristic(conversationText);
+	const assemble = (head: string, tail: string): string => {
+		const elided = conversationText.length - head.length - tail.length;
+		return `${head}\n\n[... ${elided} characters of older conversation elided so this summarization request fits within the model context window ...]\n\n${tail}`;
+	};
+	const bareMarker = assemble("", "");
+	const fitsBudget = (candidate: string) => estimateTextTokensHeuristic(candidate) <= inputBudgetTokens;
+	if (inputBudgetTokens <= 0) {
+		// A window this small cannot fit any excerpt (not even the marker).
+		// Fail closed with an empty excerpt rather than submitting text into a
+		// request that is guaranteed to overflow.
+		return "";
+	}
+	if (totalEstimatedTokens <= inputBudgetTokens) return conversationText;
+
+	// Derive the character budget from the text's own measured token density
+	// instead of assuming 4 chars/token: a CJK-heavy conversation runs near
+	// 1 token/char, and a fixed 4-chars/token cut would overshoot the budget
+	// by up to ~4x — re-overflowing the very request this bound protects.
+	// Verify the complete assembled candidate (elision marker included)
+	// against the estimator and shrink until it fits.
+	const charsPerToken = conversationText.length / totalEstimatedTokens;
+	let budgetChars = Math.floor(inputBudgetTokens * charsPerToken);
+	for (let attempt = 0; attempt < 12 && budgetChars > 0; attempt++) {
+		const headChars = Math.floor(budgetChars * 0.35);
+		const tailChars = Math.max(0, budgetChars - headChars);
+		const head = conversationText.slice(0, headChars);
+		const tail = tailChars > 0 ? conversationText.slice(conversationText.length - tailChars) : "";
+		const assembled = assemble(head, tail);
+		if (fitsBudget(assembled)) return assembled;
+		budgetChars = Math.floor(budgetChars * 0.8);
+	}
+	// All attempts overshot (adversarially non-uniform density, or a budget
+	// smaller than the marker itself). Fail closed: return the bare marker
+	// only when it fits the budget, else an empty excerpt — never an
+	// over-budget result.
+	return fitsBudget(bareMarker) ? bareMarker : "";
 }
 
 export async function generateSummary(