@sayknow-cli/agent-core 0.3.12 → 0.3.13

package/CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ## [Unreleased]
 
+## [0.10.0] - 2026-07-12
+
+### Fixed
+
+- The native-free token heuristic is now script-aware: common-BMP CJK characters (Hangul, unified/compat Han, Kana, CJK punctuation, full-width forms) are charged at 1 token each (measured o200k_base upper bound 0.96 tokens/char) and supplementary code points (surrogate pairs: rare Han extensions, emoji) at 1 token per code point, instead of chars/4 for everything. The old estimate undercounted Korean/CJK-heavy unsent context by 2–4x and could delay threshold compaction past the provider window; ASCII estimates are unchanged. `boundConversationTextForSummary` now derives its truncation cut from the text's own estimated token density, validates the complete assembled excerpt (elision marker included) against the estimator, and fails closed — bare marker only when the marker itself fits the budget, otherwise an empty excerpt, including when the computed input budget is non-positive — instead of assuming 4 chars/token and returning over-budget or unbounded text.
+
+- A tool call for a name absent from the active tool set now appends a recovery hint pointing at `search_tool_bm25` (gated on a callable `search_tool_bm25`, matched by internal name or `customWireName`), so a model no longer abandons a discoverable tool such as `task` after a bare "Tool <name> not found"; the base error wording stays byte-for-byte stable when discovery is unavailable (#2042).
+
 ## [0.9.2] - 2026-07-09
 
 ### Fixed

package/package.json

@@ -1,7 +1,7 @@
 {
 	"type": "module",
 	"name": "@sayknow-cli/agent-core",
-	"version": "0.3.12",
+	"version": "0.3.13",
 	"description": "General-purpose agent with transport abstraction, state management, and attachment support",
 	"homepage": "https://sayknow-cli.com",
 	"author": "jaybeyond",
@@ -25,7 +25,7 @@
 		"state-management"
 	],
 	"main": "./src/index.ts",
-	"types": "./dist/types/index.d.ts",
+	"types": "./src/index.ts",
 	"scripts": {
 		"check": "biome check . && bun run check:types",
 		"check:types": "tsgo -p tsconfig.json --noEmit",
@@ -35,9 +35,9 @@
 		"fmt": "biome format --write ."
 	},
 	"dependencies": {
-		"@sayknow-cli/ai": "0.3.12",
-		"@sayknow-cli/natives": "0.3.12",
-		"@sayknow-cli/utils": "0.3.12",
+		"@sayknow-cli/ai": "0.3.13",
+		"@sayknow-cli/natives": "0.3.13",
+		"@sayknow-cli/utils": "0.3.13",
 		"@opentelemetry/api": "^1.9.0"
 	},
 	"devDependencies": {
@@ -51,24 +51,23 @@
 	"files": [
 		"src",
 		"README.md",
-		"CHANGELOG.md",
-		"dist/types"
+		"CHANGELOG.md"
 	],
 	"exports": {
 		".": {
-			"types": "./dist/types/index.d.ts",
+			"types": "./src/index.ts",
 			"import": "./src/index.ts"
 		},
 		"./compaction": {
-			"types": "./dist/types/compaction.d.ts",
+			"types": "./src/compaction.ts",
 			"import": "./src/compaction.ts"
 		},
 		"./compaction/*": {
-			"types": "./dist/types/compaction/*.d.ts",
+			"types": "./src/compaction/*.ts",
 			"import": "./src/compaction/*.ts"
 		},
 		"./*": {
-			"types": "./dist/types/*.d.ts",
+			"types": "./src/*.ts",
 			"import": "./src/*.ts"
 		}
 	}

package/src/agent-loop.ts

@@ -1092,6 +1092,17 @@ function emitAbortedAssistantMessage(
 	return abortedMessage;
 }
 
+/**
+ * Match a tool against the model-visible call name. Tools emitted via OpenAI's
+ * custom-tool path (e.g. `apply_patch` on GPT-5) arrive under their wire-level
+ * name, which may differ from the harness-internal `name`, so dispatch and any
+ * "is this tool callable" check must consider both. Internal `name` takes
+ * precedence when a caller needs a single match.
+ */
+function toolMatchesCallName(tool: { name: string; customWireName?: string }, callName: string): boolean {
+	return tool.name === callName || (tool.customWireName !== undefined && tool.customWireName === callName);
+}
+
 /**
  * Execute tool calls from an assistant message.
  */
@@ -1273,7 +1284,22 @@ async function executeToolCalls(
 							`Re-issue the call with complete arguments, splitting the work into smaller steps if needed.`,
 					);
 				}
-				if (!tool) throw new Error(`Tool ${toolCall.name} not found`);
+				if (!tool) {
+					// A discoverable tool that hasn't been activated yet resolves to
+					// undefined here. The model often "remembers" such a tool (e.g.
+					// `task`) from earlier context and calls it by name without first
+					// re-discovering it. Point it at tool discovery so it can activate
+					// the tool and retry instead of giving up on the capability. The
+					// base wording stays byte-for-byte stable for downstream consumers;
+					// the period and hint are appended only when discovery is callable.
+					const base = `Tool ${toolCall.name} not found`;
+					const hasToolDiscovery = tools?.some(t => toolMatchesCallName(t, "search_tool_bm25")) ?? false;
+					throw new Error(
+						hasToolDiscovery
+							? `${base}. If you are unsure whether this tool exists or how to use it, call \`search_tool_bm25\` to discover and activate the matching tool, then retry.`
+							: base,
+					);
+				}
 
 				let effectiveArgs: Record<string, unknown>;
 				try {

package/src/agent.ts

@@ -53,6 +53,16 @@ function assertUserImagePlaceholdersHavePayload(messages: readonly AgentMessage[
 	}
 }
 
+/**
+ * Whether persisted history ends at a point where a new model turn can resume.
+ * Assistant-ended histories require an in-memory queued message and are handled
+ * separately by `Agent.continue()`.
+ */
+export function canContinuePersistedHistory(messages: readonly AgentMessage[]): boolean {
+	const lastMessage = messages.at(-1);
+	return lastMessage !== undefined && lastMessage.role !== "assistant";
+}
+
 /**
  * Default convertToLlm: Keep only LLM-compatible messages, convert attachments.
  */
@@ -1171,6 +1181,10 @@ export class Agent {
 			throw new Error("Cannot continue from message role: assistant");
 		}
 
+		if (!canContinuePersistedHistory(messages)) {
+			throw new Error("No messages to continue from");
+		}
+
 		await this.#runLoop(undefined);
 	}
 

package/src/compaction/compaction.ts

@@ -405,7 +405,61 @@ function countCollectedMessageFragments(collected: { fragments: string[]; extra:
 const HEURISTIC_BYTES_PER_TOKEN = 4;
 
 /**
- * Native-free chars/4 token estimate for a message. This is the only message
+ * Token-dense character weight for the script-aware heuristic.
+ *
+ * Common-BMP CJK blocks (Hangul, unified/compat Han, Kana, CJK punctuation,
+ * full-width forms) tokenize at ~0.6–1.0 tokens per character under
+ * o200k-class BPE vocabularies (measured o200k_base: Hangul prose 0.604,
+ * spaceless Hangul 0.964, Han 0.793, Kana 0.740 tokens/char — versus the
+ * 0.25 the chars/4 heuristic assumes). Each such character is charged 1
+ * token: an upper bound for these measured blocks whose only failure mode is
+ * compacting slightly early, while undercounting risks overflowing the
+ * provider window.
+ *
+ * Surrogate code units are charged 0.5 each, i.e. 1 token per supplementary
+ * code point (supplementary Han extensions, emoji, and other astral chars).
+ * That is a floor rather than an upper bound — rare ideographs and emoji can
+ * cost several tokens — but it is strictly safer than the 0.5-per-pair the
+ * plain chars/4 rule produced.
+ */
+function tokenDenseCharWeight(text: string): { weight: number; units: number } {
+	let weight = 0;
+	let units = 0;
+	for (let i = 0; i < text.length; i++) {
+		const c = text.charCodeAt(i);
+		if (
+			(c >= 0x1100 && c <= 0x11ff) || // Hangul Jamo
+			(c >= 0x3000 && c <= 0x303f) || // CJK symbols & punctuation
+			(c >= 0x3040 && c <= 0x30ff) || // Hiragana & Katakana
+			(c >= 0x3130 && c <= 0x318f) || // Hangul compatibility Jamo
+			(c >= 0x3400 && c <= 0x4dbf) || // CJK ideographs extension A
+			(c >= 0x4e00 && c <= 0x9fff) || // CJK unified ideographs
+			(c >= 0xac00 && c <= 0xd7af) || // Hangul syllables
+			(c >= 0xf900 && c <= 0xfaff) || // CJK compatibility ideographs
+			(c >= 0xff00 && c <= 0xffef) // Half/full-width forms
+		) {
+			weight += 1;
+			units += 1;
+		} else if (c >= 0xd800 && c <= 0xdfff) {
+			// Surrogate half: a supplementary code point contributes two units.
+			weight += 0.5;
+			units += 1;
+		}
+	}
+	return { weight, units };
+}
+
+/**
+ * Script-aware native-free token estimate for a plain string fragment:
+ * token-dense characters cost ~1 token each, everything else chars/4.
+ */
+function estimateFragmentTokensHeuristic(fragment: string): { dense: number; otherChars: number } {
+	const { weight, units } = tokenDenseCharWeight(fragment);
+	return { dense: weight, otherChars: fragment.length - units };
+}
+
+/**
+ * Native-free token estimate for a message. This is the only message
  * token estimator: provider usage (see {@link calculatePromptTokens}) anchors
  * the already-sent context, and this covers unsent/trailing deltas, per-entry
  * budgeting, and display surfaces. Callers add a conservative inflation factor
@@ -413,24 +467,23 @@ const HEURISTIC_BYTES_PER_TOKEN = 4;
  */
 export function estimateMessageTokensHeuristic(message: AgentMessage): number {
 	const { fragments, extra } = collectMessageFragments(message);
-	let bytes = 0;
-	for (const fragment of fragments) {
-		bytes += fragment.length;
-	}
-	return extra + Math.ceil(bytes / HEURISTIC_BYTES_PER_TOKEN);
+	return extra + estimateTextTokensHeuristic(fragments);
 }
 
 /**
- * Native-free chars/4 token estimate for plain string fragments. Fragment-level
- * counterpart of {@link estimateMessageTokensHeuristic}.
+ * Script-aware native-free token estimate for plain string fragments.
+ * Fragment-level counterpart of {@link estimateMessageTokensHeuristic}.
  */
 export function estimateTextTokensHeuristic(fragments: string | readonly string[]): number {
-	if (typeof fragments === "string") return Math.ceil(fragments.length / HEURISTIC_BYTES_PER_TOKEN);
-	let bytes = 0;
-	for (const fragment of fragments) {
-		bytes += fragment.length;
+	const list = typeof fragments === "string" ? [fragments] : fragments;
+	let dense = 0;
+	let otherChars = 0;
+	for (const fragment of list) {
+		const counts = estimateFragmentTokensHeuristic(fragment);
+		dense += counts.dense;
+		otherChars += counts.otherChars;
 	}
-	return Math.ceil(bytes / HEURISTIC_BYTES_PER_TOKEN);
+	return Math.ceil(dense + Math.max(0, otherChars) / HEURISTIC_BYTES_PER_TOKEN);
 }
 
 /** Shared content walk for both the native and heuristic estimators. */
@@ -760,8 +813,8 @@ export interface SummaryOptions {
  * on the very overflow the recovery was meant to absorb.
  *
  * The budget reserves the summary's own output tokens plus prompt/system/template
- * overhead, and applies a conservative safety factor because the chars/4 heuristic
- * undercounts dense or CJK text (the reason the original overflow was missed).
+ * overhead, and applies a conservative safety factor for estimator error on
+ * dense text (the reason the original overflow was missed).
  * Truncation keeps the head (origin/goals) and the tail (most recent state) and
  * elides the middle; it is a last resort that only triggers when the input would
  * otherwise not fit.
@@ -779,16 +832,43 @@ export function boundConversationTextForSummary(
 	const inputBudgetTokens = Math.floor(
 		(contextWindow - Math.max(0, outputMaxTokens) - OVERHEAD_TOKENS) * SAFETY_FACTOR,
 	);
-	if (inputBudgetTokens <= 0) return conversationText;
-	if (estimateTextTokensHeuristic(conversationText) <= inputBudgetTokens) return conversationText;
-
-	const budgetChars = inputBudgetTokens * HEURISTIC_BYTES_PER_TOKEN;
-	const headChars = Math.floor(budgetChars * 0.35);
-	const tailChars = Math.max(0, budgetChars - headChars);
-	const head = conversationText.slice(0, headChars);
-	const tail = tailChars > 0 ? conversationText.slice(conversationText.length - tailChars) : "";
-	const elided = conversationText.length - head.length - tail.length;
-	return `${head}\n\n[... ${elided} characters of older conversation elided so this summarization request fits within the model context window ...]\n\n${tail}`;
+	const totalEstimatedTokens = estimateTextTokensHeuristic(conversationText);
+	const assemble = (head: string, tail: string): string => {
+		const elided = conversationText.length - head.length - tail.length;
+		return `${head}\n\n[... ${elided} characters of older conversation elided so this summarization request fits within the model context window ...]\n\n${tail}`;
+	};
+	const bareMarker = assemble("", "");
+	const fitsBudget = (candidate: string) => estimateTextTokensHeuristic(candidate) <= inputBudgetTokens;
+	if (inputBudgetTokens <= 0) {
+		// A window this small cannot fit any excerpt (not even the marker).
+		// Fail closed with an empty excerpt rather than submitting text into a
+		// request that is guaranteed to overflow.
+		return "";
+	}
+	if (totalEstimatedTokens <= inputBudgetTokens) return conversationText;
+
+	// Derive the character budget from the text's own measured token density
+	// instead of assuming 4 chars/token: a CJK-heavy conversation runs near
+	// 1 token/char, and a fixed 4-chars/token cut would overshoot the budget
+	// by up to ~4x — re-overflowing the very request this bound protects.
+	// Verify the complete assembled candidate (elision marker included)
+	// against the estimator and shrink until it fits.
+	const charsPerToken = conversationText.length / totalEstimatedTokens;
+	let budgetChars = Math.floor(inputBudgetTokens * charsPerToken);
+	for (let attempt = 0; attempt < 12 && budgetChars > 0; attempt++) {
+		const headChars = Math.floor(budgetChars * 0.35);
+		const tailChars = Math.max(0, budgetChars - headChars);
+		const head = conversationText.slice(0, headChars);
+		const tail = tailChars > 0 ? conversationText.slice(conversationText.length - tailChars) : "";
+		const assembled = assemble(head, tail);
+		if (fitsBudget(assembled)) return assembled;
+		budgetChars = Math.floor(budgetChars * 0.8);
+	}
+	// All attempts overshot (adversarially non-uniform density, or a budget
+	// smaller than the marker itself). Fail closed: return the bare marker
+	// only when it fits the budget, else an empty excerpt — never an
+	// over-budget result.
+	return fitsBudget(bareMarker) ? bareMarker : "";
 }
 
 export async function generateSummary(

package/dist/types/agent-loop.d.ts

@@ -1,56 +0,0 @@
-/**
- * Agent loop that works with AgentMessage throughout.
- * Transforms to Message[] only at the LLM call boundary.
- */
-import { type Context, EventStream } from "@sayknow-cli/ai";
-import { type AgentRunCoverage, type AgentRunSummary } from "./run-collector";
-import type { AgentContext, AgentEvent, AgentLoopConfig, AgentMessage, StreamFn } from "./types";
-/**
- * Start an agent loop with a new prompt message.
- * The prompt is added to the context and events are emitted for it.
- */
-export declare function agentLoop(prompts: AgentMessage[], context: AgentContext, config: AgentLoopConfig, signal?: AbortSignal, streamFn?: StreamFn): EventStream<AgentEvent, AgentMessage[]>;
-/**
- * Continue an agent loop from the current context without adding a new message.
- * Used for retries - context already has user message or tool results.
- *
- * **Important:** The last message in context must convert to a `user` or `toolResult` message
- * via `convertToLlm`. If it doesn't, the LLM provider will reject the request.
- * This cannot be validated here since `convertToLlm` is only called once per turn.
- */
-export declare function agentLoopContinue(context: AgentContext, config: AgentLoopConfig, signal?: AbortSignal, streamFn?: StreamFn): EventStream<AgentEvent, AgentMessage[]>;
-/**
- * Detailed-result handle returned by {@link agentLoopDetailed}. Adds the
- * run-level telemetry/coverage rollup to the existing `AgentMessage[]`
- * payload without changing the resolved type of `stream.result()`.
- */
-export interface AgentLoopDetailedResult {
-    readonly messages: AgentMessage[];
-    readonly telemetry: AgentRunSummary | undefined;
-    readonly coverage: AgentRunCoverage | undefined;
-}
-/**
- * Convenience wrapper over {@link agentLoop} that exposes the run-level
- * summary + coverage alongside the messages. The returned `stream` is the
- * same `EventStream` callers already consume; `detailed()` awaits the
- * stream's `agent_end` event and returns the additive fields.
- *
- * Existing `stream.result()` semantics are preserved — it still resolves to
- * `AgentMessage[]`. Use {@link agentLoopDetailed} when you need the rollup;
- * use {@link agentLoop} when you do not.
- */
-export declare function agentLoopDetailed(prompts: AgentMessage[], context: AgentContext, config: AgentLoopConfig, signal?: AbortSignal, streamFn?: StreamFn): {
-    readonly stream: EventStream<AgentEvent, AgentMessage[]>;
-    readonly detailed: () => Promise<AgentLoopDetailedResult>;
-};
-/**
- * Like {@link agentLoopDetailed} but built on top of
- * {@link agentLoopContinue}.
- */
-export declare function agentLoopContinueDetailed(context: AgentContext, config: AgentLoopConfig, signal?: AbortSignal, streamFn?: StreamFn): {
-    readonly stream: EventStream<AgentEvent, AgentMessage[]>;
-    readonly detailed: () => Promise<AgentLoopDetailedResult>;
-};
-export declare function normalizeMessagesForProvider(messages: Context["messages"], model: AgentLoopConfig["model"]): Context["messages"];
-export declare const INTENT_FIELD = "_i";
-export declare function normalizeTools(tools: AgentContext["tools"], injectIntent: boolean): Context["tools"];