@selesai/code 0.13.40 → 0.14.3

package/dist/extensions/skills-index/index.ts

@@ -0,0 +1,105 @@
+import { readFile } from "node:fs/promises";
+import { dirname } from "node:path";
+import { Type } from "typebox";
+import { truncateHead, type ExtensionAPI, type ResolvedSkillInfo } from "@selesai/code";
+import type { Skill } from "../../core/skills.ts";
+
+export const SKILLS_INDEX_INSTRUCTION =
+	"Skills are reusable procedures. Use skill_show without a name to list them, or pass query to filter names and descriptions. Pass an exact listed name to load its instructions before applying it. References in a loaded skill are relative to its directory. Tool discovery is separate: use tool_search or codemode searchTools().";
+
+/** The host owns trust and settings filtering. Never scan additional directories. */
+export function eligibleSkills(skills: readonly ResolvedSkillInfo[]): ResolvedSkillInfo[] {
+	const seen = new Set<string>();
+	return skills.filter((skill) => {
+		if (skill.disableModelInvocation || seen.has(skill.name)) return false;
+		seen.add(skill.name);
+		return true;
+	});
+}
+
+/** An excluded show tool must not strand the model behind an unusable synthetic index. */
+export function compactSkillIndex(skills: Skill[], showAvailable: boolean): Skill[] {
+	if (!showAvailable || skills.length === 0) return skills;
+	return [{
+		name: "skills-index",
+		description: SKILLS_INDEX_INSTRUCTION,
+		filePath: "<skills-index>",
+		baseDir: "<skills-index>",
+		sourceInfo: { path: "<skills-index>", source: "builtin", scope: "user", origin: "top-level" },
+		disableModelInvocation: false,
+	}];
+}
+
+export interface SkillShowArgs {
+	name?: string;
+	query?: string;
+	limit?: number;
+}
+
+function escapeAttribute(value: string): string {
+	return value.replace(/&/g, "&amp;").replace(/"/g, "&quot;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+}
+
+export type SkillShowDetails =
+	| { loaded: true; complete: boolean; skill: string; filePath: string }
+	| { loaded: false; entries: Array<{ name: string; description: string }>; total: number; truncated: boolean };
+
+export interface SkillShowResult {
+	content: Array<{ type: "text"; text: string }>;
+	details: SkillShowDetails;
+}
+
+/** List metadata or load one exact resolved skill; no classification or tool activation. */
+export async function showSkill(skills: readonly ResolvedSkillInfo[], args: SkillShowArgs, signal?: AbortSignal): Promise<SkillShowResult> {
+	signal?.throwIfAborted();
+	const eligible = eligibleSkills(skills);
+	if (args.name !== undefined) {
+		const skill = eligible.find((entry) => entry.name === args.name);
+		if (!skill) throw new Error(`Unknown or model-ineligible skill "${args.name}". Use skill_show without a name to list eligible skills.`);
+		const body = await readFile(skill.filePath, { encoding: "utf8", signal });
+		const output = truncateHead(body);
+		const guidance = output.truncated
+			? `Instructions are incomplete (50KB/2000-line limit). Read the remaining instructions from ${skill.filePath} before applying this skill.`
+			: "The full instructions are embedded below; do not read the file again.";
+		return {
+			content: [{ type: "text" as const, text: `<skill name="${escapeAttribute(skill.name)}" location="${escapeAttribute(skill.filePath)}">\n${guidance}\nReferences are relative to ${dirname(skill.filePath)}.\n\n${output.content}\n</skill>` }],
+			details: { loaded: true, complete: !output.truncated, skill: skill.name, filePath: skill.filePath },
+		};
+	}
+
+	const terms = (args.query ?? "").toLocaleLowerCase().split(/\s+/u).filter(Boolean);
+	const matches = eligible.filter((skill) => {
+		const text = `${skill.name} ${skill.description}`.toLocaleLowerCase();
+		return terms.every((term) => text.includes(term));
+	});
+	const limit = Math.max(1, Math.min(50, Math.floor(args.limit ?? 20)));
+	const entries = matches.slice(0, limit).map(({ name, description }) => ({ name, description }));
+	const listing = entries.map((entry) => `${entry.name} — ${entry.description}`).join("\n");
+	const output = truncateHead(listing);
+	return {
+		content: [{ type: "text" as const, text: matches.length === 0 ? "No eligible skills match." : `${output.content}\n\n${entries.length} of ${matches.length} matches. Pass an exact name to load instructions.${matches.length > entries.length || output.truncated ? " Narrow query to see remaining matches." : ""}` }],
+		details: { loaded: false, entries, total: matches.length, truncated: output.truncated || matches.length > entries.length },
+	};
+}
+
+export default function skillsIndexExtension(pi: ExtensionAPI): void {
+	pi.registerTool({
+		name: "skill_show",
+		label: "Skills",
+		exposure: "direct",
+		description: "List or filter the resolved skill index, or load one skill's instructions by exact name. Omit name to list; query filters names/descriptions using all supplied words. Output is limited to 50KB or 2000 lines; incomplete instructions must be read from the returned file path before use. Does not discover or activate tools.",
+		promptSnippet: "List reusable procedures or load one skill's instructions",
+		promptGuidelines: ["Use skill_show for the skills-index entry; <skills-index> is a synthetic index, not a file to read."],
+		parameters: Type.Object({
+			name: Type.Optional(Type.String({ minLength: 1, description: "Exact listed skill name to load." })),
+			query: Type.Optional(Type.String({ minLength: 1, description: "Filter names/descriptions when listing." })),
+			limit: Type.Optional(Type.Integer({ minimum: 1, maximum: 50, description: "Maximum listed matches (default 20)." })),
+		}),
+		execute: async (_id, args, signal) => showSkill(pi.getResolvedSkills(), args, signal),
+	});
+	// Runtime APIs are bound at session_start. The callback reads the current loadout,
+	// so later explicit exclusions restore the ordinary index rather than hide skills.
+	pi.on("session_start", () => {
+		pi.setSkillsIndexFilter((skills) => compactSkillIndex(skills, pi.getActiveTools().includes("skill_show")));
+	});
+}

package/dist/extensions/skills-index/package.json

@@ -0,0 +1,9 @@
+{
+	"name": "@selesai/skills-index",
+	"version": "0.0.0",
+	"private": true,
+	"type": "module",
+	"description": "Minimal skill-only prompt index and instruction loader; no tool activation or routing.",
+	"pi": { "extensions": ["./index.ts"] },
+	"peerDependencies": { "@selesai/code": "*", "typebox": "*" }
+}

package/dist/extensions/tokenin-onboarding.ts

@@ -12,22 +12,21 @@
 import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { dirname, join } from "node:path";
 import { getAgentDir, getModelsPath } from "@selesai/code";
-import type { AuthStorage, ExtensionAPI, ExtensionCommandContext, ExtensionContext, SessionStartEvent } from "@selesai/code";
+import type { ExtensionAPI, ExtensionCommandContext, ExtensionContext, SessionStartEvent } from "@selesai/code";
 import {
-	calculateCost,
-	contentText,
 	createAssistantMessageEventStream,
-	getSystemMessageText,
 	lazyStream,
-	type AssistantMessage,
 	type AssistantMessageEvent,
 	type AssistantMessageEventStream,
 	type Context,
 	type Model,
 	type SimpleStreamOptions,
-	type TextContent,
 } from "@earendil-works/pi-ai";
 import { getApiProvider } from "@earendil-works/pi-ai/compat";
+import { normalizeContext } from "@earendil-works/pi-ai/utils/transcript";
+import { classifyTokenIn, TOKENIN_CLASSIFIERS } from "./jev/classifier.ts";
+
+type TokenInCredentialStore = ExtensionContext["modelRegistry"]["authStorage"];
 
 export const TOKEN_IN_PROVIDER = "tokenin";
 export const TOKEN_IN_DASHBOARD_URL = "https://token.selesai.in/dashboard/tokens";
@@ -82,13 +81,13 @@ export function removeTokenInApiKey(models: Record<string, unknown>): Record<str
 	return next;
 }
 
-export function getStoredTokenInApiKey(authStorage: AuthStorage): string | undefined {
-	const credential = authStorage.get(TOKEN_IN_PROVIDER);
+export async function getStoredTokenInApiKey(authStorage: TokenInCredentialStore): Promise<string | undefined> {
+	const credential = await authStorage.read(TOKEN_IN_PROVIDER);
 	return credential?.type === "api_key" ? credential.key : undefined;
 }
 
-export function setStoredTokenInApiKey(authStorage: AuthStorage, token: string): void {
-	authStorage.set(TOKEN_IN_PROVIDER, { type: "api_key", key: token });
+export async function setStoredTokenInApiKey(authStorage: TokenInCredentialStore, token: string): Promise<void> {
+	await authStorage.modify(TOKEN_IN_PROVIDER, async () => ({ type: "api_key", key: token }));
 }
 
 /** Validate a pasted token. Returns normalized token or an error message. */
@@ -199,21 +198,22 @@ export function saveTokenInAccount(
 /** Write the given account into auth.json and update tokenin-auth activeId. */
 export function applyTokenInAccountToAuth(
 	account: TokenInAccount,
-	authStorage: AuthStorage,
+	authStorage: TokenInCredentialStore,
 	authPath: string = getTokenInAuthPath(),
-): void {
-	setStoredTokenInApiKey(authStorage, account.apiKey);
+): Promise<void> {
+	// Record the active account synchronously; the credential write is async.
 	const auth = readTokenInAuth(authPath);
 	auth.activeId = account.id;
 	writeTokenInAuth(auth, authPath);
+	return setStoredTokenInApiKey(authStorage, account.apiKey);
 }
 
 /** Return the id of the account whose apiKey matches auth.json, or null. */
-export function getActiveTokenInAccountId(
-	authStorage: AuthStorage,
+export async function getActiveTokenInAccountId(
+	authStorage: TokenInCredentialStore,
 	authPath: string = getTokenInAuthPath(),
-): string | null {
-	const apiKey = getStoredTokenInApiKey(authStorage);
+): Promise<string | null> {
+	const apiKey = await getStoredTokenInApiKey(authStorage);
 	if (!apiKey) return null;
 	const auth = readTokenInAuth(authPath);
 	return auth.accounts.find((a) => a.apiKey === apiKey)?.id ?? null;
@@ -340,178 +340,20 @@ function persistActiveTokenInAccount(account: TokenInAccount, authPath: string =
 	}
 }
 
-/**
- * The decisions deployments Token-In serves (Jev).
- *
- * These answer exactly one non-streaming JSON completion, and the gateway's
- * Jev deployment rejects the SSE path outright, so the normal OpenAI-compatible
- * streaming adapter can only turn a working decision into an HTTP 500. Every
- * other tokenin model keeps the untouched streaming path below.
- */
+/** Legacy Jev IDs are excluded from chat catalog discovery; Jev uses classifiers. */
 export const TOKEN_IN_DECISIONS_MODEL_PATTERN = /^jev-/;
 
 export function isTokenInDecisionsModel(model: { id: string }): boolean {
 	return TOKEN_IN_DECISIONS_MODEL_PATTERN.test(model.id);
 }
 
-function emptyUsage(): AssistantMessage["usage"] {
-	return {
-		input: 0,
-		output: 0,
-		cacheRead: 0,
-		cacheWrite: 0,
-		totalTokens: 0,
-		cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
-	};
-}
-
-/**
- * The plain-text chat messages a decisions request carries: the transcript's
- * text, and nothing else. Decisions deployments take no tools, no images, and
- * no thinking blocks, so anything richer is dropped rather than mis-sent.
- */
-export function tokenInDecisionMessages(context: Context): Array<{ role: "system" | "user" | "assistant"; content: string }> {
-	const messages: Array<{ role: "system" | "user" | "assistant"; content: string }> = [];
-	for (const message of context.messages) {
-		if (message.role === "system") {
-			const content = getSystemMessageText(message).trim();
-			if (content) messages.push({ role: "system", content });
-			continue;
-		}
-		if (message.role === "assistant") {
-			const content = contentText(message.content).trim();
-			if (content) messages.push({ role: "assistant", content });
-			continue;
-		}
-		// User and tool-result text both travel as user turns: a decisions model has
-		// no tool protocol to answer through.
-		const content = contentText(message.content).trim();
-		if (content) messages.push({ role: "user", content });
-	}
-	return messages;
-}
-
-/** The decisions completion's text, when the body is OpenAI-shaped. */
-function decisionCompletion(body: unknown): { text: string; finishReason: unknown; usage: Record<string, unknown> } | undefined {
-	if (typeof body !== "object" || body === null) return undefined;
-	const choices = (body as { choices?: unknown }).choices;
-	if (!Array.isArray(choices) || choices.length === 0) return undefined;
-	const choice = choices[0] as { message?: { content?: unknown }; finish_reason?: unknown };
-	const content = choice?.message?.content;
-	const text =
-		typeof content === "string"
-			? content
-			: Array.isArray(content)
-				? contentText(content as readonly { type: "text"; text: string }[])
-				: undefined;
-	if (text === undefined) return undefined;
-	const usage = (body as { usage?: unknown }).usage;
-	return {
-		text,
-		finishReason: choice?.finish_reason,
-		usage: typeof usage === "object" && usage !== null ? (usage as Record<string, unknown>) : {},
-	};
-}
-
-function usageNumber(value: unknown): number {
-	return typeof value === "number" && Number.isFinite(value) ? value : 0;
-}
-
-/**
- * One non-streaming decisions call, rendered into the normal event protocol so
- * every pi consumer (and cost accounting) sees an ordinary assistant message.
- * The returned `errorMessage` feeds the caller's account-rotation decision.
- */
-async function requestTokenInDecision(
-	model: Model<any>,
-	context: Context,
-	options: SimpleStreamOptions,
-): Promise<{ stream: AssistantMessageEventStream; errorMessage?: string }> {
-	const stream = createAssistantMessageEventStream();
-	const message: AssistantMessage = {
-		role: "assistant",
-		content: [],
-		api: model.api,
-		provider: model.provider,
-		model: model.id,
-		usage: emptyUsage(),
-		stopReason: "pending",
-		timestamp: Date.now(),
-	};
-	const fail = (text: string): { stream: AssistantMessageEventStream; errorMessage: string } => {
-		const aborted = options.signal?.aborted === true;
-		message.stopReason = aborted ? "aborted" : "error";
-		message.errorMessage = text;
-		stream.push({ type: "error", reason: message.stopReason, error: message });
-		stream.end(message);
-		return { stream, errorMessage: text };
-	};
-
-	const requested: Record<string, unknown> = {
-		model: model.id,
-		messages: tokenInDecisionMessages(context),
-	};
-	if (options.maxTokens !== undefined) requested.max_tokens = options.maxTokens;
-	const payload = (await options.onPayload?.(requested, model)) ?? requested;
-
-	let response: Response;
-	try {
-		response = await (options.fetch ?? fetch)(`${model.baseUrl.replace(/\/+$/, "")}/chat/completions`, {
-			method: "POST",
-			headers: { "Content-Type": "application/json", ...(options.headers ?? {}) },
-			body: JSON.stringify(payload),
-			...(options.signal ? { signal: options.signal } : {}),
-		});
-	} catch (error) {
-		return fail(error instanceof Error ? error.message : String(error));
-	}
-	await options.onResponse?.(
-		{ status: response.status, headers: Object.fromEntries(response.headers.entries()) },
-		model,
-	);
-
-	const raw = await response.text();
-	if (!response.ok) return fail(`HTTP ${response.status}: ${raw.slice(0, 400)}`);
-
-	let body: unknown;
-	try {
-		body = JSON.parse(raw);
-	} catch {
-		return fail(`Malformed decisions response: ${raw.slice(0, 200)}`);
-	}
-	const completion = decisionCompletion(body);
-	if (!completion) return fail(`Unexpected decisions response: ${raw.slice(0, 200)}`);
-
-	const input = usageNumber(completion.usage.prompt_tokens ?? completion.usage.input_tokens);
-	const output = usageNumber(completion.usage.completion_tokens ?? completion.usage.output_tokens);
-	message.usage = {
-		...emptyUsage(),
-		input,
-		output,
-		totalTokens: usageNumber(completion.usage.total_tokens) || input + output,
-	};
-	calculateCost(model, message.usage);
-	message.stopReason = completion.finishReason === "length" ? "length" : "stop";
-
-	stream.push({ type: "start", partial: { ...message } });
-	const block: TextContent = { type: "text", text: "" };
-	message.content = [block];
-	stream.push({ type: "text_start", contentIndex: 0, partial: { ...message } });
-	block.text = completion.text;
-	stream.push({ type: "text_delta", contentIndex: 0, delta: completion.text, partial: { ...message } });
-	stream.push({ type: "text_end", contentIndex: 0, content: completion.text, partial: { ...message } });
-	stream.push({ type: "done", reason: message.stopReason === "length" ? "length" : "stop", message });
-	stream.end(message);
-	return { stream };
-}
-
 /**
  * Custom streamSimple implementation for tokenin provider that wraps OpenAI completions
  * and automatically rotates to alternative saved keys on 401/429/budget exceeded errors.
  */
 export function createTokenInStreamSimple(options?: {
 	authPath?: string;
-	getAuthStorage?: () => AuthStorage | undefined;
+	getAuthStorage?: () => TokenInCredentialStore | undefined;
 	onRotate?: (failedAccount: TokenInAccount, nextAccount: TokenInAccount, reason: string) => void;
 	streamSimple?: (model: Model<any>, context: Context, options?: SimpleStreamOptions) => AssistantMessageEventStream;
 }) {
@@ -522,7 +364,7 @@ export function createTokenInStreamSimple(options?: {
 	const activate = (account: TokenInAccount): void => {
 		const authStorage = options?.getAuthStorage?.();
 		if (authStorage) {
-			applyTokenInAccountToAuth(account, authStorage, authPath);
+			void applyTokenInAccountToAuth(account, authStorage, authPath).catch(() => undefined);
 		} else {
 			persistActiveTokenInAccount(account, authPath);
 		}
@@ -550,7 +392,7 @@ export function createTokenInStreamSimple(options?: {
 
 			// If no accounts saved, fall back directly to base implementation
 			if (accounts.length === 0) {
-				return streamSimple(model, context, streamOptions);
+				return streamSimple(model, normalizeContext(context), streamOptions);
 			}
 
 			// Order accounts: active account first, followed by remaining accounts
@@ -577,24 +419,9 @@ export function createTokenInStreamSimple(options?: {
 
 				const requestOptions = withAccountAuthorization(streamOptions, currentAccount);
 
-				// Decisions deployments never stream: one POST, one text block, and only
-				// a rotateable failure moves on to the next account.
-				if (isTokenInDecisionsModel(model)) {
-					const decided = await requestTokenInDecision(model, context, requestOptions);
-					if (decided.errorMessage === undefined) return decided.stream;
-					if (isRotateableTokenInError(decided.errorMessage) && i + 1 < candidateAccounts.length) {
-						setTokenInCooldown(currentAccount.id);
-						const nextAccount = candidateAccounts[i + 1]!;
-						activate(nextAccount);
-						options?.onRotate?.(currentAccount, nextAccount, decided.errorMessage);
-						continue;
-					}
-					return decided.stream;
-				}
-
 				let underlyingStream: AssistantMessageEventStream;
 				try {
-					underlyingStream = streamSimple(model, context, requestOptions);
+					underlyingStream = streamSimple(model, normalizeContext(context), requestOptions);
 				} catch (err) {
 					lastThrownError = err;
 					if (isRotateableTokenInError(err) && i + 1 < candidateAccounts.length) {
@@ -691,7 +518,7 @@ export function createTokenInStreamSimple(options?: {
 				throw lastThrownError;
 			}
 
-			return streamSimple(model, context, streamOptions);
+			return streamSimple(model, normalizeContext(context), streamOptions);
 		});
 	};
 }
@@ -975,7 +802,7 @@ async function handleTokenInAdd(pi: ExtensionAPI, ctx: ExtensionCommandContext):
 
 	if (wasEmpty) {
 		const saved = readTokenInAuth().accounts.find((a) => a.id === token)!;
-		applyTokenInAccountToAuth(saved, ctx.modelRegistry.authStorage);
+		await applyTokenInAccountToAuth(saved, ctx.modelRegistry.authStorage);
 		try {
 			await ctx.modelRegistry.refresh();
 		} catch {
@@ -1018,7 +845,7 @@ async function handleTokenInSwitch(ctx: ExtensionCommandContext): Promise<void>
 	const account = await pickAccount(ctx, "Select TokenIN account", auth);
 	if (!account) return;
 
-	applyTokenInAccountToAuth(account, ctx.modelRegistry.authStorage);
+	await applyTokenInAccountToAuth(account, ctx.modelRegistry.authStorage);
 	try {
 		await ctx.modelRegistry.refresh();
 	} catch {
@@ -1028,13 +855,13 @@ async function handleTokenInSwitch(ctx: ExtensionCommandContext): Promise<void>
 }
 
 async function handleTokenInUsage(ctx: ExtensionCommandContext): Promise<void> {
-	const apiKey = getStoredTokenInApiKey(ctx.modelRegistry.authStorage);
+	const apiKey = await getStoredTokenInApiKey(ctx.modelRegistry.authStorage);
 	if (!apiKey) {
 		ctx.ui.notify("No active Token-In token. Use /tokenin add first.", "warning");
 		return;
 	}
 	const auth = readTokenInAuth();
-	const activeId = getActiveTokenInAccountId(ctx.modelRegistry.authStorage);
+	const activeId = await getActiveTokenInAccountId(ctx.modelRegistry.authStorage);
 	const account = auth.accounts.find((a) => a.id === activeId);
 	const baseUrl = account?.baseUrl ?? getTokenInBaseUrl(readModelsJson()) ?? TOKEN_IN_DEFAULT_BASE_URL;
 	try {
@@ -1066,7 +893,7 @@ async function handleTokenInRemove(ctx: ExtensionCommandContext): Promise<void>
 	const account = await pickAccount(ctx, "Select account to remove", auth);
 	if (!account) return;
 
-	const activeId = getActiveTokenInAccountId(ctx.modelRegistry.authStorage);
+	const activeId = await getActiveTokenInAccountId(ctx.modelRegistry.authStorage);
 	if (account.id === activeId) {
 		ctx.ui.notify("Cannot remove the active account. Use /tokenin switch first.", "warning");
 		return;
@@ -1083,7 +910,7 @@ async function handleTokenInRemove(ctx: ExtensionCommandContext): Promise<void>
 async function runOnboarding(pi: ExtensionAPI, ctx: ExtensionContext): Promise<void> {
 	const modelsPath = getModelsPath();
 	const models = readModelsJson(modelsPath);
-	const storedKey = getStoredTokenInApiKey(ctx.modelRegistry.authStorage);
+	const storedKey = await getStoredTokenInApiKey(ctx.modelRegistry.authStorage);
 	const legacyModelsKey = getTokenInApiKey(models);
 	const currentKey = storedKey ?? legacyModelsKey;
 
@@ -1091,7 +918,7 @@ async function runOnboarding(pi: ExtensionAPI, ctx: ExtensionContext): Promise<v
 	// into auth.json so Token-In models stay package-local.
 	if (!isPlaceholderToken(currentKey)) {
 		if (!storedKey && typeof legacyModelsKey === "string") {
-			setStoredTokenInApiKey(ctx.modelRegistry.authStorage, legacyModelsKey);
+			await setStoredTokenInApiKey(ctx.modelRegistry.authStorage, legacyModelsKey);
 			saveTokenInAccount(legacyModelsKey, { baseUrl: getTokenInBaseUrl(models), setActive: true });
 			// legacyModelsKey is only non-empty when models.json parsed successfully,
 			// so the file always exists here.
@@ -1139,7 +966,7 @@ async function runOnboarding(pi: ExtensionAPI, ctx: ExtensionContext): Promise<v
 	}
 
 	const token = validated.token!;
-	setStoredTokenInApiKey(ctx.modelRegistry.authStorage, token);
+	await setStoredTokenInApiKey(ctx.modelRegistry.authStorage, token);
 	markOnboardingComplete();
 
 	// Seed tokenin-auth.json so /tokenin switch and /tokenin remove work after onboarding.
@@ -1156,14 +983,27 @@ async function runOnboarding(pi: ExtensionAPI, ctx: ExtensionContext): Promise<v
 }
 
 export default function tokenInOnboardingExtension(pi: ExtensionAPI): void {
-	let authStorage: AuthStorage | undefined;
-	pi.registerProvider("tokenin", {
-		api: "openai-completions",
+	let authStorage: TokenInCredentialStore | undefined;
+	const provider = {
+		api: "openai-completions" as const,
 		streamSimple: createTokenInStreamSimple({ getAuthStorage: () => authStorage }),
-	});
+		classifiers: { "typesafe-system-one": { classify: classifyTokenIn } },
+	};
+	pi.registerProvider("tokenin", provider);
 
 	pi.on("session_start", async (event: SessionStartEvent, ctx: ExtensionContext) => {
 		authStorage = ctx.modelRegistry.authStorage;
+		// Explicit model lists replace the catalog across ALL operations. Snapshot
+		// chat/image/classifier entries, not only Jev or credential-filtered models.
+		const models = [
+			...ctx.modelRegistry.getModelsOfType("chat", "tokenin"),
+			...ctx.modelRegistry.getModelsOfType("image", "tokenin"),
+			...ctx.modelRegistry.getModelsOfType("classifier", "tokenin"),
+		];
+		for (const classifier of TOKENIN_CLASSIFIERS) {
+			if (!models.some((model) => model.type === "classifier" && model.id === classifier.id)) models.push(classifier);
+		}
+		pi.registerProvider("tokenin", { ...provider, models });
 		if (event.reason !== "startup") return;
 		if (isOnboardingComplete()) return;
 

package/dist/extensions/tool-search/index.d.ts

@@ -0,0 +1,11 @@
+/**
+ * The `tool_search` tool as an extension. The CLI loads it as a built-in extension; SDK users add
+ * `createToolSearchExtension()` to their extension factories.
+ *
+ * `tool_search` is registered inactive. Activate it with `--tools`, the `defaultTools` setting, or
+ * `setActiveTools()`.
+ */
+import type { ExtensionFactory } from "../../core/extensions/types.ts";
+export declare function createToolSearchExtension(): ExtensionFactory;
+declare const _default: ExtensionFactory;
+export default _default;

package/dist/extensions/tool-search/index.js

@@ -0,0 +1,14 @@
+/**
+ * The `tool_search` tool as an extension. The CLI loads it as a built-in extension; SDK users add
+ * `createToolSearchExtension()` to their extension factories.
+ *
+ * `tool_search` is registered inactive. Activate it with `--tools`, the `defaultTools` setting, or
+ * `setActiveTools()`.
+ */
+import { createToolSearchToolDefinition } from "./tool.js";
+export function createToolSearchExtension() {
+    return (pi) => {
+        pi.registerTool({ ...createToolSearchToolDefinition({ tools: pi }), defaultActive: false });
+    };
+}
+export default createToolSearchExtension();

package/dist/extensions/tool-search/tool.d.ts

@@ -0,0 +1,91 @@
+/**
+ * Tool discovery: a BM25 ranker over tool metadata, shared by `searchTools()` in codemode scripts and
+ * the optional `tool_search` tool.
+ *
+ * `tool_search` searches tools that are not declared to
+ * the model (`codemode` and `deferred` exposure) and loads the matches, so they are declared for the
+ * next model call. Loading goes through the active tool set, so it is recorded in the transcript
+ * like any other tool change and survives `/tree`, resume, and fork on that branch.
+ */
+import { type Static, Type } from "typebox";
+import type { ExtensionAPI, ExtensionContext, ToolDefinition, ToolInfo, ToolNamespace } from "../../core/extensions/types.ts";
+export declare const TOOL_SEARCH_TOOL_NAME = "tool_search";
+export declare const DEFAULT_TOOL_SEARCH_LIMIT = 8;
+/** A tool as the ranker sees it: its name and the text built by {@link createToolSearchDocument}. */
+export interface ToolSearchDocument {
+    name: string;
+    text: string;
+    /** Short human-readable form for rankers that read tools as text (name, first paragraph of the description). */
+    summary?: string;
+}
+export interface ToolSearchMatch {
+    name: string;
+    score: number;
+}
+/** What a ranker may use besides the query: the caller's cancellation and session (model access). */
+export interface ToolRankOptions {
+    signal?: AbortSignal;
+    /** Absent when the caller has no session. A ranker that needs one falls back to its own default. */
+    ctx?: ExtensionContext;
+}
+/**
+ * Ranks tools for a query. BM25 is the built-in; an extension can replace it with
+ * {@link registerToolRanker}, for example a model-backed ranker. Rankers may be asynchronous and
+ * should degrade to a local ranking rather than throw.
+ */
+export interface ToolRanker {
+    rank(query: string, documents: readonly ToolSearchDocument[], limit: number, options?: ToolRankOptions): ToolSearchMatch[] | Promise<ToolSearchMatch[]>;
+}
+/** Lowercase terms, split at camelCase boundaries and non-alphanumerics, without stop words. */
+export declare function tokenize(text: string): string[];
+/**
+ * Search text of a tool: the name, the name with `_`
+ * as spaces, the description, schema descriptions and property names, and the namespace with its
+ * description and instructions.
+ */
+export declare function createToolSearchDocument(tool: Pick<ToolInfo, "name" | "description" | "parameters">, namespace?: ToolNamespace): ToolSearchDocument;
+/** Okapi BM25 with the usual parameters. Ties keep document order. */
+export declare class Bm25Ranker implements ToolRanker {
+    private readonly k1;
+    private readonly b;
+    constructor(options?: {
+        k1?: number;
+        b?: number;
+    });
+    rank(query: string, documents: readonly ToolSearchDocument[], limit: number): ToolSearchMatch[];
+}
+/**
+ * Make `ranker` the one `tool_search` and codemode's `searchTools()` use, until the returned function
+ * is called. Kept on `globalThis` so every copy of this module sees the same ranker.
+ */
+export declare function registerToolRanker(ranker: ToolRanker): () => void;
+/** The registered ranker, or BM25. */
+export declare function getToolRanker(): ToolRanker;
+export declare const toolSearchSchema: Type.TObject<{
+    query: Type.TString;
+    limit: Type.TOptional<Type.TNumber>;
+}>;
+export type ToolSearchInput = Static<typeof toolSearchSchema>;
+/** Whether the tool is this `tool_search`, not another extension's tool of the same name. */
+export declare function isToolSearchTool(tool: Pick<ToolInfo, "name" | "parameters">): boolean;
+export interface ToolSearchResultTool {
+    name: string;
+    description: string;
+}
+export interface ToolSearchToolDetails {
+    /** Tools loaded by this call. */
+    loaded: string[];
+}
+export interface ToolSearchToolOptions {
+    /**
+     * The session's tools. `tool_search` searches the tools that are not declared to the model and
+     * activates the matches. Without it, the tool finds nothing. An `ExtensionAPI` fits.
+     */
+    tools?: Pick<ExtensionAPI, "getAllTools" | "getActiveTools" | "setActiveTools">;
+}
+/**
+ * The `tool_search` description. It does not list the searchable tools or their namespaces, so it
+ * stays the same while tools are registered, for example when MCP servers connect.
+ */
+export declare const TOOL_SEARCH_DESCRIPTION = "# Tool discovery\n\nSearches over deferred tool metadata and exposes matching tools for the next model call.\n\nSome of the tools, such as tools of MCP servers, may not have been provided to you upfront, and you should use this tool (`tool_search`) to search for the required tools. For MCP tool discovery, always use `tool_search`.";
+export declare function createToolSearchToolDefinition(options?: ToolSearchToolOptions): ToolDefinition<typeof toolSearchSchema, ToolSearchToolDetails>;