@sayknow-cli/coding-agent 0.5.21 → 0.5.23

package/CHANGELOG.md

@@ -2,6 +2,23 @@
 
 ## [Unreleased]
 
+## [0.5.23] - 2026-09-20
+
+## [0.5.22] - 2026-09-19
+
+### Added
+
+- TypeSafe (hosted System One `jev`) as a typed-decision backend. Add the key with `TYPESAFE_API_KEY=<key> skc setup typesafe`; it is verified against the live API before being stored, because the decision service fails open and an unverified bad key would be swallowed silently forever. Removing it (`--remove`) falls back to your logged-in model. The key is read from the environment, never a flag — this repo already refuses raw `--api-key` values because they leak into shell history and the process list.
+- Decision backends now resolve in order: TypeSafe when a key exists (the only backend that returns calibrated probabilities), otherwise the model you are already logged into. TypeSafe is deliberately **not** registered in the chat-provider registry: it has no stream, no messages and no text output, so giving it a `Model` shape would put a non-chat endpoint in the model picker.
+- Typed decisions (`decisions.enabled`, default off): a small, reusable service that asks the model you are already logged into a typed question and gets back a value your code can branch on. Type safety comes from a forced tool call with enum-constrained fields, so an option outside the declared set cannot reach the caller. No extra vendor, no extra key.
+- Workflow-skill routing now has a semantic second stage. The literal keyword table stays first and free; the model is consulted only when it matches nothing. Measured over 23 prompts × 3 runs against `claude-opus-5`: keyword-only **0/27 in Korean** (43% overall), hybrid **69/69 (100%)** with zero false activations on the 18 negative cases, p50 1.46s. Reproduce with `bun scripts/eval-skill-routing.ts --repeat 3`.
+
+### Fixed
+
+- `NativeRuntimeCompatibilityError` now lists only the causes that actually fired and names the file the process loaded `@sayknow-cli/natives` from. A pure version mismatch used to read "required workflow arbitration methods are **available**", which described a healthy runtime while refusing to start every session and extension.
+- `dev:link` / `dev:doctor` fail on (and `dev:link` removes) nested installs under `packages/<pkg>/node_modules` that shadow a workspace package, and verify that `@sayknow-cli/natives` resolves to the same version as the `coding-agent` runtime that loads it. A published `@sayknow-cli/natives` copy left inside `packages/coding-agent/node_modules` wins resolution over the workspace link and breaks every SDK session.
+- `dev:doctor` accepts this checkout's bun-linked `bin/skc.js` wrapper — the one `install:dev` itself creates — instead of reporting it as drift, but only when the wrapper is byte-identical to the expected workspace wrapper and `@sayknow-cli/coding-agent/cli` resolves back to this checkout's `src/cli.ts`.
+
 ## [0.5.21] - 2026-09-17
 
 ### Changed

package/dist/types/cli/setup-cli.d.ts

@@ -5,7 +5,7 @@
  */
 import { AuthStorage, SqliteAuthCredentialStore } from "@sayknow-cli/ai";
 import { runExternalCredentialAutoImport } from "../setup/credential-auto-import";
-export type SetupComponent = "claude" | "codex" | "credentials" | "defaults" | "hermes" | "hooks" | "provider" | "python" | "stt" | "ui-skills";
+export type SetupComponent = "claude" | "codex" | "credentials" | "defaults" | "hermes" | "hooks" | "provider" | "python" | "stt" | "typesafe" | "ui-skills";
 export interface SetupCommandArgs {
     component: SetupComponent;
     flags: {
@@ -37,6 +37,8 @@ export interface SetupCommandArgs {
         yes?: boolean;
         dryRun?: boolean;
         keychain?: boolean;
+        skipVerify?: boolean;
+        remove?: boolean;
     };
 }
 /**
@@ -71,3 +73,15 @@ export declare function handleCredentialsSetup(flags: {
  * Print setup command help.
  */
 export declare function printSetupHelp(): void;
+/**
+ * `skc setup typesafe` — enable the hosted System One model for typed decisions.
+ *
+ * The key is read from `TYPESAFE_API_KEY`, never from a flag: this repo already refuses
+ * raw `--api-key` values because they land in shell history and in the process list of
+ * every user on the machine. Same rule applies here.
+ *
+ * Verified against the live API before storing. The decision service fails open, so an
+ * unverified bad key would be swallowed forever — the user would believe TypeSafe was
+ * active while every decision quietly came from their own model.
+ */
+export declare function handleTypeSafeSetup(flags: SetupCommandArgs["flags"]): Promise<void>;

package/dist/types/commands/setup.d.ts

@@ -100,6 +100,12 @@ export default class Setup extends Command {
         "dry-run": import("@sayknow-cli/utils/cli").FlagDescriptor<"boolean"> & {
             description: string;
         };
+        "skip-verify": import("@sayknow-cli/utils/cli").FlagDescriptor<"boolean"> & {
+            description: string;
+        };
+        remove: import("@sayknow-cli/utils/cli").FlagDescriptor<"boolean"> & {
+            description: string;
+        };
     };
     run(): Promise<void>;
 }

package/dist/types/config/settings-schema.d.ts

@@ -2412,6 +2412,15 @@ export declare const SETTINGS_SCHEMA: {
         readonly type: "number";
         readonly default: 16000;
     };
+    readonly "decisions.enabled": {
+        readonly type: "boolean";
+        readonly default: false;
+        readonly ui: {
+            readonly tab: "context";
+            readonly label: "Typed decisions";
+            readonly description: "Add a model-backed second stage to workflow routing. The keyword table already runs on every turn and costs nothing; this handles the phrasings it cannot express, which is most wording that is not a literal match. Costs one small model call, and only on turns the keyword table did not already answer. Any failure falls back to keyword-only behaviour, but a successful answer can also select a different workflow than the deep-interview ambiguity detector would have.";
+        };
+    };
     readonly "ttsr.enabled": {
         readonly type: "boolean";
         readonly default: true;

package/dist/types/decisions/index.d.ts

@@ -0,0 +1,18 @@
+import { type LlmBackendDeps } from "./llm-backend";
+import type { DecisionBackend, DecisionRequest, DecisionResult } from "./types";
+export { createLlmDecisionBackend } from "./llm-backend";
+export * from "./types";
+export { createTypeSafeDecisionBackend, TYPESAFE_PROVIDER } from "./typesafe-backend";
+export interface DecisionServiceOptions extends LlmBackendDeps {
+    /** Off by default; callers opt in per feature. */
+    enabled?: boolean;
+    timeoutMs?: number;
+    /** Injection point for tests and for the self-hosted/hosted backends. */
+    backends?: DecisionBackend[];
+}
+export interface DecisionService {
+    readonly enabled: boolean;
+    /** Resolves null when disabled, unavailable, timed out, or the model misbehaved. */
+    decide(request: DecisionRequest): Promise<DecisionResult | null>;
+}
+export declare function createDecisionService(options: DecisionServiceOptions): DecisionService;

package/dist/types/decisions/llm-backend.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Decision backend that runs on the model the user is already logged into.
+ *
+ * No extra API key, no extra vendor, no data leaving the providers the user already
+ * trusts. The type safety comes from a **forced tool call with enum-constrained
+ * properties**: the provider itself rejects any value outside the declared set, so a
+ * malformed or hallucinated option cannot reach our code — the same guarantee the
+ * hosted System One model gives, enforced one layer up.
+ *
+ * Two deliberate omissions:
+ *
+ * 1. We never ask the model to emit probabilities. Measured elsewhere on this exact
+ *    task shape, writing probabilities collapses accuracy (~0.35 vs ~0.90 for picking
+ *    a constrained option), and the numbers are not calibrated anyway. Answers from
+ *    this backend carry `calibrated: false` and no `probabilities` map.
+ * 2. Every question goes in **one** call. Splitting them multiplies cost and latency
+ *    while the enum constraint already keeps each field independent.
+ */
+import { type Api, type Model } from "@sayknow-cli/ai";
+import type { ModelRegistry } from "../config/model-registry";
+import type { Settings } from "../config/settings";
+import { type DecisionBackend } from "./types";
+export interface LlmBackendDeps {
+    /**
+     * Refuse to spend more than this per million input tokens on a decision.
+     *
+     * The whole premise of a typed-decision service is judgment cheap enough to put in
+     * places you could not previously afford it. Routing a prompt through a frontier
+     * model inverts that: the deterministic path it replaces costs effectively nothing
+     * (the routing rules already sit in the cached system prompt), so a decision call on
+     * an expensive model is a pure cost *increase* for a few points of accuracy.
+     *
+     * Measured: a routing decision on claude-opus-5 costs ~$0.0063 and 1.46s; the same
+     * decision on the hosted System One model costs ~$0.000018 and 0.31s.
+     *
+     * Above the cap this backend declines, which leaves routing to the system prompt —
+     * exactly the behaviour before typed decisions existed. Configure a `smol` role with
+     * a cheap model to turn it back on.
+     */
+    maxInputCostPerMTok?: number;
+    /** Injected in tests to make the local-runtime probe deterministic. */
+    fetchImpl?: typeof fetch;
+    registry: ModelRegistry;
+    settings: Settings;
+    sessionId?: string;
+    /** Overrides role resolution; used by callers that already picked a model. */
+    model?: Model<Api>;
+}
+/** Reset between tests; also lets a caller force a fresh probe after starting a runtime. */
+export declare function clearLocalRuntimeLivenessCache(): void;
+export declare function createLlmDecisionBackend(deps: LlmBackendDeps): DecisionBackend;

package/dist/types/decisions/skill-routing.d.ts

@@ -0,0 +1,10 @@
+import { type CanonicalSkcWorkflowSkill } from "../skill-state/active-state";
+import type { DecisionService } from "./index";
+/** Exported so tests can assert the contract the model is actually given. */
+export declare function buildRoutingCriteria(): Record<string, string>;
+export type SkillRouter = (text: string) => Promise<CanonicalSkcWorkflowSkill | null>;
+/**
+ * Build the semantic router. Returns null-resolving function when the service is
+ * disabled so the caller keeps its existing behaviour with no branching.
+ */
+export declare function createSemanticSkillRouter(service: DecisionService): SkillRouter;

package/dist/types/decisions/types.d.ts

@@ -0,0 +1,91 @@
+/**
+ * Typed decisions — "Jev-shaped" structured judgments for code to branch on.
+ *
+ * The request/response shape follows TypeSafe's System One API so a backend can be
+ * swapped without touching call sites: the hosted `jev` model, a self-hosted OpenJev
+ * daemon, or — the default — the model the user is already logged into.
+ *
+ * What this is NOT: a probability oracle. Only the hosted model returns calibrated
+ * probabilities. Backends that constrain an ordinary LLM return an ordinal value and
+ * report `calibrated: false`; treat those numbers as a ranking, never as P(correct).
+ */
+/** Pick exactly one option from a closed set. */
+export interface ChoiceQuestion {
+    type: "choice";
+    instructions: string;
+    /** option id -> what that option means. At least two. */
+    criteria: Record<string, string>;
+}
+/** Rate the state against ordered levels. Level 0 is the lowest. */
+export interface ScoreQuestion {
+    type: "score";
+    instructions: string;
+    /** Ordered level descriptions, lowest first. At least two. */
+    criteria: string[];
+}
+/** Is this statement true of the state? */
+export interface NoulQuestion {
+    type: "noul";
+    instructions: string;
+}
+export type Question = ChoiceQuestion | ScoreQuestion | NoulQuestion;
+export interface ChoiceAnswer {
+    type: "choice";
+    /** The selected option id. Always one of the declared `criteria` keys. */
+    choice: string;
+    /** Present only when the backend exposes a distribution. */
+    probabilities?: Record<string, number>;
+    /**
+     * How certain the backend is, 0..1. Only meaningful when the result reports
+     * `calibrated: true` — that is the difference between a number you can threshold on
+     * and a number that merely ranks. Absent when the backend cannot supply one.
+     */
+    confidence?: number;
+}
+export interface ScoreAnswer {
+    type: "score";
+    /** Level index. Fractional only when the backend returns a distribution. */
+    score: number;
+    /** Selected level index. */
+    level: number;
+    legend: Record<string, string>;
+    probabilities?: Record<string, number>;
+    /** See {@link ChoiceAnswer.confidence}. */
+    confidence?: number;
+}
+export interface NoulAnswer {
+    type: "noul";
+    /** 0 (no) .. 1 (yes). Ordinal unless `calibrated` is true. */
+    noul: number;
+}
+export type Answer = ChoiceAnswer | ScoreAnswer | NoulAnswer;
+export interface DecisionResult {
+    answers: Record<string, Answer>;
+    /** Which backend answered, for logging and A/B comparison. */
+    backend: string;
+    /** Model identifier the backend used. */
+    model: string;
+    /**
+     * False means the numbers are ordinal rankings, not probabilities.
+     * Only the hosted System One model reports true.
+     */
+    calibrated: boolean;
+    durationMs: number;
+}
+export interface DecisionRequest {
+    /** The content to judge. Plain text, or JSON-serialisable structured state. */
+    state: string | Record<string, unknown> | unknown[];
+    /** Question id -> question. Answers come back under the same ids. */
+    questions: Record<string, Question>;
+    signal?: AbortSignal;
+}
+export interface DecisionBackend {
+    readonly name: string;
+    /** Resolves null when the backend is unavailable (no credentials, offline, disabled). */
+    decide(request: DecisionRequest): Promise<DecisionResult | null>;
+}
+export declare const MIN_OPTIONS = 2;
+/** Matches OpenJev's letter-slot ceiling so a graph stays portable across backends. */
+export declare const MAX_OPTIONS = 16;
+export declare function validateQuestions(questions: Record<string, Question>): void;
+export declare function stateToText(state: DecisionRequest["state"]): string;

package/dist/types/decisions/typesafe-backend.d.ts

@@ -0,0 +1,15 @@
+import type { ModelRegistry } from "../config/model-registry";
+import { type DecisionBackend } from "./types";
+/** Provider id under which the key is stored and surfaced in the model list. */
+export declare const TYPESAFE_PROVIDER = "typesafe";
+export interface TypeSafeBackendDeps {
+    registry: ModelRegistry;
+    sessionId?: string;
+    /** Override for self-hosted or proxied deployments. */
+    baseUrl?: string;
+    /** Model id sent in the request body. Named to avoid colliding with the LLM backend's `model`. */
+    modelId?: string;
+    /** Injected in tests; defaults to global fetch. */
+    fetchImpl?: typeof fetch;
+}
+export declare function createTypeSafeDecisionBackend(deps: TypeSafeBackendDeps): DecisionBackend;

package/dist/types/hooks/skill-state.d.ts

@@ -38,6 +38,12 @@ export interface RecordSkillActivationInput {
     turnId?: string;
     nowIso?: string;
     stateDir?: string;
+    /**
+     * Semantic fallback, consulted only when no keyword matched. Supplying it turns the
+     * literal keyword table into a two-stage router; omitting it keeps the historical
+     * keyword-only behaviour byte for byte.
+     */
+    resolveSkillSemantically?: (text: string) => Promise<SkcWorkflowSkill | null>;
 }
 export interface StopHookInput {
     cwd: string;

package/dist/types/modes/components/provider-onboarding-selector.d.ts

@@ -1,5 +1,5 @@
 import { Container } from "@sayknow-cli/tui";
-export type ProviderOnboardingAction = "custom-provider-wizard" | "oauth-login" | "import-credentials" | "api-guide";
+export type ProviderOnboardingAction = "custom-provider-wizard" | "oauth-login" | "import-credentials" | "api-guide" | "typesafe-key";
 export declare class ProviderOnboardingSelectorComponent extends Container {
     #private;
     constructor(onSelect: (action: ProviderOnboardingAction) => void, onCancel: () => void);

package/dist/types/modes/components/typesafe-key-prompt.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Key entry for TypeSafe, reachable from the same place models are added.
+ *
+ * TypeSafe is not a chat model, so it never appears in the model picker — but the place
+ * users look when they want to "add a model with a key" is this onboarding list, and
+ * making them find a CLI subcommand instead would mean most users never enable it.
+ *
+ * The key is taken through {@link SecretInput} and consumed once: it is never rendered,
+ * never placed in a flag, and never written anywhere but the credential store.
+ */
+import { Container } from "@sayknow-cli/tui";
+export interface TypeSafeKeyPromptResult {
+    apiKey: string;
+}
+export declare class TypeSafeKeyPromptComponent extends Container {
+    #private;
+    constructor(onSubmit: (result: TypeSafeKeyPromptResult) => void, onCancel: () => void, onRender?: () => void);
+    /** Shown while the key is being checked against the live API. */
+    setBusy(busy: boolean): void;
+    /** Keeps the prompt open so a rejected key can be corrected without restarting. */
+    setError(message: string): void;
+    handleInput(keyData: string): void;
+}

package/dist/types/modes/controllers/selector-controller.d.ts

@@ -81,6 +81,15 @@ export declare class SelectorController {
     }): void;
     showCommandPalette(commands: SlashCommand[], actions: CommandPaletteAction[], executeSlashCommand: (name: string) => Promise<void>): void;
     showProviderOnboarding(): void;
+    /**
+     * Take a TypeSafe key and verify it before storing.
+     *
+     * Verification is not optional here. The decision service fails open by design, so an
+     * unverified bad key produces no error anywhere: decisions silently keep coming from
+     * the user's own model while the UI claims TypeSafe is on. Better to keep the prompt
+     * open and say the key was rejected.
+     */
+    showTypeSafeKeyPrompt(): void;
     showCustomModelPresetWizard(snapshot: ModelProfileConfig): void;
     showCustomProviderWizard(): void;
     showEffortSelector(): void;

package/dist/types/modes/interactive-mode.d.ts

@@ -264,6 +264,7 @@ export declare class InteractiveMode implements InteractiveModeContext {
         temporaryOnly?: boolean;
     }): void;
     showEffortSelector(): void;
+    showTypeSafeKeyPrompt(): void;
     showProviderOnboarding(): void;
     showPluginSelector(mode?: "install" | "uninstall"): void;
     showUserMessageSelector(): void;

package/dist/types/modes/types.d.ts

@@ -280,6 +280,8 @@ export interface InteractiveModeContext {
     }): void;
     showEffortSelector(): void;
     showProviderOnboarding(): void;
+    /** Open the TypeSafe key prompt (typed decisions; not a chat model). */
+    showTypeSafeKeyPrompt(): void;
     showPluginSelector(mode?: "install" | "uninstall"): void;
     showUserMessageSelector(): void;
     showTreeSelector(): void;

package/dist/types/sdk/bus/native-runtime-compatibility.d.ts

@@ -2,12 +2,14 @@ export declare class NativeRuntimeCompatibilityError extends Error {
     readonly runtimeVersion: string;
     readonly nativeVersion: string;
     readonly workflowArbitrationAvailable: boolean;
+    readonly nativeModulePath: string | null;
     readonly code = "native_runtime_incompatible";
     readonly retryable = false;
-    constructor(runtimeVersion: string, nativeVersion: string, workflowArbitrationAvailable: boolean);
+    constructor(runtimeVersion: string, nativeVersion: string, workflowArbitrationAvailable: boolean, nativeModulePath?: string | null);
 }
 export declare function assertNativeRuntimeCompatibility(input: {
     runtimeVersion: string;
     nativeVersion: string;
     notificationServer: unknown;
+    nativeModulePath?: string | null;
 }): void;

package/dist/types/session/agent-session.d.ts

@@ -872,15 +872,6 @@ export declare class AgentSession {
     get customCommands(): ReadonlyArray<LoadedCustomCommand>;
     /** Update the MCP prompt commands list. Called when server prompts are (re)loaded. */
     setMCPPromptCommands(commands: LoadedCustomCommand[]): void;
-    /**
-     * Send a prompt to the agent.
-     * - Handles extension commands (registered via pi.registerCommand) immediately, even during streaming
-     * - Expands file-based prompt templates by default
-     * - During streaming, queues via steer() or followUp() based on streamingBehavior option
-     * - Validates model and API key before sending (when not streaming)
-     * @throws Error if streaming and no streamingBehavior specified
-     * @throws Error if no model selected or no API key available (when not streaming)
-     */
     prompt(text: string, options?: PromptOptions): Promise<void>;
     promptCustomMessage<T = unknown>(message: Pick<CustomMessage<T>, "customType" | "content" | "display" | "details" | "attribution">, options?: Pick<PromptOptions, "streamingBehavior" | "toolChoice" | "followUpQueuePolicy" | "onPreflightAccepted" | "onPreflightAcceptCommit">): Promise<void>;
     /**

package/dist/types/setup/decision-provider.d.ts

@@ -0,0 +1,24 @@
+export interface TypeSafeKeySetupResult {
+    provider: string;
+    verified: boolean;
+    /** Present when verification ran and failed; the key is not stored in that case. */
+    error?: string;
+}
+/**
+ * Verify a key against the live API before storing it.
+ *
+ * Storing an unverified key is worse than refusing it: the decision service fails open,
+ * so a bad key produces no error anywhere — it silently falls back to the user's own
+ * model forever, and the user believes TypeSafe is active.
+ */
+export declare function verifyTypeSafeKey(apiKey: string, fetchImpl?: typeof fetch): Promise<string | null>;
+export interface SetTypeSafeKeyOptions {
+    apiKey: string;
+    /** Skip the live check. Only for offline setup; the key may be wrong. */
+    skipVerify?: boolean;
+    fetchImpl?: typeof fetch;
+    dbPath?: string;
+}
+export declare function setTypeSafeKey(options: SetTypeSafeKeyOptions): Promise<TypeSafeKeySetupResult>;
+export declare function removeTypeSafeKey(dbPath?: string): Promise<void>;
+export declare function formatTypeSafeKeyResult(result: TypeSafeKeySetupResult): string;

package/dist/types/setup/model-onboarding-guidance.d.ts

@@ -2,6 +2,11 @@ export declare const MODEL_ONBOARDING_API_PROVIDER_COMMAND = "/provider add --co
 export declare const MODEL_ONBOARDING_PROVIDER_PRESET_COMMAND = "/provider add --preset <minimax|minimax-cn|glm>";
 export declare const MODEL_ONBOARDING_SETUP_COMMAND = "skc setup provider";
 export declare const MODEL_ONBOARDING_OAUTH_COMMAND = "/provider login [provider-id] or /login [provider-id]";
+/**
+ * TypeSafe is not a chat model and never appears in the model list, so the only way a
+ * user learns it exists is from the surfaces where they go to add credentials.
+ */
+export declare const MODEL_ONBOARDING_TYPESAFE_COMMAND = "/provider typesafe";
 export declare function formatModelOnboardingGuidance(): string;
 export declare function formatModelOnboardingInlineHint(): string;
 export declare function formatNoModelOnboardingError(): string;

package/package.json

@@ -1,7 +1,7 @@
 {
 	"type": "module",
 	"name": "@sayknow-cli/coding-agent",
-	"version": "0.5.21",
+	"version": "0.5.23",
 	"description": "Sayknow-CLI CLI with read, bash, edit, write tools and session management",
 	"homepage": "https://sayknow-cli.com",
 	"author": "jaybeyond",
@@ -54,12 +54,12 @@
 		"@agentclientprotocol/sdk": "1.3.0",
 		"@babel/parser": "^7.29.3",
 		"@mozilla/readability": "^0.6.0",
-		"@sayknow-cli/stats": "0.5.21",
-		"@sayknow-cli/agent-core": "0.5.21",
-		"@sayknow-cli/ai": "0.5.21",
-		"@sayknow-cli/natives": "0.5.21",
-		"@sayknow-cli/tui": "0.5.21",
-		"@sayknow-cli/utils": "0.5.21",
+		"@sayknow-cli/stats": "0.5.23",
+		"@sayknow-cli/agent-core": "0.5.23",
+		"@sayknow-cli/ai": "0.5.23",
+		"@sayknow-cli/natives": "0.5.23",
+		"@sayknow-cli/tui": "0.5.23",
+		"@sayknow-cli/utils": "0.5.23",
 		"@puppeteer/browsers": "^2.13.0",
 		"@types/turndown": "5.0.6",
 		"@xterm/headless": "^6.0.0",

package/scripts/eval-skill-routing.ts

@@ -0,0 +1,173 @@
+#!/usr/bin/env bun
+/**
+ * Measure workflow routing against the model the user is logged into.
+ *
+ * Runs the real `decisions/llm-backend` — same forced tool call, same enum
+ * constraint, same model role resolution the session uses — over a fixed set of
+ * prompts, and compares it with the deterministic keyword stage it is meant to
+ * back up.
+ *
+ *   bun scripts/eval-skill-routing.ts [--repeat N] [--json out.json] [--backend typesafe|llm]
+ *
+ * `--backend` pins one backend so the two can be compared head to head. Without it the
+ * normal resolution order applies (TypeSafe when a key exists, else the logged-in model).
+ *
+ * The keyword baseline is recomputed here rather than quoted, so the comparison
+ * can never drift away from what `skill-keywords.ts` currently contains.
+ */
+import { ModelRegistry } from "../src/config/model-registry";
+import { resolveRoleSelection } from "../src/config/model-resolver";
+import { Settings } from "../src/config/settings";
+import { createDecisionService, createLlmDecisionBackend, createTypeSafeDecisionBackend } from "../src/decisions";
+import { createSemanticSkillRouter } from "../src/decisions/skill-routing";
+import { detectPrimarySkillKeyword } from "../src/hooks/skill-state";
+import { discoverAuthStorage } from "../src/sdk";
+
+type Expected = "deep-interview" | "ralplan" | "ultragoal" | "team" | null;
+interface Case {
+	prompt: string;
+	expect: Expected;
+	lang: "ko" | "en";
+}
+
+const CASES: Case[] = [
+	{ prompt: "요구사항이 아직 흐릿한데 나한테 질문해서 스펙을 뽑아줘", expect: "deep-interview", lang: "ko" },
+	{ prompt: "뭘 만들지 정리가 안 됐어. 인터뷰하듯 파고들어줘", expect: "deep-interview", lang: "ko" },
+	{ prompt: "추측하지 말고 모르는 건 다 물어봐", expect: "deep-interview", lang: "ko" },
+	{ prompt: "Ask me questions until the requirements are actually clear", expect: "deep-interview", lang: "en" },
+	{ prompt: "don't assume anything, dig into what I actually need", expect: "deep-interview", lang: "en" },
+	{ prompt: "이거 아키텍처 리스크 커. 실행 전에 합의된 계획부터 세워줘", expect: "ralplan", lang: "ko" },
+	{ prompt: "여러 안 비교해서 검토받을 계획서 만들어줘", expect: "ralplan", lang: "ko" },
+	{ prompt: "Draft a deliberate plan and stop for my approval before touching code", expect: "ralplan", lang: "en" },
+	{ prompt: "consensus plan for the migration", expect: "ralplan", lang: "en" },
+	{ prompt: "이 목표 끝까지 추적해줘. 중간에 잊지 말고", expect: "ultragoal", lang: "ko" },
+	{ prompt: "장기 목표로 등록해두고 진행상황 계속 관리해", expect: "ultragoal", lang: "ko" },
+	{ prompt: "Track this objective until every deliverable is verified", expect: "ultragoal", lang: "en" },
+	{ prompt: "ultragoal this and keep the ledger updated", expect: "ultragoal", lang: "en" },
+	{ prompt: "작업 크니까 워커 여러 개로 나눠서 병렬로 돌려줘", expect: "team", lang: "ko" },
+	{ prompt: "팀 구성해서 각자 파트 맡아 진행하게 해", expect: "team", lang: "ko" },
+	{ prompt: "Spin up coordinated workers for these three slices", expect: "team", lang: "en" },
+	{ prompt: "coordinated team run on the backlog", expect: "team", lang: "en" },
+	{ prompt: "이 테스트 왜 깨지는지 봐줘", expect: null, lang: "ko" },
+	{ prompt: "README 오타 하나 고쳐", expect: null, lang: "ko" },
+	{ prompt: "이 함수 뭐하는 건지 설명해줘", expect: null, lang: "ko" },
+	{ prompt: "우리 서비스에 이 모델 붙이면 뭐가 좋아?", expect: null, lang: "ko" },
+	{ prompt: "fix the failing lint rule in src/utils.ts", expect: null, lang: "en" },
+	{ prompt: "what does this regex do?", expect: null, lang: "en" },
+];
+
+function pct(hit: number, total: number): string {
+	return total === 0 ? "n/a" : `${hit}/${total} (${((hit / total) * 100).toFixed(0)}%)`;
+}
+
+async function main(): Promise<void> {
+	const repeat = Math.max(1, Number(Bun.argv[Bun.argv.indexOf("--repeat") + 1]) || 1);
+	const jsonIndex = Bun.argv.indexOf("--json");
+	const jsonPath = jsonIndex > 0 ? Bun.argv[jsonIndex + 1] : undefined;
+
+	const settings = await Settings.init();
+	const registry = new ModelRegistry(await discoverAuthStorage());
+	await registry.refresh();
+	registry.applyConfiguredModelBindings(settings);
+	const backendIndex = Bun.argv.indexOf("--backend");
+	const pinned = backendIndex > 0 ? Bun.argv[backendIndex + 1] : undefined;
+	const model = resolveRoleSelection(["smol", "default"], settings, registry.getAvailable(), registry)?.model;
+	if (!model && pinned !== "typesafe") throw new Error("no model available — log in first");
+
+	// `--model provider/id` pins one concrete model so candidates can be compared on
+	// measured accuracy and latency instead of on guesses about what "small" means.
+	const modelIndex = Bun.argv.indexOf("--model");
+	const wanted = modelIndex > 0 ? Bun.argv[modelIndex + 1] : undefined;
+	const forced = wanted
+		? registry.getAvailable().find(m => `${m.provider}/${m.id}` === wanted || m.id === wanted)
+		: undefined;
+	if (wanted && !forced) throw new Error(`model not available: ${wanted}`);
+
+	const backends =
+		pinned === "typesafe"
+			? [createTypeSafeDecisionBackend({ registry })]
+			: pinned === "llm" || forced
+				? [createLlmDecisionBackend({ registry, settings, model: forced })]
+				: undefined;
+	// The llm backend now selects its own small model, so the script cannot label the run
+	// from role resolution — doing so reported opus while a 4B model actually answered.
+	const label =
+		pinned === "typesafe" ? "typesafe/jev" : pinned === "llm" ? "llm/auto-small" : `${model?.provider}/${model?.id}`;
+	console.log(`backend: ${pinned ?? "auto"}   model: ${label}   repeat: ${repeat}\n`);
+
+	const route = createSemanticSkillRouter(
+		createDecisionService({ registry, settings, enabled: true, timeoutMs: 30_000, backends }),
+	);
+
+	const rows: Array<{ case: Case; keyword: Expected; semantic: Expected; ms: number }> = [];
+	for (const testCase of CASES) {
+		for (let run = 0; run < repeat; run++) {
+			const keyword = (detectPrimarySkillKeyword(testCase.prompt)?.skill ?? null) as Expected;
+			const started = Date.now();
+			const semantic = (await route(testCase.prompt)) as Expected;
+			rows.push({ case: testCase, keyword, semantic, ms: Date.now() - started });
+			const hybrid = keyword ?? semantic;
+			const mark = hybrid === testCase.expect ? "OK  " : "MISS";
+			console.log(
+				`${mark} [${testCase.lang}] want=${testCase.expect ?? "none"} kw=${keyword ?? "-"} sem=${semantic ?? "none"} ${Date.now() - started}ms :: ${testCase.prompt.slice(0, 44)}`,
+			);
+		}
+	}
+
+	const score = (pick: (row: (typeof rows)[number]) => Expected, filter: (row: (typeof rows)[number]) => boolean) => {
+		const subset = rows.filter(filter);
+		return [subset.filter(row => pick(row) === row.case.expect).length, subset.length] as const;
+	};
+	const positives = (row: (typeof rows)[number]) => row.case.expect !== null;
+	const negatives = (row: (typeof rows)[number]) => row.case.expect === null;
+	const ko = (row: (typeof rows)[number]) => positives(row) && row.case.lang === "ko";
+	const en = (row: (typeof rows)[number]) => positives(row) && row.case.lang === "en";
+	const hybrid = (row: (typeof rows)[number]) => row.keyword ?? row.semantic;
+
+	console.log("\n=== stage comparison ===");
+	for (const [label, pick] of [
+		["keyword only (Codex hook)", (row: (typeof rows)[number]) => row.keyword],
+		["semantic only (SHIPPED)", (row: (typeof rows)[number]) => row.semantic],
+		["keyword+semantic (upper bound)", hybrid],
+	] as const) {
+		console.log(
+			`${label.padEnd(32)} all ${pct(...score(pick, () => true))}  ko ${pct(...score(pick, ko))}  en ${pct(...score(pick, en))}  clean-negatives ${pct(...score(pick, negatives))}`,
+		);
+	}
+
+	const latencies = rows.map(row => row.ms).sort((a, b) => a - b);
+	console.log(
+		`\nlatency p50 ${latencies[Math.floor(latencies.length / 2)]}ms  p95 ${latencies[Math.max(0, Math.ceil(latencies.length * 0.95) - 1)]}ms  max ${latencies.at(-1)}ms`,
+	);
+
+	// Report against what actually ships in this host, not against the upper bound.
+	const misses = rows.filter(row => row.semantic !== row.case.expect);
+	if (misses.length > 0) {
+		console.log("\n=== misses in shipped configuration (semantic only) ===");
+		for (const row of misses)
+			console.log(
+				`  [${row.case.lang}] want=${row.case.expect ?? "none"} got=${row.semantic ?? "none"} :: ${row.case.prompt}`,
+			);
+	}
+
+	if (jsonPath) {
+		await Bun.write(
+			jsonPath,
+			JSON.stringify(
+				{
+					// Must be the backend that actually answered, not the chat model that was
+					// resolved for the fallback path — a mislabelled run poisons later comparisons.
+					model: label,
+					backend: pinned ?? "auto",
+					repeat,
+					rows: rows.map(r => ({ ...r.case, ...r, case: undefined })),
+				},
+				null,
+				2,
+			),
+		);
+		console.log(`\nwrote ${jsonPath}`);
+	}
+}
+
+await main();