@ultimat3/ai 1.2.0 → 3.0.0

package/src/error-body.ts

@@ -0,0 +1,44 @@
+// Single responsibility: what a provider's own failure body says, and the credential that must
+// never survive into it.
+//
+// Shared by both transports rather than copied: every endpoint this package speaks to reports a
+// failure in the same `{ error: { message } }` envelope, and every one of them carries a key in a
+// header a proxy can echo into its own 4xx body. Two copies of either rule is two behaviours to
+// keep in step — and the scrub was on one provider only until 2026-08.
+
+import { REDACTED } from '@ultimat3/core';
+
+/** Enough of an error body to name the field that was wrong, not enough to fill a log. */
+const DETAIL_LIMIT = 300;
+
+/**
+ * The provider's own message, when it sent one — it names the offending field, we name the fix.
+ * Falls back to the raw text because a proxy or a gateway timeout page is not JSON and is still
+ * the best evidence there is.
+ */
+export async function detailOf(response: Response): Promise<string> {
+  const body = await response.text().catch(() => '');
+  try {
+    const parsed: unknown = JSON.parse(body);
+    if (typeof parsed === 'object' && parsed !== null) {
+      const error = (parsed as Record<string, unknown>)['error'];
+      if (typeof error === 'object' && error !== null) {
+        const message = (error as Record<string, unknown>)['message'];
+        if (typeof message === 'string') return message.slice(0, DETAIL_LIMIT);
+      }
+    }
+  } catch {
+    // Not JSON — a proxy or a gateway timeout page. The raw text is still the best evidence.
+  }
+  return body === '' ? response.statusText : body.slice(0, DETAIL_LIMIT);
+}
+
+/**
+ * Every occurrence of the credential replaced with `[redacted]`. Cheap, and the one leak path:
+ * a proxy that echoes the request headers into its own 400 body puts the key in an error, and an
+ * error reaches a log index, a span and an HTTP problem document.
+ */
+export function withoutKey(detail: string, apiKey: string): string {
+  if (apiKey === '') return detail;
+  return detail.split(apiKey).join(REDACTED);
+}

package/src/errors.ts

@@ -10,9 +10,15 @@ export const AI_ERROR_CODES = [
   'X_AI_BUDGET_EXCEEDED',
   'X_AI_GATEWAY_MISSING',
   'X_AI_PROMPT_VERSION',
+  'X_AI_MODEL_UNKNOWN',
+  'X_AI_PROMPT_SECRET',
   'X_LLM_OUTPUT_INVALID',
   'X_LLM_REFUSED',
   'X_LLM_TRUNCATED',
+  'X_LLM_STREAM_INVALID',
+  'X_AGENT_MAX_TURNS',
+  'X_AGENT_TOOL_UNEXPOSED',
+  'X_HIVE_EMPTY',
   'X_EVAL_THRESHOLD',
   'X_EVAL_BASELINE_MISSING',
   'X_EVAL_BASELINE_INVALID',
@@ -32,9 +38,15 @@ export const AI_ERROR_TITLES: Readonly<Record<AiErrorCode, string>> = {
   X_AI_BUDGET_EXCEEDED: 'a model call would exceed its budget',
   X_AI_GATEWAY_MISSING: 'an llm() action ran with no gateway installed',
   X_AI_PROMPT_VERSION: 'prompt version or slots are wrong',
+  X_AI_MODEL_UNKNOWN: 'a model id nothing registered in the catalogue',
+  X_AI_PROMPT_SECRET: 'a Secret was about to be rendered into a prompt',
   X_LLM_OUTPUT_INVALID: 'structured output failed its schema on the answer and the repair turn',
   X_LLM_REFUSED: 'the model declined the request',
   X_LLM_TRUNCATED: 'the answer hit its maxTokens ceiling before it was complete',
+  X_LLM_STREAM_INVALID: 'a streamed answer failed its output schema, and a stream cannot repair',
+  X_AGENT_MAX_TURNS: 'an agent hit its turn ceiling without answering',
+  X_AGENT_TOOL_UNEXPOSED: 'an agent lists a tool that is not an MCP-exposed action',
+  X_HIVE_EMPTY: 'a hive split produced no members, so the run did nothing',
   X_EVAL_THRESHOLD: 'an eval scored below its tolerance',
   X_EVAL_BASELINE_MISSING: 'an eval has no recorded baseline to gate against',
   X_EVAL_BASELINE_INVALID: 'a recorded baseline cannot be read',
@@ -108,6 +120,63 @@ export class AiGatewayMissingError extends UltimateError {
   }
 }
 
+/**
+ * A model id nothing put in the catalogue. This is what replaced the closed `ModelId` union: the
+ * union made a company's own model id inexpressible, so the only way past `tsc` was to claim a
+ * Claude id — and then `costOf` priced an internal model at Anthropic list rates and the budget
+ * ledger reserved against a number belonging to a model nobody ran. A wrong id is still refused;
+ * it is refused HERE, at the first read of the spec, instead of by making a right one impossible.
+ */
+export class AiModelUnknownError extends UltimateError {
+  constructor(input: { model: string; registered: readonly string[] }) {
+    super({
+      code: 'X_AI_MODEL_UNKNOWN',
+      cause:
+        `model "${input.model}" has no registered spec, so nothing can price it ` +
+        `(registered: ${input.registered.length > 0 ? input.registered.join(', ') : 'none'})`,
+      // The `errors` gate blanks every interpolation, so the literal half alone has to name the
+      // call. Which ids ARE registered is a fact of the failure, and cause is where facts live.
+      fix: 'registerModel({ id, contextWindow, maxOutput, inputPerMillion, outputPerMillion, cacheMinimumTokens, reasoning }) at boot, before configureAi',
+      docs: docsFor('X_AI_MODEL_UNKNOWN'),
+      meta: { model: input.model },
+    });
+  }
+}
+
+/**
+ * A `Secret` reached a prompt variable. `Secret` redacts by VALUE, so this would not have leaked
+ * — it would have rendered `[redacted]` into the template and asked the model to reason about it,
+ * which is a prompt that reads fine and means something else. The same class of failure as an
+ * unfilled `{{slot}}`, and refused for the same reason: loudly, before a token is spent.
+ */
+export class AiPromptSecretError extends UltimateError {
+  constructor(input: { ref: string; keys: readonly string[] }) {
+    super({
+      code: 'X_AI_PROMPT_SECRET',
+      cause: `prompt "${input.ref}" was given a Secret in vars(): ${input.keys.join(', ')}`,
+      fix: 'drop the key from vars() and from the template, or revealSecret(value) in vars() if the model genuinely has to read it',
+      docs: docsFor('X_AI_PROMPT_SECRET'),
+    });
+  }
+}
+
+/**
+ * A streamed answer did not satisfy its `output` schema. Distinct from `X_LLM_OUTPUT_INVALID`
+ * because there is no repair turn to have failed: the consumer has already read the tokens, and
+ * replaying a second answer over the top is two answers to one question. So a stream gets one
+ * attempt, and the fix is either a looser schema or the non-streaming call that CAN repair.
+ */
+export class LlmStreamInvalidError extends UltimateError {
+  constructor(input: { prompt: string; issues: string }) {
+    super({
+      code: 'X_LLM_STREAM_INVALID',
+      cause: `streamed answer to prompt "${input.prompt}" failed its output schema: ${input.issues}`,
+      fix: 'call the action instead of .stream() when the answer must satisfy a structured schema — a stream has already delivered its tokens and cannot take a repair turn',
+      docs: docsFor('X_LLM_STREAM_INVALID'),
+    });
+  }
+}
+
 /**
  * The model's answer failed the action's `output` schema on the first turn AND on the repair
  * turn that followed. Two failures is a disagreement between the prompt and the schema, not a
@@ -126,6 +195,50 @@ export class LlmOutputInvalidError extends UltimateError {
   }
 }
 
+/**
+ * An `agent()` ran out of turns with no answer. Never a partial one: the loop's whole contract is
+ * that it either satisfies `output` or says it did not, and a half-finished transcript returned as
+ * a result is a model's working notes presented as a decision.
+ *
+ * Reaching the ceiling almost always means the loop has no exit condition — a tool that answers
+ * the same thing every turn, or a prompt that never tells the model to finish. Raising the
+ * ceiling on that spends more money on the same non-answer, which is why the fix names the prompt
+ * before it names the number.
+ */
+export class AgentMaxTurnsError extends UltimateError {
+  constructor(input: { agent: string; turns: number; calls: number }) {
+    super({
+      code: 'X_AGENT_MAX_TURNS',
+      cause:
+        `agent "${input.agent}" used all ${input.turns} turns and ${input.calls} tool calls ` +
+        `without calling the respond tool`,
+      fix: 'tell the template when to stop and answer through the respond tool, then bump its version — raise maxTurns only once the run demonstrably converges',
+      docs: docsFor('X_AGENT_MAX_TURNS'),
+      meta: { agent: input.agent, turns: input.turns },
+    });
+  }
+}
+
+/**
+ * An `agent()` lists an action that is not an MCP-exposed tool. Refused at DECLARATION rather
+ * than filtered at the call, because a silently dropped tool is the worst of both: the
+ * declaration reads as if the model can call it, and the model is never offered it.
+ *
+ * `isMcpExposed` is the one predicate — an in-app agent and an external MCP client see exactly
+ * the same catalogue, which is what keeps "there is no second authz system" true of the catalogue
+ * too.
+ */
+export class AgentToolUnexposedError extends UltimateError {
+  constructor(input: { agent: string; tools: readonly string[] }) {
+    super({
+      code: 'X_AGENT_TOOL_UNEXPOSED',
+      cause: `agent "${input.agent}" lists tools no MCP surface exposes: ${input.tools.join(', ')}`,
+      fix: 'add mcp: { expose: true } to the action named in cause, or drop it from the agent tools list',
+      docs: docsFor('X_AGENT_TOOL_UNEXPOSED'),
+    });
+  }
+}
+
 /**
  * The provider's safety classifiers declined the request. A refusal is a 200 with no answer in
  * it, so it has to become an error HERE or it becomes an empty string somewhere downstream that
@@ -137,8 +250,13 @@ export class LlmRefusedError extends UltimateError {
   constructor(input: {
     prompt: string;
     model: string;
-    /** A blessed model that is NOT the one that refused — the fix has to be pasteable. */
-    alternative: string;
+    /**
+     * A blessed model MORE capable than the one that refused, or `undefined` when the refusal
+     * came from the most capable one this build knows. Retrying a refusal on a weaker model is
+     * the one retry that cannot help, so the fix line drops the suggestion rather than inventing
+     * a downgrade.
+     */
+    alternative: string | undefined;
     category: string | undefined;
     explanation: string | undefined;
   }) {
@@ -148,7 +266,10 @@ export class LlmRefusedError extends UltimateError {
         `model "${input.model}" declined prompt "${input.prompt}"` +
         `${input.category === undefined ? '' : ` (${input.category})`}` +
         `${input.explanation === undefined ? '' : `: ${input.explanation}`}`,
-      fix: `set model: '${input.alternative}' on the llm() declaration, or edit the template in definePrompt('${input.prompt}') and bump its version`,
+      fix:
+        input.alternative === undefined
+          ? `edit the template in definePrompt('${input.prompt}') and bump its version — no blessed model is more capable than '${input.model}'`
+          : `set model: '${input.alternative}' on the llm() declaration, or edit the template in definePrompt('${input.prompt}') and bump its version`,
       docs: docsFor('X_LLM_REFUSED'),
       meta: { model: input.model, category: input.category },
     });
@@ -201,100 +322,15 @@ export class AiPromptRenderError extends UltimateError {
   }
 }
 
-/**
- * An eval scored further below its recorded baseline than its tolerance allows. The gate is the
- * DROP, not an absolute number — a model that got marginally worse everywhere is not the same
- * event as a prompt edit that broke one case, and only the second one is anybody's fault.
- *
- * This is a test failure, not a warning.
- */
-export class EvalThresholdError extends UltimateError {
-  constructor(input: {
-    eval: string;
-    score: number;
-    baseline: number;
-    tolerance: number;
-    promptVersion: string;
-    regressed: readonly string[];
-  }) {
-    super({
-      code: 'X_EVAL_THRESHOLD',
-      cause:
-        `eval "${input.eval}" scored ${input.score.toFixed(3)} against a recorded baseline of ` +
-        `${input.baseline.toFixed(3)} (tolerance ${input.tolerance.toFixed(3)}) on prompt ` +
-        `version ${input.promptVersion}; regressed: ${input.regressed.join(', ')}`,
-      fix: `x test ${input.eval} to see per-case scores, then fix the prompt — or ULTIMATE_EVAL_RECORD=1 x test eval to accept the new numbers as a reviewed diff`,
-      docs: docsFor('X_EVAL_THRESHOLD'),
-    });
-  }
-}
-
-/**
- * An eval declared a baseline that has never been recorded. Not a pass: an eval with nothing to
- * compare against gates on nothing, and a step that cannot fail is a step that is not running.
- */
-export class EvalBaselineMissingError extends UltimateError {
-  constructor(input: { eval: string; path: string; reason: string; fix?: string }) {
-    super({
-      code: 'X_EVAL_BASELINE_MISSING',
-      cause: `eval "${input.eval}" gates against ${input.path}, which ${input.reason}`,
-      fix: input.fix ?? `ULTIMATE_EVAL_RECORD=1 x test eval, then commit ${input.path}`,
-      docs: docsFor('X_EVAL_BASELINE_MISSING'),
-    });
-  }
-}
-
-/** A recorded baseline that cannot be read. Never treated as absent — that would erase a gate. */
-export class EvalBaselineInvalidError extends UltimateError {
-  constructor(input: { path: string; problem: string }) {
-    super({
-      code: 'X_EVAL_BASELINE_INVALID',
-      cause: `the recorded baseline ${input.path} ${input.problem}`,
-      fix: `ULTIMATE_EVAL_RECORD=1 x test eval to re-record ${input.path}`,
-      docs: docsFor('X_EVAL_BASELINE_INVALID'),
-    });
-  }
-}
-
-/**
- * A registered prompt that no eval names. An unevaluated prompt is untested code that costs
- * money and answers users, so the gate fails on it exactly like an untyped module.
- */
-export class EvalMissingError extends UltimateError {
-  constructor(input: { prompt: string; id: string }) {
-    super({
-      code: 'X_EVAL_MISSING',
-      cause: `prompt "${input.prompt}" has no eval`,
-      fix: `defineEval({ name: '${input.id}', prompt, cases, scorers, tolerance, baseline }) beside the prompt, then ULTIMATE_EVAL_RECORD=1 x test eval`,
-      docs: docsFor('X_EVAL_MISSING'),
-    });
-  }
-}
-
-/**
- * The gate ran with baseline recording switched on. Recording makes every eval write the numbers
- * it just measured and pass, so a `x verify` that inherited the flag reports green over scores
- * nothing compared — and rewrites the committed baselines on its way through, which is the half
- * a red step alone would not undo. Recording is a deliberate, reviewable diff, never a gate run.
- */
-export class EvalRecordingError extends UltimateError {
-  constructor(input: { env: string }) {
-    super({
-      code: 'X_EVAL_RECORDING',
-      cause: `${input.env} is set, so every eval would re-record its baseline instead of gating on it`,
-      fix: `env -u ${input.env} x verify`,
-      docs: docsFor('X_EVAL_RECORDING'),
-    });
-  }
-}
-
 /** A vector's length does not match the store's declared dimension. */
 export class VectorDimMismatchError extends UltimateError {
   constructor(input: { store: string; expected: number; received: number }) {
     super({
       code: 'X_VECTOR_DIM_MISMATCH',
       cause: `store "${input.store}" expects ${input.expected} dimensions, got ${input.received}`,
-      fix: 'use the same embedder that created the store, or x ai reindex to rebuild it',
+      // Not `x ai reindex`: that command is PLANNED and throws, so a fix line naming it sends an
+      // operator to a wall. A fix has to be performable today, which here means app code.
+      fix: 'use the same embedder that created the store, or re-embed every record at the new width and upsert it',
       docs: docsFor('X_VECTOR_DIM_MISMATCH'),
     });
   }
@@ -330,7 +366,7 @@ export class EmbedderDimMismatchError extends UltimateError {
       cause:
         `embedder "${input.embedder}" is declared with ${input.expected} dimensions but the ` +
         `provider returned ${input.received}`,
-      fix: `set dimension: ${input.received} on the embedder, then x ai reindex to rebuild the store`,
+      fix: `set dimension: ${input.received} on the embedder, then re-embed every record at that width and upsert it`,
       docs: docsFor('X_VECTOR_DIM_MISMATCH'),
     });
   }
@@ -393,13 +429,23 @@ export class AiRequestInvalidError extends UltimateError {
 export class AiTransportError extends UltimateError {
   readonly status: number | undefined;
 
-  constructor(input: { provider: string; status?: number | undefined; detail: string }) {
+  constructor(input: {
+    provider: string;
+    status?: number | undefined;
+    detail: string;
+    /**
+     * The env var holding THIS provider's key. Passed on the HTTP path, where a 401 has one fix:
+     * a hardcoded `ANTHROPIC_API_KEY` was the whole fix line, so an OpenAI-format endpoint
+     * rejecting a key sent an operator to set a variable it never reads.
+     */
+    envVar?: string | undefined;
+  }) {
     super({
       code: 'X_AI_PROVIDER_UNAVAILABLE',
       cause: `provider "${input.provider}" ${
         input.status === undefined ? 'failed' : `returned ${input.status}`
       }: ${input.detail}`,
-      fix: fixForStatus(input.status),
+      fix: fixForStatus(input.status, input.envVar),
       docs: docsFor('X_AI_PROVIDER_UNAVAILABLE'),
       meta: { provider: input.provider, status: input.status },
     });
@@ -407,9 +453,11 @@ export class AiTransportError extends UltimateError {
   }
 }
 
-function fixForStatus(status: number | undefined): string {
+function fixForStatus(status: number | undefined, envVar: string | undefined): string {
   if (status === 401 || status === 403) {
-    return 'export ANTHROPIC_API_KEY=<key> with a key that is active for this model';
+    return envVar === undefined
+      ? 'export the API key env var of the provider named in cause, with a key that is active for this model'
+      : `export ${envVar}=<key> with a key that is active for this model`;
   }
   if (status === 429) {
     return 'lower concurrency or raise the provider rate limit; the gateway already backs off';

package/src/eval-baseline.ts

@@ -9,7 +9,7 @@
 // not a measurement.
 
 import { isAbsolute } from 'node:path';
-import { EvalBaselineInvalidError, EvalBaselineMissingError } from './errors';
+import { EvalBaselineInvalidError, EvalBaselineMissingError } from './eval-errors';
 
 export interface EvalBaseline {
   readonly eval: string;

package/src/eval-errors.ts

@@ -0,0 +1,98 @@
+// The five X_EVAL_* codes, apart from ./errors only because one file has one job and the catalogue
+// outgrew its ceiling. The codes themselves, their titles and the single `registerErrorCodes` call
+// stay in ./errors — one owner, one registration, one place a duplicate can surface.
+
+import { UltimateError } from '@ultimat3/core';
+import type { AiErrorCode } from './errors';
+
+const docsFor = (code: AiErrorCode): string => `https://ultimate.dev/errors/${code}`;
+
+/**
+ * An eval scored further below its recorded baseline than its tolerance allows. The gate is the
+ * DROP, not an absolute number — a model that got marginally worse everywhere is not the same
+ * event as a prompt edit that broke one case, and only the second one is anybody's fault.
+ *
+ * This is a test failure, not a warning.
+ */
+export class EvalThresholdError extends UltimateError {
+  constructor(input: {
+    eval: string;
+    score: number;
+    baseline: number;
+    tolerance: number;
+    promptVersion: string;
+    regressed: readonly string[];
+  }) {
+    super({
+      code: 'X_EVAL_THRESHOLD',
+      cause:
+        `eval "${input.eval}" scored ${input.score.toFixed(3)} against a recorded baseline of ` +
+        `${input.baseline.toFixed(3)} (tolerance ${input.tolerance.toFixed(3)}) on prompt ` +
+        `version ${input.promptVersion}; regressed: ${input.regressed.join(', ')}`,
+      // `x test eval --filter <name>`, never `x test <name>`: `x test`'s positional is a TestType,
+      // so the eval's own name there is `X_CLI_BAD_FLAG` ("not a test type") — a fix line that
+      // cannot be run is axiom 4 broken at the one moment it is needed.
+      fix: `x test eval --filter ${input.eval} to see per-case scores, then fix the prompt — or ULTIMATE_EVAL_RECORD=1 x test eval to accept the new numbers as a reviewed diff`,
+      docs: docsFor('X_EVAL_THRESHOLD'),
+    });
+  }
+}
+
+/**
+ * An eval declared a baseline that has never been recorded. Not a pass: an eval with nothing to
+ * compare against gates on nothing, and a step that cannot fail is a step that is not running.
+ */
+export class EvalBaselineMissingError extends UltimateError {
+  constructor(input: { eval: string; path: string; reason: string; fix?: string }) {
+    super({
+      code: 'X_EVAL_BASELINE_MISSING',
+      cause: `eval "${input.eval}" gates against ${input.path}, which ${input.reason}`,
+      fix: input.fix ?? `ULTIMATE_EVAL_RECORD=1 x test eval, then commit ${input.path}`,
+      docs: docsFor('X_EVAL_BASELINE_MISSING'),
+    });
+  }
+}
+
+/** A recorded baseline that cannot be read. Never treated as absent — that would erase a gate. */
+export class EvalBaselineInvalidError extends UltimateError {
+  constructor(input: { path: string; problem: string }) {
+    super({
+      code: 'X_EVAL_BASELINE_INVALID',
+      cause: `the recorded baseline ${input.path} ${input.problem}`,
+      fix: `ULTIMATE_EVAL_RECORD=1 x test eval to re-record ${input.path}`,
+      docs: docsFor('X_EVAL_BASELINE_INVALID'),
+    });
+  }
+}
+
+/**
+ * A registered prompt that no eval names. An unevaluated prompt is untested code that costs
+ * money and answers users, so the gate fails on it exactly like an untyped module.
+ */
+export class EvalMissingError extends UltimateError {
+  constructor(input: { prompt: string; id: string }) {
+    super({
+      code: 'X_EVAL_MISSING',
+      cause: `prompt "${input.prompt}" has no eval`,
+      fix: `defineEval({ name: '${input.id}', prompt, cases, scorers, tolerance, baseline }) beside the prompt, then ULTIMATE_EVAL_RECORD=1 x test eval`,
+      docs: docsFor('X_EVAL_MISSING'),
+    });
+  }
+}
+
+/**
+ * The gate ran with baseline recording switched on. Recording makes every eval write the numbers
+ * it just measured and pass, so a `x verify` that inherited the flag reports green over scores
+ * nothing compared — and rewrites the committed baselines on its way through, which is the half
+ * a red step alone would not undo. Recording is a deliberate, reviewable diff, never a gate run.
+ */
+export class EvalRecordingError extends UltimateError {
+  constructor(input: { env: string }) {
+    super({
+      code: 'X_EVAL_RECORDING',
+      cause: `${input.env} is set, so every eval would re-record its baseline instead of gating on it`,
+      fix: `env -u ${input.env} x verify`,
+      docs: docsFor('X_EVAL_RECORDING'),
+    });
+  }
+}

package/src/evals.ts

@@ -11,7 +11,6 @@
 // Every result is filed against a prompt's content hash, so a score is always attributable
 // to an exact prompt rather than "whatever was in main that day".
 
-import { EvalBaselineMissingError, EvalThresholdError } from './errors';
 import type { EvalBaseline, Regression } from './eval-baseline';
 import {
   baselinePath,
@@ -21,6 +20,7 @@ import {
   regressionsAgainst,
   writeBaseline,
 } from './eval-baseline';
+import { EvalBaselineMissingError, EvalThresholdError } from './eval-errors';
 import type { Gateway } from './gateway';
 import type { Prompt, PromptVars } from './prompt';
 import { describePrompts } from './prompt';

package/src/fix-line.evals.ts

@@ -0,0 +1,35 @@
+/**
+ * The eval attached to `fixLinePrompt`. Cases live here rather than in the test file, matching
+ * every app's own prompts — a fixture-driven, deterministic `exact` scorer, no judge, because
+ * this is a proof of the eval mechanism itself and a proof that can drift is not one.
+ */
+
+import { defineEval } from './evals';
+import { fixLinePrompt } from './fix-line';
+import { exact } from './scorers';
+
+export const fixLineCases = [
+  {
+    name: 'runnable/x-command',
+    vars: { fixLine: 'bun run scripts/verify.ts --only lint,boundaries' },
+    expected: 'runnable',
+  },
+  { name: 'runnable/cli-flag', vars: { fixLine: 'x db migrate' }, expected: 'runnable' },
+  {
+    name: 'vague/check',
+    vars: { fixLine: 'check the configuration and try again' },
+    expected: 'vague',
+  },
+  { name: 'vague/see-docs', vars: { fixLine: 'see the docs for details' }, expected: 'vague' },
+];
+
+export const fixLineEval = defineEval({
+  name: 'ai.fix-line-runnable',
+  prompt: fixLinePrompt,
+  // The gate is the drop from this recorded score, never an absolute number: models drift,
+  // prompts should not. Accepting a new number is a diff in the committed baseline file.
+  baseline: import.meta.resolve('./fix-line.v1.baseline.json'),
+  tolerance: 0.05,
+  scorers: [exact],
+  cases: fixLineCases,
+});

package/src/fix-line.ts

@@ -0,0 +1,27 @@
+/**
+ * The prompt behind `fix-line` — `@ultimat3/ai`'s own dogfood eval, and the package's first
+ * framework-level `*.eval.test.ts`. Not an app feature: every app that declares a prompt is
+ * required to pair it with an eval and a committed baseline (`defineEval`, `X_EVAL_MISSING`,
+ * `X_EVAL_BASELINE_MISSING`), and this proves that whole convention actually catches a
+ * regression from inside the package that owns it — through the real opt-in suite `x verify`'s
+ * `eval` step runs, not only through `evals.test.ts`'s unit fixtures against a temp-dir baseline.
+ *
+ * The task is small on purpose: classify whether an error's `fix:` line is a runnable command —
+ * axiom 4, "errors are instructions" — or vague guidance ("check the config", "see the docs").
+ */
+
+import { definePrompt } from './prompt';
+
+export const fixLinePrompt = definePrompt<{ fixLine: string }>({
+  id: 'ai.fix-line-runnable',
+  version: '1',
+  template:
+    'Reply with exactly one word: "runnable" if the fix line below names a command to run, or ' +
+    '"vague" if it only describes what to do without naming one.\nFix line: {{fixLine}}',
+  input: {
+    type: 'object',
+    properties: { fixLine: { type: 'string' } },
+    required: ['fixLine'],
+  },
+  output: { type: 'string', enum: ['runnable', 'vague'] },
+});

package/src/fix-line.v1.baseline.json

@@ -0,0 +1,12 @@
+{
+  "eval": "ai.fix-line-runnable",
+  "prompt": "ai.fix-line-runnable@1",
+  "promptHash": "6b788c934d335044043d13c71c1c7b49",
+  "score": 1,
+  "cases": {
+    "runnable/cli-flag": 1,
+    "runnable/x-command": 1,
+    "vague/check": 1,
+    "vague/see-docs": 1
+  }
+}