@ultimat3/ai 1.2.0 → 2.0.0

package/src/fix-line.evals.ts

@@ -0,0 +1,35 @@
+/**
+ * The eval attached to `fixLinePrompt`. Cases live here rather than in the test file, matching
+ * every app's own prompts — a fixture-driven, deterministic `exact` scorer, no judge, because
+ * this is a proof of the eval mechanism itself and a proof that can drift is not one.
+ */
+
+import { defineEval } from './evals';
+import { fixLinePrompt } from './fix-line';
+import { exact } from './scorers';
+
+export const fixLineCases = [
+  {
+    name: 'runnable/x-command',
+    vars: { fixLine: 'bun run scripts/verify.ts --only lint,boundaries' },
+    expected: 'runnable',
+  },
+  { name: 'runnable/cli-flag', vars: { fixLine: 'x db migrate' }, expected: 'runnable' },
+  {
+    name: 'vague/check',
+    vars: { fixLine: 'check the configuration and try again' },
+    expected: 'vague',
+  },
+  { name: 'vague/see-docs', vars: { fixLine: 'see the docs for details' }, expected: 'vague' },
+];
+
+export const fixLineEval = defineEval({
+  name: 'ai.fix-line-runnable',
+  prompt: fixLinePrompt,
+  // The gate is the drop from this recorded score, never an absolute number: models drift,
+  // prompts should not. Accepting a new number is a diff in the committed baseline file.
+  baseline: import.meta.resolve('./fix-line.v1.baseline.json'),
+  tolerance: 0.05,
+  scorers: [exact],
+  cases: fixLineCases,
+});

package/src/fix-line.ts

@@ -0,0 +1,27 @@
+/**
+ * The prompt behind `fix-line` — `@ultimat3/ai`'s own dogfood eval, and the package's first
+ * framework-level `*.eval.test.ts`. Not an app feature: every app that declares a prompt is
+ * required to pair it with an eval and a committed baseline (`defineEval`, `X_EVAL_MISSING`,
+ * `X_EVAL_BASELINE_MISSING`), and this proves that whole convention actually catches a
+ * regression from inside the package that owns it — through the real opt-in suite `x verify`'s
+ * `eval` step runs, not only through `evals.test.ts`'s unit fixtures against a temp-dir baseline.
+ *
+ * The task is small on purpose: classify whether an error's `fix:` line is a runnable command —
+ * axiom 4, "errors are instructions" — or vague guidance ("check the config", "see the docs").
+ */
+
+import { definePrompt } from './prompt';
+
+export const fixLinePrompt = definePrompt<{ fixLine: string }>({
+  id: 'ai.fix-line-runnable',
+  version: '1',
+  template:
+    'Reply with exactly one word: "runnable" if the fix line below names a command to run, or ' +
+    '"vague" if it only describes what to do without naming one.\nFix line: {{fixLine}}',
+  input: {
+    type: 'object',
+    properties: { fixLine: { type: 'string' } },
+    required: ['fixLine'],
+  },
+  output: { type: 'string', enum: ['runnable', 'vague'] },
+});

package/src/fix-line.v1.baseline.json

@@ -0,0 +1,12 @@
+{
+  "eval": "ai.fix-line-runnable",
+  "prompt": "ai.fix-line-runnable@1",
+  "promptHash": "6b788c934d335044043d13c71c1c7b49",
+  "score": 1,
+  "cases": {
+    "runnable/cli-flag": 1,
+    "runnable/x-command": 1,
+    "vague/check": 1,
+    "vague/see-docs": 1
+  }
+}

package/src/gateway.ts

@@ -98,10 +98,20 @@ class GatewayImpl implements Gateway {
     // cheap-in-tokens call on an expensive model is still a cost cap the app declared.
     // `record` below replaces the estimate with the provider's real counts.
     const ledger = currentBudget();
-    await ledger?.reserve(estimateSpend(resolved));
-
-    const result = await this.attempt(model, (provider) => provider.generate(resolved));
-    await ledger?.record(result.usage, result.cost);
+    // The estimate is DEBITED here, not merely checked: three concurrent calls under one ledger
+    // all read the same `spent()` otherwise, all pass, and all three record against a ceiling
+    // only one of them fitted.
+    const reservation = await ledger?.reserve(estimateSpend(resolved));
+
+    let result: GenerateResult;
+    try {
+      result = await this.attempt(model, (provider) => provider.generate(resolved));
+    } catch (error) {
+      // A call that never landed must not go on holding its reservation.
+      await ledger?.release(reservation);
+      throw error;
+    }
+    await ledger?.record(result.usage, result.cost, reservation);
     // A refusal is not an answer, so it is not cached. Storing one would keep serving a decision
     // the classifier might not make twice, long after the prompt that provoked it was fixed.
     if (result.stopReason !== 'refusal') {
@@ -114,14 +124,26 @@ class GatewayImpl implements Gateway {
     const model = request.model ?? this.config.defaultModel ?? DEFAULT_MODEL;
     const resolved: GenerateRequest = { ...request, model };
     const ledger = currentBudget();
-    await ledger?.reserve(estimateSpend(resolved));
+    const reservation = await ledger?.reserve(estimateSpend(resolved));
 
     // A stream is not retried mid-flight: the consumer has already seen tokens, and
     // replaying from the top would duplicate them. Only the handshake retries.
     const provider = this.providerFor(model);
-    for await (const chunk of provider.stream(resolved)) {
-      if (chunk.type === 'done') await ledger?.record(chunk.result.usage, chunk.result.cost);
-      yield chunk;
+    let settled = false;
+    try {
+      for await (const chunk of provider.stream(resolved)) {
+        if (chunk.type !== 'done') {
+          yield chunk;
+          continue;
+        }
+        settled = true;
+        await ledger?.record(chunk.result.usage, chunk.result.cost, reservation);
+        yield { type: 'done', result: { ...chunk.result, provider: provider.name } };
+      }
+    } finally {
+      // A stream that threw, or that its consumer abandoned, never reached `done` — so nothing
+      // reconciled the reservation and it would hold the ceiling for the rest of the window.
+      if (!settled) await ledger?.release(reservation);
     }
   }
 
@@ -140,8 +162,16 @@ class GatewayImpl implements Gateway {
    * Try every provider that serves `model`, retrying each on a retryable failure with
    * exponential backoff plus full jitter (jitter matters: synchronised retries from N
    * workers reproduce the rate limit they are backing off from).
+   *
+   * Fallback here is across PROVIDERS serving one model, never across models — a silent model
+   * swap changes what answered, what it cost and which eval baseline the answer belongs to. The
+   * provider that did answer is stamped onto the result, so the fallback that DOES exist reaches
+   * the span instead of being invisible.
    */
-  private async attempt<T>(model: ModelId, call: (provider: Provider) => Promise<T>): Promise<T> {
+  private async attempt(
+    model: ModelId,
+    call: (provider: Provider) => Promise<GenerateResult>,
+  ): Promise<GenerateResult> {
     const candidates = this.config.providers.filter((p) => p.models.includes(model));
     const failures: string[] = [];
     if (candidates.length === 0) {
@@ -151,7 +181,7 @@ class GatewayImpl implements Gateway {
     for (const provider of candidates) {
       for (let attempt = 1; attempt <= this.retry.attempts; attempt += 1) {
         try {
-          return await call(provider);
+          return { ...(await call(provider)), provider: provider.name };
         } catch (error) {
           failures.push(`${provider.name}#${attempt}: ${messageOf(error)}`);
           if (!isRetryable(error) || attempt === this.retry.attempts) break;

package/src/index.ts

@@ -4,6 +4,8 @@
 /** Re-exported so an `llm` file needs one import, not two. Same object as schema's. */
 export type { Infer } from '@ultimat3/schema';
 export { t } from '@ultimat3/schema';
+export type { AgentBudget, AgentDef, AgentVarsArgs } from './agent';
+export { agent } from './agent';
 export type {
   BudgetLedgerInput,
   BudgetLimits,
@@ -30,24 +32,24 @@ export {
 } from './embeddings';
 export type { AiErrorCode } from './errors';
 export {
+  AgentMaxTurnsError,
+  AgentToolUnexposedError,
   AI_ERROR_CODES,
   AI_ERROR_TITLES,
   AiBudgetExceededError,
   AiGatewayMissingError,
   AiKeyMissingError,
+  AiModelUnknownError,
   AiPromptRenderError,
+  AiPromptSecretError,
   AiPromptVersionError,
   AiProviderUnavailableError,
   AiRequestInvalidError,
   AiTransportError,
   EmbedderDimMismatchError,
-  EvalBaselineInvalidError,
-  EvalBaselineMissingError,
-  EvalMissingError,
-  EvalRecordingError,
-  EvalThresholdError,
   LlmOutputInvalidError,
   LlmRefusedError,
+  LlmStreamInvalidError,
   LlmTruncatedError,
   VectorDimMismatchError,
   VectorScopeWidenedError,
@@ -63,6 +65,13 @@ export {
   regressionsAgainst,
   writeBaseline,
 } from './eval-baseline';
+export {
+  EvalBaselineInvalidError,
+  EvalBaselineMissingError,
+  EvalMissingError,
+  EvalRecordingError,
+  EvalThresholdError,
+} from './eval-errors';
 export type {
   CaseResult,
   DefineEvalInput,
@@ -81,6 +90,7 @@ export {
 export type { CreateGatewayInput, Gateway, GatewayCache, RetryPolicy } from './gateway';
 export { backoffMs, cacheKeyFor, createGateway, DEFAULT_RETRY, isRetryable } from './gateway';
 export type {
+  LlmAction,
   LlmBudget,
   LlmCache,
   LlmDef,
@@ -88,8 +98,25 @@ export type {
   LlmVarsArgs,
 } from './llm';
 export { llm } from './llm';
+export type { LlmStreamChunk } from './llm-stream';
 export type { Effort, ModelId, ModelReasoning, ModelSpec, ThinkingMode } from './models';
-export { DEFAULT_MODEL, EFFORTS, MODEL_IDS, MODELS, reasoningBody } from './models';
+export {
+  ANTHROPIC_MODEL_IDS,
+  assertModel,
+  DEFAULT_MODEL,
+  EFFORTS,
+  isModelRegistered,
+  modelIds,
+  modelSpec,
+  moreCapableThan,
+  reasoningBody,
+  registeredModels,
+  registerModel,
+  resetModels,
+} from './models';
+export { OPENAI_MODEL_IDS, registerOpenAiModels } from './openai-models';
+export type { OpenAiProviderInput } from './openai-provider';
+export { openAiProvider } from './openai-provider';
 export type { PgVectorStoreInput } from './pg-vector';
 export { PgVectorStore } from './pg-vector';
 export type {
@@ -118,6 +145,7 @@ export {
   resetPrompts,
 } from './prompt';
 export type {
+  AiContentBlock,
   AiMessage,
   AnthropicProviderInput,
   EchoProviderInput,
@@ -137,6 +165,7 @@ export {
   estimateInputTokens,
   estimateTextTokens,
   estimateTokens,
+  messageText,
   parseMessage,
   requiresStreaming,
   STREAM_ONLY_MAX_TOKENS,
@@ -150,10 +179,18 @@ export type {
   RetrieveInput,
 } from './rag';
 export { assembleContext, chunk, indexDocument, passthroughReranker, retrieve } from './rag';
+export { assertNoSecrets } from './redaction';
 export type { RemoteEmbedderInput } from './remote-embedder';
 export { RemoteEmbedder } from './remote-embedder';
-export type { AiRuntimeInput } from './runtime';
-export { aiEmbedder, aiGateway, configureAi, resetAiRuntime, semanticCacheFor } from './runtime';
+export type { AiRuntimeInput, Redactor } from './runtime';
+export {
+  aiEmbedder,
+  aiGateway,
+  aiRedactor,
+  configureAi,
+  resetAiRuntime,
+  semanticCacheFor,
+} from './runtime';
 export type { Scorer } from './scorers';
 export {
   contains,

package/src/llm-stream.ts

@@ -0,0 +1,171 @@
+/**
+ * The streaming half of `llm()`. Same action, same policy, same budget — one transport swapped.
+ *
+ * `.stream()` re-enters the action through its ordinary call path, marking the invocation with an
+ * ambient sink; `llm.ts` sees the sink and drives `gateway.stream` instead of `gateway.generate`.
+ * That is why the policy, the input parse, the audit record, the rate limit, the semantic cache,
+ * the span and the manifest row all still apply: there is no second execution path to keep in
+ * step, only a second way for the answer to arrive.
+ *
+ * Two questions a streamed model call forces, both answered here:
+ *
+ *  1. OUTPUT SCHEMA. A schema cannot be checked until the last token has landed, so a stream
+ *     yields UNVALIDATED text and one final `done` carrying the value that DID satisfy `output`.
+ *     There is no repair turn: the consumer has already read the tokens, and replaying a second
+ *     answer over the top is two answers to one question. One attempt, then
+ *     `X_LLM_STREAM_INVALID`, whose fix is the non-streaming call that can repair.
+ *  2. BUDGET. Unchanged, and still reserved before the provider is touched: `Gateway.stream`
+ *     debits the worst-case estimate on the first pull and reconciles it against real usage at
+ *     `done`, releasing it in a `finally` when a stream throws or is abandoned. The whole stream
+ *     is driven INSIDE the action's handler, so the reservation and its reconciliation sit on one
+ *     async chain — abandoning the iterator stops delivery, never the accounting.
+ */
+
+import { AsyncLocalStorage } from 'node:async_hooks';
+import { AiTransportError } from './errors';
+import type { Gateway } from './gateway';
+import type { GenerateRequest, GenerateResult } from './provider';
+
+/**
+ * One increment of a streamed answer. `thinking` is a separate kind on purpose — the wire layer
+ * refuses to fold reasoning into `text`, and a consumer concatenating every chunk must not end up
+ * shipping it to the user.
+ */
+export type LlmStreamChunk<T> =
+  | { readonly type: 'text'; readonly text: string }
+  | { readonly type: 'thinking'; readonly text: string }
+  | { readonly type: 'done'; readonly value: T };
+
+type Resolver = () => void;
+
+/**
+ * The buffer between the handler pushing chunks and the caller pulling them. Unbounded: an LLM
+ * answer is bounded by `maxTokens` and a slow reader must never stall a call whose budget
+ * reservation is already open.
+ */
+export class LlmSink {
+  private readonly queue: LlmStreamChunk<unknown>[] = [];
+  private wake: Resolver | undefined;
+  private ended = false;
+  private failure: unknown;
+  private failed = false;
+  private abandoned = false;
+
+  emit(chunk: LlmStreamChunk<unknown>): void {
+    // A consumer that stopped reading gets nothing more, but the call it authorised runs to
+    // completion — half a reconciliation holds a budget ceiling for the rest of the window.
+    if (this.abandoned) return;
+    this.queue.push(chunk);
+    this.release();
+  }
+
+  finish(value: unknown): void {
+    this.emit({ type: 'done', value });
+    this.ended = true;
+    this.release();
+  }
+
+  fail(error: unknown): void {
+    this.failure = error;
+    this.failed = true;
+    this.ended = true;
+    this.release();
+  }
+
+  close(): void {
+    this.abandoned = true;
+    this.queue.length = 0;
+    this.release();
+  }
+
+  async *drain(): AsyncGenerator<LlmStreamChunk<unknown>> {
+    while (true) {
+      const next = this.queue.shift();
+      if (next !== undefined) {
+        yield next;
+        continue;
+      }
+      // The original throwable, rethrown rather than wrapped: it is already an `UltimateError`
+      // with the code, the cause and the fix the caller needs, and re-boxing it loses all three.
+      if (this.failed) throw this.failure;
+      if (this.ended || this.abandoned) return;
+      await new Promise<void>((resolve) => {
+        this.wake = resolve;
+      });
+    }
+  }
+
+  private release(): void {
+    const waiter = this.wake;
+    this.wake = undefined;
+    waiter?.();
+  }
+}
+
+const sinks = new AsyncLocalStorage<LlmSink>();
+
+/** Mark everything `fn` awaits as a streamed invocation. */
+export function withLlmSink<T>(sink: LlmSink, fn: () => Promise<T>): Promise<T> {
+  return sinks.run(sink, fn);
+}
+
+/** The sink of the streamed invocation this call belongs to, or `undefined` for a plain one. */
+export function currentLlmSink(): LlmSink | undefined {
+  return sinks.getStore();
+}
+
+/**
+ * Turn one invocation into an iterable of chunks. LAZY, like `Gateway.stream`: nothing is
+ * authorised, budgeted or sent until the first pull, so a `.stream()` handed to a consumer that
+ * never reads it spends nothing and denies nobody.
+ */
+export function llmStream<T>(run: (sink: LlmSink) => Promise<T>): AsyncIterable<LlmStreamChunk<T>> {
+  return {
+    async *[Symbol.asyncIterator](): AsyncGenerator<LlmStreamChunk<T>> {
+      const sink = new LlmSink();
+      // Never rejects — every outcome lands on the sink, which is where the consumer looks.
+      void run(sink).then(
+        (value) => {
+          sink.finish(value);
+        },
+        (error: unknown) => {
+          sink.fail(error);
+        },
+      );
+      try {
+        for await (const chunk of sink.drain()) yield chunk as LlmStreamChunk<T>;
+      } finally {
+        sink.close();
+      }
+    },
+  };
+}
+
+/**
+ * Drive one streamed turn: forward every increment to the consumer, keep the assembled result.
+ *
+ * A `tool-call` chunk cannot arrive here — the streaming path offers no tools, precisely so there
+ * ARE text deltas to stream — and is dropped rather than thrown on, because a provider that sends
+ * one has not broken the answer the `done` chunk carries.
+ */
+export async function streamOneTurn(
+  gateway: Gateway,
+  request: GenerateRequest,
+  sink: LlmSink,
+): Promise<GenerateResult> {
+  let assembled: GenerateResult | undefined;
+  for await (const chunk of gateway.stream(request)) {
+    if (chunk.type === 'text') sink.emit({ type: 'text', text: chunk.text });
+    else if (chunk.type === 'thinking') sink.emit({ type: 'thinking', text: chunk.text });
+    else if (chunk.type === 'done') assembled = chunk.result;
+  }
+  if (assembled === undefined) {
+    // Not "an empty answer": the transport ended without the frame that carries usage and stop
+    // reason, so nothing downstream could tell a complete answer from a cut one.
+    throw new AiTransportError({
+      provider: 'gateway',
+      detail: 'the stream ended without a done chunk, so the answer and its usage are unknown',
+    });
+  }
+  return assembled;
+}