@ultimat3/ai 1.2.0 → 3.0.0

package/src/provider.ts

@@ -4,16 +4,63 @@
 // catalogue and the per-model rules, ./wire owns the response half.
 
 import type { Money } from '@ultimat3/money';
+import { detailOf, withoutKey } from './error-body';
 import { AiKeyMissingError, AiTransportError } from './errors';
 import type { Effort, ModelId, ThinkingMode } from './models';
-import { DEFAULT_MODEL, MODEL_IDS, MODELS, reasoningBody } from './models';
+import { ANTHROPIC_MODEL_IDS, DEFAULT_MODEL, modelIds, modelSpec, reasoningBody } from './models';
 import { readSse } from './sse';
 import type { LlmTool, LlmToolCall } from './tools';
-import { MessageStream, parseStopDetails, parseStopReason, parseUsage } from './wire';
+import {
+  asToolInput,
+  MessageStream,
+  parseStopDetails,
+  parseStopReason,
+  parseUsage,
+  throwInBandError,
+} from './wire';
+
+/**
+ * One block of a structured message. A plain string message is still the common case and still
+ * legal; blocks exist because a tool loop cannot be expressed without them — a `tool_result` has
+ * to name the `tool_use` it answers, and a string has nowhere to put the id.
+ *
+ * The field names are the Messages API's, not a translated set, so `body()` passes a block
+ * through untouched. A second vocabulary here would be a mapping table to keep in step with a
+ * wire format we do not own.
+ */
+export type AiContentBlock =
+  | { readonly type: 'text'; readonly text: string }
+  | {
+      readonly type: 'tool_use';
+      readonly id: string;
+      readonly name: string;
+      readonly input: Record<string, unknown>;
+    }
+  | {
+      readonly type: 'tool_result';
+      readonly tool_use_id: string;
+      readonly content: string;
+      readonly is_error?: boolean;
+    };
 
 export interface AiMessage {
   readonly role: 'user' | 'assistant';
-  readonly content: string;
+  readonly content: string | readonly AiContentBlock[];
+}
+
+/**
+ * The readable text of a message, blocks flattened. For ESTIMATING and for the echo provider —
+ * never for building a request, which sends `content` as it stands.
+ */
+export function messageText(message: AiMessage): string {
+  if (typeof message.content === 'string') return message.content;
+  return message.content
+    .map((block) => {
+      if (block.type === 'text') return block.text;
+      if (block.type === 'tool_result') return block.content;
+      return JSON.stringify(block.input);
+    })
+    .join(' ');
 }
 
 export interface GenerateRequest {
@@ -26,6 +73,15 @@ export interface GenerateRequest {
   readonly thinking?: ThinkingMode;
   readonly tools?: readonly LlmTool[];
   readonly stopSequences?: readonly string[];
+  /**
+   * The caller's abort signal, forwarded to the socket by every provider in this package.
+   *
+   * Deliberately absent from `cacheKeyFor` and from every estimate: it says whether a request was
+   * ABANDONED, never what it asked for, so two calls that differ only in it are the same call and
+   * must share a cache entry. Omitted means the call runs to completion — there is no ambient
+   * default, because a timeout the caller did not ask for is a truncated answer nothing reports.
+   */
+  readonly signal?: AbortSignal;
 }
 
 export interface TokenUsage {
@@ -61,6 +117,14 @@ export interface StopDetails {
 
 export interface GenerateResult {
   readonly model: ModelId;
+  /**
+   * Which provider actually answered. Stamped by the GATEWAY, not by the provider: falling back
+   * from one provider to the next is a gateway concept, and a `Provider` implementation an app
+   * wrote cannot be asked to report on a decision it did not make. Optional for exactly that
+   * reason — a provider's own result has not been through the router yet. `llm()` puts it on the
+   * span as `llm.provider`, which is what makes a fallback visible rather than silent.
+   */
+  readonly provider?: string;
   readonly text: string;
   readonly toolCalls: readonly LlmToolCall[];
   readonly stopReason: StopReason;
@@ -91,7 +155,7 @@ export interface Provider {
  * away is money the framework silently absorbs, and under-reporting spend defeats a budget.
  */
 export function costOf(model: ModelId, usage: TokenUsage): Money {
-  const spec = MODELS[model];
+  const spec = modelSpec(model);
   // Cache reads are ~0.1x input; cache writes ~1.25x. Scaled by 10 to stay integral.
   const inputUnits =
     usage.inputTokens * 10 + usage.cacheReadTokens * 1 + Math.ceil(usage.cacheWriteTokens * 12.5);
@@ -121,8 +185,6 @@ export interface AnthropicProviderInput {
 
 const ANTHROPIC_VERSION = '2023-06-01';
 const API_KEY_ENV = 'ANTHROPIC_API_KEY';
-/** Enough of an error body to name the field that was wrong, not enough to fill a log. */
-const DETAIL_LIMIT = 300;
 
 /**
  * Above this ceiling a non-streaming request sits on an open socket past the HTTP timeout and
@@ -134,7 +196,7 @@ export const STREAM_ONLY_MAX_TOKENS = 16_000;
 /** Whether this request has to go over the streaming transport to arrive at all. */
 export function requiresStreaming(request: GenerateRequest): boolean {
   const model = request.model ?? DEFAULT_MODEL;
-  return Math.min(request.maxTokens, MODELS[model].maxOutput) > STREAM_ONLY_MAX_TOKENS;
+  return Math.min(request.maxTokens, modelSpec(model).maxOutput) > STREAM_ONLY_MAX_TOKENS;
 }
 
 /**
@@ -146,7 +208,8 @@ export function requiresStreaming(request: GenerateRequest): boolean {
  */
 export class AnthropicProvider implements Provider {
   readonly name = 'anthropic';
-  readonly models = MODEL_IDS;
+  /** Its own list, never the registry's: an app's internal model must not be routed here. */
+  readonly models: readonly ModelId[] = ANTHROPIC_MODEL_IDS;
   private readonly config: AnthropicProviderInput;
 
   constructor(config: AnthropicProviderInput = {}) {
@@ -162,7 +225,7 @@ export class AnthropicProvider implements Provider {
    */
   async generate(request: GenerateRequest): Promise<GenerateResult> {
     if (requiresStreaming(request)) return this.assemble(request);
-    const response = await this.send({ ...this.body(request), stream: false });
+    const response = await this.send({ ...this.body(request), stream: false }, request.signal);
     const raw = (await response.json()) as Record<string, unknown>;
     return parseMessage(request.model ?? DEFAULT_MODEL, raw);
   }
@@ -185,7 +248,7 @@ export class AnthropicProvider implements Provider {
    */
   async *stream(request: GenerateRequest): AsyncIterable<StreamChunk> {
     const model = request.model ?? DEFAULT_MODEL;
-    const response = await this.send({ ...this.body(request), stream: true });
+    const response = await this.send({ ...this.body(request), stream: true }, request.signal);
     if (response.body === null) {
       throw new AiTransportError({
         provider: this.name,
@@ -217,7 +280,7 @@ export class AnthropicProvider implements Provider {
     const model = request.model ?? DEFAULT_MODEL;
     const body: Record<string, unknown> = {
       model,
-      max_tokens: Math.min(request.maxTokens, MODELS[model].maxOutput),
+      max_tokens: Math.min(request.maxTokens, modelSpec(model).maxOutput),
       messages: request.messages.map((m) => ({ role: m.role, content: m.content })),
       ...reasoningBody(model, request.effort, request.thinking),
     };
@@ -232,7 +295,10 @@ export class AnthropicProvider implements Provider {
    * carrying its status, because the gateway decides whether to retry from that status and a
    * body parsed as if it were a message would read as an empty, successful answer.
    */
-  private async send(body: Record<string, unknown>): Promise<Response> {
+  private async send(
+    body: Record<string, unknown>,
+    signal: AbortSignal | undefined,
+  ): Promise<Response> {
     const apiKey = this.config.apiKey ?? Bun.env[API_KEY_ENV];
     if (apiKey === undefined || apiKey === '') {
       throw new AiKeyMissingError({ provider: this.name, envVar: API_KEY_ENV });
@@ -248,38 +314,32 @@ export class AnthropicProvider implements Provider {
         accept: body['stream'] === true ? 'text/event-stream' : 'application/json',
       },
       body: JSON.stringify(body),
+      // Attached only when the caller has one: `exactOptionalPropertyTypes`, and an explicit
+      // `signal: undefined` is a different value to some fetch implementations.
+      ...(signal === undefined ? {} : { signal }),
     });
     if (!response.ok) {
       throw new AiTransportError({
         provider: this.name,
         status: response.status,
-        detail: await detailOf(response),
+        // The endpoint's own message, with the credential scrubbed out of it — the same rule the
+        // OpenAI-format provider follows, and for the same reason: a proxy echoing the request
+        // headers into its 4xx body is the one path by which `x-api-key` reaches an error.
+        detail: withoutKey(await detailOf(response), apiKey),
+        envVar: API_KEY_ENV,
       });
     }
     return response;
   }
 }
 
-/** The provider's own message, when it sent one — it names the offending field, we name the fix. */
-async function detailOf(response: Response): Promise<string> {
-  const body = await response.text().catch(() => '');
-  try {
-    const parsed: unknown = JSON.parse(body);
-    if (typeof parsed === 'object' && parsed !== null) {
-      const error = (parsed as Record<string, unknown>)['error'];
-      if (typeof error === 'object' && error !== null) {
-        const message = (error as Record<string, unknown>)['message'];
-        if (typeof message === 'string') return message.slice(0, DETAIL_LIMIT);
-      }
-    }
-  } catch {
-    // Not JSON — a proxy or a gateway timeout page. The raw text is still the best evidence.
-  }
-  return body === '' ? response.statusText : body.slice(0, DETAIL_LIMIT);
-}
-
 /** Map a Messages API response onto `GenerateResult`. Exported so tests can drive it. */
 export function parseMessage(model: ModelId, raw: Record<string, unknown>): GenerateResult {
+  // A 200 carrying an `error` object instead of an answer — how a gateway in front of a model
+  // reports a fault it noticed after the headers were sent. Read as a message it is an empty,
+  // successful answer, which is the one outcome nothing downstream can detect; the STREAMED half
+  // of this same provider has always refused it.
+  throwInBandError(raw);
   const content = Array.isArray(raw['content']) ? raw['content'] : [];
   let text = '';
   const toolCalls: LlmToolCall[] = [];
@@ -291,17 +351,24 @@ export function parseMessage(model: ModelId, raw: Record<string, unknown>): Gene
       toolCalls.push({
         id: b['id'],
         name: b['name'],
-        input: (b['input'] ?? {}) as Record<string, unknown>,
+        // Parsed, never cast: `input` is untrusted, and a string under `Record<string, unknown>`
+        // is a lie every later reader indexes into. The streamed half already parses it.
+        input: asToolInput(b['input']),
       });
     }
   }
   const usage = parseUsage(raw['usage']);
+  const stopDetails = parseStopDetails(raw['stop_details']);
   return {
     model,
     text,
     toolCalls,
-    stopReason: parseStopReason(raw['stop_reason']),
-    stopDetails: parseStopDetails(raw['stop_details']),
+    // A refusal detail is a refusal whatever the stop field says: `parseStopReason` answers
+    // `end_turn` for a spelling this build has never seen, and `llm()` branches on the REASON —
+    // `stopDetails` has no reader that can refuse — so the pair would arrive as a complete answer
+    // that happens to be empty. The OpenAI-format read has always forced it.
+    stopReason: stopDetails === undefined ? parseStopReason(raw['stop_reason']) : 'refusal',
+    stopDetails,
     usage,
     cost: costOf(model, usage),
   };
@@ -324,7 +391,14 @@ export interface EchoProviderInput {
  */
 export class EchoProvider implements Provider {
   readonly name = 'echo';
-  readonly models = MODEL_IDS;
+  /**
+   * A getter over the whole registry, not a snapshot: the test double has to serve whatever the
+   * test registered, and a field read at construction time would miss a model registered after.
+   */
+  get models(): readonly ModelId[] {
+    return modelIds();
+  }
+
   private readonly config: EchoProviderInput;
 
   constructor(config: EchoProviderInput = {}) {
@@ -365,7 +439,7 @@ export class EchoProvider implements Provider {
 function lastUserMessage(messages: readonly AiMessage[]): string {
   for (let i = messages.length - 1; i >= 0; i -= 1) {
     const message = messages[i];
-    if (message !== undefined && message.role === 'user') return message.content;
+    if (message !== undefined && message.role === 'user') return messageText(message);
   }
   return '';
 }
@@ -381,7 +455,7 @@ export function estimateTokens(request: GenerateRequest): number {
 
 /** The prompt half alone — what the provider bills at the input rate. */
 export function estimateInputTokens(request: GenerateRequest): number {
-  const body = request.messages.map((m) => m.content).join(' ');
+  const body = request.messages.map(messageText).join(' ');
   return estimateTextTokens(body) + estimateTextTokens(request.system ?? '');
 }
 
@@ -394,7 +468,7 @@ export function estimateCost(request: GenerateRequest): Money {
   const model = request.model ?? DEFAULT_MODEL;
   return costOf(model, {
     inputTokens: estimateInputTokens(request),
-    outputTokens: Math.min(request.maxTokens, MODELS[model].maxOutput),
+    outputTokens: Math.min(request.maxTokens, modelSpec(model).maxOutput),
     cacheReadTokens: 0,
     cacheWriteTokens: 0,
   });

package/src/rag.ts

@@ -156,6 +156,28 @@ export interface AssembledContext {
   readonly dropped: readonly string[];
 }
 
+const BLOCK_OPEN = '<document id=';
+const BLOCK_CLOSE = '</document>';
+
+/**
+ * One retrieved document, fenced and labelled with the id it came from. A bare separator was not
+ * a boundary: it is a string a document can simply contain, and the assembled text lands in the
+ * `user` message indistinguishable from the author's own instructions — while a tool RESULT
+ * carries provenance and this carried none. So the fence is neutralised inside the payload, the
+ * shape `cdata()` uses in `@ultimat3/seo`'s `xml.ts`: the marker is broken, never deleted, so the
+ * model still reads every word the document actually said.
+ *
+ * Influence only, and deliberately not sold as more: the actor is `ctx.actor` and tool dispatch is
+ * matched against `def.tools`, so retrieved text can persuade a model and can never authorise it.
+ */
+function documentBlock(id: string, text: string): string {
+  const label = id.replaceAll('"', "'").replaceAll('>', ')').replaceAll('<', '(');
+  const body = text
+    .replaceAll(BLOCK_CLOSE, '<\\/document>')
+    .replaceAll(BLOCK_OPEN, '<\\document id=');
+  return `${BLOCK_OPEN}"${label}">\n${body}\n${BLOCK_CLOSE}`;
+}
+
 /**
  * Fill a context window in rank order until the budget is reached. Skips oversized hits
  * rather than stopping, so one long chunk does not starve every shorter one behind it.
@@ -163,9 +185,10 @@ export interface AssembledContext {
 export function assembleContext(input: {
   readonly hits: readonly SearchHit[];
   readonly maxTokens: number;
+  /** Between BLOCKS, never inside one — the block fence is what separates documents. */
   readonly separator?: string;
 }): AssembledContext {
-  const separator = input.separator ?? '\n\n---\n\n';
+  const separator = input.separator ?? '\n\n';
   const separatorTokens = estimateChunkTokens(separator);
   const parts: string[] = [];
   const used: string[] = [];
@@ -173,12 +196,13 @@ export function assembleContext(input: {
   let tokens = 0;
 
   for (const hit of input.hits) {
-    const cost = estimateChunkTokens(hit.text) + (parts.length === 0 ? 0 : separatorTokens);
+    const block = documentBlock(hit.id, hit.text);
+    const cost = estimateChunkTokens(block) + (parts.length === 0 ? 0 : separatorTokens);
     if (tokens + cost > input.maxTokens) {
       dropped.push(hit.id);
       continue;
     }
-    parts.push(hit.text);
+    parts.push(block);
     used.push(hit.id);
     tokens += cost;
   }

package/src/redaction.ts

@@ -0,0 +1,22 @@
+// The one gate between `vars()` and the provider: a `Secret` never reaches a prompt.
+//
+// Not a leak check — `Secret` redacts by value, so the string would have arrived as `[redacted]`.
+// It is a CORRECTNESS check, and the same one `render()` already makes for an unfilled `{{slot}}`:
+// a prompt that reads fine and means something else is the failure nobody sees, and here it is
+// also a token bill for an answer about a placeholder.
+
+import { isSecret } from '@ultimat3/core';
+import { AiPromptSecretError } from './errors';
+
+/**
+ * Refuse a `Secret` among a prompt's variables, naming every key that carries one. Runs whether
+ * or not an app installed a redactor: the redactor is the app's policy, this is the framework's
+ * invariant, and an invariant that only holds when something optional is configured is not one.
+ *
+ * Structural (`isSecret` reads the shared brand), so two copies of `@ultimat3/core` in one tree
+ * still recognise each other's secrets.
+ */
+export function assertNoSecrets(ref: string, vars: Readonly<Record<string, unknown>>): void {
+  const keys = Object.keys(vars).filter((key) => isSecret(vars[key]));
+  if (keys.length > 0) throw new AiPromptSecretError({ ref, keys: keys.sort() });
+}

package/src/remote-embedder.ts

@@ -6,6 +6,7 @@
 // vendor: `baseUrl` selects the provider and nothing else changes. A second class per vendor
 // would be a second thing to learn for a difference that does not exist on the wire.
 
+import { readWithinLimit } from '@ultimat3/core';
 import type { Embedder } from './embeddings';
 import { normalize } from './embeddings';
 import { AiKeyMissingError, AiTransportError, EmbedderDimMismatchError } from './errors';
@@ -15,6 +16,14 @@ const DEFAULT_BASE_URL = 'https://api.voyageai.com/v1';
 /** Providers cap a batch around 128 inputs; 96 leaves headroom for long texts. */
 const DEFAULT_BATCH_SIZE = 96;
 const DETAIL_LIMIT = 300;
+/**
+ * A hosted endpoint is a third party and `baseUrl` is app config, so neither its latency nor its
+ * response size is ours to assume. 30s is longer than any healthy embedding batch and shorter than
+ * a job lease; 32 MiB is an order of magnitude past the worst legitimate batch (96 inputs x 3072
+ * dimensions of JSON floats is under 4 MiB), so nothing real hits it and nothing unreal is held.
+ */
+const DEFAULT_TIMEOUT_MS = 30_000;
+const DEFAULT_MAX_RESPONSE_BYTES = 32 * 1024 * 1024;
 
 export interface RemoteEmbedderInput {
   /** The provider's model id. Doubles as the embedder name, so a store records what wrote it. */
@@ -30,6 +39,10 @@ export interface RemoteEmbedderInput {
   readonly baseUrl?: string;
   /** Inputs per request. Larger calls are split; the provider's own cap is not the caller's. */
   readonly batchSize?: number;
+  /** Deadline for ONE batch request. Defaults to 30s; `AbortSignal.timeout` enforces it. */
+  readonly timeoutMs?: number;
+  /** Bytes this process will hold of one response. Defaults to 32 MiB. */
+  readonly maxResponseBytes?: number;
   /** Injectable so a test can assert the request body without a network. Defaults to `fetch`. */
   readonly fetch?: typeof fetch;
 }
@@ -65,19 +78,53 @@ export class RemoteEmbedder implements Embedder {
       throw new AiKeyMissingError({ provider: this.name, envVar: API_KEY_ENV });
     }
     const doFetch = this.config.fetch ?? fetch;
-    const response = await doFetch(`${this.config.baseUrl ?? DEFAULT_BASE_URL}/embeddings`, {
-      method: 'POST',
-      headers: { authorization: `Bearer ${apiKey}`, 'content-type': 'application/json' },
-      body: JSON.stringify({ model: this.name, input: texts }),
-    });
+    const timeoutMs = this.config.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    const url = `${this.config.baseUrl ?? DEFAULT_BASE_URL}/embeddings`;
+    let response: Response;
+    try {
+      response = await doFetch(url, {
+        method: 'POST',
+        headers: { authorization: `Bearer ${apiKey}`, 'content-type': 'application/json' },
+        body: JSON.stringify({ model: this.name, input: texts }),
+        // Without this the call has no deadline at all: the per-request budget produces a
+        // `ctx.signal` that never reaches here, so a provider that accepts the connection and
+        // never answers holds a worker for as long as the socket stays open.
+        signal: AbortSignal.timeout(timeoutMs),
+      });
+    } catch (error) {
+      throw new AiTransportError({
+        provider: this.name,
+        detail: `${error instanceof Error ? error.message : 'the request failed before a response'} — no answer within ${timeoutMs}ms (deadline, egress, DNS or TLS)`,
+        envVar: API_KEY_ENV,
+      });
+    }
     if (!response.ok) {
       throw new AiTransportError({
         provider: this.name,
         status: response.status,
         detail: (await response.text().catch(() => '')).slice(0, DETAIL_LIMIT),
+        envVar: API_KEY_ENV,
       });
     }
-    return this.decode((await response.json()) as unknown, texts.length);
+    // Read through core's counting reader rather than `response.json()`: a body is buffered whole
+    // before anything measures it otherwise, and a `content-length` a remote wrote is not a bound.
+    const maxBytes = this.config.maxResponseBytes ?? DEFAULT_MAX_RESPONSE_BYTES;
+    const read = await readWithinLimit(response.body, maxBytes);
+    if ('over' in read) {
+      throw new AiTransportError({
+        provider: this.name,
+        status: response.status,
+        detail: `response body is at least ${read.over} bytes, limit is ${maxBytes}`,
+        envVar: API_KEY_ENV,
+      });
+    }
+    let payload: unknown;
+    try {
+      payload = JSON.parse(new TextDecoder().decode(read.bytes));
+    } catch {
+      throw this.malformed('the response body is not valid JSON');
+    }
+    return this.decode(payload, texts.length);
   }
 
   private decode(payload: unknown, expected: number): readonly Float32Array[] {

package/src/runtime.ts

@@ -24,12 +24,35 @@ export interface AiRuntimeInput {
   readonly embedder?: Embedder;
   /** One cache per scope key. Defaults to the in-memory driver; pgvector in production. */
   readonly semanticCache?: (scope: string) => SemanticCache;
+  /**
+   * The last thing that runs over a prompt before it leaves the process. Declared here, in the
+   * one place the framework already owns, because `vars()` is the one declared place a model call
+   * loads data — the framework knows exactly where the row enters the prompt, and until now
+   * nothing sat between that and a third-party endpoint.
+   *
+   * WHAT to remove is the app's decision, not the framework's: a PII classifier is a model choice
+   * and ships in no framework (axiom 8). Ultimate ships the seam, the span attribute
+   * (`llm.redacted`) and the one rule it can enforce structurally — a `Secret` reaching `vars()`
+   * is `X_AI_PROMPT_SECRET`, whether or not a redactor is installed.
+   */
+  readonly redact?: Redactor;
 }
 
+/**
+ * Rewrites a prompt on its way to the provider. Sees the whole RENDERED text, system prompt
+ * included — template as well as values, because a redactor shown only the values cannot tell
+ * a name in a data slot from the same name in an instruction.
+ */
+export type Redactor = (text: string) => string;
+
+/** No app redactor installed. Named rather than inline so `llm.redacted` has one thing to mean. */
+const noRedaction: Redactor = (text) => text;
+
 interface AiRuntime {
   readonly gateway: Gateway;
   readonly embedder: Embedder;
   readonly semanticCache: (scope: string) => SemanticCache;
+  readonly redact: Redactor;
 }
 
 let runtime: AiRuntime | undefined;
@@ -40,6 +63,7 @@ export function configureAi(input: AiRuntimeInput): void {
     gateway: input.gateway,
     embedder: input.embedder ?? new HashEmbedder(),
     semanticCache: input.semanticCache ?? (() => createMemorySemanticCache()),
+    redact: input.redact ?? noRedaction,
   };
   // A new runtime means a new embedder and a new gateway; vectors from the old one are not
   // comparable to vectors from the new one, and a stale hit would answer the wrong question.
@@ -56,6 +80,11 @@ export function aiEmbedder(): Embedder {
   return runtime?.embedder ?? new HashEmbedder();
 }
 
+/** The installed redactor, or the identity. Never absent, so the call site has no branch. */
+export function aiRedactor(): Redactor {
+  return runtime?.redact ?? noRedaction;
+}
+
 /**
  * The cache for one scope. Scopes are separate CACHE INSTANCES, never a filter over a shared
  * one: cosine similarity has no notion of a tenant, so two tenants asking near-identical