@zerotal/ai 1.5.0

package/src/schema.ts

@@ -0,0 +1,292 @@
+/**
+ * `@zerotal/validator` schema → JSON Schema, for structured output and tool inputs.
+ *
+ * The supported JSON Schema subset is narrow — narrower than most people expect,
+ * and the API rejects anything outside it at *request* time, which is the worst
+ * possible place to find out. So this module makes one decision and holds it:
+ *
+ * > **Strip what the API cannot express, then re-check it client-side.**
+ *
+ * `min(2)`, `max(140)`, `regex(...)` and friends are dropped from the emitted
+ * schema and re-applied by {@link recheckAgainstSchema} once the model answers.
+ * We already own a validator; running it over the parsed object costs nothing and
+ * keeps the constraint honest instead of decorative. The alternative — refusing
+ * the schema at definition time — would make `rule.string().min(2)` unusable for
+ * structured output, which is a worse deal for a constraint we can enforce
+ * ourselves.
+ *
+ * The exception is anything with no client-side rescue: a **recursive** schema
+ * cannot be expressed at all, and a **file** field has no JSON representation.
+ * Those throw here, at definition time, where the stack trace points at the
+ * schema rather than at a 400 from the provider.
+ */
+import type { FieldRule, FieldRuleDefinition, Schema } from "@zerotal/validator";
+import { runValidation } from "@zerotal/validator";
+import { AiSchemaError } from "./errors.ts";
+import type { JsonSchema } from "./types.ts";
+
+/** Either shape callers have on hand: the builder map, or the raw definitions. */
+export type SchemaInput = Record<string, FieldRule | FieldRuleDefinition>;
+
+/**
+ * Rule names that map onto a supported JSON Schema keyword. Everything else is
+ * stripped and re-checked. Kept as a map rather than a switch so the supported
+ * set is one readable list.
+ */
+const FORMAT_RULES: Record<string, string> = {
+  email: "email",
+  url: "uri",
+  ip: "ipv4",
+};
+
+/** Unwrap a builder rule to its definition. */
+function defOf(value: FieldRule | FieldRuleDefinition): FieldRuleDefinition {
+  return "_def" in value ? value._def : value;
+}
+
+/** Normalise either input shape to raw definitions. */
+export function toSchema(input: SchemaInput): Schema {
+  const out: Schema = {};
+  for (const [key, value] of Object.entries(input)) out[key] = defOf(value);
+  return out;
+}
+
+/**
+ * Translate a validator schema into the JSON Schema the providers accept.
+ *
+ * Every object gets `additionalProperties: false` — required, not optional, and
+ * the single most common reason a hand-written schema is rejected.
+ *
+ * Optional and nullable fields are emitted the same way: listed in `required`,
+ * with the value type widened to `anyOf: [<type>, {type: "null"}]`. That is the
+ * form both Anthropic and OpenAI's strict mode accept, and
+ * {@link recheckAgainstSchema} drops a `null` that stood in for "absent" before
+ * re-validating, so the round trip preserves the original meaning.
+ *
+ * @throws {AiSchemaError} for a recursive schema or a `file` field.
+ *
+ * @example
+ * translateSchema({
+ *   title:  rule.string().min(3),          // min stripped, re-checked after
+ *   tags:   rule.array(rule.string()),
+ *   author: rule.string().optional(),      // → anyOf [string, null]
+ * });
+ */
+export function translateSchema(input: SchemaInput): JsonSchema {
+  const schema = toSchema(input);
+  return objectSchema(schema, new Set());
+}
+
+/** Build the `{type: "object", ...}` node for a map of field definitions. */
+function objectSchema(schema: Schema, path: Set<FieldRuleDefinition>): JsonSchema {
+  const properties: Record<string, JsonSchema> = {};
+  const required: string[] = [];
+
+  for (const [name, def] of Object.entries(schema)) {
+    properties[name] = fieldSchema(def, name, path);
+    // Every property is listed. Optionality lives in the null branch of anyOf,
+    // because a subset `required` is rejected by strict tool use.
+    required.push(name);
+  }
+
+  return {
+    type: "object",
+    properties,
+    required,
+    additionalProperties: false,
+  };
+}
+
+/** Translate one field, widening to `anyOf` when it may be absent or null. */
+function fieldSchema(
+  def: FieldRuleDefinition,
+  name: string,
+  path: Set<FieldRuleDefinition>,
+): JsonSchema {
+  if (path.has(def)) {
+    throw new AiSchemaError(
+      `The schema for '${name}' is recursive. Structured output does not support recursive ` +
+        `schemas — flatten the shape, or bound the depth by declaring each level explicitly.`,
+      { field: name },
+    );
+  }
+
+  const next = new Set(path).add(def);
+  const base = baseSchema(def, name, next);
+
+  const mayBeNull = def.nullable || !def.required || def.sometimes === true;
+  if (!mayBeNull) return base;
+
+  return { anyOf: [base, { type: "null" }] };
+}
+
+/** The un-widened schema for a field's declared type. */
+function baseSchema(
+  def: FieldRuleDefinition,
+  name: string,
+  path: Set<FieldRuleDefinition>,
+): JsonSchema {
+  switch (def.type) {
+    case "string":
+      return withStringKeywords({ type: "string" }, def);
+
+    case "number":
+      return { type: def.rules.some((r) => r.name === "integer") ? "integer" : "number" };
+
+    case "boolean":
+      return { type: "boolean" };
+
+    case "date":
+      // No native date type in JSON Schema; `date-time` is in the supported
+      // format list, and the model returns an ISO-8601 string.
+      return { type: "string", format: "date-time" };
+
+    case "array": {
+      if (!def.children) {
+        throw new AiSchemaError(
+          `The array field '${name}' has no item type. Declare one: rule.array(rule.string()).`,
+          { field: name },
+        );
+      }
+      return { type: "array", items: fieldSchema(def.children, `${name}[]`, path) };
+    }
+
+    case "object": {
+      if (!def.shape) {
+        throw new AiSchemaError(
+          `The object field '${name}' has no shape. Declare one: rule.object({ … }).`,
+          { field: name },
+        );
+      }
+      const properties: Record<string, JsonSchema> = {};
+      const required: string[] = [];
+      for (const [key, child] of Object.entries(def.shape)) {
+        properties[key] = fieldSchema(child, `${name}.${key}`, path);
+        required.push(key);
+      }
+      return { type: "object", properties, required, additionalProperties: false };
+    }
+
+    case "file":
+      throw new AiSchemaError(
+        `The field '${name}' is a file. A model cannot return a file through structured output — ` +
+          `ask for a filename or an identifier instead.`,
+        { field: name },
+      );
+
+    default:
+      throw new AiSchemaError(
+        `The field '${name}' has an unsupported type '${String(def.type)}'.`,
+        {
+          field: name,
+          type: def.type,
+        },
+      );
+  }
+}
+
+/** Apply the string rules that survive translation: `enum` and `format`. */
+function withStringKeywords(base: JsonSchema, def: FieldRuleDefinition): JsonSchema {
+  const out: JsonSchema = { ...base };
+
+  for (const rule of def.rules) {
+    if (rule.name === "in" && Array.isArray(rule.args[0])) {
+      // `enum` narrows harder than any format, so it wins outright.
+      return { enum: (rule.args[0] as string[]).slice() };
+    }
+    const format = FORMAT_RULES[rule.name];
+    if (format) out.format = format;
+  }
+
+  return out;
+}
+
+/**
+ * Which of a schema's constraints this translation could not express.
+ *
+ * Useful in a test or a boot-time audit: it names exactly what
+ * {@link recheckAgainstSchema} will be carrying, so a schema whose important
+ * constraint is invisible to the model is a thing you can notice rather than
+ * discover.
+ *
+ * @example
+ * strippedConstraints({ title: rule.string().min(3).max(80) });
+ * // → ["title: min", "title: max"]
+ */
+export function strippedConstraints(input: SchemaInput): string[] {
+  const out: string[] = [];
+  walk(toSchema(input), "", out, new Set());
+  return out;
+}
+
+function walk(schema: Schema, prefix: string, out: string[], path: Set<FieldRuleDefinition>): void {
+  for (const [name, def] of Object.entries(schema)) {
+    if (path.has(def)) continue;
+    const next = new Set(path).add(def);
+    const label = prefix ? `${prefix}.${name}` : name;
+
+    for (const rule of def.rules) {
+      if (rule.name === "in" || FORMAT_RULES[rule.name] || rule.name === "integer") continue;
+      out.push(`${label}: ${rule.name}`);
+    }
+
+    if (def.shape) walk(def.shape, label, out, next);
+    if (def.children) walk({ [`${name}[]`]: def.children }, prefix, out, next);
+  }
+}
+
+/**
+ * Re-apply the constraints translation had to drop.
+ *
+ * Called on the model's parsed answer. A `null` standing in for an absent
+ * optional field is removed first — the wire form widened it to null, and the
+ * validator would otherwise reject a value the caller never asked for.
+ *
+ * @returns The validated object.
+ * @throws {AiSchemaError} listing every field that failed.
+ *
+ * @internal
+ */
+export function recheckAgainstSchema<T>(input: SchemaInput, value: unknown): T {
+  const schema = toSchema(input);
+
+  if (typeof value !== "object" || value === null || Array.isArray(value)) {
+    throw new AiSchemaError(
+      `The model returned ${Array.isArray(value) ? "an array" : typeof value}, not an object.`,
+      { received: value },
+    );
+  }
+
+  const cleaned = dropAbsentNulls(schema, value as Record<string, unknown>);
+  const result = runValidation(schema, cleaned);
+
+  if (!result.success) {
+    const detail = Object.entries(result.errors)
+      .map(([field, message]) => `${field}: ${message}`)
+      .join("; ");
+    throw new AiSchemaError(
+      `The model's answer does not satisfy the schema — ${detail}. These constraints are enforced ` +
+        `here rather than by the provider, because structured output cannot express them.`,
+      { errors: result.errors },
+    );
+  }
+
+  return result.data as T;
+}
+
+/** Remove `null`s that stood in for "absent" on optional, non-nullable fields. */
+function dropAbsentNulls(schema: Schema, value: Record<string, unknown>): Record<string, unknown> {
+  const out: Record<string, unknown> = { ...value };
+
+  for (const [name, def] of Object.entries(schema)) {
+    if (out[name] === null && !def.nullable) {
+      delete out[name];
+      continue;
+    }
+    if (def.shape && out[name] && typeof out[name] === "object") {
+      out[name] = dropAbsentNulls(def.shape, out[name] as Record<string, unknown>);
+    }
+  }
+
+  return out;
+}

package/src/spend.ts

@@ -0,0 +1,100 @@
+/**
+ * Spend ceilings and the running spend ledger.
+ *
+ * Two guards, both cheap, both approximate on purpose:
+ *
+ * - **Per-request** is checked *before* the call, from a token estimate, because
+ *   after the call the money is already spent. It bounds the blast radius of one
+ *   runaway prompt.
+ * - **Per-day** is checked from actual reported usage, accumulated in-process.
+ *   In-process is a real limitation — N workers get N ceilings — and it is
+ *   stated rather than hidden, because the alternative (a shared counter behind
+ *   the cache) buys precision for an availability dependency on the hot path.
+ *
+ * A model with no price contributes 0 and is never blocked. See `pricing.ts`.
+ */
+import { AiSpendLimitError } from "./errors.ts";
+import { estimateCost, modelPrice } from "./pricing.ts";
+import type { AiLimitsConfigShape, AiUsage } from "./types.ts";
+
+/** Today's spend, keyed by UTC date so the window rolls without a timer. */
+let _day = "";
+let _spentUsd = 0;
+
+/** UTC `YYYY-MM-DD`. UTC, not local, so a deploy across zones agrees with itself. */
+function today(): string {
+  return new Date().toISOString().slice(0, 10);
+}
+
+function roll(): void {
+  const now = today();
+  if (now !== _day) {
+    _day = now;
+    _spentUsd = 0;
+  }
+}
+
+/** USD recorded so far today, in this process. */
+export function spentToday(): number {
+  roll();
+  return _spentUsd;
+}
+
+/** Add a completed request's cost to today's total. */
+export function recordSpend(model: string, usage: AiUsage): number {
+  roll();
+  const cost = estimateCost(model, usage);
+  _spentUsd += cost;
+  return cost;
+}
+
+/** Reset the ledger. Tests, and the `ai:spend --reset` path. */
+export function resetSpend(): void {
+  _day = today();
+  _spentUsd = 0;
+}
+
+/**
+ * Refuse a request that would breach a ceiling.
+ *
+ * @param model - The model about to be called.
+ * @param estimatedInputTokens - Counted, not guessed, where the driver can.
+ * @param maxOutputTokens - The request's own ceiling; the worst case we can bound.
+ *
+ * @throws {AiSpendLimitError} when either ceiling would be breached.
+ */
+export function assertWithinLimits(
+  limits: AiLimitsConfigShape,
+  model: string,
+  estimatedInputTokens: number,
+  maxOutputTokens: number,
+): void {
+  if (limits.perDayUsd > 0 && spentToday() >= limits.perDayUsd) {
+    throw new AiSpendLimitError(
+      `Daily AI spend ceiling reached: $${spentToday().toFixed(2)} of $${limits.perDayUsd.toFixed(2)}. ` +
+        `Raise limits.perDayUsd in config/ai.ts, or wait for the UTC day to roll.`,
+      { model, spentUsd: spentToday(), limitUsd: limits.perDayUsd },
+    );
+  }
+
+  if (limits.perRequestUsd <= 0) return;
+  // Unpriced model: the ceiling has nothing to compare against, so it stands
+  // aside rather than blocking every request to a model we simply don't know.
+  if (!modelPrice(model)) return;
+
+  const worstCase = estimateCost(model, {
+    inputTokens: estimatedInputTokens,
+    outputTokens: maxOutputTokens,
+    cacheReadTokens: 0,
+    cacheWriteTokens: 0,
+  });
+
+  if (worstCase > limits.perRequestUsd) {
+    throw new AiSpendLimitError(
+      `This request could cost up to $${worstCase.toFixed(4)}, over the per-request ceiling of ` +
+        `$${limits.perRequestUsd.toFixed(4)}. Shorten the prompt, lower maxTokens, or raise ` +
+        `limits.perRequestUsd in config/ai.ts.`,
+      { model, estimatedInputTokens, maxOutputTokens, worstCaseUsd: worstCase },
+    );
+  }
+}

package/src/stats.ts

@@ -0,0 +1,121 @@
+/**
+ * In-process counters behind the monitor's AI section.
+ *
+ * A ring buffer, not a table: the panel answers "what is this app doing with the
+ * model right now", and the durable record of that is the provider's own
+ * dashboard plus whatever the app writes down deliberately. Keeping prompts in
+ * a framework-owned store would make every app that installs this package a
+ * place user prompts accumulate, which is not a decision this package gets to
+ * make for them.
+ */
+
+/** One recorded generation. */
+export interface AiDelivery {
+  at: number;
+  driver: string;
+  model: string;
+  operation: string;
+  inputTokens: number;
+  outputTokens: number;
+  cacheReadTokens: number;
+  durationMs: number;
+  costUsd: number;
+  ok: boolean;
+  refused: boolean;
+  /** Already redacted by the manager — never the raw prompt. */
+  preview: string;
+  error?: string;
+}
+
+/** Rolled-up figures for one model. */
+export interface ModelStat {
+  model: string;
+  calls: number;
+  inputTokens: number;
+  outputTokens: number;
+  cacheReadTokens: number;
+  costUsd: number;
+  failures: number;
+  refusals: number;
+  /** Median latency, in milliseconds. */
+  p50: number;
+  /** 95th-percentile latency, in milliseconds. */
+  p95: number;
+}
+
+/** How many generations are kept. Roughly a working session's worth. */
+const CAPACITY = 200;
+
+const _deliveries: AiDelivery[] = [];
+
+/** Record one generation. Called from the observability bridge. @internal */
+export function recordDelivery(delivery: AiDelivery): void {
+  _deliveries.push(delivery);
+  if (_deliveries.length > CAPACITY) _deliveries.splice(0, _deliveries.length - CAPACITY);
+}
+
+/** The most recent generations, newest first. */
+export function recentGenerations(limit = 50): AiDelivery[] {
+  return _deliveries.slice(-limit).reverse();
+}
+
+/** Per-model roll-up over everything still in the buffer. */
+export function modelStats(): ModelStat[] {
+  const byModel = new Map<string, { rows: AiDelivery[]; latencies: number[] }>();
+
+  for (const delivery of _deliveries) {
+    const entry = byModel.get(delivery.model) ?? { rows: [], latencies: [] };
+    entry.rows.push(delivery);
+    if (delivery.ok) entry.latencies.push(delivery.durationMs);
+    byModel.set(delivery.model, entry);
+  }
+
+  const out: ModelStat[] = [];
+  for (const [model, { rows, latencies }] of byModel) {
+    latencies.sort((a, b) => a - b);
+    out.push({
+      model,
+      calls: rows.length,
+      inputTokens: sum(rows, (r) => r.inputTokens),
+      outputTokens: sum(rows, (r) => r.outputTokens),
+      cacheReadTokens: sum(rows, (r) => r.cacheReadTokens),
+      costUsd: sum(rows, (r) => r.costUsd),
+      failures: rows.filter((r) => !r.ok).length,
+      refusals: rows.filter((r) => r.refused).length,
+      p50: percentile(latencies, 0.5),
+      p95: percentile(latencies, 0.95),
+    });
+  }
+
+  return out.sort((a, b) => b.costUsd - a.costUsd || b.calls - a.calls);
+}
+
+/** Share of recorded calls that the provider declined, 0–1. */
+export function refusalRate(): number {
+  if (_deliveries.length === 0) return 0;
+  return _deliveries.filter((d) => d.refused).length / _deliveries.length;
+}
+
+/** Reset the buffer. Tests. */
+export function resetStats(): void {
+  _deliveries.length = 0;
+}
+
+function sum(rows: AiDelivery[], pick: (row: AiDelivery) => number): number {
+  let total = 0;
+  for (const row of rows) total += pick(row);
+  return total;
+}
+
+/**
+ * Nearest-rank percentile over a sorted list.
+ *
+ * Nearest-rank rather than interpolated: with a couple of dozen samples the
+ * interpolation invents a latency nothing actually took, and the point of a p95
+ * here is to name a request that really happened.
+ */
+function percentile(sorted: number[], fraction: number): number {
+  if (sorted.length === 0) return 0;
+  const rank = Math.ceil(fraction * sorted.length);
+  return Math.round(sorted[Math.min(rank, sorted.length) - 1] ?? 0);
+}

package/src/tool.ts

@@ -0,0 +1,98 @@
+import { RuleBuilder } from "@zerotal/validator";
+import type { FieldRule } from "@zerotal/validator";
+import { recheckAgainstSchema, translateSchema, type SchemaInput } from "./schema.ts";
+import type { AiTool, AiToolContext } from "./types.ts";
+
+/**
+ * Define a tool the model can call, with its input described by the validator
+ * you already use for forms.
+ *
+ * The schema is translated to JSON Schema for the provider and re-checked on the
+ * way back in, so the handler receives input that has actually been validated —
+ * not merely input the provider promised to shape. Constraints the provider
+ * cannot express (`min`, `regex`, …) are enforced by that second pass; see
+ * `schema.ts` for why that is the deal.
+ *
+ * @example
+ * const lookupOrder = tool({
+ *   name: "lookup_order",
+ *   description:
+ *     "Fetch one order by its id. Call this whenever the user refers to an order " +
+ *     "number — do not answer from the conversation alone.",
+ *   input: (rule) => ({ id: rule.string().uuid() }),
+ *   async handle({ id }) {
+ *     return await Order.find(id);
+ *   },
+ * });
+ *
+ * await Ai.agent({ prompt: "Where is order 2f1c…?", tools: [lookupOrder] });
+ */
+export function tool<I extends Record<string, unknown> = Record<string, unknown>>(options: {
+  /** Snake_case, specific: `lookup_order` beats `orders`. */
+  name: string;
+  /**
+   * What it does *and when to call it*. The trigger condition is the half that
+   * moves the call rate — a description that only states what the tool does
+   * leaves the model guessing about when it applies.
+   */
+  description: string;
+  /** The input shape, as a validator schema. */
+  input: ((rule: RuleBuilder) => Record<string, FieldRule>) | SchemaInput;
+  /** Runs when the model calls the tool. Return anything JSON-serializable. */
+  handle: (input: I, ctx: AiToolContext) => Promise<unknown> | unknown;
+}): AiTool {
+  const schema: SchemaInput =
+    typeof options.input === "function" ? options.input(new RuleBuilder()) : options.input;
+
+  return {
+    name: options.name,
+    description: options.description,
+    inputSchema: translateSchema(schema),
+    handler: (raw, ctx) => options.handle(recheckAgainstSchema<I>(schema, raw), ctx),
+  };
+}
+// The return is `AiTool`, not `AiTool<I>`, on purpose. `I` appears in a
+// parameter position, so `AiTool<{a, b}>` is *not* assignable to
+// `AiTool<Record<string, unknown>>` — and `tools: [add]` would be a type error
+// at the one call site every user writes. The narrowing still happens: `handle`
+// receives `I`, validated by `recheckAgainstSchema`, and only the stored
+// signature is widened.
+
+/**
+ * Run a tool's handler and turn whatever it returns — or throws — into the
+ * string the provider expects back.
+ *
+ * A throwing handler must not end the run: the model is perfectly capable of
+ * trying something else once it is told the call failed, and killing the turn
+ * denies it that. So the error becomes an error-flagged result instead.
+ *
+ * @internal
+ */
+export async function runTool(
+  t: AiTool,
+  input: Record<string, unknown>,
+  ctx: AiToolContext,
+): Promise<{ content: string; isError: boolean }> {
+  try {
+    const value = await t.handler(input, ctx);
+    return { content: stringifyToolResult(value), isError: false };
+  } catch (error) {
+    return {
+      content: error instanceof Error ? error.message : String(error),
+      isError: true,
+    };
+  }
+}
+
+/** Strings pass through; everything else is JSON. Undefined becomes a stated no-op. */
+function stringifyToolResult(value: unknown): string {
+  if (typeof value === "string") return value;
+  if (value === undefined) return "(no output)";
+  try {
+    return JSON.stringify(value) ?? String(value);
+  } catch {
+    // A cycle or a BigInt. Better a named failure the model can react to than a
+    // thrown error that ends the turn.
+    return "(tool output could not be serialized to JSON)";
+  }
+}