@voltro/ai 0.27.0 → 0.29.0

package/dist/index.d.ts

@@ -16,6 +16,7 @@ import { QueryDescriptor } from '@voltro/database';
 import { QueryProcedureDescriptor } from '@voltro/protocol';
 import { Runtime } from 'effect';
 import { Schema } from 'effect';
+import { SemanticCacheShape } from '@voltro/cache';
 import { simulateStreamingMiddleware } from 'ai';
 import { SpeechModelV4 } from '@ai-sdk/provider';
 import { Stream } from 'effect';
@@ -468,6 +469,11 @@ export declare interface AppToolPolicy {
  */
 export declare const appTools: (entries: ReadonlyArray<AppToolEntry>, policy?: AppToolPolicy) => ReadonlyArray<AnyTool>;
 
+export declare interface AssertionFailure {
+    readonly assertion: EvalAssertion;
+    readonly detail: string;
+}
+
 /** Render the capability map into a system prompt that enumerates EXACTLY the
  *  tables + columns the model may use — the soft in-grammar steer that pairs
  *  with the hard validator. */
@@ -651,6 +657,9 @@ declare const DEFAULT_INPUT: Schema.Struct<{
     prompt: typeof Schema.String;
 }>;
 
+/** The default judge threshold when a judge is configured without one. */
+export declare const DEFAULT_JUDGE_THRESHOLD = 0.7;
+
 /** Default input schema type when `input` is omitted (`{ prompt: string }`). */
 export declare type DefaultAgentInputSchema = typeof DEFAULT_INPUT;
 
@@ -701,6 +710,25 @@ export declare const defineAgent: <S extends Schema.Schema.Any = DefaultAgentInp
  */
 export declare const defineAgentExecutor: <S extends Schema.Schema.Any = DefaultAgentInputSchema>(descriptor: AgentDescriptor<S>, config: AgentExecutorConfig<AgentInput<S>>) => AgentExecutor<S>;
 
+/**
+ * Declare an eval suite. Validated at module load — an empty case list, a
+ * duplicate case id, or a nonsensical threshold throws loudly so a broken eval
+ * fails discovery, never at replay time. Call it as the default export of a
+ * `*.eval.ts` file; `voltro eval` discovers + runs it.
+ */
+export declare const defineEval: (input: DefineEvalInput) => EvalDefinition;
+
+export declare interface DefineEvalInput {
+    readonly name: string;
+    readonly cases: ReadonlyArray<EvalCase>;
+    /** Assertions applied to EVERY case. */
+    readonly assert?: ReadonlyArray<EvalAssertion>;
+    readonly judge?: EvalJudgeConfig;
+    /** Tool names the replay may call (resolved by the runner against the app's
+     *  registered `*.tool.tsx`). Empty = a bare generate. */
+    readonly tools?: ReadonlyArray<string>;
+}
+
 export declare const defineTool: <A, I, O>(def: {
     readonly name: string;
     readonly description: string;
@@ -760,6 +788,113 @@ export declare const estimateCostUsd: (usage: AiUsage, opts: {
     readonly actualCostUsd?: number;
 }) => CostBreakdown;
 
+/**
+ * A HARD assertion over the replayed output — a pure predicate, no model in the
+ * loop. These are the deterministic half of the gate: an LLM judge decides "is
+ * this answer good", an assertion decides "does it literally contain the order id
+ * / never leak the system prompt / come back non-empty within the latency bound".
+ */
+export declare type EvalAssertion = {
+    readonly kind: 'contains';
+    readonly value: string;
+    readonly caseInsensitive?: boolean;
+} | {
+    readonly kind: 'notContains';
+    readonly value: string;
+    readonly caseInsensitive?: boolean;
+} | {
+    readonly kind: 'matches';
+    readonly pattern: string;
+    readonly flags?: string;
+} | {
+    readonly kind: 'equals';
+    readonly value: string;
+    readonly trim?: boolean;
+} | {
+    readonly kind: 'maxLatencyMs';
+    readonly value: number;
+} | {
+    readonly kind: 'nonEmpty';
+};
+
+export declare interface EvalCase {
+    /** Stable id — names the case in the report + the branch. */
+    readonly id: string;
+    readonly input: EvalInput;
+    /** The output the ORIGINAL recorded run produced — the baseline the judge
+     *  compares the replay against. Optional: a case can gate on assertions alone. */
+    readonly golden?: string;
+    /** Assertions for THIS case (run in addition to the eval-wide `assert`). */
+    readonly assert?: ReadonlyArray<EvalAssertion>;
+}
+
+export declare interface EvalCaseResult {
+    readonly id: string;
+    readonly pass: boolean;
+    readonly assertionFailures: ReadonlyArray<AssertionFailure>;
+    /** Present when the eval declared a judge. */
+    readonly judge?: JudgeVerdict;
+    readonly output: EvalReplayOutput;
+}
+
+export declare interface EvalDefinition extends DefineEvalInput {
+    readonly __eval: true;
+}
+
+/** The captured inputs of a recorded run — what the replay re-sends. `messages`
+ *  carries the full prior conversation for a multi-turn recorded run; `prompt` is
+ *  the single user turn. */
+export declare interface EvalInput {
+    readonly prompt: string;
+    readonly system?: string;
+    readonly messages?: ReadonlyArray<ModelMessage>;
+}
+
+/** Grade one replay. Injected so the pure runner never needs a provider. */
+export declare type EvalJudge = (args: {
+    readonly case: EvalCase;
+    readonly output: EvalReplayOutput;
+    readonly config: EvalJudgeConfig;
+}) => Promise<JudgeVerdict>;
+
+/** LLM-judge configuration. The judge scores the replay against the rubric (and,
+ *  when present, the golden baseline); a score below `threshold` fails the case. */
+export declare interface EvalJudgeConfig {
+    /** What "good" means for these cases — the grading rubric handed to the judge. */
+    readonly rubric: string;
+    /** Minimum score (0..1) to pass. Default 0.7. */
+    readonly threshold?: number;
+}
+
+/** Re-run a single case. Injected by the caller — the CLI wires the app's
+ *  provider (against a branched store); a test passes a deterministic fake. */
+export declare type EvalReplay = (args: {
+    readonly case: EvalCase;
+    readonly def: EvalDefinition;
+}) => Promise<EvalReplayOutput>;
+
+/** What one replay produced. `latencyMs` feeds the `maxLatencyMs` assertion. */
+export declare interface EvalReplayOutput {
+    readonly text: string;
+    readonly latencyMs: number;
+}
+
+export declare interface EvalReport {
+    readonly name: string;
+    readonly ok: boolean;
+    readonly passed: number;
+    readonly failed: number;
+    readonly cases: ReadonlyArray<EvalCaseResult>;
+    /** The isolated data branch this suite's replays target — reused branching
+     *  machinery (`branchNamespaceName`). Set by the CLI when `--branch` is used. */
+    readonly branch?: {
+        readonly id: string;
+    };
+}
+
+/** Run every assertion against one output; return only the failures. Pure. */
+export declare const evaluateAssertions: (out: EvalReplayOutput, assertions: ReadonlyArray<EvalAssertion>) => ReadonlyArray<AssertionFailure>;
+
 declare const EXECUTOR_BRAND: "@voltro/ai/AgentExecutor";
 
 /** The `exposeAsTool` annotation shape (mirrors @voltro/protocol's, structural
@@ -1116,12 +1251,30 @@ export declare const isAgentDescriptor: (x: unknown) => x is AnyAgentDescriptor;
 /** Discovery predicate for `*.agent.server.tsx` default exports. */
 export declare const isAgentExecutor: (x: unknown) => x is AnyAgentExecutor;
 
+export declare const isEvalDefinition: (value: unknown) => value is EvalDefinition;
+
 /** Whether a stream with this id is currently registered (in flight). */
 export declare const isStreamActive: (streamId: string) => boolean;
 
 /** Is `x` a `defineTool(...)` result? */
 export declare const isTool: (x: unknown) => x is AnyTool;
 
+export declare interface JudgeVerdict {
+    readonly pass: boolean;
+    /** 0..1. Below `judge.threshold` fails the case even when `pass` is true. */
+    readonly score: number;
+    readonly reason: string;
+}
+
+/**
+ * The real LLM judge: asks the model for a structured `{ pass, score, reason }`
+ * verdict via `generateObject` (schema-constrained, so a malformed grade surfaces
+ * as a typed error rather than a parse crash). A judge failure is turned into a
+ * failing verdict rather than throwing, so one flaky grade fails ONE case instead
+ * of aborting the whole eval — the report stays a complete deploy-gate signal.
+ */
+export declare const llmJudge: EvalJudge;
+
 /** The synthesized `<name>.messages` body — load the thread's messages. */
 export declare const loadAgentMessages: (store: DataStore, threadId: string) => Effect.Effect<ReadonlyArray<AgentMessage>>;
 
@@ -1410,12 +1563,47 @@ export declare const providerFromEnv: (env?: Record<string, string | undefined>)
 
 export declare type ProviderName = 'mock' | 'anthropic' | 'openai' | 'gateway';
 
+/** The minimal store surface {@link recordReads} needs — any object with a
+ *  `query(descriptor)` that returns rows. Structurally a subset of the
+ *  framework `DataStore`, so no `@voltro/runtime` import is required. */
+export declare interface QueryableStore {
+    query(descriptor: {
+        readonly table: string;
+    } & Record<string, unknown>): Promise<ReadonlyArray<Record<string, unknown>>>;
+}
+
+/** A store wrapper that records the dependency set of everything read through
+ *  it — the provenance the semantic cache tags entries with. */
+export declare interface ReadRecorder<S extends QueryableStore> {
+    /** Pass THIS where the real store would go; every `query()` it serves records
+     *  the table + each returned row's id. All other methods pass through. */
+    readonly store: S;
+    /** The accumulated dependency tags (deduped): `table:<t>` for each table
+     *  queried + `row:<t>:<id>` for each row returned that carries an `id`. Feed
+     *  this to a call's `deps`. */
+    deps(): ReadonlyArray<string>;
+    /** Drop everything recorded so far (reuse the recorder for a new answer). */
+    reset(): void;
+}
+
 /**
  * Price a usage tally and write a ledger row. Returns the computed
  * `CostBreakdown` so the caller can surface the cost without a re-query.
  */
 export declare const recordAiUsage: (store: DataStore, record: AiUsageRecordInput) => Effect.Effect<CostBreakdown>;
 
+/**
+ * Wrap a store so the rows it serves become a dependency set — automatic
+ * provenance capture with no runtime edit. Every `query()` records
+ * `tableDep(table)` and, for each returned row with an `id`, `rowDep(table,
+ * id)`. Read your data through `rec.store`, build the prompt from the rows,
+ * then pass `rec.deps()` as the call's dependency set.
+ *
+ * The wrapper is a `Proxy`, so the returned `store` has the SAME type + full
+ * method surface as the input — only `query` is intercepted.
+ */
+export declare const recordReads: <S extends QueryableStore>(store: S) => ReadRecorder<S>;
+
 /**
  * Redis-backed resumable-stream store — the production "resumable via
  * Redis-Replay" path. The append-only log is a Redis LIST per `streamId`
@@ -1690,6 +1878,102 @@ export declare interface RunAssistantInput {
  */
 export declare const runDataCopilot: (question: string, schema: CopilotSchema, deps: DataCopilotDeps) => Promise<CopilotValidation>;
 
+/**
+ * Replay + score every case in an eval. The orchestrator is pure over its
+ * injected `replay` / `judge`, so it runs identically under a real provider and
+ * under a deterministic test fake. `ok` is the deploy-gate signal: false iff any
+ * case failed.
+ */
+export declare const runEval: (def: EvalDefinition, deps: RunEvalDeps) => Promise<EvalReport>;
+
+export declare interface RunEvalDeps {
+    readonly replay: EvalReplay;
+    /** Omitted → assertions-only gating (no LLM in the loop). */
+    readonly judge?: EvalJudge;
+    readonly branchId?: string;
+}
+
+/**
+ * Combine assertion failures + the judge verdict into one pass/fail. Pure. A case
+ * passes iff no assertion failed AND (no judge, or the judge's score clears the
+ * threshold). Assertions are deterministic and always decisive; the judge only
+ * ever ADDS a reason to fail, never rescues an assertion failure.
+ */
+export declare const scoreCase: (c: EvalCase, out: EvalReplayOutput, evalWideAssertions: ReadonlyArray<EvalAssertion>, judgeConfig: EvalJudgeConfig | undefined, judgeVerdict: JudgeVerdict | undefined) => EvalCaseResult;
+
+/** Injectable collaborators for a semantic-cached call. */
+export declare interface SemanticCacheDeps {
+    /** The semantic cache (build with `makeSemanticCache(store)` from
+     *  `@voltro/cache` over a memory / RESP `CacheStore`). */
+    readonly cache: SemanticCacheShape;
+    /** Embed a prompt to a vector. Defaults to `@voltro/ai`'s `embed` (the
+     *  env-configured provider). Injectable for tests + provider choice. */
+    readonly embed?: (text: string) => Effect.Effect<ReadonlyArray<number>, AiError>;
+}
+
+/** Per-call cache policy. */
+export declare interface SemanticCachePolicy {
+    /** Minimum cosine similarity ([-1, 1]) for a hit. Default 0.95 — high, so
+     *  only genuine near-duplicates share an answer. */
+    readonly threshold?: number;
+    /** TTL floor in ms — the answer expires after this even if nothing it
+     *  depends on changes. Omit for "live until a dependency changes". */
+    readonly ttlMs?: number;
+    /** The dependency set: the source rows/tables this answer read, as tags
+     *  (`rowDep` / `tableDep`, or {@link recordReads}). A change to any of them
+     *  evicts the entry. Empty ⇒ cached but TTL-only (never dependency-evicted)
+     *  — correct only for an answer that reads no data. */
+    readonly deps: ReadonlyArray<string>;
+}
+
+/**
+ * `generateObject`, semantically cached + dependency-evicted. Same contract as
+ * {@link semanticGenerateText} for the structured surface.
+ *
+ * Note on serialization: the DECODED object is stored. With the memory backend
+ * it round-trips by reference; with a RESP backend it is JSON — so a schema
+ * whose decoded form is not JSON-safe (class instances, non-plain branded
+ * carriers) will not survive a cross-process hit. For those, cache the text
+ * form or key on a JSON-safe projection.
+ */
+export declare const semanticGenerateObject: <A, I>(options: GenerateObjectOptions<A, I>, policy: SemanticCachePolicy, deps: SemanticCacheDeps) => Effect.Effect<SemanticResult<A>, AiError>;
+
+/**
+ * `generateText`, semantically cached + dependency-evicted.
+ *
+ * On a near-duplicate prompt (embedding within `threshold`) returns the cached
+ * text with `cached: true` and zero usage. On a miss it generates, records the
+ * answer under its dependency set, and returns `cached: false`.
+ *
+ * ```ts
+ * const rec = recordReads(ctx.store)
+ * const docs = yield* Effect.promise(() => rec.store.query(docsQuery))
+ * const res = yield* semanticGenerateText(
+ *   { prompt: buildPrompt(docs) },
+ *   { deps: rec.deps() },
+ *   { cache },
+ * )
+ * // res.cached === true on the next near-identical prompt — until a doc changes.
+ * ```
+ */
+export declare const semanticGenerateText: (options: GenerateTextOptions, policy: SemanticCachePolicy, deps: SemanticCacheDeps) => Effect.Effect<SemanticResult<string>, AiError>;
+
+/** The outcome of a semantic-cached generation. */
+export declare interface SemanticResult<A> {
+    readonly value: A;
+    /** `true` when served from cache (zero tokens spent). */
+    readonly cached: boolean;
+    /** Underlying generation usage — all-zero when `cached`. */
+    readonly usage: SemanticUsage;
+}
+
+/** Usage tally shape shared by both generate results; all-zero on a cache hit. */
+export declare interface SemanticUsage {
+    readonly inputTokens: number | undefined;
+    readonly outputTokens: number | undefined;
+    readonly totalTokens: number | undefined;
+}
+
 /**
  * One persisted event of a RESUMABLE stream: an `AgentEvent` plus its monotonic
  * 1-based position. `seq` is the resume cursor — a client that has rendered up