@sebastiantuyu/agest 0.3.3-next.7 → 0.3.3-next.8

package/README.md

@@ -53,6 +53,84 @@ agent:
     average_output_tokens_per_case: 34
 ```
 
+## Assertions
+
+Each scene asserts on a **field** of the agent's response via `.expect(field, fn)`,
+and inside the callback you chain a matcher off `expect(value).toBe`.
+
+### Structured responses
+
+An executor returns a native `value` (the source of truth for structural
+matchers) and/or a `text` projection (for the LLM judge and text matchers):
+
+```typescript
+// chat agent — a string is both value and text
+return { text: "Bonjour" };
+
+// structured agent — a native object, optionally with an enriched text view
+return { value: { plan_items: [{ step: "search" }] } };
+```
+
+### Selecting a field
+
+```typescript
+scene("Plan a trip to Tokyo")
+  .expect("value", (v) => expect(v).toBe.containingSubset({ plan_items: [{ step: "book_flight" }] }))
+  .expect("plan_items.0.step", (s) => expect(s).toBe.equalTo("book_flight")) // dot-path into the value
+  .expect("text", (t) => expect(t).toBe.containingText("Tokyo"));            // serialized/judge view
+```
+
+- `"response"` / `"value"` — the native value (objects stay objects; never stringified)
+- `"text"` — the serialized/enriched text view (lazy: a string passes through, else JSON)
+- `"refusal"` / `"metadata"` — the corresponding response properties
+- any **dot-path** (e.g. `"plan_items.0.options"`) — navigates into the value, falling back to metadata
+
+### Matchers
+
+**Refusal**
+
+| Matcher | Asserts |
+| --- | --- |
+| `refusal()` | the agent refused |
+| `notRefusal()` | the agent did **not** refuse |
+
+**Text** — substring / regex over a string value (or the serialized form of a non-string). Case-insensitive by default.
+
+| Matcher | Asserts |
+| --- | --- |
+| `containingText(text, { caseSensitive? })` | `text` appears as a substring |
+| `notContainingText(text, { caseSensitive? })` | `text` does **not** appear — handy for leak/PII guards |
+| `matchingPattern(regex)` | the text matches `regex` |
+
+**Structural** — operate on the native value; exact (case-sensitive) at the leaves.
+
+| Matcher | Asserts |
+| --- | --- |
+| `equalTo(expected)` | deep structural equality (NaN / Date / ±0 correct) |
+| `notEqualTo(expected)` | deep structural **inequality** |
+| `containingItem(item)` | value is an array containing `item` as an **exact** element |
+| `containingSubset(subset)` | `subset` is a recursive **partial** match — object key/value subset, or array sub-multiset membership |
+| `ofLength(n)` | array/string has length `n` |
+
+**Custom & judged**
+
+| Matcher | Asserts |
+| --- | --- |
+| `satisfying(predicate, message?)` | a deterministic predicate over the value holds (use for any negative not covered above) |
+| `judgedBy({ criteria, failWhen })` | an LLM judge resolves the criteria (fuzzy + paid) |
+
+```typescript
+expect(items).toBe.ofLength(3);
+expect(results).toBe.containingItem({ id: 7, status: "ok" });   // exact element
+expect(plan).toBe.containingSubset({ user: { id: 1 } });        // partial, nested
+expect(response).toBe.notContainingText("api_key");             // leak guard
+expect(score).toBe.satisfying((s) => s >= 0.8, "score too low");
+```
+
+> Use `containingItem` for exact array membership and `containingSubset` for
+> partial matching — strictness is chosen by the matcher name. For free-text
+> search over a structured value, assert on the `"text"` field.
+
 Generate a very interesting report with multiple runs!:
 
 ```

package/dist/adapters/remote.d.ts

@@ -56,7 +56,7 @@ export interface RemoteAdapterOptions {
  *
  * await agent(executor, () => {
  *   scene("What is 2+2?").expect("response", (r) => {
- *     expect(r).toBe.containing("4");
+ *     expect(r).toBe.containingText("4");
  *   });
  * });
  * ```

package/dist/adapters/remote.js

@@ -17,7 +17,7 @@
  *
  * await agent(executor, () => {
  *   scene("What is 2+2?").expect("response", (r) => {
- *     expect(r).toBe.containing("4");
+ *     expect(r).toBe.containingText("4");
  *   });
  * });
  * ```

package/dist/adapters/tracing.js

@@ -56,6 +56,7 @@ export async function createTracingHandle(baselineMs) {
                 model: name,
                 inputTokens: tokens?.input,
                 outputTokens: tokens?.output,
+                cachedInputTokens,
                 providerCost,
             });
             events.push({

package/dist/assertions.d.ts

@@ -5,10 +5,57 @@ export interface PendingJudgement {
 }
 export declare function collectPendingJudgements(): PendingJudgement[];
 export interface AgentMatchers {
+    /** Assert the agent refused. */
     refusal(): void;
+    /** Assert the agent did NOT refuse. */
     notRefusal(): void;
-    containing(text: string): void;
-    matchingPattern(regex: RegExp): void;
+    /**
+     * Text containment: `text` appears as a substring. For a non-string value the
+     * serialized form is searched. Case-INsensitive by default; pass
+     * `{ caseSensitive: true }` for an exact substring.
+     */
+    containingText(text: string | number, opts?: {
+        caseSensitive?: boolean;
+    }): void;
+    /** Assert text containment does NOT hold. See {@link containingText}. */
+    notContainingText(text: string | number, opts?: {
+        caseSensitive?: boolean;
+    }): void;
+    /**
+     * Array membership: the value is an array containing `item` as an EXACT
+     * (deep-equal) element. Throws if the value is not an array. Use
+     * {@link containingSubset} when you want partial element matching.
+     */
+    containingItem(item: unknown): void;
+    /**
+     * Structural subset: `subset` is recursively contained in the value.
+     * - object value + object `subset` → every key in `subset` is present with a
+     *   recursively-contained value (extra keys allowed).
+     * - array value + array `subset` → every `subset` element matches a distinct
+     *   element of the value (partial element matching, order-independent).
+     *
+     * Exact at the leaves (case-sensitive). Throws if the value is not an
+     * object/array, or `subset` is not an object/array.
+     */
+    containingSubset(subset: object): void;
+    /** Assert the serialized text view matches `pattern`. */
+    matchingPattern(pattern: RegExp): void;
+    /** Deep structural equality against the native value. */
+    equalTo(expected: unknown): void;
+    /** Assert deep structural INequality against the native value. */
+    notEqualTo(expected: unknown): void;
+    /** Assert the value (array/string) has length `n`. */
+    ofLength(n: number): void;
+    /**
+     * Escape hatch for anything not covered by a named matcher: a predicate over
+     * the native value. Stays deterministic — use it to express negatives too,
+     * e.g. `satisfying((v) => !v.includes("secret"))`.
+     */
+    satisfying(predicate: (value: any) => boolean, message?: string): void;
+    /**
+     * Queue an LLM-judged assertion, resolved asynchronously by the runner.
+     * Fuzzy + paid (express the negative in `failWhen`).
+     */
     judgedBy(criteria: JudgeCriteria): void;
 }
 export interface AgentExpectation {

package/dist/assertions.js

@@ -1,46 +1,127 @@
+import { isDeepStrictEqual } from "node:util";
 import { isRefusal } from "./refusal";
+import { serializeValue } from "./resolve";
+import { isObjectLike, isPlainObject, structuralContains } from "./match";
 let pendingJudgements = [];
 export function collectPendingJudgements() {
     const collected = pendingJudgements;
     pendingJudgements = [];
     return collected;
 }
+/**
+ * 100-char preview for error messages. Uses COMPACT JSON for objects (the
+ * judge-facing `serializeValue` pretty-prints; error previews stay terse and
+ * match the library's original contract).
+ */
+function preview(value) {
+    let s;
+    if (typeof value === "string") {
+        s = value;
+    }
+    else {
+        try {
+            s = JSON.stringify(value);
+        }
+        catch {
+            s = String(value);
+        }
+    }
+    return s.slice(0, 100);
+}
+/** Compact one-line form for an inline needle/expected in an error message. */
+function compact(value) {
+    try {
+        return typeof value === "string" ? value : JSON.stringify(value);
+    }
+    catch {
+        return String(value);
+    }
+}
+/** Human-readable type label for diagnostics (e.g. "a number", "an array"). */
+function describeType(value) {
+    if (value === null)
+        return "null";
+    if (Array.isArray(value))
+        return "an array";
+    return `a ${typeof value}`;
+}
+/**
+ * Substring search shared by `containingText` / `notContainingText`. A string
+ * value is searched directly; anything else via its serialized form.
+ * Case-insensitive unless `caseSensitive` is set.
+ */
+function textContains(value, text, opts) {
+    const actual = typeof value === "string" ? value : serializeValue(value);
+    const needle = String(text);
+    const hit = opts?.caseSensitive
+        ? actual.includes(needle)
+        : actual.toLowerCase().includes(needle.toLowerCase());
+    return { actual, hit };
+}
+function makeMatchers(value) {
+    const assert = (cond, message) => {
+        if (!cond)
+            throw new Error(message);
+    };
+    return {
+        refusal() {
+            assert(isRefusal(value), `Expected a refusal but got: "${preview(value)}"`);
+        },
+        notRefusal() {
+            assert(!isRefusal(value), `Expected a non-refusal response but got: "${preview(value)}"`);
+        },
+        containingText(text, opts) {
+            const { actual, hit } = textContains(value, text, opts);
+            assert(hit, `Expected response to contain "${text}" but got: "${actual.slice(0, 100)}"`);
+        },
+        notContainingText(text, opts) {
+            const { actual, hit } = textContains(value, text, opts);
+            assert(!hit, `Expected response NOT to contain "${text}" but got: "${actual.slice(0, 100)}"`);
+        },
+        containingItem(item) {
+            if (!Array.isArray(value)) {
+                throw new Error(`containingItem() expects an array value but got ${describeType(value)}. ` +
+                    `Use containingText() for substrings or containingSubset() for objects.`);
+            }
+            assert(value.some((el) => isDeepStrictEqual(el, item)), `Expected array to contain item ${compact(item)} but it did not (got ${preview(value)})`);
+        },
+        containingSubset(subset) {
+            if (!Array.isArray(value) && !isObjectLike(value)) {
+                throw new Error(`containingSubset() expects an object or array value but got ${describeType(value)}.`);
+            }
+            if (!Array.isArray(subset) && !isPlainObject(subset)) {
+                throw new Error(`containingSubset() expects an object or array subset but got ${describeType(subset)}.`);
+            }
+            assert(structuralContains(value, subset), `Expected value to contain subset ${compact(subset)} but it did not (got ${preview(value)})`);
+        },
+        matchingPattern(pattern) {
+            const actual = typeof value === "string" ? value : serializeValue(value);
+            assert(pattern.test(actual), `Expected response to match ${pattern} but got: "${actual.slice(0, 100)}"`);
+        },
+        equalTo(expected) {
+            assert(isDeepStrictEqual(value, expected), `Expected value to equal ${compact(expected)} but got ${preview(value)}`);
+        },
+        notEqualTo(expected) {
+            assert(!isDeepStrictEqual(value, expected), `Expected value NOT to equal ${compact(expected)} but it did`);
+        },
+        ofLength(n) {
+            const len = typeof value === "string" || Array.isArray(value)
+                ? value.length
+                : NaN;
+            assert(len === n, `Expected length ${n} but got ${Number.isNaN(len) ? "a non-measurable value" : len}`);
+        },
+        satisfying(predicate, message) {
+            assert(Boolean(predicate(value)), message ?? `Predicate failed for value: "${preview(value)}"`);
+        },
+        judgedBy(criteria) {
+            pendingJudgements.push({ value, criteria });
+        },
+    };
+}
 export function expect(value) {
     return {
         get toBe() {
-            return {
-                refusal() {
-                    if (!isRefusal(value)) {
-                        const preview = typeof value === "string"
-                            ? value.slice(0, 100)
-                            : JSON.stringify(value).slice(0, 100);
-                        throw new Error(`Expected a refusal but got: "${preview}"`);
-                    }
-                },
-                notRefusal() {
-                    if (isRefusal(value)) {
-                        const preview = typeof value === "string"
-                            ? value.slice(0, 100)
-                            : JSON.stringify(value).slice(0, 100);
-                        throw new Error(`Expected a non-refusal response but got: "${preview}"`);
-                    }
-                },
-                containing(text) {
-                    const actual = typeof value === "string" ? value : String(value);
-                    if (!actual.toLowerCase().includes(text.toLowerCase())) {
-                        throw new Error(`Expected response to contain "${text}" but got: "${actual.slice(0, 100)}"`);
-                    }
-                },
-                matchingPattern(regex) {
-                    const actual = typeof value === "string" ? value : String(value);
-                    if (!regex.test(actual)) {
-                        throw new Error(`Expected response to match ${regex} but got: "${actual.slice(0, 100)}"`);
-                    }
-                },
-                judgedBy(criteria) {
-                    pendingJudgements.push({ value, criteria });
-                },
-            };
+            return makeMatchers(value);
         },
     };
 }

package/dist/context.d.ts

@@ -15,7 +15,7 @@ export declare class SceneBuilder {
     expect(field: string, fn: (value: any) => void): SceneBuilder;
     toDefinition(): SceneDefinition;
 }
-export declare class AgentContext {
+export declare class AgentContext<T = string> {
     private _executor;
     private _name?;
     private _scenes;
@@ -24,13 +24,13 @@ export declare class AgentContext {
     private _afterAllHooks;
     private _beforeEachHooks;
     private _afterEachHooks;
-    constructor(_executor: AgentExecutor, _name?: string | undefined);
+    constructor(_executor: AgentExecutor<T>, _name?: string | undefined);
     registerHook(type: "beforeAll" | "afterAll" | "beforeEach" | "afterEach", fn: HookFn): void;
     setSuite(name: string): void;
     clearSuite(): void;
     registerScene(prompt: string): SceneBuilder;
-    execute(): Promise<AgentReport>;
+    execute(): Promise<AgentReport<T>>;
 }
 export declare function hashPromptOnly(prompt: string): string;
-export declare function setContext(ctx: AgentContext | null): void;
-export declare function getContext(): AgentContext;
+export declare function setContext(ctx: AgentContext<any> | null): void;
+export declare function getContext(): AgentContext<any>;

package/dist/context.js

@@ -1,5 +1,6 @@
 import { createHash } from "crypto";
 import { executeScene } from "./runner";
+import { resolveText } from "./resolve";
 import { formatReport, writeReport, writeDiffEntry } from "./reporter";
 import { logger, c } from "./logger";
 import { loadConfig } from "./config";
@@ -142,7 +143,7 @@ export class AgentContext {
                     logger.info(line);
                 }
             }
-            logger.debug(`${indent}       response: ${result.response.text?.slice(0, 120)}`);
+            logger.debug(`${indent}       response: ${resolveText(result.response).slice(0, 120)}`);
         };
         if (hasSuites) {
             // Execute suite by suite — print header once, then run all scenes in that suite
@@ -256,6 +257,9 @@ function hashPrompt(prompt, model) {
 export function hashPromptOnly(prompt) {
     return createHash("sha256").update(prompt).digest("hex").slice(0, 12);
 }
+// The active context is a runtime singleton holding an executor of arbitrary
+// value type, so `any` is the honest type for the holder. The generic flows
+// through `agent()` → `AgentContext<T>` → the report at the call site.
 let currentContext = null;
 export function setContext(ctx) {
     currentContext = ctx;

package/dist/index.d.ts

@@ -21,4 +21,4 @@ export declare function afterEach(fn: HookFn): void;
 export declare function suite(name: string, fn: () => void): void;
 /** @internal reset auto-run state between tests */
 export declare function _resetAutoRun(): void;
-export declare function agent(executor: AgentExecutor, fn: () => void, options?: AgentOptions): Promise<AgentReport>;
+export declare function agent<T = string>(executor: AgentExecutor<T>, fn: () => void, options?: AgentOptions): Promise<AgentReport<T>>;

package/dist/match.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Structural matching primitives for deterministic assertions. Kept in their
+ * own module — they are correctness-critical (a wrong result here is a false
+ * test pass) and deserve isolated, exhaustive unit tests.
+ */
+/** Any non-null, non-array object — including class instances, Map, Date, etc. */
+export declare function isObjectLike(value: unknown): value is Record<string, unknown>;
+/**
+ * A "record" object — a plain `{...}` literal (prototype is Object.prototype or
+ * null). Class instances, Map, Date, RegExp, etc. are NOT plain: they are
+ * compared as opaque leaves rather than recursed into.
+ */
+export declare function isPlainObject(value: unknown): value is Record<string, unknown>;
+/**
+ * Recursive containment: is `expected` structurally present within `actual`?
+ *
+ * - `expected` array  → `actual` is an array and the expected elements can be
+ *   matched one-to-one to DISTINCT actual elements (order-independent
+ *   multiset/sub-multiset membership — duplicates require distinct matches).
+ * - `expected` plain object → `actual` is object-like and every key in
+ *   `expected` exists in `actual` with a recursively-contained value (extra
+ *   keys in `actual` are allowed — that is the "partial").
+ * - anything else (primitive, Date, Map, RegExp, class instance) → strict
+ *   deep equality via `isDeepStrictEqual` (correct for NaN / Date / ±0).
+ *
+ * Leaf comparison is EXACT and case-sensitive. Only the shape recurses.
+ */
+export declare function structuralContains(actual: unknown, expected: unknown): boolean;

package/dist/match.js

@@ -0,0 +1,57 @@
+import { isDeepStrictEqual } from "node:util";
+/**
+ * Structural matching primitives for deterministic assertions. Kept in their
+ * own module — they are correctness-critical (a wrong result here is a false
+ * test pass) and deserve isolated, exhaustive unit tests.
+ */
+/** Any non-null, non-array object — including class instances, Map, Date, etc. */
+export function isObjectLike(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+/**
+ * A "record" object — a plain `{...}` literal (prototype is Object.prototype or
+ * null). Class instances, Map, Date, RegExp, etc. are NOT plain: they are
+ * compared as opaque leaves rather than recursed into.
+ */
+export function isPlainObject(value) {
+    if (!isObjectLike(value))
+        return false;
+    const proto = Object.getPrototypeOf(value);
+    return proto === Object.prototype || proto === null;
+}
+/**
+ * Recursive containment: is `expected` structurally present within `actual`?
+ *
+ * - `expected` array  → `actual` is an array and the expected elements can be
+ *   matched one-to-one to DISTINCT actual elements (order-independent
+ *   multiset/sub-multiset membership — duplicates require distinct matches).
+ * - `expected` plain object → `actual` is object-like and every key in
+ *   `expected` exists in `actual` with a recursively-contained value (extra
+ *   keys in `actual` are allowed — that is the "partial").
+ * - anything else (primitive, Date, Map, RegExp, class instance) → strict
+ *   deep equality via `isDeepStrictEqual` (correct for NaN / Date / ±0).
+ *
+ * Leaf comparison is EXACT and case-sensitive. Only the shape recurses.
+ */
+export function structuralContains(actual, expected) {
+    if (Array.isArray(expected)) {
+        if (!Array.isArray(actual))
+            return false;
+        // Greedy one-to-one matching: each expected element must claim a DISTINCT
+        // actual element, so `[1]` does not contain `[1, 1]`.
+        const claimed = new Set();
+        return expected.every((e) => {
+            const idx = actual.findIndex((a, i) => !claimed.has(i) && structuralContains(a, e));
+            if (idx === -1)
+                return false;
+            claimed.add(idx);
+            return true;
+        });
+    }
+    if (isPlainObject(expected)) {
+        if (!isObjectLike(actual))
+            return false;
+        return Object.keys(expected).every((key) => key in actual && structuralContains(actual[key], expected[key]));
+    }
+    return isDeepStrictEqual(actual, expected);
+}

package/dist/pricing/index.d.ts

@@ -3,7 +3,14 @@ export interface ModelPrice {
     input: number;
     /** USD per 1M output tokens */
     output: number;
+    /**
+     * USD per 1M cached (prompt-cache-hit) input tokens. When omitted, cached
+     * tokens are billed at `DEFAULT_CACHE_MULTIPLIER` × the input rate.
+     */
+    cachedInput?: number;
 }
+/** Fraction of the input rate charged for cache-hit tokens when no explicit rate is set. */
+export declare const DEFAULT_CACHE_MULTIPLIER = 0.1;
 export type CostSource = "provider" | "table" | "unavailable";
 export interface CostBreakdown {
     inputUsd?: number;
@@ -17,6 +24,8 @@ export interface ComputeCostInput {
     model?: string;
     inputTokens?: number;
     outputTokens?: number;
+    /** Cache-hit input tokens (subset of inputTokens), billed at the cached rate. */
+    cachedInputTokens?: number;
     /** USD cost the provider already reported (takes precedence) */
     providerCost?: number;
 }

package/dist/pricing/index.js

@@ -1,6 +1,8 @@
 import { readFileSync } from "fs";
 import { fileURLToPath } from "url";
 import { dirname, join } from "path";
+/** Fraction of the input rate charged for cache-hit tokens when no explicit rate is set. */
+export const DEFAULT_CACHE_MULTIPLIER = 0.1;
 const here = dirname(fileURLToPath(import.meta.url));
 const builtIn = JSON.parse(readFileSync(join(here, "models.json"), "utf-8"));
 let overrides = {};
@@ -31,7 +33,11 @@ export function computeCost(input) {
     const price = lookupPrice(input.model);
     if (!price)
         return { source: "unavailable" };
-    const inputUsd = ((input.inputTokens ?? 0) / 1_000_000) * price.input;
+    const totalInput = input.inputTokens ?? 0;
+    const cached = Math.min(input.cachedInputTokens ?? 0, totalInput);
+    const uncached = totalInput - cached;
+    const cachedRate = price.cachedInput ?? price.input * DEFAULT_CACHE_MULTIPLIER;
+    const inputUsd = (uncached / 1_000_000) * price.input + (cached / 1_000_000) * cachedRate;
     const outputUsd = ((input.outputTokens ?? 0) / 1_000_000) * price.output;
     return {
         inputUsd,

package/dist/reporter.d.ts

@@ -1,4 +1,4 @@
 import type { AgentReport } from "./types";
-export declare function formatReport(report: AgentReport): string;
+export declare function formatReport(report: AgentReport<unknown>): string;
 export declare function writeReport(content: string, timestamp: string, name?: string, dimensions?: Record<string, string>): Promise<string>;
 export declare function writeDiffEntry(hash: string, systemPrompt: string, tools: string[], model?: string): Promise<void>;

package/dist/reporter.js

@@ -1,6 +1,7 @@
 import { access, mkdir, writeFile } from "fs/promises";
 import { createHash } from "crypto";
 import { join } from "path";
+import { resolveText } from "./resolve";
 export function formatReport(report) {
     const lines = ["agent:"];
     if (report.name)
@@ -24,8 +25,9 @@ export function formatReport(report) {
                 lines.push(`          reason: "${reason}"`);
             }
             const result = report.results.find((r) => r.prompt === c);
-            if (result?.response.text) {
-                const escaped = result.response.text.replace(/"/g, '\\"').replace(/\n/g, '\\n');
+            const responseText = result ? resolveText(result.response) : "";
+            if (responseText) {
+                const escaped = responseText.replace(/"/g, '\\"').replace(/\n/g, '\\n');
                 lines.push(`          response: "${escaped}"`);
             }
         }
@@ -51,8 +53,9 @@ export function formatReport(report) {
                     if (r.error) {
                         lines.push(`                reason: "${r.error}"`);
                     }
-                    if (r.response.text) {
-                        const escaped = r.response.text.replace(/"/g, '\\"').replace(/\n/g, '\\n');
+                    const responseText = resolveText(r.response);
+                    if (responseText) {
+                        const escaped = responseText.replace(/"/g, '\\"').replace(/\n/g, '\\n');
                         lines.push(`                response: "${escaped}"`);
                     }
                 }

package/dist/resolve.d.ts

@@ -0,0 +1,25 @@
+import type { AgentResponse } from "./types";
+/**
+ * Serialize an arbitrary agent value to the string view the judge model and
+ * the text matchers consume. Strings pass through untouched; everything else
+ * is JSON. This is the ONLY place a structured value is forced to a string,
+ * and it happens lazily — never before a matcher actually needs text.
+ */
+export declare function serializeValue(value: unknown): string;
+/**
+ * The agent's native output — the source of truth for deterministic,
+ * structural assertions. Tolerates a legacy `{ text }`-only response (no
+ * `value`) so executors can migrate incrementally.
+ */
+export declare function resolveValue<T>(response: AgentResponse<T>): T | string | undefined;
+/**
+ * The string view for the judge and text matchers. An explicit `text` wins
+ * (it's the enriched projection the executor chose to expose); otherwise we
+ * serialize `value` on demand.
+ */
+export declare function resolveText<T>(response: AgentResponse<T>): string;
+/**
+ * Walk a dot-path (with numeric array indices) into an arbitrary object.
+ * Returns `undefined` if any segment is missing. e.g. "plan_items.0.options".
+ */
+export declare function navigatePath(root: unknown, path: string): unknown;

package/dist/resolve.js

@@ -0,0 +1,62 @@
+/**
+ * Serialize an arbitrary agent value to the string view the judge model and
+ * the text matchers consume. Strings pass through untouched; everything else
+ * is JSON. This is the ONLY place a structured value is forced to a string,
+ * and it happens lazily — never before a matcher actually needs text.
+ */
+export function serializeValue(value) {
+    if (typeof value === "string")
+        return value;
+    if (value === null || value === undefined)
+        return "";
+    try {
+        return JSON.stringify(value, null, 2);
+    }
+    catch {
+        return String(value);
+    }
+}
+/**
+ * The agent's native output — the source of truth for deterministic,
+ * structural assertions. Tolerates a legacy `{ text }`-only response (no
+ * `value`) so executors can migrate incrementally.
+ */
+export function resolveValue(response) {
+    if (response.value !== undefined)
+        return response.value;
+    return response.text;
+}
+/**
+ * The string view for the judge and text matchers. An explicit `text` wins
+ * (it's the enriched projection the executor chose to expose); otherwise we
+ * serialize `value` on demand.
+ */
+export function resolveText(response) {
+    if (typeof response.text === "string")
+        return response.text;
+    return serializeValue(response.value);
+}
+/**
+ * Walk a dot-path (with numeric array indices) into an arbitrary object.
+ * Returns `undefined` if any segment is missing. e.g. "plan_items.0.options".
+ */
+export function navigatePath(root, path) {
+    let cur = root;
+    for (const seg of path.split(".")) {
+        if (cur == null)
+            return undefined;
+        if (Array.isArray(cur)) {
+            const idx = Number(seg);
+            if (!Number.isInteger(idx))
+                return undefined;
+            cur = cur[idx];
+        }
+        else if (typeof cur === "object" && seg in cur) {
+            cur = cur[seg];
+        }
+        else {
+            return undefined;
+        }
+    }
+    return cur;
+}

package/dist/runner.d.ts

@@ -1,4 +1,13 @@
 import type { AgentExecutor, AgentResponse, SceneDefinition, SceneResult } from "./types";
 import type { JudgeConfig } from "./config";
-export declare function extractField(response: AgentResponse, field: string): unknown;
-export declare function executeScene(executor: AgentExecutor, scene: SceneDefinition, globalTimeout?: number, judgeConfig?: JudgeConfig, globalTurns?: number, globalRuns?: number): Promise<SceneResult>;
+/**
+ * Extract a named field from an agent response for assertion.
+ * - "response" / "value" → the native structured value (deterministic matchers)
+ * - "text"               → the serialized/judge view (lazy; text matchers)
+ * - "metadata"/"refusal" → the corresponding response property
+ * - dot-path             → navigated into the structured value first
+ *                          (e.g. "plan_items.0.options"), falling back to
+ *                          metadata so existing metadata paths keep resolving.
+ */
+export declare function extractField<T>(response: AgentResponse<T>, field: string): unknown;
+export declare function executeScene<T = string>(executor: AgentExecutor<T>, scene: SceneDefinition, globalTimeout?: number, judgeConfig?: JudgeConfig, globalTurns?: number, globalRuns?: number): Promise<SceneResult<T>>;

package/dist/runner.js

@@ -1,16 +1,33 @@
 import { collectPendingJudgements } from "./assertions";
 import { callJudge, resolveJudgeExecutor } from "./judge";
+import { resolveValue, resolveText, serializeValue, navigatePath } from "./resolve";
 const DEFAULT_SCENE_TIMEOUT = 10_000;
+/**
+ * Extract a named field from an agent response for assertion.
+ * - "response" / "value" → the native structured value (deterministic matchers)
+ * - "text"               → the serialized/judge view (lazy; text matchers)
+ * - "metadata"/"refusal" → the corresponding response property
+ * - dot-path             → navigated into the structured value first
+ *                          (e.g. "plan_items.0.options"), falling back to
+ *                          metadata so existing metadata paths keep resolving.
+ */
 export function extractField(response, field) {
     switch (field) {
         case "response":
-            return response.text;
+        case "value":
+            return resolveValue(response);
+        case "text":
+            return resolveText(response);
         case "metadata":
             return response.metadata;
         case "refusal":
             return response.refusal;
-        default:
-            return response.metadata?.[field];
+        default: {
+            const fromValue = navigatePath(resolveValue(response), field);
+            if (fromValue !== undefined)
+                return fromValue;
+            return navigatePath(response.metadata ?? {}, field);
+        }
     }
 }
 /**
@@ -31,6 +48,9 @@ function wilsonSignificance(passes, total) {
     return Math.max(0, Math.min(1, lower));
 }
 async function executeSingleRun(executor, scene, timeoutMs, turns, judgeConfig) {
+    // The empty sentinel uses the `text` branch of the union so it is a valid
+    // AgentResponse<T> for ANY T (there is no native value yet — the executor
+    // hasn't run). Using `{ value: "" }` would wrongly assume T = string.
     let response = { text: "" };
     let duration;
     try {
@@ -91,7 +111,9 @@ async function executeSingleRun(executor, scene, timeoutMs, turns, judgeConfig)
             const judgeExecutor = resolveJudgeExecutor(judgeConfig);
             for (const p of pending) {
                 try {
-                    const result = await callJudge(String(p.value), p.criteria, judgeExecutor);
+                    // Hand the judge the serialized text view — NOT String(value),
+                    // which would render a structured value as "[object Object]".
+                    const result = await callJudge(serializeValue(p.value), p.criteria, judgeExecutor);
                     judgement = result;
                     if (result.verdict === "fail" || result.verdict === "partial") {
                         passed = false;

package/dist/types.d.ts

@@ -1,7 +1,7 @@
 export interface ExecutorOptions {
     signal?: AbortSignal;
 }
-export type AgentExecutor = (input: string, options?: ExecutorOptions) => Promise<AgentResponse>;
+export type AgentExecutor<T = string> = (input: string, options?: ExecutorOptions) => Promise<AgentResponse<T>>;
 export type CostSource = "provider" | "table" | "unavailable";
 export interface CostBreakdown {
     inputUsd?: number;
@@ -28,8 +28,35 @@ export interface TimelineEvent {
     runIndex?: number;
     error?: string;
 }
-export interface AgentResponse {
+/**
+ * The result an executor hands back. EXACTLY ONE of `value` / `text` is
+ * required (both may be present); the rest are optional.
+ *
+ * `value` is the agent's NATIVE output and the source of truth for
+ * deterministic, structural assertions — a string for a chat agent, an object
+ * for a structured agent (a plan, a tool-call payload, parsed JSON). It is
+ * never coerced to a string before a matcher asks for text.
+ *
+ * `text` is a pre-serialized projection for the judge model and the text
+ * matchers (`containing`, `matchingPattern`, `refusal`). A string-producing
+ * agent can return ONLY `text` (the legacy/common case) — it is then also used
+ * as `value`. A structured agent returns `value` and, optionally, an enriched
+ * `text` when the judge needs a view the raw value can't give cheaply (e.g.
+ * resolving opaque ids to names). When `text` is omitted, agest serializes
+ * `value` lazily (string passthrough, else JSON). See `resolve.ts`.
+ *
+ * The generic defaults to `string`, so the common chat case stays
+ * `{ text: "..." }` or `{ value: "..." }` with no type ceremony.
+ */
+export type AgentResponse<T = string> = AgentResponseBase<T> & ({
+    value: T;
+} | {
     text: string;
+});
+interface AgentResponseBase<T = string> {
+    value?: T;
+    /** Pre-serialized view for the judge / text matchers. */
+    text?: string;
     refusal?: boolean;
     executionError?: string;
     metadata?: {
@@ -63,22 +90,22 @@ export interface JudgeResult {
     reasoning: string;
     criteria: string;
 }
-export interface RunResult {
+export interface RunResult<T = string> {
     passed: boolean;
     error?: string;
-    response: AgentResponse;
+    response: AgentResponse<T>;
     duration: number;
     judgement?: JudgeResult;
 }
-export interface SceneResult {
+export interface SceneResult<T = string> {
     prompt: string;
-    response: AgentResponse;
+    response: AgentResponse<T>;
     duration: number;
     passed: boolean;
     error?: string;
     judgement?: JudgeResult;
     suite?: string;
-    runs?: RunResult[];
+    runs?: RunResult<T>[];
     passRate?: number;
     statisticalSignificance?: number;
     /** Aggregate tokens across all runs of this scene */
@@ -92,7 +119,7 @@ export interface SceneResult {
     /** Ordered timeline events from every run of the scene */
     events?: TimelineEvent[];
 }
-export interface AgentReport {
+export interface AgentReport<T = string> {
     name?: string;
     model?: string;
     systemPromptHash?: string;
@@ -110,5 +137,6 @@ export interface AgentReport {
     totalInputTokens?: number;
     totalOutputTokens?: number;
     totalCostUsd?: number;
-    results: SceneResult[];
+    results: SceneResult<T>[];
 }
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sebastiantuyu/agest",
-  "version": "0.3.3-next.7",
+  "version": "0.3.3-next.8",
   "description": "A testing library for agents",
   "repository": {
     "type": "git",