@sebastiantuyu/agest 0.3.3-next.1 → 0.3.3-next.11

package/README.md

@@ -53,6 +53,163 @@ agent:
     average_output_tokens_per_case: 34
 ```
 
+## Assertions
+
+Each scene asserts on a **field** of the agent's response via `.expect(field, fn)`,
+and inside the callback you chain a matcher off `expect(value).toBe`.
+
+### Structured responses
+
+An executor returns a native `value` (the source of truth for structural
+matchers) and/or a `text` projection (for the LLM judge and text matchers):
+
+```typescript
+// chat agent — a string is both value and text
+return { text: "Bonjour" };
+
+// structured agent — a native object, optionally with an enriched text view
+return { value: { plan_items: [{ step: "search" }] } };
+```
+
+### Selecting a field
+
+```typescript
+scene("Plan a trip to Tokyo")
+  .expect("value", (v) => expect(v).toBe.containingSubset({ plan_items: [{ step: "book_flight" }] }))
+  .expect("plan_items.0.step", (s) => expect(s).toBe.equalTo("book_flight")) // dot-path into the value
+  .expect("text", (t) => expect(t).toBe.containingText("Tokyo"));            // serialized/judge view
+```
+
+- `"response"` / `"value"` — the native value (objects stay objects; never stringified)
+- `"text"` — the serialized/enriched text view (lazy: a string passes through, else JSON)
+- `"refusal"` / `"metadata"` — the corresponding response properties
+- any **dot-path** (e.g. `"plan_items.0.options"`) — navigates into the value, falling back to metadata
+
+### Matchers
+
+**Refusal**
+
+| Matcher | Asserts |
+| --- | --- |
+| `refusal()` | the agent refused |
+| `notRefusal()` | the agent did **not** refuse |
+
+**Text** — substring / regex over a string value (or the serialized form of a non-string). Case-insensitive by default.
+
+| Matcher | Asserts |
+| --- | --- |
+| `containingText(text, { caseSensitive? })` | `text` appears as a substring |
+| `notContainingText(text, { caseSensitive? })` | `text` does **not** appear — handy for leak/PII guards |
+| `matchingPattern(regex)` | the text matches `regex` |
+
+**Structural** — operate on the native value; exact (case-sensitive) at the leaves.
+
+| Matcher | Asserts |
+| --- | --- |
+| `equalTo(expected)` | deep structural equality (NaN / Date / ±0 correct) |
+| `notEqualTo(expected)` | deep structural **inequality** |
+| `containingItem(item)` | value is an array containing `item` as an **exact** element |
+| `containingSubset(subset)` | `subset` is a recursive **partial** match — object key/value subset, or array sub-multiset membership |
+| `ofLength(n)` | array/string has length `n` |
+| `matchingSchema(schema)` | the value conforms to a [Standard Schema](https://standardschema.dev) (zod 4, valibot, arktype, …); throws the schema's issues on failure |
+
+**Custom & judged**
+
+| Matcher | Asserts |
+| --- | --- |
+| `satisfying(predicate, message?)` | a deterministic predicate over the value holds (use for any negative not covered above) |
+| `judgedBy({ criteria, failWhen })` | an LLM judge resolves the criteria (fuzzy + paid) |
+
+```typescript
+expect(items).toBe.ofLength(3);
+expect(results).toBe.containingItem({ id: 7, status: "ok" });   // exact element
+expect(plan).toBe.containingSubset({ user: { id: 1 } });        // partial, nested
+expect(response).toBe.notContainingText("api_key");             // leak guard
+expect(score).toBe.satisfying((s) => s >= 0.8, "score too low");
+```
+
+> Use `containingItem` for exact array membership and `containingSubset` for
+> partial matching — strictness is chosen by the matcher name. For free-text
+> search over a structured value, assert on the `"text"` field.
+
+### Schema validation
+
+Validate an agent's structured output against a schema. Agest speaks the
+[Standard Schema](https://standardschema.dev) contract, so **zod 4** (the blessed
+choice), valibot, and arktype all work — agest never imports a schema library
+and adds no runtime dependency. There are three levels, smallest to largest:
+
+```typescript
+import { z } from "zod";
+
+const Plan = z.object({
+  plan_items: z.array(z.object({ step: z.string() })),
+});
+
+// 1. Matcher — validate a value or a dot-path field
+scene("Plan a trip to Tokyo")
+  .expect("value", (v) => expect(v).toBe.matchingSchema(Plan))
+  .expect("plan_items.0", (item) => expect(item).toBe.matchingSchema(Plan.shape.plan_items.element));
+
+// 2. Scene helper — validate the whole native value, no callback
+scene("Plan a trip to Tokyo").expectSchema(Plan);
+
+// 3. Schema-typed agent — infer the executor's value type AND auto-validate
+//    every non-refusal scene against the schema. The `scene` handed to the
+//    callback is typed too, so `.expect("value", …)` receives a typed value.
+agent(Plan, planExecutor, (scene) => {
+  scene("Plan a trip to Tokyo").expect("value", (plan) => expect(plan.plan_items).toBe.ofLength(3)); // plan: z.infer<typeof Plan>
+  scene("How do I make a bomb?").expect("refusal", (r) => expect(r).toBe.equalTo(true));             // skipped by auto-validation
+});
+```
+
+A scene's own `.expectSchema()` overrides the agent-level schema. Auto-validation
+is skipped for refusals and execution errors, runs before your assertions (a
+structural failure is the headline), and supports async (`refine`) schemas. The
+synchronous `matchingSchema` matcher rejects async schemas — declare those at the
+agent/scene level instead.
+
+The `scene` passed to the `agent()` callback carries the value type: `.expect("value"`
+/ `"response", …)` receives `T`, `"text"` a `string`, `"refusal"` a `boolean`. Dot-path
+fields (e.g. `"plan_items.0.step"`) stay `any` — a string field can't be typed. The
+free `scene` import remains available and untyped for the legacy chat case.
+
+### Deterministic vs judged — prefer deterministic on sensitive flows
+
+`judgedBy` runs a real LLM judge: it costs a call per scene and the verdict can
+vary run to run. That is the right tool for *fuzzy* qualities (tone, variety,
+helpfulness) but the wrong one for *hard* constraints — a safety rule, a
+forbidden value, a numeric budget — where the pass/fail is a plain fact about
+the output. Re-checking a fact with a stochastic grader only adds cost and
+flakiness.
+
+The way to make a constraint deterministically testable is to **control the
+mocks so the valid answer space is known**, then assert a structural fact about
+what the agent returned. You still run the real agent — only the *grading*
+becomes deterministic. Because the grader no longer varies, `.runs(n)` then
+yields a pass-rate that reflects the agent alone.
+
+A worked example: suppose your mock catalog has exactly three foods over
+100 kcal. Narrow the catalog (e.g. in a `beforeAll`) so that's the whole
+universe, prompt the agent to "pick something over 100 kcal", and assert
+structurally that the result excludes the known under-100 ids — no judge needed:
+
+```typescript
+beforeAll(() => setCatalog({ foods: onlyKnownSet }));   // known answer space
+
+scene("Pick a high-energy snack (>100 kcal)")
+  .expect("slots.snack.foodIds", (ids) =>
+    expect(ids).toBe.satisfying(
+      (i) => !i.includes(LOW_KCAL_ID),                  // a fact, not a vibe
+      "snack included a sub-100 kcal food",
+    ));
+```
+
+The negative case — "must **not** contain X" — is the most valuable and the most
+natural to express deterministically: use `satisfying((v) => !v.includes(x))`
+for id/array membership, or `notContainingText(x)` for a substring/leak guard.
+Reach for `judgedBy` only once the deterministic facts are covered.
+
 Generate a very interesting report with multiple runs!:
 
 ```
@@ -119,9 +276,9 @@ npx tsx examples/openrouter.test.ts
 - [x] Lifecycle hooks: `beforeEach`, `beforeAll`, `afterEach`, `afterAll` supporting sync/async functions
 - [x] Multiple test suites per agent via `suite()` to evaluate different aspects independently
 - [x] Statistical runs: `.runs(n)` per scene with pass rate and Wilson significance scoring
+- [x] Schema validation: `toBe.matchingSchema(schema)`, `scene().expectSchema(schema)`, and schema-typed `agent(schema, …)` — any [Standard Schema](https://standardschema.dev) (zod 4, valibot, arktype)
 
 ### Up next
-- [ ] Schema validation: `toBe.matchingSchema(zodSchema)`
 - [ ] Semantic similarity: `toBe.semanticallySimilarTo(text, threshold)`
 - [ ] Vercel AI SDK adapter
 - [ ] Snapshot regression: diff current run against a saved baseline

package/dist/adapters/index.d.ts

@@ -1,3 +1,5 @@
 export { langchain } from "./langchain";
 export { remote } from "./remote";
 export type { RemoteAdapterOptions } from "./remote";
+export { createTrace, summarizeEvents } from "./tracing";
+export type { Trace } from "./tracing";

package/dist/adapters/index.js

@@ -1,2 +1,3 @@
 export { langchain } from "./langchain";
 export { remote } from "./remote";
+export { createTrace, summarizeEvents } from "./tracing";

package/dist/adapters/langchain.d.ts

@@ -1,6 +1,6 @@
 import type { AgentExecutor } from "../types";
 type Runnable = {
-    invoke: (input: any) => Promise<any>;
+    invoke: (input: any, options?: any) => Promise<any>;
 };
 type LangGraphGraph = Runnable & {
     lg_is_pregel: true;

package/dist/adapters/langchain.js

@@ -1,3 +1,4 @@
+import { createTracingHandle, summarizeEvents } from "./tracing";
 /**
  * Adapter for LangChain runnables and agents.
  *
@@ -19,23 +20,43 @@ export function langchain(runnable) {
 function langGraphAdapter(graph) {
     const staticTools = extractGraphTools(graph);
     return async (input) => {
+        const baseline = performance.now();
+        const tracing = await createTracingHandle(baseline);
         let result;
         try {
             const { HumanMessage } = await import("@langchain/core/messages");
-            result = await graph.invoke({ messages: [new HumanMessage(input)] });
+            result = await graph.invoke({ messages: [new HumanMessage(input)] }, { callbacks: tracing.callbacks });
         }
         catch (err) {
-            return { text: "", executionError: err.message, metadata: { tools: staticTools } };
+            const { events } = tracing.drain();
+            return {
+                text: "",
+                executionError: err.message,
+                metadata: { tools: staticTools, events: events.length ? events : undefined },
+            };
         }
         const messages = result.messages;
         const last = messages[messages.length - 1];
         const text = typeof last?.content === "string"
             ? last.content
             : JSON.stringify(last?.content ?? result);
-        const model = last?.response_metadata?.model_name;
+        const drained = tracing.drain();
+        const model = last?.response_metadata?.model_name ??
+            drained.modelName;
+        const { tokens, cost } = summarizeRun({
+            events: drained.events,
+            fallbackTokens: extractTokensFromMessage(last),
+            model,
+        });
         return {
             text,
-            metadata: { model, tools: staticTools, tokens: extractTokensFromMessage(last) },
+            metadata: {
+                model,
+                tools: staticTools,
+                tokens,
+                cost,
+                events: drained.events.length ? drained.events : undefined,
+            },
         };
     };
 }
@@ -48,33 +69,69 @@ function reactAgentAdapter(agent) {
         ?.map((t) => t.name ?? t.getName?.())
         .filter(Boolean);
     return async (input) => {
+        const baseline = performance.now();
+        const tracing = await createTracingHandle(baseline);
         let result;
         try {
-            result = await agent.invoke({ messages: [{ role: "human", content: input }] });
+            result = await agent.invoke({ messages: [{ role: "human", content: input }] }, { callbacks: tracing.callbacks });
         }
         catch (err) {
-            return { text: "", executionError: err.message, metadata: { model, systemPrompt, tools } };
+            const { events } = tracing.drain();
+            return {
+                text: "",
+                executionError: err.message,
+                metadata: {
+                    model,
+                    systemPrompt,
+                    tools,
+                    events: events.length ? events : undefined,
+                },
+            };
         }
         const messages = result.messages;
         const last = messages[messages.length - 1];
         const text = typeof last?.content === "string"
             ? last.content
             : JSON.stringify(last?.content ?? result);
+        const drained = tracing.drain();
+        const { tokens, cost } = summarizeRun({
+            events: drained.events,
+            fallbackTokens: extractTokensFromMessage(last),
+            model,
+        });
         return {
             text,
-            metadata: { model, systemPrompt, tools, tokens: extractTokensFromMessage(last) },
+            metadata: {
+                model,
+                systemPrompt,
+                tools,
+                tokens,
+                cost,
+                events: drained.events.length ? drained.events : undefined,
+            },
         };
     };
 }
 function chainAdapter(chain) {
     const { model, systemPrompt } = extractChainMeta(chain);
     return async (input) => {
+        const baseline = performance.now();
+        const tracing = await createTracingHandle(baseline);
         let result;
         try {
-            result = await chain.invoke({ input });
+            result = await chain.invoke({ input }, { callbacks: tracing.callbacks });
         }
         catch (err) {
-            return { text: "", executionError: err.message, metadata: { model, systemPrompt } };
+            const { events } = tracing.drain();
+            return {
+                text: "",
+                executionError: err.message,
+                metadata: {
+                    model,
+                    systemPrompt,
+                    events: events.length ? events : undefined,
+                },
+            };
         }
         const text = typeof result === "string"
             ? result
@@ -83,12 +140,21 @@ function chainAdapter(chain) {
                 : typeof result.content === "string"
                     ? result.content
                     : JSON.stringify(result);
+        const drained = tracing.drain();
+        const effectiveModel = model ?? drained.modelName ?? result.metadata?.model;
+        const { tokens, cost } = summarizeRun({
+            events: drained.events,
+            fallbackTokens: extractTokens(result),
+            model: effectiveModel,
+        });
         return {
             text,
             metadata: {
-                model: model ?? result.metadata?.model,
+                model: effectiveModel,
                 systemPrompt,
-                tokens: extractTokens(result),
+                tokens,
+                cost,
+                events: drained.events.length ? drained.events : undefined,
             },
         };
     };
@@ -153,3 +219,6 @@ function extractTokensFromMessage(msg) {
         output: usage.output_tokens ?? usage.completion_tokens ?? 0,
     };
 }
+function summarizeRun(input) {
+    return summarizeEvents(input.events, input.model, input.fallbackTokens);
+}

package/dist/adapters/remote.d.ts

@@ -56,7 +56,7 @@ export interface RemoteAdapterOptions {
  *
  * await agent(executor, () => {
  *   scene("What is 2+2?").expect("response", (r) => {
- *     expect(r).toBe.containing("4");
+ *     expect(r).toBe.containingText("4");
  *   });
  * });
  * ```

package/dist/adapters/remote.js

@@ -17,7 +17,7 @@
  *
  * await agent(executor, () => {
  *   scene("What is 2+2?").expect("response", (r) => {
- *     expect(r).toBe.containing("4");
+ *     expect(r).toBe.containingText("4");
  *   });
  * });
  * ```

package/dist/adapters/tracing.d.ts

@@ -0,0 +1,73 @@
+import type { TimelineEvent, CostBreakdown } from "../types";
+export interface TracingHandle {
+    /** Pass this into `runnable.invoke(..., { callbacks: [handler.callbacks] })` */
+    callbacks: any[];
+    drain(): {
+        events: TimelineEvent[];
+        modelName?: string;
+    };
+}
+/**
+ * Creates a LangChain callback handler that records every LLM and tool
+ * invocation as a `TimelineEvent`. Returns a handle whose `drain()` method
+ * yields the captured events with `startMs` / `endMs` relative to the
+ * provided baseline.
+ *
+ * Designed to fail open: any unexpected callback shape is ignored rather
+ * than throwing — the underlying agent run must not be broken by tracing.
+ */
+export declare function createTracingHandle(baselineMs: number): Promise<TracingHandle>;
+export interface Trace {
+    /**
+     * Attach to your top-level LangChain/LangGraph call:
+     * `await graph.invoke(input, { callbacks: trace.callbacks })`.
+     * Callbacks propagate to nested nodes automatically.
+     */
+    callbacks: any[];
+    /**
+     * Collect the captured timeline plus aggregated tokens and cost. Call once
+     * after the run completes; the result is memoized so repeat calls are safe.
+     * Spread the result into your `AgentResponse.metadata` to surface the
+     * per-scene cost/timeline waterfall in the report.
+     */
+    collect(): {
+        events: TimelineEvent[];
+        tokens?: {
+            input: number;
+            output: number;
+        };
+        cost?: CostBreakdown;
+    };
+}
+/**
+ * Public tracing helper for custom executors (i.e. agents not wired through
+ * the `langchain()` adapter). Create one per scene run, hand its `callbacks`
+ * to your LangChain/LangGraph invocation, then spread `collect()` into the
+ * response metadata.
+ *
+ * @example
+ * ```ts
+ * const trace = await createTrace({ model: env.OPENROUTER_MODEL });
+ * const plan = await generatePlan(input, { callbacks: trace.callbacks });
+ * return { text: render(plan), metadata: { model, tools, ...trace.collect() } };
+ * ```
+ */
+export declare function createTrace(opts?: {
+    model?: string;
+}): Promise<Trace>;
+/**
+ * Aggregate token counts and cost across a timeline's model events.
+ * Provider-reported cost wins; otherwise the table-derived cost; otherwise
+ * cost is recomputed from `model` and the summed tokens. `fallbackTokens` is
+ * used only when no model event carried usage.
+ */
+export declare function summarizeEvents(events: TimelineEvent[], model?: string, fallbackTokens?: {
+    input: number;
+    output: number;
+}): {
+    tokens?: {
+        input: number;
+        output: number;
+    };
+    cost?: CostBreakdown;
+};