@sebastiantuyu/agest 0.1.0

package/README.md

@@ -0,0 +1,127 @@
+# Agest
+
+A quantitative testing library for agents using a Jest-like syntax. 
+Batteries included.
+
+Main purpose is to provide helpful benchmarks with minimum API for quick iteration and evaluation of
+different system prompts, models and tools considering their impact on the agent's performance.
+
+
+## Basic usage
+
+A language-learning assistant that should refuse off-topic questions, tested with a real LLM via OpenRouter.
+
+```typescript
+import "dotenv/config";
+import { agent, scene, expect } from "@sebastiantuyu/agest";
+import { createAgent } from "langchain";
+
+const reactAgent = createAgent({
+    model: "openai/gpt-4.1-mini",
+    systemPrompt: "You are a language learning assistant. Refuse all off-topic questions.",
+})
+
+await agent(reactAgent, () => {
+  scene("What is the weather like today?")
+    .expect("response", (response) => {
+      expect(response).toBe.refusal();
+    });
+
+  scene("How do you say 'good morning' in Japanese?")
+    .expect("response", (response) => {
+      expect(response).toBe.notRefusal();
+    });
+});
+```
+
+This produces a scored report:
+
+```
+agent: 
+    model: "openai/gpt-4.1-mini"
+    system_prompt: <check_sum>
+    tools: []
+    success_rate: 1
+    failed_cases:
+        (none)
+    timestamp: "2025-01-01T00:00:00.000Z"
+    duration: 3421
+    total_cases: 2
+    average_input_tokens_per_case: 87
+    average_output_tokens_per_case: 34
+```
+
+Generate a very interesting report with multiple runs!:
+
+```
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+  AGEST STATS  ·  5 reports found
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+  Success Rate
+  ────────────────────────────────────────────────────────────
+  anthropic/claude-haiku-4-5  ███████████████████░   93%
+  google/gemini-2.0-flash-li  ███████████████████░   93%
+  openai/gpt-4.1-nano (1x)    ████████████████░░░░   80%
+  meta-llama/llama-3.1-8b-in  ███████████████░░░░░   73%
+  mistralai/ministral-8b-251  ████████████░░░░░░░░   60%
+
+  Avg Input Tokens / Case
+  ────────────────────────────────────────────────────────────
+  anthropic/claude-haiku-4-5  ████████████████████   1021
+  google/gemini-2.0-flash-li  ██████░░░░░░░░░░░░░░    311
+  openai/gpt-4.1-nano         ███████░░░░░░░░░░░░░    335
+  meta-llama/llama-3.1-8b-in  ██████████████░░░░░░    711
+  mistralai/ministral-8b-251  █████████░░░░░░░░░░░    482
+
+  Avg Output Tokens / Case
+  ────────────────────────────────────────────────────────────
+  anthropic/claude-haiku-4-5  ████████████████████    103
+  google/gemini-2.0-flash-li  █████░░░░░░░░░░░░░░░     24
+  openai/gpt-4.1-nano         ██████░░░░░░░░░░░░░░     33
+  meta-llama/llama-3.1-8b-in  ███████░░░░░░░░░░░░░     37
+  mistralai/ministral-8b-251  ██████████░░░░░░░░░░     54
+
+  Avg Duration / Run  (fastest first)
+  ────────────────────────────────────────────────────────────
+  meta-llama/llama-3.1-8b-in  ██░░░░░░░░░░░░░░░░░░      8.6s
+  google/gemini-2.0-flash-li  ███░░░░░░░░░░░░░░░░░     14.2s
+  openai/gpt-4.1-nano (1x)    █████░░░░░░░░░░░░░░░     20.3s
+  mistralai/ministral-8b-251  ███████░░░░░░░░░░░░░     30.1s
+  anthropic/claude-haiku-4-5  ████████████████████     1m24s
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+  5 models · 5 total runs
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+```
+
+## Running the real example
+
+Copy `.env.example` to `.env` and add your [OpenRouter](https://openrouter.ai) API key:
+
+```sh
+cp .env.example .env
+# edit .env and set OPENROUTER_API_KEY
+npx tsx examples/openrouter.test.ts
+```
+
+
+## Roadmap
+
+- [ ] Multi-run support: `.runs(n)` per scene for statistical significance
+- [ ] Suite-level runs: `agent(exec, { runs: 3 }, () => { ... })` for overall stability benchmarks
+- [ ] Additional matchers: `toBe.semanticallySimilarTo(text, threshold)`, `toBe.matchingSchema(zodSchema)`
+- [ ] JSON/file reporters for persisting reports to disk
+- [ ] Snapshot comparison: diff reports across runs to track agent regression
+- [ ] More adapters: Vercel AI SDK, OpenAI Agents SDK, raw API calls
+
+## Development requirements
+- Node 22+
+- pnpm
+
+## Build
+
+```sh
+pnpm install
+pnpm build
+```

package/dist/adapters/index.d.ts

@@ -0,0 +1 @@
+export { langchain } from "./langchain";

package/dist/adapters/index.js

@@ -0,0 +1 @@
+export { langchain } from "./langchain";

package/dist/adapters/langchain.d.ts

@@ -0,0 +1,30 @@
+import type { AgentExecutor } from "../types";
+type Runnable = {
+    invoke: (input: any) => Promise<any>;
+};
+type LangGraphGraph = Runnable & {
+    lg_is_pregel: true;
+    nodes?: Record<string, any>;
+};
+type LangChainReactAgent = Runnable & {
+    options: {
+        model?: string | any;
+        tools?: any[];
+        systemPrompt?: string;
+        prompt?: string;
+    };
+};
+type SimpleChain = Runnable & {
+    steps?: any[];
+};
+/**
+ * Adapter for LangChain runnables and agents.
+ *
+ * Supported inputs:
+ * - `createAgent(...)` from `langchain` — meta extracted from `agent.options`
+ * - `createReactAgent(...)` from `@langchain/langgraph/prebuilt` — tools from
+ *   `graph.nodes.tools`, model from response_metadata
+ * - Simple chain (`prompt.pipe(model)`) — meta extracted from `steps[]`
+ */
+export declare function langchain(runnable: LangGraphGraph | LangChainReactAgent | SimpleChain): AgentExecutor;
+export {};

package/dist/adapters/langchain.js

@@ -0,0 +1,155 @@
+/**
+ * Adapter for LangChain runnables and agents.
+ *
+ * Supported inputs:
+ * - `createAgent(...)` from `langchain` — meta extracted from `agent.options`
+ * - `createReactAgent(...)` from `@langchain/langgraph/prebuilt` — tools from
+ *   `graph.nodes.tools`, model from response_metadata
+ * - Simple chain (`prompt.pipe(model)`) — meta extracted from `steps[]`
+ */
+export function langchain(runnable) {
+    if (isLangGraphGraph(runnable)) {
+        return langGraphAdapter(runnable);
+    }
+    if (isReactAgent(runnable)) {
+        return reactAgentAdapter(runnable);
+    }
+    return chainAdapter(runnable);
+}
+function langGraphAdapter(graph) {
+    const staticTools = extractGraphTools(graph);
+    return async (input) => {
+        let result;
+        try {
+            const { HumanMessage } = await import("@langchain/core/messages");
+            result = await graph.invoke({ messages: [new HumanMessage(input)] });
+        }
+        catch (err) {
+            return { text: "", executionError: err.message, metadata: { tools: staticTools } };
+        }
+        const messages = result.messages;
+        const last = messages[messages.length - 1];
+        const text = typeof last?.content === "string"
+            ? last.content
+            : JSON.stringify(last?.content ?? result);
+        const model = last?.response_metadata?.model_name;
+        return {
+            text,
+            metadata: { model, tools: staticTools, tokens: extractTokensFromMessage(last) },
+        };
+    };
+}
+function reactAgentAdapter(agent) {
+    const model = typeof agent.options.model === "string"
+        ? agent.options.model
+        : agent.options.model?.modelName ?? agent.options.model?.model;
+    const systemPrompt = agent.options.systemPrompt ?? agent.options.prompt ?? undefined;
+    const tools = agent.options.tools
+        ?.map((t) => t.name ?? t.getName?.())
+        .filter(Boolean);
+    return async (input) => {
+        let result;
+        try {
+            result = await agent.invoke({ messages: [{ role: "human", content: input }] });
+        }
+        catch (err) {
+            return { text: "", executionError: err.message, metadata: { model, systemPrompt, tools } };
+        }
+        const messages = result.messages;
+        const last = messages[messages.length - 1];
+        const text = typeof last?.content === "string"
+            ? last.content
+            : JSON.stringify(last?.content ?? result);
+        return {
+            text,
+            metadata: { model, systemPrompt, tools, tokens: extractTokensFromMessage(last) },
+        };
+    };
+}
+function chainAdapter(chain) {
+    const { model, systemPrompt } = extractChainMeta(chain);
+    return async (input) => {
+        let result;
+        try {
+            result = await chain.invoke({ input });
+        }
+        catch (err) {
+            return { text: "", executionError: err.message, metadata: { model, systemPrompt } };
+        }
+        const text = typeof result === "string"
+            ? result
+            : typeof result.output === "string"
+                ? result.output
+                : typeof result.content === "string"
+                    ? result.content
+                    : JSON.stringify(result);
+        return {
+            text,
+            metadata: {
+                model: model ?? result.metadata?.model,
+                systemPrompt,
+                tokens: extractTokens(result),
+            },
+        };
+    };
+}
+function isLangGraphGraph(r) {
+    return r.lg_is_pregel === true;
+}
+function isReactAgent(r) {
+    return r.options !== undefined && typeof r.options === "object" && !Array.isArray(r.options);
+}
+function extractGraphTools(graph) {
+    const tools = graph.nodes?.["tools"]?.bound?.tools;
+    if (!Array.isArray(tools) || tools.length === 0)
+        return undefined;
+    return tools.map((t) => t.name ?? t.getName?.()).filter(Boolean);
+}
+function extractChainMeta(chain) {
+    function fromSteps(steps) {
+        let model;
+        let systemPrompt;
+        for (const step of steps ?? []) {
+            if (!model && (step.modelName || step.model)) {
+                model = step.modelName ?? step.model;
+            }
+            if (!systemPrompt && Array.isArray(step.promptMessages)) {
+                for (const msg of step.promptMessages) {
+                    const name = msg?.constructor?.name ?? "";
+                    if (name.toLowerCase().includes("system")) {
+                        systemPrompt = msg?.prompt?.template ?? msg?.template;
+                        break;
+                    }
+                }
+            }
+            if (step.steps) {
+                const nested = fromSteps(step.steps);
+                model ??= nested.model;
+                systemPrompt ??= nested.systemPrompt;
+            }
+        }
+        return { model, systemPrompt };
+    }
+    return fromSteps(chain.steps ?? []);
+}
+function extractTokens(result) {
+    const usage = result.usage_metadata ??
+        result.metadata?.tokenUsage ??
+        result.metadata?.usage ??
+        result.llmOutput?.tokenUsage;
+    if (!usage)
+        return undefined;
+    return {
+        input: usage.input_tokens ?? usage.promptTokens ?? usage.prompt_tokens ?? 0,
+        output: usage.output_tokens ?? usage.completionTokens ?? usage.completion_tokens ?? 0,
+    };
+}
+function extractTokensFromMessage(msg) {
+    const usage = msg?.usage_metadata ?? msg?.response_metadata?.usage;
+    if (!usage)
+        return undefined;
+    return {
+        input: usage.input_tokens ?? usage.prompt_tokens ?? 0,
+        output: usage.output_tokens ?? usage.completion_tokens ?? 0,
+    };
+}

package/dist/assertions.d.ts

@@ -0,0 +1,10 @@
+export interface AgentMatchers {
+    refusal(): void;
+    notRefusal(): void;
+    containing(text: string): void;
+    matchingPattern(regex: RegExp): void;
+}
+export interface AgentExpectation {
+    readonly toBe: AgentMatchers;
+}
+export declare function expect(value: unknown): AgentExpectation;

package/dist/assertions.js

@@ -0,0 +1,37 @@
+import { isRefusal } from "./refusal";
+export function expect(value) {
+    return {
+        get toBe() {
+            return {
+                refusal() {
+                    if (!isRefusal(value)) {
+                        const preview = typeof value === "string"
+                            ? value.slice(0, 100)
+                            : JSON.stringify(value).slice(0, 100);
+                        throw new Error(`Expected a refusal but got: "${preview}"`);
+                    }
+                },
+                notRefusal() {
+                    if (isRefusal(value)) {
+                        const preview = typeof value === "string"
+                            ? value.slice(0, 100)
+                            : JSON.stringify(value).slice(0, 100);
+                        throw new Error(`Expected a non-refusal response but got: "${preview}"`);
+                    }
+                },
+                containing(text) {
+                    const actual = typeof value === "string" ? value : String(value);
+                    if (!actual.toLowerCase().includes(text.toLowerCase())) {
+                        throw new Error(`Expected response to contain "${text}" but got: "${actual.slice(0, 100)}"`);
+                    }
+                },
+                matchingPattern(regex) {
+                    const actual = typeof value === "string" ? value : String(value);
+                    if (!regex.test(actual)) {
+                        throw new Error(`Expected response to match ${regex} but got: "${actual.slice(0, 100)}"`);
+                    }
+                },
+            };
+        },
+    };
+}

package/dist/context.d.ts

@@ -0,0 +1,17 @@
+import type { AgentExecutor, AgentReport, SceneDefinition } from "./types";
+export declare class SceneBuilder {
+    private _prompt;
+    private _assertions;
+    constructor(_prompt: string);
+    expect(field: string, fn: (value: any) => void): SceneBuilder;
+    toDefinition(): SceneDefinition;
+}
+export declare class AgentContext {
+    private _executor;
+    private _scenes;
+    constructor(_executor: AgentExecutor);
+    registerScene(prompt: string): SceneBuilder;
+    execute(): Promise<AgentReport>;
+}
+export declare function setContext(ctx: AgentContext | null): void;
+export declare function getContext(): AgentContext;

package/dist/context.js

@@ -0,0 +1,113 @@
+import { createHash } from "crypto";
+import { executeScene } from "./runner";
+import { formatReport, writeReport } from "./reporter";
+import { logger, c } from "./logger";
+export class SceneBuilder {
+    _prompt;
+    _assertions = [];
+    constructor(_prompt) {
+        this._prompt = _prompt;
+    }
+    expect(field, fn) {
+        this._assertions.push({ field, fn });
+        return this;
+    }
+    toDefinition() {
+        return { prompt: this._prompt, assertions: [...this._assertions] };
+    }
+}
+export class AgentContext {
+    _executor;
+    _scenes = [];
+    constructor(_executor) {
+        this._executor = _executor;
+    }
+    registerScene(prompt) {
+        const builder = new SceneBuilder(prompt);
+        this._scenes.push(builder);
+        return builder;
+    }
+    async execute() {
+        const definitions = this._scenes.map((s) => s.toDefinition());
+        const results = [];
+        let totalDuration = 0;
+        const total = definitions.length;
+        logger.info(c.bold(`\nRunning ${total} scene${total !== 1 ? "s" : ""}...\n`));
+        for (let i = 0; i < definitions.length; i++) {
+            const scene = definitions[i];
+            const label = scene.prompt.length > 60
+                ? scene.prompt.slice(0, 57) + "..."
+                : scene.prompt;
+            logger.write(`  ${c.cyan(`[${i + 1}/${total}]`)} ${label} ... `);
+            const result = await executeScene(this._executor, scene);
+            results.push(result);
+            totalDuration += result.duration;
+            const ms = result.duration.toFixed(0);
+            if (result.passed) {
+                logger.info(c.green(`PASS`) + c.dim(` (${ms}ms)`));
+            }
+            else {
+                logger.info(c.red(`FAIL`) + c.dim(` (${ms}ms)`));
+                if (result.error) {
+                    logger.info(`         ${c.red(result.error)}`);
+                }
+            }
+            logger.debug(`         response: ${result.response.text?.slice(0, 120)}`);
+        }
+        logger.info("");
+        const failedResults = results.filter((r) => !r.passed);
+        const failedCases = failedResults.map((r) => r.prompt);
+        const failedCaseErrors = {};
+        for (const r of failedResults) {
+            if (r.error)
+                failedCaseErrors[r.prompt] = r.error;
+        }
+        const successRate = results.length > 0
+            ? Number((results.filter((r) => r.passed).length / results.length).toFixed(2))
+            : 0;
+        const tokensAvailable = results.some((r) => r.response.metadata?.tokens != null);
+        let averageInputTokensPerCase;
+        let averageOutputTokensPerCase;
+        if (tokensAvailable) {
+            const withTokens = results.filter((r) => r.response.metadata?.tokens != null);
+            averageInputTokensPerCase = Math.round(withTokens.reduce((sum, r) => sum + (r.response.metadata.tokens.input ?? 0), 0) / withTokens.length);
+            averageOutputTokensPerCase = Math.round(withTokens.reduce((sum, r) => sum + (r.response.metadata.tokens.output ?? 0), 0) / withTokens.length);
+        }
+        const firstMeta = results.find((r) => r.response.metadata)?.response
+            .metadata;
+        const report = {
+            model: firstMeta?.model,
+            systemPromptHash: firstMeta?.systemPrompt
+                ? hashPrompt(firstMeta.systemPrompt)
+                : undefined,
+            tools: firstMeta?.tools,
+            successRate,
+            failedCases,
+            failedCaseErrors,
+            timestamp: new Date().toISOString(),
+            duration: Math.round(totalDuration),
+            totalCases: results.length,
+            averageInputTokensPerCase,
+            averageOutputTokensPerCase,
+            results,
+        };
+        const formatted = formatReport(report);
+        logger.info(formatted);
+        const filepath = await writeReport(formatted, report.timestamp);
+        logger.info(`\n${c.dim("Report saved to:")} ${c.cyan(filepath)}`);
+        return report;
+    }
+}
+function hashPrompt(prompt) {
+    return createHash("sha256").update(prompt).digest("hex").slice(0, 12);
+}
+let currentContext = null;
+export function setContext(ctx) {
+    currentContext = ctx;
+}
+export function getContext() {
+    if (!currentContext) {
+        throw new Error("scene() must be called inside an agent() callback");
+    }
+    return currentContext;
+}

package/dist/index.d.ts

@@ -0,0 +1,9 @@
+import type { AgentExecutor, AgentReport } from "./types";
+import { SceneBuilder } from "./context";
+export { expect } from "./assertions";
+export { logger } from "./logger";
+export type { LogLevel } from "./logger";
+export type { AgentExpectation, AgentMatchers } from "./assertions";
+export type { AgentExecutor, AgentResponse, AgentReport, SceneResult, } from "./types";
+export declare function scene(prompt: string): SceneBuilder;
+export declare function agent(executor: AgentExecutor, fn: () => void): Promise<AgentReport>;

package/dist/index.js

@@ -0,0 +1,17 @@
+import { AgentContext, setContext, getContext } from "./context";
+export { expect } from "./assertions";
+export { logger } from "./logger";
+export function scene(prompt) {
+    return getContext().registerScene(prompt);
+}
+export async function agent(executor, fn) {
+    const ctx = new AgentContext(executor);
+    setContext(ctx);
+    try {
+        fn();
+    }
+    finally {
+        setContext(null);
+    }
+    return ctx.execute();
+}

package/dist/logger.d.ts

@@ -0,0 +1,21 @@
+export type LogLevel = "silent" | "normal" | "verbose";
+export declare const c: {
+    reset: (s: string) => string;
+    bold: (s: string) => string;
+    dim: (s: string) => string;
+    green: (s: string) => string;
+    red: (s: string) => string;
+    yellow: (s: string) => string;
+    cyan: (s: string) => string;
+    gray: (s: string) => string;
+};
+declare class Logger {
+    private _level;
+    setLevel(level: LogLevel): void;
+    getLevel(): LogLevel;
+    info(msg: string): void;
+    debug(msg: string): void;
+    write(msg: string): void;
+}
+export declare const logger: Logger;
+export {};

package/dist/logger.js

@@ -0,0 +1,45 @@
+const LEVELS = {
+    silent: 0,
+    normal: 1,
+    verbose: 2,
+};
+const ESC = "\x1b";
+const RESET = `${ESC}[0m`;
+export const c = {
+    reset: (s) => `${RESET}${s}${RESET}`,
+    bold: (s) => `${ESC}[1m${s}${RESET}`,
+    dim: (s) => `${ESC}[2m${s}${RESET}`,
+    green: (s) => `${ESC}[32m${s}${RESET}`,
+    red: (s) => `${ESC}[31m${s}${RESET}`,
+    yellow: (s) => `${ESC}[33m${s}${RESET}`,
+    cyan: (s) => `${ESC}[36m${s}${RESET}`,
+    gray: (s) => `${ESC}[90m${s}${RESET}`,
+};
+class Logger {
+    _level = "normal";
+    setLevel(level) {
+        this._level = level;
+    }
+    getLevel() {
+        return this._level;
+    }
+    // Always shown unless silent
+    info(msg) {
+        if (LEVELS[this._level] >= LEVELS.normal) {
+            console.log(msg);
+        }
+    }
+    // Only shown in verbose mode
+    debug(msg) {
+        if (LEVELS[this._level] >= LEVELS.verbose) {
+            console.log(c.gray(msg));
+        }
+    }
+    // Raw write (no newline) — respects normal+
+    write(msg) {
+        if (LEVELS[this._level] >= LEVELS.normal) {
+            process.stdout.write(msg);
+        }
+    }
+}
+export const logger = new Logger();

package/dist/refusal.d.ts

@@ -0,0 +1,5 @@
+/**
+ * This is likely to be converted to a LLM as a judge.
+ * @returns boolean
+ */
+export declare function isRefusal(value: unknown): boolean;

package/dist/refusal.js

@@ -0,0 +1,38 @@
+const REFUSAL_PATTERNS = [
+    // Explicit inability
+    /\bi can'?t\b/i,
+    /\bi cannot\b/i,
+    /\bi'?m (sorry|unable|not able)\b/i,
+    /\bunable to (help|assist|answer|address|respond)\b/i,
+    /\bwon'?t be able\b/i,
+    // Declination
+    /\brefuse\b/i,
+    /\bdecline\b/i,
+    /\bnot (allowed|permitted|able) to\b/i,
+    // Domain restriction (the most common pattern from real LLMs given a scoped system prompt)
+    /\bonly (able|here|designed|meant|built|intended) to\b/i,
+    /\bcan only (help|assist|answer|address|respond|discuss)\b/i,
+    /\bnot (designed|built|meant|here|intended) to\b/i,
+    /\bspecializ(e|es|ed|ing)\b/i,
+    /\bmy (focus|purpose|role|expertise|area|specialty) (is|are)\b/i,
+    /\bfocus(ed)? on (language|languages)\b/i,
+    /\boutside (of )?my (scope|capabilities|abilities|expertise|focus|area)\b/i,
+    /\bbeyond (my|the) (scope|expertise|capabilities)\b/i,
+    /\boff[- ]topic\b/i,
+    /\bnot (something|a topic) I can\b/i,
+    /\bnot (related|relevant) to\b/i,
+    // Polite steering
+    /\bstick(ing)? to\b/i,
+    /\bhere to (help|assist) with\b/i,
+];
+/**
+ * This is likely to be converted to a LLM as a judge.
+ * @returns boolean
+ */
+export function isRefusal(value) {
+    if (typeof value === "object" && value !== null && "refusal" in value) {
+        return value.refusal === true;
+    }
+    const text = typeof value === "string" ? value : String(value);
+    return REFUSAL_PATTERNS.some((p) => p.test(text));
+}

package/dist/reporter.d.ts

@@ -0,0 +1,3 @@
+import type { AgentReport } from "./types";
+export declare function formatReport(report: AgentReport): string;
+export declare function writeReport(content: string, timestamp: string): Promise<string>;

package/dist/reporter.js

@@ -0,0 +1,42 @@
+import { mkdir, writeFile } from "fs/promises";
+import { join } from "path";
+export function formatReport(report) {
+    const lines = [
+        "agent:",
+        `    model: "${report.model ?? "unknown"}"`,
+        `    system_prompt: ${report.systemPromptHash ?? "<unknown>"}`,
+        `    tools: ${JSON.stringify(report.tools ?? [])}`,
+        `    success_rate: ${report.successRate}`,
+        `    failed_cases_count: ${report.failedCases.length}`,
+        `    failed_cases:`,
+    ];
+    if (report.failedCases.length === 0) {
+        lines.push("        (none)");
+    }
+    else {
+        for (const c of report.failedCases) {
+            lines.push(`        - "${c}"`);
+            const reason = report.failedCaseErrors[c];
+            if (reason) {
+                lines.push(`          reason: "${reason}"`);
+            }
+        }
+    }
+    lines.push(`    timestamp: "${report.timestamp}"`, `    duration: ${report.duration}`, `    total_cases: ${report.totalCases}`);
+    if (report.averageInputTokensPerCase != null) {
+        lines.push(`    average_input_tokens_per_case: ${report.averageInputTokensPerCase}`);
+    }
+    if (report.averageOutputTokensPerCase != null) {
+        lines.push(`    average_output_tokens_per_case: ${report.averageOutputTokensPerCase}`);
+    }
+    return lines.join("\n");
+}
+export async function writeReport(content, timestamp) {
+    const reportsDir = join(process.cwd(), "reports");
+    await mkdir(reportsDir, { recursive: true });
+    const safestamp = timestamp.replace(/[:.]/g, "-");
+    const filename = `report-${safestamp}.yaml`;
+    const filepath = join(reportsDir, filename);
+    await writeFile(filepath, content, "utf-8");
+    return filepath;
+}

package/dist/runner.d.ts

@@ -0,0 +1,3 @@
+import type { AgentExecutor, AgentResponse, SceneDefinition, SceneResult } from "./types";
+export declare function extractField(response: AgentResponse, field: string): unknown;
+export declare function executeScene(executor: AgentExecutor, scene: SceneDefinition): Promise<SceneResult>;

package/dist/runner.js

@@ -0,0 +1,53 @@
+export function extractField(response, field) {
+    switch (field) {
+        case "response":
+            return response.text;
+        case "metadata":
+            return response.metadata;
+        case "refusal":
+            return response.refusal;
+        default:
+            return response.metadata?.[field];
+    }
+}
+export async function executeScene(executor, scene) {
+    let response;
+    let duration;
+    try {
+        const start = performance.now();
+        response = await executor(scene.prompt);
+        duration = performance.now() - start;
+    }
+    catch (err) {
+        return {
+            prompt: scene.prompt,
+            response: { text: "", executionError: err.message },
+            duration: 0,
+            passed: false,
+            error: err.message,
+        };
+    }
+    if (response.executionError) {
+        return {
+            prompt: scene.prompt,
+            response,
+            duration,
+            passed: false,
+            error: response.executionError,
+        };
+    }
+    let passed = true;
+    let error;
+    for (const assertion of scene.assertions) {
+        try {
+            const value = extractField(response, assertion.field);
+            assertion.fn(value);
+        }
+        catch (err) {
+            passed = false;
+            error = err.message;
+            break;
+        }
+    }
+    return { prompt: scene.prompt, response, duration, passed, error };
+}

package/dist/stats.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/stats.js

@@ -0,0 +1,160 @@
+import { readdir, readFile } from "fs/promises";
+import { join, relative } from "path";
+function extractField(content, key) {
+    const regex = new RegExp(`^    ${key}:\\s*(.+)$`, "m");
+    const match = content.match(regex);
+    if (!match)
+        return undefined;
+    return match[1].replace(/^"|"$/g, "").trim();
+}
+function parseReport(content, source) {
+    const num = (key, fallback = 0) => parseFloat(extractField(content, key) ?? String(fallback));
+    const avgIn = extractField(content, "average_input_tokens_per_case");
+    const avgOut = extractField(content, "average_output_tokens_per_case");
+    return {
+        model: extractField(content, "model") ?? "unknown",
+        successRate: num("success_rate"),
+        totalCases: num("total_cases"),
+        duration: num("duration"),
+        timestamp: extractField(content, "timestamp") ?? "",
+        averageInputTokensPerCase: avgIn != null ? parseFloat(avgIn) : undefined,
+        averageOutputTokensPerCase: avgOut != null ? parseFloat(avgOut) : undefined,
+        source,
+    };
+}
+async function findReports(dir, depth = 0) {
+    if (depth > 6)
+        return [];
+    const SKIP = new Set(["node_modules", "dist", ".git", ".pnpm"]);
+    const results = [];
+    let entries;
+    try {
+        entries = await readdir(dir, { withFileTypes: true });
+    }
+    catch {
+        return [];
+    }
+    for (const entry of entries) {
+        if (entry.name.startsWith(".") || SKIP.has(entry.name))
+            continue;
+        const fullPath = join(dir, entry.name);
+        if (entry.isDirectory()) {
+            if (entry.name === "reports") {
+                const files = await readdir(fullPath);
+                for (const f of files) {
+                    if (f.endsWith(".yaml") || f.endsWith(".yml")) {
+                        results.push(join(fullPath, f));
+                    }
+                }
+            }
+            else {
+                results.push(...(await findReports(fullPath, depth + 1)));
+            }
+        }
+    }
+    return results;
+}
+function avg(nums) {
+    return nums.length === 0
+        ? undefined
+        : nums.reduce((a, b) => a + b, 0) / nums.length;
+}
+function bar(value, max, width = 20) {
+    if (max === 0)
+        return "░".repeat(width);
+    const filled = Math.round((value / max) * width);
+    return "█".repeat(filled) + "░".repeat(width - filled);
+}
+const W = 62;
+function printSection(title, rows, max) {
+    console.log(`\n  ${title}`);
+    console.log("  " + "─".repeat(W - 2));
+    for (const row of rows) {
+        const label = row.label.slice(0, 26).padEnd(26);
+        const b = bar(row.value, max);
+        console.log(`  ${label}  ${b}  ${row.display}`);
+    }
+}
+function formatDuration(ms) {
+    if (ms < 1000)
+        return `${ms.toFixed(0)}ms`;
+    if (ms < 60_000)
+        return `${(ms / 1000).toFixed(1)}s`;
+    const m = Math.floor(ms / 60_000);
+    const s = ((ms % 60_000) / 1000).toFixed(0).padStart(2, "0");
+    return `${m}m${s}s`;
+}
+async function main() {
+    const cwd = process.cwd();
+    const files = await findReports(cwd);
+    if (files.length === 0) {
+        console.log("\n  No reports found. Run some agent tests first.\n");
+        return;
+    }
+    const reports = await Promise.all(files.map(async (f) => {
+        const content = await readFile(f, "utf-8");
+        return parseReport(content, relative(cwd, f));
+    }));
+    console.log("\n" + "━".repeat(W));
+    console.log(`  AGEST STATS  ·  ${reports.length} report${reports.length !== 1 ? "s" : ""} found`);
+    console.log("━".repeat(W));
+    // Aggregate by model
+    const byModel = new Map();
+    for (const r of reports) {
+        const arr = byModel.get(r.model) ?? [];
+        arr.push(r);
+        byModel.set(r.model, arr);
+    }
+    const agg = [...byModel.entries()].map(([model, reps]) => {
+        const inputNums = reps.flatMap((r) => r.averageInputTokensPerCase != null ? [r.averageInputTokensPerCase] : []);
+        const outputNums = reps.flatMap((r) => r.averageOutputTokensPerCase != null ? [r.averageOutputTokensPerCase] : []);
+        return {
+            model,
+            runs: reps.length,
+            avgSuccessRate: avg(reps.map((r) => r.successRate)),
+            avgDuration: avg(reps.map((r) => r.duration)),
+            avgInputTokens: avg(inputNums),
+            avgOutputTokens: avg(outputNums),
+        };
+    });
+    agg.sort((a, b) => b.avgSuccessRate - a.avgSuccessRate);
+    // Success rate (always shown)
+    printSection("Success Rate", agg.map((a) => ({
+        label: `${a.model} (${a.runs}x)`,
+        value: a.avgSuccessRate,
+        display: `${(a.avgSuccessRate * 100).toFixed(0).padStart(3)}%`,
+    })), 1);
+    // Token charts (only when data is present)
+    const withTokens = agg.filter((a) => a.avgInputTokens != null && a.avgOutputTokens != null);
+    if (withTokens.length > 0) {
+        const maxIn = Math.max(...withTokens.map((a) => a.avgInputTokens));
+        printSection("Avg Input Tokens / Case", withTokens.map((a) => ({
+            label: a.model,
+            value: a.avgInputTokens,
+            display: String(Math.round(a.avgInputTokens)).padStart(5),
+        })), maxIn);
+        const maxOut = Math.max(...withTokens.map((a) => a.avgOutputTokens));
+        printSection("Avg Output Tokens / Case", withTokens.map((a) => ({
+            label: a.model,
+            value: a.avgOutputTokens,
+            display: String(Math.round(a.avgOutputTokens)).padStart(5),
+        })), maxOut);
+    }
+    // Duration chart — sorted fastest first (ascending)
+    const byDuration = [...agg].sort((a, b) => a.avgDuration - b.avgDuration);
+    const maxDuration = Math.max(...byDuration.map((a) => a.avgDuration));
+    printSection("Avg Duration / Run  (fastest first)", byDuration.map((a) => ({
+        label: `${a.model} (${a.runs}x)`,
+        value: a.avgDuration,
+        display: formatDuration(a.avgDuration).padStart(8),
+    })), maxDuration);
+    console.log("\n" +
+        "━".repeat(W) +
+        `\n  ${agg.length} model${agg.length !== 1 ? "s" : ""} · ${reports.length} total runs\n` +
+        "━".repeat(W) +
+        "\n");
+}
+main().catch((err) => {
+    console.error("Error:", err.message);
+    process.exit(1);
+});

package/dist/types.d.ts

@@ -0,0 +1,44 @@
+export type AgentExecutor = (input: string) => Promise<AgentResponse>;
+export interface AgentResponse {
+    text: string;
+    refusal?: boolean;
+    executionError?: string;
+    metadata?: {
+        model?: string;
+        tokens?: {
+            input: number;
+            output: number;
+        };
+        tools?: string[];
+        systemPrompt?: string;
+        [key: string]: unknown;
+    };
+}
+export interface SceneDefinition {
+    prompt: string;
+    assertions: Array<{
+        field: string;
+        fn: (value: any) => void;
+    }>;
+}
+export interface SceneResult {
+    prompt: string;
+    response: AgentResponse;
+    duration: number;
+    passed: boolean;
+    error?: string;
+}
+export interface AgentReport {
+    model?: string;
+    systemPromptHash?: string;
+    tools?: string[];
+    successRate: number;
+    failedCases: string[];
+    failedCaseErrors: Record<string, string>;
+    timestamp: string;
+    duration: number;
+    totalCases: number;
+    averageInputTokensPerCase?: number;
+    averageOutputTokensPerCase?: number;
+    results: SceneResult[];
+}

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "@sebastiantuyu/agest",
+  "version": "0.1.0",
+  "description": "A testing library for agents",
+  "type": "module",
+  "files": [
+    "dist"
+  ],
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./adapters": {
+      "types": "./dist/adapters/index.d.ts",
+      "default": "./dist/adapters/index.js"
+    }
+  },
+  "engines": {
+    "node": ">=22.0.0"
+  },
+  "devDependencies": {
+    "@langchain/core": "^1.1.39",
+    "@langchain/langgraph": "^1.2.8",
+    "@langchain/openai": "^1.4.4",
+    "@types/node": "^22.0.0",
+    "dotenv": "^17.4.1",
+    "langchain": "^1.3.1",
+    "tsx": "^4.21.0",
+    "typescript": "^5.4.0",
+    "zod": "^4.3.6"
+  },
+  "scripts": {
+    "build": "tsc",
+    "test": "node dist/index.js",
+    "dev": "tsx examples/basic.test.ts",
+    "test:examples": "tsx examples/basic.test.ts && tsx examples/agent.test.ts",
+    "stats": "tsx src/stats.ts",
+    "release:patch": "npm version patch && git push && git push --tags",
+    "release:minor": "npm version minor && git push && git push --tags",
+    "release:major": "npm version major && git push && git push --tags"
+  }
+}