@sebastiantuyu/agest 0.2.1 → 0.3.0

package/README.md

@@ -110,12 +110,28 @@ npx tsx examples/openrouter.test.ts
 
 ## Roadmap
 
-- [ ] Multi-run support: `.runs(n)` per scene for statistical significance
-- [ ] Suite-level runs: `agent(exec, { runs: 3 }, () => { ... })` for overall stability benchmarks
-- [ ] Additional matchers: `toBe.semanticallySimilarTo(text, threshold)`, `toBe.matchingSchema(zodSchema)`
-- [ ] JSON/file reporters for persisting reports to disk
-- [ ] Snapshot comparison: diff reports across runs to track agent regression
-- [ ] More adapters: Vercel AI SDK, OpenAI Agents SDK, raw API calls
+### Shipped
+- [x] Multi-turn support: `.turns(n)` per scene
+- [x] LLM-as-judge: `.judgedBy({ criteria, failWhen })`
+- [x] Remote HTTP adapter for framework-agnostic testing
+- [x] Report persistence to `.reports/` with YAML format
+- [x] Stats CLI with multi-model comparison and dimension analysis
+
+### Up next
+- [ ] Schema validation: `toBe.matchingSchema(zodSchema)`
+- [ ] Semantic similarity: `toBe.semanticallySimilarTo(text, threshold)`
+- [ ] Statistical runs: `.runs(n)` per scene with mean/stddev reporting
+- [ ] Vercel AI SDK adapter
+- [ ] Snapshot regression: diff current run against a saved baseline
+
+### Planned
+- [ ] Cost estimation per scene (token count to dollar cost)
+- [ ] CI/CD reporter (GitHub Actions PR comments)
+- [ ] Tool-call trajectory assertions
+- [ ] Watch mode for TDD-style iteration
+- [ ] OpenAI Agents SDK adapter
+- [ ] Webhook/n8n adapter for no-code agent sources
+- [ ] Jest/Vitest custom matcher export
 
 ## Development requirements
 - Node 22+

package/dist/adapters/remote.d.ts

@@ -4,6 +4,12 @@ export interface RemoteAdapterOptions {
     headers?: Record<string, string>;
     /** HTTP method, defaults to POST */
     method?: "POST" | "PUT" | "GET";
+    /**
+     * Extra fields merged into the request body.
+     * Merged *under* the output of `buildRequest`, so `buildRequest` wins on conflicts.
+     * Ignored when method is GET.
+     */
+    body?: Record<string, unknown>;
     /**
      * Build the request body from the input prompt.
      * Defaults to `{ prompt: input }`.
@@ -14,7 +20,7 @@ export interface RemoteAdapterOptions {
      * When omitted the adapter tries common shapes:
      * - `{ text }` / `{ response }` / `{ output }` / `{ message }` / plain string
      */
-    parseResponse?: (body: unknown) => AgentResponse;
+    parseResponse?: <TBody = unknown>(body: TBody) => AgentResponse;
     /**
      * Static metadata for this remote agent.
      * Because the remote endpoint is opaque, metadata like model name,

package/dist/adapters/remote.js

@@ -23,7 +23,7 @@
  * ```
  */
 export function remote(endpoint, options = {}) {
-    const { headers = {}, method = "POST", buildRequest = defaultBuildRequest, parseResponse, metadata: staticMetadata, } = options;
+    const { headers = {}, method = "POST", body: extraBody, buildRequest = defaultBuildRequest, parseResponse, metadata: staticMetadata, } = options;
     return async (input) => {
         let res;
         try {
@@ -32,7 +32,13 @@ export function remote(endpoint, options = {}) {
                 headers: { "Content-Type": "application/json", ...headers },
             };
             if (method !== "GET") {
-                fetchOptions.body = JSON.stringify(buildRequest(input));
+                const built = buildRequest(input);
+                const merged = extraBody && typeof built === "object" && built !== null
+                    ? { ...extraBody, ...built }
+                    : extraBody && typeof built !== "object"
+                        ? { ...extraBody, prompt: built }
+                        : built;
+                fetchOptions.body = JSON.stringify(merged);
             }
             res = await fetch(endpoint, fetchOptions);
         }

package/dist/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/cli.js

@@ -0,0 +1,43 @@
+#!/usr/bin/env node
+import { spawn } from "child_process";
+import { main as stats } from "./stats.js";
+import { main as preview } from "./preview.js";
+const command = process.argv[2];
+async function run() {
+    const files = process.argv.slice(3);
+    if (files.length === 0) {
+        console.error("  Usage: agest run <file...>");
+        process.exit(1);
+    }
+    for (const file of files) {
+        const child = spawn("npx", ["tsx", file], {
+            stdio: "inherit",
+            shell: true,
+        });
+        const code = await new Promise((resolve) => child.on("close", (c) => resolve(c ?? 1)));
+        if (code !== 0)
+            process.exit(code);
+    }
+}
+const commands = {
+    stats,
+    preview,
+    run,
+};
+if (!command || !commands[command]) {
+    console.log(`
+  Usage: agest <command>
+
+  Commands:
+    run        Run test file(s)    agest run tests/*.test.ts
+    stats      Show aggregated test statistics
+    preview    Generate an HTML report preview
+`);
+    process.exit(command ? 1 : 0);
+}
+// Forward remaining args so subcommands see them at process.argv[2+]
+process.argv = [process.argv[0], process.argv[1], ...process.argv.slice(3)];
+commands[command]().catch((err) => {
+    console.error("Error:", err.message);
+    process.exit(1);
+});

package/dist/index.d.ts

@@ -12,4 +12,6 @@ export interface AgentOptions {
     name?: string;
 }
 export declare function scene(prompt: string): SceneBuilder;
+/** @internal reset auto-run state between tests */
+export declare function _resetAutoRun(): void;
 export declare function agent(executor: AgentExecutor, fn: () => void, options?: AgentOptions): Promise<AgentReport>;

package/dist/index.js

@@ -5,14 +5,36 @@ export { defineConfig } from "./config";
 export function scene(prompt) {
     return getContext().registerScene(prompt);
 }
-export async function agent(executor, fn, options) {
+const pendingAgents = [];
+let autoRunScheduled = false;
+/** @internal reset auto-run state between tests */
+export function _resetAutoRun() {
+    pendingAgents.length = 0;
+    autoRunScheduled = false;
+}
+export function agent(executor, fn, options) {
     const ctx = new AgentContext(executor, options?.name);
     setContext(ctx);
     try {
         fn();
     }
-    finally {
+    catch (err) {
         setContext(null);
+        return Promise.reject(err);
+    }
+    setContext(null);
+    const promise = ctx.execute();
+    pendingAgents.push(promise);
+    if (!autoRunScheduled) {
+        autoRunScheduled = true;
+        process.nextTick(async () => {
+            try {
+                await Promise.all(pendingAgents);
+            }
+            catch {
+                process.exitCode = 1;
+            }
+        });
     }
-    return ctx.execute();
+    return promise;
 }

package/dist/preview.d.ts

@@ -1 +1,2 @@
-export {};
+declare function main(): Promise<void>;
+export { main };

package/dist/preview.js

@@ -771,7 +771,4 @@ async function main() {
     console.log(`\n  Preview: ${tmpPath}\n`);
     openBrowser(tmpPath);
 }
-main().catch((err) => {
-    console.error("Error:", err.message);
-    process.exit(1);
-});
+export { main };

package/dist/runner.js

@@ -22,9 +22,12 @@ export async function executeScene(executor, scene, globalTimeout, judgeConfig,
         const start = performance.now();
         let input = scene.prompt;
         for (let t = 0; t < turns; t++) {
+            let timer;
             response = await Promise.race([
-                executor(input),
-                new Promise((_, reject) => setTimeout(() => reject(new Error(`Scene timed out after ${timeoutMs}ms`)), timeoutMs)),
+                executor(input).finally(() => clearTimeout(timer)),
+                new Promise((_, reject) => {
+                    timer = setTimeout(() => reject(new Error(`Scene timed out after ${timeoutMs}ms`)), timeoutMs);
+                }),
             ]);
             if (response.executionError)
                 break;

package/dist/stats.d.ts

@@ -1 +1,2 @@
-export {};
+declare function main(): Promise<void>;
+export { main };

package/dist/stats.js

@@ -311,7 +311,4 @@ async function main() {
         "━".repeat(W) +
         "\n");
 }
-main().catch((err) => {
-    console.error("Error:", err.message);
-    process.exit(1);
-});
+export { main };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sebastiantuyu/agest",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "description": "A testing library for agents",
   "repository": {
     "type": "git",
@@ -10,6 +10,9 @@
   "files": [
     "dist"
   ],
+  "bin": {
+    "agest": "dist/cli.js"
+  },
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "exports": {
@@ -29,8 +32,8 @@
     "test:coverage": "vitest run --coverage",
     "dev": "tsx examples/basic.test.ts",
     "test:examples": "tsx examples/basic.test.ts && tsx examples/agent.test.ts",
-    "stats": "tsx src/stats.ts",
-    "preview": "tsx src/preview.ts",
+    "stats": "tsx src/cli.ts stats",
+    "preview": "tsx src/cli.ts preview",
     "site:preview": "npx serve site -p 3000",
     "release:patch": "npm version patch && git push && git push --tags",
     "release:minor": "npm version minor && git push && git push --tags",