@sebastiantuyu/agest 0.2.1 → 0.2.2

package/README.md

@@ -110,12 +110,28 @@ npx tsx examples/openrouter.test.ts
 
 ## Roadmap
 
-- [ ] Multi-run support: `.runs(n)` per scene for statistical significance
-- [ ] Suite-level runs: `agent(exec, { runs: 3 }, () => { ... })` for overall stability benchmarks
-- [ ] Additional matchers: `toBe.semanticallySimilarTo(text, threshold)`, `toBe.matchingSchema(zodSchema)`
-- [ ] JSON/file reporters for persisting reports to disk
-- [ ] Snapshot comparison: diff reports across runs to track agent regression
-- [ ] More adapters: Vercel AI SDK, OpenAI Agents SDK, raw API calls
+### Shipped
+- [x] Multi-turn support: `.turns(n)` per scene
+- [x] LLM-as-judge: `.judgedBy({ criteria, failWhen })`
+- [x] Remote HTTP adapter for framework-agnostic testing
+- [x] Report persistence to `.reports/` with YAML format
+- [x] Stats CLI with multi-model comparison and dimension analysis
+
+### Up next
+- [ ] Schema validation: `toBe.matchingSchema(zodSchema)`
+- [ ] Semantic similarity: `toBe.semanticallySimilarTo(text, threshold)`
+- [ ] Statistical runs: `.runs(n)` per scene with mean/stddev reporting
+- [ ] Vercel AI SDK adapter
+- [ ] Snapshot regression: diff current run against a saved baseline
+
+### Planned
+- [ ] Cost estimation per scene (token count to dollar cost)
+- [ ] CI/CD reporter (GitHub Actions PR comments)
+- [ ] Tool-call trajectory assertions
+- [ ] Watch mode for TDD-style iteration
+- [ ] OpenAI Agents SDK adapter
+- [ ] Webhook/n8n adapter for no-code agent sources
+- [ ] Jest/Vitest custom matcher export
 
 ## Development requirements
 - Node 22+

package/dist/adapters/remote.d.ts

@@ -4,6 +4,12 @@ export interface RemoteAdapterOptions {
     headers?: Record<string, string>;
     /** HTTP method, defaults to POST */
     method?: "POST" | "PUT" | "GET";
+    /**
+     * Extra fields merged into the request body.
+     * Merged *under* the output of `buildRequest`, so `buildRequest` wins on conflicts.
+     * Ignored when method is GET.
+     */
+    body?: Record<string, unknown>;
     /**
      * Build the request body from the input prompt.
      * Defaults to `{ prompt: input }`.

package/dist/adapters/remote.js

@@ -23,7 +23,7 @@
  * ```
  */
 export function remote(endpoint, options = {}) {
-    const { headers = {}, method = "POST", buildRequest = defaultBuildRequest, parseResponse, metadata: staticMetadata, } = options;
+    const { headers = {}, method = "POST", body: extraBody, buildRequest = defaultBuildRequest, parseResponse, metadata: staticMetadata, } = options;
     return async (input) => {
         let res;
         try {
@@ -32,7 +32,13 @@ export function remote(endpoint, options = {}) {
                 headers: { "Content-Type": "application/json", ...headers },
             };
             if (method !== "GET") {
-                fetchOptions.body = JSON.stringify(buildRequest(input));
+                const built = buildRequest(input);
+                const merged = extraBody && typeof built === "object" && built !== null
+                    ? { ...extraBody, ...built }
+                    : extraBody && typeof built !== "object"
+                        ? { ...extraBody, prompt: built }
+                        : built;
+                fetchOptions.body = JSON.stringify(merged);
             }
             res = await fetch(endpoint, fetchOptions);
         }

package/dist/index.d.ts

@@ -12,4 +12,6 @@ export interface AgentOptions {
     name?: string;
 }
 export declare function scene(prompt: string): SceneBuilder;
+/** @internal reset auto-run state between tests */
+export declare function _resetAutoRun(): void;
 export declare function agent(executor: AgentExecutor, fn: () => void, options?: AgentOptions): Promise<AgentReport>;

package/dist/index.js

@@ -5,14 +5,36 @@ export { defineConfig } from "./config";
 export function scene(prompt) {
     return getContext().registerScene(prompt);
 }
-export async function agent(executor, fn, options) {
+const pendingAgents = [];
+let autoRunScheduled = false;
+/** @internal reset auto-run state between tests */
+export function _resetAutoRun() {
+    pendingAgents.length = 0;
+    autoRunScheduled = false;
+}
+export function agent(executor, fn, options) {
     const ctx = new AgentContext(executor, options?.name);
     setContext(ctx);
     try {
         fn();
     }
-    finally {
+    catch (err) {
         setContext(null);
+        return Promise.reject(err);
+    }
+    setContext(null);
+    const promise = ctx.execute();
+    pendingAgents.push(promise);
+    if (!autoRunScheduled) {
+        autoRunScheduled = true;
+        process.nextTick(async () => {
+            try {
+                await Promise.all(pendingAgents);
+            }
+            catch {
+                process.exitCode = 1;
+            }
+        });
     }
-    return ctx.execute();
+    return promise;
 }

package/dist/runner.js

@@ -22,9 +22,12 @@ export async function executeScene(executor, scene, globalTimeout, judgeConfig,
         const start = performance.now();
         let input = scene.prompt;
         for (let t = 0; t < turns; t++) {
+            let timer;
             response = await Promise.race([
-                executor(input),
-                new Promise((_, reject) => setTimeout(() => reject(new Error(`Scene timed out after ${timeoutMs}ms`)), timeoutMs)),
+                executor(input).finally(() => clearTimeout(timer)),
+                new Promise((_, reject) => {
+                    timer = setTimeout(() => reject(new Error(`Scene timed out after ${timeoutMs}ms`)), timeoutMs);
+                }),
             ]);
             if (response.executionError)
                 break;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sebastiantuyu/agest",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "description": "A testing library for agents",
   "repository": {
     "type": "git",