@sebastiantuyu/agest 0.1.6 → 0.2.1

package/dist/stats.js

@@ -1,59 +1,6 @@
-import { readdir, readFile } from "fs/promises";
+import { readdir, readFile, rm } from "fs/promises";
 import { join, relative } from "path";
-function extractField(content, key) {
-    const regex = new RegExp(`^    ${key}:\\s*(.+)$`, "m");
-    const match = content.match(regex);
-    if (!match)
-        return undefined;
-    return match[1].replace(/^"|"$/g, "").trim();
-}
-function parseReport(content, source) {
-    const num = (key, fallback = 0) => parseFloat(extractField(content, key) ?? String(fallback));
-    const avgIn = extractField(content, "average_input_tokens_per_case");
-    const avgOut = extractField(content, "average_output_tokens_per_case");
-    return {
-        model: extractField(content, "model") ?? "unknown",
-        successRate: num("success_rate"),
-        totalCases: num("total_cases"),
-        duration: num("duration"),
-        timestamp: extractField(content, "timestamp") ?? "",
-        averageInputTokensPerCase: avgIn != null ? parseFloat(avgIn) : undefined,
-        averageOutputTokensPerCase: avgOut != null ? parseFloat(avgOut) : undefined,
-        source,
-    };
-}
-async function findReports(dir, depth = 0) {
-    if (depth > 6)
-        return [];
-    const SKIP = new Set(["node_modules", "dist", ".git", ".pnpm"]);
-    const results = [];
-    let entries;
-    try {
-        entries = await readdir(dir, { withFileTypes: true });
-    }
-    catch {
-        return [];
-    }
-    for (const entry of entries) {
-        if (entry.name.startsWith(".") || SKIP.has(entry.name))
-            continue;
-        const fullPath = join(dir, entry.name);
-        if (entry.isDirectory()) {
-            if (entry.name === "reports") {
-                const files = await readdir(fullPath);
-                for (const f of files) {
-                    if (f.endsWith(".yaml") || f.endsWith(".yml")) {
-                        results.push(join(fullPath, f));
-                    }
-                }
-            }
-            else {
-                results.push(...(await findReports(fullPath, depth + 1)));
-            }
-        }
-    }
-    return results;
-}
+import { parseReport, findReports, loadDiffEntry, computeDiff, formatDuration, ensureDimensions, findVaryingDimensions, groupByDimension, findControlledPairs, diffConfigs, } from "./reports.js";
 function avg(nums) {
     return nums.length === 0
         ? undefined
@@ -75,28 +22,207 @@ function printSection(title, rows, max) {
         console.log(`  ${label}  ${b}  ${row.display}`);
     }
 }
-function formatDuration(ms) {
-    if (ms < 1000)
-        return `${ms.toFixed(0)}ms`;
-    if (ms < 60_000)
-        return `${(ms / 1000).toFixed(1)}s`;
-    const m = Math.floor(ms / 60_000);
-    const s = ((ms % 60_000) / 1000).toFixed(0).padStart(2, "0");
-    return `${m}m${s}s`;
+function formatDelta(prev, curr) {
+    const d = (curr - prev) * 100;
+    if (Math.abs(d) < 0.5)
+        return "  =  ";
+    const sign = d > 0 ? "+" : "";
+    return `${sign}${d.toFixed(0)}%`.padStart(5);
+}
+function shortDimLabel(dim, val, maxLen = 20) {
+    const short = val.length > maxLen ? val.slice(0, maxLen - 1) + "…" : val;
+    return `${dim}:${short}`;
+}
+// ---------------------------------------------------------------------------
+// Per-dimension evolution: group by held dims, show evolution along varied
+// ---------------------------------------------------------------------------
+async function printDimensionEvolution(name, runs, primaryDim, varyingDims) {
+    const groups = groupByDimension(runs, primaryDim);
+    for (const [groupVal, groupRuns] of groups) {
+        const sorted = [...groupRuns].sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime());
+        console.log(`\n  ${name} / ${primaryDim}: ${groupVal}`);
+        console.log("  " + "─".repeat(W - 2));
+        for (let i = 0; i < sorted.length; i++) {
+            const r = sorted[i];
+            const runNum = `#${i + 1}`.padStart(3);
+            const pct = `${(r.successRate * 100).toFixed(0).padStart(3)}%`;
+            const b = bar(r.successRate, 1, 16);
+            const delta = i === 0 ? "     " : formatDelta(sorted[i - 1].successRate, r.successRate);
+            // Show other varying dimensions for this run
+            const otherDims = varyingDims
+                .filter((d) => d !== primaryDim)
+                .map((d) => shortDimLabel(d, r.dimensions?.[d] ?? "?", 12))
+                .join("  ");
+            console.log(`  ${runNum}  ${b}  ${pct}  ${delta}  ${otherDims}`);
+            // Show what changed from previous run (within this group)
+            if (i > 0) {
+                const prev = sorted[i - 1];
+                const diff = diffConfigs(prev.dimensions ?? {}, r.dimensions ?? {});
+                const changedLabels = Object.entries(diff.varied)
+                    .filter(([k]) => k !== primaryDim)
+                    .map(([k, v]) => `${k}: ${v.from} → ${v.to}`)
+                    .slice(0, 3);
+                for (const l of changedLabels) {
+                    console.log(`        ${l}`);
+                }
+                // Show prompt diff if prompt changed
+                if (diff.varied["prompt"] && prev.systemPromptHash && r.systemPromptHash) {
+                    const [prevEntry, currEntry] = await Promise.all([
+                        loadDiffEntry(prev.systemPromptHash),
+                        loadDiffEntry(r.systemPromptHash),
+                    ]);
+                    if (prevEntry && currEntry) {
+                        const promptDiff = computeDiff(prevEntry, currEntry)
+                            .filter((l) => l.startsWith("prompt:") || l.startsWith("tools:"));
+                        for (const l of promptDiff)
+                            console.log(`        ${l}`);
+                    }
+                }
+            }
+        }
+    }
+}
+// ---------------------------------------------------------------------------
+// Cross-dimension comparison
+// ---------------------------------------------------------------------------
+function printCrossComparison(name, runs, dim) {
+    // Find configs that appear across multiple values of `dim`
+    const otherDims = Object.keys(runs[0]?.dimensions ?? {}).filter((d) => d !== dim);
+    const configKey = (r) => otherDims.map((d) => `${d}:${r.dimensions?.[d] ?? "?"}`).join("|");
+    const byConfig = new Map();
+    for (const r of runs) {
+        const ck = configKey(r);
+        const inner = byConfig.get(ck) ?? new Map();
+        const dimVal = r.dimensions?.[dim] ?? "?";
+        // Keep the latest run per (config, dimValue) pair
+        const existing = inner.get(dimVal);
+        if (!existing || new Date(r.timestamp) > new Date(existing.timestamp)) {
+            inner.set(dimVal, r);
+        }
+        byConfig.set(ck, inner);
+    }
+    // Only show configs tested across 2+ values of the dimension
+    const comparable = [...byConfig.entries()].filter(([, m]) => m.size > 1);
+    if (comparable.length === 0)
+        return;
+    console.log(`\n  Cross-${dim} comparison: ${name}`);
+    console.log("  " + "─".repeat(W - 2));
+    for (const [ck, dimMap] of comparable) {
+        const configLabel = ck
+            .split("|")
+            .map((s) => s.length > 30 ? s.slice(0, 29) + "…" : s)
+            .join(" + ");
+        console.log(`  ${configLabel}`);
+        const entries = [...dimMap.entries()].sort((a, b) => b[1].successRate - a[1].successRate);
+        const best = entries[0][1].successRate;
+        for (const [dimVal, r] of entries) {
+            const pct = `${(r.successRate * 100).toFixed(0)}%`.padStart(4);
+            const label = dimVal.length > 28 ? dimVal.slice(0, 27) + "…" : dimVal;
+            const vs = r.successRate === best ? "" : `  (${((r.successRate - best) * 100).toFixed(0)}%)`;
+            console.log(`    ${label.padEnd(30)} ${pct}${vs}`);
+        }
+    }
+}
+// ---------------------------------------------------------------------------
+// Attribution summary
+// ---------------------------------------------------------------------------
+function printAttribution(name, runs) {
+    const pairs = findControlledPairs(runs);
+    if (pairs.length === 0)
+        return;
+    // Group by dimension
+    const byDim = new Map();
+    for (const p of pairs) {
+        const entry = byDim.get(p.variedDimension) ?? { deltas: [], examples: [] };
+        entry.deltas.push(p.delta);
+        if (entry.examples.length < 2) {
+            const d = (p.delta * 100).toFixed(0);
+            const sign = p.delta > 0 ? "+" : "";
+            entry.examples.push(`${p.variedFrom} → ${p.variedTo}: ${sign}${d}%`);
+        }
+        byDim.set(p.variedDimension, entry);
+    }
+    console.log(`\n  Dimension Impact: ${name}`);
+    console.log("  " + "─".repeat(W - 2));
+    const sorted = [...byDim.entries()].sort((a, b) => Math.max(...b[1].deltas.map(Math.abs)) - Math.max(...a[1].deltas.map(Math.abs)));
+    for (const [dim, { deltas, examples }] of sorted) {
+        const avgDelta = deltas.reduce((a, b) => a + b, 0) / deltas.length;
+        const sign = avgDelta > 0 ? "+" : "";
+        const avgStr = `${sign}${(avgDelta * 100).toFixed(0)}%`;
+        console.log(`  ${dim.padEnd(12)} ${avgStr.padStart(6)} avg  (${deltas.length} comparison${deltas.length !== 1 ? "s" : ""})`);
+        for (const ex of examples) {
+            console.log(`    ${ex}`);
+        }
+    }
+}
+// ---------------------------------------------------------------------------
+// Purge
+// ---------------------------------------------------------------------------
+async function purge(cwd) {
+    const SKIP = new Set(["node_modules", "dist", ".git", ".pnpm"]);
+    let count = 0;
+    async function walk(dir, depth = 0) {
+        if (depth > 6)
+            return;
+        let entries;
+        try {
+            entries = await readdir(dir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const entry of entries) {
+            if (SKIP.has(entry.name))
+                continue;
+            const fullPath = join(dir, entry.name);
+            if (entry.isDirectory()) {
+                if (entry.name === ".reports" || entry.name === ".diff") {
+                    await rm(fullPath, { recursive: true, force: true });
+                    console.log(`  removed  ${relative(cwd, fullPath)}/`);
+                    count++;
+                }
+                else if (!entry.name.startsWith(".")) {
+                    await walk(fullPath, depth + 1);
+                }
+            }
+        }
+    }
+    await walk(cwd);
+    console.log(`\n  Purged ${count} director${count !== 1 ? "ies" : "y"}.\n`);
 }
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
 async function main() {
+    const args = process.argv.slice(2);
+    const agentFlagIdx = args.indexOf("--agent");
+    const agentFilter = agentFlagIdx !== -1 ? args[agentFlagIdx + 1] : undefined;
+    if (args.includes("--purge")) {
+        await purge(process.cwd());
+        return;
+    }
     const cwd = process.cwd();
     const files = await findReports(cwd);
     if (files.length === 0) {
         console.log("\n  No reports found. Run some agent tests first.\n");
         return;
     }
-    const reports = await Promise.all(files.map(async (f) => {
+    let reports = await Promise.all(files.map(async (f) => {
         const content = await readFile(f, "utf-8");
         return parseReport(content, relative(cwd, f));
     }));
+    // Ensure all reports have dimensions (backward compat)
+    await Promise.all(reports.map((r) => ensureDimensions(r)));
+    if (agentFilter) {
+        reports = reports.filter((r) => r.name?.toLowerCase() === agentFilter.toLowerCase());
+        if (reports.length === 0) {
+            console.log(`\n  No reports found for agent "${agentFilter}".\n`);
+            return;
+        }
+    }
     console.log("\n" + "━".repeat(W));
-    console.log(`  AGEST STATS  ·  ${reports.length} report${reports.length !== 1 ? "s" : ""} found`);
+    const filterLabel = agentFilter ? `  ·  agent: ${agentFilter}` : "";
+    console.log(`  AGEST STATS  ·  ${reports.length} report${reports.length !== 1 ? "s" : ""} found${filterLabel}`);
     console.log("━".repeat(W));
     // Aggregate by model
     const byModel = new Map();
@@ -148,6 +274,37 @@ async function main() {
         value: a.avgDuration,
         display: formatDuration(a.avgDuration).padStart(8),
     })), maxDuration);
+    // Dimension-aware evolution — named agents with more than one run
+    const named = reports.filter((r) => r.name);
+    const byAgentName = new Map();
+    for (const r of named) {
+        const arr = byAgentName.get(r.name) ?? [];
+        arr.push(r);
+        byAgentName.set(r.name, arr);
+    }
+    const evolvingAgents = [...byAgentName.entries()].filter(([, runs]) => runs.length > 1);
+    if (evolvingAgents.length > 0) {
+        console.log(`\n  ${"─".repeat(W - 2)}`);
+        console.log(`  EVOLUTION  ·  dimension-aware grouping`);
+        for (const [name, runs] of evolvingAgents) {
+            const varyingDims = findVaryingDimensions(runs);
+            if (varyingDims.length === 0) {
+                // All runs have identical config — just show flat timeline
+                await printDimensionEvolution(name, runs, "model", []);
+            }
+            else {
+                // Group by the primary varying dimension (most unique values)
+                const primaryDim = varyingDims[0];
+                await printDimensionEvolution(name, runs, primaryDim, varyingDims);
+                // Cross-comparison for the primary varying dimension
+                if (varyingDims.length >= 2) {
+                    printCrossComparison(name, runs, primaryDim);
+                }
+            }
+            // Attribution summary from controlled pairs
+            printAttribution(name, runs);
+        }
+    }
     console.log("\n" +
         "━".repeat(W) +
         `\n  ${agg.length} model${agg.length !== 1 ? "s" : ""} · ${reports.length} total runs\n` +

package/dist/types.d.ts

@@ -20,6 +20,14 @@ export interface SceneDefinition {
         field: string;
         fn: (value: any) => void;
     }>;
+    timeout?: number;
+    turns?: number;
+}
+export type JudgeVerdict = "pass" | "fail" | "partial";
+export interface JudgeResult {
+    verdict: JudgeVerdict;
+    reasoning: string;
+    criteria: string;
 }
 export interface SceneResult {
     prompt: string;
@@ -27,10 +35,14 @@ export interface SceneResult {
     duration: number;
     passed: boolean;
     error?: string;
+    judgement?: JudgeResult;
 }
 export interface AgentReport {
+    name?: string;
     model?: string;
     systemPromptHash?: string;
+    promptHash?: string;
+    dimensions?: Record<string, string>;
     tools?: string[];
     successRate: number;
     failedCases: string[];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sebastiantuyu/agest",
-  "version": "0.1.6",
+  "version": "0.2.1",
   "description": "A testing library for agents",
   "repository": {
     "type": "git",
@@ -23,11 +23,15 @@
     }
   },
   "scripts": {
-    "build": "tsc",
-    "test": "node dist/index.js",
+    "build": "tsc -p tsconfig.build.json",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:coverage": "vitest run --coverage",
     "dev": "tsx examples/basic.test.ts",
     "test:examples": "tsx examples/basic.test.ts && tsx examples/agent.test.ts",
     "stats": "tsx src/stats.ts",
+    "preview": "tsx src/preview.ts",
+    "site:preview": "npx serve site -p 3000",
     "release:patch": "npm version patch && git push && git push --tags",
     "release:minor": "npm version minor && git push && git push --tags",
     "release:major": "npm version major && git push && git push --tags"
@@ -40,10 +44,15 @@
     "@langchain/langgraph": "^1.2.8",
     "@langchain/openai": "^1.4.4",
     "@types/node": "^22.0.0",
+    "@vitest/coverage-v8": "^3",
     "dotenv": "^17.4.1",
     "langchain": "^1.3.1",
     "tsx": "^4.21.0",
     "typescript": "^5.4.0",
+    "vitest": "^3",
     "zod": "^4.3.6"
+  },
+  "dependencies": {
+    "@supercharge/promise-pool": "^3.3.0"
   }
 }