@sebastiantuyu/agest 0.1.0 → 0.2.1

package/dist/reporter.d.ts

@@ -1,3 +1,4 @@
 import type { AgentReport } from "./types";
 export declare function formatReport(report: AgentReport): string;
-export declare function writeReport(content: string, timestamp: string): Promise<string>;
+export declare function writeReport(content: string, timestamp: string, name?: string, dimensions?: Record<string, string>): Promise<string>;
+export declare function writeDiffEntry(hash: string, systemPrompt: string, tools: string[], model?: string): Promise<void>;

package/dist/reporter.js

@@ -1,15 +1,18 @@
-import { mkdir, writeFile } from "fs/promises";
+import { access, mkdir, writeFile } from "fs/promises";
+import { createHash } from "crypto";
 import { join } from "path";
 export function formatReport(report) {
-    const lines = [
-        "agent:",
-        `    model: "${report.model ?? "unknown"}"`,
-        `    system_prompt: ${report.systemPromptHash ?? "<unknown>"}`,
-        `    tools: ${JSON.stringify(report.tools ?? [])}`,
-        `    success_rate: ${report.successRate}`,
-        `    failed_cases_count: ${report.failedCases.length}`,
-        `    failed_cases:`,
-    ];
+    const lines = ["agent:"];
+    if (report.name)
+        lines.push(`    name: "${report.name}"`);
+    lines.push(`    model: "${report.model ?? "unknown"}"`, `    system_prompt: ${report.systemPromptHash ?? "<unknown>"}`, `    prompt_hash: ${report.promptHash ?? "<unknown>"}`, `    tools: ${JSON.stringify(report.tools ?? [])}`);
+    if (report.dimensions && Object.keys(report.dimensions).length > 0) {
+        lines.push(`    dimensions:`);
+        for (const [key, value] of Object.entries(report.dimensions)) {
+            lines.push(`        ${key}: "${value}"`);
+        }
+    }
+    lines.push(`    success_rate: ${report.successRate}`, `    failed_cases_count: ${report.failedCases.length}`, `    failed_cases:`);
     if (report.failedCases.length === 0) {
         lines.push("        (none)");
     }
@@ -31,12 +34,44 @@ export function formatReport(report) {
     }
     return lines.join("\n");
 }
-export async function writeReport(content, timestamp) {
-    const reportsDir = join(process.cwd(), "reports");
+export async function writeReport(content, timestamp, name, dimensions) {
+    const reportsDir = join(process.cwd(), ".reports");
     await mkdir(reportsDir, { recursive: true });
-    const safestamp = timestamp.replace(/[:.]/g, "-");
-    const filename = `report-${safestamp}.yaml`;
+    const safename = name ? `-${name.replace(/[^a-zA-Z0-9_-]/g, "_")}` : "";
+    let filename;
+    if (dimensions && Object.keys(dimensions).length > 0) {
+        const sorted = Object.entries(dimensions).sort(([a], [b]) => a.localeCompare(b));
+        const dimHash = createHash("sha256").update(JSON.stringify(sorted)).digest("hex").slice(0, 8);
+        filename = `report${safename}-${dimHash}.yaml`;
+    }
+    else {
+        const safestamp = timestamp.replace(/[:.]/g, "-");
+        filename = `report${safename}-${safestamp}.yaml`;
+    }
     const filepath = join(reportsDir, filename);
+    try {
+        await access(filepath);
+        console.warn(`\x1b[33m⚠ Overwriting previous report for ${name ?? "unnamed"} (same config)\x1b[0m`);
+    }
+    catch { }
     await writeFile(filepath, content, "utf-8");
     return filepath;
 }
+export async function writeDiffEntry(hash, systemPrompt, tools, model) {
+    const diffDir = join(process.cwd(), ".diff");
+    await mkdir(diffDir, { recursive: true });
+    const filepath = join(diffDir, `${hash}.yaml`);
+    try {
+        await access(filepath);
+        return; // already exists — skip
+    }
+    catch { }
+    const lines = [
+        `system_prompt: |`,
+        ...systemPrompt.split("\n").map((l) => `  ${l}`),
+        `tools: ${JSON.stringify(tools)}`,
+    ];
+    if (model)
+        lines.push(`model: "${model}"`);
+    await writeFile(filepath, lines.join("\n"), "utf-8");
+}

package/dist/reports.d.ts

@@ -0,0 +1,78 @@
+export interface ParsedReport {
+    name?: string;
+    systemPromptHash?: string;
+    promptHash?: string;
+    dimensions?: Record<string, string>;
+    tools?: string[];
+    model: string;
+    successRate: number;
+    totalCases: number;
+    failedCasesCount: number;
+    failedCases: Array<{
+        prompt: string;
+        reason?: string;
+    }>;
+    duration: number;
+    timestamp: string;
+    averageInputTokensPerCase?: number;
+    averageOutputTokensPerCase?: number;
+    source: string;
+}
+export interface DiffEntry {
+    systemPrompt: string;
+    tools: string[];
+    model?: string;
+}
+export declare function extractField(content: string, key: string): string | undefined;
+export declare function parseFailedCases(content: string): Array<{
+    prompt: string;
+    reason?: string;
+}>;
+export declare function parseDimensions(content: string): Record<string, string> | undefined;
+export declare function parseReport(content: string, source: string): ParsedReport;
+export declare function findReports(dir: string, depth?: number): Promise<string[]>;
+export declare function loadDiffEntry(hash: string): Promise<DiffEntry | null>;
+export declare function computeDiff(a: DiffEntry, b: DiffEntry): string[];
+export interface ConfigDiff {
+    held: Record<string, string>;
+    varied: Record<string, {
+        from: string;
+        to: string;
+    }>;
+    changedCount: number;
+}
+export interface ControlledComparison {
+    a: ParsedReport;
+    b: ParsedReport;
+    variedDimension: string;
+    variedFrom: string;
+    variedTo: string;
+    delta: number;
+}
+/**
+ * Reconstruct dimensions from legacy report fields (backward compat).
+ * For old reports that lack the `dimensions` block, we build one from
+ * model, promptHash (or systemPromptHash + diff entry), and tools.
+ */
+export declare function ensureDimensions(report: ParsedReport): Promise<Record<string, string>>;
+/**
+ * Diff two config maps generically. Returns which dimensions were
+ * held constant vs varied, without knowing what the dimensions are.
+ */
+export declare function diffConfigs(a: Record<string, string>, b: Record<string, string>): ConfigDiff;
+/**
+ * Find all report pairs within the same agent where exactly one
+ * dimension differs. These are "controlled comparisons" — the delta
+ * can be cleanly attributed to the single varied dimension.
+ */
+export declare function findControlledPairs(reports: ParsedReport[]): ControlledComparison[];
+/**
+ * Detect which dimensions vary across a set of reports.
+ * Returns dimension names sorted by number of unique values (most varying first).
+ */
+export declare function findVaryingDimensions(reports: ParsedReport[]): string[];
+/**
+ * Group reports by the value of a specific dimension.
+ */
+export declare function groupByDimension(reports: ParsedReport[], dimension: string): Map<string, ParsedReport[]>;
+export declare function formatDuration(ms: number): string;

package/dist/reports.js

@@ -0,0 +1,278 @@
+import { createHash } from "crypto";
+import { readdir, readFile } from "fs/promises";
+import { join } from "path";
+export function extractField(content, key) {
+    const regex = new RegExp(`^    ${key}:\\s*(.+)$`, "m");
+    const match = content.match(regex);
+    if (!match)
+        return undefined;
+    return match[1].replace(/^"|"$/g, "").trim();
+}
+export function parseFailedCases(content) {
+    const lines = content.split("\n");
+    const startIdx = lines.findIndex((l) => l.trimStart().startsWith("failed_cases:"));
+    if (startIdx === -1)
+        return [];
+    const cases = [];
+    for (let i = startIdx + 1; i < lines.length; i++) {
+        const line = lines[i];
+        if (!line.startsWith("        "))
+            break;
+        const promptMatch = line.match(/^\s+- "(.+)"$/);
+        if (promptMatch) {
+            const next = lines[i + 1];
+            const reasonMatch = next?.match(/^\s+reason: "(.+)"$/);
+            cases.push({ prompt: promptMatch[1], reason: reasonMatch?.[1] });
+        }
+    }
+    return cases;
+}
+export function parseDimensions(content) {
+    const lines = content.split("\n");
+    const startIdx = lines.findIndex((l) => l.trimStart().startsWith("dimensions:"));
+    if (startIdx === -1)
+        return undefined;
+    const dims = {};
+    for (let i = startIdx + 1; i < lines.length; i++) {
+        const line = lines[i];
+        if (!line.startsWith("        "))
+            break;
+        const match = line.match(/^\s+(\w+):\s*"?(.+?)"?\s*$/);
+        if (match) {
+            dims[match[1]] = match[2];
+        }
+    }
+    return Object.keys(dims).length > 0 ? dims : undefined;
+}
+export function parseReport(content, source) {
+    const num = (key, fallback = 0) => parseFloat(extractField(content, key) ?? String(fallback));
+    const avgIn = extractField(content, "average_input_tokens_per_case");
+    const avgOut = extractField(content, "average_output_tokens_per_case");
+    const toolsRaw = extractField(content, "tools");
+    const tools = toolsRaw
+        ? (() => {
+            try {
+                return JSON.parse(toolsRaw);
+            }
+            catch {
+                return undefined;
+            }
+        })()
+        : undefined;
+    const model = extractField(content, "model") ?? "unknown";
+    const promptHash = extractField(content, "prompt_hash");
+    const systemPromptHash = extractField(content, "system_prompt");
+    const dimensions = parseDimensions(content);
+    return {
+        name: extractField(content, "name"),
+        systemPromptHash,
+        promptHash,
+        dimensions,
+        tools,
+        model,
+        successRate: num("success_rate"),
+        totalCases: num("total_cases"),
+        failedCasesCount: parseInt(extractField(content, "failed_cases_count") ?? "0", 10),
+        failedCases: parseFailedCases(content),
+        duration: num("duration"),
+        timestamp: extractField(content, "timestamp") ?? "",
+        averageInputTokensPerCase: avgIn != null ? parseFloat(avgIn) : undefined,
+        averageOutputTokensPerCase: avgOut != null ? parseFloat(avgOut) : undefined,
+        source,
+    };
+}
+export async function findReports(dir, depth = 0) {
+    if (depth > 6)
+        return [];
+    const SKIP = new Set(["node_modules", "dist", ".git", ".pnpm"]);
+    const results = [];
+    let entries;
+    try {
+        entries = await readdir(dir, { withFileTypes: true });
+    }
+    catch {
+        return [];
+    }
+    for (const entry of entries) {
+        if (SKIP.has(entry.name))
+            continue;
+        const fullPath = join(dir, entry.name);
+        if (entry.isDirectory()) {
+            if (entry.name === ".reports") {
+                const files = await readdir(fullPath);
+                for (const f of files) {
+                    if (f.endsWith(".yaml") || f.endsWith(".yml")) {
+                        results.push(join(fullPath, f));
+                    }
+                }
+            }
+            else if (!entry.name.startsWith(".")) {
+                results.push(...(await findReports(fullPath, depth + 1)));
+            }
+        }
+    }
+    return results;
+}
+export async function loadDiffEntry(hash) {
+    try {
+        const content = await readFile(join(process.cwd(), ".diff", `${hash}.yaml`), "utf-8");
+        const promptMatch = content.match(/^system_prompt: \|\n([\s\S]*?)(?=\ntools:)/m);
+        const toolsMatch = content.match(/^tools: (.+)$/m);
+        const modelMatch = content.match(/^model: "(.+)"$/m);
+        return {
+            systemPrompt: promptMatch
+                ? promptMatch[1].replace(/^  /gm, "").trimEnd()
+                : "",
+            tools: toolsMatch ? JSON.parse(toolsMatch[1]) : [],
+            model: modelMatch ? modelMatch[1] : undefined,
+        };
+    }
+    catch {
+        return null;
+    }
+}
+export function computeDiff(a, b) {
+    const lines = [];
+    if (a.model !== b.model) {
+        if (a.model)
+            lines.push(`model: - "${a.model}"`);
+        if (b.model)
+            lines.push(`model: + "${b.model}"`);
+    }
+    const added = b.tools.filter((t) => !a.tools.includes(t));
+    const removed = a.tools.filter((t) => !b.tools.includes(t));
+    if (added.length)
+        lines.push(`tools: +[${added.join(", ")}]`);
+    if (removed.length)
+        lines.push(`tools: -[${removed.join(", ")}]`);
+    const aLines = new Set(a.systemPrompt.split("\n").map((l) => l.trim()).filter(Boolean));
+    const bLines = new Set(b.systemPrompt.split("\n").map((l) => l.trim()).filter(Boolean));
+    const addedLines = [...bLines].filter((l) => !aLines.has(l)).slice(0, 3);
+    const removedLines = [...aLines].filter((l) => !bLines.has(l)).slice(0, 3);
+    for (const l of addedLines)
+        lines.push(`prompt: + "${l.slice(0, 60)}"`);
+    for (const l of removedLines)
+        lines.push(`prompt: - "${l.slice(0, 60)}"`);
+    return lines;
+}
+/**
+ * Reconstruct dimensions from legacy report fields (backward compat).
+ * For old reports that lack the `dimensions` block, we build one from
+ * model, promptHash (or systemPromptHash + diff entry), and tools.
+ */
+export async function ensureDimensions(report) {
+    if (report.dimensions)
+        return report.dimensions;
+    const dims = {};
+    dims.model = report.model;
+    if (report.promptHash) {
+        dims.prompt = report.promptHash;
+    }
+    else if (report.systemPromptHash) {
+        // Derive prompt-only hash from diff entry
+        const entry = await loadDiffEntry(report.systemPromptHash);
+        if (entry) {
+            dims.prompt = createHash("sha256").update(entry.systemPrompt).digest("hex").slice(0, 12);
+            report.promptHash = dims.prompt;
+        }
+    }
+    if (report.tools?.length) {
+        dims.tools = [...report.tools].sort().join(",");
+    }
+    else {
+        dims.tools = "none";
+    }
+    report.dimensions = dims;
+    return dims;
+}
+/**
+ * Diff two config maps generically. Returns which dimensions were
+ * held constant vs varied, without knowing what the dimensions are.
+ */
+export function diffConfigs(a, b) {
+    const allKeys = new Set([...Object.keys(a), ...Object.keys(b)]);
+    const held = {};
+    const varied = {};
+    for (const key of allKeys) {
+        const av = a[key] ?? "(absent)";
+        const bv = b[key] ?? "(absent)";
+        if (av === bv) {
+            held[key] = av;
+        }
+        else {
+            varied[key] = { from: av, to: bv };
+        }
+    }
+    return { held, varied, changedCount: Object.keys(varied).length };
+}
+/**
+ * Find all report pairs within the same agent where exactly one
+ * dimension differs. These are "controlled comparisons" — the delta
+ * can be cleanly attributed to the single varied dimension.
+ */
+export function findControlledPairs(reports) {
+    const pairs = [];
+    for (let i = 0; i < reports.length; i++) {
+        for (let j = i + 1; j < reports.length; j++) {
+            const a = reports[i];
+            const b = reports[j];
+            if (!a.dimensions || !b.dimensions)
+                continue;
+            const diff = diffConfigs(a.dimensions, b.dimensions);
+            if (diff.changedCount !== 1)
+                continue;
+            const [dimName, { from, to }] = Object.entries(diff.varied)[0];
+            pairs.push({
+                a,
+                b,
+                variedDimension: dimName,
+                variedFrom: from,
+                variedTo: to,
+                delta: b.successRate - a.successRate,
+            });
+        }
+    }
+    return pairs.sort((a, b) => Math.abs(b.delta) - Math.abs(a.delta));
+}
+/**
+ * Detect which dimensions vary across a set of reports.
+ * Returns dimension names sorted by number of unique values (most varying first).
+ */
+export function findVaryingDimensions(reports) {
+    const valuesByDim = new Map();
+    for (const r of reports) {
+        if (!r.dimensions)
+            continue;
+        for (const [key, val] of Object.entries(r.dimensions)) {
+            const set = valuesByDim.get(key) ?? new Set();
+            set.add(val);
+            valuesByDim.set(key, set);
+        }
+    }
+    return [...valuesByDim.entries()]
+        .filter(([, vals]) => vals.size > 1)
+        .sort((a, b) => b[1].size - a[1].size)
+        .map(([key]) => key);
+}
+/**
+ * Group reports by the value of a specific dimension.
+ */
+export function groupByDimension(reports, dimension) {
+    const groups = new Map();
+    for (const r of reports) {
+        const val = r.dimensions?.[dimension] ?? "(unknown)";
+        const arr = groups.get(val) ?? [];
+        arr.push(r);
+        groups.set(val, arr);
+    }
+    return groups;
+}
+export function formatDuration(ms) {
+    if (ms < 1000)
+        return `${ms.toFixed(0)}ms`;
+    if (ms < 60_000)
+        return `${(ms / 1000).toFixed(1)}s`;
+    const m = Math.floor(ms / 60_000);
+    const s = ((ms % 60_000) / 1000).toFixed(0).padStart(2, "0");
+    return `${m}m${s}s`;
+}

package/dist/runner.d.ts

@@ -1,3 +1,4 @@
 import type { AgentExecutor, AgentResponse, SceneDefinition, SceneResult } from "./types";
+import type { JudgeConfig } from "./config";
 export declare function extractField(response: AgentResponse, field: string): unknown;
-export declare function executeScene(executor: AgentExecutor, scene: SceneDefinition): Promise<SceneResult>;
+export declare function executeScene(executor: AgentExecutor, scene: SceneDefinition, globalTimeout?: number, judgeConfig?: JudgeConfig, globalTurns?: number): Promise<SceneResult>;

package/dist/runner.js

@@ -1,3 +1,6 @@
+import { collectPendingJudgements } from "./assertions";
+import { callJudge, resolveJudgeExecutor } from "./judge";
+const DEFAULT_SCENE_TIMEOUT = 10_000;
 export function extractField(response, field) {
     switch (field) {
         case "response":
@@ -10,12 +13,24 @@ export function extractField(response, field) {
             return response.metadata?.[field];
     }
 }
-export async function executeScene(executor, scene) {
-    let response;
+export async function executeScene(executor, scene, globalTimeout, judgeConfig, globalTurns) {
+    let response = { text: "" };
     let duration;
+    const timeoutMs = scene.timeout ?? globalTimeout ?? DEFAULT_SCENE_TIMEOUT;
+    const turns = scene.turns ?? globalTurns ?? 1;
     try {
         const start = performance.now();
-        response = await executor(scene.prompt);
+        let input = scene.prompt;
+        for (let t = 0; t < turns; t++) {
+            response = await Promise.race([
+                executor(input),
+                new Promise((_, reject) => setTimeout(() => reject(new Error(`Scene timed out after ${timeoutMs}ms`)), timeoutMs)),
+            ]);
+            if (response.executionError)
+                break;
+            if (t < turns - 1)
+                input = response.text;
+        }
         duration = performance.now() - start;
     }
     catch (err) {
@@ -38,6 +53,7 @@ export async function executeScene(executor, scene) {
     }
     let passed = true;
     let error;
+    let judgement;
     for (const assertion of scene.assertions) {
         try {
             const value = extractField(response, assertion.field);
@@ -49,5 +65,31 @@ export async function executeScene(executor, scene) {
             break;
         }
     }
-    return { prompt: scene.prompt, response, duration, passed, error };
+    const pending = collectPendingJudgements();
+    if (pending.length > 0 && passed) {
+        if (!judgeConfig) {
+            passed = false;
+            error = "judgedBy() requires a judge configured in agest.config.ts";
+        }
+        else {
+            const judgeExecutor = resolveJudgeExecutor(judgeConfig);
+            for (const p of pending) {
+                try {
+                    const result = await callJudge(String(p.value), p.criteria, judgeExecutor);
+                    judgement = result;
+                    if (result.verdict === "fail" || result.verdict === "partial") {
+                        passed = false;
+                        error = `Judge verdict: ${result.verdict} — ${result.reasoning}`;
+                        break;
+                    }
+                }
+                catch (err) {
+                    passed = false;
+                    error = `Judge error: ${err.message}`;
+                    break;
+                }
+            }
+        }
+    }
+    return { prompt: scene.prompt, response, duration, passed, error, judgement };
 }