@ikarmic/forge-eval 0.0.0-stage → 4.5.0

package/dist/report.js

@@ -0,0 +1,120 @@
+/** Pretty-print a model identifier the same way the CLI does it. */
+function modelLabel(model) {
+    return `${model.provider}/${model.model}`;
+}
+/** Render a human-friendly markdown table for a suite run. */
+export function formatMarkdownReport(suite) {
+    const lines = [];
+    lines.push(`# Forge eval: ${modelLabel(suite.model)}`);
+    lines.push('');
+    lines.push(`Started:  ${suite.startedAt}`);
+    lines.push(`Finished: ${suite.finishedAt}`);
+    lines.push('');
+    lines.push(`**${suite.summary.passed}/${suite.summary.total}** tasks passed ` +
+        `(${pct(suite.summary.passRate)}%) in ${(suite.summary.totalDurationMs / 1000).toFixed(1)}s.`);
+    if (suite.summary.totalCostUsd > 0) {
+        lines.push(`Total cost: $${suite.summary.totalCostUsd.toFixed(4)}`);
+    }
+    lines.push('');
+    // Per-task table.
+    lines.push('| # | Task | Category | Result | Duration | Graders |');
+    lines.push('|---|------|----------|--------|----------|---------|');
+    for (const [i, r] of suite.results.entries()) {
+        const idx = String(i + 1).padStart(2, ' ');
+        const result = r.error
+            ? `error: ${truncate(r.error, 40)}`
+            : r.passed
+                ? 'pass'
+                : `fail (${r.graderResults.filter((g) => !g.passed).length}/${r.graderResults.length})`;
+        const dur = `${(r.durationMs / 1000).toFixed(1)}s`;
+        const passedCount = r.graderResults.filter((g) => g.passed).length;
+        const graderStr = r.graderResults.length === 0 ? '-' : `${passedCount}/${r.graderResults.length}`;
+        lines.push(`| ${idx} | ${r.task.id} | ${r.task.category} | ${result} | ${dur} | ${graderStr} |`);
+    }
+    lines.push('');
+    // Failing-task detail blocks: only the ones worth reading.
+    for (const r of suite.results) {
+        if (r.passed && !r.error)
+            continue;
+        lines.push(`## ${r.task.id}: ${r.task.name}`);
+        if (r.error) {
+            lines.push('');
+            lines.push(`Run error: \`${r.error}\``);
+            lines.push('');
+            continue;
+        }
+        lines.push('');
+        lines.push(`Reason: \`${r.agentResult.reason}\` · iterations: ${r.agentResult.iterations}`);
+        lines.push('');
+        lines.push('Graders:');
+        lines.push('');
+        for (const [i, g] of r.graderResults.entries()) {
+            const mark = g.passed ? '✓' : '✗';
+            lines.push(`- ${mark} **[${i + 1}]** ${g.message}`);
+            if (!g.passed && g.details) {
+                for (const dl of g.details.split('\n'))
+                    lines.push(`  ${dl}`);
+            }
+        }
+        lines.push('');
+        lines.push('<details><summary>Final text</summary>');
+        lines.push('');
+        lines.push('```');
+        lines.push(truncate(r.agentResult.finalText, 1200));
+        lines.push('```');
+        lines.push('</details>');
+        lines.push('');
+    }
+    return lines.join('\n');
+}
+/** Render a compact single-line summary for the terminal. */
+export function formatTerminalSummary(suite) {
+    const s = suite.summary;
+    const lines = [
+        `forge-eval: ${s.passed}/${s.total} passed (${pct(s.passRate)}%) in ${(s.totalDurationMs / 1000).toFixed(1)}s`,
+        `  model: ${modelLabel(suite.model)}`,
+    ];
+    if (s.totalCostUsd > 0) {
+        lines.push(`  cost:  $${s.totalCostUsd.toFixed(4)}`);
+    }
+    for (const r of suite.results) {
+        const mark = r.passed ? '✓' : '✗';
+        const dur = `${(r.durationMs / 1000).toFixed(1)}s`.padStart(6, ' ');
+        lines.push(`  ${mark} ${r.task.id.padEnd(34, ' ')} ${dur}`);
+    }
+    return lines.join('\n');
+}
+export function formatJsonReport(suite) {
+    return {
+        model: suite.model,
+        startedAt: suite.startedAt,
+        finishedAt: suite.finishedAt,
+        summary: suite.summary,
+        tasks: suite.results.map((r) => ({
+            id: r.task.id,
+            name: r.task.name,
+            category: r.task.category,
+            passed: r.passed,
+            durationMs: r.durationMs,
+            iterations: r.agentResult.iterations,
+            reason: r.agentResult.reason,
+            graderResults: r.graderResults,
+            toolInvocations: r.toolInvocations.map((t) => ({
+                name: t.name,
+                args: t.args,
+                isError: t.isError,
+            })),
+            finalText: r.agentResult.finalText,
+            ...(r.error ? { error: r.error } : {}),
+        })),
+    };
+}
+function pct(rate) {
+    return (rate * 100).toFixed(0);
+}
+function truncate(s, max) {
+    if (s.length <= max)
+        return s;
+    return s.slice(0, max) + '…';
+}
+//# sourceMappingURL=report.js.map

package/dist/report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.js","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAEA,oEAAoE;AACpE,SAAS,UAAU,CAAC,KAA0C;IAC5D,OAAO,GAAG,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;AAC5C,CAAC;AAED,8DAA8D;AAC9D,MAAM,UAAU,oBAAoB,CAAC,KAAqB;IACxD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,iBAAiB,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACvD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,aAAa,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;IAC3C,KAAK,CAAC,IAAI,CAAC,aAAa,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC;IAC5C,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CACR,KAAK,KAAK,CAAC,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,kBAAkB;QAChE,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAChG,CAAC;IACF,IAAI,KAAK,CAAC,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;QACnC,KAAK,CAAC,IAAI,CAAC,gBAAgB,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACtE,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,kBAAkB;IAClB,KAAK,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;IACpE,KAAK,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;IACpE,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QAC7C,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK;YACpB,CAAC,CAAC,UAAU,QAAQ,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE;YACnC,CAAC,CAAC,CAAC,CAAC,MAAM;gBACR,CAAC,CAAC,MAAM;gBACR,CAAC,CAAC,SAAS,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC;QAC5F,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;QACnD,MAAM,WAAW,GAAG,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QACnE,MAAM,SAAS,GAAG,CAAC,CAAC,aAAa,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,WAAW,IAAI,CAAC,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;QAClG,KAAK,CAAC,IAAI,CACR,KAAK,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,MAAM,MAAM,MAAM,GAAG,MAAM,SAAS,IAAI,CACrF,CAAC;IACJ,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,2DAA2D;IAC3D,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAC9B,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,KAAK;YAAE,SAAS;QACnC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9C,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;YACZ,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;YACxC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,SAAS;QACX,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,WAAW,CAAC,MAAM,oBAAoB,CAAC,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC;QAC5F,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC;YAC/C,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YACpD,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;gBAC3B,KAAK,MAAM,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC;oBAAE,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;QACrD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC;QACpD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,6DAA6D;AAC7D,MAAM,UAAU,qBAAqB,CAAC,KAAqB;IACzD,MAAM,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC;IACxB,MAAM,KAAK,GAAG;QACZ,eAAe,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,YAAY,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,eAAe,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;QAC9G,YAAY,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;KACtC,CAAC;IACF,IAAI,CAAC,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACvD,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QAClC,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACpE,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC;IAC9D,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAuBD,MAAM,UAAU,gBAAgB,CAAC,KAAqB;IACpD,OAAO;QACL,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC/B,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE;YACb,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI;YACjB,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ;YACzB,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,UAAU,EAAE,CAAC,CAAC,UAAU;YACxB,UAAU,EAAE,CAAC,CAAC,WAAW,CAAC,UAAU;YACpC,MAAM,EAAE,CAAC,CAAC,WAAW,CAAC,MAAM;YAC5B,aAAa,EAAE,CAAC,CAAC,aAAa;YAC9B,eAAe,EAAE,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC7C,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC,CAAC;YACH,SAAS,EAAE,CAAC,CAAC,WAAW,CAAC,SAAS;YAClC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACvC,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC;AAED,SAAS,GAAG,CAAC,IAAY;IACvB,OAAO,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACjC,CAAC;AAED,SAAS,QAAQ,CAAC,CAAS,EAAE,GAAW;IACtC,IAAI,CAAC,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,CAAC,CAAC;IAC9B,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,GAAG,CAAC;AAC/B,CAAC"}

package/dist/runner.d.ts

@@ -0,0 +1,59 @@
+import { createLanguageModel } from '@ikarmic/forge-core';
+import type { AgentEvent, AgentRunResult, LanguageModelFactory, ModelConfig } from '@ikarmic/forge-core';
+import type { GraderResult, TaskDefinition, TaskInfo, ToolInvocation } from './graders/index.js';
+/** What a single task run produced: passed to the report formatter. */
+export interface TaskRunResult {
+    task: TaskInfo;
+    passed: boolean;
+    /** Per-grader results, in the order the task defined them. */
+    graderResults: GraderResult[];
+    /** The agent's raw run result (kept for the JSON report). */
+    agentResult: AgentRunResult;
+    /** Tool invocations the agent made (also embedded in grader context). */
+    toolInvocations: ToolInvocation[];
+    /** Total wall-clock time for setup + run + grading. */
+    durationMs: number;
+    /** The cwd the task ran in: useful for debugging a failing run. */
+    cwd: string;
+    /** Error message if the run itself blew up (vs. just one grader failing). */
+    error?: string;
+}
+/** What a whole batch run produced. */
+export interface SuiteRunResult {
+    model: ModelConfig;
+    startedAt: string;
+    finishedAt: string;
+    /** One entry per task that ran (skipped tasks are not included). */
+    results: TaskRunResult[];
+    /** Summary numbers: precomputed so the CLI doesn't re-iterate. */
+    summary: {
+        total: number;
+        passed: number;
+        failed: number;
+        passRate: number;
+        totalDurationMs: number;
+        totalCostUsd: number;
+    };
+}
+/** Options for {@link runTask}. */
+export interface RunTaskOptions {
+    /** Override the model factory (defaults to `createLanguageModel`). */
+    modelFactory?: LanguageModelFactory;
+    /** Override the per-task cwd (otherwise a fresh temp dir is created). */
+    cwd?: string;
+    /** Stream events for this run (used by the CLI to mirror to stdout). */
+    onEvent?: (event: AgentEvent) => void;
+}
+/**
+ * Run a single task end-to-end: create cwd, run setup, execute the agent,
+ * grade the result, return everything. Always cleans up the temp dir on
+ * exit (success or failure): the CLI uses the cwd only during the run.
+ */
+export declare function runTask(task: TaskDefinition, model: ModelConfig, options?: RunTaskOptions): Promise<TaskRunResult>;
+/** Run every task in `tasks` (or the whole catalog if omitted) sequentially. */
+export declare function runSuite(tasks: readonly TaskDefinition[], model: ModelConfig, options?: RunTaskOptions): Promise<SuiteRunResult>;
+/** Resolve a ModelConfig using the same precedence as the CLI. */
+export declare function resolveModel(override?: string): ModelConfig;
+/** Re-export for the CLI to use directly. */
+export { createLanguageModel };
+//# sourceMappingURL=runner.d.ts.map

package/dist/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAIA,OAAO,EAAS,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AACjE,OAAO,KAAK,EAEV,UAAU,EACV,cAAc,EACd,oBAAoB,EACpB,WAAW,EACZ,MAAM,qBAAqB,CAAC;AAE7B,OAAO,KAAK,EAEV,YAAY,EACZ,cAAc,EACd,QAAQ,EACR,cAAc,EACf,MAAM,oBAAoB,CAAC;AAE5B,uEAAuE;AACvE,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,OAAO,CAAC;IAChB,8DAA8D;IAC9D,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,6DAA6D;IAC7D,WAAW,EAAE,cAAc,CAAC;IAC5B,yEAAyE;IACzE,eAAe,EAAE,cAAc,EAAE,CAAC;IAClC,uDAAuD;IACvD,UAAU,EAAE,MAAM,CAAC;IACnB,mEAAmE;IACnE,GAAG,EAAE,MAAM,CAAC;IACZ,6EAA6E;IAC7E,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,uCAAuC;AACvC,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,WAAW,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,oEAAoE;IACpE,OAAO,EAAE,aAAa,EAAE,CAAC;IACzB,kEAAkE;IAClE,OAAO,EAAE;QACP,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,eAAe,EAAE,MAAM,CAAC;QACxB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;CACH;AAED,mCAAmC;AACnC,MAAM,WAAW,cAAc;IAC7B,sEAAsE;IACtE,YAAY,CAAC,EAAE,oBAAoB,CAAC;IACpC,yEAAyE;IACzE,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,wEAAwE;IACxE,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,IAAI,CAAC;CACvC;AAED;;;;GAIG;AACH,wBAAsB,OAAO,CAC3B,IAAI,EAAE,cAAc,EACpB,KAAK,EAAE,WAAW,EAClB,OAAO,GAAE,cAAmB,GAC3B,OAAO,CAAC,aAAa,CAAC,CA8GxB;AAED,gFAAgF;AAChF,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,SAAS,cAAc,EAAE,EAChC,KAAK,EAAE,WAAW,EAClB,OAAO,GAAE,cAAmB,GAC3B,OAAO,CAAC,cAAc,CAAC,CAgCzB;AAED,kEAAkE;AAClE,wBAAgB,YAAY,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,WAAW,CAW3D;AAED,6CAA6C;AAC7C,OAAO,EAAE,mBAAmB,EAAE,CAAC"}

package/dist/runner.js

@@ -0,0 +1,217 @@
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { Agent, createLanguageModel } from '@ikarmic/forge-core';
+/**
+ * Run a single task end-to-end: create cwd, run setup, execute the agent,
+ * grade the result, return everything. Always cleans up the temp dir on
+ * exit (success or failure): the CLI uses the cwd only during the run.
+ */
+export async function runTask(task, model, options = {}) {
+    const start = Date.now();
+    const ownsCwd = options.cwd === undefined;
+    const cwd = options.cwd ?? (await mkdtemp(join(tmpdir(), `forge-eval-${task.id}-`)));
+    const toolInvocations = [];
+    const pendingCalls = new Map();
+    let agentResult;
+    let error;
+    let graderResults = [];
+    let passed = false;
+    // A memory task must not read the host's ~/.forge/MEMORY.md.
+    const prevGlobalMemory = process.env.FORGE_GLOBAL_MEMORY_DIR;
+    if (task.memory)
+        process.env.FORGE_GLOBAL_MEMORY_DIR = join(cwd, '.no-global-memory');
+    try {
+        // Setup the fixture files (if any).
+        if (task.setup) {
+            await task.setup(cwd);
+        }
+        // Build an Agent with everything disabled that would add noise: cost
+        // tracking (we sum it ourselves), memory (per-task temp dir makes
+        // persistent state confusing; a task can opt back in), cache (each task is unique).
+        const config = {
+            model,
+            cwd,
+            auto: true, // evals are autonomous: no interactive approval prompts
+            allowDestructive: true, // same reason
+            disableCost: true,
+            disableMemory: task.memory !== true,
+            disableCache: true,
+            ...(options.modelFactory ? { modelFactory: options.modelFactory } : {}),
+        };
+        const agent = new Agent(config);
+        agentResult = await agent.run({
+            task: task.task,
+            onEvent: (event) => {
+                if (event.type === 'tool_call') {
+                    pendingCalls.set(event.id, { name: event.name, args: event.args, result: undefined, isError: false });
+                }
+                else if (event.type === 'tool_result') {
+                    const pending = pendingCalls.get(event.id);
+                    if (pending) {
+                        pending.result = event.result;
+                        pending.isError = event.isError;
+                        toolInvocations.push(pending);
+                        pendingCalls.delete(event.id);
+                    }
+                }
+                options.onEvent?.(event);
+            },
+        });
+        // Run graders *before* cleaning up the temp dir so filesystem graders
+        // can inspect files the agent created.
+        const ctx = {
+            cwd,
+            task: taskInfo(task),
+            result: agentResult,
+            toolInvocations,
+        };
+        passed = true;
+        for (const grader of task.graders) {
+            try {
+                const r = await grader(ctx);
+                graderResults.push(r);
+                if (!r.passed)
+                    passed = false;
+            }
+            catch (err) {
+                const message = err instanceof Error ? err.message : String(err);
+                graderResults.push({ passed: false, message: 'grader threw', details: message });
+                passed = false;
+            }
+        }
+    }
+    catch (err) {
+        error = err instanceof Error ? err.message : String(err);
+    }
+    finally {
+        if (task.memory) {
+            if (prevGlobalMemory === undefined)
+                delete process.env.FORGE_GLOBAL_MEMORY_DIR;
+            else
+                process.env.FORGE_GLOBAL_MEMORY_DIR = prevGlobalMemory;
+        }
+        if (ownsCwd) {
+            await rm(cwd, { recursive: true, force: true });
+        }
+    }
+    if (error) {
+        return {
+            task: taskInfo(task),
+            passed: false,
+            graderResults,
+            agentResult: agentResult ?? emptyAgentResult(),
+            toolInvocations,
+            durationMs: Date.now() - start,
+            cwd,
+            error,
+        };
+    }
+    return {
+        task: taskInfo(task),
+        passed,
+        graderResults,
+        agentResult,
+        toolInvocations,
+        durationMs: Date.now() - start,
+        cwd,
+    };
+}
+/** Run every task in `tasks` (or the whole catalog if omitted) sequentially. */
+export async function runSuite(tasks, model, options = {}) {
+    const startedAt = new Date().toISOString();
+    const results = [];
+    let totalCostUsd = 0;
+    for (const task of tasks) {
+        const result = await runTask(task, model, options);
+        results.push(result);
+        if (result.agentResult.costKnown) {
+            totalCostUsd += result.agentResult.costUsd;
+        }
+    }
+    const finishedAt = new Date().toISOString();
+    const passed = results.filter((r) => r.passed).length;
+    const failed = results.length - passed;
+    const totalDurationMs = results.reduce((sum, r) => sum + r.durationMs, 0);
+    return {
+        model,
+        startedAt,
+        finishedAt,
+        results,
+        summary: {
+            total: results.length,
+            passed,
+            failed,
+            passRate: results.length === 0 ? 0 : passed / results.length,
+            totalDurationMs,
+            totalCostUsd,
+        },
+    };
+}
+/** Resolve a ModelConfig using the same precedence as the CLI. */
+export function resolveModel(override) {
+    if (override) {
+        const [provider, model] = override.split('/');
+        if (!provider || !model) {
+            throw new Error(`Invalid --model "${override}". Expected "<provider>/<model>" e.g. "anthropic/claude-3-5-haiku-latest".`);
+        }
+        return { provider: provider, model };
+    }
+    return createLanguageModelFallback();
+}
+/** Re-export for the CLI to use directly. */
+export { createLanguageModel };
+/**
+ * A tiny wrapper around `createLanguageModel` so we don't have to import the
+ * core types directly here. Keeps the runner decoupled from the provider
+ * implementation.
+ */
+function createLanguageModelFallback() {
+    // Same precedence as the CLI: env vars, then Ollama probe.
+    // We delegate to the core helper when available; fall back to a hard error.
+    const provider = process.env.FORGE_DEFAULT_PROVIDER;
+    if (provider === 'ollama') {
+        return { provider: 'ollama', model: process.env.FORGE_DEFAULT_MODEL ?? 'llama3.1' };
+    }
+    const envModel = process.env.FORGE_DEFAULT_MODEL;
+    if (envModel && envModel.includes('/')) {
+        const [p, m] = envModel.split('/');
+        if (!p || !m) {
+            throw new Error(`Invalid FORGE_DEFAULT_MODEL "${envModel}". Expected "<provider>/<model>".`);
+        }
+        return { provider: p, model: m };
+    }
+    // Try Anthropic, OpenAI, Google, OpenRouter in that order: same as core.
+    if (process.env.ANTHROPIC_API_KEY)
+        return { provider: 'anthropic', model: envModel ?? 'claude-3-5-sonnet-latest' };
+    if (process.env.OPENAI_API_KEY)
+        return { provider: 'openai', model: envModel ?? 'gpt-4o-mini' };
+    if (process.env.GOOGLE_GENERATIVE_AI_API_KEY)
+        return { provider: 'google', model: envModel ?? 'gemini-2.0-flash-exp' };
+    if (process.env.OPENROUTER_API_KEY)
+        return { provider: 'openrouter', model: envModel ?? 'anthropic/claude-3.5-haiku' };
+    throw new Error('No model resolved. Pass --model <provider>/<model> or set FORGE_DEFAULT_MODEL and a provider API key ' +
+        '(ANTHROPIC_API_KEY, OPENAI_API_KEY, GOOGLE_GENERATIVE_AI_API_KEY, OPENROUTER_API_KEY) or run `ollama serve`.');
+}
+function taskInfo(task) {
+    return {
+        id: task.id,
+        name: task.name,
+        category: task.category,
+        description: task.description,
+    };
+}
+function emptyAgentResult() {
+    return {
+        finalText: '',
+        iterations: 0,
+        inputTokens: 0,
+        outputTokens: 0,
+        durationMs: 0,
+        reason: 'error',
+        costUsd: 0,
+        costKnown: false,
+        messages: [],
+    };
+}
+//# sourceMappingURL=runner.js.map

package/dist/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,KAAK,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AA+DjE;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,IAAoB,EACpB,KAAkB,EAClB,UAA0B,EAAE;IAE5B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,KAAK,SAAS,CAAC;IAC1C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,cAAc,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IACrF,MAAM,eAAe,GAAqB,EAAE,CAAC;IAC7C,MAAM,YAAY,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEvD,IAAI,WAA4B,CAAC;IACjC,IAAI,KAAyB,CAAC;IAC9B,IAAI,aAAa,GAAmB,EAAE,CAAC;IACvC,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,6DAA6D;IAC7D,MAAM,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC;IAC7D,IAAI,IAAI,CAAC,MAAM;QAAE,OAAO,CAAC,GAAG,CAAC,uBAAuB,GAAG,IAAI,CAAC,GAAG,EAAE,mBAAmB,CAAC,CAAC;IAEtF,IAAI,CAAC;QACH,oCAAoC;QACpC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;QAED,qEAAqE;QACrE,kEAAkE;QAClE,oFAAoF;QACpF,MAAM,MAAM,GAAgB;YAC1B,KAAK;YACL,GAAG;YACH,IAAI,EAAE,IAAI,EAAE,wDAAwD;YACpE,gBAAgB,EAAE,IAAI,EAAE,cAAc;YACtC,WAAW,EAAE,IAAI;YACjB,aAAa,EAAE,IAAI,CAAC,MAAM,KAAK,IAAI;YACnC,YAAY,EAAE,IAAI;YAClB,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACxE,CAAC;QACF,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC;QAEhC,WAAW,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC;YAC5B,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,OAAO,EAAE,CAAC,KAAiB,EAAE,EAAE;gBAC7B,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;oBAC/B,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;gBACxG,CAAC;qBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;oBACxC,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;oBAC3C,IAAI,OAAO,EAAE,CAAC;wBACZ,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;wBAC9B,OAAO,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;wBAChC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;wBAC9B,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;oBAChC,CAAC;gBACH,CAAC;gBACD,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;YAC3B,CAAC;SACF,CAAC,CAAC;QAEH,sEAAsE;QACtE,uCAAuC;QACvC,MAAM,GAAG,GAAkB;YACzB,GAAG;YACH,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC;YACpB,MAAM,EAAE,WAAW;YACnB,eAAe;SAChB,CAAC;QAEF,MAAM,GAAG,IAAI,CAAC;QACd,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,IAAI,CAAC;gBACH,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC5B,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACtB,IAAI,CAAC,CAAC,CAAC,MAAM;oBAAE,MAAM,GAAG,KAAK,CAAC;YAChC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACjE,aAAa,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;gBACjF,MAAM,GAAG,KAAK,CAAC;YACjB,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC3D,CAAC;YAAS,CAAC;QACT,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,IAAI,gBAAgB,KAAK,SAAS;gBAAE,OAAO,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC;;gBAC1E,OAAO,CAAC,GAAG,CAAC,uBAAuB,GAAG,gBAAgB,CAAC;QAC9D,CAAC;QACD,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,IAAI,KAAK,EAAE,CAAC;QACV,OAAO;YACL,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC;YACpB,MAAM,EAAE,KAAK;YACb,aAAa;YACb,WAAW,EAAE,WAAW,IAAI,gBAAgB,EAAE;YAC9C,eAAe;YACf,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC9B,GAAG;YACH,KAAK;SACN,CAAC;IACJ,CAAC;IAED,OAAO;QACL,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC;QACpB,MAAM;QACN,aAAa;QACb,WAAW;QACX,eAAe;QACf,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;QAC9B,GAAG;KACJ,CAAC;AACJ,CAAC;AAED,gFAAgF;AAChF,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAgC,EAChC,KAAkB,EAClB,UAA0B,EAAE;IAE5B,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QACnD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,MAAM,CAAC,WAAW,CAAC,SAAS,EAAE,CAAC;YACjC,YAAY,IAAI,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC5C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACtD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;IACvC,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IAE1E,OAAO;QACL,KAAK;QACL,SAAS;QACT,UAAU;QACV,OAAO;QACP,OAAO,EAAE;YACP,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,MAAM;YACN,MAAM;YACN,QAAQ,EAAE,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;YAC5D,eAAe;YACf,YAAY;SACb;KACF,CAAC;AACJ,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,YAAY,CAAC,QAAiB;IAC5C,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC9C,IAAI,CAAC,QAAQ,IAAI,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CACb,oBAAoB,QAAQ,4EAA4E,CACzG,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,QAAmC,EAAE,KAAK,EAAE,CAAC;IAClE,CAAC;IACD,OAAO,2BAA2B,EAAE,CAAC;AACvC,CAAC;AAED,6CAA6C;AAC7C,OAAO,EAAE,mBAAmB,EAAE,CAAC;AAE/B;;;;GAIG;AACH,SAAS,2BAA2B;IAClC,2DAA2D;IAC3D,4EAA4E;IAC5E,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IACpD,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1B,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,UAAU,EAAE,CAAC;IACtF,CAAC;IACD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;IACjD,IAAI,QAAQ,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACvC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,gCAAgC,QAAQ,mCAAmC,CAAC,CAAC;QAC/F,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,CAA4B,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IAC9D,CAAC;IACD,yEAAyE;IACzE,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB;QAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,IAAI,0BAA0B,EAAE,CAAC;IACnH,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc;QAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,IAAI,aAAa,EAAE,CAAC;IAChG,IAAI,OAAO,CAAC,GAAG,CAAC,4BAA4B;QAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,IAAI,sBAAsB,EAAE,CAAC;IACvH,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB;QAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,KAAK,EAAE,QAAQ,IAAI,4BAA4B,EAAE,CAAC;IACvH,MAAM,IAAI,KAAK,CACb,uGAAuG;QACrG,8GAA8G,CACjH,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,IAAoB;IACpC,OAAO;QACL,EAAE,EAAE,IAAI,CAAC,EAAE;QACX,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,WAAW,EAAE,IAAI,CAAC,WAAW;KAC9B,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB;IACvB,OAAO;QACL,SAAS,EAAE,EAAE;QACb,UAAU,EAAE,CAAC;QACb,WAAW,EAAE,CAAC;QACd,YAAY,EAAE,CAAC;QACf,UAAU,EAAE,CAAC;QACb,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC;QACV,SAAS,EAAE,KAAK;QAChB,QAAQ,EAAE,EAAE;KACb,CAAC;AACJ,CAAC"}

package/dist/tasks/edit.remove-section.d.ts

@@ -0,0 +1,15 @@
+/**
+ * edit / remove-section: two-step task: identify a section by name, then
+ * remove it. Verifies the model handles multiple edits coherently.
+ */
+declare const _default: {
+    id: string;
+    name: string;
+    category: "edit";
+    description: string;
+    task: string;
+    setup: (cwd: string) => Promise<void>;
+    graders: import("../index.js").Grader[];
+};
+export default _default;
+//# sourceMappingURL=edit.remove-section.d.ts.map

package/dist/tasks/edit.remove-section.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"edit.remove-section.d.ts","sourceRoot":"","sources":["../../src/tasks/edit.remove-section.ts"],"names":[],"mappings":"AAKA;;;GAGG;;;;;;;;;;AACH,wBA8BG"}

package/dist/tasks/edit.remove-section.js

@@ -0,0 +1,31 @@
+import { mkdir, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { defineTask, fileMatches, fileMissing, toolWasCalled, completed } from '../graders/index.js';
+/**
+ * edit / remove-section: two-step task: identify a section by name, then
+ * remove it. Verifies the model handles multiple edits coherently.
+ */
+export default defineTask({
+    id: 'edit-remove-section',
+    name: 'Remove a section from config.json',
+    category: 'edit',
+    description: 'Agent removes a named section from a JSON config file.',
+    task: 'The file `config.json` exists in the current directory. It has two top-level ' +
+        'keys: `keep` and `stale`. Remove the entire `stale` key (and its value) using ' +
+        'edit_file. Keep `keep` untouched. Then verify the file is still valid JSON.',
+    setup: async (cwd) => {
+        await mkdir(cwd, { recursive: true });
+        await writeFile(join(cwd, 'config.json'), JSON.stringify({ keep: { value: 42 }, stale: { value: 'old', note: 'remove me' } }, null, 2));
+    },
+    graders: [
+        completed(),
+        toolWasCalled('edit_file'),
+        fileMatches('config.json', '"value": 42'),
+        fileMatches('config.json', 'keep'),
+        // The `stale` key should be gone: we check that the substring "stale"
+        // (which only appears inside the key) is absent.
+        fileMatches('config.json', { equals: JSON.stringify({ keep: { value: 42 } }, null, 2) }),
+        fileMissing('stale-backup.json'),
+    ],
+});
+//# sourceMappingURL=edit.remove-section.js.map

package/dist/tasks/edit.remove-section.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"edit.remove-section.js","sourceRoot":"","sources":["../../src/tasks/edit.remove-section.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,WAAW,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAErG;;;GAGG;AACH,eAAe,UAAU,CAAC;IACxB,EAAE,EAAE,qBAAqB;IACzB,IAAI,EAAE,mCAAmC;IACzC,QAAQ,EAAE,MAAM;IAChB,WAAW,EAAE,wDAAwD;IACrE,IAAI,EACF,+EAA+E;QAC/E,gFAAgF;QAChF,6EAA6E;IAC/E,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACnB,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACtC,MAAM,SAAS,CACb,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,EACxB,IAAI,CAAC,SAAS,CACZ,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,EACnE,IAAI,EACJ,CAAC,CACF,CACF,CAAC;IACJ,CAAC;IACD,OAAO,EAAE;QACP,SAAS,EAAE;QACX,aAAa,CAAC,WAAW,CAAC;QAC1B,WAAW,CAAC,aAAa,EAAE,aAAa,CAAC;QACzC,WAAW,CAAC,aAAa,EAAE,MAAM,CAAC;QAClC,sEAAsE;QACtE,iDAAiD;QACjD,WAAW,CAAC,aAAa,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;QACxF,WAAW,CAAC,mBAAmB,CAAC;KACjC;CACF,CAAC,CAAC"}

package/dist/tasks/edit.rename-symbol.d.ts

@@ -0,0 +1,16 @@
+/**
+ * edit / rename-symbol: multi-file rename. Agent must find files containing
+ * the symbol `foo` and rename it to `bar` consistently. This is a real-world
+ * refactor that touches multiple files and demands care.
+ */
+declare const _default: {
+    id: string;
+    name: string;
+    category: "edit";
+    description: string;
+    task: string;
+    setup: (cwd: string) => Promise<void>;
+    graders: import("../index.js").Grader[];
+};
+export default _default;
+//# sourceMappingURL=edit.rename-symbol.d.ts.map

package/dist/tasks/edit.rename-symbol.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"edit.rename-symbol.d.ts","sourceRoot":"","sources":["../../src/tasks/edit.rename-symbol.ts"],"names":[],"mappings":"AAKA;;;;GAIG;;;;;;;;;;AACH,wBAyBG"}

package/dist/tasks/edit.rename-symbol.js

@@ -0,0 +1,34 @@
+import { mkdir, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { defineTask, fileMatches, toolWasCalled, completed } from '../graders/index.js';
+/**
+ * edit / rename-symbol: multi-file rename. Agent must find files containing
+ * the symbol `foo` and rename it to `bar` consistently. This is a real-world
+ * refactor that touches multiple files and demands care.
+ */
+export default defineTask({
+    id: 'edit-rename-symbol',
+    name: 'Rename symbol foo -> bar',
+    category: 'edit',
+    description: 'Agent renames a symbol across multiple files.',
+    task: 'Search the current directory for files that contain the JavaScript variable ' +
+        'name `foo`. Rename every occurrence to `bar` in each file using edit_file. ' +
+        'Do not change any other words. Leave any file that does not contain `foo` ' +
+        'untouched.',
+    setup: async (cwd) => {
+        await mkdir(cwd, { recursive: true });
+        await writeFile(join(cwd, 'a.ts'), 'export const foo = 1;\nexport function useFoo() { return foo; }\n');
+        await writeFile(join(cwd, 'b.ts'), 'import { foo } from "./a";\nexport const doubled = foo * 2;\n');
+        await writeFile(join(cwd, 'c.ts'), '// no relevant symbols here\nexport const baz = 9;\n');
+    },
+    graders: [
+        completed(),
+        toolWasCalled('search_files'),
+        toolWasCalled('edit_file'),
+        fileMatches('a.ts', 'bar'),
+        fileMatches('b.ts', 'bar'),
+        // c.ts didn't have foo, must be untouched.
+        fileMatches('c.ts', 'baz'),
+    ],
+});
+//# sourceMappingURL=edit.rename-symbol.js.map

package/dist/tasks/edit.rename-symbol.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"edit.rename-symbol.js","sourceRoot":"","sources":["../../src/tasks/edit.rename-symbol.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAExF;;;;GAIG;AACH,eAAe,UAAU,CAAC;IACxB,EAAE,EAAE,oBAAoB;IACxB,IAAI,EAAE,0BAA0B;IAChC,QAAQ,EAAE,MAAM;IAChB,WAAW,EAAE,+CAA+C;IAC5D,IAAI,EACF,8EAA8E;QAC9E,6EAA6E;QAC7E,4EAA4E;QAC5E,YAAY;IACd,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACnB,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACtC,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,EAAE,mEAAmE,CAAC,CAAC;QACxG,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,EAAE,+DAA+D,CAAC,CAAC;QACpG,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,EAAE,sDAAsD,CAAC,CAAC;IAC7F,CAAC;IACD,OAAO,EAAE;QACP,SAAS,EAAE;QACX,aAAa,CAAC,cAAc,CAAC;QAC7B,aAAa,CAAC,WAAW,CAAC;QAC1B,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC;QAC1B,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC;QAC1B,2CAA2C;QAC3C,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC;KAC3B;CACF,CAAC,CAAC"}

package/dist/tasks/edit.replace-greeting.d.ts

@@ -0,0 +1,16 @@
+/**
+ * edit / replace-greeting: agent modifies an existing file in place.
+ * Verifies the model uses edit_file (precise replacement) rather than
+ * rewriting the whole file with write_file.
+ */
+declare const _default: {
+    id: string;
+    name: string;
+    category: "edit";
+    description: string;
+    task: string;
+    setup: (cwd: string) => Promise<void>;
+    graders: import("../index.js").Grader[];
+};
+export default _default;
+//# sourceMappingURL=edit.replace-greeting.d.ts.map

package/dist/tasks/edit.replace-greeting.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"edit.replace-greeting.d.ts","sourceRoot":"","sources":["../../src/tasks/edit.replace-greeting.ts"],"names":[],"mappings":"AAKA;;;;GAIG;;;;;;;;;;AACH,wBAoBG"}

package/dist/tasks/edit.replace-greeting.js

@@ -0,0 +1,29 @@
+import { mkdir, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { defineTask, fileMatches, toolWasCalled, completed } from '../graders/index.js';
+/**
+ * edit / replace-greeting: agent modifies an existing file in place.
+ * Verifies the model uses edit_file (precise replacement) rather than
+ * rewriting the whole file with write_file.
+ */
+export default defineTask({
+    id: 'edit-replace-greeting',
+    name: 'Replace greeting in file',
+    category: 'edit',
+    description: 'Agent edits greeting.txt to swap the greeting word.',
+    task: 'The file `greeting.txt` exists in the current directory and contains the word ' +
+        '"Hello". Change just that one word to "Bonjour" using the edit_file tool: do ' +
+        'not rewrite the whole file.',
+    setup: async (cwd) => {
+        await mkdir(cwd, { recursive: true });
+        await writeFile(join(cwd, 'greeting.txt'), 'Hello, world!\nA second line.\n');
+    },
+    graders: [
+        completed(),
+        toolWasCalled('edit_file'),
+        fileMatches('greeting.txt', 'Bonjour'),
+        // Make sure the model didn't nuke the rest of the file.
+        fileMatches('greeting.txt', 'A second line.'),
+    ],
+});
+//# sourceMappingURL=edit.replace-greeting.js.map

package/dist/tasks/edit.replace-greeting.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"edit.replace-greeting.js","sourceRoot":"","sources":["../../src/tasks/edit.replace-greeting.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAExF;;;;GAIG;AACH,eAAe,UAAU,CAAC;IACxB,EAAE,EAAE,uBAAuB;IAC3B,IAAI,EAAE,0BAA0B;IAChC,QAAQ,EAAE,MAAM;IAChB,WAAW,EAAE,qDAAqD;IAClE,IAAI,EACF,gFAAgF;QAChF,+EAA+E;QAC/E,6BAA6B;IAC/B,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACnB,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACtC,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,EAAE,iCAAiC,CAAC,CAAC;IAChF,CAAC;IACD,OAAO,EAAE;QACP,SAAS,EAAE;QACX,aAAa,CAAC,WAAW,CAAC;QAC1B,WAAW,CAAC,cAAc,EAAE,SAAS,CAAC;QACtC,wDAAwD;QACxD,WAAW,CAAC,cAAc,EAAE,gBAAgB,CAAC;KAC9C;CACF,CAAC,CAAC"}

package/dist/tasks/edit.write-hello.d.ts

@@ -0,0 +1,15 @@
+/**
+ * edit / write-hello: agent creates a fresh file with specific content.
+ * Verifies the model uses write_file (not edit_file on a non-existent file).
+ */
+declare const _default: {
+    id: string;
+    name: string;
+    category: "edit";
+    description: string;
+    task: string;
+    setup: () => Promise<void>;
+    graders: import("../index.js").Grader[];
+};
+export default _default;
+//# sourceMappingURL=edit.write-hello.d.ts.map

package/dist/tasks/edit.write-hello.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"edit.write-hello.d.ts","sourceRoot":"","sources":["../../src/tasks/edit.write-hello.ts"],"names":[],"mappings":"AAEA;;;GAGG;;;;;;;;;;AACH,wBAgBG"}

package/dist/tasks/edit.write-hello.js

@@ -0,0 +1,22 @@
+import { defineTask, fileMatches, toolWasCalled, completed } from '../graders/index.js';
+/**
+ * edit / write-hello: agent creates a fresh file with specific content.
+ * Verifies the model uses write_file (not edit_file on a non-existent file).
+ */
+export default defineTask({
+    id: 'edit-write-hello',
+    name: 'Write hello.txt',
+    category: 'edit',
+    description: 'Agent creates hello.txt with a specific greeting.',
+    task: 'Create a new file called `hello.txt` in the current directory whose entire ' +
+        'contents are exactly the text: `Hello, Forge!` (one line, no trailing newline needed).',
+    setup: async () => {
+        // No fixtures: fresh dir.
+    },
+    graders: [
+        completed(),
+        toolWasCalled('write_file'),
+        fileMatches('hello.txt', { equals: 'Hello, Forge!' }),
+    ],
+});
+//# sourceMappingURL=edit.write-hello.js.map

package/dist/tasks/edit.write-hello.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"edit.write-hello.js","sourceRoot":"","sources":["../../src/tasks/edit.write-hello.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAExF;;;GAGG;AACH,eAAe,UAAU,CAAC;IACxB,EAAE,EAAE,kBAAkB;IACtB,IAAI,EAAE,iBAAiB;IACvB,QAAQ,EAAE,MAAM;IAChB,WAAW,EAAE,mDAAmD;IAChE,IAAI,EACF,6EAA6E;QAC7E,wFAAwF;IAC1F,KAAK,EAAE,KAAK,IAAI,EAAE;QAChB,0BAA0B;IAC5B,CAAC;IACD,OAAO,EAAE;QACP,SAAS,EAAE;QACX,aAAa,CAAC,YAAY,CAAC;QAC3B,WAAW,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC;KACtD;CACF,CAAC,CAAC"}