@ikarmic/forge-eval 0.0.0-stage → 4.5.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 IKarmic
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,3 +1,96 @@
-# Temporary Holding Version
+# @ikarmic/forge-eval
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Eval harness for [IKarmic AI Forge](https://github.com/ikarmicai/ikarmic-forge). Runs agent
+tasks against isolated workspaces and grades the results with deterministic checks: 
+no API keys needed to *grade*, only to *run*.
+
+## What's in the box
+
+- **5 free graders**: `regex`, `contains`, `equals`, `file-exists`, `command-runs`.
+  Deterministic, no LLM-as-judge, no API spend.
+- **10 curated tasks** spanning three categories:
+  - **read**: list / read / search style tasks
+  - **edit**: write / modify / delete style tasks
+  - **multi-step**: tasks that need search + edit, run + report, or read + summarize
+- **Per-task isolated cwd**: every task runs in its own temp directory so tasks can't
+  affect each other or your real project.
+- **CLI**: `forge-eval run [--model X] [--task Y] [--json]` plus
+  `forge-eval list` for browsing the task catalog.
+- **Markdown + JSON reports** so you can diff runs over time.
+
+## Install / build
+
+This package lives in the Forge monorepo. From the repo root:
+
+```bash
+pnpm -r build
+pnpm --filter @ikarmic/forge-eval test
+```
+
+## CLI usage
+
+```bash
+# Run the full suite against whatever model FORGE_DEFAULT_* resolves to.
+pnpm forge-eval run
+
+# Run a single task by id.
+pnpm forge-eval run --task read-package-version
+
+# Pin a specific model.
+pnpm forge-eval run --model anthropic/claude-3-5-haiku-latest
+
+# JSON output (one event per line) for piping into scripts.
+pnpm forge-eval run --json | tee run.jsonl
+
+# Browse the task catalog.
+pnpm forge-eval list
+```
+
+The CLI exits 0 if all tasks passed, 1 if any failed, 2 if the run couldn't start
+(missing model, bad task id, etc.).
+
+## Task authoring
+
+A task is just an object: setup writes fixture files into a temp dir, the agent runs
+against that dir, then graders inspect the result + filesystem.
+
+```ts
+import { defineTask, regex, fileExists, commandRuns } from '@ikarmic/forge-eval';
+
+export default defineTask({
+  id: 'my-task',
+  name: 'My task',
+  category: 'edit',
+  description: 'Does X then Y',
+  task: 'Write "hi" to greeting.txt',
+  setup: async (cwd) => {
+    await fs.writeFile(path.join(cwd, '.gitkeep'), '');
+  },
+  graders: [
+    fileExists('greeting.txt'),
+    commandRuns({ command: 'node -e "require(\'fs\').readFileSync(\'greeting.txt\',\'utf8\')"', contains: 'hi' }),
+  ],
+});
+```
+
+Drop the task file in `src/tasks/your-task.ts`, register it in `src/tasks/index.ts`, done.
+
+## Grader reference
+
+| Grader | What it checks |
+|---|---|
+| `regex(pattern, opts?)` | `result.finalText` matches a regex (and optionally case-insensitive / multiline) |
+| `contains(substring)` | `result.finalText` contains a substring (case-sensitive) |
+| `equals(text)` | `result.finalText.trim()` equals `text.trim()` |
+| `fileExists(relPath)` | A file exists at `cwd/relPath` |
+| `commandRuns(opts)` | Runs a shell command via `child_process.exec` in `cwd`; checks exit code 0 + optional stdout predicate |
+
+All graders receive `{ cwd, task, result, toolInvocations }` so you can also write
+custom graders against `result.messages` or the recorded tool calls.
+
+## Limitations
+
+- Free graders only: no LLM-as-judge in v1 (that needs API keys per task).
+- 10 tasks in v1: the catalog is intentionally small. More tasks are welcome
+  via PR.
+- Tasks assume a writable temp dir and a working `node` on PATH (for `command-runs`).

package/dist/bin/forge-eval.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=forge-eval.d.ts.map

package/dist/bin/forge-eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"forge-eval.d.ts","sourceRoot":"","sources":["../../src/bin/forge-eval.ts"],"names":[],"mappings":""}

package/dist/bin/forge-eval.js

@@ -0,0 +1,138 @@
+#!/usr/bin/env node
+import process from 'node:process';
+import { findTask, TASKS } from '../tasks/index.js';
+import { formatJsonReport, formatMarkdownReport, formatTerminalSummary } from '../report.js';
+import { resolveModel, runSuite } from '../runner.js';
+function parseArgs(argv) {
+    const args = argv.slice(2);
+    const out = {
+        command: 'run',
+        json: false,
+        markdown: false,
+        help: false,
+    };
+    const takeNext = (flag) => {
+        const v = args.shift();
+        if (v === undefined || v.startsWith('--')) {
+            throw new Error(`${flag} requires a value`);
+        }
+        return v;
+    };
+    // Subcommand.
+    if (args[0] === 'list') {
+        args.shift();
+        out.command = 'list';
+    }
+    else if (args[0] === 'help' || args[0] === '--help' || args[0] === '-h') {
+        args.shift();
+        out.command = 'help';
+        out.help = true;
+    }
+    else if (args[0] === 'run') {
+        args.shift();
+        out.command = 'run';
+    }
+    while (args.length > 0) {
+        const a = args.shift();
+        if (a === '--model') {
+            out.model = takeNext('--model');
+        }
+        else if (a === '--task') {
+            out.task = takeNext('--task');
+        }
+        else if (a === '--json') {
+            out.json = true;
+        }
+        else if (a === '--markdown') {
+            out.markdown = true;
+        }
+        else if (a === '--help' || a === '-h') {
+            out.help = true;
+        }
+        else {
+            throw new Error(`Unknown argument: ${a}`);
+        }
+    }
+    return out;
+}
+function printHelp() {
+    process.stdout.write(`forge-eval: run Forge agent tasks and grade the results.\n\n` +
+        `Usage:\n` +
+        `  forge-eval run [--model <provider>/<model>] [--task <id>] [--json | --markdown]\n` +
+        `  forge-eval list\n` +
+        `  forge-eval help\n\n` +
+        `Flags:\n` +
+        `  --model <p>/<m>   Override the model. Defaults to FORGE_DEFAULT_MODEL / provider env vars.\n` +
+        `  --task <id>       Run only the named task (see \`forge-eval list\`).\n` +
+        `  --json            Emit a single JSON report on stdout.\n` +
+        `  --markdown        Emit a markdown report on stdout (in addition to terminal summary).\n\n` +
+        `Exit codes: 0 = all passed, 1 = any failed, 2 = run couldn't start.\n`);
+}
+function printList() {
+    process.stdout.write(`# Tasks (${TASKS.length})\n\n`);
+    for (const t of TASKS) {
+        process.stdout.write(`## ${t.id}: ${t.name} [${t.category}]\n${t.description}\n\n`);
+    }
+}
+async function main() {
+    const args = parseArgs(process.argv);
+    if (args.command === 'help' || args.help) {
+        printHelp();
+        return;
+    }
+    if (args.command === 'list') {
+        printList();
+        return;
+    }
+    // command === 'run' from here on
+    // Task lookup first: it doesn't need a model, so we can fail fast on a
+    // bogus --task id before bothering to resolve credentials.
+    let tasks;
+    if (args.task) {
+        const t = findTask(args.task);
+        if (!t) {
+            process.stderr.write(`forge-eval: unknown task id "${args.task}". Run \`forge-eval list\` to see options.\n`);
+            process.exit(2);
+            return;
+        }
+        tasks = [t];
+    }
+    else {
+        tasks = TASKS;
+    }
+    let model;
+    try {
+        model = resolveModel(args.model);
+    }
+    catch (err) {
+        process.stderr.write(`forge-eval: ${err.message}\n`);
+        process.exit(2);
+        return;
+    }
+    if (!args.json) {
+        process.stderr.write(`forge-eval: running ${tasks.length} task(s) against ${model.provider}/${model.model}...\n`);
+    }
+    try {
+        const suite = await runSuite(tasks, model);
+        if (args.json) {
+            const report = formatJsonReport(suite);
+            process.stdout.write(JSON.stringify(report, null, 2) + '\n');
+        }
+        else {
+            process.stdout.write('\n' + formatTerminalSummary(suite) + '\n');
+            if (args.markdown) {
+                process.stdout.write('\n' + formatMarkdownReport(suite) + '\n');
+            }
+        }
+        process.exit(suite.summary.failed === 0 ? 0 : 1);
+    }
+    catch (err) {
+        process.stderr.write(`forge-eval: ${err.message}\n`);
+        process.exit(2);
+    }
+}
+main().catch((err) => {
+    process.stderr.write(`forge-eval: ${err.message}\n`);
+    process.exit(1);
+});
+//# sourceMappingURL=forge-eval.js.map

package/dist/bin/forge-eval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"forge-eval.js","sourceRoot":"","sources":["../../src/bin/forge-eval.ts"],"names":[],"mappings":";AACA,OAAO,OAAO,MAAM,cAAc,CAAC;AAEnC,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAC7F,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAWtD,SAAS,SAAS,CAAC,IAAc;IAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC3B,MAAM,GAAG,GAAY;QACnB,OAAO,EAAE,KAAK;QACd,IAAI,EAAE,KAAK;QACX,QAAQ,EAAE,KAAK;QACf,IAAI,EAAE,KAAK;KACZ,CAAC;IAEF,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAU,EAAE;QACxC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QACvB,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1C,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,mBAAmB,CAAC,CAAC;QAC9C,CAAC;QACD,OAAO,CAAC,CAAC;IACX,CAAC,CAAC;IAEF,cAAc;IACd,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,KAAK,EAAE,CAAC;QACb,GAAG,CAAC,OAAO,GAAG,MAAM,CAAC;IACvB,CAAC;SAAM,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC1E,IAAI,CAAC,KAAK,EAAE,CAAC;QACb,GAAG,CAAC,OAAO,GAAG,MAAM,CAAC;QACrB,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;IAClB,CAAC;SAAM,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,KAAK,EAAE,CAAC;QAC7B,IAAI,CAAC,KAAK,EAAE,CAAC;QACb,GAAG,CAAC,OAAO,GAAG,KAAK,CAAC;IACtB,CAAC;IAED,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QACvB,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;YACpB,GAAG,CAAC,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;QAClC,CAAC;aAAM,IAAI,CAAC,KAAK,QAAQ,EAAE,CAAC;YAC1B,GAAG,CAAC,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAChC,CAAC;aAAM,IAAI,CAAC,KAAK,QAAQ,EAAE,CAAC;YAC1B,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;QAClB,CAAC;aAAM,IAAI,CAAC,KAAK,YAAY,EAAE,CAAC;YAC9B,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC;QACtB,CAAC;aAAM,IAAI,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;YACxC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC;QAClB,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,SAAS;IAChB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,8DAA8D;QAC5D,UAAU;QACV,qFAAqF;QACrF,qBAAqB;QACrB,uBAAuB;QACvB,UAAU;QACV,gGAAgG;QAChG,0EAA0E;QAC1E,4DAA4D;QAC5D,6FAA6F;QAC7F,uEAAuE,CAC1E,CAAC;AACJ,CAAC;AAED,SAAS,SAAS;IAChB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,KAAK,CAAC,MAAM,OAAO,CAAC,CAAC;IACtD,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,QAAQ,MAAM,CAAC,CAAC,WAAW,MAAM,CAAC,CAAC;IACtF,CAAC;AACH,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAErC,IAAI,IAAI,CAAC,OAAO,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACzC,SAAS,EAAE,CAAC;QACZ,OAAO;IACT,CAAC;IACD,IAAI,IAAI,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;QAC5B,SAAS,EAAE,CAAC;QACZ,OAAO;IACT,CAAC;IAED,iCAAiC;IACjC,uEAAuE;IACvE,2DAA2D;IAC3D,IAAI,KAA8D,CAAC;IACnE,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9B,IAAI,CAAC,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,IAAI,CAAC,IAAI,8CAA8C,CAAC,CAAC;YAC9G,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChB,OAAO;QACT,CAAC;QACD,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;IACd,CAAC;SAAM,CAAC;QACN,KAAK,GAAG,KAAK,CAAC;IAChB,CAAC;IAED,IAAI,KAAK,CAAC;IACV,IAAI,CAAC;QACH,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,eAAgB,GAAa,CAAC,OAAO,IAAI,CAAC,CAAC;QAChE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChB,OAAO;IACT,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,uBAAuB,KAAK,CAAC,MAAM,oBAAoB,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,KAAK,OAAO,CAC5F,CAAC;IACJ,CAAC;IAED,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAE3C,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;YACvC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;QAC/D,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,qBAAqB,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC;YACjE,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,oBAAoB,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC;YAClE,CAAC;QACH,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,eAAgB,GAAa,CAAC,OAAO,IAAI,CAAC,CAAC;QAChE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAU,EAAE,EAAE;IAC1B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC;IACrD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}

package/dist/graders/behavior.d.ts

@@ -0,0 +1,57 @@
+import type { Grader } from './types.js';
+/**
+ * Grader that passes when the agent called a given tool at least `min` times
+ * during the run. Use this to assert the model took the right path (vs. just
+ * guessing in its final text).
+ */
+export declare function toolWasCalled(name: string, min?: number): Grader;
+/**
+ * Grader that passes when the agent never called a given tool. Useful for
+ * "the agent should have done this without writing a file" tasks.
+ */
+export declare function toolNotCalled(name: string): Grader;
+/**
+ * Grader that passes when `finalText` is non-empty. Catches "model went
+ * silent" runs that some graders above would also catch indirectly.
+ */
+export declare function finalTextNonEmpty(): Grader;
+/**
+ * Grader that passes when the agent finished with `reason === 'completed'`.
+ * Catches runs that aborted early (max_iterations, error, etc.): even if
+ * the other graders happened to pass.
+ */
+export declare function completed(): Grader;
+/**
+ * Options for {@link toolCount}.
+ *
+ * At least one of `min` / `max` / `exact` must be provided. If
+ * multiple are passed they're AND-ed together: `exact` is the
+ * strongest constraint.
+ */
+export interface ToolCountOptions {
+    min?: number;
+    max?: number;
+    exact?: number;
+}
+/**
+ * Grader that passes when the call count for `name` falls within
+ * `min` / `max` (inclusive). Use this to assert a budget
+ * ("didn't spam the read tool") or a minimum ("read at least once
+ * before guessing").
+ */
+export declare function toolCount(name: string, options?: ToolCountOptions): Grader;
+/**
+ * Grader that passes when at least one call to `name` had
+ * `predicate(args)` return true. Use this to assert the agent
+ * invoked a tool with the right arguments (not just the right tool
+ * name): e.g. "the agent called `edit_file` with a non-empty diff"
+ * vs. "the agent called `edit_file` 5 times on autopilot".
+ */
+export declare function toolCalledWith(name: string, predicate: (args: unknown) => boolean): Grader;
+/**
+ * Grader that passes when no tool call during the run returned an
+ * error. Useful as a baseline guard: any task that says "agent did
+ * something" should also assert it didn't fall over partway.
+ */
+export declare function noToolErrors(): Grader;
+//# sourceMappingURL=behavior.d.ts.map

package/dist/graders/behavior.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"behavior.d.ts","sourceRoot":"","sources":["../../src/graders/behavior.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAA+B,MAAM,YAAY,CAAC;AAEtE;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,SAAI,GAAG,MAAM,CAO3D;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAOlD;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,IAAI,MAAM,CAO1C;AAED;;;;GAIG;AACH,wBAAgB,SAAS,IAAI,MAAM,CAMlC;AAED;;;;;;GAMG;AACH,MAAM,WAAW,gBAAgB;IAC/B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;CAEhB;AAED;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,MAAM,CA0B9E;AAED;;;;;;GAMG;AACH,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,OAAO,GACpC,MAAM,CAsBR;AAED;;;;GAIG;AACH,wBAAgB,YAAY,IAAI,MAAM,CAcrC"}

package/dist/graders/behavior.js

@@ -0,0 +1,154 @@
+/**
+ * Grader that passes when the agent called a given tool at least `min` times
+ * during the run. Use this to assert the model took the right path (vs. just
+ * guessing in its final text).
+ */
+export function toolWasCalled(name, min = 1) {
+    return (ctx) => {
+        const calls = ctx.toolInvocations.filter((t) => t.name === name).length;
+        return calls >= min
+            ? { passed: true, message: `tool ${name} called ${calls}x` }
+            : { passed: false, message: `tool ${name} called ${calls}x (need ${min})` };
+    };
+}
+/**
+ * Grader that passes when the agent never called a given tool. Useful for
+ * "the agent should have done this without writing a file" tasks.
+ */
+export function toolNotCalled(name) {
+    return (ctx) => {
+        const calls = ctx.toolInvocations.filter((t) => t.name === name).length;
+        return calls === 0
+            ? { passed: true, message: `tool ${name} was not called` }
+            : { passed: false, message: `tool ${name} was called ${calls}x but should not have been` };
+    };
+}
+/**
+ * Grader that passes when `finalText` is non-empty. Catches "model went
+ * silent" runs that some graders above would also catch indirectly.
+ */
+export function finalTextNonEmpty() {
+    return (ctx) => {
+        const text = ctx.result.finalText?.trim() ?? '';
+        return text.length > 0
+            ? { passed: true, message: 'finalText is non-empty' }
+            : { passed: false, message: 'finalText is empty' };
+    };
+}
+/**
+ * Grader that passes when the agent finished with `reason === 'completed'`.
+ * Catches runs that aborted early (max_iterations, error, etc.): even if
+ * the other graders happened to pass.
+ */
+export function completed() {
+    return (ctx) => {
+        return ctx.result.reason === 'completed'
+            ? { passed: true, message: `reason: completed` }
+            : { passed: false, message: `reason: ${ctx.result.reason}` };
+    };
+}
+/**
+ * Grader that passes when the call count for `name` falls within
+ * `min` / `max` (inclusive). Use this to assert a budget
+ * ("didn't spam the read tool") or a minimum ("read at least once
+ * before guessing").
+ */
+export function toolCount(name, options = {}) {
+    if (options.min === undefined && options.max === undefined && options.exact === undefined) {
+        throw new Error('toolCount requires at least one of min/max/exact');
+    }
+    const { min = 0, max = Number.POSITIVE_INFINITY, exact } = options;
+    const effectiveMin = exact ?? min;
+    const effectiveMax = exact ?? max;
+    return (ctx) => {
+        const calls = ctx.toolInvocations.filter((t) => t.name === name).length;
+        if (calls < effectiveMin) {
+            return {
+                passed: false,
+                message: `tool ${name} called ${calls}x (need at least ${effectiveMin})`,
+            };
+        }
+        if (calls > effectiveMax) {
+            return {
+                passed: false,
+                message: `tool ${name} called ${calls}x (at most ${effectiveMax})`,
+            };
+        }
+        return {
+            passed: true,
+            message: `tool ${name} called ${calls}x (in [${effectiveMin}, ${effectiveMax}])`,
+        };
+    };
+}
+/**
+ * Grader that passes when at least one call to `name` had
+ * `predicate(args)` return true. Use this to assert the agent
+ * invoked a tool with the right arguments (not just the right tool
+ * name): e.g. "the agent called `edit_file` with a non-empty diff"
+ * vs. "the agent called `edit_file` 5 times on autopilot".
+ */
+export function toolCalledWith(name, predicate) {
+    return (ctx) => {
+        const calls = ctx.toolInvocations.filter((t) => t.name === name);
+        if (calls.length === 0) {
+            return { passed: false, message: `tool ${name} was never called` };
+        }
+        const matching = calls.filter((t) => predicate(t.args));
+        if (matching.length > 0) {
+            return {
+                passed: true,
+                message: `tool ${name} called with matching args ${matching.length}x of ${calls.length}`,
+            };
+        }
+        return {
+            passed: false,
+            message: `tool ${name} called ${calls.length}x but no call's args matched predicate`,
+            details: `observed arg samples: ${calls
+                .slice(0, 3)
+                .map((c) => summarizeArgs(c.args))
+                .join(' | ')}`,
+        };
+    };
+}
+/**
+ * Grader that passes when no tool call during the run returned an
+ * error. Useful as a baseline guard: any task that says "agent did
+ * something" should also assert it didn't fall over partway.
+ */
+export function noToolErrors() {
+    return (ctx) => {
+        const errs = ctx.toolInvocations.filter((t) => t.isError);
+        if (errs.length === 0) {
+            return { passed: true, message: 'no tool errors' };
+        }
+        return {
+            passed: false,
+            message: `${errs.length} tool call(s) errored`,
+            details: errs
+                .map((e) => `  - ${e.name}: ${shortError(e.result)}`)
+                .join('\n'),
+        };
+    };
+}
+function shortError(result) {
+    if (result instanceof Error)
+        return result.message;
+    if (typeof result === 'string')
+        return result;
+    try {
+        return JSON.stringify(result).slice(0, 200);
+    }
+    catch {
+        return '<unserializable>';
+    }
+}
+function summarizeArgs(args) {
+    try {
+        const s = JSON.stringify(args);
+        return s.length > 80 ? s.slice(0, 77) + '…' : s;
+    }
+    catch {
+        return '<unserializable>';
+    }
+}
+//# sourceMappingURL=behavior.js.map

package/dist/graders/behavior.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"behavior.js","sourceRoot":"","sources":["../../src/graders/behavior.ts"],"names":[],"mappings":"AAEA;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,IAAY,EAAE,GAAG,GAAG,CAAC;IACjD,OAAO,CAAC,GAAkB,EAAgB,EAAE;QAC1C,MAAM,KAAK,GAAG,GAAG,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;QACxE,OAAO,KAAK,IAAI,GAAG;YACjB,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,IAAI,WAAW,KAAK,GAAG,EAAE;YAC5D,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,IAAI,WAAW,KAAK,WAAW,GAAG,GAAG,EAAE,CAAC;IAChF,CAAC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,OAAO,CAAC,GAAkB,EAAgB,EAAE;QAC1C,MAAM,KAAK,GAAG,GAAG,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;QACxE,OAAO,KAAK,KAAK,CAAC;YAChB,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,IAAI,iBAAiB,EAAE;YAC1D,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,IAAI,eAAe,KAAK,4BAA4B,EAAE,CAAC;IAC/F,CAAC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB;IAC/B,OAAO,CAAC,GAAkB,EAAgB,EAAE;QAC1C,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAChD,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC;YACpB,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,wBAAwB,EAAE;YACrD,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,oBAAoB,EAAE,CAAC;IACvD,CAAC,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,SAAS;IACvB,OAAO,CAAC,GAAkB,EAAgB,EAAE;QAC1C,OAAO,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,WAAW;YACtC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,mBAAmB,EAAE;YAChD,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;IACjE,CAAC,CAAC;AACJ,CAAC;AAgBD;;;;;GAKG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,UAA4B,EAAE;IACpE,IAAI,OAAO,CAAC,GAAG,KAAK,SAAS,IAAI,OAAO,CAAC,GAAG,KAAK,SAAS,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC1F,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;IACtE,CAAC;IACD,MAAM,EAAE,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,MAAM,CAAC,iBAAiB,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;IACnE,MAAM,YAAY,GAAG,KAAK,IAAI,GAAG,CAAC;IAClC,MAAM,YAAY,GAAG,KAAK,IAAI,GAAG,CAAC;IAClC,OAAO,CAAC,GAAkB,EAAgB,EAAE;QAC1C,MAAM,KAAK,GAAG,GAAG,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;QACxE,IAAI,KAAK,GAAG,YAAY,EAAE,CAAC;YACzB,OAAO;gBACL,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,QAAQ,IAAI,WAAW,KAAK,oBAAoB,YAAY,GAAG;aACzE,CAAC;QACJ,CAAC;QACD,IAAI,KAAK,GAAG,YAAY,EAAE,CAAC;YACzB,OAAO;gBACL,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,QAAQ,IAAI,WAAW,KAAK,cAAc,YAAY,GAAG;aACnE,CAAC;QACJ,CAAC;QACD,OAAO;YACL,MAAM,EAAE,IAAI;YACZ,OAAO,EAAE,QAAQ,IAAI,WAAW,KAAK,UAAU,YAAY,KAAK,YAAY,IAAI;SACjF,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAC5B,IAAY,EACZ,SAAqC;IAErC,OAAO,CAAC,GAAkB,EAAgB,EAAE;QAC1C,MAAM,KAAK,GAAG,GAAG,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;QACjE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,IAAI,mBAAmB,EAAE,CAAC;QACrE,CAAC;QACD,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACxD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,OAAO;gBACL,MAAM,EAAE,IAAI;gBACZ,OAAO,EAAE,QAAQ,IAAI,8BAA8B,QAAQ,CAAC,MAAM,QAAQ,KAAK,CAAC,MAAM,EAAE;aACzF,CAAC;QACJ,CAAC;QACD,OAAO;YACL,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,QAAQ,IAAI,WAAW,KAAK,CAAC,MAAM,wCAAwC;YACpF,OAAO,EAAE,yBAAyB,KAAK;iBACpC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;iBACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;iBACjC,IAAI,CAAC,KAAK,CAAC,EAAE;SACjB,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY;IAC1B,OAAO,CAAC,GAAkB,EAAgB,EAAE;QAC1C,MAAM,IAAI,GAAG,GAAG,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC1D,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC;QACrD,CAAC;QACD,OAAO;YACL,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,uBAAuB;YAC9C,OAAO,EAAE,IAAI;iBACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;iBACpD,IAAI,CAAC,IAAI,CAAC;SACd,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,MAAe;IACjC,IAAI,MAAM,YAAY,KAAK;QAAE,OAAO,MAAM,CAAC,OAAO,CAAC;IACnD,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC;IAC9C,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,kBAAkB,CAAC;IAC5B,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,IAAa;IAClC,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC/B,OAAO,CAAC,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,kBAAkB,CAAC;IAC5B,CAAC;AACH,CAAC"}

package/dist/graders/content.d.ts

@@ -0,0 +1,59 @@
+/**
+ * Structured content extraction graders. v2.13.
+ *
+ * Helpers for tasks where the agent's final text is expected to
+ * contain extractable structured content (a JSON object, a fenced
+ * code block in a specific language). These complement the
+ * full-schema graders in `./schema.ts`: where `jsonSchema` requires a
+ * Zod schema, `jsonContains` accepts a plain predicate so you don't
+ * need to write a schema for "the response is roughly shaped like X".
+ */
+import type { Grader } from './types.js';
+/** Options for {@link languageBlock}. */
+export interface LanguageBlockOptions {
+    /**
+     * Truncate the extracted block content in failure details to this
+     * many characters. Default 320.
+     */
+    maxDetailChars?: number;
+    /**
+     * When true (default), the grader fails when the fenced block is
+     * missing. When false, a missing block is acceptable and yields a
+     * pass.
+     */
+    requireBlock?: boolean;
+}
+/**
+ * Grader that extracts the first <code>```&lt;lang&gt; ... ```</code>
+ * fenced block from `finalText` and passes when `predicate(block)`
+ * returns true. Language matching is case-insensitive.
+ *
+ * Use this for tasks like "agent produced a runnable Python snippet
+ * with a `solve()` function" without parsing the whole response
+ * manually.
+ *
+ * ```ts
+ * graders: [
+ *   languageBlock('ts', (code) => /export function/.test(code)),
+ * ]
+ * ```
+ */
+export declare function languageBlock(lang: string, predicate: (code: string) => boolean, options?: LanguageBlockOptions): Grader;
+/**
+ * Predicate-driven JSON shape check. Parses `finalText` as JSON: raw
+ * text first, then a single fenced block: then calls
+ * `predicate(value)`. Passes when the predicate accepts.
+ *
+ * Use this instead of {@link jsonSchema} when the field set is loose,
+ * you want to assert a single property, or you don't want a Zod
+ * dependency in this task's grading.
+ *
+ * ```ts
+ * jsonContains((v) => {
+ *   return v && typeof v === 'object' &&
+ *     'name' in v && 'price' in v && (v as any).price > 0;
+ * })
+ * ```
+ */
+export declare function jsonContains(predicate: (value: unknown) => boolean): Grader;
+//# sourceMappingURL=content.d.ts.map

package/dist/graders/content.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"content.d.ts","sourceRoot":"","sources":["../../src/graders/content.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,KAAK,EAAE,MAAM,EAA+B,MAAM,YAAY,CAAC;AAEtE,yCAAyC;AACzC,MAAM,WAAW,oBAAoB;IACnC;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;OAIG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,aAAa,CAC3B,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,EACpC,OAAO,GAAE,oBAAyB,GACjC,MAAM,CAuBR;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,YAAY,CAAC,SAAS,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,GAAG,MAAM,CAmB3E"}