@ikarmic/forge-eval 0.0.0-stage → 4.5.0

package/dist/tasks/index.d.ts

@@ -0,0 +1,9 @@
+import type { TaskDefinition } from '../graders/index.js';
+/**
+ * The full task catalog. Order is the default run order. Keep grouped by
+ * category so the report reads naturally.
+ */
+export declare const TASKS: readonly TaskDefinition[];
+/** Lookup by id; returns undefined if missing. */
+export declare function findTask(id: string): TaskDefinition | undefined;
+//# sourceMappingURL=index.d.ts.map

package/dist/tasks/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tasks/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAc1D;;;GAGG;AACH,eAAO,MAAM,KAAK,EAAE,SAAS,cAAc,EAgB1C,CAAC;AAEF,kDAAkD;AAClD,wBAAgB,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAE/D"}

package/dist/tasks/index.js

@@ -0,0 +1,37 @@
+import readListFiles from './read.list-files.js';
+import readPackageVersion from './read.package-version.js';
+import readSearchTodos from './read.search-todos.js';
+import editWriteHello from './edit.write-hello.js';
+import editReplaceGreeting from './edit.replace-greeting.js';
+import editRemoveSection from './edit.remove-section.js';
+import editRenameSymbol from './edit.rename-symbol.js';
+import multiRunAndReport from './multi.run-and-report.js';
+import multiRefactorMove from './multi.refactor-move.js';
+import multiSummarize from './multi.summarize.js';
+import memoryRecallRelevant from './memory.recall-relevant.js';
+/**
+ * The full task catalog. Order is the default run order. Keep grouped by
+ * category so the report reads naturally.
+ */
+export const TASKS = [
+    // read
+    readListFiles,
+    readPackageVersion,
+    readSearchTodos,
+    // edit
+    editWriteHello,
+    editReplaceGreeting,
+    editRemoveSection,
+    editRenameSymbol,
+    // multi-step
+    multiRunAndReport,
+    multiRefactorMove,
+    multiSummarize,
+    // memory (appended so the indexes of the tasks above stay stable)
+    memoryRecallRelevant,
+];
+/** Lookup by id; returns undefined if missing. */
+export function findTask(id) {
+    return TASKS.find((t) => t.id === id);
+}
+//# sourceMappingURL=index.js.map

package/dist/tasks/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/tasks/index.ts"],"names":[],"mappings":"AAEA,OAAO,aAAa,MAAM,sBAAsB,CAAC;AACjD,OAAO,kBAAkB,MAAM,2BAA2B,CAAC;AAC3D,OAAO,eAAe,MAAM,wBAAwB,CAAC;AACrD,OAAO,cAAc,MAAM,uBAAuB,CAAC;AACnD,OAAO,mBAAmB,MAAM,4BAA4B,CAAC;AAC7D,OAAO,iBAAiB,MAAM,0BAA0B,CAAC;AACzD,OAAO,gBAAgB,MAAM,yBAAyB,CAAC;AACvD,OAAO,iBAAiB,MAAM,2BAA2B,CAAC;AAC1D,OAAO,iBAAiB,MAAM,0BAA0B,CAAC;AACzD,OAAO,cAAc,MAAM,sBAAsB,CAAC;AAClD,OAAO,oBAAoB,MAAM,6BAA6B,CAAC;AAE/D;;;GAGG;AACH,MAAM,CAAC,MAAM,KAAK,GAA8B;IAC9C,OAAO;IACP,aAAa;IACb,kBAAkB;IAClB,eAAe;IACf,OAAO;IACP,cAAc;IACd,mBAAmB;IACnB,iBAAiB;IACjB,gBAAgB;IAChB,aAAa;IACb,iBAAiB;IACjB,iBAAiB;IACjB,cAAc;IACd,kEAAkE;IAClE,oBAAoB;CACrB,CAAC;AAEF,kDAAkD;AAClD,MAAM,UAAU,QAAQ,CAAC,EAAU;IACjC,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;AACxC,CAAC"}

package/dist/tasks/memory.recall-relevant.d.ts

@@ -0,0 +1,19 @@
+/**
+ * memory / recall-relevant: the project's memory is larger than the prompt
+ * budget, so only an index plus the entries relevant to the question reach the
+ * model. The answer lives in one `correction` section ("use pnpm, never npm").
+ * Another section names a file that no longer exists, so it is injected with
+ * the "possibly stale" marker. Opts into memory (the only task that does).
+ */
+declare const _default: {
+    id: string;
+    name: string;
+    category: "read";
+    description: string;
+    task: string;
+    memory: true;
+    setup: (cwd: string) => Promise<void>;
+    graders: import("../index.js").Grader[];
+};
+export default _default;
+//# sourceMappingURL=memory.recall-relevant.d.ts.map

package/dist/tasks/memory.recall-relevant.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"memory.recall-relevant.d.ts","sourceRoot":"","sources":["../../src/tasks/memory.recall-relevant.ts"],"names":[],"mappings":"AAOA;;;;;;GAMG;;;;;;;;;;;AACH,wBAuBG"}

package/dist/tasks/memory.recall-relevant.js

@@ -0,0 +1,36 @@
+import { mkdir, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { defineTask, regex, toolNotCalled, completed } from '../graders/index.js';
+const FILLER_SENTENCE = 'Notes about this subsystem that have nothing to do with how anything is built or run. ';
+/**
+ * memory / recall-relevant: the project's memory is larger than the prompt
+ * budget, so only an index plus the entries relevant to the question reach the
+ * model. The answer lives in one `correction` section ("use pnpm, never npm").
+ * Another section names a file that no longer exists, so it is injected with
+ * the "possibly stale" marker. Opts into memory (the only task that does).
+ */
+export default defineTask({
+    id: 'memory-recall-relevant',
+    name: 'Recall the right memory entry',
+    category: 'read',
+    description: 'Agent answers from project memory: pnpm correction wins; the stale entry is flagged, not trusted.',
+    task: 'Which command runs the tests in this project? Reply with just the command, nothing else.',
+    memory: true,
+    setup: async (cwd) => {
+        await mkdir(join(cwd, '.forge'), { recursive: true });
+        await writeFile(join(cwd, 'package.json'), JSON.stringify({ name: 'sample', version: '1.0.0' }, null, 2));
+        const filler = Array.from({ length: 10 }, (_, i) => {
+            const n = String(i + 1).padStart(2, '0');
+            return `## Subsystem ${n}\n\n<!-- forge: type=convention, updated=2026-09-01 -->\n${FILLER_SENTENCE.repeat(10)}`;
+        });
+        const memory = [
+            '# Project Memory',
+            ...filler,
+            '## Package manager\n\n<!-- forge: type=correction, updated=2026-10-01, refs=package.json -->\nUse pnpm, never npm. Run `pnpm test` to run the tests in this workspace.',
+            '## Legacy API\n\n<!-- forge: type=decision, updated=2026-08-01, refs=src/legacy/api.ts, pin=true -->\nNever call the legacy API directly; go through the gateway.',
+        ].join('\n\n');
+        await writeFile(join(cwd, '.forge', 'MEMORY.md'), `${memory}\n`);
+    },
+    graders: [completed(), regex(/pnpm test/), toolNotCalled('run_command')],
+});
+//# sourceMappingURL=memory.recall-relevant.js.map

package/dist/tasks/memory.recall-relevant.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"memory.recall-relevant.js","sourceRoot":"","sources":["../../src/tasks/memory.recall-relevant.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAElF,MAAM,eAAe,GAAG,wFAAwF,CAAC;AAEjH;;;;;;GAMG;AACH,eAAe,UAAU,CAAC;IACxB,EAAE,EAAE,wBAAwB;IAC5B,IAAI,EAAE,+BAA+B;IACrC,QAAQ,EAAE,MAAM;IAChB,WAAW,EAAE,mGAAmG;IAChH,IAAI,EAAE,0FAA0F;IAChG,MAAM,EAAE,IAAI;IACZ,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACnB,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC1G,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACjD,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YACzC,OAAO,gBAAgB,CAAC,4DAA4D,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACnH,CAAC,CAAC,CAAC;QACH,MAAM,MAAM,GAAG;YACb,kBAAkB;YAClB,GAAG,MAAM;YACT,wKAAwK;YACxK,mKAAmK;SACpK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACf,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,WAAW,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC;IACnE,CAAC;IACD,OAAO,EAAE,CAAC,SAAS,EAAE,EAAE,KAAK,CAAC,WAAW,CAAC,EAAE,aAAa,CAAC,aAAa,CAAC,CAAC;CACzE,CAAC,CAAC"}

package/dist/tasks/multi.refactor-move.d.ts

@@ -0,0 +1,16 @@
+/**
+ * multi-step / refactor-move: agent must move a function from one file to
+ * another and remove it from the source. Tests reading + writing + editing
+ * in a coordinated way.
+ */
+declare const _default: {
+    id: string;
+    name: string;
+    category: "multi-step";
+    description: string;
+    task: string;
+    setup: (cwd: string) => Promise<void>;
+    graders: import("../index.js").Grader[];
+};
+export default _default;
+//# sourceMappingURL=multi.refactor-move.d.ts.map

package/dist/tasks/multi.refactor-move.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"multi.refactor-move.d.ts","sourceRoot":"","sources":["../../src/tasks/multi.refactor-move.ts"],"names":[],"mappings":"AAKA;;;;GAIG;;;;;;;;;;AACH,wBAkCG"}

package/dist/tasks/multi.refactor-move.js

@@ -0,0 +1,37 @@
+import { mkdir, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { defineTask, fileMatches, fileMissing, toolWasCalled, completed, commandRuns } from '../graders/index.js';
+/**
+ * multi-step / refactor-move: agent must move a function from one file to
+ * another and remove it from the source. Tests reading + writing + editing
+ * in a coordinated way.
+ */
+export default defineTask({
+    id: 'multi-refactor-move',
+    name: 'Move function between files',
+    category: 'multi-step',
+    description: 'Agent moves a function from a.ts to b.ts and removes it from a.ts.',
+    task: 'The current directory has two files: `a.ts` and `b.ts`. `a.ts` exports a ' +
+        'function called `greet`. Move the `greet` function (and only that function) ' +
+        'into `b.ts`. Update `a.ts` so the function is no longer defined there, but ' +
+        'leave any other exports in `a.ts` alone. `b.ts` should end up declaring and ' +
+        'exporting `greet`.',
+    setup: async (cwd) => {
+        await mkdir(cwd, { recursive: true });
+        await writeFile(join(cwd, 'a.ts'), "export function greet(name: string): string { return `Hello, ${name}!`; }\n" +
+            "export const VERSION = '1.0';\n");
+        await writeFile(join(cwd, 'b.ts'), "// b.ts placeholder\nexport const PLACEHOLDER = true;\n");
+    },
+    graders: [
+        completed(),
+        toolWasCalled('read_file'),
+        fileMatches('b.ts', 'function greet'),
+        fileMatches('a.ts', 'VERSION'),
+        // `a.ts` must no longer define the function. Graded on the outcome, so write_file and edit_file both count.
+        commandRuns({
+            command: ['node', '-e', "process.exit(require('fs').readFileSync('a.ts', 'utf8').includes('function greet') ? 1 : 0)"],
+        }),
+        fileMissing('a-greet-backup.ts'),
+    ],
+});
+//# sourceMappingURL=multi.refactor-move.js.map

package/dist/tasks/multi.refactor-move.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"multi.refactor-move.js","sourceRoot":"","sources":["../../src/tasks/multi.refactor-move.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,WAAW,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAElH;;;;GAIG;AACH,eAAe,UAAU,CAAC;IACxB,EAAE,EAAE,qBAAqB;IACzB,IAAI,EAAE,6BAA6B;IACnC,QAAQ,EAAE,YAAY;IACtB,WAAW,EAAE,oEAAoE;IACjF,IAAI,EACF,2EAA2E;QAC3E,8EAA8E;QAC9E,6EAA6E;QAC7E,8EAA8E;QAC9E,oBAAoB;IACtB,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACnB,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACtC,MAAM,SAAS,CACb,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,EACjB,6EAA6E;YAC3E,iCAAiC,CACpC,CAAC;QACF,MAAM,SAAS,CACb,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,EACjB,yDAAyD,CAC1D,CAAC;IACJ,CAAC;IACD,OAAO,EAAE;QACP,SAAS,EAAE;QACX,aAAa,CAAC,WAAW,CAAC;QAC1B,WAAW,CAAC,MAAM,EAAE,gBAAgB,CAAC;QACrC,WAAW,CAAC,MAAM,EAAE,SAAS,CAAC;QAC9B,4GAA4G;QAC5G,WAAW,CAAC;YACV,OAAO,EAAE,CAAC,MAAM,EAAE,IAAI,EAAE,6FAA6F,CAAC;SACvH,CAAC;QACF,WAAW,CAAC,mBAAmB,CAAC;KACjC;CACF,CAAC,CAAC"}

package/dist/tasks/multi.run-and-report.d.ts

@@ -0,0 +1,16 @@
+/**
+ * multi-step / run-and-report: agent must run a command and report its
+ * output. Verifies the model uses run_command (not just hallucinating the
+ * answer) and that it surfaces the result correctly.
+ */
+declare const _default: {
+    id: string;
+    name: string;
+    category: "multi-step";
+    description: string;
+    task: string;
+    setup: () => Promise<void>;
+    graders: import("../index.js").Grader[];
+};
+export default _default;
+//# sourceMappingURL=multi.run-and-report.d.ts.map

package/dist/tasks/multi.run-and-report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"multi.run-and-report.d.ts","sourceRoot":"","sources":["../../src/tasks/multi.run-and-report.ts"],"names":[],"mappings":"AAEA;;;;GAIG;;;;;;;;;;AACH,wBAmBG"}

package/dist/tasks/multi.run-and-report.js

@@ -0,0 +1,26 @@
+import { defineTask, commandRuns, contains, toolWasCalled, completed } from '../graders/index.js';
+/**
+ * multi-step / run-and-report: agent must run a command and report its
+ * output. Verifies the model uses run_command (not just hallucinating the
+ * answer) and that it surfaces the result correctly.
+ */
+export default defineTask({
+    id: 'multi-run-and-report',
+    name: 'Run a command and report output',
+    category: 'multi-step',
+    description: 'Agent runs a shell command and reports its stdout.',
+    task: 'Run the command `node -e "console.log(2 + 2)"` in the current directory using ' +
+        'the run_command tool and tell me what it printed.',
+    setup: async () => {
+        // No fixtures needed.
+    },
+    graders: [
+        completed(),
+        toolWasCalled('run_command'),
+        // Sanity: the command grader proves the command actually works in cwd.
+        commandRuns({ command: ['node', '-e', 'console.log(2 + 2)'], contains: '4' }),
+        // The model's reply should mention 4 too.
+        contains('4'),
+    ],
+});
+//# sourceMappingURL=multi.run-and-report.js.map

package/dist/tasks/multi.run-and-report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"multi.run-and-report.js","sourceRoot":"","sources":["../../src/tasks/multi.run-and-report.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAElG;;;;GAIG;AACH,eAAe,UAAU,CAAC;IACxB,EAAE,EAAE,sBAAsB;IAC1B,IAAI,EAAE,iCAAiC;IACvC,QAAQ,EAAE,YAAY;IACtB,WAAW,EAAE,oDAAoD;IACjE,IAAI,EACF,gFAAgF;QAChF,mDAAmD;IACrD,KAAK,EAAE,KAAK,IAAI,EAAE;QAChB,sBAAsB;IACxB,CAAC;IACD,OAAO,EAAE;QACP,SAAS,EAAE;QACX,aAAa,CAAC,aAAa,CAAC;QAC5B,uEAAuE;QACvE,WAAW,CAAC,EAAE,OAAO,EAAE,CAAC,MAAM,EAAE,IAAI,EAAE,oBAAoB,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC;QAC7E,0CAA0C;QAC1C,QAAQ,CAAC,GAAG,CAAC;KACd;CACF,CAAC,CAAC"}

package/dist/tasks/multi.summarize.d.ts

@@ -0,0 +1,16 @@
+/**
+ * multi-step / summarize: agent must read two files and produce a one-line
+ * summary that includes specific facts from both. Verifies it actually
+ * read each file (not just guessed) by checking tool calls + output tokens.
+ */
+declare const _default: {
+    id: string;
+    name: string;
+    category: "multi-step";
+    description: string;
+    task: string;
+    setup: (cwd: string) => Promise<void>;
+    graders: import("../index.js").Grader[];
+};
+export default _default;
+//# sourceMappingURL=multi.summarize.d.ts.map

package/dist/tasks/multi.summarize.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"multi.summarize.d.ts","sourceRoot":"","sources":["../../src/tasks/multi.summarize.ts"],"names":[],"mappings":"AAKA;;;;GAIG;;;;;;;;;;AACH,wBAgCG"}

package/dist/tasks/multi.summarize.js

@@ -0,0 +1,35 @@
+import { mkdir, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { defineTask, regex, contains, toolWasCalled, completed } from '../graders/index.js';
+/**
+ * multi-step / summarize: agent must read two files and produce a one-line
+ * summary that includes specific facts from both. Verifies it actually
+ * read each file (not just guessed) by checking tool calls + output tokens.
+ */
+export default defineTask({
+    id: 'multi-summarize-two-files',
+    name: 'Summarize two files',
+    category: 'multi-step',
+    description: 'Agent reads two files and summarizes them in one sentence.',
+    task: 'Read the files `alpha.txt` and `beta.txt` in the current directory. In a ' +
+        'single short sentence (no more than 30 words), summarize what each file ' +
+        'is about. Mention the project name "Astra" and the year "2030" in your reply.',
+    setup: async (cwd) => {
+        await mkdir(cwd, { recursive: true });
+        await writeFile(join(cwd, 'alpha.txt'), 'Project Astra is the company-wide initiative launched in 2030 to unify our ' +
+            'disconnected tools into one platform.\n');
+        await writeFile(join(cwd, 'beta.txt'), 'The 2030 roadmap for Astra focuses on three pillars: developer experience, ' +
+            'security, and cost.\n');
+    },
+    graders: [
+        completed(),
+        toolWasCalled('read_file', 2),
+        // Both files were read: check the model called read_file at least twice
+        // (the tool was called) AND that the result mentions both key terms.
+        contains('Astra'),
+        contains('2030'),
+        // The reply should be reasonably short: not a multi-page essay.
+        regex(/\b\w+\b.*\b\w+\b/s),
+    ],
+});
+//# sourceMappingURL=multi.summarize.js.map

package/dist/tasks/multi.summarize.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"multi.summarize.js","sourceRoot":"","sources":["../../src/tasks/multi.summarize.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAE5F;;;;GAIG;AACH,eAAe,UAAU,CAAC;IACxB,EAAE,EAAE,2BAA2B;IAC/B,IAAI,EAAE,qBAAqB;IAC3B,QAAQ,EAAE,YAAY;IACtB,WAAW,EAAE,4DAA4D;IACzE,IAAI,EACF,2EAA2E;QAC3E,0EAA0E;QAC1E,+EAA+E;IACjF,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACnB,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACtC,MAAM,SAAS,CACb,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,EACtB,6EAA6E;YAC3E,yCAAyC,CAC5C,CAAC;QACF,MAAM,SAAS,CACb,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,EACrB,6EAA6E;YAC3E,uBAAuB,CAC1B,CAAC;IACJ,CAAC;IACD,OAAO,EAAE;QACP,SAAS,EAAE;QACX,aAAa,CAAC,WAAW,EAAE,CAAC,CAAC;QAC7B,wEAAwE;QACxE,qEAAqE;QACrE,QAAQ,CAAC,OAAO,CAAC;QACjB,QAAQ,CAAC,MAAM,CAAC;QAChB,gEAAgE;QAChE,KAAK,CAAC,mBAAmB,CAAC;KAC3B;CACF,CAAC,CAAC"}

package/dist/tasks/read.list-files.d.ts

@@ -0,0 +1,15 @@
+/**
+ * read / list-files: ask the agent to enumerate TypeScript files in the
+ * project root and report what it found.
+ */
+declare const _default: {
+    id: string;
+    name: string;
+    category: "read";
+    description: string;
+    task: string;
+    setup: (cwd: string) => Promise<void>;
+    graders: import("../index.js").Grader[];
+};
+export default _default;
+//# sourceMappingURL=read.list-files.d.ts.map

package/dist/tasks/read.list-files.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"read.list-files.d.ts","sourceRoot":"","sources":["../../src/tasks/read.list-files.ts"],"names":[],"mappings":"AAKA;;;GAGG;;;;;;;;;;AACH,wBAoBG"}

package/dist/tasks/read.list-files.js

@@ -0,0 +1,28 @@
+import { mkdir, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { defineTask, regex, toolWasCalled, completed } from '../graders/index.js';
+/**
+ * read / list-files: ask the agent to enumerate TypeScript files in the
+ * project root and report what it found.
+ */
+export default defineTask({
+    id: 'read-list-typescript-files',
+    name: 'List TypeScript files',
+    category: 'read',
+    description: 'Agent enumerates .ts files under the project root.',
+    task: 'Use the list_files tool to find all TypeScript files (extension `.ts`) in the ' +
+        'current directory and report their names. Do not recurse into node_modules.',
+    setup: async (cwd) => {
+        await mkdir(cwd, { recursive: true });
+        await writeFile(join(cwd, 'a.ts'), "export const a = 1;\n");
+        await writeFile(join(cwd, 'b.ts'), "export const b = 2;\n");
+        await writeFile(join(cwd, 'README.md'), '# not ts\n');
+        await writeFile(join(cwd, 'package.json'), '{}\n');
+    },
+    graders: [
+        completed(),
+        toolWasCalled('list_files'),
+        regex(/\.ts\b/),
+    ],
+});
+//# sourceMappingURL=read.list-files.js.map

package/dist/tasks/read.list-files.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"read.list-files.js","sourceRoot":"","sources":["../../src/tasks/read.list-files.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAElF;;;GAGG;AACH,eAAe,UAAU,CAAC;IACxB,EAAE,EAAE,4BAA4B;IAChC,IAAI,EAAE,uBAAuB;IAC7B,QAAQ,EAAE,MAAM;IAChB,WAAW,EAAE,oDAAoD;IACjE,IAAI,EACF,gFAAgF;QAChF,6EAA6E;IAC/E,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACnB,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACtC,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,EAAE,uBAAuB,CAAC,CAAC;QAC5D,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,EAAE,uBAAuB,CAAC,CAAC;QAC5D,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,EAAE,YAAY,CAAC,CAAC;QACtD,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,EAAE;QACP,SAAS,EAAE;QACX,aAAa,CAAC,YAAY,CAAC;QAC3B,KAAK,CAAC,QAAQ,CAAC;KAChB;CACF,CAAC,CAAC"}

package/dist/tasks/read.package-version.d.ts

@@ -0,0 +1,15 @@
+/**
+ * read / package-version: the agent must read package.json and surface the
+ * version field. Verifies the model actually reads the file (not just guesses).
+ */
+declare const _default: {
+    id: string;
+    name: string;
+    category: "read";
+    description: string;
+    task: string;
+    setup: (cwd: string) => Promise<void>;
+    graders: import("../index.js").Grader[];
+};
+export default _default;
+//# sourceMappingURL=read.package-version.d.ts.map

package/dist/tasks/read.package-version.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"read.package-version.d.ts","sourceRoot":"","sources":["../../src/tasks/read.package-version.ts"],"names":[],"mappings":"AAKA;;;GAGG;;;;;;;;;;AACH,wBAoBG"}

package/dist/tasks/read.package-version.js

@@ -0,0 +1,25 @@
+import { mkdir, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { defineTask, regex, toolWasCalled, completed } from '../graders/index.js';
+/**
+ * read / package-version: the agent must read package.json and surface the
+ * version field. Verifies the model actually reads the file (not just guesses).
+ */
+export default defineTask({
+    id: 'read-package-version',
+    name: 'Read package.json version',
+    category: 'read',
+    description: 'Agent reads package.json and reports the version string.',
+    task: 'Read the file `package.json` in the current directory and tell me the value of ' +
+        'the "version" field. Reply with just the version string, nothing else.',
+    setup: async (cwd) => {
+        await mkdir(cwd, { recursive: true });
+        await writeFile(join(cwd, 'package.json'), JSON.stringify({ name: 'sample', version: '4.2.1', description: 'x' }, null, 2));
+    },
+    graders: [
+        completed(),
+        toolWasCalled('read_file'),
+        regex(/4\.2\.1/),
+    ],
+});
+//# sourceMappingURL=read.package-version.js.map

package/dist/tasks/read.package-version.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"read.package-version.js","sourceRoot":"","sources":["../../src/tasks/read.package-version.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAElF;;;GAGG;AACH,eAAe,UAAU,CAAC;IACxB,EAAE,EAAE,sBAAsB;IAC1B,IAAI,EAAE,2BAA2B;IACjC,QAAQ,EAAE,MAAM;IAChB,WAAW,EAAE,0DAA0D;IACvE,IAAI,EACF,iFAAiF;QACjF,wEAAwE;IAC1E,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACnB,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACtC,MAAM,SAAS,CACb,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,EACzB,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAChF,CAAC;IACJ,CAAC;IACD,OAAO,EAAE;QACP,SAAS,EAAE;QACX,aAAa,CAAC,WAAW,CAAC;QAC1B,KAAK,CAAC,SAAS,CAAC;KACjB;CACF,CAAC,CAAC"}

package/dist/tasks/read.search-todos.d.ts

@@ -0,0 +1,16 @@
+/**
+ * read / search-todos: agent searches the workspace for TODO comments.
+ * We seed three TODO markers in three files (plus a decoy without one)
+ * so a model that only checks one file fails.
+ */
+declare const _default: {
+    id: string;
+    name: string;
+    category: "read";
+    description: string;
+    task: string;
+    setup: (cwd: string) => Promise<void>;
+    graders: import("../index.js").Grader[];
+};
+export default _default;
+//# sourceMappingURL=read.search-todos.d.ts.map

package/dist/tasks/read.search-todos.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"read.search-todos.d.ts","sourceRoot":"","sources":["../../src/tasks/read.search-todos.ts"],"names":[],"mappings":"AAKA;;;;GAIG;;;;;;;;;;AACH,wBAqBG"}

package/dist/tasks/read.search-todos.js

@@ -0,0 +1,30 @@
+import { mkdir, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { defineTask, regex, toolWasCalled, completed } from '../graders/index.js';
+/**
+ * read / search-todos: agent searches the workspace for TODO comments.
+ * We seed three TODO markers in three files (plus a decoy without one)
+ * so a model that only checks one file fails.
+ */
+export default defineTask({
+    id: 'read-search-todos',
+    name: 'Search for TODO comments',
+    category: 'read',
+    description: 'Agent searches for TODO comments across files.',
+    task: 'Search the current directory for files containing the text "TODO" and list ' +
+        'the file paths you find. Use the search_files tool: do not read files manually.',
+    setup: async (cwd) => {
+        await mkdir(cwd, { recursive: true });
+        await writeFile(join(cwd, 'alpha.ts'), '// TODO: implement alpha\nconst a = 1;\n');
+        await writeFile(join(cwd, 'beta.ts'), 'export function beta() { /* TODO later */ }\n');
+        await writeFile(join(cwd, 'gamma.ts'), '// no todos here\nconst g = 3;\n');
+        await writeFile(join(cwd, 'README.md'), '# Project\n');
+    },
+    graders: [
+        completed(),
+        toolWasCalled('search_files'),
+        regex(/alpha\.ts/),
+        regex(/beta\.ts/),
+    ],
+});
+//# sourceMappingURL=read.search-todos.js.map

package/dist/tasks/read.search-todos.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"read.search-todos.js","sourceRoot":"","sources":["../../src/tasks/read.search-todos.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAElF;;;;GAIG;AACH,eAAe,UAAU,CAAC;IACxB,EAAE,EAAE,mBAAmB;IACvB,IAAI,EAAE,0BAA0B;IAChC,QAAQ,EAAE,MAAM;IAChB,WAAW,EAAE,gDAAgD;IAC7D,IAAI,EACF,6EAA6E;QAC7E,iFAAiF;IACnF,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACnB,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACtC,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,EAAE,0CAA0C,CAAC,CAAC;QACnF,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,EAAE,+CAA+C,CAAC,CAAC;QACvF,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,EAAE,kCAAkC,CAAC,CAAC;QAC3E,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,EAAE,aAAa,CAAC,CAAC;IACzD,CAAC;IACD,OAAO,EAAE;QACP,SAAS,EAAE;QACX,aAAa,CAAC,cAAc,CAAC;QAC7B,KAAK,CAAC,WAAW,CAAC;QAClB,KAAK,CAAC,UAAU,CAAC;KAClB;CACF,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,60 @@
 {
   "name": "@ikarmic/forge-eval",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "version": "4.5.0",
+  "description": "Eval harness for IKarmic AI Forge. Runs agent tasks against isolated workspaces and grades the results with deterministic checks.",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "bin": {
+    "forge-eval": "./dist/bin/forge-eval.js"
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "keywords": [
+    "ai",
+    "eval",
+    "benchmark",
+    "agent",
+    "llm",
+    "code-assistant",
+    "testing"
+  ],
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/ikarmicai/ikarmic-forge"
+  },
+  "bugs": {
+    "url": "https://github.com/ikarmicai/ikarmic-forge/issues"
+  },
+  "homepage": "https://github.com/ikarmicai/ikarmic-forge/tree/main/packages/eval#readme",
+  "engines": {
+    "node": ">=22"
+  },
+  "dependencies": {
+    "ai": "^7.0.0",
+    "zod": "^3.23.8",
+    "@ikarmic/forge-core": "4.5.0"
+  },
+  "devDependencies": {
+    "@types/node": "^22.7.0",
+    "tsx": "^4.19.2",
+    "typescript": "^5.6.3",
+    "vitest": "^5.0.2"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "dev": "tsx src/bin/forge-eval.ts",
+    "test": "vitest run",
+    "typecheck": "tsc -p tsconfig.json --noEmit",
+    "clean": "rm -rf dist *.tsbuildinfo"
+  }
 }