@ikarmic/forge-eval 0.0.0-stage → 4.4.0

package/dist/graders/content.js

@@ -0,0 +1,135 @@
+/**
+ * Grader that extracts the first <code>```&lt;lang&gt; ... ```</code>
+ * fenced block from `finalText` and passes when `predicate(block)`
+ * returns true. Language matching is case-insensitive.
+ *
+ * Use this for tasks like "agent produced a runnable Python snippet
+ * with a `solve()` function" without parsing the whole response
+ * manually.
+ *
+ * ```ts
+ * graders: [
+ *   languageBlock('ts', (code) => /export function/.test(code)),
+ * ]
+ * ```
+ */
+export function languageBlock(lang, predicate, options = {}) {
+    const maxDetailChars = options.maxDetailChars ?? 320;
+    const requireBlock = options.requireBlock ?? true;
+    return (ctx) => {
+        const text = ctx.result.finalText ?? '';
+        const block = extractFencedBlock(text, lang);
+        if (block === null) {
+            return requireBlock
+                ? {
+                    passed: false,
+                    message: `no \`\`\`${lang} ... \`\`\` block found in finalText`,
+                    details: truncate(text, maxDetailChars),
+                }
+                : { passed: true, message: `no \`\`\`${lang} block (not required)` };
+        }
+        return predicate(block)
+            ? { passed: true, message: `\`\`\`${lang} block matched predicate` }
+            : {
+                passed: false,
+                message: `\`\`\`${lang} block did not match predicate`,
+                details: truncate(block, maxDetailChars),
+            };
+    };
+}
+/**
+ * Predicate-driven JSON shape check. Parses `finalText` as JSON: raw
+ * text first, then a single fenced block: then calls
+ * `predicate(value)`. Passes when the predicate accepts.
+ *
+ * Use this instead of {@link jsonSchema} when the field set is loose,
+ * you want to assert a single property, or you don't want a Zod
+ * dependency in this task's grading.
+ *
+ * ```ts
+ * jsonContains((v) => {
+ *   return v && typeof v === 'object' &&
+ *     'name' in v && 'price' in v && (v as any).price > 0;
+ * })
+ * ```
+ */
+export function jsonContains(predicate) {
+    return (ctx) => {
+        const text = ctx.result.finalText ?? '';
+        const parsed = tryParseJsonLoose(text);
+        if (parsed.parseError) {
+            return {
+                passed: false,
+                message: 'finalText is not valid JSON',
+                details: `${parsed.parseError}\n\ntext: ${truncate(text, 240)}`,
+            };
+        }
+        return predicate(parsed.value)
+            ? { passed: true, message: 'JSON matched predicate' }
+            : {
+                passed: false,
+                message: 'JSON did not match predicate',
+                details: `value: ${truncate(JSON.stringify(parsed.value), 240)}`,
+            };
+    };
+}
+// Helpers
+/**
+ * Extract the first fenced code block matching the language tag.
+ * Case-insensitive. The same regex is used for both the typed
+ * (`` ```ts ```) and generic (`` ``` ```) cases: passing an empty
+ * `lang` matches any fenced block, which `jsonContains` uses as a
+ * fallback when the raw text isn't valid JSON.
+ */
+function extractFencedBlock(text, lang) {
+    if (lang === '') {
+        const fallback = text.match(/```[^\n]*\n?([\s\S]*?)```/);
+        if (fallback && typeof fallback[1] === 'string')
+            return fallback[1].trim();
+        return null;
+    }
+    const pattern = new RegExp('```' + escapeRegex(lang.toLowerCase()) + '[\\s\\S]*?\\n([\\s\\S]*?)```', 'i');
+    const m = text.match(pattern);
+    if (m && typeof m[1] === 'string')
+        return m[1].trim();
+    return null;
+}
+/**
+ * Try to parse `text` as JSON. Falls back to extracting a fenced
+ * block first. Mirrors `validateFinalText` from `forge-core`'s
+ * structured module so the two stay in sync structurally (without
+ * forcing this leaf package to depend on core).
+ */
+function tryParseJsonLoose(text) {
+    const trimmed = text.trim();
+    if (trimmed.length === 0)
+        return { value: undefined, parseError: 'response was empty' };
+    try {
+        return { value: JSON.parse(trimmed), parseError: null };
+    }
+    catch {
+        /* try fenced block */
+    }
+    const fence = extractFencedBlock(trimmed, '');
+    if (fence !== null) {
+        try {
+            return { value: JSON.parse(fence), parseError: null };
+        }
+        catch (err) {
+            return {
+                value: undefined,
+                parseError: `fenced block not valid JSON: ${err.message}`,
+            };
+        }
+    }
+    return { value: undefined, parseError: 'no JSON found (and no fenced block)' };
+}
+function escapeRegex(s) {
+    return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+function truncate(s, max) {
+    if (s.length <= max)
+        return s;
+    return s.slice(0, max) + '…';
+}
+//# sourceMappingURL=content.js.map

package/dist/graders/content.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"content.js","sourceRoot":"","sources":["../../src/graders/content.ts"],"names":[],"mappings":"AA2BA;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,aAAa,CAC3B,IAAY,EACZ,SAAoC,EACpC,UAAgC,EAAE;IAElC,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,GAAG,CAAC;IACrD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC;IAClD,OAAO,CAAC,GAAkB,EAAgB,EAAE;QAC1C,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC7C,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACnB,OAAO,YAAY;gBACjB,CAAC,CAAC;oBACE,MAAM,EAAE,KAAK;oBACb,OAAO,EAAE,YAAY,IAAI,sCAAsC;oBAC/D,OAAO,EAAE,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;iBACxC;gBACH,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,YAAY,IAAI,uBAAuB,EAAE,CAAC;QACzE,CAAC;QACD,OAAO,SAAS,CAAC,KAAK,CAAC;YACrB,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,IAAI,0BAA0B,EAAE;YACpE,CAAC,CAAC;gBACE,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,SAAS,IAAI,gCAAgC;gBACtD,OAAO,EAAE,QAAQ,CAAC,KAAK,EAAE,cAAc,CAAC;aACzC,CAAC;IACR,CAAC,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,YAAY,CAAC,SAAsC;IACjE,OAAO,CAAC,GAAkB,EAAgB,EAAE;QAC1C,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YACtB,OAAO;gBACL,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,6BAA6B;gBACtC,OAAO,EAAE,GAAG,MAAM,CAAC,UAAU,aAAa,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE;aAChE,CAAC;QACJ,CAAC;QACD,OAAO,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC;YAC5B,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,wBAAwB,EAAE;YACrD,CAAC,CAAC;gBACE,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,8BAA8B;gBACvC,OAAO,EAAE,UAAU,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,EAAE;aACjE,CAAC;IACR,CAAC,CAAC;AACJ,CAAC;AAED,UAAU;AAEV;;;;;;GAMG;AACH,SAAS,kBAAkB,CAAC,IAAY,EAAE,IAAY;IACpD,IAAI,IAAI,KAAK,EAAE,EAAE,CAAC;QAChB,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;QACzD,IAAI,QAAQ,IAAI,OAAO,QAAQ,CAAC,CAAC,CAAC,KAAK,QAAQ;YAAE,OAAO,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3E,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,MAAM,CACxB,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,GAAG,8BAA8B,EACxE,GAAG,CACJ,CAAC;IACF,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC9B,IAAI,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,QAAQ;QAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACtD,OAAO,IAAI,CAAC;AACd,CAAC;AAOD;;;;;GAKG;AACH,SAAS,iBAAiB,CAAC,IAAY;IACrC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,oBAAoB,EAAE,CAAC;IACxF,IAAI,CAAC;QACH,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IAC1D,CAAC;IAAC,MAAM,CAAC;QACP,sBAAsB;IACxB,CAAC;IACD,MAAM,KAAK,GAAG,kBAAkB,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAC9C,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QACnB,IAAI,CAAC;YACH,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;QACxD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO;gBACL,KAAK,EAAE,SAAS;gBAChB,UAAU,EAAE,gCAAiC,GAAa,CAAC,OAAO,EAAE;aACrE,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,qCAAqC,EAAE,CAAC;AACjF,CAAC;AAED,SAAS,WAAW,CAAC,CAAS;IAC5B,OAAO,CAAC,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,QAAQ,CAAC,CAAS,EAAE,GAAW;IACtC,IAAI,CAAC,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,CAAC,CAAC;IAC9B,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,GAAG,CAAC;AAC/B,CAAC"}

package/dist/graders/fs.d.ts

@@ -0,0 +1,24 @@
+import type { Grader } from './types.js';
+/**
+ * Grader that passes when `cwd/<relPath>` exists. Resolves to an absolute
+ * path so paths with `..` can't escape the task's cwd.
+ */
+export declare function fileExists(relPath: string): Grader;
+/**
+ * Grader that passes when `cwd/<relPath>` exists AND its contents match the
+ * given predicate (substring, regex, or equality).
+ */
+export declare function fileMatches(relPath: string, predicate: string | RegExp | {
+    equals: string;
+}): Grader;
+/**
+ * Grader that passes when `cwd/<relPath>` does NOT exist. Inverse of
+ * {@link fileExists} for tasks like "delete stale.txt".
+ */
+export declare function fileMissing(relPath: string): Grader;
+/**
+ * Grader that passes when `cwd/<relPath>` is a directory containing at least
+ * `min` files matching the (optional) extension filter.
+ */
+export declare function directoryHas(relPath: string, min?: number, ext?: string): Grader;
+//# sourceMappingURL=fs.d.ts.map

package/dist/graders/fs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fs.d.ts","sourceRoot":"","sources":["../../src/graders/fs.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,EAA+B,MAAM,YAAY,CAAC;AAEtE;;;GAGG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAmBlD;AAED;;;GAGG;AACH,wBAAgB,WAAW,CACzB,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,GAC9C,MAAM,CAsCR;AAED;;;GAGG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAUnD;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,SAAI,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM,CA0B3E"}

package/dist/graders/fs.js

@@ -0,0 +1,134 @@
+import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
+import { resolve, sep } from 'node:path';
+/**
+ * Grader that passes when `cwd/<relPath>` exists. Resolves to an absolute
+ * path so paths with `..` can't escape the task's cwd.
+ */
+export function fileExists(relPath) {
+    return (ctx) => {
+        const abs = safeResolve(ctx.cwd, relPath);
+        if (abs === null) {
+            return {
+                passed: false,
+                message: `path "${relPath}" escaped task cwd`,
+                details: `cwd: ${ctx.cwd}`,
+            };
+        }
+        if (existsSync(abs)) {
+            return { passed: true, message: `file exists: ${relPath}` };
+        }
+        return {
+            passed: false,
+            message: `file missing: ${relPath}`,
+            details: `expected at: ${abs}`,
+        };
+    };
+}
+/**
+ * Grader that passes when `cwd/<relPath>` exists AND its contents match the
+ * given predicate (substring, regex, or equality).
+ */
+export function fileMatches(relPath, predicate) {
+    return (ctx) => {
+        const abs = safeResolve(ctx.cwd, relPath);
+        if (abs === null) {
+            return { passed: false, message: `path "${relPath}" escaped task cwd` };
+        }
+        if (!existsSync(abs)) {
+            return { passed: false, message: `file missing: ${relPath}`, details: `expected at: ${abs}` };
+        }
+        const text = readFileSync(abs, 'utf8');
+        if (typeof predicate === 'string') {
+            return text.includes(predicate)
+                ? { passed: true, message: `file ${relPath} contains "${predicate}"` }
+                : {
+                    passed: false,
+                    message: `file ${relPath} does not contain "${predicate}"`,
+                    details: truncate(text),
+                };
+        }
+        if (predicate instanceof RegExp) {
+            return predicate.test(text)
+                ? { passed: true, message: `file ${relPath} matches ${predicate}` }
+                : {
+                    passed: false,
+                    message: `file ${relPath} does not match ${predicate}`,
+                    details: truncate(text),
+                };
+        }
+        const want = predicate.equals.trim();
+        const got = text.trim();
+        return got === want
+            ? { passed: true, message: `file ${relPath} equals expected` }
+            : {
+                passed: false,
+                message: `file ${relPath} content mismatch`,
+                details: `expected: ${JSON.stringify(want)}\nactual:   ${JSON.stringify(got)}`,
+            };
+    };
+}
+/**
+ * Grader that passes when `cwd/<relPath>` does NOT exist. Inverse of
+ * {@link fileExists} for tasks like "delete stale.txt".
+ */
+export function fileMissing(relPath) {
+    return (ctx) => {
+        const abs = safeResolve(ctx.cwd, relPath);
+        if (abs === null) {
+            return { passed: true, message: `path "${relPath}" not in cwd (treated as missing)` };
+        }
+        return !existsSync(abs)
+            ? { passed: true, message: `file absent: ${relPath}` }
+            : { passed: false, message: `file should be missing: ${relPath}`, details: `still exists at: ${abs}` };
+    };
+}
+/**
+ * Grader that passes when `cwd/<relPath>` is a directory containing at least
+ * `min` files matching the (optional) extension filter.
+ */
+export function directoryHas(relPath, min = 1, ext) {
+    return (ctx) => {
+        const abs = safeResolve(ctx.cwd, relPath);
+        if (abs === null) {
+            return { passed: false, message: `path "${relPath}" escaped task cwd` };
+        }
+        if (!existsSync(abs)) {
+            return { passed: false, message: `directory missing: ${relPath}` };
+        }
+        const st = statSync(abs);
+        if (!st.isDirectory()) {
+            return { passed: false, message: `not a directory: ${relPath}` };
+        }
+        const entries = readdirSync(abs);
+        const matched = ext ? entries.filter((e) => e.endsWith(ext)) : entries;
+        if (matched.length >= min) {
+            return {
+                passed: true,
+                message: `directory ${relPath} has ${matched.length} entries${ext ? ` (${ext})` : ''}`,
+            };
+        }
+        return {
+            passed: false,
+            message: `directory ${relPath} has only ${matched.length} entries (need ${min}${ext ? ` ${ext}` : ''})`,
+        };
+    };
+}
+/**
+ * Resolve `relPath` against `cwd` and reject any escape attempts (`..`
+ * segments, absolute paths, symlink games are not in scope: we trust the
+ * grader author to use safe paths; this just defends against typos).
+ */
+function safeResolve(cwd, relPath) {
+    const abs = resolve(cwd, relPath);
+    const root = resolve(cwd);
+    if (abs !== root && !abs.startsWith(root + sep)) {
+        return null;
+    }
+    return abs;
+}
+function truncate(s, max = 240) {
+    if (s.length <= max)
+        return s;
+    return s.slice(0, max) + '…';
+}
+//# sourceMappingURL=fs.js.map

package/dist/graders/fs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fs.js","sourceRoot":"","sources":["../../src/graders/fs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC1E,OAAO,EAAQ,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAI/C;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,OAAe;IACxC,OAAO,CAAC,GAAkB,EAAgB,EAAE;QAC1C,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC1C,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YACjB,OAAO;gBACL,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,SAAS,OAAO,oBAAoB;gBAC7C,OAAO,EAAE,QAAQ,GAAG,CAAC,GAAG,EAAE;aAC3B,CAAC;QACJ,CAAC;QACD,IAAI,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACpB,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,gBAAgB,OAAO,EAAE,EAAE,CAAC;QAC9D,CAAC;QACD,OAAO;YACL,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,iBAAiB,OAAO,EAAE;YACnC,OAAO,EAAE,gBAAgB,GAAG,EAAE;SAC/B,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CACzB,OAAe,EACf,SAA+C;IAE/C,OAAO,CAAC,GAAkB,EAAgB,EAAE;QAC1C,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC1C,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YACjB,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,OAAO,oBAAoB,EAAE,CAAC;QAC1E,CAAC;QACD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACrB,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,iBAAiB,OAAO,EAAE,EAAE,OAAO,EAAE,gBAAgB,GAAG,EAAE,EAAE,CAAC;QAChG,CAAC;QACD,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACvC,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;YAClC,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAC7B,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,OAAO,cAAc,SAAS,GAAG,EAAE;gBACtE,CAAC,CAAC;oBACE,MAAM,EAAE,KAAK;oBACb,OAAO,EAAE,QAAQ,OAAO,sBAAsB,SAAS,GAAG;oBAC1D,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC;iBACxB,CAAC;QACR,CAAC;QACD,IAAI,SAAS,YAAY,MAAM,EAAE,CAAC;YAChC,OAAO,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;gBACzB,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,OAAO,YAAY,SAAS,EAAE,EAAE;gBACnE,CAAC,CAAC;oBACE,MAAM,EAAE,KAAK;oBACb,OAAO,EAAE,QAAQ,OAAO,mBAAmB,SAAS,EAAE;oBACtD,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC;iBACxB,CAAC;QACR,CAAC;QACD,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QACxB,OAAO,GAAG,KAAK,IAAI;YACjB,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,OAAO,kBAAkB,EAAE;YAC9D,CAAC,CAAC;gBACE,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,QAAQ,OAAO,mBAAmB;gBAC3C,OAAO,EAAE,aAAa,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE;aAC/E,CAAC;IACR,CAAC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,OAAe;IACzC,OAAO,CAAC,GAAkB,EAAgB,EAAE;QAC1C,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC1C,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YACjB,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,OAAO,mCAAmC,EAAE,CAAC;QACxF,CAAC;QACD,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;YACrB,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,gBAAgB,OAAO,EAAE,EAAE;YACtD,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,2BAA2B,OAAO,EAAE,EAAE,OAAO,EAAE,oBAAoB,GAAG,EAAE,EAAE,CAAC;IAC3G,CAAC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,OAAe,EAAE,GAAG,GAAG,CAAC,EAAE,GAAY;IACjE,OAAO,CAAC,GAAkB,EAAgB,EAAE;QAC1C,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC1C,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YACjB,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,OAAO,oBAAoB,EAAE,CAAC;QAC1E,CAAC;QACD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACrB,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,sBAAsB,OAAO,EAAE,EAAE,CAAC;QACrE,CAAC;QACD,MAAM,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;YACtB,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,oBAAoB,OAAO,EAAE,EAAE,CAAC;QACnE,CAAC;QACD,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QACjC,MAAM,OAAO,GAAG,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QACvE,IAAI,OAAO,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;YAC1B,OAAO;gBACL,MAAM,EAAE,IAAI;gBACZ,OAAO,EAAE,aAAa,OAAO,QAAQ,OAAO,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;aACvF,CAAC;QACJ,CAAC;QACD,OAAO;YACL,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,aAAa,OAAO,aAAa,OAAO,CAAC,MAAM,kBAAkB,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG;SACxG,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,WAAW,CAAC,GAAW,EAAE,OAAe;IAC/C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IAC1B,IAAI,GAAG,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;QAChD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,QAAQ,CAAC,CAAS,EAAE,GAAG,GAAG,GAAG;IACpC,IAAI,CAAC,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,CAAC,CAAC;IAC9B,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,GAAG,CAAC;AAC/B,CAAC"}

package/dist/graders/index.d.ts

@@ -0,0 +1,15 @@
+export type { Grader, GraderContext, GraderResult, TaskCategory, TaskDefinition, TaskInfo, ToolInvocation, } from './types.js';
+export { regex, contains, equals, type RegexGraderOptions } from './text.js';
+export { fileExists, fileMatches, fileMissing, directoryHas } from './fs.js';
+export { commandRuns, type CommandRunsOptions } from './shell.js';
+export { toolWasCalled, toolNotCalled, toolCalledWith, toolCount, noToolErrors, finalTextNonEmpty, completed, type ToolCountOptions, } from './behavior.js';
+export { jsonSchema, jsonSchemaPartial, type JsonSchemaGraderOptions } from './schema.js';
+export { tokensUnder, costUnder, durationUnder, iterationsUnder, type TokensUnderOptions, } from './metrics.js';
+export { languageBlock, jsonContains, type LanguageBlockOptions, } from './content.js';
+/**
+ * Helper for task authors: stamps a `TaskDefinition` with sensible defaults
+ * and gives the call site a single import. Functionally identical to the
+ * raw object: purely ergonomic.
+ */
+export declare function defineTask<T extends import('./types.js').TaskDefinition>(def: T): T;
+//# sourceMappingURL=index.d.ts.map

package/dist/graders/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/graders/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,MAAM,EACN,aAAa,EACb,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,QAAQ,EACR,cAAc,GACf,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,kBAAkB,EAAE,MAAM,WAAW,CAAC;AAC7E,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,WAAW,EAAE,KAAK,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAClE,OAAO,EACL,aAAa,EACb,aAAa,EACb,cAAc,EACd,SAAS,EACT,YAAY,EACZ,iBAAiB,EACjB,SAAS,EACT,KAAK,gBAAgB,GACtB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,KAAK,uBAAuB,EAAE,MAAM,aAAa,CAAC;AAE1F,OAAO,EACL,WAAW,EACX,SAAS,EACT,aAAa,EACb,eAAe,EACf,KAAK,kBAAkB,GACxB,MAAM,cAAc,CAAC;AAEtB,OAAO,EACL,aAAa,EACb,YAAY,EACZ,KAAK,oBAAoB,GAC1B,MAAM,cAAc,CAAC;AAEtB;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,CAAC,SAAS,OAAO,YAAY,EAAE,cAAc,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAEnF"}

package/dist/graders/index.js

@@ -0,0 +1,19 @@
+export { regex, contains, equals } from './text.js';
+export { fileExists, fileMatches, fileMissing, directoryHas } from './fs.js';
+export { commandRuns } from './shell.js';
+export { toolWasCalled, toolNotCalled, toolCalledWith, toolCount, noToolErrors, finalTextNonEmpty, completed, } from './behavior.js';
+// v2.12: JSON-schema / Zod-schema graders for structured-output tasks.
+export { jsonSchema, jsonSchemaPartial } from './schema.js';
+// v2.13: Budget / throughput graders over AgentRunResult aggregates.
+export { tokensUnder, costUnder, durationUnder, iterationsUnder, } from './metrics.js';
+// v2.13: Content extraction graders (fenced code blocks, predicate JSON).
+export { languageBlock, jsonContains, } from './content.js';
+/**
+ * Helper for task authors: stamps a `TaskDefinition` with sensible defaults
+ * and gives the call site a single import. Functionally identical to the
+ * raw object: purely ergonomic.
+ */
+export function defineTask(def) {
+    return def;
+}
+//# sourceMappingURL=index.js.map

package/dist/graders/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/graders/index.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAA2B,MAAM,WAAW,CAAC;AAC7E,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,WAAW,EAA2B,MAAM,YAAY,CAAC;AAClE,OAAO,EACL,aAAa,EACb,aAAa,EACb,cAAc,EACd,SAAS,EACT,YAAY,EACZ,iBAAiB,EACjB,SAAS,GAEV,MAAM,eAAe,CAAC;AACvB,uEAAuE;AACvE,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAgC,MAAM,aAAa,CAAC;AAC1F,qEAAqE;AACrE,OAAO,EACL,WAAW,EACX,SAAS,EACT,aAAa,EACb,eAAe,GAEhB,MAAM,cAAc,CAAC;AACtB,0EAA0E;AAC1E,OAAO,EACL,aAAa,EACb,YAAY,GAEb,MAAM,cAAc,CAAC;AAEtB;;;;GAIG;AACH,MAAM,UAAU,UAAU,CAAgD,GAAM;IAC9E,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/graders/metrics.d.ts

@@ -0,0 +1,59 @@
+/**
+ * Budget / throughput graders for the eval harness. v2.13.
+ *
+ * These graders inspect {@link AgentRunResult} aggregate fields
+ * (tokens, cost, duration, iterations) and pass when they fall under
+ * a caller-supplied ceiling. They let eval tasks answer questions
+ * like "did the agent come in under budget?" without writing custom
+ * predicates against raw numbers.
+ *
+ * Why a budget grader family?
+ *   - Cost is a first-class eval signal for an LLM agent. If a task
+ *     routinely hits 10x expected cost, that's a regression worth
+ *     catching even if the answer is technically correct.
+ *   - Latency ceilinging catches "agent goes into a long loop" bugs
+ *     that correctness graders don't.
+ *   - Pair `tokensUnder` / `costUnder` with `completed()` to assert
+ *     "this task finishes correctly AND cheaply".
+ */
+import type { Grader } from './types.js';
+/** Options for {@link tokensUnder}. */
+export interface TokensUnderOptions {
+    /**
+     * Which token counter to budget against.
+     *   - `'total'` (default): input + output, the most useful general signal
+     *   - `'input'`: "agent didn't shovel huge context"
+     *   - `'output'`: "agent didn't ramble"
+     */
+    kind?: 'input' | 'output' | 'total';
+}
+/**
+ * Grader that passes when the chosen token counter is `<= max`.
+ * The cap is a soft ceiling; the grader doesn't measure cost, just
+ * raw token counts. Use {@link costUnder} for budget enforcement in
+ * dollars.
+ *
+ * ```ts
+ * graders: [
+ *   completed(),
+ *   tokensUnder(2000),                  // total <= 2000
+ *   tokensUnder(800, { kind: 'input' }),// input alone <= 800
+ * ],
+ * ```
+ */
+export declare function tokensUnder(max: number, options?: TokensUnderOptions): Grader;
+/**
+ * Grader that passes when total cost is `<= maxUsd`. Fails when the
+ * agent's model has no known pricing (the cost isn't knowable, so
+ * "under budget" can't be verified).
+ *
+ * Used with cost-tracking models; for free-tier / on-device models
+ * pricing is unknown and this grader will always fail, so omit it
+ * from those tasks.
+ */
+export declare function costUnder(maxUsd: number): Grader;
+/** Grader that passes when the wall-clock run duration is `<= maxMs`. */
+export declare function durationUnder(maxMs: number): Grader;
+/** Grader that passes when the agent finished in `<= max` iterations. */
+export declare function iterationsUnder(max: number): Grader;
+//# sourceMappingURL=metrics.d.ts.map

package/dist/graders/metrics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../src/graders/metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AACH,OAAO,KAAK,EAAE,MAAM,EAA+B,MAAM,YAAY,CAAC;AAEtE,uCAAuC;AACvC,MAAM,WAAW,kBAAkB;IACjC;;;;;OAKG;IACH,IAAI,CAAC,EAAE,OAAO,GAAG,QAAQ,GAAG,OAAO,CAAC;CACrC;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,kBAAuB,GAAG,MAAM,CAUjF;AAED;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAgBhD;AAED,yEAAyE;AACzE,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAOnD;AAED,yEAAyE;AACzE,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAOnD"}

package/dist/graders/metrics.js

@@ -0,0 +1,70 @@
+/**
+ * Grader that passes when the chosen token counter is `<= max`.
+ * The cap is a soft ceiling; the grader doesn't measure cost, just
+ * raw token counts. Use {@link costUnder} for budget enforcement in
+ * dollars.
+ *
+ * ```ts
+ * graders: [
+ *   completed(),
+ *   tokensUnder(2000),                  // total <= 2000
+ *   tokensUnder(800, { kind: 'input' }),// input alone <= 800
+ * ],
+ * ```
+ */
+export function tokensUnder(max, options = {}) {
+    const kind = options.kind ?? 'total';
+    return (ctx) => {
+        const i = ctx.result.inputTokens;
+        const o = ctx.result.outputTokens;
+        const actual = kind === 'input' ? i : kind === 'output' ? o : i + o;
+        return actual <= max
+            ? { passed: true, message: `${kind} tokens ${actual} <= ${max}` }
+            : { passed: false, message: `${kind} tokens ${actual} exceeded ${max}` };
+    };
+}
+/**
+ * Grader that passes when total cost is `<= maxUsd`. Fails when the
+ * agent's model has no known pricing (the cost isn't knowable, so
+ * "under budget" can't be verified).
+ *
+ * Used with cost-tracking models; for free-tier / on-device models
+ * pricing is unknown and this grader will always fail, so omit it
+ * from those tasks.
+ */
+export function costUnder(maxUsd) {
+    return (ctx) => {
+        const r = ctx.result;
+        if (!r.costKnown) {
+            return {
+                passed: false,
+                message: `cost unknown for model: cannot enforce <= $${maxUsd}`,
+            };
+        }
+        return r.costUsd <= maxUsd
+            ? { passed: true, message: `cost $${r.costUsd.toFixed(6)} <= $${maxUsd}` }
+            : {
+                passed: false,
+                message: `cost $${r.costUsd.toFixed(6)} exceeded $${maxUsd}`,
+            };
+    };
+}
+/** Grader that passes when the wall-clock run duration is `<= maxMs`. */
+export function durationUnder(maxMs) {
+    return (ctx) => {
+        const actual = ctx.result.durationMs;
+        return actual <= maxMs
+            ? { passed: true, message: `duration ${actual}ms <= ${maxMs}ms` }
+            : { passed: false, message: `duration ${actual}ms exceeded ${maxMs}ms` };
+    };
+}
+/** Grader that passes when the agent finished in `<= max` iterations. */
+export function iterationsUnder(max) {
+    return (ctx) => {
+        const actual = ctx.result.iterations;
+        return actual <= max
+            ? { passed: true, message: `iterations ${actual} <= ${max}` }
+            : { passed: false, message: `iterations ${actual} exceeded ${max}` };
+    };
+}
+//# sourceMappingURL=metrics.js.map

package/dist/graders/metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../src/graders/metrics.ts"],"names":[],"mappings":"AA+BA;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,WAAW,CAAC,GAAW,EAAE,UAA8B,EAAE;IACvE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC;IACrC,OAAO,CAAC,GAAkB,EAAgB,EAAE;QAC1C,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC;QACjC,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC;QAClC,MAAM,MAAM,GAAG,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACpE,OAAO,MAAM,IAAI,GAAG;YAClB,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,WAAW,MAAM,OAAO,GAAG,EAAE,EAAE;YACjE,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,IAAI,WAAW,MAAM,aAAa,GAAG,EAAE,EAAE,CAAC;IAC7E,CAAC,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,SAAS,CAAC,MAAc;IACtC,OAAO,CAAC,GAAkB,EAAgB,EAAE;QAC1C,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC;QACrB,IAAI,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;YACjB,OAAO;gBACL,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,8CAA8C,MAAM,EAAE;aAChE,CAAC;QACJ,CAAC;QACD,OAAO,CAAC,CAAC,OAAO,IAAI,MAAM;YACxB,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,MAAM,EAAE,EAAE;YAC1E,CAAC,CAAC;gBACE,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,SAAS,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,MAAM,EAAE;aAC7D,CAAC;IACR,CAAC,CAAC;AACJ,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,aAAa,CAAC,KAAa;IACzC,OAAO,CAAC,GAAkB,EAAgB,EAAE;QAC1C,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC;QACrC,OAAO,MAAM,IAAI,KAAK;YACpB,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,YAAY,MAAM,SAAS,KAAK,IAAI,EAAE;YACjE,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,YAAY,MAAM,eAAe,KAAK,IAAI,EAAE,CAAC;IAC7E,CAAC,CAAC;AACJ,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,OAAO,CAAC,GAAkB,EAAgB,EAAE;QAC1C,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC;QACrC,OAAO,MAAM,IAAI,GAAG;YAClB,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,cAAc,MAAM,OAAO,GAAG,EAAE,EAAE;YAC7D,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,cAAc,MAAM,aAAa,GAAG,EAAE,EAAE,CAAC;IACzE,CAAC,CAAC;AACJ,CAAC"}

package/dist/graders/schema.d.ts

@@ -0,0 +1,87 @@
+/**
+ * JSON-schema / Zod-schema grader for the eval harness. v2.12.
+ *
+ * Validates `AgentRunResult.finalText` against a user-supplied Zod
+ * schema. Pairs naturally with the v2.8 `--schema` mode: run the
+ * agent in structured-output mode and grade the result with
+ * `jsonSchema` for a fully deterministic CI pipeline (no LLM-as-judge
+ * for tasks that have a clear JSON contract).
+ *
+ * The grader accepts the schema in two forms:
+ *   - A Zod schema object (`z.object({...})`): the most ergonomic
+ *     option, full TypeScript inference, no extra parse step.
+ *   - A path to a `.json` / `.ts` / `.mjs` schema file: the most
+ *     portable, lets the same schema be reused by the agent's
+ *     `--schema` flag AND the grader.
+ *
+ * Extraction follows the same rules as the agent's structured-output
+ * mode: try the raw text first, then a single ```json ... ``` fenced
+ * block. See {@link validateFinalText} in `@ikarmic/forge-core` for
+ * the reference implementation: this grader re-implements it
+ * directly so the eval harness stays a leaf package (no dep on the
+ * core's internal helpers).
+ */
+import type { z } from 'zod';
+import type { Grader } from './types.js';
+/** Options for {@link jsonSchema}. */
+export interface JsonSchemaGraderOptions {
+    /**
+     * Only relevant when `schema` is a string. Default true: the
+     * string is treated as a path to a schema file (`.ts` / `.js` /
+     * `.mjs` / `.cjs` / `.json`) and loaded via {@link loadSchemaFile}
+     * from `@ikarmic/forge-core` (dynamic import for JS/TS modules,
+     * JSON-Schema-to-Zod conversion for `.json`). Set to false to make
+     * the grader throw immediately instead of attempting to load the
+     * string as a path.
+     */
+    schemaIsPath?: boolean;
+    /**
+     * Truncate the raw `finalText` in failure details to this many
+     * characters. Default 400: long enough to see the offending JSON,
+     * short enough to keep reports scannable.
+     */
+    maxDetailChars?: number;
+}
+/**
+ * Grader that parses `finalText` as JSON and validates it against the
+ * supplied schema. Returns `{ passed: true }` on a successful
+ * validation, `{ passed: false, details }` with the validation error
+ * + truncated text otherwise.
+ *
+ * ```ts
+ * import { z } from 'zod';
+ * import { jsonSchema } from '@ikarmic/forge-eval';
+ *
+ * const prSchema = z.object({
+ *   title: z.string(),
+ *   body: z.string().min(20),
+ * });
+ *
+ * const task = defineTask({
+ *   id: 'summarize-pr',
+ *   graders: [jsonSchema(prSchema)],
+ *   ...
+ * });
+ *
+ * // Or load the same schema the agent uses via --schema:
+ * const task = defineTask({
+ *   id: 'extract-pr',
+ *   graders: [jsonSchema('./schemas/pr-summary.ts')],
+ *   ...
+ * });
+ * ```
+ */
+export declare function jsonSchema(schema: z.ZodTypeAny | string, options?: JsonSchemaGraderOptions): Grader;
+/**
+ * Grader that validates the parsed JSON's *subset* of fields: useful
+ * when the agent may add extra fields you don't want to grade but
+ * the core fields must match. Behaves like {@link jsonSchema} but
+ * wraps the user schema in `passthrough()` before validating, so any
+ * extra fields are ignored rather than triggering a strict failure.
+ *
+ * Note: when `schema` is a path, it can't be inspected synchronously,
+ * so the `passthrough()` wrapping is only applied to a Zod schema
+ * object passed directly: a path-loaded schema is graded as-is.
+ */
+export declare function jsonSchemaPartial(schema: z.ZodTypeAny | string, options?: JsonSchemaGraderOptions): Grader;
+//# sourceMappingURL=schema.d.ts.map

package/dist/graders/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/graders/schema.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAI7B,OAAO,KAAK,EAAE,MAAM,EAA+B,MAAM,YAAY,CAAC;AAEtE,sCAAsC;AACtC,MAAM,WAAW,uBAAuB;IACtC;;;;;;;;OAQG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAgB,UAAU,CACxB,MAAM,EAAE,CAAC,CAAC,UAAU,GAAG,MAAM,EAC7B,OAAO,GAAE,uBAA4B,GACpC,MAAM,CAgCR;AA2BD;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,CAAC,CAAC,UAAU,GAAG,MAAM,EAC7B,OAAO,GAAE,uBAA4B,GACpC,MAAM,CAUR"}