@devflow-tools/benchmark 0.16.21 → 0.16.22

package/dist/graders/deterministic.d.ts

@@ -0,0 +1,15 @@
+import type { GraderDefinition, GraderResult } from './types.js';
+export declare const DETERMINISTIC_GRADERS: GraderDefinition[];
+export declare function commandExitStatusGrader(): GraderDefinition;
+export declare function runBoundedCommand(file: string, args: string[], options: {
+    cwd: string;
+    timeoutMs?: number;
+    maxOutputBytes?: number;
+}): Promise<{
+    exitCode: number;
+    timedOut: boolean;
+    outputHash: string;
+    truncated: boolean;
+}>;
+export declare function result(graderId: string, status: GraderResult['status'], reason: string, startedAt: number, evidence: GraderResult['evidence'], score?: number, threshold?: number): GraderResult;
+//# sourceMappingURL=deterministic.d.ts.map

package/dist/graders/deterministic.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"deterministic.d.ts","sourceRoot":"","sources":["../../src/graders/deterministic.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAiB,gBAAgB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAKhF,eAAO,MAAM,qBAAqB,EAAE,gBAAgB,EAWnD,CAAC;AAEF,wBAAgB,uBAAuB,IAAI,gBAAgB,CAuC1D;AAED,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAAC,cAAc,CAAC,EAAE,MAAM,CAAA;CAAE,GACpE,OAAO,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,OAAO,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,OAAO,CAAA;CAAE,CAAC,CAgC1F;AAmHD,wBAAgB,MAAM,CACpB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,YAAY,CAAC,QAAQ,CAAC,EAC9B,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,YAAY,CAAC,UAAU,CAAC,EAClC,KAAK,CAAC,EAAE,MAAM,EACd,SAAS,CAAC,EAAE,MAAM,GACjB,YAAY,CAMd"}

package/dist/graders/deterministic.js

@@ -0,0 +1,178 @@
+import { execFile } from 'node:child_process';
+import { createHash } from 'node:crypto';
+const MAX_COMMAND_OUTPUT_BYTES = 32 * 1024;
+const DEFAULT_COMMAND_TIMEOUT_MS = 30_000;
+export const DETERMINISTIC_GRADERS = [
+    evidenceBooleanGrader('build_test_result', 'buildTestPassed', ['build', 'test', 'release']),
+    evidenceBooleanGrader('expected_file_state', 'expectedFileState', ['modify', 'release']),
+    evidenceBooleanGrader('forbidden_diff', 'forbiddenDiffClean', ['modify', 'release']),
+    receiptGrader(),
+    evidenceBooleanGrader('target_selection', 'targetSelectionValid', ['retrieval', 'analysis', 'release']),
+    evidenceBooleanGrader('schema_validation', 'schemaValid', ['tool', 'release']),
+    evidenceBooleanGrader('version_compatibility', 'versionCompatible', ['release', 'dependency']),
+    terminalStateGrader(),
+    evidenceBooleanGrader('workflow_evidence', 'workflowVerified', ['workflow', 'release']),
+    memoryOutcomeGrader(),
+];
+export function commandExitStatusGrader() {
+    return {
+        id: 'command_exit_status',
+        kind: 'deterministic',
+        taskKinds: ['build', 'test', 'release', 'workflow'],
+        requiredEvidence: ['command'],
+        timeoutMs: DEFAULT_COMMAND_TIMEOUT_MS,
+        cost: 'low',
+        blocking: true,
+        grade: async (context) => {
+            const startedAt = Date.now();
+            const command = context.evidence.command;
+            if (!isRecord(command) || typeof command.file !== 'string' || !command.file.trim()) {
+                return result('command_exit_status', 'unavailable', 'Structured command evidence is unavailable.', startedAt, []);
+            }
+            const args = Array.isArray(command.args)
+                ? command.args.filter((item) => typeof item === 'string')
+                : [];
+            const timeoutMs = boundedNumber(command.timeoutMs, 250, DEFAULT_COMMAND_TIMEOUT_MS, DEFAULT_COMMAND_TIMEOUT_MS);
+            const maxOutputBytes = boundedNumber(command.maxOutputBytes, 1_024, MAX_COMMAND_OUTPUT_BYTES, MAX_COMMAND_OUTPUT_BYTES);
+            const execution = await runBoundedCommand(command.file, args, {
+                cwd: typeof command.cwd === 'string' ? command.cwd : context.projectRoot,
+                timeoutMs,
+                maxOutputBytes,
+            });
+            return result('command_exit_status', execution.exitCode === 0 && !execution.timedOut ? 'passed' : 'failed', execution.timedOut ? 'Command timed out.' : `Command exited with code ${execution.exitCode}.`, startedAt, [
+                { type: 'command', value: [command.file, ...args].join(' ') },
+                { type: 'output_hash', value: execution.outputHash },
+            ], execution.exitCode === 0 && !execution.timedOut ? 1 : 0, 1);
+        },
+    };
+}
+export async function runBoundedCommand(file, args, options) {
+    const timeoutMs = options.timeoutMs ?? DEFAULT_COMMAND_TIMEOUT_MS;
+    const maxOutputBytes = options.maxOutputBytes ?? MAX_COMMAND_OUTPUT_BYTES;
+    return new Promise(resolve => {
+        let stdout = Buffer.alloc(0);
+        let stderr = Buffer.alloc(0);
+        let timedOut = false;
+        const child = execFile(file, args, {
+            cwd: options.cwd,
+            timeout: timeoutMs,
+            maxBuffer: maxOutputBytes * 2,
+            encoding: 'buffer',
+        }, (error, out, err) => {
+            stdout = Buffer.isBuffer(out) ? out : Buffer.from(out ?? '');
+            stderr = Buffer.isBuffer(err) ? err : Buffer.from(err ?? '');
+            const combined = Buffer.concat([stdout, Buffer.from('\n'), stderr]);
+            const bounded = combined.subarray(0, maxOutputBytes);
+            const code = typeof error?.code === 'number'
+                ? Number(error.code)
+                : error ? 1 : 0;
+            resolve({
+                exitCode: code,
+                timedOut,
+                outputHash: createHash('sha256').update(bounded).digest('hex'),
+                truncated: combined.length > bounded.length,
+            });
+        });
+        child.once('error', () => { });
+        child.once('exit', (_code, signal) => {
+            if (signal === 'SIGTERM' || signal === 'SIGKILL')
+                timedOut = true;
+        });
+    });
+}
+function evidenceBooleanGrader(id, key, taskKinds) {
+    return {
+        id,
+        kind: 'deterministic',
+        taskKinds,
+        requiredEvidence: [key],
+        timeoutMs: 1_000,
+        cost: 'none',
+        blocking: true,
+        grade: context => {
+            const startedAt = Date.now();
+            const value = context.evidence[key];
+            if (typeof value !== 'boolean') {
+                return result(id, 'unavailable', `Required evidence ${key} is unavailable.`, startedAt, []);
+            }
+            return result(id, value ? 'passed' : 'failed', value ? `${key} passed.` : `${key} failed.`, startedAt, [{ type: key, value: String(value) }], value ? 1 : 0, 1);
+        },
+    };
+}
+function receiptGrader() {
+    return {
+        id: 'receipt_validity',
+        kind: 'deterministic',
+        taskKinds: ['memory', 'workflow', 'release', 'tool'],
+        requiredEvidence: ['receipt'],
+        timeoutMs: 1_000,
+        cost: 'none',
+        blocking: true,
+        grade: context => {
+            const startedAt = Date.now();
+            const receipt = context.evidence.receipt;
+            if (typeof receipt !== 'string')
+                return result('receipt_validity', 'unavailable', 'Receipt is unavailable.', startedAt, []);
+            const valid = /^[a-z][a-z0-9_-]*:[A-Za-z0-9:._-]{8,}$/u.test(receipt);
+            return result('receipt_validity', valid ? 'passed' : 'failed', valid ? 'Receipt shape is valid.' : 'Receipt shape is invalid.', startedAt, [{ type: 'receipt_hash', value: createHash('sha256').update(receipt).digest('hex') }], valid ? 1 : 0, 1);
+        },
+    };
+}
+function terminalStateGrader() {
+    const terminal = new Set(['completed', 'failed', 'verified', 'cancelled', 'degraded', 'closed', 'closed_with_pending_work']);
+    return {
+        id: 'lifecycle_terminal_state',
+        kind: 'deterministic',
+        taskKinds: ['workflow', 'release', 'session'],
+        requiredEvidence: ['lifecycleState'],
+        timeoutMs: 1_000,
+        cost: 'none',
+        blocking: true,
+        grade: context => {
+            const startedAt = Date.now();
+            const state = context.evidence.lifecycleState;
+            if (typeof state !== 'string')
+                return result('lifecycle_terminal_state', 'unavailable', 'Lifecycle state is unavailable.', startedAt, []);
+            const valid = terminal.has(state);
+            return result('lifecycle_terminal_state', valid ? 'passed' : 'failed', valid ? `Lifecycle reached ${state}.` : `Lifecycle remains non-terminal at ${state}.`, startedAt, [{ type: 'lifecycle_state', value: state }], valid ? 1 : 0, 1);
+        },
+    };
+}
+function memoryOutcomeGrader() {
+    return {
+        id: 'memory_outcome',
+        kind: 'deterministic',
+        taskKinds: ['memory', 'release'],
+        requiredEvidence: ['memoryOutcome'],
+        timeoutMs: 1_000,
+        cost: 'none',
+        blocking: true,
+        grade: context => {
+            const startedAt = Date.now();
+            const outcome = context.evidence.memoryOutcome;
+            if (!isRecord(outcome))
+                return result('memory_outcome', 'unavailable', 'Memory outcome is unavailable.', startedAt, []);
+            const selected = Number(outcome.selected ?? 0);
+            const adopted = Number(outcome.adopted ?? 0);
+            const status = outcome.status;
+            const valid = (status === 'evaluated' || status === 'positive') && selected >= 0 && adopted >= 0;
+            return result('memory_outcome', valid ? 'passed' : 'failed', valid ? `Memory outcome evaluated (${adopted}/${selected} adopted).` : 'Memory outcome was not verified.', startedAt, [{ type: 'memory_outcome', value: JSON.stringify({ status, selected, adopted }) }], valid ? 1 : 0, 1);
+        },
+    };
+}
+export function result(graderId, status, reason, startedAt, evidence, score, threshold) {
+    const durationMs = Math.max(0, Date.now() - startedAt);
+    const receipt = `grader:${createHash('sha256').update(JSON.stringify({
+        graderId, status, reason, evidence, score, threshold,
+    })).digest('hex').slice(0, 24)}`;
+    return { graderId, kind: 'deterministic', status, score, threshold, evidence, reason, durationMs, receipt };
+}
+function isRecord(value) {
+    return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
+}
+function boundedNumber(value, minimum, maximum, fallback) {
+    return typeof value === 'number' && Number.isFinite(value)
+        ? Math.max(minimum, Math.min(maximum, Math.floor(value)))
+        : fallback;
+}
+//# sourceMappingURL=deterministic.js.map

package/dist/graders/deterministic.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"deterministic.js","sourceRoot":"","sources":["../../src/graders/deterministic.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC,MAAM,wBAAwB,GAAG,EAAE,GAAG,IAAI,CAAC;AAC3C,MAAM,0BAA0B,GAAG,MAAM,CAAC;AAE1C,MAAM,CAAC,MAAM,qBAAqB,GAAuB;IACvD,qBAAqB,CAAC,mBAAmB,EAAE,iBAAiB,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAC3F,qBAAqB,CAAC,qBAAqB,EAAE,mBAAmB,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IACxF,qBAAqB,CAAC,gBAAgB,EAAE,oBAAoB,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IACpF,aAAa,EAAE;IACf,qBAAqB,CAAC,kBAAkB,EAAE,sBAAsB,EAAE,CAAC,WAAW,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;IACvG,qBAAqB,CAAC,mBAAmB,EAAE,aAAa,EAAE,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAC9E,qBAAqB,CAAC,uBAAuB,EAAE,mBAAmB,EAAE,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IAC9F,mBAAmB,EAAE;IACrB,qBAAqB,CAAC,mBAAmB,EAAE,kBAAkB,EAAE,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;IACvF,mBAAmB,EAAE;CACtB,CAAC;AAEF,MAAM,UAAU,uBAAuB;IACrC,OAAO;QACL,EAAE,EAAE,qBAAqB;QACzB,IAAI,EAAE,eAAe;QACrB,SAAS,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,CAAC;QACnD,gBAAgB,EAAE,CAAC,SAAS,CAAC;QAC7B,SAAS,EAAE,0BAA0B;QACrC,IAAI,EAAE,KAAK;QACX,QAAQ,EAAE,IAAI;QACd,KAAK,EAAE,KAAK,EAAC,OAAO,EAAC,EAAE;YACrB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;YACzC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;gBACnF,OAAO,MAAM,CAAC,qBAAqB,EAAE,aAAa,EAAE,6CAA6C,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;YACpH,CAAC;YACD,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC;gBACtC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC;gBACzE,CAAC,CAAC,EAAE,CAAC;YACP,MAAM,SAAS,GAAG,aAAa,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,EAAE,0BAA0B,EAAE,0BAA0B,CAAC,CAAC;YAChH,MAAM,cAAc,GAAG,aAAa,CAAC,OAAO,CAAC,cAAc,EAAE,KAAK,EAAE,wBAAwB,EAAE,wBAAwB,CAAC,CAAC;YACxH,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE;gBAC5D,GAAG,EAAE,OAAO,OAAO,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW;gBACxE,SAAS;gBACT,cAAc;aACf,CAAC,CAAC;YACH,OAAO,MAAM,CACX,qBAAqB,EACrB,SAAS,CAAC,QAAQ,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EACrE,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,4BAA4B,SAAS,CAAC,QAAQ,GAAG,EAC7F,SAAS,EACT;gBACE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;gBAC7D,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,SAAS,CAAC,UAAU,EAAE;aACrD,EACD,SAAS,CAAC,QAAQ,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACvD,CAAC,CACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,IAAc,EACd,OAAqE;IAErE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,0BAA0B,CAAC;IAClE,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,wBAAwB,CAAC;IAC1E,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE;QAC3B,IAAI,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC7B,IAAI,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC7B,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE;YACjC,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,OAAO,EAAE,SAAS;YAClB,SAAS,EAAE,cAAc,GAAG,CAAC;YAC7B,QAAQ,EAAE,QAAQ;SACnB,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE;YACrB,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC;YAC7D,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC;YAC7D,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;YACpE,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;YACrD,MAAM,IAAI,GAAG,OAAQ,KAAsC,EAAE,IAAI,KAAK,QAAQ;gBAC5E,CAAC,CAAC,MAAM,CAAE,KAA+B,CAAC,IAAI,CAAC;gBAC/C,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAClB,OAAO,CAAC;gBACN,QAAQ,EAAE,IAAI;gBACd,QAAQ;gBACR,UAAU,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;gBAC9D,SAAS,EAAE,QAAQ,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;aAC5C,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;YACnC,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,SAAS;gBAAE,QAAQ,GAAG,IAAI,CAAC;QACpE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,qBAAqB,CAAC,EAAU,EAAE,GAAW,EAAE,SAAmB;IACzE,OAAO;QACL,EAAE;QACF,IAAI,EAAE,eAAe;QACrB,SAAS;QACT,gBAAgB,EAAE,CAAC,GAAG,CAAC;QACvB,SAAS,EAAE,KAAK;QAChB,IAAI,EAAE,MAAM;QACZ,QAAQ,EAAE,IAAI;QACd,KAAK,EAAE,OAAO,CAAC,EAAE;YACf,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YACpC,IAAI,OAAO,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC/B,OAAO,MAAM,CAAC,EAAE,EAAE,aAAa,EAAE,qBAAqB,GAAG,kBAAkB,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;YAC9F,CAAC;YACD,OAAO,MAAM,CACX,EAAE,EACF,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAC3B,KAAK,CAAC,CAAC,CAAC,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,GAAG,GAAG,UAAU,EAC3C,SAAS,EACT,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,EACrC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACb,CAAC,CACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,aAAa;IACpB,OAAO;QACL,EAAE,EAAE,kBAAkB;QACtB,IAAI,EAAE,eAAe;QACrB,SAAS,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,CAAC;QACpD,gBAAgB,EAAE,CAAC,SAAS,CAAC;QAC7B,SAAS,EAAE,KAAK;QAChB,IAAI,EAAE,MAAM;QACZ,QAAQ,EAAE,IAAI;QACd,KAAK,EAAE,OAAO,CAAC,EAAE;YACf,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;YACzC,IAAI,OAAO,OAAO,KAAK,QAAQ;gBAAE,OAAO,MAAM,CAAC,kBAAkB,EAAE,aAAa,EAAE,yBAAyB,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;YAC5H,MAAM,KAAK,GAAG,yCAAyC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtE,OAAO,MAAM,CACX,kBAAkB,EAClB,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAC3B,KAAK,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,2BAA2B,EAC/D,SAAS,EACT,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,EACrF,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACb,CAAC,CACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB;IAC1B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,0BAA0B,CAAC,CAAC,CAAC;IAC7H,OAAO;QACL,EAAE,EAAE,0BAA0B;QAC9B,IAAI,EAAE,eAAe;QACrB,SAAS,EAAE,CAAC,UAAU,EAAE,SAAS,EAAE,SAAS,CAAC;QAC7C,gBAAgB,EAAE,CAAC,gBAAgB,CAAC;QACpC,SAAS,EAAE,KAAK;QAChB,IAAI,EAAE,MAAM;QACZ,QAAQ,EAAE,IAAI;QACd,KAAK,EAAE,OAAO,CAAC,EAAE;YACf,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;YAC9C,IAAI,OAAO,KAAK,KAAK,QAAQ;gBAAE,OAAO,MAAM,CAAC,0BAA0B,EAAE,aAAa,EAAE,iCAAiC,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;YAC1I,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAClC,OAAO,MAAM,CACX,0BAA0B,EAC1B,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAC3B,KAAK,CAAC,CAAC,CAAC,qBAAqB,KAAK,GAAG,CAAC,CAAC,CAAC,qCAAqC,KAAK,GAAG,EACrF,SAAS,EACT,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAC3C,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACb,CAAC,CACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB;IAC1B,OAAO;QACL,EAAE,EAAE,gBAAgB;QACpB,IAAI,EAAE,eAAe;QACrB,SAAS,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC;QAChC,gBAAgB,EAAE,CAAC,eAAe,CAAC;QACnC,SAAS,EAAE,KAAK;QAChB,IAAI,EAAE,MAAM;QACZ,QAAQ,EAAE,IAAI;QACd,KAAK,EAAE,OAAO,CAAC,EAAE;YACf,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;YAC/C,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAAE,OAAO,MAAM,CAAC,gBAAgB,EAAE,aAAa,EAAE,gCAAgC,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;YACxH,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;YAC/C,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC;YAC7C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;YAC9B,MAAM,KAAK,GAAG,CAAC,MAAM,KAAK,WAAW,IAAI,MAAM,KAAK,UAAU,CAAC,IAAI,QAAQ,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,CAAC;YACjG,OAAO,MAAM,CACX,gBAAgB,EAChB,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAC3B,KAAK,CAAC,CAAC,CAAC,6BAA6B,OAAO,IAAI,QAAQ,YAAY,CAAC,CAAC,CAAC,kCAAkC,EACzG,SAAS,EACT,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,EAClF,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACb,CAAC,CACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,MAAM,CACpB,QAAgB,EAChB,MAA8B,EAC9B,MAAc,EACd,SAAiB,EACjB,QAAkC,EAClC,KAAc,EACd,SAAkB;IAElB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC;IACvD,MAAM,OAAO,GAAG,UAAU,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC;QACnE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS;KACrD,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;IACjC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,eAAe,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC;AAC9G,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,aAAa,CAAC,KAAc,EAAE,OAAe,EAAE,OAAe,EAAE,QAAgB;IACvF,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QACxD,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QACzD,CAAC,CAAC,QAAQ,CAAC;AACf,CAAC"}

package/dist/graders/index.d.ts

@@ -0,0 +1,4 @@
+export { DETERMINISTIC_GRADERS, commandExitStatusGrader, runBoundedCommand } from './deterministic.js';
+export { GraderRegistry, modelResultCannotOverrideDeterministicFailure, summarizeGraderResults, } from './registry.js';
+export type { GraderContext, GraderDefinition, GraderEvidenceReference, GraderKind, GraderResult, GraderRunSummary, GraderStatus, } from './types.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/graders/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/graders/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvG,OAAO,EACL,cAAc,EACd,6CAA6C,EAC7C,sBAAsB,GACvB,MAAM,eAAe,CAAC;AACvB,YAAY,EACV,aAAa,EACb,gBAAgB,EAChB,uBAAuB,EACvB,UAAU,EACV,YAAY,EACZ,gBAAgB,EAChB,YAAY,GACb,MAAM,YAAY,CAAC"}

package/dist/graders/index.js

@@ -0,0 +1,3 @@
+export { DETERMINISTIC_GRADERS, commandExitStatusGrader, runBoundedCommand } from './deterministic.js';
+export { GraderRegistry, modelResultCannotOverrideDeterministicFailure, summarizeGraderResults, } from './registry.js';
+//# sourceMappingURL=index.js.map

package/dist/graders/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/graders/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvG,OAAO,EACL,cAAc,EACd,6CAA6C,EAC7C,sBAAsB,GACvB,MAAM,eAAe,CAAC"}

package/dist/graders/registry.d.ts

@@ -0,0 +1,11 @@
+import type { GraderContext, GraderDefinition, GraderResult, GraderRunSummary } from './types.js';
+export declare class GraderRegistry {
+    private readonly definitions;
+    constructor(definitions?: GraderDefinition[]);
+    register(definition: GraderDefinition): void;
+    list(taskKind?: string): GraderDefinition[];
+    run(context: GraderContext, graderIds?: string[]): Promise<GraderRunSummary>;
+}
+export declare function summarizeGraderResults(definitions: GraderDefinition[], results: GraderResult[]): GraderRunSummary;
+export declare function modelResultCannotOverrideDeterministicFailure(deterministic: GraderResult[], model: GraderResult): GraderResult;
+//# sourceMappingURL=registry.d.ts.map

package/dist/graders/registry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../../src/graders/registry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,aAAa,EACb,gBAAgB,EAChB,YAAY,EACZ,gBAAgB,EACjB,MAAM,YAAY,CAAC;AAGpB,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAuC;gBAEvD,WAAW,GAAE,gBAAgB,EAA0D;IAInG,QAAQ,CAAC,UAAU,EAAE,gBAAgB,GAAG,IAAI;IAM5C,IAAI,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,gBAAgB,EAAE;IAKrC,GAAG,CAAC,OAAO,EAAE,aAAa,EAAE,SAAS,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAuBnF;AAED,wBAAgB,sBAAsB,CACpC,WAAW,EAAE,gBAAgB,EAAE,EAC/B,OAAO,EAAE,YAAY,EAAE,GACtB,gBAAgB,CAgBlB;AAED,wBAAgB,6CAA6C,CAC3D,aAAa,EAAE,YAAY,EAAE,EAC7B,KAAK,EAAE,YAAY,GAClB,YAAY,CAUd"}

package/dist/graders/registry.js

@@ -0,0 +1,85 @@
+import { commandExitStatusGrader, DETERMINISTIC_GRADERS } from './deterministic.js';
+export class GraderRegistry {
+    definitions = new Map();
+    constructor(definitions = [commandExitStatusGrader(), ...DETERMINISTIC_GRADERS]) {
+        for (const definition of definitions)
+            this.register(definition);
+    }
+    register(definition) {
+        if (!definition.id.trim())
+            throw new Error('Grader ID is required');
+        if (this.definitions.has(definition.id))
+            throw new Error(`Grader ${definition.id} is already registered`);
+        this.definitions.set(definition.id, definition);
+    }
+    list(taskKind) {
+        return [...this.definitions.values()].filter(definition => !taskKind || definition.taskKinds.includes('*') || definition.taskKinds.includes(taskKind));
+    }
+    async run(context, graderIds) {
+        const selected = graderIds
+            ? graderIds.map(id => this.definitions.get(id)).filter((item) => Boolean(item))
+            : this.list(context.taskKind);
+        const results = [];
+        for (const grader of selected) {
+            const missing = grader.requiredEvidence.filter(key => context.evidence[key] === undefined);
+            if (missing.length > 0) {
+                results.push({
+                    graderId: grader.id,
+                    kind: grader.kind,
+                    status: 'unavailable',
+                    evidence: missing.map(value => ({ type: 'missing_evidence', value })),
+                    reason: `Missing required evidence: ${missing.join(', ')}.`,
+                    durationMs: 0,
+                    receipt: `grader:unavailable:${grader.id}`,
+                });
+                continue;
+            }
+            results.push(await withTimeout(grader, context));
+        }
+        return summarizeGraderResults(selected, results);
+    }
+}
+export function summarizeGraderResults(definitions, results) {
+    const byId = new Map(definitions.map(definition => [definition.id, definition]));
+    const blockingFailures = results
+        .filter(result => result.status === 'failed' && result.kind === 'deterministic' && byId.get(result.graderId)?.blocking)
+        .map(result => result.graderId);
+    const unavailableRequired = results
+        .filter(result => result.status === 'unavailable' && byId.get(result.graderId)?.blocking)
+        .map(result => result.graderId);
+    const status = blockingFailures.length > 0
+        ? 'failed'
+        : unavailableRequired.length > 0
+            ? 'unavailable'
+            : results.some(result => result.status === 'failed')
+                ? 'failed'
+                : 'passed';
+    return { status, results, blockingFailures, unavailableRequired };
+}
+export function modelResultCannotOverrideDeterministicFailure(deterministic, model) {
+    const blocking = deterministic.find(result => result.kind === 'deterministic' && result.status === 'failed');
+    if (!blocking || model.status !== 'passed')
+        return model;
+    return {
+        ...model,
+        status: 'failed',
+        score: 0,
+        reason: `Model pass cannot override deterministic failure ${blocking.graderId}.`,
+        receipt: `grader:model-blocked:${blocking.graderId}`,
+    };
+}
+async function withTimeout(definition, context) {
+    const timeout = new Promise(resolve => {
+        setTimeout(() => resolve({
+            graderId: definition.id,
+            kind: definition.kind,
+            status: 'unavailable',
+            evidence: [],
+            reason: `Grader timed out after ${definition.timeoutMs}ms.`,
+            durationMs: definition.timeoutMs,
+            receipt: `grader:timeout:${definition.id}`,
+        }), definition.timeoutMs).unref();
+    });
+    return Promise.race([Promise.resolve(definition.grade(context)), timeout]);
+}
+//# sourceMappingURL=registry.js.map

package/dist/graders/registry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"registry.js","sourceRoot":"","sources":["../../src/graders/registry.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,uBAAuB,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAEpF,MAAM,OAAO,cAAc;IACR,WAAW,GAAG,IAAI,GAAG,EAA4B,CAAC;IAEnE,YAAY,cAAkC,CAAC,uBAAuB,EAAE,EAAE,GAAG,qBAAqB,CAAC;QACjG,KAAK,MAAM,UAAU,IAAI,WAAW;YAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;IAClE,CAAC;IAED,QAAQ,CAAC,UAA4B;QACnC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QACpE,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,UAAU,UAAU,CAAC,EAAE,wBAAwB,CAAC,CAAC;QAC1G,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IAClD,CAAC;IAED,IAAI,CAAC,QAAiB;QACpB,OAAO,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CACxD,CAAC,QAAQ,IAAI,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IAChG,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,OAAsB,EAAE,SAAoB;QACpD,MAAM,QAAQ,GAAG,SAAS;YACxB,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAA4B,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACzG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAChC,MAAM,OAAO,GAAmB,EAAE,CAAC;QACnC,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;YAC9B,MAAM,OAAO,GAAG,MAAM,CAAC,gBAAgB,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,SAAS,CAAC,CAAC;YAC3F,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,OAAO,CAAC,IAAI,CAAC;oBACX,QAAQ,EAAE,MAAM,CAAC,EAAE;oBACnB,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,MAAM,EAAE,aAAa;oBACrB,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,KAAK,EAAE,CAAC,CAAC;oBACrE,MAAM,EAAE,8BAA8B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;oBAC3D,UAAU,EAAE,CAAC;oBACb,OAAO,EAAE,sBAAsB,MAAM,CAAC,EAAE,EAAE;iBAC3C,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,MAAM,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;QACnD,CAAC;QACD,OAAO,sBAAsB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACnD,CAAC;CACF;AAED,MAAM,UAAU,sBAAsB,CACpC,WAA+B,EAC/B,OAAuB;IAEvB,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC;IACjF,MAAM,gBAAgB,GAAG,OAAO;SAC7B,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,IAAI,KAAK,eAAe,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC;SACtH,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAClC,MAAM,mBAAmB,GAAG,OAAO;SAChC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,aAAa,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC;SACxF,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAClC,MAAM,MAAM,GAAG,gBAAgB,CAAC,MAAM,GAAG,CAAC;QACxC,CAAC,CAAC,QAAQ;QACV,CAAC,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC;YAC9B,CAAC,CAAC,aAAa;YACf,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC;gBAClD,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,QAAQ,CAAC;IACjB,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,CAAC;AACpE,CAAC;AAED,MAAM,UAAU,6CAA6C,CAC3D,aAA6B,EAC7B,KAAmB;IAEnB,MAAM,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,eAAe,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;IAC7G,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACzD,OAAO;QACL,GAAG,KAAK;QACR,MAAM,EAAE,QAAQ;QAChB,KAAK,EAAE,CAAC;QACR,MAAM,EAAE,oDAAoD,QAAQ,CAAC,QAAQ,GAAG;QAChF,OAAO,EAAE,wBAAwB,QAAQ,CAAC,QAAQ,EAAE;KACrD,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,UAA4B,EAAE,OAAsB;IAC7E,MAAM,OAAO,GAAG,IAAI,OAAO,CAAe,OAAO,CAAC,EAAE;QAClD,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC;YACvB,QAAQ,EAAE,UAAU,CAAC,EAAE;YACvB,IAAI,EAAE,UAAU,CAAC,IAAI;YACrB,MAAM,EAAE,aAAa;YACrB,QAAQ,EAAE,EAAE;YACZ,MAAM,EAAE,0BAA0B,UAAU,CAAC,SAAS,KAAK;YAC3D,UAAU,EAAE,UAAU,CAAC,SAAS;YAChC,OAAO,EAAE,kBAAkB,UAAU,CAAC,EAAE,EAAE;SAC3C,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC;IACpC,CAAC,CAAC,CAAC;IACH,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;AAC7E,CAAC"}

package/dist/graders/types.d.ts

@@ -0,0 +1,41 @@
+export type GraderKind = 'deterministic' | 'model' | 'human';
+export type GraderStatus = 'passed' | 'failed' | 'unavailable';
+export interface GraderEvidenceReference {
+    type: string;
+    value: string;
+}
+export interface GraderResult {
+    graderId: string;
+    kind: GraderKind;
+    status: GraderStatus;
+    score?: number;
+    threshold?: number;
+    evidence: GraderEvidenceReference[];
+    reason: string;
+    durationMs: number;
+    receipt: string;
+}
+export interface GraderContext {
+    taskId: string;
+    taskKind: string;
+    projectRoot: string;
+    acceptanceCriteria: string[];
+    evidence: Record<string, unknown>;
+}
+export interface GraderDefinition {
+    id: string;
+    kind: GraderKind;
+    taskKinds: string[];
+    requiredEvidence: string[];
+    timeoutMs: number;
+    cost: 'none' | 'low' | 'medium' | 'high';
+    blocking: boolean;
+    grade(context: GraderContext): Promise<GraderResult> | GraderResult;
+}
+export interface GraderRunSummary {
+    status: GraderStatus;
+    results: GraderResult[];
+    blockingFailures: string[];
+    unavailableRequired: string[];
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/graders/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/graders/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,UAAU,GAAG,eAAe,GAAG,OAAO,GAAG,OAAO,CAAC;AAC7D,MAAM,MAAM,YAAY,GAAG,QAAQ,GAAG,QAAQ,GAAG,aAAa,CAAC;AAE/D,MAAM,WAAW,uBAAuB;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,UAAU,CAAC;IACjB,MAAM,EAAE,YAAY,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,uBAAuB,EAAE,CAAC;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,UAAU,CAAC;IACjB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACzC,QAAQ,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,GAAG,YAAY,CAAC;CACrE;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,mBAAmB,EAAE,MAAM,EAAE,CAAC;CAC/B"}

package/dist/graders/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/graders/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/graders/types.ts"],"names":[],"mappings":""}

package/dist/index.d.ts

@@ -13,4 +13,7 @@ export { runRetrievalQualityBenchmark } from './retrieval-quality.js';
 export type { RetrievalQualityCase, RetrievalQualityFixture, RetrievalQualityReport, RetrievalQualityThresholds, } from './retrieval-quality.js';
 export { runRetrievalQualityCorpus } from './retrieval-quality-corpus.js';
 export type { RetrievalQualityCorpusFixture, RetrievalQualityCorpusProject, RetrievalQualityCorpusReport, } from './retrieval-quality-corpus.js';
+export * from './graders/index.js';
+export { aggregateRepeatedTrials, createTrialReceipt, renderRepeatedTrialReport, } from './repeated-trials.js';
+export type { RepeatedTrial, RepeatedTrialReport, RepeatedTrialTaskReport, } from './repeated-trials.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,YAAY,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAC1E,OAAO,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,YAAY,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,YAAY,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,YAAY,EAAE,wBAAwB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACnF,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,YAAY,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAClE,OAAO,EAAE,4BAA4B,EAAE,MAAM,wBAAwB,CAAC;AACtE,YAAY,EACV,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,0BAA0B,GAC3B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAC;AAC1E,YAAY,EACV,6BAA6B,EAC7B,6BAA6B,EAC7B,4BAA4B,GAC7B,MAAM,+BAA+B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,YAAY,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAC1E,OAAO,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,YAAY,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,YAAY,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,YAAY,EAAE,wBAAwB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACnF,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,YAAY,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAClE,OAAO,EAAE,4BAA4B,EAAE,MAAM,wBAAwB,CAAC;AACtE,YAAY,EACV,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,0BAA0B,GAC3B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAC;AAC1E,YAAY,EACV,6BAA6B,EAC7B,6BAA6B,EAC7B,4BAA4B,GAC7B,MAAM,+BAA+B,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,OAAO,EACL,uBAAuB,EACvB,kBAAkB,EAClB,yBAAyB,GAC1B,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,aAAa,EACb,mBAAmB,EACnB,uBAAuB,GACxB,MAAM,sBAAsB,CAAC"}

package/dist/index.js

@@ -6,4 +6,6 @@ export { runBenchmarkSuite } from "./suite-runner.js";
 export { renderBenchmarkReport } from "./report-renderer.js";
 export { runRetrievalQualityBenchmark } from './retrieval-quality.js';
 export { runRetrievalQualityCorpus } from './retrieval-quality-corpus.js';
+export * from './graders/index.js';
+export { aggregateRepeatedTrials, createTrialReceipt, renderRepeatedTrialReport, } from './repeated-trials.js';
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,OAAO,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAE7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAE7D,OAAO,EAAE,4BAA4B,EAAE,MAAM,wBAAwB,CAAC;AAOtE,OAAO,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,OAAO,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAE7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAE7D,OAAO,EAAE,4BAA4B,EAAE,MAAM,wBAAwB,CAAC;AAOtE,OAAO,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAC;AAM1E,cAAc,oBAAoB,CAAC;AACnC,OAAO,EACL,uBAAuB,EACvB,kBAAkB,EAClB,yBAAyB,GAC1B,MAAM,sBAAsB,CAAC"}

package/dist/repeated-trials.d.ts

@@ -0,0 +1,64 @@
+import type { GraderResult } from './graders/index.js';
+export interface RepeatedTrial {
+    trialId: string;
+    executionId: string;
+    taskId: string;
+    projectRevision: string;
+    fixtureVersion: string;
+    taskInputHash: string;
+    toolProfileHash: string;
+    acceptanceHash: string;
+    status: 'passed' | 'failed' | 'unavailable';
+    graderResults: GraderResult[];
+    evidenceReceipt: string;
+    durationMs: number;
+    tokensUsed: number;
+    fallbackCount: number;
+    failureCategory?: string;
+    simulated?: boolean;
+}
+export interface RepeatedTrialTaskReport {
+    taskId: string;
+    attemptedTrials: number;
+    successfulTrials: number;
+    unavailableTrials: number;
+    passAtK: 0 | 1;
+    passPowerK: 0 | 1;
+    trialIds: string[];
+}
+export interface RepeatedTrialReport {
+    runId: string;
+    suiteId: string;
+    suiteVersion: string;
+    projectRoot: string;
+    commit: string | null;
+    createdAt: number;
+    status: 'completed';
+    k: 1 | 3;
+    tasks: RepeatedTrialTaskReport[];
+    trials: RepeatedTrial[];
+    summary: {
+        attemptedTasks: number;
+        attemptedTrials: number;
+        successfulTrials: number;
+        passAt1: number;
+        passAtK: number;
+        passPowerK: number;
+        avgDurationMs: number;
+        avgTokensUsed: number;
+        fallbackRate: number;
+        graderStatus: 'passed' | 'failed' | 'unavailable';
+        failureCategories: Record<string, number>;
+    };
+}
+export declare function aggregateRepeatedTrials(input: {
+    suiteId: string;
+    suiteVersion: string;
+    projectRoot: string;
+    commit: string | null;
+    k: 1 | 3;
+    trials: RepeatedTrial[];
+}): RepeatedTrialReport;
+export declare function createTrialReceipt(input: Omit<RepeatedTrial, 'evidenceReceipt'>): string;
+export declare function renderRepeatedTrialReport(report: RepeatedTrialReport, format?: 'text' | 'json'): string;
+//# sourceMappingURL=repeated-trials.d.ts.map

package/dist/repeated-trials.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"repeated-trials.d.ts","sourceRoot":"","sources":["../src/repeated-trials.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,aAAa,CAAC;IAC5C,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,uBAAuB;IACtC,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC;IACf,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC;IAClB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,WAAW,CAAC;IACpB,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;IACT,KAAK,EAAE,uBAAuB,EAAE,CAAC;IACjC,MAAM,EAAE,aAAa,EAAE,CAAC;IACxB,OAAO,EAAE;QACP,cAAc,EAAE,MAAM,CAAC;QACvB,eAAe,EAAE,MAAM,CAAC;QACxB,gBAAgB,EAAE,MAAM,CAAC;QACzB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;QAChB,UAAU,EAAE,MAAM,CAAC;QACnB,aAAa,EAAE,MAAM,CAAC;QACtB,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,QAAQ,GAAG,QAAQ,GAAG,aAAa,CAAC;QAClD,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAC3C,CAAC;CACH;AAED,wBAAgB,uBAAuB,CAAC,KAAK,EAAE;IAC7C,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;IACT,MAAM,EAAE,aAAa,EAAE,CAAC;CACzB,GAAG,mBAAmB,CAyDtB;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,IAAI,CAAC,aAAa,EAAE,iBAAiB,CAAC,GAAG,MAAM,CAExF;AAED,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,mBAAmB,EAAE,MAAM,GAAE,MAAM,GAAG,MAAe,GAAG,MAAM,CAc/G"}

package/dist/repeated-trials.js

@@ -0,0 +1,136 @@
+import { createHash, randomUUID } from 'node:crypto';
+export function aggregateRepeatedTrials(input) {
+    validateTrials(input.trials, input.k);
+    const grouped = new Map();
+    for (const trial of input.trials) {
+        const list = grouped.get(trial.taskId) ?? [];
+        list.push(trial);
+        grouped.set(trial.taskId, list);
+    }
+    const tasks = [...grouped.entries()].sort(([left], [right]) => left.localeCompare(right)).map(([taskId, trials]) => {
+        if (trials.length !== input.k)
+            throw new Error(`Task ${taskId} requires exactly ${input.k} independent trials`);
+        const successfulTrials = trials.filter(trial => trial.status === 'passed').length;
+        return {
+            taskId,
+            attemptedTrials: trials.length,
+            successfulTrials,
+            unavailableTrials: trials.filter(trial => trial.status === 'unavailable').length,
+            passAtK: successfulTrials > 0 ? 1 : 0,
+            passPowerK: successfulTrials === input.k ? 1 : 0,
+            trialIds: trials.map(trial => trial.trialId),
+        };
+    });
+    const attemptedTrials = input.trials.length;
+    const successfulTrials = input.trials.filter(trial => trial.status === 'passed').length;
+    const failureCategories = {};
+    for (const trial of input.trials) {
+        if (!trial.failureCategory)
+            continue;
+        failureCategories[trial.failureCategory] = (failureCategories[trial.failureCategory] ?? 0) + 1;
+    }
+    const unavailable = input.trials.some(trial => trial.status === 'unavailable');
+    const failed = input.trials.some(trial => trial.status === 'failed');
+    return {
+        runId: `eval:${Date.now()}:${randomUUID().slice(0, 8)}`,
+        suiteId: input.suiteId,
+        suiteVersion: input.suiteVersion,
+        projectRoot: input.projectRoot,
+        commit: input.commit,
+        createdAt: Date.now(),
+        status: 'completed',
+        k: input.k,
+        tasks,
+        trials: input.trials,
+        summary: {
+            attemptedTasks: tasks.length,
+            attemptedTrials,
+            successfulTrials,
+            passAt1: attemptedTrials > 0 ? successfulTrials / attemptedTrials : 0,
+            passAtK: average(tasks.map(task => task.passAtK)),
+            passPowerK: average(tasks.map(task => task.passPowerK)),
+            avgDurationMs: average(input.trials.map(trial => trial.durationMs)),
+            avgTokensUsed: average(input.trials.map(trial => trial.tokensUsed)),
+            fallbackRate: attemptedTrials > 0
+                ? input.trials.filter(trial => trial.fallbackCount > 0).length / attemptedTrials
+                : 0,
+            graderStatus: failed ? 'failed' : unavailable ? 'unavailable' : 'passed',
+            failureCategories,
+        },
+    };
+}
+export function createTrialReceipt(input) {
+    return `trial:${createHash('sha256').update(stableJson(input)).digest('hex').slice(0, 24)}`;
+}
+export function renderRepeatedTrialReport(report, format = 'text') {
+    if (format === 'json')
+        return JSON.stringify(report, null, 2);
+    return [
+        `DevFlow Evaluation: ${report.suiteId} (${report.suiteVersion})`,
+        `Run: ${report.runId}`,
+        `Project: ${report.projectRoot}`,
+        `Revision: ${report.commit ?? 'unknown'}`,
+        `Trials: ${report.summary.attemptedTrials} (${report.summary.attemptedTasks} task(s) x ${report.k})`,
+        `pass@1: ${fraction(report.summary.successfulTrials, report.summary.attemptedTrials)}`,
+        `pass@${report.k}: ${fraction(report.tasks.filter(task => task.passAtK === 1).length, report.summary.attemptedTasks)}`,
+        `pass^${report.k}: ${fraction(report.tasks.filter(task => task.passPowerK === 1).length, report.summary.attemptedTasks)}`,
+        `Grader status: ${report.summary.graderStatus}`,
+        `Fallback rate: ${(report.summary.fallbackRate * 100).toFixed(1)}%`,
+    ].join('\n');
+}
+function validateTrials(trials, k) {
+    if (trials.length === 0)
+        throw new Error('Repeated evaluation requires real trials');
+    const trialIds = new Set();
+    const executionIds = new Set();
+    const identityByTask = new Map();
+    for (const trial of trials) {
+        if (trial.simulated)
+            throw new Error(`Simulated trial ${trial.trialId} is not accepted`);
+        if (!trial.evidenceReceipt.trim() || trial.graderResults.length === 0) {
+            throw new Error(`Trial ${trial.trialId} is missing evidence-bound grader results`);
+        }
+        if (trialIds.has(trial.trialId))
+            throw new Error(`Duplicate trial ID ${trial.trialId}`);
+        if (executionIds.has(trial.executionId))
+            throw new Error(`Duplicate execution ID ${trial.executionId}`);
+        trialIds.add(trial.trialId);
+        executionIds.add(trial.executionId);
+        const identity = stableJson({
+            projectRevision: trial.projectRevision,
+            fixtureVersion: trial.fixtureVersion,
+            taskInputHash: trial.taskInputHash,
+            toolProfileHash: trial.toolProfileHash,
+            acceptanceHash: trial.acceptanceHash,
+        });
+        const existing = identityByTask.get(trial.taskId);
+        if (existing && existing !== identity)
+            throw new Error(`Mixed trial identity for task ${trial.taskId}`);
+        identityByTask.set(trial.taskId, identity);
+    }
+    for (const [taskId, count] of countByTask(trials)) {
+        if (count !== k)
+            throw new Error(`Task ${taskId} requires exactly ${k} trials; received ${count}`);
+    }
+}
+function countByTask(trials) {
+    const counts = new Map();
+    for (const trial of trials)
+        counts.set(trial.taskId, (counts.get(trial.taskId) ?? 0) + 1);
+    return counts;
+}
+function stableJson(value) {
+    if (Array.isArray(value))
+        return `[${value.map(stableJson).join(',')}]`;
+    if (!value || typeof value !== 'object')
+        return JSON.stringify(value) ?? 'null';
+    const record = value;
+    return `{${Object.keys(record).sort().map(key => `${JSON.stringify(key)}:${stableJson(record[key])}`).join(',')}}`;
+}
+function average(values) {
+    return values.length > 0 ? values.reduce((sum, value) => sum + value, 0) / values.length : 0;
+}
+function fraction(numerator, denominator) {
+    return `${numerator}/${denominator} (${denominator > 0 ? (numerator / denominator * 100).toFixed(1) : '0.0'}%)`;
+}
+//# sourceMappingURL=repeated-trials.js.map

package/dist/repeated-trials.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"repeated-trials.js","sourceRoot":"","sources":["../src/repeated-trials.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AA0DrD,MAAM,UAAU,uBAAuB,CAAC,KAOvC;IACC,cAAc,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IACtC,MAAM,OAAO,GAAG,IAAI,GAAG,EAA2B,CAAC;IACnD,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAC7C,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAClC,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE;QACjH,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,MAAM,qBAAqB,KAAK,CAAC,CAAC,qBAAqB,CAAC,CAAC;QAChH,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;QAClF,OAAO;YACL,MAAM;YACN,eAAe,EAAE,MAAM,CAAC,MAAM;YAC9B,gBAAgB;YAChB,iBAAiB,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,aAAa,CAAC,CAAC,MAAM;YAChF,OAAO,EAAE,gBAAgB,GAAG,CAAC,CAAC,CAAC,CAAC,CAAU,CAAC,CAAC,CAAC,CAAU;YACvD,UAAU,EAAE,gBAAgB,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAU,CAAC,CAAC,CAAC,CAAU;YAClE,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC;SAC7C,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,MAAM,eAAe,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;IAC5C,MAAM,gBAAgB,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;IACxF,MAAM,iBAAiB,GAA2B,EAAE,CAAC;IACrD,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACjC,IAAI,CAAC,KAAK,CAAC,eAAe;YAAE,SAAS;QACrC,iBAAiB,CAAC,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,iBAAiB,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IACjG,CAAC;IACD,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,aAAa,CAAC,CAAC;IAC/E,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;IACrE,OAAO;QACL,KAAK,EAAE,QAAQ,IAAI,CAAC,GAAG,EAAE,IAAI,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;QACvD,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;QACrB,MAAM,EAAE,WAAW;QACnB,CAAC,EAAE,KAAK,CAAC,CAAC;QACV,KAAK;QACL,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,OAAO,EAAE;YACP,cAAc,EAAE,KAAK,CAAC,MAAM;YAC5B,eAAe;YACf,gBAAgB;YAChB,OAAO,EAAE,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;YACrE,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACjD,UAAU,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACvD,aAAa,EAAE,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACnE,aAAa,EAAE,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACnE,YAAY,EAAE,eAAe,GAAG,CAAC;gBAC/B,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,MAAM,GAAG,eAAe;gBAChF,CAAC,CAAC,CAAC;YACL,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ;YACxE,iBAAiB;SAClB;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,KAA6C;IAC9E,OAAO,SAAS,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;AAC9F,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,MAA2B,EAAE,SAA0B,MAAM;IACrG,IAAI,MAAM,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAC9D,OAAO;QACL,uBAAuB,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC,YAAY,GAAG;QAChE,QAAQ,MAAM,CAAC,KAAK,EAAE;QACtB,YAAY,MAAM,CAAC,WAAW,EAAE;QAChC,aAAa,MAAM,CAAC,MAAM,IAAI,SAAS,EAAE;QACzC,WAAW,MAAM,CAAC,OAAO,CAAC,eAAe,KAAK,MAAM,CAAC,OAAO,CAAC,cAAc,cAAc,MAAM,CAAC,CAAC,GAAG;QACpG,WAAW,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,gBAAgB,EAAE,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE;QACtF,QAAQ,MAAM,CAAC,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE;QACtH,QAAQ,MAAM,CAAC,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE;QACzH,kBAAkB,MAAM,CAAC,OAAO,CAAC,YAAY,EAAE;QAC/C,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;KACpE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,MAAuB,EAAE,CAAS;IACxD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IACrF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IACvC,MAAM,cAAc,GAAG,IAAI,GAAG,EAAkB,CAAC;IACjD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;QACzF,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,EAAE,IAAI,KAAK,CAAC,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtE,MAAM,IAAI,KAAK,CAAC,SAAS,KAAK,CAAC,OAAO,2CAA2C,CAAC,CAAC;QACrF,CAAC;QACD,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACxF,IAAI,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;QACxG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC5B,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QACpC,MAAM,QAAQ,GAAG,UAAU,CAAC;YAC1B,eAAe,EAAE,KAAK,CAAC,eAAe;YACtC,cAAc,EAAE,KAAK,CAAC,cAAc;YACpC,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,eAAe,EAAE,KAAK,CAAC,eAAe;YACtC,cAAc,EAAE,KAAK,CAAC,cAAc;SACrC,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAClD,IAAI,QAAQ,IAAI,QAAQ,KAAK,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QACxG,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC7C,CAAC;IACD,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;QAClD,IAAI,KAAK,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,MAAM,qBAAqB,CAAC,qBAAqB,KAAK,EAAE,CAAC,CAAC;IACrG,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,MAAuB;IAC1C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,KAAK,IAAI,MAAM;QAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1F,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;IACxE,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC;IAChF,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;AACrH,CAAC;AAED,SAAS,OAAO,CAAC,MAAgB;IAC/B,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAC/F,CAAC;AAED,SAAS,QAAQ,CAAC,SAAiB,EAAE,WAAmB;IACtD,OAAO,GAAG,SAAS,IAAI,WAAW,KAAK,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC;AAClH,CAAC"}

package/dist/report-renderer.js

@@ -23,6 +23,9 @@ function renderText(report) {
         report.regressions.length === 0
             ? "Regressions >10%: none"
             : `Regressions >10%: ${report.regressions.map((item) => item.message).join("; ")}`,
+        ...(report.evaluation ? [
+            `Evaluation: ${report.evaluation.status} (${report.evaluation.passedTasks} passed, ${report.evaluation.failedTasks} failed, ${report.evaluation.unavailableTasks} unavailable)`,
+        ] : []),
     ];
     return lines.join("\n");
 }
@@ -60,6 +63,12 @@ function renderMarkdown(report) {
         "## Regressions Over 10%",
         "",
         regressions,
+        ...(report.evaluation ? [
+            '',
+            '## Evaluation',
+            '',
+            `Status: **${report.evaluation.status}**. Passed ${report.evaluation.passedTasks}, failed ${report.evaluation.failedTasks}, unavailable ${report.evaluation.unavailableTasks}.`,
+        ] : []),
         "",
         "The baseline leg is deterministic lexical repository search. The DevFlow leg uses task-context retrieval from the initialized project CodeGraph. No estimated or simulated scores are included.",
         "",

package/dist/report-renderer.js.map

@@ -1 +1 @@
-{"version":3,"file":"report-renderer.js","sourceRoot":"","sources":["../src/report-renderer.ts"],"names":[],"mappings":"AAIA,MAAM,UAAU,qBAAqB,CACnC,MAA4B,EAC5B,MAA6B;IAE7B,IAAI,MAAM,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAC9D,IAAI,MAAM,KAAK,UAAU;QAAE,OAAO,cAAc,CAAC,MAAM,CAAC,CAAC;IACzD,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,UAAU,CAAC,MAA4B;IAC9C,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC;IACpD,MAAM,KAAK,GAAG;QACZ,sBAAsB,MAAM,CAAC,SAAS,KAAK,MAAM,CAAC,YAAY,GAAG;QACjE,QAAQ,MAAM,CAAC,KAAK,EAAE;QACtB,YAAY,MAAM,CAAC,WAAW,EAAE;QAChC,WAAW,MAAM,CAAC,MAAM,IAAI,SAAS,EAAE;QACvC,UAAU,MAAM,CAAC,KAAK,CAAC,MAAM,MAAM,MAAM,CAAC,MAAM,WAAW;QAC3D,EAAE;QACF,wDAAwD;QACxD,wBAAwB,MAAM,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,OAAO,CAAC,KAAK,CAAC,kBAAkB,CAAC,EAAE;QACtK,wBAAwB,MAAM,CAAC,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,OAAO,CAAC,KAAK,CAAC,iBAAiB,CAAC,EAAE;QACzK,wBAAwB,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,OAAO,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,aAAa,CAAC,KAAK,CAAC,eAAe,CAAC,EAAE;QACjL,wBAAwB,MAAM,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE;QACjK,EAAE;QACF,MAAM,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC;YAC7B,CAAC,CAAC,wBAAwB;YAC1B,CAAC,CAAC,qBAAqB,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;KACrF,CAAC;IACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,cAAc,CAAC,MAA4B;IAClD,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC;IACpD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACzC,KAAK,IAAI,CAAC,MAAM,MAAM,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,IAAI,CAChR,CAAC;IACF,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC;QACjD,CAAC,CAAC,OAAO;QACT,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAErE,OAAO;QACL,gCAAgC,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;QACtF,EAAE;QACF,cAAc,MAAM,CAAC,OAAO,OAAO,MAAM,CAAC,YAAY,GAAG;QACzD,YAAY,MAAM,CAAC,KAAK,IAAI;QAC5B,gBAAgB,MAAM,CAAC,WAAW,IAAI;QACtC,eAAe,MAAM,CAAC,MAAM,IAAI,SAAS,IAAI;QAC7C,aAAa,MAAM,CAAC,MAAM,EAAE;QAC5B,gBAAgB,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,EAAE;QAC1D,EAAE;QACF,YAAY;QACZ,EAAE;QACF,iDAAiD;QACjD,8BAA8B;QAC9B,mBAAmB,MAAM,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,MAAM,OAAO,CAAC,KAAK,CAAC,kBAAkB,CAAC,cAAc;QAC/I,qBAAqB,MAAM,CAAC,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC,MAAM,OAAO,CAAC,KAAK,CAAC,iBAAiB,CAAC,cAAc;QACpJ,yBAAyB,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,mBAAmB,CAAC,MAAM,aAAa,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI;QACtJ,gBAAgB,MAAM,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC,SAAS,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,SAAS,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI;QACnI,EAAE;QACF,UAAU;QACV,EAAE;QACF,qHAAqH;QACrH,mDAAmD;QACnD,GAAG,QAAQ;QACX,EAAE;QACF,yBAAyB;QACzB,EAAE;QACF,WAAW;QACX,EAAE;QACF,iMAAiM;QACjM,EAAE;KACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,SAAS,MAAM,CAAC,KAAa,EAAE,MAAc;IAC3C,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;AAC/B,CAAC;AAED,SAAS,OAAO,CAAC,KAAa;IAC5B,OAAO,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACxC,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IAClC,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;IAChC,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,QAAQ,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;AAC/C,CAAC"}
+{"version":3,"file":"report-renderer.js","sourceRoot":"","sources":["../src/report-renderer.ts"],"names":[],"mappings":"AAIA,MAAM,UAAU,qBAAqB,CACnC,MAA4B,EAC5B,MAA6B;IAE7B,IAAI,MAAM,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAC9D,IAAI,MAAM,KAAK,UAAU;QAAE,OAAO,cAAc,CAAC,MAAM,CAAC,CAAC;IACzD,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,UAAU,CAAC,MAA4B;IAC9C,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC;IACpD,MAAM,KAAK,GAAG;QACZ,sBAAsB,MAAM,CAAC,SAAS,KAAK,MAAM,CAAC,YAAY,GAAG;QACjE,QAAQ,MAAM,CAAC,KAAK,EAAE;QACtB,YAAY,MAAM,CAAC,WAAW,EAAE;QAChC,WAAW,MAAM,CAAC,MAAM,IAAI,SAAS,EAAE;QACvC,UAAU,MAAM,CAAC,KAAK,CAAC,MAAM,MAAM,MAAM,CAAC,MAAM,WAAW;QAC3D,EAAE;QACF,wDAAwD;QACxD,wBAAwB,MAAM,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,OAAO,CAAC,KAAK,CAAC,kBAAkB,CAAC,EAAE;QACtK,wBAAwB,MAAM,CAAC,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,OAAO,CAAC,KAAK,CAAC,iBAAiB,CAAC,EAAE;QACzK,wBAAwB,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,OAAO,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,aAAa,CAAC,KAAK,CAAC,eAAe,CAAC,EAAE;QACjL,wBAAwB,MAAM,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE;QACjK,EAAE;QACF,MAAM,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC;YAC7B,CAAC,CAAC,wBAAwB;YAC1B,CAAC,CAAC,qBAAqB,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACpF,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;YACtB,eAAe,MAAM,CAAC,UAAU,CAAC,MAAM,KAAK,MAAM,CAAC,UAAU,CAAC,WAAW,YAAY,MAAM,CAAC,UAAU,CAAC,WAAW,YAAY,MAAM,CAAC,UAAU,CAAC,gBAAgB,eAAe;SAChL,CAAC,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;IACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,cAAc,CAAC,MAA4B;IAClD,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC;IACpD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACzC,KAAK,IAAI,CAAC,MAAM,MAAM,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,IAAI,CAChR,CAAC;IACF,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC;QACjD,CAAC,CAAC,OAAO;QACT,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAErE,OAAO;QACL,gCAAgC,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;QACtF,EAAE;QACF,cAAc,MAAM,CAAC,OAAO,OAAO,MAAM,CAAC,YAAY,GAAG;QACzD,YAAY,MAAM,CAAC,KAAK,IAAI;QAC5B,gBAAgB,MAAM,CAAC,WAAW,IAAI;QACtC,eAAe,MAAM,CAAC,MAAM,IAAI,SAAS,IAAI;QAC7C,aAAa,MAAM,CAAC,MAAM,EAAE;QAC5B,gBAAgB,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,EAAE;QAC1D,EAAE;QACF,YAAY;QACZ,EAAE;QACF,iDAAiD;QACjD,8BAA8B;QAC9B,mBAAmB,MAAM,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,MAAM,OAAO,CAAC,KAAK,CAAC,kBAAkB,CAAC,cAAc;QAC/I,qBAAqB,MAAM,CAAC,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC,MAAM,OAAO,CAAC,KAAK,CAAC,iBAAiB,CAAC,cAAc;QACpJ,yBAAyB,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,mBAAmB,CAAC,MAAM,aAAa,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI;QACtJ,gBAAgB,MAAM,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC,SAAS,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,SAAS,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI;QACnI,EAAE;QACF,UAAU;QACV,EAAE;QACF,qHAAqH;QACrH,mDAAmD;QACnD,GAAG,QAAQ;QACX,EAAE;QACF,yBAAyB;QACzB,EAAE;QACF,WAAW;QACX,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;YACtB,EAAE;YACF,eAAe;YACf,EAAE;YACF,aAAa,MAAM,CAAC,UAAU,CAAC,MAAM,cAAc,MAAM,CAAC,UAAU,CAAC,WAAW,YAAY,MAAM,CAAC,UAAU,CAAC,WAAW,iBAAiB,MAAM,CAAC,UAAU,CAAC,gBAAgB,GAAG;SAChL,CAAC,CAAC,CAAC,EAAE,CAAC;QACP,EAAE;QACF,iMAAiM;QACjM,EAAE;KACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,SAAS,MAAM,CAAC,KAAa,EAAE,MAAc;IAC3C,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;AAC/B,CAAC;AAED,SAAS,OAAO,CAAC,KAAa;IAC5B,OAAO,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACxC,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IAClC,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;IAChC,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,QAAQ,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;AAC/C,CAAC"}

package/dist/suite-runner.d.ts

@@ -1,10 +1,13 @@
-import type { BenchmarkSuite, BenchmarkSuiteReport, BenchmarkSuiteTask } from "@devflow-tools/sdk";
+import type { BenchmarkSuite, BenchmarkSuiteReport, BenchmarkSuiteTask, BenchmarkTaskComparison } from "@devflow-tools/sdk";
 import { type BenchmarkRetrievalResult } from "./lexical-runner.js";
+import type { GraderContext, GraderRegistry } from './graders/index.js';
 export type BenchmarkRetrievalRunner = (task: BenchmarkSuiteTask) => Promise<BenchmarkRetrievalResult>;
 export interface SuiteRunOptions {
     rounds?: number;
     baselineRunner?: BenchmarkRetrievalRunner;
     devflowRunner?: BenchmarkRetrievalRunner;
+    graderRegistry?: GraderRegistry;
+    graderContextFactory?: (task: BenchmarkSuiteTask, comparison: BenchmarkTaskComparison) => GraderContext;
 }
 export declare function runBenchmarkSuite(projectRoot: string, suite: BenchmarkSuite, options?: SuiteRunOptions): Promise<BenchmarkSuiteReport>;
 //# sourceMappingURL=suite-runner.d.ts.map

package/dist/suite-runner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"suite-runner.d.ts","sourceRoot":"","sources":["../src/suite-runner.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAKV,cAAc,EACd,oBAAoB,EACpB,kBAAkB,EAEnB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAEL,KAAK,wBAAwB,EAC9B,MAAM,qBAAqB,CAAC;AAE7B,MAAM,MAAM,wBAAwB,GAAG,CACrC,IAAI,EAAE,kBAAkB,KACrB,OAAO,CAAC,wBAAwB,CAAC,CAAC;AAEvC,MAAM,WAAW,eAAe;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,wBAAwB,CAAC;IAC1C,aAAa,CAAC,EAAE,wBAAwB,CAAC;CAC1C;AAOD,wBAAsB,iBAAiB,CACrC,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,cAAc,EACrB,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,oBAAoB,CAAC,CA0C/B"}
+{"version":3,"file":"suite-runner.d.ts","sourceRoot":"","sources":["../src/suite-runner.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAKV,cAAc,EACd,oBAAoB,EACpB,kBAAkB,EAClB,uBAAuB,EACxB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAEL,KAAK,wBAAwB,EAC9B,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAExE,MAAM,MAAM,wBAAwB,GAAG,CACrC,IAAI,EAAE,kBAAkB,KACrB,OAAO,CAAC,wBAAwB,CAAC,CAAC;AAEvC,MAAM,WAAW,eAAe;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,wBAAwB,CAAC;IAC1C,aAAa,CAAC,EAAE,wBAAwB,CAAC;IACzC,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,oBAAoB,CAAC,EAAE,CACrB,IAAI,EAAE,kBAAkB,EACxB,UAAU,EAAE,uBAAuB,KAChC,aAAa,CAAC;CACpB;AAOD,wBAAsB,iBAAiB,CACrC,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,cAAc,EACrB,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,oBAAoB,CAAC,CA6D/B"}

package/dist/suite-runner.js

@@ -12,13 +12,17 @@ export async function runBenchmarkSuite(projectRoot, suite, options = {}) {
         for (const task of suite.tasks) {
             const baseline = await runLeg("baseline", task, baselineRunner, rounds);
             const devflow = await runLeg("devflow", task, devflowRunner, rounds);
-            tasks.push({
+            const comparison = {
                 taskId: task.id,
                 prompt: task.prompt,
                 baseline,
                 devflow,
                 delta: calculateDelta(baseline, devflow),
-            });
+            };
+            if (options.graderRegistry && options.graderContextFactory) {
+                comparison.evaluation = await options.graderRegistry.run(options.graderContextFactory(task, comparison));
+            }
+            tasks.push(comparison);
         }
     }
     finally {
@@ -27,6 +31,7 @@ export async function runBenchmarkSuite(projectRoot, suite, options = {}) {
     const baseline = aggregate(tasks.map((task) => task.baseline));
     const devflow = aggregate(tasks.map((task) => task.devflow));
     const delta = calculateAggregateDelta(baseline, devflow);
+    const taskEvaluations = tasks.flatMap(task => task.evaluation ? [task.evaluation] : []);
     return {
         runId: `benchmark:${Date.now()}:${randomUUID().slice(0, 8)}`,
         suiteId: suite.id,
@@ -40,6 +45,18 @@ export async function runBenchmarkSuite(projectRoot, suite, options = {}) {
         tasks,
         summary: { baseline, devflow, delta },
         regressions: detectRegressions(baseline, devflow),
+        ...(taskEvaluations.length > 0 ? {
+            evaluation: {
+                status: taskEvaluations.some(item => item.status === 'failed')
+                    ? 'failed'
+                    : taskEvaluations.some(item => item.status === 'unavailable')
+                        ? 'unavailable'
+                        : 'passed',
+                passedTasks: taskEvaluations.filter(item => item.status === 'passed').length,
+                failedTasks: taskEvaluations.filter(item => item.status === 'failed').length,
+                unavailableTasks: taskEvaluations.filter(item => item.status === 'unavailable').length,
+            },
+        } : {}),
     };
 }
 async function createDevFlowRunner(projectRoot) {

package/dist/suite-runner.js.map

@@ -1 +1 @@
-{"version":3,"file":"suite-runner.js","sourceRoot":"","sources":["../src/suite-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAWzC,OAAO,EACL,sBAAsB,GAEvB,MAAM,qBAAqB,CAAC;AAiB7B,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,WAAmB,EACnB,KAAqB,EACrB,UAA2B,EAAE;IAE7B,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,sBAAsB,CAAC,WAAW,CAAC,CAAC;IACxF,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IAChF,MAAM,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,mBAAmB,CAAC,WAAW,CAAC,CAAC;IAC7F,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,cAAe,CAAC,GAAG,CAAC;IACnE,MAAM,KAAK,GAA8B,EAAE,CAAC;IAE5C,IAAI,CAAC;QACH,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YAC/B,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,UAAU,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,CAAC,CAAC;YACxE,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC;YACrE,KAAK,CAAC,IAAI,CAAC;gBACT,MAAM,EAAE,IAAI,CAAC,EAAE;gBACf,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,QAAQ;gBACR,OAAO;gBACP,KAAK,EAAE,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC;aACzC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;YAAS,CAAC;QACT,cAAc,EAAE,KAAK,EAAE,CAAC;IAC1B,CAAC;IAED,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC/D,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IAC7D,MAAM,KAAK,GAAG,uBAAuB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAEzD,OAAO;QACL,KAAK,EAAE,aAAa,IAAI,CAAC,GAAG,EAAE,IAAI,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;QAC5D,OAAO,EAAE,KAAK,CAAC,EAAE;QACjB,SAAS,EAAE,KAAK,CAAC,IAAI;QACrB,YAAY,EAAE,KAAK,CAAC,OAAO;QAC3B,WAAW;QACX,MAAM,EAAE,UAAU,CAAC,WAAW,CAAC;QAC/B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;QACrB,MAAM;QACN,MAAM,EAAE,WAAW;QACnB,KAAK;QACL,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE;QACrC,WAAW,EAAE,iBAAiB,CAAC,QAAQ,EAAE,OAAO,CAAC;KAClD,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mBAAmB,CAChC,WAAmB;IAEnB,MAAM,WAAW,GAAG,+BAA+B,CAAC;IACpD,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,WAAW,CAAmD,CAAC;IACtG,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,WAAW,EAAE,EAAE,iBAAiB,EAAE,KAAK,EAAE,CAAC,CAAC;IAC/E,OAAO;QACL,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YAClB,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YACpC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,CAAC,MAAM,EAAE;gBACjE,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,IAAI;gBACf,SAAS,EAAE,aAAa,IAAI,CAAC,EAAE,EAAE;aAClC,CAAC,CAAC;YACH,OAAO;gBACL,KAAK,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC;wBACjB,GAAG,MAAM,CAAC,aAAa,CAAC,KAAK;wBAC7B,GAAG,MAAM,CAAC,aAAa,CAAC,KAAK;qBAC9B,CAAC,CAAC;gBACH,OAAO,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,MAAwB,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;gBACvF,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,UAAU,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS;aAC1C,CAAC;QACJ,CAAC;QACD,KAAK,EAAE,GAAG,EAAE;YACV,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACxB,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YACvB,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;YAC1B,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;YAC1B,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QAC5B,CAAC;KACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,MAAM,CACnB,IAAmB,EACnB,IAAwB,EACxB,MAAgC,EAChC,MAAc;IAEd,MAAM,OAAO,GAA+B,EAAE,CAAC;IAC/C,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC/C,OAAO,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IACnC,CAAC;IAED,MAAM,YAAY,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC7E,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC1E,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;IAEzD,OAAO;QACL,IAAI;QACJ,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC/D,YAAY;QACZ,gBAAgB,EAAE,KAAK,CAAC,KAAK;QAC7B,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC/D,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,cAAc,EAAE,KAAK,CAAC,cAAc;QACpC,MAAM;KACP,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CACpB,IAAwB,EACxB,KAAe,EACf,OAAiB;IAEjB,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QAC3D,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QAClF,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9B,CAAC,CAAC,CAAC;IACH,MAAM,cAAc,GAAG,CAAC,IAAI,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QACvE,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;QAClF,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9B,CAAC,CAAC,CAAC;IACH,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;IACtF,MAAM,YAAY,GAAG,YAAY,CAAC,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC;IACjE,OAAO;QACL,KAAK,EAAE,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;QAC3D,YAAY;QACZ,cAAc;KACf,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,IAA2B;IAC5C,OAAO;QACL,aAAa,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACzD,eAAe,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACpE,mBAAmB,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QACrE,aAAa,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;KAC1D,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,QAA6B,EAAE,OAA4B;IACjF,OAAO;QACL,kBAAkB,EAAE,SAAS,CAAC,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC;QACtE,iBAAiB,EAAE,SAAS,CAAC,QAAQ,CAAC,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC;QACvF,eAAe,EAAE,OAAO,CAAC,gBAAgB,GAAG,QAAQ,CAAC,gBAAgB;QACrE,aAAa,EAAE,SAAS,CAAC,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC;KAClE,CAAC;AACJ,CAAC;AAED,SAAS,uBAAuB,CAC9B,QAAmC,EACnC,OAAkC;IAElC,OAAO;QACL,kBAAkB,EAAE,SAAS,CAAC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC;QAC5E,iBAAiB,EAAE,SAAS,CAAC,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC,eAAe,CAAC;QAC/E,eAAe,EAAE,OAAO,CAAC,mBAAmB,GAAG,QAAQ,CAAC,mBAAmB;QAC3E,aAAa,EAAE,SAAS,CAAC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC;KACxE,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CACxB,QAAmC,EACnC,OAAkC;IAElC,MAAM,SAAS,GAAG,GAAG,CAAC;IACtB,MAAM,UAAU,GAAsC;QACpD,UAAU,CAAC,YAAY,EAAE,QAAQ,CAAC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,EAAE,SAAS,CAAC;QAC5F,UAAU,CAAC,cAAc,EAAE,QAAQ,CAAC,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC,eAAe,CAAC,EAAE,SAAS,CAAC;QAClG,UAAU,CAAC,YAAY,EAAE,QAAQ,CAAC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,EAAE,SAAS,CAAC;QAC5F,UAAU,CACR,kBAAkB,EAClB,SAAS,CAAC,QAAQ,CAAC,mBAAmB,EAAE,OAAO,CAAC,mBAAmB,CAAC,EACpE,SAAS,CACV;KACF,CAAC;IACF,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAA+B,EAAE,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;AACjF,CAAC;AAED,SAAS,UAAU,CACjB,MAAqC,EACrC,UAAkB,EAClB,SAAiB;IAEjB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,UAAU,IAAI,SAAS;QAAE,OAAO,IAAI,CAAC;IACzE,OAAO;QACL,MAAM;QACN,UAAU;QACV,SAAS;QACT,OAAO,EAAE,GAAG,MAAM,iBAAiB,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;KACjH,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,KAAK,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;AACnD,CAAC;AAED,SAAS,OAAO,CAAC,MAAgB;IAC/B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AACvE,CAAC;AAED,SAAS,SAAS,CAAC,QAAgB,EAAE,SAAiB;IACpD,OAAO,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,SAAS,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;AAC9D,CAAC;AAED,SAAS,QAAQ,CAAC,QAAgB,EAAE,SAAiB;IACnD,OAAO,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,QAAQ,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAClF,CAAC;AAED,SAAS,eAAe,CAAC,MAA0B;IACjD,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,EAAE,EAAE,CAAC;QAC3D,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IAC1E,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,UAAU,CAAC,WAAmB;IACrC,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,KAAK,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,EAAE;YAChD,GAAG,EAAE,WAAW;YAChB,QAAQ,EAAE,MAAM;SACjB,CAAC,CAAC,IAAI,EAAE,CAAC;IACZ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
+{"version":3,"file":"suite-runner.js","sourceRoot":"","sources":["../src/suite-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAWzC,OAAO,EACL,sBAAsB,GAEvB,MAAM,qBAAqB,CAAC;AAuB7B,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,WAAmB,EACnB,KAAqB,EACrB,UAA2B,EAAE;IAE7B,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,sBAAsB,CAAC,WAAW,CAAC,CAAC;IACxF,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IAChF,MAAM,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,mBAAmB,CAAC,WAAW,CAAC,CAAC;IAC7F,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,cAAe,CAAC,GAAG,CAAC;IACnE,MAAM,KAAK,GAA8B,EAAE,CAAC;IAE5C,IAAI,CAAC;QACH,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YAC/B,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,UAAU,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,CAAC,CAAC;YACxE,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC;YACrE,MAAM,UAAU,GAA4B;gBAC1C,MAAM,EAAE,IAAI,CAAC,EAAE;gBACf,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,QAAQ;gBACR,OAAO;gBACP,KAAK,EAAE,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC;aACzC,CAAC;YACF,IAAI,OAAO,CAAC,cAAc,IAAI,OAAO,CAAC,oBAAoB,EAAE,CAAC;gBAC3D,UAAU,CAAC,UAAU,GAAG,MAAM,OAAO,CAAC,cAAc,CAAC,GAAG,CACtD,OAAO,CAAC,oBAAoB,CAAC,IAAI,EAAE,UAAU,CAAC,CAC/C,CAAC;YACJ,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;YAAS,CAAC;QACT,cAAc,EAAE,KAAK,EAAE,CAAC;IAC1B,CAAC;IAED,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC/D,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IAC7D,MAAM,KAAK,GAAG,uBAAuB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAEzD,MAAM,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACxF,OAAO;QACL,KAAK,EAAE,aAAa,IAAI,CAAC,GAAG,EAAE,IAAI,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;QAC5D,OAAO,EAAE,KAAK,CAAC,EAAE;QACjB,SAAS,EAAE,KAAK,CAAC,IAAI;QACrB,YAAY,EAAE,KAAK,CAAC,OAAO;QAC3B,WAAW;QACX,MAAM,EAAE,UAAU,CAAC,WAAW,CAAC;QAC/B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;QACrB,MAAM;QACN,MAAM,EAAE,WAAW;QACnB,KAAK;QACL,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE;QACrC,WAAW,EAAE,iBAAiB,CAAC,QAAQ,EAAE,OAAO,CAAC;QACjD,GAAG,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;YAC/B,UAAU,EAAE;gBACV,MAAM,EAAE,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC;oBAC5D,CAAC,CAAC,QAAiB;oBACnB,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,aAAa,CAAC;wBAC3D,CAAC,CAAC,aAAsB;wBACxB,CAAC,CAAC,QAAiB;gBACvB,WAAW,EAAE,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM;gBAC5E,WAAW,EAAE,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM;gBAC5E,gBAAgB,EAAE,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,aAAa,CAAC,CAAC,MAAM;aACvF;SACF,CAAC,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mBAAmB,CAChC,WAAmB;IAEnB,MAAM,WAAW,GAAG,+BAA+B,CAAC;IACpD,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,WAAW,CAAmD,CAAC;IACtG,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,WAAW,EAAE,EAAE,iBAAiB,EAAE,KAAK,EAAE,CAAC,CAAC;IAC/E,OAAO;QACL,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YAClB,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YACpC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,CAAC,MAAM,EAAE;gBACjE,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,IAAI;gBACf,SAAS,EAAE,aAAa,IAAI,CAAC,EAAE,EAAE;aAClC,CAAC,CAAC;YACH,OAAO;gBACL,KAAK,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC;wBACjB,GAAG,MAAM,CAAC,aAAa,CAAC,KAAK;wBAC7B,GAAG,MAAM,CAAC,aAAa,CAAC,KAAK;qBAC9B,CAAC,CAAC;gBACH,OAAO,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,MAAwB,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;gBACvF,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,UAAU,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS;aAC1C,CAAC;QACJ,CAAC;QACD,KAAK,EAAE,GAAG,EAAE;YACV,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACxB,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YACvB,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;YAC1B,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;YAC1B,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QAC5B,CAAC;KACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,MAAM,CACnB,IAAmB,EACnB,IAAwB,EACxB,MAAgC,EAChC,MAAc;IAEd,MAAM,OAAO,GAA+B,EAAE,CAAC;IAC/C,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC/C,OAAO,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IACnC,CAAC;IAED,MAAM,YAAY,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC7E,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC1E,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;IAEzD,OAAO;QACL,IAAI;QACJ,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC/D,YAAY;QACZ,gBAAgB,EAAE,KAAK,CAAC,KAAK;QAC7B,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC/D,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,cAAc,EAAE,KAAK,CAAC,cAAc;QACpC,MAAM;KACP,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CACpB,IAAwB,EACxB,KAAe,EACf,OAAiB;IAEjB,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QAC3D,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QAClF,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9B,CAAC,CAAC,CAAC;IACH,MAAM,cAAc,GAAG,CAAC,IAAI,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QACvE,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;QAClF,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9B,CAAC,CAAC,CAAC;IACH,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;IACtF,MAAM,YAAY,GAAG,YAAY,CAAC,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC;IACjE,OAAO;QACL,KAAK,EAAE,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;QAC3D,YAAY;QACZ,cAAc;KACf,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,IAA2B;IAC5C,OAAO;QACL,aAAa,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACzD,eAAe,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACpE,mBAAmB,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QACrE,aAAa,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;KAC1D,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,QAA6B,EAAE,OAA4B;IACjF,OAAO;QACL,kBAAkB,EAAE,SAAS,CAAC,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC;QACtE,iBAAiB,EAAE,SAAS,CAAC,QAAQ,CAAC,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC;QACvF,eAAe,EAAE,OAAO,CAAC,gBAAgB,GAAG,QAAQ,CAAC,gBAAgB;QACrE,aAAa,EAAE,SAAS,CAAC,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC;KAClE,CAAC;AACJ,CAAC;AAED,SAAS,uBAAuB,CAC9B,QAAmC,EACnC,OAAkC;IAElC,OAAO;QACL,kBAAkB,EAAE,SAAS,CAAC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC;QAC5E,iBAAiB,EAAE,SAAS,CAAC,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC,eAAe,CAAC;QAC/E,eAAe,EAAE,OAAO,CAAC,mBAAmB,GAAG,QAAQ,CAAC,mBAAmB;QAC3E,aAAa,EAAE,SAAS,CAAC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC;KACxE,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CACxB,QAAmC,EACnC,OAAkC;IAElC,MAAM,SAAS,GAAG,GAAG,CAAC;IACtB,MAAM,UAAU,GAAsC;QACpD,UAAU,CAAC,YAAY,EAAE,QAAQ,CAAC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,EAAE,SAAS,CAAC;QAC5F,UAAU,CAAC,cAAc,EAAE,QAAQ,CAAC,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC,eAAe,CAAC,EAAE,SAAS,CAAC;QAClG,UAAU,CAAC,YAAY,EAAE,QAAQ,CAAC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,EAAE,SAAS,CAAC;QAC5F,UAAU,CACR,kBAAkB,EAClB,SAAS,CAAC,QAAQ,CAAC,mBAAmB,EAAE,OAAO,CAAC,mBAAmB,CAAC,EACpE,SAAS,CACV;KACF,CAAC;IACF,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAA+B,EAAE,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;AACjF,CAAC;AAED,SAAS,UAAU,CACjB,MAAqC,EACrC,UAAkB,EAClB,SAAiB;IAEjB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,UAAU,IAAI,SAAS;QAAE,OAAO,IAAI,CAAC;IACzE,OAAO;QACL,MAAM;QACN,UAAU;QACV,SAAS;QACT,OAAO,EAAE,GAAG,MAAM,iBAAiB,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;KACjH,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,KAAK,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;AACnD,CAAC;AAED,SAAS,OAAO,CAAC,MAAgB;IAC/B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AACvE,CAAC;AAED,SAAS,SAAS,CAAC,QAAgB,EAAE,SAAiB;IACpD,OAAO,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,SAAS,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;AAC9D,CAAC;AAED,SAAS,QAAQ,CAAC,QAAgB,EAAE,SAAiB;IACnD,OAAO,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,QAAQ,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAClF,CAAC;AAED,SAAS,eAAe,CAAC,MAA0B;IACjD,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,EAAE,EAAE,CAAC;QAC3D,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IAC1E,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,UAAU,CAAC,WAAmB;IACrC,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,KAAK,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,EAAE;YAChD,GAAG,EAAE,WAAW;YAChB,QAAQ,EAAE,MAAM;SACjB,CAAC,CAAC,IAAI,EAAE,CAAC;IACZ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devflow-tools/benchmark",
-  "version": "0.16.21",
+  "version": "0.16.22",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -19,9 +19,9 @@
   },
   "dependencies": {
     "@colbymchenry/codegraph": "^1.1.1",
-    "@devflow-tools/context-engine": "0.16.21",
-    "@devflow-tools/database": "0.16.21",
-    "@devflow-tools/sdk": "0.16.21"
+    "@devflow-tools/context-engine": "0.16.22",
+    "@devflow-tools/database": "0.16.22",
+    "@devflow-tools/sdk": "0.16.22"
   },
   "devDependencies": {
     "typescript": "^5.5.0",