@wix/pathgrade 1.0.13 → 1.0.15

package/dist/providers/scripted-mcp-mock-host.d.ts

@@ -0,0 +1,39 @@
+import type { ToolEvent } from '../tool-events.js';
+import { type CompiledMcpMockSession } from '../sdk/mcp-mock-approvals.js';
+export interface ScriptedMcpDecision {
+    decision: 'approve' | 'deny';
+    outcome: 'matched' | 'unmatched' | 'protocol_error';
+    reason: 'rule_match' | 'no_matching_rule' | 'protocol_error';
+    ruleIndex?: number;
+    ruleLabel?: string;
+}
+export interface ScriptedMcpMockHost {
+    readonly claudeServers: Record<string, {
+        type: 'http';
+        url: string;
+        headers: Record<string, string>;
+    }>;
+    readonly codexConfig: {
+        mcp_servers: Record<string, {
+            url: string;
+            http_headers: Record<string, string>;
+        }>;
+    };
+    beginTurn(turnNumber: number): void;
+    decide(serverName: string, toolName: string, args: Record<string, unknown>): ScriptedMcpDecision;
+    isReadOnly(serverName: string, toolName: string): boolean;
+    failProtocol(message: string): void;
+    settleEvents(events: ToolEvent[]): {
+        events: ToolEvent[];
+        error?: Error;
+    };
+    dispose(): Promise<void>;
+}
+type Observer = (identity: {
+    serverName: string;
+    toolName: string;
+}) => void;
+/** Non-public acceptance seam. It observes only authenticated manifest-member calls. */
+export declare function __setScriptedMcpMockObserverForTesting(observer: Observer | null): void;
+export declare function startScriptedMcpMockHost(plan: CompiledMcpMockSession): Promise<ScriptedMcpMockHost>;
+export {};

package/dist/providers/scripted-mcp-mock-host.js

@@ -0,0 +1,368 @@
+import { createServer } from 'node:http';
+import { randomBytes } from 'node:crypto';
+import { Worker } from 'node:worker_threads';
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
+import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js';
+import { compileScriptedMcpSchemaForProvider } from '../core/mcp-schema-profile.js';
+import { MCP_ANNOTATION_PROTOCOL_FLOOR } from '../core/generated-mcp-protocol.js';
+import { getOriginalMcpInput } from '../sdk/mcp-event-input.js';
+import { createMcpArgumentDigest, createMcpReceiptKey, matchesMcpApprovalArguments, } from '../sdk/mcp-mock-approvals.js';
+let testObserver = null;
+/** Non-public acceptance seam. It observes only authenticated manifest-member calls. */
+export function __setScriptedMcpMockObserverForTesting(observer) {
+    testObserver = observer;
+}
+const MAX_BODY_BYTES = 1024 * 1024;
+const MATCH_TIMEOUT_MS = 250;
+function isRecord(value) {
+    if (!value || typeof value !== 'object' || Array.isArray(value))
+        return false;
+    const prototype = Object.getPrototypeOf(value);
+    return prototype === Object.prototype || prototype === null;
+}
+function asJsonObject(value) {
+    return isRecord(value) ? value : undefined;
+}
+function formatResponse(value) {
+    return typeof value === 'string' ? value : JSON.stringify(value);
+}
+class MatcherWorker {
+    worker;
+    nextId = 1;
+    pending = new Map();
+    constructor() {
+        this.worker = new Worker(`
+            const { parentPort } = require('node:worker_threads');
+            parentPort.on('message', ({ id, cases, input }) => {
+                try {
+                    let fallback;
+                    for (const item of cases) {
+                        if (item.when) {
+                            if (new RegExp(item.when, 'i').test(input)) return parentPort.postMessage({ id, response: item.response });
+                        } else if (fallback === undefined) fallback = item.response;
+                    }
+                    parentPort.postMessage({ id, response: fallback, missing: fallback === undefined });
+                } catch (error) {
+                    parentPort.postMessage({ id, error: error instanceof Error ? error.message : String(error) });
+                }
+            });
+        `, { eval: true });
+        this.worker.on('message', (message) => {
+            const pending = this.pending.get(message.id);
+            if (!pending)
+                return;
+            this.pending.delete(message.id);
+            clearTimeout(pending.timer);
+            if (message.error)
+                pending.reject(new Error(`scripted MCP matcher failed: ${message.error}`));
+            else
+                pending.resolve(message.missing ? undefined : message.response);
+        });
+        this.worker.on('error', (error) => this.failAll(error instanceof Error ? error : new Error(String(error))));
+        this.worker.on('exit', (code) => {
+            if (code !== 0)
+                this.failAll(new Error(`scripted MCP matcher exited with code ${code}`));
+        });
+    }
+    match(cases, input) {
+        const id = this.nextId++;
+        return new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                this.pending.delete(id);
+                reject(new Error('scripted MCP matcher deadline exceeded'));
+                void this.worker.terminate();
+            }, MATCH_TIMEOUT_MS);
+            this.pending.set(id, { resolve, reject, timer });
+            this.worker.postMessage({ id, cases, input });
+        });
+    }
+    async dispose() {
+        this.failAll(new Error('scripted MCP matcher disposed'));
+        await this.worker.terminate();
+    }
+    failAll(error) {
+        for (const pending of this.pending.values()) {
+            clearTimeout(pending.timer);
+            pending.reject(error);
+        }
+        this.pending.clear();
+    }
+}
+export async function startScriptedMcpMockHost(plan) {
+    const token = randomBytes(32).toString('base64url');
+    const receiptKey = createMcpReceiptKey();
+    const matcher = new MatcherWorker();
+    const routeToServer = new Map();
+    for (const serverName of plan.servers.keys())
+        routeToServer.set(`/${randomBytes(18).toString('base64url')}`, serverName);
+    const activeTransports = new Set();
+    let sequence = 0;
+    let currentTurn = 0;
+    let fatal;
+    let authorizations = [];
+    let receipts = [];
+    let closing = false;
+    const latch = (error) => { fatal ??= error; };
+    const httpServer = createServer(async (req, res) => {
+        try {
+            if (closing)
+                return sendHttp(res, 503, 'closing');
+            if (req.headers.host !== '127.0.0.1' && !req.headers.host?.startsWith('127.0.0.1:')) {
+                return sendHttp(res, 400, 'invalid host');
+            }
+            const authorization = req.headers.authorization;
+            if (Array.isArray(authorization) || authorization !== `Bearer ${token}`) {
+                return sendHttp(res, 401, 'unauthorized');
+            }
+            if (fatal)
+                return sendHttp(res, 500, 'scripted MCP session failed');
+            if (req.method !== 'POST')
+                return sendHttp(res, 405, 'method not allowed');
+            const serverName = routeToServer.get(req.url ?? '');
+            const compiled = serverName ? plan.servers.get(serverName) : undefined;
+            if (!compiled)
+                return sendHttp(res, 404, 'not found');
+            const body = await readJsonBody(req);
+            if (body === undefined)
+                return sendHttp(res, 400, 'invalid JSON');
+            const requestedVersion = isRecord(body) && body.method === 'initialize'
+                ? isRecord(body.params) ? body.params.protocolVersion : undefined
+                : undefined;
+            if (typeof requestedVersion === 'string' && requestedVersion < MCP_ANNOTATION_PROTOCOL_FLOOR) {
+                return sendJsonRpcError(res, isRecord(body) ? body.id : null, -32602, 'MCP protocol predates Tool Annotations');
+            }
+            const server = new Server({ name: compiled.name, version: '1.0.0' }, { capabilities: { tools: {} }, ...(compiled.instructions ? { instructions: compiled.instructions } : {}) });
+            server.setRequestHandler(ListToolsRequestSchema, async () => ({
+                tools: [...compiled.tools.values()].map((tool) => ({
+                    name: tool.name,
+                    description: tool.description ?? `Mock tool: ${tool.name}`,
+                    inputSchema: compileScriptedMcpSchemaForProvider(tool.inputSchema),
+                    ...(tool.readOnlyHint !== undefined ? { annotations: { readOnlyHint: tool.readOnlyHint } } : {}),
+                })),
+            }));
+            server.setRequestHandler(CallToolRequestSchema, async (request) => {
+                const tool = compiled.tools.get(request.params.name);
+                if (!tool)
+                    throw new Error(`Unknown tool: ${request.params.name}`);
+                const args = asJsonObject(request.params.arguments ?? {});
+                if (!args)
+                    throw new Error('Tool arguments must be a JSON object');
+                if (currentTurn === 0) {
+                    latch(new Error(`MCP call arrived outside an active turn for ${compiled.name}.${tool.name}`));
+                    throw new Error('MCP call arrived outside an active turn');
+                }
+                const digest = createMcpArgumentDigest(receiptKey, args);
+                const receipt = { serverName: compiled.name, toolName: tool.name, digest, sequence: ++sequence };
+                if (tool.readOnlyHint !== true) {
+                    const authorization = authorizations.find((entry) => !entry.consumed
+                        && entry.serverName === compiled.name && entry.toolName === tool.name && entry.digest === digest);
+                    if (!authorization || authorization.sequence >= receipt.sequence) {
+                        latch(new Error(`MCP receipt arrived without earlier authorization for ${compiled.name}.${tool.name}`));
+                        throw new Error('MCP call was not authorized');
+                    }
+                    authorization.consumed = true;
+                }
+                receipts.push(receipt);
+                testObserver?.({ serverName: compiled.name, toolName: tool.name });
+                const response = await matcher.match(tool.cases, JSON.stringify(args));
+                if (response === undefined)
+                    throw new Error(`Unknown tool: ${tool.name}`);
+                return { content: [{ type: 'text', text: formatResponse(response) }] };
+            });
+            const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined });
+            activeTransports.add(transport);
+            transport.onclose = () => activeTransports.delete(transport);
+            await server.connect(transport);
+            await transport.handleRequest(req, res, body);
+        }
+        catch (error) {
+            latch(error instanceof Error ? error : new Error(String(error)));
+            if (!res.headersSent)
+                sendHttp(res, 500, 'scripted MCP host failure');
+            else
+                res.end();
+        }
+    });
+    await new Promise((resolve, reject) => {
+        httpServer.once('error', reject);
+        httpServer.listen(0, '127.0.0.1', () => resolve());
+    });
+    const address = httpServer.address();
+    if (!address || typeof address === 'string' || address.address !== '127.0.0.1') {
+        await closeHttpServer(httpServer);
+        throw new Error('scripted MCP host did not bind IPv4 loopback');
+    }
+    const headers = { Authorization: `Bearer ${token}` };
+    const claudeServers = {};
+    const codexServers = {};
+    for (const [route, serverName] of routeToServer) {
+        const url = `http://127.0.0.1:${address.port}${route}`;
+        claudeServers[serverName] = { type: 'http', url, headers };
+        codexServers[serverName] = { url, http_headers: headers };
+    }
+    return {
+        claudeServers,
+        codexConfig: { mcp_servers: codexServers },
+        beginTurn(turnNumber) {
+            if (closing)
+                throw new Error('scripted MCP host is closing');
+            if (fatal)
+                throw fatal;
+            currentTurn = turnNumber;
+            authorizations = [];
+            receipts = [];
+        },
+        decide(serverName, toolName, rawArgs) {
+            if (fatal)
+                return { decision: 'deny', outcome: 'protocol_error', reason: 'protocol_error' };
+            const args = asJsonObject(rawArgs);
+            const tool = plan.servers.get(serverName)?.tools.get(toolName);
+            if (!args || !tool || tool.readOnlyHint === true || currentTurn === 0) {
+                latch(new Error('invalid scripted MCP approval request'));
+                return { decision: 'deny', outcome: 'protocol_error', reason: 'protocol_error' };
+            }
+            const ruleIndex = plan.rules.findIndex((rule) => rule.serverName === serverName
+                && rule.toolName === toolName && matchesMcpApprovalArguments(args, rule.argumentsContaining));
+            if (ruleIndex < 0) {
+                latch(new Error(`no scripted MCP approval rule matched ${serverName}.${toolName}`));
+                return { decision: 'deny', outcome: 'unmatched', reason: 'no_matching_rule' };
+            }
+            const rule = plan.rules[ruleIndex];
+            if (rule.decision === 'approve') {
+                authorizations.push({
+                    serverName,
+                    toolName,
+                    digest: createMcpArgumentDigest(receiptKey, args),
+                    sequence: ++sequence,
+                });
+            }
+            return {
+                decision: rule.decision,
+                outcome: 'matched',
+                reason: 'rule_match',
+                ruleIndex,
+                ...(rule.label ? { ruleLabel: rule.label } : {}),
+            };
+        },
+        isReadOnly(serverName, toolName) {
+            return plan.servers.get(serverName)?.tools.get(toolName)?.readOnlyHint === true;
+        },
+        failProtocol(message) {
+            latch(new Error(message.slice(0, 500)));
+        },
+        settleEvents(events) {
+            const projected = projectReceipts(events, receiptKey, receipts, latch);
+            for (const authorization of authorizations) {
+                if (!authorization.consumed)
+                    latch(new Error(`approved MCP call was not invoked: ${authorization.serverName}.${authorization.toolName}`));
+            }
+            if (receipts.some((receipt) => !receipt.consumed))
+                latch(new Error('surplus authenticated MCP receipt'));
+            const error = fatal;
+            currentTurn = 0;
+            authorizations = [];
+            receipts = [];
+            return { events: projected, ...(error ? { error } : {}) };
+        },
+        async dispose() {
+            if (closing)
+                return;
+            closing = true;
+            await Promise.all([...activeTransports].map((transport) => transport.close().catch(() => undefined)));
+            await matcher.dispose().catch(() => undefined);
+            await closeHttpServer(httpServer);
+            receiptKey.fill(0);
+        },
+    };
+}
+function projectReceipts(events, key, receipts, latch) {
+    const cohorts = new Map();
+    for (const [index, event] of events.entries()) {
+        if (event.action !== 'mcp_tool_call')
+            continue;
+        const identity = eventIdentity(event);
+        const json = eventInput(event);
+        if (!identity || !json)
+            continue;
+        const digest = createMcpArgumentDigest(key, json);
+        const cohortKey = `${identity.serverName}\0${identity.toolName}\0${digest}`;
+        const cohort = cohorts.get(cohortKey) ?? { eventIndexes: [], receipts: [], identity };
+        cohort.eventIndexes.push(index);
+        cohorts.set(cohortKey, cohort);
+    }
+    for (const receipt of receipts) {
+        const cohort = cohorts.get(`${receipt.serverName}\0${receipt.toolName}\0${receipt.digest}`);
+        if (cohort)
+            cohort.receipts.push(receipt);
+    }
+    const projected = [...events];
+    for (const cohort of cohorts.values()) {
+        const canonicalEvents = cohort.eventIndexes.filter((index) => events[index].mcp?.invocation !== 'not_invoked');
+        const deniedEvents = cohort.eventIndexes.filter((index) => events[index].mcp?.invocation === 'not_invoked');
+        if (deniedEvents.length > 0 && cohort.receipts.length > 0) {
+            latch(new Error(`denied MCP call produced a receipt for ${cohort.identity.serverName}.${cohort.identity.toolName}`));
+        }
+        if (canonicalEvents.length === 0)
+            continue;
+        if (canonicalEvents.length !== cohort.receipts.length) {
+            latch(new Error(`MCP lifecycle/receipt count mismatch for ${cohort.identity.serverName}.${cohort.identity.toolName}`));
+            for (const index of canonicalEvents)
+                projected[index] = withInvocation(events[index], cohort.identity, 'unknown');
+            continue;
+        }
+        for (const [offset, index] of canonicalEvents.entries()) {
+            cohort.receipts[offset].consumed = true;
+            projected[index] = withInvocation(events[index], cohort.identity, 'confirmed');
+        }
+    }
+    return projected;
+}
+function eventIdentity(event) {
+    const args = event.arguments ?? {};
+    return typeof args.server === 'string' && typeof args.tool === 'string'
+        ? { serverName: args.server, toolName: args.tool } : undefined;
+}
+function eventInput(event) {
+    const original = getOriginalMcpInput(event);
+    if (original)
+        return asJsonObject(original);
+    const args = event.arguments ?? {};
+    return asJsonObject(Object.fromEntries(Object.entries(args).filter(([name]) => !['server', 'tool', 'status'].includes(name))));
+}
+function withInvocation(event, identity, invocation) {
+    const status = String(event.arguments?.status ?? 'unknown').toLowerCase();
+    return invocation === 'confirmed'
+        ? { ...event, mcp: { ...identity, invocation, outcome: status.includes('fail') || status.includes('error') ? 'tool_error' : 'completed' } }
+        : { ...event, mcp: { ...identity, invocation, outcome: 'unknown' } };
+}
+async function readJsonBody(req) {
+    const chunks = [];
+    let size = 0;
+    for await (const chunk of req) {
+        const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+        size += buffer.length;
+        if (size > MAX_BODY_BYTES)
+            throw new Error('scripted MCP request body too large');
+        chunks.push(buffer);
+    }
+    try {
+        return JSON.parse(Buffer.concat(chunks).toString('utf8'));
+    }
+    catch {
+        return undefined;
+    }
+}
+function sendHttp(res, status, message) {
+    res.statusCode = status;
+    res.setHeader('content-type', 'text/plain; charset=utf-8');
+    res.end(message);
+}
+function sendJsonRpcError(res, id, code, message) {
+    res.statusCode = 200;
+    res.setHeader('content-type', 'application/json');
+    res.end(JSON.stringify({ jsonrpc: '2.0', id: id ?? null, error: { code, message } }));
+}
+function closeHttpServer(server) {
+    return new Promise((resolve) => server.close(() => resolve()));
+}

package/dist/reporters/cli.js

@@ -46,9 +46,10 @@ export async function runCliPreview(resultsDir, opts) {
         console.log();
         // ── Trials
         for (const trial of trials) {
-            const tp = trial.reward >= 0.5;
-            const trialStatus = tp ? fmt.pass('PASS') : fmt.fail('FAIL');
-            const reward = fmt.bold(trial.reward.toFixed(2));
+            const evaluated = trial.reward !== undefined;
+            const tp = evaluated && trial.reward >= 0.5;
+            const trialStatus = !evaluated ? fmt.dim('N/A') : tp ? fmt.pass('PASS') : fmt.fail('FAIL');
+            const reward = fmt.bold(evaluated ? trial.reward.toFixed(2) : 'n/a');
             const dur = `${((trial.duration_ms || 0) / 1000).toFixed(1)}s`;
             const cmds = `${trial.n_commands || 0} cmds`;
             const scorers = (trial.scorer_results || []).map((g) => {

package/dist/reporters/github-comment.js

@@ -243,7 +243,7 @@ function formatGroupDetails(group) {
     for (const trial of group.trials) {
         out.push(`#### ${trial.name ?? `trial ${trial.trial_id}`}`);
         const reason = trial.diagnostics?.completionReason ?? '—';
-        out.push(`reward: **${trial.reward.toFixed(2)}**  |  duration: ${durationSeconds(trial.duration_ms)}  |  completion: \`${reason}\``);
+        out.push(`reward: **${trial.reward === undefined ? 'n/a' : trial.reward.toFixed(2)}**  |  duration: ${durationSeconds(trial.duration_ms)}  |  completion: \`${reason}\``);
         if (trial.scorer_results.length > 0) {
             out.push('');
             out.push('| Scorer | Score | Weight | Details |');

package/dist/reporters/report-summary.js

@@ -14,6 +14,7 @@ export function printReportSummary(groups, opts = {}) {
         for (const diagnostic of group.diagnostics) {
             const shouldPrintFull = opts.forceVerbose
                 || diagnostic.state !== 'passed'
+                || diagnostic.resultKind === 'synthetic_no_evaluation'
                 || diagnostic.report.completionReason === 'timeout'
                 || diagnostic.report.completionReason === 'agent_crashed';
             console.log(`    ${fmt.bold(diagnostic.caseName)}`);

package/dist/reporting/core.js

@@ -1,3 +1,4 @@
+import { buildDiagnosticsReport } from '../sdk/diagnostics.js';
 import { extractSkillsFromLog } from '../tool-events.js';
 export function buildPathgradeReport(input) {
     const warnings = [];
@@ -37,7 +38,8 @@ export function buildPathgradeReport(input) {
     }
     const scores = reportableGroups.flatMap(group => group.cases
         .filter(testCase => testCase.resultKind !== 'synthetic_no_evaluation')
-        .map(testCase => testCase.score));
+        .map(testCase => testCase.score)
+        .filter((score) => score !== undefined));
     const overallPassRate = average(scores);
     const status = input.threshold != null
         ? (overallPassRate >= input.threshold ? 'pass' : 'fail')
@@ -75,7 +77,15 @@ function toBuiltCase(testCase) {
     // rule among real evaluations without letting a later synthetic result from
     // an unevaluated sibling agent erase a legitimate score.
     const evaluation = testCase.evaluations.findLast(candidate => candidate.resultKind !== 'synthetic_no_evaluation') ?? testCase.evaluations[testCase.evaluations.length - 1];
-    const diagnostics = evaluation.diagnostics ?? testCase.diagnostics;
+    const diagnostics = evaluation.diagnostics
+        ?? testCase.diagnostics
+        ?? (evaluation.resultKind === 'synthetic_no_evaluation'
+            ? buildDiagnosticsReport({
+                score: undefined,
+                warnings: ['evaluate() was not called; no evaluation score is available'],
+                log: [],
+            })
+            : undefined);
     return {
         name: testCase.name,
         state: testCase.state,
@@ -119,7 +129,7 @@ function fallbackCase(testCase, warnings) {
 function normalizeTrial(input) {
     const base = input.trial ?? {
         trial_id: 1,
-        reward: input.score,
+        ...(input.score !== undefined ? { reward: input.score } : {}),
         scorer_results: [],
         duration_ms: input.runnerDurationMs,
         n_commands: 0,
@@ -128,8 +138,12 @@ function normalizeTrial(input) {
         session_log: [],
     };
     const skills = base.skills_used ?? extractSkillsFromLog(base.session_log);
+    const { reward: existingReward, ...baseWithoutReward } = base;
     return {
-        ...base,
+        ...baseWithoutReward,
+        ...(input.resultKind === 'synthetic_no_evaluation'
+            ? {}
+            : { reward: existingReward ?? input.score }),
         name: input.name,
         duration_ms: base.duration_ms || input.runnerDurationMs,
         diagnostics: input.diagnostics ?? base.diagnostics,
@@ -172,7 +186,7 @@ function buildSummary(groupName, cases, report) {
         average_duration_ms: average(cases.map(testCase => testCase.runnerDurationMs)),
         trial_count: cases.length,
         diagnostics: cases.flatMap(testCase => testCase.diagnostics
-            ? [{ caseName: testCase.name, state: testCase.state, report: testCase.diagnostics }]
+            ? [{ caseName: testCase.name, state: testCase.state, resultKind: testCase.resultKind, report: testCase.diagnostics }]
             : []),
     };
 }

package/dist/reporting/types.d.ts

@@ -25,7 +25,7 @@ export interface ReportCaseInput {
     diagnostics?: DiagnosticsReport;
 }
 export interface ReportEvaluationInput {
-    score: number;
+    score?: number;
     trial?: TrialResult;
     diagnostics?: DiagnosticsReport;
     resultKind?: EvaluationResultKind;
@@ -58,5 +58,6 @@ export interface ReportSummaryGroup {
 export interface ReportSummaryDiagnostics {
     caseName: string;
     state: ReportCaseState;
+    resultKind?: EvaluationResultKind;
     report: DiagnosticsReport;
 }

package/dist/runners/model-builders.js

@@ -24,7 +24,7 @@ export function buildNormalizedRunSnapshotFromReportGroups(run, groups) {
                 const evaluations = testCase.evaluations?.map((evaluation, evaluationIndex) => ({
                     id: `${attemptId}:evaluation-${evaluationIndex + 1}`,
                     attemptId,
-                    score: evaluation.score,
+                    ...(evaluation.score !== undefined ? { score: evaluation.score } : {}),
                     ...(evaluation.trial ? { trial: evaluation.trial } : {}),
                     ...(evaluation.diagnostics ? { diagnostics: evaluation.diagnostics } : {}),
                     ...(evaluation.resultKind ? { resultKind: evaluation.resultKind } : {}),

package/dist/runners/model-validation.js

@@ -162,7 +162,10 @@ function validateEvaluation(input) {
     if (input.evaluation.attemptId !== input.attemptId) {
         input.errors.push({ path: `${evaluationPath}.attemptId`, message: 'Evaluation attemptId must reference its attempt.' });
     }
-    if (!Number.isFinite(input.evaluation.score) || input.evaluation.score < 0 || input.evaluation.score > 1) {
+    const scoreMissingForSynthetic = input.evaluation.resultKind === 'synthetic_no_evaluation'
+        && input.evaluation.score === undefined;
+    if (!scoreMissingForSynthetic
+        && (!Number.isFinite(input.evaluation.score) || input.evaluation.score < 0 || input.evaluation.score > 1)) {
         input.errors.push({ path: `${evaluationPath}.score`, message: 'Evaluation score must be a finite number in [0, 1].' });
     }
     validateNativeReferences(input.evaluation.nativeReferences, `${evaluationPath}.nativeReferences`, input.errors);

package/dist/runners/model.d.ts

@@ -79,7 +79,7 @@ export interface AttemptRecord {
 export interface EvaluationRecord {
     id: string;
     attemptId: string;
-    score: number;
+    score?: number;
     name?: string;
     diagnostics?: DiagnosticsReport;
     trial?: TrialResult;

package/dist/runners/report-projection.js

@@ -37,7 +37,7 @@ function totalDurationMs(runCase) {
 function projectedEvaluations(runCase) {
     if (runCase.scoringPolicy.kind === 'non-scoring') {
         return runCase.attempts.flatMap(attempt => (attempt.evaluations ?? []).map(evaluation => ({
-            score: evaluation.score,
+            ...(evaluation.score !== undefined ? { score: evaluation.score } : {}),
             ...(evaluation.trial ? { trial: evaluation.trial } : {}),
             ...(evaluation.diagnostics ? { diagnostics: evaluation.diagnostics } : {}),
             ...(evaluation.resultKind ? { resultKind: evaluation.resultKind } : {}),
@@ -53,7 +53,7 @@ function projectedEvaluations(runCase) {
     if (!evaluation)
         return [];
     return [{
-            score: evaluation.score,
+            ...(evaluation.score !== undefined ? { score: evaluation.score } : {}),
             ...(evaluation.trial ? { trial: evaluation.trial } : {}),
             ...(evaluation.diagnostics ? { diagnostics: evaluation.diagnostics } : {}),
             ...(evaluation.resultKind ? { resultKind: evaluation.resultKind } : {}),

package/dist/runners/vitest-adapter.js

@@ -86,7 +86,7 @@ function toReportCaseInput(testCase, groupName) {
         filePath,
         groupName,
         evaluations: pathgradeMeta?.map(entry => ({
-            score: entry.score,
+            ...(entry.score !== undefined ? { score: entry.score } : {}),
             trial: entry.trial,
             diagnostics: entry.diagnostics,
             resultKind: entry.resultKind,