@wix/pathgrade 1.0.29 → 1.0.31

package/dist/adapters/jest/results.d.ts

@@ -23,6 +23,7 @@ export declare function normalizeJestRunResults(input: {
     run: AdapterRunHandle;
     results: JestAggregatedResult;
     metadataByCaseId?: Map<string, PathgradeTestMeta[]>;
+    cwd?: string;
 }): NormalizedRunSnapshot;
 export declare function jestCaseId(input: {
     filePath: string;

package/dist/adapters/jest/results.js

@@ -1,3 +1,4 @@
+import { resolveSourceMetadata } from '../../reporting/source-metadata.js';
 export function normalizeJestRunResults(input) {
     const runId = `${input.run.adapterName}:run`;
     const files = input.results.testResults ?? [];
@@ -5,6 +6,7 @@ export function normalizeJestRunResults(input) {
         id: unitId(index),
         runId,
         displayName: fileResult.testFilePath,
+        ...resolveSourceMetadata(fileResult.testFilePath, input.cwd ?? process.cwd()),
         groupingHints: [{
                 kind: 'source',
                 key: fileResult.testFilePath,
@@ -68,6 +70,10 @@ function runCase(input) {
             : {}),
         ...(entry.recordedAt ? { recordedAt: entry.recordedAt } : {}),
         ...(entry.agent ? { agent: entry.agent } : {}),
+        ...(entry.evaluationDefinitionKey
+            ? { evaluationDefinitionKey: entry.evaluationDefinitionKey }
+            : {}),
+        ...(entry.scorerRevision ? { scorerRevision: entry.scorerRevision } : {}),
     }));
     const failureMessage = input.assertion.failureMessages?.find(message => message.trim().length > 0);
     return {

package/dist/adapters/jest/runner-adapter.js

@@ -1,9 +1,11 @@
 import { normalizeJestRunResults } from './results.js';
 import { discoverPathgradeEvalFiles, } from '@wix/pathgrade/adapter-kit';
 export function createJestAdapter(options = {}) {
+    let discoveryCwd = process.cwd();
     return {
         name: 'jest',
         async discover(input) {
+            discoveryCwd = input.cwd;
             const include = input.include ?? ['**/*.eval.ts'];
             const exclude = input.exclude ?? [];
             const files = discoverPathgradeEvalFiles({
@@ -34,6 +36,7 @@ export function createJestAdapter(options = {}) {
                 run,
                 results: native.results ?? { testResults: [] },
                 metadataByCaseId: native.metadataByCaseId,
+                cwd: discoveryCwd,
             });
         },
     };

package/dist/adapters/node-test/index.js

@@ -85,6 +85,8 @@ function toReportEvaluations(evaluations) {
         scoringDurationMs: entry.scoringDurationMs,
         recordedAt: entry.recordedAt,
         agent: entry.agent,
+        evaluationDefinitionKey: entry.evaluationDefinitionKey,
+        scorerRevision: entry.scorerRevision,
     }));
 }
 function writeCase(testCase) {

package/dist/adapters/node-test/runner-adapter.js

@@ -37,7 +37,7 @@ export function createNodeTestAdapter() {
                     adapterName: this.name,
                     status: 'completed',
                     exitCode: 0,
-                    native: { resultsPath, resultsDir },
+                    native: { resultsPath, resultsDir, cwd },
                 };
             }
             const exitCode = await spawnNodeTest({
@@ -54,7 +54,7 @@ export function createNodeTestAdapter() {
                 adapterName: this.name,
                 status: input.signal?.aborted ? 'cancelled' : (exitCode === 0 ? 'completed' : 'failed'),
                 exitCode,
-                native: { resultsPath, resultsDir },
+                native: { resultsPath, resultsDir, cwd },
             };
         },
         async collectNormalizedRunSnapshot(run) {
@@ -71,7 +71,7 @@ export function createNodeTestAdapter() {
                 return buildNormalizedRunSnapshotFromReportGroups(run, Array.from(groupMap.entries()).map(([groupName, groupedCases]) => ({
                     groupName,
                     cases: groupedCases,
-                })));
+                })), { cwd: native.cwd });
             }
             finally {
                 if (native.resultsDir)
@@ -103,10 +103,11 @@ function readNodeTestRunNative(run) {
     if (typeof run.native === 'object'
         && run.native !== null
         && typeof run.native.resultsPath === 'string'
-        && typeof run.native.resultsDir === 'string') {
+        && typeof run.native.resultsDir === 'string'
+        && typeof run.native.cwd === 'string') {
         return run.native;
     }
-    return { resultsPath: '', resultsDir: '' };
+    return { resultsPath: '', resultsDir: '', cwd: process.cwd() };
 }
 async function readCases(resultsPath) {
     if (!resultsPath || !(await fs.pathExists(resultsPath)))

package/dist/reporting/comparison-contract.d.ts

@@ -0,0 +1,10 @@
+import type { ComparisonContract, EvalReport } from '../types.js';
+import type { Scorer } from '../sdk/types.js';
+import type { ReportGroupInput } from './types.js';
+export declare function buildComparisonContract(input: {
+    group: ReportGroupInput;
+    report: EvalReport;
+    attemptsRequested: number;
+    attemptsCompleted: number;
+}): ComparisonContract;
+export declare function createScorerRevision(scorers: readonly Scorer[]): string | undefined;

package/dist/reporting/comparison-contract.js

@@ -0,0 +1,127 @@
+import { createHash } from 'node:crypto';
+import { canonicalizeJson } from '../core/canonical-json.js';
+export function buildComparisonContract(input) {
+    const unavailable = new Set();
+    const evaluations = input.group.cases.flatMap(testCase => {
+        const caseIdentity = testCase.repeatKey ?? testCase.caseId ?? testCase.name;
+        return (testCase.attempts?.flatMap(attempt => attempt.evaluations ?? [])
+            ?? testCase.evaluations
+            ?? [])
+            .filter(evaluation => evaluation.resultKind !== 'synthetic_no_evaluation')
+            .map((evaluation, index) => ({
+            case: caseIdentity,
+            definition: evaluation.evaluationDefinitionKey ?? `position:${index}`,
+            scorer: evaluation.scorerRevision,
+            runtime: runtimeIdentity(evaluation.agent ?? evaluation.trial?.agent),
+        }));
+    });
+    const definitionRevision = input.group.sourceRevision
+        ? revision('definition', {
+            source: input.group.sourceRevision,
+            evaluations: uniqueSorted(evaluations.map(evaluation => ({
+                case: evaluation.case, definition: evaluation.definition,
+            }))),
+        })
+        : undefined;
+    if (!definitionRevision)
+        unavailable.add('definition_metadata_missing');
+    const scorerRevision = evaluations.length > 0 && evaluations.every(evaluation => evaluation.scorer)
+        ? revision('scorers', uniqueSorted(evaluations.map(evaluation => ({
+            case: evaluation.case,
+            definition: evaluation.definition,
+            scorer: evaluation.scorer,
+        }))))
+        : undefined;
+    if (!scorerRevision)
+        unavailable.add('scorer_metadata_missing');
+    const runtimeComplete = evaluations.length > 0
+        && evaluations.every(evaluation => Object.values(evaluation.runtime)
+            .every(value => value !== undefined));
+    const runtimeRevision = runtimeComplete
+        ? revision('runtime', uniqueSorted(evaluations.map(evaluation => ({
+            case: evaluation.case,
+            definition: evaluation.definition,
+            runtime: evaluation.runtime,
+        }))))
+        : undefined;
+    if (!runtimeRevision)
+        unavailable.add('runtime_metadata_missing');
+    const reportableCases = input.group.cases.filter(testCase => testCase.reportable !== false
+        && testCase.state !== 'skipped' && testCase.state !== 'pending');
+    const samplingComplete = input.attemptsCompleted === input.attemptsRequested
+        && reportableCases.every(testCase => (testCase.attempts?.length ?? (testCase.evaluations === undefined ? 0 : 1)) === input.attemptsRequested)
+        && reportableCases.every(testCase => testCase.attempts
+            ? testCase.attempts.every(attempt => hasTerminalEvaluation(attempt.evaluations))
+            : hasTerminalEvaluation(testCase.evaluations));
+    const samplingRevision = samplingComplete
+        ? revision('sampling', {
+            attempts: input.attemptsRequested,
+            cases: reportableCases.map(testCase => testCase.repeatKey ?? testCase.caseId ?? testCase.name).toSorted(),
+        })
+        : undefined;
+    if (!samplingRevision)
+        unavailable.add('sampling_incomplete');
+    return {
+        version: 1,
+        ...(definitionRevision ? { definition_revision: definitionRevision } : {}),
+        ...(scorerRevision ? { scorer_revision: scorerRevision } : {}),
+        ...(runtimeRevision ? { runtime_revision: runtimeRevision } : {}),
+        ...(samplingRevision ? { sampling_revision: samplingRevision } : {}),
+        report_schema_revision: 'pathgrade-results-v2',
+        ...(unavailable.size > 0 ? { unavailable_reasons: [...unavailable].sort() } : {}),
+    };
+}
+function hasTerminalEvaluation(evaluations) {
+    return evaluations?.some(evaluation => evaluation.resultKind !== 'synthetic_no_evaluation') === true;
+}
+export function createScorerRevision(scorers) {
+    try {
+        return revision('scorer-declaration', scorers
+            .map(scorer => canonicalizeJson(scorerDeclaration(scorer))).toSorted());
+    }
+    catch {
+        return undefined;
+    }
+}
+function runtimeIdentity(agent) {
+    return {
+        name: agent?.name,
+        model: agent?.resolvedModel,
+        transport: agent?.transport,
+        interaction: agent?.interactionMode,
+    };
+}
+function scorerDeclaration(scorer) {
+    const common = { type: scorer.type, name: scorer.name.normalize('NFC'), weight: scorer.weight };
+    if (scorer.type === 'check' || scorer.type === 'score') {
+        return { ...common, function: scorer.fn.toString() };
+    }
+    if (scorer.type === 'tool_usage') {
+        return {
+            ...common,
+            expectations: scorer.expectations
+                .toSorted((left, right) => canonicalizeJson(left).localeCompare(canonicalizeJson(right))),
+        };
+    }
+    return {
+        ...common,
+        rubric: scorer.rubric,
+        model: scorer.model ?? null,
+        retry: scorer.retry ?? null,
+        includeToolEvents: scorer.includeToolEvents ?? null,
+        input: typeof scorer.input === 'function' ? scorer.input.toString() : scorer.input ?? null,
+        tools: scorer.tools?.toSorted() ?? null,
+        maxRounds: scorer.maxRounds ?? null,
+        cacheControl: scorer.cacheControl ?? null,
+    };
+}
+function revision(namespace, value) {
+    return `sha256:${createHash('sha256')
+        .update(canonicalizeJson(['pathgrade-comparison', 1, namespace, value]))
+        .digest('hex')}`;
+}
+function uniqueSorted(values) {
+    return [...new Map(values.map(value => [canonicalizeJson(value), value])).entries()]
+        .toSorted(([left], [right]) => left.localeCompare(right))
+        .map(([, value]) => value);
+}

package/dist/reporting/core.js

@@ -1,5 +1,6 @@
 import { buildDiagnosticsReport } from '../sdk/diagnostics.js';
 import { extractSkillsFromLog } from '../tool-events.js';
+import { buildComparisonContract } from './comparison-contract.js';
 export function buildPathgradeReport(input) {
     const warnings = [];
     const consolidatedGroups = [];
@@ -8,6 +9,7 @@ export function buildPathgradeReport(input) {
     const attemptsRequested = input.attemptsRequested ?? 1;
     const attemptsCompleted = input.attemptsCompleted ?? attemptsRequested;
     const builtGroups = input.groups.map(group => ({
+        input: group,
         groupName: group.groupName,
         cases: group.cases.map(toBuiltCase),
     }));
@@ -32,7 +34,18 @@ export function buildPathgradeReport(input) {
             return rest;
         });
         const { trials: _trials, ...rest } = report;
-        consolidatedGroups.push({ ...rest, trials: strippedTrials, trace_file: traceFile });
+        consolidatedGroups.push({
+            ...rest,
+            trials: strippedTrials,
+            trace_file: traceFile,
+            ...(group.input.sourceFile ? { source_file: group.input.sourceFile } : {}),
+            comparison_contract: buildComparisonContract({
+                group: group.input,
+                report,
+                attemptsRequested,
+                attemptsCompleted,
+            }),
+        });
     }
     const scores = reportableGroups.flatMap(group => group.cases.flatMap(testCase => (testCase.attempts
         .filter(attempt => attempt.resultKind !== 'synthetic_no_evaluation')

package/dist/reporting/report-parser.d.ts

@@ -0,0 +1,2 @@
+import type { PathgradeReport } from '../types.js';
+export declare function parsePathgradeReport(value: unknown): PathgradeReport;

package/dist/reporting/report-parser.js

@@ -0,0 +1,112 @@
+export function parsePathgradeReport(value) {
+    if (!isRecord(value)
+        || (value.version !== 1 && value.version !== 2)
+        || typeof value.timestamp !== 'string'
+        || typeof value.overall_pass_rate !== 'number'
+        || (value.version === 2 && (typeof value.overall_mean_reward !== 'number' || !validAttemptCounts(value)))
+        || !optionalNumber(value.threshold)
+        || (value.status !== 'pass' && value.status !== 'fail')
+        || (value.run_kind !== undefined && value.run_kind !== 'evaluation' && value.run_kind !== 'no-affected')
+        || !Array.isArray(value.groups)
+        || !value.groups.every(group => isReportGroup(group, value.version))
+        || !isSelection(value.selection)) {
+        throw new Error('PathGrade results.json is missing or has an unsupported schema');
+    }
+    return value;
+}
+function isReportGroup(value, version) {
+    return isRecord(value)
+        && typeof value.task === 'string'
+        && (version === 1 ? legacyMetrics(value) : schemaV2Metrics(value))
+        && optionalString(value.source_file)
+        && typeof value.trace_file === 'string'
+        && Array.isArray(value.skills_used)
+        && value.skills_used.every(item => typeof item === 'string')
+        && Array.isArray(value.trials)
+        && value.trials.every(isTrial)
+        && (value.comparison_contract === undefined || isComparisonContract(value.comparison_contract));
+}
+function isTrial(value) {
+    return isRecord(value)
+        && typeof value.trial_id === 'number'
+        && typeof value.duration_ms === 'number'
+        && typeof value.input_tokens === 'number'
+        && typeof value.output_tokens === 'number'
+        && optionalNumber(value.reward)
+        && optionalNumber(value.conversation_input_tokens)
+        && optionalNumber(value.conversation_output_tokens)
+        && optionalNumber(value.conversation_cost_usd)
+        && Array.isArray(value.scorer_results)
+        && value.scorer_results.every(scorer => isRecord(scorer)
+            && typeof scorer.scorer_type === 'string'
+            && typeof scorer.score === 'number'
+            && typeof scorer.weight === 'number');
+}
+function legacyMetrics(value) {
+    return typeof value.pass_rate === 'number'
+        && typeof value.pass_at_k === 'number'
+        && typeof value.pass_pow_k === 'number';
+}
+function schemaV2Metrics(value) {
+    return (value.status === 'pass' || value.status === 'fail')
+        && typeof value.mean_reward === 'number'
+        && optionalNumber(value.success_rate)
+        && (value.pass_at_k === undefined || (isRecord(value.pass_at_k)
+            && Object.values(value.pass_at_k).every(item => typeof item === 'number')))
+        && (value.pass_at_k_method === undefined || value.pass_at_k_method === 'finite_sample_unbiased')
+        && (value.pass_at_k_unavailable_reason === undefined
+            || value.pass_at_k_unavailable_reason === 'non_binary_reward'
+            || value.pass_at_k_unavailable_reason === 'incomplete_attempts'
+            || value.pass_at_k_unavailable_reason === 'case_identity_mismatch')
+        && optionalNonNegativeInteger(value.eligible_case_count)
+        && optionalPositiveInteger(value.attempts_per_case);
+}
+function validAttemptCounts(value) {
+    return typeof value.attempts_requested === 'number'
+        && Number.isSafeInteger(value.attempts_requested)
+        && value.attempts_requested >= 1
+        && typeof value.attempts_completed === 'number'
+        && Number.isSafeInteger(value.attempts_completed)
+        && value.attempts_completed >= 0
+        && value.attempts_completed <= value.attempts_requested;
+}
+function isSelection(value) {
+    return value === undefined || (isRecord(value)
+        && typeof value.base_ref === 'string'
+        && typeof value.changed_files_count === 'number'
+        && Array.isArray(value.selected)
+        && value.selected.every(item => typeof item === 'string')
+        && Array.isArray(value.skipped)
+        && value.skipped.every(item => isRecord(item)
+            && typeof item.file === 'string'
+            && item.reason === 'no-matching-deps'));
+}
+function isComparisonContract(value) {
+    return isRecord(value)
+        && value.version === 1
+        && optionalString(value.definition_revision)
+        && optionalString(value.scorer_revision)
+        && optionalString(value.runtime_revision)
+        && optionalString(value.sampling_revision)
+        && value.report_schema_revision === 'pathgrade-results-v2'
+        && (value.unavailable_reasons === undefined || (Array.isArray(value.unavailable_reasons)
+            && value.unavailable_reasons.every(reason => reason === 'definition_metadata_missing'
+                || reason === 'scorer_metadata_missing'
+                || reason === 'runtime_metadata_missing'
+                || reason === 'sampling_incomplete')));
+}
+function optionalString(value) {
+    return value === undefined || typeof value === 'string';
+}
+function optionalNumber(value) {
+    return value === undefined || typeof value === 'number';
+}
+function optionalNonNegativeInteger(value) {
+    return value === undefined || (typeof value === 'number' && Number.isSafeInteger(value) && value >= 0);
+}
+function optionalPositiveInteger(value) {
+    return value === undefined || (typeof value === 'number' && Number.isSafeInteger(value) && value >= 1);
+}
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}

package/dist/reporting/source-metadata.d.ts

@@ -0,0 +1,5 @@
+export interface SourceMetadata {
+    sourceFile?: string;
+    sourceRevision?: string;
+}
+export declare function resolveSourceMetadata(filePath: string | undefined, cwd: string): SourceMetadata;

package/dist/reporting/source-metadata.js

@@ -0,0 +1,20 @@
+import { createHash } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+import * as path from 'node:path';
+export function resolveSourceMetadata(filePath, cwd) {
+    if (!filePath)
+        return {};
+    const absolute = path.resolve(cwd, filePath);
+    const relative = path.relative(cwd, absolute).replaceAll(path.sep, '/');
+    if (!relative || relative === '..' || relative.startsWith('../'))
+        return {};
+    try {
+        return {
+            sourceFile: relative,
+            sourceRevision: `sha256:${createHash('sha256').update(readFileSync(absolute)).digest('hex')}`,
+        };
+    }
+    catch {
+        return { sourceFile: relative };
+    }
+}

package/dist/reporting/types.d.ts

@@ -12,6 +12,8 @@ export interface ReportRunInput {
 }
 export interface ReportGroupInput {
     groupName: string;
+    sourceFile?: string;
+    sourceRevision?: string;
     cases: ReportCaseInput[];
 }
 export interface ReportCaseInput {
@@ -45,6 +47,8 @@ export interface ReportEvaluationInput {
     scoringDurationMs?: number;
     recordedAt?: string;
     agent?: AgentExecutionMetadata;
+    evaluationDefinitionKey?: string;
+    scorerRevision?: string;
 }
 export interface PathgradeReportBuildResult {
     report: PathgradeReport;

package/dist/runners/model-builders.d.ts

@@ -1,4 +1,6 @@
 import type { ReportGroupInput } from '../reporting/types.js';
 import type { AdapterRunHandle } from './adapter.js';
 import type { NormalizedRunSnapshot } from './model.js';
-export declare function buildNormalizedRunSnapshotFromReportGroups(run: AdapterRunHandle, groups: ReportGroupInput[]): NormalizedRunSnapshot;
+export declare function buildNormalizedRunSnapshotFromReportGroups(run: AdapterRunHandle, groups: ReportGroupInput[], options?: {
+    cwd?: string;
+}): NormalizedRunSnapshot;

package/dist/runners/model-builders.js

@@ -1,4 +1,6 @@
-export function buildNormalizedRunSnapshotFromReportGroups(run, groups) {
+import { resolveSourceMetadata } from '../reporting/source-metadata.js';
+export function buildNormalizedRunSnapshotFromReportGroups(run, groups, options = {}) {
+    const cwd = options.cwd ?? process.cwd();
     return {
         version: 1,
         completeness: 'final',
@@ -12,12 +14,18 @@ export function buildNormalizedRunSnapshotFromReportGroups(run, groups) {
                         message: diagnostic.message,
                     })) } : {}),
             },
-            units: groups.map((group, index) => ({
-                id: unitId(group.groupName, index),
-                runId: `${run.adapterName}:run`,
-                displayName: group.groupName,
-                groupingHints: [{ kind: 'suite', key: group.groupName, label: group.groupName, order: index }],
-            })),
+            units: groups.map((group, index) => {
+                const source = group.sourceFile && group.sourceRevision
+                    ? { sourceFile: group.sourceFile, sourceRevision: group.sourceRevision }
+                    : resolveSourceMetadata(group.sourceFile ?? group.cases.map(testCase => testCase.filePath ?? testCase.sourceRef).find(Boolean), cwd);
+                return {
+                    id: unitId(group.groupName, index),
+                    runId: `${run.adapterName}:run`,
+                    displayName: group.groupName,
+                    ...source,
+                    groupingHints: [{ kind: 'suite', key: group.groupName, label: group.groupName, order: index }],
+                };
+            }),
             cases: groups.flatMap((group, groupIndex) => group.cases.map((testCase, caseIndex) => {
                 const caseId = testCase.caseId ?? `${unitId(group.groupName, groupIndex)}:case-${caseIndex + 1}`;
                 const attemptId = `${caseId}:attempt-1`;
@@ -33,6 +41,10 @@ export function buildNormalizedRunSnapshotFromReportGroups(run, groups) {
                         : {}),
                     ...(evaluation.recordedAt ? { recordedAt: evaluation.recordedAt } : {}),
                     ...(evaluation.agent ? { agent: evaluation.agent } : {}),
+                    ...(evaluation.evaluationDefinitionKey
+                        ? { evaluationDefinitionKey: evaluation.evaluationDefinitionKey }
+                        : {}),
+                    ...(evaluation.scorerRevision ? { scorerRevision: evaluation.scorerRevision } : {}),
                 }));
                 return {
                     id: caseId,

package/dist/runners/model.d.ts

@@ -25,6 +25,8 @@ export interface EvalUnitRecord {
     id: string;
     runId: string;
     displayName: string;
+    sourceFile?: string;
+    sourceRevision?: string;
     diagnostics?: Diagnostic[];
     nativeReferences?: NativeReference[];
     groupingHints?: GroupingHint[];
@@ -92,6 +94,8 @@ export interface EvaluationRecord {
     scoringDurationMs?: number;
     recordedAt?: string;
     agent?: import('../sdk/types.js').AgentExecutionMetadata;
+    evaluationDefinitionKey?: string;
+    scorerRevision?: string;
 }
 export interface AssertionRecord {
     id: string;

package/dist/runners/report-projection.js

@@ -16,9 +16,15 @@ export function projectNormalizedRunSnapshotToReportInput(snapshot, options = {}
             ?? preferredGroupingLabel(unit?.groupingHints)
             ?? unit?.displayName;
         const key = groupName ?? snapshot.model.run.adapterName;
-        const group = groups.get(key) ?? { groupName: key, cases: [] };
+        const group = groups.get(key) ?? {
+            groupName: key,
+            ...(unit?.sourceFile ? { sourceFile: unit.sourceFile } : {}),
+            ...(unit?.sourceRevision ? { sourceRevision: unit.sourceRevision } : {}),
+            cases: [],
+        };
         group.cases.push({
             caseId: runCase.id,
+            ...(runCase.repeatKey ? { repeatKey: runCase.repeatKey } : {}),
             name: runCase.name,
             state: runCase.state,
             ...(runCase.state === 'skipped' || runCase.state === 'pending' ? { reportable: repeated || runCase.scoringPolicy.kind !== 'non-scoring' } : {}),
@@ -53,6 +59,10 @@ function projectedEvaluations(runCase, attemptEvaluations) {
             ...(evaluation.scoringDurationMs !== undefined ? { scoringDurationMs: evaluation.scoringDurationMs } : {}),
             ...(evaluation.recordedAt ? { recordedAt: evaluation.recordedAt } : {}),
             ...(evaluation.agent ? { agent: evaluation.agent } : {}),
+            ...(evaluation.evaluationDefinitionKey
+                ? { evaluationDefinitionKey: evaluation.evaluationDefinitionKey }
+                : {}),
+            ...(evaluation.scorerRevision ? { scorerRevision: evaluation.scorerRevision } : {}),
         }));
     }
     if (runCase.scoringPolicy.kind === 'score') {
@@ -70,6 +80,10 @@ function projectedEvaluations(runCase, attemptEvaluations) {
             ...(evaluation.scoringDurationMs !== undefined ? { scoringDurationMs: evaluation.scoringDurationMs } : {}),
             ...(evaluation.recordedAt ? { recordedAt: evaluation.recordedAt } : {}),
             ...(evaluation.agent ? { agent: evaluation.agent } : {}),
+            ...(evaluation.evaluationDefinitionKey
+                ? { evaluationDefinitionKey: evaluation.evaluationDefinitionKey }
+                : {}),
+            ...(evaluation.scorerRevision ? { scorerRevision: evaluation.scorerRevision } : {}),
         }];
 }
 function preferredGroupingLabel(hints) {

package/dist/runners/vitest-adapter.js

@@ -2,9 +2,11 @@ import { discoverPathgradeEvalFiles } from '../evals/discovery.js';
 import { buildNormalizedRunSnapshotFromReportGroups } from './model-builders.js';
 import { buildDiagnosticsReport } from '../sdk/diagnostics.js';
 export function createVitestAdapter(options = {}) {
+    let discoveryCwd = process.cwd();
     return {
         name: 'vitest',
         async discover(input) {
+            discoveryCwd = input.cwd;
             const include = input.include ?? ['**/*.eval.ts'];
             const exclude = input.exclude ?? [];
             const files = discoverPathgradeEvalFiles({
@@ -28,11 +30,12 @@ export function createVitestAdapter(options = {}) {
                 exitCode: input.signal?.aborted ? 1 : 0,
                 native: {
                     testModules: options.testModules ?? [],
+                    cwd: discoveryCwd,
                 },
             };
         },
         async collectNormalizedRunSnapshot(run) {
-            return buildNormalizedRunSnapshotFromReportGroups(run, collectVitestReportGroups(readVitestRunNative(run).testModules));
+            return buildNormalizedRunSnapshotFromReportGroups(run, collectVitestReportGroups(readVitestRunNative(run).testModules), { cwd: readVitestRunNative(run).cwd });
         },
     };
 }
@@ -56,12 +59,13 @@ export function collectVitestReportGroups(testModules) {
 function readVitestRunNative(run) {
     if (isVitestRunNative(run.native))
         return run.native;
-    return { testModules: [] };
+    return { testModules: [], cwd: process.cwd() };
 }
 function isVitestRunNative(native) {
     return typeof native === 'object'
         && native !== null
-        && Array.isArray(native.testModules);
+        && Array.isArray(native.testModules)
+        && typeof native.cwd === 'string';
 }
 function getGroupName(testCase) {
     const modulePath = testCase.module.relativeModuleId;
@@ -94,6 +98,8 @@ function toReportCaseInput(testCase, groupName) {
             scoringDurationMs: entry.scoringDurationMs,
             recordedAt: entry.recordedAt,
             agent: entry.agent,
+            evaluationDefinitionKey: entry.evaluationDefinitionKey,
+            scorerRevision: entry.scorerRevision,
         })),
         diagnostics: normalized.diagnostics,
     };

package/dist/sdk/evaluate.js

@@ -1,4 +1,5 @@
 import { countShellCommandsFromLog, extractSkillsFromLog, extractToolEventsFromLog } from '../tool-events.js';
+import { createScorerRevision } from '../reporting/comparison-contract.js';
 import { getRuntime } from './eval-runtime.js';
 import { emitEvalResult } from './result-capture.js';
 import { runJudgePipeline } from './judge-pipeline.js';
@@ -73,6 +74,7 @@ function makeEvaluateAgent() {
             : undefined;
         if (isFirstEval)
             conversationAttributed.add(agent);
+        const scorerRevision = createScorerRevision(scorers);
         const recordedResult = {
             ...evalResult,
             tokenUsage: deltaTokenUsage,
@@ -82,6 +84,7 @@ function makeEvaluateAgent() {
             ...(opts?.evaluationDefinitionKey
                 ? { evaluationDefinitionKey: opts.evaluationDefinitionKey }
                 : {}),
+            ...(scorerRevision ? { scorerRevision } : {}),
             trial: buildTrialResult(agent.log, { ...evalResult, tokenUsage: deltaTokenUsage }, agent.scenarioEvidence, conversationTokens, conversationCost),
         };
         emitEvalResult({ result: recordedResult, agent });
@@ -120,6 +123,7 @@ async function fromSnapshot(snapshotPath, scorers, opts) {
     };
     const scoringStartedAt = performance.now();
     const evalResult = await evaluateWithContext(ctx, scorers, { ...opts, llm: trackedLLM });
+    const scorerRevision = createScorerRevision(scorers);
     const recordedResult = {
         ...evalResult,
         resultKind: 'snapshot',
@@ -128,6 +132,7 @@ async function fromSnapshot(snapshotPath, scorers, opts) {
         ...(opts?.evaluationDefinitionKey
             ? { evaluationDefinitionKey: opts.evaluationDefinitionKey }
             : {}),
+        ...(scorerRevision ? { scorerRevision } : {}),
         trial: buildTrialResult(snapshot.log, evalResult, snapshot.scenarioEvidence),
     };
     maybeThrowOnScorerErrors(recordedResult, opts?.onScorerError ?? 'skip');

package/dist/sdk/index.d.ts

@@ -21,6 +21,7 @@ export { DEFAULT_COPY_IGNORE } from '../providers/copy-filter.js';
 export { extractToolEventsFromLog } from '../tool-events.js';
 export { collectSensitiveEnvValues, sanitizePersistenceValue, } from '../tool-event-results.js';
 export { parseEnvFile } from '../utils/env.js';
+export { parsePathgradeReport } from '../reporting/report-parser.js';
 export { createAskBus, requireAskBusForLiveBatches, AskBusTimeoutError } from './ask-bus/bus.js';
 export { toAskUserToolEvent } from './ask-bus/projection.js';
 export type { AskUserToolEvent, AskUserToolEventArguments, AskUserToolEventQuestionArgument, } from './ask-bus/projection.js';
@@ -40,6 +41,7 @@ export type { DiagnosticsReport } from './diagnostics.js';
 export type { ExpectedMcpStartupStatus, ExpectedMcpToolCall, McpStartupStatusEvidence, McpToolCallEvidence, McpApprovalEvidence, ExpectedMcpApproval, McpInvocationResult, } from './mcp-evidence.js';
 export type { McpPolicyDenialReason, McpToolCallRequest, McpToolPolicyDecision, } from './mcp-safety.js';
 export type { ToolEvent, McpToolCallClassification } from '../tool-events.js';
+export type { ComparisonContract, ComparisonUnavailableReason, PathgradeGroupReport, PathgradeReport, StrippedTrialResult, } from '../types.js';
 export type { LLMPort, EvalRuntime } from './eval-runtime.js';
 export { createAgentLLM, createLLMClient, ProviderNotSupportedError } from '../utils/llm.js';
 export type { CreateLLMClientOptions, LLMProviderAdapter, TokenUsage as LLMTokenUsage } from '../utils/llm.js';

package/dist/sdk/index.js

@@ -19,6 +19,7 @@ export { DEFAULT_COPY_IGNORE } from '../providers/copy-filter.js';
 export { extractToolEventsFromLog } from '../tool-events.js';
 export { collectSensitiveEnvValues, sanitizePersistenceValue, } from '../tool-event-results.js';
 export { parseEnvFile } from '../utils/env.js';
+export { parsePathgradeReport } from '../reporting/report-parser.js';
 export { createAskBus, requireAskBusForLiveBatches, AskBusTimeoutError } from './ask-bus/bus.js';
 export { toAskUserToolEvent } from './ask-bus/projection.js';
 export { buildAskBatchLogEntries } from './agent-result-log.js';

package/dist/sdk/lifecycle.js

@@ -81,6 +81,7 @@ function recordResult(result, agent, attribution) {
             ...(result.evaluationDefinitionKey
                 ? { evaluationDefinitionKey: result.evaluationDefinitionKey }
                 : {}),
+            ...(result.scorerRevision ? { scorerRevision: result.scorerRevision } : {}),
             trial: result.trial,
             resultKind: result.resultKind ?? 'evaluated',
             ...(result.scoringDurationMs !== undefined

package/dist/sdk/scorer-utils.js

@@ -2,10 +2,20 @@ export function clamp(n) {
     return Math.max(0, Math.min(1, n));
 }
 export function getErrorMessage(error) {
-    if (error instanceof Error) {
-        return error.message;
+    if (!(error instanceof Error)) {
+        return String(error);
     }
-    return String(error);
+    const seen = new Set();
+    let message = error.message;
+    let cause = error.cause;
+    while (cause instanceof Error && !seen.has(cause)) {
+        seen.add(cause);
+        if (cause.message && !message.includes(cause.message)) {
+            message = message ? `${message}: ${cause.message}` : cause.message;
+        }
+        cause = cause.cause;
+    }
+    return message || String(error);
 }
 export function matchesExpectation(event, expectation) {
     if (event.action !== expectation.action)

package/dist/sdk/types.d.ts

@@ -372,6 +372,8 @@ export interface RecordedEvalResult extends Omit<EvalResult, 'score'> {
     scoringDurationMs?: number;
     recordedAt?: string;
     evaluationDefinitionKey?: string;
+    /** Privacy-safe revision of the declared scorers used by this evaluation. */
+    scorerRevision?: string;
 }
 export type EvaluationResultKind = 'evaluated' | 'synthetic_no_evaluation' | 'snapshot';
 export interface ScorerResultEntry {
@@ -393,6 +395,7 @@ export interface PathgradeTestMeta {
     scorers: ScorerResultEntry[];
     /** Stable identity for this evaluation definition across separate runs. */
     evaluationDefinitionKey?: string;
+    scorerRevision?: string;
     trial?: TrialResult;
     diagnostics?: DiagnosticsReport;
     resultKind?: EvaluationResultKind;

package/dist/types.d.ts

@@ -219,6 +219,16 @@ export interface EvalReport {
     trials: TrialResult[];
     skills_used: string[];
 }
+export type ComparisonUnavailableReason = 'definition_metadata_missing' | 'scorer_metadata_missing' | 'runtime_metadata_missing' | 'sampling_incomplete';
+export interface ComparisonContract {
+    version: 1;
+    definition_revision?: string;
+    scorer_revision?: string;
+    runtime_revision?: string;
+    sampling_revision?: string;
+    report_schema_revision: 'pathgrade-results-v2';
+    unavailable_reasons?: ComparisonUnavailableReason[];
+}
 /**
  * TrialResult with `session_log` and `conversation` stripped. These fields
  * live only in the per-group trace files; the consolidated results.json keeps
@@ -232,6 +242,8 @@ export type StrippedTrialResult = Omit<TrialResult, 'session_log' | 'conversatio
 export type PathgradeGroupReport = Omit<EvalReport, 'trials' | 'status'> & {
     status?: 'pass' | 'fail';
     trials: StrippedTrialResult[];
+    source_file?: string;
+    comparison_contract?: ComparisonContract;
     /** Relative path from `.pathgrade/` to the trace file for this group. */
     trace_file: string;
 };
@@ -277,6 +289,8 @@ export interface PathgradeReport {
      * in every group passed its vitest test.
      */
     status: 'pass' | 'fail';
+    /** Agent Evals compatibility manifest when changed selection finds no runnable evals. */
+    run_kind?: 'evaluation' | 'no-affected';
     groups: PathgradeGroupReport[];
     /**
      * Present when `pathgrade run --changed` produced the run. Absent on

package/dist/utils/llm-providers/anthropic.js

@@ -73,7 +73,7 @@ async function postAnthropic(apiKey, useCache, body, env) {
     });
     if (!response.ok) {
         const errBody = await response.text().catch(() => '');
-        throw new Error(`Anthropic API error (${response.status}): ${errBody.slice(0, 300)}`);
+        throw new Error(`Anthropic API error (${response.status}) for model ${String(body.model)}: ${errBody.slice(0, 300)}`);
     }
     return await response.json();
 }
@@ -116,7 +116,10 @@ export const anthropicProvider = {
             };
         }
         catch (error) {
-            throw new Error(`Anthropic API call failed`, { cause: error });
+            // Reporting seams only propagate error.message, never error.cause.
+            const reason = error instanceof Error ? error.message : String(error);
+            const modelTag = reason.includes(model) ? '' : ` (model ${model})`;
+            throw new Error(`Anthropic API call failed${modelTag}: ${reason}`, { cause: error });
         }
     },
     async callWithTools(messages, opts) {

package/dist/utils/llm.js

@@ -114,6 +114,7 @@ export function createLLMClient(arg, legacyAgentName) {
             const toTry = narrowed.length > 0 ? narrowed : candidates;
             let lastError;
             let lastFailed;
+            const failures = [];
             for (const provider of toTry) {
                 if (!await provider.isAvailable(opts.env))
                     continue;
@@ -128,6 +129,7 @@ export function createLLMClient(arg, legacyAgentName) {
                 catch (error) {
                     lastError = error;
                     lastFailed = { name: provider.name, reason: lastError.message };
+                    failures.push(lastFailed);
                     // Fall through to next provider
                 }
             }
@@ -149,11 +151,15 @@ export function createLLMClient(arg, legacyAgentName) {
                     catch (error) {
                         lastError = error;
                         lastFailed = { name: provider.name, reason: lastError.message };
+                        failures.push(lastFailed);
                     }
                 }
             }
-            if (lastError)
-                throw lastError;
+            if (lastError) {
+                if (failures.length <= 1)
+                    throw lastError;
+                throw new Error(`All LLM providers failed: ${failures.map((f) => `${f.name}: ${f.reason}`).join('; ')}`, { cause: lastError });
+            }
             if (agentName === 'claude') {
                 throw new Error('Claude CLI not found on PATH. Install with: npm install -g @anthropic-ai/claude-code');
             }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wix/pathgrade",
-  "version": "1.0.29",
+  "version": "1.0.31",
   "packageManager": "yarn@4.12.0",
   "description": "Evaluate whether AI agents discover and use your skills correctly",
   "exports": {
@@ -140,5 +140,5 @@
     "typescript": "^5.9.3",
     "zod": "4.3.6"
   },
-  "falconPackageHash": "126198d020df14b3e6fceef3b0325ac5e84411c5e1b544a0167de604"
+  "falconPackageHash": "1f94e7e2c7f2f999b9ba92d6b76b362a5adf549800b96e08994d6c49"
 }