@wix/pathgrade 1.0.38 → 1.0.40

package/dist/reporters/loader.js

@@ -34,11 +34,18 @@ export async function loadReports(resultsDir, opts) {
             if (raw.version !== 1 && raw.version !== 2)
                 continue;
             for (const group of raw.groups) {
+                const runnerStatus = groupRunnerStatus(group);
+                const thresholdStatus = groupThresholdStatus(raw, group);
                 const report = {
                     file,
                     timestamp: raw.timestamp,
+                    threshold: raw.threshold,
                     ...group,
-                    status: groupStatus(raw, group),
+                    runner_status: runnerStatus,
+                    ...(raw.runner_status === 'pass' || raw.runner_status === 'fail'
+                        ? { run_runner_status: raw.runner_status } : {}),
+                    threshold_status: thresholdStatus,
+                    status: runnerStatus === 'fail' || thresholdStatus === 'fail' ? 'fail' : 'pass',
                 };
                 if (!opts?.skipTraces && group.trace_file) {
                     await hydrateTraces(report, group.trace_file, resolved);
@@ -50,15 +57,25 @@ export async function loadReports(resultsDir, opts) {
     }
     return results;
 }
-function groupStatus(raw, group) {
+function groupRunnerStatus(group) {
+    if (group.runner_status === 'pass' || group.runner_status === 'fail')
+        return group.runner_status;
+    const outcomes = Array.isArray(group.trials)
+        ? group.trials.map((trial) => trial.runner_outcome).filter((outcome) => outcome !== undefined)
+        : [];
+    if (outcomes.length > 0) {
+        return outcomes.every((outcome) => outcome === 'passed') ? 'pass' : 'fail';
+    }
     if (group.status === 'pass' || group.status === 'fail')
         return group.status;
     const meanReward = group.mean_reward ?? group.pass_rate ?? 0;
-    if (typeof raw.threshold === 'number')
-        return meanReward >= raw.threshold ? 'pass' : 'fail';
-    const outcomes = Array.isArray(group.trials) ? group.trials.map((trial) => trial.runner_outcome) : [];
-    if (outcomes.length > 0 && outcomes.every((outcome) => (outcome === 'passed' || outcome === 'failed' || outcome === 'not-run'))) {
-        return outcomes.every((outcome) => outcome === 'passed') ? 'pass' : 'fail';
-    }
-    return meanReward >= 0.5 ? 'pass' : 'fail';
+    return meanReward === 1 ? 'pass' : 'fail';
+}
+function groupThresholdStatus(raw, group) {
+    if (group.threshold_status === 'pass' || group.threshold_status === 'fail'
+        || group.threshold_status === 'not_configured')
+        return group.threshold_status;
+    if (typeof raw.threshold !== 'number')
+        return 'not_configured';
+    return (group.mean_reward ?? group.pass_rate ?? 0) >= raw.threshold ? 'pass' : 'fail';
 }

package/dist/reporting/comparison-contract.d.ts

@@ -1,10 +1,11 @@
-import type { ComparisonContract, EvalReport } from '../types.js';
+import type { EvalReport } from '../types.js';
 import type { Scorer } from '../sdk/types.js';
 import type { ReportGroupInput } from './types.js';
+import type { ComparisonContractV2 } from './reliability-contract.js';
 export declare function buildComparisonContract(input: {
     group: ReportGroupInput;
     report: EvalReport;
     attemptsRequested: number;
     attemptsCompleted: number;
-}): ComparisonContract;
+}): ComparisonContractV2;
 export declare function createScorerRevision(scorers: readonly Scorer[]): string | undefined;

package/dist/reporting/comparison-contract.js

@@ -1,7 +1,6 @@
 import { createHash } from 'node:crypto';
 import { canonicalizeJson } from '../core/canonical-json.js';
 export function buildComparisonContract(input) {
-    const unavailable = new Set();
     const evaluations = input.group.cases.flatMap(testCase => {
         const caseIdentity = testCase.repeatKey ?? testCase.caseId ?? testCase.name;
         return (testCase.attempts?.flatMap(attempt => attempt.evaluations ?? [])
@@ -15,16 +14,16 @@ export function buildComparisonContract(input) {
             runtime: runtimeIdentity(evaluation),
         }));
     });
-    const definitionRevision = input.group.sourceRevision
+    const comparisonInputs = input.group.comparisonInputs ?? { state: 'missing' };
+    const definitionRevision = input.group.sourceRevision && comparisonInputs.state === 'resolved'
         ? revision('definition', {
             source: input.group.sourceRevision,
+            comparison_inputs: comparisonInputs.revision,
             evaluations: uniqueSorted(evaluations.map(evaluation => ({
                 case: evaluation.case, definition: evaluation.definition,
             }))),
         })
         : undefined;
-    if (!definitionRevision)
-        unavailable.add('definition_metadata_missing');
     const scorerRevision = evaluations.length > 0 && evaluations.every(evaluation => evaluation.scorer)
         ? revision('scorers', uniqueSorted(evaluations.map(evaluation => ({
             case: evaluation.case,
@@ -32,20 +31,10 @@ export function buildComparisonContract(input) {
             scorer: evaluation.scorer,
         }))))
         : undefined;
-    if (!scorerRevision)
-        unavailable.add('scorer_metadata_missing');
     const runtimeComplete = evaluations.length > 0
-        && evaluations.every(evaluation => Object.values(evaluation.runtime)
-            .every(value => value !== undefined));
-    const runtimeRevision = runtimeComplete
-        ? revision('runtime', uniqueSorted(evaluations.map(evaluation => ({
-            case: evaluation.case,
-            definition: evaluation.definition,
-            runtime: evaluation.runtime,
-        }))))
-        : undefined;
-    if (!runtimeRevision)
-        unavailable.add('runtime_metadata_missing');
+        && evaluations.every(evaluation => runtimeIdentityComplete(evaluation.runtime));
+    const runtimeComponents = runtimeComplete ? runtimeComponentRevisions(evaluations) : undefined;
+    const runtimeRevision = runtimeComponents ? revision('runtime', runtimeComponents) : undefined;
     const reportableCases = input.group.cases.filter(testCase => testCase.reportable !== false
         && testCase.state !== 'skipped' && testCase.state !== 'pending');
     const samplingComplete = input.attemptsCompleted === input.attemptsRequested
@@ -59,16 +48,41 @@ export function buildComparisonContract(input) {
             cases: reportableCases.map(testCase => testCase.repeatKey ?? testCase.caseId ?? testCase.name).toSorted(),
         })
         : undefined;
-    if (!samplingRevision)
-        unavailable.add('sampling_incomplete');
     return {
-        version: 1,
-        ...(definitionRevision ? { definition_revision: definitionRevision } : {}),
-        ...(scorerRevision ? { scorer_revision: scorerRevision } : {}),
-        ...(runtimeRevision ? { runtime_revision: runtimeRevision } : {}),
-        ...(samplingRevision ? { sampling_revision: samplingRevision } : {}),
-        report_schema_revision: 'pathgrade-results-v2',
-        ...(unavailable.size > 0 ? { unavailable_reasons: [...unavailable].sort() } : {}),
+        version: 2,
+        ...(definitionRevision
+            ? { comparison_inputs: comparisonInputs, definition_revision: definitionRevision }
+            : { comparison_inputs: comparisonInputs }),
+        ...(scorerRevision
+            ? { scorer_revision_state: { state: 'complete' }, scorer_revision: scorerRevision }
+            : { scorer_revision_state: { state: 'incomplete', reason: 'explicit-revision-required' } }),
+        ...(runtimeRevision && runtimeComponents
+            ? {
+                runtime_revision_state: { state: 'complete' },
+                runtime_revision: runtimeRevision,
+                runtime_components: runtimeComponents,
+            }
+            : { runtime_revision_state: { state: 'incomplete', reason: 'metadata-missing' } }),
+        ...(samplingRevision
+            ? { sampling_revision_state: { state: 'complete' }, sampling_revision: samplingRevision }
+            : { sampling_revision_state: { state: 'incomplete', reason: 'sampling-incomplete' } }),
+        report_schema_revision: 'pathgrade-results-v3',
+    };
+}
+function runtimeComponentRevisions(evaluations) {
+    const component = (name) => revision(`runtime-${name}`, uniqueSorted(evaluations.map(evaluation => ({
+        case: evaluation.case,
+        definition: evaluation.definition,
+        value: evaluation.runtime[name] ?? null,
+    }))));
+    return {
+        agent_name: component('agent_name'),
+        model: component('model'),
+        transport: component('transport'),
+        interaction_mode: component('interaction_mode'),
+        ...(evaluations.some(evaluation => evaluation.runtime.flow)
+            ? { flow: component('flow') }
+            : {}),
     };
 }
 function hasTerminalEvaluation(evaluations) {
@@ -85,16 +99,20 @@ export function createScorerRevision(scorers) {
 }
 function runtimeIdentity(evaluation) {
     const agent = evaluation.agent ?? evaluation.trial?.agent;
-    if (agent) {
+    if (agent)
         return {
-            name: agent.name,
+            agent_name: agent.name,
             model: agent.resolvedModel,
             transport: agent.transport,
-            interaction: agent.interactionMode,
+            interaction_mode: agent.interactionMode,
+            flow: undefined,
         };
-    }
     const flow = evaluation.trial?.flow_trace;
     return {
+        agent_name: undefined,
+        model: undefined,
+        transport: undefined,
+        interaction_mode: undefined,
         flow: flow?.completeness.runtimeIdentity === 'complete'
             && flow.participants.every(participant => participant.runtime) ? {
             ...(flow.protocol ? { protocol: flow.protocol } : {}),
@@ -105,10 +123,22 @@ function runtimeIdentity(evaluation) {
         } : undefined,
     };
 }
+function runtimeIdentityComplete(identity) {
+    return identity.flow !== undefined || [
+        identity.agent_name, identity.model, identity.transport, identity.interaction_mode,
+    ].every(value => value !== undefined);
+}
 function scorerDeclaration(scorer) {
-    const common = { type: scorer.type, name: scorer.name.normalize('NFC'), weight: scorer.weight };
+    const common = {
+        type: scorer.type,
+        name: scorer.name.normalize('NFC'),
+        weight: scorer.weight,
+        ...(scorer.revision === undefined ? {} : { revision: scorer.revision }),
+    };
     if (scorer.type === 'check' || scorer.type === 'score') {
-        return { ...common, function: scorer.fn.toString() };
+        if (scorer.revision === undefined)
+            throw new Error('explicit scorer revision required');
+        return common;
     }
     if (scorer.type === 'tool_usage') {
         return {
@@ -117,13 +147,16 @@ function scorerDeclaration(scorer) {
                 .toSorted((left, right) => canonicalizeJson(left).localeCompare(canonicalizeJson(right))),
         };
     }
+    if (typeof scorer.input === 'function' && scorer.revision === undefined) {
+        throw new Error('explicit scorer revision required');
+    }
     return {
         ...common,
         rubric: scorer.rubric,
         model: scorer.model ?? null,
         retry: scorer.retry ?? null,
         includeToolEvents: scorer.includeToolEvents ?? null,
-        input: typeof scorer.input === 'function' ? scorer.input.toString() : scorer.input ?? null,
+        input: typeof scorer.input === 'function' ? null : scorer.input ?? null,
         tools: scorer.tools?.toSorted() ?? null,
         maxRounds: scorer.maxRounds ?? null,
         cacheControl: scorer.cacheControl ?? null,

package/dist/reporting/core.js

@@ -1,6 +1,7 @@
 import { buildDiagnosticsReport } from '../sdk/diagnostics.js';
 import { extractSkillsFromLog } from '../tool-events.js';
 import { buildComparisonContract } from './comparison-contract.js';
+import { TASK_INVENTORY_VERSION } from './reliability-contract.js';
 export function buildPathgradeReport(input) {
     const warnings = [];
     const consolidatedGroups = [];
@@ -52,20 +53,34 @@ export function buildPathgradeReport(input) {
         .map(attempt => attempt.score)
         .filter((score) => score !== undefined))));
     const overallMeanReward = average(scores);
-    const status = input.threshold != null
-        ? (overallMeanReward >= input.threshold ? 'pass' : 'fail')
-        : (reportableGroups.every(group => group.cases.every(testCase => testCase.state === 'passed')) ? 'pass' : 'fail');
+    const runnerStatus = (input.runStatus === undefined || input.runStatus === 'completed')
+        && reportableGroups.every(group => group.cases.every(testCase => (testCase.state !== 'failed'
+            && testCase.attempts.every(attempt => attempt.outcome.kind === 'passed')))) ? 'pass' : 'fail';
+    const thresholdStatus = input.threshold == null
+        ? 'not_configured'
+        : overallMeanReward >= input.threshold ? 'pass' : 'fail';
     return {
         report: {
-            version: 2,
+            version: 3,
             timestamp: new Date().toISOString(),
             ...(input.threshold != null ? { threshold: input.threshold } : {}),
             attempts_requested: attemptsRequested,
             attempts_completed: attemptsCompleted,
             overall_mean_reward: overallMeanReward,
             overall_pass_rate: overallMeanReward,
-            status,
+            runner_status: runnerStatus,
+            threshold_status: thresholdStatus,
+            status: runnerStatus === 'fail' || thresholdStatus === 'fail' ? 'fail' : 'pass',
             groups: consolidatedGroups,
+            task_inventory: input.taskInventory ?? {
+                version: TASK_INVENTORY_VERSION,
+                files: input.groups.flatMap(group => group.sourceFile ? [{
+                        eval_file: group.sourceFile,
+                        completeness: 'incomplete',
+                        reason: 'adapter-cannot-prove-completeness',
+                        tasks: [taskInventoryEntry(group)],
+                    }] : []),
+            },
             ...(input.selection ? { selection: input.selection } : {}),
         },
         traces,
@@ -73,6 +88,13 @@ export function buildPathgradeReport(input) {
         warnings,
     };
 }
+function taskInventoryEntry(group) {
+    const scored = group.cases.some(testCase => testCase.state !== 'skipped' && testCase.state !== 'pending');
+    if (scored)
+        return { task_key: group.groupName, state: 'scored' };
+    const reason = group.cases.every(testCase => testCase.state === 'skipped') ? 'skipped' : 'pending';
+    return { task_key: group.groupName, state: 'not-scored', reason };
+}
 function reportableBuiltCase(testCase) {
     if (testCase.reportable !== undefined)
         return testCase.reportable;
@@ -185,15 +207,20 @@ function buildEvalReport(groupName, cases, run) {
         .filter((score) => score !== undefined);
     const meanReward = average(rewards);
     const metrics = binaryMetrics(cases, run);
+    const runnerStatus = cases.every(testCase => (testCase.state !== 'failed'
+        && testCase.attempts.every(attempt => attempt.outcome.kind === 'passed'))) ? 'pass' : 'fail';
+    const thresholdStatus = run.threshold == null
+        ? 'not_configured'
+        : meanReward >= run.threshold ? 'pass' : 'fail';
     const skills = new Set();
     for (const trial of trials)
         for (const skill of trial.skills_used ?? [])
             skills.add(skill);
     return {
         task: groupName,
-        status: run.threshold != null
-            ? (meanReward >= run.threshold ? 'pass' : 'fail')
-            : (cases.every(testCase => testCase.state === 'passed') ? 'pass' : 'fail'),
+        runner_status: runnerStatus,
+        threshold_status: thresholdStatus,
+        status: runnerStatus === 'fail' || thresholdStatus === 'fail' ? 'fail' : 'pass',
         mean_reward: meanReward,
         ...(metrics.successRate !== undefined ? { success_rate: metrics.successRate } : {}),
         ...(metrics.passAtK ? {

package/dist/reporting/reliability-contract.d.ts

@@ -53,6 +53,7 @@ export interface RuntimeComponentRevisions {
     model: string;
     transport: string;
     interaction_mode: string;
+    flow?: string;
 }
 type DefinitionRevisionContract = {
     comparison_inputs: Extract<ComparisonInputsState, {

package/dist/reporting/report-parser.js

@@ -1,30 +1,87 @@
 export function parsePathgradeReport(value) {
     if (!isRecord(value)
-        || (value.version !== 1 && value.version !== 2)
+        || (value.version !== 1 && value.version !== 2 && value.version !== 3)
         || typeof value.timestamp !== 'string'
         || typeof value.overall_pass_rate !== 'number'
-        || (value.version === 2 && (typeof value.overall_mean_reward !== 'number' || !validAttemptCounts(value)))
+        || (value.version !== 1 && (typeof value.overall_mean_reward !== 'number' || !validAttemptCounts(value)))
         || !optionalNumber(value.threshold)
         || (value.status !== 'pass' && value.status !== 'fail')
+        || !optionalGateStatus(value.runner_status)
+        || !optionalThresholdStatus(value.threshold_status)
         || (value.run_kind !== undefined && value.run_kind !== 'evaluation' && value.run_kind !== 'no-affected')
         || !Array.isArray(value.groups)
         || !value.groups.every(group => isReportGroup(group, value.version))
+        || (value.version === 3 && !isTaskInventory(value.task_inventory))
         || !isSelection(value.selection)) {
         throw new Error('PathGrade results.json is missing or has an unsupported schema');
     }
-    return value;
+    const threshold = value.threshold;
+    const groups = value.groups.map(group => {
+        const runnerStatus = deriveRunnerStatus(group);
+        const thresholdStatus = deriveThresholdStatus(group.mean_reward ?? group.pass_rate, threshold);
+        return {
+            ...group,
+            runner_status: runnerStatus,
+            threshold_status: thresholdStatus,
+            status: composeStatus(runnerStatus, thresholdStatus),
+        };
+    });
+    const legacyFailure = value.runner_status === undefined && value.status === 'fail' && groups.length === 0;
+    const runnerStatus = value.runner_status !== 'fail' && !legacyFailure
+        && (groups.length > 0 || value.runner_status === 'pass' || value.status === 'pass')
+        && groups.every(group => group.runner_status === 'pass') ? 'pass' : 'fail';
+    const thresholdStatus = deriveThresholdStatus(value.overall_mean_reward ?? value.overall_pass_rate, threshold);
+    return {
+        ...value,
+        groups,
+        runner_status: runnerStatus,
+        threshold_status: thresholdStatus,
+        status: composeStatus(runnerStatus, thresholdStatus),
+    };
 }
 function isReportGroup(value, version) {
     return isRecord(value)
         && typeof value.task === 'string'
         && (version === 1 ? legacyMetrics(value) : schemaV2Metrics(value))
+        && optionalGateStatus(value.runner_status)
+        && optionalThresholdStatus(value.threshold_status)
         && optionalString(value.source_file)
         && typeof value.trace_file === 'string'
         && Array.isArray(value.skills_used)
         && value.skills_used.every(item => typeof item === 'string')
         && Array.isArray(value.trials)
         && value.trials.every(isTrial)
-        && (value.comparison_contract === undefined || isComparisonContract(value.comparison_contract));
+        && (version === 3
+            ? isComparisonContractV2(value.comparison_contract)
+            : value.comparison_contract === undefined || isComparisonContract(value.comparison_contract));
+}
+function deriveRunnerStatus(value) {
+    if (value.runner_status === 'fail')
+        return 'fail';
+    const outcomes = value.trials
+        .map(trial => trial.runner_outcome)
+        .filter(outcome => outcome !== undefined);
+    if (outcomes.length > 0)
+        return outcomes.every(outcome => outcome === 'passed') ? 'pass' : 'fail';
+    if (value.runner_status === 'pass')
+        return value.runner_status;
+    if (value.status === 'pass' || value.status === 'fail')
+        return value.status;
+    return value.pass_rate === 1 ? 'pass' : 'fail';
+}
+function deriveThresholdStatus(reward, threshold) {
+    if (threshold === undefined)
+        return 'not_configured';
+    return typeof reward === 'number' && reward >= threshold ? 'pass' : 'fail';
+}
+function composeStatus(runnerStatus, thresholdStatus) {
+    return runnerStatus === 'fail' || thresholdStatus === 'fail' ? 'fail' : 'pass';
+}
+function optionalGateStatus(value) {
+    return value === undefined || value === 'pass' || value === 'fail';
+}
+function optionalThresholdStatus(value) {
+    return optionalGateStatus(value) || value === 'not_configured';
 }
 function isTrial(value) {
     return isRecord(value)
@@ -40,7 +97,8 @@ function isTrial(value) {
         && value.scorer_results.every(scorer => isRecord(scorer)
             && typeof scorer.scorer_type === 'string'
             && typeof scorer.score === 'number'
-            && typeof scorer.weight === 'number');
+            && typeof scorer.weight === 'number'
+            && (scorer.status === undefined || scorer.status === 'ok' || scorer.status === 'error' || scorer.status === 'skipped'));
 }
 function legacyMetrics(value) {
     return typeof value.pass_rate === 'number'
@@ -82,6 +140,9 @@ function isSelection(value) {
             && item.reason === 'no-matching-deps'));
 }
 function isComparisonContract(value) {
+    return isComparisonContractV1(value) || isComparisonContractV2(value);
+}
+function isComparisonContractV1(value) {
     return isRecord(value)
         && value.version === 1
         && optionalString(value.definition_revision)
@@ -95,6 +156,65 @@ function isComparisonContract(value) {
                 || reason === 'runtime_metadata_missing'
                 || reason === 'sampling_incomplete')));
 }
+function isComparisonContractV2(value) {
+    if (!isRecord(value) || value.version !== 2 || value.report_schema_revision !== 'pathgrade-results-v3')
+        return false;
+    const comparisonInputs = value.comparison_inputs;
+    const definitionComplete = isRecord(comparisonInputs) && comparisonInputs.state === 'resolved';
+    const scorerComplete = isState(value.scorer_revision_state, 'complete');
+    const runtimeComplete = isState(value.runtime_revision_state, 'complete');
+    const samplingComplete = isState(value.sampling_revision_state, 'complete');
+    return isComparisonInputs(comparisonInputs)
+        && (definitionComplete ? typeof value.definition_revision === 'string' : value.definition_revision === undefined)
+        && isRevisionState(value.scorer_revision_state, ['explicit-revision-required', 'metadata-missing'])
+        && (scorerComplete ? typeof value.scorer_revision === 'string' : value.scorer_revision === undefined)
+        && isRevisionState(value.runtime_revision_state, ['metadata-missing'])
+        && (runtimeComplete
+            ? typeof value.runtime_revision === 'string' && isRuntimeComponents(value.runtime_components)
+            : value.runtime_revision === undefined && value.runtime_components === undefined)
+        && isRevisionState(value.sampling_revision_state, ['sampling-incomplete'])
+        && (samplingComplete ? typeof value.sampling_revision === 'string' : value.sampling_revision === undefined);
+}
+function isComparisonInputs(value) {
+    if (!isRecord(value))
+        return false;
+    if (value.state === 'missing')
+        return true;
+    if (value.state === 'invalid') {
+        return value.reason === 'malformed-declaration' || value.reason === 'no-matches'
+            || value.reason === 'outside-repository' || value.reason === 'unreadable-input';
+    }
+    return value.state === 'resolved'
+        && Array.isArray(value.declarations) && value.declarations.every(item => typeof item === 'string')
+        && Array.isArray(value.files) && value.files.every(file => isRecord(file)
+        && typeof file.path === 'string' && typeof file.revision === 'string')
+        && typeof value.revision === 'string';
+}
+function isRevisionState(value, reasons) {
+    return isState(value, 'complete') || (isRecord(value) && value.state === 'incomplete'
+        && typeof value.reason === 'string' && reasons.includes(value.reason));
+}
+function isState(value, state) {
+    return isRecord(value) && value.state === state;
+}
+function isRuntimeComponents(value) {
+    return isRecord(value) && ['agent_name', 'model', 'transport', 'interaction_mode']
+        .every(key => typeof value[key] === 'string')
+        && optionalString(value.flow);
+}
+function isTaskInventory(value) {
+    return isRecord(value) && value.version === 1 && Array.isArray(value.files)
+        && value.files.every(file => isRecord(file)
+            && typeof file.eval_file === 'string'
+            && (file.completeness === 'complete' || (file.completeness === 'incomplete'
+                && (file.reason === 'adapter-cannot-prove-completeness'
+                    || file.reason === 'collection-failed' || file.reason === 'run-incomplete')))
+            && Array.isArray(file.tasks)
+            && file.tasks.every(task => isRecord(task) && typeof task.task_key === 'string'
+                && (task.state === 'scored' || (task.state === 'not-scored'
+                    && (task.reason === 'skipped' || task.reason === 'pending'
+                        || task.reason === 'timed-out' || task.reason === 'failed-before-evaluation')))));
+}
 function optionalString(value) {
     return value === undefined || typeof value === 'string';
 }

package/dist/reporting/source-metadata.d.ts

@@ -1,5 +1,8 @@
+import type { ComparisonInputsState } from './reliability-contract.js';
 export interface SourceMetadata {
     sourceFile?: string;
     sourceRevision?: string;
+    comparisonInputs?: ComparisonInputsState;
 }
 export declare function resolveSourceMetadata(filePath: string | undefined, cwd: string): SourceMetadata;
+export declare function createSourceMetadataResolver(cwd: string): (filePath: string | undefined) => SourceMetadata;

package/dist/reporting/source-metadata.js

@@ -1,7 +1,18 @@
 import { createHash } from 'node:crypto';
 import { readFileSync } from 'node:fs';
+import { readdirSync } from 'node:fs';
 import * as path from 'node:path';
+import picomatch from 'picomatch';
+import { canonicalizeJson } from '../core/canonical-json.js';
+import { parsePathgradeMeta } from '../affected/meta.js';
 export function resolveSourceMetadata(filePath, cwd) {
+    return resolveSourceMetadataWithFiles(filePath, cwd, () => listRepositoryFiles(cwd));
+}
+export function createSourceMetadataResolver(cwd) {
+    let repositoryFiles;
+    return filePath => resolveSourceMetadataWithFiles(filePath, cwd, () => repositoryFiles ??= listRepositoryFiles(cwd));
+}
+function resolveSourceMetadataWithFiles(filePath, cwd, repositoryFiles) {
     if (!filePath)
         return {};
     const absolute = path.resolve(cwd, filePath);
@@ -11,10 +22,64 @@ export function resolveSourceMetadata(filePath, cwd) {
     try {
         return {
             sourceFile: relative,
-            sourceRevision: `sha256:${createHash('sha256').update(readFileSync(absolute)).digest('hex')}`,
+            sourceRevision: digest(readFileSync(absolute)),
+            comparisonInputs: resolveComparisonInputs(absolute, cwd, repositoryFiles),
+        };
+    }
+    catch {
+        return { sourceFile: relative, comparisonInputs: { state: 'invalid', reason: 'unreadable-input' } };
+    }
+}
+function resolveComparisonInputs(evalFile, cwd, repositoryFiles) {
+    let declarations;
+    try {
+        declarations = parsePathgradeMeta(evalFile)?.comparisonInputs;
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : '';
+        return {
+            state: 'invalid',
+            reason: message.includes('escapes the repo root') ? 'outside-repository' : 'malformed-declaration',
+        };
+    }
+    if (declarations === undefined)
+        return { state: 'missing' };
+    const normalized = declarations.map(value => value.replaceAll('\\', '/').normalize('NFC'));
+    const matched = [...new Set(normalized.flatMap(declaration => {
+            const matches = picomatch(declaration, { dot: true });
+            return repositoryFiles().filter(file => matches(file));
+        }))].toSorted();
+    if (normalized.length > 0 && matched.length === 0)
+        return { state: 'invalid', reason: 'no-matches' };
+    try {
+        const files = matched.map(file => ({ path: file, revision: digest(readFileSync(path.join(cwd, file))) }));
+        return {
+            state: 'resolved',
+            declarations: normalized,
+            files,
+            revision: digest(canonicalizeJson({ declarations: normalized, files })),
         };
     }
     catch {
-        return { sourceFile: relative };
+        return { state: 'invalid', reason: 'unreadable-input' };
     }
 }
+function listRepositoryFiles(cwd) {
+    const files = [];
+    const walk = (directory, relative) => {
+        for (const entry of readdirSync(directory, { withFileTypes: true })) {
+            const childRelative = relative ? `${relative}/${entry.name}` : entry.name;
+            if (!relative && (entry.name === '.git' || entry.name === '.pathgrade'))
+                continue;
+            if (entry.isDirectory())
+                walk(path.join(directory, entry.name), childRelative);
+            else if (entry.isFile())
+                files.push(childRelative);
+        }
+    };
+    walk(path.resolve(cwd), '');
+    return files.toSorted();
+}
+function digest(value) {
+    return `sha256:${createHash('sha256').update(value).digest('hex')}`;
+}

package/dist/reporting/types.d.ts

@@ -1,19 +1,23 @@
 import type { DiagnosticsReport } from '../sdk/diagnostics.js';
 import type { AgentExecutionMetadata, EvaluationResultKind } from '../sdk/types.js';
-import type { AttemptOutcome } from '../runners/model.js';
+import type { AttemptOutcome, RunStatus } from '../runners/model.js';
 import type { PathgradeReport, PathgradeSelectionReport, TrialResult } from '../types.js';
+import type { ComparisonInputsState, TaskInventory } from './reliability-contract.js';
 export type ReportCaseState = 'passed' | 'failed' | 'skipped' | 'pending';
 export interface ReportRunInput {
+    runStatus?: RunStatus;
     threshold?: number;
     selection?: PathgradeSelectionReport;
     attemptsRequested?: number;
     attemptsCompleted?: number;
+    taskInventory?: TaskInventory;
     groups: ReportGroupInput[];
 }
 export interface ReportGroupInput {
     groupName: string;
     sourceFile?: string;
     sourceRevision?: string;
+    comparisonInputs?: ComparisonInputsState;
     cases: ReportCaseInput[];
 }
 export interface ReportCaseInput {

package/dist/runners/model-builders.d.ts

@@ -3,4 +3,5 @@ import type { AdapterRunHandle } from './adapter.js';
 import type { NormalizedRunSnapshot } from './model.js';
 export declare function buildNormalizedRunSnapshotFromReportGroups(run: AdapterRunHandle, groups: ReportGroupInput[], options?: {
     cwd?: string;
+    discoveredFiles?: readonly string[];
 }): NormalizedRunSnapshot;