@wix/pathgrade 1.0.39 → 1.0.41

package/README.md

@@ -178,7 +178,7 @@ Use `check()` for binary requirements, `score()` for partial credit, `judge()` f
 check('tests-pass', async ({ runCommand }) => {
     const { exitCode } = await runCommand('npm test');
     return exitCode === 0;
-});
+}, { revision: { contract: 1 } });
 ```
 
 ### `score()` - Partial credit
@@ -187,7 +187,7 @@ check('tests-pass', async ({ runCommand }) => {
 score('coverage', async ({ runCommand }) => {
     const { stdout } = await runCommand('npx coverage-summary');
     return parseFloat(stdout) / 100;
-});
+}, { revision: { contract: 1 } });
 ```
 
 ### `judge()` - Rubric evaluation
@@ -199,6 +199,7 @@ Was the fix minimal and correct? (0-0.5)`,
 });
 ```
 
+Function-bearing `check()` and `score()` scorers require explicit canonical-JSON `revision` data for authoritative baseline comparisons. A `judge()` with a function-valued `input` requires the same declaration. Imported scorer helpers belong in `comparisonInputs`; closed-over values belong in `revision`.
 Judge scorers also support:
 
 - `retry` for transient judge failures
@@ -320,7 +321,7 @@ Pathgrade exposes a few useful features that are easy to miss from the basic exa
 - `createAgent({ skillDir, workspace })` stages a real skill and a fixture workspace into the sandbox, which is how Pathgrade's skill examples are evaluated.
 - `createAgent({ debug: true })` preserves the final workspace under the backward-compatible `pathgrade-debug/<test-name>/` path; when you use `runConversation()`, it also writes `run-snapshot.json`.
 - `createAgent({ debug: { retainRuns: 5 } })` opts into managed run retention under `pathgrade-debug/runs/<run-id>/<test-name>/`. Use `pathgrade clean --debug`, `--keep=N`, and `--dry-run` to clean marked, inactive debug runs safely.
-- `evaluate.fromSnapshot(snapshotPath, scorers)` re-runs grading against a saved snapshot without re-running the agent.
+- `evaluate.fromSnapshot(snapshotPath, scorers)` re-runs grading against a saved snapshot without re-running the agent. For trusted live deterministic scoring, `evaluate(agent, scorers, { deterministicToolEvidence: 'live' })` exposes exact transient MCP arguments only through `ctx.toolEvents`; `agent.log` stays sanitized, and redaction-blocked argument predicates fail closed.
 - `previewReactions(messages, reactions)` lets you inspect which scripted reactions would fire offline.
 - `conversationWindow` on agents and personas keeps long transcripts bounded with summarization instead of sending the full conversation every turn.
 - `copyIgnore` and `DEFAULT_COPY_IGNORE` let you control what gets copied into the sandbox when seeding from large fixtures or skill directories.
@@ -440,9 +441,8 @@ Useful details:
 - `pathgrade preview browser` starts a local viewer on `http://localhost:3847`.
 - `pathgrade report` posts or updates a PR comment in GitHub Actions; locally it prints the markdown report and then the numeric pass rate. Provider orchestrators can add a `--details-url`, suppress stale updates with `--expected-head-sha`, and opt into surfaced API failures with `--strict`.
 - `pathgrade validate --affected` is a strict mode for CI: every discovered eval must either live under a `SKILL.md` anchor or export valid `__pathgradeMeta`.
-
+Declare methodology files separately from affected-selection dependencies with `__pathgradeMeta.comparisonInputs`, for example `comparisonInputs: ['evals/fixtures/**', 'evals/scorers/**']`. An empty array explicitly declares a self-contained eval. Pathgrade hashes the raw eval source plus each declared glob, matched repository-relative path, and raw file bytes; `.git` internals and `.pathgrade` outputs are excluded. Missing, invalid, unmatched, escaping, or unreadable inputs suppress numeric deltas. `deps`, `extraDeps`, inferred skill roots, and global triggers affect selection only and never definition identity.
 Run `pathgrade --help` for the full help text.
-
 ## Configuration
 
 ```typescript
@@ -461,7 +461,7 @@ export default {
 
 Pathgrade reads `pathgrade.config.*` for CLI and affected-selection behavior. `runner.adapter` and `--adapter=<name|path>` select the runner; `--adapter` wins over config. `attempts` defaults to `1` and must be a positive integer. Built-in adapters `vitest`, `jest`, and `node-test` support repeated attempts; third-party invocation adapters must advertise `supportsRepeatedAttempts: true` and produce the normalized child snapshot contract.
 
-New reports use schema version 2. They preserve every attempt with `case_id`, `attempt_id`, and `attempt_index`; include the display-safe `runner_outcome` that determined binary success; publish `mean_reward` for partial scores; and publish finite-sample pass@k only for complete binary attempts. `runner_status` and `threshold_status` expose the two gates independently, and canonical `status` passes only when runner assertions pass and any configured threshold passes. Runner assertions, runner diagnostics, and native runner references remain in the normalized run model and are not copied into public report artifacts; evaluation diagnostics retain their existing report behavior. A partial reward or incomplete attempt makes pass@k explicitly unavailable. Version-1 and older version-2 reports remain readable with missing gate fields derived during loading, but Pathgrade no longer computes pass@k or pass^k by pooling heterogeneous cases.
+New reports use schema version 3. They add a versioned task inventory with every selected eval file, collection completeness, stable task keys, and bounded non-scoring reasons. They also preserve every attempt with `case_id`, `attempt_id`, and `attempt_index`; include the display-safe `runner_outcome`; publish `mean_reward`; and publish finite-sample pass@k only for complete binary attempts. `runner_status` and `threshold_status` expose the two gates independently, and canonical `status` passes only when runner assertions pass and any configured threshold passes. Runner assertions, runner diagnostics, native runner references, and full flow traces remain transient or in trace artifacts rather than being copied into consolidated public reports. A partial reward or incomplete attempt makes pass@k explicitly unavailable. Report versions 1 and 2 remain readable, but they cannot be promoted as authoritative task-accounted baselines and must be rerun.
 
 Third-party runner adapters are supported through `@wix/pathgrade/adapter-kit`. Adapter names resolve as follows:
 

package/dist/adapters/jest/results.d.ts

@@ -23,6 +23,7 @@ export declare function normalizeJestRunResults(input: {
     run: AdapterRunHandle;
     results: JestAggregatedResult;
     metadataByCaseId?: Map<string, PathgradeTestMeta[]>;
+    discoveredFiles?: readonly string[];
     cwd?: string;
 }): NormalizedRunSnapshot;
 export declare function jestCaseId(input: {

package/dist/adapters/jest/results.js

@@ -1,20 +1,39 @@
-import { resolveSourceMetadata } from '../../reporting/source-metadata.js';
+import { createSourceMetadataResolver } from '../../reporting/source-metadata.js';
 export function normalizeJestRunResults(input) {
     const runId = `${input.run.adapterName}:run`;
+    const cwd = input.cwd ?? process.cwd();
+    const resolveSourceMetadata = createSourceMetadataResolver(cwd);
     const files = input.results.testResults ?? [];
-    const units = files.map((fileResult, index) => ({
-        id: unitId(index),
-        runId,
-        displayName: fileResult.testFilePath,
-        ...resolveSourceMetadata(fileResult.testFilePath, input.cwd ?? process.cwd()),
-        groupingHints: [{
-                kind: 'source',
-                key: fileResult.testFilePath,
-                label: fileResult.testFilePath,
-                order: index,
-            }],
-        nativeReferences: [{ kind: 'jest-file', id: fileResult.testFilePath }],
-    }));
+    const collection = input.run.status === 'completed'
+        ? { state: 'complete' }
+        : { state: 'incomplete', reason: 'run-incomplete' };
+    const units = [
+        ...files.map((fileResult, index) => ({
+            id: unitId(index),
+            runId,
+            displayName: fileResult.testFilePath,
+            collection,
+            ...resolveSourceMetadata(fileResult.testFilePath),
+            groupingHints: [{
+                    kind: 'source',
+                    key: fileResult.testFilePath,
+                    label: fileResult.testFilePath,
+                    order: index,
+                }],
+            nativeReferences: [{ kind: 'jest-file', id: fileResult.testFilePath }],
+        })),
+        ...(input.discoveredFiles ?? [])
+            .filter(file => !files.some(result => normalizePath(result.testFilePath, resolveSourceMetadata) === file))
+            .map((file, index) => ({
+            id: unitId(files.length + index),
+            runId,
+            displayName: file,
+            collection,
+            ...resolveSourceMetadata(file),
+            groupingHints: [{ kind: 'source', key: file, label: file, order: files.length + index }],
+            nativeReferences: [{ kind: 'jest-file', id: file }],
+        })),
+    ];
     const occurrenceCounts = new Map();
     return {
         version: 1,
@@ -45,6 +64,9 @@ export function normalizeJestRunResults(input) {
         },
     };
 }
+function normalizePath(file, resolveSourceMetadata) {
+    return resolveSourceMetadata(file).sourceFile ?? file.replaceAll('\\', '/');
+}
 function assertionsForFile(fileResult) {
     return fileResult.assertionResults ?? fileResult.testResults ?? [];
 }

package/dist/adapters/jest/runner-adapter.d.ts

@@ -3,6 +3,7 @@ import { type PathgradeTestMeta, RunnerAdapter } from '@wix/pathgrade/adapter-ki
 interface JestRunNative {
     results?: JestAggregatedResult;
     metadataByCaseId?: Map<string, PathgradeTestMeta[]>;
+    discoveredFiles?: string[];
 }
 export declare function createJestAdapter(options?: JestRunNative): RunnerAdapter;
 export declare function createPathgradeAdapter(): RunnerAdapter;

package/dist/adapters/jest/runner-adapter.js

@@ -27,7 +27,10 @@ export function createJestAdapter(options = {}) {
                 adapterName: this.name,
                 status: input.signal?.aborted ? 'cancelled' : 'completed',
                 exitCode: input.signal?.aborted ? 1 : 0,
-                native: options,
+                native: {
+                    ...options,
+                    discoveredFiles: input.discovered.units.flatMap(unit => unit.sourceRef ? [unit.sourceRef] : []),
+                },
             };
         },
         async collectNormalizedRunSnapshot(run) {
@@ -36,6 +39,7 @@ export function createJestAdapter(options = {}) {
                 run,
                 results: native.results ?? { testResults: [] },
                 metadataByCaseId: native.metadataByCaseId,
+                discoveredFiles: native.discoveredFiles,
                 cwd: discoveryCwd,
             });
         },

package/dist/adapters/node-test/runner-adapter.js

@@ -37,7 +37,7 @@ export function createNodeTestAdapter() {
                     adapterName: this.name,
                     status: 'completed',
                     exitCode: 0,
-                    native: { resultsPath, resultsDir, cwd },
+                    native: { resultsPath, resultsDir, cwd, discoveredFiles: files },
                 };
             }
             const exitCode = await spawnNodeTest({
@@ -54,7 +54,7 @@ export function createNodeTestAdapter() {
                 adapterName: this.name,
                 status: input.signal?.aborted ? 'cancelled' : (exitCode === 0 ? 'completed' : 'failed'),
                 exitCode,
-                native: { resultsPath, resultsDir, cwd },
+                native: { resultsPath, resultsDir, cwd, discoveredFiles: files },
             };
         },
         async collectNormalizedRunSnapshot(run) {
@@ -71,7 +71,7 @@ export function createNodeTestAdapter() {
                 return buildNormalizedRunSnapshotFromReportGroups(run, Array.from(groupMap.entries()).map(([groupName, groupedCases]) => ({
                     groupName,
                     cases: groupedCases,
-                })), { cwd: native.cwd });
+                })), { cwd: native.cwd, discoveredFiles: native.discoveredFiles });
             }
             finally {
                 if (native.resultsDir)
@@ -104,10 +104,11 @@ function readNodeTestRunNative(run) {
         && run.native !== null
         && typeof run.native.resultsPath === 'string'
         && typeof run.native.resultsDir === 'string'
-        && typeof run.native.cwd === 'string') {
+        && typeof run.native.cwd === 'string'
+        && Array.isArray(run.native.discoveredFiles)) {
         return run.native;
     }
-    return { resultsPath: '', resultsDir: '', cwd: process.cwd() };
+    return { resultsPath: '', resultsDir: '', cwd: process.cwd(), discoveredFiles: [] };
 }
 async function readCases(resultsPath) {
     if (!resultsPath || !(await fs.pathExists(resultsPath)))

package/dist/affected/meta.d.ts

@@ -17,10 +17,13 @@
 export interface ParsedMeta {
     deps?: string[];
     extraDeps?: string[];
+    comparisonInputs?: string[];
     alwaysRun?: boolean;
 }
 /**
  * Parse `__pathgradeMeta` from an eval file's AST. Returns `null` if the
  * export is not present.
  */
-export declare function parsePathgradeMeta(evalFile: string): ParsedMeta | null;
+export declare function parsePathgradeMeta(evalFile: string, options?: {
+    validateComparisonInputs?: boolean;
+}): ParsedMeta | null;

package/dist/affected/meta.js

@@ -21,14 +21,14 @@ import picomatch from 'picomatch';
  * Parse `__pathgradeMeta` from an eval file's AST. Returns `null` if the
  * export is not present.
  */
-export function parsePathgradeMeta(evalFile) {
+export function parsePathgradeMeta(evalFile, options = {}) {
     const source = fs.readFileSync(evalFile, 'utf-8');
     const sourceFile = ts.createSourceFile(evalFile, source, ts.ScriptTarget.Latest, 
     /* setParentNodes */ true, ts.ScriptKind.TS);
     for (const stmt of sourceFile.statements) {
         const initializer = findMetaInitializer(stmt);
         if (initializer) {
-            return extractMeta(initializer, evalFile);
+            return extractMeta(initializer, evalFile, options);
         }
     }
     return null;
@@ -50,7 +50,7 @@ function findMetaInitializer(stmt) {
     }
     return null;
 }
-function extractMeta(expr, evalFile) {
+function extractMeta(expr, evalFile, options) {
     // Optionally unwrap a type assertion: `{...} as PathgradeMeta` / `<PathgradeMeta>{...}`.
     let node = expr;
     while (ts.isAsExpression(node) || ts.isTypeAssertionExpression(node) || ts.isSatisfiesExpression(node)) {
@@ -64,7 +64,9 @@ function extractMeta(expr, evalFile) {
         if (!ts.isPropertyAssignment(prop))
             continue;
         const name = propertyKeyName(prop.name);
-        if (name === 'deps' || name === 'extraDeps') {
+        if (name === 'deps' || name === 'extraDeps' || name === 'comparisonInputs') {
+            if (name === 'comparisonInputs' && options.validateComparisonInputs === false)
+                continue;
             const globs = extractStringArray(prop.initializer, evalFile, name);
             for (const g of globs)
                 validateGlob(g, evalFile, name);
@@ -89,7 +91,9 @@ function validateGlob(glob, evalFile, field) {
     if (glob.length === 0) {
         throw new Error(`${evalFile}: __pathgradeMeta.${field} contains an empty glob.`);
     }
-    if (glob.startsWith('../') || glob.includes('/../') || glob === '..') {
+    const normalized = glob.replaceAll('\\', '/');
+    if (normalized.startsWith('/') || /^[A-Za-z]:\//.test(normalized)
+        || normalized.startsWith('../') || normalized.includes('/../') || normalized === '..') {
         throw new Error(`${evalFile}: __pathgradeMeta.${field} entry "${glob}" escapes the repo root — ` +
             `globs must be repo-root-relative.`);
     }

package/dist/affected/select.js

@@ -44,7 +44,7 @@ export function selectAffected(input) {
     for (const evalFile of evalFiles) {
         const absEval = path.resolve(repoRoot, evalFile);
         const skillRoot = findSkillRoot(absEval, repoRoot);
-        const meta = parsePathgradeMeta(absEval);
+        const meta = parsePathgradeMeta(absEval, { validateComparisonInputs: false });
         // Precedence #2: alwaysRun wins over dep matching.
         if (meta?.alwaysRun === true) {
             selected.push({ file: evalFile, reason: 'always-run' });

package/dist/commands/report.js

@@ -44,9 +44,8 @@ async function loadReport(resolvedPath) {
     if (!(await fs.pathExists(resolvedPath))) {
         throw new Error(`results file not found at ${resolvedPath}`);
     }
-    const raw = await fs.readJSON(resolvedPath);
     try {
-        return parsePathgradeReport(raw);
+        return parsePathgradeReport(await fs.readJSON(resolvedPath));
     }
     catch {
         throw new Error(`results file at ${resolvedPath} is not a valid pathgrade report`);

package/dist/reporting/comparison-contract.d.ts

@@ -1,10 +1,11 @@
-import type { ComparisonContract, EvalReport } from '../types.js';
+import type { EvalReport } from '../types.js';
 import type { Scorer } from '../sdk/types.js';
 import type { ReportGroupInput } from './types.js';
+import type { ComparisonContractV2 } from './reliability-contract.js';
 export declare function buildComparisonContract(input: {
     group: ReportGroupInput;
     report: EvalReport;
     attemptsRequested: number;
     attemptsCompleted: number;
-}): ComparisonContract;
+}): ComparisonContractV2;
 export declare function createScorerRevision(scorers: readonly Scorer[]): string | undefined;

package/dist/reporting/comparison-contract.js

@@ -1,7 +1,6 @@
 import { createHash } from 'node:crypto';
 import { canonicalizeJson } from '../core/canonical-json.js';
 export function buildComparisonContract(input) {
-    const unavailable = new Set();
     const evaluations = input.group.cases.flatMap(testCase => {
         const caseIdentity = testCase.repeatKey ?? testCase.caseId ?? testCase.name;
         return (testCase.attempts?.flatMap(attempt => attempt.evaluations ?? [])
@@ -15,16 +14,16 @@ export function buildComparisonContract(input) {
             runtime: runtimeIdentity(evaluation),
         }));
     });
-    const definitionRevision = input.group.sourceRevision
+    const comparisonInputs = input.group.comparisonInputs ?? { state: 'missing' };
+    const definitionRevision = input.group.sourceRevision && comparisonInputs.state === 'resolved'
         ? revision('definition', {
             source: input.group.sourceRevision,
+            comparison_inputs: comparisonInputs.revision,
             evaluations: uniqueSorted(evaluations.map(evaluation => ({
                 case: evaluation.case, definition: evaluation.definition,
             }))),
         })
         : undefined;
-    if (!definitionRevision)
-        unavailable.add('definition_metadata_missing');
     const scorerRevision = evaluations.length > 0 && evaluations.every(evaluation => evaluation.scorer)
         ? revision('scorers', uniqueSorted(evaluations.map(evaluation => ({
             case: evaluation.case,
@@ -32,20 +31,10 @@ export function buildComparisonContract(input) {
             scorer: evaluation.scorer,
         }))))
         : undefined;
-    if (!scorerRevision)
-        unavailable.add('scorer_metadata_missing');
     const runtimeComplete = evaluations.length > 0
-        && evaluations.every(evaluation => Object.values(evaluation.runtime)
-            .every(value => value !== undefined));
-    const runtimeRevision = runtimeComplete
-        ? revision('runtime', uniqueSorted(evaluations.map(evaluation => ({
-            case: evaluation.case,
-            definition: evaluation.definition,
-            runtime: evaluation.runtime,
-        }))))
-        : undefined;
-    if (!runtimeRevision)
-        unavailable.add('runtime_metadata_missing');
+        && evaluations.every(evaluation => runtimeIdentityComplete(evaluation.runtime));
+    const runtimeComponents = runtimeComplete ? runtimeComponentRevisions(evaluations) : undefined;
+    const runtimeRevision = runtimeComponents ? revision('runtime', runtimeComponents) : undefined;
     const reportableCases = input.group.cases.filter(testCase => testCase.reportable !== false
         && testCase.state !== 'skipped' && testCase.state !== 'pending');
     const samplingComplete = input.attemptsCompleted === input.attemptsRequested
@@ -59,16 +48,41 @@ export function buildComparisonContract(input) {
             cases: reportableCases.map(testCase => testCase.repeatKey ?? testCase.caseId ?? testCase.name).toSorted(),
         })
         : undefined;
-    if (!samplingRevision)
-        unavailable.add('sampling_incomplete');
     return {
-        version: 1,
-        ...(definitionRevision ? { definition_revision: definitionRevision } : {}),
-        ...(scorerRevision ? { scorer_revision: scorerRevision } : {}),
-        ...(runtimeRevision ? { runtime_revision: runtimeRevision } : {}),
-        ...(samplingRevision ? { sampling_revision: samplingRevision } : {}),
-        report_schema_revision: 'pathgrade-results-v2',
-        ...(unavailable.size > 0 ? { unavailable_reasons: [...unavailable].sort() } : {}),
+        version: 2,
+        ...(definitionRevision
+            ? { comparison_inputs: comparisonInputs, definition_revision: definitionRevision }
+            : { comparison_inputs: comparisonInputs }),
+        ...(scorerRevision
+            ? { scorer_revision_state: { state: 'complete' }, scorer_revision: scorerRevision }
+            : { scorer_revision_state: { state: 'incomplete', reason: 'explicit-revision-required' } }),
+        ...(runtimeRevision && runtimeComponents
+            ? {
+                runtime_revision_state: { state: 'complete' },
+                runtime_revision: runtimeRevision,
+                runtime_components: runtimeComponents,
+            }
+            : { runtime_revision_state: { state: 'incomplete', reason: 'metadata-missing' } }),
+        ...(samplingRevision
+            ? { sampling_revision_state: { state: 'complete' }, sampling_revision: samplingRevision }
+            : { sampling_revision_state: { state: 'incomplete', reason: 'sampling-incomplete' } }),
+        report_schema_revision: 'pathgrade-results-v3',
+    };
+}
+function runtimeComponentRevisions(evaluations) {
+    const component = (name) => revision(`runtime-${name}`, uniqueSorted(evaluations.map(evaluation => ({
+        case: evaluation.case,
+        definition: evaluation.definition,
+        value: evaluation.runtime[name] ?? null,
+    }))));
+    return {
+        agent_name: component('agent_name'),
+        model: component('model'),
+        transport: component('transport'),
+        interaction_mode: component('interaction_mode'),
+        ...(evaluations.some(evaluation => evaluation.runtime.flow)
+            ? { flow: component('flow') }
+            : {}),
     };
 }
 function hasTerminalEvaluation(evaluations) {
@@ -85,16 +99,20 @@ export function createScorerRevision(scorers) {
 }
 function runtimeIdentity(evaluation) {
     const agent = evaluation.agent ?? evaluation.trial?.agent;
-    if (agent) {
+    if (agent)
         return {
-            name: agent.name,
+            agent_name: agent.name,
             model: agent.resolvedModel,
             transport: agent.transport,
-            interaction: agent.interactionMode,
+            interaction_mode: agent.interactionMode,
+            flow: undefined,
         };
-    }
     const flow = evaluation.trial?.flow_trace;
     return {
+        agent_name: undefined,
+        model: undefined,
+        transport: undefined,
+        interaction_mode: undefined,
         flow: flow?.completeness.runtimeIdentity === 'complete'
             && flow.participants.every(participant => participant.runtime) ? {
             ...(flow.protocol ? { protocol: flow.protocol } : {}),
@@ -105,10 +123,22 @@ function runtimeIdentity(evaluation) {
         } : undefined,
     };
 }
+function runtimeIdentityComplete(identity) {
+    return identity.flow !== undefined || [
+        identity.agent_name, identity.model, identity.transport, identity.interaction_mode,
+    ].every(value => value !== undefined);
+}
 function scorerDeclaration(scorer) {
-    const common = { type: scorer.type, name: scorer.name.normalize('NFC'), weight: scorer.weight };
+    const common = {
+        type: scorer.type,
+        name: scorer.name.normalize('NFC'),
+        weight: scorer.weight,
+        ...(scorer.revision === undefined ? {} : { revision: scorer.revision }),
+    };
     if (scorer.type === 'check' || scorer.type === 'score') {
-        return { ...common, function: scorer.fn.toString() };
+        if (scorer.revision === undefined)
+            throw new Error('explicit scorer revision required');
+        return common;
     }
     if (scorer.type === 'tool_usage') {
         return {
@@ -117,13 +147,16 @@ function scorerDeclaration(scorer) {
                 .toSorted((left, right) => canonicalizeJson(left).localeCompare(canonicalizeJson(right))),
         };
     }
+    if (typeof scorer.input === 'function' && scorer.revision === undefined) {
+        throw new Error('explicit scorer revision required');
+    }
     return {
         ...common,
         rubric: scorer.rubric,
         model: scorer.model ?? null,
         retry: scorer.retry ?? null,
         includeToolEvents: scorer.includeToolEvents ?? null,
-        input: typeof scorer.input === 'function' ? scorer.input.toString() : scorer.input ?? null,
+        input: typeof scorer.input === 'function' ? null : scorer.input ?? null,
         tools: scorer.tools?.toSorted() ?? null,
         maxRounds: scorer.maxRounds ?? null,
         cacheControl: scorer.cacheControl ?? null,

package/dist/reporting/core.js

@@ -1,6 +1,7 @@
 import { buildDiagnosticsReport } from '../sdk/diagnostics.js';
 import { extractSkillsFromLog } from '../tool-events.js';
 import { buildComparisonContract } from './comparison-contract.js';
+import { TASK_INVENTORY_VERSION } from './reliability-contract.js';
 export function buildPathgradeReport(input) {
     const warnings = [];
     const consolidatedGroups = [];
@@ -60,7 +61,7 @@ export function buildPathgradeReport(input) {
         : overallMeanReward >= input.threshold ? 'pass' : 'fail';
     return {
         report: {
-            version: 2,
+            version: 3,
             timestamp: new Date().toISOString(),
             ...(input.threshold != null ? { threshold: input.threshold } : {}),
             attempts_requested: attemptsRequested,
@@ -71,6 +72,15 @@ export function buildPathgradeReport(input) {
             threshold_status: thresholdStatus,
             status: runnerStatus === 'fail' || thresholdStatus === 'fail' ? 'fail' : 'pass',
             groups: consolidatedGroups,
+            task_inventory: input.taskInventory ?? {
+                version: TASK_INVENTORY_VERSION,
+                files: input.groups.flatMap(group => group.sourceFile ? [{
+                        eval_file: group.sourceFile,
+                        completeness: 'incomplete',
+                        reason: 'adapter-cannot-prove-completeness',
+                        tasks: [taskInventoryEntry(group)],
+                    }] : []),
+            },
             ...(input.selection ? { selection: input.selection } : {}),
         },
         traces,
@@ -78,6 +88,13 @@ export function buildPathgradeReport(input) {
         warnings,
     };
 }
+function taskInventoryEntry(group) {
+    const scored = group.cases.some(testCase => testCase.state !== 'skipped' && testCase.state !== 'pending');
+    if (scored)
+        return { task_key: group.groupName, state: 'scored' };
+    const reason = group.cases.every(testCase => testCase.state === 'skipped') ? 'skipped' : 'pending';
+    return { task_key: group.groupName, state: 'not-scored', reason };
+}
 function reportableBuiltCase(testCase) {
     if (testCase.reportable !== undefined)
         return testCase.reportable;

package/dist/reporting/reliability-contract.d.ts

@@ -53,6 +53,7 @@ export interface RuntimeComponentRevisions {
     model: string;
     transport: string;
     interaction_mode: string;
+    flow?: string;
 }
 type DefinitionRevisionContract = {
     comparison_inputs: Extract<ComparisonInputsState, {

package/dist/reporting/report-parser.js

@@ -1,9 +1,9 @@
 export function parsePathgradeReport(value) {
     if (!isRecord(value)
-        || (value.version !== 1 && value.version !== 2)
+        || (value.version !== 1 && value.version !== 2 && value.version !== 3)
         || typeof value.timestamp !== 'string'
         || typeof value.overall_pass_rate !== 'number'
-        || (value.version === 2 && (typeof value.overall_mean_reward !== 'number' || !validAttemptCounts(value)))
+        || (value.version !== 1 && (typeof value.overall_mean_reward !== 'number' || !validAttemptCounts(value)))
         || !optionalNumber(value.threshold)
         || (value.status !== 'pass' && value.status !== 'fail')
         || !optionalGateStatus(value.runner_status)
@@ -11,6 +11,7 @@ export function parsePathgradeReport(value) {
         || (value.run_kind !== undefined && value.run_kind !== 'evaluation' && value.run_kind !== 'no-affected')
         || !Array.isArray(value.groups)
         || !value.groups.every(group => isReportGroup(group, value.version))
+        || (value.version === 3 && !isTaskInventory(value.task_inventory))
         || !isSelection(value.selection)) {
         throw new Error('PathGrade results.json is missing or has an unsupported schema');
     }
@@ -50,7 +51,9 @@ function isReportGroup(value, version) {
         && value.skills_used.every(item => typeof item === 'string')
         && Array.isArray(value.trials)
         && value.trials.every(isTrial)
-        && (value.comparison_contract === undefined || isComparisonContract(value.comparison_contract));
+        && (version === 3
+            ? isComparisonContractV2(value.comparison_contract)
+            : value.comparison_contract === undefined || isComparisonContract(value.comparison_contract));
 }
 function deriveRunnerStatus(value) {
     if (value.runner_status === 'fail')
@@ -137,6 +140,9 @@ function isSelection(value) {
             && item.reason === 'no-matching-deps'));
 }
 function isComparisonContract(value) {
+    return isComparisonContractV1(value) || isComparisonContractV2(value);
+}
+function isComparisonContractV1(value) {
     return isRecord(value)
         && value.version === 1
         && optionalString(value.definition_revision)
@@ -150,6 +156,65 @@ function isComparisonContract(value) {
                 || reason === 'runtime_metadata_missing'
                 || reason === 'sampling_incomplete')));
 }
+function isComparisonContractV2(value) {
+    if (!isRecord(value) || value.version !== 2 || value.report_schema_revision !== 'pathgrade-results-v3')
+        return false;
+    const comparisonInputs = value.comparison_inputs;
+    const definitionComplete = isRecord(comparisonInputs) && comparisonInputs.state === 'resolved';
+    const scorerComplete = isState(value.scorer_revision_state, 'complete');
+    const runtimeComplete = isState(value.runtime_revision_state, 'complete');
+    const samplingComplete = isState(value.sampling_revision_state, 'complete');
+    return isComparisonInputs(comparisonInputs)
+        && (definitionComplete ? typeof value.definition_revision === 'string' : value.definition_revision === undefined)
+        && isRevisionState(value.scorer_revision_state, ['explicit-revision-required', 'metadata-missing'])
+        && (scorerComplete ? typeof value.scorer_revision === 'string' : value.scorer_revision === undefined)
+        && isRevisionState(value.runtime_revision_state, ['metadata-missing'])
+        && (runtimeComplete
+            ? typeof value.runtime_revision === 'string' && isRuntimeComponents(value.runtime_components)
+            : value.runtime_revision === undefined && value.runtime_components === undefined)
+        && isRevisionState(value.sampling_revision_state, ['sampling-incomplete'])
+        && (samplingComplete ? typeof value.sampling_revision === 'string' : value.sampling_revision === undefined);
+}
+function isComparisonInputs(value) {
+    if (!isRecord(value))
+        return false;
+    if (value.state === 'missing')
+        return true;
+    if (value.state === 'invalid') {
+        return value.reason === 'malformed-declaration' || value.reason === 'no-matches'
+            || value.reason === 'outside-repository' || value.reason === 'unreadable-input';
+    }
+    return value.state === 'resolved'
+        && Array.isArray(value.declarations) && value.declarations.every(item => typeof item === 'string')
+        && Array.isArray(value.files) && value.files.every(file => isRecord(file)
+        && typeof file.path === 'string' && typeof file.revision === 'string')
+        && typeof value.revision === 'string';
+}
+function isRevisionState(value, reasons) {
+    return isState(value, 'complete') || (isRecord(value) && value.state === 'incomplete'
+        && typeof value.reason === 'string' && reasons.includes(value.reason));
+}
+function isState(value, state) {
+    return isRecord(value) && value.state === state;
+}
+function isRuntimeComponents(value) {
+    return isRecord(value) && ['agent_name', 'model', 'transport', 'interaction_mode']
+        .every(key => typeof value[key] === 'string')
+        && optionalString(value.flow);
+}
+function isTaskInventory(value) {
+    return isRecord(value) && value.version === 1 && Array.isArray(value.files)
+        && value.files.every(file => isRecord(file)
+            && typeof file.eval_file === 'string'
+            && (file.completeness === 'complete' || (file.completeness === 'incomplete'
+                && (file.reason === 'adapter-cannot-prove-completeness'
+                    || file.reason === 'collection-failed' || file.reason === 'run-incomplete')))
+            && Array.isArray(file.tasks)
+            && file.tasks.every(task => isRecord(task) && typeof task.task_key === 'string'
+                && (task.state === 'scored' || (task.state === 'not-scored'
+                    && (task.reason === 'skipped' || task.reason === 'pending'
+                        || task.reason === 'timed-out' || task.reason === 'failed-before-evaluation')))));
+}
 function optionalString(value) {
     return value === undefined || typeof value === 'string';
 }

package/dist/reporting/source-metadata.d.ts

@@ -1,5 +1,8 @@
+import type { ComparisonInputsState } from './reliability-contract.js';
 export interface SourceMetadata {
     sourceFile?: string;
     sourceRevision?: string;
+    comparisonInputs?: ComparisonInputsState;
 }
 export declare function resolveSourceMetadata(filePath: string | undefined, cwd: string): SourceMetadata;
+export declare function createSourceMetadataResolver(cwd: string): (filePath: string | undefined) => SourceMetadata;