@wix/pathgrade 1.0.1 → 1.0.3

package/dist/adapter-kit/index.d.ts

@@ -8,7 +8,7 @@ export { runWithAdapter, type PathgradeRunOptions, type AdapterReporterMode } fr
 export { createRunnerLifecycleHooks } from '../runners/lifecycle-hooks.js';
 export { discoverPathgradeEvalFiles } from '../evals/discovery.js';
 export { DEFAULT_EVAL_EXCLUDE, DEFAULT_EVAL_INCLUDE, defaultPathgradeConfig, resolvePathgradeConfig, type PathgradeConfig, type ResolvedPathgradeConfig, } from '../config/pathgrade.js';
-export { readSidecar } from '../affected/sidecar.js';
+export { readSidecar, readSidecarForInvocation } from '../affected/sidecar.js';
 export { getPathgradeDir } from '../reporters/results-path.js';
 export { printReportSummary } from '../reporters/report-summary.js';
 export { fmt } from '../utils/cli.js';

package/dist/adapter-kit/index.js

@@ -6,7 +6,7 @@ export { runWithAdapter } from '../runners/orchestrator.js';
 export { createRunnerLifecycleHooks } from '../runners/lifecycle-hooks.js';
 export { discoverPathgradeEvalFiles } from '../evals/discovery.js';
 export { DEFAULT_EVAL_EXCLUDE, DEFAULT_EVAL_INCLUDE, defaultPathgradeConfig, resolvePathgradeConfig, } from '../config/pathgrade.js';
-export { readSidecar } from '../affected/sidecar.js';
+export { readSidecar, readSidecarForInvocation } from '../affected/sidecar.js';
 export { getPathgradeDir } from '../reporters/results-path.js';
 export { printReportSummary } from '../reporters/report-summary.js';
 export { fmt } from '../utils/cli.js';

package/dist/adapters/jest/invocation-adapter.js

@@ -2,7 +2,7 @@ import * as fs from 'node:fs';
 import * as path from 'node:path';
 import { spawn } from 'node:child_process';
 import { createRequire } from 'node:module';
-import { getPathgradeDir, printReportSummary, readSidecar, runWithAdapter, } from '@wix/pathgrade/adapter-kit';
+import { getPathgradeDir, printReportSummary, readSidecarForInvocation, runWithAdapter, } from '@wix/pathgrade/adapter-kit';
 import { removeJestMetadata, withJestMetadataEnv } from './metadata.js';
 import { createJestAdapter } from './runner-adapter.js';
 export function createJestInvocationAdapter(input) {
@@ -10,7 +10,7 @@ export function createJestInvocationAdapter(input) {
     return {
         name: 'jest',
         async run(runInput) {
-            const selection = await readSidecar(runInput.cwd, msg => {
+            const selection = await readSidecarForInvocation(runInput.cwd, runInput.env.PATHGRADE_SELECTION_INVOCATION_ID, msg => {
                 process.stderr.write(`[pathgrade] ${msg}\n`);
             }) ?? undefined;
             const include = runInput.selectedFiles && runInput.selectedFiles.length > 0

package/dist/adapters/jest/reporter.js

@@ -1,6 +1,6 @@
 import * as path from 'node:path';
 import { execSync } from 'node:child_process';
-import { fmt, getPathgradeDir, printReportSummary, readSidecar, resolvePathgradeConfig, runWithAdapter, } from '@wix/pathgrade/adapter-kit';
+import { fmt, getPathgradeDir, printReportSummary, readSidecarForInvocation, resolvePathgradeConfig, runWithAdapter, } from '@wix/pathgrade/adapter-kit';
 import { getJestLifecycleMetadata } from './lifecycle.js';
 import { readJestMetadata, removeJestMetadata } from './metadata.js';
 import { createJestAdapter } from './runner-adapter.js';
@@ -30,7 +30,7 @@ export default class PathgradeJestReporter {
                 writeEmptyReport: false,
                 warn: warning => console.warn(warning),
                 log: () => console.log(`\n  ${fmt.dim('Results written to')} ${outputDir}\n`),
-                loadSelection: async () => (await readSidecar(cwd, msg => {
+                loadSelection: async () => (await readSidecarForInvocation(cwd, process.env.PATHGRADE_SELECTION_INVOCATION_ID, msg => {
                     console.warn(`[pathgrade] ${msg}`);
                 })) ?? undefined,
                 printSummary: summaries => {

package/dist/adapters/jest/results.js

@@ -61,6 +61,12 @@ function runCase(input) {
         score: entry.score,
         ...(entry.trial ? { trial: entry.trial } : {}),
         ...(entry.diagnostics ? { diagnostics: entry.diagnostics } : {}),
+        ...(entry.resultKind ? { resultKind: entry.resultKind } : {}),
+        ...(entry.scoringDurationMs !== undefined
+            ? { scoringDurationMs: entry.scoringDurationMs }
+            : {}),
+        ...(entry.recordedAt ? { recordedAt: entry.recordedAt } : {}),
+        ...(entry.agent ? { agent: entry.agent } : {}),
     }));
     const failureMessage = input.assertion.failureMessages?.find(message => message.trim().length > 0);
     return {

package/dist/adapters/node-test/index.js

@@ -70,6 +70,10 @@ function toReportEvaluations(evaluations) {
         score: entry.score,
         trial: entry.trial,
         diagnostics: entry.diagnostics,
+        resultKind: entry.resultKind,
+        scoringDurationMs: entry.scoringDurationMs,
+        recordedAt: entry.recordedAt,
+        agent: entry.agent,
     }));
 }
 function writeCase(testCase) {

package/dist/adapters/node-test/invocation-adapter.js

@@ -1,4 +1,4 @@
-import { readSidecar } from '../../affected/sidecar.js';
+import { readSidecarForInvocation } from '../../affected/sidecar.js';
 import { getPathgradeDir } from '../../reporters/results-path.js';
 import { printReportSummary } from '../../reporters/report-summary.js';
 import { runWithAdapter } from '../../runners/orchestrator.js';
@@ -7,7 +7,7 @@ export function createNodeTestInvocationAdapter(input) {
     return {
         name: 'node-test',
         async run(runInput) {
-            const selection = await readSidecar(runInput.cwd, msg => {
+            const selection = await readSidecarForInvocation(runInput.cwd, runInput.env.PATHGRADE_SELECTION_INVOCATION_ID, msg => {
                 process.stderr.write(`[pathgrade] ${msg}\n`);
             }) ?? undefined;
             const include = runInput.selectedFiles && runInput.selectedFiles.length > 0

package/dist/adapters/vitest/reporter.js

@@ -2,7 +2,7 @@ import * as path from 'path';
 import { execSync } from 'child_process';
 import { fmt } from '../../utils/cli.js';
 import { getPathgradeDir } from '../../reporters/results-path.js';
-import { readSidecar } from '../../affected/sidecar.js';
+import { readSidecarForInvocation } from '../../affected/sidecar.js';
 import { printReportSummary } from '../../reporters/report-summary.js';
 import { createVitestAdapter } from '../../runners/vitest-adapter.js';
 import { runWithAdapter } from '../../runners/orchestrator.js';
@@ -35,7 +35,7 @@ export class PathgradeReporter {
                 writeEmptyReport: false,
                 warn: warning => console.warn(warning),
                 log: () => console.log(`\n  ${fmt.dim('Results written to')} ${outputDir}\n`),
-                loadSelection: async () => (await readSidecar(cwd, msg => {
+                loadSelection: async () => (await readSidecarForInvocation(cwd, process.env.PATHGRADE_SELECTION_INVOCATION_ID, msg => {
                     console.warn(`[pathgrade] ${msg}`);
                 })) ?? undefined,
                 printSummary: summaries => {

package/dist/affected/sidecar.d.ts

@@ -21,12 +21,14 @@ export declare function getSidecarPath(cwd: string): string;
  * `.pathgrade/selection.json` (and merged into `results.json` as
  * `PathgradeReport.selection`).
  */
-export declare function toSelectionReport(result: SelectionResult): PathgradeSelectionReport;
-export declare function writeSidecar(cwd: string, result: SelectionResult): Promise<void>;
+export declare function toSelectionReport(result: SelectionResult, invocationId?: string): PathgradeSelectionReport;
+export declare function writeSidecar(cwd: string, result: SelectionResult, invocationId?: string): Promise<void>;
 /**
  * Read the sidecar if present. Returns `null` when missing, and `null`
  * with an optional warning when the file exists but is malformed — the
  * reporter is expected to tolerate corruption rather than break the run.
  */
 export declare function readSidecar(cwd: string, onWarning?: (msg: string) => void): Promise<PathgradeSelectionReport | null>;
+/** Reads selection only when it belongs to the currently spawned runner. */
+export declare function readSidecarForInvocation(cwd: string, invocationId: string | undefined, onWarning?: (msg: string) => void): Promise<PathgradeSelectionReport | null>;
 export declare function clearSidecar(cwd: string): Promise<void>;

package/dist/affected/sidecar.js

@@ -24,8 +24,9 @@ export function getSidecarPath(cwd) {
  * `.pathgrade/selection.json` (and merged into `results.json` as
  * `PathgradeReport.selection`).
  */
-export function toSelectionReport(result) {
+export function toSelectionReport(result, invocationId) {
     const report = {
+        ...(invocationId ? { invocation_id: invocationId } : {}),
         base_ref: result.baseRef,
         changed_files_count: result.changedFiles.length,
         selected: result.selected.map(s => s.file).sort(),
@@ -38,10 +39,10 @@ export function toSelectionReport(result) {
         report.global_match = result.globalMatch;
     return report;
 }
-export async function writeSidecar(cwd, result) {
+export async function writeSidecar(cwd, result, invocationId) {
     const sidecarPath = getSidecarPath(cwd);
     await fs.ensureDir(path.dirname(sidecarPath));
-    await fs.writeJSON(sidecarPath, toSelectionReport(result), { spaces: 2 });
+    await fs.writeJSON(sidecarPath, toSelectionReport(result, invocationId), { spaces: 2 });
 }
 /**
  * Read the sidecar if present. Returns `null` when missing, and `null`
@@ -66,6 +67,13 @@ export async function readSidecar(cwd, onWarning) {
         return null;
     }
 }
+/** Reads selection only when it belongs to the currently spawned runner. */
+export async function readSidecarForInvocation(cwd, invocationId, onWarning) {
+    if (!invocationId)
+        return null;
+    const report = await readSidecar(cwd, onWarning);
+    return report?.invocation_id === invocationId ? report : null;
+}
 export async function clearSidecar(cwd) {
     const sidecarPath = getSidecarPath(cwd);
     try {

package/dist/commands/run-changed.js

@@ -13,6 +13,7 @@
  * only producer of the file list here.
  */
 import * as fs from 'fs';
+import { randomUUID } from 'node:crypto';
 import { selectAffected } from '../affected/select.js';
 import { resolveBaseRef, computeChangedFiles } from '../affected/git.js';
 import { writeSidecar } from '../affected/sidecar.js';
@@ -21,10 +22,12 @@ import { resolvePathgradeConfig } from '../config/pathgrade.js';
 import { loadRunnerInvocationAdapter } from '../runners/adapter-loader.js';
 export async function runChanged(opts) {
     const { cwd, parsed } = opts;
+    const selectionInvocationId = randomUUID();
     const runnerEnv = {
         ...process.env,
         ...(parsed.forceDiagnostics ? { PATHGRADE_DIAGNOSTICS: '1' } : {}),
         ...(parsed.forceVerbose ? { PATHGRADE_VERBOSE: '1' } : {}),
+        PATHGRADE_SELECTION_INVOCATION_ID: selectionInvocationId,
     };
     const configPath = findVitestConfigArg(parsed.runnerArgs);
     let config;
@@ -111,7 +114,7 @@ export async function runChanged(opts) {
     // Persist the sidecar immediately — even on empty selection, so the
     // reporter (if invoked later by another workflow step) has a coherent
     // record. A subsequent plain `pathgrade run` clears it.
-    await writeSidecar(cwd, result);
+    await writeSidecar(cwd, result, selectionInvocationId);
     if (result.selected.length === 0) {
         if (!parsed.quiet) {
             process.stderr.write(`pathgrade: no affected evals; nothing to run.\n`);

package/dist/reporting/core.js

@@ -35,7 +35,9 @@ export function buildPathgradeReport(input) {
             trace_file: traceFile,
         });
     }
-    const scores = reportableGroups.flatMap(group => group.cases.map(testCase => testCase.score));
+    const scores = reportableGroups.flatMap(group => group.cases
+        .filter(testCase => testCase.resultKind !== 'synthetic_no_evaluation')
+        .map(testCase => testCase.score));
     const overallPassRate = average(scores);
     const status = input.threshold != null
         ? (overallPassRate >= input.threshold ? 'pass' : 'fail')
@@ -69,7 +71,10 @@ function toBuiltCase(testCase) {
             `empty results for "${testCase.name}" — evaluate() may not have been called`,
         ]);
     }
-    const evaluation = testCase.evaluations[testCase.evaluations.length - 1];
+    // A case can own multiple agents. Preserve the existing "last evaluation"
+    // rule among real evaluations without letting a later synthetic result from
+    // an unevaluated sibling agent erase a legitimate score.
+    const evaluation = testCase.evaluations.findLast(candidate => candidate.resultKind !== 'synthetic_no_evaluation') ?? testCase.evaluations[testCase.evaluations.length - 1];
     const diagnostics = evaluation.diagnostics ?? testCase.diagnostics;
     return {
         name: testCase.name,
@@ -78,12 +83,17 @@ function toBuiltCase(testCase) {
         runnerDurationMs: testCase.runnerDurationMs,
         diagnostics,
         reportable: testCase.reportable,
+        resultKind: evaluation.resultKind,
         trial: normalizeTrial({
             name: testCase.name,
             score: evaluation.score,
             runnerDurationMs: testCase.runnerDurationMs,
             trial: evaluation.trial,
             diagnostics,
+            resultKind: evaluation.resultKind,
+            scoringDurationMs: evaluation.scoringDurationMs,
+            recordedAt: evaluation.recordedAt,
+            agent: evaluation.agent,
         }),
         warnings: [],
     };
@@ -123,6 +133,12 @@ function normalizeTrial(input) {
         name: input.name,
         duration_ms: base.duration_ms || input.runnerDurationMs,
         diagnostics: input.diagnostics ?? base.diagnostics,
+        ...(input.resultKind ? { result_kind: input.resultKind } : {}),
+        ...(input.scoringDurationMs !== undefined
+            ? { scoring_duration_ms: input.scoringDurationMs }
+            : {}),
+        ...(input.recordedAt ? { recorded_at: input.recordedAt } : {}),
+        ...(input.agent ? { agent: input.agent } : {}),
         ...(skills.length > 0 ? { skills_used: skills } : {}),
     };
 }

package/dist/reporting/types.d.ts

@@ -1,4 +1,5 @@
 import type { DiagnosticsReport } from '../sdk/diagnostics.js';
+import type { AgentExecutionMetadata, EvaluationResultKind } from '../sdk/types.js';
 import type { PathgradeReport, PathgradeSelectionReport, TrialResult } from '../types.js';
 export type ReportCaseState = 'passed' | 'failed' | 'skipped' | 'pending';
 export interface ReportRunInput {
@@ -27,6 +28,10 @@ export interface ReportEvaluationInput {
     score: number;
     trial?: TrialResult;
     diagnostics?: DiagnosticsReport;
+    resultKind?: EvaluationResultKind;
+    scoringDurationMs?: number;
+    recordedAt?: string;
+    agent?: AgentExecutionMetadata;
 }
 export interface PathgradeReportBuildResult {
     report: PathgradeReport;

package/dist/runners/model-builders.js

@@ -27,6 +27,12 @@ export function buildNormalizedRunSnapshotFromReportGroups(run, groups) {
                     score: evaluation.score,
                     ...(evaluation.trial ? { trial: evaluation.trial } : {}),
                     ...(evaluation.diagnostics ? { diagnostics: evaluation.diagnostics } : {}),
+                    ...(evaluation.resultKind ? { resultKind: evaluation.resultKind } : {}),
+                    ...(evaluation.scoringDurationMs !== undefined
+                        ? { scoringDurationMs: evaluation.scoringDurationMs }
+                        : {}),
+                    ...(evaluation.recordedAt ? { recordedAt: evaluation.recordedAt } : {}),
+                    ...(evaluation.agent ? { agent: evaluation.agent } : {}),
                 }));
                 return {
                     id: caseId,

package/dist/runners/model.d.ts

@@ -84,6 +84,10 @@ export interface EvaluationRecord {
     diagnostics?: DiagnosticsReport;
     trial?: TrialResult;
     nativeReferences?: NativeReference[];
+    resultKind?: import('../sdk/types.js').EvaluationResultKind;
+    scoringDurationMs?: number;
+    recordedAt?: string;
+    agent?: import('../sdk/types.js').AgentExecutionMetadata;
 }
 export interface AssertionRecord {
     id: string;

package/dist/runners/report-projection.js

@@ -40,6 +40,10 @@ function projectedEvaluations(runCase) {
             score: evaluation.score,
             ...(evaluation.trial ? { trial: evaluation.trial } : {}),
             ...(evaluation.diagnostics ? { diagnostics: evaluation.diagnostics } : {}),
+            ...(evaluation.resultKind ? { resultKind: evaluation.resultKind } : {}),
+            ...(evaluation.scoringDurationMs !== undefined ? { scoringDurationMs: evaluation.scoringDurationMs } : {}),
+            ...(evaluation.recordedAt ? { recordedAt: evaluation.recordedAt } : {}),
+            ...(evaluation.agent ? { agent: evaluation.agent } : {}),
         })));
     }
     if (runCase.scoringPolicy.kind === 'score') {
@@ -52,6 +56,10 @@ function projectedEvaluations(runCase) {
             score: evaluation.score,
             ...(evaluation.trial ? { trial: evaluation.trial } : {}),
             ...(evaluation.diagnostics ? { diagnostics: evaluation.diagnostics } : {}),
+            ...(evaluation.resultKind ? { resultKind: evaluation.resultKind } : {}),
+            ...(evaluation.scoringDurationMs !== undefined ? { scoringDurationMs: evaluation.scoringDurationMs } : {}),
+            ...(evaluation.recordedAt ? { recordedAt: evaluation.recordedAt } : {}),
+            ...(evaluation.agent ? { agent: evaluation.agent } : {}),
         }];
 }
 function preferredGroupingLabel(hints) {

package/dist/runners/vitest-adapter.js

@@ -89,6 +89,10 @@ function toReportCaseInput(testCase, groupName) {
             score: entry.score,
             trial: entry.trial,
             diagnostics: entry.diagnostics,
+            resultKind: entry.resultKind,
+            scoringDurationMs: entry.scoringDurationMs,
+            recordedAt: entry.recordedAt,
+            agent: entry.agent,
         })),
         diagnostics: normalized.diagnostics,
     };

package/dist/runners/vitest-invocation.d.ts

@@ -8,3 +8,4 @@ export type SpawnVitest = (req: SpawnVitestRequest) => Promise<number> | number;
 export declare function createVitestInvocationAdapter(input?: {
     spawnVitest?: SpawnVitest;
 }): RunnerInvocationAdapter;
+export declare function resolveVitestCli(cwd: string): string;

package/dist/runners/vitest-invocation.js

@@ -1,3 +1,6 @@
+import fs from 'node:fs';
+import { createRequire } from 'node:module';
+import path from 'node:path';
 export function createVitestInvocationAdapter(input = {}) {
     const spawnVitest = input.spawnVitest ?? defaultSpawnVitest;
     return {
@@ -32,13 +35,25 @@ function hasPassWithNoTests(args) {
 }
 async function defaultSpawnVitest(req) {
     const { spawn } = await import('child_process');
+    const vitestCli = resolveVitestCli(req.cwd);
     return await new Promise(resolve => {
-        const child = spawn('npx', ['vitest', ...req.argv], {
+        const child = spawn(process.execPath, [vitestCli, ...req.argv], {
             stdio: 'inherit',
             env: req.env,
             cwd: req.cwd,
-            shell: true,
         });
         child.on('close', code => resolve(code ?? 0));
     });
 }
+export function resolveVitestCli(cwd) {
+    const requireFromCwd = createRequire(path.join(cwd, 'package.json'));
+    const packageJsonPath = requireFromCwd.resolve('vitest/package.json');
+    const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
+    const bin = typeof packageJson.bin === 'string'
+        ? packageJson.bin
+        : packageJson.bin?.vitest;
+    if (!bin) {
+        throw new Error(`Unable to resolve the Vitest CLI from ${packageJsonPath}`);
+    }
+    return path.resolve(path.dirname(packageJsonPath), bin);
+}

package/dist/sdk/agent-result-log.js

@@ -20,6 +20,15 @@ export function buildModelAgentResultLogEntry(params) {
         // never produce a zero-valued field that could be mistaken for
         // a free turn.
         ...(params.turnResult.costUsd !== undefined ? { cost_usd: params.turnResult.costUsd } : {}),
+        ...(params.turnResult.cacheCreationInputTokens !== undefined
+            ? { cache_creation_input_tokens: params.turnResult.cacheCreationInputTokens }
+            : {}),
+        ...(params.turnResult.cacheReadInputTokens !== undefined
+            ? { cache_read_input_tokens: params.turnResult.cacheReadInputTokens }
+            : {}),
+        ...(params.turnResult.errorSubtype
+            ? { error_subtype: params.turnResult.errorSubtype }
+            : {}),
         ...getOutputMetrics(assistantMessage),
     };
 }

package/dist/sdk/agent.js

@@ -64,6 +64,19 @@ class AgentImpl {
     get workspace() {
         return this.ws.path;
     }
+    get executionMetadata() {
+        const interactionMode = this.interactionMode === 'startChat'
+            ? 'start_chat'
+            : this.interactionMode === 'runConversation'
+                ? 'conversation'
+                : this.interactionMode ?? undefined;
+        return {
+            name: this.agentName,
+            ...(this.modelOpt ? { requestedModel: this.modelOpt } : {}),
+            ...(this.transport ? { transport: this.transport } : {}),
+            ...(interactionMode ? { interactionMode } : {}),
+        };
+    }
     guardInteraction(mode) {
         if (this.interactionMode && this.interactionMode !== mode) {
             throw new Error(`Cannot use ${mode}() after ${this.interactionMode}() — an agent supports only one interaction method`);

package/dist/sdk/evaluate.d.ts

@@ -1,12 +1,12 @@
-import type { Agent, Scorer, EvalResult, RecordedEvalResult, ScorerResultEntry, EvaluateOptions } from './types.js';
+import type { Agent, Scorer, RecordedEvalResult, ScorerResultEntry, EvaluateOptions } from './types.js';
 export type OnScorerErrorMode = NonNullable<EvaluateOptions['onScorerError']>;
 export declare class EvalScorerError extends Error {
     readonly scorerErrors: ScorerResultEntry[];
     readonly result: RecordedEvalResult;
     constructor(result: RecordedEvalResult, scorerErrors: ScorerResultEntry[]);
 }
-type EvaluateFromSnapshot = (snapshotPath: string, scorers: Scorer[], opts?: EvaluateOptions) => Promise<EvalResult>;
-type EvaluateFn = ((agent: Agent, scorers: Scorer[], opts?: EvaluateOptions) => Promise<EvalResult>) & {
+type EvaluateFromSnapshot = (snapshotPath: string, scorers: Scorer[], opts?: EvaluateOptions) => Promise<RecordedEvalResult>;
+type EvaluateFn = ((agent: Agent, scorers: Scorer[], opts?: EvaluateOptions) => Promise<RecordedEvalResult>) & {
     fromSnapshot: EvaluateFromSnapshot;
 };
 export declare const evaluate: EvaluateFn;

package/dist/sdk/evaluate.js

@@ -54,6 +54,7 @@ function makeEvaluateAgent() {
         // accumulated agent-turn cost.
         const beforeCostUsd = trackedLLM.costUsd ?? 0;
         // measure() returns the delta consumed by this evaluate call.
+        const scoringStartedAt = performance.now();
         const { result: evalResult, tokens: deltaTokenUsage } = trackedLLM.measure
             ? await trackedLLM.measure(() => evaluateWithContext(ctx, scorers, { ...opts, llm: trackedLLM }))
             : await (async () => {
@@ -75,6 +76,12 @@ function makeEvaluateAgent() {
         const recordedResult = {
             ...evalResult,
             tokenUsage: deltaTokenUsage,
+            resultKind: 'evaluated',
+            scoringDurationMs: Math.max(0, performance.now() - scoringStartedAt),
+            recordedAt: new Date().toISOString(),
+            ...(opts?.evaluationDefinitionKey
+                ? { evaluationDefinitionKey: opts.evaluationDefinitionKey }
+                : {}),
             trial: buildTrialResult(agent.log, { ...evalResult, tokenUsage: deltaTokenUsage }, conversationTokens, conversationCost),
         };
         emitEvalResult({ result: recordedResult, agent });
@@ -106,9 +113,16 @@ async function fromSnapshot(snapshotPath, scorers, opts) {
         },
         artifacts: createSessionArtifacts(snapshot.workspace ?? '', snapshot.toolEvents),
     };
+    const scoringStartedAt = performance.now();
     const evalResult = await evaluateWithContext(ctx, scorers, { ...opts, llm: trackedLLM });
     const recordedResult = {
         ...evalResult,
+        resultKind: 'snapshot',
+        scoringDurationMs: Math.max(0, performance.now() - scoringStartedAt),
+        recordedAt: new Date().toISOString(),
+        ...(opts?.evaluationDefinitionKey
+            ? { evaluationDefinitionKey: opts.evaluationDefinitionKey }
+            : {}),
         trial: buildTrialResult(snapshot.log, evalResult),
     };
     maybeThrowOnScorerErrors(recordedResult, opts?.onScorerError ?? 'skip');

package/dist/sdk/index.d.ts

@@ -21,13 +21,14 @@ export { emitEvalResult, resetAllResultObserversForTests, resetUserResultObserve
 export { getAgentCapabilities } from './types.js';
 export type { AgentTransport, AgentCapabilities, AgentName, McpRunMode, McpSafetyOptions, McpToolPolicy, McpToolPolicyRule, } from './types.js';
 export type { AskBus, AskBatch, AskQuestion, AskOption, AskAnswer, AskResolution, AskBatchSnapshot, AskAnswerSnapshot, AskResolutionSnapshot, AskHandle, AskHandler, AskSource, AskLifecycle, AskAnswerSource, Unsubscribe as AskBusUnsubscribe, } from './ask-bus/types.js';
-export type { Agent, AgentOptions, Message, Scorer, CheckScorer, ScoreScorer, JudgeScorer, ToolUsageScorer, ScorerContext, EvalResult, ScorerResultEntry, ScorerStatus, ChatSession, ConversationResult, ConverseOptions, UntilPredicate, UntilContext, Reaction, TextReaction, AskUserReaction, AskUserQuestion, AskUserOption, ReactionPreviewEntry, TextReactionPreviewEntry, AskUserReactionPreviewEntry, ReactionPreviewResult, ReactionPreviewTurn, StepScorer, Persona, PersonaConfig, ConversationWindowConfig, TurnDetail, ReactionFiredEntry, PathgradePluginOptions, PathgradeMeta, TurnTiming, TokenUsage, EvaluateOptions, ReactionPreviewStatus, ScoreResult, JudgeInput, CodeJudgeToolName, ToolExpectation, SessionArtifactMatchOptions, SessionArtifactContent, SessionArtifacts, RecordedEvalResult, PathgradeTestMeta, } from './types.js';
+export type { Agent, AgentOptions, Message, Scorer, CheckScorer, ScoreScorer, JudgeScorer, ToolUsageScorer, ScorerContext, EvalResult, ScorerResultEntry, ScorerStatus, ChatSession, ConversationResult, ConverseOptions, UntilPredicate, UntilContext, Reaction, TextReaction, AskUserReaction, AskUserQuestion, AskUserOption, ReactionPreviewEntry, TextReactionPreviewEntry, AskUserReactionPreviewEntry, ReactionPreviewResult, ReactionPreviewTurn, StepScorer, Persona, PersonaConfig, ConversationWindowConfig, TurnDetail, ReactionFiredEntry, PathgradePluginOptions, PathgradeMeta, TurnTiming, TokenUsage, EvaluateOptions, ReactionPreviewStatus, ScoreResult, JudgeInput, CodeJudgeToolName, ToolExpectation, SessionArtifactMatchOptions, SessionArtifactContent, SessionArtifacts, RecordedEvalResult, PathgradeTestMeta, EvaluationResultKind, AgentExecutionMetadata, AgentInteractionMode, } from './types.js';
 export type { ConversationWindow, ConversationWindowOptions } from './conversation-window.js';
 export type { JudgePipelineOptions } from './judge-pipeline.js';
 export type { RunScorerOptions } from './run-scorer.js';
 export type { OnScorerErrorMode } from './evaluate.js';
 export type { EvalResultEvent, EvalResultObserver, ResultObserverHandle, ResultObserverOptions, ResultObserverOwner, } from './result-capture.js';
 export type { RunSnapshot } from './snapshots.js';
+export type { DiagnosticsReport } from './diagnostics.js';
 export type { ExpectedMcpStartupStatus, ExpectedMcpToolCall, McpStartupStatusEvidence, McpToolCallEvidence, } from './mcp-evidence.js';
 export type { McpPolicyDenialReason, McpToolCallRequest, McpToolPolicyDecision, } from './mcp-safety.js';
 export type { ToolEvent } from '../tool-events.js';

package/dist/sdk/lifecycle.js

@@ -73,6 +73,12 @@ function recordResult(result, agent, attribution) {
             score: result.score,
             scorers: result.scorers,
             trial: result.trial,
+            resultKind: result.resultKind ?? 'evaluated',
+            ...(result.scoringDurationMs !== undefined
+                ? { scoringDurationMs: result.scoringDurationMs }
+                : {}),
+            ...(result.recordedAt ? { recordedAt: result.recordedAt } : {}),
+            ...(agent.executionMetadata ? { agent: agent.executionMetadata } : {}),
             diagnostics: buildDiagnosticsReport({
                 completionReason,
                 completionDetail: conversationEnd?.completion_detail,
@@ -129,14 +135,18 @@ function synthesizeTrialFromAgent(agent) {
     return {
         score: 1,
         scorers: [],
+        resultKind: 'synthetic_no_evaluation',
+        ...(agent.executionMetadata ? { agent: agent.executionMetadata } : {}),
         trial: {
             trial_id: 0,
             reward: 1,
             scorer_results: [],
             duration_ms: 0,
             n_commands: nCommands,
-            input_tokens: tokenUsage?.inputTokens ?? 0,
-            output_tokens: tokenUsage?.outputTokens ?? 0,
+            input_tokens: 0,
+            output_tokens: 0,
+            conversation_input_tokens: tokenUsage?.inputTokens ?? 0,
+            conversation_output_tokens: tokenUsage?.outputTokens ?? 0,
             session_log: [...agent.log],
         },
         diagnostics: buildDiagnosticsReport({

package/dist/sdk/result-capture.d.ts

@@ -1,7 +1,9 @@
 import type { Agent, RecordedEvalResult } from './types.js';
+import { type CaseContext } from './case-context.js';
 export interface EvalResultEvent {
     readonly result: RecordedEvalResult;
     readonly agent: Agent;
+    readonly case?: CaseContext;
 }
 export type EvalResultObserver = (event: EvalResultEvent) => void;
 export type ResultObserverOwner = 'user' | 'adapter' | 'test';

package/dist/sdk/result-capture.js

@@ -1,3 +1,4 @@
+import { getCurrentCaseContext } from './case-context.js';
 const observers = new Set();
 export function subscribeToEvalResults(observer, options = {}) {
     if (options.owner === 'adapter' && options.key) {
@@ -20,9 +21,16 @@ export function subscribeToEvalResults(observer, options = {}) {
     };
 }
 export function emitEvalResult(event) {
+    const currentCase = getCurrentCaseContext();
+    const deliveredEvent = event.case || currentCase.status !== 'active'
+        ? event
+        : { ...event, case: currentCase.context };
+    // Deliver against a stable snapshot. Observers are allowed to subscribe or
+    // unsubscribe from inside a callback, but those mutations must only affect
+    // subsequent events.
     for (const subscription of [...observers]) {
         try {
-            subscription.observer(event);
+            subscription.observer(deliveredEvent);
         }
         catch {
             // Result capture should be best-effort for every observer; one

package/dist/sdk/types.d.ts

@@ -6,6 +6,15 @@ import type { DiagnosticsReport } from './diagnostics.js';
 import type { LLMPort } from '../utils/llm-types.js';
 import type { McpSafetyOptions } from './mcp-safety.js';
 export type AgentName = 'claude' | 'codex' | 'cursor';
+export type AgentInteractionMode = 'prompt' | 'start_chat' | 'conversation';
+/** Privacy-safe execution dimensions exposed to result observers. */
+export interface AgentExecutionMetadata {
+    name: AgentName;
+    /** Model override requested by the caller; not necessarily the provider-resolved model. */
+    requestedModel?: string;
+    transport?: AgentTransport;
+    interactionMode?: AgentInteractionMode;
+}
 export interface AgentOptions {
     agent?: AgentName;
     model?: string;
@@ -61,6 +70,8 @@ export interface Agent {
     readonly messages: Message[];
     readonly log: LogEntry[];
     readonly workspace: string;
+    /** Provider/configuration metadata only; never contains prompts or paths. */
+    readonly executionMetadata?: AgentExecutionMetadata;
     dispose(): Promise<void>;
 }
 export interface ConverseOptions {
@@ -318,6 +329,8 @@ export interface EvaluateOptions {
     failFast?: boolean;
     llm?: LLMPort;
     onScorerError?: 'skip' | 'zero' | 'fail';
+    /** Stable identity for this evaluation definition across separate runs. */
+    evaluationDefinitionKey?: string;
 }
 export interface TokenUsage {
     inputTokens: number;
@@ -331,7 +344,12 @@ export interface EvalResult {
 }
 export interface RecordedEvalResult extends EvalResult {
     trial?: TrialResult;
+    resultKind?: EvaluationResultKind;
+    scoringDurationMs?: number;
+    recordedAt?: string;
+    evaluationDefinitionKey?: string;
 }
+export type EvaluationResultKind = 'evaluated' | 'synthetic_no_evaluation' | 'snapshot';
 export interface ScorerResultEntry {
     name: string;
     type: 'check' | 'score' | 'judge' | 'tool_usage';
@@ -351,6 +369,10 @@ export interface PathgradeTestMeta {
     scorers: ScorerResultEntry[];
     trial?: TrialResult;
     diagnostics?: DiagnosticsReport;
+    resultKind?: EvaluationResultKind;
+    scoringDurationMs?: number;
+    recordedAt?: string;
+    agent?: AgentExecutionMetadata;
 }
 export interface AgentCapabilities {
     mcp: boolean;

package/dist/types.d.ts

@@ -109,6 +109,9 @@ export interface LogEntry {
      * carries a `costUsd` value; absent for agents that do not expose cost.
      */
     cost_usd?: number;
+    cache_creation_input_tokens?: number;
+    cache_read_input_tokens?: number;
+    error_subtype?: AgentTurnResult['errorSubtype'];
     completion_reason?: string;
     completion_detail?: string;
     turn_timings?: Array<{
@@ -174,6 +177,10 @@ export interface TrialResult {
     session_log: LogEntry[];
     skills_used?: string[];
     diagnostics?: DiagnosticsReport;
+    result_kind?: import('./sdk/types.js').EvaluationResultKind;
+    scoring_duration_ms?: number;
+    recorded_at?: string;
+    agent?: import('./sdk/types.js').AgentExecutionMetadata;
     conversation?: {
         turns: ConversationTurn[];
         total_turns: number;
@@ -213,6 +220,8 @@ export type PathgradeGroupReport = Omit<EvalReport, 'trials'> & {
  * source of truth — new variants added there propagate here automatically).
  */
 export interface PathgradeSelectionReport {
+    /** Binds this sidecar to the runner process spawned by `run --changed`. */
+    invocation_id?: string;
     base_ref: string;
     changed_files_count: number;
     global_match?: string;

package/package.json

@@ -1,15 +1,8 @@
 {
   "name": "@wix/pathgrade",
-  "version": "1.0.1",
+  "version": "1.0.3",
   "packageManager": "yarn@4.12.0",
   "description": "Evaluate whether AI agents discover and use your skills correctly",
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/wix-incubator/pathgrade.git"
-  },
-  "installConfig": {
-    "selfReferences": true
-  },
   "exports": {
     ".": {
       "types": "./dist/sdk/index.d.ts",
@@ -63,18 +56,9 @@
     "!dist/**/*.d.ts.map",
     "bin/",
     "templates/",
-    "README.md",
-    "LICENSE"
+    "README.md"
   ],
   "scripts": {
-    "lint": "oxlint src tests evals examples scripts --deny-warnings --disable-unicorn-plugin --disable-oxc-plugin --allow no-unused-vars --allow no-underscore-dangle --allow no-shadow --allow preserve-caught-error --allow no-unsafe-optional-chaining --allow no-unreachable --ignore-pattern tests/fixtures --ignore-pattern .worktrees",
-    "knip": "yarn quality:deps",
-    "knip:unused": "knip --exclude binaries --no-config-hints --max-show-issues 20",
-    "quality:deps": "knip --dependencies --exclude binaries --no-config-hints --max-show-issues 20",
-    "quality:arch": "sentrux check .",
-    "quality:boundaries": "scripts/check-quality-boundaries.sh",
-    "quality:max-lines": "scripts/check-max-lines.sh .",
-    "quality": "yarn lint && yarn quality:deps && yarn quality:arch && yarn quality:boundaries && yarn quality:max-lines",
     "test": "yarn build && vitest run && yarn test:runner-cli-smoke",
     "test:runner-cli-smoke": "node tests/runner-cli-smoke/run-smokes.mjs",
     "test:evals": "vitest run --config evals/vitest.config.mts",
@@ -95,11 +79,21 @@
     "llm",
     "testing"
   ],
+  "publishConfig": {
+    "registry": "https://registry.npmjs.org/",
+    "access": "public"
+  },
+  "wix": {
+    "artifact": {
+      "groupId": "com.wixpress",
+      "artifactId": "pathgrade"
+    }
+  },
   "author": "Nadav Lachish",
   "license": "MIT",
   "type": "module",
   "engines": {
-    "node": ">=20.11.0"
+    "node": ">=20.19.0"
   },
   "peerDependencies": {
     "jest": "^30.0.0",
@@ -117,13 +111,13 @@
     "@types/fs-extra": "^11.0.4",
     "@types/jest": "^30.0.0",
     "@types/picomatch": "^4.0.2",
-    "@vitest/coverage-v8": "4.1.7",
+    "@vitest/coverage-v8": "4.1.10",
     "jest": "^30.0.0",
-    "vitest": "4.1.7"
+    "vitest": "4.1.10"
   },
   "dependencies": {
-    "@anthropic-ai/claude-agent-sdk": "0.2.116",
-    "@modelcontextprotocol/sdk": "1.29.0",
+    "@anthropic-ai/claude-agent-sdk": "0.2.141",
+    "@modelcontextprotocol/sdk": "1.30.0",
     "@types/node": "25.6.0",
     "fs-extra": "11.3.3",
     "jiti": "2.6.1",
@@ -132,7 +126,5 @@
     "typescript": "^5.9.3",
     "zod": "4.3.6"
   },
-  "resolutions": {
-    "fast-uri@npm:^3.0.1": "npm:3.1.2"
-  }
-}
+  "falconPackageHash": "939cc25b6013ca70146b099b6e06801ff4fd21aed2d63ebdb5bf721a"
+}

package/LICENSE

@@ -1,24 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2025 Nadav Lachish
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
-NOTICE: Pathgrade was originally forked from skillgrade by Minko Gechev,
-which is MIT licensed.