@wix/pathgrade 1.0.25 → 1.0.27

package/dist/pathgrade.js

@@ -156,10 +156,27 @@ export async function runPathgradeCli(options = {}) {
         const reportArgs = args.slice(1);
         const resultsPathFlag = reportArgs.find(a => a.startsWith('--results-path='));
         const commentIdFlag = reportArgs.find(a => a.startsWith('--comment-id='));
+        const detailsUrlFlag = reportArgs.find(a => a.startsWith('--details-url='));
+        const noticeFlag = reportArgs.find(a => a.startsWith('--notice='));
+        const expectedHeadShaFlag = reportArgs.find(a => a.startsWith('--expected-head-sha='));
         const noComment = reportArgs.includes('--no-comment');
+        const strict = reportArgs.includes('--strict');
         const resultsPath = resultsPathFlag ? resultsPathFlag.split('=').slice(1).join('=') : undefined;
         const commentId = commentIdFlag ? commentIdFlag.split('=').slice(1).join('=') : undefined;
-        await runReport(process.cwd(), { resultsPath, commentId, noComment });
+        const detailsUrl = detailsUrlFlag ? detailsUrlFlag.split('=').slice(1).join('=') : undefined;
+        const notice = noticeFlag ? noticeFlag.split('=').slice(1).join('=') : undefined;
+        const expectedHeadSha = expectedHeadShaFlag
+            ? expectedHeadShaFlag.split('=').slice(1).join('=')
+            : undefined;
+        await runReport(process.cwd(), {
+            resultsPath,
+            commentId,
+            detailsUrl,
+            notice,
+            expectedHeadSha,
+            noComment,
+            strict,
+        });
         return;
     }
     if (command === 'run' || !command || command.startsWith('-')) {
@@ -234,6 +251,10 @@ function printHelp(cliName) {
                      [--results-path=<path>]   Override results.json location
                      [--no-comment]            Print markdown to stdout; do not post
                      [--comment-id=<id>]       Override comment marker (default: $GITHUB_WORKFLOW:$GITHUB_JOB)
+                     [--details-url=<url>]     Add a provider-neutral full-report link
+                     [--notice=<text>]         Add a provider-neutral report status notice
+                     [--expected-head-sha=<sha>] Skip stale PR comment updates
+                     [--strict]                Fail on missing context, results, or GitHub errors
     ${cliName} affected               Print eval files affected by a change-set (one per line)
                      [--since=<ref>]           Diff <ref>...HEAD (overrides git auto-detection)
                      [--changed-files=<path>]  Newline-delimited repo-relative file list

package/dist/reporters/github-comment.d.ts

@@ -10,6 +10,10 @@
 import type { PathgradeReport, PathgradeSelectionReport } from '../types.js';
 export interface FormatOptions {
     commentId: string;
+    /** Provider-neutral URL for a richer external report. */
+    detailsUrl?: string;
+    /** Provider-neutral status notice rendered above report details. */
+    notice?: string;
 }
 /** Minimal body posted when `.pathgrade/results.json` is missing. */
 export declare const MISSING_RESULTS_BODY = "Pathgrade evals did not produce results. Check the workflow logs.";
@@ -55,14 +59,19 @@ export interface PostOptions {
     commentId: string;
     /** Body WITHOUT the marker — the marker is prepended if not already present. */
     body: string;
+    /** Throw instead of swallowing GitHub API failures. */
+    strict?: boolean;
+    /** Skip the update if the pull request has advanced to another head. */
+    expectedHeadSha?: string;
 }
+export type PostResult = 'created' | 'updated' | 'stale';
 /**
  * Find an existing PR comment carrying `<!-- pathgrade:${commentId} -->`
- * and update it; otherwise create a new one. Swallows all errors (logs
- * to stderr) — `pathgrade report` must never fail CI.
+ * and update it; otherwise create a new one. Default mode logs and swallows
+ * transport errors; strict mode rethrows them for an owning orchestrator.
  *
  * Bodies longer than GitHub's 65,536-character limit are truncated with a
  * sentinel pointing to workflow artifacts, preserving the leading dedup
  * marker so subsequent runs still find and update this comment.
  */
-export declare function postOrUpdateComment(ctx: PrContext, opts: PostOptions): Promise<void>;
+export declare function postOrUpdateComment(ctx: PrContext, opts: PostOptions): Promise<PostResult | undefined>;

package/dist/reporters/github-comment.js

@@ -26,6 +26,12 @@ export function formatNoAffectedEvalsMarkdown(selection) {
     return lines.join('\n');
 }
 export function commentMarker(commentId) {
+    if (commentId.length === 0
+        || commentId.length > 128
+        || /[<>\r\n]/.test(commentId)
+        || commentId.includes('--')) {
+        throw new Error('Pathgrade comment id must be 1-128 characters and cannot contain HTML comment delimiters');
+    }
     return `<!-- pathgrade:${commentId} -->`;
 }
 function pct(n) {
@@ -59,9 +65,19 @@ export function formatReportMarkdown(report, opts) {
     lines.push('');
     lines.push(`### ${icon} Pathgrade report`);
     lines.push('');
+    lines.push(`**Last run:** \`${report.timestamp}\``);
+    lines.push('');
     lines.push(`**Pass rate:** ${pct(p)}  |  ` +
         `**pass@${totalTrials}:** ${pct(overallPassAtK)}  |  ` +
         `**pass^${totalTrials}:** ${pct(overallPassPowK)}`);
+    if (opts.detailsUrl) {
+        lines.push('');
+        lines.push(`**Details:** [Open the full report](${formatLinkDestination(opts.detailsUrl)})`);
+    }
+    if (opts.notice) {
+        lines.push('');
+        lines.push(`> ${opts.notice.replace(/[\r\n]+/g, ' ').trim()}`);
+    }
     if (report.threshold != null) {
         lines.push('');
         lines.push(`Threshold: ${pct(report.threshold)} — ${report.status.toUpperCase()}`);
@@ -100,14 +116,14 @@ export function formatSelectionSection(selection) {
     out.push('### Selection');
     const total = selection.selected.length + selection.skipped.length;
     if (selection.global_match) {
-        out.push(`Ran **all ${total}** evals — global trigger \`${selection.global_match}\` matched.`);
+        out.push(`Selected **all ${total}** eval files — global trigger \`${selection.global_match}\` matched.`);
         return out.join('\n');
     }
     if (selection.skipped.length === 0 && selection.selected.length > 0) {
-        out.push(`Ran **all ${total}** evals — every eval had a matching change.`);
+        out.push(`Selected **all ${total}** eval files — every eval had a matching change.`);
         return out.join('\n');
     }
-    out.push(`Ran **${selection.selected.length} of ${total}** evals based on changes vs \`${selection.base_ref}\`.`);
+    out.push(`Selected **${selection.selected.length} of ${total}** eval files based on changes vs \`${selection.base_ref}\`.`);
     if (selection.skipped.length > 0) {
         out.push('');
         out.push(`<details><summary>${selection.skipped.length} skipped (unaffected)</summary>`);
@@ -171,13 +187,36 @@ function apiHeaders(token) {
     };
 }
 async function listPrComments(ctx) {
-    const url = `https://api.github.com/repos/${ctx.owner}/${ctx.repo}/issues/${ctx.prNumber}/comments?per_page=100`;
+    const comments = [];
+    for (let page = 1; page <= 100; page += 1) {
+        const url = `https://api.github.com/repos/${ctx.owner}/${ctx.repo}/issues/${ctx.prNumber}/comments?per_page=100&page=${page}`;
+        const res = await fetch(url, { headers: apiHeaders(ctx.token) });
+        if (!res.ok) {
+            throw new Error(`GET ${url} returned ${res.status}`);
+        }
+        const data = await res.json();
+        if (!Array.isArray(data)) {
+            throw new Error(`GET ${url} returned a non-array response`);
+        }
+        for (const value of data) {
+            if (isGithubComment(value))
+                comments.push(value);
+        }
+        if (data.length < 100)
+            break;
+    }
+    return comments;
+}
+async function currentPrHeadSha(ctx) {
+    const url = `https://api.github.com/repos/${ctx.owner}/${ctx.repo}/pulls/${ctx.prNumber}`;
     const res = await fetch(url, { headers: apiHeaders(ctx.token) });
-    if (!res.ok) {
+    if (!res.ok)
         throw new Error(`GET ${url} returned ${res.status}`);
+    const data = await res.json();
+    if (!isRecord(data) || !isRecord(data.head) || typeof data.head.sha !== 'string') {
+        throw new Error(`GET ${url} returned no pull-request head SHA`);
     }
-    const data = (await res.json());
-    return Array.isArray(data) ? data : [];
+    return data.head.sha;
 }
 async function createPrComment(ctx, body) {
     const url = `https://api.github.com/repos/${ctx.owner}/${ctx.repo}/issues/${ctx.prNumber}/comments`;
@@ -206,8 +245,8 @@ const GITHUB_COMMENT_MAX = 65536;
 const TRUNCATION_SENTINEL = '\n\n_…output truncated to fit GitHub\'s comment size limit — see the workflow artifacts for the full report._';
 /**
  * Find an existing PR comment carrying `<!-- pathgrade:${commentId} -->`
- * and update it; otherwise create a new one. Swallows all errors (logs
- * to stderr) — `pathgrade report` must never fail CI.
+ * and update it; otherwise create a new one. Default mode logs and swallows
+ * transport errors; strict mode rethrows them for an owning orchestrator.
  *
  * Bodies longer than GitHub's 65,536-character limit are truncated with a
  * sentinel pointing to workflow artifacts, preserving the leading dedup
@@ -221,19 +260,51 @@ export async function postOrUpdateComment(ctx, opts) {
         : withMarker.slice(0, GITHUB_COMMENT_MAX - TRUNCATION_SENTINEL.length) +
             TRUNCATION_SENTINEL;
     try {
+        if (opts.expectedHeadSha) {
+            const currentHead = await currentPrHeadSha(ctx);
+            if (currentHead !== opts.expectedHeadSha)
+                return 'stale';
+        }
         const existing = await listPrComments(ctx);
         const match = existing.find((c) => typeof c.body === 'string' && c.body.includes(marker));
         if (match) {
             await updatePrComment(ctx, match.id, body);
+            return 'updated';
         }
         else {
             await createPrComment(ctx, body);
+            return 'created';
         }
     }
     catch (err) {
         const message = err instanceof Error ? err.message : String(err);
         console.error(`pathgrade report: failed to post PR comment — ${message}`);
+        if (opts.strict)
+            throw err;
+    }
+}
+function formatLinkDestination(value) {
+    let url;
+    try {
+        url = new URL(value);
     }
+    catch {
+        throw new Error('Pathgrade details URL must be an absolute HTTP(S) URL');
+    }
+    if (url.protocol !== 'https:' && url.protocol !== 'http:') {
+        throw new Error('Pathgrade details URL must be an absolute HTTP(S) URL');
+    }
+    return url.toString().replace(/\)/g, '%29');
+}
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function isGithubComment(value) {
+    return isRecord(value)
+        && typeof value.id === 'number'
+        && Number.isSafeInteger(value.id)
+        && value.id > 0
+        && typeof value.body === 'string';
 }
 function formatGroupDetails(group) {
     const out = [];

package/dist/reporting/artifacts.js

@@ -1,18 +1,21 @@
 import path from 'node:path';
 import fs from 'fs-extra';
+import { collectSensitiveEnvValues, sanitizePersistenceValue, } from '../tool-event-results.js';
 export async function writePathgradeArtifacts(artifactRoot, built) {
     await fs.ensureDir(path.join(artifactRoot, 'traces'));
+    const sensitiveValues = collectSensitiveEnvValues(process.env);
+    const report = sanitizePersistenceValue(built.report, sensitiveValues);
     const gitignorePath = path.join(artifactRoot, '.gitignore');
     if (!(await fs.pathExists(gitignorePath))) {
         await fs.writeFile(gitignorePath, '*\n');
     }
     const traceFiles = [];
     for (const trace of built.traces) {
-        await fs.writeJson(path.join(artifactRoot, trace.traceFile), trace.trials, { spaces: 2 });
+        await fs.writeJson(path.join(artifactRoot, trace.traceFile), sanitizePersistenceValue(trace.trials, sensitiveValues), { spaces: 2 });
         traceFiles.push(trace.traceFile);
     }
     const resultsPath = path.join(artifactRoot, 'results.json');
-    await fs.writeJson(resultsPath, built.report, { spaces: 2 });
+    await fs.writeJson(resultsPath, report, { spaces: 2 });
     return {
         resultsPath,
         traceFiles,

package/dist/sdk/agent.js

@@ -4,7 +4,7 @@ import { lifecycleCore } from './lifecycle.js';
 import { ChatSessionImpl } from './chat.js';
 import { runConversation } from './converse.js';
 import { createPersona } from './persona.js';
-import { evaluate } from './evaluate.js';
+import { evaluateStepScorers } from './evaluate.js';
 import { createManagedSession } from './managed-session.js';
 import { createAgentLLM } from '../utils/llm.js';
 import { buildRunSnapshot } from './snapshots.js';
@@ -19,6 +19,10 @@ import { cleanDebugRuns, DEFAULT_DEBUG_RETAIN_RUNS, prepareManagedDebugRun, } fr
 import { collectOpenCodeMcpToolNames, validateOpenCodeDeclaration } from '../agents/opencode/contract.js';
 import { collectSensitiveEnvValues } from '../tool-event-results.js';
 import { compileMcpMockApprovalSession, } from './mcp-mock-approvals.js';
+import { compileScenario } from './scenario-machine-v2.js';
+import { startPublicCodexScenarioRuntime } from '../agents/codex-app-server/scenario-mount.js';
+import { createPublicOpenCodeScenarioRuntimeFactory } from '../agents/opencode/scenario.js';
+import { createPublicCursorScenarioRuntimeFactory } from '../agents/cursor-scenario.js';
 import { createAskUserHandler } from './ask-bus/handler.js';
 /**
  * Test-only injection point: override the sink used by the next emitter
@@ -35,6 +39,7 @@ class AgentImpl {
     llm;
     timeoutSetting;
     modelOpt;
+    resolvedModel;
     conversationWindowOpt;
     interactionMode = null;
     _messages = [];
@@ -52,6 +57,9 @@ class AgentImpl {
     activeChatSession;
     scriptedMcp;
     sensitiveValues;
+    scenarioArtifact;
+    scenarioRuntimeFactory;
+    _scenarioEvidence = [];
     constructor(opts) {
         this.ws = opts.workspace;
         this.agentName = opts.agentName;
@@ -69,6 +77,8 @@ class AgentImpl {
         this.opencodeMcpToolNames = opts.opencodeMcpToolNames;
         this.scriptedMcp = opts.scriptedMcp;
         this.sensitiveValues = opts.sensitiveValues;
+        this.scenarioArtifact = opts.scenarioArtifact;
+        this.scenarioRuntimeFactory = opts.scenarioRuntimeFactory;
     }
     get messages() {
         return this._messages;
@@ -79,6 +89,9 @@ class AgentImpl {
     get workspace() {
         return this.ws.path;
     }
+    get scenarioEvidence() {
+        return structuredClone(this._scenarioEvidence);
+    }
     get executionMetadata() {
         const interactionMode = this.interactionMode === 'startChat'
             ? 'start_chat'
@@ -88,7 +101,10 @@ class AgentImpl {
         return {
             name: this.agentName,
             ...(this.modelOpt ? { requestedModel: this.modelOpt } : {}),
-            transport: resolveExecutionTransport(this.agentName, this.transport),
+            ...(this.resolvedModel ? { resolvedModel: this.resolvedModel } : {}),
+            transport: this.scenarioArtifact && (this.agentName === 'opencode' || this.agentName === 'cursor')
+                ? 'acp'
+                : resolveExecutionTransport(this.agentName, this.transport),
             ...(interactionMode ? { interactionMode } : {}),
         };
     }
@@ -114,6 +130,9 @@ class AgentImpl {
             ...(this.opencodeExecutable !== undefined ? { opencodeExecutable: this.opencodeExecutable } : {}),
             ...(this.opencodeMcpToolNames !== undefined ? { opencodeMcpToolNames: this.opencodeMcpToolNames } : {}),
             ...(this.scriptedMcp !== undefined ? { scriptedMcp: this.scriptedMcp } : {}),
+            ...(this.scenarioArtifact !== undefined ? { scenarioArtifact: this.scenarioArtifact } : {}),
+            ...(this.scenarioRuntimeFactory !== undefined ? { scenarioRuntimeFactory: this.scenarioRuntimeFactory } : {}),
+            onScenarioEvidence: (evidence) => { this._scenarioEvidence = structuredClone(evidence); },
             sensitiveValues: this.sensitiveValues,
         });
     }
@@ -128,6 +147,9 @@ class AgentImpl {
         return Math.ceil((turns * 80_000 + 200_000) / 1000);
     }
     accumulateTurnUsage(turnResult) {
+        if (turnResult.resolvedModel) {
+            this.resolvedModel = turnResult.resolvedModel;
+        }
         if (turnResult.inputTokens || turnResult.outputTokens) {
             this.llm.addTokens?.(turnResult.inputTokens ?? 0, turnResult.outputTokens ?? 0);
         }
@@ -286,7 +308,7 @@ class AgentImpl {
         // scorer judge calls accumulate on the same tracker.
         const agent = this;
         const runStepScorers = async (scorers) => {
-            const result = await evaluate(agent, scorers, { llm: this.llm });
+            const result = await evaluateStepScorers(agent, scorers, { llm: this.llm });
             const score = result.score;
             if (score === undefined) {
                 throw new Error('Step scorer evaluation did not produce a score');
@@ -362,6 +384,7 @@ class AgentImpl {
                         conversationResult: this.lastConversationResult,
                         workspace: dest,
                         sensitiveValues: this.sensitiveValues,
+                        scenarioEvidence: this._scenarioEvidence,
                     });
                     await fs.writeJSON(path.join(dest, 'run-snapshot.json'), snapshot, { spaces: 2 });
                 }
@@ -409,10 +432,37 @@ export async function createAgent(opts) {
         }
     }
     const agentName = resolveAgentName(opts, process.env);
+    let scenarioArtifact;
+    if (opts.mcpScenario !== undefined) {
+        if (opts.agent === undefined)
+            throw new Error('mcpScenario requires an explicit agent');
+        if (opts.mcpMock !== undefined || opts.mcpMockApprovalRules !== undefined || opts.mcpConfigFile !== undefined || opts.mcpSafety !== undefined) {
+            throw new Error('mcpScenario cannot be combined with legacy or live MCP options');
+        }
+        const compiled = compileScenario(opts.mcpScenario);
+        if (!compiled.ok)
+            throw new Error(`Scenario compilation failed: ${JSON.stringify(compiled.diagnostics)}`);
+        scenarioArtifact = compiled.artifact;
+    }
     validateOpenCodeDeclaration(agentName, opts);
     const transport = agentName === 'codex'
         ? resolveCodexTransport(opts, process.env)
         : undefined;
+    if (scenarioArtifact && agentName === 'codex' && transport !== 'app-server') {
+        throw new Error('mcpScenario requires Codex transport app-server; Codex exec is unsupported');
+    }
+    const scenarioRuntimeFactory = !scenarioArtifact
+        ? undefined
+        : agentName === 'codex'
+            ? startPublicCodexScenarioRuntime
+            : agentName === 'opencode'
+                ? createPublicOpenCodeScenarioRuntimeFactory({
+                    opencodeExecutable: opts.opencodeExecutable,
+                    requestedModel: opts.model,
+                })
+                : agentName === 'cursor'
+                    ? createPublicCursorScenarioRuntimeFactory({ requestedModel: opts.model })
+                    : undefined;
     let scriptedMcp;
     if (opts.mcpMockApprovalRules !== undefined) {
         if (opts.mcpMock === undefined)
@@ -434,7 +484,7 @@ export async function createAgent(opts) {
     const timeoutSetting = opts.timeout ?? 300;
     // Capture runner context now; adapters own installation and restoration.
     const testCtx = opts.debug ? resolveCaseDebugContext() : { name: '', dir: '' };
-    const { timeout: _, mcpMock, mcpConfigFile, mcpMockApprovalRules: ________, agent: __, debug: ___, transport: _____, mcpSafety: ______, opencodeExecutable, ...rest } = opts;
+    const { timeout: _, mcpMock, mcpScenario: _________, mcpConfigFile, mcpMockApprovalRules: ________, agent: __, debug: ___, transport: _____, mcpSafety: ______, opencodeExecutable, ...rest } = opts;
     const workspace = await prepareWorkspace({
         ...rest,
         agent: agentName,
@@ -471,6 +521,8 @@ export async function createAgent(opts) {
         opencodeExecutable,
         opencodeMcpToolNames: agentName === 'opencode' ? collectOpenCodeMcpToolNames(mcpMock) : undefined,
         scriptedMcp,
+        scenarioArtifact,
+        scenarioRuntimeFactory,
         sensitiveValues: workspace.sensitiveValues ?? collectSensitiveEnvValues(workspace.env),
     });
     lifecycleCore.registerAgent(agent);

package/dist/sdk/evaluate.d.ts

@@ -9,5 +9,7 @@ type EvaluateFromSnapshot = (snapshotPath: string, scorers: Scorer[], opts?: Eva
 type EvaluateFn = ((agent: Agent, scorers: Scorer[], opts?: EvaluateOptions) => Promise<RecordedEvalResult>) & {
     fromSnapshot: EvaluateFromSnapshot;
 };
+/** Internal runConversation hook: step scorers must not consume final-run attribution. */
+export declare function evaluateStepScorers(agent: Agent, scorers: Scorer[], opts?: EvaluateOptions): Promise<RecordedEvalResult>;
 export declare const evaluate: EvaluateFn;
 export {};

package/dist/sdk/evaluate.js

@@ -1,4 +1,4 @@
-import { extractSkillsFromLog, extractToolEventsFromLog } from '../tool-events.js';
+import { countShellCommandsFromLog, extractSkillsFromLog, extractToolEventsFromLog } from '../tool-events.js';
 import { getRuntime } from './eval-runtime.js';
 import { emitEvalResult } from './result-capture.js';
 import { runJudgePipeline } from './judge-pipeline.js';
@@ -33,7 +33,7 @@ export class EvalScorerError extends Error {
 // Local per-call: no module-level state, no cleanup export needed.
 function makeEvaluateAgent() {
     const conversationAttributed = new WeakSet();
-    return async function evaluateAgent(agent, scorers, opts) {
+    return async function evaluateAgent(agent, scorers, opts, attributeConversation = true) {
         const toolEvents = extractToolEventsFromLog(agent.log);
         const ctx = {
             workspace: agent.workspace,
@@ -42,15 +42,16 @@ function makeEvaluateAgent() {
             toolEvents,
             runCommand: (cmd) => agent.exec(cmd),
             artifacts: createSessionArtifacts(agent.workspace, toolEvents),
+            scenarioEvidence: agent.scenarioEvidence,
         };
         const trackedLLM = opts?.llm ?? agent.llm;
         // Snapshot conversation tokens BEFORE running scorers, for first-eval attribution.
-        const before = trackedLLM.tokenUsage ?? { inputTokens: 0, outputTokens: 0 };
+        const before = agent.llm.tokenUsage ?? { inputTokens: 0, outputTokens: 0 };
         // Snapshot conversation cost too. AgentImpl's `sendTurn` accumulates
         // per-turn `costUsd` onto `trackedLLM` via `addCost`, so by the time
         // `evaluate()` runs the pre-evaluate cost is the conversation's
         // accumulated agent-turn cost.
-        const beforeCostUsd = trackedLLM.costUsd ?? 0;
+        const beforeCostUsd = agent.llm.costUsd ?? 0;
         // measure() returns the delta consumed by this evaluate call.
         const scoringStartedAt = performance.now();
         const { result: evalResult, tokens: deltaTokenUsage } = trackedLLM.measure
@@ -59,12 +60,13 @@ function makeEvaluateAgent() {
                 const r = await evaluateWithContext(ctx, scorers, { ...opts, llm: trackedLLM });
                 return { result: r, tokens: r.tokenUsage ?? { inputTokens: 0, outputTokens: 0 } };
             })();
-        // Attribute conversation tokens on the first evaluate() for this agent.
-        const isFirstEval = !conversationAttributed.has(agent);
+        // Attribute conversation tokens on the first public evaluate() for this agent.
+        const isFirstEval = attributeConversation && !conversationAttributed.has(agent);
         const conversationTokens = isFirstEval && (before.inputTokens > 0 || before.outputTokens > 0)
             ? { conversation_input_tokens: before.inputTokens, conversation_output_tokens: before.outputTokens }
             : undefined;
-        // Same first-eval attribution rule for cost. Omitted entirely when
+        // Agent usage is independent of the optional scorer LLM. Same
+        // first-eval attribution rule for cost; omit it when unavailable.
         // no conversation cost was captured (Codex / Cursor today).
         const conversationCost = isFirstEval && beforeCostUsd > 0
             ? { conversation_cost_usd: beforeCostUsd }
@@ -80,7 +82,7 @@ function makeEvaluateAgent() {
             ...(opts?.evaluationDefinitionKey
                 ? { evaluationDefinitionKey: opts.evaluationDefinitionKey }
                 : {}),
-            trial: buildTrialResult(agent.log, { ...evalResult, tokenUsage: deltaTokenUsage }, conversationTokens, conversationCost),
+            trial: buildTrialResult(agent.log, { ...evalResult, tokenUsage: deltaTokenUsage }, agent.scenarioEvidence, conversationTokens, conversationCost),
         };
         emitEvalResult({ result: recordedResult, agent });
         maybeThrowOnScorerErrors(recordedResult, opts?.onScorerError ?? 'skip');
@@ -88,6 +90,10 @@ function makeEvaluateAgent() {
     };
 }
 const evaluateAgent = makeEvaluateAgent();
+/** Internal runConversation hook: step scorers must not consume final-run attribution. */
+export function evaluateStepScorers(agent, scorers, opts) {
+    return evaluateAgent(agent, scorers, opts, false);
+}
 async function fromSnapshot(snapshotPath, scorers, opts) {
     const snapshot = await loadRunSnapshot(snapshotPath);
     const trackedLLM = opts?.llm ?? createLLMClient({ adapters: [{
@@ -110,6 +116,7 @@ async function fromSnapshot(snapshotPath, scorers, opts) {
             return sandboxExec(cmd, { cwd: workspace, env: getProcessEnv() });
         },
         artifacts: createSessionArtifacts(snapshot.workspace ?? '', snapshot.toolEvents),
+        scenarioEvidence: snapshot.scenarioEvidence ?? [],
     };
     const scoringStartedAt = performance.now();
     const evalResult = await evaluateWithContext(ctx, scorers, { ...opts, llm: trackedLLM });
@@ -121,7 +128,7 @@ async function fromSnapshot(snapshotPath, scorers, opts) {
         ...(opts?.evaluationDefinitionKey
             ? { evaluationDefinitionKey: opts.evaluationDefinitionKey }
             : {}),
-        trial: buildTrialResult(snapshot.log, evalResult),
+        trial: buildTrialResult(snapshot.log, evalResult, snapshot.scenarioEvidence),
     };
     maybeThrowOnScorerErrors(recordedResult, opts?.onScorerError ?? 'skip');
     return recordedResult;
@@ -265,8 +272,8 @@ function matchesArtifactPattern(artifactPath, pattern) {
         return artifactPath.includes(pattern);
     return pattern.test(artifactPath);
 }
-function buildTrialResult(log, result, conversationTokens, conversationCost) {
-    const nCommands = log.filter((entry) => entry.type === 'command').length;
+function buildTrialResult(log, result, scenarioEvidence, conversationTokens, conversationCost) {
+    const nCommands = countShellCommandsFromLog(log);
     const skills = extractSkillsFromLog(log);
     return {
         trial_id: 0,
@@ -285,6 +292,7 @@ function buildTrialResult(log, result, conversationTokens, conversationCost) {
         // field.
         ...conversationCost,
         session_log: [...log],
+        ...(scenarioEvidence && scenarioEvidence.length > 0 ? { scenario_evidence: structuredClone(scenarioEvidence) } : {}),
         ...(skills.length > 0 ? { skills_used: skills } : {}),
     };
 }
@@ -297,7 +305,7 @@ function getProcessEnv() {
     }
     return env;
 }
-export const evaluate = Object.assign(evaluateAgent, {
+export const evaluate = Object.assign((agent, scorers, opts) => evaluateAgent(agent, scorers, opts), {
     fromSnapshot,
 });
 function toTrialScorerResult(result) {

package/dist/sdk/index.d.ts

@@ -2,8 +2,12 @@ export { createAgent } from './agent.js';
 export { resolveAgentName, resolveCodexTransport, InvalidTransportEnvError, } from './agent-resolution.js';
 export { AgentCrashError } from './agent-crash.js';
 export { check, score, judge, toolUsage } from './scorers.js';
+export { compileScenario, compileScenarioText } from './scenario-machine-v2.js';
+export { findScenarioCaseMatches, getFinalScenarioState, getScenarioStateTimeline, ScenarioEvidenceError, wasScenarioCaseMatched, } from './scenario-evidence.js';
 export { getMcpToolCall, isMcpToolCall, findMcpToolCalls, getMcpStartupStatus, isMcpStartupStatus, getMcpApproval, isMcpApproval, findMcpApprovals, getMcpInvocation, wasMcpToolInvoked, } from './mcp-evidence.js';
 export type { McpMockApprovalRule, McpMockJsonValue } from './mcp-mock-approvals.js';
+export type { CompileScenarioResult, EvidenceEnvelope, JsonObject, JsonPrimitive, JsonValue, MatchExpr, MatchPredicate, McpContent, ScenarioArtifact, ScenarioCase, ScenarioDiagnostic, ScenarioEvidence, ScenarioMachineV2, ScenarioRejection, ScenarioSuccess, ScenarioToolError, } from './scenario-machine-v2.js';
+export type { ExpectedScenarioCaseMatch, ScenarioCaseMatchEvidence, ScenarioStateTransitionEvidence, } from './scenario-evidence.js';
 export { decideMcpToolCall, redactMcpSecrets, } from './mcp-safety.js';
 export { evaluate, EvalScorerError } from './evaluate.js';
 export { RUN_SNAPSHOT_VERSION, buildRunSnapshot, loadRunSnapshot, SnapshotParseError, SnapshotVersionError, WorkspaceMissingError, } from './snapshots.js';
@@ -15,6 +19,8 @@ export { previewReactions } from './reaction-preview.js';
 export { setRuntime, resetRuntime } from './eval-runtime.js';
 export { DEFAULT_COPY_IGNORE } from '../providers/copy-filter.js';
 export { extractToolEventsFromLog } from '../tool-events.js';
+export { collectSensitiveEnvValues, sanitizePersistenceValue, } from '../tool-event-results.js';
+export { parseEnvFile } from '../utils/env.js';
 export { createAskBus, requireAskBusForLiveBatches, AskBusTimeoutError } from './ask-bus/bus.js';
 export { toAskUserToolEvent } from './ask-bus/projection.js';
 export type { AskUserToolEvent, AskUserToolEventArguments, AskUserToolEventQuestionArgument, } from './ask-bus/projection.js';

package/dist/sdk/index.js

@@ -3,6 +3,8 @@ export { createAgent } from './agent.js';
 export { resolveAgentName, resolveCodexTransport, InvalidTransportEnvError, } from './agent-resolution.js';
 export { AgentCrashError } from './agent-crash.js';
 export { check, score, judge, toolUsage } from './scorers.js';
+export { compileScenario, compileScenarioText } from './scenario-machine-v2.js';
+export { findScenarioCaseMatches, getFinalScenarioState, getScenarioStateTimeline, ScenarioEvidenceError, wasScenarioCaseMatched, } from './scenario-evidence.js';
 export { getMcpToolCall, isMcpToolCall, findMcpToolCalls, getMcpStartupStatus, isMcpStartupStatus, getMcpApproval, isMcpApproval, findMcpApprovals, getMcpInvocation, wasMcpToolInvoked, } from './mcp-evidence.js';
 export { decideMcpToolCall, redactMcpSecrets, } from './mcp-safety.js';
 export { evaluate, EvalScorerError } from './evaluate.js';
@@ -15,6 +17,8 @@ export { previewReactions } from './reaction-preview.js';
 export { setRuntime, resetRuntime } from './eval-runtime.js';
 export { DEFAULT_COPY_IGNORE } from '../providers/copy-filter.js';
 export { extractToolEventsFromLog } from '../tool-events.js';
+export { collectSensitiveEnvValues, sanitizePersistenceValue, } from '../tool-event-results.js';
+export { parseEnvFile } from '../utils/env.js';
 export { createAskBus, requireAskBusForLiveBatches, AskBusTimeoutError } from './ask-bus/bus.js';
 export { toAskUserToolEvent } from './ask-bus/projection.js';
 export { buildAskBatchLogEntries } from './agent-result-log.js';

package/dist/sdk/lifecycle.js

@@ -1,5 +1,6 @@
 import { getCurrentCaseContext } from './case-context.js';
 import { buildDiagnosticsReport } from './diagnostics.js';
+import { countShellCommandsFromLog } from '../tool-events.js';
 const pendingAgents = new Set();
 const agentOwners = new WeakMap();
 const agentResults = new WeakMap();
@@ -131,7 +132,7 @@ async function flushCase(input) {
 function synthesizeTrialFromAgent(agent) {
     if (agent.log.length === 0)
         return null;
-    const nCommands = agent.log.filter((entry) => entry.type === 'command').length;
+    const nCommands = countShellCommandsFromLog(agent.log);
     const tokenUsage = agent.llm.tokenUsage;
     const conversationEnd = [...agent.log].reverse().find((entry) => entry.type === 'conversation_end');
     const completionReason = conversationEnd?.completion_reason ?? (agent.log.some((entry) => entry.type === 'agent_result') ? 'completed' : undefined);
@@ -147,8 +148,12 @@ function synthesizeTrialFromAgent(agent) {
             n_commands: nCommands,
             input_tokens: 0,
             output_tokens: 0,
-            conversation_input_tokens: tokenUsage?.inputTokens ?? 0,
-            conversation_output_tokens: tokenUsage?.outputTokens ?? 0,
+            ...(tokenUsage && (tokenUsage.inputTokens > 0 || tokenUsage.outputTokens > 0)
+                ? {
+                    conversation_input_tokens: tokenUsage.inputTokens,
+                    conversation_output_tokens: tokenUsage.outputTokens,
+                }
+                : {}),
             session_log: [...agent.log],
         },
         diagnostics: buildDiagnosticsReport({

package/dist/sdk/managed-session.d.ts

@@ -5,6 +5,8 @@ import type { McpSafetyOptions } from './mcp-safety.js';
 import type { LLMPort } from '../utils/llm-types.js';
 import type { AskBus } from './ask-bus/types.js';
 import type { CompiledMcpMockSession } from './mcp-mock-approvals.js';
+import { type PublicScenarioRuntimeFactory } from '../internal/direct-mcp-v2/public-scenario-runtime.js';
+import type { EvidenceEnvelope, ScenarioArtifact } from '../internal/direct-mcp-v2/types.js';
 export interface ManagedSessionDeps {
     ws: Workspace;
     agentName: AgentName;
@@ -36,6 +38,10 @@ export interface ManagedSessionDeps {
     opencodeMcpToolNames?: string[];
     /** Trusted pre-workspace compiled generated-MCP declaration. */
     scriptedMcp?: CompiledMcpMockSession;
+    scenarioArtifact?: ScenarioArtifact;
+    /** Provider adapter entrypoint; defaults to the preserved Claude bridge. */
+    scenarioRuntimeFactory?: PublicScenarioRuntimeFactory;
+    onScenarioEvidence?: (evidence: readonly EvidenceEnvelope[]) => void;
     /** Runtime-only values that must not enter session logs or persisted results. */
     sensitiveValues?: readonly string[];
 }