@wix/pathgrade 1.0.26 → 1.0.27

package/README.md

@@ -428,14 +428,14 @@ pathgrade analyze [--skill=<name>] [--dir=<path>]
 pathgrade affected [--since=<ref>] [--changed-files=<path>] [--explain] [--json]
 pathgrade preview [browser] [--last=N] [--filter=text]
 pathgrade preview-reactions --snapshot <run-snapshot.json> --reactions <file.ts>
-pathgrade report [--results-path=<path>] [--no-comment] [--comment-id=<id>]
+pathgrade report [--results-path=<path>] [--no-comment] [--comment-id=<id>] [--details-url=<url>] [--expected-head-sha=<sha>] [--strict]
 ```
 
 Useful details:
 
 - `pathgrade run --changed` computes affected evals first, writes selection metadata to `.pathgrade/selection.json`, and only then launches the selected runner adapter. Vitest is the default adapter; Jest is selected with `runner.adapter: 'jest'` or `--adapter=jest`.
 - `pathgrade preview browser` starts a local viewer on `http://localhost:3847`.
-- `pathgrade report` posts or updates a PR comment in GitHub Actions; locally it prints the markdown report and then the numeric pass rate.
+- `pathgrade report` posts or updates a PR comment in GitHub Actions; locally it prints the markdown report and then the numeric pass rate. Provider orchestrators can add a `--details-url`, suppress stale updates with `--expected-head-sha`, and opt into surfaced API failures with `--strict`.
 - `pathgrade validate --affected` is a strict mode for CI: every discovered eval must either live under a `SKILL.md` anchor or export valid `__pathgradeMeta`.
 
 Run `pathgrade --help` for the full help text.

package/dist/agents/claude/sdk-message-projector.js

@@ -30,6 +30,7 @@ import { parseClaudeSdkMcpToolName } from './mcp-tool-name.js';
 import { applyObservedToolResult, extractObservedToolResults, } from './tool-results.js';
 export function projectSdkMessages(input) {
     let sessionId;
+    let resolvedModel;
     let initSkills;
     let assistantText = '';
     let resultText = '';
@@ -52,6 +53,9 @@ export function projectSdkMessages(input) {
                     sessionId = sid;
                 const sub = msg.subtype;
                 if (sub === 'init') {
+                    const model = msg.model;
+                    if (typeof model === 'string' && model)
+                        resolvedModel = model;
                     const skills = msg.skills;
                     if (Array.isArray(skills))
                         initSkills = skills.filter((s) => typeof s === 'string');
@@ -153,6 +157,7 @@ export function projectSdkMessages(input) {
         traceOutput,
         toolEvents: finalToolEvents,
         runtimePoliciesApplied: [],
+        ...(resolvedModel ? { resolvedModel } : {}),
         inputTokens,
         outputTokens,
         ...(cacheCreationInputTokens !== undefined ? { cacheCreationInputTokens } : {}),

package/dist/agents/codex-app-server/agent.js

@@ -397,7 +397,7 @@ export class CodexAppServerAgent extends BaseAgent {
                         message: info.message ?? 'app-server exited',
                         pid: info.pid,
                         signal: info.signal,
-                        sensitiveValues,
+                        sensitiveValues, resolvedModel: model,
                     });
                 }
                 if (turn.turnFailed) {
@@ -406,14 +406,14 @@ export class CodexAppServerAgent extends BaseAgent {
                         activeTurn: turn,
                         exitCode: 1,
                         message: turn.failureMessage ?? 'turn failed',
-                        sensitiveValues,
+                        sensitiveValues, resolvedModel: model,
                     });
                 }
                 await authoritativeTurnReady;
                 if (turn.turnFailed) {
                     return assembleTurnResult({
                         askBus, activeTurn: turn, exitCode: 1,
-                        message: turn.failureMessage ?? 'turn failed', sensitiveValues,
+                        message: turn.failureMessage ?? 'turn failed', sensitiveValues, resolvedModel: model,
                     });
                 }
                 return assembleTurnResult({
@@ -421,7 +421,7 @@ export class CodexAppServerAgent extends BaseAgent {
                     activeTurn: turn,
                     exitCode: 0,
                     message: turn.assistantMessageParts.join('\n\n'),
-                    sensitiveValues,
+                    sensitiveValues, resolvedModel: model,
                 });
             }
             finally {
@@ -612,7 +612,7 @@ function shouldEnableMcpElicitations(options) {
     return options?.mcpConfigOrigin === 'generated_mock';
 }
 function assembleTurnResult(args) {
-    const { askBus, activeTurn, exitCode, message, pid, signal, sensitiveValues = [] } = args;
+    const { askBus, activeTurn, exitCode, message, pid, signal, sensitiveValues = [], resolvedModel } = args;
     for (const pending of activeTurn.pendingMcpDenials.splice(0)) {
         activeTurn.nonAskToolEvents.push(buildPolicyDeniedMcpToolEvent(activeTurn.turnNumber, pending));
     }
@@ -641,7 +641,7 @@ function assembleTurnResult(args) {
         visibleAssistantMessageSource: 'assistant_message',
         exitCode,
         traceOutput: rawOutput,
-        toolEvents,
+        toolEvents, resolvedModel,
         ...(exitCode !== 0
             ? {
                 crashInfo: {

package/dist/agents/codex.js

@@ -25,6 +25,7 @@ export class CodexAgent extends TranscriptAgent {
             exitCode: result.exitCode,
             traceOutput: rawOutput,
             toolEvents,
+            resolvedModel: resolveCodexModel(options?.model),
         };
     }
 }

package/dist/agents/cursor.js

@@ -241,6 +241,7 @@ export class CursorAgent extends BaseAgent {
             timedOut: result.timedOut,
             toolEvents,
             runtimePoliciesApplied: appliedRuntimePolicies,
+            ...(options?.model ? { resolvedModel: options.model } : {}),
             ...(parsed.tokenUsage
                 ? { inputTokens: parsed.tokenUsage.inputTokens, outputTokens: parsed.tokenUsage.outputTokens }
                 : {}),

package/dist/agents/opencode.js

@@ -6,7 +6,7 @@ import { buildSummary, enrichSkillEvents } from '../tool-events.js';
 import { collectSensitiveEnvValues, sanitizePersistenceValue, sanitizeToolEventResult, } from '../tool-event-results.js';
 import { readStagedMcpServers } from '../providers/mcp-config.js';
 import { removeSandboxRoot } from '../providers/sandbox-lifecycle.js';
-import { attachTurnResultSensitiveValues } from '../sdk/turn-result-secrets.js';
+import { attachTurnResultSensitiveValues, cloneTurnResultWithSensitiveValues } from '../sdk/turn-result-secrets.js';
 import { attachOriginalMcpInput } from '../sdk/mcp-event-input.js';
 import { attachToolEventSensitiveValues } from '../sdk/tool-event-secrets.js';
 import { currentOpenCodePlatformKey, OPENCODE_RUNTIME_LOCK, } from './opencode/contract.js';
@@ -451,7 +451,9 @@ class OpenCodeSession {
                 throw new Error('OpenCode protocol error: resumed session ID changed');
             }
             this.sessionId = parsed.sessionId;
-            return parsed.result;
+            return cloneTurnResultWithSensitiveValues(parsed.result, {
+                resolvedModel: this.runtimePolicy.model,
+            });
         }
         catch (error) {
             this.failed = true;

package/dist/commands/report.d.ts

@@ -24,6 +24,14 @@ export interface ReportOptions {
     noComment?: boolean;
     /** Override the comment-id used in the dedup marker. See `resolveCommentId`. */
     commentId?: string;
+    /** Provider-neutral URL for a richer external report. */
+    detailsUrl?: string;
+    /** Provider-neutral status notice included in the report body. */
+    notice?: string;
+    /** Surface missing context, malformed results, and GitHub API failures. */
+    strict?: boolean;
+    /** Skip a comment update when the pull request has advanced. */
+    expectedHeadSha?: string;
 }
 /**
  * Resolve the comment-id used for the dedup marker.
@@ -33,7 +41,7 @@ export interface ReportOptions {
  */
 export declare function resolveCommentId(explicit?: string): string;
 /**
- * Entry point for `pathgrade report`. Never throws and never sets a
- * failing exit code — even on malformed inputs or network errors.
+ * Entry point for `pathgrade report`. The default display-only mode never
+ * fails CI. Strict mode is intended for orchestrators that own publication.
  */
 export declare function runReport(cwd: string, opts?: ReportOptions): Promise<void>;

package/dist/commands/report.js

@@ -66,8 +66,8 @@ function printMarkdownAndPassRate(markdown, passRate) {
     console.log(String(passRate));
 }
 /**
- * Entry point for `pathgrade report`. Never throws and never sets a
- * failing exit code — even on malformed inputs or network errors.
+ * Entry point for `pathgrade report`. The default display-only mode never
+ * fails CI. Strict mode is intended for orchestrators that own publication.
  */
 export async function runReport(cwd, opts = {}) {
     const resolvedPath = path.resolve(cwd, opts.resultsPath ?? DEFAULT_RESULTS_PATH);
@@ -77,6 +77,9 @@ export async function runReport(cwd, opts = {}) {
     // Branch point: when true, post the comment and print only the pass rate.
     const shouldPost = inCI && !noComment;
     const prContext = shouldPost ? resolvePrContext(process.env) : null;
+    if (opts.strict && shouldPost && !prContext) {
+        throw new Error('pathgrade report: strict PR commenting requires GITHUB_TOKEN, GITHUB_REPOSITORY, and pull-request context');
+    }
     let report = null;
     let loadError = null;
     try {
@@ -95,6 +98,8 @@ export async function runReport(cwd, opts = {}) {
                 await postOrUpdateComment(prContext, {
                     commentId,
                     body: markdown,
+                    strict: opts.strict,
+                    expectedHeadSha: opts.expectedHeadSha,
                 });
                 console.log('0');
             }
@@ -108,15 +113,31 @@ export async function runReport(cwd, opts = {}) {
             await postOrUpdateComment(prContext, {
                 commentId,
                 body: MISSING_RESULTS_BODY,
+                strict: opts.strict,
+                expectedHeadSha: opts.expectedHeadSha,
             });
         }
         // Always emit a pass rate line so downstream capture doesn't explode.
         console.log('0');
+        if (opts.strict)
+            throw new Error(`pathgrade report: ${loadError}`);
         return;
     }
-    const markdown = formatReportMarkdown(report, { commentId });
+    const markdown = formatReportMarkdown(report, {
+        commentId,
+        detailsUrl: opts.detailsUrl,
+        notice: opts.notice,
+    });
     if (prContext) {
-        await postOrUpdateComment(prContext, { commentId, body: markdown });
+        const postResult = await postOrUpdateComment(prContext, {
+            commentId,
+            body: markdown,
+            strict: opts.strict,
+            expectedHeadSha: opts.expectedHeadSha,
+        });
+        if (postResult === 'stale') {
+            console.error('pathgrade report: pull-request head advanced; skipped stale comment update');
+        }
         // Only the pass rate goes to stdout — the markdown lives on the PR.
         console.log(String(report.overall_pass_rate));
         return;

package/dist/pathgrade.js

@@ -156,10 +156,27 @@ export async function runPathgradeCli(options = {}) {
         const reportArgs = args.slice(1);
         const resultsPathFlag = reportArgs.find(a => a.startsWith('--results-path='));
         const commentIdFlag = reportArgs.find(a => a.startsWith('--comment-id='));
+        const detailsUrlFlag = reportArgs.find(a => a.startsWith('--details-url='));
+        const noticeFlag = reportArgs.find(a => a.startsWith('--notice='));
+        const expectedHeadShaFlag = reportArgs.find(a => a.startsWith('--expected-head-sha='));
         const noComment = reportArgs.includes('--no-comment');
+        const strict = reportArgs.includes('--strict');
         const resultsPath = resultsPathFlag ? resultsPathFlag.split('=').slice(1).join('=') : undefined;
         const commentId = commentIdFlag ? commentIdFlag.split('=').slice(1).join('=') : undefined;
-        await runReport(process.cwd(), { resultsPath, commentId, noComment });
+        const detailsUrl = detailsUrlFlag ? detailsUrlFlag.split('=').slice(1).join('=') : undefined;
+        const notice = noticeFlag ? noticeFlag.split('=').slice(1).join('=') : undefined;
+        const expectedHeadSha = expectedHeadShaFlag
+            ? expectedHeadShaFlag.split('=').slice(1).join('=')
+            : undefined;
+        await runReport(process.cwd(), {
+            resultsPath,
+            commentId,
+            detailsUrl,
+            notice,
+            expectedHeadSha,
+            noComment,
+            strict,
+        });
         return;
     }
     if (command === 'run' || !command || command.startsWith('-')) {
@@ -234,6 +251,10 @@ function printHelp(cliName) {
                      [--results-path=<path>]   Override results.json location
                      [--no-comment]            Print markdown to stdout; do not post
                      [--comment-id=<id>]       Override comment marker (default: $GITHUB_WORKFLOW:$GITHUB_JOB)
+                     [--details-url=<url>]     Add a provider-neutral full-report link
+                     [--notice=<text>]         Add a provider-neutral report status notice
+                     [--expected-head-sha=<sha>] Skip stale PR comment updates
+                     [--strict]                Fail on missing context, results, or GitHub errors
     ${cliName} affected               Print eval files affected by a change-set (one per line)
                      [--since=<ref>]           Diff <ref>...HEAD (overrides git auto-detection)
                      [--changed-files=<path>]  Newline-delimited repo-relative file list

package/dist/reporters/github-comment.d.ts

@@ -10,6 +10,10 @@
 import type { PathgradeReport, PathgradeSelectionReport } from '../types.js';
 export interface FormatOptions {
     commentId: string;
+    /** Provider-neutral URL for a richer external report. */
+    detailsUrl?: string;
+    /** Provider-neutral status notice rendered above report details. */
+    notice?: string;
 }
 /** Minimal body posted when `.pathgrade/results.json` is missing. */
 export declare const MISSING_RESULTS_BODY = "Pathgrade evals did not produce results. Check the workflow logs.";
@@ -55,14 +59,19 @@ export interface PostOptions {
     commentId: string;
     /** Body WITHOUT the marker — the marker is prepended if not already present. */
     body: string;
+    /** Throw instead of swallowing GitHub API failures. */
+    strict?: boolean;
+    /** Skip the update if the pull request has advanced to another head. */
+    expectedHeadSha?: string;
 }
+export type PostResult = 'created' | 'updated' | 'stale';
 /**
  * Find an existing PR comment carrying `<!-- pathgrade:${commentId} -->`
- * and update it; otherwise create a new one. Swallows all errors (logs
- * to stderr) — `pathgrade report` must never fail CI.
+ * and update it; otherwise create a new one. Default mode logs and swallows
+ * transport errors; strict mode rethrows them for an owning orchestrator.
  *
  * Bodies longer than GitHub's 65,536-character limit are truncated with a
  * sentinel pointing to workflow artifacts, preserving the leading dedup
  * marker so subsequent runs still find and update this comment.
  */
-export declare function postOrUpdateComment(ctx: PrContext, opts: PostOptions): Promise<void>;
+export declare function postOrUpdateComment(ctx: PrContext, opts: PostOptions): Promise<PostResult | undefined>;

package/dist/reporters/github-comment.js

@@ -26,6 +26,12 @@ export function formatNoAffectedEvalsMarkdown(selection) {
     return lines.join('\n');
 }
 export function commentMarker(commentId) {
+    if (commentId.length === 0
+        || commentId.length > 128
+        || /[<>\r\n]/.test(commentId)
+        || commentId.includes('--')) {
+        throw new Error('Pathgrade comment id must be 1-128 characters and cannot contain HTML comment delimiters');
+    }
     return `<!-- pathgrade:${commentId} -->`;
 }
 function pct(n) {
@@ -59,9 +65,19 @@ export function formatReportMarkdown(report, opts) {
     lines.push('');
     lines.push(`### ${icon} Pathgrade report`);
     lines.push('');
+    lines.push(`**Last run:** \`${report.timestamp}\``);
+    lines.push('');
     lines.push(`**Pass rate:** ${pct(p)}  |  ` +
         `**pass@${totalTrials}:** ${pct(overallPassAtK)}  |  ` +
         `**pass^${totalTrials}:** ${pct(overallPassPowK)}`);
+    if (opts.detailsUrl) {
+        lines.push('');
+        lines.push(`**Details:** [Open the full report](${formatLinkDestination(opts.detailsUrl)})`);
+    }
+    if (opts.notice) {
+        lines.push('');
+        lines.push(`> ${opts.notice.replace(/[\r\n]+/g, ' ').trim()}`);
+    }
     if (report.threshold != null) {
         lines.push('');
         lines.push(`Threshold: ${pct(report.threshold)} — ${report.status.toUpperCase()}`);
@@ -100,14 +116,14 @@ export function formatSelectionSection(selection) {
     out.push('### Selection');
     const total = selection.selected.length + selection.skipped.length;
     if (selection.global_match) {
-        out.push(`Ran **all ${total}** evals — global trigger \`${selection.global_match}\` matched.`);
+        out.push(`Selected **all ${total}** eval files — global trigger \`${selection.global_match}\` matched.`);
         return out.join('\n');
     }
     if (selection.skipped.length === 0 && selection.selected.length > 0) {
-        out.push(`Ran **all ${total}** evals — every eval had a matching change.`);
+        out.push(`Selected **all ${total}** eval files — every eval had a matching change.`);
         return out.join('\n');
     }
-    out.push(`Ran **${selection.selected.length} of ${total}** evals based on changes vs \`${selection.base_ref}\`.`);
+    out.push(`Selected **${selection.selected.length} of ${total}** eval files based on changes vs \`${selection.base_ref}\`.`);
     if (selection.skipped.length > 0) {
         out.push('');
         out.push(`<details><summary>${selection.skipped.length} skipped (unaffected)</summary>`);
@@ -171,13 +187,36 @@ function apiHeaders(token) {
     };
 }
 async function listPrComments(ctx) {
-    const url = `https://api.github.com/repos/${ctx.owner}/${ctx.repo}/issues/${ctx.prNumber}/comments?per_page=100`;
+    const comments = [];
+    for (let page = 1; page <= 100; page += 1) {
+        const url = `https://api.github.com/repos/${ctx.owner}/${ctx.repo}/issues/${ctx.prNumber}/comments?per_page=100&page=${page}`;
+        const res = await fetch(url, { headers: apiHeaders(ctx.token) });
+        if (!res.ok) {
+            throw new Error(`GET ${url} returned ${res.status}`);
+        }
+        const data = await res.json();
+        if (!Array.isArray(data)) {
+            throw new Error(`GET ${url} returned a non-array response`);
+        }
+        for (const value of data) {
+            if (isGithubComment(value))
+                comments.push(value);
+        }
+        if (data.length < 100)
+            break;
+    }
+    return comments;
+}
+async function currentPrHeadSha(ctx) {
+    const url = `https://api.github.com/repos/${ctx.owner}/${ctx.repo}/pulls/${ctx.prNumber}`;
     const res = await fetch(url, { headers: apiHeaders(ctx.token) });
-    if (!res.ok) {
+    if (!res.ok)
         throw new Error(`GET ${url} returned ${res.status}`);
+    const data = await res.json();
+    if (!isRecord(data) || !isRecord(data.head) || typeof data.head.sha !== 'string') {
+        throw new Error(`GET ${url} returned no pull-request head SHA`);
     }
-    const data = (await res.json());
-    return Array.isArray(data) ? data : [];
+    return data.head.sha;
 }
 async function createPrComment(ctx, body) {
     const url = `https://api.github.com/repos/${ctx.owner}/${ctx.repo}/issues/${ctx.prNumber}/comments`;
@@ -206,8 +245,8 @@ const GITHUB_COMMENT_MAX = 65536;
 const TRUNCATION_SENTINEL = '\n\n_…output truncated to fit GitHub\'s comment size limit — see the workflow artifacts for the full report._';
 /**
  * Find an existing PR comment carrying `<!-- pathgrade:${commentId} -->`
- * and update it; otherwise create a new one. Swallows all errors (logs
- * to stderr) — `pathgrade report` must never fail CI.
+ * and update it; otherwise create a new one. Default mode logs and swallows
+ * transport errors; strict mode rethrows them for an owning orchestrator.
  *
  * Bodies longer than GitHub's 65,536-character limit are truncated with a
  * sentinel pointing to workflow artifacts, preserving the leading dedup
@@ -221,19 +260,51 @@ export async function postOrUpdateComment(ctx, opts) {
         : withMarker.slice(0, GITHUB_COMMENT_MAX - TRUNCATION_SENTINEL.length) +
             TRUNCATION_SENTINEL;
     try {
+        if (opts.expectedHeadSha) {
+            const currentHead = await currentPrHeadSha(ctx);
+            if (currentHead !== opts.expectedHeadSha)
+                return 'stale';
+        }
         const existing = await listPrComments(ctx);
         const match = existing.find((c) => typeof c.body === 'string' && c.body.includes(marker));
         if (match) {
             await updatePrComment(ctx, match.id, body);
+            return 'updated';
         }
         else {
             await createPrComment(ctx, body);
+            return 'created';
         }
     }
     catch (err) {
         const message = err instanceof Error ? err.message : String(err);
         console.error(`pathgrade report: failed to post PR comment — ${message}`);
+        if (opts.strict)
+            throw err;
+    }
+}
+function formatLinkDestination(value) {
+    let url;
+    try {
+        url = new URL(value);
     }
+    catch {
+        throw new Error('Pathgrade details URL must be an absolute HTTP(S) URL');
+    }
+    if (url.protocol !== 'https:' && url.protocol !== 'http:') {
+        throw new Error('Pathgrade details URL must be an absolute HTTP(S) URL');
+    }
+    return url.toString().replace(/\)/g, '%29');
+}
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function isGithubComment(value) {
+    return isRecord(value)
+        && typeof value.id === 'number'
+        && Number.isSafeInteger(value.id)
+        && value.id > 0
+        && typeof value.body === 'string';
 }
 function formatGroupDetails(group) {
     const out = [];

package/dist/reporting/artifacts.js

@@ -1,18 +1,21 @@
 import path from 'node:path';
 import fs from 'fs-extra';
+import { collectSensitiveEnvValues, sanitizePersistenceValue, } from '../tool-event-results.js';
 export async function writePathgradeArtifacts(artifactRoot, built) {
     await fs.ensureDir(path.join(artifactRoot, 'traces'));
+    const sensitiveValues = collectSensitiveEnvValues(process.env);
+    const report = sanitizePersistenceValue(built.report, sensitiveValues);
     const gitignorePath = path.join(artifactRoot, '.gitignore');
     if (!(await fs.pathExists(gitignorePath))) {
         await fs.writeFile(gitignorePath, '*\n');
     }
     const traceFiles = [];
     for (const trace of built.traces) {
-        await fs.writeJson(path.join(artifactRoot, trace.traceFile), trace.trials, { spaces: 2 });
+        await fs.writeJson(path.join(artifactRoot, trace.traceFile), sanitizePersistenceValue(trace.trials, sensitiveValues), { spaces: 2 });
         traceFiles.push(trace.traceFile);
     }
     const resultsPath = path.join(artifactRoot, 'results.json');
-    await fs.writeJson(resultsPath, built.report, { spaces: 2 });
+    await fs.writeJson(resultsPath, report, { spaces: 2 });
     return {
         resultsPath,
         traceFiles,

package/dist/sdk/agent.js

@@ -4,7 +4,7 @@ import { lifecycleCore } from './lifecycle.js';
 import { ChatSessionImpl } from './chat.js';
 import { runConversation } from './converse.js';
 import { createPersona } from './persona.js';
-import { evaluate } from './evaluate.js';
+import { evaluateStepScorers } from './evaluate.js';
 import { createManagedSession } from './managed-session.js';
 import { createAgentLLM } from '../utils/llm.js';
 import { buildRunSnapshot } from './snapshots.js';
@@ -39,6 +39,7 @@ class AgentImpl {
     llm;
     timeoutSetting;
     modelOpt;
+    resolvedModel;
     conversationWindowOpt;
     interactionMode = null;
     _messages = [];
@@ -100,6 +101,7 @@ class AgentImpl {
         return {
             name: this.agentName,
             ...(this.modelOpt ? { requestedModel: this.modelOpt } : {}),
+            ...(this.resolvedModel ? { resolvedModel: this.resolvedModel } : {}),
             transport: this.scenarioArtifact && (this.agentName === 'opencode' || this.agentName === 'cursor')
                 ? 'acp'
                 : resolveExecutionTransport(this.agentName, this.transport),
@@ -145,6 +147,9 @@ class AgentImpl {
         return Math.ceil((turns * 80_000 + 200_000) / 1000);
     }
     accumulateTurnUsage(turnResult) {
+        if (turnResult.resolvedModel) {
+            this.resolvedModel = turnResult.resolvedModel;
+        }
         if (turnResult.inputTokens || turnResult.outputTokens) {
             this.llm.addTokens?.(turnResult.inputTokens ?? 0, turnResult.outputTokens ?? 0);
         }
@@ -303,7 +308,7 @@ class AgentImpl {
         // scorer judge calls accumulate on the same tracker.
         const agent = this;
         const runStepScorers = async (scorers) => {
-            const result = await evaluate(agent, scorers, { llm: this.llm });
+            const result = await evaluateStepScorers(agent, scorers, { llm: this.llm });
             const score = result.score;
             if (score === undefined) {
                 throw new Error('Step scorer evaluation did not produce a score');

package/dist/sdk/evaluate.d.ts

@@ -9,5 +9,7 @@ type EvaluateFromSnapshot = (snapshotPath: string, scorers: Scorer[], opts?: Eva
 type EvaluateFn = ((agent: Agent, scorers: Scorer[], opts?: EvaluateOptions) => Promise<RecordedEvalResult>) & {
     fromSnapshot: EvaluateFromSnapshot;
 };
+/** Internal runConversation hook: step scorers must not consume final-run attribution. */
+export declare function evaluateStepScorers(agent: Agent, scorers: Scorer[], opts?: EvaluateOptions): Promise<RecordedEvalResult>;
 export declare const evaluate: EvaluateFn;
 export {};

package/dist/sdk/evaluate.js

@@ -1,4 +1,4 @@
-import { extractSkillsFromLog, extractToolEventsFromLog } from '../tool-events.js';
+import { countShellCommandsFromLog, extractSkillsFromLog, extractToolEventsFromLog } from '../tool-events.js';
 import { getRuntime } from './eval-runtime.js';
 import { emitEvalResult } from './result-capture.js';
 import { runJudgePipeline } from './judge-pipeline.js';
@@ -33,7 +33,7 @@ export class EvalScorerError extends Error {
 // Local per-call: no module-level state, no cleanup export needed.
 function makeEvaluateAgent() {
     const conversationAttributed = new WeakSet();
-    return async function evaluateAgent(agent, scorers, opts) {
+    return async function evaluateAgent(agent, scorers, opts, attributeConversation = true) {
         const toolEvents = extractToolEventsFromLog(agent.log);
         const ctx = {
             workspace: agent.workspace,
@@ -46,12 +46,12 @@ function makeEvaluateAgent() {
         };
         const trackedLLM = opts?.llm ?? agent.llm;
         // Snapshot conversation tokens BEFORE running scorers, for first-eval attribution.
-        const before = trackedLLM.tokenUsage ?? { inputTokens: 0, outputTokens: 0 };
+        const before = agent.llm.tokenUsage ?? { inputTokens: 0, outputTokens: 0 };
         // Snapshot conversation cost too. AgentImpl's `sendTurn` accumulates
         // per-turn `costUsd` onto `trackedLLM` via `addCost`, so by the time
         // `evaluate()` runs the pre-evaluate cost is the conversation's
         // accumulated agent-turn cost.
-        const beforeCostUsd = trackedLLM.costUsd ?? 0;
+        const beforeCostUsd = agent.llm.costUsd ?? 0;
         // measure() returns the delta consumed by this evaluate call.
         const scoringStartedAt = performance.now();
         const { result: evalResult, tokens: deltaTokenUsage } = trackedLLM.measure
@@ -60,12 +60,13 @@ function makeEvaluateAgent() {
                 const r = await evaluateWithContext(ctx, scorers, { ...opts, llm: trackedLLM });
                 return { result: r, tokens: r.tokenUsage ?? { inputTokens: 0, outputTokens: 0 } };
             })();
-        // Attribute conversation tokens on the first evaluate() for this agent.
-        const isFirstEval = !conversationAttributed.has(agent);
+        // Attribute conversation tokens on the first public evaluate() for this agent.
+        const isFirstEval = attributeConversation && !conversationAttributed.has(agent);
         const conversationTokens = isFirstEval && (before.inputTokens > 0 || before.outputTokens > 0)
             ? { conversation_input_tokens: before.inputTokens, conversation_output_tokens: before.outputTokens }
             : undefined;
-        // Same first-eval attribution rule for cost. Omitted entirely when
+        // Agent usage is independent of the optional scorer LLM. Same
+        // first-eval attribution rule for cost; omit it when unavailable.
         // no conversation cost was captured (Codex / Cursor today).
         const conversationCost = isFirstEval && beforeCostUsd > 0
             ? { conversation_cost_usd: beforeCostUsd }
@@ -89,6 +90,10 @@ function makeEvaluateAgent() {
     };
 }
 const evaluateAgent = makeEvaluateAgent();
+/** Internal runConversation hook: step scorers must not consume final-run attribution. */
+export function evaluateStepScorers(agent, scorers, opts) {
+    return evaluateAgent(agent, scorers, opts, false);
+}
 async function fromSnapshot(snapshotPath, scorers, opts) {
     const snapshot = await loadRunSnapshot(snapshotPath);
     const trackedLLM = opts?.llm ?? createLLMClient({ adapters: [{
@@ -268,7 +273,7 @@ function matchesArtifactPattern(artifactPath, pattern) {
     return pattern.test(artifactPath);
 }
 function buildTrialResult(log, result, scenarioEvidence, conversationTokens, conversationCost) {
-    const nCommands = log.filter((entry) => entry.type === 'command').length;
+    const nCommands = countShellCommandsFromLog(log);
     const skills = extractSkillsFromLog(log);
     return {
         trial_id: 0,
@@ -300,7 +305,7 @@ function getProcessEnv() {
     }
     return env;
 }
-export const evaluate = Object.assign(evaluateAgent, {
+export const evaluate = Object.assign((agent, scorers, opts) => evaluateAgent(agent, scorers, opts), {
     fromSnapshot,
 });
 function toTrialScorerResult(result) {

package/dist/sdk/index.d.ts

@@ -19,6 +19,8 @@ export { previewReactions } from './reaction-preview.js';
 export { setRuntime, resetRuntime } from './eval-runtime.js';
 export { DEFAULT_COPY_IGNORE } from '../providers/copy-filter.js';
 export { extractToolEventsFromLog } from '../tool-events.js';
+export { collectSensitiveEnvValues, sanitizePersistenceValue, } from '../tool-event-results.js';
+export { parseEnvFile } from '../utils/env.js';
 export { createAskBus, requireAskBusForLiveBatches, AskBusTimeoutError } from './ask-bus/bus.js';
 export { toAskUserToolEvent } from './ask-bus/projection.js';
 export type { AskUserToolEvent, AskUserToolEventArguments, AskUserToolEventQuestionArgument, } from './ask-bus/projection.js';

package/dist/sdk/index.js

@@ -17,6 +17,8 @@ export { previewReactions } from './reaction-preview.js';
 export { setRuntime, resetRuntime } from './eval-runtime.js';
 export { DEFAULT_COPY_IGNORE } from '../providers/copy-filter.js';
 export { extractToolEventsFromLog } from '../tool-events.js';
+export { collectSensitiveEnvValues, sanitizePersistenceValue, } from '../tool-event-results.js';
+export { parseEnvFile } from '../utils/env.js';
 export { createAskBus, requireAskBusForLiveBatches, AskBusTimeoutError } from './ask-bus/bus.js';
 export { toAskUserToolEvent } from './ask-bus/projection.js';
 export { buildAskBatchLogEntries } from './agent-result-log.js';

package/dist/sdk/lifecycle.js

@@ -1,5 +1,6 @@
 import { getCurrentCaseContext } from './case-context.js';
 import { buildDiagnosticsReport } from './diagnostics.js';
+import { countShellCommandsFromLog } from '../tool-events.js';
 const pendingAgents = new Set();
 const agentOwners = new WeakMap();
 const agentResults = new WeakMap();
@@ -131,7 +132,7 @@ async function flushCase(input) {
 function synthesizeTrialFromAgent(agent) {
     if (agent.log.length === 0)
         return null;
-    const nCommands = agent.log.filter((entry) => entry.type === 'command').length;
+    const nCommands = countShellCommandsFromLog(agent.log);
     const tokenUsage = agent.llm.tokenUsage;
     const conversationEnd = [...agent.log].reverse().find((entry) => entry.type === 'conversation_end');
     const completionReason = conversationEnd?.completion_reason ?? (agent.log.some((entry) => entry.type === 'agent_result') ? 'completed' : undefined);
@@ -147,8 +148,12 @@ function synthesizeTrialFromAgent(agent) {
             n_commands: nCommands,
             input_tokens: 0,
             output_tokens: 0,
-            conversation_input_tokens: tokenUsage?.inputTokens ?? 0,
-            conversation_output_tokens: tokenUsage?.outputTokens ?? 0,
+            ...(tokenUsage && (tokenUsage.inputTokens > 0 || tokenUsage.outputTokens > 0)
+                ? {
+                    conversation_input_tokens: tokenUsage.inputTokens,
+                    conversation_output_tokens: tokenUsage.outputTokens,
+                }
+                : {}),
             session_log: [...agent.log],
         },
         diagnostics: buildDiagnosticsReport({

package/dist/sdk/types.d.ts

@@ -17,6 +17,8 @@ export interface AgentExecutionMetadata {
     name: AgentName;
     /** Model override requested by the caller; not necessarily the provider-resolved model. */
     requestedModel?: string;
+    /** Concrete model observed or selected for the executed turn. */
+    resolvedModel?: string;
     transport?: AgentExecutionTransport;
     interactionMode?: AgentInteractionMode;
 }

package/dist/tool-event-results.d.ts

@@ -1,6 +1,6 @@
 import type { ToolEventResult } from './tool-events.js';
 export declare const TOOL_RESULT_MAX_CHARS: number;
-export declare function collectSensitiveEnvValues(env?: Readonly<Record<string, string>>): string[];
+export declare function collectSensitiveEnvValues(env?: Readonly<Record<string, string | undefined>>): string[];
 /**
  * Clone a persistence payload while removing secrets from both structured
  * containers and any other strings that repeat their values. The second pass

package/dist/tool-event-results.js

@@ -41,6 +41,7 @@ const NON_SECRET_ENVIRONMENT_KEY_NAMES = new Set(['tokencount', 'tokenizersparal
 const BOUNDARY_ONLY_SENSITIVE_VALUE_MAX_LENGTH = 1;
 export function collectSensitiveEnvValues(env) {
     return [...new Set(Object.entries(env ?? {})
+            .filter((entry) => typeof entry[1] === 'string')
             .filter(([key, value]) => isSecretEnvironmentKey(key) && value.length > 0)
             .map(([, value]) => value))]
         .sort((a, b) => b.length - a.length);
@@ -153,7 +154,7 @@ function redactCredentialShapes(value, sensitiveValues, explicitSensitiveValues)
         redacted = redacted.replace(/([a-z][a-z0-9+.-]*:\/\/[^\s:/@]+:)[^\s/@]+(@)/gi, '$1<redacted>$2');
     }
     if (redacted.includes('=') || redacted.includes(':')) {
-        redacted = redacted.replace(/(\b(?:api[_-]?key|access[_-]?token|refresh[_-]?token|client[_-]?secret|password|passwd|session(?:id)?|cookie)\s*[=:]\s*)[^\s,;"'}]+/gi, '$1<redacted>');
+        redacted = redacted.replace(/(\b(?:api[_-]?key|access[_-]?token|refresh[_-]?token|client[_-]?secret|token|secret|auth|password|passwd|session(?:id)?|cookie)\s*[=:]\s*)[^\s,;"'}]+/gi, '$1<redacted>');
     }
     return redacted;
 }

package/dist/tool-events.d.ts

@@ -64,6 +64,11 @@ export declare function extractToolEventsFromLog(log: ReadonlyArray<{
     type: string;
     tool_event?: ToolEvent;
 }>): ToolEvent[];
+/** Count shell commands across legacy command logs and provider tool events. */
+export declare function countShellCommandsFromLog(log: ReadonlyArray<{
+    type: string;
+    tool_event?: ToolEvent;
+}>): number;
 /**
  * Extract deduplicated skill names from tool events that have action 'use_skill'.
  */

package/dist/tool-events.js

@@ -101,6 +101,11 @@ export function extractToolEventsFromLog(log) {
         .filter((e) => e.type === 'tool_event' && e.tool_event != null)
         .map((e) => e.tool_event);
 }
+/** Count shell commands across legacy command logs and provider tool events. */
+export function countShellCommandsFromLog(log) {
+    return log.filter((entry) => entry.type === 'command'
+        || (entry.type === 'tool_event' && entry.tool_event?.action === 'run_shell')).length;
+}
 /**
  * Extract deduplicated skill names from tool events that have action 'use_skill'.
  */

package/dist/types.d.ts

@@ -291,6 +291,8 @@ export interface AgentTurnResult {
     timedOut?: boolean;
     toolEvents: import('./tool-events.js').ToolEvent[];
     runtimePoliciesApplied?: RuntimePolicyDescriptor[];
+    /** Concrete model that executed this turn, as resolved by the provider or runner. */
+    resolvedModel?: string;
     inputTokens?: number;
     outputTokens?: number;
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wix/pathgrade",
-  "version": "1.0.26",
+  "version": "1.0.27",
   "packageManager": "yarn@4.12.0",
   "description": "Evaluate whether AI agents discover and use your skills correctly",
   "exports": {
@@ -140,5 +140,5 @@
     "typescript": "^5.9.3",
     "zod": "4.3.6"
   },
-  "falconPackageHash": "23390180e5694ad3ec546fb637b601045310b46b4d7f10927a63e0db"
+  "falconPackageHash": "5c7c088ce2f8c51308dde9f92d3f259465bbf34021afddb3af39d2d9"
 }