@wix/pathgrade 1.0.26 → 1.0.28

package/dist/sdk/evaluate.js

@@ -1,4 +1,4 @@
-import { extractSkillsFromLog, extractToolEventsFromLog } from '../tool-events.js';
+import { countShellCommandsFromLog, extractSkillsFromLog, extractToolEventsFromLog } from '../tool-events.js';
 import { getRuntime } from './eval-runtime.js';
 import { emitEvalResult } from './result-capture.js';
 import { runJudgePipeline } from './judge-pipeline.js';
@@ -33,7 +33,7 @@ export class EvalScorerError extends Error {
 // Local per-call: no module-level state, no cleanup export needed.
 function makeEvaluateAgent() {
     const conversationAttributed = new WeakSet();
-    return async function evaluateAgent(agent, scorers, opts) {
+    return async function evaluateAgent(agent, scorers, opts, attributeConversation = true) {
         const toolEvents = extractToolEventsFromLog(agent.log);
         const ctx = {
             workspace: agent.workspace,
@@ -46,12 +46,12 @@ function makeEvaluateAgent() {
         };
         const trackedLLM = opts?.llm ?? agent.llm;
         // Snapshot conversation tokens BEFORE running scorers, for first-eval attribution.
-        const before = trackedLLM.tokenUsage ?? { inputTokens: 0, outputTokens: 0 };
+        const before = agent.llm.tokenUsage ?? { inputTokens: 0, outputTokens: 0 };
         // Snapshot conversation cost too. AgentImpl's `sendTurn` accumulates
         // per-turn `costUsd` onto `trackedLLM` via `addCost`, so by the time
         // `evaluate()` runs the pre-evaluate cost is the conversation's
         // accumulated agent-turn cost.
-        const beforeCostUsd = trackedLLM.costUsd ?? 0;
+        const beforeCostUsd = agent.llm.costUsd ?? 0;
         // measure() returns the delta consumed by this evaluate call.
         const scoringStartedAt = performance.now();
         const { result: evalResult, tokens: deltaTokenUsage } = trackedLLM.measure
@@ -60,12 +60,13 @@ function makeEvaluateAgent() {
                 const r = await evaluateWithContext(ctx, scorers, { ...opts, llm: trackedLLM });
                 return { result: r, tokens: r.tokenUsage ?? { inputTokens: 0, outputTokens: 0 } };
             })();
-        // Attribute conversation tokens on the first evaluate() for this agent.
-        const isFirstEval = !conversationAttributed.has(agent);
+        // Attribute conversation tokens on the first public evaluate() for this agent.
+        const isFirstEval = attributeConversation && !conversationAttributed.has(agent);
         const conversationTokens = isFirstEval && (before.inputTokens > 0 || before.outputTokens > 0)
             ? { conversation_input_tokens: before.inputTokens, conversation_output_tokens: before.outputTokens }
             : undefined;
-        // Same first-eval attribution rule for cost. Omitted entirely when
+        // Agent usage is independent of the optional scorer LLM. Same
+        // first-eval attribution rule for cost; omit it when unavailable.
         // no conversation cost was captured (Codex / Cursor today).
         const conversationCost = isFirstEval && beforeCostUsd > 0
             ? { conversation_cost_usd: beforeCostUsd }
@@ -89,6 +90,10 @@ function makeEvaluateAgent() {
     };
 }
 const evaluateAgent = makeEvaluateAgent();
+/** Internal runConversation hook: step scorers must not consume final-run attribution. */
+export function evaluateStepScorers(agent, scorers, opts) {
+    return evaluateAgent(agent, scorers, opts, false);
+}
 async function fromSnapshot(snapshotPath, scorers, opts) {
     const snapshot = await loadRunSnapshot(snapshotPath);
     const trackedLLM = opts?.llm ?? createLLMClient({ adapters: [{
@@ -268,7 +273,7 @@ function matchesArtifactPattern(artifactPath, pattern) {
     return pattern.test(artifactPath);
 }
 function buildTrialResult(log, result, scenarioEvidence, conversationTokens, conversationCost) {
-    const nCommands = log.filter((entry) => entry.type === 'command').length;
+    const nCommands = countShellCommandsFromLog(log);
     const skills = extractSkillsFromLog(log);
     return {
         trial_id: 0,
@@ -300,7 +305,7 @@ function getProcessEnv() {
     }
     return env;
 }
-export const evaluate = Object.assign(evaluateAgent, {
+export const evaluate = Object.assign((agent, scorers, opts) => evaluateAgent(agent, scorers, opts), {
     fromSnapshot,
 });
 function toTrialScorerResult(result) {

package/dist/sdk/index.d.ts

@@ -19,6 +19,8 @@ export { previewReactions } from './reaction-preview.js';
 export { setRuntime, resetRuntime } from './eval-runtime.js';
 export { DEFAULT_COPY_IGNORE } from '../providers/copy-filter.js';
 export { extractToolEventsFromLog } from '../tool-events.js';
+export { collectSensitiveEnvValues, sanitizePersistenceValue, } from '../tool-event-results.js';
+export { parseEnvFile } from '../utils/env.js';
 export { createAskBus, requireAskBusForLiveBatches, AskBusTimeoutError } from './ask-bus/bus.js';
 export { toAskUserToolEvent } from './ask-bus/projection.js';
 export type { AskUserToolEvent, AskUserToolEventArguments, AskUserToolEventQuestionArgument, } from './ask-bus/projection.js';

package/dist/sdk/index.js

@@ -17,6 +17,8 @@ export { previewReactions } from './reaction-preview.js';
 export { setRuntime, resetRuntime } from './eval-runtime.js';
 export { DEFAULT_COPY_IGNORE } from '../providers/copy-filter.js';
 export { extractToolEventsFromLog } from '../tool-events.js';
+export { collectSensitiveEnvValues, sanitizePersistenceValue, } from '../tool-event-results.js';
+export { parseEnvFile } from '../utils/env.js';
 export { createAskBus, requireAskBusForLiveBatches, AskBusTimeoutError } from './ask-bus/bus.js';
 export { toAskUserToolEvent } from './ask-bus/projection.js';
 export { buildAskBatchLogEntries } from './agent-result-log.js';

package/dist/sdk/lifecycle.js

@@ -1,8 +1,14 @@
 import { getCurrentCaseContext } from './case-context.js';
 import { buildDiagnosticsReport } from './diagnostics.js';
-const pendingAgents = new Set();
-const agentOwners = new WeakMap();
-const agentResults = new WeakMap();
+import { countShellCommandsFromLog } from '../tool-events.js';
+const lifecycleStateKey = Symbol.for('@wix/pathgrade/lifecycle-state');
+const sharedLifecycle = globalThis;
+const lifecycleState = sharedLifecycle[lifecycleStateKey] ?? (sharedLifecycle[lifecycleStateKey] = {
+    pendingAgents: new Set(),
+    agentOwners: new WeakMap(),
+    agentResults: new WeakMap(),
+});
+const { pendingAgents, agentOwners, agentResults } = lifecycleState;
 function currentAgentOwner() {
     const current = getCurrentCaseContext();
     if (current.status !== 'active')
@@ -131,7 +137,7 @@ async function flushCase(input) {
 function synthesizeTrialFromAgent(agent) {
     if (agent.log.length === 0)
         return null;
-    const nCommands = agent.log.filter((entry) => entry.type === 'command').length;
+    const nCommands = countShellCommandsFromLog(agent.log);
     const tokenUsage = agent.llm.tokenUsage;
     const conversationEnd = [...agent.log].reverse().find((entry) => entry.type === 'conversation_end');
     const completionReason = conversationEnd?.completion_reason ?? (agent.log.some((entry) => entry.type === 'agent_result') ? 'completed' : undefined);
@@ -147,8 +153,12 @@ function synthesizeTrialFromAgent(agent) {
             n_commands: nCommands,
             input_tokens: 0,
             output_tokens: 0,
-            conversation_input_tokens: tokenUsage?.inputTokens ?? 0,
-            conversation_output_tokens: tokenUsage?.outputTokens ?? 0,
+            ...(tokenUsage && (tokenUsage.inputTokens > 0 || tokenUsage.outputTokens > 0)
+                ? {
+                    conversation_input_tokens: tokenUsage.inputTokens,
+                    conversation_output_tokens: tokenUsage.outputTokens,
+                }
+                : {}),
             session_log: [...agent.log],
         },
         diagnostics: buildDiagnosticsReport({

package/dist/sdk/result-capture.js

@@ -1,5 +1,8 @@
 import { getCurrentCaseContext } from './case-context.js';
-const observers = new Set();
+const observerRegistryKey = Symbol.for('@wix/pathgrade/eval-result-observers');
+const globalRegistry = globalThis;
+const observers = globalRegistry[observerRegistryKey]
+    ?? (globalRegistry[observerRegistryKey] = new Set());
 export function subscribeToEvalResults(observer, options = {}) {
     if (options.owner === 'adapter' && options.key) {
         removeObserverByOwnerAndKey(options.owner, options.key);

package/dist/sdk/types.d.ts

@@ -17,6 +17,8 @@ export interface AgentExecutionMetadata {
     name: AgentName;
     /** Model override requested by the caller; not necessarily the provider-resolved model. */
     requestedModel?: string;
+    /** Concrete model observed or selected for the executed turn. */
+    resolvedModel?: string;
     transport?: AgentExecutionTransport;
     interactionMode?: AgentInteractionMode;
 }

package/dist/tool-event-results.d.ts

@@ -1,6 +1,6 @@
 import type { ToolEventResult } from './tool-events.js';
 export declare const TOOL_RESULT_MAX_CHARS: number;
-export declare function collectSensitiveEnvValues(env?: Readonly<Record<string, string>>): string[];
+export declare function collectSensitiveEnvValues(env?: Readonly<Record<string, string | undefined>>): string[];
 /**
  * Clone a persistence payload while removing secrets from both structured
  * containers and any other strings that repeat their values. The second pass

package/dist/tool-event-results.js

@@ -41,6 +41,7 @@ const NON_SECRET_ENVIRONMENT_KEY_NAMES = new Set(['tokencount', 'tokenizersparal
 const BOUNDARY_ONLY_SENSITIVE_VALUE_MAX_LENGTH = 1;
 export function collectSensitiveEnvValues(env) {
     return [...new Set(Object.entries(env ?? {})
+            .filter((entry) => typeof entry[1] === 'string')
             .filter(([key, value]) => isSecretEnvironmentKey(key) && value.length > 0)
             .map(([, value]) => value))]
         .sort((a, b) => b.length - a.length);
@@ -153,7 +154,7 @@ function redactCredentialShapes(value, sensitiveValues, explicitSensitiveValues)
         redacted = redacted.replace(/([a-z][a-z0-9+.-]*:\/\/[^\s:/@]+:)[^\s/@]+(@)/gi, '$1<redacted>$2');
     }
     if (redacted.includes('=') || redacted.includes(':')) {
-        redacted = redacted.replace(/(\b(?:api[_-]?key|access[_-]?token|refresh[_-]?token|client[_-]?secret|password|passwd|session(?:id)?|cookie)\s*[=:]\s*)[^\s,;"'}]+/gi, '$1<redacted>');
+        redacted = redacted.replace(/(\b(?:api[_-]?key|access[_-]?token|refresh[_-]?token|client[_-]?secret|token|secret|auth|password|passwd|session(?:id)?|cookie)\s*[=:]\s*)[^\s,;"'}]+/gi, '$1<redacted>');
     }
     return redacted;
 }

package/dist/tool-events.d.ts

@@ -64,6 +64,11 @@ export declare function extractToolEventsFromLog(log: ReadonlyArray<{
     type: string;
     tool_event?: ToolEvent;
 }>): ToolEvent[];
+/** Count shell commands across legacy command logs and provider tool events. */
+export declare function countShellCommandsFromLog(log: ReadonlyArray<{
+    type: string;
+    tool_event?: ToolEvent;
+}>): number;
 /**
  * Extract deduplicated skill names from tool events that have action 'use_skill'.
  */

package/dist/tool-events.js

@@ -101,6 +101,11 @@ export function extractToolEventsFromLog(log) {
         .filter((e) => e.type === 'tool_event' && e.tool_event != null)
         .map((e) => e.tool_event);
 }
+/** Count shell commands across legacy command logs and provider tool events. */
+export function countShellCommandsFromLog(log) {
+    return log.filter((entry) => entry.type === 'command'
+        || (entry.type === 'tool_event' && entry.tool_event?.action === 'run_shell')).length;
+}
 /**
  * Extract deduplicated skill names from tool events that have action 'use_skill'.
  */

package/dist/types.d.ts

@@ -150,6 +150,14 @@ export interface LogEntry {
 }
 export interface TrialResult {
     trial_id: number;
+    /** Stable semantic case identity in schema-v2 reports. */
+    case_id?: string;
+    /** Unique identity for one runner-owned execution of the case. */
+    attempt_id?: string;
+    /** One-based invocation index. */
+    attempt_index?: number;
+    /** Display-safe runner outcome for this specific attempt. */
+    runner_outcome?: import('./runners/model.js').AttemptOutcome['kind'];
     name?: string;
     reward?: number;
     scorer_results: ScorerResult[];
@@ -192,9 +200,22 @@ export interface TrialResult {
 }
 export interface EvalReport {
     task: string;
-    pass_rate: number;
-    pass_at_k: number;
-    pass_pow_k: number;
+    /** Canonical status for this report or consolidated group when available. */
+    status?: 'pass' | 'fail';
+    /** Schema-v2 canonical average score. */
+    mean_reward?: number;
+    /** Binary success frequency, absent when attempts are ineligible. */
+    success_rate?: number;
+    /** Finite-sample pass@k values keyed by k in schema v2; number only for v1 reads. */
+    pass_at_k?: Record<string, number> | number;
+    pass_at_k_method?: 'finite_sample_unbiased';
+    pass_at_k_unavailable_reason?: PassAtKUnavailableReason;
+    eligible_case_count?: number;
+    attempts_per_case?: number;
+    /** @deprecated Version-1 artifact compatibility only. */
+    pass_rate?: number;
+    /** @deprecated Version-1 artifact compatibility only. New writers omit it. */
+    pass_pow_k?: number;
     trials: TrialResult[];
     skills_used: string[];
 }
@@ -208,7 +229,8 @@ export type StrippedTrialResult = Omit<TrialResult, 'session_log' | 'conversatio
 /**
  * Per-group entry in the consolidated `.pathgrade/results.json` report.
  */
-export type PathgradeGroupReport = Omit<EvalReport, 'trials'> & {
+export type PathgradeGroupReport = Omit<EvalReport, 'trials' | 'status'> & {
+    status?: 'pass' | 'fail';
     trials: StrippedTrialResult[];
     /** Relative path from `.pathgrade/` to the trace file for this group. */
     trace_file: string;
@@ -239,12 +261,16 @@ export interface PathgradeSelectionReport {
  * external consumers can gate on schema revisions.
  */
 export interface PathgradeReport {
-    version: 1;
+    version: 1 | 2;
     timestamp: string;
     /** `ci.threshold` from Pathgrade config or legacy plugin config, if configured. */
     threshold?: number;
     /** Weighted average of every individual trial score across all groups. */
     overall_pass_rate: number;
+    /** Canonical schema-v2 average reward. */
+    overall_mean_reward?: number;
+    attempts_requested?: number;
+    attempts_completed?: number;
     /**
      * Threshold check result. When `threshold` is set: `'pass'` iff
      * `overall_pass_rate >= threshold`. Otherwise: `'pass'` iff every trial
@@ -259,6 +285,7 @@ export interface PathgradeReport {
      */
     selection?: PathgradeSelectionReport;
 }
+export type PassAtKUnavailableReason = 'non_binary_reward' | 'incomplete_attempts' | 'case_identity_mismatch';
 export interface TrialPaths {
     root: string;
     workspace: string;
@@ -291,6 +318,8 @@ export interface AgentTurnResult {
     timedOut?: boolean;
     toolEvents: import('./tool-events.js').ToolEvent[];
     runtimePoliciesApplied?: RuntimePolicyDescriptor[];
+    /** Concrete model that executed this turn, as resolved by the provider or runner. */
+    resolvedModel?: string;
     inputTokens?: number;
     outputTokens?: number;
     /**

package/dist/viewer.html

@@ -838,12 +838,13 @@
     applyTheme(getTheme());
 
     // ─── State ─────────────────────────────────────────────────
-    let reports = [];
-    let sortKey = 'date';
-    let sortDir = 'desc';
+    let reports = [], sortKey = 'date', sortDir = 'desc';
     const $ = id => document.getElementById(id);
     const esc = s => s ? String(s).replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;') : '';
-
+    const meanReward = r => r.mean_reward ?? r.pass_rate ?? 0;
+    const reportStatus = r => r.status === 'pass' || r.status === 'fail' ? r.status : (meanReward(r) >= 0.5 ? 'pass' : 'fail');
+    const passAtKSortValue = r => typeof r.pass_at_k === 'number' ? r.pass_at_k : (r.pass_at_k ? Math.max(...Object.values(r.pass_at_k)) : -1);
+    const passAtKDisplay = r => typeof r.pass_at_k === 'number' ? (r.pass_at_k * 100).toFixed(0) + '% v1' : r.pass_at_k ? Object.entries(r.pass_at_k).map(([k, value]) => '@' + k + ' ' + (value * 100).toFixed(0) + '%').join(' · ') : r.pass_at_k_unavailable_reason ? '— ' + r.pass_at_k_unavailable_reason.replaceAll('_', ' ') : '—';
     function parseDate(file) {
       // filename format: 2026-04-08T14-25-03-134Z  →  2026-04-08T14:25:03.134
       const m = file.match(/(\d{4}-\d{2}-\d{2})T(\d{2})-(\d{2})-(\d{2})-(\d+)/);
@@ -877,10 +878,10 @@
         switch (sortKey) {
           case 'task': va = (a.task || '').toLowerCase(); vb = (b.task || '').toLowerCase(); break;
           case 'date': va = parseDate(a.file); vb = parseDate(b.file); break;
-          case 'pass_rate': va = a.pass_rate || 0; vb = b.pass_rate || 0; break;
-          case 'pass_at_k': va = a.pass_at_k ?? -1; vb = b.pass_at_k ?? -1; break;
-          case 'pass_pow_k': va = a.pass_pow_k ?? -1; vb = b.pass_pow_k ?? -1; break;
-          case 'status': va = a.pass_rate >= 0.5 ? 1 : 0; vb = b.pass_rate >= 0.5 ? 1 : 0; break;
+          case 'mean_reward': va = meanReward(a); vb = meanReward(b); break;
+          case 'success_rate': va = a.success_rate ?? -1; vb = b.success_rate ?? -1; break;
+          case 'pass_at_k': va = passAtKSortValue(a); vb = passAtKSortValue(b); break;
+          case 'status': va = reportStatus(a) === 'pass' ? 1 : 0; vb = reportStatus(b) === 'pass' ? 1 : 0; break;
           default: return 0;
         }
         if (va < vb) return sortDir === 'asc' ? -1 : 1;
@@ -936,24 +937,23 @@
       const header = `<div class="reports-header">
         ${colHeader('task', 'Task')}
         ${colHeader('date', 'Date')}
-        ${colHeader('pass_rate', 'Pass Rate', 'center')}
+        ${colHeader('mean_reward', 'Mean Reward', 'center')}
+        ${colHeader('success_rate', 'Success Rate', 'center')}
         ${colHeader('pass_at_k', 'pass@k', 'center')}
-        ${colHeader('pass_pow_k', 'pass^k', 'center')}
         ${colHeader('status', 'Status', 'center')}
         <div></div>
       </div>`;
 
       const rows = sorted.map(r => {
-        const pr = (r.pass_rate * 100).toFixed(0);
-        const isPass = r.pass_rate >= 0.5;
+        const reward = (meanReward(r) * 100).toFixed(0), isPass = reportStatus(r) === 'pass';
         const ts = r.timestamp || parseDate(r.file);
 
         return `<a class="report-row" onclick="navigate('detail','${esc(r.file)}')" href="javascript:void(0)">
       <div class="report-task">${esc(r.task)}</div>
       <div class="report-date">${formatDate(ts)}</div>
-      <div class="report-metric"><div class="val">${pr}%</div></div>
-      <div class="report-metric"><div class="val">${r.pass_at_k != null ? (r.pass_at_k * 100).toFixed(0) + '%' : '—'}</div></div>
-      <div class="report-metric"><div class="val">${r.pass_pow_k != null ? (r.pass_pow_k * 100).toFixed(0) + '%' : '—'}</div></div>
+      <div class="report-metric"><div class="val">${reward}%</div></div>
+      <div class="report-metric"><div class="val">${r.success_rate != null ? (r.success_rate * 100).toFixed(0) + '%' : '—'}</div></div>
+      <div class="report-metric"><div class="val">${esc(passAtKDisplay(r))}</div></div>
       <div style="text-align:center"><span class="badge ${isPass ? 'badge-pass' : 'badge-fail'}">${isPass ? 'PASS' : 'FAIL'}</span></div>
       <button class="delete-btn" onclick="event.preventDefault();event.stopPropagation();deleteReport('${esc(r.file)}')" title="Delete report">&#x2715;</button>
     </a>`;
@@ -975,9 +975,9 @@
       const totalTokens = r.trials.reduce((s, t) => s + (t.input_tokens || 0) + (t.output_tokens || 0) + (t.conversation_input_tokens || 0) + (t.conversation_output_tokens || 0), 0);
 
       const stats = `
-    <div class="stat-pill"><div class="val">${(r.pass_rate * 100).toFixed(1)}%</div><div class="lbl">Pass Rate</div></div>
-    <div class="stat-pill"><div class="val">${r.pass_at_k != null ? (r.pass_at_k * 100).toFixed(1) + '%' : '—'}</div><div class="lbl">pass@k</div></div>
-    <div class="stat-pill"><div class="val">${r.pass_pow_k != null ? (r.pass_pow_k * 100).toFixed(1) + '%' : '—'}</div><div class="lbl">pass^k</div></div>
+    <div class="stat-pill"><div class="val">${(meanReward(r) * 100).toFixed(1)}%</div><div class="lbl">Mean Reward</div></div>
+    <div class="stat-pill"><div class="val">${r.success_rate != null ? (r.success_rate * 100).toFixed(1) + '%' : '—'}</div><div class="lbl">Success Rate</div></div>
+    <div class="stat-pill"><div class="val">${esc(passAtKDisplay(r))}</div><div class="lbl">pass@k</div></div>
     <div class="stat-pill"><div class="val">${(avgDur / 1000).toFixed(1)}s</div><div class="lbl">Avg Duration</div></div>
     <div class="stat-pill"><div class="val">~${totalTokens}</div><div class="lbl">Total Tokens</div></div>
     <div class="stat-pill"><div class="val">${r.skills_used?.join(', ') || 'none'}</div><div class="lbl">Skills</div></div>
@@ -999,7 +999,7 @@
     }
     function renderTrial(t, idx) {
       const evaluated = t.reward !== undefined;
-      const pass = evaluated && t.reward >= 0.5;
+      const pass = evaluated && (t.runner_outcome === undefined ? t.reward >= 0.5 : t.runner_outcome === 'passed' && t.reward === 1);
       const dur = ((t.duration_ms || 0) / 1000).toFixed(1);
       const tokens = (t.input_tokens || 0) + (t.output_tokens || 0);
       const convTokens = (t.conversation_input_tokens || 0) + (t.conversation_output_tokens || 0);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wix/pathgrade",
-  "version": "1.0.26",
+  "version": "1.0.28",
   "packageManager": "yarn@4.12.0",
   "description": "Evaluate whether AI agents discover and use your skills correctly",
   "exports": {
@@ -140,5 +140,5 @@
     "typescript": "^5.9.3",
     "zod": "4.3.6"
   },
-  "falconPackageHash": "23390180e5694ad3ec546fb637b601045310b46b4d7f10927a63e0db"
+  "falconPackageHash": "f8df1d8950a1ce263fb945ab13722fdb8328a2f761153de0df90d0e9"
 }