@wix/pathgrade 1.0.27 → 1.0.29

package/dist/types.d.ts

@@ -150,6 +150,14 @@ export interface LogEntry {
 }
 export interface TrialResult {
     trial_id: number;
+    /** Stable semantic case identity in schema-v2 reports. */
+    case_id?: string;
+    /** Unique identity for one runner-owned execution of the case. */
+    attempt_id?: string;
+    /** One-based invocation index. */
+    attempt_index?: number;
+    /** Display-safe runner outcome for this specific attempt. */
+    runner_outcome?: import('./runners/model.js').AttemptOutcome['kind'];
     name?: string;
     reward?: number;
     scorer_results: ScorerResult[];
@@ -192,9 +200,22 @@ export interface TrialResult {
 }
 export interface EvalReport {
     task: string;
-    pass_rate: number;
-    pass_at_k: number;
-    pass_pow_k: number;
+    /** Canonical status for this report or consolidated group when available. */
+    status?: 'pass' | 'fail';
+    /** Schema-v2 canonical average score. */
+    mean_reward?: number;
+    /** Binary success frequency, absent when attempts are ineligible. */
+    success_rate?: number;
+    /** Finite-sample pass@k values keyed by k in schema v2; number only for v1 reads. */
+    pass_at_k?: Record<string, number> | number;
+    pass_at_k_method?: 'finite_sample_unbiased';
+    pass_at_k_unavailable_reason?: PassAtKUnavailableReason;
+    eligible_case_count?: number;
+    attempts_per_case?: number;
+    /** @deprecated Version-1 artifact compatibility only. */
+    pass_rate?: number;
+    /** @deprecated Version-1 artifact compatibility only. New writers omit it. */
+    pass_pow_k?: number;
     trials: TrialResult[];
     skills_used: string[];
 }
@@ -208,7 +229,8 @@ export type StrippedTrialResult = Omit<TrialResult, 'session_log' | 'conversatio
 /**
  * Per-group entry in the consolidated `.pathgrade/results.json` report.
  */
-export type PathgradeGroupReport = Omit<EvalReport, 'trials'> & {
+export type PathgradeGroupReport = Omit<EvalReport, 'trials' | 'status'> & {
+    status?: 'pass' | 'fail';
     trials: StrippedTrialResult[];
     /** Relative path from `.pathgrade/` to the trace file for this group. */
     trace_file: string;
@@ -239,12 +261,16 @@ export interface PathgradeSelectionReport {
  * external consumers can gate on schema revisions.
  */
 export interface PathgradeReport {
-    version: 1;
+    version: 1 | 2;
     timestamp: string;
     /** `ci.threshold` from Pathgrade config or legacy plugin config, if configured. */
     threshold?: number;
     /** Weighted average of every individual trial score across all groups. */
     overall_pass_rate: number;
+    /** Canonical schema-v2 average reward. */
+    overall_mean_reward?: number;
+    attempts_requested?: number;
+    attempts_completed?: number;
     /**
      * Threshold check result. When `threshold` is set: `'pass'` iff
      * `overall_pass_rate >= threshold`. Otherwise: `'pass'` iff every trial
@@ -259,6 +285,7 @@ export interface PathgradeReport {
      */
     selection?: PathgradeSelectionReport;
 }
+export type PassAtKUnavailableReason = 'non_binary_reward' | 'incomplete_attempts' | 'case_identity_mismatch';
 export interface TrialPaths {
     root: string;
     workspace: string;
@@ -396,6 +423,8 @@ export interface AgentSessionOptions {
     scriptedMcpHost?: import('./providers/scripted-mcp-mock-host.js').ScriptedMcpMockHost;
     /** Parent-owned public ScenarioMachineV2 bridge; never serialized into child config. */
     scenarioMcpServers?: Record<string, import('@anthropic-ai/claude-agent-sdk').McpSdkServerConfigWithInstance>;
+    /** RuntimeLock-admitted Claude executable; wins over developer overrides for ScenarioMachineV2 sessions. */
+    admittedClaudeCodeExecutable?: string;
     /** Parent-owned admitted Codex app-server mount; projected only into thread/start memory. */
     codexScenarioMount?: import('./agents/codex-app-server/scenario-mount.js').CodexScenarioMount;
     /** Runtime-only values inherited by drivers and persistence sinks for redaction. */

package/dist/viewer.html

@@ -838,12 +838,13 @@
     applyTheme(getTheme());
 
     // ─── State ─────────────────────────────────────────────────
-    let reports = [];
-    let sortKey = 'date';
-    let sortDir = 'desc';
+    let reports = [], sortKey = 'date', sortDir = 'desc';
     const $ = id => document.getElementById(id);
     const esc = s => s ? String(s).replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;') : '';
-
+    const meanReward = r => r.mean_reward ?? r.pass_rate ?? 0;
+    const reportStatus = r => r.status === 'pass' || r.status === 'fail' ? r.status : (meanReward(r) >= 0.5 ? 'pass' : 'fail');
+    const passAtKSortValue = r => typeof r.pass_at_k === 'number' ? r.pass_at_k : (r.pass_at_k ? Math.max(...Object.values(r.pass_at_k)) : -1);
+    const passAtKDisplay = r => typeof r.pass_at_k === 'number' ? (r.pass_at_k * 100).toFixed(0) + '% v1' : r.pass_at_k ? Object.entries(r.pass_at_k).map(([k, value]) => '@' + k + ' ' + (value * 100).toFixed(0) + '%').join(' · ') : r.pass_at_k_unavailable_reason ? '— ' + r.pass_at_k_unavailable_reason.replaceAll('_', ' ') : '—';
     function parseDate(file) {
       // filename format: 2026-04-08T14-25-03-134Z  →  2026-04-08T14:25:03.134
       const m = file.match(/(\d{4}-\d{2}-\d{2})T(\d{2})-(\d{2})-(\d{2})-(\d+)/);
@@ -877,10 +878,10 @@
         switch (sortKey) {
           case 'task': va = (a.task || '').toLowerCase(); vb = (b.task || '').toLowerCase(); break;
           case 'date': va = parseDate(a.file); vb = parseDate(b.file); break;
-          case 'pass_rate': va = a.pass_rate || 0; vb = b.pass_rate || 0; break;
-          case 'pass_at_k': va = a.pass_at_k ?? -1; vb = b.pass_at_k ?? -1; break;
-          case 'pass_pow_k': va = a.pass_pow_k ?? -1; vb = b.pass_pow_k ?? -1; break;
-          case 'status': va = a.pass_rate >= 0.5 ? 1 : 0; vb = b.pass_rate >= 0.5 ? 1 : 0; break;
+          case 'mean_reward': va = meanReward(a); vb = meanReward(b); break;
+          case 'success_rate': va = a.success_rate ?? -1; vb = b.success_rate ?? -1; break;
+          case 'pass_at_k': va = passAtKSortValue(a); vb = passAtKSortValue(b); break;
+          case 'status': va = reportStatus(a) === 'pass' ? 1 : 0; vb = reportStatus(b) === 'pass' ? 1 : 0; break;
           default: return 0;
         }
         if (va < vb) return sortDir === 'asc' ? -1 : 1;
@@ -936,24 +937,23 @@
       const header = `<div class="reports-header">
         ${colHeader('task', 'Task')}
         ${colHeader('date', 'Date')}
-        ${colHeader('pass_rate', 'Pass Rate', 'center')}
+        ${colHeader('mean_reward', 'Mean Reward', 'center')}
+        ${colHeader('success_rate', 'Success Rate', 'center')}
         ${colHeader('pass_at_k', 'pass@k', 'center')}
-        ${colHeader('pass_pow_k', 'pass^k', 'center')}
         ${colHeader('status', 'Status', 'center')}
         <div></div>
       </div>`;
 
       const rows = sorted.map(r => {
-        const pr = (r.pass_rate * 100).toFixed(0);
-        const isPass = r.pass_rate >= 0.5;
+        const reward = (meanReward(r) * 100).toFixed(0), isPass = reportStatus(r) === 'pass';
         const ts = r.timestamp || parseDate(r.file);
 
         return `<a class="report-row" onclick="navigate('detail','${esc(r.file)}')" href="javascript:void(0)">
       <div class="report-task">${esc(r.task)}</div>
       <div class="report-date">${formatDate(ts)}</div>
-      <div class="report-metric"><div class="val">${pr}%</div></div>
-      <div class="report-metric"><div class="val">${r.pass_at_k != null ? (r.pass_at_k * 100).toFixed(0) + '%' : '—'}</div></div>
-      <div class="report-metric"><div class="val">${r.pass_pow_k != null ? (r.pass_pow_k * 100).toFixed(0) + '%' : '—'}</div></div>
+      <div class="report-metric"><div class="val">${reward}%</div></div>
+      <div class="report-metric"><div class="val">${r.success_rate != null ? (r.success_rate * 100).toFixed(0) + '%' : '—'}</div></div>
+      <div class="report-metric"><div class="val">${esc(passAtKDisplay(r))}</div></div>
       <div style="text-align:center"><span class="badge ${isPass ? 'badge-pass' : 'badge-fail'}">${isPass ? 'PASS' : 'FAIL'}</span></div>
       <button class="delete-btn" onclick="event.preventDefault();event.stopPropagation();deleteReport('${esc(r.file)}')" title="Delete report">&#x2715;</button>
     </a>`;
@@ -975,9 +975,9 @@
       const totalTokens = r.trials.reduce((s, t) => s + (t.input_tokens || 0) + (t.output_tokens || 0) + (t.conversation_input_tokens || 0) + (t.conversation_output_tokens || 0), 0);
 
       const stats = `
-    <div class="stat-pill"><div class="val">${(r.pass_rate * 100).toFixed(1)}%</div><div class="lbl">Pass Rate</div></div>
-    <div class="stat-pill"><div class="val">${r.pass_at_k != null ? (r.pass_at_k * 100).toFixed(1) + '%' : '—'}</div><div class="lbl">pass@k</div></div>
-    <div class="stat-pill"><div class="val">${r.pass_pow_k != null ? (r.pass_pow_k * 100).toFixed(1) + '%' : '—'}</div><div class="lbl">pass^k</div></div>
+    <div class="stat-pill"><div class="val">${(meanReward(r) * 100).toFixed(1)}%</div><div class="lbl">Mean Reward</div></div>
+    <div class="stat-pill"><div class="val">${r.success_rate != null ? (r.success_rate * 100).toFixed(1) + '%' : '—'}</div><div class="lbl">Success Rate</div></div>
+    <div class="stat-pill"><div class="val">${esc(passAtKDisplay(r))}</div><div class="lbl">pass@k</div></div>
     <div class="stat-pill"><div class="val">${(avgDur / 1000).toFixed(1)}s</div><div class="lbl">Avg Duration</div></div>
     <div class="stat-pill"><div class="val">~${totalTokens}</div><div class="lbl">Total Tokens</div></div>
     <div class="stat-pill"><div class="val">${r.skills_used?.join(', ') || 'none'}</div><div class="lbl">Skills</div></div>
@@ -999,7 +999,7 @@
     }
     function renderTrial(t, idx) {
       const evaluated = t.reward !== undefined;
-      const pass = evaluated && t.reward >= 0.5;
+      const pass = evaluated && (t.runner_outcome === undefined ? t.reward >= 0.5 : t.runner_outcome === 'passed' && t.reward === 1);
       const dur = ((t.duration_ms || 0) / 1000).toFixed(1);
       const tokens = (t.input_tokens || 0) + (t.output_tokens || 0);
       const convTokens = (t.conversation_input_tokens || 0) + (t.conversation_output_tokens || 0);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wix/pathgrade",
-  "version": "1.0.27",
+  "version": "1.0.29",
   "packageManager": "yarn@4.12.0",
   "description": "Evaluate whether AI agents discover and use your skills correctly",
   "exports": {
@@ -140,5 +140,5 @@
     "typescript": "^5.9.3",
     "zod": "4.3.6"
   },
-  "falconPackageHash": "5c7c088ce2f8c51308dde9f92d3f259465bbf34021afddb3af39d2d9"
+  "falconPackageHash": "126198d020df14b3e6fceef3b0325ac5e84411c5e1b544a0167de604"
 }