@wix/pathgrade 1.0.26 → 1.0.28

package/dist/reporters/github-comment.js

@@ -26,11 +26,24 @@ export function formatNoAffectedEvalsMarkdown(selection) {
     return lines.join('\n');
 }
 export function commentMarker(commentId) {
+    if (commentId.length === 0
+        || commentId.length > 128
+        || /[<>\r\n]/.test(commentId)
+        || commentId.includes('--')) {
+        throw new Error('Pathgrade comment id must be 1-128 characters and cannot contain HTML comment delimiters');
+    }
     return `<!-- pathgrade:${commentId} -->`;
 }
 function pct(n) {
     return `${(n * 100).toFixed(1)}%`;
 }
+function formatPassAtK(value, reason) {
+    if (typeof value === 'number')
+        return `${pct(value)} (legacy v1)`;
+    if (value)
+        return Object.entries(value).map(([k, metric]) => `pass@${k} ${pct(metric)}`).join(', ');
+    return reason ? `Unavailable (${reason.replaceAll('_', ' ')})` : 'Unavailable';
+}
 function durationSeconds(ms) {
     return `${(ms / 1000).toFixed(1)}s`;
 }
@@ -49,30 +62,36 @@ function escapeTableCell(value) {
  * posted as a GitHub issue comment body.
  */
 export function formatReportMarkdown(report, opts) {
-    const totalTrials = report.groups.reduce((n, g) => n + g.trials.length, 0);
-    const p = report.overall_pass_rate;
-    const overallPassAtK = totalTrials > 0 ? 1 - Math.pow(1 - p, totalTrials) : 0;
-    const overallPassPowK = totalTrials > 0 ? Math.pow(p, totalTrials) : 0;
+    const meanReward = report.overall_mean_reward ?? report.overall_pass_rate;
     const icon = report.status === 'pass' ? '✅' : '❌';
     const lines = [];
     lines.push(commentMarker(opts.commentId));
     lines.push('');
     lines.push(`### ${icon} Pathgrade report`);
     lines.push('');
-    lines.push(`**Pass rate:** ${pct(p)}  |  ` +
-        `**pass@${totalTrials}:** ${pct(overallPassAtK)}  |  ` +
-        `**pass^${totalTrials}:** ${pct(overallPassPowK)}`);
+    lines.push(`**Last run:** \`${report.timestamp}\``);
+    lines.push('');
+    lines.push(`**Mean reward:** ${pct(meanReward)}` +
+        (report.attempts_requested ? `  |  **Attempts:** ${report.attempts_completed ?? 0}/${report.attempts_requested}` : ''));
+    if (opts.detailsUrl) {
+        lines.push('');
+        lines.push(`**Details:** [Open the full report](${formatLinkDestination(opts.detailsUrl)})`);
+    }
+    if (opts.notice) {
+        lines.push('');
+        lines.push(`> ${opts.notice.replace(/[\r\n]+/g, ' ').trim()}`);
+    }
     if (report.threshold != null) {
         lines.push('');
         lines.push(`Threshold: ${pct(report.threshold)} — ${report.status.toUpperCase()}`);
     }
     lines.push('');
-    lines.push('| Group | Pass rate | pass@k | pass^k | Skills | Avg duration |');
+    lines.push('| Group | Mean reward | Success rate | pass@k | Skills | Avg duration |');
     lines.push('|---|---|---|---|---|---|');
     for (const group of report.groups) {
         const skills = group.skills_used.length > 0 ? group.skills_used.join(', ') : '—';
         const avg = computeAvgDuration(group.trials);
-        lines.push(`| ${escapeTableCell(group.task)} | ${pct(group.pass_rate)} | ${pct(group.pass_at_k)} | ${pct(group.pass_pow_k)} | ${escapeTableCell(skills)} | ${durationSeconds(avg)} |`);
+        lines.push(`| ${escapeTableCell(group.task)} | ${pct(group.mean_reward ?? group.pass_rate ?? 0)} | ${group.success_rate === undefined ? '—' : pct(group.success_rate)} | ${escapeTableCell(formatPassAtK(group.pass_at_k, group.pass_at_k_unavailable_reason))} | ${escapeTableCell(skills)} | ${durationSeconds(avg)} |`);
     }
     if (report.selection) {
         lines.push('');
@@ -100,14 +119,14 @@ export function formatSelectionSection(selection) {
     out.push('### Selection');
     const total = selection.selected.length + selection.skipped.length;
     if (selection.global_match) {
-        out.push(`Ran **all ${total}** evals — global trigger \`${selection.global_match}\` matched.`);
+        out.push(`Selected **all ${total}** eval files — global trigger \`${selection.global_match}\` matched.`);
         return out.join('\n');
     }
     if (selection.skipped.length === 0 && selection.selected.length > 0) {
-        out.push(`Ran **all ${total}** evals — every eval had a matching change.`);
+        out.push(`Selected **all ${total}** eval files — every eval had a matching change.`);
         return out.join('\n');
     }
-    out.push(`Ran **${selection.selected.length} of ${total}** evals based on changes vs \`${selection.base_ref}\`.`);
+    out.push(`Selected **${selection.selected.length} of ${total}** eval files based on changes vs \`${selection.base_ref}\`.`);
     if (selection.skipped.length > 0) {
         out.push('');
         out.push(`<details><summary>${selection.skipped.length} skipped (unaffected)</summary>`);
@@ -171,13 +190,36 @@ function apiHeaders(token) {
     };
 }
 async function listPrComments(ctx) {
-    const url = `https://api.github.com/repos/${ctx.owner}/${ctx.repo}/issues/${ctx.prNumber}/comments?per_page=100`;
+    const comments = [];
+    for (let page = 1; page <= 100; page += 1) {
+        const url = `https://api.github.com/repos/${ctx.owner}/${ctx.repo}/issues/${ctx.prNumber}/comments?per_page=100&page=${page}`;
+        const res = await fetch(url, { headers: apiHeaders(ctx.token) });
+        if (!res.ok) {
+            throw new Error(`GET ${url} returned ${res.status}`);
+        }
+        const data = await res.json();
+        if (!Array.isArray(data)) {
+            throw new Error(`GET ${url} returned a non-array response`);
+        }
+        for (const value of data) {
+            if (isGithubComment(value))
+                comments.push(value);
+        }
+        if (data.length < 100)
+            break;
+    }
+    return comments;
+}
+async function currentPrHeadSha(ctx) {
+    const url = `https://api.github.com/repos/${ctx.owner}/${ctx.repo}/pulls/${ctx.prNumber}`;
     const res = await fetch(url, { headers: apiHeaders(ctx.token) });
-    if (!res.ok) {
+    if (!res.ok)
         throw new Error(`GET ${url} returned ${res.status}`);
+    const data = await res.json();
+    if (!isRecord(data) || !isRecord(data.head) || typeof data.head.sha !== 'string') {
+        throw new Error(`GET ${url} returned no pull-request head SHA`);
     }
-    const data = (await res.json());
-    return Array.isArray(data) ? data : [];
+    return data.head.sha;
 }
 async function createPrComment(ctx, body) {
     const url = `https://api.github.com/repos/${ctx.owner}/${ctx.repo}/issues/${ctx.prNumber}/comments`;
@@ -206,8 +248,8 @@ const GITHUB_COMMENT_MAX = 65536;
 const TRUNCATION_SENTINEL = '\n\n_…output truncated to fit GitHub\'s comment size limit — see the workflow artifacts for the full report._';
 /**
  * Find an existing PR comment carrying `<!-- pathgrade:${commentId} -->`
- * and update it; otherwise create a new one. Swallows all errors (logs
- * to stderr) — `pathgrade report` must never fail CI.
+ * and update it; otherwise create a new one. Default mode logs and swallows
+ * transport errors; strict mode rethrows them for an owning orchestrator.
  *
  * Bodies longer than GitHub's 65,536-character limit are truncated with a
  * sentinel pointing to workflow artifacts, preserving the leading dedup
@@ -221,19 +263,51 @@ export async function postOrUpdateComment(ctx, opts) {
         : withMarker.slice(0, GITHUB_COMMENT_MAX - TRUNCATION_SENTINEL.length) +
             TRUNCATION_SENTINEL;
     try {
+        if (opts.expectedHeadSha) {
+            const currentHead = await currentPrHeadSha(ctx);
+            if (currentHead !== opts.expectedHeadSha)
+                return 'stale';
+        }
         const existing = await listPrComments(ctx);
         const match = existing.find((c) => typeof c.body === 'string' && c.body.includes(marker));
         if (match) {
             await updatePrComment(ctx, match.id, body);
+            return 'updated';
         }
         else {
             await createPrComment(ctx, body);
+            return 'created';
         }
     }
     catch (err) {
         const message = err instanceof Error ? err.message : String(err);
         console.error(`pathgrade report: failed to post PR comment — ${message}`);
+        if (opts.strict)
+            throw err;
+    }
+}
+function formatLinkDestination(value) {
+    let url;
+    try {
+        url = new URL(value);
     }
+    catch {
+        throw new Error('Pathgrade details URL must be an absolute HTTP(S) URL');
+    }
+    if (url.protocol !== 'https:' && url.protocol !== 'http:') {
+        throw new Error('Pathgrade details URL must be an absolute HTTP(S) URL');
+    }
+    return url.toString().replace(/\)/g, '%29');
+}
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function isGithubComment(value) {
+    return isRecord(value)
+        && typeof value.id === 'number'
+        && Number.isSafeInteger(value.id)
+        && value.id > 0
+        && typeof value.body === 'string';
 }
 function formatGroupDetails(group) {
     const out = [];

package/dist/reporters/loader.d.ts

@@ -2,6 +2,7 @@ import type { EvalReport } from '../types.js';
 export interface LoadedReport extends EvalReport {
     file: string;
     timestamp?: string;
+    status?: 'pass' | 'fail';
 }
 export declare function loadReports(resultsDir: string, opts?: {
     skipTraces?: boolean;

package/dist/reporters/loader.js

@@ -7,7 +7,13 @@ async function hydrateTraces(report, traceFile, resolved) {
     const traceTrials = await fs.readJSON(tracePath);
     if (!Array.isArray(traceTrials) || !Array.isArray(report.trials))
         return;
-    report.trials = report.trials.map((t, i) => ({ ...t, ...traceTrials[i] }));
+    const tracesByAttemptId = new Map(traceTrials
+        .filter((trial) => typeof trial?.attempt_id === 'string')
+        .map((trial) => [trial.attempt_id, trial]));
+    report.trials = report.trials.map((trial, index) => ({
+        ...trial,
+        ...(trial.attempt_id ? tracesByAttemptId.get(trial.attempt_id) : traceTrials[index]),
+    }));
 }
 export async function loadReports(resultsDir, opts) {
     const resolved = path.resolve(resultsDir);
@@ -25,8 +31,15 @@ export async function loadReports(resultsDir, opts) {
                 results.push({ file, ...raw });
                 continue;
             }
+            if (raw.version !== 1 && raw.version !== 2)
+                continue;
             for (const group of raw.groups) {
-                const report = { file, timestamp: raw.timestamp, ...group };
+                const report = {
+                    file,
+                    timestamp: raw.timestamp,
+                    ...group,
+                    status: groupStatus(raw, group),
+                };
                 if (!opts?.skipTraces && group.trace_file) {
                     await hydrateTraces(report, group.trace_file, resolved);
                 }
@@ -37,3 +50,15 @@ export async function loadReports(resultsDir, opts) {
     }
     return results;
 }
+function groupStatus(raw, group) {
+    if (group.status === 'pass' || group.status === 'fail')
+        return group.status;
+    const meanReward = group.mean_reward ?? group.pass_rate ?? 0;
+    if (typeof raw.threshold === 'number')
+        return meanReward >= raw.threshold ? 'pass' : 'fail';
+    const outcomes = Array.isArray(group.trials) ? group.trials.map((trial) => trial.runner_outcome) : [];
+    if (outcomes.length > 0 && outcomes.every((outcome) => (outcome === 'passed' || outcome === 'failed' || outcome === 'not-run'))) {
+        return outcomes.every((outcome) => outcome === 'passed') ? 'pass' : 'fail';
+    }
+    return meanReward >= 0.5 ? 'pass' : 'fail';
+}

package/dist/reporters/report-summary.js

@@ -3,13 +3,21 @@ import { formatDiagnostics, formatDiagnosticsSummary } from './diagnostics.js';
 export function printReportSummary(groups, opts = {}) {
     console.log(`\n${fmt.bold('── pathgrade summary ')}${fmt.dim('─'.repeat(40))}\n`);
     for (const group of groups) {
-        const prColor = group.pass_rate >= 0.5 ? fmt.green : fmt.red;
+        const rewardColor = group.mean_reward >= 0.5 ? fmt.green : fmt.red;
         console.log(`  ${fmt.bold(group.task)}`);
-        console.log(`    ${fmt.dim('pass rate'.padEnd(12))} ${prColor((group.pass_rate * 100).toFixed(1) + '%')}`);
-        console.log(`    ${fmt.dim(`pass@${group.trial_count}`.padEnd(12))} ${(group.pass_at_k * 100).toFixed(1)}%`);
-        console.log(`    ${fmt.dim(`pass^${group.trial_count}`.padEnd(12))} ${(group.pass_pow_k * 100).toFixed(1)}%`);
+        console.log(`    ${fmt.dim('mean reward'.padEnd(12))} ${rewardColor((group.mean_reward * 100).toFixed(1) + '%')}`);
+        console.log(`    ${fmt.dim('success'.padEnd(12))} ${group.success_rate === undefined ? '—' : `${(group.success_rate * 100).toFixed(1)}%`}`);
+        if (group.pass_at_k) {
+            for (const [k, value] of Object.entries(group.pass_at_k)) {
+                console.log(`    ${fmt.dim(`pass@${k}`.padEnd(12))} ${(value * 100).toFixed(1)}%`);
+            }
+        }
+        else {
+            const reason = group.pass_at_k_unavailable_reason?.replaceAll('_', ' ') ?? 'repeated binary attempts required';
+            console.log(`    ${fmt.dim('pass@k'.padEnd(12))} — (${reason})`);
+        }
         console.log(`    ${fmt.dim('avg time'.padEnd(12))} ${(group.average_duration_ms / 1000).toFixed(1)}s`);
-        console.log(`    ${fmt.dim('trials'.padEnd(12))} ${group.trial_count}`);
+        console.log(`    ${fmt.dim('attempts'.padEnd(12))} ${group.trial_count}`);
         console.log();
         for (const diagnostic of group.diagnostics) {
             const shouldPrintFull = opts.forceVerbose

package/dist/reporting/artifacts.js

@@ -1,18 +1,21 @@
 import path from 'node:path';
 import fs from 'fs-extra';
+import { collectSensitiveEnvValues, sanitizePersistenceValue, } from '../tool-event-results.js';
 export async function writePathgradeArtifacts(artifactRoot, built) {
     await fs.ensureDir(path.join(artifactRoot, 'traces'));
+    const sensitiveValues = collectSensitiveEnvValues(process.env);
+    const report = sanitizePersistenceValue(built.report, sensitiveValues);
     const gitignorePath = path.join(artifactRoot, '.gitignore');
     if (!(await fs.pathExists(gitignorePath))) {
         await fs.writeFile(gitignorePath, '*\n');
     }
     const traceFiles = [];
     for (const trace of built.traces) {
-        await fs.writeJson(path.join(artifactRoot, trace.traceFile), trace.trials, { spaces: 2 });
+        await fs.writeJson(path.join(artifactRoot, trace.traceFile), sanitizePersistenceValue(trace.trials, sensitiveValues), { spaces: 2 });
         traceFiles.push(trace.traceFile);
     }
     const resultsPath = path.join(artifactRoot, 'results.json');
-    await fs.writeJson(resultsPath, built.report, { spaces: 2 });
+    await fs.writeJson(resultsPath, report, { spaces: 2 });
     return {
         resultsPath,
         traceFiles,

package/dist/reporting/core.d.ts

@@ -1,2 +1,3 @@
 import type { PathgradeReportBuildResult, ReportRunInput } from './types.js';
 export declare function buildPathgradeReport(input: ReportRunInput): PathgradeReportBuildResult;
+export declare function finiteSamplePassAtK(n: number, successes: number, k: number): number;