@wix/pathgrade 0.37.0 → 1.0.0

package/dist/reporting/core.js

@@ -0,0 +1,173 @@
+import { extractSkillsFromLog } from '../tool-events.js';
+export function buildPathgradeReport(input) {
+    const warnings = [];
+    const consolidatedGroups = [];
+    const traces = [];
+    const summaries = [];
+    const builtGroups = input.groups.map(group => ({
+        groupName: group.groupName,
+        cases: group.cases.map(toBuiltCase),
+    }));
+    for (const group of builtGroups) {
+        for (const testCase of group.cases) {
+            warnings.push(...testCase.warnings);
+        }
+    }
+    const reportableGroups = builtGroups
+        .map(group => ({
+        ...group,
+        cases: group.cases.filter(reportableBuiltCase),
+    }))
+        .filter(group => group.cases.length > 0);
+    for (const group of reportableGroups) {
+        const report = buildEvalReport(group.groupName, group.cases);
+        summaries.push(buildSummary(group.groupName, group.cases, report));
+        const traceFile = `traces/${slug(group.groupName)}.json`;
+        traces.push({ traceFile, trials: report.trials });
+        const strippedTrials = report.trials.map(trial => {
+            const { session_log, conversation, ...rest } = trial;
+            return rest;
+        });
+        const { trials: _trials, ...rest } = report;
+        consolidatedGroups.push({
+            ...rest,
+            trials: strippedTrials,
+            trace_file: traceFile,
+        });
+    }
+    const scores = reportableGroups.flatMap(group => group.cases.map(testCase => testCase.score));
+    const overallPassRate = average(scores);
+    const status = input.threshold != null
+        ? (overallPassRate >= input.threshold ? 'pass' : 'fail')
+        : (reportableGroups.every(group => group.cases.every(testCase => testCase.state === 'passed')) ? 'pass' : 'fail');
+    return {
+        report: {
+            version: 1,
+            timestamp: new Date().toISOString(),
+            ...(input.threshold != null ? { threshold: input.threshold } : {}),
+            overall_pass_rate: overallPassRate,
+            status,
+            groups: consolidatedGroups,
+            ...(input.selection ? { selection: input.selection } : {}),
+        },
+        traces,
+        summaries,
+        warnings,
+    };
+}
+function reportableBuiltCase(testCase) {
+    if (testCase.reportable !== undefined)
+        return testCase.reportable;
+    return testCase.state !== 'skipped' && testCase.state !== 'pending';
+}
+function toBuiltCase(testCase) {
+    if (testCase.evaluations === undefined) {
+        return fallbackCase(testCase, []);
+    }
+    if (testCase.evaluations.length === 0) {
+        return fallbackCase(testCase, [
+            `empty results for "${testCase.name}" — evaluate() may not have been called`,
+        ]);
+    }
+    const evaluation = testCase.evaluations[testCase.evaluations.length - 1];
+    const diagnostics = evaluation.diagnostics ?? testCase.diagnostics;
+    return {
+        name: testCase.name,
+        state: testCase.state,
+        score: evaluation.score,
+        runnerDurationMs: testCase.runnerDurationMs,
+        diagnostics,
+        reportable: testCase.reportable,
+        trial: normalizeTrial({
+            name: testCase.name,
+            score: evaluation.score,
+            runnerDurationMs: testCase.runnerDurationMs,
+            trial: evaluation.trial,
+            diagnostics,
+        }),
+        warnings: [],
+    };
+}
+function fallbackCase(testCase, warnings) {
+    const score = testCase.state === 'passed' ? 1 : 0;
+    return {
+        name: testCase.name,
+        state: testCase.state,
+        score,
+        runnerDurationMs: testCase.runnerDurationMs,
+        diagnostics: testCase.diagnostics,
+        reportable: testCase.reportable,
+        trial: normalizeTrial({
+            name: testCase.name,
+            score,
+            runnerDurationMs: testCase.runnerDurationMs,
+            diagnostics: testCase.diagnostics,
+        }),
+        warnings,
+    };
+}
+function normalizeTrial(input) {
+    const base = input.trial ?? {
+        trial_id: 1,
+        reward: input.score,
+        scorer_results: [],
+        duration_ms: input.runnerDurationMs,
+        n_commands: 0,
+        input_tokens: 0,
+        output_tokens: 0,
+        session_log: [],
+    };
+    const skills = base.skills_used ?? extractSkillsFromLog(base.session_log);
+    return {
+        ...base,
+        name: input.name,
+        duration_ms: base.duration_ms || input.runnerDurationMs,
+        diagnostics: input.diagnostics ?? base.diagnostics,
+        ...(skills.length > 0 ? { skills_used: skills } : {}),
+    };
+}
+function buildEvalReport(groupName, cases) {
+    const trials = cases.map((testCase, index) => ({
+        ...testCase.trial,
+        trial_id: index + 1,
+    }));
+    const passRate = average(cases.map(testCase => testCase.state === 'passed' ? 1 : 0));
+    const skills = new Set();
+    for (const trial of trials) {
+        for (const skill of trial.skills_used ?? []) {
+            skills.add(skill);
+        }
+    }
+    return {
+        task: groupName,
+        pass_rate: passRate,
+        pass_at_k: 1 - Math.pow(1 - passRate, trials.length),
+        pass_pow_k: Math.pow(passRate, trials.length),
+        trials,
+        skills_used: [...skills],
+    };
+}
+function buildSummary(groupName, cases, report) {
+    return {
+        task: groupName,
+        pass_rate: report.pass_rate,
+        pass_at_k: report.pass_at_k,
+        pass_pow_k: report.pass_pow_k,
+        average_duration_ms: average(cases.map(testCase => testCase.runnerDurationMs)),
+        trial_count: cases.length,
+        diagnostics: cases.flatMap(testCase => testCase.diagnostics
+            ? [{ caseName: testCase.name, state: testCase.state, report: testCase.diagnostics }]
+            : []),
+    };
+}
+function average(values) {
+    if (values.length === 0)
+        return 0;
+    return values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+function slug(value) {
+    return value
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, '-')
+        .replace(/^-+|-+$/g, '') || 'report';
+}

package/dist/reporting/types.d.ts

@@ -0,0 +1,57 @@
+import type { DiagnosticsReport } from '../sdk/diagnostics.js';
+import type { PathgradeReport, PathgradeSelectionReport, TrialResult } from '../types.js';
+export type ReportCaseState = 'passed' | 'failed' | 'skipped' | 'pending';
+export interface ReportRunInput {
+    threshold?: number;
+    selection?: PathgradeSelectionReport;
+    groups: ReportGroupInput[];
+}
+export interface ReportGroupInput {
+    groupName: string;
+    cases: ReportCaseInput[];
+}
+export interface ReportCaseInput {
+    caseId?: string;
+    name: string;
+    state: ReportCaseState;
+    runnerDurationMs: number;
+    sourceRef?: string;
+    filePath?: string;
+    groupName?: string;
+    runnerCaseId?: string;
+    reportable?: boolean;
+    evaluations?: readonly ReportEvaluationInput[];
+    diagnostics?: DiagnosticsReport;
+}
+export interface ReportEvaluationInput {
+    score: number;
+    trial?: TrialResult;
+    diagnostics?: DiagnosticsReport;
+}
+export interface PathgradeReportBuildResult {
+    report: PathgradeReport;
+    traces: Array<{
+        traceFile: string;
+        trials: TrialResult[];
+    }>;
+    summaries: ReportSummaryGroup[];
+    warnings: string[];
+}
+export interface ArtifactWriteResult {
+    resultsPath: string;
+    traceFiles: string[];
+}
+export interface ReportSummaryGroup {
+    task: string;
+    pass_rate: number;
+    pass_at_k: number;
+    pass_pow_k: number;
+    average_duration_ms: number;
+    trial_count: number;
+    diagnostics: ReportSummaryDiagnostics[];
+}
+export interface ReportSummaryDiagnostics {
+    caseName: string;
+    state: ReportCaseState;
+    report: DiagnosticsReport;
+}

package/dist/reporting/types.js

@@ -0,0 +1 @@
+export {};

package/dist/reporting/vitest-edge.d.ts

@@ -0,0 +1 @@
+export { collectVitestReportGroups } from '../runners/vitest-adapter.js';

package/dist/reporting/vitest-edge.js

@@ -0,0 +1 @@
+export { collectVitestReportGroups } from '../runners/vitest-adapter.js';

package/dist/runners/adapter-loader.d.ts

@@ -0,0 +1,17 @@
+import type { ResolvedPathgradeConfig } from '../config/pathgrade.js';
+import { type SpawnVitest } from './vitest-invocation.js';
+import type { RunnerAdapter } from './adapter.js';
+import type { RunnerInvocationAdapter } from './invocation.js';
+export interface ExternalAdapterFactoryInput {
+    config: ResolvedPathgradeConfig;
+}
+export declare function loadRunnerAdapter(input: {
+    adapterName?: string;
+    cwd?: string;
+}): Promise<RunnerAdapter>;
+export declare function loadRunnerInvocationAdapter(input: {
+    adapterName?: string;
+    cwd?: string;
+    config: ResolvedPathgradeConfig;
+    spawnVitest?: SpawnVitest;
+}): Promise<RunnerInvocationAdapter>;

package/dist/runners/adapter-loader.js

@@ -0,0 +1,67 @@
+import { createRequire } from 'node:module';
+import path from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { createJestInvocationAdapter } from '../adapters/jest/invocation-adapter.js';
+import { createJestAdapter } from '../adapters/jest/runner-adapter.js';
+import { createNodeTestInvocationAdapter } from '../adapters/node-test/invocation-adapter.js';
+import { createVitestInvocationAdapter } from './vitest-invocation.js';
+import { resolveRunnerAdapter } from './selection.js';
+export async function loadRunnerAdapter(input) {
+    const name = input.adapterName ?? 'vitest';
+    if (name === 'vitest' || name === 'node-test') {
+        return resolveRunnerAdapter({ adapterName: name });
+    }
+    if (name === 'jest')
+        return createJestAdapter();
+    const mod = await importExternalAdapter({
+        adapterName: name,
+        cwd: input.cwd ?? process.cwd(),
+    });
+    if (typeof mod.createPathgradeAdapter !== 'function') {
+        throw new Error(`Pathgrade adapter package "${adapterPackageSpecifier(name)}" must export createPathgradeAdapter().`);
+    }
+    return mod.createPathgradeAdapter();
+}
+export async function loadRunnerInvocationAdapter(input) {
+    const name = input.adapterName ?? 'vitest';
+    if (name === 'vitest')
+        return createVitestInvocationAdapter({ spawnVitest: input.spawnVitest });
+    if (name === 'node-test')
+        return createNodeTestInvocationAdapter({ config: input.config });
+    if (name === 'jest')
+        return createJestInvocationAdapter({ config: input.config });
+    const mod = await importExternalAdapter({
+        adapterName: name,
+        cwd: input.cwd ?? process.cwd(),
+    });
+    if (typeof mod.createPathgradeInvocationAdapter !== 'function') {
+        throw new Error(`Pathgrade adapter package "${adapterPackageSpecifier(name)}" must export createPathgradeInvocationAdapter().`);
+    }
+    return mod.createPathgradeInvocationAdapter({ config: input.config });
+}
+function adapterPackageSpecifier(adapterName) {
+    if (adapterName.startsWith('.') || path.isAbsolute(adapterName) || adapterName.includes('/')) {
+        return adapterName;
+    }
+    return `@wix/pathgrade-adapter-${adapterName}`;
+}
+function resolveAdapterSpecifier(input) {
+    if (input.adapterName.startsWith('.') || path.isAbsolute(input.adapterName)) {
+        return path.resolve(input.cwd, input.adapterName);
+    }
+    const specifier = adapterPackageSpecifier(input.adapterName);
+    return createRequire(path.join(input.cwd, 'package.json')).resolve(specifier);
+}
+async function importExternalAdapter(input) {
+    try {
+        const resolved = resolveAdapterSpecifier(input);
+        return await import(pathToFileURL(resolved).href);
+    }
+    catch (err) {
+        throw new Error(`Unsupported Pathgrade runner adapter: ${input.adapterName}. ` +
+            `Tried to load ${adapterPackageSpecifier(input.adapterName)} from ${input.cwd}: ${errMsg(err)}`);
+    }
+}
+function errMsg(err) {
+    return err instanceof Error ? err.message : String(err);
+}

package/dist/runners/adapter.d.ts

@@ -0,0 +1,65 @@
+import type { PathgradeSelectionReport } from '../types.js';
+import type { Agent, PathgradeTestMeta, RecordedEvalResult } from '../sdk/types.js';
+import type { CaseContextScope } from '../sdk/case-context.js';
+import type { NormalizedRunSnapshot } from './model.js';
+export declare const runnerAdapterContractVersion = 1;
+export interface RunnerAdapter {
+    readonly name: string;
+    discover(input: AdapterDiscoveryInput): Promise<AdapterDiscoveryResult>;
+    invoke(input: AdapterInvocationInput): Promise<AdapterRunHandle>;
+    collectNormalizedRunSnapshot(run: AdapterRunHandle): Promise<NormalizedRunSnapshot>;
+}
+export interface AdapterDiscoveryInput {
+    cwd: string;
+    include?: string[];
+    exclude?: string[];
+    selection?: PathgradeSelectionReport;
+}
+export interface AdapterDiscoveryResult {
+    units: AdapterEvalUnit[];
+    diagnostics?: AdapterDiagnostic[];
+}
+export interface AdapterEvalUnit {
+    id: string;
+    displayName: string;
+    sourceRef?: string;
+    native?: unknown;
+}
+export interface AdapterDiagnostic {
+    severity: 'info' | 'warning' | 'error';
+    message: string;
+    sourceRef?: string;
+}
+export interface AdapterInvocationInput {
+    discovered: AdapterDiscoveryResult;
+    argv: string[];
+    env: NodeJS.ProcessEnv;
+    lifecycle: AdapterLifecycleHooks;
+    signal?: AbortSignal;
+}
+export interface AdapterRunHandle {
+    adapterName: string;
+    status: AdapterRunStatus;
+    exitCode: number;
+    native?: unknown;
+    diagnostics?: AdapterDiagnostic[];
+}
+export type AdapterRunStatus = 'completed' | 'failed' | 'cancelled' | 'timed_out' | 'parked';
+export interface AdapterLifecycleHooks {
+    onResult(event: EvalResultEvent): void;
+    withCaseContext<T>(context: AdapterCaseContext, run: () => T): T;
+    flushCase(caseId: string): Promise<PathgradeTestMeta[]>;
+    cleanupRun(): Promise<void>;
+}
+export interface AdapterCaseContext {
+    caseId: string;
+    caseName: string;
+    filePath?: string;
+    sourceRef?: string;
+    runnerNativeId?: string;
+    scope: CaseContextScope;
+}
+export interface EvalResultEvent {
+    result: RecordedEvalResult;
+    agent: Agent;
+}

package/dist/runners/adapter.js

@@ -0,0 +1 @@
+export const runnerAdapterContractVersion = 1;

package/dist/runners/invocation.d.ts

@@ -0,0 +1,10 @@
+export interface RunnerInvocationAdapter {
+    readonly name: string;
+    run(input: RunnerInvocationInput): Promise<number>;
+}
+export interface RunnerInvocationInput {
+    cwd: string;
+    runnerArgs: string[];
+    selectedFiles?: string[];
+    env: NodeJS.ProcessEnv;
+}

package/dist/runners/invocation.js

@@ -0,0 +1 @@
+export {};

package/dist/runners/lifecycle-hooks.d.ts

@@ -0,0 +1,2 @@
+import type { AdapterLifecycleHooks } from './adapter.js';
+export declare function createRunnerLifecycleHooks(): AdapterLifecycleHooks;

package/dist/runners/lifecycle-hooks.js

@@ -0,0 +1,18 @@
+import { lifecycleCore } from '../sdk/lifecycle.js';
+import { runWithCaseContext } from '../sdk/case-context.js';
+export function createRunnerLifecycleHooks() {
+    return {
+        onResult: event => lifecycleCore.recordResult(event.result, event.agent),
+        withCaseContext: (context, run) => runWithCaseContext(toCaseContext(context), run),
+        flushCase: caseId => lifecycleCore.flushCase({ caseId }),
+        cleanupRun: () => lifecycleCore.cleanupAll(),
+    };
+}
+function toCaseContext(context) {
+    return {
+        caseId: context.caseId,
+        caseName: context.caseName,
+        filePath: context.filePath ?? context.sourceRef ?? '',
+        scope: context.scope,
+    };
+}

package/dist/runners/model-builders.d.ts

@@ -0,0 +1,4 @@
+import type { ReportGroupInput } from '../reporting/types.js';
+import type { AdapterRunHandle } from './adapter.js';
+import type { NormalizedRunSnapshot } from './model.js';
+export declare function buildNormalizedRunSnapshotFromReportGroups(run: AdapterRunHandle, groups: ReportGroupInput[]): NormalizedRunSnapshot;

package/dist/runners/model-builders.js

@@ -0,0 +1,67 @@
+export function buildNormalizedRunSnapshotFromReportGroups(run, groups) {
+    return {
+        version: 1,
+        completeness: 'final',
+        model: {
+            run: {
+                id: `${run.adapterName}:run`,
+                adapterName: run.adapterName,
+                status: run.status,
+                ...(run.diagnostics ? { diagnostics: run.diagnostics.map(diagnostic => ({
+                        severity: diagnostic.severity,
+                        message: diagnostic.message,
+                    })) } : {}),
+            },
+            units: groups.map((group, index) => ({
+                id: unitId(group.groupName, index),
+                runId: `${run.adapterName}:run`,
+                displayName: group.groupName,
+                groupingHints: [{ kind: 'suite', key: group.groupName, label: group.groupName, order: index }],
+            })),
+            cases: groups.flatMap((group, groupIndex) => group.cases.map((testCase, caseIndex) => {
+                const caseId = testCase.caseId ?? `${unitId(group.groupName, groupIndex)}:case-${caseIndex + 1}`;
+                const attemptId = `${caseId}:attempt-1`;
+                const evaluations = testCase.evaluations?.map((evaluation, evaluationIndex) => ({
+                    id: `${attemptId}:evaluation-${evaluationIndex + 1}`,
+                    attemptId,
+                    score: evaluation.score,
+                    ...(evaluation.trial ? { trial: evaluation.trial } : {}),
+                    ...(evaluation.diagnostics ? { diagnostics: evaluation.diagnostics } : {}),
+                }));
+                return {
+                    id: caseId,
+                    runId: `${run.adapterName}:run`,
+                    unitId: unitId(group.groupName, groupIndex),
+                    name: testCase.name,
+                    state: testCase.state,
+                    scoringPolicy: scoringPolicyForCase(testCase.state, evaluations),
+                    groupingHints: [{ kind: 'suite', key: group.groupName, label: group.groupName, order: groupIndex }],
+                    attempts: [{
+                            id: attemptId,
+                            caseId,
+                            outcome: outcomeForCase(testCase.state),
+                            durationMs: testCase.runnerDurationMs,
+                            ...(evaluations ? { evaluations } : {}),
+                        }],
+                };
+            })),
+        },
+    };
+}
+function unitId(groupName, index) {
+    return `unit-${index + 1}:${groupName}`;
+}
+function scoringPolicyForCase(state, evaluations) {
+    if (state === 'skipped' || state === 'pending')
+        return { kind: 'non-scoring', reason: state };
+    if (evaluations && evaluations.length > 0)
+        return { kind: 'from-evaluations' };
+    return { kind: 'score', score: state === 'passed' ? 1 : 0 };
+}
+function outcomeForCase(state) {
+    if (state === 'passed')
+        return { kind: 'passed' };
+    if (state === 'failed')
+        return { kind: 'failed' };
+    return { kind: 'not-run', reason: state };
+}

package/dist/runners/model-validation.d.ts

@@ -0,0 +1,12 @@
+import type { NormalizedRunSnapshot, SnapshotCompleteness } from './model.js';
+export interface NormalizedRunValidationResult {
+    ok: boolean;
+    errors: NormalizedRunValidationError[];
+}
+export interface NormalizedRunValidationError {
+    path: string;
+    message: string;
+}
+export declare function validateNormalizedRunSnapshot(snapshot: NormalizedRunSnapshot, options: {
+    completeness: SnapshotCompleteness;
+}): NormalizedRunValidationResult;