@wix/pathgrade 0.29.0

package/dist/sdk/scorers.d.ts

@@ -0,0 +1,64 @@
+import type { CheckScorer, CodeJudgeToolName, ScorerContext, JudgeScorer, ScoreScorer, ScoreResult, ToolExpectation, ToolUsageScorer } from './types.js';
+/**
+ * Create a check scorer — a boolean gate that passes (1.0) or fails (0.0).
+ */
+export declare function check(name: string, fn: (ctx: ScorerContext) => boolean | Promise<boolean>, opts?: {
+    weight?: number;
+}): CheckScorer;
+/**
+ * Create a score scorer — returns a number (0-1) or { score, details } for partial credit.
+ */
+export declare function score(name: string, fn: (ctx: ScorerContext) => number | ScoreResult | Promise<number | ScoreResult>, opts?: {
+    weight?: number;
+}): ScoreScorer;
+/**
+ * Create a judge scorer — uses an LLM to evaluate against a rubric.
+ * Transcript is auto-included. Tool events opt-in via includeToolEvents.
+ *
+ * @param opts.input - Additional context to include in the judge prompt alongside
+ *   the transcript. Each key becomes a section heading; string values are rendered
+ *   as-is, objects are JSON-stringified. Use this to pass file contents, command
+ *   output, or other artifacts that the agent produced but may not appear in the
+ *   transcript. The judge LLM only sees the conversation transcript by default —
+ *   if the agent wrote a file, the judge cannot verify its contents unless you read
+ *   the file and pass it here.
+ *
+ *   @example
+ *   // Read the file the agent created and pass it to the judge
+ *   const content = await agent.exec('cat output.md');
+ *   judge('file quality', {
+ *     rubric: 'The output file should contain a valid markdown document...',
+ *     input: { 'output.md contents': content.stdout },
+ *   })
+ *   // Or derive the judge input from scorer context
+ *   judge('file quality', {
+ *     rubric: 'The output file should contain a valid markdown document...',
+ *     input: async (ctx) => ({ 'output.md contents': await ctx.artifacts.read('output.md') }),
+ *   })
+ */
+export declare function judge(name: string, opts: {
+    rubric: string;
+    weight?: number;
+    model?: string;
+    retry?: boolean | number;
+    includeToolEvents?: boolean;
+    /** Additional context for the judge. See function docs for usage pattern. */
+    input?: import('./types.js').JudgeInput;
+    /**
+     * Opt-in allowlist of tools the judge LLM may call during a multi-turn
+     * tool-use loop. When set to a non-empty array, the judge runs in a
+     * tool-use loop against a provider that supports callWithTools. When
+     * omitted, behavior is byte-for-byte unchanged.
+     */
+    tools?: CodeJudgeToolName[];
+    /** Cap on LLM calls per tool-use judge; default 10. */
+    maxRounds?: number;
+    /** Anthropic prompt caching. Default: true when tools is set, unchanged otherwise. */
+    cacheControl?: boolean;
+}): JudgeScorer;
+/**
+ * Create a tool usage scorer — matches tool events against expectations.
+ */
+export declare function toolUsage(name: string, expectations: ToolExpectation[], opts?: {
+    weight?: number;
+}): ToolUsageScorer;

package/dist/sdk/scorers.js

@@ -0,0 +1,80 @@
+/**
+ * Create a check scorer — a boolean gate that passes (1.0) or fails (0.0).
+ */
+export function check(name, fn, opts) {
+    return {
+        type: 'check',
+        name,
+        weight: opts?.weight ?? 1,
+        fn,
+    };
+}
+/**
+ * Create a score scorer — returns a number (0-1) or { score, details } for partial credit.
+ */
+export function score(name, fn, opts) {
+    return {
+        type: 'score',
+        name,
+        weight: opts?.weight ?? 1,
+        fn,
+    };
+}
+/**
+ * Create a judge scorer — uses an LLM to evaluate against a rubric.
+ * Transcript is auto-included. Tool events opt-in via includeToolEvents.
+ *
+ * @param opts.input - Additional context to include in the judge prompt alongside
+ *   the transcript. Each key becomes a section heading; string values are rendered
+ *   as-is, objects are JSON-stringified. Use this to pass file contents, command
+ *   output, or other artifacts that the agent produced but may not appear in the
+ *   transcript. The judge LLM only sees the conversation transcript by default —
+ *   if the agent wrote a file, the judge cannot verify its contents unless you read
+ *   the file and pass it here.
+ *
+ *   @example
+ *   // Read the file the agent created and pass it to the judge
+ *   const content = await agent.exec('cat output.md');
+ *   judge('file quality', {
+ *     rubric: 'The output file should contain a valid markdown document...',
+ *     input: { 'output.md contents': content.stdout },
+ *   })
+ *   // Or derive the judge input from scorer context
+ *   judge('file quality', {
+ *     rubric: 'The output file should contain a valid markdown document...',
+ *     input: async (ctx) => ({ 'output.md contents': await ctx.artifacts.read('output.md') }),
+ *   })
+ */
+export function judge(name, opts) {
+    const tools = opts.tools && opts.tools.length > 0 ? [...opts.tools] : undefined;
+    const includeToolEvents = opts.includeToolEvents
+        ?? (tools?.includes('getToolEvents') ? true : undefined);
+    // When tools is set, default cacheControl to true (mitigates the
+    // 10–70× input-token inflation measured in the spike). Explicit
+    // false opts out; explicit true is a no-op.
+    const cacheControl = opts.cacheControl ?? (tools ? true : undefined);
+    return {
+        type: 'judge',
+        name,
+        weight: opts.weight ?? 1,
+        rubric: opts.rubric,
+        model: opts.model,
+        retry: opts.retry,
+        includeToolEvents,
+        input: opts.input,
+        tools,
+        maxRounds: opts.maxRounds,
+        cacheControl,
+    };
+}
+/**
+ * Create a tool usage scorer — matches tool events against expectations.
+ */
+export function toolUsage(name, expectations, opts) {
+    return {
+        type: 'tool_usage',
+        name,
+        weight: opts?.weight ?? 1,
+        expectations,
+    };
+}

package/dist/sdk/snapshots.d.ts

@@ -0,0 +1,41 @@
+import type { LogEntry } from '../types.js';
+import type { ToolEvent } from '../tool-events.js';
+import type { AgentName, ConversationResult, Message, TurnTiming } from './types.js';
+export declare const RUN_SNAPSHOT_VERSION = 1;
+export interface RunSnapshot {
+    version: 1;
+    timestamp: string;
+    agent: AgentName;
+    messages: Message[];
+    log: LogEntry[];
+    toolEvents: ToolEvent[];
+    turnTimings: TurnTiming[];
+    conversationResult: {
+        turns: number;
+        completionReason: ConversationResult['completionReason'];
+        completionDetail?: string;
+        turnTimings: TurnTiming[];
+    };
+    workspace: string | null;
+}
+export declare function buildRunSnapshot(params: {
+    agent: AgentName;
+    messages: Message[];
+    log: LogEntry[];
+    conversationResult: ConversationResult;
+    workspace: string | null;
+    timestamp?: string;
+}): RunSnapshot;
+export declare class SnapshotParseError extends Error {
+    constructor(message: string, options?: {
+        cause?: unknown;
+    });
+}
+export declare class SnapshotVersionError extends Error {
+    constructor(version: number);
+}
+export declare class WorkspaceMissingError extends Error {
+    constructor(message: string);
+}
+export declare function buildTranscript(messages: Message[]): string;
+export declare function loadRunSnapshot(snapshotPath: string): Promise<RunSnapshot>;

package/dist/sdk/snapshots.js

@@ -0,0 +1,148 @@
+import fs from 'fs-extra';
+export const RUN_SNAPSHOT_VERSION = 1;
+export function buildRunSnapshot(params) {
+    const { agent, messages, log, conversationResult, workspace, timestamp } = params;
+    const toolEvents = log
+        .filter((entry) => entry.type === 'tool_event' && entry.tool_event)
+        .map((entry) => entry.tool_event);
+    return {
+        version: RUN_SNAPSHOT_VERSION,
+        timestamp: timestamp ?? new Date().toISOString(),
+        agent,
+        messages: [...messages],
+        log: [...log],
+        toolEvents,
+        turnTimings: [...conversationResult.turnTimings],
+        conversationResult: {
+            turns: conversationResult.turns,
+            completionReason: conversationResult.completionReason,
+            ...(conversationResult.completionDetail ? { completionDetail: conversationResult.completionDetail } : {}),
+            turnTimings: [...conversationResult.turnTimings],
+        },
+        workspace,
+    };
+}
+export class SnapshotParseError extends Error {
+    constructor(message, options) {
+        super(message);
+        this.name = 'SnapshotParseError';
+        if (options?.cause !== undefined) {
+            this.cause = options.cause;
+        }
+    }
+}
+export class SnapshotVersionError extends Error {
+    constructor(version) {
+        super(`Unsupported snapshot version: ${version}`);
+        this.name = 'SnapshotVersionError';
+    }
+}
+export class WorkspaceMissingError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'WorkspaceMissingError';
+    }
+}
+const ASK_SOURCE_VALUES = new Set(['claude', 'codex-app-server', 'cursor']);
+const ASK_LIFECYCLE_VALUES = new Set(['live', 'post-hoc']);
+function validateLogEntries(entries) {
+    for (let i = 0; i < entries.length; i++) {
+        const entry = entries[i];
+        if (!entry || typeof entry !== 'object') {
+            throw new SnapshotParseError(`Snapshot log entry ${i} must be an object`);
+        }
+        const record = entry;
+        if (record.type === 'ask_batch') {
+            if (typeof record.batch_id !== 'string') {
+                throw new SnapshotParseError(`ask_batch log entry ${i} is missing required field: batch_id`);
+            }
+            if (typeof record.turn_number !== 'number') {
+                throw new SnapshotParseError(`ask_batch log entry ${i} is missing required field: turn_number`);
+            }
+            if (typeof record.source !== 'string' || !ASK_SOURCE_VALUES.has(record.source)) {
+                throw new SnapshotParseError(`ask_batch log entry ${i} has invalid source: ${String(record.source)}`);
+            }
+            if (typeof record.lifecycle !== 'string' || !ASK_LIFECYCLE_VALUES.has(record.lifecycle)) {
+                throw new SnapshotParseError(`ask_batch log entry ${i} has invalid lifecycle: ${String(record.lifecycle)}`);
+            }
+            if (typeof record.source_tool !== 'string') {
+                throw new SnapshotParseError(`ask_batch log entry ${i} is missing required field: source_tool`);
+            }
+            if (typeof record.question_count !== 'number') {
+                throw new SnapshotParseError(`ask_batch log entry ${i} is missing required field: question_count`);
+            }
+        }
+        // Legacy blocked_prompt_* fields on agent_result entries remain accepted
+        // without additional validation — their schema is owned by the writer.
+    }
+}
+export function buildTranscript(messages) {
+    return messages
+        .map((msg) => `[${msg.role === 'user' ? 'User' : 'Agent'}]\n${msg.content}`)
+        .join('\n\n');
+}
+export async function loadRunSnapshot(snapshotPath) {
+    let parsed;
+    try {
+        parsed = await fs.readJSON(snapshotPath);
+    }
+    catch (error) {
+        throw new SnapshotParseError(`Failed to read snapshot JSON from ${snapshotPath}`, { cause: error });
+    }
+    return validateRunSnapshot(parsed);
+}
+function validateRunSnapshot(input) {
+    if (!input || typeof input !== 'object') {
+        throw new SnapshotParseError('Snapshot must be a JSON object');
+    }
+    const snapshot = input;
+    if (typeof snapshot.version !== 'number') {
+        throw new SnapshotParseError('Snapshot is missing required field: version');
+    }
+    if (snapshot.version > RUN_SNAPSHOT_VERSION) {
+        throw new SnapshotVersionError(snapshot.version);
+    }
+    if (!Array.isArray(snapshot.messages)) {
+        throw new SnapshotParseError('Snapshot is missing required field: messages');
+    }
+    if (!Array.isArray(snapshot.log)) {
+        throw new SnapshotParseError('Snapshot is missing required field: log');
+    }
+    validateLogEntries(snapshot.log);
+    if (!Array.isArray(snapshot.toolEvents)) {
+        throw new SnapshotParseError('Snapshot is missing required field: toolEvents');
+    }
+    if (!snapshot.conversationResult || typeof snapshot.conversationResult !== 'object') {
+        throw new SnapshotParseError('Snapshot is missing required field: conversationResult');
+    }
+    if (!Array.isArray(snapshot.conversationResult.turnTimings)) {
+        throw new SnapshotParseError('Snapshot conversationResult is missing required field: turnTimings');
+    }
+    if (typeof snapshot.conversationResult.turns !== 'number') {
+        throw new SnapshotParseError('Snapshot conversationResult is missing required field: turns');
+    }
+    if (typeof snapshot.conversationResult.completionReason !== 'string') {
+        throw new SnapshotParseError('Snapshot conversationResult is missing required field: completionReason');
+    }
+    const turnTimings = Array.isArray(snapshot.turnTimings)
+        ? snapshot.turnTimings
+        : snapshot.conversationResult.turnTimings;
+    return {
+        version: RUN_SNAPSHOT_VERSION,
+        timestamp: typeof snapshot.timestamp === 'string' ? snapshot.timestamp : new Date(0).toISOString(),
+        agent: snapshot.agent === 'claude' || snapshot.agent === 'codex' || snapshot.agent === 'cursor' ? snapshot.agent : 'claude',
+        messages: snapshot.messages,
+        log: snapshot.log,
+        toolEvents: snapshot.toolEvents,
+        turnTimings,
+        conversationResult: {
+            turns: snapshot.conversationResult.turns,
+            completionReason: snapshot.conversationResult.completionReason,
+            ...(snapshot.conversationResult.completionDetail
+                ? { completionDetail: snapshot.conversationResult.completionDetail }
+                : {}),
+            turnTimings: snapshot.conversationResult.turnTimings,
+        },
+        workspace: typeof snapshot.workspace === 'string' ? snapshot.workspace : null,
+    };
+}