@wix/pathgrade 0.29.0

package/dist/sdk/evaluate.js

@@ -0,0 +1,325 @@
+import { extractSkillsFromLog } from '../tool-events.js';
+import { getRuntime } from './eval-runtime.js';
+import { runJudgePipeline } from './judge-pipeline.js';
+import { runScorer } from './run-scorer.js';
+import { createLLMClient } from '../utils/llm.js';
+import { sandboxExec } from '../providers/sandbox-exec.js';
+import { buildTranscript, loadRunSnapshot, WorkspaceMissingError } from './snapshots.js';
+import fs from 'fs-extra';
+import path from 'path';
+export class EvalScorerError extends Error {
+    scorerErrors;
+    result;
+    constructor(result, scorerErrors) {
+        super(formatScorerErrorMessage(scorerErrors));
+        this.name = 'EvalScorerError';
+        this.result = result;
+        this.scorerErrors = scorerErrors;
+    }
+}
+/**
+ * Run scorers against a trial and compute a weighted average score.
+ *
+ * Pipeline phases:
+ *   1. check + score scorers (parallel, deterministic)
+ *   2. judge scorers (parallel, LLM calls) — skipped if fail-fast triggered
+ *   3. toolUsage scorers (parallel) — skipped if fail-fast triggered
+ *
+ * Fail-fast: if any check scores 0 and failFast !== false, phases 2-3 are skipped.
+ * score() returning 0 does NOT trigger fail-fast.
+ */
+// Track which agents have already had conversation tokens attributed.
+// Local per-call: no module-level state, no cleanup export needed.
+function makeEvaluateAgent() {
+    const conversationAttributed = new WeakSet();
+    return async function evaluateAgent(agent, scorers, opts) {
+        const toolEvents = agent.log
+            .filter((e) => e.type === 'tool_event' && e.tool_event)
+            .map((e) => e.tool_event);
+        const ctx = {
+            workspace: agent.workspace,
+            log: agent.log,
+            transcript: agent.transcript(),
+            toolEvents,
+            runCommand: (cmd) => agent.exec(cmd),
+            artifacts: createSessionArtifacts(agent.workspace, toolEvents),
+        };
+        const trackedLLM = opts?.llm ?? agent.llm;
+        // Snapshot conversation tokens BEFORE running scorers, for first-eval attribution.
+        const before = trackedLLM.tokenUsage ?? { inputTokens: 0, outputTokens: 0 };
+        // Snapshot conversation cost too. AgentImpl's `sendTurn` accumulates
+        // per-turn `costUsd` onto `trackedLLM` via `addCost`, so by the time
+        // `evaluate()` runs the pre-evaluate cost is the conversation's
+        // accumulated agent-turn cost.
+        const beforeCostUsd = trackedLLM.costUsd ?? 0;
+        // measure() returns the delta consumed by this evaluate call.
+        const { result: evalResult, tokens: deltaTokenUsage } = trackedLLM.measure
+            ? await trackedLLM.measure(() => evaluateWithContext(ctx, scorers, { ...opts, llm: trackedLLM }))
+            : await (async () => {
+                const r = await evaluateWithContext(ctx, scorers, { ...opts, llm: trackedLLM });
+                return { result: r, tokens: r.tokenUsage ?? { inputTokens: 0, outputTokens: 0 } };
+            })();
+        // Attribute conversation tokens on the first evaluate() for this agent.
+        const isFirstEval = !conversationAttributed.has(agent);
+        const conversationTokens = isFirstEval && (before.inputTokens > 0 || before.outputTokens > 0)
+            ? { conversation_input_tokens: before.inputTokens, conversation_output_tokens: before.outputTokens }
+            : undefined;
+        // Same first-eval attribution rule for cost. Omitted entirely when
+        // no conversation cost was captured (Codex / Cursor today).
+        const conversationCost = isFirstEval && beforeCostUsd > 0
+            ? { conversation_cost_usd: beforeCostUsd }
+            : undefined;
+        if (isFirstEval)
+            conversationAttributed.add(agent);
+        const recordedResult = {
+            ...evalResult,
+            tokenUsage: deltaTokenUsage,
+            trial: buildTrialResult(agent.log, { ...evalResult, tokenUsage: deltaTokenUsage }, conversationTokens, conversationCost),
+        };
+        getRuntime().onResult(recordedResult, agent);
+        maybeThrowOnScorerErrors(recordedResult, opts?.onScorerError ?? 'skip');
+        return recordedResult;
+    };
+}
+const evaluateAgent = makeEvaluateAgent();
+async function fromSnapshot(snapshotPath, scorers, opts) {
+    const snapshot = await loadRunSnapshot(snapshotPath);
+    const trackedLLM = opts?.llm ?? createLLMClient({ adapters: [{
+                name: 'runtime', isAvailable: async () => true,
+                call: (prompt, callOpts) => getRuntime().llm.call(prompt, callOpts),
+            }] });
+    const ctx = {
+        workspace: snapshot.workspace ?? '',
+        log: snapshot.log,
+        transcript: buildTranscript(snapshot.messages),
+        toolEvents: snapshot.toolEvents,
+        runCommand: async (cmd) => {
+            const workspace = snapshot.workspace;
+            if (!workspace) {
+                throw new WorkspaceMissingError('Snapshot does not include a workspace path');
+            }
+            if (!(await fs.pathExists(workspace))) {
+                throw new WorkspaceMissingError(`Snapshot workspace does not exist on disk: ${workspace}`);
+            }
+            return sandboxExec(cmd, { cwd: workspace, env: getProcessEnv() });
+        },
+        artifacts: createSessionArtifacts(snapshot.workspace ?? '', snapshot.toolEvents),
+    };
+    const evalResult = await evaluateWithContext(ctx, scorers, { ...opts, llm: trackedLLM });
+    const recordedResult = {
+        ...evalResult,
+        trial: buildTrialResult(snapshot.log, evalResult),
+    };
+    maybeThrowOnScorerErrors(recordedResult, opts?.onScorerError ?? 'skip');
+    return recordedResult;
+}
+async function evaluateWithContext(ctx, scorers, opts) {
+    const failFast = opts?.failFast ?? true;
+    const trackedLLM = opts?.llm ?? createLLMClient({ adapters: [{
+                name: 'runtime', isAvailable: async () => true,
+                call: (prompt, callOpts) => getRuntime().llm.call(prompt, callOpts),
+            }] });
+    const onScorerError = opts?.onScorerError ?? 'skip';
+    const phase1 = [];
+    const phase2 = [];
+    const phase3 = [];
+    for (const g of scorers) {
+        switch (g.type) {
+            case 'check':
+            case 'score':
+                phase1.push(g);
+                break;
+            case 'judge':
+                phase2.push(g);
+                break;
+            case 'tool_usage':
+                phase3.push(g);
+                break;
+        }
+    }
+    const results = [];
+    const phase1Results = await Promise.all(phase1.map((g) => runScorer(g, ctx)));
+    results.push(...phase1Results);
+    const anyCheckFailed = failFast && phase1Results.some((r) => r.type === 'check' && r.status !== 'error' && r.score === 0);
+    if (anyCheckFailed) {
+        for (const g of [...phase2, ...phase3]) {
+            results.push(makeSkipped(g));
+        }
+    }
+    else {
+        const judgeResults = await runJudgePipeline(phase2, ctx, { llm: trackedLLM });
+        results.push(...judgeResults);
+        const phase3Results = await Promise.all(phase3.map((g) => runScorer(g, ctx)));
+        results.push(...phase3Results);
+    }
+    const tokenUsage = trackedLLM.tokenUsage ?? { inputTokens: 0, outputTokens: 0 };
+    const scoringResults = onScorerError === 'skip'
+        ? results.filter((result) => result.status !== 'error')
+        : results;
+    const totalWeight = scoringResults.reduce((sum, r) => sum + r.weight, 0);
+    const weightedSum = scoringResults.reduce((sum, r) => sum + r.score * r.weight, 0);
+    const score = totalWeight > 0 ? weightedSum / totalWeight : 0;
+    return {
+        score,
+        scorers: results,
+        tokenUsage,
+    };
+}
+function maybeThrowOnScorerErrors(result, mode) {
+    if (mode !== 'fail')
+        return;
+    const scorerErrors = result.scorers.filter((entry) => entry.status === 'error');
+    if (scorerErrors.length > 0) {
+        throw new EvalScorerError(result, scorerErrors);
+    }
+}
+function createSessionArtifacts(workspace, toolEvents) {
+    const matchingArtifacts = (opts) => {
+        const actions = opts?.actions ?? ['write_file', 'edit_file'];
+        const seen = new Set();
+        const matches = [];
+        for (const event of toolEvents) {
+            if (!actions.includes(event.action))
+                continue;
+            const absolutePath = extractArtifactPath(event);
+            if (!absolutePath)
+                continue;
+            const relativePath = normalizeArtifactPath(workspace, absolutePath);
+            if (!matchesArtifactPattern(relativePath, opts?.pattern))
+                continue;
+            if (seen.has(relativePath))
+                continue;
+            seen.add(relativePath);
+            matches.push({ path: relativePath, absolutePath });
+        }
+        return matches;
+    };
+    return {
+        list: (opts) => matchingArtifacts(opts).map((artifact) => artifact.path),
+        read: async (artifactPath) => {
+            const absolutePath = resolveArtifactPath(workspace, artifactPath);
+            return fs.readFile(absolutePath, 'utf8');
+        },
+        latest: async (opts) => {
+            const matches = matchingArtifacts(opts);
+            const latest = matches[matches.length - 1];
+            if (!latest)
+                return null;
+            return {
+                path: latest.path,
+                content: await fs.readFile(latest.absolutePath, 'utf8'),
+            };
+        },
+    };
+}
+function extractArtifactPath(event) {
+    const candidates = [
+        event.arguments?.path,
+        event.arguments?.file,
+        event.arguments?.file_path,
+    ];
+    for (const candidate of candidates) {
+        if (typeof candidate === 'string' && candidate.trim().length > 0) {
+            return candidate;
+        }
+    }
+    const summaryMatch = event.summary.match(/^(?:read_file|write_file|edit_file)\s+(.+)$/);
+    return summaryMatch?.[1] ?? null;
+}
+function normalizeArtifactPath(workspace, artifactPath) {
+    if (!workspace || !path.isAbsolute(artifactPath)) {
+        return artifactPath;
+    }
+    const relativePath = path.relative(workspace, artifactPath);
+    if (relativePath.startsWith('..') || path.isAbsolute(relativePath)) {
+        return artifactPath;
+    }
+    return relativePath;
+}
+function resolveArtifactPath(workspace, artifactPath) {
+    if (path.isAbsolute(artifactPath)) {
+        return artifactPath;
+    }
+    if (!workspace) {
+        throw new WorkspaceMissingError('Session artifact reads require a workspace for relative paths');
+    }
+    return path.join(workspace, artifactPath);
+}
+function matchesArtifactPattern(artifactPath, pattern) {
+    if (!pattern)
+        return true;
+    if (typeof pattern === 'string')
+        return artifactPath.includes(pattern);
+    return pattern.test(artifactPath);
+}
+function buildTrialResult(log, result, conversationTokens, conversationCost) {
+    const nCommands = log.filter((entry) => entry.type === 'command').length;
+    const skills = extractSkillsFromLog(log);
+    return {
+        trial_id: 0,
+        reward: result.score,
+        scorer_results: result.scorers.map(toTrialScorerResult),
+        duration_ms: 0,
+        n_commands: nCommands,
+        input_tokens: result.tokenUsage?.inputTokens ?? 0,
+        output_tokens: result.tokenUsage?.outputTokens ?? 0,
+        ...conversationTokens,
+        // `total_cost_usd` is intentionally NOT emitted here: total cost is
+        // conservative — emitted only when every included component has a
+        // known cost. Today judge LLM providers expose no cost, so a partial
+        // total would mislead consumers. `conversation_cost_usd` is the only
+        // guaranteed cost surface; future judge-cost work unlocks the total
+        // field.
+        ...conversationCost,
+        session_log: [...log],
+        ...(skills.length > 0 ? { skills_used: skills } : {}),
+    };
+}
+function getProcessEnv() {
+    const env = {};
+    for (const [key, value] of Object.entries(process.env)) {
+        if (typeof value === 'string') {
+            env[key] = value;
+        }
+    }
+    return env;
+}
+export const evaluate = Object.assign(evaluateAgent, {
+    fromSnapshot,
+});
+function toTrialScorerResult(result) {
+    return {
+        scorer_type: mapScorerType(result.type),
+        score: result.score,
+        weight: result.weight,
+        details: result.details ?? '',
+        status: result.status,
+    };
+}
+function mapScorerType(type) {
+    switch (type) {
+        case 'check':
+        case 'score':
+            return 'deterministic';
+        case 'judge':
+            return 'llm_rubric';
+        case 'tool_usage':
+            return 'tool_usage';
+    }
+}
+function makeSkipped(scorer) {
+    return {
+        name: scorer.name,
+        type: scorer.type,
+        score: 0,
+        weight: scorer.weight ?? 1,
+        details: 'skipped (fail-fast)',
+        status: 'skipped',
+    };
+}
+function formatScorerErrorMessage(scorerErrors) {
+    const joined = scorerErrors
+        .map((result) => `${result.name}: ${result.details ?? 'unknown error'}`)
+        .join('; ');
+    return `Scorer evaluation failed: ${joined}`;
+}

package/dist/sdk/index.d.ts

@@ -0,0 +1,31 @@
+export { createAgent } from './agent.js';
+export { resolveAgentName, resolveCodexTransport, InvalidTransportEnvError, } from './agent-resolution.js';
+export { AgentCrashError } from './agent-crash.js';
+export { check, score, judge, toolUsage } from './scorers.js';
+export { evaluate, EvalScorerError } from './evaluate.js';
+export { RUN_SNAPSHOT_VERSION, buildRunSnapshot, loadRunSnapshot, SnapshotParseError, SnapshotVersionError, WorkspaceMissingError, } from './snapshots.js';
+export { createPersona } from './persona.js';
+export { createConversationWindow } from './conversation-window.js';
+export { runJudgePipeline } from './judge-pipeline.js';
+export { runScorer } from './run-scorer.js';
+export { previewReactions } from './reaction-preview.js';
+export { setRuntime, resetRuntime } from './eval-runtime.js';
+export { DEFAULT_COPY_IGNORE } from '../providers/copy-filter.js';
+export { createAskBus, requireAskBusForLiveBatches, AskBusTimeoutError } from './ask-bus/bus.js';
+export { toAskUserToolEvent } from './ask-bus/projection.js';
+export type { AskUserToolEvent, AskUserToolEventArguments, AskUserToolEventQuestionArgument, } from './ask-bus/projection.js';
+export { buildAskBatchLogEntries } from './agent-result-log.js';
+export { getAgentCapabilities } from './types.js';
+export type { AgentTransport, AgentCapabilities, AgentName } from './types.js';
+export type { AskBus, AskBatch, AskQuestion, AskOption, AskAnswer, AskResolution, AskBatchSnapshot, AskAnswerSnapshot, AskResolutionSnapshot, AskHandle, AskHandler, AskSource, AskLifecycle, AskAnswerSource, Unsubscribe as AskBusUnsubscribe, } from './ask-bus/types.js';
+export type { Agent, AgentOptions, Message, Scorer, CheckScorer, ScoreScorer, JudgeScorer, ToolUsageScorer, ScorerContext, EvalResult, ScorerResultEntry, ScorerStatus, ChatSession, ConversationResult, ConverseOptions, Reaction, TextReaction, AskUserReaction, AskUserQuestion, AskUserOption, ReactionPreviewEntry, TextReactionPreviewEntry, AskUserReactionPreviewEntry, ReactionPreviewResult, ReactionPreviewTurn, Persona, PersonaConfig, ConversationWindowConfig, PathgradePluginOptions, PathgradeMeta, TurnTiming, TokenUsage, EvaluateOptions, ReactionPreviewStatus, } from './types.js';
+export type { ConversationWindow, ConversationWindowOptions } from './conversation-window.js';
+export type { JudgePipelineOptions } from './judge-pipeline.js';
+export type { RunScorerOptions } from './run-scorer.js';
+export type { OnScorerErrorMode } from './evaluate.js';
+export type { RunSnapshot } from './snapshots.js';
+export type { LLMPort, EvalRuntime } from './eval-runtime.js';
+export { createLLMClient, ProviderNotSupportedError } from '../utils/llm.js';
+export type { CreateLLMClientOptions, LLMProviderAdapter, TokenUsage as LLMTokenUsage } from '../utils/llm.js';
+export { createMockLLM } from '../utils/llm-mocks.js';
+export type { CreateMockLLMOptions, MockResponse, MockLLM } from '../utils/llm-mocks.js';

package/dist/sdk/index.js

@@ -0,0 +1,20 @@
+// @wix/pathgrade — public API for writing evaluations
+export { createAgent } from './agent.js';
+export { resolveAgentName, resolveCodexTransport, InvalidTransportEnvError, } from './agent-resolution.js';
+export { AgentCrashError } from './agent-crash.js';
+export { check, score, judge, toolUsage } from './scorers.js';
+export { evaluate, EvalScorerError } from './evaluate.js';
+export { RUN_SNAPSHOT_VERSION, buildRunSnapshot, loadRunSnapshot, SnapshotParseError, SnapshotVersionError, WorkspaceMissingError, } from './snapshots.js';
+export { createPersona } from './persona.js';
+export { createConversationWindow } from './conversation-window.js';
+export { runJudgePipeline } from './judge-pipeline.js';
+export { runScorer } from './run-scorer.js';
+export { previewReactions } from './reaction-preview.js';
+export { setRuntime, resetRuntime } from './eval-runtime.js';
+export { DEFAULT_COPY_IGNORE } from '../providers/copy-filter.js';
+export { createAskBus, requireAskBusForLiveBatches, AskBusTimeoutError } from './ask-bus/bus.js';
+export { toAskUserToolEvent } from './ask-bus/projection.js';
+export { buildAskBatchLogEntries } from './agent-result-log.js';
+export { getAgentCapabilities } from './types.js';
+export { createLLMClient, ProviderNotSupportedError } from '../utils/llm.js';
+export { createMockLLM } from '../utils/llm-mocks.js';

package/dist/sdk/judge-pipeline.d.ts

@@ -0,0 +1,7 @@
+import type { JudgeScorer, ScorerContext, ScorerResultEntry } from './types.js';
+import type { LLMPort } from '../utils/llm-types.js';
+export interface JudgePipelineOptions {
+    /** Override the LLM. Defaults to getRuntime().llm at call time. */
+    llm?: LLMPort;
+}
+export declare function runJudgePipeline(judges: JudgeScorer | JudgeScorer[], ctx: ScorerContext, opts?: JudgePipelineOptions): Promise<ScorerResultEntry[]>;

package/dist/sdk/judge-pipeline.js

@@ -0,0 +1,178 @@
+import { getRuntime } from './eval-runtime.js';
+import { runJudgeWithTools } from './judge-tool-runner.js';
+import { buildBatchedJudgePrompt, buildJudgePrompt } from './judge-prompt-builder.js';
+import { clamp, getErrorMessage } from './scorer-utils.js';
+export async function runJudgePipeline(judges, ctx, opts) {
+    const list = Array.isArray(judges) ? judges : [judges];
+    const llm = opts?.llm ?? getRuntime().llm;
+    // Partition BEFORE model grouping: tool-using judges always run individually
+    // against the tool-use runner. Batching would silently produce wrong scores
+    // for a tool-using judge sharing a model group with a plain judge.
+    const toolJudges = list.filter(hasTools);
+    const plainJudges = list.filter((j) => !hasTools(j));
+    const toolResultsPromise = Promise.all(toolJudges.map((j) => runJudgeWithToolsEntry(j, ctx, llm)));
+    const plainResultsPromise = runPlainJudges(plainJudges, ctx, llm);
+    const [toolResults, plainResults] = await Promise.all([toolResultsPromise, plainResultsPromise]);
+    // Preserve original order.
+    const byName = new Map();
+    for (const r of [...toolResults, ...plainResults])
+        byName.set(r.name, r);
+    return list.map((j) => byName.get(j.name));
+}
+function hasTools(j) {
+    return Array.isArray(j.tools) && j.tools.length > 0;
+}
+async function runJudgeWithToolsEntry(scorer, ctx, llm) {
+    const result = await runJudgeWithTools(scorer, ctx, llm);
+    return result.entry;
+}
+async function runPlainJudges(judges, ctx, llm) {
+    if (judges.length === 0)
+        return [];
+    if (judges.length <= 1) {
+        return Promise.all(judges.map((g) => runJudgeScorer(g, ctx, llm)));
+    }
+    // Group judges by model — only batch judges with the same model
+    const byModel = new Map();
+    for (const j of judges) {
+        const key = j.model ?? '__default__';
+        const group = byModel.get(key) ?? [];
+        group.push(j);
+        byModel.set(key, group);
+    }
+    // If all judges have different models, run individually
+    if (byModel.size === judges.length) {
+        return Promise.all(judges.map((g) => runJudgeScorer(g, ctx, llm)));
+    }
+    const groupResults = await Promise.all([...byModel.values()].map(async (group) => {
+        if (group.length === 1) {
+            return [await runJudgeScorer(group[0], ctx, llm)];
+        }
+        // Attempt batched execution
+        const batchResults = await runBatchedJudges(group, ctx, llm);
+        if (batchResults) {
+            return batchResults;
+        }
+        // Fallback to individual calls
+        return Promise.all(group.map((g) => runJudgeScorer(g, ctx, llm)));
+    }));
+    return groupResults.flat();
+}
+// --- Internal helpers ---
+async function runBatchedJudges(judges, ctx, llm) {
+    const resolvedInputs = await Promise.all(judges.map((judge) => resolveJudgeInput(judge, ctx)));
+    const prompt = buildBatchedJudgePrompt(judges, ctx, resolvedInputs);
+    try {
+        const response = await llm.call(prompt, { model: judges[0].model, cacheControl: true });
+        const cleaned = response.text.replace(/```(?:json)?\s*/g, '').replace(/```/g, '').trim();
+        const arrayMatch = cleaned.match(/\[[\s\S]*\]/);
+        if (!arrayMatch)
+            return null;
+        const parsed = JSON.parse(arrayMatch[0]);
+        if (!Array.isArray(parsed) || parsed.length !== judges.length)
+            return null;
+        return judges.map((j, i) => {
+            const entry = parsed[i];
+            return {
+                name: j.name,
+                type: 'judge',
+                score: clamp(parseFloat(String(entry.score)) || 0),
+                weight: j.weight,
+                details: entry.details || entry.reasoning || 'No details provided',
+                status: 'ok',
+            };
+        });
+    }
+    catch {
+        return null;
+    }
+}
+async function runJudgeScorer(scorer, ctx, llm) {
+    const input = await resolveJudgeInput(scorer, ctx);
+    const prompt = buildJudgePrompt(scorer, ctx, input);
+    const retries = getRetryCount(scorer.retry);
+    for (let attempt = 0; attempt <= retries; attempt++) {
+        try {
+            const response = await llm.call(prompt, { model: scorer.model, cacheControl: true });
+            return parseJudgeResponse(response.text, scorer);
+        }
+        catch (error) {
+            if (attempt === retries) {
+                return {
+                    name: scorer.name,
+                    type: 'judge',
+                    score: 0,
+                    weight: scorer.weight,
+                    details: getErrorMessage(error),
+                    status: 'error',
+                };
+            }
+            await wait(1000 * (2 ** attempt));
+        }
+    }
+    return {
+        name: scorer.name,
+        type: 'judge',
+        score: 0,
+        weight: scorer.weight,
+        details: 'Judge retry loop exited unexpectedly',
+        status: 'error',
+    };
+}
+function parseJudgeResponse(text, scorer) {
+    try {
+        const cleaned = text.replace(/```(?:json)?\s*/g, '').replace(/```/g, '').trim();
+        const jsonMatch = cleaned.match(/\{[\s\S]*\}/);
+        if (jsonMatch) {
+            const parsed = JSON.parse(jsonMatch[0]);
+            return {
+                name: scorer.name,
+                type: 'judge',
+                score: clamp(parseFloat(parsed.score) || 0),
+                weight: scorer.weight,
+                details: parsed.details || parsed.reasoning || 'No details provided',
+                status: 'ok',
+            };
+        }
+    }
+    catch {
+        const scoreMatch = text.match(/"score"\s*:\s*([\d.]+)/);
+        if (scoreMatch) {
+            return {
+                name: scorer.name,
+                type: 'judge',
+                score: clamp(parseFloat(scoreMatch[1]) || 0),
+                weight: scorer.weight,
+                details: 'Parsed score from truncated LLM response',
+                status: 'ok',
+            };
+        }
+    }
+    return {
+        name: scorer.name,
+        type: 'judge',
+        score: 0,
+        weight: scorer.weight,
+        details: `Failed to parse LLM response: ${text.substring(0, 200)}`,
+    };
+}
+async function resolveJudgeInput(scorer, ctx) {
+    if (!scorer.input)
+        return undefined;
+    if (typeof scorer.input === 'function') {
+        return scorer.input(ctx);
+    }
+    return scorer.input;
+}
+function getRetryCount(retry) {
+    if (retry === true) {
+        return 1;
+    }
+    if (typeof retry === 'number' && Number.isFinite(retry)) {
+        return Math.max(0, Math.floor(retry));
+    }
+    return 0;
+}
+function wait(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/dist/sdk/judge-prompt-builder.d.ts

@@ -0,0 +1,7 @@
+import type { JudgeScorer, ScorerContext } from './types.js';
+export declare function buildJudgePrompt(scorer: JudgeScorer, ctx: ScorerContext, input?: Record<string, unknown>): string;
+export declare function buildBatchedJudgePrompt(judges: JudgeScorer[], ctx: ScorerContext, inputs: Array<Record<string, unknown> | undefined>): string;
+export declare function buildToolUseJudgePrompt(scorer: JudgeScorer, ctx: ScorerContext): {
+    system: string;
+    user: string;
+};

package/dist/sdk/judge-prompt-builder.js

@@ -0,0 +1,76 @@
+export function buildJudgePrompt(scorer, ctx, input) {
+    const sections = [];
+    sections.push(`## Session Transcript\n${ctx.transcript}`);
+    if (scorer.includeToolEvents && ctx.toolEvents.length > 0) {
+        sections.push(`## Tool Events\n${formatToolEvents(ctx)}`);
+    }
+    if (input) {
+        for (const [key, value] of Object.entries(input)) {
+            const body = typeof value === 'string' ? value : JSON.stringify(value, null, 2);
+            sections.push(`## ${key}\n${body}`);
+        }
+    }
+    return `You are an evaluation judge. Score the following agent session on a scale from 0.0 to 1.0 based on the rubric below.
+
+${sections.join('\n\n')}
+
+## Rubric
+${scorer.rubric}
+
+Respond with ONLY a JSON object: {"score": <number>, "details": "<brief explanation>"}`;
+}
+function formatToolEvents(ctx) {
+    return ctx.toolEvents
+        .map((event) => {
+        const turn = event.turnNumber ? `turn ${event.turnNumber}` : 'instruction';
+        return `- ${turn}: ${event.action} via ${event.providerToolName} (${event.provider})`;
+    })
+        .join('\n');
+}
+export function buildBatchedJudgePrompt(judges, ctx, inputs) {
+    const sections = [];
+    sections.push(`## Session Transcript\n${ctx.transcript}`);
+    if (judges.some((j) => j.includeToolEvents) && ctx.toolEvents.length > 0) {
+        sections.push(`## Tool Events\n${formatToolEvents(ctx)}`);
+    }
+    const rubrics = judges.map((j, i) => {
+        const parts = [`### Rubric ${i + 1}: "${j.name}"\n${j.rubric}`];
+        const input = inputs[i];
+        if (input) {
+            for (const [key, value] of Object.entries(input)) {
+                const body = typeof value === 'string' ? value : JSON.stringify(value, null, 2);
+                parts.push(`#### ${key}\n${body}`);
+            }
+        }
+        return parts.join('\n\n');
+    }).join('\n\n');
+    return `You are an evaluation judge. Score the following agent session on each rubric below, from 0.0 to 1.0.
+
+${sections.join('\n\n')}
+
+${rubrics}
+
+Respond with ONLY a JSON array, one entry per rubric in order:
+[{"scorer_name": "<name>", "score": <number>, "details": "<brief explanation>"}, ...]`;
+}
+export function buildToolUseJudgePrompt(scorer, ctx) {
+    const system = [
+        'You are an evaluation judge with access to workspace-reading tools.',
+        'Use the tools to gather the evidence you need before scoring.',
+        'When you have enough evidence, reply with a final score as a fenced JSON block:',
+        '```json',
+        '{"score": <number 0..1>, "details": "<one-paragraph rationale citing evidence>"}',
+        '```',
+        'Do not include any other text after the JSON block.',
+    ].join('\n');
+    const parts = [];
+    parts.push('## Session Transcript');
+    parts.push(ctx.transcript);
+    if (scorer.includeToolEvents && ctx.toolEvents.length > 0) {
+        parts.push('## Tool Events');
+        parts.push(formatToolEvents(ctx));
+    }
+    parts.push('## Rubric');
+    parts.push(scorer.rubric);
+    return { system, user: parts.join('\n\n') };
+}

package/dist/sdk/judge-tool-runner.d.ts

@@ -0,0 +1,19 @@
+import type { ToolCapableLLMPort } from '../utils/llm-types.js';
+import type { JudgeScorer, ScorerContext, ScorerResultEntry, TokenUsage } from './types.js';
+import { type JudgeErrorCode, type JudgeToolCallRecord, type ToolExecutor } from './judge-tool-session.js';
+export type { JudgeErrorCode, JudgeToolCallRecord };
+export interface RunJudgeWithToolsResult {
+    entry: ScorerResultEntry;
+    rounds: number;
+    toolCalls: JudgeToolCallRecord[];
+    tokenUsage: TokenUsage;
+    wallTimeMs: number;
+}
+export interface RunJudgeWithToolsOptions {
+    maxRounds?: number;
+    /** Override the per-tool timeout (ms). Default 10_000. */
+    toolTimeoutMs?: number;
+    /** Override tool execution; used by tests that need to simulate slow or buggy tools. */
+    toolExecutor?: ToolExecutor;
+}
+export declare function runJudgeWithTools(scorer: JudgeScorer, ctx: ScorerContext, llm: ToolCapableLLMPort, opts?: RunJudgeWithToolsOptions): Promise<RunJudgeWithToolsResult>;