@wix/pathgrade 0.29.0

package/dist/sdk/agent.js

@@ -0,0 +1,354 @@
+import { prepareWorkspace } from '../providers/workspace.js';
+import { resolveAgentName, resolveCodexTransport } from './agent-resolution.js';
+import { lifecycle } from '../plugin/lifecycle.js';
+import { ChatSessionImpl } from './chat.js';
+import { runConversation } from './converse.js';
+import { createPersona } from './persona.js';
+import { evaluate } from './evaluate.js';
+import { createManagedSession } from './managed-session.js';
+import { createAgentLLM } from '../utils/llm.js';
+import { buildRunSnapshot } from './snapshots.js';
+import { buildModelAgentResultLogEntry } from './agent-result-log.js';
+import { getVisibleAssistantMessage } from './visible-turn.js';
+import { createVerboseEmitter } from '../reporters/verbose-emitter.js';
+import fs from 'fs-extra';
+import * as path from 'path';
+/**
+ * Test-only injection point: override the sink used by the next emitter
+ * built inside `createAgent`. Pass `null` to restore the default (stderr).
+ * Intended for unit tests only — do not use in production code paths.
+ */
+let verboseSinkOverride = null;
+export function __setVerboseSinkForTesting(sink) {
+    verboseSinkOverride = sink;
+}
+class AgentImpl {
+    ws;
+    agentName;
+    llm;
+    timeoutSetting;
+    modelOpt;
+    conversationWindowOpt;
+    interactionMode = null;
+    _messages = [];
+    _log = [];
+    disposed = false;
+    debugOpt;
+    debugName;
+    debugBaseDir;
+    lastConversationResult = null;
+    verbose;
+    transport;
+    constructor(ws, agentName, llm, timeoutSetting, conversationWindow, modelOpt, debugOpt, debugName, debugBaseDir, verbose, transport) {
+        this.ws = ws;
+        this.agentName = agentName;
+        this.llm = llm;
+        this.timeoutSetting = timeoutSetting;
+        this.modelOpt = modelOpt;
+        this.conversationWindowOpt = conversationWindow;
+        this.debugOpt = debugOpt;
+        this.debugName = debugName;
+        this.debugBaseDir = debugBaseDir;
+        this.verbose = verbose;
+        this.transport = transport;
+    }
+    get messages() {
+        return this._messages;
+    }
+    get log() {
+        return this._log;
+    }
+    get workspace() {
+        return this.ws.path;
+    }
+    guardInteraction(mode) {
+        if (this.interactionMode && this.interactionMode !== mode) {
+            throw new Error(`Cannot use ${mode}() after ${this.interactionMode}() — an agent supports only one interaction method`);
+        }
+        this.interactionMode = mode;
+    }
+    createSession(timeoutSec, askUserTimeoutMs) {
+        return createManagedSession({
+            ws: this.ws,
+            agentName: this.agentName,
+            timeoutSec,
+            messages: this._messages,
+            log: this._log,
+            model: this.modelOpt,
+            conversationWindow: this.conversationWindowOpt,
+            llm: this.llm,
+            ...(askUserTimeoutMs !== undefined ? { askUserTimeoutMs } : {}),
+            ...(this.transport !== undefined ? { transport: this.transport } : {}),
+        });
+    }
+    resolveTimeoutSec(mode, maxTurns) {
+        if (this.timeoutSetting !== 'auto') {
+            return this.timeoutSetting;
+        }
+        if (mode !== 'runConversation') {
+            throw new Error("timeout: 'auto' is only supported for runConversation()");
+        }
+        const turns = maxTurns ?? 30;
+        return Math.ceil((turns * 80_000 + 200_000) / 1000);
+    }
+    async executeLoggedTurnResult(session, message, turnNumber, kind) {
+        const timestamp = () => new Date().toISOString();
+        this._messages.push({ role: 'user', content: message });
+        this._log.push({
+            type: kind,
+            timestamp: timestamp(),
+            instruction: message,
+            ...(kind === 'user_reply' ? { turn_number: turnNumber } : {}),
+        });
+        this.verbose.turnStart({ turn: turnNumber, kind, message });
+        const turnStart = Date.now();
+        const turnResult = await session.executeTurn(message);
+        const response = getVisibleAssistantMessage(turnResult);
+        const durationMs = Date.now() - turnStart;
+        this._log.push(buildModelAgentResultLogEntry({
+            timestamp: timestamp(),
+            turnNumber,
+            durationMs,
+            turnResult,
+            assistantMessage: response,
+        }));
+        for (const toolEvent of turnResult.toolEvents) {
+            this._log.push({
+                type: 'tool_event',
+                timestamp: timestamp(),
+                tool_event: toolEvent,
+            });
+            this.verbose.toolEvent({ action: toolEvent.action, summary: toolEvent.summary });
+        }
+        this._messages.push({ role: 'agent', content: response });
+        this.verbose.turnEnd({
+            turn: turnNumber,
+            durationMs,
+            outputLines: response.split('\n').length,
+            messagePreview: response,
+        });
+        if (turnResult.exitCode !== 0) {
+            const timeoutSec = this.resolveTimeoutSec(this.interactionMode ?? 'prompt');
+            if (turnResult.timedOut) {
+                throw new Error(`Agent (limit: ${timeoutSec}s) timed out (agent killed)`);
+            }
+            throw new Error(`Agent exited with code ${turnResult.exitCode}`);
+        }
+        return turnResult;
+    }
+    async executeLoggedTurn(session, message, turnNumber, kind) {
+        const turnResult = await this.executeLoggedTurnResult(session, message, turnNumber, kind);
+        return getVisibleAssistantMessage(turnResult);
+    }
+    async prompt(message) {
+        this.guardInteraction('prompt');
+        if (this._messages.length > 0) {
+            throw new Error('prompt() can only be called once per agent');
+        }
+        const ms = this.createSession(this.resolveTimeoutSec('prompt'));
+        try {
+            return await this.executeLoggedTurn(ms, message, 1, 'agent_start');
+        }
+        finally {
+            await ms.dispose?.();
+        }
+    }
+    async startChat(firstMessage) {
+        this.guardInteraction('startChat');
+        if (this._messages.length > 0) {
+            throw new Error('startChat() can only be called once per agent');
+        }
+        const timeoutSec = this.resolveTimeoutSec('startChat');
+        const ms = this.createSession(timeoutSec);
+        const turnResult = await this.executeLoggedTurnResult(ms, firstMessage, 1, 'agent_start');
+        return new ChatSessionImpl(turnResult, {
+            messages: this._messages,
+            log: this._log,
+            exec: (cmd) => this.exec(cmd),
+            sendTurn: (message) => ms.executeTurn(message),
+            verbose: this.verbose,
+        });
+    }
+    async runConversation(opts) {
+        this.guardInteraction('runConversation');
+        if (this._messages.length > 0) {
+            throw new Error('runConversation() can only be called once per agent');
+        }
+        const timeoutSec = this.resolveTimeoutSec('runConversation', opts.maxTurns);
+        const ms = this.createSession(timeoutSec, opts.askUserTimeoutMs);
+        let turnNumber = 0;
+        const sendTurn = async (message) => {
+            const turnResult = await ms.executeTurn(message);
+            turnNumber++;
+            // Accumulate agent turn tokens (from CLI stream-json) on the shared tracker
+            if (turnResult.inputTokens || turnResult.outputTokens) {
+                this.llm.addTokens?.(turnResult.inputTokens ?? 0, turnResult.outputTokens ?? 0);
+            }
+            // Accumulate agent turn cost on the same shared tracker when
+            // the upstream provider reports it (Claude SDK populates
+            // `costUsd` from `total_cost_usd`; Codex/Cursor leave it
+            // undefined, in which case this is a no-op).
+            if (turnResult.costUsd !== undefined) {
+                this.llm.addCost?.(turnResult.costUsd);
+            }
+            for (const toolEvent of turnResult.toolEvents) {
+                this._log.push({
+                    type: 'tool_event',
+                    timestamp: new Date().toISOString(),
+                    tool_event: toolEvent,
+                });
+            }
+            // Exit-code failures are no longer thrown here. The runConversation
+            // loop projects the partial-turn through `pushModelAgentMessage`
+            // first (so `model_agent_result`, `ask_batch`, turn timings/details
+            // all capture the failed turn) and then throws — preserving
+            // observability into what the agent attempted before the failure.
+            return turnResult;
+        };
+        // Set up persona reply callback if persona config provided
+        let personaReply;
+        if (opts.persona) {
+            const personaWindowConfig = opts.persona.conversationWindow !== undefined
+                ? opts.persona.conversationWindow
+                : this.conversationWindowOpt;
+            const persona = createPersona({
+                ...opts.persona,
+                llm: opts.persona.llm ?? this.llm,
+                conversationWindow: personaWindowConfig,
+            });
+            personaReply = async () => {
+                const fakeChatSession = {
+                    turn: turnNumber,
+                    done: false,
+                    lastMessage: this._messages[this._messages.length - 1]?.content ?? '',
+                    messages: this._messages,
+                    reply: async () => { },
+                    hasFile: async () => false,
+                    end: () => { },
+                };
+                return persona.reply(fakeChatSession);
+            };
+        }
+        // Set up step scorer runner — pass this.llm explicitly so step
+        // scorer judge calls accumulate on the same tracker.
+        const agent = this;
+        const runStepScorers = async (scorers) => {
+            return evaluate(agent, scorers, { llm: this.llm });
+        };
+        try {
+            const result = await runConversation(opts, {
+                sendTurn,
+                hasFile: async (pattern) => {
+                    const result = await this.exec(`ls -d ${pattern} 2>/dev/null`);
+                    return result.stdout.trim().length > 0;
+                },
+                workspace: this.workspace,
+                messages: this._messages,
+                log: this._log,
+                personaReply,
+                runStepScorers,
+                verbose: this.verbose,
+                askBus: ms.askBus,
+                agentName: this.agentName,
+                agentTimeoutSec: timeoutSec,
+                ...(this.transport !== undefined ? { transport: this.transport } : {}),
+            });
+            this.lastConversationResult = result;
+            return result;
+        }
+        finally {
+            await ms.dispose?.();
+        }
+    }
+    async exec(cmd) {
+        return this.ws.exec(cmd);
+    }
+    transcript() {
+        const parts = [];
+        for (const msg of this._messages) {
+            const label = msg.role === 'user' ? 'User' : 'Agent';
+            parts.push(`[${label}]\n${msg.content}`);
+        }
+        return parts.join('\n\n');
+    }
+    async dispose() {
+        if (this.disposed)
+            return;
+        this.disposed = true;
+        // Do NOT call lifecycle.untrackAgent here — flush() in afterEach needs
+        // the agent to still be in pendingAgents so it can collect results
+        // (token usage, scorers, diagnostics) into task.meta.pathgrade.
+        if (this.debugOpt) {
+            const dest = typeof this.debugOpt === 'string'
+                ? this.debugOpt
+                : path.join(this.debugBaseDir, 'pathgrade-debug', this.debugName);
+            await fs.remove(dest);
+            await fs.copy(this.ws.path, dest);
+            if (this.interactionMode === 'runConversation' && this.lastConversationResult) {
+                const snapshot = buildRunSnapshot({
+                    agent: this.agentName,
+                    messages: this._messages,
+                    log: this._log,
+                    conversationResult: this.lastConversationResult,
+                    workspace: dest,
+                });
+                await fs.writeJSON(path.join(dest, 'run-snapshot.json'), snapshot, { spaces: 2 });
+            }
+        }
+        await this.ws.dispose();
+    }
+}
+/**
+ * Create an isolated trial for agent evaluation.
+ *
+ * Workspace is set up immediately (files copied, skills staged, MCP configured).
+ * The agent process is NOT started — it's spawned lazily on first instruct()/chat()/converse().
+ */
+function slugify(s) {
+    return s
+        .replace(/[^a-zA-Z0-9_-]/g, '-')
+        .replace(/-+/g, '-')
+        .replace(/^-|-$/g, '')
+        .toLowerCase();
+}
+function resolveTestContext() {
+    try {
+        const worker = globalThis.__vitest_worker__;
+        const name = worker?.current?.name ? slugify(worker.current.name) : '';
+        const dir = worker?.filepath ? path.dirname(worker.filepath) : '';
+        return { name, dir };
+    }
+    catch {
+        return { name: '', dir: '' };
+    }
+}
+export async function createAgent(opts) {
+    const agentName = resolveAgentName(opts, process.env);
+    const transport = agentName === 'codex'
+        ? resolveCodexTransport(opts, process.env)
+        : undefined;
+    const timeoutSetting = opts.timeout ?? 300;
+    // Capture test context now, while vitest state is available
+    const testCtx = opts.debug ? resolveTestContext() : { name: '', dir: '' };
+    const { timeout: _, mcpMock, agent: __, debug: ___, model: ____, transport: _____, ...rest } = opts;
+    const workspace = await prepareWorkspace({
+        ...rest,
+        agent: agentName,
+        mcp: mcpMock ? { mock: mcpMock } : undefined,
+    });
+    // Create agent LLM once, using the fully-resolved sandbox env (includes
+    // keychain OAuth tokens, API keys, safe host vars).
+    const llm = createAgentLLM(agentName, workspace.env);
+    // Fall back to sandbox dir name if no test name resolved
+    const debugName = testCtx.name || path.basename(path.dirname(workspace.path));
+    // Default debug dir is next to the eval file, fallback to cwd
+    const debugBaseDir = testCtx.dir || process.cwd();
+    const verbose = createVerboseEmitter({
+        enabled: process.env.PATHGRADE_VERBOSE === '1',
+        sink: verboseSinkOverride ?? undefined,
+        testName: testCtx.name || undefined,
+    });
+    const agent = new AgentImpl(workspace, agentName, llm, timeoutSetting, opts.conversationWindow, opts.model, opts.debug, debugName, debugBaseDir, verbose, transport);
+    lifecycle.trackAgent(agent);
+    return agent;
+}

package/dist/sdk/ask-bus/bus.d.ts

@@ -0,0 +1,21 @@
+import type { AskBus } from './types.js';
+interface AskBusConfig {
+    askUserTimeoutMs: number;
+}
+export declare class AskBusTimeoutError extends Error {
+    readonly batchId: string;
+    readonly turnNumber: number;
+    constructor(batchId: string, turnNumber: number, timeoutMs: number);
+}
+export declare function createAskBus(config: AskBusConfig): AskBus;
+export interface AgentSessionOptionsWithBus {
+    askBus?: AskBus;
+}
+/**
+ * Helper for drivers that MUST have a real subscriber because they emit
+ * `lifecycle: 'live'` batches. Throws at driver construction when the bus is
+ * missing — silently resolving to `null` would send an empty answer map on the
+ * wire, which is a worse failure mode than refusing to start.
+ */
+export declare function requireAskBusForLiveBatches(sessionOptions: AgentSessionOptionsWithBus | undefined, driverName: string): AskBus;
+export {};

package/dist/sdk/ask-bus/bus.js

@@ -0,0 +1,129 @@
+export class AskBusTimeoutError extends Error {
+    batchId;
+    turnNumber;
+    constructor(batchId, turnNumber, timeoutMs) {
+        super(`ask_user batch ${batchId} on turn ${turnNumber} did not resolve within ${timeoutMs}ms`);
+        this.name = 'AskBusTimeoutError';
+        this.batchId = batchId;
+        this.turnNumber = turnNumber;
+    }
+}
+export function createAskBus(config) {
+    const { askUserTimeoutMs } = config;
+    const handlers = new Set();
+    const pending = [];
+    const redactAnswers = (answers, questions) => {
+        const secretIds = new Set(questions.filter((q) => q.isSecret).map((q) => q.id));
+        return answers.map((answer) => {
+            const isSecret = secretIds.has(answer.questionId);
+            const shouldRedact = isSecret && (answer.source === 'reaction' || answer.source === 'fallback');
+            return {
+                questionId: answer.questionId,
+                values: shouldRedact ? ['<redacted>'] : [...answer.values],
+                source: answer.source,
+            };
+        });
+    };
+    const toSnapshot = (entry) => {
+        const resolution = entry.resolution
+            ? { answers: redactAnswers(entry.resolution.answers, entry.batch.questions) }
+            : null;
+        return { ...entry.batch, resolution };
+    };
+    return {
+        emit(batch) {
+            const entry = { batch, resolution: null };
+            pending.push(entry);
+            if (batch.lifecycle === 'post-hoc') {
+                let warned = false;
+                let captured = false;
+                for (const handler of handlers) {
+                    try {
+                        handler(batch, (resolution) => {
+                            if (!captured) {
+                                entry.resolution = resolution;
+                                captured = true;
+                            }
+                            if (!warned) {
+                                warned = true;
+                                console.warn(`AskBus: responded to post-hoc batch ${batch.batchId} — ignored at the wire; use reactions for live batches`);
+                            }
+                        });
+                    }
+                    catch (err) {
+                        console.error('AskBus handler threw', err);
+                    }
+                }
+                return {
+                    batchId: batch.batchId,
+                    resolution: Promise.resolve(null),
+                };
+            }
+            // lifecycle === 'live'
+            let resolveFn;
+            let rejectFn;
+            const resolutionPromise = new Promise((resolve, reject) => {
+                resolveFn = resolve;
+                rejectFn = reject;
+            });
+            let settled = false;
+            const timer = setTimeout(() => {
+                if (settled)
+                    return;
+                settled = true;
+                rejectFn(new AskBusTimeoutError(batch.batchId, batch.turnNumber, askUserTimeoutMs));
+            }, askUserTimeoutMs);
+            if (typeof timer.unref === 'function') {
+                timer.unref();
+            }
+            const respond = (resolution) => {
+                if (settled) {
+                    console.debug(`AskBus: late respond() on batch ${batch.batchId} — first-wins, ignored`);
+                    return;
+                }
+                settled = true;
+                clearTimeout(timer);
+                entry.resolution = resolution;
+                resolveFn(resolution);
+            };
+            for (const handler of handlers) {
+                try {
+                    handler(batch, respond);
+                }
+                catch (err) {
+                    console.error('AskBus handler threw', err);
+                }
+            }
+            return {
+                batchId: batch.batchId,
+                resolution: resolutionPromise,
+            };
+        },
+        onAsk(handler) {
+            handlers.add(handler);
+            return () => {
+                handlers.delete(handler);
+            };
+        },
+        snapshot(turnNumber) {
+            const filtered = turnNumber === undefined
+                ? pending
+                : pending.filter((entry) => entry.batch.turnNumber === turnNumber);
+            return filtered.map(toSnapshot);
+        },
+    };
+}
+/**
+ * Helper for drivers that MUST have a real subscriber because they emit
+ * `lifecycle: 'live'` batches. Throws at driver construction when the bus is
+ * missing — silently resolving to `null` would send an empty answer map on the
+ * wire, which is a worse failure mode than refusing to start.
+ */
+export function requireAskBusForLiveBatches(sessionOptions, driverName) {
+    const bus = sessionOptions?.askBus;
+    if (!bus) {
+        throw new Error(`${driverName} requires AgentSessionOptions.askBus for live ask_user batches. ` +
+            `Construct the session via createManagedSession or pass an AskBus explicitly.`);
+    }
+    return bus;
+}

package/dist/sdk/ask-bus/handler.d.ts

@@ -0,0 +1,31 @@
+import type { AskHandler } from './types.js';
+import type { AskUserReactionPreviewEntry, Reaction } from '../types.js';
+export type UnmatchedAskUserDisposition = 'error' | 'first-option' | 'decline';
+export interface AskUserHandlerContext {
+    reactions: readonly Reaction[];
+    onUnmatchedAskUser: UnmatchedAskUserDisposition;
+    firedOnce: Set<number>;
+}
+export interface AskUserUnmatchedSignal {
+    batchId: string;
+    turnNumber: number;
+    /**
+     * The question text(s) on the live batch that triggered the unmatched
+     * signal — populated whether the trigger is `'error'` outright or a
+     * `'first-option'` degradation (free-text / isSecret). Consumers compose
+     * the user-facing completion detail from at least the first entry.
+     */
+    questionTexts: string[];
+}
+export interface AskUserHandlerApi {
+    readonly handler: AskHandler;
+    /** Preview entries recorded per batchId (live AND post-hoc). */
+    readonly previewByBatch: Map<string, AskUserReactionPreviewEntry[]>;
+    /**
+     * Set when a live batch falls through to `onUnmatchedAskUser: 'error'`.
+     * Caller inspects after each turn and ends the conversation with
+     * `completionReason: 'error'`.
+     */
+    getUnmatchedError(): AskUserUnmatchedSignal | null;
+}
+export declare function createAskUserHandler(ctx: AskUserHandlerContext): AskUserHandlerApi;