@wix/pathgrade 0.29.0

package/dist/utils/llm.js

@@ -0,0 +1,209 @@
+import { cliProvider } from './llm-providers/cli.js';
+import { anthropicProvider } from './llm-providers/anthropic.js';
+import { openaiProvider } from './llm-providers/openai.js';
+export class ProviderNotSupportedError extends Error {
+    name = 'ProviderNotSupportedError';
+    adapterName;
+    constructor(adapterName) {
+        super(`Adapter '${adapterName}' does not implement callWithTools. `
+            + 'Use an adapter that supports tool use (e.g., anthropicAdapter when '
+            + 'ANTHROPIC_API_KEY is set) or remove tools from this judge.');
+        this.adapterName = adapterName;
+    }
+}
+// Re-export provider utilities that callsites depend on
+export { isClaudeCliAvailable, isCodexCliAvailable, resetCliCache, extractStructuredOutput, parseCliEnvelope } from './llm-providers/cli.js';
+function inferProviderFromModel(model) {
+    const normalized = model?.trim().toLowerCase();
+    if (!normalized)
+        return undefined;
+    if (normalized.startsWith('claude'))
+        return 'anthropic';
+    if (normalized.startsWith('gpt-')
+        || normalized.startsWith('chatgpt-')
+        || normalized.startsWith('o1')
+        || normalized.startsWith('o3')
+        || normalized.startsWith('o4')) {
+        return 'openai';
+    }
+    return undefined;
+}
+export function createLLMClient(arg, legacyAgentName) {
+    const options = Array.isArray(arg)
+        ? { adapters: arg, agent: legacyAgentName }
+        : arg;
+    const providers = options.adapters ?? [];
+    const agentName = options.agent;
+    const silent = options.silent === true;
+    const warnedTransitions = new Set();
+    let inputTokens = 0;
+    let outputTokens = 0;
+    let costUsd = 0;
+    let lastProvider;
+    const warnFallthrough = (from, to, reason) => {
+        if (silent)
+            return;
+        const key = `${from}→${to}`;
+        if (warnedTransitions.has(key))
+            return;
+        warnedTransitions.add(key);
+        console.warn(`pathgrade: LLM provider fallthrough: ${from} failed (${reason}), using ${to}`);
+    };
+    const accumulate = (result) => {
+        inputTokens += result.inputTokens ?? 0;
+        outputTokens += result.outputTokens ?? 0;
+    };
+    const toolCapable = providers.find((p) => typeof p.callWithTools === 'function');
+    const port = {
+        get tokenUsage() {
+            return { inputTokens, outputTokens };
+        },
+        get costUsd() {
+            return costUsd;
+        },
+        get supportsToolUse() {
+            return !!toolCapable;
+        },
+        get lastProvider() {
+            return lastProvider;
+        },
+        addTokens(input, output) {
+            inputTokens += input;
+            outputTokens += output;
+        },
+        addCost(usd) {
+            // Defensive: ignore non-finite or negative deltas. The SDK
+            // reports a non-negative finite total_cost_usd in practice;
+            // anything else is corrupt and should not poison the tracker.
+            if (!Number.isFinite(usd) || usd < 0)
+                return;
+            costUsd += usd;
+        },
+        async measure(fn) {
+            const inBefore = inputTokens;
+            const outBefore = outputTokens;
+            const costBefore = costUsd;
+            const result = await fn();
+            return {
+                result,
+                tokens: {
+                    inputTokens: inputTokens - inBefore,
+                    outputTokens: outputTokens - outBefore,
+                },
+                costUsd: costUsd - costBefore,
+            };
+        },
+        async call(prompt, opts = {}) {
+            const requestedProviderType = inferProviderFromModel(opts.model);
+            // Filter to providers that support the requested model
+            let candidates = opts.model
+                ? providers.filter((p) => !p.supportsModel || p.supportsModel(opts.model))
+                : providers;
+            // If model explicitly targets a provider type (e.g., "gpt-4o" → openai),
+            // narrow to only providers of that type
+            const narrowed = requestedProviderType
+                ? candidates.filter((p) => p.name === requestedProviderType)
+                : candidates;
+            // Try narrowed set first, then fall back to all candidates
+            const toTry = narrowed.length > 0 ? narrowed : candidates;
+            let lastError;
+            let lastFailed;
+            for (const provider of toTry) {
+                if (!await provider.isAvailable(opts.env))
+                    continue;
+                try {
+                    const result = await provider.call(prompt, opts);
+                    if (lastFailed)
+                        warnFallthrough(lastFailed.name, provider.name, lastFailed.reason);
+                    accumulate(result);
+                    lastProvider = provider.name;
+                    return result;
+                }
+                catch (error) {
+                    lastError = error;
+                    lastFailed = { name: provider.name, reason: lastError.message };
+                    // Fall through to next provider
+                }
+            }
+            // If narrowed set failed/was unavailable, fall back to all candidates
+            if (toTry !== candidates) {
+                for (const provider of candidates) {
+                    if (toTry.includes(provider))
+                        continue; // already tried
+                    if (!await provider.isAvailable(opts.env))
+                        continue;
+                    try {
+                        const result = await provider.call(prompt, opts);
+                        if (lastFailed)
+                            warnFallthrough(lastFailed.name, provider.name, lastFailed.reason);
+                        accumulate(result);
+                        lastProvider = provider.name;
+                        return result;
+                    }
+                    catch (error) {
+                        lastError = error;
+                        lastFailed = { name: provider.name, reason: lastError.message };
+                    }
+                }
+            }
+            if (lastError)
+                throw lastError;
+            if (agentName === 'claude') {
+                throw new Error('Claude CLI not found on PATH. Install with: npm install -g @anthropic-ai/claude-code');
+            }
+            if (agentName === 'codex') {
+                throw new Error('Codex CLI not found on PATH. Install with: npm install -g @openai/codex');
+            }
+            if (agentName === 'cursor') {
+                throw new Error('Cursor eval requires a judge LLM; set ANTHROPIC_API_KEY or install Claude CLI, or override the judge model.');
+            }
+            throw new Error('No LLM backend available. Install Claude CLI (claude.ai) or set ANTHROPIC_API_KEY or OPENAI_API_KEY.');
+        },
+    };
+    if (toolCapable) {
+        port.callWithTools = async (messages, opts) => {
+            if (!await toolCapable.isAvailable(opts.env)) {
+                throw new Error('Tool-use judge requires ANTHROPIC_API_KEY; the Anthropic HTTP provider is not available.');
+            }
+            const result = await toolCapable.callWithTools(messages, opts);
+            accumulate(result);
+            lastProvider = toolCapable.name;
+            return result;
+        };
+    }
+    return port;
+}
+const defaultClient = createLLMClient([cliProvider, anthropicProvider, openaiProvider]);
+export async function callLLM(prompt, opts = {}) {
+    return defaultClient.call(prompt, opts);
+}
+/**
+ * Create an LLM client scoped to the providers that match the given agent.
+ *
+ * - claude → CLI + Anthropic API (no OpenAI fallthrough)
+ * - codex  → OpenAI API
+ *
+ * If `agentEnv` is provided, it is merged into every LLM call so that
+ * the agent's env propagates to persona/judge/summarization calls.
+ */
+export function createAgentLLM(agentName, agentEnv) {
+    // Tool-using judges need a provider that implements callWithTools.
+    // For claude, anthropicProvider is added as a tool-use-capable fallback
+    // alongside the CLI. The CLI still wins for plain call() when available.
+    // Cursor inherits the Claude chain by design (judge consistency across
+    // harnesses — see PRD §"LLM-backend routing"). Cursor evals therefore
+    // depend on Claude CLI or ANTHROPIC_API_KEY being available at judge time.
+    const baseAdapters = agentName === 'codex'
+        ? [openaiProvider]
+        : [cliProvider, anthropicProvider];
+    const adapters = agentEnv && Object.keys(agentEnv).length > 0
+        ? baseAdapters.map((a) => ({
+            ...a,
+            call: (prompt, opts) => a.call(prompt, { ...opts, env: { ...agentEnv, ...opts.env } }),
+            ...(a.callWithTools
+                ? { callWithTools: (messages, opts) => a.callWithTools(messages, { ...opts, env: { ...agentEnv, ...opts.env } }) }
+                : {}),
+        }))
+        : baseAdapters;
+    return createLLMClient({ adapters, agent: agentName });
+}

package/dist/utils/shutdown.d.ts

@@ -0,0 +1,15 @@
+type CleanupFn = () => Promise<void>;
+export declare class ShutdownManager {
+    private nextId;
+    private cleanups;
+    private shuttingDown;
+    private signalHandler;
+    register(cleanup: CleanupFn): number;
+    unregister(id: number): void;
+    shutdownAll(): Promise<void>;
+    install(): void;
+    uninstall(): void;
+}
+/** Global singleton for the CLI process */
+export declare const shutdown: ShutdownManager;
+export {};

package/dist/utils/shutdown.js

@@ -0,0 +1,41 @@
+export class ShutdownManager {
+    nextId = 0;
+    cleanups = new Map();
+    shuttingDown = false;
+    signalHandler;
+    register(cleanup) {
+        const id = this.nextId++;
+        this.cleanups.set(id, cleanup);
+        return id;
+    }
+    unregister(id) {
+        this.cleanups.delete(id);
+    }
+    async shutdownAll() {
+        if (this.shuttingDown)
+            return;
+        this.shuttingDown = true;
+        const tasks = [...this.cleanups.values()];
+        this.cleanups.clear();
+        await Promise.allSettled(tasks.map(fn => fn()));
+    }
+    install() {
+        if (this.signalHandler)
+            return;
+        this.signalHandler = () => {
+            this.shutdownAll().finally(() => process.exit(130));
+        };
+        process.on('SIGINT', this.signalHandler);
+        process.on('SIGTERM', this.signalHandler);
+    }
+    uninstall() {
+        if (this.signalHandler) {
+            process.removeListener('SIGINT', this.signalHandler);
+            process.removeListener('SIGTERM', this.signalHandler);
+            this.signalHandler = undefined;
+        }
+        this.shuttingDown = false;
+    }
+}
+/** Global singleton for the CLI process */
+export const shutdown = new ShutdownManager();

package/dist/utils/timeout.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Shared timeout utilities for agent and scorer execution.
+ */
+/**
+ * Run an async function with an AbortSignal that fires after timeoutMs.
+ * The signal is passed to `run` so it can propagate cancellation to child processes.
+ */
+export declare function withAbortTimeout<T>(run: (signal: AbortSignal) => Promise<T>, timeoutMs: number, label: string): Promise<T>;

package/dist/utils/timeout.js

@@ -0,0 +1,32 @@
+/**
+ * Shared timeout utilities for agent and scorer execution.
+ */
+/**
+ * Run an async function with an AbortSignal that fires after timeoutMs.
+ * The signal is passed to `run` so it can propagate cancellation to child processes.
+ */
+export function withAbortTimeout(run, timeoutMs, label) {
+    return new Promise((resolve, reject) => {
+        const controller = new AbortController();
+        let timedOut = false;
+        const timer = setTimeout(() => {
+            timedOut = true;
+            controller.abort();
+        }, timeoutMs);
+        run(controller.signal).then((val) => {
+            clearTimeout(timer);
+            if (timedOut || controller.signal.aborted) {
+                reject(new Error(`${label} timed out after ${timeoutMs / 1000}s`));
+                return;
+            }
+            resolve(val);
+        }, (err) => {
+            clearTimeout(timer);
+            if (timedOut || controller.signal.aborted) {
+                reject(new Error(`${label} timed out after ${timeoutMs / 1000}s`));
+                return;
+            }
+            reject(err);
+        });
+    });
+}