eqho-eval 0.5.0

package/dist/cli/commands/action-eval.js

@@ -0,0 +1,133 @@
+import { Command } from "commander";
+import chalk from "chalk";
+import ora from "ora";
+import fs from "node:fs";
+import path from "node:path";
+import { EqhoClient, EqhoApiError } from "../../core/eqho-client.js";
+import { fetchCampaignAgents, resolveSystemPromptSections, } from "../../core/agent-fetcher.js";
+import { assemblePrompt } from "../../core/prompt-assembler.js";
+import { generateActionEvalConfig } from "../../core/action-eval-builder.js";
+import { loadAuth } from "../auth-store.js";
+import { getProxyConfig } from "../../core/provider-mapper.js";
+import { getId } from "../../types/helpers.js";
+export const actionEvalCommand = new Command("action-eval")
+    .description("Generate eval config for testing live (in-call) action/tool usage")
+    .option("-c, --campaign <id>", "Campaign ID (reads from eqho.config.json if omitted)")
+    .option("-a, --agent <id>", "Specific agent ID (default: first agent in campaign)")
+    .option("-n, --calls <count>", "Number of real calls to use as ground truth", "0")
+    .option("-o, --output <dir>", "Output directory", ".")
+    .option("-d, --dir <dir>", "Project directory (for reading eqho.config.json)", ".")
+    .action(async (opts) => {
+    const auth = loadAuth();
+    if (!auth) {
+        console.log(chalk.red("Not authenticated. Run: eqho-eval auth"));
+        process.exit(1);
+    }
+    const client = new EqhoClient(auth);
+    let campaignId = opts.campaign;
+    if (!campaignId) {
+        const configPath = path.resolve(opts.dir, "eqho.config.json");
+        if (fs.existsSync(configPath)) {
+            try {
+                const config = JSON.parse(fs.readFileSync(configPath, "utf-8"));
+                campaignId = config.campaignId;
+            }
+            catch {
+                // ignore
+            }
+        }
+    }
+    if (!campaignId) {
+        console.log(chalk.red("No campaign ID. Use --campaign <id> or run from an init'd project."));
+        process.exit(1);
+    }
+    const spinner = ora({ text: "Fetching campaign and agent data...", stream: process.stderr }).start();
+    let campaign;
+    try {
+        campaign = await client.getCampaign(campaignId);
+    }
+    catch (err) {
+        spinner.fail(`Failed to fetch campaign: ${err instanceof EqhoApiError ? err.message : String(err)}`);
+        process.exit(1);
+    }
+    const { agents, errors } = await fetchCampaignAgents(client, campaign, {
+        agentId: opts.agent,
+        onProgress: (msg) => {
+            spinner.text = msg;
+        },
+    });
+    if (errors.length) {
+        for (const e of errors)
+            console.log(chalk.yellow(`  Warning: ${e}`));
+    }
+    if (!agents.length) {
+        spinner.fail("No agents loaded from campaign.");
+        process.exit(1);
+    }
+    const agentData = agents[0];
+    const { agent } = agentData;
+    const { sections, seeder } = resolveSystemPromptSections(agent, campaign);
+    spinner.text = `Assembling prompt for ${agent.name}...`;
+    const assembled = assemblePrompt({
+        agent,
+        campaign,
+        roles: agent.roles,
+        actions: agent.actions,
+        scripts: agent.scripts,
+        systemPromptSections: sections,
+        conversationSeeder: seeder,
+    });
+    let calls;
+    const callCount = parseInt(opts.calls, 10);
+    if (callCount > 0) {
+        spinner.text = `Fetching ${callCount} real calls...`;
+        try {
+            const { calls: rawCalls } = await client.listCalls(campaignId, {
+                limit: callCount,
+            });
+            const fullCalls = [];
+            for (const call of rawCalls) {
+                spinner.text = `Fetching transcript ${fullCalls.length + 1}/${rawCalls.length}...`;
+                try {
+                    const full = await client.getCall(getId(call));
+                    if (full.transcript?.length)
+                        fullCalls.push(full);
+                }
+                catch {
+                    // skip
+                }
+            }
+            calls = fullCalls;
+            spinner.text = `Loaded ${fullCalls.length} calls with transcripts`;
+        }
+        catch (err) {
+            console.log(chalk.yellow(`  Warning: Failed to fetch calls: ${err instanceof EqhoApiError ? err.message : String(err)}`));
+        }
+    }
+    spinner.text = "Generating action eval config...";
+    const evalDir = path.join(path.resolve(opts.output), "action-eval");
+    const files = generateActionEvalConfig({
+        assembled,
+        actions: agent.actions,
+        campaignName: campaign.name,
+        calls,
+        proxy: getProxyConfig(loadAuth()),
+    });
+    let filesWritten = 0;
+    for (const [filePath, content] of Object.entries(files)) {
+        const fullPath = path.join(evalDir, filePath);
+        const dir = path.dirname(fullPath);
+        if (!fs.existsSync(dir))
+            fs.mkdirSync(dir, { recursive: true });
+        fs.writeFileSync(fullPath, content, "utf-8");
+        filesWritten++;
+    }
+    const liveCount = agent.actions.filter((a) => !a.execution_type || a.execution_type === "live").length;
+    spinner.succeed(`Generated ${filesWritten} files for ${campaign.name} (${liveCount} live actions)`);
+    console.log(chalk.dim(`  Action eval → ${evalDir}/`));
+    console.log();
+    console.log(chalk.bold("Next steps:"));
+    console.log(chalk.dim("  cd action-eval && npx promptfoo eval"));
+    console.log(chalk.dim("  npx promptfoo view"));
+});
+//# sourceMappingURL=action-eval.js.map

package/dist/cli/commands/action-eval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"action-eval.js","sourceRoot":"","sources":["../../../src/cli/commands/action-eval.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACrE,OAAO,EACL,mBAAmB,EACnB,2BAA2B,GAC5B,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAChE,OAAO,EAAE,wBAAwB,EAAE,MAAM,mCAAmC,CAAC;AAC7E,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAE/D,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAE/C,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,OAAO,CAAC,aAAa,CAAC;KACxD,WAAW,CACV,mEAAmE,CACpE;KACA,MAAM,CACL,qBAAqB,EACrB,sDAAsD,CACvD;KACA,MAAM,CACL,kBAAkB,EAClB,sDAAsD,CACvD;KACA,MAAM,CACL,qBAAqB,EACrB,6CAA6C,EAC7C,GAAG,CACJ;KACA,MAAM,CAAC,oBAAoB,EAAE,kBAAkB,EAAE,GAAG,CAAC;KACrD,MAAM,CACL,iBAAiB,EACjB,kDAAkD,EAClD,GAAG,CACJ;KACA,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACrB,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;IACxB,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC,CAAC;QACjE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;IACpC,IAAI,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC;IAE/B,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,kBAAkB,CAAC,CAAC;QAC9D,IAAI,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC9B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAmB,IAAI,CAAC,KAAK,CACvC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CACrC,CAAC;gBACF,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;YACjC,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,GAAG,CACP,oEAAoE,CACrE,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,OAAO,GAAG,GAAG,CAAC,EAAE,IAAI,EAAE,qCAAqC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;IAErG,IAAI,QAAQ,CAAC;IACb,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IAClD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,IAAI,CACV,6BAA6B,GAAG,YAAY,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACvF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE,QAAQ,EAAE;QACrE,OAAO,EAAE,IAAI,CAAC,KAAK;QACnB,UAAU,EAAE,CAAC,GAAG,EAAE,EAAE;YAClB,OAAO,CAAC,IAAI,GAAG,GAAG,CAAC;QACrB,CAAC;KACF,CAAC,CAAC;IAEH,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,KAAK,MAAM,CAAC,IAAI,MAAM;YACpB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC;IACjD,CAAC;IAED,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,OAAO,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;QAChD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;IAC7B,MAAM,EAAE,KAAK,EAAE,GAAG,SAAS,CAAC;IAE5B,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,2BAA2B,CACtD,KAAK,EACL,QAAQ,CACT,CAAC;IAEF,OAAO,CAAC,IAAI,GAAG,yBAAyB,KAAK,CAAC,IAAI,KAAK,CAAC;IAExD,MAAM,SAAS,GAAG,cAAc,CAAC;QAC/B,KAAK;QACL,QAAQ;QACR,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,oBAAoB,EAAE,QAAQ;QAC9B,kBAAkB,EAAE,MAAM;KAC3B,CAAC,CAAC;IAEH,IAAI,KAAK,CAAC;IACV,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC3C,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QAClB,OAAO,CAAC,IAAI,GAAG,YAAY,SAAS,gBAAgB,CAAC;QACrD,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,UAAU,EAAE;gBAC7D,KAAK,EAAE,SAAS;aACjB,CAAC,CAAC;YACH,MAAM,SAAS,GAAG,EAAE,CAAC;YACrB,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,GAAG,uBAAuB,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACnF,IAAI,CAAC;oBACH,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;oBAC/C,IAAI,IAAI,CAAC,UAAU,EAAE,MAAM;wBAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpD,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO;gBACT,CAAC;YACH,CAAC;YACD,KAAK,GAAG,SAAS,CAAC;YAClB,OAAO,CAAC,IAAI,GAAG,UAAU,SAAS,CAAC,MAAM,yBAAyB,CAAC;QACrE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,MAAM,CACV,qCAAqC,GAAG,YAAY,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAC/F,CACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,CAAC,IAAI,GAAG,kCAAkC,CAAC;IAElD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,aAAa,CAAC,CAAC;IACpE,MAAM,KAAK,GAAG,wBAAwB,CAAC;QACrC,SAAS;QACT,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,YAAY,EAAE,QAAQ,CAAC,IAAI;QAC3B,KAAK;QACL,KAAK,EAAE,cAAc,CAAC,QAAQ,EAAE,CAAC;KAClC,CAAC,CAAC;IAEH,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QAC9C,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAChE,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAC7C,YAAY,EAAE,CAAC;IACjB,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CACpC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,cAAc,IAAI,CAAC,CAAC,cAAc,KAAK,MAAM,CACxD,CAAC,MAAM,CAAC;IAET,OAAO,CAAC,OAAO,CACb,aAAa,YAAY,cAAc,QAAQ,CAAC,IAAI,KAAK,SAAS,gBAAgB,CACnF,CAAC;IAEF,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,GAAG,CAAC,mBAAmB,OAAO,GAAG,CAAC,CACzC,CAAC;IACF,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC;IACvC,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,GAAG,CAAC,wCAAwC,CAAC,CACpD,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC"}

package/dist/cli/commands/auth.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare const authCommand: Command;
+//# sourceMappingURL=auth.d.ts.map

package/dist/cli/commands/auth.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"auth.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/auth.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAUpC,eAAO,MAAM,WAAW,SA+EpB,CAAC"}

package/dist/cli/commands/auth.js

@@ -0,0 +1,156 @@
+import { Command } from "commander";
+import { input } from "@inquirer/prompts";
+import chalk from "chalk";
+import ora from "ora";
+import axios from "axios";
+import { EqhoClient } from "../../core/eqho-client.js";
+import { saveAuth, loadAuth, clearAuth } from "../auth-store.js";
+const DEFAULT_BACKEND = "https://evals.eqho-solutions.dev";
+export const authCommand = new Command("auth")
+    .description("Configure Eqho API authentication")
+    .option("--key <apiKey>", "Eqho API key (or enter interactively)")
+    .option("--login", "Sign in via browser (SSO) — enables org switching")
+    .option("--url <baseUrl>", "Eqho API base URL", "https://api.eqho.ai")
+    .option("--backend <url>", "eqho-eval backend URL", DEFAULT_BACKEND)
+    .option("--logout", "Remove stored credentials")
+    .action(async (opts) => {
+    if (opts.logout) {
+        clearAuth();
+        console.log(chalk.green("Credentials removed."));
+        return;
+    }
+    // ── SSO browser login flow ────────────────────────────────────────
+    if (opts.login) {
+        await handleLogin(opts.backend);
+        return;
+    }
+    // ── API key flow (existing) ───────────────────────────────────────
+    const existing = loadAuth();
+    if (existing && !opts.key) {
+        if (existing.authMode === "bearer") {
+            console.log(chalk.dim(`Current auth: SSO (${existing.email ?? "bearer"})`));
+        }
+        else {
+            console.log(chalk.dim(`Current API key: ${(existing.apiKey ?? "").slice(0, 8)}...`));
+        }
+        if (existing.backendToken) {
+            console.log(chalk.dim(`Backend: ${existing.backendUrl || DEFAULT_BACKEND}`));
+        }
+    }
+    const apiKey = opts.key ||
+        (await input({
+            message: "Eqho API key:",
+            validate: (v) => (v.trim().length > 0 ? true : "Required"),
+        }));
+    const spinner = ora({ text: "Validating connection...", stream: process.stderr }).start();
+    const client = new EqhoClient({
+        apiKey: apiKey.trim(),
+        baseUrl: opts.url,
+    });
+    const valid = await client.validateConnection();
+    if (!valid) {
+        spinner.fail("Could not connect to Eqho API. Check your API key.");
+        process.exit(1);
+    }
+    spinner.succeed("Authenticated with Eqho API");
+    let backendToken;
+    const backendUrl = opts.backend;
+    spinner.start("Registering with eqho-eval backend...");
+    try {
+        const res = await axios.post(`${backendUrl}/api/auth/token`, {
+            apiKey: apiKey.trim(),
+            apiUrl: opts.url,
+        }, { timeout: 10_000 });
+        backendToken = res.data.token;
+        spinner.succeed("Registered with backend (model proxy enabled)");
+    }
+    catch {
+        spinner.warn("Backend unavailable — direct mode (requires OPENAI_API_KEY)");
+    }
+    saveAuth({
+        authMode: "api_key",
+        apiKey: apiKey.trim(),
+        apiBaseUrl: opts.url,
+        backendUrl: backendToken ? backendUrl : undefined,
+        backendToken,
+    });
+    console.log(chalk.dim("Config saved to ~/.eqho-eval/config.json"));
+});
+async function handleLogin(backendUrl) {
+    const spinner = ora({ text: "Initiating browser login...", stream: process.stderr }).start();
+    let deviceCode;
+    let verificationUrl;
+    try {
+        const res = await axios.post(`${backendUrl}/api/auth/device`, {
+            backendUrl,
+        }, { timeout: 10_000 });
+        deviceCode = res.data.device_code;
+        verificationUrl = res.data.verification_url;
+    }
+    catch (err) {
+        spinner.fail("Could not reach eqho-eval backend.");
+        const msg = err instanceof Error ? err.message : "";
+        if (msg)
+            console.log(chalk.dim(`  ${msg}`));
+        process.exit(1);
+    }
+    spinner.succeed("Opening browser to authenticate with Eqho...");
+    console.log();
+    console.log(chalk.dim("  If the browser doesn't open, visit:"));
+    console.log(chalk.cyan(`  ${verificationUrl}`));
+    console.log();
+    // Open the browser
+    try {
+        const { exec } = await import("node:child_process");
+        const platform = process.platform;
+        const cmd = platform === "darwin" ? "open" :
+            platform === "win32" ? "start" :
+                "xdg-open";
+        exec(`${cmd} "${verificationUrl}"`);
+    }
+    catch { /* non-critical */ }
+    // Poll for completion
+    const pollSpinner = ora({ text: "Waiting for authentication...", stream: process.stderr }).start();
+    const maxAttempts = 150; // 10 min at 4s intervals
+    const pollInterval = 4_000;
+    for (let i = 0; i < maxAttempts; i++) {
+        await new Promise((r) => setTimeout(r, pollInterval));
+        try {
+            const res = await axios.post(`${backendUrl}/api/auth/device/poll`, {
+                device_code: deviceCode,
+            }, { timeout: 10_000 });
+            if (res.data.status === "completed") {
+                pollSpinner.succeed("Authenticated with Eqho");
+                const email = res.data.email;
+                const orgId = res.data.orgId;
+                if (email) {
+                    console.log(chalk.green(`  Signed in as ${email}`));
+                }
+                saveAuth({
+                    authMode: "bearer",
+                    apiBaseUrl: "https://api.eqho.ai",
+                    email,
+                    orgId,
+                    backendUrl,
+                    backendToken: res.data.token,
+                });
+                console.log(chalk.dim("  Config saved to ~/.eqho-eval/config.json"));
+                return;
+            }
+            if (res.data.status === "expired") {
+                pollSpinner.fail("Login session expired. Please try again.");
+                process.exit(1);
+            }
+        }
+        catch (err) {
+            const axErr = err;
+            if (axErr.response?.status === 400) {
+                pollSpinner.fail("Login session expired. Please try again.");
+                process.exit(1);
+            }
+        }
+    }
+    pollSpinner.fail("Login timed out. Please try again.");
+    process.exit(1);
+}
+//# sourceMappingURL=auth.js.map

package/dist/cli/commands/auth.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"auth.js","sourceRoot":"","sources":["../../../src/cli/commands/auth.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC1C,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAEjE,MAAM,eAAe,GAAG,kCAAkC,CAAC;AAE3D,MAAM,CAAC,MAAM,WAAW,GAAG,IAAI,OAAO,CAAC,MAAM,CAAC;KAC3C,WAAW,CAAC,mCAAmC,CAAC;KAChD,MAAM,CAAC,gBAAgB,EAAE,uCAAuC,CAAC;KACjE,MAAM,CAAC,SAAS,EAAE,mDAAmD,CAAC;KACtE,MAAM,CAAC,iBAAiB,EAAE,mBAAmB,EAAE,qBAAqB,CAAC;KACrE,MAAM,CAAC,iBAAiB,EAAE,uBAAuB,EAAE,eAAe,CAAC;KACnE,MAAM,CAAC,UAAU,EAAE,2BAA2B,CAAC;KAC/C,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACrB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,SAAS,EAAE,CAAC;QACZ,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC;QACjD,OAAO;IACT,CAAC;IAED,qEAAqE;IACrE,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QACf,MAAM,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAChC,OAAO;IACT,CAAC;IAED,qEAAqE;IACrE,MAAM,QAAQ,GAAG,QAAQ,EAAE,CAAC;IAC5B,IAAI,QAAQ,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;QAC1B,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,sBAAsB,QAAQ,CAAC,KAAK,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAC;QAC9E,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,oBAAoB,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QACvF,CAAC;QACD,IAAI,QAAQ,CAAC,YAAY,EAAE,CAAC;YAC1B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,QAAQ,CAAC,UAAU,IAAI,eAAe,EAAE,CAAC,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GACV,IAAI,CAAC,GAAG;QACR,CAAC,MAAM,KAAK,CAAC;YACX,OAAO,EAAE,eAAe;YACxB,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC;SAC3D,CAAC,CAAC,CAAC;IAEN,MAAM,OAAO,GAAG,GAAG,CAAC,EAAE,IAAI,EAAE,0BAA0B,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;IAE1F,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC;QAC5B,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE;QACrB,OAAO,EAAE,IAAI,CAAC,GAAG;KAClB,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,kBAAkB,EAAE,CAAC;IAChD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC;QACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,OAAO,CAAC,6BAA6B,CAAC,CAAC;IAE/C,IAAI,YAAgC,CAAC;IACrC,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC;IAEhC,OAAO,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC;IACvD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,UAAU,iBAAiB,EAAE;YAC3D,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE;YACrB,MAAM,EAAE,IAAI,CAAC,GAAG;SACjB,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QACxB,YAAY,GAAG,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC;QAC9B,OAAO,CAAC,OAAO,CAAC,+CAA+C,CAAC,CAAC;IACnE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,IAAI,CAAC,6DAA6D,CAAC,CAAC;IAC9E,CAAC;IAED,QAAQ,CAAC;QACP,QAAQ,EAAE,SAAS;QACnB,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE;QACrB,UAAU,EAAE,IAAI,CAAC,GAAG;QACpB,UAAU,EAAE,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;QACjD,YAAY;KACb,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC,CAAC;AACrE,CAAC,CAAC,CAAC;AAEL,KAAK,UAAU,WAAW,CAAC,UAAkB;IAC3C,MAAM,OAAO,GAAG,GAAG,CAAC,EAAE,IAAI,EAAE,6BAA6B,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;IAE7F,IAAI,UAAkB,CAAC;IACvB,IAAI,eAAuB,CAAC;IAE5B,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,UAAU,kBAAkB,EAAE;YAC5D,UAAU;SACX,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAExB,UAAU,GAAG,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC;QAClC,eAAe,GAAG,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC;IAC9C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;QACnD,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QACpD,IAAI,GAAG;YAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC;QAC5C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,OAAO,CAAC,8CAA8C,CAAC,CAAC;IAChE,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC,CAAC;IAChE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,eAAe,EAAE,CAAC,CAAC,CAAC;IAChD,OAAO,CAAC,GAAG,EAAE,CAAC;IAEd,mBAAmB;IACnB,IAAI,CAAC;QACH,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;QACpD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QAClC,MAAM,GAAG,GACP,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAChC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;gBAChC,UAAU,CAAC;QACb,IAAI,CAAC,GAAG,GAAG,KAAK,eAAe,GAAG,CAAC,CAAC;IACtC,CAAC;IAAC,MAAM,CAAC,CAAC,kBAAkB,CAAC,CAAC;IAE9B,sBAAsB;IACtB,MAAM,WAAW,GAAG,GAAG,CAAC,EAAE,IAAI,EAAE,+BAA+B,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;IACnG,MAAM,WAAW,GAAG,GAAG,CAAC,CAAC,yBAAyB;IAClD,MAAM,YAAY,GAAG,KAAK,CAAC;IAE3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,CAAC;QAEtD,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,UAAU,uBAAuB,EAAE;gBACjE,WAAW,EAAE,UAAU;aACxB,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;YAExB,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBACpC,WAAW,CAAC,OAAO,CAAC,yBAAyB,CAAC,CAAC;gBAE/C,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,KAA2B,CAAC;gBACnD,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,KAA2B,CAAC;gBAEnD,IAAI,KAAK,EAAE,CAAC;oBACV,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,kBAAkB,KAAK,EAAE,CAAC,CAAC,CAAC;gBACtD,CAAC;gBAED,QAAQ,CAAC;oBACP,QAAQ,EAAE,QAAQ;oBAClB,UAAU,EAAE,qBAAqB;oBACjC,KAAK;oBACL,KAAK;oBACL,UAAU;oBACV,YAAY,EAAE,GAAG,CAAC,IAAI,CAAC,KAAK;iBAC7B,CAAC,CAAC;gBAEH,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC,CAAC;gBACrE,OAAO;YACT,CAAC;YAED,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAClC,WAAW,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAC;gBAC7D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,GAAwC,CAAC;YACvD,IAAI,KAAK,CAAC,QAAQ,EAAE,MAAM,KAAK,GAAG,EAAE,CAAC;gBACnC,WAAW,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAC;gBAC7D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;IACH,CAAC;IAED,WAAW,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;IACvD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC"}

package/dist/cli/commands/cache.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare const cacheCommand: Command;
+//# sourceMappingURL=cache.d.ts.map

package/dist/cli/commands/cache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/cache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIpC,eAAO,MAAM,YAAY,SAqCrB,CAAC"}

package/dist/cli/commands/cache.js

@@ -0,0 +1,43 @@
+import { Command } from "commander";
+import chalk from "chalk";
+import { clearCache, cacheStats } from "../../core/api-cache.js";
+export const cacheCommand = new Command("cache")
+    .description("Manage API response cache")
+    .argument("<action>", "Action: clear, stats")
+    .action(async (action, _opts, cmd) => {
+    const jsonMode = cmd.parent?.opts().json;
+    switch (action) {
+        case "clear": {
+            const { cleared } = clearCache();
+            if (jsonMode) {
+                console.log(JSON.stringify({ cleared }));
+            }
+            else {
+                console.log(chalk.green(`  ✓ Cleared ${cleared} cached entries`));
+            }
+            break;
+        }
+        case "stats": {
+            const stats = cacheStats();
+            if (jsonMode) {
+                console.log(JSON.stringify(stats, null, 2));
+            }
+            else {
+                console.log();
+                console.log(chalk.bold("  Cache stats:"));
+                console.log(chalk.dim(`    Entries: ${stats.entries}`));
+                console.log(chalk.dim(`    Size:    ${(stats.sizeBytes / 1024).toFixed(1)} KB`));
+                if (stats.oldestMs !== null) {
+                    const mins = Math.floor(stats.oldestMs / 60_000);
+                    console.log(chalk.dim(`    Oldest:  ${mins}m ago`));
+                }
+                console.log();
+            }
+            break;
+        }
+        default:
+            console.log(chalk.red(`Unknown action: ${action}. Use: clear, stats`));
+            process.exit(1);
+    }
+});
+//# sourceMappingURL=cache.js.map

package/dist/cli/commands/cache.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../../src/cli/commands/cache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAEjE,MAAM,CAAC,MAAM,YAAY,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC;KAC7C,WAAW,CAAC,2BAA2B,CAAC;KACxC,QAAQ,CAAC,UAAU,EAAE,sBAAsB,CAAC;KAC5C,MAAM,CAAC,KAAK,EAAE,MAAc,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;IAC3C,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,IAAI,CAAC;IAEzC,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,MAAM,EAAE,OAAO,EAAE,GAAG,UAAU,EAAE,CAAC;YACjC,IAAI,QAAQ,EAAE,CAAC;gBACb,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,eAAe,OAAO,iBAAiB,CAAC,CAAC,CAAC;YACpE,CAAC;YACD,MAAM;QACR,CAAC;QACD,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,MAAM,KAAK,GAAG,UAAU,EAAE,CAAC;YAC3B,IAAI,QAAQ,EAAE,CAAC;gBACb,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAC9C,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,EAAE,CAAC;gBACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC;gBAC1C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,gBAAgB,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;gBACxD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,gBAAgB,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;gBACjF,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;oBAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,GAAG,MAAM,CAAC,CAAC;oBACjD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,gBAAgB,IAAI,OAAO,CAAC,CAAC,CAAC;gBACtD,CAAC;gBACD,OAAO,CAAC,GAAG,EAAE,CAAC;YAChB,CAAC;YACD,MAAM;QACR,CAAC;QACD;YACE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,mBAAmB,MAAM,qBAAqB,CAAC,CAAC,CAAC;YACvE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;AACH,CAAC,CAAC,CAAC"}

package/dist/cli/commands/ci.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare const ciCommand: Command;
+//# sourceMappingURL=ci.d.ts.map

package/dist/cli/commands/ci.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ci.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/ci.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAoGpC,eAAO,MAAM,SAAS,SAmClB,CAAC"}

package/dist/cli/commands/ci.js

@@ -0,0 +1,124 @@
+import { Command } from "commander";
+import chalk from "chalk";
+import fs from "node:fs";
+import path from "node:path";
+const WORKFLOW_TEMPLATE = `name: Eqho Agent Eval
+
+on:
+  pull_request:
+    paths:
+      - 'prompts/**'
+      - 'tools/**'
+      - 'promptfooconfig.yaml'
+      - 'tests/**'
+  workflow_dispatch:
+    inputs:
+      campaign_id:
+        description: 'Eqho Campaign ID'
+        required: false
+      threshold:
+        description: 'Minimum pass rate (0-100)'
+        required: false
+        default: '80'
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  eval:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run promptfoo eval
+        env:
+          OPENAI_API_KEY: \${{ secrets.OPENAI_API_KEY }}
+          EQHO_API_KEY: \${{ secrets.EQHO_API_KEY }}
+        run: npx promptfoo eval --no-cache -o output/eval-results.json
+
+      - name: Check pass rate
+        run: |
+          THRESHOLD=\${{ github.event.inputs.threshold || '80' }}
+          RESULT=$(node -e "
+            const r = require('./output/eval-results.json');
+            const s = r.results?.stats || r.stats;
+            const total = s.successes + s.failures + (s.errors || 0);
+            const pct = total > 0 ? (s.successes / total * 100) : 0;
+            console.log(JSON.stringify({ pct: pct.toFixed(1), pass: s.successes, fail: s.failures, errors: s.errors || 0, total }));
+            process.exit(pct < $THRESHOLD ? 1 : 0);
+          ")
+          echo "::notice::Eval results: $RESULT"
+
+      - name: Comment PR with results
+        if: github.event_name == 'pull_request' && always()
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            let body = '## Eqho Agent Eval Results\\n\\n';
+            try {
+              const r = JSON.parse(fs.readFileSync('output/eval-results.json', 'utf8'));
+              const s = r.results?.stats || r.stats;
+              const total = s.successes + s.failures + (s.errors || 0);
+              const pct = total > 0 ? (s.successes / total * 100).toFixed(1) : '0';
+              const emoji = pct >= 80 ? '✅' : pct >= 60 ? '⚠️' : '❌';
+              body += \`\${emoji} **\${pct}%** pass rate (\${s.successes}/\${total})\\n\\n\`;
+              body += \`| Metric | Count |\\n|--------|-------|\\n\`;
+              body += \`| Passed | \${s.successes} |\\n\`;
+              body += \`| Failed | \${s.failures} |\\n\`;
+              body += \`| Errors | \${s.errors || 0} |\\n\`;
+            } catch (e) {
+              body += '❌ Failed to parse eval results.\\n';
+            }
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body
+            });
+
+      - name: Upload results artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: eval-results
+          path: output/
+          retention-days: 30
+`;
+export const ciCommand = new Command("ci")
+    .description("Generate a GitHub Actions workflow for automated eval on PR")
+    .option("-o, --output <dir>", "Output directory", ".")
+    .option("--threshold <pct>", "Default pass rate threshold", "80")
+    .action(async (opts) => {
+    const outDir = path.resolve(opts.output);
+    const workflowDir = path.join(outDir, ".github", "workflows");
+    if (!fs.existsSync(workflowDir)) {
+        fs.mkdirSync(workflowDir, { recursive: true });
+    }
+    const workflowPath = path.join(workflowDir, "eqho-eval.yml");
+    const content = WORKFLOW_TEMPLATE.replace("default: '80'", `default: '${opts.threshold}'`);
+    fs.writeFileSync(workflowPath, content, "utf-8");
+    console.log(chalk.green(`GitHub Actions workflow written to ${workflowPath}`));
+    console.log();
+    console.log(chalk.bold("Required secrets:"));
+    console.log(chalk.dim("  OPENAI_API_KEY  — OpenAI API key for running evals"));
+    console.log(chalk.dim("  EQHO_API_KEY    — Eqho API key (only for sync commands)"));
+    console.log();
+    console.log(chalk.bold("Triggers:"));
+    console.log(chalk.dim("  - On PR when prompts/, tools/, or config files change"));
+    console.log(chalk.dim("  - Manual dispatch with optional campaign_id and threshold"));
+    console.log();
+    console.log(chalk.dim(`  Pass rate threshold: ${opts.threshold}% (PRs below this will show a warning)`));
+});
+//# sourceMappingURL=ci.js.map

package/dist/cli/commands/ci.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ci.js","sourceRoot":"","sources":["../../../src/cli/commands/ci.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,MAAM,iBAAiB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6FzB,CAAC;AAEF,MAAM,CAAC,MAAM,SAAS,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;KACvC,WAAW,CAAC,6DAA6D,CAAC;KAC1E,MAAM,CAAC,oBAAoB,EAAE,kBAAkB,EAAE,GAAG,CAAC;KACrD,MAAM,CAAC,mBAAmB,EAAE,6BAA6B,EAAE,IAAI,CAAC;KAChE,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACrB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACzC,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;IAE9D,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAChC,EAAE,CAAC,SAAS,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACjD,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;IAC7D,MAAM,OAAO,GAAG,iBAAiB,CAAC,OAAO,CACvC,eAAe,EACf,aAAa,IAAI,CAAC,SAAS,GAAG,CAC/B,CAAC;IAEF,EAAE,CAAC,aAAa,CAAC,YAAY,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAEjD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,sCAAsC,YAAY,EAAE,CAAC,CAAC,CAAC;IAC/E,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC;IAC7C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,sDAAsD,CAAC,CAAC,CAAC;IAC/E,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,2DAA2D,CAAC,CAAC,CAAC;IACpF,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;IACrC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,yDAAyD,CAAC,CAAC,CAAC;IAClF,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,6DAA6D,CAAC,CAAC,CAAC;IACtF,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,GAAG,CACP,0BAA0B,IAAI,CAAC,SAAS,wCAAwC,CACjF,CACF,CAAC;AACJ,CAAC,CAAC,CAAC"}

package/dist/cli/commands/conversations.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare const conversationsCommand: Command;
+//# sourceMappingURL=conversations.d.ts.map

package/dist/cli/commands/conversations.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"conversations.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/conversations.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAWpC,eAAO,MAAM,oBAAoB,SAiG7B,CAAC"}

package/dist/cli/commands/conversations.js

@@ -0,0 +1,89 @@
+import { Command } from "commander";
+import chalk from "chalk";
+import ora from "ora";
+import fs from "node:fs";
+import path from "node:path";
+import { EqhoClient } from "../../core/eqho-client.js";
+import { generateConversationTestsYaml } from "../../core/conversation-loader.js";
+import { loadAuth } from "../auth-store.js";
+import { getId } from "../../types/helpers.js";
+export const conversationsCommand = new Command("conversations")
+    .description("Pull real conversations from Eqho as test cases")
+    .option("-c, --campaign <id>", "Campaign ID (reads from eqho.config.json if omitted)")
+    .option("-n, --last <count>", "Number of recent calls to pull", "25")
+    .option("-s, --status <filter>", "Filter by call status")
+    .option("-o, --output <file>", "Output file", "tests/conversations.yaml")
+    .option("-d, --dir <dir>", "Project directory", ".")
+    .action(async (opts, cmd) => {
+    const jsonMode = cmd.parent?.opts().json;
+    const auth = loadAuth();
+    if (!auth) {
+        if (jsonMode) {
+            console.log(JSON.stringify({ error: "Not authenticated", fix: "eqho-eval auth --key <key>" }));
+        }
+        else {
+            console.log(chalk.red("Not authenticated. Run: eqho-eval auth"));
+        }
+        process.exit(1);
+    }
+    let campaignId = opts.campaign;
+    if (!campaignId) {
+        const configPath = path.resolve(opts.dir, "eqho.config.json");
+        if (fs.existsSync(configPath)) {
+            const config = JSON.parse(fs.readFileSync(configPath, "utf-8"));
+            campaignId = config.campaignId;
+        }
+    }
+    if (!campaignId) {
+        console.log(chalk.red("No campaign ID. Use --campaign <id> or run from an init'd project."));
+        process.exit(1);
+    }
+    const client = new EqhoClient(auth);
+    const spinner = ora({
+        text: `Pulling last ${opts.last} calls from campaign...`,
+        stream: process.stderr,
+    }).start();
+    const { calls } = await client.listCalls(campaignId, {
+        limit: parseInt(opts.last, 10),
+        status: opts.status,
+    });
+    if (!calls.length) {
+        spinner.warn("No calls found for this campaign.");
+        return;
+    }
+    // Fetch full transcripts for each call
+    const fullCalls = [];
+    for (const call of calls) {
+        spinner.text = `Fetching transcript ${fullCalls.length + 1}/${calls.length}...`;
+        try {
+            const full = await client.getCall(getId(call));
+            if (full.transcript?.length)
+                fullCalls.push(full);
+        }
+        catch {
+            // skip calls that fail to fetch
+        }
+    }
+    spinner.text = "Generating test cases...";
+    const yamlContent = generateConversationTestsYaml(fullCalls);
+    const outPath = path.resolve(opts.dir, opts.output);
+    const outDir = path.dirname(outPath);
+    if (!fs.existsSync(outDir))
+        fs.mkdirSync(outDir, { recursive: true });
+    fs.writeFileSync(outPath, yamlContent, "utf-8");
+    const lineCount = yamlContent.split("\n").length;
+    spinner.succeed(`Generated ${lineCount} lines of test cases from ${fullCalls.length} calls → ${opts.output}`);
+    if (jsonMode) {
+        console.log(JSON.stringify({
+            outputFile: outPath,
+            callsProcessed: fullCalls.length,
+            linesGenerated: lineCount,
+        }));
+    }
+    else {
+        console.log();
+        console.log(chalk.bold("To use in your eval:"));
+        console.log(chalk.dim("  Add to promptfooconfig.yaml:\n  tests: file://tests/conversations.yaml"));
+    }
+});
+//# sourceMappingURL=conversations.js.map

package/dist/cli/commands/conversations.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"conversations.js","sourceRoot":"","sources":["../../../src/cli/commands/conversations.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AACvD,OAAO,EAAE,6BAA6B,EAAE,MAAM,mCAAmC,CAAC;AAClF,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAE5C,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAE/C,MAAM,CAAC,MAAM,oBAAoB,GAAG,IAAI,OAAO,CAAC,eAAe,CAAC;KAC7D,WAAW,CAAC,iDAAiD,CAAC;KAC9D,MAAM,CAAC,qBAAqB,EAAE,sDAAsD,CAAC;KACrF,MAAM,CAAC,oBAAoB,EAAE,gCAAgC,EAAE,IAAI,CAAC;KACpE,MAAM,CAAC,uBAAuB,EAAE,uBAAuB,CAAC;KACxD,MAAM,CAAC,qBAAqB,EAAE,aAAa,EAAE,0BAA0B,CAAC;KACxE,MAAM,CAAC,iBAAiB,EAAE,mBAAmB,EAAE,GAAG,CAAC;KACnD,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE;IAC1B,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,IAAI,CAAC;IAEzC,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;IACxB,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,IAAI,QAAQ,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,mBAAmB,EAAE,GAAG,EAAE,4BAA4B,EAAE,CAAC,CAAC,CAAC;QACjG,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC,CAAC;QACnE,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC;IAE/B,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,kBAAkB,CAAC,CAAC;QAC9D,IAAI,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAmB,IAAI,CAAC,KAAK,CACvC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CACrC,CAAC;YACF,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;QACjC,CAAC;IACH,CAAC;IAED,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,GAAG,CACP,oEAAoE,CACrE,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;IACpC,MAAM,OAAO,GAAG,GAAG,CAAC;QAClB,IAAI,EAAE,gBAAgB,IAAI,CAAC,IAAI,yBAAyB;QACxD,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC,CAAC,KAAK,EAAE,CAAC;IAEX,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,UAAU,EAAE;QACnD,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;QAC9B,MAAM,EAAE,IAAI,CAAC,MAAM;KACpB,CAAC,CAAC;IAEH,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QAClB,OAAO,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;QAClD,OAAO;IACT,CAAC;IAED,uCAAuC;IACvC,MAAM,SAAS,GAAG,EAAE,CAAC;IACrB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,OAAO,CAAC,IAAI,GAAG,uBAAuB,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAChF,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YAC/C,IAAI,IAAI,CAAC,UAAU,EAAE,MAAM;gBAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpD,CAAC;QAAC,MAAM,CAAC;YACP,gCAAgC;QAClC,CAAC;IACH,CAAC;IAED,OAAO,CAAC,IAAI,GAAG,0BAA0B,CAAC;IAC1C,MAAM,WAAW,GAAG,6BAA6B,CAAC,SAAS,CAAC,CAAC;IAE7D,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACpD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtE,EAAE,CAAC,aAAa,CAAC,OAAO,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;IAEhD,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IACjD,OAAO,CAAC,OAAO,CACb,aAAa,SAAS,6BAA6B,SAAS,CAAC,MAAM,YAAY,IAAI,CAAC,MAAM,EAAE,CAC7F,CAAC;IAEF,IAAI,QAAQ,EAAE,CAAC;QACb,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC;YACzB,UAAU,EAAE,OAAO;YACnB,cAAc,EAAE,SAAS,CAAC,MAAM;YAChC,cAAc,EAAE,SAAS;SAC1B,CAAC,CAAC,CAAC;IACN,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC,CAAC;QAChD,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,GAAG,CACP,0EAA0E,CAC3E,CACF,CAAC;IACJ,CAAC;AACH,CAAC,CAAC,CAAC"}

package/dist/cli/commands/diff.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare const diffCommand: Command;
+//# sourceMappingURL=diff.d.ts.map

package/dist/cli/commands/diff.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"diff.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/diff.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAuBpC,eAAO,MAAM,WAAW,SA2IpB,CAAC"}