@root-signals/scorable-cli 0.1.0

package/dist/commands/prompt-test/run.js

@@ -0,0 +1,194 @@
+import { readFileSync, writeFileSync } from "node:fs";
+import yaml from "js-yaml";
+import Table from "cli-table3";
+import { requireApiKey } from "../../auth.js";
+import { apiRequest } from "../../client.js";
+import { printInfo, printSuccess, printWarning, printError } from "../../output.js";
+import { CliError } from "../../types.js";
+function isPromptTestComplete(exp) {
+    if (!exp.tasks.length)
+        return false;
+    return exp.tasks.every((t) => t.status === "completed" || t.status === "failed");
+}
+function displayProgressTable(experiments) {
+    const table = new Table({
+        head: ["Prompt Test ID", "Status", "Tasks Completed"],
+        style: { head: ["cyan"] },
+    });
+    for (const exp of experiments) {
+        const done = exp.tasks.filter((t) => t.status === "completed" || t.status === "failed").length;
+        const total = exp.tasks.length;
+        table.push([
+            exp.id,
+            isPromptTestComplete(exp) ? "✅ Completed" : "⏳ Running",
+            `${done}/${total}`,
+        ]);
+    }
+    console.log(table.toString());
+}
+function displayAggregatedResults(experiments) {
+    if (!experiments.length) {
+        printWarning("No prompt test results to display.");
+        return;
+    }
+    const allEvaluators = new Map();
+    for (const exp of experiments) {
+        for (const e of exp.evaluators) {
+            if (!allEvaluators.has(e.id))
+                allEvaluators.set(e.id, e.name);
+        }
+    }
+    const sortedEvalIds = [...allEvaluators.keys()].sort();
+    const table = new Table({
+        head: [
+            "Inputs",
+            "Prompt",
+            "Model",
+            "Cost",
+            "Latency (s)",
+            "Output",
+            ...sortedEvalIds.map((id) => allEvaluators.get(id)),
+        ],
+        style: { head: ["cyan"] },
+        wordWrap: true,
+    });
+    for (const exp of experiments) {
+        for (const task of exp.tasks) {
+            const inputsStr = Object.entries(task.variables)
+                .map(([k, v]) => `${k}: ${v}`)
+                .join("\n");
+            const row = [
+                inputsStr,
+                exp.prompt,
+                exp.model,
+                task.cost ?? "N/A",
+                task.model_call_duration != null ? task.model_call_duration.toFixed(3) : "N/A",
+                task.llm_output ?? "",
+            ];
+            const scores = new Map(task.evaluation_results.map((r) => [r.id, r.score]));
+            for (const evalId of sortedEvalIds) {
+                const score = scores.get(evalId);
+                row.push(score != null ? String(score) : "N/A");
+            }
+            table.push(row);
+        }
+    }
+    console.log(table.toString());
+}
+export async function runPromptTests(outputFile, configPath, sleep = (ms) => new Promise((r) => setTimeout(r, ms))) {
+    let rawConfig;
+    try {
+        rawConfig = yaml.load(readFileSync(configPath, "utf8"));
+    }
+    catch {
+        printError(`'${configPath}' not found. Please run \`pt init\` first or specify a different config file with -c.`);
+        throw new CliError(1, `Config file not found: ${configPath}`);
+    }
+    let config;
+    try {
+        config = rawConfig;
+        if (!config.prompts || !config.models || !config.evaluators) {
+            throw new Error("Missing required fields: prompts, models, evaluators");
+        }
+    }
+    catch (e) {
+        printError(`Error reading or validating '${configPath}': ${e instanceof Error ? e.message : String(e)}`);
+        throw new CliError(1, "Invalid config");
+    }
+    const apiKey = await requireApiKey();
+    printInfo("Starting prompt tests");
+    const experiments = {};
+    for (const prompt of config.prompts) {
+        for (const model of config.models) {
+            const evaluators = config.evaluators.map((e) => {
+                const entry = {};
+                if (e.id)
+                    entry["id"] = e.id;
+                if (e.name)
+                    entry["name"] = e.name;
+                if (e.version_id)
+                    entry["version_id"] = e.version_id;
+                return entry;
+            });
+            const payload = {
+                prompt,
+                inputs: config.inputs.map((i) => i.vars),
+                model,
+                evaluators,
+            };
+            if (config.response_schema)
+                payload["response_schema"] = config.response_schema;
+            if (config.dataset_id)
+                payload["dataset_id"] = config.dataset_id;
+            const result = (await apiRequest("POST", "prompt-tests", {
+                payload,
+                apiKey,
+            }));
+            if (result?.id) {
+                experiments[result.id] = result;
+                printSuccess(`Successfully created prompt test for model '${model}' with ID: ${result.id}`);
+            }
+            else {
+                printWarning(`Failed to create prompt test for model '${model}' with prompt: ${prompt}`);
+            }
+        }
+    }
+    if (!Object.keys(experiments).length) {
+        printError("No prompt tests were created. Aborting.");
+        throw new CliError(1, "No prompt tests created");
+    }
+    printInfo("Waiting for prompt tests to complete...");
+    const completed = {};
+    while (Object.keys(completed).length < Object.keys(experiments).length) {
+        for (const expId of Object.keys(experiments)) {
+            if (completed[expId])
+                continue;
+            const expData = (await apiRequest("GET", `prompt-tests/${expId}`, {
+                apiKey,
+            }));
+            if (!expData) {
+                printWarning(`Could not retrieve status for prompt test ${expId}`);
+                continue;
+            }
+            experiments[expId] = expData;
+            if (isPromptTestComplete(expData)) {
+                completed[expId] = expData;
+                printSuccess(`Prompt test ${expId} completed.`);
+            }
+        }
+        displayProgressTable(Object.values(experiments));
+        if (Object.keys(completed).length < Object.keys(experiments).length) {
+            await sleep(1000);
+        }
+    }
+    printSuccess("All prompt tests completed.");
+    const finalTests = Object.values(completed).sort((a, b) => a.id.localeCompare(b.id));
+    displayAggregatedResults(finalTests);
+    if (outputFile) {
+        try {
+            writeFileSync(outputFile, JSON.stringify(finalTests, null, 2));
+            printSuccess(`Results saved to ${outputFile}`);
+        }
+        catch (e) {
+            printError(`Failed to write results to ${outputFile}: ${e instanceof Error ? e.message : String(e)}`);
+        }
+    }
+    const ids = finalTests.map((t) => t.id).join(",");
+    printInfo(`\nView full results in the browser:\nhttps://scorable.ai/prompt-testing/compare?ids=${ids}`);
+}
+export function registerRunCommand(pt) {
+    pt.command("run")
+        .description("Runs prompt tests from the prompt-tests.yaml file")
+        .option("-o, --output <path>", "Output file path to save prompt test results as JSON")
+        .option("-c, --config <path>", "Path to prompt testing configuration file", "prompt-tests.yaml")
+        .action(async (opts) => {
+        try {
+            await runPromptTests(opts.output, opts.config);
+        }
+        catch (e) {
+            if (e instanceof CliError)
+                throw e;
+            throw new CliError(1, e instanceof Error ? e.message : String(e));
+        }
+    });
+}

package/dist/commands/prompt-test/run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.js","sourceRoot":"","sources":["../../../src/commands/prompt-test/run.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACtD,OAAO,IAAI,MAAM,SAAS,CAAC;AAC3B,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EACL,SAAS,EACT,YAAY,EACZ,YAAY,EACZ,UAAU,GACX,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAG1C,SAAS,oBAAoB,CAAC,GAAe;IAC3C,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IACpC,OAAO,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,WAAW,IAAI,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;AACnF,CAAC;AAED,SAAS,oBAAoB,CAAC,WAAyB;IACrD,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC;QACtB,IAAI,EAAE,CAAC,gBAAgB,EAAE,QAAQ,EAAE,iBAAiB,CAAC;QACrD,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,MAAM,CAAC,EAAE;KAC1B,CAAC,CAAC;IAEH,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,WAAW,IAAI,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;QAC/F,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,oBAAoB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,IAAI,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC;IACpG,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;AAChC,CAAC;AAED,SAAS,wBAAwB,CAAC,WAAyB;IACzD,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;QACxB,YAAY,CAAC,oCAAoC,CAAC,CAAC;QACnD,OAAO;IACT,CAAC;IAED,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAChD,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;YAC/B,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAAE,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IACD,MAAM,aAAa,GAAG,CAAC,GAAG,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAEvD,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC;QACtB,IAAI,EAAE;YACJ,QAAQ;YACR,QAAQ;YACR,OAAO;YACP,MAAM;YACN,aAAa;YACb,QAAQ;YACR,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;SACrD;QACD,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,MAAM,CAAC,EAAE;QACzB,QAAQ,EAAE,IAAI;KACf,CAAC,CAAC;IAEH,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QAC9B,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC;iBAC7C,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;iBAC7B,IAAI,CAAC,IAAI,CAAC,CAAC;YAEd,MAAM,GAAG,GAAa;gBACpB,SAAS;gBACT,GAAG,CAAC,MAAM;gBACV,GAAG,CAAC,KAAK;gBACT,IAAI,CAAC,IAAI,IAAI,KAAK;gBAClB,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK;gBAC9E,IAAI,CAAC,UAAU,IAAI,EAAE;aACtB,CAAC;YAEF,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC5E,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;gBACnC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBACjC,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAClD,CAAC;YAED,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,UAA8B,EAC9B,UAAkB,EAClB,QAAuC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAEpF,IAAI,SAAkB,CAAC;IACvB,IAAI,CAAC;QACH,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;IAC1D,CAAC;IAAC,MAAM,CAAC;QACP,UAAU,CACR,IAAI,UAAU,uFAAuF,CACtG,CAAC;QACF,MAAM,IAAI,QAAQ,CAAC,CAAC,EAAE,0BAA0B,UAAU,EAAE,CAAC,CAAC;IAChE,CAAC;IAED,IAAI,MAAwB,CAAC;IAC7B,IAAI,CAAC;QACH,MAAM,GAAG,SAA6B,CAAC;QACvC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;YAC5D,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,UAAU,CAAC,gCAAgC,UAAU,MAAM,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACzG,MAAM,IAAI,QAAQ,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC;IAC1C,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,aAAa,EAAE,CAAC;IACrC,SAAS,CAAC,uBAAuB,CAAC,CAAC;IAEnC,MAAM,WAAW,GAA+B,EAAE,CAAC;IAEnD,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACpC,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAClC,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBAC7C,MAAM,KAAK,GAA4B,EAAE,CAAC;gBAC1C,IAAI,CAAC,CAAC,EAAE;oBAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBAC7B,IAAI,CAAC,CAAC,IAAI;oBAAE,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBACnC,IAAI,CAAC,CAAC,UAAU;oBAAE,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC;gBACrD,OAAO,KAAK,CAAC;YACf,CAAC,CAAC,CAAC;YAEH,MAAM,OAAO,GAA4B;gBACvC,MAAM;gBACN,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;gBACxC,KAAK;gBACL,UAAU;aACX,CAAC;YACF,IAAI,MAAM,CAAC,eAAe;gBAAE,OAAO,CAAC,iBAAiB,CAAC,GAAG,MAAM,CAAC,eAAe,CAAC;YAChF,IAAI,MAAM,CAAC,UAAU;gBAAE,OAAO,CAAC,YAAY,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC;YAEjE,MAAM,MAAM,GAAG,CAAC,MAAM,UAAU,CAAC,MAAM,EAAE,cAAc,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAsB,CAAC;YACpG,IAAI,MAAM,EAAE,EAAE,EAAE,CAAC;gBACf,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC;gBAChC,YAAY,CAAC,+CAA+C,KAAK,cAAc,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;YAC9F,CAAC;iBAAM,CAAC;gBACN,YAAY,CAAC,2CAA2C,KAAK,kBAAkB,MAAM,EAAE,CAAC,CAAC;YAC3F,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,EAAE,CAAC;QACrC,UAAU,CAAC,yCAAyC,CAAC,CAAC;QACtD,MAAM,IAAI,QAAQ,CAAC,CAAC,EAAE,yBAAyB,CAAC,CAAC;IACnD,CAAC;IAED,SAAS,CAAC,yCAAyC,CAAC,CAAC;IACrD,MAAM,SAAS,GAA+B,EAAE,CAAC;IAEjD,OAAO,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,EAAE,CAAC;QACvE,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;YAC7C,IAAI,SAAS,CAAC,KAAK,CAAC;gBAAE,SAAS;YAE/B,MAAM,OAAO,GAAG,CAAC,MAAM,UAAU,CAAC,KAAK,EAAE,gBAAgB,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAsB,CAAC;YACpG,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,YAAY,CAAC,6CAA6C,KAAK,EAAE,CAAC,CAAC;gBACnE,SAAS;YACX,CAAC;YAED,WAAW,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC;YAE7B,IAAI,oBAAoB,CAAC,OAAO,CAAC,EAAE,CAAC;gBAClC,SAAS,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC;gBAC3B,YAAY,CAAC,eAAe,KAAK,aAAa,CAAC,CAAC;YAClD,CAAC;QACH,CAAC;QAED,oBAAoB,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;QAEjD,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,EAAE,CAAC;YACpE,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED,YAAY,CAAC,6BAA6B,CAAC,CAAC;IAC5C,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACrF,wBAAwB,CAAC,UAAU,CAAC,CAAC;IAErC,IAAI,UAAU,EAAE,CAAC;QACf,IAAI,CAAC;YACH,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAC/D,YAAY,CAAC,oBAAoB,UAAU,EAAE,CAAC,CAAC;QACjD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,UAAU,CAAC,8BAA8B,UAAU,KAAK,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACxG,CAAC;IACH,CAAC;IAED,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClD,SAAS,CAAC,uFAAuF,GAAG,EAAE,CAAC,CAAC;AAC1G,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,EAAW;IAC5C,EAAE;SACC,OAAO,CAAC,KAAK,CAAC;SACd,WAAW,CAAC,mDAAmD,CAAC;SAChE,MAAM,CAAC,qBAAqB,EAAE,sDAAsD,CAAC;SACrF,MAAM,CAAC,qBAAqB,EAAE,2CAA2C,EAAE,mBAAmB,CAAC;SAC/F,MAAM,CAAC,KAAK,EAAE,IAAyC,EAAE,EAAE;QAC1D,IAAI,CAAC;YACH,MAAM,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACjD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,YAAY,QAAQ;gBAAE,MAAM,CAAC,CAAC;YACnC,MAAM,IAAI,QAAQ,CAAC,CAAC,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACpE,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function createCli(): Command;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIpC,wBAAgB,SAAS,IAAI,OAAO,CAUnC"}

package/dist/index.js

@@ -0,0 +1,29 @@
+import { Command } from "commander";
+import { registerJudgeCommands } from "./commands/judge/index.js";
+import { registerPromptTestCommands } from "./commands/prompt-test/index.js";
+export function createCli() {
+    const program = new Command()
+        .name("scorable-cli")
+        .description("A CLI tool to interact with the Scorable API")
+        .exitOverride();
+    registerJudgeCommands(program);
+    registerPromptTestCommands(program);
+    return program;
+}
+if (import.meta.url === `file://${process.argv[1]}`) {
+    createCli()
+        .parseAsync(process.argv)
+        .catch((e) => {
+        if (e && typeof e === "object" && "exitCode" in e) {
+            process.exit(e.exitCode);
+        }
+        if (e &&
+            typeof e === "object" &&
+            "code" in e &&
+            String(e.code).startsWith("commander.")) {
+            process.exit(e.exitCode ?? 1);
+        }
+        process.stderr.write(String(e) + "\n");
+        process.exit(1);
+    });
+}

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAC;AAClE,OAAO,EAAE,0BAA0B,EAAE,MAAM,iCAAiC,CAAC;AAE7E,MAAM,UAAU,SAAS;IACvB,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE;SAC1B,IAAI,CAAC,cAAc,CAAC;SACpB,WAAW,CAAC,8CAA8C,CAAC;SAC3D,YAAY,EAAE,CAAC;IAElB,qBAAqB,CAAC,OAAO,CAAC,CAAC;IAC/B,0BAA0B,CAAC,OAAO,CAAC,CAAC;IAEpC,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,UAAU,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IACpD,SAAS,EAAE;SACR,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC;SACxB,KAAK,CAAC,CAAC,CAAU,EAAE,EAAE;QACpB,sEAAsE;QACtE,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;YAClD,OAAO,CAAC,IAAI,CAAE,CAA0B,CAAC,QAAQ,CAAC,CAAC;QACrD,CAAC;QACD,qEAAqE;QACrE,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,IAAI,CAAC,IAAI,MAAM,CAAE,CAAuB,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;YAChH,OAAO,CAAC,IAAI,CAAE,CAA2B,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;QAC3D,CAAC;QACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;QACvC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/output.d.ts

@@ -0,0 +1,9 @@
+import type { Judge } from "./types.js";
+export declare function printJson(data: unknown): void;
+export declare function printError(msg: string): void;
+export declare function printSuccess(msg: string): void;
+export declare function printInfo(msg: string): void;
+export declare function printWarning(msg: string): void;
+export declare function printMessage(msg: string): void;
+export declare function printJudgeTable(judges: Judge[], nextCursor?: string): void;
+//# sourceMappingURL=output.d.ts.map

package/dist/output.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"output.d.ts","sourceRoot":"","sources":["../src/output.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAExC,wBAAgB,SAAS,CAAC,IAAI,EAAE,OAAO,GAAG,IAAI,CAE7C;AAED,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAE5C;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAE9C;AAED,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAE3C;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAE9C;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAE9C;AAMD,wBAAgB,eAAe,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAmB1E"}

package/dist/output.js

@@ -0,0 +1,40 @@
+import chalk from "chalk";
+import Table from "cli-table3";
+export function printJson(data) {
+    console.log(JSON.stringify(data, null, 2));
+}
+export function printError(msg) {
+    console.log(chalk.bold.red("Error:") + " " + msg);
+}
+export function printSuccess(msg) {
+    console.log(chalk.bold.green("Success:") + " " + msg);
+}
+export function printInfo(msg) {
+    console.log(chalk.bold.blue("Info:") + " " + msg);
+}
+export function printWarning(msg) {
+    console.log(chalk.bold.yellow("Warning:") + " " + msg);
+}
+export function printMessage(msg) {
+    console.log(msg);
+}
+function truncate(s, max) {
+    return s.length > max ? s.slice(0, max - 1) + "…" : s;
+}
+export function printJudgeTable(judges, nextCursor) {
+    const table = new Table({
+        head: ["ID", "Name", "Intent", "Created At", "Status"],
+        style: { head: ["cyan"] },
+        colWidths: [38, 30, 52, 12, 10],
+        wordWrap: true,
+    });
+    for (const j of judges) {
+        const date = (j.created_at ?? "").slice(0, 10);
+        table.push([j.id, j.name, truncate(j.intent ?? "", 50), date, j.status ?? ""]);
+    }
+    console.log(table.toString());
+    if (nextCursor) {
+        const cursor = nextCursor.split("cursor=")[1] ?? nextCursor;
+        printInfo(`Next page available. Use --cursor "${cursor}"`);
+    }
+}

package/dist/output.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"output.js","sourceRoot":"","sources":["../src/output.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,KAAK,MAAM,YAAY,CAAC;AAG/B,MAAM,UAAU,SAAS,CAAC,IAAa;IACrC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAW;IACpC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC;AACpD,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC;AACxD,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,GAAW;IACnC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC;AACpD,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AACnB,CAAC;AAED,SAAS,QAAQ,CAAC,CAAS,EAAE,GAAW;IACtC,OAAO,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,MAAe,EAAE,UAAmB;IAClE,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC;QACtB,IAAI,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,QAAQ,CAAC;QACtD,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,MAAM,CAAC,EAAE;QACzB,SAAS,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC;QAC/B,QAAQ,EAAE,IAAI;KACf,CAAC,CAAC;IAEH,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC/C,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,EAAE,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC;IACjF,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;IAE9B,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC;QAC5D,SAAS,CAAC,sCAAsC,MAAM,GAAG,CAAC,CAAC;IAC7D,CAAC;AACH,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,62 @@
+export interface Judge {
+    id: string;
+    name: string;
+    intent: string;
+    created_at: string;
+    status?: string;
+    stage?: string;
+    evaluator_references?: Array<{
+        id: string;
+    }>;
+}
+export interface JudgeListResponse {
+    results: Judge[];
+    next?: string;
+}
+export interface PromptTestTask {
+    id: string;
+    status: string;
+    cost?: string;
+    llm_output?: string;
+    model_call_duration?: number;
+    evaluation_results: Array<{
+        id: string;
+        name: string;
+        score: number | null;
+        justification: string | null;
+    }>;
+    variables: Record<string, unknown>;
+}
+export interface PromptTest {
+    id: string;
+    model: string;
+    prompt: string;
+    tasks: PromptTestTask[];
+    avg_cost?: string;
+    avg_model_call_duration?: number;
+    evaluators: Array<{
+        id: string;
+        name: string;
+    }>;
+}
+export interface EvaluatorConfig {
+    id?: string;
+    name?: string;
+    version_id?: string;
+}
+export interface PromptTestInput {
+    vars: Record<string, unknown>;
+}
+export interface PromptTestConfig {
+    prompts: string[];
+    inputs: PromptTestInput[];
+    models: string[];
+    evaluators: EvaluatorConfig[];
+    response_schema?: Record<string, unknown>;
+    dataset_id?: string;
+}
+export declare class CliError extends Error {
+    readonly exitCode: number;
+    constructor(exitCode: number, message: string);
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,KAAK;IACpB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,oBAAoB,CAAC,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC9C;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,KAAK,EAAE,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,kBAAkB,EAAE,KAAK,CAAC;QACxB,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QACrB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;KAC9B,CAAC,CAAC;IACH,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,UAAU,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACjD;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC1C,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,qBAAa,QAAS,SAAQ,KAAK;aAEf,QAAQ,EAAE,MAAM;gBAAhB,QAAQ,EAAE,MAAM,EAChC,OAAO,EAAE,MAAM;CAKlB"}

package/dist/types.js

@@ -0,0 +1,8 @@
+export class CliError extends Error {
+    exitCode;
+    constructor(exitCode, message) {
+        super(message);
+        this.exitCode = exitCode;
+        this.name = "CliError";
+    }
+}

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AA2DA,MAAM,OAAO,QAAS,SAAQ,KAAK;IAEf;IADlB,YACkB,QAAgB,EAChC,OAAe;QAEf,KAAK,CAAC,OAAO,CAAC,CAAC;QAHC,aAAQ,GAAR,QAAQ,CAAQ;QAIhC,IAAI,CAAC,IAAI,GAAG,UAAU,CAAC;IACzB,CAAC;CACF"}

package/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "@root-signals/scorable-cli",
+  "version": "0.1.0",
+  "description": "CLI for Scorable",
+  "license": "Apache-2.0",
+  "author": "Scorable",
+  "bin": {
+    "scorable": "./dist/index.js"
+  },
+  "files": [
+    "dist/**/*.js",
+    "dist/**/*.js.map",
+    "dist/**/*.d.ts",
+    "dist/**/*.d.ts.map",
+    "README.md",
+    "LICENSE"
+  ],
+  "type": "module",
+  "main": "./dist/index.js",
+  "scripts": {
+    "build": "tsc --project tsconfig.build.json",
+    "bundle": "esbuild src/cli.ts --bundle --platform=node --target=node18 --format=cjs --minify --banner:js='#!/usr/bin/env node' --log-override:empty-import-meta=silent --outfile=dist/scorable.cjs && chmod +x dist/scorable.cjs",
+    "check-all": "npm run typecheck && npm run lint && npm run fmt:check && npm run test",
+    "prepublishOnly": "npm run check-all && npm run build",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "lint": "oxlint src tests",
+    "fmt": "oxfmt .",
+    "fmt:check": "oxfmt . --check"
+  },
+  "dependencies": {
+    "@inquirer/prompts": "^8.3.2",
+    "@root-signals/scorable": "^0.2.1",
+    "chalk": "^5.6.2",
+    "cli-table3": "^0.6.5",
+    "commander": "^14.0.3",
+    "js-yaml": "^4.1.1",
+    "ora": "^9.3.0"
+  },
+  "devDependencies": {
+    "@types/js-yaml": "^4.0.9",
+    "@types/node": "^25.5.0",
+    "esbuild": "^0.25.0",
+    "oxfmt": "^0.41.0",
+    "oxlint": "^1.56.0",
+    "typescript": "^5.9.3",
+    "vitest": "^4.1.0"
+  },
+  "engines": {
+    "node": ">=20"
+  }
+}