@root-signals/scorable-cli 0.1.0

package/README.md

@@ -0,0 +1,206 @@
+<h1 align="center">
+  <img width="600" alt="Scorable logo" src="https://scorable.ai/images/scorable-color.svg" loading="lazy">
+</h1>
+
+<p align="center" class="large-text">
+  <i><strong>Measurement & Control for LLM Automations</strong></i>
+</p>
+
+<p align="center">
+  <a href="https://scorable.ai/register">
+    <img src="https://img.shields.io/badge/Get_Started-2E6AFB?style=for-the-badge&logo=rocket&logoColor=white&scale=2" />
+  </a>
+
+  <a href="https://huggingface.co/root-signals">
+    <img src="https://img.shields.io/badge/HuggingFace-FF9D00?style=for-the-badge&logo=huggingface&logoColor=white&scale=2" />
+  </a>
+
+  <a href="https://sdk.rootsignals.ai/en/latest/">
+    <img src="https://img.shields.io/badge/Documentation-E53935?style=for-the-badge&logo=readthedocs&logoColor=white&scale=2" />
+  </a>
+
+  <a href="https://scorable.ai/demo-user">
+    <img src="https://img.shields.io/badge/Temporary_API_Key-15a20b?style=for-the-badge&logo=keycdn&logoColor=white&scale=2" />
+  </a>
+</p>
+
+The `scorable` CLI is a command-line tool for interacting with the Scorable API. It lets you manage and execute Judges and run prompt testing experiments directly from the terminal.
+
+Requires **Node.js 20 or higher**.
+
+## Installation
+
+```bash
+curl -sSL https://scorable.ai/cli/install.sh | sh
+```
+
+Or install directly with npm:
+
+```bash
+npm install -g @root-signals/scorable-cli
+```
+
+Or run without installing via npx:
+
+```bash
+npx @root-signals/scorable-cli judge list
+```
+
+## Authentication
+
+Set your Scorable API key as an environment variable:
+
+```bash
+export SCORABLE_API_KEY="your-api-key"
+```
+
+Get a free API key at [scorable.ai/register](https://scorable.ai/register).
+
+### Temporary API keys
+
+If no API key is set, the CLI will offer to create a temporary key interactively (in TTY sessions) and save it to `~/.scorable/settings.json`. The `SCORABLE_API_KEY` environment variable always takes precedence.
+
+## Judge Management
+
+### List judges
+
+```bash
+scorable judge list
+```
+
+Options: `--page-size`, `--cursor`, `--search`, `--name`, `--ordering`, `--is-preset / --not-is-preset`, `--is-public / --not-is-public`, `--show-global / --not-show-global`
+
+### Get a judge
+
+```bash
+scorable judge get <judge_id>
+```
+
+### Create a judge
+
+```bash
+scorable judge create --name "My Judge" --intent "Evaluate response quality."
+```
+
+Options: `--name` (required), `--intent` (required), `--stage`, `--evaluator-references` (JSON string, e.g. `'[{"id": "eval-id"}]'`)
+
+### Update a judge
+
+```bash
+scorable judge update <judge_id> --name "Updated Name"
+```
+
+Options: `--name`, `--stage`, `--evaluator-references` (use `"[]"` to clear)
+
+### Delete a judge
+
+```bash
+scorable judge delete <judge_id>
+```
+
+Prompts for confirmation. Use `--yes` to skip.
+
+### Duplicate a judge
+
+```bash
+scorable judge duplicate <judge_id>
+```
+
+## Judge Execution
+
+### Execute by ID
+
+```bash
+scorable judge execute <judge_id> --request "What is the capital of France?" --response "Paris"
+```
+
+Options: `--request`, `--response`, `--contexts` (JSON list), `--expected-output`, `--tag` (repeatable), `--user-id`, `--session-id`, `--system-prompt`
+
+Pipe input via stdin:
+
+```bash
+echo "Paris" | scorable judge execute <judge_id> --request "What is the capital of France?"
+cat response.txt | scorable judge execute <judge_id>
+```
+
+### Execute by name
+
+```bash
+scorable judge execute-by-name "My Judge" --request "What is the capital of France?" --response "Paris"
+```
+
+Accepts the same options as `execute`. Stdin piping works the same way.
+
+### Execute via OpenAI-compatible API
+
+```bash
+scorable judge exec-openai <judge_id> \
+  --model gpt-4o \
+  --messages '[{"role": "user", "content": "Hello"}, {"role": "assistant", "content": "Hi there"}]'
+```
+
+Generic variant (judge ID in the `model` field):
+
+```bash
+scorable judge exec-openai-generic \
+  --model <judge_id_or_name> \
+  --messages '[{"role": "user", "content": "Hello"}, {"role": "assistant", "content": "Hi there"}]'
+```
+
+## Prompt Testing
+
+Initialize a config file and run experiments:
+
+```bash
+scorable pt init
+scorable pt run
+```
+
+Use a custom config path:
+
+```bash
+scorable pt run --config path/to/prompt-tests.yaml
+```
+
+The `prompt-test` command is an alias for `pt`.
+
+### Config file format
+
+```yaml
+prompts:
+  - "Extract info from: {{text}}"
+
+inputs:
+  - vars:
+      text: "John Doe, john@example.com"
+
+# Or use a dataset instead of inline inputs:
+# dataset_id: "<uuid>"
+
+models:
+  - gpt-4o-mini
+  - gemini-2.5-flash-lite
+
+evaluators:
+  - name: Precision
+  - name: Confidentiality
+
+# Optional: enforce structured output
+# response_schema:
+#   type: object
+#   properties:
+#     name: { type: string }
+```
+
+Results are displayed in a table and a browser link is printed for the full comparison view.
+
+## Development
+
+```bash
+npm install
+npm run build       # compile TypeScript
+npm test            # run tests
+npm run typecheck   # type-check without emitting
+npm run lint        # lint with oxlint
+npm run fmt         # format with oxfmt
+```

package/dist/auth.d.ts

@@ -0,0 +1,7 @@
+import { Scorable } from "@root-signals/scorable";
+export declare function saveSettings(settings: Record<string, unknown>): void;
+export declare function getApiKey(): string | undefined;
+export declare function getBaseUrl(): string;
+export declare function getSdkClient(apiKey: string): Scorable;
+export declare function requireApiKey(): Promise<string>;
+//# sourceMappingURL=auth.d.ts.map

package/dist/auth.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"auth.d.ts","sourceRoot":"","sources":["../src/auth.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAuBlD,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAQpE;AAED,wBAAgB,SAAS,IAAI,MAAM,GAAG,SAAS,CAK9C;AAED,wBAAgB,UAAU,IAAI,MAAM,CAEnC;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,QAAQ,CAErD;AAED,wBAAsB,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC,CA2DrD"}

package/dist/auth.js

@@ -0,0 +1,103 @@
+import { readFileSync, writeFileSync, mkdirSync, existsSync } from "node:fs";
+import { join } from "node:path";
+import { homedir } from "node:os";
+import { Scorable } from "@root-signals/scorable";
+import { CliError } from "./types.js";
+import { printError, printInfo, printSuccess } from "./output.js";
+function configDir() {
+    return join(homedir(), ".scorable");
+}
+function settingsPath() {
+    return join(configDir(), "settings.json");
+}
+function loadSettings() {
+    try {
+        const path = settingsPath();
+        if (!existsSync(path))
+            return {};
+        const data = JSON.parse(readFileSync(path, "utf8"));
+        return typeof data === "object" && data !== null ? data : {};
+    }
+    catch {
+        return {};
+    }
+}
+export function saveSettings(settings) {
+    try {
+        const dir = configDir();
+        mkdirSync(dir, { recursive: true, mode: 0o700 });
+        writeFileSync(settingsPath(), JSON.stringify(settings, null, 2), { mode: 0o600 });
+    }
+    catch {
+    }
+}
+export function getApiKey() {
+    const fromEnv = process.env["SCORABLE_API_KEY"];
+    if (fromEnv)
+        return fromEnv;
+    const settings = loadSettings();
+    return settings["temporary_api_key"];
+}
+export function getBaseUrl() {
+    return process.env["SCORABLE_API_URL"] ?? "https://api.scorable.ai";
+}
+export function getSdkClient(apiKey) {
+    return new Scorable({ apiKey, baseUrl: getBaseUrl() });
+}
+export async function requireApiKey() {
+    const key = getApiKey();
+    if (key)
+        return key;
+    printError("SCORABLE_API_KEY environment variable not set.");
+    const shell = process.env["SHELL"] ?? "";
+    if (shell.includes("fish")) {
+        printInfo("Run: set -x SCORABLE_API_KEY <your_key>");
+    }
+    else {
+        printInfo("Run: export SCORABLE_API_KEY='<your_key>'");
+    }
+    if (process.stdin.isTTY && process.stdout.isTTY) {
+        const { confirm } = await import("@inquirer/prompts");
+        const answer = await confirm({
+            message: "No API key found. Create a temporary key now?",
+            default: true,
+        });
+        if (!answer) {
+            printInfo("Aborted. Please set SCORABLE_API_KEY and try again.");
+            throw new CliError(1, "Aborted");
+        }
+        try {
+            const resp = await fetch(`${getBaseUrl()}/create-demo-user/`, { method: "POST" });
+            if (!resp.ok)
+                throw new Error(`HTTP ${resp.status}`);
+            const data = (await resp.json());
+            const tempKey = data["api_key"];
+            if (!tempKey) {
+                printError("Temporary key response did not include 'api_key'.");
+                throw new CliError(1, "Missing api_key in demo response");
+            }
+            process.env["SCORABLE_API_KEY"] = tempKey;
+            const settings = loadSettings();
+            settings["temporary_api_key"] = tempKey;
+            saveSettings(settings);
+            printSuccess("Temporary API key saved to ~/.scorable/settings.json");
+            if (shell.includes("fish")) {
+                printInfo("To persist in your shell: set -x SCORABLE_API_KEY <paste_key_here>");
+            }
+            else {
+                printInfo("To persist in your shell: export SCORABLE_API_KEY='<paste_key_here>'");
+            }
+            return tempKey;
+        }
+        catch (e) {
+            if (e instanceof CliError)
+                throw e;
+            printError(`Failed to create temporary API key: ${e instanceof Error ? e.message : String(e)}`);
+            throw new CliError(1, "Failed to create temporary API key");
+        }
+    }
+    else {
+        printInfo("Set SCORABLE_API_KEY and retry. Non-interactive session cannot prompt.");
+        throw new CliError(1, "Missing API key");
+    }
+}

package/dist/auth.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"auth.js","sourceRoot":"","sources":["../src/auth.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAElE,SAAS,SAAS;IAChB,OAAO,IAAI,CAAC,OAAO,EAAE,EAAE,WAAW,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,YAAY;IACnB,OAAO,IAAI,CAAC,SAAS,EAAE,EAAE,eAAe,CAAC,CAAC;AAC5C,CAAC;AAED,SAAS,YAAY;IACnB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,YAAY,EAAE,CAAC;QAC5B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,OAAO,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAY,CAAC;QAC/D,OAAO,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,CAAC,CAAC,CAAE,IAAgC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5F,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,QAAiC;IAC5D,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,SAAS,EAAE,CAAC;QACxB,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACjD,aAAa,CAAC,YAAY,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACpF,CAAC;IAAC,MAAM,CAAC;QACP,iCAAiC;IACnC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,SAAS;IACvB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;IAChD,IAAI,OAAO;QAAE,OAAO,OAAO,CAAC;IAC5B,MAAM,QAAQ,GAAG,YAAY,EAAE,CAAC;IAChC,OAAO,QAAQ,CAAC,mBAAmB,CAAuB,CAAC;AAC7D,CAAC;AAED,MAAM,UAAU,UAAU;IACxB,OAAO,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,IAAI,yBAAyB,CAAC;AACtE,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAc;IACzC,OAAO,IAAI,QAAQ,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,EAAE,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa;IACjC,MAAM,GAAG,GAAG,SAAS,EAAE,CAAC;IACxB,IAAI,GAAG;QAAE,OAAO,GAAG,CAAC;IAEpB,UAAU,CAAC,gDAAgD,CAAC,CAAC;IAC7D,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IACzC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3B,SAAS,CAAC,yCAAyC,CAAC,CAAC;IACvD,CAAC;SAAM,CAAC;QACN,SAAS,CAAC,2CAA2C,CAAC,CAAC;IACzD,CAAC;IAED,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAChD,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;QACtD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC;YAC3B,OAAO,EAAE,+CAA+C;YACxD,OAAO,EAAE,IAAI;SACd,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,SAAS,CAAC,qDAAqD,CAAC,CAAC;YACjE,MAAM,IAAI,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QACnC,CAAC;QAED,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,GAAG,UAAU,EAAE,oBAAoB,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;YAClF,IAAI,CAAC,IAAI,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YACrD,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAA4B,CAAC;YAC5D,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAuB,CAAC;YAEtD,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,UAAU,CAAC,mDAAmD,CAAC,CAAC;gBAChE,MAAM,IAAI,QAAQ,CAAC,CAAC,EAAE,kCAAkC,CAAC,CAAC;YAC5D,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,GAAG,OAAO,CAAC;YAC1C,MAAM,QAAQ,GAAG,YAAY,EAAE,CAAC;YAChC,QAAQ,CAAC,mBAAmB,CAAC,GAAG,OAAO,CAAC;YACxC,YAAY,CAAC,QAAQ,CAAC,CAAC;YACvB,YAAY,CAAC,sDAAsD,CAAC,CAAC;YAErE,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC3B,SAAS,CAAC,oEAAoE,CAAC,CAAC;YAClF,CAAC;iBAAM,CAAC;gBACN,SAAS,CAAC,sEAAsE,CAAC,CAAC;YACpF,CAAC;YAED,OAAO,OAAO,CAAC;QACjB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,YAAY,QAAQ;gBAAE,MAAM,CAAC,CAAC;YACnC,UAAU,CAAC,uCAAuC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAChG,MAAM,IAAI,QAAQ,CAAC,CAAC,EAAE,oCAAoC,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;SAAM,CAAC;QACN,SAAS,CAAC,wEAAwE,CAAC,CAAC;QACpF,MAAM,IAAI,QAAQ,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC"}

package/dist/cli.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=cli.d.ts.map

package/dist/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":""}

package/dist/cli.js

@@ -0,0 +1,16 @@
+import { createCli } from "./index.js";
+createCli()
+    .parseAsync(process.argv)
+    .catch((e) => {
+    if (e && typeof e === "object" && "exitCode" in e) {
+        process.exit(e.exitCode);
+    }
+    if (e &&
+        typeof e === "object" &&
+        "code" in e &&
+        String(e.code).startsWith("commander.")) {
+        process.exit(e.exitCode ?? 1);
+    }
+    process.stderr.write(String(e) + "\n");
+    process.exit(1);
+});

package/dist/client.d.ts

@@ -0,0 +1,7 @@
+export declare function apiRequest(method: string, endpoint: string, options?: {
+    payload?: Record<string, unknown>;
+    params?: Record<string, unknown>;
+    apiKey?: string;
+    baseUrl?: string;
+}): Promise<unknown>;
+//# sourceMappingURL=client.d.ts.map

package/dist/client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAGA,wBAAsB,UAAU,CAC9B,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE;IACR,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,GACA,OAAO,CAAC,OAAO,CAAC,CAoDlB"}

package/dist/client.js

@@ -0,0 +1,52 @@
+import { getApiKey, getBaseUrl } from "./auth.js";
+import { printError, printJson } from "./output.js";
+export async function apiRequest(method, endpoint, options) {
+    const apiKey = options?.apiKey ?? getApiKey();
+    const baseUrl = options?.baseUrl ?? getBaseUrl();
+    const seg = endpoint.endsWith("/") ? endpoint : `${endpoint}/`;
+    const url = new URL(`${baseUrl}/${seg}`);
+    if (options?.params) {
+        for (const [k, v] of Object.entries(options.params)) {
+            if (v !== undefined && v !== null) {
+                url.searchParams.set(k, String(v));
+            }
+        }
+    }
+    const headers = {
+        Authorization: `Api-Key ${apiKey}`,
+        "Content-Type": "application/json",
+        Accept: "application/json",
+        "User-Agent": "scorable-cli/1.0",
+    };
+    let response;
+    try {
+        response = await fetch(url.toString(), {
+            method,
+            headers,
+            body: options?.payload ? JSON.stringify(options.payload) : undefined,
+        });
+    }
+    catch (e) {
+        printError(`Request failed: ${e instanceof Error ? e.message : String(e)}`);
+        return null;
+    }
+    if (response.status === 204)
+        return null;
+    if (!response.ok) {
+        printError(`API Error: ${response.status} for ${method} ${url}`);
+        try {
+            printJson(await response.json());
+        }
+        catch {
+            printError(`Response content: ${await response.text().catch(() => "")}`);
+        }
+        return null;
+    }
+    try {
+        return await response.json();
+    }
+    catch {
+        printError(`Failed to decode JSON response from API for ${url}.`);
+        return null;
+    }
+}

package/dist/client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.js","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAEpD,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,MAAc,EACd,QAAgB,EAChB,OAKC;IAED,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,IAAI,SAAS,EAAE,CAAC;IAC9C,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,UAAU,EAAE,CAAC;IAEjD,MAAM,GAAG,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,GAAG,CAAC;IAC/D,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,OAAO,IAAI,GAAG,EAAE,CAAC,CAAC;IAEzC,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QACpB,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YACpD,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;gBAClC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAA2B;QACtC,aAAa,EAAE,WAAW,MAAM,EAAE;QAClC,cAAc,EAAE,kBAAkB;QAClC,MAAM,EAAE,kBAAkB;QAC1B,YAAY,EAAE,kBAAkB;KACjC,CAAC;IAEF,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE;YACrC,MAAM;YACN,OAAO;YACP,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS;SACrE,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,UAAU,CAAC,mBAAmB,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC5E,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC;IAEzC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,UAAU,CAAC,cAAc,QAAQ,CAAC,MAAM,QAAQ,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC;QACjE,IAAI,CAAC;YACH,SAAS,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QACnC,CAAC;QAAC,MAAM,CAAC;YACP,UAAU,CAAC,qBAAqB,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAC3E,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,UAAU,CAAC,+CAA+C,GAAG,GAAG,CAAC,CAAC;QAClE,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}

package/dist/commands/judge/create.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerCreateCommand(judge: Command): void;
+//# sourceMappingURL=create.d.ts.map

package/dist/commands/judge/create.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"create.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAgD1D"}

package/dist/commands/judge/create.js

@@ -0,0 +1,40 @@
+import { requireApiKey, getSdkClient } from "../../auth.js";
+import { printInfo, printSuccess, printError, printJson } from "../../output.js";
+import { CliError } from "../../types.js";
+export function registerCreateCommand(judge) {
+    judge
+        .command("create")
+        .description("Create a new judge")
+        .requiredOption("--name <name>", "The name for the new judge")
+        .requiredOption("--intent <intent>", "The intent for the new judge")
+        .option("--stage <stage>", "The stage for the new judge")
+        .option("--evaluator-references <json>", 'JSON string for evaluator references. E.g., \'[{"id": "eval-id"}]\'')
+        .action(async (opts) => {
+        const apiKey = await requireApiKey();
+        const payload = { name: opts.name, intent: opts.intent };
+        if (opts.stage)
+            payload.stage = opts.stage;
+        if (opts.evaluatorReferences) {
+            try {
+                payload.evaluator_references = JSON.parse(opts.evaluatorReferences);
+            }
+            catch {
+                printError("Invalid JSON format for --evaluator-references.");
+                return;
+            }
+        }
+        printInfo("Attempting to create judge with payload:");
+        printJson(payload);
+        try {
+            const client = getSdkClient(apiKey);
+            const result = await client.judges.create(payload);
+            printSuccess("Judge created successfully!");
+            printJson(result);
+        }
+        catch (e) {
+            if (e instanceof CliError)
+                throw e;
+            printError(e instanceof Error ? e.message : String(e));
+        }
+    });
+}

package/dist/commands/judge/create.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"create.js","sourceRoot":"","sources":["../../../src/commands/judge/create.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjF,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAG1C,MAAM,UAAU,qBAAqB,CAAC,KAAc;IAClD,KAAK;SACF,OAAO,CAAC,QAAQ,CAAC;SACjB,WAAW,CAAC,oBAAoB,CAAC;SACjC,cAAc,CAAC,eAAe,EAAE,4BAA4B,CAAC;SAC7D,cAAc,CAAC,mBAAmB,EAAE,8BAA8B,CAAC;SACnE,MAAM,CAAC,iBAAiB,EAAE,6BAA6B,CAAC;SACxD,MAAM,CAAC,+BAA+B,EAAE,uEAAuE,CAAC;SAChH,MAAM,CAAC,KAAK,EAAE,IAAoF,EAAE,EAAE;QACrG,MAAM,MAAM,GAAG,MAAM,aAAa,EAAE,CAAC;QAErC,MAAM,OAAO,GAAoB,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;QAC1E,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QAE3C,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC7B,IAAI,CAAC;gBACH,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAA0B,CAAC;YAC/F,CAAC;YAAC,MAAM,CAAC;gBACP,UAAU,CAAC,iDAAiD,CAAC,CAAC;gBAC9D,OAAO;YACT,CAAC;QACH,CAAC;QAED,SAAS,CAAC,0CAA0C,CAAC,CAAC;QACtD,SAAS,CAAC,OAAO,CAAC,CAAC;QAEnB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;YACpC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACnD,YAAY,CAAC,6BAA6B,CAAC,CAAC;YAC5C,SAAS,CAAC,MAAM,CAAC,CAAC;QACpB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,YAAY,QAAQ;gBAAE,MAAM,CAAC,CAAC;YACnC,UAAU,CAAC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACzD,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/commands/judge/delete.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerDeleteCommand(judge: Command): void;
+//# sourceMappingURL=delete.d.ts.map

package/dist/commands/judge/delete.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"delete.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/delete.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CA8B1D"}

package/dist/commands/judge/delete.js

@@ -0,0 +1,33 @@
+import { requireApiKey, getSdkClient } from "../../auth.js";
+import { printInfo, printSuccess } from "../../output.js";
+import { CliError } from "../../types.js";
+export function registerDeleteCommand(judge) {
+    judge
+        .command("delete <judgeId>")
+        .description("Delete a judge by its ID")
+        .option("--yes", "Skip confirmation prompt")
+        .action(async (judgeId, opts) => {
+        const apiKey = await requireApiKey();
+        if (!opts.yes) {
+            const { confirm } = await import("@inquirer/prompts");
+            const ok = await confirm({
+                message: "Are you sure you want to delete this judge?",
+                default: false,
+            });
+            if (!ok) {
+                throw new CliError(1, "Aborted");
+            }
+        }
+        printInfo(`Deleting judge ${judgeId}...`);
+        try {
+            const client = getSdkClient(apiKey);
+            await client.judges.delete(judgeId);
+            printSuccess(`Judge ${judgeId} deleted successfully.`);
+        }
+        catch (e) {
+            if (e instanceof CliError)
+                throw e;
+            throw new CliError(1, e instanceof Error ? e.message : String(e));
+        }
+    });
+}

package/dist/commands/judge/delete.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"delete.js","sourceRoot":"","sources":["../../../src/commands/judge/delete.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC1D,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE1C,MAAM,UAAU,qBAAqB,CAAC,KAAc;IAClD,KAAK;SACF,OAAO,CAAC,kBAAkB,CAAC;SAC3B,WAAW,CAAC,0BAA0B,CAAC;SACvC,MAAM,CAAC,OAAO,EAAE,0BAA0B,CAAC;SAC3C,MAAM,CAAC,KAAK,EAAE,OAAe,EAAE,IAAuB,EAAE,EAAE;QACzD,MAAM,MAAM,GAAG,MAAM,aAAa,EAAE,CAAC;QAErC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;YACd,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;YACtD,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC;gBACvB,OAAO,EAAE,6CAA6C;gBACtD,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;YACH,IAAI,CAAC,EAAE,EAAE,CAAC;gBACR,MAAM,IAAI,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;QAED,SAAS,CAAC,kBAAkB,OAAO,KAAK,CAAC,CAAC;QAE1C,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;YACpC,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACpC,YAAY,CAAC,SAAS,OAAO,wBAAwB,CAAC,CAAC;QACzD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,YAAY,QAAQ;gBAAE,MAAM,CAAC,CAAC;YACnC,MAAM,IAAI,QAAQ,CAAC,CAAC,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACpE,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/commands/judge/duplicate.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerDuplicateCommand(judge: Command): void;
+//# sourceMappingURL=duplicate.d.ts.map

package/dist/commands/judge/duplicate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"duplicate.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/duplicate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAkB7D"}

package/dist/commands/judge/duplicate.js

@@ -0,0 +1,23 @@
+import { requireApiKey, getSdkClient } from "../../auth.js";
+import { printInfo, printSuccess, printError, printJson } from "../../output.js";
+import { CliError } from "../../types.js";
+export function registerDuplicateCommand(judge) {
+    judge
+        .command("duplicate <judgeId>")
+        .description("Duplicate an existing judge")
+        .action(async (judgeId) => {
+        const apiKey = await requireApiKey();
+        printInfo(`Duplicating judge ID: ${judgeId}...`);
+        try {
+            const client = getSdkClient(apiKey);
+            const result = await client.judges.duplicate(judgeId);
+            printSuccess(`Judge ${judgeId} duplicated successfully!`);
+            printJson(result);
+        }
+        catch (e) {
+            if (e instanceof CliError)
+                throw e;
+            printError(e instanceof Error ? e.message : String(e));
+        }
+    });
+}

package/dist/commands/judge/duplicate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"duplicate.js","sourceRoot":"","sources":["../../../src/commands/judge/duplicate.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjF,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE1C,MAAM,UAAU,wBAAwB,CAAC,KAAc;IACrD,KAAK;SACF,OAAO,CAAC,qBAAqB,CAAC;SAC9B,WAAW,CAAC,6BAA6B,CAAC;SAC1C,MAAM,CAAC,KAAK,EAAE,OAAe,EAAE,EAAE;QAChC,MAAM,MAAM,GAAG,MAAM,aAAa,EAAE,CAAC;QACrC,SAAS,CAAC,yBAAyB,OAAO,KAAK,CAAC,CAAC;QAEjD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;YACpC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YACtD,YAAY,CAAC,SAAS,OAAO,2BAA2B,CAAC,CAAC;YAC1D,SAAS,CAAC,MAAM,CAAC,CAAC;QACpB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,YAAY,QAAQ;gBAAE,MAAM,CAAC,CAAC;YACnC,UAAU,CAAC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACzD,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/commands/judge/exec-openai-generic.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerExecOpenaiGenericCommand(judge: Command): void;
+//# sourceMappingURL=exec-openai-generic.d.ts.map

package/dist/commands/judge/exec-openai-generic.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"exec-openai-generic.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/exec-openai-generic.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,gCAAgC,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CA8CrE"}

package/dist/commands/judge/exec-openai-generic.js

@@ -0,0 +1,50 @@
+import { requireApiKey } from "../../auth.js";
+import { printInfo, printSuccess, printError, printWarning, printJson } from "../../output.js";
+import { CliError } from "../../types.js";
+import { apiRequest } from "../../client.js";
+export function registerExecOpenaiGenericCommand(judge) {
+    judge
+        .command("exec-openai-generic")
+        .description("Execute a judge via the generic OpenAI API (judge is in the 'model' field)")
+        .requiredOption("--model <model>", "Judge ID (or name) to use as the 'model' field")
+        .requiredOption("--messages <json>", "JSON string of the messages payload")
+        .option("--extra-body <json>", "Optional JSON string for extra_body parameters")
+        .action(async (opts) => {
+        const apiKey = await requireApiKey();
+        let messages;
+        try {
+            messages = JSON.parse(opts.messages);
+        }
+        catch {
+            printError("Invalid JSON for --messages. Aborting.");
+            return;
+        }
+        const payload = { model: opts.model, messages };
+        if (opts.extraBody) {
+            try {
+                payload["extra_body"] = JSON.parse(opts.extraBody);
+            }
+            catch {
+                printWarning("Invalid JSON for --extra-body. Skipping.");
+            }
+        }
+        printInfo(`Executing a Judge using generic OpenAI endpoint. Judge ID/Name: ${opts.model}`);
+        printInfo("Attempting to execute with OpenAI compatible payload:");
+        printJson(payload);
+        try {
+            const result = await apiRequest("POST", "judges/openai/chat/completions", {
+                payload,
+                apiKey,
+            });
+            if (result) {
+                printSuccess("OpenAI compatible execution successful!");
+                printJson(result);
+            }
+        }
+        catch (e) {
+            if (e instanceof CliError)
+                throw e;
+            printError(e instanceof Error ? e.message : String(e));
+        }
+    });
+}

package/dist/commands/judge/exec-openai-generic.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"exec-openai-generic.js","sourceRoot":"","sources":["../../../src/commands/judge/exec-openai-generic.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC/F,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAE7C,MAAM,UAAU,gCAAgC,CAAC,KAAc;IAC7D,KAAK;SACF,OAAO,CAAC,qBAAqB,CAAC;SAC9B,WAAW,CAAC,4EAA4E,CAAC;SACzF,cAAc,CAAC,iBAAiB,EAAE,gDAAgD,CAAC;SACnF,cAAc,CAAC,mBAAmB,EAAE,qCAAqC,CAAC;SAC1E,MAAM,CAAC,qBAAqB,EAAE,gDAAgD,CAAC;SAC/E,MAAM,CAAC,KAAK,EAAE,IAA6D,EAAE,EAAE;QAC9E,MAAM,MAAM,GAAG,MAAM,aAAa,EAAE,CAAC;QAErC,IAAI,QAAiB,CAAC;QACtB,IAAI,CAAC;YACH,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvC,CAAC;QAAC,MAAM,CAAC;YACP,UAAU,CAAC,wCAAwC,CAAC,CAAC;YACrD,OAAO;QACT,CAAC;QAED,MAAM,OAAO,GAA4B,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC;QAEzE,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,IAAI,CAAC;gBACH,OAAO,CAAC,YAAY,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACrD,CAAC;YAAC,MAAM,CAAC;gBACP,YAAY,CAAC,0CAA0C,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;QAED,SAAS,CAAC,mEAAmE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;QAC3F,SAAS,CAAC,uDAAuD,CAAC,CAAC;QACnE,SAAS,CAAC,OAAO,CAAC,CAAC;QAEnB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,EAAE,gCAAgC,EAAE;gBACxE,OAAO;gBACP,MAAM;aACP,CAAC,CAAC;YACH,IAAI,MAAM,EAAE,CAAC;gBACX,YAAY,CAAC,yCAAyC,CAAC,CAAC;gBACxD,SAAS,CAAC,MAAM,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,YAAY,QAAQ;gBAAE,MAAM,CAAC,CAAC;YACnC,UAAU,CAAC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACzD,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/commands/judge/exec-openai.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerExecOpenaiCommand(judge: Command): void;
+//# sourceMappingURL=exec-openai.d.ts.map

package/dist/commands/judge/exec-openai.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"exec-openai.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/exec-openai.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAyD9D"}