@root-signals/scorable-cli 0.12.0 → 0.14.0

package/dist/commands/project/get.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerGetCommand(project: Command): void;
+//# sourceMappingURL=get.d.ts.map

package/dist/commands/project/get.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"get.d.ts","sourceRoot":"","sources":["../../../src/commands/project/get.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAkBzD"}

package/dist/commands/project/get.js

@@ -0,0 +1,23 @@
+import ora from "ora";
+import { requireApiKey, getSdkClient } from "../../auth.js";
+import { printSuccess, printJson, handleSdkError } from "../../output.js";
+export function registerGetCommand(project) {
+    project
+        .command("get <projectId>")
+        .description("Get a specific project by its ID")
+        .action(async (projectId) => {
+        const apiKey = await requireApiKey();
+        const spinner = ora("Fetching...").start();
+        try {
+            const client = getSdkClient(apiKey);
+            const result = await client.projects.retrieve(projectId);
+            spinner.stop();
+            printSuccess(`Project '${result.name}' details:`);
+            printJson(result);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+}

package/dist/commands/project/index.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerProjectCommands(program: Command): void;
+//# sourceMappingURL=index.d.ts.map

package/dist/commands/project/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/commands/project/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAQpC,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAW9D"}

package/dist/commands/project/index.js

@@ -0,0 +1,17 @@
+import { Command } from "commander";
+import { registerListCommand } from "./list.js";
+import { registerGetCommand } from "./get.js";
+import { registerCreateCommand } from "./create.js";
+import { registerUpdateCommand } from "./update.js";
+import { registerDeleteCommand } from "./delete.js";
+import { registerSetDefaultCommand } from "./set-default.js";
+export function registerProjectCommands(program) {
+    const project = new Command("project").description("Project management commands");
+    registerListCommand(project);
+    registerGetCommand(project);
+    registerCreateCommand(project);
+    registerUpdateCommand(project);
+    registerDeleteCommand(project);
+    registerSetDefaultCommand(project);
+    program.addCommand(project);
+}

package/dist/commands/project/list.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerListCommand(project: Command): void;
+//# sourceMappingURL=list.d.ts.map

package/dist/commands/project/list.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"list.d.ts","sourceRoot":"","sources":["../../../src/commands/project/list.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAoC1D"}

package/dist/commands/project/list.js

@@ -0,0 +1,39 @@
+import ora from "ora";
+import { requireApiKey, getSdkClient } from "../../auth.js";
+import { printMessage, printProjectTable, handleSdkError } from "../../output.js";
+export function registerListCommand(project) {
+    project
+        .command("list")
+        .description("List projects in your organization")
+        .option("--page-size <number>", "Number of results to return per page", (v) => parseInt(v, 10))
+        .option("--cursor <cursor>", "The pagination cursor value")
+        .option("--search <term>", "A search term to filter by")
+        .option("--ordering <field>", "Which field to use for ordering the results")
+        .action(async (opts) => {
+        const apiKey = await requireApiKey();
+        const params = {};
+        if (opts.pageSize !== undefined)
+            params.page_size = opts.pageSize;
+        if (opts.cursor !== undefined)
+            params.cursor = opts.cursor;
+        if (opts.search !== undefined)
+            params.search = opts.search;
+        if (opts.ordering !== undefined)
+            params.ordering = opts.ordering;
+        const spinner = ora("Fetching...").start();
+        try {
+            const client = getSdkClient(apiKey);
+            const response = await client.projects.list(params);
+            spinner.stop();
+            if (!response.results.length) {
+                printMessage("No projects found.");
+                return;
+            }
+            printProjectTable(response.results, response.next);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+}

package/dist/commands/project/set-default.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerSetDefaultCommand(project: Command): void;
+//# sourceMappingURL=set-default.d.ts.map

package/dist/commands/project/set-default.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"set-default.d.ts","sourceRoot":"","sources":["../../../src/commands/project/set-default.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAkBhE"}

package/dist/commands/project/set-default.js

@@ -0,0 +1,23 @@
+import ora from "ora";
+import { requireApiKey, getSdkClient } from "../../auth.js";
+import { printSuccess, printJson, handleSdkError } from "../../output.js";
+export function registerSetDefaultCommand(project) {
+    project
+        .command("set-default <projectId>")
+        .description("Promote a project to the org default. Atomically clears the previous default.")
+        .action(async (projectId) => {
+        const apiKey = await requireApiKey();
+        const spinner = ora("Updating default project...").start();
+        try {
+            const client = getSdkClient(apiKey);
+            const result = await client.projects.update(projectId, { is_default: true });
+            spinner.stop();
+            printSuccess(`Project ${projectId} is now the org default.`);
+            printJson(result);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+}

package/dist/commands/project/update.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerUpdateCommand(project: Command): void;
+//# sourceMappingURL=update.d.ts.map

package/dist/commands/project/update.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"update.d.ts","sourceRoot":"","sources":["../../../src/commands/project/update.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAyC5D"}

package/dist/commands/project/update.js

@@ -0,0 +1,37 @@
+import ora from "ora";
+import { requireApiKey, getSdkClient } from "../../auth.js";
+import { printSuccess, printInfo, printJson, handleSdkError } from "../../output.js";
+export function registerUpdateCommand(project) {
+    project
+        .command("update <projectId>")
+        .description("Update an existing project (PATCH)")
+        .option("--name <name>", "New project name")
+        .option("--description <text>", "New project description")
+        .option("--is-default", "Promote this project to org default. Atomically clears the previous default. Clearing the default directly is not supported by the backend; promote another project instead.", false)
+        .action(async (projectId, opts) => {
+        const apiKey = await requireApiKey();
+        const payload = {};
+        if (opts.name !== undefined)
+            payload.name = opts.name;
+        if (opts.description !== undefined)
+            payload.description = opts.description;
+        if (opts.isDefault)
+            payload.is_default = true;
+        if (Object.keys(payload).length === 0) {
+            printInfo("No update parameters provided. Aborting.");
+            return;
+        }
+        const spinner = ora("Updating...").start();
+        try {
+            const client = getSdkClient(apiKey);
+            const result = await client.projects.update(projectId, payload);
+            spinner.stop();
+            printSuccess(`Project ${projectId} updated successfully!`);
+            printJson(result);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+}

package/dist/commands/prompt-test/init.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../../src/commands/prompt-test/init.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAqDpC,wBAAgB,mBAAmB,CAAC,EAAE,EAAE,OAAO,GAAG,IAAI,CA6BrD"}
+{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../../src/commands/prompt-test/init.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AA4DpC,wBAAgB,mBAAmB,CAAC,EAAE,EAAE,OAAO,GAAG,IAAI,CA8BrD"}

package/dist/commands/prompt-test/init.js

@@ -1,5 +1,6 @@
 import { writeFileSync, existsSync } from "node:fs";
 import { printWarning, printSuccess, printInfo, printError } from "../../output.js";
+import { PROJECT_ID_FLAG_DESC } from "../../lib/project-id.js";
 const TEMPLATE = `# Prompt Testing Configuration
 # This file defines a test suite of prompt and model combinations, with optional evaluators.
 
@@ -48,10 +49,16 @@ evaluators:
 #       description: "The username of the user. Must start with @"
 #   additionalProperties: false
 `;
+function buildTemplate(projectId) {
+    if (!projectId)
+        return TEMPLATE;
+    return `${TEMPLATE}\n# Project to assign prompt-test runs to. Override per-run with \`--project-id\`.\nproject_id: "${projectId}"\n`;
+}
 export function registerInitCommand(pt) {
     pt.command("init")
         .description("Initializes a new prompt-tests.yaml file in the current directory")
-        .action(async () => {
+        .option("--project-id <uuid>", PROJECT_ID_FLAG_DESC)
+        .action(async (opts) => {
         const configPath = "prompt-tests.yaml";
         if (existsSync(configPath)) {
             printWarning(`'${configPath}' already exists in the current directory.`);
@@ -66,7 +73,7 @@ export function registerInitCommand(pt) {
             }
         }
         try {
-            writeFileSync(configPath, TEMPLATE);
+            writeFileSync(configPath, buildTemplate(opts.projectId));
             printSuccess(`'${configPath}' created successfully.`);
             printInfo("Update the file with your prompt test details and run `pt run`.");
         }

package/dist/commands/prompt-test/run.d.ts

@@ -1,4 +1,4 @@
 import { Command } from "commander";
-export declare function runPromptTests(outputFile: string | undefined, configPath: string, sleep?: (ms: number) => Promise<void>): Promise<void>;
+export declare function runPromptTests(outputFile: string | undefined, configPath: string, sleep?: (ms: number) => Promise<void>, projectIdOverride?: string): Promise<void>;
 export declare function registerRunCommand(pt: Command): void;
 //# sourceMappingURL=run.d.ts.map

package/dist/commands/prompt-test/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/commands/prompt-test/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AA6GpC,wBAAsB,cAAc,CAClC,UAAU,EAAE,MAAM,GAAG,SAAS,EAC9B,UAAU,EAAE,MAAM,EAClB,KAAK,GAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAiD,GACnF,OAAO,CAAC,IAAI,CAAC,CAiHf;AAED,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,OAAO,GAAG,IAAI,CAYpD"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/commands/prompt-test/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AA8GpC,wBAAsB,cAAc,CAClC,UAAU,EAAE,MAAM,GAAG,SAAS,EAC9B,UAAU,EAAE,MAAM,EAClB,KAAK,GAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAiD,EACpF,iBAAiB,CAAC,EAAE,MAAM,GACzB,OAAO,CAAC,IAAI,CAAC,CA2Hf;AAED,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,OAAO,GAAG,IAAI,CAapD"}

package/dist/commands/prompt-test/run.js

@@ -23,6 +23,7 @@ const UNICODE_CHARS = {
     middle: "│",
 };
 import { CliError } from "../../types.js";
+import { resolveProjectIdValue, PROJECT_ID_FLAG_DESC } from "../../lib/project-id.js";
 function isPromptTestComplete(exp) {
     if (!exp.tasks.length)
         return false;
@@ -93,7 +94,7 @@ function displayAggregatedResults(experiments) {
     }
     console.log(table.toString());
 }
-export async function runPromptTests(outputFile, configPath, sleep = (ms) => new Promise((r) => setTimeout(r, ms))) {
+export async function runPromptTests(outputFile, configPath, sleep = (ms) => new Promise((r) => setTimeout(r, ms)), projectIdOverride) {
     let rawConfig;
     try {
         rawConfig = yaml.load(readFileSync(configPath, "utf8"));
@@ -118,6 +119,9 @@ export async function runPromptTests(outputFile, configPath, sleep = (ms) => new
     }
     const apiKey = await requireApiKey();
     printInfo("Starting prompt tests");
+    const resolvedProjectId = projectIdOverride !== undefined
+        ? resolveProjectIdValue(projectIdOverride)
+        : (config.project_id ?? resolveProjectIdValue(undefined));
     const experiments = {};
     for (const prompt of config.prompts) {
         for (const model of config.models) {
@@ -141,6 +145,8 @@ export async function runPromptTests(outputFile, configPath, sleep = (ms) => new
                 payload["response_schema"] = config.response_schema;
             if (config.dataset_id)
                 payload["dataset_id"] = config.dataset_id;
+            if (resolvedProjectId !== undefined)
+                payload["project_id"] = resolvedProjectId;
             const result = (await apiRequest("POST", "prompt-tests", {
                 payload,
                 apiKey,
@@ -200,9 +206,10 @@ export function registerRunCommand(pt) {
         .description("Runs prompt tests from the prompt-tests.yaml file")
         .option("-o, --output <path>", "Output file path to save prompt test results as JSON")
         .option("-c, --config <path>", "Path to prompt testing configuration file", "prompt-tests.yaml")
+        .option("--project-id <uuid>", PROJECT_ID_FLAG_DESC + " Overrides project_id from config file.")
         .action(async (opts) => {
         try {
-            await runPromptTests(opts.output, opts.config);
+            await runPromptTests(opts.output, opts.config, undefined, opts.projectId);
         }
         catch (e) {
             handleSdkError(e);

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAIA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAoDpC,wBAAgB,SAAS,IAAI,OAAO,CA+BnC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAIA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAqDpC,wBAAgB,SAAS,IAAI,OAAO,CAgCnC"}

package/dist/index.js

@@ -14,6 +14,7 @@ import { registerFileCommands } from "./commands/file/index.js";
 import { registerOtelFilterCommands } from "./commands/otel-filter/index.js";
 import { registerOtelTraceCommands } from "./commands/otel-trace/index.js";
 import { registerModelCommands } from "./commands/model/index.js";
+import { registerProjectCommands } from "./commands/project/index.js";
 const { version } = createRequire(import.meta.url)("../package.json");
 function buildBanner(ver) {
     const logo = chalk.hex("#4D9FFF");
@@ -76,6 +77,7 @@ export function createCli() {
     registerOtelFilterCommands(program);
     registerOtelTraceCommands(program);
     registerModelCommands(program);
+    registerProjectCommands(program);
     return program;
 }
 if (realpathSync(fileURLToPath(import.meta.url)) === realpathSync(process.argv[1])) {

package/dist/lib/filter-yaml.d.ts

@@ -1,4 +1,12 @@
 import { z } from "zod";
+export declare const Match: z.ZodObject<{
+    conditions: z.ZodDefault<z.ZodArray<z.ZodObject<{
+        column: z.ZodString;
+        operator: z.ZodString;
+        value: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
+        key: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>>>;
+}, z.core.$strict>;
 export declare const FilterYamlSchema: z.ZodObject<{
     name: z.ZodString;
     evaluator_id: z.ZodOptional<z.ZodString>;

package/dist/lib/filter-yaml.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"filter-yaml.d.ts","sourceRoot":"","sources":["../../src/lib/filter-yaml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAyFxB,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAczB,CAAC;AAEL,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU,CAczD"}
+{"version":3,"file":"filter-yaml.d.ts","sourceRoot":"","sources":["../../src/lib/filter-yaml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAYxB,eAAO,MAAM,KAAK;;;;;;;kBAIP,CAAC;AAyEZ,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAczB,CAAC;AAEL,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU,CAczD"}

package/dist/lib/filter-yaml.js

@@ -8,7 +8,7 @@ const MatchCondition = z
     key: z.string().optional(),
 })
     .strict();
-const Match = z
+export const Match = z
     .object({
     conditions: z.array(MatchCondition).default([]),
 })

package/dist/lib/project-id.d.ts

@@ -0,0 +1,9 @@
+export type ProjectIdSource = "flag" | "env" | "settings" | "none";
+export interface ResolvedProjectId {
+    value: string | undefined;
+    source: ProjectIdSource;
+}
+export declare function resolveProjectId(flagValue?: string): ResolvedProjectId;
+export declare function resolveProjectIdValue(flagValue?: string): string | undefined;
+export declare const PROJECT_ID_FLAG_DESC = "Project UUID to scope this operation. Pass \"\" to opt out of inherited defaults from SCORABLE_PROJECT_ID or ~/.scorable/settings.json.";
+//# sourceMappingURL=project-id.d.ts.map

package/dist/lib/project-id.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"project-id.d.ts","sourceRoot":"","sources":["../../src/lib/project-id.ts"],"names":[],"mappings":"AAEA,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,KAAK,GAAG,UAAU,GAAG,MAAM,CAAC;AAEnE,MAAM,WAAW,iBAAiB;IAKhC,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,MAAM,EAAE,eAAe,CAAC;CACzB;AAYD,wBAAgB,gBAAgB,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,iBAAiB,CAqBtE;AAKD,wBAAgB,qBAAqB,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAE5E;AAKD,eAAO,MAAM,oBAAoB,4IACwG,CAAC"}

package/dist/lib/project-id.js

@@ -0,0 +1,23 @@
+import { loadSettings } from "../auth.js";
+export function resolveProjectId(flagValue) {
+    if (flagValue !== undefined) {
+        if (flagValue === "") {
+            return { value: undefined, source: "flag" };
+        }
+        return { value: flagValue, source: "flag" };
+    }
+    const fromEnv = process.env["SCORABLE_PROJECT_ID"];
+    if (fromEnv !== undefined && fromEnv !== "") {
+        return { value: fromEnv, source: "env" };
+    }
+    const settings = loadSettings();
+    const fromSettings = settings["project_id"];
+    if (typeof fromSettings === "string" && fromSettings !== "") {
+        return { value: fromSettings, source: "settings" };
+    }
+    return { value: undefined, source: "none" };
+}
+export function resolveProjectIdValue(flagValue) {
+    return resolveProjectId(flagValue).value;
+}
+export const PROJECT_ID_FLAG_DESC = 'Project UUID to scope this operation. Pass "" to opt out of inherited defaults from SCORABLE_PROJECT_ID or ~/.scorable/settings.json.';

package/dist/output.d.ts

@@ -1,4 +1,4 @@
-import type { Judge, EvaluatorListItem, ExecutionLogList, ModelList } from "@root-signals/scorable";
+import type { Judge, EvaluatorListItem, ExecutionLogList, ModelList, Project } from "@root-signals/scorable";
 export declare function printJson(data: unknown): void;
 export declare function printError(msg: string): void;
 export declare function handleSdkError(e: unknown): never;
@@ -8,6 +8,7 @@ export declare function printWarning(msg: string): void;
 export declare function printMessage(msg: string): void;
 export declare function printJudgeTable(judges: Judge[], nextCursor?: string): void;
 export declare function printEvaluatorTable(evaluators: EvaluatorListItem[], nextCursor?: string): void;
+export declare function printProjectTable(projects: Project[], nextCursor?: string): void;
 export declare function printModelTable(models: ModelList[], nextCursor?: string): void;
 export declare function printExecutionLogTable(logs: ExecutionLogList[], nextCursor?: string): void;
 //# sourceMappingURL=output.d.ts.map

package/dist/output.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"output.d.ts","sourceRoot":"","sources":["../src/output.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,KAAK,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAoBpG,wBAAgB,SAAS,CAAC,IAAI,EAAE,OAAO,GAAG,IAAI,CAE7C;AAED,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAE5C;AAED,wBAAgB,cAAc,CAAC,CAAC,EAAE,OAAO,GAAG,KAAK,CAKhD;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAE9C;AAED,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAE3C;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAE9C;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAE9C;AAMD,wBAAgB,eAAe,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAmB1E;AAED,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,iBAAiB,EAAE,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAmB9F;AAED,wBAAgB,eAAe,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAmB9E;AAED,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,gBAAgB,EAAE,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAoB1F"}
+{"version":3,"file":"output.d.ts","sourceRoot":"","sources":["../src/output.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,KAAK,EACL,iBAAiB,EACjB,gBAAgB,EAChB,SAAS,EACT,OAAO,EACR,MAAM,wBAAwB,CAAC;AAoBhC,wBAAgB,SAAS,CAAC,IAAI,EAAE,OAAO,GAAG,IAAI,CAE7C;AAED,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAE5C;AAmBD,wBAAgB,cAAc,CAAC,CAAC,EAAE,OAAO,GAAG,KAAK,CAgBhD;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAE9C;AAED,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAE3C;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAE9C;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAE9C;AAUD,wBAAgB,eAAe,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAmB1E;AAED,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,iBAAiB,EAAE,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAmB9F;AAED,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAoBhF;AAED,wBAAgB,eAAe,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAmB9E;AAED,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,gBAAgB,EAAE,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CA6B1F"}

package/dist/output.js

@@ -1,5 +1,6 @@
 import chalk from "chalk";
 import Table from "cli-table3";
+import { ScorableError } from "@root-signals/scorable";
 import { CliError } from "./types.js";
 const UNICODE_CHARS = {
     top: "─",
@@ -24,12 +25,42 @@ export function printJson(data) {
 export function printError(msg) {
     console.error(chalk.red("✖") + " " + msg);
 }
+function extractBackendMessage(details) {
+    if (!details || typeof details !== "object")
+        return undefined;
+    const d = details;
+    if (typeof d["detail"] === "string")
+        return d["detail"];
+    if (typeof d["title"] === "string")
+        return d["title"];
+    if (typeof d["error"] === "string")
+        return d["error"];
+    if (typeof d["message"] === "string")
+        return d["message"];
+    for (const v of Object.values(d)) {
+        if (Array.isArray(v) && v.length && typeof v[0] === "string") {
+            return v[0];
+        }
+    }
+    return undefined;
+}
 export function handleSdkError(e) {
     if (e instanceof CliError)
         throw e;
-    const message = e instanceof Error ? e.message || String(e) : String(e);
-    printError(message);
-    throw new CliError(1, message);
+    let message;
+    if (e instanceof ScorableError) {
+        const detail = extractBackendMessage(e.details);
+        message = detail ?? e.detail ?? e.title ?? e.message ?? `API Error ${e.status}: ${e.code}`;
+    }
+    else if (e instanceof Error) {
+        message = e.message || String(e);
+    }
+    else {
+        message = String(e);
+    }
+    const oneLine = message.replace(/\s*\n\s*/g, " ").trim();
+    printError(oneLine);
+    throw new CliError(1, oneLine);
 }
 export function printSuccess(msg) {
     console.log(chalk.green("✔") + " " + msg);
@@ -46,16 +77,19 @@ export function printMessage(msg) {
 function truncate(s, max) {
     return s.length > max ? s.slice(0, max - 1) + "…" : s;
 }
+function projectIdCell(value) {
+    return value ?? "";
+}
 export function printJudgeTable(judges, nextCursor) {
     const table = new Table({
-        head: ["ID", "Name", "Intent", "Created At"].map((h) => chalk.bold.cyan(h)),
+        head: ["ID", "Name", "Intent", "Project ID", "Created At"].map((h) => chalk.bold.cyan(h)),
         chars: UNICODE_CHARS,
-        colWidths: [38, 30, 52, 12],
+        colWidths: [38, 24, 40, 38, 12],
         wordWrap: true,
     });
     for (const j of judges) {
         const date = (j.created_at ?? "").slice(0, 10);
-        table.push([j.id, j.name, truncate(j.intent ?? "", 50), date]);
+        table.push([j.id, j.name, truncate(j.intent ?? "", 38), projectIdCell(j.project_id), date]);
     }
     console.log(table.toString());
     if (nextCursor) {
@@ -65,14 +99,32 @@ export function printJudgeTable(judges, nextCursor) {
 }
 export function printEvaluatorTable(evaluators, nextCursor) {
     const table = new Table({
-        head: ["ID", "Name", "Created At"].map((h) => chalk.bold.cyan(h)),
+        head: ["ID", "Name", "Project ID", "Created At"].map((h) => chalk.bold.cyan(h)),
         chars: UNICODE_CHARS,
-        colWidths: [38, 30, 12],
+        colWidths: [38, 30, 38, 12],
         wordWrap: true,
     });
     for (const e of evaluators) {
         const date = (e.created_at ?? "").slice(0, 10);
-        table.push([e.id, e.name, date]);
+        table.push([e.id, e.name, projectIdCell(e.project_id), date]);
+    }
+    console.log(table.toString());
+    if (nextCursor) {
+        const cursor = nextCursor.split("cursor=")[1] ?? nextCursor;
+        printInfo(`Next page available. Use --cursor "${cursor}"`);
+    }
+}
+export function printProjectTable(projects, nextCursor) {
+    const table = new Table({
+        head: ["ID", "Name", "Description", "Created At"].map((h) => chalk.bold.cyan(h)),
+        chars: UNICODE_CHARS,
+        colWidths: [38, 30, 40, 12],
+        wordWrap: true,
+    });
+    for (const p of projects) {
+        const date = (p.created_at ?? "").slice(0, 10);
+        const name = p.is_default ? `${p.name} (default)` : p.name;
+        table.push([p.id, name, truncate(p.description ?? "", 38), date]);
     }
     console.log(table.toString());
     if (nextCursor) {
@@ -99,15 +151,22 @@ export function printModelTable(models, nextCursor) {
 }
 export function printExecutionLogTable(logs, nextCursor) {
     const table = new Table({
-        head: ["ID", "Item Name", "Type", "Score", "Created At"].map((h) => chalk.bold.cyan(h)),
+        head: ["ID", "Item Name", "Type", "Score", "Project ID", "Created At"].map((h) => chalk.bold.cyan(h)),
         chars: UNICODE_CHARS,
-        colWidths: [38, 30, 15, 8, 12],
+        colWidths: [38, 24, 15, 8, 38, 12],
         wordWrap: true,
     });
     for (const l of logs) {
         const date = (l.created_at ?? "").slice(0, 10);
         const score = l.score != null ? String(l.score) : "";
-        table.push([l.id, truncate(l.executed_item_name, 28), l.execution_type, score, date]);
+        table.push([
+            l.id,
+            truncate(l.executed_item_name, 22),
+            l.execution_type,
+            score,
+            projectIdCell(l.project_id),
+            date,
+        ]);
     }
     console.log(table.toString());
     if (nextCursor) {

package/dist/types.d.ts

@@ -39,6 +39,7 @@ export interface PromptTestConfig {
     evaluators: EvaluatorConfig[];
     response_schema?: Record<string, unknown>;
     dataset_id?: string;
+    project_id?: string;
 }
 export declare class CliError extends Error {
     readonly exitCode: number;

package/dist/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,kBAAkB,EAAE,KAAK,CAAC;QACxB,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QACrB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;KAC9B,CAAC,CAAC;IACH,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,UAAU,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACjD;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC1C,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,qBAAa,QAAS,SAAQ,KAAK;aAEf,QAAQ,EAAE,MAAM;gBAAhB,QAAQ,EAAE,MAAM,EAChC,OAAO,EAAE,MAAM;CAKlB"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,kBAAkB,EAAE,KAAK,CAAC;QACxB,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QACrB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;KAC9B,CAAC,CAAC;IACH,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,UAAU,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACjD;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC1C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,qBAAa,QAAS,SAAQ,KAAK;aAEf,QAAQ,EAAE,MAAM;gBAAhB,QAAQ,EAAE,MAAM,EAChC,OAAO,EAAE,MAAM;CAKlB"}

package/dist/utils.d.ts

@@ -1,5 +1,14 @@
+import { type ZodType } from "zod";
 import type { Turn } from "@root-signals/scorable";
 export declare function isTurnArray(v: unknown): v is Turn[];
+export declare function isToolCatalog(v: unknown): v is Record<string, unknown>[];
 export declare function isStringArray(v: unknown): v is string[];
 export declare function isStringRecord(v: unknown): v is Record<string, string>;
+export type JsonArgResult<T> = {
+    ok: true;
+    value: T;
+} | {
+    ok: false;
+};
+export declare function parseJsonArg<T>(raw: string, schema: ZodType<T>): JsonArgResult<T>;
 //# sourceMappingURL=utils.d.ts.map

package/dist/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AAEnD,wBAAgB,WAAW,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,IAAI,IAAI,EAAE,CAanD;AAED,wBAAgB,aAAa,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,IAAI,MAAM,EAAE,CAEvD;AAED,wBAAgB,cAAc,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAOtE"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAK,KAAK,OAAO,EAAE,MAAM,KAAK,CAAC;AACtC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AAanD,wBAAgB,WAAW,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,IAAI,IAAI,EAAE,CAEnD;AAED,wBAAgB,aAAa,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAExE;AAED,wBAAgB,aAAa,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,IAAI,MAAM,EAAE,CAEvD;AAED,wBAAgB,cAAc,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAOtE;AAID,MAAM,MAAM,aAAa,CAAC,CAAC,IAAI;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,KAAK,EAAE,CAAC,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAA;CAAE,CAAC;AAEtE,wBAAgB,YAAY,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,CAUjF"}

package/dist/utils.js

@@ -1,11 +1,18 @@
+import { z } from "zod";
+const TurnSchema = z.object({
+    role: z.enum(["user", "assistant", "tool"]),
+    content: z.string().nullable().optional(),
+    contexts: z.array(z.string()).nullable().optional(),
+    tool_calls: z.array(z.record(z.string(), z.unknown())).nullable().optional(),
+    tool_call_id: z.string().nullable().optional(),
+});
+const TurnArraySchema = z.array(TurnSchema);
+const ToolCatalogSchema = z.array(z.record(z.string(), z.unknown()));
 export function isTurnArray(v) {
-    return (Array.isArray(v) &&
-        v.every((t) => typeof t === "object" &&
-            t !== null &&
-            "role" in t &&
-            typeof t.role === "string" &&
-            "content" in t &&
-            typeof t.content === "string"));
+    return TurnArraySchema.safeParse(v).success;
+}
+export function isToolCatalog(v) {
+    return ToolCatalogSchema.safeParse(v).success;
 }
 export function isStringArray(v) {
     return Array.isArray(v) && v.every((s) => typeof s === "string");
@@ -16,3 +23,16 @@ export function isStringRecord(v) {
         !Array.isArray(v) &&
         Object.values(v).every((val) => typeof val === "string"));
 }
+export function parseJsonArg(raw, schema) {
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch {
+        return { ok: false };
+    }
+    const result = schema.safeParse(parsed);
+    if (!result.success)
+        return { ok: false };
+    return { ok: true, value: result.data };
+}