@root-signals/scorable-cli 0.1.3 → 0.2.1

package/dist/commands/evaluator/index.js

@@ -0,0 +1,21 @@
+import { Command } from "commander";
+import { registerListCommand } from "./list.js";
+import { registerGetCommand } from "./get.js";
+import { registerCreateCommand } from "./create.js";
+import { registerUpdateCommand } from "./update.js";
+import { registerDeleteCommand } from "./delete.js";
+import { registerExecuteCommand } from "./execute.js";
+import { registerExecuteByNameCommand } from "./execute-by-name.js";
+import { registerDuplicateCommand } from "./duplicate.js";
+export function registerEvaluatorCommands(program) {
+    const evaluator = new Command("evaluator").description("Evaluator management commands");
+    registerListCommand(evaluator);
+    registerGetCommand(evaluator);
+    registerCreateCommand(evaluator);
+    registerUpdateCommand(evaluator);
+    registerDeleteCommand(evaluator);
+    registerExecuteCommand(evaluator);
+    registerExecuteByNameCommand(evaluator);
+    registerDuplicateCommand(evaluator);
+    program.addCommand(evaluator);
+}

package/dist/commands/evaluator/list.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerListCommand(evaluator: Command): void;
+//# sourceMappingURL=list.d.ts.map

package/dist/commands/evaluator/list.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"list.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/list.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CA2C5D"}

package/dist/commands/evaluator/list.js

@@ -0,0 +1,39 @@
+import { requireApiKey, getSdkClient } from "../../auth.js";
+import { printInfo, printMessage, printEvaluatorTable, handleSdkError } from "../../output.js";
+export function registerListCommand(evaluator) {
+    evaluator
+        .command("list")
+        .description("List evaluators with optional filters")
+        .option("--page-size <number>", "Number of results to return per page", parseInt)
+        .option("--cursor <cursor>", "The pagination cursor value")
+        .option("--search <term>", "A search term to filter by")
+        .option("--name <name>", "Filter by exact evaluator name")
+        .option("--ordering <field>", "Which field to use for ordering the results")
+        .action(async (opts) => {
+        const apiKey = await requireApiKey();
+        const params = {};
+        if (opts.pageSize !== undefined)
+            params.page_size = opts.pageSize;
+        if (opts.cursor !== undefined)
+            params.cursor = opts.cursor;
+        if (opts.search !== undefined)
+            params.search = opts.search;
+        if (opts.name !== undefined)
+            params.name = opts.name;
+        if (opts.ordering !== undefined)
+            params.ordering = opts.ordering;
+        printInfo(`Fetching evaluators with params: ${JSON.stringify(params)}...`);
+        try {
+            const client = getSdkClient(apiKey);
+            const response = await client.evaluators.list(params);
+            if (!response.results.length) {
+                printMessage("No evaluators found.");
+                return;
+            }
+            printEvaluatorTable(response.results, response.next);
+        }
+        catch (e) {
+            handleSdkError(e);
+        }
+    });
+}

package/dist/commands/evaluator/update.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerUpdateCommand(evaluator: Command): void;
+//# sourceMappingURL=update.d.ts.map

package/dist/commands/evaluator/update.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"update.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/update.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CA2D9D"}

package/dist/commands/evaluator/update.js

@@ -0,0 +1,51 @@
+import { requireApiKey, getSdkClient } from "../../auth.js";
+import { printInfo, printSuccess, printError, printJson, handleSdkError } from "../../output.js";
+export function registerUpdateCommand(evaluator) {
+    evaluator
+        .command("update <evaluatorId>")
+        .description("Update an existing evaluator (PATCH)")
+        .option("--name <name>", "The new name for the evaluator")
+        .option("--scoring-criteria <text>", "The new scoring criteria (prompt text)")
+        .option("--system-message <text>", "The new system message")
+        .option("--models <json>", "JSON array of model names. E.g., '[\"gpt-4\"]'")
+        .option("--objective-id <id>", "The new objective ID")
+        .option("--objective-version-id <id>", "The new objective version ID")
+        .action(async (evaluatorId, opts) => {
+        const apiKey = await requireApiKey();
+        const payload = {};
+        if (opts.name !== undefined)
+            payload.name = opts.name;
+        if (opts.scoringCriteria !== undefined)
+            payload.prompt = opts.scoringCriteria;
+        if (opts.systemMessage !== undefined)
+            payload.system_message = opts.systemMessage;
+        if (opts.objectiveId !== undefined)
+            payload.objective_id = opts.objectiveId;
+        if (opts.objectiveVersionId !== undefined)
+            payload.objective_version_id = opts.objectiveVersionId;
+        if (opts.models !== undefined) {
+            try {
+                payload.models = JSON.parse(opts.models);
+            }
+            catch {
+                printError("Invalid JSON format for --models.");
+                return;
+            }
+        }
+        if (Object.keys(payload).length === 0) {
+            printInfo("No update parameters provided. Aborting.");
+            return;
+        }
+        printInfo(`Attempting to update evaluator ${evaluatorId} with PATCH payload:`);
+        printJson(payload);
+        try {
+            const client = getSdkClient(apiKey);
+            const result = await client.evaluators.update(evaluatorId, payload);
+            printSuccess(`Evaluator ${evaluatorId} updated successfully!`);
+            printJson(result);
+        }
+        catch (e) {
+            handleSdkError(e);
+        }
+    });
+}

package/dist/commands/execution-log/get.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerGetCommand(executionLog: Command): void;
+//# sourceMappingURL=get.d.ts.map

package/dist/commands/execution-log/get.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"get.d.ts","sourceRoot":"","sources":["../../../src/commands/execution-log/get.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIpC,wBAAgB,kBAAkB,CAAC,YAAY,EAAE,OAAO,GAAG,IAAI,CAgB9D"}

package/dist/commands/execution-log/get.js

@@ -0,0 +1,19 @@
+import { requireApiKey, getSdkClient } from "../../auth.js";
+import { printInfo, printJson, handleSdkError } from "../../output.js";
+export function registerGetCommand(executionLog) {
+    executionLog
+        .command("get <logId>")
+        .description("Get a specific execution log by its ID")
+        .action(async (logId) => {
+        const apiKey = await requireApiKey();
+        printInfo(`Fetching execution log with ID: ${logId}...`);
+        try {
+            const client = getSdkClient(apiKey);
+            const result = await client.executionLogs.get(logId);
+            printJson(result);
+        }
+        catch (e) {
+            handleSdkError(e);
+        }
+    });
+}

package/dist/commands/execution-log/index.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerExecutionLogCommands(program: Command): void;
+//# sourceMappingURL=index.d.ts.map

package/dist/commands/execution-log/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/commands/execution-log/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIpC,wBAAgB,4BAA4B,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAOnE"}

package/dist/commands/execution-log/index.js

@@ -0,0 +1,9 @@
+import { Command } from "commander";
+import { registerListCommand } from "./list.js";
+import { registerGetCommand } from "./get.js";
+export function registerExecutionLogCommands(program) {
+    const executionLog = new Command("execution-log").description("Execution log commands");
+    registerListCommand(executionLog);
+    registerGetCommand(executionLog);
+    program.addCommand(executionLog);
+}

package/dist/commands/execution-log/list.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerListCommand(executionLog: Command): void;
+//# sourceMappingURL=list.d.ts.map

package/dist/commands/execution-log/list.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"list.d.ts","sourceRoot":"","sources":["../../../src/commands/execution-log/list.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,wBAAgB,mBAAmB,CAAC,YAAY,EAAE,OAAO,GAAG,IAAI,CAgE/D"}

package/dist/commands/execution-log/list.js

@@ -0,0 +1,60 @@
+import { requireApiKey, getSdkClient } from "../../auth.js";
+import { printInfo, printMessage, printExecutionLogTable, handleSdkError } from "../../output.js";
+export function registerListCommand(executionLog) {
+    executionLog
+        .command("list")
+        .description("List execution logs with optional filters")
+        .option("--page-size <number>", "Number of results to return per page", parseInt)
+        .option("--cursor <cursor>", "The pagination cursor value")
+        .option("--search <term>", "Search term to filter by evaluator id or name")
+        .option("--evaluator-id <id>", "Filter by evaluator ID")
+        .option("--judge-id <id>", "Filter by judge ID")
+        .option("--model <model>", "Filter by model name")
+        .option("--tags <tags>", "Filter by tags (comma-separated)")
+        .option("--score-min <number>", "Minimum score filter", parseFloat)
+        .option("--score-max <number>", "Maximum score filter", parseFloat)
+        .option("--created-at-after <date>", "Filter logs created after this date (ISO 8601)")
+        .option("--created-at-before <date>", "Filter logs created before this date (ISO 8601)")
+        .option("--owner-email <email>", "Filter by owner email")
+        .action(async (opts) => {
+        const apiKey = await requireApiKey();
+        const params = {};
+        if (opts.pageSize !== undefined)
+            params.page_size = opts.pageSize;
+        if (opts.cursor !== undefined)
+            params.cursor = opts.cursor;
+        if (opts.search !== undefined)
+            params.search = opts.search;
+        if (opts.evaluatorId !== undefined)
+            params.evaluator_id = opts.evaluatorId;
+        if (opts.judgeId !== undefined)
+            params.judge_id = opts.judgeId;
+        if (opts.model !== undefined)
+            params.model = opts.model;
+        if (opts.tags !== undefined)
+            params.tags = opts.tags;
+        if (opts.scoreMin !== undefined)
+            params.score_min = opts.scoreMin;
+        if (opts.scoreMax !== undefined)
+            params.score_max = opts.scoreMax;
+        if (opts.createdAtAfter !== undefined)
+            params.created_at_after = opts.createdAtAfter;
+        if (opts.createdAtBefore !== undefined)
+            params.created_at_before = opts.createdAtBefore;
+        if (opts.ownerEmail !== undefined)
+            params.owner__email = opts.ownerEmail;
+        printInfo(`Fetching execution logs...`);
+        try {
+            const client = getSdkClient(apiKey);
+            const response = await client.executionLogs.list(params);
+            if (!response.results.length) {
+                printMessage("No execution logs found.");
+                return;
+            }
+            printExecutionLogTable(response.results, response.next);
+        }
+        catch (e) {
+            handleSdkError(e);
+        }
+    });
+}

package/dist/commands/judge/create.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"create.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAgD1D"}
+{"version":3,"file":"create.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CA+C1D"}

package/dist/commands/judge/create.js

@@ -1,6 +1,5 @@
 import { requireApiKey, getSdkClient } from "../../auth.js";
-import { printInfo, printSuccess, printError, printJson } from "../../output.js";
-import { CliError } from "../../types.js";
+import { printInfo, printSuccess, printError, printJson, handleSdkError } from "../../output.js";
 export function registerCreateCommand(judge) {
     judge
         .command("create")
@@ -32,9 +31,7 @@ export function registerCreateCommand(judge) {
             printJson(result);
         }
         catch (e) {
-            if (e instanceof CliError)
-                throw e;
-            printError(e instanceof Error ? e.message : String(e));
+            handleSdkError(e);
         }
     });
 }

package/dist/commands/judge/delete.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"delete.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/delete.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CA8B1D"}
+{"version":3,"file":"delete.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/delete.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAmC1D"}

package/dist/commands/judge/delete.js

@@ -1,5 +1,5 @@
 import { requireApiKey, getSdkClient } from "../../auth.js";
-import { printInfo, printSuccess } from "../../output.js";
+import { printInfo, printSuccess, handleSdkError } from "../../output.js";
 import { CliError } from "../../types.js";
 export function registerDeleteCommand(judge) {
     judge
@@ -9,6 +9,9 @@ export function registerDeleteCommand(judge) {
         .action(async (judgeId, opts) => {
         const apiKey = await requireApiKey();
         if (!opts.yes) {
+            if (!process.stdin.isTTY && !process.stdout.isTTY) {
+                throw new CliError(1, "Refusing to prompt for confirmation in a non-interactive environment. Pass --yes to skip the prompt.");
+            }
             const { confirm } = await import("@inquirer/prompts");
             const ok = await confirm({
                 message: "Are you sure you want to delete this judge?",
@@ -25,9 +28,7 @@ export function registerDeleteCommand(judge) {
             printSuccess(`Judge ${judgeId} deleted successfully.`);
         }
         catch (e) {
-            if (e instanceof CliError)
-                throw e;
-            throw new CliError(1, e instanceof Error ? e.message : String(e));
+            handleSdkError(e);
         }
     });
 }

package/dist/commands/judge/duplicate.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"duplicate.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/duplicate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAkB7D"}
+{"version":3,"file":"duplicate.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/duplicate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIpC,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAiB7D"}

package/dist/commands/judge/duplicate.js

@@ -1,6 +1,5 @@
 import { requireApiKey, getSdkClient } from "../../auth.js";
-import { printInfo, printSuccess, printError, printJson } from "../../output.js";
-import { CliError } from "../../types.js";
+import { printInfo, printSuccess, printJson, handleSdkError } from "../../output.js";
 export function registerDuplicateCommand(judge) {
     judge
         .command("duplicate <judgeId>")
@@ -15,9 +14,7 @@ export function registerDuplicateCommand(judge) {
             printJson(result);
         }
         catch (e) {
-            if (e instanceof CliError)
-                throw e;
-            printError(e instanceof Error ? e.message : String(e));
+            handleSdkError(e);
         }
     });
 }

package/dist/commands/judge/exec-openai-generic.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"exec-openai-generic.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/exec-openai-generic.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,gCAAgC,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CA8CrE"}
+{"version":3,"file":"exec-openai-generic.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/exec-openai-generic.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAYpC,wBAAgB,gCAAgC,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CA6CrE"}

package/dist/commands/judge/exec-openai-generic.js

@@ -1,6 +1,5 @@
 import { requireApiKey } from "../../auth.js";
-import { printInfo, printSuccess, printError, printWarning, printJson } from "../../output.js";
-import { CliError } from "../../types.js";
+import { printInfo, printSuccess, printError, printWarning, printJson, handleSdkError, } from "../../output.js";
 import { apiRequest } from "../../client.js";
 export function registerExecOpenaiGenericCommand(judge) {
     judge
@@ -42,9 +41,7 @@ export function registerExecOpenaiGenericCommand(judge) {
             }
         }
         catch (e) {
-            if (e instanceof CliError)
-                throw e;
-            printError(e instanceof Error ? e.message : String(e));
+            handleSdkError(e);
         }
     });
 }

package/dist/commands/judge/exec-openai.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"exec-openai.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/exec-openai.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAyD9D"}
+{"version":3,"file":"exec-openai.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/exec-openai.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAYpC,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAwD9D"}

package/dist/commands/judge/exec-openai.js

@@ -1,6 +1,5 @@
 import { requireApiKey } from "../../auth.js";
-import { printInfo, printSuccess, printError, printWarning, printJson } from "../../output.js";
-import { CliError } from "../../types.js";
+import { printInfo, printSuccess, printError, printWarning, printJson, handleSdkError, } from "../../output.js";
 import { apiRequest } from "../../client.js";
 export function registerExecOpenaiCommand(judge) {
     judge
@@ -42,9 +41,7 @@ export function registerExecOpenaiCommand(judge) {
             }
         }
         catch (e) {
-            if (e instanceof CliError)
-                throw e;
-            printError(e instanceof Error ? e.message : String(e));
+            handleSdkError(e);
         }
     });
 }

package/dist/commands/judge/execute-by-name.d.ts

@@ -3,6 +3,7 @@ declare function readStdinDefault(): Promise<string>;
 export declare function executeJudgeByName(judgeName: string, opts: {
     request?: string;
     response?: string;
+    turns?: string;
     contexts?: string;
     expectedOutput?: string;
     tag?: string[];

package/dist/commands/judge/execute-by-name.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"execute-by-name.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/execute-by-name.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,iBAAe,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAIjD;AAED,wBAAsB,kBAAkB,CACtC,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE;IACJ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,EACD,SAAS,0BAAmB,GAC3B,OAAO,CAAC,IAAI,CAAC,CAyCf;AAED,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAyBjE"}
+{"version":3,"file":"execute-by-name.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/execute-by-name.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,iBAAe,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAIjD;AAED,wBAAsB,kBAAkB,CACtC,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE;IACJ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,EACD,SAAS,0BAAmB,GAC3B,OAAO,CAAC,IAAI,CAAC,CAyDf;AAED,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAkEjE"}

package/dist/commands/judge/execute-by-name.js

@@ -1,6 +1,6 @@
 import { requireApiKey, getSdkClient } from "../../auth.js";
-import { printInfo, printSuccess, printError, printJson } from "../../output.js";
-import { CliError } from "../../types.js";
+import { printInfo, printSuccess, printError, printJson, handleSdkError } from "../../output.js";
+import { isTurnArray, isStringArray } from "../../utils.js";
 async function readStdinDefault() {
     const chunks = [];
     for await (const chunk of process.stdin)
@@ -10,31 +10,50 @@ async function readStdinDefault() {
 export async function executeJudgeByName(judgeName, opts, readStdin = readStdinDefault) {
     const apiKey = await requireApiKey();
     let response = opts.response;
-    if (!opts.request && !response && !process.stdin.isTTY) {
+    if (!opts.request && !response && !opts.turns && !process.stdin.isTTY) {
         response = await readStdin();
     }
-    if (!opts.request && !response) {
-        printError("Either --request or --response must be provided.");
+    if (!opts.request && !response && !opts.turns) {
+        printError("Either --request, --response, or --turns must be provided.");
         return;
     }
     const payload = {};
     if (opts.request)
-        payload["request"] = opts.request;
+        payload.request = opts.request;
     if (response)
-        payload["response"] = response;
+        payload.response = response;
     if (opts.expectedOutput)
-        payload["expected_output"] = opts.expectedOutput;
+        payload.expected_output = opts.expectedOutput;
     if (opts.tag?.length)
-        payload["tags"] = opts.tag;
+        payload.tags = opts.tag;
     if (opts.userId)
-        payload["user_id"] = opts.userId;
+        payload.user_id = opts.userId;
     if (opts.sessionId)
-        payload["session_id"] = opts.sessionId;
+        payload.session_id = opts.sessionId;
     if (opts.systemPrompt)
-        payload["system_prompt"] = opts.systemPrompt;
+        payload.system_prompt = opts.systemPrompt;
+    if (opts.turns) {
+        try {
+            const parsed = JSON.parse(opts.turns);
+            if (!isTurnArray(parsed)) {
+                printError("Invalid JSON for --turns.");
+                return;
+            }
+            payload.turns = parsed;
+        }
+        catch {
+            printError("Invalid JSON for --turns.");
+            return;
+        }
+    }
     if (opts.contexts) {
         try {
-            payload["contexts"] = JSON.parse(opts.contexts);
+            const parsed = JSON.parse(opts.contexts);
+            if (!isStringArray(parsed)) {
+                printError("Invalid JSON for --contexts. Skipping.");
+                return;
+            }
+            payload.contexts = parsed;
         }
         catch {
             printError("Invalid JSON for --contexts. Skipping.");
@@ -60,14 +79,33 @@ export function registerExecuteByNameCommand(judge) {
         .option("--user-id <id>", "User identifier for tracking purposes")
         .option("--session-id <id>", "Session identifier for tracking purposes")
         .option("--system-prompt <text>", "System prompt that was used for the LLM call")
+        .option("--turns <json>", 'JSON array of conversation turns. E.g., \'[{"role":"user","content":"Hello"}]\'')
+        .addHelpText("after", `
+Examples:
+  # Evaluate a customer service response
+  $ scorable judge execute-by-name "Customer Service Judge" \\
+      --request "My order hasn't arrived and it's been two weeks." \\
+      --response "I completely understand your frustration. Let me immediately look into your order status and arrange expedited shipping at no extra cost."
+
+  # With context documents
+  $ scorable judge execute-by-name "E-commerce Support Judge" \\
+      --request "What's your return policy for electronics?" \\
+      --response "You can return electronics within 30 days, unused and in original packaging." \\
+      --contexts '["Returns for electronics: 30 days, unused, original packaging, valid receipt required."]'
+
+  # Multi-turn conversation
+  $ scorable judge execute-by-name "Customer Service Judge" \\
+      --turns '[{"role":"user","content":"I haven\\'t received my refund and it\\'s been 10 days."},{"role":"assistant","content":"Thank you for contacting us. I will investigate this immediately and provide an update within 24 hours."}]'
+
+  # Pipe the LLM response from stdin
+  $ echo "We offer a 30-day money-back guarantee on all purchases." | \\
+      scorable judge execute-by-name "Custom Returns Policy Judge" --request "Do you offer refunds?"`)
         .action(async (judgeName, opts) => {
         try {
             await executeJudgeByName(judgeName, opts);
         }
         catch (e) {
-            if (e instanceof CliError)
-                throw e;
-            printError(e instanceof Error ? e.message : String(e));
+            handleSdkError(e);
         }
     });
 }

package/dist/commands/judge/execute.d.ts

@@ -3,6 +3,7 @@ declare function readStdinDefault(): Promise<string>;
 export declare function executeJudge(judgeId: string, opts: {
     request?: string;
     response?: string;
+    turns?: string;
     contexts?: string;
     expectedOutput?: string;
     tag?: string[];

package/dist/commands/judge/execute.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"execute.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/execute.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,iBAAe,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAIjD;AAED,wBAAsB,YAAY,CAChC,OAAO,EAAE,MAAM,EACf,IAAI,EAAE;IACJ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,EACD,SAAS,0BAAmB,GAC3B,OAAO,CAAC,IAAI,CAAC,CAsCf;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAyB3D"}
+{"version":3,"file":"execute.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/execute.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,iBAAe,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAIjD;AAED,wBAAsB,YAAY,CAChC,OAAO,EAAE,MAAM,EACf,IAAI,EAAE;IACJ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,EACD,SAAS,0BAAmB,GAC3B,OAAO,CAAC,IAAI,CAAC,CAyDf;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAyE3D"}

package/dist/commands/judge/execute.js

@@ -1,6 +1,6 @@
 import { requireApiKey, getSdkClient } from "../../auth.js";
-import { printInfo, printSuccess, printError, printJson } from "../../output.js";
-import { CliError } from "../../types.js";
+import { printInfo, printSuccess, printError, printJson, handleSdkError } from "../../output.js";
+import { isTurnArray, isStringArray } from "../../utils.js";
 async function readStdinDefault() {
     const chunks = [];
     for await (const chunk of process.stdin)
@@ -10,31 +10,50 @@ async function readStdinDefault() {
 export async function executeJudge(judgeId, opts, readStdin = readStdinDefault) {
     const apiKey = await requireApiKey();
     let response = opts.response;
-    if (!opts.request && !response && !process.stdin.isTTY) {
+    if (!opts.request && !response && !opts.turns && !process.stdin.isTTY) {
         response = await readStdin();
     }
-    if (!opts.request && !response) {
-        printError("Either --request or --response must be provided.");
+    if (!opts.request && !response && !opts.turns) {
+        printError("Either --request, --response, or --turns must be provided.");
         return;
     }
     const payload = {};
     if (opts.request)
-        payload["request"] = opts.request;
+        payload.request = opts.request;
     if (response)
-        payload["response"] = response;
+        payload.response = response;
     if (opts.expectedOutput)
-        payload["expected_output"] = opts.expectedOutput;
+        payload.expected_output = opts.expectedOutput;
     if (opts.tag?.length)
-        payload["tags"] = opts.tag;
+        payload.tags = opts.tag;
     if (opts.userId)
-        payload["user_id"] = opts.userId;
+        payload.user_id = opts.userId;
     if (opts.sessionId)
-        payload["session_id"] = opts.sessionId;
+        payload.session_id = opts.sessionId;
     if (opts.systemPrompt)
-        payload["system_prompt"] = opts.systemPrompt;
+        payload.system_prompt = opts.systemPrompt;
+    if (opts.turns) {
+        try {
+            const parsed = JSON.parse(opts.turns);
+            if (!isTurnArray(parsed)) {
+                printError("Invalid JSON for --turns.");
+                return;
+            }
+            payload.turns = parsed;
+        }
+        catch {
+            printError("Invalid JSON for --turns.");
+            return;
+        }
+    }
     if (opts.contexts) {
         try {
-            payload["contexts"] = JSON.parse(opts.contexts);
+            const parsed = JSON.parse(opts.contexts);
+            if (!isStringArray(parsed)) {
+                printError("Invalid JSON for --contexts. Skipping.");
+                return;
+            }
+            payload.contexts = parsed;
         }
         catch {
             printError("Invalid JSON for --contexts. Skipping.");
@@ -60,14 +79,40 @@ export function registerExecuteCommand(judge) {
         .option("--user-id <id>", "User identifier for tracking purposes")
         .option("--session-id <id>", "Session identifier for tracking purposes")
         .option("--system-prompt <text>", "System prompt that was used for the LLM call")
+        .option("--turns <json>", 'JSON array of conversation turns. E.g., \'[{"role":"user","content":"Hello"}]\'')
+        .addHelpText("after", `
+Examples:
+  # Evaluate a customer service response about return policy
+  $ scorable judge execute <judgeId> \\
+      --request "What's your return policy?" \\
+      --response "We have a 30-day return policy. If you're not satisfied, you can return within 30 days for a full refund."
+
+  # RAG evaluation with context documents
+  $ scorable judge execute <judgeId> \\
+      --request "What's your return policy for electronics?" \\
+      --response "You can return electronics within 30 days, unused and in original packaging." \\
+      --contexts '["Returns for electronics are accepted within 30 days. Item must be unused, in original packaging, with valid receipt."]'
+
+  # Multi-turn conversation evaluation
+  $ scorable judge execute <judgeId> \\
+      --turns '[{"role":"user","content":"Hello, I need help with my order"},{"role":"assistant","content":"I\\'d be happy to help! What\\'s your order number?"},{"role":"user","content":"It\\'s ORDER-12345"},{"role":"assistant","content":"I found your order. It\\'s currently in transit."}]'
+
+  # With tracking metadata and tags
+  $ scorable judge execute <judgeId> \\
+      --request "How do I reset my password?" \\
+      --response "Click the Forgot Password link on the login page and follow the instructions." \\
+      --user-id user_123 --session-id session_abc \\
+      --tag production --tag v1.23
+
+  # Pipe the LLM response from stdin
+  $ echo "You can cancel your subscription from account settings under billing." | \\
+      scorable judge execute <judgeId> --request "How do I cancel my subscription?"`)
         .action(async (judgeId, opts) => {
         try {
             await executeJudge(judgeId, opts);
         }
         catch (e) {
-            if (e instanceof CliError)
-                throw e;
-            printError(e instanceof Error ? e.message : String(e));
+            handleSdkError(e);
         }
     });
 }