@root-signals/scorable-cli 0.1.3 → 0.2.0

package/README.md

@@ -24,7 +24,7 @@
   </a>
 </p>
 
-The `scorable` CLI is a command-line tool for interacting with the Scorable API. It lets you manage and execute Judges and run prompt testing experiments directly from the terminal.
+The `scorable` CLI is a command-line tool for interacting with the Scorable API. It lets you manage and execute Judges and Evaluators, view execution logs, and run prompt testing experiments directly from the terminal.
 
 Requires **Node.js 20 or higher**.
 
@@ -68,7 +68,7 @@ If no API key is set, the CLI will offer to create a temporary key interactively
 scorable judge list
 ```
 
-Options: `--page-size`, `--cursor`, `--search`, `--name`, `--ordering`, `--is-preset / --not-is-preset`, `--is-public / --not-is-public`, `--show-global / --not-show-global`
+Options: `--page-size`, `--cursor`, `--search`, `--name`, `--ordering`
 
 ### Get a judge
 
@@ -114,37 +114,118 @@ scorable judge duplicate <judge_id>
 scorable judge execute <judge_id> --request "What is the capital of France?" --response "Paris"
 ```
 
-Options: `--request`, `--response`, `--contexts` (JSON list), `--expected-output`, `--tag` (repeatable), `--user-id`, `--session-id`, `--system-prompt`
+Options: `--request`, `--response`, `--turns` (JSON array of conversation turns), `--contexts` (JSON list), `--expected-output`, `--tag` (repeatable), `--user-id`, `--session-id`, `--system-prompt`
 
-Pipe input via stdin:
+Pipe a response via stdin:
 
 ```bash
 echo "Paris" | scorable judge execute <judge_id> --request "What is the capital of France?"
 cat response.txt | scorable judge execute <judge_id>
 ```
 
+For multi-turn conversations, pass the full history as a JSON array:
+
+```bash
+scorable judge execute <judge_id> --turns '[{"role":"user","content":"Hi"},{"role":"assistant","content":"Hello!"}]'
+```
+
 ### Execute by name
 
 ```bash
 scorable judge execute-by-name "My Judge" --request "What is the capital of France?" --response "Paris"
 ```
 
-Accepts the same options as `execute`. Stdin piping works the same way.
+Accepts the same options as `execute`. Stdin piping and `--turns` work the same way.
+
+## Evaluator Management
+
+### List evaluators
+
+```bash
+scorable evaluator list
+```
+
+Options: `--page-size`, `--cursor`, `--search`, `--name`, `--ordering`
+
+### Get an evaluator
+
+```bash
+scorable evaluator get <evaluator_id>
+```
+
+### Create an evaluator
+
+```bash
+scorable evaluator create \
+  --name "My Evaluator" \
+  --scoring-criteria "Does the {{ response }} directly answer the user's question?" \
+  --intent "Evaluate response relevance"
+```
+
+Options: `--name` (required), `--scoring-criteria` (required — must contain `{{ request }}` and/or `{{ response }}`), `--intent` or `--objective-id` (one required), `--system-message`, `--models` (JSON array), `--overwrite`, `--objective-version-id`
+
+### Update an evaluator
+
+```bash
+scorable evaluator update <evaluator_id> --name "Updated Name"
+```
+
+Options: `--name`, `--scoring-criteria`, `--system-message`, `--models` (JSON array), `--objective-id`, `--objective-version-id`
+
+### Delete an evaluator
+
+```bash
+scorable evaluator delete <evaluator_id>
+```
+
+Prompts for confirmation. Use `--yes` to skip.
+
+### Duplicate an evaluator
+
+```bash
+scorable evaluator duplicate <evaluator_id>
+```
+
+## Evaluator Execution
 
-### Execute via OpenAI-compatible API
+### Execute by ID
 
 ```bash
-scorable judge exec-openai <judge_id> \
-  --model gpt-4o \
-  --messages '[{"role": "user", "content": "Hello"}, {"role": "assistant", "content": "Hi there"}]'
+scorable evaluator execute <evaluator_id> --request "What is 2+2?" --response "4"
 ```
 
-Generic variant (judge ID in the `model` field):
+Options: `--request`, `--response`, `--turns` (JSON array of conversation turns), `--contexts` (JSON list), `--expected-output`, `--tag` (repeatable), `--user-id`, `--session-id`, `--system-prompt`, `--variables` (JSON object of extra template variables)
+
+Stdin piping and `--turns` work the same way as for judge execution.
+
+For evaluators with custom template placeholders beyond `{{request}}`/`{{response}}`:
+
+```bash
+scorable evaluator execute <evaluator_id> --request "Hello" --variables '{"lang":"EN","topic":"science"}'
+```
+
+### Execute by name
+
+```bash
+scorable evaluator execute-by-name "My Evaluator" --request "What is 2+2?" --response "4"
+```
+
+Accepts the same options as `execute`, including `--variables`.
+
+## Execution Logs
+
+### List execution logs
+
+```bash
+scorable execution-log list
+```
+
+Options: `--page-size`, `--cursor`, `--search`, `--evaluator-id`, `--judge-id`, `--model`, `--tags`, `--score-min`, `--score-max`, `--created-at-after`, `--created-at-before`, `--owner-email`
+
+### Get an execution log
 
 ```bash
-scorable judge exec-openai-generic \
-  --model <judge_id_or_name> \
-  --messages '[{"role": "user", "content": "Hello"}, {"role": "assistant", "content": "Hi there"}]'
+scorable execution-log get <log_id>
 ```
 
 ## Prompt Testing

package/dist/client.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAMA,wBAAsB,UAAU,CAC9B,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE;IACR,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,GACA,OAAO,CAAC,OAAO,CAAC,CAoDlB"}
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAQA,wBAAsB,UAAU,CAC9B,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE;IACR,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,GACA,OAAO,CAAC,OAAO,CAAC,CAoDlB"}

package/dist/commands/evaluator/create.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerCreateCommand(evaluator: Command): void;
+//# sourceMappingURL=create.d.ts.map

package/dist/commands/evaluator/create.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"create.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAkF9D"}

package/dist/commands/evaluator/create.js

@@ -0,0 +1,62 @@
+import { requireApiKey, getSdkClient } from "../../auth.js";
+import { printInfo, printSuccess, printError, printJson, handleSdkError } from "../../output.js";
+export function registerCreateCommand(evaluator) {
+    evaluator
+        .command("create")
+        .description("Create a new evaluator")
+        .requiredOption("--name <name>", "The name for the new evaluator")
+        .requiredOption("--scoring-criteria <text>", "The scoring criteria prompt. Must contain {{ request }} and/or {{ response }} as placeholders.")
+        .option("--intent <intent>", "The intent for the evaluator (mutually exclusive with --objective-id)")
+        .option("--objective-id <id>", "Objective ID (mutually exclusive with --intent)")
+        .option("--system-message <text>", "System message for the evaluator")
+        .option("--models <json>", "JSON array of model names, in priority order. E.g., '[\"gpt-5-mini\"]'")
+        .option("--overwrite", "Overwrite if evaluator with same name exists")
+        .option("--objective-version-id <id>", "Objective version ID")
+        .action(async (opts) => {
+        if (!opts.intent && !opts.objectiveId) {
+            printError("Either --intent or --objective-id is required.");
+            return;
+        }
+        if (!/\{\{\s*request\s*\}\}/.test(opts.scoringCriteria) &&
+            !/\{\{\s*response\s*\}\}/.test(opts.scoringCriteria)) {
+            printError("The --scoring-criteria must contain at least one of {{ request }} or {{ response }} as a placeholder.\n" +
+                'Example: "Does the {{ response }} directly answer the user\'s question?"');
+            return;
+        }
+        const payload = {
+            name: opts.name,
+            predicate: opts.scoringCriteria,
+        };
+        if (opts.intent)
+            payload.intent = opts.intent;
+        if (opts.objectiveId)
+            payload.objective_id = opts.objectiveId;
+        if (opts.systemMessage)
+            payload.system_message = opts.systemMessage;
+        if (opts.overwrite !== undefined)
+            payload.overwrite = opts.overwrite;
+        if (opts.objectiveVersionId)
+            payload.objective_version_id = opts.objectiveVersionId;
+        if (opts.models) {
+            try {
+                payload.models = JSON.parse(opts.models);
+            }
+            catch {
+                printError("Invalid JSON format for --models.");
+                return;
+            }
+        }
+        printInfo("Attempting to create evaluator with payload:");
+        printJson(payload);
+        try {
+            const apiKey = await requireApiKey();
+            const client = getSdkClient(apiKey);
+            const result = await client.evaluators.create(payload);
+            printSuccess("Evaluator created successfully!");
+            printJson(result);
+        }
+        catch (e) {
+            handleSdkError(e);
+        }
+    });
+}

package/dist/commands/evaluator/delete.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerDeleteCommand(evaluator: Command): void;
+//# sourceMappingURL=delete.d.ts.map

package/dist/commands/evaluator/delete.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"delete.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/delete.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAmC9D"}

package/dist/commands/evaluator/delete.js

@@ -0,0 +1,34 @@
+import { requireApiKey, getSdkClient } from "../../auth.js";
+import { printInfo, printSuccess, handleSdkError } from "../../output.js";
+import { CliError } from "../../types.js";
+export function registerDeleteCommand(evaluator) {
+    evaluator
+        .command("delete <evaluatorId>")
+        .description("Delete an evaluator by its ID")
+        .option("--yes", "Skip confirmation prompt")
+        .action(async (evaluatorId, opts) => {
+        const apiKey = await requireApiKey();
+        if (!opts.yes) {
+            if (!process.stdin.isTTY && !process.stdout.isTTY) {
+                throw new CliError(1, "Refusing to prompt for confirmation in a non-interactive environment. Pass --yes to skip the prompt.");
+            }
+            const { confirm } = await import("@inquirer/prompts");
+            const ok = await confirm({
+                message: "Are you sure you want to delete this evaluator?",
+                default: false,
+            });
+            if (!ok) {
+                throw new CliError(1, "Aborted");
+            }
+        }
+        printInfo(`Deleting evaluator ${evaluatorId}...`);
+        try {
+            const client = getSdkClient(apiKey);
+            await client.evaluators.delete(evaluatorId);
+            printSuccess(`Evaluator ${evaluatorId} deleted successfully.`);
+        }
+        catch (e) {
+            handleSdkError(e);
+        }
+    });
+}

package/dist/commands/evaluator/duplicate.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerDuplicateCommand(evaluator: Command): void;
+//# sourceMappingURL=duplicate.d.ts.map

package/dist/commands/evaluator/duplicate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"duplicate.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/duplicate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIpC,wBAAgB,wBAAwB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAiBjE"}

package/dist/commands/evaluator/duplicate.js

@@ -0,0 +1,20 @@
+import { requireApiKey, getSdkClient } from "../../auth.js";
+import { printInfo, printSuccess, printJson, handleSdkError } from "../../output.js";
+export function registerDuplicateCommand(evaluator) {
+    evaluator
+        .command("duplicate <evaluatorId>")
+        .description("Duplicate an existing evaluator")
+        .action(async (evaluatorId) => {
+        const apiKey = await requireApiKey();
+        printInfo(`Duplicating evaluator ID: ${evaluatorId}...`);
+        try {
+            const client = getSdkClient(apiKey);
+            const result = await client.evaluators.duplicate(evaluatorId);
+            printSuccess(`Evaluator ${evaluatorId} duplicated successfully!`);
+            printJson(result);
+        }
+        catch (e) {
+            handleSdkError(e);
+        }
+    });
+}

package/dist/commands/evaluator/execute-by-name.d.ts

@@ -0,0 +1,17 @@
+import { Command } from "commander";
+declare function readStdinDefault(): Promise<string>;
+export declare function executeEvaluatorByName(evaluatorName: string, opts: {
+    request?: string;
+    response?: string;
+    turns?: string;
+    contexts?: string;
+    expectedOutput?: string;
+    tag?: string[];
+    userId?: string;
+    sessionId?: string;
+    systemPrompt?: string;
+    variables?: string;
+}, readStdin?: typeof readStdinDefault): Promise<void>;
+export declare function registerExecuteByNameCommand(evaluator: Command): void;
+export {};
+//# sourceMappingURL=execute-by-name.d.ts.map

package/dist/commands/evaluator/execute-by-name.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"execute-by-name.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/execute-by-name.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,iBAAe,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAIjD;AAED,wBAAsB,sBAAsB,CAC1C,aAAa,EAAE,MAAM,EACrB,IAAI,EAAE;IACJ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,EACD,SAAS,0BAAmB,GAC3B,OAAO,CAAC,IAAI,CAAC,CAuEf;AAED,wBAAgB,4BAA4B,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CA4ErE"}

package/dist/commands/evaluator/execute-by-name.js

@@ -0,0 +1,131 @@
+import { requireApiKey, getSdkClient } from "../../auth.js";
+import { printInfo, printSuccess, printError, printJson, handleSdkError } from "../../output.js";
+import { isTurnArray, isStringArray, isStringRecord } from "../../utils.js";
+async function readStdinDefault() {
+    const chunks = [];
+    for await (const chunk of process.stdin)
+        chunks.push(chunk);
+    return Buffer.concat(chunks).toString().trim();
+}
+export async function executeEvaluatorByName(evaluatorName, opts, readStdin = readStdinDefault) {
+    const apiKey = await requireApiKey();
+    let response = opts.response;
+    if (!opts.request && !response && !opts.turns && !process.stdin.isTTY) {
+        response = await readStdin();
+    }
+    if (!opts.request && !response && !opts.turns) {
+        printError("Either --request, --response, or --turns must be provided.");
+        return;
+    }
+    const payload = {};
+    if (opts.request)
+        payload.request = opts.request;
+    if (response)
+        payload.response = response;
+    if (opts.expectedOutput)
+        payload.expected_output = opts.expectedOutput;
+    if (opts.tag?.length)
+        payload.tags = opts.tag;
+    if (opts.userId)
+        payload.user_id = opts.userId;
+    if (opts.sessionId)
+        payload.session_id = opts.sessionId;
+    if (opts.systemPrompt)
+        payload.system_prompt = opts.systemPrompt;
+    if (opts.turns) {
+        try {
+            const parsed = JSON.parse(opts.turns);
+            if (!isTurnArray(parsed)) {
+                printError("Invalid JSON for --turns.");
+                return;
+            }
+            payload.turns = parsed;
+        }
+        catch {
+            printError("Invalid JSON for --turns.");
+            return;
+        }
+    }
+    if (opts.contexts) {
+        try {
+            const parsed = JSON.parse(opts.contexts);
+            if (!isStringArray(parsed)) {
+                printError("Invalid JSON for --contexts. Skipping.");
+                return;
+            }
+            payload.contexts = parsed;
+        }
+        catch {
+            printError("Invalid JSON for --contexts. Skipping.");
+            return;
+        }
+    }
+    if (opts.variables) {
+        try {
+            const parsed = JSON.parse(opts.variables);
+            if (!isStringRecord(parsed)) {
+                printError("Invalid JSON for --variables.");
+                return;
+            }
+            payload.variables = parsed;
+        }
+        catch {
+            printError("Invalid JSON for --variables.");
+            return;
+        }
+    }
+    printInfo(`Attempting to execute evaluator '${evaluatorName}' with payload:`);
+    printJson(payload);
+    const client = getSdkClient(apiKey);
+    const result = await client.evaluators.executeByName(evaluatorName, payload);
+    printSuccess("Evaluator execution by name successful!");
+    printJson(result);
+}
+export function registerExecuteByNameCommand(evaluator) {
+    evaluator
+        .command("execute-by-name <evaluatorName>")
+        .description("Execute an evaluator by name with interaction details")
+        .option("--request <text>", "Request text")
+        .option("--response <text>", "Response text to evaluate")
+        .option("--contexts <json>", "JSON list of context strings. E.g., '[\"ctx1\"]'")
+        .option("--expected-output <text>", "Expected output text")
+        .option("--tag <tag>", "Add one or more tags", (v, a) => [...a, v], [])
+        .option("--user-id <id>", "User identifier for tracking purposes")
+        .option("--session-id <id>", "Session identifier for tracking purposes")
+        .option("--system-prompt <text>", "System prompt that was used for the LLM call")
+        .option("--turns <json>", 'JSON array of conversation turns. E.g., \'[{"role":"user","content":"Hello"}]\'')
+        .option("--variables <json>", 'JSON object of extra template variables. E.g., \'{"lang":"EN"}\'')
+        .addHelpText("after", `
+Examples:
+  # Check helpfulness of a support response
+  $ scorable evaluator execute-by-name "Helpfulness" \\
+      --request "Where can I find the application instructions?" \\
+      --response "You can find the instructions from our Careers page."
+
+  # Network troubleshooting accuracy check
+  $ scorable evaluator execute-by-name "Network Troubleshooting" \\
+      --request "My internet is not working." \\
+      --response "Let's check the basics. Ensure the Ethernet cable is securely connected, then run \`ping 8.8.8.8\` and share the results."
+
+  # Faithfulness against retrieved context
+  $ scorable evaluator execute-by-name "Faithfulness" \\
+      --request "What is your return policy for electronics?" \\
+      --response "You can return electronics within 30 days, unused and in original packaging." \\
+      --contexts '["Returns for electronics: 30 days, unused, original packaging, valid receipt required."]'
+
+  # Multi-turn conversation
+  $ scorable evaluator execute-by-name "Helpfulness" \\
+      --turns '[{"role":"user","content":"Hello, I need help with my order"},{"role":"assistant","content":"I\\'d be happy to help! What\\'s your order number?"},{"role":"user","content":"It\\'s ORDER-12345"}]'
+
+  # Pipe a response from your app via stdin
+  $ ./my-app respond "Do you offer refunds?" | \\
+      scorable evaluator execute-by-name "Helpfulness" --request "Do you offer refunds?"`)
+        .action(async (evaluatorName, opts) => {
+        try {
+            await executeEvaluatorByName(evaluatorName, opts);
+        }
+        catch (e) {
+            handleSdkError(e);
+        }
+    });
+}

package/dist/commands/evaluator/execute.d.ts

@@ -0,0 +1,17 @@
+import { Command } from "commander";
+declare function readStdinDefault(): Promise<string>;
+export declare function executeEvaluator(evaluatorId: string, opts: {
+    request?: string;
+    response?: string;
+    turns?: string;
+    contexts?: string;
+    expectedOutput?: string;
+    tag?: string[];
+    userId?: string;
+    sessionId?: string;
+    systemPrompt?: string;
+    variables?: string;
+}, readStdin?: typeof readStdinDefault): Promise<void>;
+export declare function registerExecuteCommand(evaluator: Command): void;
+export {};
+//# sourceMappingURL=execute.d.ts.map

package/dist/commands/evaluator/execute.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"execute.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/execute.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,iBAAe,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAIjD;AAED,wBAAsB,gBAAgB,CACpC,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE;IACJ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,EACD,SAAS,0BAAmB,GAC3B,OAAO,CAAC,IAAI,CAAC,CAuEf;AAED,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAgF/D"}

package/dist/commands/evaluator/execute.js

@@ -0,0 +1,135 @@
+import { requireApiKey, getSdkClient } from "../../auth.js";
+import { printInfo, printSuccess, printError, printJson, handleSdkError } from "../../output.js";
+import { isTurnArray, isStringArray, isStringRecord } from "../../utils.js";
+async function readStdinDefault() {
+    const chunks = [];
+    for await (const chunk of process.stdin)
+        chunks.push(chunk);
+    return Buffer.concat(chunks).toString().trim();
+}
+export async function executeEvaluator(evaluatorId, opts, readStdin = readStdinDefault) {
+    const apiKey = await requireApiKey();
+    let response = opts.response;
+    if (!opts.request && !response && !opts.turns && !process.stdin.isTTY) {
+        response = await readStdin();
+    }
+    if (!opts.request && !response && !opts.turns) {
+        printError("Either --request, --response, or --turns must be provided.");
+        return;
+    }
+    const payload = {};
+    if (opts.request)
+        payload.request = opts.request;
+    if (response)
+        payload.response = response;
+    if (opts.expectedOutput)
+        payload.expected_output = opts.expectedOutput;
+    if (opts.tag?.length)
+        payload.tags = opts.tag;
+    if (opts.userId)
+        payload.user_id = opts.userId;
+    if (opts.sessionId)
+        payload.session_id = opts.sessionId;
+    if (opts.systemPrompt)
+        payload.system_prompt = opts.systemPrompt;
+    if (opts.turns) {
+        try {
+            const parsed = JSON.parse(opts.turns);
+            if (!isTurnArray(parsed)) {
+                printError("Invalid JSON for --turns.");
+                return;
+            }
+            payload.turns = parsed;
+        }
+        catch {
+            printError("Invalid JSON for --turns.");
+            return;
+        }
+    }
+    if (opts.contexts) {
+        try {
+            const parsed = JSON.parse(opts.contexts);
+            if (!isStringArray(parsed)) {
+                printError("Invalid JSON for --contexts. Skipping.");
+                return;
+            }
+            payload.contexts = parsed;
+        }
+        catch {
+            printError("Invalid JSON for --contexts. Skipping.");
+            return;
+        }
+    }
+    if (opts.variables) {
+        try {
+            const parsed = JSON.parse(opts.variables);
+            if (!isStringRecord(parsed)) {
+                printError("Invalid JSON for --variables.");
+                return;
+            }
+            payload.variables = parsed;
+        }
+        catch {
+            printError("Invalid JSON for --variables.");
+            return;
+        }
+    }
+    printInfo(`Attempting to execute evaluator ${evaluatorId} with payload:`);
+    printJson(payload);
+    const client = getSdkClient(apiKey);
+    const result = await client.evaluators.execute(evaluatorId, payload);
+    printSuccess("Evaluator execution successful!");
+    printJson(result);
+}
+export function registerExecuteCommand(evaluator) {
+    evaluator
+        .command("execute <evaluatorId>")
+        .description("Execute an evaluator with interaction details")
+        .option("--request <text>", "Request text")
+        .option("--response <text>", "Response text to evaluate")
+        .option("--contexts <json>", "JSON list of context strings. E.g., '[\"ctx1\"]'")
+        .option("--expected-output <text>", "Expected output text")
+        .option("--tag <tag>", "Add one or more tags", (v, a) => [...a, v], [])
+        .option("--user-id <id>", "User identifier for tracking purposes")
+        .option("--session-id <id>", "Session identifier for tracking purposes")
+        .option("--system-prompt <text>", "System prompt that was used for the LLM call")
+        .option("--turns <json>", 'JSON array of conversation turns. E.g., \'[{"role":"user","content":"Hello"}]\'')
+        .option("--variables <json>", 'JSON object of extra template variables. E.g., \'{"lang":"EN"}\'')
+        .addHelpText("after", `
+Examples:
+  # Check helpfulness of a support response
+  $ scorable evaluator execute <evaluatorId> \\
+      --request "Where can I find the application instructions?" \\
+      --response "You can find the instructions from our Careers page."
+
+  # Evaluate network troubleshooting response
+  $ scorable evaluator execute <evaluatorId> \\
+      --request "My internet is not working." \\
+      --response "Let's check the basics. Ensure the Ethernet cable is securely connected, then run \`ping 8.8.8.8\` in a terminal and share the results."
+
+  # RAG faithfulness check with context documents
+  $ scorable evaluator execute <evaluatorId> \\
+      --request "Was the number of pensioners working above 100k in 2023?" \\
+      --response "Yes, 150,000 pensioners were working in 2023." \\
+      --expected-output "In 2023, 150k pensioners were still working." \\
+      --contexts '["More than 150,000 pensioners were employed in 2023, the centre\\'s statistics reveal."]'
+
+  # Multi-turn conversation evaluation
+  $ scorable evaluator execute <evaluatorId> \\
+      --turns '[{"role":"user","content":"Hello, I need help with my order"},{"role":"assistant","content":"I\\'d be happy to help! What\\'s your order number?"},{"role":"user","content":"It\\'s ORDER-12345"},{"role":"assistant","content":"I found your order. It\\'s currently in transit."}]'
+
+  # With custom template variables and tracking
+  $ scorable evaluator execute <evaluatorId> \\
+      --request "How do I cancel my subscription?" \\
+      --response "You can cancel anytime from account settings under billing." \\
+      --variables '{"topic":"subscription"}' \\
+      --user-id user_123 --session-id session_abc --tag production`)
+        .action(async (evaluatorId, opts) => {
+        try {
+            await executeEvaluator(evaluatorId, opts);
+        }
+        catch (e) {
+            handleSdkError(e);
+        }
+    });
+}

package/dist/commands/evaluator/get.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerGetCommand(evaluator: Command): void;
+//# sourceMappingURL=get.d.ts.map

package/dist/commands/evaluator/get.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"get.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/get.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIpC,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAiB3D"}

package/dist/commands/evaluator/get.js

@@ -0,0 +1,20 @@
+import { requireApiKey, getSdkClient } from "../../auth.js";
+import { printInfo, printSuccess, printJson, handleSdkError } from "../../output.js";
+export function registerGetCommand(evaluator) {
+    evaluator
+        .command("get <evaluatorId>")
+        .description("Get a specific evaluator by its ID")
+        .action(async (evaluatorId) => {
+        const apiKey = await requireApiKey();
+        printInfo(`Fetching evaluator with ID: ${evaluatorId}...`);
+        try {
+            const client = getSdkClient(apiKey);
+            const result = await client.evaluators.get(evaluatorId);
+            printSuccess(`Evaluator '${result.name}' details:`);
+            printJson(result);
+        }
+        catch (e) {
+            handleSdkError(e);
+        }
+    });
+}

package/dist/commands/evaluator/index.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerEvaluatorCommands(program: Command): void;
+//# sourceMappingURL=index.d.ts.map

package/dist/commands/evaluator/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAUpC,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAahE"}