@root-signals/scorable-cli 0.12.0 → 0.14.0

package/dist/commands/judge/create.js

@@ -1,6 +1,10 @@
 import ora from "ora";
+import { z } from "zod";
 import { requireApiKey, getSdkClient } from "../../auth.js";
 import { printSuccess, printError, printJson, handleSdkError } from "../../output.js";
+import { parseJsonArg } from "../../utils.js";
+import { resolveProjectIdValue, PROJECT_ID_FLAG_DESC } from "../../lib/project-id.js";
+const EvaluatorRefsSchema = z.array(z.object({ id: z.string() }).passthrough());
 export function registerCreateCommand(judge) {
     judge
         .command("create")
@@ -9,20 +13,23 @@ export function registerCreateCommand(judge) {
         .requiredOption("--intent <intent>", "The intent for the new judge")
         .option("--stage <stage>", "The stage for the new judge")
         .option("--evaluator-references <json>", 'JSON string for evaluator references. E.g., \'[{"id": "eval-id"}]\'')
+        .option("--project-id <uuid>", PROJECT_ID_FLAG_DESC)
         .action(async (opts) => {
         const apiKey = await requireApiKey();
         const payload = { name: opts.name, intent: opts.intent };
         if (opts.stage)
             payload.stage = opts.stage;
         if (opts.evaluatorReferences) {
-            try {
-                payload.evaluator_references = JSON.parse(opts.evaluatorReferences);
-            }
-            catch {
-                printError("Invalid JSON format for --evaluator-references.");
+            const r = parseJsonArg(opts.evaluatorReferences, EvaluatorRefsSchema);
+            if (!r.ok) {
+                printError('Invalid JSON format for --evaluator-references. Expected an array of objects with an "id" string.');
                 return;
             }
+            payload.evaluator_references = r.value;
         }
+        const projectId = resolveProjectIdValue(opts.projectId);
+        if (projectId !== undefined)
+            payload.projectId = projectId;
         const spinner = ora("Creating...").start();
         try {
             const client = getSdkClient(apiKey);

package/dist/commands/judge/duplicate.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"duplicate.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/duplicate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAkB7D"}
+{"version":3,"file":"duplicate.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/duplicate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAuB7D"}

package/dist/commands/judge/duplicate.js

@@ -1,16 +1,19 @@
 import ora from "ora";
 import { requireApiKey, getSdkClient } from "../../auth.js";
 import { printSuccess, printJson, handleSdkError } from "../../output.js";
+import { resolveProjectIdValue, PROJECT_ID_FLAG_DESC } from "../../lib/project-id.js";
 export function registerDuplicateCommand(judge) {
     judge
         .command("duplicate <judgeId>")
         .description("Duplicate an existing judge")
-        .action(async (judgeId) => {
+        .option("--project-id <uuid>", PROJECT_ID_FLAG_DESC)
+        .action(async (judgeId, opts) => {
         const apiKey = await requireApiKey();
         const spinner = ora("Duplicating...").start();
         try {
             const client = getSdkClient(apiKey);
-            const result = await client.judges.duplicate(judgeId);
+            const projectId = resolveProjectIdValue(opts.projectId);
+            const result = await client.judges.duplicate(judgeId, projectId !== undefined ? { projectId } : {});
             spinner.stop();
             printSuccess(`Judge ${judgeId} duplicated successfully!`);
             printJson(result);

package/dist/commands/judge/exec-openai-generic.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"exec-openai-generic.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/exec-openai-generic.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAYpC,wBAAgB,gCAAgC,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CA6CrE"}
+{"version":3,"file":"exec-openai-generic.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/exec-openai-generic.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAkBpC,wBAAgB,gCAAgC,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAwDrE"}

package/dist/commands/judge/exec-openai-generic.js

@@ -1,6 +1,11 @@
+import { z } from "zod";
 import { requireApiKey } from "../../auth.js";
 import { printInfo, printSuccess, printError, printWarning, printJson, handleSdkError, } from "../../output.js";
+import { parseJsonArg } from "../../utils.js";
+import { resolveProjectIdValue, PROJECT_ID_FLAG_DESC } from "../../lib/project-id.js";
 import { apiRequest } from "../../client.js";
+const MessagesSchema = z.array(z.object({ role: z.string() }).passthrough());
+const ExtraBodySchema = z.record(z.string(), z.unknown());
 export function registerExecOpenaiGenericCommand(judge) {
     judge
         .command("exec-openai-generic")
@@ -8,32 +13,37 @@ export function registerExecOpenaiGenericCommand(judge) {
         .requiredOption("--model <model>", "Judge ID (or name) to use as the 'model' field")
         .requiredOption("--messages <json>", "JSON string of the messages payload")
         .option("--extra-body <json>", "Optional JSON string for extra_body parameters")
+        .option("--project-id <uuid>", PROJECT_ID_FLAG_DESC)
         .action(async (opts) => {
         const apiKey = await requireApiKey();
-        let messages;
-        try {
-            messages = JSON.parse(opts.messages);
-        }
-        catch {
-            printError("Invalid JSON for --messages. Aborting.");
+        const messagesResult = parseJsonArg(opts.messages, MessagesSchema);
+        if (!messagesResult.ok) {
+            printError("Invalid JSON for --messages. Expected an array of objects each with a string `role`. Aborting.");
             return;
         }
-        const payload = { model: opts.model, messages };
+        const payload = {
+            model: opts.model,
+            messages: messagesResult.value,
+        };
         if (opts.extraBody) {
-            try {
-                payload["extra_body"] = JSON.parse(opts.extraBody);
+            const extraResult = parseJsonArg(opts.extraBody, ExtraBodySchema);
+            if (extraResult.ok) {
+                payload["extra_body"] = extraResult.value;
             }
-            catch {
-                printWarning("Invalid JSON for --extra-body. Skipping.");
+            else {
+                printWarning("Invalid JSON for --extra-body (expected an object). Skipping.");
             }
         }
         printInfo(`Executing a Judge using generic OpenAI endpoint. Judge ID/Name: ${opts.model}`);
         printInfo("Attempting to execute with OpenAI compatible payload:");
         printJson(payload);
+        const projectId = resolveProjectIdValue(opts.projectId);
+        const headers = projectId !== undefined ? { "X-Project-Id": projectId } : undefined;
         try {
             const result = await apiRequest("POST", "judges/openai/chat/completions", {
                 payload,
                 apiKey,
+                headers,
             });
             if (result) {
                 printSuccess("OpenAI compatible execution successful!");

package/dist/commands/judge/exec-openai.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"exec-openai.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/exec-openai.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAYpC,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAwD9D"}
+{"version":3,"file":"exec-openai.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/exec-openai.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAkBpC,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAiE9D"}

package/dist/commands/judge/exec-openai.js

@@ -1,6 +1,11 @@
+import { z } from "zod";
 import { requireApiKey } from "../../auth.js";
 import { printInfo, printSuccess, printError, printWarning, printJson, handleSdkError, } from "../../output.js";
+import { parseJsonArg } from "../../utils.js";
+import { resolveProjectIdValue, PROJECT_ID_FLAG_DESC } from "../../lib/project-id.js";
 import { apiRequest } from "../../client.js";
+const MessagesSchema = z.array(z.object({ role: z.string() }).passthrough());
+const ExtraBodySchema = z.record(z.string(), z.unknown());
 export function registerExecOpenaiCommand(judge) {
     judge
         .command("exec-openai <judgeIdInPath>")
@@ -8,32 +13,37 @@ export function registerExecOpenaiCommand(judge) {
         .requiredOption("--model <model>", "LLM model for judge execution (e.g., gpt-5.5)")
         .requiredOption("--messages <json>", "JSON string of the messages payload")
         .option("--extra-body <json>", "Optional JSON string for extra_body parameters")
+        .option("--project-id <uuid>", PROJECT_ID_FLAG_DESC)
         .action(async (judgeIdInPath, opts) => {
         const apiKey = await requireApiKey();
-        let messages;
-        try {
-            messages = JSON.parse(opts.messages);
-        }
-        catch {
-            printError("Invalid JSON for --messages. Aborting.");
+        const messagesResult = parseJsonArg(opts.messages, MessagesSchema);
+        if (!messagesResult.ok) {
+            printError("Invalid JSON for --messages. Expected an array of objects each with a string `role`. Aborting.");
             return;
         }
-        const payload = { model: opts.model, messages };
+        const payload = {
+            model: opts.model,
+            messages: messagesResult.value,
+        };
         if (opts.extraBody) {
-            try {
-                payload["extra_body"] = JSON.parse(opts.extraBody);
+            const extraResult = parseJsonArg(opts.extraBody, ExtraBodySchema);
+            if (extraResult.ok) {
+                payload["extra_body"] = extraResult.value;
             }
-            catch {
-                printWarning("Invalid JSON for --extra-body. Skipping.");
+            else {
+                printWarning("Invalid JSON for --extra-body (expected an object). Skipping.");
             }
         }
         printInfo(`Executing Judge ID (via path): ${judgeIdInPath} using OpenAI chat completions format.`);
         printInfo("Attempting to execute with OpenAI compatible payload:");
         printJson(payload);
+        const projectId = resolveProjectIdValue(opts.projectId);
+        const headers = projectId !== undefined ? { "X-Project-Id": projectId } : undefined;
         try {
             const result = await apiRequest("POST", `judges/${judgeIdInPath}/openai/chat/completions`, {
                 payload,
                 apiKey,
+                headers,
             });
             if (result) {
                 printSuccess("OpenAI compatible execution successful!");

package/dist/commands/judge/execute-by-name.d.ts

@@ -4,12 +4,14 @@ export declare function executeJudgeByName(judgeName: string, opts: {
     request?: string;
     response?: string;
     turns?: string;
+    tools?: string;
     contexts?: string;
     expectedOutput?: string;
     tag?: string[];
     userId?: string;
     sessionId?: string;
     systemPrompt?: string;
+    projectId?: string;
 }, readStdin?: typeof readStdinDefault): Promise<void>;
 export declare function registerExecuteByNameCommand(judge: Command): void;
 export {};

package/dist/commands/judge/execute-by-name.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"execute-by-name.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/execute-by-name.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,iBAAe,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAIjD;AAED,wBAAsB,kBAAkB,CACtC,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE;IACJ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,EACD,SAAS,0BAAmB,GAC3B,OAAO,CAAC,IAAI,CAAC,CA6Df;AAED,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAkEjE"}
+{"version":3,"file":"execute-by-name.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/execute-by-name.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAQpC,iBAAe,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAIjD;AAED,wBAAsB,kBAAkB,CACtC,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE;IACJ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,EACD,SAAS,0BAAmB,GAC3B,OAAO,CAAC,IAAI,CAAC,CA8Ef;AAED,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CA2EjE"}

package/dist/commands/judge/execute-by-name.js

@@ -1,7 +1,8 @@
 import ora from "ora";
 import { requireApiKey, getSdkClient } from "../../auth.js";
 import { printSuccess, printError, printJson, handleSdkError } from "../../output.js";
-import { isTurnArray, isStringArray } from "../../utils.js";
+import { isTurnArray, isToolCatalog, isStringArray } from "../../utils.js";
+import { resolveProjectIdValue, PROJECT_ID_FLAG_DESC } from "../../lib/project-id.js";
 async function readStdinDefault() {
     const chunks = [];
     for await (const chunk of process.stdin)
@@ -47,6 +48,20 @@ export async function executeJudgeByName(judgeName, opts, readStdin = readStdinD
             return;
         }
     }
+    if (opts.tools) {
+        try {
+            const parsed = JSON.parse(opts.tools);
+            if (!isToolCatalog(parsed)) {
+                printError("Invalid JSON for --tools. Expected array of tool objects.");
+                return;
+            }
+            payload.tools = parsed;
+        }
+        catch {
+            printError("Invalid JSON for --tools.");
+            return;
+        }
+    }
     if (opts.contexts) {
         try {
             const parsed = JSON.parse(opts.contexts);
@@ -61,6 +76,9 @@ export async function executeJudgeByName(judgeName, opts, readStdin = readStdinD
             return;
         }
     }
+    const projectId = resolveProjectIdValue(opts.projectId);
+    if (projectId !== undefined)
+        payload.projectId = projectId;
     const spinner = ora("Running judge...").start();
     try {
         const client = getSdkClient(apiKey);
@@ -86,7 +104,9 @@ export function registerExecuteByNameCommand(judge) {
         .option("--user-id <id>", "User identifier for tracking purposes")
         .option("--session-id <id>", "Session identifier for tracking purposes")
         .option("--system-prompt <text>", "System prompt that was used for the LLM call")
-        .option("--turns <json>", 'JSON array of conversation turns. E.g., \'[{"role":"user","content":"Hello"}]\'')
+        .option("--turns <json>", 'JSON array of conversation turns. Roles: user|assistant|tool. Assistant turns may carry `tool_calls`; tool results use a `tool` role with `tool_call_id`. E.g., \'[{"role":"user","content":"Hello"}]\'')
+        .option("--tools <json>", "JSON array of OpenAI-style tool definitions available to the agent.")
+        .option("--project-id <uuid>", PROJECT_ID_FLAG_DESC)
         .addHelpText("after", `
 Examples:
   # Evaluate a customer service response
@@ -104,6 +124,11 @@ Examples:
   $ scorable judge execute-by-name "Customer Service Judge" \\
       --turns '[{"role":"user","content":"I haven\\'t received my refund and it\\'s been 10 days."},{"role":"assistant","content":"Thank you for contacting us. I will investigate this immediately and provide an update within 24 hours."}]'
 
+  # Tool-aware evaluation: assistant tool_calls + tool-role result + tool catalog
+  $ scorable judge execute-by-name "Agent Quality Judge" \\
+      --turns '[{"role":"user","content":"What is the weather in Paris?"},{"role":"assistant","content":null,"tool_calls":[{"id":"call_1","type":"function","function":{"name":"get_weather","arguments":"{\\"city\\":\\"Paris\\"}"}}]},{"role":"tool","tool_call_id":"call_1","content":"{\\"temp\\":12}"},{"role":"assistant","content":"It is 12C in Paris."}]' \\
+      --tools '[{"type":"function","function":{"name":"get_weather","description":"Get weather for a city","parameters":{"type":"object","properties":{"city":{"type":"string"}}}}}]'
+
   # Pipe the LLM response from stdin
   $ echo "We offer a 30-day money-back guarantee on all purchases." | \\
       scorable judge execute-by-name "Custom Returns Policy Judge" --request "Do you offer refunds?"`)

package/dist/commands/judge/execute.d.ts

@@ -4,12 +4,14 @@ export declare function executeJudge(judgeId: string, opts: {
     request?: string;
     response?: string;
     turns?: string;
+    tools?: string;
     contexts?: string;
     expectedOutput?: string;
     tag?: string[];
     userId?: string;
     sessionId?: string;
     systemPrompt?: string;
+    projectId?: string;
 }, readStdin?: typeof readStdinDefault): Promise<void>;
 export declare function registerExecuteCommand(judge: Command): void;
 export {};

package/dist/commands/judge/execute.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"execute.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/execute.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,iBAAe,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAIjD;AAED,wBAAsB,YAAY,CAChC,OAAO,EAAE,MAAM,EACf,IAAI,EAAE;IACJ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,EACD,SAAS,0BAAmB,GAC3B,OAAO,CAAC,IAAI,CAAC,CA6Df;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAyE3D"}
+{"version":3,"file":"execute.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/execute.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAQpC,iBAAe,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAIjD;AAED,wBAAsB,YAAY,CAChC,OAAO,EAAE,MAAM,EACf,IAAI,EAAE;IACJ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,EACD,SAAS,0BAAmB,GAC3B,OAAO,CAAC,IAAI,CAAC,CA8Ef;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAkF3D"}

package/dist/commands/judge/execute.js

@@ -1,7 +1,8 @@
 import ora from "ora";
 import { requireApiKey, getSdkClient } from "../../auth.js";
 import { printSuccess, printError, printJson, handleSdkError } from "../../output.js";
-import { isTurnArray, isStringArray } from "../../utils.js";
+import { isTurnArray, isToolCatalog, isStringArray } from "../../utils.js";
+import { resolveProjectIdValue, PROJECT_ID_FLAG_DESC } from "../../lib/project-id.js";
 async function readStdinDefault() {
     const chunks = [];
     for await (const chunk of process.stdin)
@@ -47,6 +48,20 @@ export async function executeJudge(judgeId, opts, readStdin = readStdinDefault)
             return;
         }
     }
+    if (opts.tools) {
+        try {
+            const parsed = JSON.parse(opts.tools);
+            if (!isToolCatalog(parsed)) {
+                printError("Invalid JSON for --tools. Expected array of tool objects.");
+                return;
+            }
+            payload.tools = parsed;
+        }
+        catch {
+            printError("Invalid JSON for --tools.");
+            return;
+        }
+    }
     if (opts.contexts) {
         try {
             const parsed = JSON.parse(opts.contexts);
@@ -61,6 +76,9 @@ export async function executeJudge(judgeId, opts, readStdin = readStdinDefault)
             return;
         }
     }
+    const projectId = resolveProjectIdValue(opts.projectId);
+    if (projectId !== undefined)
+        payload.projectId = projectId;
     const spinner = ora("Running judge...").start();
     try {
         const client = getSdkClient(apiKey);
@@ -86,7 +104,9 @@ export function registerExecuteCommand(judge) {
         .option("--user-id <id>", "User identifier for tracking purposes")
         .option("--session-id <id>", "Session identifier for tracking purposes")
         .option("--system-prompt <text>", "System prompt that was used for the LLM call")
-        .option("--turns <json>", 'JSON array of conversation turns. E.g., \'[{"role":"user","content":"Hello"}]\'')
+        .option("--turns <json>", 'JSON array of conversation turns. Roles: user|assistant|tool. Assistant turns may carry `tool_calls`; tool results use a `tool` role with `tool_call_id`. E.g., \'[{"role":"user","content":"Hello"}]\'')
+        .option("--tools <json>", "JSON array of OpenAI-style tool definitions available to the agent.")
+        .option("--project-id <uuid>", PROJECT_ID_FLAG_DESC)
         .addHelpText("after", `
 Examples:
   # Evaluate a customer service response about return policy
@@ -104,6 +124,11 @@ Examples:
   $ scorable judge execute <judgeId> \\
       --turns '[{"role":"user","content":"Hello, I need help with my order"},{"role":"assistant","content":"I\\'d be happy to help! What\\'s your order number?"},{"role":"user","content":"It\\'s ORDER-12345"},{"role":"assistant","content":"I found your order. It\\'s currently in transit."}]'
 
+  # Tool-aware evaluation: assistant tool_calls, tool-role result, and tool catalog
+  $ scorable judge execute <judgeId> \\
+      --turns '[{"role":"user","content":"What is the weather in Paris?"},{"role":"assistant","content":null,"tool_calls":[{"id":"call_1","type":"function","function":{"name":"get_weather","arguments":"{\\"city\\":\\"Paris\\"}"}}]},{"role":"tool","tool_call_id":"call_1","content":"{\\"temp\\":12}"},{"role":"assistant","content":"It is 12C in Paris."}]' \\
+      --tools '[{"type":"function","function":{"name":"get_weather","description":"Get weather for a city","parameters":{"type":"object","properties":{"city":{"type":"string"}}}}}]'
+
   # With tracking metadata and tags
   $ scorable judge execute <judgeId> \\
       --request "How do I reset my password?" \\

package/dist/commands/judge/generate.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"generate.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/generate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAepC,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CA8I5D"}
+{"version":3,"file":"generate.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/generate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAkBpC,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAkJ5D"}

package/dist/commands/judge/generate.js

@@ -1,7 +1,10 @@
 import ora from "ora";
 import chalk from "chalk";
+import { z } from "zod";
 import { requireApiKey, getSdkClient } from "../../auth.js";
 import { printSuccess, printError, printWarning, printInfo, printJson, handleSdkError, } from "../../output.js";
+import { parseJsonArg } from "../../utils.js";
+import { resolveProjectIdValue, PROJECT_ID_FLAG_DESC } from "../../lib/project-id.js";
 import { CliError } from "../../types.js";
 import { uploadFile } from "../file/upload.js";
 export function registerGenerateCommand(judge) {
@@ -26,6 +29,7 @@ offer to connect the guest with a human agent when uncertain."`)
         .option("--context-aware", "Enable context-aware evaluators for RAG applications (hallucination detection, context drift, etc.)", false)
         .option("--file <path>", "Path to a file (PDF, PNG, JPG) to upload and attach as context for the judge")
         .option("--file-id <id>", "ID of an already-uploaded file to use as context for the judge")
+        .option("--project-id <uuid>", PROJECT_ID_FLAG_DESC)
         .action(async (opts) => {
         const acceptedVisibilities = ["private", "public"];
         const visibility = opts.visibility;
@@ -40,13 +44,12 @@ offer to connect the guest with a human agent when uncertain."`)
         const apiKey = await requireApiKey();
         let extra_contexts;
         if (opts.extraContexts) {
-            try {
-                extra_contexts = JSON.parse(opts.extraContexts);
-            }
-            catch {
+            const r = parseJsonArg(opts.extraContexts, z.record(z.string(), z.string()));
+            if (!r.ok) {
                 printError("Invalid --extra-contexts JSON: must be a key-value object.");
                 throw new CliError(1, "Invalid extra-contexts JSON");
             }
+            extra_contexts = r.value;
         }
         const spinner = ora("Generating judge (this may take a moment)...").start();
         try {
@@ -56,6 +59,7 @@ offer to connect the guest with a human agent when uncertain."`)
                 fileId = uploaded.id;
             }
             const client = getSdkClient(apiKey);
+            const projectId = resolveProjectIdValue(opts.projectId);
             const result = (await client.judges.generate({
                 intent: opts.intent,
                 visibility: visibility,
@@ -66,6 +70,7 @@ offer to connect the guest with a human agent when uncertain."`)
                 file_id: fileId,
                 extra_contexts: extra_contexts ?? null,
                 enable_context_aware_evaluators: opts.contextAware || undefined,
+                ...(projectId !== undefined ? { projectId } : {}),
                 ...(opts.reasoningEffort && {
                     generating_model_params: {
                         reasoning_effort: opts.reasoningEffort,

package/dist/commands/judge/list.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"list.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/list.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CA4CxD"}
+{"version":3,"file":"list.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/list.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAgDxD"}

package/dist/commands/judge/list.js

@@ -1,6 +1,7 @@
 import ora from "ora";
 import { requireApiKey, getSdkClient } from "../../auth.js";
 import { printMessage, printJudgeTable, handleSdkError } from "../../output.js";
+import { resolveProjectIdValue, PROJECT_ID_FLAG_DESC } from "../../lib/project-id.js";
 export function registerListCommand(judge) {
     judge
         .command("list")
@@ -10,6 +11,7 @@ export function registerListCommand(judge) {
         .option("--search <term>", "A search term to filter by")
         .option("--name <name>", "Filter by exact judge name")
         .option("--ordering <field>", "Which field to use for ordering the results")
+        .option("--project-id <uuid>", PROJECT_ID_FLAG_DESC)
         .action(async (opts) => {
         const apiKey = await requireApiKey();
         const params = {};
@@ -23,6 +25,9 @@ export function registerListCommand(judge) {
             params.name = opts.name;
         if (opts.ordering !== undefined)
             params.ordering = opts.ordering;
+        const projectId = resolveProjectIdValue(opts.projectId);
+        if (projectId !== undefined)
+            params.projectId = projectId;
         const spinner = ora("Fetching...").start();
         try {
             const client = getSdkClient(apiKey);

package/dist/commands/judge/update.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"update.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/update.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAkD1D"}
+{"version":3,"file":"update.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/update.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAWpC,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CA8D1D"}

package/dist/commands/judge/update.js

@@ -1,6 +1,10 @@
 import ora from "ora";
+import { z } from "zod";
 import { requireApiKey, getSdkClient } from "../../auth.js";
 import { printInfo, printSuccess, printError, printJson, handleSdkError } from "../../output.js";
+import { parseJsonArg } from "../../utils.js";
+import { resolveProjectIdValue, PROJECT_ID_FLAG_DESC } from "../../lib/project-id.js";
+const EvaluatorRefsSchema = z.array(z.object({ id: z.string() }).passthrough());
 export function registerUpdateCommand(judge) {
     judge
         .command("update <judgeId>")
@@ -8,6 +12,7 @@ export function registerUpdateCommand(judge) {
         .option("--name <name>", "The new name for the judge")
         .option("--stage <stage>", "The new stage for the judge")
         .option("--evaluator-references <json>", 'JSON string to update evaluator references. Use "[]" to clear.')
+        .option("--project-id <uuid>", "Move the judge to this project (within the same organization). " + PROJECT_ID_FLAG_DESC)
         .action(async (judgeId, opts) => {
         const apiKey = await requireApiKey();
         const payload = {};
@@ -16,14 +21,16 @@ export function registerUpdateCommand(judge) {
         if (opts.stage !== undefined)
             payload.stage = opts.stage;
         if (opts.evaluatorReferences !== undefined) {
-            try {
-                payload.evaluator_references = JSON.parse(opts.evaluatorReferences);
-            }
-            catch {
-                printError("Invalid JSON format for --evaluator-references.");
+            const r = parseJsonArg(opts.evaluatorReferences, EvaluatorRefsSchema);
+            if (!r.ok) {
+                printError('Invalid JSON format for --evaluator-references. Expected an array of objects with an "id" string.');
                 return;
             }
+            payload.evaluator_references = r.value;
         }
+        const projectId = resolveProjectIdValue(opts.projectId);
+        if (projectId !== undefined)
+            payload.projectId = projectId;
         if (Object.keys(payload).length === 0) {
             printInfo("No update parameters provided. Aborting.");
             return;

package/dist/commands/otel-filter/create.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"create.d.ts","sourceRoot":"","sources":["../../../src/commands/otel-filter/create.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAkBpC,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,OAAO,GAAG,IAAI,CA6J/D"}
+{"version":3,"file":"create.d.ts","sourceRoot":"","sources":["../../../src/commands/otel-filter/create.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAmBpC,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,OAAO,GAAG,IAAI,CA+J/D"}

package/dist/commands/otel-filter/create.js

@@ -3,7 +3,8 @@ import ora from "ora";
 import { requireApiKey } from "../../auth.js";
 import { apiRequest } from "../../client.js";
 import { printSuccess, printError, printJson, handleSdkError } from "../../output.js";
-import { loadFilterYaml } from "../../lib/filter-yaml.js";
+import { loadFilterYaml, Match } from "../../lib/filter-yaml.js";
+import { parseJsonArg } from "../../utils.js";
 export function registerCreateCommand(otelFilter) {
     otelFilter
         .command("create")
@@ -71,13 +72,12 @@ YAML manifest (only needed for custom extractor_rules):
         }
         let filterCriteria;
         if (opts.filterCriteria) {
-            try {
-                filterCriteria = JSON.parse(opts.filterCriteria);
-            }
-            catch {
-                printError("Invalid JSON for --filter-criteria.");
+            const r = parseJsonArg(opts.filterCriteria, Match);
+            if (!r.ok) {
+                printError('Invalid JSON for --filter-criteria. Expected {"conditions":[{column,operator,value?,key?},...]}.');
                 process.exit(1);
             }
+            filterCriteria = r.value;
         }
         let samplingRate;
         if (opts.samplingRate !== undefined) {

package/dist/commands/project/create.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerCreateCommand(project: Command): void;
+//# sourceMappingURL=create.d.ts.map

package/dist/commands/project/create.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"create.d.ts","sourceRoot":"","sources":["../../../src/commands/project/create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CA8B5D"}

package/dist/commands/project/create.js

@@ -0,0 +1,31 @@
+import ora from "ora";
+import { requireApiKey, getSdkClient } from "../../auth.js";
+import { printSuccess, printJson, handleSdkError } from "../../output.js";
+export function registerCreateCommand(project) {
+    project
+        .command("create")
+        .description("Create a new project")
+        .requiredOption("--name <name>", "Project name")
+        .option("--description <text>", "Project description")
+        .option("--is-default", "Mark this project as the org default (atomically clears any previous default).", false)
+        .action(async (opts) => {
+        const apiKey = await requireApiKey();
+        const payload = { name: opts.name };
+        if (opts.description !== undefined)
+            payload.description = opts.description;
+        if (opts.isDefault)
+            payload.is_default = true;
+        const spinner = ora("Creating...").start();
+        try {
+            const client = getSdkClient(apiKey);
+            const result = await client.projects.create(payload);
+            spinner.stop();
+            printSuccess("Project created successfully!");
+            printJson(result);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+}

package/dist/commands/project/delete.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerDeleteCommand(project: Command): void;
+//# sourceMappingURL=delete.d.ts.map

package/dist/commands/project/delete.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"delete.d.ts","sourceRoot":"","sources":["../../../src/commands/project/delete.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAoC5D"}

package/dist/commands/project/delete.js

@@ -0,0 +1,37 @@
+import ora from "ora";
+import { requireApiKey, getSdkClient } from "../../auth.js";
+import { printSuccess, handleSdkError } from "../../output.js";
+import { CliError } from "../../types.js";
+export function registerDeleteCommand(project) {
+    project
+        .command("delete <projectId>")
+        .description("Delete a project by its ID")
+        .option("--yes", "Skip confirmation prompt")
+        .action(async (projectId, opts) => {
+        const apiKey = await requireApiKey();
+        if (!opts.yes) {
+            if (!process.stdin.isTTY || !process.stdout.isTTY) {
+                throw new CliError(1, "Refusing to prompt for confirmation in a non-interactive environment. Pass --yes to skip the prompt.");
+            }
+            const { confirm } = await import("@inquirer/prompts");
+            const ok = await confirm({
+                message: "Are you sure you want to delete this project?",
+                default: false,
+            });
+            if (!ok) {
+                throw new CliError(1, "Aborted");
+            }
+        }
+        const spinner = ora("Deleting...").start();
+        try {
+            const client = getSdkClient(apiKey);
+            await client.projects.delete(projectId);
+            spinner.stop();
+            printSuccess(`Project ${projectId} deleted successfully.`);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+}