@root-signals/scorable-cli 0.12.0 → 0.13.0

package/dist/commands/evaluator/create.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"create.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CA+E9D"}
+{"version":3,"file":"create.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAQpC,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAiF9D"}

package/dist/commands/evaluator/create.js

@@ -1,6 +1,8 @@
 import ora from "ora";
+import { z } from "zod";
 import { requireApiKey, getSdkClient } from "../../auth.js";
 import { printSuccess, printError, printJson, handleSdkError } from "../../output.js";
+import { parseJsonArg } from "../../utils.js";
 export function registerCreateCommand(evaluator) {
     evaluator
         .command("create")
@@ -36,13 +38,12 @@ export function registerCreateCommand(evaluator) {
         if (opts.objectiveVersionId)
             payload.objective_version_id = opts.objectiveVersionId;
         if (opts.models) {
-            try {
-                payload.models = JSON.parse(opts.models);
-            }
-            catch {
-                printError("Invalid JSON format for --models.");
+            const r = parseJsonArg(opts.models, z.array(z.string()));
+            if (!r.ok) {
+                printError("Invalid JSON format for --models. Expected an array of model name strings.");
                 return;
             }
+            payload.models = r.value;
         }
         const spinner = ora("Creating...").start();
         try {

package/dist/commands/evaluator/execute-by-name.d.ts

@@ -4,6 +4,7 @@ export declare function executeEvaluatorByName(evaluatorName: string, opts: {
     request?: string;
     response?: string;
     turns?: string;
+    tools?: string;
     contexts?: string;
     expectedOutput?: string;
     tag?: string[];

package/dist/commands/evaluator/execute-by-name.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"execute-by-name.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/execute-by-name.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,iBAAe,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAIjD;AAED,wBAAsB,sBAAsB,CAC1C,aAAa,EAAE,MAAM,EACrB,IAAI,EAAE;IACJ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,EACD,SAAS,0BAAmB,GAC3B,OAAO,CAAC,IAAI,CAAC,CA2Ef;AAED,wBAAgB,4BAA4B,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CA4ErE"}
+{"version":3,"file":"execute-by-name.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/execute-by-name.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,iBAAe,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAIjD;AAED,wBAAsB,sBAAsB,CAC1C,aAAa,EAAE,MAAM,EACrB,IAAI,EAAE;IACJ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,EACD,SAAS,0BAAmB,GAC3B,OAAO,CAAC,IAAI,CAAC,CAyFf;AAED,wBAAgB,4BAA4B,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAmFrE"}

package/dist/commands/evaluator/execute-by-name.js

@@ -1,7 +1,7 @@
 import ora from "ora";
 import { requireApiKey, getSdkClient } from "../../auth.js";
 import { printSuccess, printError, printJson, handleSdkError } from "../../output.js";
-import { isTurnArray, isStringArray, isStringRecord } from "../../utils.js";
+import { isTurnArray, isToolCatalog, isStringArray, isStringRecord } from "../../utils.js";
 async function readStdinDefault() {
     const chunks = [];
     for await (const chunk of process.stdin)
@@ -47,6 +47,20 @@ export async function executeEvaluatorByName(evaluatorName, opts, readStdin = re
             return;
         }
     }
+    if (opts.tools) {
+        try {
+            const parsed = JSON.parse(opts.tools);
+            if (!isToolCatalog(parsed)) {
+                printError("Invalid JSON for --tools. Expected array of tool objects.");
+                return;
+            }
+            payload.tools = parsed;
+        }
+        catch {
+            printError("Invalid JSON for --tools.");
+            return;
+        }
+    }
     if (opts.contexts) {
         try {
             const parsed = JSON.parse(opts.contexts);
@@ -100,7 +114,8 @@ export function registerExecuteByNameCommand(evaluator) {
         .option("--user-id <id>", "User identifier for tracking purposes")
         .option("--session-id <id>", "Session identifier for tracking purposes")
         .option("--system-prompt <text>", "System prompt that was used for the LLM call")
-        .option("--turns <json>", 'JSON array of conversation turns. E.g., \'[{"role":"user","content":"Hello"}]\'')
+        .option("--turns <json>", 'JSON array of conversation turns. Roles: user|assistant|tool. Assistant turns may carry `tool_calls`; tool results use a `tool` role with `tool_call_id`. E.g., \'[{"role":"user","content":"Hello"}]\'')
+        .option("--tools <json>", "JSON array of OpenAI-style tool definitions available to the agent.")
         .option("--variables <json>", 'JSON object of extra template variables. E.g., \'{"lang":"EN"}\'')
         .addHelpText("after", `
 Examples:
@@ -124,6 +139,11 @@ Examples:
   $ scorable evaluator execute-by-name "Helpfulness" \\
       --turns '[{"role":"user","content":"Hello, I need help with my order"},{"role":"assistant","content":"I\\'d be happy to help! What\\'s your order number?"},{"role":"user","content":"It\\'s ORDER-12345"}]'
 
+  # Tool-aware evaluation: assistant tool_calls + tool-role result + tool catalog
+  $ scorable evaluator execute-by-name "Tool Selection" \\
+      --turns '[{"role":"user","content":"What is the weather in Paris?"},{"role":"assistant","content":null,"tool_calls":[{"id":"call_1","type":"function","function":{"name":"get_weather","arguments":"{\\"city\\":\\"Paris\\"}"}}]},{"role":"tool","tool_call_id":"call_1","content":"{\\"temp\\":12}"},{"role":"assistant","content":"It is 12C in Paris."}]' \\
+      --tools '[{"type":"function","function":{"name":"get_weather","description":"Get weather for a city","parameters":{"type":"object","properties":{"city":{"type":"string"}}}}}]'
+
   # Pipe a response from your app via stdin
   $ ./my-app respond "Do you offer refunds?" | \\
       scorable evaluator execute-by-name "Helpfulness" --request "Do you offer refunds?"`)

package/dist/commands/evaluator/execute.d.ts

@@ -4,6 +4,7 @@ export declare function executeEvaluator(evaluatorId: string, opts: {
     request?: string;
     response?: string;
     turns?: string;
+    tools?: string;
     contexts?: string;
     expectedOutput?: string;
     tag?: string[];

package/dist/commands/evaluator/execute.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"execute.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/execute.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,iBAAe,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAIjD;AAED,wBAAsB,gBAAgB,CACpC,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE;IACJ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,EACD,SAAS,0BAAmB,GAC3B,OAAO,CAAC,IAAI,CAAC,CAwFf;AAED,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CA2F/D"}
+{"version":3,"file":"execute.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/execute.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,iBAAe,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAIjD;AAED,wBAAsB,gBAAgB,CACpC,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE;IACJ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,EACD,SAAS,0BAAmB,GAC3B,OAAO,CAAC,IAAI,CAAC,CAsGf;AAED,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAkG/D"}

package/dist/commands/evaluator/execute.js

@@ -1,7 +1,7 @@
 import ora from "ora";
 import { requireApiKey, getSdkClient } from "../../auth.js";
 import { printSuccess, printError, printJson, handleSdkError } from "../../output.js";
-import { isTurnArray, isStringArray, isStringRecord } from "../../utils.js";
+import { isTurnArray, isToolCatalog, isStringArray, isStringRecord } from "../../utils.js";
 async function readStdinDefault() {
     const chunks = [];
     for await (const chunk of process.stdin)
@@ -47,6 +47,20 @@ export async function executeEvaluator(evaluatorId, opts, readStdin = readStdinD
             return;
         }
     }
+    if (opts.tools) {
+        try {
+            const parsed = JSON.parse(opts.tools);
+            if (!isToolCatalog(parsed)) {
+                printError("Invalid JSON for --tools. Expected array of tool objects.");
+                return;
+            }
+            payload.tools = parsed;
+        }
+        catch {
+            printError("Invalid JSON for --tools.");
+            return;
+        }
+    }
     if (opts.contexts) {
         try {
             const parsed = JSON.parse(opts.contexts);
@@ -113,7 +127,8 @@ export function registerExecuteCommand(evaluator) {
         .option("--user-id <id>", "User identifier for tracking purposes")
         .option("--session-id <id>", "Session identifier for tracking purposes")
         .option("--system-prompt <text>", "System prompt that was used for the LLM call")
-        .option("--turns <json>", 'JSON array of conversation turns. E.g., \'[{"role":"user","content":"Hello"}]\'')
+        .option("--turns <json>", 'JSON array of conversation turns. Roles: user|assistant|tool. Assistant turns may carry `tool_calls`; tool results use a `tool` role with `tool_call_id`. E.g., \'[{"role":"user","content":"Hello"}]\'')
+        .option("--tools <json>", "JSON array of OpenAI-style tool definitions available to the agent.")
         .option("--variables <json>", 'JSON object of extra template variables. E.g., \'{"lang":"EN"}\'')
         .option("--file-ids <json>", "JSON array of file UUIDs from 'scorable file upload'. E.g., '[\"uuid1\"]'")
         .addHelpText("after", `
@@ -139,6 +154,11 @@ Examples:
   $ scorable evaluator execute <evaluatorId> \\
       --turns '[{"role":"user","content":"Hello, I need help with my order"},{"role":"assistant","content":"I\\'d be happy to help! What\\'s your order number?"},{"role":"user","content":"It\\'s ORDER-12345"},{"role":"assistant","content":"I found your order. It\\'s currently in transit."}]'
 
+  # Tool-aware evaluation: assistant tool_calls, tool-role result, and tool catalog
+  $ scorable evaluator execute <evaluatorId> \\
+      --turns '[{"role":"user","content":"What is the weather in Paris?"},{"role":"assistant","content":null,"tool_calls":[{"id":"call_1","type":"function","function":{"name":"get_weather","arguments":"{\\"city\\":\\"Paris\\"}"}}]},{"role":"tool","tool_call_id":"call_1","content":"{\\"temp\\":12}"},{"role":"assistant","content":"It is 12C in Paris."}]' \\
+      --tools '[{"type":"function","function":{"name":"get_weather","description":"Get weather for a city","parameters":{"type":"object","properties":{"city":{"type":"string"}}}}}]'
+
   # With custom template variables and tracking
   $ scorable evaluator execute <evaluatorId> \\
       --request "How do I cancel my subscription?" \\

package/dist/commands/evaluator/update.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"update.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/update.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAwD9D"}
+{"version":3,"file":"update.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/update.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAQpC,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CA0D9D"}

package/dist/commands/evaluator/update.js

@@ -1,6 +1,8 @@
 import ora from "ora";
+import { z } from "zod";
 import { requireApiKey, getSdkClient } from "../../auth.js";
 import { printInfo, printSuccess, printError, printJson, handleSdkError } from "../../output.js";
+import { parseJsonArg } from "../../utils.js";
 export function registerUpdateCommand(evaluator) {
     evaluator
         .command("update <evaluatorId>")
@@ -22,13 +24,12 @@ export function registerUpdateCommand(evaluator) {
         if (opts.objectiveVersionId !== undefined)
             payload.objective_version_id = opts.objectiveVersionId;
         if (opts.models !== undefined) {
-            try {
-                payload.models = JSON.parse(opts.models);
-            }
-            catch {
-                printError("Invalid JSON format for --models.");
+            const r = parseJsonArg(opts.models, z.array(z.string()));
+            if (!r.ok) {
+                printError("Invalid JSON format for --models. Expected an array of model name strings.");
                 return;
             }
+            payload.models = r.value;
         }
         if (Object.keys(payload).length === 0) {
             printInfo("No update parameters provided. Aborting.");

package/dist/commands/judge/create.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"create.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CA+C1D"}
+{"version":3,"file":"create.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAUpC,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CA+C1D"}

package/dist/commands/judge/create.js

@@ -1,6 +1,9 @@
 import ora from "ora";
+import { z } from "zod";
 import { requireApiKey, getSdkClient } from "../../auth.js";
 import { printSuccess, printError, printJson, handleSdkError } from "../../output.js";
+import { parseJsonArg } from "../../utils.js";
+const EvaluatorRefsSchema = z.array(z.object({ id: z.string() }).passthrough());
 export function registerCreateCommand(judge) {
     judge
         .command("create")
@@ -15,13 +18,12 @@ export function registerCreateCommand(judge) {
         if (opts.stage)
             payload.stage = opts.stage;
         if (opts.evaluatorReferences) {
-            try {
-                payload.evaluator_references = JSON.parse(opts.evaluatorReferences);
-            }
-            catch {
-                printError("Invalid JSON format for --evaluator-references.");
+            const r = parseJsonArg(opts.evaluatorReferences, EvaluatorRefsSchema);
+            if (!r.ok) {
+                printError('Invalid JSON format for --evaluator-references. Expected an array of objects with an "id" string.');
                 return;
             }
+            payload.evaluator_references = r.value;
         }
         const spinner = ora("Creating...").start();
         try {

package/dist/commands/judge/exec-openai-generic.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"exec-openai-generic.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/exec-openai-generic.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAYpC,wBAAgB,gCAAgC,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CA6CrE"}
+{"version":3,"file":"exec-openai-generic.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/exec-openai-generic.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAiBpC,wBAAgB,gCAAgC,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAiDrE"}

package/dist/commands/judge/exec-openai-generic.js

@@ -1,6 +1,10 @@
+import { z } from "zod";
 import { requireApiKey } from "../../auth.js";
 import { printInfo, printSuccess, printError, printWarning, printJson, handleSdkError, } from "../../output.js";
+import { parseJsonArg } from "../../utils.js";
 import { apiRequest } from "../../client.js";
+const MessagesSchema = z.array(z.object({ role: z.string() }).passthrough());
+const ExtraBodySchema = z.record(z.string(), z.unknown());
 export function registerExecOpenaiGenericCommand(judge) {
     judge
         .command("exec-openai-generic")
@@ -10,21 +14,22 @@ export function registerExecOpenaiGenericCommand(judge) {
         .option("--extra-body <json>", "Optional JSON string for extra_body parameters")
         .action(async (opts) => {
         const apiKey = await requireApiKey();
-        let messages;
-        try {
-            messages = JSON.parse(opts.messages);
-        }
-        catch {
-            printError("Invalid JSON for --messages. Aborting.");
+        const messagesResult = parseJsonArg(opts.messages, MessagesSchema);
+        if (!messagesResult.ok) {
+            printError("Invalid JSON for --messages. Expected an array of objects each with a string `role`. Aborting.");
             return;
         }
-        const payload = { model: opts.model, messages };
+        const payload = {
+            model: opts.model,
+            messages: messagesResult.value,
+        };
         if (opts.extraBody) {
-            try {
-                payload["extra_body"] = JSON.parse(opts.extraBody);
+            const extraResult = parseJsonArg(opts.extraBody, ExtraBodySchema);
+            if (extraResult.ok) {
+                payload["extra_body"] = extraResult.value;
             }
-            catch {
-                printWarning("Invalid JSON for --extra-body. Skipping.");
+            else {
+                printWarning("Invalid JSON for --extra-body (expected an object). Skipping.");
             }
         }
         printInfo(`Executing a Judge using generic OpenAI endpoint. Judge ID/Name: ${opts.model}`);

package/dist/commands/judge/exec-openai.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"exec-openai.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/exec-openai.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAYpC,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAwD9D"}
+{"version":3,"file":"exec-openai.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/exec-openai.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAiBpC,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CA4D9D"}

package/dist/commands/judge/exec-openai.js

@@ -1,6 +1,10 @@
+import { z } from "zod";
 import { requireApiKey } from "../../auth.js";
 import { printInfo, printSuccess, printError, printWarning, printJson, handleSdkError, } from "../../output.js";
+import { parseJsonArg } from "../../utils.js";
 import { apiRequest } from "../../client.js";
+const MessagesSchema = z.array(z.object({ role: z.string() }).passthrough());
+const ExtraBodySchema = z.record(z.string(), z.unknown());
 export function registerExecOpenaiCommand(judge) {
     judge
         .command("exec-openai <judgeIdInPath>")
@@ -10,21 +14,22 @@ export function registerExecOpenaiCommand(judge) {
         .option("--extra-body <json>", "Optional JSON string for extra_body parameters")
         .action(async (judgeIdInPath, opts) => {
         const apiKey = await requireApiKey();
-        let messages;
-        try {
-            messages = JSON.parse(opts.messages);
-        }
-        catch {
-            printError("Invalid JSON for --messages. Aborting.");
+        const messagesResult = parseJsonArg(opts.messages, MessagesSchema);
+        if (!messagesResult.ok) {
+            printError("Invalid JSON for --messages. Expected an array of objects each with a string `role`. Aborting.");
             return;
         }
-        const payload = { model: opts.model, messages };
+        const payload = {
+            model: opts.model,
+            messages: messagesResult.value,
+        };
         if (opts.extraBody) {
-            try {
-                payload["extra_body"] = JSON.parse(opts.extraBody);
+            const extraResult = parseJsonArg(opts.extraBody, ExtraBodySchema);
+            if (extraResult.ok) {
+                payload["extra_body"] = extraResult.value;
             }
-            catch {
-                printWarning("Invalid JSON for --extra-body. Skipping.");
+            else {
+                printWarning("Invalid JSON for --extra-body (expected an object). Skipping.");
             }
         }
         printInfo(`Executing Judge ID (via path): ${judgeIdInPath} using OpenAI chat completions format.`);

package/dist/commands/judge/execute-by-name.d.ts

@@ -4,6 +4,7 @@ export declare function executeJudgeByName(judgeName: string, opts: {
     request?: string;
     response?: string;
     turns?: string;
+    tools?: string;
     contexts?: string;
     expectedOutput?: string;
     tag?: string[];

package/dist/commands/judge/execute-by-name.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"execute-by-name.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/execute-by-name.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,iBAAe,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAIjD;AAED,wBAAsB,kBAAkB,CACtC,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE;IACJ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,EACD,SAAS,0BAAmB,GAC3B,OAAO,CAAC,IAAI,CAAC,CA6Df;AAED,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAkEjE"}
+{"version":3,"file":"execute-by-name.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/execute-by-name.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,iBAAe,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAIjD;AAED,wBAAsB,kBAAkB,CACtC,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE;IACJ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,EACD,SAAS,0BAAmB,GAC3B,OAAO,CAAC,IAAI,CAAC,CA2Ef;AAED,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAyEjE"}

package/dist/commands/judge/execute-by-name.js

@@ -1,7 +1,7 @@
 import ora from "ora";
 import { requireApiKey, getSdkClient } from "../../auth.js";
 import { printSuccess, printError, printJson, handleSdkError } from "../../output.js";
-import { isTurnArray, isStringArray } from "../../utils.js";
+import { isTurnArray, isToolCatalog, isStringArray } from "../../utils.js";
 async function readStdinDefault() {
     const chunks = [];
     for await (const chunk of process.stdin)
@@ -47,6 +47,20 @@ export async function executeJudgeByName(judgeName, opts, readStdin = readStdinD
             return;
         }
     }
+    if (opts.tools) {
+        try {
+            const parsed = JSON.parse(opts.tools);
+            if (!isToolCatalog(parsed)) {
+                printError("Invalid JSON for --tools. Expected array of tool objects.");
+                return;
+            }
+            payload.tools = parsed;
+        }
+        catch {
+            printError("Invalid JSON for --tools.");
+            return;
+        }
+    }
     if (opts.contexts) {
         try {
             const parsed = JSON.parse(opts.contexts);
@@ -86,7 +100,8 @@ export function registerExecuteByNameCommand(judge) {
         .option("--user-id <id>", "User identifier for tracking purposes")
         .option("--session-id <id>", "Session identifier for tracking purposes")
         .option("--system-prompt <text>", "System prompt that was used for the LLM call")
-        .option("--turns <json>", 'JSON array of conversation turns. E.g., \'[{"role":"user","content":"Hello"}]\'')
+        .option("--turns <json>", 'JSON array of conversation turns. Roles: user|assistant|tool. Assistant turns may carry `tool_calls`; tool results use a `tool` role with `tool_call_id`. E.g., \'[{"role":"user","content":"Hello"}]\'')
+        .option("--tools <json>", "JSON array of OpenAI-style tool definitions available to the agent.")
         .addHelpText("after", `
 Examples:
   # Evaluate a customer service response
@@ -104,6 +119,11 @@ Examples:
   $ scorable judge execute-by-name "Customer Service Judge" \\
       --turns '[{"role":"user","content":"I haven\\'t received my refund and it\\'s been 10 days."},{"role":"assistant","content":"Thank you for contacting us. I will investigate this immediately and provide an update within 24 hours."}]'
 
+  # Tool-aware evaluation: assistant tool_calls + tool-role result + tool catalog
+  $ scorable judge execute-by-name "Agent Quality Judge" \\
+      --turns '[{"role":"user","content":"What is the weather in Paris?"},{"role":"assistant","content":null,"tool_calls":[{"id":"call_1","type":"function","function":{"name":"get_weather","arguments":"{\\"city\\":\\"Paris\\"}"}}]},{"role":"tool","tool_call_id":"call_1","content":"{\\"temp\\":12}"},{"role":"assistant","content":"It is 12C in Paris."}]' \\
+      --tools '[{"type":"function","function":{"name":"get_weather","description":"Get weather for a city","parameters":{"type":"object","properties":{"city":{"type":"string"}}}}}]'
+
   # Pipe the LLM response from stdin
   $ echo "We offer a 30-day money-back guarantee on all purchases." | \\
       scorable judge execute-by-name "Custom Returns Policy Judge" --request "Do you offer refunds?"`)

package/dist/commands/judge/execute.d.ts

@@ -4,6 +4,7 @@ export declare function executeJudge(judgeId: string, opts: {
     request?: string;
     response?: string;
     turns?: string;
+    tools?: string;
     contexts?: string;
     expectedOutput?: string;
     tag?: string[];

package/dist/commands/judge/execute.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"execute.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/execute.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,iBAAe,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAIjD;AAED,wBAAsB,YAAY,CAChC,OAAO,EAAE,MAAM,EACf,IAAI,EAAE;IACJ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,EACD,SAAS,0BAAmB,GAC3B,OAAO,CAAC,IAAI,CAAC,CA6Df;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAyE3D"}
+{"version":3,"file":"execute.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/execute.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,iBAAe,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAIjD;AAED,wBAAsB,YAAY,CAChC,OAAO,EAAE,MAAM,EACf,IAAI,EAAE;IACJ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,EACD,SAAS,0BAAmB,GAC3B,OAAO,CAAC,IAAI,CAAC,CA2Ef;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAgF3D"}

package/dist/commands/judge/execute.js

@@ -1,7 +1,7 @@
 import ora from "ora";
 import { requireApiKey, getSdkClient } from "../../auth.js";
 import { printSuccess, printError, printJson, handleSdkError } from "../../output.js";
-import { isTurnArray, isStringArray } from "../../utils.js";
+import { isTurnArray, isToolCatalog, isStringArray } from "../../utils.js";
 async function readStdinDefault() {
     const chunks = [];
     for await (const chunk of process.stdin)
@@ -47,6 +47,20 @@ export async function executeJudge(judgeId, opts, readStdin = readStdinDefault)
             return;
         }
     }
+    if (opts.tools) {
+        try {
+            const parsed = JSON.parse(opts.tools);
+            if (!isToolCatalog(parsed)) {
+                printError("Invalid JSON for --tools. Expected array of tool objects.");
+                return;
+            }
+            payload.tools = parsed;
+        }
+        catch {
+            printError("Invalid JSON for --tools.");
+            return;
+        }
+    }
     if (opts.contexts) {
         try {
             const parsed = JSON.parse(opts.contexts);
@@ -86,7 +100,8 @@ export function registerExecuteCommand(judge) {
         .option("--user-id <id>", "User identifier for tracking purposes")
         .option("--session-id <id>", "Session identifier for tracking purposes")
         .option("--system-prompt <text>", "System prompt that was used for the LLM call")
-        .option("--turns <json>", 'JSON array of conversation turns. E.g., \'[{"role":"user","content":"Hello"}]\'')
+        .option("--turns <json>", 'JSON array of conversation turns. Roles: user|assistant|tool. Assistant turns may carry `tool_calls`; tool results use a `tool` role with `tool_call_id`. E.g., \'[{"role":"user","content":"Hello"}]\'')
+        .option("--tools <json>", "JSON array of OpenAI-style tool definitions available to the agent.")
         .addHelpText("after", `
 Examples:
   # Evaluate a customer service response about return policy
@@ -104,6 +119,11 @@ Examples:
   $ scorable judge execute <judgeId> \\
       --turns '[{"role":"user","content":"Hello, I need help with my order"},{"role":"assistant","content":"I\\'d be happy to help! What\\'s your order number?"},{"role":"user","content":"It\\'s ORDER-12345"},{"role":"assistant","content":"I found your order. It\\'s currently in transit."}]'
 
+  # Tool-aware evaluation: assistant tool_calls, tool-role result, and tool catalog
+  $ scorable judge execute <judgeId> \\
+      --turns '[{"role":"user","content":"What is the weather in Paris?"},{"role":"assistant","content":null,"tool_calls":[{"id":"call_1","type":"function","function":{"name":"get_weather","arguments":"{\\"city\\":\\"Paris\\"}"}}]},{"role":"tool","tool_call_id":"call_1","content":"{\\"temp\\":12}"},{"role":"assistant","content":"It is 12C in Paris."}]' \\
+      --tools '[{"type":"function","function":{"name":"get_weather","description":"Get weather for a city","parameters":{"type":"object","properties":{"city":{"type":"string"}}}}}]'
+
   # With tracking metadata and tags
   $ scorable judge execute <judgeId> \\
       --request "How do I reset my password?" \\

package/dist/commands/judge/generate.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"generate.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/generate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAepC,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CA8I5D"}
+{"version":3,"file":"generate.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/generate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAiBpC,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CA8I5D"}

package/dist/commands/judge/generate.js

@@ -1,7 +1,9 @@
 import ora from "ora";
 import chalk from "chalk";
+import { z } from "zod";
 import { requireApiKey, getSdkClient } from "../../auth.js";
 import { printSuccess, printError, printWarning, printInfo, printJson, handleSdkError, } from "../../output.js";
+import { parseJsonArg } from "../../utils.js";
 import { CliError } from "../../types.js";
 import { uploadFile } from "../file/upload.js";
 export function registerGenerateCommand(judge) {
@@ -40,13 +42,12 @@ offer to connect the guest with a human agent when uncertain."`)
         const apiKey = await requireApiKey();
         let extra_contexts;
         if (opts.extraContexts) {
-            try {
-                extra_contexts = JSON.parse(opts.extraContexts);
-            }
-            catch {
+            const r = parseJsonArg(opts.extraContexts, z.record(z.string(), z.string()));
+            if (!r.ok) {
                 printError("Invalid --extra-contexts JSON: must be a key-value object.");
                 throw new CliError(1, "Invalid extra-contexts JSON");
             }
+            extra_contexts = r.value;
         }
         const spinner = ora("Generating judge (this may take a moment)...").start();
         try {

package/dist/commands/judge/update.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"update.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/update.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAkD1D"}
+{"version":3,"file":"update.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/update.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAUpC,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAkD1D"}

package/dist/commands/judge/update.js

@@ -1,6 +1,9 @@
 import ora from "ora";
+import { z } from "zod";
 import { requireApiKey, getSdkClient } from "../../auth.js";
 import { printInfo, printSuccess, printError, printJson, handleSdkError } from "../../output.js";
+import { parseJsonArg } from "../../utils.js";
+const EvaluatorRefsSchema = z.array(z.object({ id: z.string() }).passthrough());
 export function registerUpdateCommand(judge) {
     judge
         .command("update <judgeId>")
@@ -16,13 +19,12 @@ export function registerUpdateCommand(judge) {
         if (opts.stage !== undefined)
             payload.stage = opts.stage;
         if (opts.evaluatorReferences !== undefined) {
-            try {
-                payload.evaluator_references = JSON.parse(opts.evaluatorReferences);
-            }
-            catch {
-                printError("Invalid JSON format for --evaluator-references.");
+            const r = parseJsonArg(opts.evaluatorReferences, EvaluatorRefsSchema);
+            if (!r.ok) {
+                printError('Invalid JSON format for --evaluator-references. Expected an array of objects with an "id" string.');
                 return;
             }
+            payload.evaluator_references = r.value;
         }
         if (Object.keys(payload).length === 0) {
             printInfo("No update parameters provided. Aborting.");

package/dist/commands/otel-filter/create.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"create.d.ts","sourceRoot":"","sources":["../../../src/commands/otel-filter/create.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAkBpC,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,OAAO,GAAG,IAAI,CA6J/D"}
+{"version":3,"file":"create.d.ts","sourceRoot":"","sources":["../../../src/commands/otel-filter/create.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAmBpC,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,OAAO,GAAG,IAAI,CA+J/D"}

package/dist/commands/otel-filter/create.js

@@ -3,7 +3,8 @@ import ora from "ora";
 import { requireApiKey } from "../../auth.js";
 import { apiRequest } from "../../client.js";
 import { printSuccess, printError, printJson, handleSdkError } from "../../output.js";
-import { loadFilterYaml } from "../../lib/filter-yaml.js";
+import { loadFilterYaml, Match } from "../../lib/filter-yaml.js";
+import { parseJsonArg } from "../../utils.js";
 export function registerCreateCommand(otelFilter) {
     otelFilter
         .command("create")
@@ -71,13 +72,12 @@ YAML manifest (only needed for custom extractor_rules):
         }
         let filterCriteria;
         if (opts.filterCriteria) {
-            try {
-                filterCriteria = JSON.parse(opts.filterCriteria);
-            }
-            catch {
-                printError("Invalid JSON for --filter-criteria.");
+            const r = parseJsonArg(opts.filterCriteria, Match);
+            if (!r.ok) {
+                printError('Invalid JSON for --filter-criteria. Expected {"conditions":[{column,operator,value?,key?},...]}.');
                 process.exit(1);
             }
+            filterCriteria = r.value;
         }
         let samplingRate;
         if (opts.samplingRate !== undefined) {

package/dist/lib/filter-yaml.d.ts

@@ -1,4 +1,12 @@
 import { z } from "zod";
+export declare const Match: z.ZodObject<{
+    conditions: z.ZodDefault<z.ZodArray<z.ZodObject<{
+        column: z.ZodString;
+        operator: z.ZodString;
+        value: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>;
+        key: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>>>;
+}, z.core.$strict>;
 export declare const FilterYamlSchema: z.ZodObject<{
     name: z.ZodString;
     evaluator_id: z.ZodOptional<z.ZodString>;

package/dist/lib/filter-yaml.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"filter-yaml.d.ts","sourceRoot":"","sources":["../../src/lib/filter-yaml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAyFxB,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAczB,CAAC;AAEL,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU,CAczD"}
+{"version":3,"file":"filter-yaml.d.ts","sourceRoot":"","sources":["../../src/lib/filter-yaml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAYxB,eAAO,MAAM,KAAK;;;;;;;kBAIP,CAAC;AAyEZ,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAczB,CAAC;AAEL,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU,CAczD"}

package/dist/lib/filter-yaml.js

@@ -8,7 +8,7 @@ const MatchCondition = z
     key: z.string().optional(),
 })
     .strict();
-const Match = z
+export const Match = z
     .object({
     conditions: z.array(MatchCondition).default([]),
 })

package/dist/utils.d.ts

@@ -1,5 +1,14 @@
+import { type ZodType } from "zod";
 import type { Turn } from "@root-signals/scorable";
 export declare function isTurnArray(v: unknown): v is Turn[];
+export declare function isToolCatalog(v: unknown): v is Record<string, unknown>[];
 export declare function isStringArray(v: unknown): v is string[];
 export declare function isStringRecord(v: unknown): v is Record<string, string>;
+export type JsonArgResult<T> = {
+    ok: true;
+    value: T;
+} | {
+    ok: false;
+};
+export declare function parseJsonArg<T>(raw: string, schema: ZodType<T>): JsonArgResult<T>;
 //# sourceMappingURL=utils.d.ts.map

package/dist/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AAEnD,wBAAgB,WAAW,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,IAAI,IAAI,EAAE,CAanD;AAED,wBAAgB,aAAa,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,IAAI,MAAM,EAAE,CAEvD;AAED,wBAAgB,cAAc,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAOtE"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAK,KAAK,OAAO,EAAE,MAAM,KAAK,CAAC;AACtC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAC;AAanD,wBAAgB,WAAW,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,IAAI,IAAI,EAAE,CAEnD;AAED,wBAAgB,aAAa,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAExE;AAED,wBAAgB,aAAa,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,IAAI,MAAM,EAAE,CAEvD;AAED,wBAAgB,cAAc,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAOtE;AAID,MAAM,MAAM,aAAa,CAAC,CAAC,IAAI;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,KAAK,EAAE,CAAC,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAA;CAAE,CAAC;AAEtE,wBAAgB,YAAY,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,CAUjF"}

package/dist/utils.js

@@ -1,11 +1,18 @@
+import { z } from "zod";
+const TurnSchema = z.object({
+    role: z.enum(["user", "assistant", "tool"]),
+    content: z.string().nullable().optional(),
+    contexts: z.array(z.string()).nullable().optional(),
+    tool_calls: z.array(z.record(z.string(), z.unknown())).nullable().optional(),
+    tool_call_id: z.string().nullable().optional(),
+});
+const TurnArraySchema = z.array(TurnSchema);
+const ToolCatalogSchema = z.array(z.record(z.string(), z.unknown()));
 export function isTurnArray(v) {
-    return (Array.isArray(v) &&
-        v.every((t) => typeof t === "object" &&
-            t !== null &&
-            "role" in t &&
-            typeof t.role === "string" &&
-            "content" in t &&
-            typeof t.content === "string"));
+    return TurnArraySchema.safeParse(v).success;
+}
+export function isToolCatalog(v) {
+    return ToolCatalogSchema.safeParse(v).success;
 }
 export function isStringArray(v) {
     return Array.isArray(v) && v.every((s) => typeof s === "string");
@@ -16,3 +23,16 @@ export function isStringRecord(v) {
         !Array.isArray(v) &&
         Object.values(v).every((val) => typeof val === "string"));
 }
+export function parseJsonArg(raw, schema) {
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch {
+        return { ok: false };
+    }
+    const result = schema.safeParse(parsed);
+    if (!result.success)
+        return { ok: false };
+    return { ok: true, value: result.data };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@root-signals/scorable-cli",
-  "version": "0.12.0",
+  "version": "0.13.0",
   "description": "CLI for Scorable",
   "license": "Apache-2.0",
   "author": "Scorable",
@@ -32,7 +32,7 @@
   },
   "dependencies": {
     "@inquirer/prompts": "^8.3.2",
-    "@root-signals/scorable": "^0.8.0",
+    "@root-signals/scorable": "^0.10.0",
     "chalk": "^5.6.2",
     "cli-table3": "^0.6.5",
     "commander": "^14.0.3",