@root-signals/scorable-cli 0.12.0 → 0.14.0

package/README.md

@@ -78,6 +78,66 @@ export SCORABLE_API_KEY="sk-your-api-key"
 
 The key lookup order is: `SCORABLE_API_KEY` env var → `api_key` in `~/.scorable/settings.json` → `temporary_api_key` in `~/.scorable/settings.json`.
 
+## Projects
+
+A **project** is a workspace inside your organization that groups related judges, evaluators, datasets, and execution logs. Every resource belongs to a project; omitting the project at create/execute time files things under the org's default project.
+
+### Manage projects
+
+```bash
+scorable project list                                   # list all projects (default marked)
+scorable project get <project_id>                       # show a single project
+scorable project create --name "Production"             # create a project
+scorable project create --name "Production" --is-default
+scorable project update <project_id> --name "Renamed"
+scorable project update <project_id> --is-default       # promote to default
+scorable project set-default <project_id>               # convenience for `update --is-default`
+scorable project delete <project_id>
+```
+
+### `--project-id` on every project-aware command
+
+Every command that creates, executes, lists, or filters a project-scoped resource accepts `--project-id <uuid>`:
+
+```bash
+# Filter list endpoints
+scorable judge list --project-id <project_id>
+scorable evaluator list --project-id <project_id>
+scorable execution-log list --project-id <project_id>
+
+# Route an execution log to a project
+scorable judge execute <judge_id> --project-id <project_id>
+scorable evaluator execute <evaluator_id> --project-id <project_id>
+
+# Pin a resource to a project at creation
+scorable judge create --name X --intent Y --project-id <project_id>
+scorable evaluator import-yaml --file evaluator.yaml --project-id <project_id>
+
+# Move a resource between projects
+scorable judge update <judge_id> --project-id <other_project_id>
+
+# OpenAI-compat (translated to X-Project-Id header)
+scorable judge exec-openai <judge_id> --project-id <project_id>
+```
+
+### Setting a default project for your shell
+
+To avoid passing `--project-id` on every command, set an env var or persist a per-machine default:
+
+```bash
+# Environment variable (great for CI)
+export SCORABLE_PROJECT_ID=<project_id>
+
+# Persistent default written to ~/.scorable/settings.json
+scorable auth set-project <project_id>
+scorable auth show-project    # see what's resolved and from where
+scorable auth unset-project   # remove the saved default
+```
+
+Resolution order: `--project-id` flag → `SCORABLE_PROJECT_ID` env var → `project_id` in `~/.scorable/settings.json` → omitted (backend resolves to org default). Pass `--project-id ""` to explicitly opt out of an inherited default for a single invocation.
+
+`scorable auth logout` clears the entire auth section of `~/.scorable/settings.json`, including the saved project_id.
+
 ## Scorable Skills for AI Coding Agents
 
 Install Scorable skills into your project so your AI coding agent (Claude Code, Cursor, etc.) can integrate evaluators automatically:

package/dist/client.d.ts

@@ -3,6 +3,7 @@ interface RequestOptions {
     params?: Record<string, unknown>;
     apiKey?: string;
     baseUrl?: string;
+    headers?: Record<string, string>;
 }
 export interface ApiRequestResult {
     ok: boolean;

package/dist/client.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAQA,UAAU,cAAc;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,OAAO,CAAC;CACf;AAQD,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,gBAAgB,CAAC,CAoD3B;AAKD,wBAAsB,UAAU,CAC9B,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,OAAO,CAAC,CAGlB"}
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAQA,UAAU,cAAc;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,OAAO,CAAC;CACf;AAQD,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,gBAAgB,CAAC,CAqD3B;AAKD,wBAAsB,UAAU,CAC9B,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,OAAO,CAAC,CAGlB"}

package/dist/client.js

@@ -15,6 +15,7 @@ export async function apiRequestStatus(method, endpoint, options) {
         }
     }
     const headers = {
+        ...options?.headers,
         Authorization: `Api-Key ${apiKey}`,
         "Content-Type": "application/json",
         Accept: "application/json",

package/dist/commands/auth/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/commands/auth/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAgE3D"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/commands/auth/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAUpC,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAsI3D"}

package/dist/commands/auth/index.js

@@ -1,7 +1,9 @@
 import ora from "ora";
 import { loadSettings, saveSettings, createDemoKey } from "../../auth.js";
-import { printSuccess, printError } from "../../output.js";
+import { printSuccess, printError, printInfo, printMessage } from "../../output.js";
+import { resolveProjectId } from "../../lib/project-id.js";
 import { CliError } from "../../types.js";
+const AUTH_SETTINGS_KEYS = ["api_key", "temporary_api_key", "project_id"];
 export function registerAuthCommands(program) {
     const auth = program.command("auth").description("Manage authentication");
     auth
@@ -39,6 +41,71 @@ export function registerAuthCommands(program) {
         }
         printSuccess("API key saved to ~/.scorable/settings.json");
     });
+    auth
+        .command("set-project <projectId>")
+        .description("Persist a project_id to ~/.scorable/settings.json")
+        .action((projectId) => {
+        if (!projectId.trim()) {
+            printError("Project ID must not be empty.");
+            throw new CliError(1, "Empty project id");
+        }
+        const settings = loadSettings();
+        settings["project_id"] = projectId.trim();
+        if (!saveSettings(settings)) {
+            printError("Failed to save project_id to ~/.scorable/settings.json");
+            throw new CliError(1, "Failed to save project_id");
+        }
+        printSuccess(`project_id saved to ~/.scorable/settings.json`);
+    });
+    auth
+        .command("unset-project")
+        .description("Remove the persisted project_id from ~/.scorable/settings.json")
+        .action(() => {
+        const settings = loadSettings();
+        if (!("project_id" in settings)) {
+            printInfo("No project_id was set.");
+            return;
+        }
+        delete settings["project_id"];
+        if (!saveSettings(settings)) {
+            printError("Failed to update ~/.scorable/settings.json");
+            throw new CliError(1, "Failed to update settings");
+        }
+        printSuccess("project_id removed from ~/.scorable/settings.json");
+    });
+    auth
+        .command("show-project")
+        .description("Print the resolved project_id and its source (flag/env/settings/none)")
+        .action(() => {
+        const resolved = resolveProjectId(undefined);
+        if (resolved.value === undefined) {
+            printMessage("project_id: <none> (source: none)");
+            return;
+        }
+        printMessage(`project_id: ${resolved.value} (source: ${resolved.source})`);
+    });
+    auth
+        .command("logout")
+        .description("Clear the auth section of ~/.scorable/settings.json (api keys + project_id)")
+        .action(() => {
+        const settings = loadSettings();
+        let touched = false;
+        for (const k of AUTH_SETTINGS_KEYS) {
+            if (k in settings) {
+                delete settings[k];
+                touched = true;
+            }
+        }
+        if (!touched) {
+            printInfo("Already logged out.");
+            return;
+        }
+        if (!saveSettings(settings)) {
+            printError("Failed to update ~/.scorable/settings.json");
+            throw new CliError(1, "Failed to update settings");
+        }
+        printSuccess("Logged out — cleared api_key, temporary_api_key, and project_id.");
+    });
     auth
         .command("demo-key")
         .description("Create a free demo API key and save it to ~/.scorable/settings.json")

package/dist/commands/evaluator/create.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"create.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CA+E9D"}
+{"version":3,"file":"create.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AASpC,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAsF9D"}

package/dist/commands/evaluator/create.js

@@ -1,6 +1,9 @@
 import ora from "ora";
+import { z } from "zod";
 import { requireApiKey, getSdkClient } from "../../auth.js";
 import { printSuccess, printError, printJson, handleSdkError } from "../../output.js";
+import { parseJsonArg } from "../../utils.js";
+import { resolveProjectIdValue, PROJECT_ID_FLAG_DESC } from "../../lib/project-id.js";
 export function registerCreateCommand(evaluator) {
     evaluator
         .command("create")
@@ -12,6 +15,7 @@ export function registerCreateCommand(evaluator) {
         .option("--models <json>", "JSON array of model names, in priority order. E.g., '[\"gpt-5-mini\"]'")
         .option("--overwrite", "Overwrite if evaluator with same name exists")
         .option("--objective-version-id <id>", "Objective version ID")
+        .option("--project-id <uuid>", PROJECT_ID_FLAG_DESC)
         .action(async (opts) => {
         if (!opts.intent && !opts.objectiveId) {
             printError("Either --intent or --objective-id is required.");
@@ -36,14 +40,16 @@ export function registerCreateCommand(evaluator) {
         if (opts.objectiveVersionId)
             payload.objective_version_id = opts.objectiveVersionId;
         if (opts.models) {
-            try {
-                payload.models = JSON.parse(opts.models);
-            }
-            catch {
-                printError("Invalid JSON format for --models.");
+            const r = parseJsonArg(opts.models, z.array(z.string()));
+            if (!r.ok) {
+                printError("Invalid JSON format for --models. Expected an array of model name strings.");
                 return;
             }
+            payload.models = r.value;
         }
+        const projectId = resolveProjectIdValue(opts.projectId);
+        if (projectId !== undefined)
+            payload.projectId = projectId;
         const spinner = ora("Creating...").start();
         try {
             const apiKey = await requireApiKey();

package/dist/commands/evaluator/duplicate.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"duplicate.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/duplicate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,wBAAgB,wBAAwB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAkBjE"}
+{"version":3,"file":"duplicate.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/duplicate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,wBAAwB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAuBjE"}

package/dist/commands/evaluator/duplicate.js

@@ -1,16 +1,19 @@
 import ora from "ora";
 import { requireApiKey, getSdkClient } from "../../auth.js";
 import { printSuccess, printJson, handleSdkError } from "../../output.js";
+import { resolveProjectIdValue, PROJECT_ID_FLAG_DESC } from "../../lib/project-id.js";
 export function registerDuplicateCommand(evaluator) {
     evaluator
         .command("duplicate <evaluatorId>")
         .description("Duplicate an existing evaluator")
-        .action(async (evaluatorId) => {
+        .option("--project-id <uuid>", PROJECT_ID_FLAG_DESC)
+        .action(async (evaluatorId, opts) => {
         const apiKey = await requireApiKey();
         const spinner = ora("Duplicating...").start();
         try {
             const client = getSdkClient(apiKey);
-            const result = await client.evaluators.duplicate(evaluatorId);
+            const projectId = resolveProjectIdValue(opts.projectId);
+            const result = await client.evaluators.duplicate(evaluatorId, projectId !== undefined ? { projectId } : {});
             spinner.stop();
             printSuccess(`Evaluator ${evaluatorId} duplicated successfully!`);
             printJson(result);

package/dist/commands/evaluator/execute-by-name.d.ts

@@ -4,6 +4,7 @@ export declare function executeEvaluatorByName(evaluatorName: string, opts: {
     request?: string;
     response?: string;
     turns?: string;
+    tools?: string;
     contexts?: string;
     expectedOutput?: string;
     tag?: string[];
@@ -11,6 +12,7 @@ export declare function executeEvaluatorByName(evaluatorName: string, opts: {
     sessionId?: string;
     systemPrompt?: string;
     variables?: string;
+    projectId?: string;
 }, readStdin?: typeof readStdinDefault): Promise<void>;
 export declare function registerExecuteByNameCommand(evaluator: Command): void;
 export {};

package/dist/commands/evaluator/execute-by-name.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"execute-by-name.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/execute-by-name.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,iBAAe,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAIjD;AAED,wBAAsB,sBAAsB,CAC1C,aAAa,EAAE,MAAM,EACrB,IAAI,EAAE;IACJ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,EACD,SAAS,0BAAmB,GAC3B,OAAO,CAAC,IAAI,CAAC,CA2Ef;AAED,wBAAgB,4BAA4B,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CA4ErE"}
+{"version":3,"file":"execute-by-name.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/execute-by-name.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAQpC,iBAAe,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAIjD;AAED,wBAAsB,sBAAsB,CAC1C,aAAa,EAAE,MAAM,EACrB,IAAI,EAAE;IACJ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,EACD,SAAS,0BAAmB,GAC3B,OAAO,CAAC,IAAI,CAAC,CA4Ff;AAED,wBAAgB,4BAA4B,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAqFrE"}

package/dist/commands/evaluator/execute-by-name.js

@@ -1,7 +1,8 @@
 import ora from "ora";
 import { requireApiKey, getSdkClient } from "../../auth.js";
 import { printSuccess, printError, printJson, handleSdkError } from "../../output.js";
-import { isTurnArray, isStringArray, isStringRecord } from "../../utils.js";
+import { isTurnArray, isToolCatalog, isStringArray, isStringRecord } from "../../utils.js";
+import { resolveProjectIdValue, PROJECT_ID_FLAG_DESC } from "../../lib/project-id.js";
 async function readStdinDefault() {
     const chunks = [];
     for await (const chunk of process.stdin)
@@ -47,6 +48,20 @@ export async function executeEvaluatorByName(evaluatorName, opts, readStdin = re
             return;
         }
     }
+    if (opts.tools) {
+        try {
+            const parsed = JSON.parse(opts.tools);
+            if (!isToolCatalog(parsed)) {
+                printError("Invalid JSON for --tools. Expected array of tool objects.");
+                return;
+            }
+            payload.tools = parsed;
+        }
+        catch {
+            printError("Invalid JSON for --tools.");
+            return;
+        }
+    }
     if (opts.contexts) {
         try {
             const parsed = JSON.parse(opts.contexts);
@@ -75,6 +90,9 @@ export async function executeEvaluatorByName(evaluatorName, opts, readStdin = re
             return;
         }
     }
+    const projectId = resolveProjectIdValue(opts.projectId);
+    if (projectId !== undefined)
+        payload.projectId = projectId;
     const spinner = ora("Running evaluator...").start();
     try {
         const client = getSdkClient(apiKey);
@@ -100,8 +118,10 @@ export function registerExecuteByNameCommand(evaluator) {
         .option("--user-id <id>", "User identifier for tracking purposes")
         .option("--session-id <id>", "Session identifier for tracking purposes")
         .option("--system-prompt <text>", "System prompt that was used for the LLM call")
-        .option("--turns <json>", 'JSON array of conversation turns. E.g., \'[{"role":"user","content":"Hello"}]\'')
+        .option("--turns <json>", 'JSON array of conversation turns. Roles: user|assistant|tool. Assistant turns may carry `tool_calls`; tool results use a `tool` role with `tool_call_id`. E.g., \'[{"role":"user","content":"Hello"}]\'')
+        .option("--tools <json>", "JSON array of OpenAI-style tool definitions available to the agent.")
         .option("--variables <json>", 'JSON object of extra template variables. E.g., \'{"lang":"EN"}\'')
+        .option("--project-id <uuid>", PROJECT_ID_FLAG_DESC)
         .addHelpText("after", `
 Examples:
   # Check helpfulness of a support response
@@ -124,6 +144,11 @@ Examples:
   $ scorable evaluator execute-by-name "Helpfulness" \\
       --turns '[{"role":"user","content":"Hello, I need help with my order"},{"role":"assistant","content":"I\\'d be happy to help! What\\'s your order number?"},{"role":"user","content":"It\\'s ORDER-12345"}]'
 
+  # Tool-aware evaluation: assistant tool_calls + tool-role result + tool catalog
+  $ scorable evaluator execute-by-name "Tool Selection" \\
+      --turns '[{"role":"user","content":"What is the weather in Paris?"},{"role":"assistant","content":null,"tool_calls":[{"id":"call_1","type":"function","function":{"name":"get_weather","arguments":"{\\"city\\":\\"Paris\\"}"}}]},{"role":"tool","tool_call_id":"call_1","content":"{\\"temp\\":12}"},{"role":"assistant","content":"It is 12C in Paris."}]' \\
+      --tools '[{"type":"function","function":{"name":"get_weather","description":"Get weather for a city","parameters":{"type":"object","properties":{"city":{"type":"string"}}}}}]'
+
   # Pipe a response from your app via stdin
   $ ./my-app respond "Do you offer refunds?" | \\
       scorable evaluator execute-by-name "Helpfulness" --request "Do you offer refunds?"`)

package/dist/commands/evaluator/execute.d.ts

@@ -4,6 +4,7 @@ export declare function executeEvaluator(evaluatorId: string, opts: {
     request?: string;
     response?: string;
     turns?: string;
+    tools?: string;
     contexts?: string;
     expectedOutput?: string;
     tag?: string[];
@@ -12,6 +13,7 @@ export declare function executeEvaluator(evaluatorId: string, opts: {
     systemPrompt?: string;
     variables?: string;
     fileIds?: string;
+    projectId?: string;
 }, readStdin?: typeof readStdinDefault): Promise<void>;
 export declare function registerExecuteCommand(evaluator: Command): void;
 export {};

package/dist/commands/evaluator/execute.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"execute.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/execute.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,iBAAe,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAIjD;AAED,wBAAsB,gBAAgB,CACpC,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE;IACJ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,EACD,SAAS,0BAAmB,GAC3B,OAAO,CAAC,IAAI,CAAC,CAwFf;AAED,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CA2F/D"}
+{"version":3,"file":"execute.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/execute.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAQpC,iBAAe,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAIjD;AAED,wBAAsB,gBAAgB,CACpC,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE;IACJ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,EACD,SAAS,0BAAmB,GAC3B,OAAO,CAAC,IAAI,CAAC,CAyGf;AAED,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAoG/D"}

package/dist/commands/evaluator/execute.js

@@ -1,7 +1,8 @@
 import ora from "ora";
 import { requireApiKey, getSdkClient } from "../../auth.js";
 import { printSuccess, printError, printJson, handleSdkError } from "../../output.js";
-import { isTurnArray, isStringArray, isStringRecord } from "../../utils.js";
+import { isTurnArray, isToolCatalog, isStringArray, isStringRecord } from "../../utils.js";
+import { resolveProjectIdValue, PROJECT_ID_FLAG_DESC } from "../../lib/project-id.js";
 async function readStdinDefault() {
     const chunks = [];
     for await (const chunk of process.stdin)
@@ -47,6 +48,20 @@ export async function executeEvaluator(evaluatorId, opts, readStdin = readStdinD
             return;
         }
     }
+    if (opts.tools) {
+        try {
+            const parsed = JSON.parse(opts.tools);
+            if (!isToolCatalog(parsed)) {
+                printError("Invalid JSON for --tools. Expected array of tool objects.");
+                return;
+            }
+            payload.tools = parsed;
+        }
+        catch {
+            printError("Invalid JSON for --tools.");
+            return;
+        }
+    }
     if (opts.contexts) {
         try {
             const parsed = JSON.parse(opts.contexts);
@@ -88,6 +103,9 @@ export async function executeEvaluator(evaluatorId, opts, readStdin = readStdinD
         }
         payload.file_ids = parsed;
     }
+    const projectId = resolveProjectIdValue(opts.projectId);
+    if (projectId !== undefined)
+        payload.projectId = projectId;
     const spinner = ora("Running evaluator...").start();
     try {
         const client = getSdkClient(apiKey);
@@ -113,9 +131,11 @@ export function registerExecuteCommand(evaluator) {
         .option("--user-id <id>", "User identifier for tracking purposes")
         .option("--session-id <id>", "Session identifier for tracking purposes")
         .option("--system-prompt <text>", "System prompt that was used for the LLM call")
-        .option("--turns <json>", 'JSON array of conversation turns. E.g., \'[{"role":"user","content":"Hello"}]\'')
+        .option("--turns <json>", 'JSON array of conversation turns. Roles: user|assistant|tool. Assistant turns may carry `tool_calls`; tool results use a `tool` role with `tool_call_id`. E.g., \'[{"role":"user","content":"Hello"}]\'')
+        .option("--tools <json>", "JSON array of OpenAI-style tool definitions available to the agent.")
         .option("--variables <json>", 'JSON object of extra template variables. E.g., \'{"lang":"EN"}\'')
         .option("--file-ids <json>", "JSON array of file UUIDs from 'scorable file upload'. E.g., '[\"uuid1\"]'")
+        .option("--project-id <uuid>", PROJECT_ID_FLAG_DESC)
         .addHelpText("after", `
 Examples:
   # Check helpfulness of a support response
@@ -139,6 +159,11 @@ Examples:
   $ scorable evaluator execute <evaluatorId> \\
       --turns '[{"role":"user","content":"Hello, I need help with my order"},{"role":"assistant","content":"I\\'d be happy to help! What\\'s your order number?"},{"role":"user","content":"It\\'s ORDER-12345"},{"role":"assistant","content":"I found your order. It\\'s currently in transit."}]'
 
+  # Tool-aware evaluation: assistant tool_calls, tool-role result, and tool catalog
+  $ scorable evaluator execute <evaluatorId> \\
+      --turns '[{"role":"user","content":"What is the weather in Paris?"},{"role":"assistant","content":null,"tool_calls":[{"id":"call_1","type":"function","function":{"name":"get_weather","arguments":"{\\"city\\":\\"Paris\\"}"}}]},{"role":"tool","tool_call_id":"call_1","content":"{\\"temp\\":12}"},{"role":"assistant","content":"It is 12C in Paris."}]' \\
+      --tools '[{"type":"function","function":{"name":"get_weather","description":"Get weather for a city","parameters":{"type":"object","properties":{"city":{"type":"string"}}}}}]'
+
   # With custom template variables and tracking
   $ scorable evaluator execute <evaluatorId> \\
       --request "How do I cancel my subscription?" \\

package/dist/commands/evaluator/export-yaml.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"export-yaml.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/export-yaml.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,wBAAgB,yBAAyB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAwBlE"}
+{"version":3,"file":"export-yaml.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/export-yaml.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAiBpC,wBAAgB,yBAAyB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAwBlE"}

package/dist/commands/evaluator/export-yaml.js

@@ -1,7 +1,16 @@
 import { writeFileSync } from "node:fs";
+import yaml from "js-yaml";
 import ora from "ora";
 import { requireApiKey, getSdkClient } from "../../auth.js";
 import { printSuccess, handleSdkError } from "../../output.js";
+function stripProjectId(yamlContent) {
+    const parsed = yaml.load(yamlContent);
+    if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+        delete parsed.project_id;
+        return yaml.dump(parsed);
+    }
+    return yamlContent;
+}
 export function registerExportYamlCommand(evaluator) {
     evaluator
         .command("export-yaml")
@@ -13,7 +22,7 @@ export function registerExportYamlCommand(evaluator) {
         try {
             const apiKey = await requireApiKey();
             const client = getSdkClient(apiKey);
-            const yaml = await client.evaluators.exportYaml(id);
+            const yaml = stripProjectId(await client.evaluators.exportYaml(id));
             spinner.stop();
             if (opts.output) {
                 writeFileSync(opts.output, yaml, "utf8");

package/dist/commands/evaluator/import-yaml.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"import-yaml.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/import-yaml.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,wBAAgB,yBAAyB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CA2ClE"}
+{"version":3,"file":"import-yaml.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/import-yaml.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,yBAAyB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAkDlE"}

package/dist/commands/evaluator/import-yaml.js

@@ -2,12 +2,14 @@ import { readFileSync } from "node:fs";
 import ora from "ora";
 import { requireApiKey, getBaseUrl } from "../../auth.js";
 import { printSuccess, printError, printJson, handleSdkError } from "../../output.js";
+import { resolveProjectIdValue, PROJECT_ID_FLAG_DESC } from "../../lib/project-id.js";
 export function registerImportYamlCommand(evaluator) {
     evaluator
         .command("import-yaml")
         .description("Import an evaluator from a YAML file")
         .requiredOption("--file <path>", "Path to the YAML file to import")
         .option("--overwrite", "Overwrite if an evaluator with the same name already exists")
+        .option("--project-id <uuid>", PROJECT_ID_FLAG_DESC)
         .action(async (opts) => {
         let yamlContent;
         try {
@@ -21,13 +23,20 @@ export function registerImportYamlCommand(evaluator) {
         try {
             const apiKey = await requireApiKey();
             const baseUrl = getBaseUrl();
+            const projectId = resolveProjectIdValue(opts.projectId);
+            const body = {
+                yaml: yamlContent,
+                overwrite: opts.overwrite ?? false,
+            };
+            if (projectId !== undefined)
+                body["project_id"] = projectId;
             const response = await fetch(`${baseUrl}/v1/evaluators/import-yaml/`, {
                 method: "POST",
                 headers: {
                     Authorization: `Api-Key ${apiKey}`,
                     "Content-Type": "application/json",
                 },
-                body: JSON.stringify({ yaml: yamlContent, overwrite: opts.overwrite ?? false }),
+                body: JSON.stringify(body),
             });
             if (!response.ok) {
                 spinner.stop();

package/dist/commands/evaluator/list.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"list.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/list.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CA4C5D"}
+{"version":3,"file":"list.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/list.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAgD5D"}

package/dist/commands/evaluator/list.js

@@ -1,6 +1,7 @@
 import ora from "ora";
 import { requireApiKey, getSdkClient } from "../../auth.js";
 import { printMessage, printEvaluatorTable, handleSdkError } from "../../output.js";
+import { resolveProjectIdValue, PROJECT_ID_FLAG_DESC } from "../../lib/project-id.js";
 export function registerListCommand(evaluator) {
     evaluator
         .command("list")
@@ -10,6 +11,7 @@ export function registerListCommand(evaluator) {
         .option("--search <term>", "A search term to filter by")
         .option("--name <name>", "Filter by exact evaluator name")
         .option("--ordering <field>", "Which field to use for ordering the results")
+        .option("--project-id <uuid>", PROJECT_ID_FLAG_DESC)
         .action(async (opts) => {
         const apiKey = await requireApiKey();
         const params = {};
@@ -23,6 +25,9 @@ export function registerListCommand(evaluator) {
             params.name = opts.name;
         if (opts.ordering !== undefined)
             params.ordering = opts.ordering;
+        const projectId = resolveProjectIdValue(opts.projectId);
+        if (projectId !== undefined)
+            params.projectId = projectId;
         const spinner = ora("Fetching...").start();
         try {
             const client = getSdkClient(apiKey);

package/dist/commands/evaluator/update.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"update.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/update.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAwD9D"}
+{"version":3,"file":"update.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/update.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AASpC,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAkE9D"}

package/dist/commands/evaluator/update.js

@@ -1,6 +1,9 @@
 import ora from "ora";
+import { z } from "zod";
 import { requireApiKey, getSdkClient } from "../../auth.js";
 import { printInfo, printSuccess, printError, printJson, handleSdkError } from "../../output.js";
+import { parseJsonArg } from "../../utils.js";
+import { resolveProjectIdValue, PROJECT_ID_FLAG_DESC } from "../../lib/project-id.js";
 export function registerUpdateCommand(evaluator) {
     evaluator
         .command("update <evaluatorId>")
@@ -10,6 +13,7 @@ export function registerUpdateCommand(evaluator) {
         .option("--models <json>", "JSON array of model names. E.g., '[\"gpt-5.5\"]'")
         .option("--objective-id <id>", "The new objective ID")
         .option("--objective-version-id <id>", "The new objective version ID")
+        .option("--project-id <uuid>", "Move the evaluator to this project (within the same organization). " + PROJECT_ID_FLAG_DESC)
         .action(async (evaluatorId, opts) => {
         const apiKey = await requireApiKey();
         const payload = {};
@@ -22,14 +26,16 @@ export function registerUpdateCommand(evaluator) {
         if (opts.objectiveVersionId !== undefined)
             payload.objective_version_id = opts.objectiveVersionId;
         if (opts.models !== undefined) {
-            try {
-                payload.models = JSON.parse(opts.models);
-            }
-            catch {
-                printError("Invalid JSON format for --models.");
+            const r = parseJsonArg(opts.models, z.array(z.string()));
+            if (!r.ok) {
+                printError("Invalid JSON format for --models. Expected an array of model name strings.");
                 return;
             }
+            payload.models = r.value;
         }
+        const projectId = resolveProjectIdValue(opts.projectId);
+        if (projectId !== undefined)
+            payload.projectId = projectId;
         if (Object.keys(payload).length === 0) {
             printInfo("No update parameters provided. Aborting.");
             return;

package/dist/commands/execution-log/list.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"list.d.ts","sourceRoot":"","sources":["../../../src/commands/execution-log/list.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,mBAAmB,CAAC,YAAY,EAAE,OAAO,GAAG,IAAI,CAiE/D"}
+{"version":3,"file":"list.d.ts","sourceRoot":"","sources":["../../../src/commands/execution-log/list.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,wBAAgB,mBAAmB,CAAC,YAAY,EAAE,OAAO,GAAG,IAAI,CAqE/D"}

package/dist/commands/execution-log/list.js

@@ -1,6 +1,7 @@
 import ora from "ora";
 import { requireApiKey, getSdkClient } from "../../auth.js";
 import { printMessage, printExecutionLogTable, handleSdkError } from "../../output.js";
+import { resolveProjectIdValue, PROJECT_ID_FLAG_DESC } from "../../lib/project-id.js";
 export function registerListCommand(executionLog) {
     executionLog
         .command("list")
@@ -17,6 +18,7 @@ export function registerListCommand(executionLog) {
         .option("--created-at-after <date>", "Filter logs created after this date (ISO 8601)")
         .option("--created-at-before <date>", "Filter logs created before this date (ISO 8601)")
         .option("--owner-email <email>", "Filter by owner email")
+        .option("--project-id <uuid>", PROJECT_ID_FLAG_DESC)
         .action(async (opts) => {
         const apiKey = await requireApiKey();
         const params = {};
@@ -44,6 +46,9 @@ export function registerListCommand(executionLog) {
             params.created_at_before = opts.createdAtBefore;
         if (opts.ownerEmail !== undefined)
             params.owner__email = opts.ownerEmail;
+        const projectId = resolveProjectIdValue(opts.projectId);
+        if (projectId !== undefined)
+            params.projectId = projectId;
         const spinner = ora("Fetching...").start();
         try {
             const client = getSdkClient(apiKey);

package/dist/commands/judge/create.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"create.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CA+C1D"}
+{"version":3,"file":"create.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAWpC,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAoD1D"}