@root-signals/scorable-cli 0.13.0 → 0.15.0

package/README.md

@@ -78,6 +78,66 @@ export SCORABLE_API_KEY="sk-your-api-key"
 
 The key lookup order is: `SCORABLE_API_KEY` env var → `api_key` in `~/.scorable/settings.json` → `temporary_api_key` in `~/.scorable/settings.json`.
 
+## Projects
+
+A **project** is a workspace inside your organization that groups related judges, evaluators, datasets, and execution logs. Every resource belongs to a project; omitting the project at create/execute time files things under the org's default project.
+
+### Manage projects
+
+```bash
+scorable project list                                   # list all projects (default marked)
+scorable project get <project_id>                       # show a single project
+scorable project create --name "Production"             # create a project
+scorable project create --name "Production" --is-default
+scorable project update <project_id> --name "Renamed"
+scorable project update <project_id> --is-default       # promote to default
+scorable project set-default <project_id>               # convenience for `update --is-default`
+scorable project delete <project_id>
+```
+
+### `--project-id` on every project-aware command
+
+Every command that creates, executes, lists, or filters a project-scoped resource accepts `--project-id <uuid>`:
+
+```bash
+# Filter list endpoints
+scorable judge list --project-id <project_id>
+scorable evaluator list --project-id <project_id>
+scorable execution-log list --project-id <project_id>
+
+# Route an execution log to a project
+scorable judge execute <judge_id> --project-id <project_id>
+scorable evaluator execute <evaluator_id> --project-id <project_id>
+
+# Pin a resource to a project at creation
+scorable judge create --name X --intent Y --project-id <project_id>
+scorable evaluator import-yaml --file evaluator.yaml --project-id <project_id>
+
+# Move a resource between projects
+scorable judge update <judge_id> --project-id <other_project_id>
+
+# OpenAI-compat (translated to X-Project-Id header)
+scorable judge exec-openai <judge_id> --project-id <project_id>
+```
+
+### Setting a default project for your shell
+
+To avoid passing `--project-id` on every command, set an env var or persist a per-machine default:
+
+```bash
+# Environment variable (great for CI)
+export SCORABLE_PROJECT_ID=<project_id>
+
+# Persistent default written to ~/.scorable/settings.json
+scorable auth set-project <project_id>
+scorable auth show-project    # see what's resolved and from where
+scorable auth unset-project   # remove the saved default
+```
+
+Resolution order: `--project-id` flag → `SCORABLE_PROJECT_ID` env var → `project_id` in `~/.scorable/settings.json` → omitted (backend resolves to org default). Pass `--project-id ""` to explicitly opt out of an inherited default for a single invocation.
+
+`scorable auth logout` clears the entire auth section of `~/.scorable/settings.json`, including the saved project_id.
+
 ## Scorable Skills for AI Coding Agents
 
 Install Scorable skills into your project so your AI coding agent (Claude Code, Cursor, etc.) can integrate evaluators automatically:
@@ -276,6 +336,59 @@ scorable evaluator execute-by-name "My Evaluator" --request "What is 2+2?" --res
 
 Accepts the same options as `execute`, including `--variables`.
 
+## Labelling & Calibration
+
+Build a labelled dataset, then calibrate a saved evaluator against it to measure how well it agrees with your expected scores.
+
+### Dataset items
+
+Populate a `test`-type dataset with request/response items.
+
+```bash
+scorable dataset-item add <dataset_id> --response "Paris" --request "Capital of France?"
+scorable dataset-item add-bulk <dataset_id> --items '[{"response":"Paris","request":"Capital of France?"}]'
+scorable dataset-item list <dataset_id>                       # latest-version items (with embedded annotations)
+scorable dataset-item list <dataset_id> --include-archived
+scorable dataset-item get <dataset_id> <item_id>
+scorable dataset-item update <dataset_id> <item_id> --response "Paris, France"
+scorable dataset-item archive <dataset_id> <item_id>
+```
+
+### Annotations
+
+Label a dataset item (or execution log) with an expected score. Omitting `--score-config-id` uses the global identity "Score" config, so a raw expected score can be set with just `--value`.
+
+```bash
+scorable annotation create --dataset-item-id <item_id> --value 0.9
+scorable annotation create --dataset-item-id <item_id> --category "👍" --score-config-id <config_id>
+scorable annotation list --dataset <dataset_id>
+scorable annotation get <annotation_id>
+scorable annotation update <annotation_id> --value 0.8
+scorable annotation delete <annotation_id>
+```
+
+### Calibration runs
+
+Start a calibration run against a labelled dataset and read its per-example results (human vs. evaluator score, disagreement, and the request/response that was scored).
+
+```bash
+scorable evaluator calibrate <evaluator_id> --dataset-id <dataset_id>
+# ...or the equivalent calibration-run command:
+scorable calibration-run create --evaluator-id <evaluator_id> --dataset-id <dataset_id>
+scorable calibration-run get <run_id>                         # poll for status + metrics
+scorable calibration-run list --evaluator-id <evaluator_id>
+scorable calibration-run items <run_id>                       # per-example results
+```
+
+### Demonstrations
+
+Attach a labelled dataset as few-shot demonstrations on an evaluator.
+
+```bash
+scorable evaluator create --name "My Evaluator" ... --demonstration-dataset <dataset_id>
+scorable evaluator update <evaluator_id> --demonstration-dataset <dataset_id>
+```
+
 ## Custom Model Management
 
 Bring your own LLM (BYO-LLM) — register a custom or self-hosted model, then reference it from evaluators and judges.

package/dist/client.d.ts

@@ -3,6 +3,7 @@ interface RequestOptions {
     params?: Record<string, unknown>;
     apiKey?: string;
     baseUrl?: string;
+    headers?: Record<string, string>;
 }
 export interface ApiRequestResult {
     ok: boolean;

package/dist/client.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAQA,UAAU,cAAc;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,OAAO,CAAC;CACf;AAQD,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,gBAAgB,CAAC,CAoD3B;AAKD,wBAAsB,UAAU,CAC9B,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,OAAO,CAAC,CAGlB"}
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAQA,UAAU,cAAc;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,OAAO,CAAC;CACf;AAQD,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,gBAAgB,CAAC,CAqD3B;AAKD,wBAAsB,UAAU,CAC9B,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,OAAO,CAAC,CAGlB"}

package/dist/client.js

@@ -15,6 +15,7 @@ export async function apiRequestStatus(method, endpoint, options) {
         }
     }
     const headers = {
+        ...options?.headers,
         Authorization: `Api-Key ${apiKey}`,
         "Content-Type": "application/json",
         Accept: "application/json",

package/dist/commands/annotation/index.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerAnnotationCommands(program: Command): void;
+//# sourceMappingURL=index.d.ts.map

package/dist/commands/annotation/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/commands/annotation/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,0BAA0B,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAiKjE"}

package/dist/commands/annotation/index.js

@@ -0,0 +1,145 @@
+import { Command } from "commander";
+import ora from "ora";
+import { requireApiKey, getSdkClient } from "../../auth.js";
+import { printSuccess, printError, printJson, handleSdkError } from "../../output.js";
+export function registerAnnotationCommands(program) {
+    const annotation = new Command("annotation").description("Annotation (labelling) commands");
+    annotation
+        .command("create")
+        .description("Create an annotation on a dataset item or execution log")
+        .option("--dataset-item-id <id>", "Dataset item to annotate (mutually exclusive with --execution-log-id)")
+        .option("--execution-log-id <id>", "Execution log to annotate (mutually exclusive with --dataset-item-id)")
+        .option("--value <number>", "Score for continuous configs", parseFloat)
+        .option("--category <label>", "Label for binary/categorical configs")
+        .option("--rationale <text>", "Optional free-text justification")
+        .option("--status <status>", "draft or published (default published)")
+        .option("--score-config-id <id>", "Score config; defaults to the global identity 'Score' config")
+        .action(async (opts) => {
+        if (!opts.datasetItemId === !opts.executionLogId) {
+            printError("Exactly one of --dataset-item-id or --execution-log-id must be provided.");
+            return;
+        }
+        const payload = {};
+        if (opts.datasetItemId)
+            payload.datasetItemId = opts.datasetItemId;
+        if (opts.executionLogId)
+            payload.executionLogId = opts.executionLogId;
+        if (opts.value !== undefined)
+            payload.value = opts.value;
+        if (opts.category !== undefined)
+            payload.category = opts.category;
+        if (opts.rationale !== undefined)
+            payload.rationale = opts.rationale;
+        if (opts.status !== undefined)
+            payload.status = opts.status;
+        if (opts.scoreConfigId)
+            payload.scoreConfigId = opts.scoreConfigId;
+        const spinner = ora("Creating annotation...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            const result = await client.annotations.create(payload);
+            spinner.stop();
+            printSuccess("Annotation created!");
+            printJson(result);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    annotation
+        .command("list")
+        .description("List annotations")
+        .option("--dataset <id>", "Filter by the dataset the annotated items belong to")
+        .option("--score-config <id>", "Filter by score config id")
+        .option("--status <status>", "Filter by draft or published")
+        .option("--dataset-item <id>", "Filter by dataset item id")
+        .option("--execution-log <id>", "Filter by execution log id")
+        .action(async (opts) => {
+        const params = {};
+        if (opts.dataset)
+            params.dataset = opts.dataset;
+        if (opts.scoreConfig)
+            params.score_config = opts.scoreConfig;
+        if (opts.status)
+            params.status = opts.status;
+        if (opts.datasetItem)
+            params.dataset_item = opts.datasetItem;
+        if (opts.executionLog)
+            params.execution_log = opts.executionLog;
+        const spinner = ora("Listing annotations...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            const result = await client.annotations.list(params);
+            spinner.stop();
+            printJson(result.results);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    annotation
+        .command("get <id>")
+        .description("Get an annotation by ID")
+        .action(async (id) => {
+        const spinner = ora("Fetching annotation...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            const result = await client.annotations.get(id);
+            spinner.stop();
+            printJson(result);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    annotation
+        .command("update <id>")
+        .description("Update an annotation (target is immutable)")
+        .option("--value <number>", "New score for continuous configs", parseFloat)
+        .option("--category <label>", "New label for binary/categorical configs")
+        .option("--rationale <text>", "New rationale")
+        .option("--status <status>", "draft or published")
+        .action(async (id, opts) => {
+        const payload = {};
+        if (opts.value !== undefined)
+            payload.value = opts.value;
+        if (opts.category !== undefined)
+            payload.category = opts.category;
+        if (opts.rationale !== undefined)
+            payload.rationale = opts.rationale;
+        if (opts.status !== undefined)
+            payload.status = opts.status;
+        const spinner = ora("Updating annotation...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            const result = await client.annotations.update(id, payload);
+            spinner.stop();
+            printSuccess("Annotation updated!");
+            printJson(result);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    annotation
+        .command("delete <id>")
+        .description("Delete an annotation")
+        .action(async (id) => {
+        const spinner = ora("Deleting annotation...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            await client.annotations.delete(id);
+            spinner.stop();
+            printSuccess(`Annotation ${id} deleted.`);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    program.addCommand(annotation);
+}

package/dist/commands/auth/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/commands/auth/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAgE3D"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/commands/auth/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAUpC,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAsI3D"}

package/dist/commands/auth/index.js

@@ -1,7 +1,9 @@
 import ora from "ora";
 import { loadSettings, saveSettings, createDemoKey } from "../../auth.js";
-import { printSuccess, printError } from "../../output.js";
+import { printSuccess, printError, printInfo, printMessage } from "../../output.js";
+import { resolveProjectId } from "../../lib/project-id.js";
 import { CliError } from "../../types.js";
+const AUTH_SETTINGS_KEYS = ["api_key", "temporary_api_key", "project_id"];
 export function registerAuthCommands(program) {
     const auth = program.command("auth").description("Manage authentication");
     auth
@@ -39,6 +41,71 @@ export function registerAuthCommands(program) {
         }
         printSuccess("API key saved to ~/.scorable/settings.json");
     });
+    auth
+        .command("set-project <projectId>")
+        .description("Persist a project_id to ~/.scorable/settings.json")
+        .action((projectId) => {
+        if (!projectId.trim()) {
+            printError("Project ID must not be empty.");
+            throw new CliError(1, "Empty project id");
+        }
+        const settings = loadSettings();
+        settings["project_id"] = projectId.trim();
+        if (!saveSettings(settings)) {
+            printError("Failed to save project_id to ~/.scorable/settings.json");
+            throw new CliError(1, "Failed to save project_id");
+        }
+        printSuccess(`project_id saved to ~/.scorable/settings.json`);
+    });
+    auth
+        .command("unset-project")
+        .description("Remove the persisted project_id from ~/.scorable/settings.json")
+        .action(() => {
+        const settings = loadSettings();
+        if (!("project_id" in settings)) {
+            printInfo("No project_id was set.");
+            return;
+        }
+        delete settings["project_id"];
+        if (!saveSettings(settings)) {
+            printError("Failed to update ~/.scorable/settings.json");
+            throw new CliError(1, "Failed to update settings");
+        }
+        printSuccess("project_id removed from ~/.scorable/settings.json");
+    });
+    auth
+        .command("show-project")
+        .description("Print the resolved project_id and its source (flag/env/settings/none)")
+        .action(() => {
+        const resolved = resolveProjectId(undefined);
+        if (resolved.value === undefined) {
+            printMessage("project_id: <none> (source: none)");
+            return;
+        }
+        printMessage(`project_id: ${resolved.value} (source: ${resolved.source})`);
+    });
+    auth
+        .command("logout")
+        .description("Clear the auth section of ~/.scorable/settings.json (api keys + project_id)")
+        .action(() => {
+        const settings = loadSettings();
+        let touched = false;
+        for (const k of AUTH_SETTINGS_KEYS) {
+            if (k in settings) {
+                delete settings[k];
+                touched = true;
+            }
+        }
+        if (!touched) {
+            printInfo("Already logged out.");
+            return;
+        }
+        if (!saveSettings(settings)) {
+            printError("Failed to update ~/.scorable/settings.json");
+            throw new CliError(1, "Failed to update settings");
+        }
+        printSuccess("Logged out — cleared api_key, temporary_api_key, and project_id.");
+    });
     auth
         .command("demo-key")
         .description("Create a free demo API key and save it to ~/.scorable/settings.json")

package/dist/commands/calibration-run/index.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerCalibrationRunCommands(program: Command): void;
+//# sourceMappingURL=index.d.ts.map

package/dist/commands/calibration-run/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/commands/calibration-run/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,8BAA8B,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAiFrE"}

package/dist/commands/calibration-run/index.js

@@ -0,0 +1,83 @@
+import { Command } from "commander";
+import ora from "ora";
+import { requireApiKey, getSdkClient } from "../../auth.js";
+import { printSuccess, printJson, handleSdkError } from "../../output.js";
+export function registerCalibrationRunCommands(program) {
+    const run = new Command("calibration-run").description("Calibration run commands");
+    run
+        .command("create")
+        .description("Start a calibration run against a labelled dataset")
+        .requiredOption("--evaluator-id <id>", "The saved evaluator to calibrate")
+        .requiredOption("--dataset-id <id>", "Dataset whose published annotations to calibrate against")
+        .option("--score-config-id <id>", "Score config (defaults to the dataset's continuous scores)")
+        .action(async (opts) => {
+        const payload = {
+            evaluatorId: opts.evaluatorId,
+            datasetId: opts.datasetId,
+        };
+        if (opts.scoreConfigId)
+            payload.scoreConfigId = opts.scoreConfigId;
+        const spinner = ora("Starting calibration run...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            const result = await client.calibrationRuns.create(payload);
+            spinner.stop();
+            printSuccess(`Calibration run ${result.id} started (status: ${result.status}).`);
+            printJson(result);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    run
+        .command("get <id>")
+        .description("Get a calibration run (poll for status and metrics)")
+        .action(async (id) => {
+        const spinner = ora("Fetching calibration run...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            const result = await client.calibrationRuns.get(id);
+            spinner.stop();
+            printJson(result);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    run
+        .command("list")
+        .description("List calibration runs")
+        .option("--evaluator-id <id>", "Filter by evaluator external id")
+        .action(async (opts) => {
+        const spinner = ora("Listing calibration runs...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            const result = await client.calibrationRuns.list(opts.evaluatorId ? { evaluatorId: opts.evaluatorId } : {});
+            spinner.stop();
+            printJson(result.results);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    run
+        .command("items <id>")
+        .description("List the per-example results of a calibration run")
+        .action(async (id) => {
+        const spinner = ora("Fetching calibration run items...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            const result = await client.calibrationRuns.listItems(id);
+            spinner.stop();
+            printJson(result.results);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    program.addCommand(run);
+}

package/dist/commands/dataset-item/index.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerDatasetItemCommands(program: Command): void;
+//# sourceMappingURL=index.d.ts.map

package/dist/commands/dataset-item/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/commands/dataset-item/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAQpC,wBAAgB,2BAA2B,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CA4KlE"}

package/dist/commands/dataset-item/index.js

@@ -0,0 +1,160 @@
+import { Command } from "commander";
+import ora from "ora";
+import { z } from "zod";
+import { requireApiKey, getSdkClient } from "../../auth.js";
+import { printSuccess, printError, printJson, handleSdkError } from "../../output.js";
+import { parseJsonArg } from "../../utils.js";
+export function registerDatasetItemCommands(program) {
+    const item = new Command("dataset-item").description("Dataset item commands");
+    item
+        .command("add <datasetId>")
+        .description("Add a single item to a dataset")
+        .requiredOption("--response <text>", "The response text")
+        .option("--request <text>", "The request/prompt text")
+        .option("--expected-output <text>", "The expected output")
+        .option("--metadata <json>", "Metadata JSON object")
+        .option("--change-note <text>", "Change note for this version")
+        .action(async (datasetId, opts) => {
+        const body = { response: opts.response };
+        if (opts.request !== undefined)
+            body.request = opts.request;
+        if (opts.expectedOutput !== undefined)
+            body.expected_output = opts.expectedOutput;
+        if (opts.changeNote !== undefined)
+            body.change_note = opts.changeNote;
+        if (opts.metadata !== undefined) {
+            const r = parseJsonArg(opts.metadata, z.record(z.string(), z.unknown()));
+            if (!r.ok) {
+                printError("Invalid JSON for --metadata. Expected an object.");
+                return;
+            }
+            body.metadata = r.value;
+        }
+        const spinner = ora("Adding item...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            const result = await client.datasets.addItem(datasetId, body);
+            spinner.stop();
+            printSuccess("Dataset item added!");
+            printJson(result);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    item
+        .command("add-bulk <datasetId>")
+        .description("Bulk add items to a dataset (at most 5000)")
+        .requiredOption("--items <json>", "JSON array of item objects")
+        .action(async (datasetId, opts) => {
+        const r = parseJsonArg(opts.items, z.array(z.record(z.string(), z.unknown())));
+        if (!r.ok) {
+            printError("Invalid JSON for --items. Expected an array of item objects.");
+            return;
+        }
+        const spinner = ora("Bulk adding items...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            const result = await client.datasets.addItems(datasetId, r.value);
+            spinner.stop();
+            printSuccess(`Added ${result.length} dataset items.`);
+            printJson(result);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    item
+        .command("list <datasetId>")
+        .description("List the latest-version items of a dataset (with embedded annotations)")
+        .option("--include-archived", "Include archived items", false)
+        .action(async (datasetId, opts) => {
+        const spinner = ora("Listing items...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            const result = await client.datasets.listItems(datasetId, {
+                includeArchived: opts.includeArchived ?? false,
+            });
+            spinner.stop();
+            printJson(result.results);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    item
+        .command("get <datasetId> <itemId>")
+        .description("Get a single dataset item")
+        .action(async (datasetId, itemId) => {
+        const spinner = ora("Fetching item...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            const result = await client.datasets.getItem(datasetId, itemId);
+            spinner.stop();
+            printJson(result);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    item
+        .command("update <datasetId> <itemId>")
+        .description("Edit a dataset item (creates a new version)")
+        .option("--response <text>", "The response text")
+        .option("--request <text>", "The request/prompt text")
+        .option("--expected-output <text>", "The expected output")
+        .option("--metadata <json>", "Metadata JSON object")
+        .option("--change-note <text>", "Change note for this version")
+        .action(async (datasetId, itemId, opts) => {
+        const body = {};
+        if (opts.response !== undefined)
+            body.response = opts.response;
+        if (opts.request !== undefined)
+            body.request = opts.request;
+        if (opts.expectedOutput !== undefined)
+            body.expected_output = opts.expectedOutput;
+        if (opts.changeNote !== undefined)
+            body.change_note = opts.changeNote;
+        if (opts.metadata !== undefined) {
+            const r = parseJsonArg(opts.metadata, z.record(z.string(), z.unknown()));
+            if (!r.ok) {
+                printError("Invalid JSON for --metadata. Expected an object.");
+                return;
+            }
+            body.metadata = r.value;
+        }
+        const spinner = ora("Updating item...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            const result = await client.datasets.updateItem(datasetId, itemId, body);
+            spinner.stop();
+            printSuccess("Dataset item updated!");
+            printJson(result);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    item
+        .command("archive <datasetId> <itemId>")
+        .description("Archive (soft-delete) a dataset item")
+        .action(async (datasetId, itemId) => {
+        const spinner = ora("Archiving item...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            await client.datasets.archiveItem(datasetId, itemId);
+            spinner.stop();
+            printSuccess(`Dataset item ${itemId} archived.`);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    program.addCommand(item);
+}

package/dist/commands/evaluator/calibrate.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerCalibrateCommand(evaluator: Command): void;
+//# sourceMappingURL=calibrate.d.ts.map

package/dist/commands/evaluator/calibrate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"calibrate.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/calibrate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,wBAAgB,wBAAwB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CA6BjE"}