@root-signals/scorable-cli 0.14.0 → 0.15.1

package/README.md

@@ -336,6 +336,59 @@ scorable evaluator execute-by-name "My Evaluator" --request "What is 2+2?" --res
 
 Accepts the same options as `execute`, including `--variables`.
 
+## Labelling & Calibration
+
+Build a labelled dataset, then calibrate a saved evaluator against it to measure how well it agrees with your expected scores.
+
+### Dataset items
+
+Populate a `test`-type dataset with request/response items.
+
+```bash
+scorable dataset-item add <dataset_id> --response "Paris" --request "Capital of France?"
+scorable dataset-item add-bulk <dataset_id> --items '[{"response":"Paris","request":"Capital of France?"}]'
+scorable dataset-item list <dataset_id>                       # latest-version items (with embedded annotations)
+scorable dataset-item list <dataset_id> --include-archived
+scorable dataset-item get <dataset_id> <item_id>
+scorable dataset-item update <dataset_id> <item_id> --response "Paris, France"
+scorable dataset-item archive <dataset_id> <item_id>
+```
+
+### Annotations
+
+Label a dataset item (or execution log) with an expected score. Omitting `--score-config-id` uses the global identity "Score" config, so a raw expected score can be set with just `--value`.
+
+```bash
+scorable annotation create --dataset-item-id <item_id> --value 0.9
+scorable annotation create --dataset-item-id <item_id> --category "👍" --score-config-id <config_id>
+scorable annotation list --dataset <dataset_id>
+scorable annotation get <annotation_id>
+scorable annotation update <annotation_id> --value 0.8
+scorable annotation delete <annotation_id>
+```
+
+### Calibration runs
+
+Start a calibration run against a labelled dataset and read its per-example results (human vs. evaluator score, disagreement, and the request/response that was scored).
+
+```bash
+scorable evaluator calibrate <evaluator_id> --dataset-id <dataset_id>
+# ...or the equivalent calibration-run command:
+scorable calibration-run create --evaluator-id <evaluator_id> --dataset-id <dataset_id>
+scorable calibration-run get <run_id>                         # poll for status + metrics
+scorable calibration-run list --evaluator-id <evaluator_id>
+scorable calibration-run items <run_id>                       # per-example results
+```
+
+### Demonstrations
+
+Attach a labelled dataset as few-shot demonstrations on an evaluator.
+
+```bash
+scorable evaluator create --name "My Evaluator" ... --demonstration-dataset <dataset_id>
+scorable evaluator update <evaluator_id> --demonstration-dataset <dataset_id>
+```
+
 ## Custom Model Management
 
 Bring your own LLM (BYO-LLM) — register a custom or self-hosted model, then reference it from evaluators and judges.

package/dist/commands/annotation/index.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerAnnotationCommands(program: Command): void;
+//# sourceMappingURL=index.d.ts.map

package/dist/commands/annotation/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/commands/annotation/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,0BAA0B,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAiKjE"}

package/dist/commands/annotation/index.js

@@ -0,0 +1,145 @@
+import { Command } from "commander";
+import ora from "ora";
+import { requireApiKey, getSdkClient } from "../../auth.js";
+import { printSuccess, printError, printJson, handleSdkError } from "../../output.js";
+export function registerAnnotationCommands(program) {
+    const annotation = new Command("annotation").description("Annotation (labelling) commands");
+    annotation
+        .command("create")
+        .description("Create an annotation on a dataset item or execution log")
+        .option("--dataset-item-id <id>", "Dataset item to annotate (mutually exclusive with --execution-log-id)")
+        .option("--execution-log-id <id>", "Execution log to annotate (mutually exclusive with --dataset-item-id)")
+        .option("--value <number>", "Score for continuous configs", parseFloat)
+        .option("--category <label>", "Label for binary/categorical configs")
+        .option("--rationale <text>", "Optional free-text justification")
+        .option("--status <status>", "draft or published (default published)")
+        .option("--score-config-id <id>", "Score config; defaults to the global identity 'Score' config")
+        .action(async (opts) => {
+        if (!opts.datasetItemId === !opts.executionLogId) {
+            printError("Exactly one of --dataset-item-id or --execution-log-id must be provided.");
+            return;
+        }
+        const payload = {};
+        if (opts.datasetItemId)
+            payload.datasetItemId = opts.datasetItemId;
+        if (opts.executionLogId)
+            payload.executionLogId = opts.executionLogId;
+        if (opts.value !== undefined)
+            payload.value = opts.value;
+        if (opts.category !== undefined)
+            payload.category = opts.category;
+        if (opts.rationale !== undefined)
+            payload.rationale = opts.rationale;
+        if (opts.status !== undefined)
+            payload.status = opts.status;
+        if (opts.scoreConfigId)
+            payload.scoreConfigId = opts.scoreConfigId;
+        const spinner = ora("Creating annotation...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            const result = await client.annotations.create(payload);
+            spinner.stop();
+            printSuccess("Annotation created!");
+            printJson(result);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    annotation
+        .command("list")
+        .description("List annotations")
+        .option("--dataset <id>", "Filter by the dataset the annotated items belong to")
+        .option("--score-config <id>", "Filter by score config id")
+        .option("--status <status>", "Filter by draft or published")
+        .option("--dataset-item <id>", "Filter by dataset item id")
+        .option("--execution-log <id>", "Filter by execution log id")
+        .action(async (opts) => {
+        const params = {};
+        if (opts.dataset)
+            params.dataset = opts.dataset;
+        if (opts.scoreConfig)
+            params.score_config = opts.scoreConfig;
+        if (opts.status)
+            params.status = opts.status;
+        if (opts.datasetItem)
+            params.dataset_item = opts.datasetItem;
+        if (opts.executionLog)
+            params.execution_log = opts.executionLog;
+        const spinner = ora("Listing annotations...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            const result = await client.annotations.list(params);
+            spinner.stop();
+            printJson(result.results);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    annotation
+        .command("get <id>")
+        .description("Get an annotation by ID")
+        .action(async (id) => {
+        const spinner = ora("Fetching annotation...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            const result = await client.annotations.get(id);
+            spinner.stop();
+            printJson(result);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    annotation
+        .command("update <id>")
+        .description("Update an annotation (target is immutable)")
+        .option("--value <number>", "New score for continuous configs", parseFloat)
+        .option("--category <label>", "New label for binary/categorical configs")
+        .option("--rationale <text>", "New rationale")
+        .option("--status <status>", "draft or published")
+        .action(async (id, opts) => {
+        const payload = {};
+        if (opts.value !== undefined)
+            payload.value = opts.value;
+        if (opts.category !== undefined)
+            payload.category = opts.category;
+        if (opts.rationale !== undefined)
+            payload.rationale = opts.rationale;
+        if (opts.status !== undefined)
+            payload.status = opts.status;
+        const spinner = ora("Updating annotation...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            const result = await client.annotations.update(id, payload);
+            spinner.stop();
+            printSuccess("Annotation updated!");
+            printJson(result);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    annotation
+        .command("delete <id>")
+        .description("Delete an annotation")
+        .action(async (id) => {
+        const spinner = ora("Deleting annotation...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            await client.annotations.delete(id);
+            spinner.stop();
+            printSuccess(`Annotation ${id} deleted.`);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    program.addCommand(annotation);
+}

package/dist/commands/calibration-run/index.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerCalibrationRunCommands(program: Command): void;
+//# sourceMappingURL=index.d.ts.map

package/dist/commands/calibration-run/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/commands/calibration-run/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,8BAA8B,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAiFrE"}

package/dist/commands/calibration-run/index.js

@@ -0,0 +1,83 @@
+import { Command } from "commander";
+import ora from "ora";
+import { requireApiKey, getSdkClient } from "../../auth.js";
+import { printSuccess, printJson, handleSdkError } from "../../output.js";
+export function registerCalibrationRunCommands(program) {
+    const run = new Command("calibration-run").description("Calibration run commands");
+    run
+        .command("create")
+        .description("Start a calibration run against a labelled dataset")
+        .requiredOption("--evaluator-id <id>", "The saved evaluator to calibrate")
+        .requiredOption("--dataset-id <id>", "Dataset whose published annotations to calibrate against")
+        .option("--score-config-id <id>", "Score config (defaults to the dataset's continuous scores)")
+        .action(async (opts) => {
+        const payload = {
+            evaluatorId: opts.evaluatorId,
+            datasetId: opts.datasetId,
+        };
+        if (opts.scoreConfigId)
+            payload.scoreConfigId = opts.scoreConfigId;
+        const spinner = ora("Starting calibration run...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            const result = await client.calibrationRuns.create(payload);
+            spinner.stop();
+            printSuccess(`Calibration run ${result.id} started (status: ${result.status}).`);
+            printJson(result);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    run
+        .command("get <id>")
+        .description("Get a calibration run (poll for status and metrics)")
+        .action(async (id) => {
+        const spinner = ora("Fetching calibration run...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            const result = await client.calibrationRuns.get(id);
+            spinner.stop();
+            printJson(result);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    run
+        .command("list")
+        .description("List calibration runs")
+        .option("--evaluator-id <id>", "Filter by evaluator external id")
+        .action(async (opts) => {
+        const spinner = ora("Listing calibration runs...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            const result = await client.calibrationRuns.list(opts.evaluatorId ? { evaluatorId: opts.evaluatorId } : {});
+            spinner.stop();
+            printJson(result.results);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    run
+        .command("items <id>")
+        .description("List the per-example results of a calibration run")
+        .action(async (id) => {
+        const spinner = ora("Fetching calibration run items...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            const result = await client.calibrationRuns.listItems(id);
+            spinner.stop();
+            printJson(result.results);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    program.addCommand(run);
+}

package/dist/commands/dataset-item/index.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerDatasetItemCommands(program: Command): void;
+//# sourceMappingURL=index.d.ts.map

package/dist/commands/dataset-item/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/commands/dataset-item/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAQpC,wBAAgB,2BAA2B,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CA4KlE"}

package/dist/commands/dataset-item/index.js

@@ -0,0 +1,160 @@
+import { Command } from "commander";
+import ora from "ora";
+import { z } from "zod";
+import { requireApiKey, getSdkClient } from "../../auth.js";
+import { printSuccess, printError, printJson, handleSdkError } from "../../output.js";
+import { parseJsonArg } from "../../utils.js";
+export function registerDatasetItemCommands(program) {
+    const item = new Command("dataset-item").description("Dataset item commands");
+    item
+        .command("add <datasetId>")
+        .description("Add a single item to a dataset")
+        .requiredOption("--response <text>", "The response text")
+        .option("--request <text>", "The request/prompt text")
+        .option("--expected-output <text>", "The expected output")
+        .option("--metadata <json>", "Metadata JSON object")
+        .option("--change-note <text>", "Change note for this version")
+        .action(async (datasetId, opts) => {
+        const body = { response: opts.response };
+        if (opts.request !== undefined)
+            body.request = opts.request;
+        if (opts.expectedOutput !== undefined)
+            body.expected_output = opts.expectedOutput;
+        if (opts.changeNote !== undefined)
+            body.change_note = opts.changeNote;
+        if (opts.metadata !== undefined) {
+            const r = parseJsonArg(opts.metadata, z.record(z.string(), z.unknown()));
+            if (!r.ok) {
+                printError("Invalid JSON for --metadata. Expected an object.");
+                return;
+            }
+            body.metadata = r.value;
+        }
+        const spinner = ora("Adding item...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            const result = await client.datasets.addItem(datasetId, body);
+            spinner.stop();
+            printSuccess("Dataset item added!");
+            printJson(result);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    item
+        .command("add-bulk <datasetId>")
+        .description("Bulk add items to a dataset (at most 5000)")
+        .requiredOption("--items <json>", "JSON array of item objects")
+        .action(async (datasetId, opts) => {
+        const r = parseJsonArg(opts.items, z.array(z.record(z.string(), z.unknown())));
+        if (!r.ok) {
+            printError("Invalid JSON for --items. Expected an array of item objects.");
+            return;
+        }
+        const spinner = ora("Bulk adding items...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            const result = await client.datasets.addItems(datasetId, r.value);
+            spinner.stop();
+            printSuccess(`Added ${result.length} dataset items.`);
+            printJson(result);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    item
+        .command("list <datasetId>")
+        .description("List the latest-version items of a dataset (with embedded annotations)")
+        .option("--include-archived", "Include archived items", false)
+        .action(async (datasetId, opts) => {
+        const spinner = ora("Listing items...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            const result = await client.datasets.listItems(datasetId, {
+                includeArchived: opts.includeArchived ?? false,
+            });
+            spinner.stop();
+            printJson(result.results);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    item
+        .command("get <datasetId> <itemId>")
+        .description("Get a single dataset item")
+        .action(async (datasetId, itemId) => {
+        const spinner = ora("Fetching item...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            const result = await client.datasets.getItem(datasetId, itemId);
+            spinner.stop();
+            printJson(result);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    item
+        .command("update <datasetId> <itemId>")
+        .description("Edit a dataset item (creates a new version)")
+        .option("--response <text>", "The response text")
+        .option("--request <text>", "The request/prompt text")
+        .option("--expected-output <text>", "The expected output")
+        .option("--metadata <json>", "Metadata JSON object")
+        .option("--change-note <text>", "Change note for this version")
+        .action(async (datasetId, itemId, opts) => {
+        const body = {};
+        if (opts.response !== undefined)
+            body.response = opts.response;
+        if (opts.request !== undefined)
+            body.request = opts.request;
+        if (opts.expectedOutput !== undefined)
+            body.expected_output = opts.expectedOutput;
+        if (opts.changeNote !== undefined)
+            body.change_note = opts.changeNote;
+        if (opts.metadata !== undefined) {
+            const r = parseJsonArg(opts.metadata, z.record(z.string(), z.unknown()));
+            if (!r.ok) {
+                printError("Invalid JSON for --metadata. Expected an object.");
+                return;
+            }
+            body.metadata = r.value;
+        }
+        const spinner = ora("Updating item...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            const result = await client.datasets.updateItem(datasetId, itemId, body);
+            spinner.stop();
+            printSuccess("Dataset item updated!");
+            printJson(result);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    item
+        .command("archive <datasetId> <itemId>")
+        .description("Archive (soft-delete) a dataset item")
+        .action(async (datasetId, itemId) => {
+        const spinner = ora("Archiving item...").start();
+        try {
+            const client = getSdkClient(await requireApiKey());
+            await client.datasets.archiveItem(datasetId, itemId);
+            spinner.stop();
+            printSuccess(`Dataset item ${itemId} archived.`);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+    program.addCommand(item);
+}

package/dist/commands/evaluator/calibrate.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerCalibrateCommand(evaluator: Command): void;
+//# sourceMappingURL=calibrate.d.ts.map

package/dist/commands/evaluator/calibrate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"calibrate.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/calibrate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKpC,wBAAgB,wBAAwB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CA6BjE"}

package/dist/commands/evaluator/calibrate.js

@@ -0,0 +1,29 @@
+import ora from "ora";
+import { requireApiKey, getSdkClient } from "../../auth.js";
+import { printSuccess, printJson, handleSdkError } from "../../output.js";
+export function registerCalibrateCommand(evaluator) {
+    evaluator
+        .command("calibrate <evaluatorId>")
+        .description("Start a calibration run for a saved evaluator against a labelled dataset")
+        .requiredOption("--dataset-id <id>", "Dataset whose published annotations to calibrate against")
+        .option("--score-config-id <id>", "Score config to calibrate against (defaults to the dataset's continuous scores)")
+        .action(async (evaluatorId, opts) => {
+        const apiKey = await requireApiKey();
+        const spinner = ora("Starting calibration run...").start();
+        try {
+            const client = getSdkClient(apiKey);
+            const run = await client.evaluators.calibrateRun(evaluatorId, {
+                datasetId: opts.datasetId,
+                ...(opts.scoreConfigId !== undefined ? { scoreConfigId: opts.scoreConfigId } : {}),
+            });
+            spinner.stop();
+            printSuccess(`Calibration run ${run.id} started (status: ${run.status}). ` +
+                `Poll 'scorable calibration-run get ${run.id}' for metrics.`);
+            printJson(run);
+        }
+        catch (e) {
+            spinner.stop();
+            handleSdkError(e);
+        }
+    });
+}

package/dist/commands/evaluator/create.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"create.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AASpC,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAsF9D"}
+{"version":3,"file":"create.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AASpC,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CA4F9D"}

package/dist/commands/evaluator/create.js

@@ -15,6 +15,7 @@ export function registerCreateCommand(evaluator) {
         .option("--models <json>", "JSON array of model names, in priority order. E.g., '[\"gpt-5-mini\"]'")
         .option("--overwrite", "Overwrite if evaluator with same name exists")
         .option("--objective-version-id <id>", "Objective version ID")
+        .option("--demonstration-dataset <id>", "Dataset of labelled examples resolved into few-shot demonstrations at evaluation time.")
         .option("--project-id <uuid>", PROJECT_ID_FLAG_DESC)
         .action(async (opts) => {
         if (!opts.intent && !opts.objectiveId) {
@@ -39,6 +40,8 @@ export function registerCreateCommand(evaluator) {
             payload.overwrite = opts.overwrite;
         if (opts.objectiveVersionId)
             payload.objective_version_id = opts.objectiveVersionId;
+        if (opts.demonstrationDataset)
+            payload.demonstration_dataset_id = opts.demonstrationDataset;
         if (opts.models) {
             const r = parseJsonArg(opts.models, z.array(z.string()));
             if (!r.ok) {

package/dist/commands/evaluator/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAYpC,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAehE"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAapC,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAgBhE"}

package/dist/commands/evaluator/index.js

@@ -9,6 +9,7 @@ import { registerExecuteByNameCommand } from "./execute-by-name.js";
 import { registerDuplicateCommand } from "./duplicate.js";
 import { registerExportYamlCommand } from "./export-yaml.js";
 import { registerImportYamlCommand } from "./import-yaml.js";
+import { registerCalibrateCommand } from "./calibrate.js";
 export function registerEvaluatorCommands(program) {
     const evaluator = new Command("evaluator").description("Evaluator management commands");
     registerListCommand(evaluator);
@@ -21,5 +22,6 @@ export function registerEvaluatorCommands(program) {
     registerDuplicateCommand(evaluator);
     registerExportYamlCommand(evaluator);
     registerImportYamlCommand(evaluator);
+    registerCalibrateCommand(evaluator);
     program.addCommand(evaluator);
 }

package/dist/commands/evaluator/update.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"update.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/update.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AASpC,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAkE9D"}
+{"version":3,"file":"update.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/update.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AASpC,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAyE9D"}

package/dist/commands/evaluator/update.js

@@ -13,6 +13,7 @@ export function registerUpdateCommand(evaluator) {
         .option("--models <json>", "JSON array of model names. E.g., '[\"gpt-5.5\"]'")
         .option("--objective-id <id>", "The new objective ID")
         .option("--objective-version-id <id>", "The new objective version ID")
+        .option("--demonstration-dataset <id>", "Dataset of labelled examples resolved into few-shot demonstrations at evaluation time.")
         .option("--project-id <uuid>", "Move the evaluator to this project (within the same organization). " + PROJECT_ID_FLAG_DESC)
         .action(async (evaluatorId, opts) => {
         const apiKey = await requireApiKey();
@@ -25,6 +26,8 @@ export function registerUpdateCommand(evaluator) {
             payload.objective_id = opts.objectiveId;
         if (opts.objectiveVersionId !== undefined)
             payload.objective_version_id = opts.objectiveVersionId;
+        if (opts.demonstrationDataset !== undefined)
+            payload.demonstration_dataset_id = opts.demonstrationDataset;
         if (opts.models !== undefined) {
             const r = parseJsonArg(opts.models, z.array(z.string()));
             if (!r.ok) {

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAIA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAqDpC,wBAAgB,SAAS,IAAI,OAAO,CAgCnC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAIA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAwDpC,wBAAgB,SAAS,IAAI,OAAO,CAmCnC"}

package/dist/index.js

@@ -15,6 +15,9 @@ import { registerOtelFilterCommands } from "./commands/otel-filter/index.js";
 import { registerOtelTraceCommands } from "./commands/otel-trace/index.js";
 import { registerModelCommands } from "./commands/model/index.js";
 import { registerProjectCommands } from "./commands/project/index.js";
+import { registerAnnotationCommands } from "./commands/annotation/index.js";
+import { registerCalibrationRunCommands } from "./commands/calibration-run/index.js";
+import { registerDatasetItemCommands } from "./commands/dataset-item/index.js";
 const { version } = createRequire(import.meta.url)("../package.json");
 function buildBanner(ver) {
     const logo = chalk.hex("#4D9FFF");
@@ -78,6 +81,9 @@ export function createCli() {
     registerOtelTraceCommands(program);
     registerModelCommands(program);
     registerProjectCommands(program);
+    registerAnnotationCommands(program);
+    registerCalibrationRunCommands(program);
+    registerDatasetItemCommands(program);
     return program;
 }
 if (realpathSync(fileURLToPath(import.meta.url)) === realpathSync(process.argv[1])) {

package/dist/lib/filter-yaml.d.ts

@@ -33,8 +33,8 @@ export declare const FilterYamlSchema: z.ZodObject<{
             }, z.core.$strict>>>;
         }, z.core.$strict>>;
         role: z.ZodEnum<{
-            user: "user";
             assistant: "assistant";
+            user: "user";
         }>;
         locator: z.ZodDiscriminatedUnion<[z.ZodObject<{
             kind: z.ZodLiteral<"span_attr">;

package/dist/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,kBAAkB,EAAE,KAAK,CAAC;QACxB,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QACrB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;KAC9B,CAAC,CAAC;IACH,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,UAAU,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACjD;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC1C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,qBAAa,QAAS,SAAQ,KAAK;aAEf,QAAQ,EAAE,MAAM;gBAAhB,QAAQ,EAAE,MAAM,EAChC,OAAO,EAAE,MAAM;CAKlB"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,kBAAkB,EAAE,KAAK,CAAC;QACxB,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QACrB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;KAC9B,CAAC,CAAC;IACH,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,UAAU,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACjD;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC1C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,qBAAa,QAAS,SAAQ,KAAK;aAEf,QAAQ,EAAE,MAAM;IADlC,YACkB,QAAQ,EAAE,MAAM,EAChC,OAAO,EAAE,MAAM,EAIhB;CACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@root-signals/scorable-cli",
-  "version": "0.14.0",
+  "version": "0.15.1",
   "description": "CLI for Scorable",
   "license": "Apache-2.0",
   "author": "Scorable",
@@ -32,7 +32,7 @@
   },
   "dependencies": {
     "@inquirer/prompts": "^8.3.2",
-    "@root-signals/scorable": "^0.11.0",
+    "@root-signals/scorable": "^0.12.1",
     "chalk": "^5.6.2",
     "cli-table3": "^0.6.5",
     "commander": "^15.0.0",
@@ -45,7 +45,7 @@
     "@types/node": "^25.5.0",
     "oxfmt": "^0.53.0",
     "oxlint": "^1.56.0",
-    "typescript": "^5.9.3",
+    "typescript": "^7.0.2",
     "vitest": "^4.1.5"
   },
   "engines": {