@root-signals/scorable-cli 0.5.0 → 0.7.0

package/README.md

@@ -289,8 +289,8 @@ inputs:
 # dataset_id: "<uuid>"
 
 models:
-  - gpt-4o-mini
-  - gemini-2.5-flash-lite
+  - gpt-5.4
+  - gemini-3-flash
 
 evaluators:
   - name: Precision

package/dist/commands/evaluator/create.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"create.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAkF9D"}
+{"version":3,"file":"create.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CA+E9D"}

package/dist/commands/evaluator/create.js

@@ -9,7 +9,6 @@ export function registerCreateCommand(evaluator) {
         .requiredOption("--scoring-criteria <text>", "The scoring criteria prompt. Must contain {{ request }} and/or {{ response }} as placeholders.")
         .option("--intent <intent>", "The intent for the evaluator (mutually exclusive with --objective-id)")
         .option("--objective-id <id>", "Objective ID (mutually exclusive with --intent)")
-        .option("--system-message <text>", "System message for the evaluator")
         .option("--models <json>", "JSON array of model names, in priority order. E.g., '[\"gpt-5-mini\"]'")
         .option("--overwrite", "Overwrite if evaluator with same name exists")
         .option("--objective-version-id <id>", "Objective version ID")
@@ -32,8 +31,6 @@ export function registerCreateCommand(evaluator) {
             payload.intent = opts.intent;
         if (opts.objectiveId)
             payload.objective_id = opts.objectiveId;
-        if (opts.systemMessage)
-            payload.system_message = opts.systemMessage;
         if (opts.overwrite !== undefined)
             payload.overwrite = opts.overwrite;
         if (opts.objectiveVersionId)

package/dist/commands/evaluator/execute.d.ts

@@ -11,6 +11,7 @@ export declare function executeEvaluator(evaluatorId: string, opts: {
     sessionId?: string;
     systemPrompt?: string;
     variables?: string;
+    fileIds?: string;
 }, readStdin?: typeof readStdinDefault): Promise<void>;
 export declare function registerExecuteCommand(evaluator: Command): void;
 export {};

package/dist/commands/evaluator/execute.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"execute.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/execute.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,iBAAe,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAIjD;AAED,wBAAsB,gBAAgB,CACpC,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE;IACJ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,EACD,SAAS,0BAAmB,GAC3B,OAAO,CAAC,IAAI,CAAC,CA2Ef;AAED,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAgF/D"}
+{"version":3,"file":"execute.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/execute.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,iBAAe,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAIjD;AAED,wBAAsB,gBAAgB,CACpC,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE;IACJ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,EACD,SAAS,0BAAmB,GAC3B,OAAO,CAAC,IAAI,CAAC,CAwFf;AAED,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CA2F/D"}

package/dist/commands/evaluator/execute.js

@@ -75,6 +75,19 @@ export async function executeEvaluator(evaluatorId, opts, readStdin = readStdinD
             return;
         }
     }
+    if (opts.fileIds) {
+        let parsed;
+        try {
+            parsed = JSON.parse(opts.fileIds);
+        }
+        catch {
+            throw new Error("Invalid JSON for --file-ids.");
+        }
+        if (!isStringArray(parsed)) {
+            throw new Error("Invalid JSON for --file-ids. Expected array of UUID strings.");
+        }
+        payload.file_ids = parsed;
+    }
     const spinner = ora("Running evaluator...").start();
     try {
         const client = getSdkClient(apiKey);
@@ -102,6 +115,7 @@ export function registerExecuteCommand(evaluator) {
         .option("--system-prompt <text>", "System prompt that was used for the LLM call")
         .option("--turns <json>", 'JSON array of conversation turns. E.g., \'[{"role":"user","content":"Hello"}]\'')
         .option("--variables <json>", 'JSON object of extra template variables. E.g., \'{"lang":"EN"}\'')
+        .option("--file-ids <json>", "JSON array of file UUIDs from 'scorable file upload'. E.g., '[\"uuid1\"]'")
         .addHelpText("after", `
 Examples:
   # Check helpfulness of a support response
@@ -130,7 +144,13 @@ Examples:
       --request "How do I cancel my subscription?" \\
       --response "You can cancel anytime from account settings under billing." \\
       --variables '{"topic":"subscription"}' \\
-      --user-id user_123 --session-id session_abc --tag production`)
+      --user-id user_123 --session-id session_abc --tag production
+
+  # Evaluate a response with an uploaded image
+  $ scorable evaluator execute <evaluatorId> \\
+      --request "Does the chart match the data?" \\
+      --response "Yes, the Q1 figures align." \\
+      --file-ids '["550e8400-e29b-41d4-a716-446655440000"]'`)
         .action(async (evaluatorId, opts) => {
         try {
             await executeEvaluator(evaluatorId, opts);

package/dist/commands/evaluator/update.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"update.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/update.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CA2D9D"}
+{"version":3,"file":"update.d.ts","sourceRoot":"","sources":["../../../src/commands/evaluator/update.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAwD9D"}

package/dist/commands/evaluator/update.js

@@ -7,7 +7,6 @@ export function registerUpdateCommand(evaluator) {
         .description("Update an existing evaluator (PATCH)")
         .option("--name <name>", "The new name for the evaluator")
         .option("--scoring-criteria <text>", "The new scoring criteria (prompt text)")
-        .option("--system-message <text>", "The new system message")
         .option("--models <json>", "JSON array of model names. E.g., '[\"gpt-4\"]'")
         .option("--objective-id <id>", "The new objective ID")
         .option("--objective-version-id <id>", "The new objective version ID")
@@ -18,8 +17,6 @@ export function registerUpdateCommand(evaluator) {
             payload.name = opts.name;
         if (opts.scoringCriteria !== undefined)
             payload.prompt = opts.scoringCriteria;
-        if (opts.systemMessage !== undefined)
-            payload.system_message = opts.systemMessage;
         if (opts.objectiveId !== undefined)
             payload.objective_id = opts.objectiveId;
         if (opts.objectiveVersionId !== undefined)

package/dist/commands/file/index.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function registerFileCommands(program: Command): void;
+//# sourceMappingURL=index.d.ts.map

package/dist/commands/file/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/commands/file/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAI3D"}

package/dist/commands/file/index.js

@@ -0,0 +1,5 @@
+import { registerUploadCommand } from "./upload.js";
+export function registerFileCommands(program) {
+    const file = program.command("file").description("Manage uploaded files");
+    registerUploadCommand(file);
+}

package/dist/commands/file/upload.d.ts

@@ -0,0 +1,4 @@
+import { Command } from "commander";
+export declare function uploadFile(filePath: string): Promise<void>;
+export declare function registerUploadCommand(file: Command): void;
+//# sourceMappingURL=upload.d.ts.map

package/dist/commands/file/upload.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"upload.d.ts","sourceRoot":"","sources":["../../../src/commands/file/upload.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAQpC,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAuBhE;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,OAAO,GAAG,IAAI,CA2BzD"}

package/dist/commands/file/upload.js

@@ -0,0 +1,54 @@
+import ora from "ora";
+import * as fs from "fs";
+import * as path from "path";
+import { requireApiKey } from "../../auth.js";
+import { printSuccess, printError, printJson, handleSdkError } from "../../output.js";
+import { getSdkClient } from "../../auth.js";
+export async function uploadFile(filePath) {
+    const apiKey = await requireApiKey();
+    if (!fs.existsSync(filePath)) {
+        printError(`File not found: ${filePath}`);
+        process.exit(1);
+    }
+    const client = getSdkClient(apiKey);
+    const fileName = path.basename(filePath);
+    const fileBuffer = fs.readFileSync(filePath);
+    const blob = new Blob([fileBuffer]);
+    const spinner = ora(`Uploading ${fileName}...`).start();
+    try {
+        const result = await client.files.upload(blob, fileName);
+        spinner.stop();
+        printSuccess("File uploaded successfully!");
+        printJson(result);
+    }
+    catch (e) {
+        spinner.stop();
+        throw e;
+    }
+}
+export function registerUploadCommand(file) {
+    file
+        .command("upload <filePath>")
+        .description("Upload a file (PDF, PNG, JPG, JPEG, WEBP, SVG) for use in evaluator execution")
+        .addHelpText("after", `
+Examples:
+  # Upload a PDF document
+  $ scorable file upload ./report.pdf
+
+  # Upload an image
+  $ scorable file upload ./chart.png
+
+  # Use the returned ID in evaluator execution
+  $ scorable evaluator execute <evaluatorId> \\
+      --request "Does the chart match?" \\
+      --response "Yes." \\
+      --file-ids '["<id from upload>"]'`)
+        .action(async (filePath) => {
+        try {
+            await uploadFile(filePath);
+        }
+        catch (e) {
+            handleSdkError(e);
+        }
+    });
+}

package/dist/commands/judge/generate.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"generate.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/generate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAcpC,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CA4H5D"}
+{"version":3,"file":"generate.d.ts","sourceRoot":"","sources":["../../../src/commands/judge/generate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAcpC,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CA2H5D"}

package/dist/commands/judge/generate.js

@@ -30,7 +30,6 @@ offer to connect the guest with a human agent when uncertain."`)
             printError(`Invalid --visibility value: "${visibility}". Accepted values: private, public.`);
             throw new CliError(1, "invalid_visibility");
         }
-        const apiVisibility = visibility === "private" ? "unlisted" : "public";
         const apiKey = await requireApiKey();
         let extra_contexts;
         if (opts.extraContexts) {
@@ -45,9 +44,9 @@ offer to connect the guest with a human agent when uncertain."`)
         const spinner = ora("Generating judge (this may take a moment)...").start();
         try {
             const client = getSdkClient(apiKey);
-            const result = await client.judges.generate({
+            const result = (await client.judges.generate({
                 intent: opts.intent,
-                visibility: apiVisibility,
+                visibility: visibility,
                 overwrite: opts.overwrite,
                 name: opts.name,
                 stage: opts.stage,
@@ -59,7 +58,7 @@ offer to connect the guest with a human agent when uncertain."`)
                         reasoning_effort: opts.reasoningEffort,
                     },
                 }),
-            });
+            }));
             spinner.stop();
             if (result.error_code === "multiple_stages") {
                 printWarning("Multiple evaluation stages detected. Each judge covers one stage.");

package/dist/commands/judge/list.js

@@ -12,7 +12,7 @@ export function registerListCommand(judge) {
         .option("--ordering <field>", "Which field to use for ordering the results")
         .action(async (opts) => {
         const apiKey = await requireApiKey();
-        const params = { is_public: false };
+        const params = {};
         if (opts.pageSize !== undefined)
             params.page_size = opts.pageSize;
         if (opts.cursor !== undefined)

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAIA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAgDpC,wBAAgB,SAAS,IAAI,OAAO,CAoBnC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAIA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAiDpC,wBAAgB,SAAS,IAAI,OAAO,CAqBnC"}

package/dist/index.js

@@ -10,6 +10,7 @@ import { registerEvaluatorCommands } from "./commands/evaluator/index.js";
 import { registerExecutionLogCommands } from "./commands/execution-log/index.js";
 import { registerAuthCommands } from "./commands/auth/index.js";
 import { registerSkillsAddCommand } from "./commands/skills-add.js";
+import { registerFileCommands } from "./commands/file/index.js";
 const { version } = createRequire(import.meta.url)("../package.json");
 function buildBanner(ver) {
     const logo = chalk.hex("#4D9FFF");
@@ -61,6 +62,7 @@ export function createCli() {
     registerExecutionLogCommands(program);
     registerAuthCommands(program);
     registerSkillsAddCommand(program);
+    registerFileCommands(program);
     return program;
 }
 if (realpathSync(fileURLToPath(import.meta.url)) === realpathSync(process.argv[1])) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@root-signals/scorable-cli",
-  "version": "0.5.0",
+  "version": "0.7.0",
   "description": "CLI for Scorable",
   "license": "Apache-2.0",
   "author": "Scorable",
@@ -31,7 +31,7 @@
   },
   "dependencies": {
     "@inquirer/prompts": "^8.3.2",
-    "@root-signals/scorable": "^0.4.0",
+    "@root-signals/scorable": "^0.6.0",
     "chalk": "^5.6.2",
     "cli-table3": "^0.6.5",
     "commander": "^14.0.3",
@@ -44,7 +44,7 @@
     "oxfmt": "^0.41.0",
     "oxlint": "^1.56.0",
     "typescript": "^5.9.3",
-    "vitest": "^4.1.0"
+    "vitest": "^4.1.5"
   },
   "engines": {
     "node": ">=20"