@2kw/ai 6.1.0-dev.7 → 6.1.0

package/README.md

@@ -99,7 +99,7 @@ kubectl-style contexts switch between organizations and environments:
 | `prompts` | Versioned prompt management, labels, compilation, testing |
 | `extractions` | Create extractions (text, file, images), list, re-run, estimate tokens |
 | `convert` | Convert PDF, DOCX, XLSX, images, and 20+ formats to Markdown/text/HTML/JSON |
-| `ai` | Chat completions, Responses (agent or direct model, streaming), embeddings, and model listing via the OpenAI-compatible gateway |
+| `ai` | Chat completions, Responses (agent or direct model, streaming), and model listing via the OpenAI-compatible gateway |
 | `transcribe` | Transcribe audio (FLAC, MP3, MP4, OGG, WAV, WebM) |
 | `datasets` | Build datasets and dataset versions for evaluation |
 | `experiments` | Run experiments against datasets |

package/dist/commands/ai.d.ts

@@ -3,7 +3,7 @@ import { Command } from "commander";
  * Reads a single text input from stdin, or `undefined` when stdin is a TTY
  * (nothing was piped in) or empty. Mirrors the `-` stdin convention used by
  * `bb convert file`, but always reads stdin implicitly (there is no `-`
- * sentinel here since both `respond` and `embed` take a plain input string).
+ * sentinel here since `respond` takes a plain input string).
  */
 export declare function readStdinInput(): string | undefined;
 /**

package/dist/commands/ai.js

@@ -9,7 +9,7 @@ import chalk from "chalk";
  * Reads a single text input from stdin, or `undefined` when stdin is a TTY
  * (nothing was piped in) or empty. Mirrors the `-` stdin convention used by
  * `bb convert file`, but always reads stdin implicitly (there is no `-`
- * sentinel here since both `respond` and `embed` take a plain input string).
+ * sentinel here since `respond` takes a plain input string).
  */
 export function readStdinInput() {
     if (process.stdin.isTTY)
@@ -157,21 +157,6 @@ async function streamResponse(command, body) {
         process.stdout.write("\n");
     }
 }
-function printEmbeddingResult(result, command) {
-    if (isJsonOutput(command)) {
-        console.log(JSON.stringify(result, null, 2));
-        return;
-    }
-    const items = result?.data ?? [];
-    console.log(chalk.green(`Received ${items.length} embedding${items.length !== 1 ? "s" : ""}.`));
-    items.forEach((item, i) => {
-        const dims = item.embedding?.length ?? 0;
-        console.log(`  [${item.index ?? i}] ${dims} dimensions`);
-    });
-    if (result?.usage) {
-        console.log(chalk.dim(`\n[${result.model} | ${result.usage.prompt_tokens} prompt / ${result.usage.total_tokens} total tokens]`));
-    }
-}
 export function makeAiCommand() {
     const cmd = new Command("ai").description("AI gateway operations");
     cmd
@@ -257,28 +242,6 @@ export function makeAiCommand() {
             }
         });
     });
-    cmd
-        .command("embed")
-        .description("Create embeddings for input text")
-        .argument("[input]", "Text to embed (reads stdin if omitted)")
-        .requiredOption("-m, --model <model>", "Model (provider/model format)")
-        .action(async (input, opts, command) => {
-        await runAction(command, async () => {
-            const inputText = input ?? readStdinInput();
-            if (!inputText) {
-                throw new Error("Input is required as an argument or via stdin.");
-            }
-            const client = getClient(command);
-            const body = { model: opts.model, input: inputText };
-            const { data } = await withSpinner("Embedding...", () => 
-            // POST /v1/embeddings has no request body in the generated spec
-            // (backend: "Not yet implemented"); cast, same as the untyped
-            // convert.ts source-conversion calls.
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            client.POST("/v1/embeddings", { body }));
-            printEmbeddingResult(data, command);
-        });
-    });
     cmd
         .command("models")
         .description("List available AI models")

package/dist/commands/queues.d.ts

@@ -0,0 +1,11 @@
+import { Command } from "commander";
+/**
+ * Annotation queues let a human reviewer work through a curated list of
+ * subjects (run results, traces, spans, dataset items). This command group
+ * covers the queue itself plus the items inside it: creating a queue,
+ * bulk-adding subjects, claiming the next eligible item, and recording the
+ * outcome — the full loop an agent or a human reviewer needs to run review
+ * end to end from the CLI.
+ */
+export declare function makeQueuesCommand(): Command;
+//# sourceMappingURL=queues.d.ts.map

package/dist/commands/queues.js

@@ -0,0 +1,244 @@
+import { Command } from "commander";
+import chalk from "chalk";
+import { getClient, runAction } from "../lib/client.js";
+import { isJsonOutput } from "../lib/config.js";
+import { formatList, formatDetail, formatSuccess } from "../lib/output.js";
+/**
+ * Annotation queues let a human reviewer work through a curated list of
+ * subjects (run results, traces, spans, dataset items). This command group
+ * covers the queue itself plus the items inside it: creating a queue,
+ * bulk-adding subjects, claiming the next eligible item, and recording the
+ * outcome — the full loop an agent or a human reviewer needs to run review
+ * end to end from the CLI.
+ */
+export function makeQueuesCommand() {
+    const cmd = new Command("queues").description("Manage annotation queues for human review");
+    // ── list ──────────────────────────────────────────────────────
+    cmd
+        .command("list")
+        .description("List annotation queues")
+        .option("--archived", "List archived queues instead of active ones", false)
+        .action(async (opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            const { data } = await client.GET("/v1/annotation-queues", {
+                params: { query: { archived: opts.archived } },
+            });
+            formatList(data, command, [
+                "id",
+                "name",
+                "description",
+                "archivedAt",
+                "createdAt",
+            ]);
+        });
+    });
+    // ── get ───────────────────────────────────────────────────────
+    cmd
+        .command("get")
+        .description("Get an annotation queue by ID")
+        .argument("<id>", "Queue ID")
+        .action(async (id, _opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            const { data } = await client.GET("/v1/annotation-queues/{id}", {
+                params: { path: { id } },
+            });
+            formatDetail(data, command);
+        });
+    });
+    // ── create ────────────────────────────────────────────────────
+    cmd
+        .command("create")
+        .description("Create a new annotation queue")
+        .requiredOption("-n, --name <name>", "Queue name")
+        .option("-d, --description <text>", "Queue description")
+        .action(async (opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            const { data } = await client.POST("/v1/annotation-queues", {
+                body: { name: opts.name, description: opts.description },
+            });
+            formatDetail(data, command);
+        });
+    });
+    // ── update ────────────────────────────────────────────────────
+    cmd
+        .command("update")
+        .description("Update a queue's name or description")
+        .argument("<id>", "Queue ID")
+        .option("-n, --name <name>", "New name")
+        .option("-d, --description <text>", "New description")
+        .action(async (id, opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            const body = {};
+            if (opts.name !== undefined)
+                body.name = opts.name;
+            if (opts.description !== undefined)
+                body.description = opts.description;
+            const { data } = await client.PATCH("/v1/annotation-queues/{id}", {
+                params: { path: { id } },
+                body: body,
+            });
+            formatDetail(data, command);
+        });
+    });
+    // ── archive ───────────────────────────────────────────────────
+    cmd
+        .command("archive")
+        .description("Archive a queue (idempotent)")
+        .argument("<id>", "Queue ID")
+        .action(async (id, _opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            await client.POST("/v1/annotation-queues/{id}/archive", {
+                params: { path: { id } },
+            });
+            formatSuccess(`Queue ${id} archived.`, command);
+        });
+    });
+    // ── by-subject ────────────────────────────────────────────────
+    cmd
+        .command("by-subject")
+        .description("Find queue items referencing a subject, across every queue")
+        .requiredOption("--subject-type <type>", "RUN_RESULT | TRACE | SPAN | DATASET_ITEM")
+        .requiredOption("--subject-id <id>", "Subject ID")
+        .action(async (opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            const { data } = await client.GET("/v1/annotation-queues/by-subject", {
+                params: {
+                    query: { subjectType: opts.subjectType, subjectId: opts.subjectId },
+                },
+            });
+            formatList(data, command, [
+                "id",
+                "queueId",
+                "subjectType",
+                "subjectId",
+                "status",
+                "assignedTo",
+            ]);
+        });
+    });
+    // ── next ──────────────────────────────────────────────────────
+    cmd
+        .command("next")
+        .description("Claim the next eligible item in a queue for the caller")
+        .argument("<id>", "Queue ID")
+        .action(async (id, _opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            const { data } = await client.GET("/v1/annotation-queues/{id}/items/next", {
+                params: { path: { id } },
+            });
+            if (!data) {
+                // 204 No Content: the queue has nothing left for this caller. A
+                // normal end state, not an error — printed plainly either way.
+                if (isJsonOutput(command)) {
+                    console.log(JSON.stringify(null));
+                }
+                else {
+                    console.log(chalk.dim("No items available to claim — the queue is empty for you."));
+                }
+                return;
+            }
+            formatDetail(data, command);
+        });
+    });
+    // ── items ─────────────────────────────────────────────────────
+    const items = new Command("items").description("Manage items within an annotation queue");
+    items
+        .command("list")
+        .description("List items in a queue")
+        .requiredOption("--queue <id>", "Queue ID")
+        .option("--status <status>", "PENDING | IN_PROGRESS | DONE | SKIPPED")
+        .option("--assigned-to <userId>", "Filter by assignee")
+        .option("-p, --page <number>", "Page number (0-indexed)", "0")
+        .option("--size <number>", "Page size", "50")
+        .action(async (opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            const { data } = await client.GET("/v1/annotation-queues/{id}/items", {
+                params: {
+                    path: { id: opts.queue },
+                    query: {
+                        status: opts.status,
+                        assignedTo: opts.assignedTo,
+                        page: opts.page !== undefined ? parseInt(opts.page, 10) : undefined,
+                        size: opts.size !== undefined ? parseInt(opts.size, 10) : undefined,
+                    },
+                },
+            });
+            formatList(data, command, [
+                "id",
+                "subjectType",
+                "subjectId",
+                "status",
+                "assignedTo",
+                "createdAt",
+            ]);
+        });
+    });
+    items
+        .command("add")
+        .description("Bulk-add subjects to a queue (duplicates skipped)")
+        .requiredOption("--queue <id>", "Queue ID")
+        .requiredOption("--items <json>", 'JSON array of {"subjectType","subjectId"} objects')
+        .action(async (opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            const parsedItems = JSON.parse(opts.items);
+            const { data } = await client.POST("/v1/annotation-queues/{id}/items", {
+                params: { path: { id: opts.queue } },
+                body: { items: parsedItems },
+            });
+            formatList(data, command, [
+                "id",
+                "subjectType",
+                "subjectId",
+                "status",
+            ]);
+        });
+    });
+    items
+        .command("get")
+        .description("Get a single queue item")
+        .argument("<itemId>", "Item ID")
+        .requiredOption("--queue <id>", "Queue ID")
+        .action(async (itemId, opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            const { data } = await client.GET("/v1/annotation-queues/{id}/items/{itemId}", {
+                params: { path: { id: opts.queue, itemId } },
+            });
+            formatDetail(data, command);
+        });
+    });
+    items
+        .command("update")
+        .description("Update a queue item's status or assignee — the outcome of a review")
+        .argument("<itemId>", "Item ID")
+        .requiredOption("--queue <id>", "Queue ID")
+        .option("--status <status>", "PENDING | IN_PROGRESS | DONE | SKIPPED")
+        .option("--assigned-to <userId>", "New assignee")
+        .action(async (itemId, opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            const body = {};
+            if (opts.status !== undefined)
+                body.status = opts.status;
+            if (opts.assignedTo !== undefined)
+                body.assignedTo = opts.assignedTo;
+            const { data } = await client.PATCH("/v1/annotation-queues/{id}/items/{itemId}", {
+                params: { path: { id: opts.queue, itemId } },
+                body: body,
+            });
+            formatDetail(data, command);
+        });
+    });
+    cmd.addCommand(items);
+    return cmd;
+}
+//# sourceMappingURL=queues.js.map

package/dist/commands/scores.js

@@ -1,14 +1,18 @@
 import { Command } from "commander";
 import { getClient, runAction } from "../lib/client.js";
-import { formatDetail } from "../lib/output.js";
+import { formatDetail, formatList } from "../lib/output.js";
 export function makeScoresCommand() {
-    const cmd = new Command("scores").description("Record evaluation scores on experiment run results");
+    const cmd = new Command("scores").description("Record and review evaluation scores on experiment run results and other subjects");
     cmd
         .command("human")
-        .description("Record a human score for a run result. Upserts by (runResultId, evaluatorId, annotatorId).")
-        .requiredOption("--run-result <id>", "Run result ID")
+        .description("Record a human score. Upserts by (subject, evaluatorId, annotatorId) — pass either " +
+        "--run-result, or --subject-type/--subject-id for a trace, span, or dataset item.")
+        .option("--run-result <id>", "Run result ID (legacy shape)")
+        .option("--subject-type <type>", "RUN_RESULT | TRACE | SPAN | DATASET_ITEM")
+        .option("--subject-id <id>", "Subject ID")
+        .option("--queue-item <id>", "Annotation queue item ID this score resolves")
         .requiredOption("--evaluator <id>", "Evaluator identifier (e.g. 'helpfulness' or a template id)")
-        .requiredOption("--score <number>", "Score between 0.0 and 1.0", parseFloat)
+        .option("--score <number>", "Score between 0.0 and 1.0", parseFloat)
         .option("--label <text>", "Short label (e.g. 'pass', 'fail', 'good')")
         .option("--comment <text>", "Optional reviewer note")
         .action(async (opts, command) => {
@@ -17,6 +21,9 @@ export function makeScoresCommand() {
             const { data } = await client.POST("/v1/evaluation-scores/human", {
                 body: {
                     runResultId: opts.runResult,
+                    subjectType: opts.subjectType,
+                    subjectId: opts.subjectId,
+                    annotationQueueItemId: opts.queueItem,
                     evaluatorId: opts.evaluator,
                     score: opts.score,
                     label: opts.label,
@@ -26,6 +33,37 @@ export function makeScoresCommand() {
             formatDetail(data, command);
         });
     });
+    cmd
+        .command("list")
+        .description("List human scores filed against a subject. Defaults to the caller's own scores — " +
+        "pass --annotator to see another reviewer's.")
+        .requiredOption("--subject-type <type>", "RUN_RESULT | TRACE | SPAN | DATASET_ITEM")
+        .requiredOption("--subject-id <id>", "Subject ID")
+        .option("--annotator <id>", "Annotator ID (defaults to the caller)")
+        .action(async (opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            const { data } = await client.GET("/v1/evaluation-scores/human", {
+                params: {
+                    query: {
+                        subjectType: opts.subjectType,
+                        subjectId: opts.subjectId,
+                        annotatorId: opts.annotator,
+                    },
+                },
+            });
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            formatList(data, command, [
+                "id",
+                "evaluatorId",
+                "score",
+                "label",
+                "annotatorId",
+                "comment",
+                "createdAt",
+            ]);
+        });
+    });
     return cmd;
 }
 //# sourceMappingURL=scores.js.map

package/dist/index.js

@@ -21,6 +21,7 @@ import { makeExperimentsCommand } from "./commands/experiments.js";
 import { makeTracingCommand } from "./commands/tracing.js";
 import { makeEvaluatorsCommand } from "./commands/evaluators.js";
 import { makeScoresCommand } from "./commands/scores.js";
+import { makeQueuesCommand } from "./commands/queues.js";
 import { makeAgentsCommand } from "./commands/agents.js";
 import { makeConversationsCommand } from "./commands/conversations.js";
 import { makeKnowledgeCommand } from "./commands/knowledge.js";
@@ -53,6 +54,7 @@ program.addCommand(makeDatasetsCommand());
 program.addCommand(makeExperimentsCommand());
 program.addCommand(makeEvaluatorsCommand());
 program.addCommand(makeScoresCommand());
+program.addCommand(makeQueuesCommand());
 program.addCommand(makeTracingCommand());
 program.addCommand(makeAgentsCommand());
 program.addCommand(makeConversationsCommand());

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@2kw/ai",
-  "version": "6.1.0-dev.7",
+  "version": "6.1.0",
   "description": "CLI for 2kw.ai — schema-driven document extraction, an OpenAI-compatible EU LLM gateway, transcription, prompts, datasets, and experiments from your terminal or agentic workflows. Ships as 2kw, backbone, and bb.",
   "keywords": [
     "cli",