@2kw/ai 6.0.0 → 6.1.0-dev.10

package/dist/commands/conversations.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function makeConversationsCommand(): Command;
+//# sourceMappingURL=conversations.d.ts.map

package/dist/commands/conversations.js

@@ -0,0 +1,81 @@
+import { Command } from "commander";
+import { getClient, runAction } from "../lib/client.js";
+import { formatDetail, formatSuccess } from "../lib/output.js";
+export function makeConversationsCommand() {
+    const cmd = new Command("conversations").description("Manage conversations");
+    cmd
+        .command("create")
+        .description("Create a conversation")
+        .option("--items <json>", "Initial items as a JSON array")
+        .option("--metadata <json>", "Metadata as JSON")
+        .action(async (opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            const { data } = await client.POST("/v1/conversations", {
+                body: {
+                    items: opts.items ? JSON.parse(opts.items) : undefined,
+                    metadata: opts.metadata ? JSON.parse(opts.metadata) : undefined,
+                },
+            });
+            formatDetail(data, command);
+        });
+    });
+    cmd
+        .command("get")
+        .description("Retrieve a conversation by id")
+        .argument("<conversationId>", "Conversation ID")
+        .action(async (conversationId, _opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            const { data } = await client.GET("/v1/conversations/{conversationId}", {
+                params: { path: { conversationId } },
+            });
+            formatDetail(data, command);
+        });
+    });
+    cmd
+        .command("update")
+        .description("Replace a conversation's metadata")
+        .argument("<conversationId>", "Conversation ID")
+        .requiredOption("--metadata <json>", "New metadata as JSON")
+        .action(async (conversationId, opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            const { data } = await client.POST("/v1/conversations/{conversationId}", {
+                params: { path: { conversationId } },
+                body: { metadata: JSON.parse(opts.metadata) },
+            });
+            formatDetail(data, command);
+        });
+    });
+    cmd
+        .command("delete")
+        .description("Delete a conversation and its recorded responses")
+        .argument("<conversationId>", "Conversation ID")
+        .action(async (conversationId, _opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            await client.DELETE("/v1/conversations/{conversationId}", {
+                params: { path: { conversationId } },
+            });
+            formatSuccess(`Conversation ${conversationId} deleted.`, command);
+        });
+    });
+    const items = new Command("items").description("List a conversation's items");
+    items
+        .command("list")
+        .description("List a conversation's items, oldest first")
+        .argument("<conversationId>", "Conversation ID")
+        .action(async (conversationId, _opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            const { data } = await client.GET("/v1/conversations/{conversationId}/items", {
+                params: { path: { conversationId } },
+            });
+            formatDetail(data, command);
+        });
+    });
+    cmd.addCommand(items);
+    return cmd;
+}
+//# sourceMappingURL=conversations.js.map

package/dist/commands/dataset-versions.js

@@ -1,7 +1,7 @@
 import { Command } from "commander";
 import { getClient, runAction } from "../lib/client.js";
 import { resolveLatestVersionId } from "../lib/datasets.js";
-import { formatPage, formatDetail } from "../lib/output.js";
+import { formatPage, formatDetail, formatSuccess } from "../lib/output.js";
 import { addPaginationOptions, paginationParams } from "../lib/pagination.js";
 export function makeDatasetVersionsCommand() {
     const cmd = new Command("versions").description("Manage dataset versions");
@@ -85,6 +85,49 @@ export function makeDatasetVersionsCommand() {
         });
     });
     cmd.addCommand(items);
+    cmd
+        .command("delete-item")
+        .description("Soft-delete an item in a dataset version (defaults to latest version)")
+        .argument("<itemId>", "Dataset item ID")
+        .requiredOption("--dataset <id>", "Dataset ID")
+        .option("--version-id <id>", "Version ID (defaults to latest)")
+        .action(async (itemId, opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            const versionId = opts.versionId ?? (await resolveLatestVersionId(client, opts.dataset));
+            await client.DELETE("/v1/datasets/{id}/versions/{versionId}/items/{itemId}", {
+                params: {
+                    // OpenAPI spec omits `id` in declared path params though `{id}` is in the URL template.
+                    path: { id: opts.dataset, versionId, itemId },
+                },
+            });
+            formatSuccess(`Item ${itemId} deleted.`, command);
+        });
+    });
+    cmd
+        .command("update-item-expected-output")
+        .description("Update the expected output of a dataset item (defaults to latest version)")
+        .argument("<itemId>", "Dataset item ID")
+        .requiredOption("--dataset <id>", "Dataset ID")
+        .option("--version-id <id>", "Version ID (defaults to latest)")
+        .option("--expected-output <json>", "New expected output (JSON); omit to clear it")
+        .action(async (itemId, opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            const versionId = opts.versionId ?? (await resolveLatestVersionId(client, opts.dataset));
+            const body = {};
+            if (opts.expectedOutput !== undefined)
+                body.expectedOutput = JSON.parse(opts.expectedOutput);
+            const { data } = await client.PATCH("/v1/datasets/{id}/versions/{versionId}/items/{itemId}/expected-output", {
+                params: {
+                    // OpenAPI spec omits `id` in declared path params though `{id}` is in the URL template.
+                    path: { id: opts.dataset, versionId, itemId },
+                },
+                body: body,
+            });
+            formatDetail(data, command);
+        });
+    });
     return cmd;
 }
 //# sourceMappingURL=dataset-versions.js.map

package/dist/commands/datasets.js

@@ -1,7 +1,7 @@
 import { Command } from "commander";
 import { getClient, runAction } from "../lib/client.js";
 import { resolveLatestVersionId } from "../lib/datasets.js";
-import { formatPage, formatDetail, formatSuccess } from "../lib/output.js";
+import { formatPage, formatDetail, formatList, formatSuccess } from "../lib/output.js";
 import { addPaginationOptions, paginationParams } from "../lib/pagination.js";
 import { makeDatasetVersionsCommand } from "./dataset-versions.js";
 export function makeDatasetsCommand() {
@@ -144,6 +144,28 @@ export function makeDatasetsCommand() {
             formatDetail(data, command);
         });
     });
+    cmd
+        .command("add-items")
+        .description("Add multiple items to a dataset in a single request (uses the latest version)")
+        .requiredOption("--dataset <id>", "Dataset ID")
+        .requiredOption("--items <json>", "Array of items (JSON), each with input and optional expectedOutput/tags/metadata")
+        .action(async (opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            const versionId = await resolveLatestVersionId(client, opts.dataset);
+            const items = JSON.parse(opts.items);
+            const { data } = await client.POST("/v1/datasets/{id}/versions/{versionId}/items/bulk", {
+                params: {
+                    // OpenAPI spec omits `id` in declared path params though `{id}` is in the URL template.
+                    path: { id: opts.dataset, versionId },
+                },
+                body: items,
+            });
+            formatList(data, command, [
+                "id", "input", "expectedOutput", "tags",
+            ]);
+        });
+    });
     cmd.addCommand(makeDatasetVersionsCommand());
     return cmd;
 }

package/dist/commands/experiments.js

@@ -1,11 +1,25 @@
 import { Command } from "commander";
 import chalk from "chalk";
 import ora from "ora";
+import Table from "cli-table3";
 import { getClient, runAction } from "../lib/client.js";
 import { formatPage, formatDetail, formatList, formatSuccess, withSpinner } from "../lib/output.js";
 import { addPaginationOptions, paginationParams } from "../lib/pagination.js";
 import { isJsonOutput } from "../lib/config.js";
 const TERMINAL_STATUSES = new Set(["COMPLETED", "FAILED"]);
+/** Summarize a single cell of the comparison matrix (a variant's result for one item). */
+function formatComparisonCell(result) {
+    if (!result)
+        return chalk.dim("—");
+    if (result.error)
+        return chalk.red("ERROR");
+    const scores = result.scores ?? [];
+    if (scores.length > 0) {
+        const avg = scores.reduce((sum, s) => sum + (s.score ?? 0), 0) / scores.length;
+        return avg.toFixed(2);
+    }
+    return result.durationMs != null ? `${result.durationMs}ms` : chalk.dim("done");
+}
 export function makeExperimentsCommand() {
     const cmd = new Command("experiments").description("Manage experiments");
     // ── list ──────────────────────────────────────────────────────
@@ -158,6 +172,53 @@ export function makeExperimentsCommand() {
             formatList(data, command, ["id", "name", "taskType", "description", "sortOrder"]);
         });
     });
+    // ── update-variant ───────────────────────────────────────────────
+    cmd
+        .command("update-variant")
+        .description("Update a variant")
+        .argument("<variantId>", "Variant ID")
+        .requiredOption("--experiment <id>", "Experiment ID")
+        .option("-n, --name <name>", "Variant name")
+        .option("--task-type <type>", "Task type")
+        .option("--config <json>", "Variant configuration (JSON)")
+        .option("-d, --description <text>", "Variant description")
+        .option("--sort-order <n>", "Sort order", parseInt)
+        .action(async (variantId, opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            const body = {};
+            if (opts.name !== undefined)
+                body.name = opts.name;
+            if (opts.taskType !== undefined)
+                body.taskType = opts.taskType;
+            if (opts.config)
+                body.configuration = JSON.parse(opts.config);
+            if (opts.description !== undefined)
+                body.description = opts.description;
+            if (opts.sortOrder !== undefined)
+                body.sortOrder = opts.sortOrder;
+            const { data } = await client.PUT("/v1/experiments/{id}/variants/{variantId}", {
+                params: { path: { id: opts.experiment, variantId } },
+                body: body,
+            });
+            formatDetail(data, command);
+        });
+    });
+    // ── delete-variant ───────────────────────────────────────────────
+    cmd
+        .command("delete-variant")
+        .description("Delete a variant")
+        .argument("<variantId>", "Variant ID")
+        .requiredOption("--experiment <id>", "Experiment ID")
+        .action(async (variantId, opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            await client.DELETE("/v1/experiments/{id}/variants/{variantId}", {
+                params: { path: { id: opts.experiment, variantId } },
+            });
+            formatSuccess(`Variant ${variantId} deleted.`, command);
+        });
+    });
     // ── run ───────────────────────────────────────────────────────
     cmd
         .command("run")
@@ -250,6 +311,154 @@ export function makeExperimentsCommand() {
         });
     });
     cmd.addCommand(results);
+    // ── set-baseline ─────────────────────────────────────────────────
+    cmd
+        .command("set-baseline")
+        .description("Mark a run as the baseline for its variant")
+        .requiredOption("--experiment <id>", "Experiment ID")
+        .requiredOption("--run <id>", "Run ID")
+        .action(async (opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            const { data } = await client.PUT("/v1/experiments/{id}/runs/{runId}/baseline", {
+                params: { path: { id: opts.experiment, runId: opts.run } },
+            });
+            formatDetail(data, command);
+        });
+    });
+    // ── clear-baseline ───────────────────────────────────────────────
+    cmd
+        .command("clear-baseline")
+        .description("Clear the baseline flag on a run")
+        .requiredOption("--experiment <id>", "Experiment ID")
+        .requiredOption("--run <id>", "Run ID")
+        .action(async (opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            const { data } = await client.DELETE("/v1/experiments/{id}/runs/{runId}/baseline", {
+                params: { path: { id: opts.experiment, runId: opts.run } },
+            });
+            formatDetail(data, command);
+        });
+    });
+    // ── comparison ───────────────────────────────────────────────────
+    const comparison = new Command("comparison").description("Get the comparison matrix for an experiment (dataset items × variants)");
+    addPaginationOptions(comparison);
+    comparison.argument("<id>", "Experiment ID");
+    comparison.action(async (id, opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            const { data } = await client.GET("/v1/experiments/{id}/comparison", {
+                params: {
+                    path: { id },
+                    query: paginationParams(opts),
+                },
+            });
+            if (isJsonOutput(command)) {
+                console.log(JSON.stringify(data, null, 2));
+                return;
+            }
+            const result = data;
+            const aggregates = result?.aggregates ?? [];
+            const page = result?.items;
+            const variantNames = new Map(aggregates.map((a) => [a.variantId, a.variantName ?? a.variantId]));
+            const variantIds = [...variantNames.keys()];
+            if (!page?.content || page.content.length === 0) {
+                console.log(chalk.dim("No comparison data found."));
+            }
+            else {
+                const table = new Table({
+                    head: ["Item", ...variantIds.map((v) => variantNames.get(v) ?? v)].map((h) => chalk.cyan(h)),
+                    wordWrap: true,
+                });
+                for (const item of page.content) {
+                    const resultsByVariant = item.resultsByVariantId ?? {};
+                    table.push([
+                        item.datasetItemId,
+                        ...variantIds.map((v) => formatComparisonCell(resultsByVariant[v])),
+                    ]);
+                }
+                console.log(table.toString());
+                console.log(chalk.dim(`Page ${(page.number ?? 0) + 1} of ${page.totalPages ?? 0} (${page.totalElements ?? 0} total)`));
+            }
+            if (aggregates.length > 0) {
+                const aggTable = new Table({
+                    head: ["Variant", "Items", "Avg Duration", "Avg Tokens", "Total Cost"].map((h) => chalk.cyan(h)),
+                    wordWrap: true,
+                });
+                for (const agg of aggregates) {
+                    aggTable.push([
+                        agg.variantName ?? agg.variantId,
+                        agg.itemCount ?? "—",
+                        agg.avgDurationMs != null ? `${Math.round(agg.avgDurationMs)}ms` : "—",
+                        agg.avgTokens != null ? Math.round(agg.avgTokens) : "—",
+                        agg.totalCost != null ? `$${agg.totalCost}` : "—",
+                    ]);
+                }
+                console.log(chalk.bold("\nAggregates:"));
+                console.log(aggTable.toString());
+            }
+        });
+    });
+    cmd.addCommand(comparison);
+    // ── regression ───────────────────────────────────────────────────
+    const regression = new Command("regression").description("Compare a run against its baseline and report regressions/improvements");
+    regression.requiredOption("--experiment <id>", "Experiment ID");
+    regression.requiredOption("--run <id>", "Run ID");
+    regression.option("--baseline-run <id>", "Baseline run ID (defaults to the run marked baseline, then the previous COMPLETED run)");
+    regression.option("--threshold <number>", "Score delta threshold for classifying a regression", parseFloat);
+    regression.action(async (opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            const { data } = await client.GET("/v1/experiments/{id}/runs/{runId}/regression", {
+                params: {
+                    path: { id: opts.experiment, runId: opts.run },
+                    query: {
+                        baselineRunId: opts.baselineRun,
+                        threshold: opts.threshold,
+                    },
+                },
+            });
+            if (isJsonOutput(command)) {
+                console.log(JSON.stringify(data, null, 2));
+                return;
+            }
+            const report = data;
+            const summary = report?.summary ?? {};
+            console.log(chalk.bold(`Regression report for run ${report?.runId}`));
+            console.log(chalk.dim(`Baseline: ${report?.baselineRunId ?? "—"} (${report?.baselineSource ?? "NONE"})` +
+                (report?.threshold != null ? `, threshold ${report.threshold}` : "")));
+            const summaryTable = new Table();
+            for (const [key, value] of Object.entries(summary)) {
+                summaryTable.push({ [chalk.cyan(key)]: value == null ? chalk.dim("—") : String(value) });
+            }
+            console.log(summaryTable.toString());
+            const groups = [
+                ["Regressed", report?.regressed ?? []],
+                ["Improved", report?.improved ?? []],
+                ["Ground Truth Changed", report?.gtChanged ?? []],
+            ];
+            for (const [label, items] of groups) {
+                if (items.length === 0)
+                    continue;
+                console.log(chalk.bold(`\n${label}:`));
+                const table = new Table({
+                    head: ["Item", "Baseline", "Current", "Delta"].map((h) => chalk.cyan(h)),
+                    wordWrap: true,
+                });
+                for (const item of items) {
+                    table.push([
+                        item.datasetItemId,
+                        item.baselineScore ?? "—",
+                        item.currentScore ?? "—",
+                        item.delta ?? "—",
+                    ]);
+                }
+                console.log(table.toString());
+            }
+        });
+    });
+    cmd.addCommand(regression);
     return cmd;
 }
 //# sourceMappingURL=experiments.js.map

package/dist/commands/extractions.js

@@ -2,7 +2,7 @@ import { Command } from "commander";
 import { readFileSync } from "node:fs";
 import { extname } from "node:path";
 import { getClient, runAction } from "../lib/client.js";
-import { formatPage, formatDetail, withSpinner } from "../lib/output.js";
+import { formatPage, formatDetail, formatSuccess, withSpinner } from "../lib/output.js";
 import { addPaginationOptions, paginationParams } from "../lib/pagination.js";
 const IMAGE_MIME_TYPES = {
     ".png": "image/png",
@@ -74,6 +74,19 @@ export function makeExtractionsCommand() {
             formatDetail(data, command);
         });
     });
+    cmd
+        .command("delete")
+        .description("Delete an extraction")
+        .argument("<id>", "Extraction ID")
+        .action(async (id, _opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            await client.DELETE("/v1/extractions/{id}", {
+                params: { path: { id } },
+            });
+            formatSuccess(`Extraction ${id} deleted.`, command);
+        });
+    });
     const list = new Command("list").description("List extractions");
     addPaginationOptions(list);
     list.option("-s, --search <term>", "Search text");

package/dist/commands/files.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function makeFilesCommand(): Command;
+//# sourceMappingURL=files.d.ts.map

package/dist/commands/files.js

@@ -0,0 +1,143 @@
+import { Command } from "commander";
+import { writeFileSync } from "node:fs";
+import chalk from "chalk";
+import { getClient, resolveAuthHeader, runAction } from "../lib/client.js";
+import { resolveConfig } from "../lib/config.js";
+import { BackboneApiError } from "../lib/errors.js";
+import { formatPage, formatDetail, formatSuccess, withSpinner } from "../lib/output.js";
+import { addPaginationOptions, paginationParams } from "../lib/pagination.js";
+import { fileToBlob } from "../lib/multipart.js";
+/**
+ * Upload a single file with purpose=agent_input|knowledge. Bypasses the typed
+ * client the same way convert.ts and transcribe.ts do: the generated types
+ * describe the multipart body as an opaque `file: string`, which is not
+ * something FormData can produce a match for.
+ */
+async function uploadFile(command, filePath, purpose) {
+    const config = resolveConfig(command);
+    const authHeader = await resolveAuthHeader(config);
+    const { blob, filename } = fileToBlob(filePath);
+    const formData = new FormData();
+    formData.append("file", blob, filename);
+    const baseUrl = config.baseUrl.replace(/\/+$/, "");
+    const url = `${baseUrl}/v1/files?purpose=${encodeURIComponent(purpose)}`;
+    const res = await fetch(url, {
+        method: "POST",
+        headers: { Authorization: authHeader },
+        body: formData,
+    });
+    if (!res.ok) {
+        let body;
+        try {
+            body = await res.json();
+        }
+        catch {
+            body = {
+                error: res.statusText,
+                message: `HTTP ${res.status}: ${res.statusText}`,
+                status: res.status,
+                timestamp: new Date().toISOString(),
+            };
+        }
+        throw new BackboneApiError(body);
+    }
+    return (await res.json());
+}
+/** Stream a file's bytes from /v1/files/{fileId}/content via a raw fetch. */
+async function downloadFile(command, fileId) {
+    const config = resolveConfig(command);
+    const authHeader = await resolveAuthHeader(config);
+    const baseUrl = config.baseUrl.replace(/\/+$/, "");
+    const url = `${baseUrl}/v1/files/${encodeURIComponent(fileId)}/content`;
+    const res = await fetch(url, {
+        headers: { Authorization: authHeader },
+    });
+    if (!res.ok) {
+        let body;
+        try {
+            body = await res.json();
+        }
+        catch {
+            body = {
+                error: res.statusText,
+                message: `HTTP ${res.status}: ${res.statusText}`,
+                status: res.status,
+                timestamp: new Date().toISOString(),
+            };
+        }
+        throw new BackboneApiError(body);
+    }
+    const disposition = res.headers.get("content-disposition") ?? "";
+    const match = /filename\*?=(?:UTF-8'')?"?([^";]+)"?/i.exec(disposition);
+    const filename = match ? decodeURIComponent(match[1]) : undefined;
+    const bytes = Buffer.from(await res.arrayBuffer());
+    return { bytes, filename, contentType: res.headers.get("content-type") ?? undefined };
+}
+export function makeFilesCommand() {
+    const cmd = new Command("files").description("Manage uploaded files");
+    const list = new Command("list").description("List files");
+    addPaginationOptions(list);
+    list.option("--purpose <purpose>", "Filter by purpose (agent_input, knowledge)");
+    list.action(async (opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            const { data } = await client.GET("/v1/files", {
+                params: { query: { purpose: opts.purpose, ...paginationParams(opts) } },
+            });
+            formatPage(data, command, ["id", "filename", "purpose", "bytes", "created_at"]);
+        });
+    });
+    cmd.addCommand(list);
+    cmd
+        .command("get")
+        .description("Retrieve a file's metadata")
+        .argument("<fileId>", "File ID")
+        .action(async (fileId, _opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            const { data } = await client.GET("/v1/files/{fileId}", {
+                params: { path: { fileId } },
+            });
+            formatDetail(data, command);
+        });
+    });
+    cmd
+        .command("upload")
+        .description("Upload a file")
+        .argument("<path>", "Local file path")
+        .requiredOption("--purpose <purpose>", "File purpose (agent_input, knowledge)")
+        .action(async (path, opts, command) => {
+        await runAction(command, async () => {
+            const data = await withSpinner("Uploading...", () => uploadFile(command, path, opts.purpose));
+            formatDetail(data, command);
+        });
+    });
+    cmd
+        .command("download")
+        .description("Download a file's content")
+        .argument("<fileId>", "File ID")
+        .option("-o, --output <path>", "Write bytes to this path instead of the file's own name")
+        .action(async (fileId, opts, command) => {
+        await runAction(command, async () => {
+            const { bytes, filename } = await withSpinner("Downloading...", () => downloadFile(command, fileId));
+            const outputPath = opts.output ?? filename ?? fileId;
+            writeFileSync(outputPath, bytes);
+            console.error(chalk.green(`Wrote ${bytes.length} bytes to ${outputPath}`));
+        });
+    });
+    cmd
+        .command("delete")
+        .description("Delete an agent_input file (knowledge files are not deletable through this API)")
+        .argument("<fileId>", "File ID")
+        .action(async (fileId, _opts, command) => {
+        await runAction(command, async () => {
+            const client = getClient(command);
+            await client.DELETE("/v1/files/{fileId}", {
+                params: { path: { fileId } },
+            });
+            formatSuccess(`File ${fileId} deleted.`, command);
+        });
+    });
+    return cmd;
+}
+//# sourceMappingURL=files.js.map

package/dist/commands/installations.d.ts

@@ -0,0 +1,3 @@
+import { Command } from "commander";
+export declare function makeInstallationsCommand(): Command;
+//# sourceMappingURL=installations.d.ts.map