codiedev 0.9.77 → 0.9.79

package/dist/cli.js

@@ -95,11 +95,14 @@ Starter packs:
 
 Traces (agents on the gateway):
   codiedev traces agents                 Agents you maintain, with saved test cases
-  codiedev traces cases <agent> [--dataset name] [--model id]
+  codiedev traces cases <agent> [--dataset name [--version n]] [--model id]
                                          Pull an agent's test cases (or one dataset)
                                          into this repo for your coding agent to
                                          run, optionally on another model
   codiedev traces report <results.json>  Send the results back to Traces
+  codiedev traces overview | runs | run | score | scores | score-kinds | queue | judge | import | add-example | versions | save-version
+                                         Agents, runs, failures, scores, review queues,
+                                         automatic scoring and datasets (codiedev traces help)
   codiedev evals report <run.json>       File one variant's eval scores under an experiment
   codiedev evals last <subject>          Latest experiment on a subject + its champion
 

package/dist/commands/traces.d.ts

@@ -27,6 +27,9 @@ interface CasesResponse {
     ok: boolean;
     /** The suite pulled, when --suite was given. */
     suite: string | null;
+    /** The saved dataset version pulled, when --version was given; cases deleted since are counted in `missing`. */
+    version?: number | null;
+    missing?: number;
     /** How the suite is judged, when it has settings. */
     judge: {
         model: string | null;
@@ -44,6 +47,7 @@ interface CasesResponse {
 export interface ResultsFile {
     agent: string;
     suite: string | null;
+    version?: number | null;
     model: string;
     judgeModel: string | null;
     harness: string | null;

package/dist/commands/traces.js

@@ -2,6 +2,7 @@
 // `codiedev traces agents`
 // `codiedev traces cases <agent> [--dataset name] [--model id] [--out dir] [--json]`
 // `codiedev traces report <results.json> [--harness name]`
+// (scores, review queues, automatic scoring and datasets: ./tracesLabel.ts)
 //
 // Test cases are runs an engineer saved from Codiedev → Traces. They run in
 // the engineer's coding agent, not in the portal: `cases` writes them into the
@@ -51,14 +52,15 @@ exports.runTraces = runTraces;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const shared_1 = require("./shared");
+const tracesLabel_1 = require("./tracesLabel");
 exports.TRACES_HELP = `
-codiedev traces — test cases saved from Codiedev → Traces
+codiedev traces — test cases, scores, review queues and automatic scoring from Codiedev → Traces
 
   codiedev traces agents                 Agents you maintain, with saved case counts
-  codiedev traces cases <agent> [--dataset name] [--case id[,id]] [--model id] [--out dir] [--json]
+  codiedev traces cases <agent> [--dataset name [--version n]] [--case id[,id]] [--model id] [--out dir] [--json]
                                          Write an agent's cases (one dataset, or just
                                          the cases named) into this repo, default
-                                         .codiedev/traces/<agent>[/<dataset>]/, then
+                                         .codiedev/traces/<agent>[/<dataset>[/v<n>]]/, then
                                          ask your coding agent to run them.
                                          --model runs them on another model, e.g.
                                          the replacement for a retiring one
@@ -66,6 +68,36 @@ codiedev traces — test cases saved from Codiedev → Traces
                                          Send the filled-in results back, so
                                          Traces shows the pass rate for that model
 
+  codiedev traces overview [--days n] [--json]
+                                         Every agent you can see: runs, failures,
+                                         spend, and what's going wrong most
+  codiedev traces runs <agent> [--failed] [--flag name] [--model id] [--verdict pass|fail] [--days n] [--limit n] [--json]
+                                         An agent's runs, newest first
+  codiedev traces run <run> [--agent name] [--json]
+                                         One run: steps, what went wrong and the fix,
+                                         scores, prompts and replies (maintainers)
+  codiedev traces score-kinds [add <name> --type number|categories|boolean [--min n --max n] [--categories a,b]]
+                                         List or add the kinds of score your team uses
+  codiedev traces queue create <name> [--scores a,b] [--description text]
+  codiedev traces score <run> <score> <value> [--agent name] [--comment text] [--step n]
+                                         Score a run (a portal run id agent~run, or
+                                         the agent's own run id with --agent)
+  codiedev traces scores <agent> [--days n] [--json]
+                                         Each score's average or mix, and the score kinds
+  codiedev traces queue list             Review queues and how many are left
+  codiedev traces queue add <queue> --agent <name> [--failed] [--days n] [--run id ...]
+                                         File runs into a queue for people to review
+  codiedev traces judge <agent> [--on | --off] [--scores a,b] [--instructions text]
+        [--model id] [--sample percent] [--daily n] [--budget usd | --key cdvm_…]
+                                         Show or set automatic scoring (a judge model
+                                         scores a sample of runs, billed to your key)
+  codiedev traces import <agent> <file.csv|file.jsonl> --dataset <name>
+                                         Add examples from a file (input, expected, name)
+  codiedev traces add-example <agent> --dataset <name> --input <text> [--expected <text>] [--name <text>]
+  codiedev traces versions <dataset>     A dataset's saved versions and what changed
+  codiedev traces save-version <dataset> [--note text]
+                                         Save the dataset as its next version
+
   <agent> is the agent's name (its API key name) or id. Only an agent's
   maintainers can pull its cases or report on them.
 `.trim();
@@ -78,7 +110,7 @@ const FLAG_LABEL = {
 };
 const TRIES = 3;
 const plural = (n, word) => `${n} ${word}${n === 1 ? "" : "s"}`;
-const recordedModel = (res) => res.cases.flatMap((c) => c.steps.map((s) => s.model))[0] ?? "the recorded model";
+const recordedModel = (res) => res.cases.flatMap((c) => c.steps.map((s) => s.model)).filter(Boolean)[0] ?? "the agent's current model";
 function slugify(name) {
     return name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "") || "agent";
 }
@@ -90,7 +122,7 @@ function casesBrief(res, casesFile, opts = {}) {
     const target = opts.model;
     const results = opts.resultsFile ? path.basename(opts.resultsFile) : "results.json";
     const lines = [
-        `# Test cases for ${res.agent.name}${res.suite ? ` · ${res.suite}` : ""}${target ? ` on ${target}` : ""}`,
+        `# Test cases for ${res.agent.name}${res.suite ? ` · ${res.suite}` : ""}${res.version ? ` v${res.version}` : ""}${target ? ` on ${target}` : ""}`,
         "",
         res.agent.description ? `${res.agent.description}\n` : "",
         `Saved from Codiedev → Traces. ${plural(res.cases.length, "case")} in \`${path.basename(casesFile)}\` (one JSON object per line):`,
@@ -101,7 +133,7 @@ function casesBrief(res, casesFile, opts = {}) {
     if (target) {
         lines.push("## This is a model check", "", `Run every case on **${target}** instead of ${recordedModel(res)} (the model the cases were recorded on). Change only the model the agent asks the gateway for; keep its prompts and tools as they are, so the result says how ${target} does on the same work.`, "");
     }
-    lines.push("## How to run them", "", "1. Find where this agent builds its model requests and calls the Codiedev gateway.", `2. For each case, replay \`steps[0].request\` (the first prompt the agent sent) through the agent's own code path, so tools and prompts are the current version, not the recorded one${target ? `, with the model set to ${target}` : ""}.`, "3. Judge each try against the case:", "   - A case's `expected`, when present, is what a passing answer looks like (a reference reply or criteria). It wins over everything below.", "   - `fix` cases: the run must not reproduce its `flags` (listed below), and must finish (no failed call, no `finish_reason: \"length\"`, no repeated identical tool call).", "   - `keep` cases: the final reply must still do what the recorded final reply did. Compare meaning, not wording.", "   - A case's `note`, when present, is the engineer's extra context.");
+    lines.push("## How to run them", "", "1. Find where this agent builds its model requests and calls the Codiedev gateway.", `2. For each case, replay \`steps[0].request\` (the first prompt the agent sent) through the agent's own code path, so tools and prompts are the current version, not the recorded one${target ? `, with the model set to ${target}` : ""}.`, "3. Judge each try against the case:", "   - A case's `expected`, when present, is what a passing answer looks like (a reference reply or criteria). It wins over everything below.", "   - `fix` cases: the run must not reproduce its `flags` (listed below), and must finish (no failed call, no `finish_reason: \"length\"`, no repeated identical tool call).", "   - `keep` cases: the final reply must still do what the recorded final reply did. Compare meaning, not wording.", "   - A case's `note`, when present, is the engineer's extra context.", "   - Cases whose `runId` starts with `manual:` or `import:` were written by hand, not recorded: `steps[0].request` is the input and there's no recorded reply, so judge them on `expected`.");
     if (res.judge?.model || res.judge?.rubric) {
         lines.push(`   - Where passing needs judgement, judge with **${res.judge.model ?? "a strong model"}** through the gateway${res.judge.rubric ? `, using this rubric: "${res.judge.rubric}"` : ""}. Use the same judge for every model so results compare.`);
     }
@@ -121,6 +153,7 @@ function resultsTemplate(res, model) {
     return {
         agent: res.agent.name,
         suite: res.suite,
+        ...(res.version ? { version: res.version } : {}),
         model: model ?? recordedModel(res),
         judgeModel: res.judge?.model ?? null,
         harness: null,
@@ -140,7 +173,7 @@ function parseCasesArgs(args) {
     for (let i = 0; i < args.length; i++) {
         const a = args[i];
         let matched = false;
-        for (const [name, key] of [["--out", "out"], ["-o", "out"], ["--dataset", "suite"], ["--suite", "suite"], ["--model", "model"], ["--case", "caseIds"]]) {
+        for (const [name, key] of [["--out", "out"], ["-o", "out"], ["--dataset", "suite"], ["--suite", "suite"], ["--model", "model"], ["--case", "caseIds"], ["--version", "version"]]) {
             const f = flag(args, i, name);
             if (f.skip >= 0) {
                 parsed[key] = f.value;
@@ -182,20 +215,26 @@ async function runAgents(config) {
     console.log("\nPull one: codiedev traces cases <agent>");
 }
 async function runCases(args, config) {
-    const { agent, suite, model, caseIds, out, json } = parseCasesArgs(args);
+    const { agent, suite, model, caseIds, version, out, json } = parseCasesArgs(args);
     if (!agent) {
-        console.error("Usage: codiedev traces cases <agent> [--dataset name] [--model id] [--out dir] [--json]");
+        console.error("Usage: codiedev traces cases <agent> [--dataset name [--version n]] [--model id] [--out dir] [--json]");
+        process.exit(1);
+    }
+    if (version !== undefined && (!suite || !/^[1-9]\d*$/.test(version))) {
+        console.error("--version takes a whole number and needs --dataset: codiedev traces cases <agent> --dataset <name> --version <n>");
         process.exit(1);
     }
     const start = Date.now();
     let res;
     try {
-        res = await (0, shared_1.apiRequest)("GET", "/api/cli/traces/cases", { config, query: { agent, ...(suite ? { suite } : {}), ...(caseIds ? { case: caseIds } : {}) } });
+        res = await (0, shared_1.apiRequest)("GET", "/api/cli/traces/cases", { config, query: { agent, ...(suite ? { suite } : {}), ...(version ? { version } : {}), ...(caseIds ? { case: caseIds } : {}) } });
     }
     catch (err) {
         const e = err;
         (0, shared_1.recordAgentEvent)(config, { tool: "codiedev_traces_cases", ok: false, latencyMs: Date.now() - start, error: e.message });
-        if (e.status === 404)
+        if (e.status === 404 && version && /No version/.test(e.message))
+            console.error(`No version ${version} of "${suite}". See its versions in Codiedev → Traces → Test cases.`);
+        else if (e.status === 404)
             console.error(`No agent called "${agent}". See your agents with: codiedev traces agents`);
         else if (e.status === 403)
             console.error(`Only ${agent}'s maintainers can pull its test cases.`);
@@ -214,7 +253,7 @@ async function runCases(args, config) {
             : `${res.agent.name} has no saved test cases. Open a run in Codiedev → Traces and click "Save as test case".`);
         return;
     }
-    const dir = path.resolve(process.cwd(), out ?? path.join(".codiedev", "traces", slugify(res.agent.name), ...(res.suite ? [slugify(res.suite)] : []), ...(caseIds && res.cases.length === 1 ? [slugify(res.cases[0].name)] : []), ...(model ? [slugify(model)] : [])));
+    const dir = path.resolve(process.cwd(), out ?? path.join(".codiedev", "traces", slugify(res.agent.name), ...(res.suite ? [slugify(res.suite)] : []), ...(res.version ? [`v${res.version}`] : []), ...(caseIds && res.cases.length === 1 ? [slugify(res.cases[0].name)] : []), ...(model ? [slugify(model)] : [])));
     fs.mkdirSync(dir, { recursive: true });
     const casesFile = path.join(dir, "cases.jsonl");
     fs.writeFileSync(casesFile, res.cases.map((c) => JSON.stringify(c)).join("\n") + "\n", "utf8");
@@ -224,7 +263,9 @@ async function runCases(args, config) {
     fs.writeFileSync(briefFile, casesBrief(res, casesFile, { model, resultsFile }), "utf8");
     const fix = res.cases.filter((c) => c.kind === "fix").length;
     const rel = (file) => path.relative(process.cwd(), file) || file;
-    console.log(`✓ ${plural(res.cases.length, "case")} for ${res.agent.name}${res.suite ? ` · ${res.suite}` : ""}${model ? ` on ${model}` : ""} (${fix} to fix, ${res.cases.length - fix} to keep passing)`);
+    console.log(`✓ ${plural(res.cases.length, "case")} for ${res.agent.name}${res.suite ? ` · ${res.suite}` : ""}${res.version ? ` v${res.version}` : ""}${model ? ` on ${model}` : ""} (${fix} to fix, ${res.cases.length - fix} to keep passing)`);
+    if (res.missing)
+        console.log(`  ${plural(res.missing, "case")} in v${res.version} were deleted since and can't be pulled.`);
     console.log(`  ${rel(casesFile)}`);
     console.log(`  ${rel(briefFile)}`);
     console.log(`  ${rel(resultsFile)}  (filled in by your coding agent, then: codiedev traces report ${rel(resultsFile)})`);
@@ -286,6 +327,30 @@ async function runTraces(args, configOverride) {
         return runCases(rest, configOverride ?? (0, shared_1.requireConfig)());
     if (sub === "report")
         return runReport(rest, configOverride ?? (0, shared_1.requireConfig)());
+    if (sub === "overview")
+        return (0, tracesLabel_1.runOverview)(rest, configOverride ?? (0, shared_1.requireConfig)());
+    if (sub === "runs")
+        return (0, tracesLabel_1.runRuns)(rest, configOverride ?? (0, shared_1.requireConfig)());
+    if (sub === "run")
+        return (0, tracesLabel_1.runRun)(rest, configOverride ?? (0, shared_1.requireConfig)());
+    if (sub === "score-kinds")
+        return (0, tracesLabel_1.runScoreKinds)(rest, configOverride ?? (0, shared_1.requireConfig)());
+    if (sub === "score")
+        return (0, tracesLabel_1.runScore)(rest, configOverride ?? (0, shared_1.requireConfig)());
+    if (sub === "scores")
+        return (0, tracesLabel_1.runScores)(rest, configOverride ?? (0, shared_1.requireConfig)());
+    if (sub === "queue")
+        return (0, tracesLabel_1.runQueue)(rest, configOverride ?? (0, shared_1.requireConfig)());
+    if (sub === "judge")
+        return (0, tracesLabel_1.runJudge)(rest, configOverride ?? (0, shared_1.requireConfig)());
+    if (sub === "import")
+        return (0, tracesLabel_1.runImport)(rest, configOverride ?? (0, shared_1.requireConfig)());
+    if (sub === "add-example")
+        return (0, tracesLabel_1.runAddExample)(rest, configOverride ?? (0, shared_1.requireConfig)());
+    if (sub === "versions")
+        return (0, tracesLabel_1.runVersions)(rest, configOverride ?? (0, shared_1.requireConfig)());
+    if (sub === "save-version")
+        return (0, tracesLabel_1.runSaveVersion)(rest, configOverride ?? (0, shared_1.requireConfig)());
     console.log(exports.TRACES_HELP);
     if (sub && sub !== "help")
         process.exit(1);

package/dist/commands/tracesLabel.d.ts

@@ -0,0 +1,24 @@
+import type { CodiedevConfig } from "../utils";
+type Parsed = {
+    positional: string[];
+    flags: Record<string, string>;
+    bools: Set<string>;
+    runs: string[];
+};
+export declare function parseArgs(args: string[]): Parsed;
+/** "4" → 4, "true" → true, anything else as text (a category). */
+export declare function scoreValue(raw: string): number | boolean | string;
+export declare function runScore(args: string[], config: CodiedevConfig): Promise<void>;
+export declare function runScores(args: string[], config: CodiedevConfig): Promise<void>;
+export declare function runQueue(args: string[], config: CodiedevConfig): Promise<void>;
+export declare function runJudge(args: string[], config: CodiedevConfig): Promise<void>;
+export declare function runImport(args: string[], config: CodiedevConfig): Promise<void>;
+export declare function runAddExample(args: string[], config: CodiedevConfig): Promise<void>;
+export declare function runVersions(args: string[], config: CodiedevConfig): Promise<void>;
+export declare function runSaveVersion(args: string[], config: CodiedevConfig): Promise<void>;
+export declare function runOverview(args: string[], config: CodiedevConfig): Promise<void>;
+export declare function runRuns(args: string[], config: CodiedevConfig): Promise<void>;
+export declare function runRun(args: string[], config: CodiedevConfig): Promise<void>;
+export declare function runScoreKinds(args: string[], config: CodiedevConfig): Promise<void>;
+export declare function runQueueCreate(args: string[], config: CodiedevConfig): Promise<void>;
+export {};

package/dist/commands/tracesLabel.js

@@ -0,0 +1,356 @@
+"use strict";
+// `codiedev traces overview | runs | run | score | scores | score-kinds |
+// queue | judge | import | add-example | versions | save-version`: what the
+// Traces portal shows and does (agents, runs, failures, scores, review
+// queues, automatic scoring, datasets), so a coding agent can do it from
+// plain words ("put this week's failed runs in the Week 44 queue"). Each
+// calls a /api/cli/traces/* route that runs the portal's own code, with the
+// same permissions.
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseArgs = parseArgs;
+exports.scoreValue = scoreValue;
+exports.runScore = runScore;
+exports.runScores = runScores;
+exports.runQueue = runQueue;
+exports.runJudge = runJudge;
+exports.runImport = runImport;
+exports.runAddExample = runAddExample;
+exports.runVersions = runVersions;
+exports.runSaveVersion = runSaveVersion;
+exports.runOverview = runOverview;
+exports.runRuns = runRuns;
+exports.runRun = runRun;
+exports.runScoreKinds = runScoreKinds;
+exports.runQueueCreate = runQueueCreate;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const shared_1 = require("./shared");
+const BOOL_FLAGS = new Set(["--failed", "--on", "--off", "--json"]);
+function parseArgs(args) {
+    const out = { positional: [], flags: {}, bools: new Set(), runs: [] };
+    for (let i = 0; i < args.length; i++) {
+        const a = args[i];
+        if (BOOL_FLAGS.has(a)) {
+            out.bools.add(a);
+            continue;
+        }
+        if (a.startsWith("--")) {
+            const eq = a.indexOf("=");
+            const name = eq > 0 ? a.slice(0, eq) : a;
+            const value = eq > 0 ? a.slice(eq + 1) : args[++i];
+            if (value === undefined)
+                throw new Error(`${name} needs a value`);
+            if (name === "--run")
+                out.runs.push(value);
+            else
+                out.flags[name] = value;
+            continue;
+        }
+        out.positional.push(a);
+    }
+    return out;
+}
+/** "4" → 4, "true" → true, anything else as text (a category). */
+function scoreValue(raw) {
+    const t = raw.trim();
+    if (/^(true|false)$/i.test(t))
+        return t.toLowerCase() === "true";
+    if (t !== "" && Number.isFinite(Number(t)))
+        return Number(t);
+    return t;
+}
+async function call(config, tool, method, route, opts) {
+    const start = Date.now();
+    try {
+        const res = await (0, shared_1.apiRequest)(method, `/api/cli/traces/${route}`, { config, ...opts });
+        (0, shared_1.recordAgentEvent)(config, { tool, ok: true, latencyMs: Date.now() - start });
+        return res;
+    }
+    catch (err) {
+        const e = err;
+        (0, shared_1.recordAgentEvent)(config, { tool, ok: false, latencyMs: Date.now() - start, error: e.message });
+        console.error(e.message);
+        process.exit(1);
+    }
+}
+function usage(text) {
+    console.error(`Usage: ${text}`);
+    process.exit(1);
+}
+async function runScore(args, config) {
+    const p = parseArgs(args);
+    const [run, name, value] = p.positional;
+    if (!run || !name || value === undefined)
+        usage("codiedev traces score <run> <score> <value> [--agent name] [--comment text] [--step n]");
+    const res = await call(config, "codiedev_traces_score", "POST", "score", {
+        body: { run, name, value: scoreValue(value), agent: p.flags["--agent"], comment: p.flags["--comment"], ...(p.flags["--step"] ? { step: Number(p.flags["--step"]) } : {}) },
+    });
+    console.log(`✓ Scored ${res.runId}: ${name} = ${value}`);
+}
+async function runScores(args, config) {
+    const p = parseArgs(args);
+    const agent = p.positional[0];
+    if (!agent)
+        usage("codiedev traces scores <agent> [--days n] [--json]");
+    const res = await call(config, "codiedev_traces_scores", "GET", "scores", { query: { agent, days: p.flags["--days"] ?? "7" } });
+    if (p.bools.has("--json")) {
+        process.stdout.write(JSON.stringify(res, null, 2) + "\n");
+        return;
+    }
+    if (!res.summary.length)
+        console.log(`No scores on ${res.agent} in the last ${p.flags["--days"] ?? 7} days.`);
+    for (const s of res.summary) {
+        const head = s.dataType === "boolean" && s.average !== null ? `${Math.round(s.average * 100)}% true`
+            : s.dataType === "numeric" && s.average !== null ? `avg ${Math.round(s.average * 100) / 100}`
+                : s.categories.map((c) => `${c.label} ${c.count}`).join(", ");
+        const from = [s.bySource.annotation ? `${s.bySource.annotation} people` : "", s.bySource.api ? `${s.bySource.api} your app` : "", s.bySource.judge ? `${s.bySource.judge} judge` : ""].filter(Boolean).join(", ");
+        console.log(`${s.name.padEnd(24)} ${head.padEnd(28)} ${s.count} scores (${from})`);
+    }
+    if (res.kinds.length)
+        console.log(`\nScore kinds: ${res.kinds.map((k) => `${k.name} (${k.dataType === "numeric" ? `${k.min ?? ""}–${k.max ?? ""}` : k.dataType === "boolean" ? "true/false" : k.categories.join("/")})`).join(", ")}`);
+}
+async function runQueue(args, config) {
+    const [verb, ...rest] = args;
+    if (verb === "list" || verb === undefined) {
+        const res = await call(config, "codiedev_traces_queue_list", "GET", "queues", {});
+        const live = res.queues.filter((q) => !q.archived);
+        if (!live.length)
+            console.log("No review queues yet. Make one in Codiedev → Traces → Review.");
+        for (const q of live)
+            console.log(`${q.name.padEnd(28)} ${String(q.pending).padStart(4)} to review  ${String(q.done).padStart(4)} done  ${q.scoreNames.join(", ")}`);
+        return;
+    }
+    if (verb === "create")
+        return runQueueCreate(rest, config);
+    if (verb !== "add")
+        usage("codiedev traces queue list | queue create <name> [--scores a,b] | queue add <queue> --agent <name> [--failed] [--days n] [--run id ...]");
+    const p = parseArgs(rest);
+    const queue = p.positional[0];
+    if (!queue || (!p.flags["--agent"] && !p.runs.length))
+        usage("codiedev traces queue add <queue> --agent <name> [--failed] [--days n] [--run id ...]");
+    const res = await call(config, "codiedev_traces_queue_add", "POST", "queue", {
+        body: { queue, agent: p.flags["--agent"], failed: p.bools.has("--failed"), ...(p.flags["--days"] ? { days: Number(p.flags["--days"]) } : {}), ...(p.runs.length ? { runs: p.runs } : {}) },
+    });
+    console.log(`✓ Added ${res.added} ${res.added === 1 ? "run" : "runs"} to ${res.queue}${res.skipped ? ` (${res.skipped} already there or not yours)` : ""}. Review them in Codiedev → Traces → Review.`);
+}
+function printJudge(agent, judge) {
+    if (!judge || judge === "none") {
+        console.log(`Automatic scoring isn't set up for ${agent}.`);
+        return;
+    }
+    console.log(`${agent}: automatic scoring ${judge.enabled ? "on" : "off"}`);
+    console.log(`  scores ${judge.scoreNames.join(", ")} · ${Math.round(judge.sampleRate * 100)}% of runs, up to ${judge.dailyMaxRuns}/day · ${judge.model}`);
+    if (judge.judgeKey)
+        console.log(`  billed to ${judge.judgeKey.name}${judge.judgeKey.monthlyBudgetUsd !== null ? ` ($${judge.judgeKey.monthlyBudgetUsd}/mo)` : ""} · ${judge.judgedToday} judged in 24h${judge.errorsToday ? `, ${judge.errorsToday} failed` : ""}`);
+    if (judge.lastError)
+        console.log(`  last problem: ${judge.lastError}`);
+}
+async function runJudge(args, config) {
+    const p = parseArgs(args);
+    const agent = p.positional[0];
+    const help = "codiedev traces judge <agent> [--on | --off] [--scores a,b] [--instructions text] [--model id] [--sample percent] [--daily n] [--budget usd | --key cdvm_…]";
+    if (!agent)
+        usage(help);
+    const changing = p.bools.has("--on") || p.bools.has("--off") || Object.keys(p.flags).length > 0;
+    if (!changing) {
+        const res = await call(config, "codiedev_traces_judge", "GET", "judge", { query: { agent } });
+        printJudge(res.agent, res.judge);
+        return;
+    }
+    const res = await call(config, "codiedev_traces_judge_set", "POST", "judge", {
+        body: {
+            agent,
+            ...(p.bools.has("--on") ? { enabled: true } : p.bools.has("--off") ? { enabled: false } : {}),
+            ...(p.flags["--scores"] ? { scores: p.flags["--scores"].split(",").map((s) => s.trim()).filter(Boolean) } : {}),
+            ...(p.flags["--instructions"] ? { instructions: p.flags["--instructions"] } : {}),
+            ...(p.flags["--model"] ? { model: p.flags["--model"] } : {}),
+            ...(p.flags["--sample"] ? { sampleRate: Number(p.flags["--sample"]) / 100 } : {}),
+            ...(p.flags["--daily"] ? { dailyMaxRuns: Number(p.flags["--daily"]) } : {}),
+            ...(p.flags["--budget"] ? { newKeyBudgetUsd: Number(p.flags["--budget"]) } : {}),
+            ...(p.flags["--key"] ? { pastedKey: p.flags["--key"] } : {}),
+        },
+    });
+    printJudge(res.agent, res.judge);
+}
+async function runImport(args, config) {
+    const p = parseArgs(args);
+    const [agent, file] = p.positional;
+    const dataset = p.flags["--dataset"];
+    if (!agent || !file || !dataset)
+        usage("codiedev traces import <agent> <file.csv|file.jsonl> --dataset <name>");
+    let text;
+    try {
+        text = fs.readFileSync(path.resolve(process.cwd(), file), "utf8");
+    }
+    catch (err) {
+        console.error(`Couldn't read ${file}: ${err.message}`);
+        process.exit(1);
+    }
+    const res = await call(config, "codiedev_traces_import", "POST", "examples", { body: { agent, dataset, text } });
+    console.log(`✓ Imported ${res.added} ${res.added === 1 ? "example" : "examples"} into ${dataset} for ${agent}.`);
+    for (const e of res.errors.slice(0, 10))
+        console.log(`  skipped${e.line ? ` line ${e.line}` : ""}: ${e.error}`);
+}
+async function runAddExample(args, config) {
+    const p = parseArgs(args);
+    const agent = p.positional[0];
+    const dataset = p.flags["--dataset"];
+    const input = p.flags["--input"];
+    if (!agent || !dataset || !input)
+        usage("codiedev traces add-example <agent> --dataset <name> --input <text> [--expected <text>] [--name <text>]");
+    await call(config, "codiedev_traces_add_example", "POST", "examples", { body: { agent, dataset, input, expected: p.flags["--expected"], name: p.flags["--name"] } });
+    console.log(`✓ Added an example to ${dataset} for ${agent}.`);
+}
+async function runVersions(args, config) {
+    const dataset = parseArgs(args).positional[0];
+    if (!dataset)
+        usage("codiedev traces versions <dataset>");
+    const res = await call(config, "codiedev_traces_versions", "GET", "versions", { query: { dataset } });
+    if (!res.versions.length)
+        console.log(`${dataset} has no saved versions. Save one with: codiedev traces save-version "${dataset}"`);
+    for (const v of res.versions)
+        console.log(`v${v.version}  ${String(v.cases).padStart(4)} cases  ${new Date(v.createdAt).toISOString().slice(0, 10)}  +${v.added} −${v.removed} ${v.changed} edited${v.note ? `  ${v.note}` : ""}`);
+    if (res.unsaved)
+        console.log(`Unsaved since v${res.versions[0].version}: +${res.unsaved.added} −${res.unsaved.removed} ${res.unsaved.changed} edited`);
+}
+async function runSaveVersion(args, config) {
+    const p = parseArgs(args);
+    const dataset = p.positional[0];
+    if (!dataset)
+        usage("codiedev traces save-version <dataset> [--note text]");
+    const res = await call(config, "codiedev_traces_save_version", "POST", "versions", { body: { dataset, note: p.flags["--note"] } });
+    console.log(`✓ Saved ${dataset} as v${res.version} (${res.cases} cases). Run it later with --dataset "${dataset}" --version ${res.version}.`);
+}
+const usd = (n) => (n === 0 ? "$0" : n < 0.01 ? "<$0.01" : `$${n.toFixed(2)}`);
+const json = (p, value) => { if (p.bools.has("--json")) {
+    process.stdout.write(JSON.stringify(value, null, 2) + "\n");
+    return true;
+} return false; };
+async function runOverview(args, config) {
+    const p = parseArgs(args);
+    const res = await call(config, "codiedev_traces_overview", "GET", "overview", { query: { days: p.flags["--days"] ?? "7" } });
+    if (json(p, res))
+        return;
+    if (!res.agents.length) {
+        console.log("No agents you can see yet. Connect one in Codiedev → Traces.");
+        return;
+    }
+    console.log(`Agents, last ${res.range}:`);
+    for (const a of res.agents)
+        console.log(`  ${a.name.padEnd(28)} ${String(a.runs).padStart(6)} runs  ${String(a.failedRuns).padStart(5)} failed  ${usd(a.spend7dUsd).padStart(9)} 7d spend  ${a.keepsPromptsDays ? `prompts ${a.keepsPromptsDays}d` : "metadata only"}${a.spike ? "  spend spike" : ""}${a.budgetBlock ? "  budget block" : ""}`);
+    if (res.issues.length) {
+        console.log("\nWhat's going wrong:");
+        for (const i of res.issues)
+            console.log(`  ${i.label.padEnd(18)} ${String(i.runs).padStart(5)} runs  ${usd(i.wastedUsd).padStart(9)} wasted  ${i.agents.join(", ")}${i.exampleRun ? `  e.g. ${i.exampleRun}` : ""}`);
+    }
+}
+async function runRuns(args, config) {
+    const p = parseArgs(args);
+    const agent = p.positional[0];
+    if (!agent)
+        usage("codiedev traces runs <agent> [--failed] [--flag errors|truncated|tool_loop|context_overflow|budget_block] [--model id] [--verdict pass|fail] [--days n] [--limit n] [--json]");
+    const res = await call(config, "codiedev_traces_runs", "GET", "runs", {
+        query: { agent, days: p.flags["--days"] ?? "7", ...(p.bools.has("--failed") ? { failed: "1" } : {}), ...(p.flags["--flag"] ? { flag: p.flags["--flag"] } : {}), ...(p.flags["--model"] ? { model: p.flags["--model"] } : {}), ...(p.flags["--verdict"] ? { verdict: p.flags["--verdict"] } : {}), limit: p.flags["--limit"] ?? "50" },
+    });
+    if (json(p, res))
+        return;
+    console.log(`${res.total} ${res.total === 1 ? "run" : "runs"} for ${res.agent}, last ${res.range}${res.total > res.runs.length ? ` (newest ${res.runs.length})` : ""}:`);
+    for (const r of res.runs)
+        console.log(`  ${r.startedAt.slice(0, 16).replace("T", " ")}  ${r.status === "failed" ? "FAIL" : " ok "}  ${r.runId.padEnd(24)} ${String(r.steps).padStart(3)} calls ${usd(r.costUsd).padStart(8)}  ${r.model ?? ""}${r.flags.length ? `  [${r.flags.join(", ")}]` : ""}${r.verdict ? `  marked ${r.verdict}` : ""}`);
+    if (res.runs.length)
+        console.log(`\nOpen one: codiedev traces run ${res.runs[0].runId} --agent "${res.agent}"`);
+}
+async function runRun(args, config) {
+    const p = parseArgs(args);
+    const run = p.positional[0];
+    if (!run)
+        usage("codiedev traces run <run> [--agent name] [--json]");
+    const res = await call(config, "codiedev_traces_run", "GET", "run", { query: { run, ...(p.flags["--agent"] ? { agent: p.flags["--agent"] } : {}) } });
+    if (json(p, res))
+        return;
+    const r = res.run;
+    console.log(`${r.title} · ${r.id}`);
+    console.log(`${new Date(r.startedAt).toISOString().slice(0, 16).replace("T", " ")} · ${r.status === "failed" ? "FAILED" : "ok"} · ${Math.round(r.durationMs / 100) / 10}s · ${usd(r.costUsd)}${r.flags.length ? ` · ${r.flags.join(", ")}` : ""}`);
+    if (r.diagnosis)
+        console.log(`\nWhat went wrong: ${r.diagnosis.title}\n  ${r.diagnosis.why}\n\nFix (for your coding agent):\n${r.diagnosis.fix.split("\n").map((l) => `  ${l}`).join("\n")}`);
+    if (r.annotation)
+        console.log(`\nMarked ${r.annotation.verdict} by ${r.annotation.by}${r.annotation.reason ? `: ${r.annotation.reason}` : ""}`);
+    if (res.scores.length)
+        console.log(`\nScores: ${res.scores.map((s) => `${s.name} ${s.text}${s.step ? ` (step ${s.step})` : ""} by ${s.by ?? s.source}${s.comment ? ` "${s.comment}"` : ""}`).join("; ")}`);
+    console.log("\nSteps:");
+    for (const s of r.spans.filter((x) => x.kind !== "run")) {
+        console.log(`  ${s.kind === "tool" ? "tool" : "call"}  ${s.name}  ${Math.round(s.durationMs)}ms${s.costUsd !== undefined ? `  ${usd(s.costUsd)}` : ""}${s.finishReason ? `  finish: ${s.finishReason}` : ""}${s.statusCode && s.statusCode >= 400 ? `  HTTP ${s.statusCode}` : ""}${s.flag ? `  [${s.flag}]` : ""}${s.note ? `  ${s.note}` : ""}`);
+        if (s.input)
+            console.log(`      prompt: ${s.input.replace(/\s+/g, " ").slice(0, 300)}`);
+        if (s.output)
+            console.log(`      reply:  ${s.output.replace(/\s+/g, " ").slice(0, 300)}`);
+    }
+    if (!r.bodiesVisible)
+        console.log("\n(Prompts and replies are shown only to the agent's maintainers.)");
+}
+async function runScoreKinds(args, config) {
+    const [verb, ...rest] = args;
+    if (verb === "add") {
+        const p = parseArgs(rest);
+        const name = p.positional[0];
+        const type = p.flags["--type"];
+        if (!name || !type)
+            usage("codiedev traces score-kinds add <name> --type number|categories|boolean [--min n --max n] [--categories a,b,c] [--description text]");
+        const dataType = type === "number" || type === "numeric" ? "numeric" : type === "categories" || type === "categorical" ? "categorical" : type === "boolean" || type === "bool" ? "boolean" : type;
+        await call(config, "codiedev_traces_score_kind_add", "POST", "score-kinds", {
+            body: { name, type: dataType, ...(p.flags["--min"] ? { min: Number(p.flags["--min"]) } : {}), ...(p.flags["--max"] ? { max: Number(p.flags["--max"]) } : {}), ...(p.flags["--categories"] ? { categories: p.flags["--categories"].split(",").map((c) => c.trim()) } : {}), ...(p.flags["--description"] ? { description: p.flags["--description"] } : {}) },
+        });
+        console.log(`✓ Added score kind ${name}.`);
+        return;
+    }
+    const p = parseArgs(verb === undefined || verb === "list" ? rest : args);
+    const res = await call(config, "codiedev_traces_score_kinds", "GET", "score-kinds", {});
+    if (json(p, res))
+        return;
+    const live = res.kinds.filter((k) => !k.archived);
+    if (!live.length)
+        console.log("No score kinds yet. Add one: codiedev traces score-kinds add helpfulness --type number --min 1 --max 5");
+    for (const k of live)
+        console.log(`  ${k.name.padEnd(24)} ${k.dataType === "numeric" ? `number ${k.min ?? ""}–${k.max ?? ""}` : k.dataType === "boolean" ? "true / false" : k.categories.join(" / ")}${k.description ? `  ${k.description}` : ""}`);
+}
+async function runQueueCreate(args, config) {
+    const p = parseArgs(args);
+    const name = p.positional[0];
+    if (!name)
+        usage("codiedev traces queue create <name> [--scores a,b] [--description text]");
+    await call(config, "codiedev_traces_queue_create", "POST", "queues", { body: { name, scores: (p.flags["--scores"] ?? "").split(",").map((s) => s.trim()).filter(Boolean), ...(p.flags["--description"] ? { description: p.flags["--description"] } : {}) } });
+    console.log(`✓ Created review queue ${name}. Add runs: codiedev traces queue add "${name}" --agent <name> --failed`);
+}

package/dist/utils.js

@@ -1681,7 +1681,13 @@ thought, use the \`codiedev\` CLI via Bash:**
 | "find something about X" / "has anyone solved Y?" | \`codiedev search "<query>" [--limit N]\` |
 | "why did we…?" / "what does the team know about X?" | \`codiedev ask "<question>"\` (synthesized answer + citations) |
 | "show my library" / "what artifacts exist?" | \`codiedev library [--scope mine\\|shared\\|all] [--kind doc\\|skill]\` |
-| "run my agent's test cases" / "check the dataset on model Y" | \`codiedev traces cases <agent> [--dataset <name>] [--model <id>]\` (writes cases + RUN_CASES.md + results.json; follow it, then \`codiedev traces report <results.json>\`) |
+| "run my agent's test cases" / "check the dataset on model Y" / "run v2 of the dataset" | \`codiedev traces cases <agent> [--dataset <name> [--version <n>]] [--model <id>]\` (writes cases + RUN_CASES.md + results.json; follow it, then \`codiedev traces report <results.json>\`) |
+| "how are my agents doing?" / "what's failing this week?" / "show the booking agent's failed runs" / "why did run X fail?" | \`codiedev traces overview\`; \`codiedev traces runs <agent> [--failed] [--flag <issue>] [--days n]\`; \`codiedev traces run <run> --agent <name>\` (add \`--json\` to read the data yourself) |
+| "add a helpfulness 1–5 score" / "make a review queue for tone" | \`codiedev traces score-kinds add <name> --type number --min 1 --max 5\`; \`codiedev traces queue create <name> --scores <a,b>\` |
+| "score that run 4 for helpfulness" / "how is my agent scoring?" | \`codiedev traces score <run> <score> <value> [--agent <name>] [--comment <why>]\`; \`codiedev traces scores <agent>\` |
+| "put this week's failed runs in the review queue" | \`codiedev traces queue list\`; \`codiedev traces queue add <queue> --agent <name> [--failed] [--days n]\` |
+| "have a judge score 10% of my agent's runs" / "turn off automatic scoring" | \`codiedev traces judge <agent> --on --scores <a,b> --sample 10 [--budget 25]\`; \`--off\`; no flags shows it |
+| "import these test cases from a CSV" / "add an example to the dataset" / "save the dataset as a version" | \`codiedev traces import <agent> <file> --dataset <name>\`; \`codiedev traces add-example <agent> --dataset <name> --input "…" [--expected "…"]\`; \`codiedev traces save-version <dataset>\`; \`codiedev traces versions <dataset>\` |
 | "what was the last eval on X?" / "file these eval results" | \`codiedev evals last <subject>\` before starting a related eval; \`codiedev evals report <run.json>\` after each variant runs (scores only — \`codiedev evals help\` for the shape) |
 
 **Just push the file — we'll figure out where it goes.** Every artifact

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codiedev",
-  "version": "0.9.77",
+  "version": "0.9.79",
   "description": "Connect Claude Code, Codex, Cursor, OpenCode, or VS Code Copilot to Codiedev for team session capture and artifact collaboration — and run the governed Codie harness",
   "bin": {
     "codiedev": "dist/cli.js",