codiedev 0.9.72 → 0.9.76

package/dist/cli.js

@@ -30,6 +30,8 @@ const share_1 = require("./commands/share");
 const send_1 = require("./commands/send");
 const capture_1 = require("./commands/capture");
 const pack_1 = require("./commands/pack");
+const traces_1 = require("./commands/traces");
+const evals_1 = require("./commands/evals");
 // --- starter packs phase 2 ---
 const hookGate_1 = require("./packs/hookGate");
 const sync_1 = require("./packs/sync");
@@ -91,6 +93,16 @@ Starter packs:
                                          detected tool
   codiedev pack status                   Installed packs, up to date or not
 
+Traces (agents on the gateway):
+  codiedev traces agents                 Agents you maintain, with saved test cases
+  codiedev traces cases <agent> [--dataset name] [--model id]
+                                         Pull an agent's test cases (or one dataset)
+                                         into this repo for your coding agent to
+                                         run, optionally on another model
+  codiedev traces report <results.json>  Send the results back to Traces
+  codiedev evals report <run.json>       File one variant's eval scores under an experiment
+  codiedev evals last <subject>          Latest experiment on a subject + its champion
+
 Discovery:
   codiedev search "<query>"              Semantic search across the team's
                                          captured sessions and artifacts
@@ -152,6 +164,8 @@ status sync never weakens it.
 `.trim();
 const COMMAND_HELP = {
     pack: pack_1.PACK_HELP,
+    traces: traces_1.TRACES_HELP,
+    evals: evals_1.EVALS_HELP,
     connect: `
 codiedev connect — link your agent CLI to Codiedev
 
@@ -515,6 +529,12 @@ async function main() {
             case "ls":
                 await (0, library_1.runLibrary)(rest);
                 return;
+            case "traces":
+                await (0, traces_1.runTraces)(rest);
+                return;
+            case "evals":
+                await (0, evals_1.runEvals)(rest);
+                return;
             case "share":
                 await (0, share_1.runShare)(rest);
                 return;

package/dist/codieLauncher.d.ts

@@ -69,10 +69,16 @@ export declare function resolveCapturePlugin({ homeDir, existsSync, }: {
     homeDir: string;
     existsSync: (candidate: string) => boolean;
 }): string;
-export declare function fetchApprovedCatalog({ baseUrl, key, fetchImpl, }: {
+/**
+ * The account's approved models. Every `codie` session starts here, so one
+ * dropped connection or slow gateway reply would end the session before it
+ * began; anything but a definite rejection is retried once.
+ */
+export declare function fetchApprovedCatalog({ baseUrl, key, fetchImpl, retryDelayMs, }: {
     baseUrl?: string;
     key: string;
     fetchImpl?: typeof fetch;
+    retryDelayMs?: number;
 }): Promise<CatalogModel[]>;
 export declare function parseCodieArgs(argv: string[]): {
     args: string[];

package/dist/codieLauncher.js

@@ -349,9 +349,29 @@ function resolveCapturePlugin({ homeDir, existsSync, }) {
     }
     return (0, url_1.pathToFileURL)(capturePath).href;
 }
-async function fetchApprovedCatalog({ baseUrl, key, fetchImpl = fetch, }) {
+// A definite answer from the gateway (harness off, token rejected): retrying
+// can't change it.
+class CatalogRejected extends Error {
+}
+/**
+ * The account's approved models. Every `codie` session starts here, so one
+ * dropped connection or slow gateway reply would end the session before it
+ * began; anything but a definite rejection is retried once.
+ */
+async function fetchApprovedCatalog({ baseUrl, key, fetchImpl = fetch, retryDelayMs = 1_000, }) {
+    try {
+        return await fetchCatalogOnce(baseUrl, key, fetchImpl);
+    }
+    catch (error) {
+        if (error instanceof CatalogRejected)
+            throw error;
+        await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
+        return fetchCatalogOnce(baseUrl, key, fetchImpl);
+    }
+}
+async function fetchCatalogOnce(baseUrl, key, fetchImpl) {
     const controller = new AbortController();
-    const timeout = setTimeout(() => controller.abort(), 5000);
+    const timeout = setTimeout(() => controller.abort(), 10_000);
     try {
         const response = await fetchImpl(`${cleanBaseUrl(baseUrl)}/models`, {
             headers: { Authorization: `Bearer ${key}` },
@@ -361,11 +381,11 @@ async function fetchApprovedCatalog({ baseUrl, key, fetchImpl = fetch, }) {
         // whose gateway is still switched off. Those developers must get an
         // answer, not an HTTP status.
         if (response.status === 403) {
-            throw new Error("The Codie harness is not enabled for your company yet. Ask your Codiedev admin to turn it on — " +
+            throw new CatalogRejected("The Codie harness is not enabled for your company yet. Ask your Codiedev admin to turn it on — " +
                 "your session capture keeps working in the meantime.");
         }
         if (response.status === 401) {
-            throw new Error("Your Codiedev token was rejected. Run `codiedev connect` to relink this machine.");
+            throw new CatalogRejected("Your Codiedev token was rejected. Run `codiedev connect` to relink this machine.");
         }
         if (!response.ok)
             throw new Error(`model catalog returned ${response.status}`);

package/dist/commands/evals.d.ts

@@ -0,0 +1,3 @@
+import type { CodiedevConfig } from "../utils";
+export declare const EVALS_HELP: string;
+export declare function runEvals(args: string[], configOverride?: CodiedevConfig): Promise<void>;

package/dist/commands/evals.js

@@ -0,0 +1,193 @@
+"use strict";
+// `codiedev evals report <run.json> [--harness name]`
+// `codiedev evals last <subject> [--json]`
+//
+// Evals are experiments (a question about a subject: an agent, or a workload
+// like session-summaries) and the runs that answer them, one per variant. The
+// eval runs wherever it runs (a script, a coding agent); `report` files the
+// scores under the experiment in Codiedev → Traces → Evals, shared with the
+// team. `last` is where to start the next one: the latest experiment on the
+// subject, its runs, and the champion the subject runs today.
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EVALS_HELP = void 0;
+exports.runEvals = runEvals;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const shared_1 = require("./shared");
+exports.EVALS_HELP = `
+codiedev evals — experiments and their runs, shared with the team
+
+  codiedev evals report <run.json> [--harness name]
+                                         File one variant's scores under an
+                                         experiment (created on first use)
+  codiedev evals last <subject> [--json] The latest experiment on a subject,
+                                         its runs, and the current champion
+
+  run.json:
+  {
+    "subject": "session-summaries",            // agent name or workload
+    "question": "Is Terra medium as good as GLM high?",
+    "experimentId": "…",                       // optional, instead of subject + question
+    "dataset": "30 Signal & Code sessions",    // optional
+    "run": {
+      "label": "terra-med-v2", "model": "openai/gpt-6.1-terra",
+      "provider": "openai", "reasoningEffort": "medium", "prompt": "summary_extract_v2",
+      "gitSha": "…", "isControl": false, "judgeModel": "anthropic/claude-sonnet-5.5",
+      "casesTotal": 30, "casesScored": 30, "errors": 0,
+      "score": 3.85, "metrics": { "fidelity": 3.6 },
+      "costPerCaseUsd": 0.026, "latencyMsAvg": 21000,
+      "wins": 20, "losses": 8, "ties": 2, "note": "…",
+      "cases": [{ "key": "case-1", "score": 4, "note": "…" }]
+    }
+  }
+
+  Scores and notes only: never put prompts, transcripts or outputs in it.
+`.trim();
+function flag(args, i, name) {
+    const a = args[i];
+    if (a === name && i + 1 < args.length)
+        return { value: args[i + 1], skip: 1 };
+    if (a.startsWith(`${name}=`))
+        return { value: a.slice(name.length + 1), skip: 0 };
+    return { skip: -1 };
+}
+function variant(run) {
+    return [run.model, run.reasoningEffort, run.prompt].filter(Boolean).join(" · ");
+}
+function fmtScore(n) {
+    return n === null ? "–" : n.toFixed(2);
+}
+function fmtCost(n) {
+    return n === null ? "–" : `$${n.toFixed(4)}`;
+}
+async function runReport(args, config) {
+    let file;
+    let harness;
+    for (let i = 0; i < args.length; i++) {
+        const f = flag(args, i, "--harness");
+        if (f.skip >= 0) {
+            harness = f.value;
+            i += f.skip;
+            continue;
+        }
+        if (!args[i].startsWith("-") && !file)
+            file = args[i];
+    }
+    if (!file) {
+        console.error("Usage: codiedev evals report <run.json> [--harness name]");
+        process.exit(1);
+    }
+    let body;
+    try {
+        body = JSON.parse(fs.readFileSync(path.resolve(process.cwd(), file), "utf8"));
+    }
+    catch (err) {
+        console.error(`Couldn't read ${file}: ${err.message}`);
+        process.exit(1);
+    }
+    if (!body.subject || !body.question || !body.run?.label || !body.run?.model || typeof body.run?.casesTotal !== "number") {
+        console.error(`${file} needs subject, question, and run.label, run.model, run.casesTotal. See: codiedev evals help`);
+        process.exit(1);
+    }
+    const start = Date.now();
+    try {
+        const res = await (0, shared_1.apiRequest)("POST", "/api/cli/evals/runs", {
+            config,
+            body: { ...body, run: { ...body.run, harness: harness ?? body.run.harness ?? undefined } },
+        });
+        (0, shared_1.recordAgentEvent)(config, { tool: "codiedev_evals_report", ok: true, latencyMs: Date.now() - start });
+        console.log(`✓ Reported ${body.run.label} on ${body.subject}${res.created ? " (new experiment)" : ""}: "${body.question}"`);
+        console.log(`  experiment ${res.experimentId}. See it in Codiedev → Traces → Evals.`);
+    }
+    catch (err) {
+        const e = err;
+        (0, shared_1.recordAgentEvent)(config, { tool: "codiedev_evals_report", ok: false, latencyMs: Date.now() - start, error: e.message });
+        console.error(`Couldn't report: ${e.message}`);
+        process.exit(1);
+    }
+}
+async function runLast(args, config) {
+    const json = args.includes("--json");
+    const subject = args.find((a) => !a.startsWith("-"));
+    if (!subject) {
+        console.error("Usage: codiedev evals last <subject> [--json]");
+        process.exit(1);
+    }
+    let res;
+    try {
+        res = await (0, shared_1.apiRequest)("GET", `/api/cli/evals/last?subject=${encodeURIComponent(subject)}`, { config });
+    }
+    catch (err) {
+        const e = err;
+        console.error(e.status === 404 ? `No evals on ${subject} yet.` : `Couldn't load evals: ${e.message}`);
+        process.exit(1);
+    }
+    if (json) {
+        console.log(JSON.stringify(res, null, 2));
+        return;
+    }
+    const { experiment, runs, champion } = res;
+    if (champion) {
+        console.log(`Champion: ${champion.run.label} (${variant(champion.run)}), score ${fmtScore(champion.run.score)}, ${fmtCost(champion.run.costPerCaseUsd)}/case`);
+        if (champion.decisionNote)
+            console.log(`  Why: ${champion.decisionNote}`);
+    }
+    else {
+        console.log("Champion: none picked yet");
+    }
+    console.log("");
+    console.log(`Latest experiment (${res.experiments} on ${experiment.subject}): ${experiment.question}`);
+    if (experiment.dataset)
+        console.log(`  Dataset: ${experiment.dataset}`);
+    console.log(`  id ${experiment.id}, by ${experiment.createdBy}, updated ${new Date(experiment.updatedAt).toISOString().slice(0, 16).replace("T", " ")}`);
+    console.log("");
+    for (const run of [...runs].sort((a, b) => a.ranAt - b.ranAt)) {
+        const h2h = run.wins !== null ? ` · ${run.wins}W/${run.losses ?? 0}L/${run.ties ?? 0}T` : "";
+        console.log(`  ${run.isControl ? "[control] " : ""}${run.label.padEnd(24)} ${fmtScore(run.score).padStart(5)}  ${fmtCost(run.costPerCaseUsd).padStart(8)}/case${h2h}  ${variant(run)}`);
+    }
+    console.log("");
+    console.log(`Next run on this experiment: put "experimentId": "${experiment.id}" in run.json, then codiedev evals report run.json`);
+}
+async function runEvals(args, configOverride) {
+    const [sub, ...rest] = args;
+    if (sub === "report")
+        return runReport(rest, configOverride ?? (0, shared_1.requireConfig)());
+    if (sub === "last")
+        return runLast(rest, configOverride ?? (0, shared_1.requireConfig)());
+    console.log(exports.EVALS_HELP);
+    if (sub && sub !== "help")
+        process.exit(1);
+}

package/dist/commands/traces.d.ts

@@ -0,0 +1,68 @@
+import type { CodiedevConfig } from "../utils";
+export declare const TRACES_HELP: string;
+interface CaseStep {
+    model: string;
+    finishReason?: string;
+    statusCode: number;
+    request?: string;
+    reply?: string;
+    toolCalls?: Array<{
+        name: string;
+        arguments: string;
+    }>;
+}
+interface CaseRow {
+    id: string;
+    name: string;
+    suite: string | null;
+    runId: string;
+    kind: "fix" | "keep";
+    flags: string[];
+    note: string | null;
+    expected: string | null;
+    createdAt: string;
+    steps: CaseStep[];
+}
+interface CasesResponse {
+    ok: boolean;
+    /** The suite pulled, when --suite was given. */
+    suite: string | null;
+    /** How the suite is judged, when it has settings. */
+    judge: {
+        model: string | null;
+        rubric: string | null;
+    } | null;
+    agent: {
+        id: string;
+        name: string;
+        description: string | null;
+        promptDays: number | null;
+    };
+    cases: CaseRow[];
+}
+/** What `report` sends, and what `cases` writes for the coding agent to fill in. */
+export interface ResultsFile {
+    agent: string;
+    suite: string | null;
+    model: string;
+    judgeModel: string | null;
+    harness: string | null;
+    results: Array<{
+        caseId: string;
+        name: string;
+        passes: number | null;
+        tries: number;
+        note: string | null;
+        output: string | null;
+    }>;
+}
+export declare function slugify(name: string): string;
+/** The brief a coding agent follows to run the cases and report back. */
+export declare function casesBrief(res: CasesResponse, casesFile: string, opts?: {
+    model?: string;
+    resultsFile?: string;
+}): string;
+/** The results.json a run starts from: every case, passes to be filled in. */
+export declare function resultsTemplate(res: CasesResponse, model: string | undefined): ResultsFile;
+export declare function runTraces(args: string[], configOverride?: CodiedevConfig): Promise<void>;
+export {};

package/dist/commands/traces.js

@@ -0,0 +1,292 @@
+"use strict";
+// `codiedev traces agents`
+// `codiedev traces cases <agent> [--dataset name] [--model id] [--out dir] [--json]`
+// `codiedev traces report <results.json> [--harness name]`
+//
+// Test cases are runs an engineer saved from Codiedev → Traces. They run in
+// the engineer's coding agent, not in the portal: `cases` writes them into the
+// repo with a brief the agent follows to replay each one against the agent's
+// own code, optionally on another model (a regression check before a model
+// swap), and a results.json for `report` to send back so Traces shows pass
+// rates by model.
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TRACES_HELP = void 0;
+exports.slugify = slugify;
+exports.casesBrief = casesBrief;
+exports.resultsTemplate = resultsTemplate;
+exports.runTraces = runTraces;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const shared_1 = require("./shared");
+exports.TRACES_HELP = `
+codiedev traces — test cases saved from Codiedev → Traces
+
+  codiedev traces agents                 Agents you maintain, with saved case counts
+  codiedev traces cases <agent> [--dataset name] [--case id[,id]] [--model id] [--out dir] [--json]
+                                         Write an agent's cases (one dataset, or just
+                                         the cases named) into this repo, default
+                                         .codiedev/traces/<agent>[/<dataset>]/, then
+                                         ask your coding agent to run them.
+                                         --model runs them on another model, e.g.
+                                         the replacement for a retiring one
+  codiedev traces report <results.json> [--harness name]
+                                         Send the filled-in results back, so
+                                         Traces shows the pass rate for that model
+
+  <agent> is the agent's name (its API key name) or id. Only an agent's
+  maintainers can pull its cases or report on them.
+`.trim();
+const FLAG_LABEL = {
+    budget_block: "refused for budget",
+    errors: "model call failed",
+    tool_loop: "repeated the same tool call",
+    truncated: "reply cut off at max_tokens",
+    context_overflow: "prompt over 70% of the context window",
+};
+const TRIES = 3;
+const plural = (n, word) => `${n} ${word}${n === 1 ? "" : "s"}`;
+const recordedModel = (res) => res.cases.flatMap((c) => c.steps.map((s) => s.model))[0] ?? "the recorded model";
+function slugify(name) {
+    return name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "") || "agent";
+}
+/** The brief a coding agent follows to run the cases and report back. */
+function casesBrief(res, casesFile, opts = {}) {
+    const fix = res.cases.filter((c) => c.kind === "fix");
+    const keep = res.cases.filter((c) => c.kind === "keep");
+    const withBodies = res.cases.filter((c) => c.steps.some((s) => s.request)).length;
+    const target = opts.model;
+    const results = opts.resultsFile ? path.basename(opts.resultsFile) : "results.json";
+    const lines = [
+        `# Test cases for ${res.agent.name}${res.suite ? ` · ${res.suite}` : ""}${target ? ` on ${target}` : ""}`,
+        "",
+        res.agent.description ? `${res.agent.description}\n` : "",
+        `Saved from Codiedev → Traces. ${plural(res.cases.length, "case")} in \`${path.basename(casesFile)}\` (one JSON object per line):`,
+        `- **${fix.length} to fix**: production runs that failed. The fix should make them pass.`,
+        `- **${keep.length} to keep passing**: good runs. They should still pass after any change.`,
+        "",
+    ];
+    if (target) {
+        lines.push("## This is a model check", "", `Run every case on **${target}** instead of ${recordedModel(res)} (the model the cases were recorded on). Change only the model the agent asks the gateway for; keep its prompts and tools as they are, so the result says how ${target} does on the same work.`, "");
+    }
+    lines.push("## How to run them", "", "1. Find where this agent builds its model requests and calls the Codiedev gateway.", `2. For each case, replay \`steps[0].request\` (the first prompt the agent sent) through the agent's own code path, so tools and prompts are the current version, not the recorded one${target ? `, with the model set to ${target}` : ""}.`, "3. Judge each try against the case:", "   - A case's `expected`, when present, is what a passing answer looks like (a reference reply or criteria). It wins over everything below.", "   - `fix` cases: the run must not reproduce its `flags` (listed below), and must finish (no failed call, no `finish_reason: \"length\"`, no repeated identical tool call).", "   - `keep` cases: the final reply must still do what the recorded final reply did. Compare meaning, not wording.", "   - A case's `note`, when present, is the engineer's extra context.");
+    if (res.judge?.model || res.judge?.rubric) {
+        lines.push(`   - Where passing needs judgement, judge with **${res.judge.model ?? "a strong model"}** through the gateway${res.judge.rubric ? `, using this rubric: "${res.judge.rubric}"` : ""}. Use the same judge for every model so results compare.`);
+    }
+    lines.push(`4. Run each case ${TRIES} times: agents are stochastic, so report a pass rate per case, not a single result.`, `5. Fill in \`${results}\`: for each case set \`passes\` (out of \`tries\`), \`output\` (the agent's final reply on one try, so it can be compared side by side with other models) and a one-line \`note\` on any failure. Set \`harness\` to the coding agent you are (claude-code, codex, cursor, copilot, …).`, `6. Send it back: \`codiedev traces report ${opts.resultsFile ? path.relative(process.cwd(), opts.resultsFile) || results : results}\`. Then show a table: case, kind, passes out of ${TRIES}, and the failure note.`, "", "Send real model calls through the gateway with the agent's usual key and `X-Codiedev-Run-Id: case-<id>-<try>`, so the replays show up in Traces too.", "");
+    if (withBodies < res.cases.length) {
+        lines.push(`Note: ${plural(res.cases.length - withBodies, "case")} ${res.cases.length - withBodies === 1 ? "has" : "have"} no recorded prompt (the agent wasn't keeping prompts then). For those, rebuild the input from the case's \`expected\` or \`note\`, or skip them and say so.`, "");
+    }
+    lines.push("## Cases", "");
+    for (const c of res.cases) {
+        const flags = c.flags.length ? c.flags.map((f) => FLAG_LABEL[f] ?? f).join(", ") : "no flags";
+        lines.push(`- \`${c.id}\` · ${c.name} · ${c.kind} · ${flags} · ${plural(c.steps.length, "model call")}${c.expected ? ` · expected: "${c.expected.replace(/\s+/g, " ").slice(0, 200)}"` : ""}${c.note ? ` · note: "${c.note}"` : ""}`);
+    }
+    return lines.filter((line, i, all) => !(line === "" && all[i - 1] === "")).join("\n") + "\n";
+}
+/** The results.json a run starts from: every case, passes to be filled in. */
+function resultsTemplate(res, model) {
+    return {
+        agent: res.agent.name,
+        suite: res.suite,
+        model: model ?? recordedModel(res),
+        judgeModel: res.judge?.model ?? null,
+        harness: null,
+        results: res.cases.map((c) => ({ caseId: c.id, name: c.name, passes: null, tries: TRIES, note: null, output: null })),
+    };
+}
+function flag(args, i, name) {
+    const a = args[i];
+    if (a === name && i + 1 < args.length)
+        return { value: args[i + 1], skip: 1 };
+    if (a.startsWith(`${name}=`))
+        return { value: a.slice(name.length + 1), skip: 0 };
+    return { skip: -1 };
+}
+function parseCasesArgs(args) {
+    const parsed = { json: false };
+    for (let i = 0; i < args.length; i++) {
+        const a = args[i];
+        let matched = false;
+        for (const [name, key] of [["--out", "out"], ["-o", "out"], ["--dataset", "suite"], ["--suite", "suite"], ["--model", "model"], ["--case", "caseIds"]]) {
+            const f = flag(args, i, name);
+            if (f.skip >= 0) {
+                parsed[key] = f.value;
+                i += f.skip;
+                matched = true;
+                break;
+            }
+        }
+        if (matched)
+            continue;
+        if (a === "--json")
+            parsed.json = true;
+        else if (!a.startsWith("-") && !parsed.agent)
+            parsed.agent = a;
+    }
+    return parsed;
+}
+async function runAgents(config) {
+    const start = Date.now();
+    let res;
+    try {
+        res = await (0, shared_1.apiRequest)("GET", "/api/cli/traces/agents", { config });
+    }
+    catch (err) {
+        const e = err;
+        (0, shared_1.recordAgentEvent)(config, { tool: "codiedev_traces_agents", ok: false, latencyMs: Date.now() - start, error: e.message });
+        console.error(`Couldn't list agents: ${e.message}`);
+        process.exit(1);
+    }
+    (0, shared_1.recordAgentEvent)(config, { tool: "codiedev_traces_agents", ok: true, latencyMs: Date.now() - start });
+    if (!res.agents.length) {
+        console.log("You don't maintain any agents yet. Connect one in Codiedev → Traces → How it works.");
+        return;
+    }
+    for (const agent of res.agents) {
+        const capture = agent.promptDays ? `prompts kept ${agent.promptDays} days` : "metadata only";
+        console.log(`${agent.name.padEnd(28)} ${plural(agent.cases, "case").padStart(9)}   ${capture}`);
+    }
+    console.log("\nPull one: codiedev traces cases <agent>");
+}
+async function runCases(args, config) {
+    const { agent, suite, model, caseIds, out, json } = parseCasesArgs(args);
+    if (!agent) {
+        console.error("Usage: codiedev traces cases <agent> [--dataset name] [--model id] [--out dir] [--json]");
+        process.exit(1);
+    }
+    const start = Date.now();
+    let res;
+    try {
+        res = await (0, shared_1.apiRequest)("GET", "/api/cli/traces/cases", { config, query: { agent, ...(suite ? { suite } : {}), ...(caseIds ? { case: caseIds } : {}) } });
+    }
+    catch (err) {
+        const e = err;
+        (0, shared_1.recordAgentEvent)(config, { tool: "codiedev_traces_cases", ok: false, latencyMs: Date.now() - start, error: e.message });
+        if (e.status === 404)
+            console.error(`No agent called "${agent}". See your agents with: codiedev traces agents`);
+        else if (e.status === 403)
+            console.error(`Only ${agent}'s maintainers can pull its test cases.`);
+        else
+            console.error(`Couldn't pull cases: ${e.message}`);
+        process.exit(1);
+    }
+    (0, shared_1.recordAgentEvent)(config, { tool: "codiedev_traces_cases", ok: true, latencyMs: Date.now() - start });
+    if (json) {
+        process.stdout.write(JSON.stringify(res, null, 2) + "\n");
+        return;
+    }
+    if (!res.cases.length) {
+        console.log(suite
+            ? `${res.agent.name} has no test cases in "${suite}". See your datasets in Codiedev → Traces → Test cases.`
+            : `${res.agent.name} has no saved test cases. Open a run in Codiedev → Traces and click "Save as test case".`);
+        return;
+    }
+    const dir = path.resolve(process.cwd(), out ?? path.join(".codiedev", "traces", slugify(res.agent.name), ...(res.suite ? [slugify(res.suite)] : []), ...(caseIds && res.cases.length === 1 ? [slugify(res.cases[0].name)] : []), ...(model ? [slugify(model)] : [])));
+    fs.mkdirSync(dir, { recursive: true });
+    const casesFile = path.join(dir, "cases.jsonl");
+    fs.writeFileSync(casesFile, res.cases.map((c) => JSON.stringify(c)).join("\n") + "\n", "utf8");
+    const resultsFile = path.join(dir, "results.json");
+    fs.writeFileSync(resultsFile, JSON.stringify(resultsTemplate(res, model), null, 2) + "\n", "utf8");
+    const briefFile = path.join(dir, "RUN_CASES.md");
+    fs.writeFileSync(briefFile, casesBrief(res, casesFile, { model, resultsFile }), "utf8");
+    const fix = res.cases.filter((c) => c.kind === "fix").length;
+    const rel = (file) => path.relative(process.cwd(), file) || file;
+    console.log(`✓ ${plural(res.cases.length, "case")} for ${res.agent.name}${res.suite ? ` · ${res.suite}` : ""}${model ? ` on ${model}` : ""} (${fix} to fix, ${res.cases.length - fix} to keep passing)`);
+    console.log(`  ${rel(casesFile)}`);
+    console.log(`  ${rel(briefFile)}`);
+    console.log(`  ${rel(resultsFile)}  (filled in by your coding agent, then: codiedev traces report ${rel(resultsFile)})`);
+    console.log(`\nNext: ask your coding agent to "run the test cases in ${rel(briefFile)}"`);
+}
+async function runReport(args, config) {
+    let file;
+    let harness;
+    for (let i = 0; i < args.length; i++) {
+        const f = flag(args, i, "--harness");
+        if (f.skip >= 0) {
+            harness = f.value;
+            i += f.skip;
+            continue;
+        }
+        if (!args[i].startsWith("-") && !file)
+            file = args[i];
+    }
+    if (!file) {
+        console.error("Usage: codiedev traces report <results.json> [--harness name]");
+        process.exit(1);
+    }
+    let body;
+    try {
+        body = JSON.parse(fs.readFileSync(path.resolve(process.cwd(), file), "utf8"));
+    }
+    catch (err) {
+        console.error(`Couldn't read ${file}: ${err.message}`);
+        process.exit(1);
+    }
+    const missing = body.results.filter((r) => typeof r.passes !== "number");
+    if (missing.length) {
+        console.error(`${plural(missing.length, "case")} in ${file} ${missing.length === 1 ? "has" : "have"} no "passes" yet: ${missing.slice(0, 5).map((r) => r.name).join(", ")}. Run them first.`);
+        process.exit(1);
+    }
+    const start = Date.now();
+    try {
+        const res = await (0, shared_1.apiRequest)("POST", "/api/cli/traces/results", {
+            config,
+            body: { ...body, harness: harness ?? body.harness ?? undefined },
+        });
+        (0, shared_1.recordAgentEvent)(config, { tool: "codiedev_traces_report", ok: true, latencyMs: Date.now() - start });
+        const passed = body.results.filter((r) => r.passes === r.tries).length;
+        console.log(`✓ Reported ${body.agent}${body.suite ? ` · ${body.suite}` : ""} on ${body.model}: ${Math.round(res.passRate * 100)}% of tries passed, ${passed}/${body.results.length} cases passed every try.`);
+        console.log("  See it next to other models in Codiedev → Traces → Test cases.");
+    }
+    catch (err) {
+        const e = err;
+        (0, shared_1.recordAgentEvent)(config, { tool: "codiedev_traces_report", ok: false, latencyMs: Date.now() - start, error: e.message });
+        console.error(e.status === 403 ? `Only ${body.agent}'s maintainers can report its results.` : `Couldn't report: ${e.message}`);
+        process.exit(1);
+    }
+}
+async function runTraces(args, configOverride) {
+    const [sub, ...rest] = args;
+    if (sub === "agents")
+        return runAgents(configOverride ?? (0, shared_1.requireConfig)());
+    if (sub === "cases")
+        return runCases(rest, configOverride ?? (0, shared_1.requireConfig)());
+    if (sub === "report")
+        return runReport(rest, configOverride ?? (0, shared_1.requireConfig)());
+    console.log(exports.TRACES_HELP);
+    if (sub && sub !== "help")
+        process.exit(1);
+}

package/dist/piRuntime.d.ts

@@ -16,6 +16,7 @@ export interface PiRuntimeDeps {
     rename?: (from: string, to: string) => void;
     remove?: (target: string) => void;
     log?: (message: string) => void;
+    sleep?: (ms: number) => void;
 }
 /**
  * The pi CLI entry to run, installing the pinned version first if it is

package/dist/piRuntime.js

@@ -70,6 +70,10 @@ function installedPiEntry(installDir, readFile = (f) => fs.readFileSync(f, "utf8
         return null;
     }
 }
+// One retry rides out a dropped connection or registry blip; a second
+// failure is a real problem (offline, blocked registry) and is reported.
+const INSTALL_ATTEMPTS = 2;
+const INSTALL_RETRY_DELAY_MS = 5_000;
 /**
  * The pi CLI entry to run, installing the pinned version first if it is
  * missing or a different version. Staged then renamed, so an interrupted
@@ -85,10 +89,10 @@ function ensurePiRuntime(deps) {
     const remove = deps.remove ?? ((target) => fs.rmSync(target, { recursive: true, force: true }));
     const rename = deps.rename ?? fs.renameSync;
     const log = deps.log ?? ((message) => console.error(message));
-    const staging = (deps.makeStagingDir ?? (() => {
+    const makeStagingDir = deps.makeStagingDir ?? (() => {
         fs.mkdirSync(deps.stateDir, { recursive: true });
         return fs.mkdtempSync(path.join(deps.stateDir, ".runtime-pi-"));
-    }))();
+    });
     const run = deps.run ?? ((args) => {
         const result = (0, child_process_1.spawnSync)(process.platform === "win32" ? "npm.cmd" : "npm", args, {
             encoding: "utf8", stdio: ["ignore", "ignore", "pipe"], shell: process.platform === "win32",
@@ -97,26 +101,35 @@ function ensurePiRuntime(deps) {
             throw result.error;
         return { status: result.status, stderr: result.stderr ?? "" };
     });
+    const sleep = deps.sleep ?? ((ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms));
     log(`Installing pi (${exports.PI_RUNTIME_SPEC}). This runs once, and only for \`codie --pi\`.`);
-    try {
-        const result = run((0, codieRuntimeInstall_1.buildRuntimeInstallArgs)(staging, exports.PI_RUNTIME_SPEC));
-        if (result.status !== 0)
-            throw new Error(result.stderr?.trim() || `npm exited with ${result.status}`);
-        // Trust the artifact, not the exit code.
-        const staged = installedPiEntry(staging, readFile);
-        if (!staged || !exists(staged)) {
-            throw new Error(`the install did not produce pi. Check that ${exports.PI_RUNTIME_SPEC} is available from your npm registry.`);
+    for (let attempt = 1;; attempt++) {
+        const staging = makeStagingDir();
+        try {
+            const result = run((0, codieRuntimeInstall_1.buildRuntimeInstallArgs)(staging, exports.PI_RUNTIME_SPEC));
+            if (result.status !== 0)
+                throw new Error(result.stderr?.trim() || `npm exited with ${result.status}`);
+            // Trust the artifact, not the exit code.
+            const staged = installedPiEntry(staging, readFile);
+            if (!staged || !exists(staged)) {
+                throw new Error(`the install did not produce pi. Check that ${exports.PI_RUNTIME_SPEC} is available from your npm registry.`);
+            }
+            remove(installDir);
+            rename(staging, installDir);
+            break;
         }
-        remove(installDir);
-        rename(staging, installDir);
-    }
-    catch (error) {
-        throw new Error(`Couldn't install pi: ${error instanceof Error ? error.message : String(error)}`);
-    }
-    finally {
-        // No-op after a successful promote; otherwise it keeps a ~200MB staging
-        // dir from being orphaned.
-        remove(staging);
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            if (attempt >= INSTALL_ATTEMPTS)
+                throw new Error(`Couldn't install pi: ${message}`);
+            log(`pi install failed (${message.split("\n")[0]}); retrying.`);
+        }
+        finally {
+            // No-op after a successful promote; otherwise it keeps a ~200MB staging
+            // dir from being orphaned.
+            remove(staging);
+        }
+        sleep(INSTALL_RETRY_DELAY_MS);
     }
     const entry = installedPiEntry(installDir, readFile);
     if (!entry)

package/dist/utils.js

@@ -1681,6 +1681,8 @@ thought, use the \`codiedev\` CLI via Bash:**
 | "find something about X" / "has anyone solved Y?" | \`codiedev search "<query>" [--limit N]\` |
 | "why did we…?" / "what does the team know about X?" | \`codiedev ask "<question>"\` (synthesized answer + citations) |
 | "show my library" / "what artifacts exist?" | \`codiedev library [--scope mine\\|shared\\|all] [--kind doc\\|skill]\` |
+| "run my agent's test cases" / "check the dataset on model Y" | \`codiedev traces cases <agent> [--dataset <name>] [--model <id>]\` (writes cases + RUN_CASES.md + results.json; follow it, then \`codiedev traces report <results.json>\`) |
+| "what was the last eval on X?" / "file these eval results" | \`codiedev evals last <subject>\` before starting a related eval; \`codiedev evals report <run.json>\` after each variant runs (scores only — \`codiedev evals help\` for the shape) |
 
 **Just push the file — we'll figure out where it goes.** Every artifact
 is classified into one of two kinds at upload time:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codiedev",
-  "version": "0.9.72",
+  "version": "0.9.76",
   "description": "Connect Claude Code, Codex, Cursor, OpenCode, or VS Code Copilot to Codiedev for team session capture and artifact collaboration — and run the governed Codie harness",
   "bin": {
     "codiedev": "dist/cli.js",