codiedev 0.9.73 → 0.9.76

package/dist/cli.js

@@ -31,6 +31,7 @@ const send_1 = require("./commands/send");
 const capture_1 = require("./commands/capture");
 const pack_1 = require("./commands/pack");
 const traces_1 = require("./commands/traces");
+const evals_1 = require("./commands/evals");
 // --- starter packs phase 2 ---
 const hookGate_1 = require("./packs/hookGate");
 const sync_1 = require("./packs/sync");
@@ -94,11 +95,13 @@ Starter packs:
 
 Traces (agents on the gateway):
   codiedev traces agents                 Agents you maintain, with saved test cases
-  codiedev traces cases <agent> [--suite name] [--model id]
-                                         Pull an agent's test cases (or one suite)
+  codiedev traces cases <agent> [--dataset name] [--model id]
+                                         Pull an agent's test cases (or one dataset)
                                          into this repo for your coding agent to
                                          run, optionally on another model
   codiedev traces report <results.json>  Send the results back to Traces
+  codiedev evals report <run.json>       File one variant's eval scores under an experiment
+  codiedev evals last <subject>          Latest experiment on a subject + its champion
 
 Discovery:
   codiedev search "<query>"              Semantic search across the team's
@@ -162,6 +165,7 @@ status sync never weakens it.
 const COMMAND_HELP = {
     pack: pack_1.PACK_HELP,
     traces: traces_1.TRACES_HELP,
+    evals: evals_1.EVALS_HELP,
     connect: `
 codiedev connect — link your agent CLI to Codiedev
 
@@ -528,6 +532,9 @@ async function main() {
             case "traces":
                 await (0, traces_1.runTraces)(rest);
                 return;
+            case "evals":
+                await (0, evals_1.runEvals)(rest);
+                return;
             case "share":
                 await (0, share_1.runShare)(rest);
                 return;

package/dist/codieLauncher.d.ts

@@ -69,10 +69,16 @@ export declare function resolveCapturePlugin({ homeDir, existsSync, }: {
     homeDir: string;
     existsSync: (candidate: string) => boolean;
 }): string;
-export declare function fetchApprovedCatalog({ baseUrl, key, fetchImpl, }: {
+/**
+ * The account's approved models. Every `codie` session starts here, so one
+ * dropped connection or slow gateway reply would end the session before it
+ * began; anything but a definite rejection is retried once.
+ */
+export declare function fetchApprovedCatalog({ baseUrl, key, fetchImpl, retryDelayMs, }: {
     baseUrl?: string;
     key: string;
     fetchImpl?: typeof fetch;
+    retryDelayMs?: number;
 }): Promise<CatalogModel[]>;
 export declare function parseCodieArgs(argv: string[]): {
     args: string[];

package/dist/codieLauncher.js

@@ -349,9 +349,29 @@ function resolveCapturePlugin({ homeDir, existsSync, }) {
     }
     return (0, url_1.pathToFileURL)(capturePath).href;
 }
-async function fetchApprovedCatalog({ baseUrl, key, fetchImpl = fetch, }) {
+// A definite answer from the gateway (harness off, token rejected): retrying
+// can't change it.
+class CatalogRejected extends Error {
+}
+/**
+ * The account's approved models. Every `codie` session starts here, so one
+ * dropped connection or slow gateway reply would end the session before it
+ * began; anything but a definite rejection is retried once.
+ */
+async function fetchApprovedCatalog({ baseUrl, key, fetchImpl = fetch, retryDelayMs = 1_000, }) {
+    try {
+        return await fetchCatalogOnce(baseUrl, key, fetchImpl);
+    }
+    catch (error) {
+        if (error instanceof CatalogRejected)
+            throw error;
+        await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
+        return fetchCatalogOnce(baseUrl, key, fetchImpl);
+    }
+}
+async function fetchCatalogOnce(baseUrl, key, fetchImpl) {
     const controller = new AbortController();
-    const timeout = setTimeout(() => controller.abort(), 5000);
+    const timeout = setTimeout(() => controller.abort(), 10_000);
     try {
         const response = await fetchImpl(`${cleanBaseUrl(baseUrl)}/models`, {
             headers: { Authorization: `Bearer ${key}` },
@@ -361,11 +381,11 @@ async function fetchApprovedCatalog({ baseUrl, key, fetchImpl = fetch, }) {
         // whose gateway is still switched off. Those developers must get an
         // answer, not an HTTP status.
         if (response.status === 403) {
-            throw new Error("The Codie harness is not enabled for your company yet. Ask your Codiedev admin to turn it on — " +
+            throw new CatalogRejected("The Codie harness is not enabled for your company yet. Ask your Codiedev admin to turn it on — " +
                 "your session capture keeps working in the meantime.");
         }
         if (response.status === 401) {
-            throw new Error("Your Codiedev token was rejected. Run `codiedev connect` to relink this machine.");
+            throw new CatalogRejected("Your Codiedev token was rejected. Run `codiedev connect` to relink this machine.");
         }
         if (!response.ok)
             throw new Error(`model catalog returned ${response.status}`);

package/dist/commands/evals.d.ts

@@ -0,0 +1,3 @@
+import type { CodiedevConfig } from "../utils";
+export declare const EVALS_HELP: string;
+export declare function runEvals(args: string[], configOverride?: CodiedevConfig): Promise<void>;

package/dist/commands/evals.js

@@ -0,0 +1,193 @@
+"use strict";
+// `codiedev evals report <run.json> [--harness name]`
+// `codiedev evals last <subject> [--json]`
+//
+// Evals are experiments (a question about a subject: an agent, or a workload
+// like session-summaries) and the runs that answer them, one per variant. The
+// eval runs wherever it runs (a script, a coding agent); `report` files the
+// scores under the experiment in Codiedev → Traces → Evals, shared with the
+// team. `last` is where to start the next one: the latest experiment on the
+// subject, its runs, and the champion the subject runs today.
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EVALS_HELP = void 0;
+exports.runEvals = runEvals;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const shared_1 = require("./shared");
+exports.EVALS_HELP = `
+codiedev evals — experiments and their runs, shared with the team
+
+  codiedev evals report <run.json> [--harness name]
+                                         File one variant's scores under an
+                                         experiment (created on first use)
+  codiedev evals last <subject> [--json] The latest experiment on a subject,
+                                         its runs, and the current champion
+
+  run.json:
+  {
+    "subject": "session-summaries",            // agent name or workload
+    "question": "Is Terra medium as good as GLM high?",
+    "experimentId": "…",                       // optional, instead of subject + question
+    "dataset": "30 Signal & Code sessions",    // optional
+    "run": {
+      "label": "terra-med-v2", "model": "openai/gpt-6.1-terra",
+      "provider": "openai", "reasoningEffort": "medium", "prompt": "summary_extract_v2",
+      "gitSha": "…", "isControl": false, "judgeModel": "anthropic/claude-sonnet-5.5",
+      "casesTotal": 30, "casesScored": 30, "errors": 0,
+      "score": 3.85, "metrics": { "fidelity": 3.6 },
+      "costPerCaseUsd": 0.026, "latencyMsAvg": 21000,
+      "wins": 20, "losses": 8, "ties": 2, "note": "…",
+      "cases": [{ "key": "case-1", "score": 4, "note": "…" }]
+    }
+  }
+
+  Scores and notes only: never put prompts, transcripts or outputs in it.
+`.trim();
+function flag(args, i, name) {
+    const a = args[i];
+    if (a === name && i + 1 < args.length)
+        return { value: args[i + 1], skip: 1 };
+    if (a.startsWith(`${name}=`))
+        return { value: a.slice(name.length + 1), skip: 0 };
+    return { skip: -1 };
+}
+function variant(run) {
+    return [run.model, run.reasoningEffort, run.prompt].filter(Boolean).join(" · ");
+}
+function fmtScore(n) {
+    return n === null ? "–" : n.toFixed(2);
+}
+function fmtCost(n) {
+    return n === null ? "–" : `$${n.toFixed(4)}`;
+}
+async function runReport(args, config) {
+    let file;
+    let harness;
+    for (let i = 0; i < args.length; i++) {
+        const f = flag(args, i, "--harness");
+        if (f.skip >= 0) {
+            harness = f.value;
+            i += f.skip;
+            continue;
+        }
+        if (!args[i].startsWith("-") && !file)
+            file = args[i];
+    }
+    if (!file) {
+        console.error("Usage: codiedev evals report <run.json> [--harness name]");
+        process.exit(1);
+    }
+    let body;
+    try {
+        body = JSON.parse(fs.readFileSync(path.resolve(process.cwd(), file), "utf8"));
+    }
+    catch (err) {
+        console.error(`Couldn't read ${file}: ${err.message}`);
+        process.exit(1);
+    }
+    if (!body.subject || !body.question || !body.run?.label || !body.run?.model || typeof body.run?.casesTotal !== "number") {
+        console.error(`${file} needs subject, question, and run.label, run.model, run.casesTotal. See: codiedev evals help`);
+        process.exit(1);
+    }
+    const start = Date.now();
+    try {
+        const res = await (0, shared_1.apiRequest)("POST", "/api/cli/evals/runs", {
+            config,
+            body: { ...body, run: { ...body.run, harness: harness ?? body.run.harness ?? undefined } },
+        });
+        (0, shared_1.recordAgentEvent)(config, { tool: "codiedev_evals_report", ok: true, latencyMs: Date.now() - start });
+        console.log(`✓ Reported ${body.run.label} on ${body.subject}${res.created ? " (new experiment)" : ""}: "${body.question}"`);
+        console.log(`  experiment ${res.experimentId}. See it in Codiedev → Traces → Evals.`);
+    }
+    catch (err) {
+        const e = err;
+        (0, shared_1.recordAgentEvent)(config, { tool: "codiedev_evals_report", ok: false, latencyMs: Date.now() - start, error: e.message });
+        console.error(`Couldn't report: ${e.message}`);
+        process.exit(1);
+    }
+}
+async function runLast(args, config) {
+    const json = args.includes("--json");
+    const subject = args.find((a) => !a.startsWith("-"));
+    if (!subject) {
+        console.error("Usage: codiedev evals last <subject> [--json]");
+        process.exit(1);
+    }
+    let res;
+    try {
+        res = await (0, shared_1.apiRequest)("GET", `/api/cli/evals/last?subject=${encodeURIComponent(subject)}`, { config });
+    }
+    catch (err) {
+        const e = err;
+        console.error(e.status === 404 ? `No evals on ${subject} yet.` : `Couldn't load evals: ${e.message}`);
+        process.exit(1);
+    }
+    if (json) {
+        console.log(JSON.stringify(res, null, 2));
+        return;
+    }
+    const { experiment, runs, champion } = res;
+    if (champion) {
+        console.log(`Champion: ${champion.run.label} (${variant(champion.run)}), score ${fmtScore(champion.run.score)}, ${fmtCost(champion.run.costPerCaseUsd)}/case`);
+        if (champion.decisionNote)
+            console.log(`  Why: ${champion.decisionNote}`);
+    }
+    else {
+        console.log("Champion: none picked yet");
+    }
+    console.log("");
+    console.log(`Latest experiment (${res.experiments} on ${experiment.subject}): ${experiment.question}`);
+    if (experiment.dataset)
+        console.log(`  Dataset: ${experiment.dataset}`);
+    console.log(`  id ${experiment.id}, by ${experiment.createdBy}, updated ${new Date(experiment.updatedAt).toISOString().slice(0, 16).replace("T", " ")}`);
+    console.log("");
+    for (const run of [...runs].sort((a, b) => a.ranAt - b.ranAt)) {
+        const h2h = run.wins !== null ? ` · ${run.wins}W/${run.losses ?? 0}L/${run.ties ?? 0}T` : "";
+        console.log(`  ${run.isControl ? "[control] " : ""}${run.label.padEnd(24)} ${fmtScore(run.score).padStart(5)}  ${fmtCost(run.costPerCaseUsd).padStart(8)}/case${h2h}  ${variant(run)}`);
+    }
+    console.log("");
+    console.log(`Next run on this experiment: put "experimentId": "${experiment.id}" in run.json, then codiedev evals report run.json`);
+}
+async function runEvals(args, configOverride) {
+    const [sub, ...rest] = args;
+    if (sub === "report")
+        return runReport(rest, configOverride ?? (0, shared_1.requireConfig)());
+    if (sub === "last")
+        return runLast(rest, configOverride ?? (0, shared_1.requireConfig)());
+    console.log(exports.EVALS_HELP);
+    if (sub && sub !== "help")
+        process.exit(1);
+}

package/dist/commands/traces.js

@@ -1,6 +1,6 @@
 "use strict";
 // `codiedev traces agents`
-// `codiedev traces cases <agent> [--suite name] [--model id] [--out dir] [--json]`
+// `codiedev traces cases <agent> [--dataset name] [--model id] [--out dir] [--json]`
 // `codiedev traces report <results.json> [--harness name]`
 //
 // Test cases are runs an engineer saved from Codiedev → Traces. They run in
@@ -55,10 +55,10 @@ exports.TRACES_HELP = `
 codiedev traces — test cases saved from Codiedev → Traces
 
   codiedev traces agents                 Agents you maintain, with saved case counts
-  codiedev traces cases <agent> [--suite name] [--case id[,id]] [--model id] [--out dir] [--json]
-                                         Write an agent's cases (one suite, or just
+  codiedev traces cases <agent> [--dataset name] [--case id[,id]] [--model id] [--out dir] [--json]
+                                         Write an agent's cases (one dataset, or just
                                          the cases named) into this repo, default
-                                         .codiedev/traces/<agent>[/<suite>]/, then
+                                         .codiedev/traces/<agent>[/<dataset>]/, then
                                          ask your coding agent to run them.
                                          --model runs them on another model, e.g.
                                          the replacement for a retiring one
@@ -140,7 +140,7 @@ function parseCasesArgs(args) {
     for (let i = 0; i < args.length; i++) {
         const a = args[i];
         let matched = false;
-        for (const [name, key] of [["--out", "out"], ["-o", "out"], ["--suite", "suite"], ["--model", "model"], ["--case", "caseIds"]]) {
+        for (const [name, key] of [["--out", "out"], ["-o", "out"], ["--dataset", "suite"], ["--suite", "suite"], ["--model", "model"], ["--case", "caseIds"]]) {
             const f = flag(args, i, name);
             if (f.skip >= 0) {
                 parsed[key] = f.value;
@@ -184,7 +184,7 @@ async function runAgents(config) {
 async function runCases(args, config) {
     const { agent, suite, model, caseIds, out, json } = parseCasesArgs(args);
     if (!agent) {
-        console.error("Usage: codiedev traces cases <agent> [--suite name] [--model id] [--out dir] [--json]");
+        console.error("Usage: codiedev traces cases <agent> [--dataset name] [--model id] [--out dir] [--json]");
         process.exit(1);
     }
     const start = Date.now();
@@ -210,7 +210,7 @@ async function runCases(args, config) {
     }
     if (!res.cases.length) {
         console.log(suite
-            ? `${res.agent.name} has no test cases in "${suite}". See your suites in Codiedev → Traces → Test cases.`
+            ? `${res.agent.name} has no test cases in "${suite}". See your datasets in Codiedev → Traces → Test cases.`
             : `${res.agent.name} has no saved test cases. Open a run in Codiedev → Traces and click "Save as test case".`);
         return;
     }

package/dist/piRuntime.d.ts

@@ -16,6 +16,7 @@ export interface PiRuntimeDeps {
     rename?: (from: string, to: string) => void;
     remove?: (target: string) => void;
     log?: (message: string) => void;
+    sleep?: (ms: number) => void;
 }
 /**
  * The pi CLI entry to run, installing the pinned version first if it is

package/dist/piRuntime.js

@@ -70,6 +70,10 @@ function installedPiEntry(installDir, readFile = (f) => fs.readFileSync(f, "utf8
         return null;
     }
 }
+// One retry rides out a dropped connection or registry blip; a second
+// failure is a real problem (offline, blocked registry) and is reported.
+const INSTALL_ATTEMPTS = 2;
+const INSTALL_RETRY_DELAY_MS = 5_000;
 /**
  * The pi CLI entry to run, installing the pinned version first if it is
  * missing or a different version. Staged then renamed, so an interrupted
@@ -85,10 +89,10 @@ function ensurePiRuntime(deps) {
     const remove = deps.remove ?? ((target) => fs.rmSync(target, { recursive: true, force: true }));
     const rename = deps.rename ?? fs.renameSync;
     const log = deps.log ?? ((message) => console.error(message));
-    const staging = (deps.makeStagingDir ?? (() => {
+    const makeStagingDir = deps.makeStagingDir ?? (() => {
         fs.mkdirSync(deps.stateDir, { recursive: true });
         return fs.mkdtempSync(path.join(deps.stateDir, ".runtime-pi-"));
-    }))();
+    });
     const run = deps.run ?? ((args) => {
         const result = (0, child_process_1.spawnSync)(process.platform === "win32" ? "npm.cmd" : "npm", args, {
             encoding: "utf8", stdio: ["ignore", "ignore", "pipe"], shell: process.platform === "win32",
@@ -97,26 +101,35 @@ function ensurePiRuntime(deps) {
             throw result.error;
         return { status: result.status, stderr: result.stderr ?? "" };
     });
+    const sleep = deps.sleep ?? ((ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms));
     log(`Installing pi (${exports.PI_RUNTIME_SPEC}). This runs once, and only for \`codie --pi\`.`);
-    try {
-        const result = run((0, codieRuntimeInstall_1.buildRuntimeInstallArgs)(staging, exports.PI_RUNTIME_SPEC));
-        if (result.status !== 0)
-            throw new Error(result.stderr?.trim() || `npm exited with ${result.status}`);
-        // Trust the artifact, not the exit code.
-        const staged = installedPiEntry(staging, readFile);
-        if (!staged || !exists(staged)) {
-            throw new Error(`the install did not produce pi. Check that ${exports.PI_RUNTIME_SPEC} is available from your npm registry.`);
+    for (let attempt = 1;; attempt++) {
+        const staging = makeStagingDir();
+        try {
+            const result = run((0, codieRuntimeInstall_1.buildRuntimeInstallArgs)(staging, exports.PI_RUNTIME_SPEC));
+            if (result.status !== 0)
+                throw new Error(result.stderr?.trim() || `npm exited with ${result.status}`);
+            // Trust the artifact, not the exit code.
+            const staged = installedPiEntry(staging, readFile);
+            if (!staged || !exists(staged)) {
+                throw new Error(`the install did not produce pi. Check that ${exports.PI_RUNTIME_SPEC} is available from your npm registry.`);
+            }
+            remove(installDir);
+            rename(staging, installDir);
+            break;
         }
-        remove(installDir);
-        rename(staging, installDir);
-    }
-    catch (error) {
-        throw new Error(`Couldn't install pi: ${error instanceof Error ? error.message : String(error)}`);
-    }
-    finally {
-        // No-op after a successful promote; otherwise it keeps a ~200MB staging
-        // dir from being orphaned.
-        remove(staging);
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            if (attempt >= INSTALL_ATTEMPTS)
+                throw new Error(`Couldn't install pi: ${message}`);
+            log(`pi install failed (${message.split("\n")[0]}); retrying.`);
+        }
+        finally {
+            // No-op after a successful promote; otherwise it keeps a ~200MB staging
+            // dir from being orphaned.
+            remove(staging);
+        }
+        sleep(INSTALL_RETRY_DELAY_MS);
     }
     const entry = installedPiEntry(installDir, readFile);
     if (!entry)

package/dist/utils.js

@@ -1681,7 +1681,8 @@ thought, use the \`codiedev\` CLI via Bash:**
 | "find something about X" / "has anyone solved Y?" | \`codiedev search "<query>" [--limit N]\` |
 | "why did we…?" / "what does the team know about X?" | \`codiedev ask "<question>"\` (synthesized answer + citations) |
 | "show my library" / "what artifacts exist?" | \`codiedev library [--scope mine\\|shared\\|all] [--kind doc\\|skill]\` |
-| "run my agent's test cases" / "check the suite on model Y" | \`codiedev traces cases <agent> [--suite <name>] [--model <id>]\` (writes cases + RUN_CASES.md + results.json; follow it, then \`codiedev traces report <results.json>\`) |
+| "run my agent's test cases" / "check the dataset on model Y" | \`codiedev traces cases <agent> [--dataset <name>] [--model <id>]\` (writes cases + RUN_CASES.md + results.json; follow it, then \`codiedev traces report <results.json>\`) |
+| "what was the last eval on X?" / "file these eval results" | \`codiedev evals last <subject>\` before starting a related eval; \`codiedev evals report <run.json>\` after each variant runs (scores only — \`codiedev evals help\` for the shape) |
 
 **Just push the file — we'll figure out where it goes.** Every artifact
 is classified into one of two kinds at upload time:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codiedev",
-  "version": "0.9.73",
+  "version": "0.9.76",
   "description": "Connect Claude Code, Codex, Cursor, OpenCode, or VS Code Copilot to Codiedev for team session capture and artifact collaboration — and run the governed Codie harness",
   "bin": {
     "codiedev": "dist/cli.js",