codiedev 0.9.77 → 0.9.78

package/dist/commands/traces.d.ts

@@ -27,6 +27,9 @@ interface CasesResponse {
     ok: boolean;
     /** The suite pulled, when --suite was given. */
     suite: string | null;
+    /** The saved dataset version pulled, when --version was given; cases deleted since are counted in `missing`. */
+    version?: number | null;
+    missing?: number;
     /** How the suite is judged, when it has settings. */
     judge: {
         model: string | null;
@@ -44,6 +47,7 @@ interface CasesResponse {
 export interface ResultsFile {
     agent: string;
     suite: string | null;
+    version?: number | null;
     model: string;
     judgeModel: string | null;
     harness: string | null;

package/dist/commands/traces.js

@@ -55,10 +55,10 @@ exports.TRACES_HELP = `
 codiedev traces — test cases saved from Codiedev → Traces
 
   codiedev traces agents                 Agents you maintain, with saved case counts
-  codiedev traces cases <agent> [--dataset name] [--case id[,id]] [--model id] [--out dir] [--json]
+  codiedev traces cases <agent> [--dataset name [--version n]] [--case id[,id]] [--model id] [--out dir] [--json]
                                          Write an agent's cases (one dataset, or just
                                          the cases named) into this repo, default
-                                         .codiedev/traces/<agent>[/<dataset>]/, then
+                                         .codiedev/traces/<agent>[/<dataset>[/v<n>]]/, then
                                          ask your coding agent to run them.
                                          --model runs them on another model, e.g.
                                          the replacement for a retiring one
@@ -78,7 +78,7 @@ const FLAG_LABEL = {
 };
 const TRIES = 3;
 const plural = (n, word) => `${n} ${word}${n === 1 ? "" : "s"}`;
-const recordedModel = (res) => res.cases.flatMap((c) => c.steps.map((s) => s.model))[0] ?? "the recorded model";
+const recordedModel = (res) => res.cases.flatMap((c) => c.steps.map((s) => s.model)).filter(Boolean)[0] ?? "the agent's current model";
 function slugify(name) {
     return name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "") || "agent";
 }
@@ -90,7 +90,7 @@ function casesBrief(res, casesFile, opts = {}) {
     const target = opts.model;
     const results = opts.resultsFile ? path.basename(opts.resultsFile) : "results.json";
     const lines = [
-        `# Test cases for ${res.agent.name}${res.suite ? ` · ${res.suite}` : ""}${target ? ` on ${target}` : ""}`,
+        `# Test cases for ${res.agent.name}${res.suite ? ` · ${res.suite}` : ""}${res.version ? ` v${res.version}` : ""}${target ? ` on ${target}` : ""}`,
         "",
         res.agent.description ? `${res.agent.description}\n` : "",
         `Saved from Codiedev → Traces. ${plural(res.cases.length, "case")} in \`${path.basename(casesFile)}\` (one JSON object per line):`,
@@ -101,7 +101,7 @@ function casesBrief(res, casesFile, opts = {}) {
     if (target) {
         lines.push("## This is a model check", "", `Run every case on **${target}** instead of ${recordedModel(res)} (the model the cases were recorded on). Change only the model the agent asks the gateway for; keep its prompts and tools as they are, so the result says how ${target} does on the same work.`, "");
     }
-    lines.push("## How to run them", "", "1. Find where this agent builds its model requests and calls the Codiedev gateway.", `2. For each case, replay \`steps[0].request\` (the first prompt the agent sent) through the agent's own code path, so tools and prompts are the current version, not the recorded one${target ? `, with the model set to ${target}` : ""}.`, "3. Judge each try against the case:", "   - A case's `expected`, when present, is what a passing answer looks like (a reference reply or criteria). It wins over everything below.", "   - `fix` cases: the run must not reproduce its `flags` (listed below), and must finish (no failed call, no `finish_reason: \"length\"`, no repeated identical tool call).", "   - `keep` cases: the final reply must still do what the recorded final reply did. Compare meaning, not wording.", "   - A case's `note`, when present, is the engineer's extra context.");
+    lines.push("## How to run them", "", "1. Find where this agent builds its model requests and calls the Codiedev gateway.", `2. For each case, replay \`steps[0].request\` (the first prompt the agent sent) through the agent's own code path, so tools and prompts are the current version, not the recorded one${target ? `, with the model set to ${target}` : ""}.`, "3. Judge each try against the case:", "   - A case's `expected`, when present, is what a passing answer looks like (a reference reply or criteria). It wins over everything below.", "   - `fix` cases: the run must not reproduce its `flags` (listed below), and must finish (no failed call, no `finish_reason: \"length\"`, no repeated identical tool call).", "   - `keep` cases: the final reply must still do what the recorded final reply did. Compare meaning, not wording.", "   - A case's `note`, when present, is the engineer's extra context.", "   - Cases whose `runId` starts with `manual:` or `import:` were written by hand, not recorded: `steps[0].request` is the input and there's no recorded reply, so judge them on `expected`.");
     if (res.judge?.model || res.judge?.rubric) {
         lines.push(`   - Where passing needs judgement, judge with **${res.judge.model ?? "a strong model"}** through the gateway${res.judge.rubric ? `, using this rubric: "${res.judge.rubric}"` : ""}. Use the same judge for every model so results compare.`);
     }
@@ -121,6 +121,7 @@ function resultsTemplate(res, model) {
     return {
         agent: res.agent.name,
         suite: res.suite,
+        ...(res.version ? { version: res.version } : {}),
         model: model ?? recordedModel(res),
         judgeModel: res.judge?.model ?? null,
         harness: null,
@@ -140,7 +141,7 @@ function parseCasesArgs(args) {
     for (let i = 0; i < args.length; i++) {
         const a = args[i];
         let matched = false;
-        for (const [name, key] of [["--out", "out"], ["-o", "out"], ["--dataset", "suite"], ["--suite", "suite"], ["--model", "model"], ["--case", "caseIds"]]) {
+        for (const [name, key] of [["--out", "out"], ["-o", "out"], ["--dataset", "suite"], ["--suite", "suite"], ["--model", "model"], ["--case", "caseIds"], ["--version", "version"]]) {
             const f = flag(args, i, name);
             if (f.skip >= 0) {
                 parsed[key] = f.value;
@@ -182,20 +183,26 @@ async function runAgents(config) {
     console.log("\nPull one: codiedev traces cases <agent>");
 }
 async function runCases(args, config) {
-    const { agent, suite, model, caseIds, out, json } = parseCasesArgs(args);
+    const { agent, suite, model, caseIds, version, out, json } = parseCasesArgs(args);
     if (!agent) {
-        console.error("Usage: codiedev traces cases <agent> [--dataset name] [--model id] [--out dir] [--json]");
+        console.error("Usage: codiedev traces cases <agent> [--dataset name [--version n]] [--model id] [--out dir] [--json]");
+        process.exit(1);
+    }
+    if (version !== undefined && (!suite || !/^[1-9]\d*$/.test(version))) {
+        console.error("--version takes a whole number and needs --dataset: codiedev traces cases <agent> --dataset <name> --version <n>");
         process.exit(1);
     }
     const start = Date.now();
     let res;
     try {
-        res = await (0, shared_1.apiRequest)("GET", "/api/cli/traces/cases", { config, query: { agent, ...(suite ? { suite } : {}), ...(caseIds ? { case: caseIds } : {}) } });
+        res = await (0, shared_1.apiRequest)("GET", "/api/cli/traces/cases", { config, query: { agent, ...(suite ? { suite } : {}), ...(version ? { version } : {}), ...(caseIds ? { case: caseIds } : {}) } });
     }
     catch (err) {
         const e = err;
         (0, shared_1.recordAgentEvent)(config, { tool: "codiedev_traces_cases", ok: false, latencyMs: Date.now() - start, error: e.message });
-        if (e.status === 404)
+        if (e.status === 404 && version && /No version/.test(e.message))
+            console.error(`No version ${version} of "${suite}". See its versions in Codiedev → Traces → Test cases.`);
+        else if (e.status === 404)
             console.error(`No agent called "${agent}". See your agents with: codiedev traces agents`);
         else if (e.status === 403)
             console.error(`Only ${agent}'s maintainers can pull its test cases.`);
@@ -214,7 +221,7 @@ async function runCases(args, config) {
             : `${res.agent.name} has no saved test cases. Open a run in Codiedev → Traces and click "Save as test case".`);
         return;
     }
-    const dir = path.resolve(process.cwd(), out ?? path.join(".codiedev", "traces", slugify(res.agent.name), ...(res.suite ? [slugify(res.suite)] : []), ...(caseIds && res.cases.length === 1 ? [slugify(res.cases[0].name)] : []), ...(model ? [slugify(model)] : [])));
+    const dir = path.resolve(process.cwd(), out ?? path.join(".codiedev", "traces", slugify(res.agent.name), ...(res.suite ? [slugify(res.suite)] : []), ...(res.version ? [`v${res.version}`] : []), ...(caseIds && res.cases.length === 1 ? [slugify(res.cases[0].name)] : []), ...(model ? [slugify(model)] : [])));
     fs.mkdirSync(dir, { recursive: true });
     const casesFile = path.join(dir, "cases.jsonl");
     fs.writeFileSync(casesFile, res.cases.map((c) => JSON.stringify(c)).join("\n") + "\n", "utf8");
@@ -224,7 +231,9 @@ async function runCases(args, config) {
     fs.writeFileSync(briefFile, casesBrief(res, casesFile, { model, resultsFile }), "utf8");
     const fix = res.cases.filter((c) => c.kind === "fix").length;
     const rel = (file) => path.relative(process.cwd(), file) || file;
-    console.log(`✓ ${plural(res.cases.length, "case")} for ${res.agent.name}${res.suite ? ` · ${res.suite}` : ""}${model ? ` on ${model}` : ""} (${fix} to fix, ${res.cases.length - fix} to keep passing)`);
+    console.log(`✓ ${plural(res.cases.length, "case")} for ${res.agent.name}${res.suite ? ` · ${res.suite}` : ""}${res.version ? ` v${res.version}` : ""}${model ? ` on ${model}` : ""} (${fix} to fix, ${res.cases.length - fix} to keep passing)`);
+    if (res.missing)
+        console.log(`  ${plural(res.missing, "case")} in v${res.version} were deleted since and can't be pulled.`);
     console.log(`  ${rel(casesFile)}`);
     console.log(`  ${rel(briefFile)}`);
     console.log(`  ${rel(resultsFile)}  (filled in by your coding agent, then: codiedev traces report ${rel(resultsFile)})`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codiedev",
-  "version": "0.9.77",
+  "version": "0.9.78",
   "description": "Connect Claude Code, Codex, Cursor, OpenCode, or VS Code Copilot to Codiedev for team session capture and artifact collaboration — and run the governed Codie harness",
   "bin": {
     "codiedev": "dist/cli.js",