codiedev 0.9.72 → 0.9.73

package/dist/cli.js

@@ -30,6 +30,7 @@ const share_1 = require("./commands/share");
 const send_1 = require("./commands/send");
 const capture_1 = require("./commands/capture");
 const pack_1 = require("./commands/pack");
+const traces_1 = require("./commands/traces");
 // --- starter packs phase 2 ---
 const hookGate_1 = require("./packs/hookGate");
 const sync_1 = require("./packs/sync");
@@ -91,6 +92,14 @@ Starter packs:
                                          detected tool
   codiedev pack status                   Installed packs, up to date or not
 
+Traces (agents on the gateway):
+  codiedev traces agents                 Agents you maintain, with saved test cases
+  codiedev traces cases <agent> [--suite name] [--model id]
+                                         Pull an agent's test cases (or one suite)
+                                         into this repo for your coding agent to
+                                         run, optionally on another model
+  codiedev traces report <results.json>  Send the results back to Traces
+
 Discovery:
   codiedev search "<query>"              Semantic search across the team's
                                          captured sessions and artifacts
@@ -152,6 +161,7 @@ status sync never weakens it.
 `.trim();
 const COMMAND_HELP = {
     pack: pack_1.PACK_HELP,
+    traces: traces_1.TRACES_HELP,
     connect: `
 codiedev connect — link your agent CLI to Codiedev
 
@@ -515,6 +525,9 @@ async function main() {
             case "ls":
                 await (0, library_1.runLibrary)(rest);
                 return;
+            case "traces":
+                await (0, traces_1.runTraces)(rest);
+                return;
             case "share":
                 await (0, share_1.runShare)(rest);
                 return;

package/dist/commands/traces.d.ts

@@ -0,0 +1,68 @@
+import type { CodiedevConfig } from "../utils";
+export declare const TRACES_HELP: string;
+interface CaseStep {
+    model: string;
+    finishReason?: string;
+    statusCode: number;
+    request?: string;
+    reply?: string;
+    toolCalls?: Array<{
+        name: string;
+        arguments: string;
+    }>;
+}
+interface CaseRow {
+    id: string;
+    name: string;
+    suite: string | null;
+    runId: string;
+    kind: "fix" | "keep";
+    flags: string[];
+    note: string | null;
+    expected: string | null;
+    createdAt: string;
+    steps: CaseStep[];
+}
+interface CasesResponse {
+    ok: boolean;
+    /** The suite pulled, when --suite was given. */
+    suite: string | null;
+    /** How the suite is judged, when it has settings. */
+    judge: {
+        model: string | null;
+        rubric: string | null;
+    } | null;
+    agent: {
+        id: string;
+        name: string;
+        description: string | null;
+        promptDays: number | null;
+    };
+    cases: CaseRow[];
+}
+/** What `report` sends, and what `cases` writes for the coding agent to fill in. */
+export interface ResultsFile {
+    agent: string;
+    suite: string | null;
+    model: string;
+    judgeModel: string | null;
+    harness: string | null;
+    results: Array<{
+        caseId: string;
+        name: string;
+        passes: number | null;
+        tries: number;
+        note: string | null;
+        output: string | null;
+    }>;
+}
+export declare function slugify(name: string): string;
+/** The brief a coding agent follows to run the cases and report back. */
+export declare function casesBrief(res: CasesResponse, casesFile: string, opts?: {
+    model?: string;
+    resultsFile?: string;
+}): string;
+/** The results.json a run starts from: every case, passes to be filled in. */
+export declare function resultsTemplate(res: CasesResponse, model: string | undefined): ResultsFile;
+export declare function runTraces(args: string[], configOverride?: CodiedevConfig): Promise<void>;
+export {};

package/dist/commands/traces.js

@@ -0,0 +1,292 @@
+"use strict";
+// `codiedev traces agents`
+// `codiedev traces cases <agent> [--suite name] [--model id] [--out dir] [--json]`
+// `codiedev traces report <results.json> [--harness name]`
+//
+// Test cases are runs an engineer saved from Codiedev → Traces. They run in
+// the engineer's coding agent, not in the portal: `cases` writes them into the
+// repo with a brief the agent follows to replay each one against the agent's
+// own code, optionally on another model (a regression check before a model
+// swap), and a results.json for `report` to send back so Traces shows pass
+// rates by model.
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TRACES_HELP = void 0;
+exports.slugify = slugify;
+exports.casesBrief = casesBrief;
+exports.resultsTemplate = resultsTemplate;
+exports.runTraces = runTraces;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const shared_1 = require("./shared");
+exports.TRACES_HELP = `
+codiedev traces — test cases saved from Codiedev → Traces
+
+  codiedev traces agents                 Agents you maintain, with saved case counts
+  codiedev traces cases <agent> [--suite name] [--case id[,id]] [--model id] [--out dir] [--json]
+                                         Write an agent's cases (one suite, or just
+                                         the cases named) into this repo, default
+                                         .codiedev/traces/<agent>[/<suite>]/, then
+                                         ask your coding agent to run them.
+                                         --model runs them on another model, e.g.
+                                         the replacement for a retiring one
+  codiedev traces report <results.json> [--harness name]
+                                         Send the filled-in results back, so
+                                         Traces shows the pass rate for that model
+
+  <agent> is the agent's name (its API key name) or id. Only an agent's
+  maintainers can pull its cases or report on them.
+`.trim();
+const FLAG_LABEL = {
+    budget_block: "refused for budget",
+    errors: "model call failed",
+    tool_loop: "repeated the same tool call",
+    truncated: "reply cut off at max_tokens",
+    context_overflow: "prompt over 70% of the context window",
+};
+const TRIES = 3;
+const plural = (n, word) => `${n} ${word}${n === 1 ? "" : "s"}`;
+const recordedModel = (res) => res.cases.flatMap((c) => c.steps.map((s) => s.model))[0] ?? "the recorded model";
+function slugify(name) {
+    return name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "") || "agent";
+}
+/** The brief a coding agent follows to run the cases and report back. */
+function casesBrief(res, casesFile, opts = {}) {
+    const fix = res.cases.filter((c) => c.kind === "fix");
+    const keep = res.cases.filter((c) => c.kind === "keep");
+    const withBodies = res.cases.filter((c) => c.steps.some((s) => s.request)).length;
+    const target = opts.model;
+    const results = opts.resultsFile ? path.basename(opts.resultsFile) : "results.json";
+    const lines = [
+        `# Test cases for ${res.agent.name}${res.suite ? ` · ${res.suite}` : ""}${target ? ` on ${target}` : ""}`,
+        "",
+        res.agent.description ? `${res.agent.description}\n` : "",
+        `Saved from Codiedev → Traces. ${plural(res.cases.length, "case")} in \`${path.basename(casesFile)}\` (one JSON object per line):`,
+        `- **${fix.length} to fix**: production runs that failed. The fix should make them pass.`,
+        `- **${keep.length} to keep passing**: good runs. They should still pass after any change.`,
+        "",
+    ];
+    if (target) {
+        lines.push("## This is a model check", "", `Run every case on **${target}** instead of ${recordedModel(res)} (the model the cases were recorded on). Change only the model the agent asks the gateway for; keep its prompts and tools as they are, so the result says how ${target} does on the same work.`, "");
+    }
+    lines.push("## How to run them", "", "1. Find where this agent builds its model requests and calls the Codiedev gateway.", `2. For each case, replay \`steps[0].request\` (the first prompt the agent sent) through the agent's own code path, so tools and prompts are the current version, not the recorded one${target ? `, with the model set to ${target}` : ""}.`, "3. Judge each try against the case:", "   - A case's `expected`, when present, is what a passing answer looks like (a reference reply or criteria). It wins over everything below.", "   - `fix` cases: the run must not reproduce its `flags` (listed below), and must finish (no failed call, no `finish_reason: \"length\"`, no repeated identical tool call).", "   - `keep` cases: the final reply must still do what the recorded final reply did. Compare meaning, not wording.", "   - A case's `note`, when present, is the engineer's extra context.");
+    if (res.judge?.model || res.judge?.rubric) {
+        lines.push(`   - Where passing needs judgement, judge with **${res.judge.model ?? "a strong model"}** through the gateway${res.judge.rubric ? `, using this rubric: "${res.judge.rubric}"` : ""}. Use the same judge for every model so results compare.`);
+    }
+    lines.push(`4. Run each case ${TRIES} times: agents are stochastic, so report a pass rate per case, not a single result.`, `5. Fill in \`${results}\`: for each case set \`passes\` (out of \`tries\`), \`output\` (the agent's final reply on one try, so it can be compared side by side with other models) and a one-line \`note\` on any failure. Set \`harness\` to the coding agent you are (claude-code, codex, cursor, copilot, …).`, `6. Send it back: \`codiedev traces report ${opts.resultsFile ? path.relative(process.cwd(), opts.resultsFile) || results : results}\`. Then show a table: case, kind, passes out of ${TRIES}, and the failure note.`, "", "Send real model calls through the gateway with the agent's usual key and `X-Codiedev-Run-Id: case-<id>-<try>`, so the replays show up in Traces too.", "");
+    if (withBodies < res.cases.length) {
+        lines.push(`Note: ${plural(res.cases.length - withBodies, "case")} ${res.cases.length - withBodies === 1 ? "has" : "have"} no recorded prompt (the agent wasn't keeping prompts then). For those, rebuild the input from the case's \`expected\` or \`note\`, or skip them and say so.`, "");
+    }
+    lines.push("## Cases", "");
+    for (const c of res.cases) {
+        const flags = c.flags.length ? c.flags.map((f) => FLAG_LABEL[f] ?? f).join(", ") : "no flags";
+        lines.push(`- \`${c.id}\` · ${c.name} · ${c.kind} · ${flags} · ${plural(c.steps.length, "model call")}${c.expected ? ` · expected: "${c.expected.replace(/\s+/g, " ").slice(0, 200)}"` : ""}${c.note ? ` · note: "${c.note}"` : ""}`);
+    }
+    return lines.filter((line, i, all) => !(line === "" && all[i - 1] === "")).join("\n") + "\n";
+}
+/** The results.json a run starts from: every case, passes to be filled in. */
+function resultsTemplate(res, model) {
+    return {
+        agent: res.agent.name,
+        suite: res.suite,
+        model: model ?? recordedModel(res),
+        judgeModel: res.judge?.model ?? null,
+        harness: null,
+        results: res.cases.map((c) => ({ caseId: c.id, name: c.name, passes: null, tries: TRIES, note: null, output: null })),
+    };
+}
+function flag(args, i, name) {
+    const a = args[i];
+    if (a === name && i + 1 < args.length)
+        return { value: args[i + 1], skip: 1 };
+    if (a.startsWith(`${name}=`))
+        return { value: a.slice(name.length + 1), skip: 0 };
+    return { skip: -1 };
+}
+function parseCasesArgs(args) {
+    const parsed = { json: false };
+    for (let i = 0; i < args.length; i++) {
+        const a = args[i];
+        let matched = false;
+        for (const [name, key] of [["--out", "out"], ["-o", "out"], ["--suite", "suite"], ["--model", "model"], ["--case", "caseIds"]]) {
+            const f = flag(args, i, name);
+            if (f.skip >= 0) {
+                parsed[key] = f.value;
+                i += f.skip;
+                matched = true;
+                break;
+            }
+        }
+        if (matched)
+            continue;
+        if (a === "--json")
+            parsed.json = true;
+        else if (!a.startsWith("-") && !parsed.agent)
+            parsed.agent = a;
+    }
+    return parsed;
+}
+async function runAgents(config) {
+    const start = Date.now();
+    let res;
+    try {
+        res = await (0, shared_1.apiRequest)("GET", "/api/cli/traces/agents", { config });
+    }
+    catch (err) {
+        const e = err;
+        (0, shared_1.recordAgentEvent)(config, { tool: "codiedev_traces_agents", ok: false, latencyMs: Date.now() - start, error: e.message });
+        console.error(`Couldn't list agents: ${e.message}`);
+        process.exit(1);
+    }
+    (0, shared_1.recordAgentEvent)(config, { tool: "codiedev_traces_agents", ok: true, latencyMs: Date.now() - start });
+    if (!res.agents.length) {
+        console.log("You don't maintain any agents yet. Connect one in Codiedev → Traces → How it works.");
+        return;
+    }
+    for (const agent of res.agents) {
+        const capture = agent.promptDays ? `prompts kept ${agent.promptDays} days` : "metadata only";
+        console.log(`${agent.name.padEnd(28)} ${plural(agent.cases, "case").padStart(9)}   ${capture}`);
+    }
+    console.log("\nPull one: codiedev traces cases <agent>");
+}
+async function runCases(args, config) {
+    const { agent, suite, model, caseIds, out, json } = parseCasesArgs(args);
+    if (!agent) {
+        console.error("Usage: codiedev traces cases <agent> [--suite name] [--model id] [--out dir] [--json]");
+        process.exit(1);
+    }
+    const start = Date.now();
+    let res;
+    try {
+        res = await (0, shared_1.apiRequest)("GET", "/api/cli/traces/cases", { config, query: { agent, ...(suite ? { suite } : {}), ...(caseIds ? { case: caseIds } : {}) } });
+    }
+    catch (err) {
+        const e = err;
+        (0, shared_1.recordAgentEvent)(config, { tool: "codiedev_traces_cases", ok: false, latencyMs: Date.now() - start, error: e.message });
+        if (e.status === 404)
+            console.error(`No agent called "${agent}". See your agents with: codiedev traces agents`);
+        else if (e.status === 403)
+            console.error(`Only ${agent}'s maintainers can pull its test cases.`);
+        else
+            console.error(`Couldn't pull cases: ${e.message}`);
+        process.exit(1);
+    }
+    (0, shared_1.recordAgentEvent)(config, { tool: "codiedev_traces_cases", ok: true, latencyMs: Date.now() - start });
+    if (json) {
+        process.stdout.write(JSON.stringify(res, null, 2) + "\n");
+        return;
+    }
+    if (!res.cases.length) {
+        console.log(suite
+            ? `${res.agent.name} has no test cases in "${suite}". See your suites in Codiedev → Traces → Test cases.`
+            : `${res.agent.name} has no saved test cases. Open a run in Codiedev → Traces and click "Save as test case".`);
+        return;
+    }
+    const dir = path.resolve(process.cwd(), out ?? path.join(".codiedev", "traces", slugify(res.agent.name), ...(res.suite ? [slugify(res.suite)] : []), ...(caseIds && res.cases.length === 1 ? [slugify(res.cases[0].name)] : []), ...(model ? [slugify(model)] : [])));
+    fs.mkdirSync(dir, { recursive: true });
+    const casesFile = path.join(dir, "cases.jsonl");
+    fs.writeFileSync(casesFile, res.cases.map((c) => JSON.stringify(c)).join("\n") + "\n", "utf8");
+    const resultsFile = path.join(dir, "results.json");
+    fs.writeFileSync(resultsFile, JSON.stringify(resultsTemplate(res, model), null, 2) + "\n", "utf8");
+    const briefFile = path.join(dir, "RUN_CASES.md");
+    fs.writeFileSync(briefFile, casesBrief(res, casesFile, { model, resultsFile }), "utf8");
+    const fix = res.cases.filter((c) => c.kind === "fix").length;
+    const rel = (file) => path.relative(process.cwd(), file) || file;
+    console.log(`✓ ${plural(res.cases.length, "case")} for ${res.agent.name}${res.suite ? ` · ${res.suite}` : ""}${model ? ` on ${model}` : ""} (${fix} to fix, ${res.cases.length - fix} to keep passing)`);
+    console.log(`  ${rel(casesFile)}`);
+    console.log(`  ${rel(briefFile)}`);
+    console.log(`  ${rel(resultsFile)}  (filled in by your coding agent, then: codiedev traces report ${rel(resultsFile)})`);
+    console.log(`\nNext: ask your coding agent to "run the test cases in ${rel(briefFile)}"`);
+}
+async function runReport(args, config) {
+    let file;
+    let harness;
+    for (let i = 0; i < args.length; i++) {
+        const f = flag(args, i, "--harness");
+        if (f.skip >= 0) {
+            harness = f.value;
+            i += f.skip;
+            continue;
+        }
+        if (!args[i].startsWith("-") && !file)
+            file = args[i];
+    }
+    if (!file) {
+        console.error("Usage: codiedev traces report <results.json> [--harness name]");
+        process.exit(1);
+    }
+    let body;
+    try {
+        body = JSON.parse(fs.readFileSync(path.resolve(process.cwd(), file), "utf8"));
+    }
+    catch (err) {
+        console.error(`Couldn't read ${file}: ${err.message}`);
+        process.exit(1);
+    }
+    const missing = body.results.filter((r) => typeof r.passes !== "number");
+    if (missing.length) {
+        console.error(`${plural(missing.length, "case")} in ${file} ${missing.length === 1 ? "has" : "have"} no "passes" yet: ${missing.slice(0, 5).map((r) => r.name).join(", ")}. Run them first.`);
+        process.exit(1);
+    }
+    const start = Date.now();
+    try {
+        const res = await (0, shared_1.apiRequest)("POST", "/api/cli/traces/results", {
+            config,
+            body: { ...body, harness: harness ?? body.harness ?? undefined },
+        });
+        (0, shared_1.recordAgentEvent)(config, { tool: "codiedev_traces_report", ok: true, latencyMs: Date.now() - start });
+        const passed = body.results.filter((r) => r.passes === r.tries).length;
+        console.log(`✓ Reported ${body.agent}${body.suite ? ` · ${body.suite}` : ""} on ${body.model}: ${Math.round(res.passRate * 100)}% of tries passed, ${passed}/${body.results.length} cases passed every try.`);
+        console.log("  See it next to other models in Codiedev → Traces → Test cases.");
+    }
+    catch (err) {
+        const e = err;
+        (0, shared_1.recordAgentEvent)(config, { tool: "codiedev_traces_report", ok: false, latencyMs: Date.now() - start, error: e.message });
+        console.error(e.status === 403 ? `Only ${body.agent}'s maintainers can report its results.` : `Couldn't report: ${e.message}`);
+        process.exit(1);
+    }
+}
+async function runTraces(args, configOverride) {
+    const [sub, ...rest] = args;
+    if (sub === "agents")
+        return runAgents(configOverride ?? (0, shared_1.requireConfig)());
+    if (sub === "cases")
+        return runCases(rest, configOverride ?? (0, shared_1.requireConfig)());
+    if (sub === "report")
+        return runReport(rest, configOverride ?? (0, shared_1.requireConfig)());
+    console.log(exports.TRACES_HELP);
+    if (sub && sub !== "help")
+        process.exit(1);
+}

package/dist/utils.js

@@ -1681,6 +1681,7 @@ thought, use the \`codiedev\` CLI via Bash:**
 | "find something about X" / "has anyone solved Y?" | \`codiedev search "<query>" [--limit N]\` |
 | "why did we…?" / "what does the team know about X?" | \`codiedev ask "<question>"\` (synthesized answer + citations) |
 | "show my library" / "what artifacts exist?" | \`codiedev library [--scope mine\\|shared\\|all] [--kind doc\\|skill]\` |
+| "run my agent's test cases" / "check the suite on model Y" | \`codiedev traces cases <agent> [--suite <name>] [--model <id>]\` (writes cases + RUN_CASES.md + results.json; follow it, then \`codiedev traces report <results.json>\`) |
 
 **Just push the file — we'll figure out where it goes.** Every artifact
 is classified into one of two kinds at upload time:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codiedev",
-  "version": "0.9.72",
+  "version": "0.9.73",
   "description": "Connect Claude Code, Codex, Cursor, OpenCode, or VS Code Copilot to Codiedev for team session capture and artifact collaboration — and run the governed Codie harness",
   "bin": {
     "codiedev": "dist/cli.js",