codiedev 0.10.0-beta.1 → 0.10.0-beta.11

package/dist/commands/evals.js

@@ -0,0 +1,187 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EVALS_HELP = void 0;
+exports.runEvals = runEvals;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const shared_1 = require("./shared");
+exports.EVALS_HELP = `
+codiedev evals — experiments and their runs, shared with the team
+
+  codiedev evals report <run.json> [--harness name]
+                                         File one variant's scores under an
+                                         experiment (created on first use)
+  codiedev evals last <subject> [--json] The latest experiment on a subject,
+                                         its runs, and the current champion
+
+  run.json:
+  {
+    "subject": "session-summaries",            // agent name or workload
+    "question": "Is Terra medium as good as GLM high?",
+    "experimentId": "…",                       // optional, instead of subject + question
+    "dataset": "30 Signal & Code sessions",    // optional
+    "run": {
+      "label": "terra-med-v2", "model": "openai/gpt-6.1-terra",
+      "provider": "openai", "reasoningEffort": "medium", "prompt": "summary_extract_v2",
+      "gitSha": "…", "isControl": false, "judgeModel": "anthropic/claude-sonnet-5.5",
+      "casesTotal": 30, "casesScored": 30, "errors": 0,
+      "scoringRevision": 1,                   // required once experiment scoring rules are defined
+      "score": 3.85, "metrics": { "fidelity": 3.6 },
+      "costPerCaseUsd": 0.026, "latencyMsAvg": 21000,
+      "wins": 20, "losses": 8, "ties": 2, "note": "…",
+      "cases": [{ "key": "case-1", "score": 4, "note": "…" }]
+    }
+  }
+
+  Read scoring rules and their current revision with: codiedev evals last <subject> --json
+
+  Scores and notes only: never put prompts, transcripts or outputs in it.
+`.trim();
+function flag(args, i, name) {
+    const a = args[i];
+    if (a === name && i + 1 < args.length)
+        return { value: args[i + 1], skip: 1 };
+    if (a.startsWith(`${name}=`))
+        return { value: a.slice(name.length + 1), skip: 0 };
+    return { skip: -1 };
+}
+function variant(run) {
+    return [run.model, run.reasoningEffort, run.prompt].filter(Boolean).join(" · ");
+}
+function fmtScore(n) {
+    return n === null ? "–" : n.toFixed(2);
+}
+function fmtCost(n) {
+    return n === null ? "–" : `$${n.toFixed(4)}`;
+}
+async function runReport(args, config) {
+    let file;
+    let harness;
+    for (let i = 0; i < args.length; i++) {
+        const f = flag(args, i, "--harness");
+        if (f.skip >= 0) {
+            harness = f.value;
+            i += f.skip;
+            continue;
+        }
+        if (!args[i].startsWith("-") && !file)
+            file = args[i];
+    }
+    if (!file) {
+        console.error("Usage: codiedev evals report <run.json> [--harness name]");
+        process.exit(1);
+    }
+    let body;
+    try {
+        body = JSON.parse(fs.readFileSync(path.resolve(process.cwd(), file), "utf8"));
+    }
+    catch (err) {
+        console.error(`Couldn't read ${file}: ${err.message}`);
+        process.exit(1);
+    }
+    if (!body.subject || !body.question || !body.run?.label || !body.run?.model || typeof body.run?.casesTotal !== "number") {
+        console.error(`${file} needs subject, question, and run.label, run.model, run.casesTotal. See: codiedev evals help`);
+        process.exit(1);
+    }
+    const start = Date.now();
+    try {
+        const res = await (0, shared_1.apiRequest)("POST", "/api/cli/evals/runs", {
+            config,
+            body: { ...body, run: { ...body.run, harness: harness ?? body.run.harness ?? undefined } },
+        });
+        (0, shared_1.recordAgentEvent)(config, { tool: "codiedev_evals_report", ok: true, latencyMs: Date.now() - start });
+        console.log(`✓ Reported ${body.run.label} on ${body.subject}${res.created ? " (new experiment)" : ""}: "${body.question}"`);
+        console.log(`  experiment ${res.experimentId}. See it in Codiedev → Traces → Evals.`);
+    }
+    catch (err) {
+        const e = err;
+        (0, shared_1.recordAgentEvent)(config, { tool: "codiedev_evals_report", ok: false, latencyMs: Date.now() - start, error: e.message });
+        console.error(`Couldn't report: ${e.message}`);
+        process.exit(1);
+    }
+}
+async function runLast(args, config) {
+    const json = args.includes("--json");
+    const subject = args.find((a) => !a.startsWith("-"));
+    if (!subject) {
+        console.error("Usage: codiedev evals last <subject> [--json]");
+        process.exit(1);
+    }
+    let res;
+    try {
+        res = await (0, shared_1.apiRequest)("GET", `/api/cli/evals/last?subject=${encodeURIComponent(subject)}`, { config });
+    }
+    catch (err) {
+        const e = err;
+        console.error(e.status === 404 ? `No evals on ${subject} yet.` : `Couldn't load evals: ${e.message}`);
+        process.exit(1);
+    }
+    if (json) {
+        console.log(JSON.stringify(res, null, 2));
+        return;
+    }
+    const { experiment, runs, champion } = res;
+    if (champion) {
+        console.log(`Champion: ${champion.run.label} (${variant(champion.run)}), score ${fmtScore(champion.run.score)}, ${fmtCost(champion.run.costPerCaseUsd)}/case`);
+        if (champion.decisionNote)
+            console.log(`  Why: ${champion.decisionNote}`);
+    }
+    else {
+        console.log("Champion: none picked yet");
+    }
+    console.log("");
+    console.log(`Latest experiment (${res.experiments} on ${experiment.subject}): ${experiment.question}`);
+    if (experiment.dataset)
+        console.log(`  Dataset: ${experiment.dataset}`);
+    console.log(`  id ${experiment.id}, by ${experiment.createdBy}, updated ${new Date(experiment.updatedAt).toISOString().slice(0, 16).replace("T", " ")}`);
+    console.log("");
+    for (const run of [...runs].sort((a, b) => a.ranAt - b.ranAt)) {
+        const h2h = run.wins !== null ? ` · ${run.wins}W/${run.losses ?? 0}L/${run.ties ?? 0}T` : "";
+        console.log(`  ${run.isControl ? "[control] " : ""}${run.label.padEnd(24)} ${fmtScore(run.score).padStart(5)}  ${fmtCost(run.costPerCaseUsd).padStart(8)}/case${h2h}  ${variant(run)}`);
+    }
+    console.log("");
+    console.log(`Next run on this experiment: put "experimentId": "${experiment.id}" in run.json, then codiedev evals report run.json`);
+}
+async function runEvals(args, configOverride) {
+    const [sub, ...rest] = args;
+    if (sub === "report")
+        return runReport(rest, configOverride ?? (0, shared_1.requireConfig)());
+    if (sub === "last")
+        return runLast(rest, configOverride ?? (0, shared_1.requireConfig)());
+    console.log(exports.EVALS_HELP);
+    if (sub && sub !== "help")
+        process.exit(1);
+}

package/dist/commands/inbox.js

@@ -70,8 +70,6 @@ async function runRead(args) {
     const config = (0, shared_1.requireConfig)();
     const start = Date.now();
     try {
-        // First, fetch inbox so we can show the full body (markRead endpoint
-        // doesn't return content).
         const listRes = await (0, shared_1.apiRequest)("GET", "/api/cli/inbox", {
             config,
             query: { limit: 100 },

package/dist/commands/library.js

@@ -1,7 +1,4 @@
 "use strict";
-// `codiedev library [--scope mine|shared|all] [--kind doc|skill] [--tag X]
-// [--type spec|...] [--limit N]` — list artifacts in your Codiedev workspace.
-// Server applies companyId scoping; --scope filters within that.
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.runLibrary = runLibrary;
 const shared_1 = require("./shared");