@runcofactor/cli 0.57.0 → 0.58.0

package/dist/{eval-HCDXL4QT.js → eval-37P36M53.js}

@@ -1,8 +1,18 @@
 import { createRequire as __cr } from "module"; import { fileURLToPath as __fup } from "url"; import { dirname as __dn } from "path"; const require = __cr(import.meta.url); const __filename = __fup(import.meta.url); const __dirname = __dn(__filename);
+import {
+  AGENT_EVAL_MAX_BUNDLE_BYTES,
+  AGENT_EVAL_MULTIPART_THRESHOLD,
+  AgentEvalTemplateSchema,
+  EvalConversationSchema,
+  loadBundle,
+  missingRequiredPaths,
+  uploadBundle,
+  uploadMultipart
+} from "./chunk-UPP4F2BE.js";
 import {
   resolveApplicationId
-} from "./chunk-YFXTMFYB.js";
-import "./chunk-SBQLD7X6.js";
+} from "./chunk-V6OC2WQL.js";
+import "./chunk-PN5IK7CJ.js";
 import {
   EVALS,
   getEval,
@@ -10,367 +20,33 @@ import {
   requestAll,
   requestText,
   requestTolerant
-} from "./chunk-XGGHOHZS.js";
+} from "./chunk-XD3PSGFG.js";
 import {
   printServedGuide
-} from "./chunk-YV4RASGX.js";
+} from "./chunk-W43YGZOP.js";
 import {
   getArgValues,
   getPositionalArg,
   getPositionalArgs,
   getScalarArgValue,
   readStdin
-} from "./chunk-72Y65NWI.js";
+} from "./chunk-WY4R4IL5.js";
 import "./chunk-S3Q7MDCS.js";
 import {
   GLOBAL_VALUE_FLAGS,
   hasFlag,
   resolveConfig
-} from "./chunk-IIDQM4MY.js";
+} from "./chunk-BMFD4OPC.js";
 import {
   writeRawOutput
-} from "./chunk-T6ZNIMI4.js";
-import "./chunk-EYHJ5FIA.js";
+} from "./chunk-LRIP5IQD.js";
+import "./chunk-C6SNLHVE.js";
 import "./chunk-TOWZAG3I.js";
 
 // src/eval.ts
-import { mkdir, readFile as readFile3, readdir as readdir2, writeFile } from "fs/promises";
-import { gunzipSync as gunzipSync2 } from "zlib";
-import { join as join2 } from "path";
-
-// ../types/src/agent-eval-template.ts
-import { z } from "zod";
-var AGENT_EVAL_MAX_BUNDLE_BYTES = 512 * 1024 ** 2;
-var AGENT_EVAL_MAX_TOTAL_BUNDLE_BYTES = 2 * 1024 ** 3;
-var ModelSchema = z.strictObject({
-  provider: z.enum(["anthropic", "azure"]),
-  model: z.string().min(1)
-});
-var EnvName = z.string().regex(/^[A-Z_][A-Z0-9_]*$/, "Must be an upper-case environment variable name").refine((name) => !name.startsWith("EVAL_"), "EVAL_ names are reserved for the attempt environment");
-var ProviderEnvSchema = z.strictObject({
-  env: z.strictObject({
-    apiKey: EnvName,
-    // gets the attempt's scoped proxy token
-    baseUrl: EnvName,
-    // gets the proxy URL for the provider's wire
-    model: EnvName
-    // gets the model id
-  }).refine((env) => new Set(Object.values(env)).size === 3, "apiKey, baseUrl and model must be different names")
-});
-var ScriptGrader = z.strictObject({
-  id: z.string().regex(/^[a-z0-9-]+$/),
-  kind: z.literal("script"),
-  command: z.string().min(1)
-});
-var JudgeGrader = z.strictObject({
-  id: z.string().regex(/^[a-z0-9-]+$/),
-  kind: z.literal("ai-judge"),
-  rubric: z.string().min(20),
-  scale: z.enum(["binary", "1-5"]),
-  passAt: z.number().int().min(2).max(5).optional(),
-  // required when scale is '1-5'
-  judge: ModelSchema
-});
-var AgentEvalTemplateSchema = z.strictObject({
-  version: z.literal(1),
-  name: z.string().regex(/^[a-z0-9-]{2,40}$/).refine((n) => n !== "runs" && n !== "compare"),
-  // reserved route segments
-  description: z.string().min(1).max(500),
-  proves: z.strictObject({ capabilities: z.array(z.string().regex(/^CAP-/)).min(1) }),
-  // resolved within the application given at push
-  runbook: z.string().min(1),
-  // path relative to this file
-  mode: z.enum(["single-turn", "multi-turn"]).default("single-turn"),
-  traces: z.strictObject({ capture: z.enum(["full", "metadata"]).default("full") }).default(() => ({ capture: "full" })),
-  run: z.string().min(1),
-  // shell command, cwd = repo root of the commit; multi-turn: the agent process, JSONL on stdin/stdout
-  simulator: ModelSchema.optional(),
-  // required when mode is multi-turn: pins the simulated-user model
-  input: z.strictObject({
-    description: z.string().min(1),
-    require: z.array(z.string()).default([])
-  }),
-  agent: z.strictObject({
-    models: z.record(z.string().regex(/^[a-z0-9-]+$/), ModelSchema).refine((m) => Object.keys(m).length >= 1),
-    default: z.string(),
-    // must be a key of models
-    providers: z.strictObject({ anthropic: ProviderEnvSchema.optional(), azure: ProviderEnvSchema.optional() }).optional()
-    // absent, or an absent entry: the standard names; every entry's provider must be used by a model
-  }),
-  graders: z.array(z.discriminatedUnion("kind", [ScriptGrader, JudgeGrader])).min(1),
-  repeats: z.number().int().min(1).max(7).default(1),
-  // attempts per case; a stochastic agent raises it together with passAtLeast
-  passAtLeast: z.number().int().min(1).default(1),
-  // <= repeats
-  passBar: z.number().min(0.1).max(1),
-  // minimum case pass rate for the eval to pass
-  attemptSeconds: z.number().int().min(30).max(1700)
-  // grader commands get a fixed 120 s
-}).superRefine((template, ctx) => {
-  if (!Object.hasOwn(template.agent.models, template.agent.default)) {
-    ctx.addIssue({
-      code: "custom",
-      path: ["agent", "default"],
-      message: "Default must name an agent model alias"
-    });
-  }
-  const used = new Set(Object.values(template.agent.models).map((m) => m.provider));
-  for (const [provider, entry] of Object.entries(template.agent.providers ?? {})) {
-    if (entry && !used.has(provider)) {
-      ctx.addIssue({
-        code: "custom",
-        path: ["agent", "providers", provider],
-        message: `No agent model uses provider ${provider}; remove this entry or add a model that uses it`
-      });
-    }
-  }
-  if (template.passAtLeast > template.repeats) {
-    ctx.addIssue({
-      code: "custom",
-      path: ["passAtLeast"],
-      message: "passAtLeast must not exceed repeats"
-    });
-  }
-  const ids = /* @__PURE__ */ new Set();
-  template.graders.forEach((grader, index) => {
-    if (ids.has(grader.id)) {
-      ctx.addIssue({
-        code: "custom",
-        path: ["graders", index, "id"],
-        message: "Grader ids must be unique"
-      });
-    }
-    ids.add(grader.id);
-    if (grader.kind === "ai-judge" && grader.scale === "1-5" && grader.passAt === void 0) {
-      ctx.addIssue({
-        code: "custom",
-        path: ["graders", index, "passAt"],
-        message: "A 1-5 judge requires passAt"
-      });
-    }
-  });
-  if (template.mode === "multi-turn" !== (template.simulator !== void 0)) {
-    ctx.addIssue({
-      code: "custom",
-      path: ["simulator"],
-      message: "A simulator is required exactly when mode is multi-turn"
-    });
-  }
-});
-var EvalConversationSchema = z.discriminatedUnion("kind", [
-  z.strictObject({ kind: z.literal("scripted"), turns: z.array(z.string().min(1)).min(1) }),
-  z.strictObject({
-    kind: z.literal("simulated"),
-    persona: z.string().min(1),
-    goal: z.string().min(1),
-    facts: z.array(z.string()),
-    stopWhen: z.string().min(1),
-    maxTurns: z.number().int().min(2).max(30)
-  })
-]);
-
-// src/eval-bundle.ts
-import { createHash } from "crypto";
-import { request as httpRequest } from "http";
-import { request as httpsRequest } from "https";
-import { lstat, readdir, readFile } from "fs/promises";
+import { mkdir, readFile as readFile2, readdir, writeFile } from "fs/promises";
+import { gunzipSync } from "zlib";
 import { join } from "path";
-import { gunzipSync, gzipSync } from "zlib";
-var BLOCK = 512;
-var MAX_RAW_BYTES = AGENT_EVAL_MAX_BUNDLE_BYTES;
-var MAX_UNPACKED_BYTES = 4 * AGENT_EVAL_MAX_BUNDLE_BYTES;
-var sha256Hex = (data) => createHash("sha256").update(data).digest("hex");
-function octal(value, width) {
-  return `${value.toString(8).padStart(width - 1, "0")}\0`;
-}
-function splitName(path) {
-  if (Buffer.byteLength(path) <= 100) return { prefix: "", name: path };
-  for (let i = path.indexOf("/"); i !== -1; i = path.indexOf("/", i + 1)) {
-    const prefix = path.slice(0, i);
-    const name = path.slice(i + 1);
-    if (Buffer.byteLength(prefix) <= 155 && Buffer.byteLength(name) <= 100) return { prefix, name };
-  }
-  throw new Error(`Path is too long for a bundle (limit 255 bytes): ${path}`);
-}
-function header(path, size, type, mode) {
-  const { prefix, name } = splitName(path);
-  const block = Buffer.alloc(BLOCK);
-  block.write(name, 0, "utf8");
-  block.write(octal(mode, 8), 100, "ascii");
-  block.write(octal(0, 8), 108, "ascii");
-  block.write(octal(0, 8), 116, "ascii");
-  block.write(octal(size, 12), 124, "ascii");
-  block.write(octal(0, 12), 136, "ascii");
-  block.fill(32, 148, 156);
-  block.write(type, 156, "ascii");
-  block.write("ustar\0", 257, "ascii");
-  block.write("00", 263, "ascii");
-  block.write(prefix, 345, "utf8");
-  const checksum = block.reduce((sum, byte) => sum + byte, 0);
-  block.write(`${checksum.toString(8).padStart(6, "0")}\0 `, 148, "ascii");
-  return block;
-}
-async function walk(root, real = "", archive = "") {
-  const names = (await readdir(real ? join(root, real) : root)).map((name) => ({ real: name, archive: name.normalize("NFC") })).sort((a, b) => Buffer.compare(Buffer.from(a.archive), Buffer.from(b.archive)));
-  const entries = [];
-  for (const name of names) {
-    const realPath = real ? `${real}/${name.real}` : name.real;
-    const path = archive ? `${archive}/${name.archive}` : name.archive;
-    const absolute = join(root, realPath);
-    const stats = await lstat(absolute);
-    if (stats.isDirectory()) {
-      entries.push({ path, absolute, directory: true, mode: 493, size: 0 });
-      entries.push(...await walk(root, realPath, path));
-    } else if (stats.isFile()) {
-      entries.push({
-        path,
-        absolute,
-        directory: false,
-        mode: stats.mode & 73 ? 493 : 420,
-        size: stats.size
-      });
-    } else {
-      throw new Error(
-        `Cannot bundle ${path}: only regular files and directories are supported (no symlinks or devices).`
-      );
-    }
-  }
-  return entries;
-}
-async function tarDirectory(root) {
-  const entries = await walk(root);
-  const total = entries.reduce((sum, entry) => sum + entry.size, 0);
-  if (total > MAX_RAW_BYTES) {
-    throw new Error(
-      `The input directory holds ${total} bytes; the CLI bundles at most ${MAX_RAW_BYTES}. Prune it.`
-    );
-  }
-  const parts = [];
-  for (const entry of entries) {
-    if (entry.directory) {
-      parts.push(header(`${entry.path}/`, 0, "5", entry.mode));
-      continue;
-    }
-    const content = await readFile(entry.absolute);
-    parts.push(header(entry.path, content.length, "0", entry.mode), content);
-    const padding = (BLOCK - content.length % BLOCK) % BLOCK;
-    if (padding) parts.push(Buffer.alloc(padding));
-  }
-  parts.push(Buffer.alloc(BLOCK * 2));
-  return Buffer.concat(parts);
-}
-function readString(block, start, length) {
-  const slice = block.subarray(start, start + length);
-  const end = slice.indexOf(0);
-  return slice.subarray(0, end === -1 ? length : end).toString("utf8");
-}
-function normalize(path) {
-  return path.replace(/^(\.\/)+/, "").replace(/\/+$/, "");
-}
-function listTarEntries(tar) {
-  const entries = [];
-  let longName = null;
-  let offset = 0;
-  while (offset + BLOCK <= tar.length) {
-    const block = tar.subarray(offset, offset + BLOCK);
-    if (block.every((byte) => byte === 0)) break;
-    const stored = Number.parseInt(readString(block, 148, 8).trim(), 8);
-    const actual = block.reduce((sum, byte, i) => sum + (i >= 148 && i < 156 ? 32 : byte), 0);
-    if (stored !== actual) throw new Error("not a valid tar archive (bad header checksum)");
-    const size = Number.parseInt(readString(block, 124, 12).trim() || "0", 8);
-    const type = String.fromCharCode(block[156] || 48);
-    const body = tar.subarray(offset + BLOCK, offset + BLOCK + size);
-    offset += BLOCK + Math.ceil(size / BLOCK) * BLOCK;
-    if (type === "L") {
-      longName = readString(body, 0, body.length);
-      continue;
-    }
-    if (type === "x") {
-      const record = body.toString("utf8").match(/\d+ path=([^\n]*)\n/);
-      if (record?.[1]) longName = record[1];
-      continue;
-    }
-    if (type === "g") continue;
-    const prefix = readString(block, 345, 155);
-    if (["1", "2", "3", "4", "6"].includes(type))
-      throw new Error(
-        `the archive holds a link or device entry (${readString(block, 0, 100)}); only files and directories are supported`
-      );
-    const name = longName ?? (prefix ? `${prefix}/${readString(block, 0, 100)}` : readString(block, 0, 100));
-    longName = null;
-    entries.push(normalize(name));
-  }
-  return entries;
-}
-async function loadBundle(input) {
-  let stats;
-  try {
-    stats = await lstat(input);
-  } catch {
-    throw new Error(`--input ${input} does not exist.`);
-  }
-  let tar;
-  if (stats.isDirectory()) {
-    tar = await tarDirectory(input);
-  } else if (stats.isFile() && /\.(tar\.gz|tgz)$/.test(input)) {
-    if (stats.size > AGENT_EVAL_MAX_BUNDLE_BYTES)
-      throw new Error(
-        `${input} is ${stats.size} bytes; a bundle is limited to ${AGENT_EVAL_MAX_BUNDLE_BYTES}. Prune it.`
-      );
-    const gzip = await readFile(input);
-    try {
-      tar = gunzipSync(gzip, { maxOutputLength: MAX_UNPACKED_BYTES });
-    } catch {
-      throw new Error(
-        `--input ${input} is not a valid gzip file, or unpacks to more than ${MAX_UNPACKED_BYTES} bytes.`
-      );
-    }
-  } else {
-    throw new Error("--input must be a directory or a .tar.gz file.");
-  }
-  const archive = gzipSync(tar, { level: 9 });
-  let entries;
-  try {
-    entries = listTarEntries(tar);
-  } catch (error) {
-    throw new Error(`--input ${input}: ${error instanceof Error ? error.message : String(error)}.`);
-  }
-  return {
-    sha256: sha256Hex(tar),
-    gzipSha256: sha256Hex(archive),
-    bytes: archive.length,
-    archive,
-    entries
-  };
-}
-function missingRequiredPaths(entries, required2) {
-  return required2.filter((path) => {
-    const wanted = normalize(path);
-    return !entries.some((entry) => entry === wanted || entry.startsWith(`${wanted}/`));
-  });
-}
-var uploadBundle = (url, headers, archive) => new Promise((resolve2, reject) => {
-  const target2 = new URL(url);
-  const request2 = target2.protocol === "https:" ? httpsRequest : httpRequest;
-  const req = request2(
-    target2,
-    { method: "PUT", headers: { ...headers, "content-length": String(archive.byteLength) } },
-    (response) => {
-      response.on("error", reject);
-      const status = response.statusCode;
-      if (status !== void 0 && status >= 200 && status < 300) {
-        response.on("end", resolve2);
-        response.resume();
-      } else {
-        response.resume();
-        req.destroy();
-        reject(new Error(`Uploading the bundle failed (${String(status)}).`));
-      }
-    }
-  );
-  req.on("error", reject);
-  req.end(archive);
-});
 
 // src/eval-run.ts
 import { setTimeout as sleep } from "timers/promises";
@@ -718,6 +394,9 @@ async function runEval(argv, name, apiUrl, token) {
   const repeats = intFlag(argv, "--repeats");
   const parallel = intFlag(argv, "--parallel");
   const evaluation = await getEval(apiUrl, token, name);
+  if (evaluation.definition.kind === "mcp" && models.length) {
+    throw new Error("MCP eval agents are declared in the template; remove --model.");
+  }
   const started = await request(
     apiUrl,
     token,
@@ -931,7 +610,7 @@ async function compareRuns(argv, ids, apiUrl, token) {
 }
 
 // src/eval-template.ts
-import { readFile as readFile2 } from "fs/promises";
+import { readFile } from "fs/promises";
 import { dirname, isAbsolute, relative, resolve, sep } from "path";
 import { parse } from "yaml";
 var RUNBOOK_TODO_MARKER = "<!-- cofactor:todo -->";
@@ -1132,7 +811,7 @@ function checkRunbook(runbook) {
 async function readPushPayload(file) {
   let source;
   try {
-    source = await readFile2(file, "utf8");
+    source = await readFile(file, "utf8");
   } catch {
     throw new Error(`Cannot read ${file}.`);
   }
@@ -1155,7 +834,7 @@ async function readPushPayload(file) {
     );
   let runbook;
   try {
-    runbook = await readFile2(runbookPath, "utf8");
+    runbook = await readFile(runbookPath, "utf8");
   } catch {
     throw new Error(
       `Cannot read the runbook ${runbookPath} (the template's runbook: ${parsed.data.runbook}).`
@@ -1295,7 +974,7 @@ function expectedArg(argv) {
 async function readJson(path, label) {
   let text;
   try {
-    text = path === "-" ? await readStdin() : await readFile3(path, "utf8");
+    text = path === "-" ? await readStdin() : await readFile2(path, "utf8");
   } catch (error) {
     throw new Error(
       `Cannot read ${label}: ${error instanceof Error ? error.message : String(error)}`
@@ -1326,17 +1005,17 @@ async function init(argv) {
     throw new Error("--capability must be a product-map code such as CAP-CMM-001.");
   const files = [
     {
-      path: join2(DEFAULT_TEMPLATE_DIR, `${name}.eval.yaml`),
+      path: join(DEFAULT_TEMPLATE_DIR, `${name}.eval.yaml`),
       text: initTemplate({
         name,
         application: getScalarArgValue(argv, "--application"),
         capability
       })
     },
-    { path: join2(DEFAULT_TEMPLATE_DIR, `${name}.runbook.md`), text: initRunbook(name) }
+    { path: join(DEFAULT_TEMPLATE_DIR, `${name}.runbook.md`), text: initRunbook(name) }
   ];
-  const gitignore = join2(DEFAULT_TEMPLATE_DIR, ".gitignore");
-  const exists = async (path) => readFile3(path).then(
+  const gitignore = join(DEFAULT_TEMPLATE_DIR, ".gitignore");
+  const exists = async (path) => readFile2(path).then(
     () => true,
     () => false
   );
@@ -1357,14 +1036,14 @@ Edit the run command and graders, write each runbook section and delete its <!--
   );
 }
 async function defaultTemplateFile() {
-  const found = (await readdir2(DEFAULT_TEMPLATE_DIR).catch(() => [])).filter(
+  const found = (await readdir(DEFAULT_TEMPLATE_DIR).catch(() => [])).filter(
     (f) => f.endsWith(".eval.yaml")
   );
   if (found.length !== 1)
     throw new Error(
       found.length === 0 ? `No ${DEFAULT_TEMPLATE_DIR}/*.eval.yaml found; pass the template file.` : `Several templates in ${DEFAULT_TEMPLATE_DIR}/ (${found.join(", ")}); pass the one to push.`
     );
-  return join2(DEFAULT_TEMPLATE_DIR, found[0]);
+  return join(DEFAULT_TEMPLATE_DIR, found[0]);
 }
 async function push({ argv, apiUrl, token, json }) {
   const file = getPositionalArg(argv, VALUE_FLAGS) ?? await defaultTemplateFile();
@@ -1398,6 +1077,17 @@ async function push({ argv, apiUrl, token, json }) {
 }
 function describeEval(detail) {
   const d = detail.definition;
+  if (d.kind === "mcp") {
+    return [
+      `${detail.name}  ${detail.id}`,
+      d.description,
+      `Application: ${d.application.id} @ ${d.application.commit}`,
+      `Client: ${d.client}`,
+      `Agents: ${d.agents.join(", ")}`,
+      `Cases: ${detail.cases.length}`,
+      `Attempt budget: ${d.attemptSeconds}s`
+    ].join("\n");
+  }
   return [
     `${detail.name}  ${detail.id}`,
     d.description,
@@ -1442,8 +1132,8 @@ async function downloadBundles(ctx, evaluation, dir) {
     if (!response.ok)
       throw new Error(`Downloading the bundle of case ${key} failed (${response.status}).`);
     await writeFile(
-      join2(dir, `${key}.tar`),
-      gunzipSync2(Buffer.from(await response.arrayBuffer()), {
+      join(dir, `${key}.tar`),
+      gunzipSync(Buffer.from(await response.arrayBuffer()), {
         maxOutputLength: DOWNLOAD_MAX_TAR_BYTES
       })
     );
@@ -1546,6 +1236,9 @@ async function addCase(ctx) {
     conversation = parsed.data;
   }
   const evaluation = await getEval(apiUrl, token, name);
+  if (evaluation.definition.kind === "mcp") {
+    throw new Error("Use cofactor mcp-eval add-case for MCP eval cases.");
+  }
   const multiTurn = evaluation.definition.mode === "multi-turn";
   if (multiTurn && !conversation)
     throw new Error(`${name} is multi-turn: pass --conversation <file.json>.`);
@@ -1558,7 +1251,7 @@ async function addCase(ctx) {
       `The bundle is missing required input path(s): ${missing.join(", ")} (the template's input.require).`
     );
   const evalPath = `${EVALS}/${encodeURIComponent(evaluation.id)}`;
-  const upload = await request(apiUrl, token, "POST", `${evalPath}/uploads`, {
+  const upload = bundle.bytes > AGENT_EVAL_MULTIPART_THRESHOLD ? (await uploadMultipart(apiUrl, token, evalPath, bundle), { status: "multipart" }) : await request(apiUrl, token, "POST", `${evalPath}/uploads`, {
     sha256: bundle.sha256,
     gzipSha256: bundle.gzipSha256,
     bytes: bundle.bytes
@@ -1648,4 +1341,4 @@ async function runEvalCommand(argv) {
 export {
   runEvalCommand
 };
-//# sourceMappingURL=eval-HCDXL4QT.js.map
+//# sourceMappingURL=eval-37P36M53.js.map