@garuda-agent/garuda 0.16.1 → 0.18.0

package/dist/chunk-FOPBOB7W.js

@@ -0,0 +1,332 @@
+#!/usr/bin/env node
+import {
+  compareReports,
+  costPerPass,
+  formatComparison,
+  taskHash
+} from "./chunk-DOCDCQVM.js";
+
+// src/evals/tune.ts
+import { readFile } from "fs/promises";
+var ADDENDUM_MAX_CHARS = 4e3;
+var CHECK_TAIL_LINES = 40;
+var REJECTED = [
+  /\b(skip|disable|ignore|bypass|weaken)\b[^.\n]{0,40}\b(tests?|checks?|assert\w*|sandbox|protect\w*)/i,
+  /\b(edit|change|modify|delete|remove|rewrite)\b[^.\n]{0,40}\b(the )?(tests?|test files?|checks?|protected files?)\b/i,
+  /\boutside_sandbox\b/i,
+  /\b(hard-?code|special-?case)\b[^.\n]{0,40}\b(expected|test|check)/i,
+  /\.check\//i
+];
+async function readAddendum(path, missingIsEmpty = false) {
+  let text;
+  try {
+    text = await readFile(path, "utf8");
+  } catch (error) {
+    if (missingIsEmpty && error.code === "ENOENT") return { text: "" };
+    return { problem: `--addendum ${path}: ${error.message}` };
+  }
+  if (text.length > ADDENDUM_MAX_CHARS) {
+    return {
+      problem: `--addendum ${path}: ${text.length} characters; the limit is ${ADDENDUM_MAX_CHARS}.`
+    };
+  }
+  return { text };
+}
+function rejectReason(text) {
+  if (text.trim() === "") return "the addendum is empty";
+  if (text.length > ADDENDUM_MAX_CHARS) {
+    return `the addendum has ${text.length} characters; the limit is ${ADDENDUM_MAX_CHARS}`;
+  }
+  for (const clause of text.split(/[.;!?](?=\s|$)|\n/)) {
+    for (const pattern of REJECTED) {
+      const hit = pattern.exec(clause);
+      if (hit === null || NEGATION.test(clause.slice(0, hit.index + hit[0].length))) continue;
+      return `the addendum asks to get around the grading ("${hit[0]}")`;
+    }
+  }
+  return void 0;
+}
+var NEGATION = /\b(never|not|don't|do not|no|without)\b/i;
+function groupRuns(results) {
+  const out = /* @__PURE__ */ new Map();
+  for (const r of results) out.set(r.id, [...out.get(r.id) ?? [], r]);
+  return out;
+}
+var asReport = (path, results, tasks) => ({
+  path,
+  schema: 2,
+  model: void 0,
+  seed: void 0,
+  tasks: tasks === void 0 ? void 0 : Object.fromEntries(tasks.map((t) => [t.id, taskHash(t)])),
+  results: [...results]
+});
+function decide(current, candidate, options) {
+  const passesCurrent = current.filter((r) => r.passed).length;
+  const passesCandidate = candidate.filter((r) => r.passed).length;
+  const runs = candidate.length;
+  const base = { passesCurrent, passesCandidate, runs };
+  const before = groupRuns(current);
+  const after = groupRuns(candidate);
+  const lost = [...before.entries()].filter(([, rs]) => rs.length > 0 && rs.every((r) => r.passed)).filter(([id]) => {
+    const now = after.get(id) ?? [];
+    return now.length > 0 && now.every((r) => !r.passed);
+  }).map(([id]) => id);
+  if (lost.length > 0) {
+    return {
+      ...base,
+      kept: false,
+      reason: `dropped: ${lost.join(", ")} went from all runs passed to none`
+    };
+  }
+  const compared = compareReports(asReport("current", current), asReport("candidate", candidate), {
+    seed: options.seed
+  });
+  const ratio = "problems" in compared || compared.comparison.costRatio === void 0 ? void 0 : compared.comparison.costRatio;
+  const costRatio = ratio === void 0 ? void 0 : { value: ratio.value, low: ratio.low, high: ratio.high };
+  const withCost = costRatio === void 0 ? base : { ...base, costRatio };
+  const gain = passesCandidate - passesCurrent;
+  if (gain >= options.margin) {
+    return {
+      ...withCost,
+      kept: true,
+      reason: `kept: ${gain} more passing run(s) (margin ${options.margin})`
+    };
+  }
+  if (gain >= 0 && costRatio !== void 0 && costRatio.high < 1) {
+    return {
+      ...withCost,
+      kept: true,
+      reason: `kept: as many passes or more, and cost per accepted change \xD7${costRatio.value.toFixed(2)} (95% interval \xD7${costRatio.low.toFixed(2)} to \xD7${costRatio.high.toFixed(2)})`
+    };
+  }
+  const cost = costRatio === void 0 ? "cost ratio unknown" : `cost \xD7${costRatio.value.toFixed(2)} (\xD7${costRatio.low.toFixed(2)} to \xD7${costRatio.high.toFixed(2)})`;
+  return {
+    ...withCost,
+    kept: false,
+    reason: `dropped: ${gain >= 0 ? "+" : ""}${gain} passing run(s) (margin ${options.margin}), ${cost}`
+  };
+}
+function proposerInput(addendum, tasks, results, history) {
+  const prompts = new Map(tasks.map((t) => [t.id, t.prompt]));
+  const lines = [
+    "# The current addendum",
+    "",
+    addendum.trim() === "" ? "(empty)" : addendum.trim(),
+    ""
+  ];
+  const failed = results.filter((r) => !r.passed);
+  if (failed.length > 0) {
+    lines.push(`# Failed runs (${failed.length} of ${results.length})`, "");
+    for (const r of failed) {
+      const tail = (r.reason ?? "").split("\n").slice(-CHECK_TAIL_LINES).join("\n");
+      lines.push(
+        `## ${r.id} (run ${r.attempt ?? 1}): ${r.stopReason}, ${r.steps} steps`,
+        "",
+        "Task prompt:",
+        prompts.get(r.id) ?? "(unknown)",
+        "",
+        "Why it failed (last lines):",
+        "```",
+        tail,
+        "```",
+        ...r.unexpected !== void 0 && r.unexpected.length > 0 ? ["", `Unexpected changes: ${r.unexpected.join(", ")}`] : [],
+        ""
+      );
+    }
+  } else {
+    lines.push(`# Every run passed (${results.length}). Lower the cost: what each task took`, "");
+    for (const [id, rs] of groupRuns(results)) {
+      const mean = (f) => rs.reduce((s, r) => s + f(r), 0) / rs.length;
+      const costliest = [...rs].sort((a, b) => (b.costUsd ?? 0) - (a.costUsd ?? 0))[0];
+      lines.push(
+        `- ${id}: ${mean((r) => r.steps).toFixed(1)} steps, ${Math.round(mean((r) => r.tokens))} tokens, $${mean((r) => r.costUsd ?? 0).toFixed(4)} a run; tools of the costliest run: ${(costliest?.tools ?? []).join(", ") || "(none recorded)"}`
+      );
+    }
+    lines.push("");
+  }
+  if (history.length > 0) {
+    lines.push("# Earlier proposals", "");
+    for (const h of history) {
+      lines.push(
+        `- round ${h.round} (${h.version}): ${h.rejected === void 0 ? h.decision?.reason ?? "" : `rejected before a run: ${h.rejected}`}. The proposer's reason: ${h.reason}`
+      );
+    }
+    lines.push("");
+  }
+  return lines.join("\n");
+}
+var PROPOSER_SYSTEM = [
+  "You improve a coding agent. The agent gets a short text, the addendum, at the end of its system",
+  "prompt. You see the current addendum and the agent's results on development tasks. Write a",
+  "complete new addendum that should make the agent pass more tasks, or pass them at a lower cost.",
+  "",
+  "Rules:",
+  `- At most ${ADDENDUM_MAX_CHARS} characters. General advice about how to work, not about one task:`,
+  "  the addendum is measured later on tasks you never see.",
+  "- Never tell the agent to skip, weaken or edit tests or checks, to touch protected files, or to",
+  "  leave the sandbox. Such a text is refused without a run.",
+  "",
+  "Answer in exactly this form:",
+  "<addendum>",
+  "the complete new addendum",
+  "</addendum>",
+  "<reason>one paragraph: what you changed and why</reason>"
+].join("\n");
+function parseProposal(text) {
+  const addendum = /<addendum>\n?([\s\S]*?)\n?<\/addendum>/.exec(text)?.[1];
+  const reason = /<reason>([\s\S]*?)<\/reason>/.exec(text)?.[1];
+  if (addendum === void 0) return void 0;
+  return { addendum: addendum.trim(), reason: (reason ?? "").trim() };
+}
+var costOf = (results) => {
+  const c = costPerPass(results);
+  if (c.kind === "unknown") return void 0;
+  return results.reduce((s, r) => s + (r.costUsd ?? 0), 0);
+};
+async function tune(options) {
+  const history = [];
+  let spent = 0;
+  let current = options.start;
+  let version = 0;
+  await options.save("v0.md", current);
+  options.log(`Scoring v0 on ${options.devTasks.length} dev task(s) \xD7 ${options.repeat}.`);
+  let currentResults = await options.score(current, options.devTasks, options.repeat, "v0");
+  let lastScoreCost = costOf(currentResults);
+  if (lastScoreCost === void 0) {
+    return { final: current, history, spentUsd: spent, stopped: "unknown cost" };
+  }
+  spent += lastScoreCost;
+  let stopped = "rounds";
+  let lastProposalCost = 0;
+  for (let round = 1; round <= options.rounds; round++) {
+    if (spent + lastScoreCost + lastProposalCost > options.maxCostUsd) {
+      stopped = "cost";
+      options.log(
+        `Stopping before round ${round}: $${spent.toFixed(2)} spent, and the next round would pass --max-cost $${options.maxCostUsd}.`
+      );
+      break;
+    }
+    const input = proposerInput(current, options.devTasks, currentResults, history);
+    const name = `v${round}`;
+    await options.save(`${name}.input.md`, input);
+    let proposal = await options.propose(input);
+    if (proposal.costUsd === void 0) {
+      stopped = "unknown cost";
+      break;
+    }
+    let proposalCost = proposal.costUsd;
+    let rejected = rejectReason(proposal.addendum);
+    let refusedFirst;
+    if (rejected !== void 0) {
+      refusedFirst = rejected;
+      await options.save(`${name}.refused.md`, proposal.addendum);
+      options.log(`Round ${round}: ${name} refused (${rejected}); asking once more.`);
+      const again = await options.propose(
+        `${input}
+# Your last answer was refused
+
+${rejected}. Write the complete addendum again without that phrase. Do not tell the agent to skip, change or delete tests or checks, even as an example of what to avoid.
+`
+      );
+      if (again.costUsd === void 0) {
+        stopped = "unknown cost";
+        break;
+      }
+      proposalCost += again.costUsd;
+      proposal = again;
+      rejected = rejectReason(proposal.addendum);
+    }
+    spent += proposalCost;
+    lastProposalCost = proposalCost;
+    await options.save(`${name}.md`, proposal.addendum);
+    if (rejected !== void 0) {
+      history.push({
+        round,
+        version: name,
+        reason: proposal.reason,
+        rejected,
+        costUsd: proposalCost
+      });
+      options.log(`Round ${round}: ${name} rejected before a run: ${rejected}.`);
+      continue;
+    }
+    options.log(`Round ${round}: scoring ${name}.`);
+    const candidate = await options.score(
+      proposal.addendum,
+      options.devTasks,
+      options.repeat,
+      name
+    );
+    const scoreCost = costOf(candidate);
+    if (scoreCost === void 0) {
+      stopped = "unknown cost";
+      break;
+    }
+    spent += scoreCost;
+    lastScoreCost = scoreCost;
+    const decision = decide(currentResults, candidate, {
+      margin: options.margin,
+      seed: options.seed
+    });
+    history.push({
+      round,
+      version: name,
+      reason: proposal.reason,
+      decision,
+      costUsd: proposalCost + scoreCost,
+      ...refusedFirst === void 0 ? {} : { refusedFirst }
+    });
+    options.log(`Round ${round}: ${name} ${decision.reason}.`);
+    if (decision.kept) {
+      current = proposal.addendum;
+      currentResults = candidate;
+      version = round;
+    }
+    await options.save("history.json", `${JSON.stringify(history, null, 2)}
+`);
+  }
+  await options.save("history.json", `${JSON.stringify(history, null, 2)}
+`);
+  await options.save("final.md", current);
+  let heldOut;
+  if (options.heldOutTasks.length > 0 && version > 0) {
+    options.log(
+      `Held-out: the empty addendum (A) against v${version} (B), \xD7 ${options.heldoutRepeat}.`
+    );
+    const a = await options.score("", options.heldOutTasks, options.heldoutRepeat, "heldout-empty");
+    const b = await options.score(
+      current,
+      options.heldOutTasks,
+      options.heldoutRepeat,
+      "heldout-final"
+    );
+    spent += (costOf(a) ?? 0) + (costOf(b) ?? 0);
+    const ra = asReport("held-out, empty addendum", a, options.heldOutTasks);
+    const rb = asReport(`held-out, v${version}`, b, options.heldOutTasks);
+    const compared = compareReports(ra, rb, { seed: options.seed });
+    heldOut = "problems" in compared ? compared.problems.join("\n") : formatComparison(ra, rb, compared.comparison);
+    await options.save("heldout.txt", `${heldOut}
+`);
+  } else if (version === 0) {
+    options.log("No version was kept: no held-out measurement.");
+  }
+  return {
+    final: current,
+    history,
+    spentUsd: spent,
+    stopped,
+    ...heldOut === void 0 ? {} : { heldOut }
+  };
+}
+
+export {
+  ADDENDUM_MAX_CHARS,
+  readAddendum,
+  rejectReason,
+  decide,
+  proposerInput,
+  PROPOSER_SYSTEM,
+  parseProposal,
+  tune
+};
+//# sourceMappingURL=chunk-FOPBOB7W.js.map