@garuda-agent/garuda 0.18.0 → 0.18.2

package/dist/chunk-DOCDCQVM.js

@@ -1,484 +0,0 @@
-#!/usr/bin/env node
-import {
-  VERSION
-} from "./chunk-C57GHLKP.js";
-import {
-  git
-} from "./chunk-6XHON3Q6.js";
-import {
-  HostExecutor
-} from "./chunk-B4SJY7KT.js";
-
-// src/buildInfo.ts
-var BUILD_COMMIT = true ? "d30d3d872ea1fd3a455eb8d6461d78efda70ba5b" : void 0;
-
-// src/evals/stats.ts
-var BOOTSTRAP_DRAWS = 1e4;
-var DEFAULT_SEED = 1;
-function seeded(seed) {
-  let a = seed >>> 0;
-  return () => {
-    a = a + 1831565813 >>> 0;
-    let t = a;
-    t = Math.imul(t ^ t >>> 15, t | 1);
-    t ^= t + Math.imul(t ^ t >>> 7, t | 61);
-    return ((t ^ t >>> 14) >>> 0) / 4294967296;
-  };
-}
-function shuffled(items, seed) {
-  const out = [...items];
-  const random = seeded(seed);
-  for (let i = out.length - 1; i > 0; i--) {
-    const j = Math.floor(random() * (i + 1));
-    [out[i], out[j]] = [out[j], out[i]];
-  }
-  return out;
-}
-function byTask(results) {
-  const map = /* @__PURE__ */ new Map();
-  for (const r of results) {
-    let t = map.get(r.id);
-    if (t === void 0) {
-      t = { id: r.id, runs: 0, passes: 0, errors: 0, results: [] };
-      map.set(r.id, t);
-    }
-    t.runs++;
-    if (r.passed) t.passes++;
-    if (r.stopReason === "error") t.errors++;
-    t.results.push(r);
-  }
-  return [...map.values()];
-}
-function passInterval(results, options = {}) {
-  const tasks = byTask(results);
-  const runs = tasks.reduce((s, t) => s + t.runs, 0);
-  if (tasks.length === 0 || runs === 0) return void 0;
-  const seed = options.seed ?? DEFAULT_SEED;
-  const draws = options.draws ?? BOOTSTRAP_DRAWS;
-  const level = options.level ?? 0.95;
-  const random = seeded(seed);
-  const rates = new Float64Array(draws);
-  for (let d = 0; d < draws; d++) {
-    let passes = 0;
-    let n = 0;
-    for (let i = 0; i < tasks.length; i++) {
-      const t = tasks[Math.floor(random() * tasks.length)];
-      passes += t.passes;
-      n += t.runs;
-    }
-    rates[d] = passes / n;
-  }
-  rates.sort();
-  const tail = (1 - level) / 2;
-  return {
-    rate: tasks.reduce((s, t) => s + t.passes, 0) / runs,
-    low: quantile(rates, tail),
-    high: quantile(rates, 1 - tail),
-    level,
-    draws,
-    seed,
-    tasks: tasks.length
-  };
-}
-function consistency(results) {
-  const repeated = byTask(results).filter((t) => t.runs >= 2);
-  return {
-    repeated: repeated.length,
-    allPassed: repeated.filter((t) => t.passes === t.runs).map((t) => t.id),
-    nonePassed: repeated.filter((t) => t.passes === 0 && t.errors < t.runs).map((t) => t.id),
-    onlyErrors: repeated.filter((t) => t.errors === t.runs).map((t) => t.id)
-  };
-}
-function spread(results) {
-  const scored = results.filter((r) => r.stopReason !== "error");
-  if (scored.length === 0) return void 0;
-  const of = (f) => {
-    const sorted = Float64Array.from(scored.map(f)).sort();
-    return { median: quantile(sorted, 0.5), p95: quantile(sorted, 0.95) };
-  };
-  return {
-    runs: scored.length,
-    steps: of((r) => r.steps),
-    tokens: of((r) => r.tokens),
-    durationMs: of((r) => r.durationMs)
-  };
-}
-function stepsSd(runs) {
-  const steps = runs.filter((r) => r.stopReason !== "error").map((r) => r.steps);
-  if (steps.length < 2) return void 0;
-  const mean = steps.reduce((s, x) => s + x, 0) / steps.length;
-  return Math.sqrt(steps.reduce((s, x) => s + (x - mean) ** 2, 0) / (steps.length - 1));
-}
-function costPerPass(results) {
-  if (results.some((r) => r.costUsd === void 0)) return { kind: "unknown" };
-  const totalUsd = results.reduce((s, r) => s + (r.costUsd ?? 0), 0);
-  const passes = results.filter((r) => r.passed).length;
-  if (passes === 0) return { kind: "none", totalUsd };
-  return {
-    kind: "value",
-    usd: totalUsd / passes,
-    minimum: results.some((r) => r.costIncomplete === true),
-    totalUsd,
-    passes
-  };
-}
-function unexpectedChanges(results) {
-  const measured = results.filter((r) => r.unexpected !== void 0);
-  return {
-    runs: measured.filter((r) => (r.unexpected?.length ?? 0) > 0).length,
-    files: measured.reduce((s, r) => s + (r.unexpected?.length ?? 0), 0),
-    measured: measured.length
-  };
-}
-function quantile(sorted, q) {
-  if (sorted.length === 0) return Number.NaN;
-  const pos = (sorted.length - 1) * q;
-  const lo = Math.floor(pos);
-  const hi = Math.ceil(pos);
-  const a = sorted[lo];
-  const b = sorted[hi];
-  return a + (b - a) * (pos - lo);
-}
-function mcpUse(results) {
-  const measured = results.filter((r) => r.mcpCalls !== void 0);
-  if (measured.length === 0) return void 0;
-  return {
-    calls: measured.reduce((sum, r) => sum + (r.mcpCalls ?? 0), 0),
-    runs: measured.filter((r) => (r.mcpCalls ?? 0) > 0).length,
-    measured: measured.length
-  };
-}
-function mcpUseText(use) {
-  return `${use.calls} in ${use.runs} of ${use.measured} runs${use.runs === 0 ? " (the agent never used the server)" : ""}`;
-}
-
-// src/evals/compare.ts
-import { readFile, stat } from "fs/promises";
-import { join } from "path";
-async function loadReport(path) {
-  const file = (await stat(path)).isDirectory() ? join(path, "report.json") : path;
-  const data = JSON.parse(await readFile(file, "utf8"));
-  return parseReport(data, file);
-}
-function parseReport(data, path) {
-  if (!Array.isArray(data.results)) throw new Error(`${path} is not an eval report (no results).`);
-  const schema = typeof data.schema === "number" ? data.schema : 1;
-  const tasks = schema >= 2 && data.tasks !== null && typeof data.tasks === "object" ? data.tasks : void 0;
-  return {
-    path,
-    schema,
-    model: typeof data.spec === "string" ? data.spec : typeof data.model === "string" ? data.model : void 0,
-    seed: typeof data.seed === "number" ? data.seed : void 0,
-    tasks,
-    results: data.results,
-    mcp: mcpServerOf(data.mcp)
-  };
-}
-function mcpServerOf(value) {
-  if (value === null || typeof value !== "object") return void 0;
-  const server = value.server;
-  return typeof server === "string" ? server : void 0;
-}
-function compareReports(a, b, options = {}) {
-  const problems = [];
-  const notes = [];
-  const idsA = [...new Set(a.results.map((r) => r.id))].sort();
-  const idsB = [...new Set(b.results.map((r) => r.id))].sort();
-  if (idsA.join("\n") !== idsB.join("\n")) {
-    const onlyA = idsA.filter((id) => !idsB.includes(id));
-    const onlyB = idsB.filter((id) => !idsA.includes(id));
-    problems.push(
-      `The reports ran different tasks${onlyA.length > 0 ? `; only in A: ${onlyA.join(", ")}` : ""}${onlyB.length > 0 ? `; only in B: ${onlyB.join(", ")}` : ""}.`
-    );
-  }
-  if (a.tasks !== void 0 && b.tasks !== void 0) {
-    const changed = idsA.filter((id) => idsB.includes(id) && a.tasks?.[id] !== b.tasks?.[id]);
-    if (changed.length > 0) {
-      problems.push(
-        `These tasks changed between the runs (their hashes differ): ${changed.join(", ")}. Run both arms on the same tasks.`
-      );
-    }
-  } else {
-    const old = [a, b].filter((r) => r.tasks === void 0).map((r) => r.path);
-    notes.push(
-      `${old.join(" and ")} ${old.length === 1 ? "has" : "have"} no task hashes (a report before 0.17): Garuda cannot check that the tasks were the same.`
-    );
-  }
-  if (problems.length > 0) return { problems };
-  if (idsA.length === 0) return { problems: ["The reports have no runs."] };
-  const tasksA = new Map(byTask(a.results).map((t) => [t.id, t]));
-  const tasksB = new Map(byTask(b.results).map((t) => [t.id, t]));
-  const rows = idsA.map((id) => {
-    const ta = tasksA.get(id);
-    const tb = tasksB.get(id);
-    const arm = (t) => {
-      const results = t?.results ?? [];
-      return {
-        passes: t?.passes ?? 0,
-        runs: t?.runs ?? 0,
-        costUsd: results.some((r) => r.costUsd === void 0) ? void 0 : results.reduce((s, r) => s + (r.costUsd ?? 0), 0),
-        minimum: results.some((r) => r.costIncomplete === true)
-      };
-    };
-    return { id, a: arm(ta), b: arm(tb) };
-  });
-  const rate = (side, list) => list.reduce((s, r) => s + r[side].passes, 0) / list.reduce((s, r) => s + r[side].runs, 0);
-  const rateA = rate("a", rows);
-  const rateB = rate("b", rows);
-  const seed = options.seed ?? DEFAULT_SEED;
-  const draws = options.draws ?? BOOTSTRAP_DRAWS;
-  const level = options.level ?? 0.95;
-  const random = seeded(seed);
-  const diffs = new Float64Array(draws);
-  for (let d = 0; d < draws; d++) {
-    let pa = 0;
-    let na = 0;
-    let pb = 0;
-    let nb = 0;
-    for (let i = 0; i < rows.length; i++) {
-      const r = rows[Math.floor(random() * rows.length)];
-      pa += r.a.passes;
-      na += r.a.runs;
-      pb += r.b.passes;
-      nb += r.b.runs;
-    }
-    diffs[d] = pb / nb - pa / na;
-  }
-  diffs.sort();
-  const tail = (1 - level) / 2;
-  const low = quantile(diffs, tail);
-  const high = quantile(diffs, 1 - tail);
-  const verdict = low > 0 ? "B passes more" : high < 0 ? "B passes less" : "no clear difference";
-  if (low === high) {
-    notes.push(
-      "Every draw gave the same pass-rate difference, so the interval shows no uncertainty: more tasks, or tasks that sometimes fail, are needed to measure it."
-    );
-  }
-  if (rows.length < 10) notes.push(roughNote(rows.length));
-  if (a.seed !== void 0 && b.seed !== void 0 && a.seed !== b.seed) {
-    notes.push(
-      `The arms ran with different seeds (${a.seed} and ${b.seed}): their run orders differ.`
-    );
-  }
-  if (a.mcp !== b.mcp) {
-    notes.push(
-      `The arms differ in the MCP server: A ${a.mcp ?? "none"}, B ${b.mcp ?? "none"}. For an MCP A/B, keep every other option the same.`
-    );
-  }
-  const cost = costRatio(rows, seed, draws, level);
-  if (typeof cost === "string") notes.push(cost);
-  else if (cost.skipped > draws * 0.05) {
-    notes.push(
-      `The cost interval skipped ${cost.skipped} of ${draws} draws (an arm had no pass in them).`
-    );
-  }
-  if (rows.some((r) => r.a.minimum || r.b.minimum)) {
-    notes.push("A run's cost is a minimum (no reported usage): the cost ratio is approximate.");
-  }
-  const runsA = rows.map((r) => r.a.runs);
-  const runsB = rows.map((r) => r.b.runs);
-  if (Math.min(...runsA, ...runsB) < 3) {
-    notes.push(
-      "Some tasks have fewer than 3 runs in an arm: repeat each task 3 times or more (--repeat 3)."
-    );
-  }
-  return {
-    comparison: {
-      rows,
-      rateA,
-      rateB,
-      difference: { value: rateB - rateA, low, high, level, draws, seed },
-      verdict,
-      ...typeof cost === "string" ? {} : { costRatio: cost },
-      notes
-    }
-  };
-}
-function costRatio(rows, seed, draws, level) {
-  if (rows.some((r) => r.a.costUsd === void 0 || r.b.costUsd === void 0)) {
-    return "A run's cost is unknown: no cost comparison.";
-  }
-  const per = (list, side) => {
-    const passes = list.reduce((s, r) => s + r[side].passes, 0);
-    const cost = list.reduce((s, r) => s + (r[side].costUsd ?? 0), 0);
-    return passes === 0 ? void 0 : cost / passes;
-  };
-  const a = per(rows, "a");
-  const b = per(rows, "b");
-  if (a === void 0 || b === void 0 || a === 0) {
-    return "An arm has no pass (or no cost): no cost ratio.";
-  }
-  const random = seeded(seed);
-  const ratios = [];
-  const drawn = new Array(rows.length);
-  for (let d = 0; d < draws; d++) {
-    for (let i = 0; i < rows.length; i++) {
-      drawn[i] = rows[Math.floor(random() * rows.length)];
-    }
-    const ca = per(drawn, "a");
-    const cb = per(drawn, "b");
-    if (ca !== void 0 && cb !== void 0 && ca > 0) ratios.push(cb / ca);
-  }
-  const sorted = Float64Array.from(ratios).sort();
-  const tail = (1 - level) / 2;
-  const low = quantile(sorted, tail);
-  const high = quantile(sorted, 1 - tail);
-  return {
-    value: b / a,
-    low,
-    high,
-    draws,
-    skipped: draws - ratios.length,
-    verdict: low > 1 ? "B costs more per accepted change" : high < 1 ? "B costs less per accepted change" : "no clear difference"
-  };
-}
-function roughNote(tasks) {
-  return `Only ${tasks} task${tasks === 1 ? "" : "s"}: with fewer than 10 tasks the interval is a rough guide.`;
-}
-var pct = (x) => `${Math.round(x * 100)}%`;
-var signed = (x) => {
-  const points = Math.round(Math.abs(x) * 100);
-  return points === 0 ? "0 pts" : `${x > 0 ? "+" : "\u2212"}${points} pts`;
-};
-function formatComparison(a, b, c) {
-  const width = Math.max(4, ...c.rows.map((r) => r.id.length));
-  const lines = [
-    `A: ${a.path}${a.model === void 0 ? "" : ` (${a.model})`}`,
-    `B: ${b.path}${b.model === void 0 ? "" : ` (${b.model})`}`,
-    "",
-    `${"Task".padEnd(width)}  A        B`,
-    ...c.rows.map(
-      (r) => `${r.id.padEnd(width)}  ${`${r.a.passes}/${r.a.runs}`.padEnd(7)}  ${r.b.passes}/${r.b.runs}`
-    ),
-    "",
-    `Pass rate over all attempts: A ${pct(c.rateA)}, B ${pct(c.rateB)}; B \u2212 A ${signed(c.difference.value)} (95% paired interval ${signed(c.difference.low)} to ${signed(c.difference.high)}; ${c.difference.draws} draws, seed ${c.difference.seed})`,
-    `Verdict: ${c.verdict}${c.verdict === "no clear difference" ? " (the interval contains 0)" : ""}.`
-  ];
-  for (const [name, report] of [
-    ["A", a],
-    ["B", b]
-  ]) {
-    const cost = costPerPass(report.results);
-    const u = unexpectedChanges(report.results);
-    const s = spread(report.results);
-    const mcp = mcpUse(report.results);
-    lines.push(
-      `${name}: cost per accepted change ${cost.kind === "unknown" ? "unknown" : cost.kind === "none" ? "no passes" : `${cost.minimum ? "\u2265 " : ""}$${cost.usd.toFixed(4)}`} \xB7 unexpected changes ${u.measured === 0 ? "not measured" : `${u.runs} of ${u.measured} runs`} \xB7 median time ${s === void 0 ? "\u2013" : `${Math.round(s.durationMs.median / 1e3)} s`}${mcp === void 0 ? "" : ` \xB7 MCP tool calls ${mcpUseText(mcp)}`}`
-    );
-  }
-  if (c.costRatio !== void 0) {
-    const x = (v) => `\xD7${v.toFixed(2)}`;
-    lines.push(
-      `Cost per accepted change: B/A ${x(c.costRatio.value)} (95% paired interval ${x(c.costRatio.low)} to ${x(c.costRatio.high)}). Verdict: ${c.costRatio.verdict}${c.costRatio.verdict === "no clear difference" ? " (the interval contains \xD71.00)" : ""}.`
-    );
-  }
-  for (const note of c.notes) lines.push(`Note: ${note}`);
-  return lines.join("\n");
-}
-
-// src/evals/report.ts
-import { createHash } from "crypto";
-import { existsSync } from "fs";
-import { arch, platform, release } from "os";
-import { dirname, join as join2 } from "path";
-import { fileURLToPath } from "url";
-var REPORT_SCHEMA = 2;
-function taskHash(task) {
-  const sorted = (record) => Object.fromEntries(Object.entries(record).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0));
-  const data = {
-    prompt: task.prompt,
-    files: sorted(task.files),
-    check: task.check,
-    protect: task.protect ?? null,
-    solution: sorted(task.solution),
-    allowChanges: task.allowChanges ?? null,
-    repo: task.repo === void 0 ? null : { base: task.repo.base, sha: task.repo.sha ?? null, tests: task.repo.tests }
-  };
-  return createHash("sha256").update(JSON.stringify(data)).digest("hex");
-}
-async function garudaCommit() {
-  let dir;
-  try {
-    dir = dirname(fileURLToPath(import.meta.url));
-  } catch {
-    return void 0;
-  }
-  for (let i = 0; i < 4; i++) {
-    if (existsSync(join2(dir, ".git"))) {
-      const r = await git(new HostExecutor(), dir, ["rev-parse", "HEAD"], { check: false });
-      return r.exitCode === 0 ? r.stdout.trim() : void 0;
-    }
-    dir = dirname(dir);
-  }
-  return void 0;
-}
-function buildFields(build, checkout) {
-  const commit = build ?? checkout;
-  return {
-    version: VERSION,
-    ...commit === void 0 ? {} : { commit },
-    ...build !== void 0 && checkout !== void 0 && checkout !== build ? { checkout } : {}
-  };
-}
-function staleBuildNote(build, checkout) {
-  if (build === void 0 || checkout === void 0 || build === checkout) return void 0;
-  return `This build is from commit ${build.slice(0, 7)}, but the checkout is at ${checkout.slice(0, 7)}. The tasks and the code that run are the older ones: run pnpm build first.`;
-}
-async function reportV2(meta, tasks, results) {
-  return {
-    schema: REPORT_SCHEMA,
-    garuda: buildFields(BUILD_COMMIT, await garudaCommit()),
-    spec: meta.model,
-    agent: meta.agent ?? { name: "garuda" },
-    provider: meta.provider,
-    // Evals set no effort: each model's default applies.
-    effort: "model default",
-    isolation: meta.isolation,
-    os: { platform: platform(), release: release(), arch: arch(), node: process.versions.node },
-    seed: meta.seed,
-    shuffle: meta.shuffle,
-    split: meta.split ?? "dev",
-    addendum: meta.addendum === void 0 ? null : {
-      file: meta.addendum.file,
-      chars: meta.addendum.text.length,
-      sha256: createHash("sha256").update(meta.addendum.text).digest("hex")
-    },
-    mcp: meta.mcp === void 0 ? null : {
-      server: meta.mcp.server,
-      ...meta.mcp.prepare === void 0 ? {} : { prepare: meta.mcp.prepare }
-    },
-    order: results.map((r) => ({ id: r.id, attempt: r.attempt ?? 1 })),
-    tasks: Object.fromEntries(tasks.map((t) => [t.id, taskHash(t)])),
-    evidence: {
-      interval: passInterval(results, { seed: meta.seed }) ?? null,
-      consistency: consistency(results),
-      spread: spread(results) ?? null,
-      stepsSd: Object.fromEntries(byTask(results).map((t) => [t.id, stepsSd(t.results) ?? null])),
-      costPerAcceptedChange: costPerPass(results),
-      unexpected: unexpectedChanges(results)
-    }
-  };
-}
-
-export {
-  BUILD_COMMIT,
-  DEFAULT_SEED,
-  shuffled,
-  passInterval,
-  consistency,
-  spread,
-  stepsSd,
-  costPerPass,
-  unexpectedChanges,
-  mcpUse,
-  mcpUseText,
-  loadReport,
-  compareReports,
-  roughNote,
-  formatComparison,
-  taskHash,
-  garudaCommit,
-  staleBuildNote,
-  reportV2
-};
-//# sourceMappingURL=chunk-DOCDCQVM.js.map

package/dist/tune-YVI2KWQF.js

@@ -1,31 +0,0 @@
-#!/usr/bin/env node
-import {
-  ADDENDUM_MAX_CHARS,
-  PROPOSER_SYSTEM,
-  decide,
-  parseProposal,
-  proposerInput,
-  readAddendum,
-  rejectReason,
-  tune
-} from "./chunk-FOPBOB7W.js";
-import "./chunk-DOCDCQVM.js";
-import "./chunk-C57GHLKP.js";
-import "./chunk-6XHON3Q6.js";
-import "./chunk-B4SJY7KT.js";
-import "./chunk-BHZ6G65V.js";
-import "./chunk-P6XWWIEK.js";
-import "./chunk-UIRJOQQS.js";
-import "./chunk-4LFJFDCF.js";
-import "./chunk-3NJNVZTK.js";
-export {
-  ADDENDUM_MAX_CHARS,
-  PROPOSER_SYSTEM,
-  decide,
-  parseProposal,
-  proposerInput,
-  readAddendum,
-  rejectReason,
-  tune
-};
-//# sourceMappingURL=tune-YVI2KWQF.js.map