lastlight-evals 0.1.0

package/dist/html-report.js

@@ -0,0 +1,325 @@
+/**
+ * Self-contained HTML scorecard, styled to match the Last Light site
+ * (~/work/lastlight-www): dark navy theme, gold/orange/teal accents,
+ * Inter + JetBrains Mono. No build step, no external assets beyond Google
+ * Fonts — written straight to `evals/results/<tiers>/index.html`.
+ *
+ * One report, one tier per tab (radio-driven CSS, no JS). Each tab leads with
+ * a model-comparison table — models as rows, sorted by the tier's primary
+ * metric, with inline bar charts and best-in-column highlighting — followed by
+ * the per-instance detail rows.
+ */
+import { writeFileSync, mkdirSync, renameSync } from "node:fs";
+import { join } from "node:path";
+import { summarizeModels } from "./report.js";
+const esc = (s) => String(s ?? "")
+    .replace(/&/g, "&amp;")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;");
+function pill(kind, label) {
+    return `<span class="pill ${kind}">${esc(label)}</span>`;
+}
+function fmtMs(ms) {
+    if (!ms)
+        return "—";
+    if (ms < 1000)
+        return `${ms}ms`;
+    const s = Math.round(ms / 1000);
+    return s < 60 ? `${s}s` : `${Math.floor(s / 60)}m${String(s % 60).padStart(2, "0")}`;
+}
+function tierMetric(tier) {
+    if (tier === "code-fix") {
+        return {
+            label: "resolved",
+            rate: (m) => (m.codeFixTotal ? m.codeFixResolved / m.codeFixTotal : 0),
+            frac: (m) => (m.codeFixTotal ? `${m.codeFixResolved}/${m.codeFixTotal}` : "—"),
+        };
+    }
+    return {
+        label: "behavioral",
+        rate: (m) => (m.behavioralTotal ? m.behavioralOk / m.behavioralTotal : 0),
+        frac: (m) => (m.behavioralTotal ? `${m.behavioralOk}/${m.behavioralTotal}` : "—"),
+    };
+}
+/** A horizontal bar (0..1 fill) with a value label beside it. */
+function bar(frac, value, klass, best) {
+    const pct = Math.max(0, Math.min(1, frac)) * 100;
+    return `<div class="bar${best ? " best" : ""}"><span class="bar-track"><span class="bar-fill ${klass}" style="width:${pct.toFixed(1)}%"></span></span><span class="bar-val">${esc(value)}</span></div>`;
+}
+function compareTable(models, tier, labels) {
+    const metric = tierMetric(tier);
+    const ranked = [...models].sort((a, b) => metric.rate(b) - metric.rate(a) || a.totalCostUsd - b.totalCostUsd);
+    // Best-in-column references. Cost/latency only consider models that actually
+    // produced a run (cost > 0) so an errored 0-cost row isn't crowned "cheapest".
+    const bestRate = Math.max(0, ...ranked.map(metric.rate));
+    const costs = ranked.filter((m) => m.totalCostUsd > 0).map((m) => m.totalCostUsd);
+    const lats = ranked.filter((m) => m.p50DurationMs > 0).map((m) => m.p50DurationMs);
+    const minCost = costs.length ? Math.min(...costs) : Infinity;
+    const minLat = lats.length ? Math.min(...lats) : Infinity;
+    const maxCost = costs.length ? Math.max(...costs) : 1;
+    const maxLat = lats.length ? Math.max(...lats) : 1;
+    const rows = ranked
+        .map((m, i) => {
+        const rate = metric.rate(m);
+        const isBestRate = metric.frac(m) !== "—" && rate >= bestRate && bestRate > 0;
+        const isBestCost = m.totalCostUsd > 0 && m.totalCostUsd === minCost;
+        const isBestLat = m.p50DurationMs > 0 && m.p50DurationMs === minLat;
+        return `
+      <tr>
+        <td class="rank">${i + 1}</td>
+        <td class="mono model">${esc(labels[m.model] ?? m.model)}</td>
+        <td class="barcell">${bar(rate, metric.frac(m), "fill-gold", isBestRate)}</td>
+        <td class="barcell">${bar(maxCost ? m.totalCostUsd / maxCost : 0, `$${m.totalCostUsd.toFixed(3)}`, "fill-teal", isBestCost)}</td>
+        <td class="barcell">${bar(maxLat ? m.p50DurationMs / maxLat : 0, fmtMs(m.p50DurationMs), "fill-orange", isBestLat)}</td>
+        <td class="mono num">${Math.round(m.avgInputTokens)}<span class="muted">/</span>${Math.round(m.avgOutputTokens)}</td>
+        <td class="mono num ${m.errors ? "fail-fg" : "muted"}">${m.errors}</td>
+      </tr>`;
+    })
+        .join("\n");
+    return `
+    <table class="cmp">
+      <thead>
+        <tr>
+          <th class="rank">#</th><th>model</th>
+          <th>${esc(metric.label)} →</th><th>total cost ↓</th><th>p50 latency ↓</th>
+          <th class="num">avg in/out tok</th><th class="num">err</th>
+        </tr>
+      </thead>
+      <tbody>${rows}</tbody>
+    </table>`;
+}
+function checksHtml(r) {
+    const checks = r.behavioral?.checks ?? [];
+    if (!checks.length && r.resolved === undefined)
+        return '<span class="muted">—</span>';
+    const chips = checks
+        .map((c) => `<span class="chip ${c.ok ? "ok" : "no"}" title="${esc(c.detail ?? "")}">${esc(c.name)}</span>`)
+        .join("");
+    return chips || '<span class="muted">—</span>';
+}
+function detailRows(results, tier, labels) {
+    const showCodeFix = tier === "code-fix";
+    // "2/3" pass-count when a case aggregates multiple trials (worst-case verdict).
+    const frac = (pass, n) => pass !== undefined && n && n > 1 ? ` <span class="frac">${pass}/${n}</span>` : "";
+    return results
+        .map((r) => {
+        const resolved = (r.resolved === undefined ? pill("na", "—") : r.resolved ? pill("pass", "resolved") : pill("fail", "unresolved")) +
+            frac(r.resolvedPass, r.trials);
+        const beh = (r.error
+            ? pill("fail", "error")
+            : r.behavioral
+                ? r.behavioral.ok
+                    ? pill("pass", "ok")
+                    : pill("fail", "miss")
+                : pill("na", "—")) + (r.error ? "" : frac(r.behavioralPass, r.trials));
+        return `
+      <tr>
+        <td class="mono">${esc(r.instance_id)}</td>
+        <td class="mono muted">${esc(labels[r.model] ?? r.model)}</td>
+        ${showCodeFix ? `<td>${resolved}</td>` : ""}
+        <td>${beh}</td>
+        <td class="checks">${checksHtml(r)}</td>
+        <td class="mono num">$${r.costUsd.toFixed(4)}</td>
+        <td class="mono num">${fmtMs(r.durationMs)}</td>
+      </tr>${r.error ? `<tr class="errrow"><td colspan="${showCodeFix ? 7 : 6}" class="mono err">${esc(r.error)}</td></tr>` : ""}`;
+    })
+        .join("\n");
+}
+/** Rows for cases still running / queued (live runs only). */
+function pendingRows(pending, tier, labels) {
+    const showCodeFix = tier === "code-fix";
+    // running before queued so the in-flight ones sit nearest the finished rows.
+    const ordered = [...pending].sort((a, b) => (a.status === b.status ? 0 : a.status === "running" ? -1 : 1));
+    return ordered
+        .map((pn) => {
+        const cell = pn.status === "running"
+            ? '<span class="pill run">running…</span>'
+            : '<span class="pill wait">queued</span>';
+        const dash = '<span class="muted">—</span>';
+        return `
+      <tr class="pendingrow">
+        <td class="mono muted">${esc(pn.instance_id)}</td>
+        <td class="mono muted">${esc(labels[pn.model] ?? pn.model)}</td>
+        ${showCodeFix ? `<td>${cell}</td><td>${dash}</td>` : `<td>${cell}</td>`}
+        <td>${dash}</td>
+        <td class="mono num muted">—</td>
+        <td class="mono num muted">—</td>
+      </tr>`;
+    })
+        .join("\n");
+}
+function tierPane(tier, results, pending, labels) {
+    const models = summarizeModels(results);
+    const showCodeFix = tier === "code-fix";
+    const detailHead = showCodeFix
+        ? "<th>instance</th><th>model</th><th>code-fix</th><th>behavioral</th><th>checks</th><th>cost</th><th>latency</th>"
+        : "<th>instance</th><th>model</th><th>behavioral</th><th>checks</th><th>cost</th><th>latency</th>";
+    return `
+      <h2>Model comparison <span class="muted">— ${esc(tier)}</span></h2>
+      ${compareTable(models, tier, labels)}
+      <h2>Per-instance results</h2>
+      <table>
+        <thead><tr>${detailHead}</tr></thead>
+        <tbody>${detailRows(results, tier, labels)}${pending.length ? pendingRows(pending, tier, labels) : ""}</tbody>
+      </table>`;
+}
+export function renderHtml(card, meta) {
+    const labels = meta.labels ?? {};
+    const pending = meta.pending ?? [];
+    // Preserve the requested tier order; keep tiers that have results OR pending
+    // work (so a tier's tab shows up before its first case finishes).
+    const tiers = meta.tiers.filter((t) => card.results.some((r) => (r.tier ?? meta.tiers[0]) === t) || pending.some((pn) => pn.tier === t));
+    const useTiers = tiers.length ? tiers : meta.tiers.slice(0, 1);
+    const radios = useTiers.map((_, i) => `<input type="radio" name="tier" id="t${i}"${i === 0 ? " checked" : ""}>`).join("\n  ");
+    const tabs = useTiers.map((t, i) => `<label for="t${i}">${esc(t)}</label>`).join("\n      ");
+    const panes = useTiers
+        .map((t, i) => {
+        const rs = card.results.filter((r) => (r.tier ?? useTiers[0]) === t);
+        const pn = pending.filter((x) => x.tier === t);
+        return `<section class="pane pane-${i}">${tierPane(t, rs, pn, labels)}</section>`;
+    })
+        .join("\n    ");
+    // Per-index CSS so the checked radio reveals its pane + lights its tab.
+    const tabCss = useTiers
+        .map((_, i) => `#t${i}:checked ~ .pane-${i}{display:block;} #t${i}:checked ~ .tabs label[for=t${i}]{background:var(--gold);color:var(--navy);border-color:var(--gold);}`)
+        .join("\n  ");
+    return `<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8" />
+<meta name="viewport" content="width=device-width, initial-scale=1" />
+${meta.live ? '<meta http-equiv="refresh" content="3" />' : ""}
+<title>Last Light — Eval Scorecard</title>
+<link rel="preconnect" href="https://fonts.googleapis.com" />
+<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
+<link href="https://fonts.googleapis.com/css2?family=JetBrains+Mono:wght@400;600&family=Inter:wght@400;500;600;700&display=swap" rel="stylesheet" />
+<style>
+  :root {
+    --gold:#F0B429; --orange:#E8752A; --teal:#1A7A8A; --teal-dark:#135E6B;
+    --navy:#1B2735; --bg:#0C1117; --bg-card:#141B24; --bg-card-hover:#1A2332;
+    --border:#1E2A38; --text:#C9D1D9; --text-muted:#7D8694; --text-bright:#ECEFF4;
+    --pass:#3FB950; --fail:#E5534B;
+    --mono:'JetBrains Mono',monospace; --sans:'Inter',-apple-system,sans-serif;
+  }
+  * { box-sizing:border-box; }
+  body { margin:0; background:var(--bg); color:var(--text); font-family:var(--sans); line-height:1.55; }
+  .wrap { max-width:1100px; margin:0 auto; padding:48px 24px 80px; }
+  header { border-bottom:1px solid var(--border); padding-bottom:24px; margin-bottom:24px; }
+  h1 { color:var(--text-bright); font-size:30px; margin:0 0 6px; letter-spacing:-0.02em; }
+  h1 .accent { color:var(--gold); }
+  h2 { color:var(--text-bright); font-size:18px; margin:32px 0 14px; }
+  h2 .muted { font-weight:400; }
+  .meta { color:var(--text-muted); font-size:13px; font-family:var(--mono); }
+  .meta b { color:var(--text); font-weight:600; }
+
+  /* radio-driven tabs (no JS) */
+  input[name=tier] { position:absolute; opacity:0; pointer-events:none; }
+  .tabs { display:flex; gap:8px; margin-bottom:8px; flex-wrap:wrap; }
+  .tabs label { cursor:pointer; font-family:var(--mono); font-size:13px; font-weight:600;
+    padding:8px 16px; border-radius:8px; border:1px solid var(--border); color:var(--text-muted);
+    background:var(--bg-card); user-select:none; }
+  .tabs label:hover { color:var(--text-bright); border-color:var(--teal); }
+  .pane { display:none; }
+  ${tabCss}
+
+  table { width:100%; border-collapse:collapse; background:var(--bg-card); border:1px solid var(--border); border-radius:12px; overflow:hidden; }
+  th { text-align:left; color:var(--text-muted); font-size:11px; text-transform:uppercase; letter-spacing:0.05em; font-weight:600; padding:11px 14px; border-bottom:1px solid var(--border); background:var(--navy); }
+  td { padding:10px 14px; border-bottom:1px solid var(--border); font-size:13px; vertical-align:middle; }
+  tr:last-child td { border-bottom:none; }
+  .mono { font-family:var(--mono); }
+  .num { text-align:right; white-space:nowrap; }
+  .muted { color:var(--text-muted); }
+  .fail-fg { color:var(--fail); }
+
+  /* comparison table */
+  table.cmp td { vertical-align:middle; }
+  .cmp .rank { width:30px; color:var(--text-muted); font-family:var(--mono); text-align:center; }
+  .cmp .model { color:var(--gold); font-weight:600; white-space:nowrap; }
+  .cmp .barcell { width:22%; min-width:140px; }
+  .bar { display:flex; align-items:center; gap:8px; }
+  .bar-track { flex:1; height:8px; background:var(--navy); border-radius:999px; overflow:hidden; }
+  .bar-fill { display:block; height:100%; border-radius:999px; }
+  .fill-gold { background:var(--gold); } .fill-teal { background:var(--teal); } .fill-orange { background:var(--orange); }
+  .bar-val { font-family:var(--mono); font-size:12px; min-width:48px; text-align:right; color:var(--text); }
+  .bar.best .bar-val { color:var(--pass); font-weight:600; }
+  .bar.best .bar-val::after { content:" ★"; color:var(--gold); font-size:10px; }
+
+  .pill { display:inline-block; font-family:var(--mono); font-size:11px; font-weight:600; padding:3px 9px; border-radius:999px; }
+  .pill.pass { background:rgba(63,185,80,0.14); color:var(--pass); }
+  .pill.fail { background:rgba(229,83,75,0.14); color:var(--fail); }
+  .pill.na { background:var(--navy); color:var(--text-muted); }
+  .pill.run { background:rgba(240,180,41,0.14); color:var(--gold); animation:pulse 1.4s ease-in-out infinite; }
+  .pill.wait { background:var(--navy); color:var(--text-muted); }
+  .pendingrow td { opacity:0.72; }
+  .frac { font-family:var(--mono); font-size:11px; color:var(--text-muted); }
+  .checks { line-height:2; }
+  .chip { display:inline-block; font-family:var(--mono); font-size:10.5px; padding:2px 7px; border-radius:5px; margin:0 4px 4px 0; border:1px solid var(--border); }
+  .chip.ok { color:var(--pass); border-color:rgba(63,185,80,0.4); }
+  .chip.no { color:var(--fail); border-color:rgba(229,83,75,0.4); }
+  .errrow td { color:var(--fail); background:rgba(229,83,75,0.06); }
+  .err { white-space:pre-wrap; }
+  footer { margin-top:40px; color:var(--text-muted); font-size:12px; font-family:var(--mono); }
+  a { color:var(--gold); }
+  .live { font-family:var(--mono); font-size:13px; color:var(--gold); font-weight:600; letter-spacing:0; vertical-align:middle; margin-left:10px; animation:pulse 1.4s ease-in-out infinite; }
+  @keyframes pulse { 0%,100%{opacity:1;} 50%{opacity:0.35;} }
+</style>
+</head>
+<body>
+  <div class="wrap">
+    <header>
+      <h1>Last Light <span class="accent">·</span> Eval Scorecard${meta.live ? ` <span class="live">● live${meta.progress ? ` ${esc(meta.progress)}` : ""}</span>` : ""}</h1>
+      <div class="meta">
+        <b>${esc(useTiers.join(" + "))}</b> &nbsp;·&nbsp;
+        models: <b>${esc(meta.models.join(", "))}</b> &nbsp;·&nbsp;
+        ${esc(card.results.length)}${pending.length ? ` / ${esc(card.results.length + pending.length)}` : ""} cases${meta.runs && meta.runs > 1
+        ? ` &nbsp;·&nbsp; <b>${esc(meta.runs)}×</b> per case <span class="muted">(worst-case verdict · mean cost)</span>`
+        : ""} &nbsp;·&nbsp;
+        ${esc(meta.generatedAt)}
+      </div>
+    </header>
+
+    <div class="tabwrap">
+      ${radios}
+      <nav class="tabs">
+      ${tabs}
+      </nav>
+    ${panes}
+    </div>
+
+    <footer>
+      Real production workflows · mocked GitHub · deterministic grading. ★ = best in column.
+      Generated by <span class="mono">npm run eval</span>. Also: scorecard.json · predictions.jsonl.
+    </footer>
+  </div>
+  <script>
+    // Preserve the active tab + scroll position across the live auto-refresh.
+    (function () {
+      try {
+        var K = "ll-eval-tab", S = "ll-eval-scroll";
+        var saved = sessionStorage.getItem(K);
+        if (saved) { var r = document.getElementById(saved); if (r) r.checked = true; }
+        document.querySelectorAll('input[name=tier]').forEach(function (radio) {
+          radio.addEventListener("change", function () { sessionStorage.setItem(K, radio.id); });
+        });
+        var y = sessionStorage.getItem(S);
+        if (y) window.scrollTo(0, parseInt(y, 10) || 0);
+        window.addEventListener("scroll", function () { sessionStorage.setItem(S, String(window.scrollY)); }, { passive: true });
+      } catch (e) {}
+    })();
+  </script>
+</body>
+</html>
+`;
+}
+export function writeHtml(dir, card, meta) {
+    mkdirSync(dir, { recursive: true });
+    const file = join(dir, "index.html");
+    // Write-then-rename so a browser auto-refreshing during a live run never
+    // reads a half-written file (rename is atomic on the same filesystem).
+    const tmp = join(dir, ".index.html.tmp");
+    writeFileSync(tmp, renderHtml(card, meta));
+    renameSync(tmp, file);
+    return file;
+}
+//# sourceMappingURL=html-report.js.map

package/dist/html-report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-report.js","sourceRoot":"","sources":["../src/html-report.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAC/D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,eAAe,EAAqC,MAAM,aAAa,CAAC;AA4BjF,MAAM,GAAG,GAAG,CAAC,CAAU,EAAU,EAAE,CACjC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;KACZ,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;KACtB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;KACrB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;KACrB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;AAE7B,SAAS,IAAI,CAAC,IAA4B,EAAE,KAAa;IACvD,OAAO,qBAAqB,IAAI,KAAK,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC;AAC3D,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,IAAI,CAAC,EAAE;QAAE,OAAO,GAAG,CAAC;IACpB,IAAI,EAAE,GAAG,IAAI;QAAE,OAAO,GAAG,EAAE,IAAI,CAAC;IAChC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC;IAChC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;AACvF,CAAC;AAYD,SAAS,UAAU,CAAC,IAAY;IAC9B,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;QACxB,OAAO;YACL,KAAK,EAAE,UAAU;YACjB,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;YACtE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,eAAe,IAAI,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;SAC/E,CAAC;IACJ,CAAC;IACD,OAAO;QACL,KAAK,EAAE,YAAY;QACnB,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;QACzE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;KAClF,CAAC;AACJ,CAAC;AAED,iEAAiE;AACjE,SAAS,GAAG,CAAC,IAAY,EAAE,KAAa,EAAE,KAAa,EAAE,IAAa;IACpE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC;IACjD,OAAO,kBAAkB,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,mDAAmD,KAAK,kBAAkB,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,0CAA0C,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC;AAC1M,CAAC;AAED,SAAS,YAAY,CAAC,MAAsB,EAAE,IAAY,EAAE,MAA8B;IACxF,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAChC,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC;IAE9G,6EAA6E;IAC7E,+EAA+E;IAC/E,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IACzD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;IAClF,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC;IACnF,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC1D,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEnD,MAAM,IAAI,GAAG,MAAM;SAChB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACZ,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,IAAI,IAAI,QAAQ,IAAI,QAAQ,GAAG,CAAC,CAAC;QAC9E,MAAM,UAAU,GAAG,CAAC,CAAC,YAAY,GAAG,CAAC,IAAI,CAAC,CAAC,YAAY,KAAK,OAAO,CAAC;QACpE,MAAM,SAAS,GAAG,CAAC,CAAC,aAAa,GAAG,CAAC,IAAI,CAAC,CAAC,aAAa,KAAK,MAAM,CAAC;QACpE,OAAO;;2BAEc,CAAC,GAAG,CAAC;iCACC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC;8BAClC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,UAAU,CAAC;8BAClD,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,WAAW,EAAE,UAAU,CAAC;8BACrG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,EAAE,aAAa,EAAE,SAAS,CAAC;+BAC3F,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,+BAA+B,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC;8BACzF,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,MAAM;YAC7D,CAAC;IACT,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,OAAO;;;;;gBAKO,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC;;;;eAIlB,IAAI;aACN,CAAC;AACd,CAAC;AAED,SAAS,UAAU,CAAC,CAAiB;IACnC,MAAM,MAAM,GAAG,CAAC,CAAC,UAAU,EAAE,MAAM,IAAI,EAAE,CAAC;IAC1C,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS;QAAE,OAAO,8BAA8B,CAAC;IACtF,MAAM,KAAK,GAAG,MAAM;SACjB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,qBAAqB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;SAC3G,IAAI,CAAC,EAAE,CAAC,CAAC;IACZ,OAAO,KAAK,IAAI,8BAA8B,CAAC;AACjD,CAAC;AAED,SAAS,UAAU,CAAC,OAAyB,EAAE,IAAY,EAAE,MAA8B;IACzF,MAAM,WAAW,GAAG,IAAI,KAAK,UAAU,CAAC;IACxC,gFAAgF;IAChF,MAAM,IAAI,GAAG,CAAC,IAAa,EAAE,CAAU,EAAE,EAAE,CACzC,IAAI,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,uBAAuB,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;IACpF,OAAO,OAAO;SACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,QAAQ,GACZ,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;YACjH,IAAI,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,GAAG,GACP,CAAC,CAAC,CAAC,KAAK;YACN,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC;YACvB,CAAC,CAAC,CAAC,CAAC,UAAU;gBACZ,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE;oBACf,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;oBACpB,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC;gBACxB,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QAC7E,OAAO;;2BAEc,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC;iCACZ,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC;UACtD,WAAW,CAAC,CAAC,CAAC,OAAO,QAAQ,OAAO,CAAC,CAAC,CAAC,EAAE;cACrC,GAAG;6BACY,UAAU,CAAC,CAAC,CAAC;gCACV,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;+BACrB,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC;aACrC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,mCAAmC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,sBAAsB,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IAC/H,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,8DAA8D;AAC9D,SAAS,WAAW,CAAC,OAAsB,EAAE,IAAY,EAAE,MAA8B;IACvF,MAAM,WAAW,GAAG,IAAI,KAAK,UAAU,CAAC;IACxC,6EAA6E;IAC7E,MAAM,OAAO,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3G,OAAO,OAAO;SACX,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QACV,MAAM,IAAI,GACR,EAAE,CAAC,MAAM,KAAK,SAAS;YACrB,CAAC,CAAC,wCAAwC;YAC1C,CAAC,CAAC,uCAAuC,CAAC;QAC9C,MAAM,IAAI,GAAG,8BAA8B,CAAC;QAC5C,OAAO;;iCAEoB,GAAG,CAAC,EAAE,CAAC,WAAW,CAAC;iCACnB,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;UACxD,WAAW,CAAC,CAAC,CAAC,OAAO,IAAI,YAAY,IAAI,OAAO,CAAC,CAAC,CAAC,OAAO,IAAI,OAAO;cACjE,IAAI;;;YAGN,CAAC;IACT,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,QAAQ,CACf,IAAY,EACZ,OAAyB,EACzB,OAAsB,EACtB,MAA8B;IAE9B,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IACxC,MAAM,WAAW,GAAG,IAAI,KAAK,UAAU,CAAC;IACxC,MAAM,UAAU,GAAG,WAAW;QAC5B,CAAC,CAAC,iHAAiH;QACnH,CAAC,CAAC,gGAAgG,CAAC;IACrG,OAAO;mDAC0C,GAAG,CAAC,IAAI,CAAC;QACpD,YAAY,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC;;;qBAGrB,UAAU;iBACd,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;eAC9F,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAe,EAAE,IAAc;IACxD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;IACjC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;IACnC,6EAA6E;IAC7E,kEAAkE;IAClE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAC7B,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,CAAC,CACxG,CAAC;IACF,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAE/D,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,wCAAwC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9H,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC7F,MAAM,KAAK,GAAG,QAAQ;SACnB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACZ,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QACrE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC;QAC/C,OAAO,6BAA6B,CAAC,KAAK,QAAQ,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,CAAC,YAAY,CAAC;IACpF,CAAC,CAAC;SACD,IAAI,CAAC,QAAQ,CAAC,CAAC;IAElB,wEAAwE;IACxE,MAAM,MAAM,GAAG,QAAQ;SACpB,GAAG,CACF,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,KAAK,CAAC,oBAAoB,CAAC,sBAAsB,CAAC,+BAA+B,CAAC,uEAAuE,CAC5J;SACA,IAAI,CAAC,MAAM,CAAC,CAAC;IAEhB,OAAO;;;;;EAKP,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,2CAA2C,CAAC,CAAC,CAAC,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAgC1D,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mEAgDF,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,6BAA6B,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,EACpG;;aAEO,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;qBACjB,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;UACtC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAClG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC;QACxB,CAAC,CAAC,qBAAqB,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,4EAA4E;QACjH,CAAC,CAAC,EACN;UACE,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC;;;;;QAKvB,MAAM;;QAEN,IAAI;;MAEN,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;CA0BV,CAAC;AACF,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,GAAW,EAAE,IAAe,EAAE,IAAc;IACpE,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACpC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IACrC,yEAAyE;IACzE,uEAAuE;IACvE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,iBAAiB,CAAC,CAAC;IACzC,aAAa,CAAC,GAAG,EAAE,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IAC3C,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IACtB,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/init.js

@@ -0,0 +1,122 @@
+/**
+ * `lastlight-evals init [dir]` — scaffold a fresh overlay + evals repo, seeded
+ * from the shipped defaults, then optionally version it as a private GitHub repo.
+ *
+ * The result is a self-contained deployment overlay: it carries its own
+ * `workflows/` / `skills/` / `agent-context/` (which shadow the built-ins by
+ * logical name) AND its own `evals/datasets/` + `evals/models.json`. Point the
+ * runner at it with `lastlight-evals run --overlay <dir>` (or run from inside
+ * it with `--overlay .`). It's pure data/config — no node deps; the globally
+ * installed CLI supplies the runtime.
+ *
+ * Git/GitHub bootstrap reuses core's `detectGh` + `bootstrapOverlayRepo` (the
+ * same flow behind `lastlight server setup`) via the `lastlight/evals` barrel —
+ * git init + initial commit + an offered `gh repo create --private`.
+ */
+import { existsSync, mkdirSync, writeFileSync, cpSync } from "node:fs";
+import { join, resolve, basename } from "node:path";
+import * as p from "@clack/prompts";
+import chalk from "chalk";
+import { detectGh, bootstrapOverlayRepo } from "lastlight/evals";
+import { builtinDatasetsRoot, builtinModelsPath } from "./paths.js";
+const CONFIG_YAML = `# Last Light — eval/overlay config (merged over the core config/default.yaml).
+# Arrays replace, maps deep-merge, env vars override. See the lastlight docs.
+#
+# managedRepos:
+#   - your-org/your-repo
+#
+# models:
+#   default: openai/gpt-5.5
+`;
+const GITIGNORE = `# Eval run output (scorecards, predictions) — regenerated, never commit.
+eval-results/
+
+# Local secrets / env.
+.env
+secrets/*
+!secrets/.env.example
+*.pem
+`;
+function readmeFor(name) {
+    return `# ${name}
+
+A Last Light **eval + overlay** workspace, created by \`lastlight-evals init\`.
+
+## Layout
+
+- \`workflows/\`, \`skills/\`, \`agent-context/\` — overlay assets. Files here
+  **shadow** the built-in Last Light assets by name, or add new ones.
+- \`evals/datasets/<tier>/\` — your eval datasets (\`instances.json\` +
+  \`tier.json\`, plus \`repos/\` + \`tests/\` for code-fix tiers). Seeded with the
+  shipped \`triage\` / \`code-fix\` samples — edit or replace them.
+- \`evals/models.json\` — the model registry these evals run against.
+- \`config.yaml\` — overlay config (managed repos, models, …).
+
+## Run
+
+\`\`\`bash
+# from inside this repo
+lastlight-evals run --overlay .
+# or a specific tier
+lastlight-evals run code-fix --overlay .
+\`\`\`
+
+Override which core build the evals test with \`LASTLIGHT_CORE_DIR=/path/to/lastlight\`.
+`;
+}
+/** Write the default files/dirs into `dir`, never overwriting what exists. */
+function scaffoldEvalRepo(dir) {
+    const created = [];
+    const writeIfMissing = (rel, content) => {
+        const abs = join(dir, rel);
+        if (existsSync(abs))
+            return;
+        mkdirSync(resolve(abs, ".."), { recursive: true });
+        writeFileSync(abs, content, "utf8");
+        created.push(rel);
+    };
+    // Overlay asset dirs (empty placeholders the user fills in).
+    for (const d of ["workflows", "skills", "agent-context"]) {
+        const keep = join(d, ".gitkeep");
+        writeIfMissing(keep, "");
+    }
+    // Seed datasets + models from the shipped defaults (a starting point to edit).
+    const datasetsDst = join(dir, "evals", "datasets");
+    if (!existsSync(datasetsDst)) {
+        mkdirSync(datasetsDst, { recursive: true });
+        cpSync(builtinDatasetsRoot(), datasetsDst, { recursive: true });
+        created.push("evals/datasets/ (seeded triage + code-fix)");
+    }
+    const modelsDst = join(dir, "evals", "models.json");
+    if (!existsSync(modelsDst)) {
+        cpSync(builtinModelsPath(), modelsDst);
+        created.push("evals/models.json");
+    }
+    writeIfMissing("config.yaml", CONFIG_YAML);
+    writeIfMissing(".gitignore", GITIGNORE);
+    writeIfMissing("README.md", readmeFor(basename(dir)));
+    return created;
+}
+export async function runInit(dirArg) {
+    p.intro(chalk.bold(`Last Light ${chalk.yellow("·")} eval init`));
+    const dir = resolve(dirArg ?? "lastlight-evals-workspace");
+    p.log.step(`Scaffolding ${chalk.cyan(dir)}`);
+    const created = scaffoldEvalRepo(dir);
+    if (created.length) {
+        p.log.success(`Created:\n  ${created.join("\n  ")}`);
+    }
+    else {
+        p.log.info("Nothing to scaffold — every default file already existed.");
+    }
+    // Offer to version it + create a private GitHub repo (reuses core's flow).
+    try {
+        const gh = await detectGh();
+        await bootstrapOverlayRepo(dir, { gh });
+    }
+    catch (err) {
+        p.log.warn(`Skipped git/GitHub bootstrap: ${err.message}`);
+    }
+    p.outro(chalk.green(`done — run it with: ${chalk.bold(`lastlight-evals run --overlay ${dir === process.cwd() ? "." : dir}`)}`));
+    return 0;
+}
+//# sourceMappingURL=init.js.map

package/dist/init.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AACH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACvE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAEpD,OAAO,KAAK,CAAC,MAAM,gBAAgB,CAAC;AACpC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAEjE,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAEpE,MAAM,WAAW,GAAG;;;;;;;;CAQnB,CAAC;AAEF,MAAM,SAAS,GAAG;;;;;;;;CAQjB,CAAC;AAEF,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,KAAK,IAAI;;;;;;;;;;;;;;;;;;;;;;;;CAwBjB,CAAC;AACF,CAAC;AAED,8EAA8E;AAC9E,SAAS,gBAAgB,CAAC,GAAW;IACnC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,cAAc,GAAG,CAAC,GAAW,EAAE,OAAe,EAAQ,EAAE;QAC5D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAC3B,IAAI,UAAU,CAAC,GAAG,CAAC;YAAE,OAAO;QAC5B,SAAS,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,aAAa,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QACpC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACpB,CAAC,CAAC;IAEF,6DAA6D;IAC7D,KAAK,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,EAAE,eAAe,CAAC,EAAE,CAAC;QACzD,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;QACjC,cAAc,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IAC3B,CAAC;IAED,+EAA+E;IAC/E,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;IACnD,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC7B,SAAS,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5C,MAAM,CAAC,mBAAmB,EAAE,EAAE,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAChE,OAAO,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAC;IAC7D,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC;IACpD,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3B,MAAM,CAAC,iBAAiB,EAAE,EAAE,SAAS,CAAC,CAAC;QACvC,OAAO,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACpC,CAAC;IAED,cAAc,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;IAC3C,cAAc,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;IACxC,cAAc,CAAC,WAAW,EAAE,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACtD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,MAAe;IAC3C,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC;IAEjE,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,IAAI,2BAA2B,CAAC,CAAC;IAC3D,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC7C,MAAM,OAAO,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;IACtC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,eAAe,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACvD,CAAC;SAAM,CAAC;QACN,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,2DAA2D,CAAC,CAAC;IAC1E,CAAC;IAED,2EAA2E;IAC3E,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,MAAM,QAAQ,EAAE,CAAC;QAC5B,MAAM,oBAAoB,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;IAC1C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,iCAAkC,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,CAAC,CAAC,KAAK,CACL,KAAK,CAAC,KAAK,CACT,uBAAuB,KAAK,CAAC,IAAI,CAAC,iCAAiC,GAAG,KAAK,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAC1G,CACF,CAAC;IACF,OAAO,CAAC,CAAC;AACX,CAAC"}

package/dist/mechanism.test.js

@@ -0,0 +1,108 @@
+/**
+ * Deterministic, AI-free tests for the eval harness mechanism. These run in the
+ * DEFAULT `npm test` suite (not the paid `*.eval.test.ts` suite) so the mock
+ * plumbing is regression-guarded for free:
+ *
+ *   - the fake GitHub speaks enough REST for the real github_* tools;
+ *   - agentic-pi's `githubApiBaseUrl` seam actually routes Octokit at it;
+ *   - workspace seeding + execution grading flip red→green correctly.
+ */
+import { describe, it, expect } from "vitest";
+import { GitHubClient } from "agentic-pi/dist/extensions/github/client.js";
+import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { startFakeGitHub } from "./fake-github.js";
+import { seedWorkspace } from "./seed.js";
+import { gradeExecution, gradeBehavioral, gradeTriage } from "./grade.js";
+const staticAuth = { getToken: async () => "fake-token", expiresAt: null };
+describe("fake GitHub + agentic-pi github tools (baseUrl seam)", () => {
+    it("serves seeded issues and records mutations made through the real GitHubClient", async () => {
+        const fake = await startFakeGitHub({
+            owner: "acme",
+            repo: "widget",
+            issues: [{ number: 101, title: "Crash on empty config", body: "boom", labels: [] }],
+        });
+        try {
+            // The REAL agentic-pi client, pointed at the fake via the released seam.
+            const gh = new GitHubClient(staticAuth, { baseUrl: fake.url });
+            const issue = (await gh.getIssue("acme", "widget", 101));
+            expect(issue.number).toBe(101);
+            expect(issue.title).toBe("Crash on empty config");
+            await gh.createLabel("acme", "widget", "bug", "d73a4a");
+            await gh.addLabels("acme", "widget", 101, ["bug", "ready-for-agent"]);
+            await gh.addIssueComment("acme", "widget", 101, "Triaged — needs a repro first.");
+            expect(fake.labelsOn(101)).toEqual(expect.arrayContaining(["bug", "ready-for-agent"]));
+            expect(fake.commentsOn(101).some((c) => /repro/i.test(c))).toBe(true);
+            expect(fake.calls.some((c) => c.method === "POST" && c.path.endsWith("/labels"))).toBe(true);
+        }
+        finally {
+            await fake.close();
+        }
+    });
+    it("behavioral + triage grades read the recorded GitHub state", async () => {
+        const fake = await startFakeGitHub({
+            owner: "acme",
+            repo: "widget",
+            issues: [{ number: 7, title: "Q", body: "how?", labels: [] }],
+        });
+        try {
+            const gh = new GitHubClient(staticAuth, { baseUrl: fake.url });
+            await gh.addLabels("acme", "widget", 7, ["question"]);
+            const beh = gradeBehavioral({ labels_added: ["question"], labels_absent: ["ready-for-agent"] }, fake, { issueNumber: 7, branch: "main" });
+            expect(beh.ok).toBe(true);
+            const tri = gradeTriage({ category: "question" }, fake, 7);
+            expect(tri.ok).toBe(true);
+            const miss = gradeTriage({ state: "ready-for-agent" }, fake, 7);
+            expect(miss.ok).toBe(false);
+        }
+        finally {
+            await fake.close();
+        }
+    });
+});
+describe("workspace seed + execution grade (SWE-bench resolved)", () => {
+    it("flips red→green when the bug is fixed, and detects PASS_TO_PASS regressions", () => {
+        const stateDir = mkdtempSync(join(tmpdir(), "ll-eval-mech-"));
+        try {
+            const fixtureDir = join(stateDir, "fixture");
+            mkdirSync(join(fixtureDir, "src"), { recursive: true });
+            // Buggy: off-by-one (returns n + 2).
+            writeFileSync(join(fixtureDir, "src", "counter.ts"), "export const next = (n: number): number => n + 2;\n");
+            const seeded = seedWorkspace({ stateDir, taskId: "mech-task", fixtureDir });
+            expect(seeded.baseCommit).toHaveLength(40);
+            // Held-out test the agent never saw.
+            const heldOutDir = join(stateDir, "held");
+            mkdirSync(heldOutDir, { recursive: true });
+            writeFileSync(join(heldOutDir, "counter.test.ts"), [
+                'import { test } from "node:test";',
+                'import assert from "node:assert/strict";',
+                'import { next } from "./src/counter.ts";',
+                'test("increments by one", () => { assert.equal(next(1), 2); });',
+                'test("stays numeric", () => { assert.equal(typeof next(3), "number"); });',
+            ].join("\n") + "\n");
+            // Before the fix → FAIL_TO_PASS test is red → not resolved.
+            const before = gradeExecution({
+                workDir: seeded.workDir,
+                heldOutDir,
+                failToPass: ["increments by one"],
+                passToPass: ["stays numeric"],
+            });
+            expect(before.resolved).toBe(false);
+            expect(before.failToPass.find((t) => t.id === "increments by one")?.pass).toBe(false);
+            expect(before.passToPass.find((t) => t.id === "stays numeric")?.pass).toBe(true);
+            // Apply the fix → both green → resolved.
+            writeFileSync(join(seeded.workDir, "src", "counter.ts"), "export const next = (n: number): number => n + 1;\n");
+            const after = gradeExecution({
+                workDir: seeded.workDir,
+                failToPass: ["increments by one"],
+                passToPass: ["stays numeric"],
+            });
+            expect(after.resolved).toBe(true);
+        }
+        finally {
+            rmSync(stateDir, { recursive: true, force: true });
+        }
+    }, 60_000);
+});
+//# sourceMappingURL=mechanism.test.js.map

package/dist/mechanism.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mechanism.test.js","sourceRoot":"","sources":["../src/mechanism.test.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,6CAA6C,CAAC;AAC3E,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACxE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAEjC,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE1E,MAAM,UAAU,GAAG,EAAE,QAAQ,EAAE,KAAK,IAAI,EAAE,CAAC,YAAY,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;AAE3E,QAAQ,CAAC,sDAAsD,EAAE,GAAG,EAAE;IACpE,EAAE,CAAC,+EAA+E,EAAE,KAAK,IAAI,EAAE;QAC7F,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC;YACjC,KAAK,EAAE,MAAM;YACb,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,uBAAuB,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;SACpF,CAAC,CAAC;QACH,IAAI,CAAC;YACH,yEAAyE;YACzE,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,UAAU,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YAE/D,MAAM,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAsC,CAAC;YAC9F,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC/B,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YAElD,MAAM,EAAE,CAAC,WAAW,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;YACxD,MAAM,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC,CAAC;YACtE,MAAM,EAAE,CAAC,eAAe,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,gCAAgC,CAAC,CAAC;YAElF,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC;YACvF,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/F,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;QACzE,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC;YACjC,KAAK,EAAE,MAAM;YACb,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;SAC9D,CAAC,CAAC;QACH,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,UAAU,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YAC/D,MAAM,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC;YAEtD,MAAM,GAAG,GAAG,eAAe,CAAC,EAAE,YAAY,EAAE,CAAC,UAAU,CAAC,EAAE,aAAa,EAAE,CAAC,iBAAiB,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;YAC1I,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAE1B,MAAM,GAAG,GAAG,WAAW,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAC3D,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAE1B,MAAM,IAAI,GAAG,WAAW,CAAC,EAAE,KAAK,EAAE,iBAAiB,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAChE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,uDAAuD,EAAE,GAAG,EAAE;IACrE,EAAE,CAAC,6EAA6E,EAAE,GAAG,EAAE;QACrF,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,eAAe,CAAC,CAAC,CAAC;QAC9D,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAC7C,SAAS,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACxD,qCAAqC;YACrC,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,YAAY,CAAC,EAAE,qDAAqD,CAAC,CAAC;YAE5G,MAAM,MAAM,GAAG,aAAa,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,CAAC;YAC5E,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;YAE3C,qCAAqC;YACrC,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAC1C,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC3C,aAAa,CACX,IAAI,CAAC,UAAU,EAAE,iBAAiB,CAAC,EACnC;gBACE,mCAAmC;gBACnC,0CAA0C;gBAC1C,0CAA0C;gBAC1C,iEAAiE;gBACjE,2EAA2E;aAC5E,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CACpB,CAAC;YAEF,4DAA4D;YAC5D,MAAM,MAAM,GAAG,cAAc,CAAC;gBAC5B,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,UAAU;gBACV,UAAU,EAAE,CAAC,mBAAmB,CAAC;gBACjC,UAAU,EAAE,CAAC,eAAe,CAAC;aAC9B,CAAC,CAAC;YACH,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,mBAAmB,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACtF,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,eAAe,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAEjF,yCAAyC;YACzC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,EAAE,YAAY,CAAC,EAAE,qDAAqD,CAAC,CAAC;YAChH,MAAM,KAAK,GAAG,cAAc,CAAC;gBAC3B,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,UAAU,EAAE,CAAC,mBAAmB,CAAC;gBACjC,UAAU,EAAE,CAAC,eAAe,CAAC;aAC9B,CAAC,CAAC;YACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpC,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,CAAC;IACH,CAAC,EAAE,MAAM,CAAC,CAAC;AACb,CAAC,CAAC,CAAC"}