@hackerrank/astra-cli 0.1.20 → 0.1.21

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hackerrank/astra-cli",
-  "version": "0.1.20",
+  "version": "0.1.21",
   "description": "Minimal zero-dependency AI coding agent for the HackerRank AI Gateway.",
   "type": "module",
   "bin": {

package/src/agent.js

@@ -26,6 +26,7 @@ import {
   buildSystemPrompt,
   INSTANCE_TEMPLATE,
   FORMAT_ERROR,
+  TRUNCATED_COMMAND,
   OBSERVATION_TEMPLATE,
   render,
 } from "./prompts.js";
@@ -162,6 +163,14 @@ export class Agent {
       if (this.mode === "interactive" && err.code === "NO_COMMAND") {
         return { kind: "chat", content };
       }
+      if (isTruncatedShellResponse(content, usage)) {
+        // A provider length limit can cut off a large heredoc before the
+        // closing fence. No command has run, so recover with smaller writes
+        // rather than consuming the terminal format-error budget.
+        this.formatErrorStreak = 0;
+        this.add("user", TRUNCATED_COMMAND);
+        return { kind: "format_error", error: "truncated shell response" };
+      }
       this.formatErrorStreak++;
       this.nFormatErrors++;
       if (
@@ -352,6 +361,12 @@ export function parseCommand(text, { allowMultiple = false } = {}) {
   return cmd;
 }
 
+function isTruncatedShellResponse(content, usage) {
+  if (usage?.finish_reason !== "length") return false;
+  const fences = String(content || "").match(/^```(?:bash|sh|shell|zsh)?[ \t]*$/gim) || [];
+  return fences.length % 2 === 1;
+}
+
 /** Returns the submission string if the sentinel was echoed, else null. */
 export function checkSubmitted(output) {
   if (output.returncode !== 0) return null;

package/src/model.js

@@ -48,7 +48,7 @@ export class GatewayModel {
     this.maxTokens = maxTokens;
     this.baseUrl = (baseUrl || process.env.ASTRA_GATEWAY_BASE_URL || DEFAULT_BASE_URL).replace(/\/+$/, "");
     this.apiKey = apiKey || process.env.ASTRA_GATEWAY_API_KEY || "";
-    this.modelKwargs = modelKwargs;
+    this.modelKwargs = sanitizeModelKwargs(model, modelKwargs);
     this.maxRetries = maxRetries;
     this.requestTimeoutMs = requestTimeoutMs;
     this.onRetry = onRetry || (() => {});
@@ -144,6 +144,7 @@ export class GatewayModel {
         this.nCalls++;
         const content = data?.choices?.[0]?.message?.content ?? "";
         const usage = normalizeUsage(data?.usage);
+        usage.finish_reason = data?.choices?.[0]?.finish_reason ?? null;
         this.totalPromptTokens += usage.prompt_tokens;
         this.totalCompletionTokens += usage.completion_tokens;
         this.totalReasoningTokens += usage.reasoning_tokens || 0;
@@ -338,3 +339,12 @@ export function normalizeUsage(usage) {
 function num(v) {
   return Number.isFinite(v) ? v : 0;
 }
+
+function sanitizeModelKwargs(model, modelKwargs) {
+  const kwargs = { ...(modelKwargs || {}) };
+  // Gemini's OpenAI-compatible chat endpoint rejects this Anthropic-style
+  // cache-routing field. Keep the gateway client safe even when a caller or
+  // persisted configuration passes provider-specific kwargs through.
+  if (/^gemini(?:-|$)/i.test(model)) delete kwargs.prompt_cache_key;
+  return kwargs;
+}

package/src/prompts.js

@@ -39,6 +39,9 @@ Autonomous mode:
   environment variables, and config files needed to complete the task.
 - In autonomous mode, you may provide multiple fenced bash blocks when they
   form one coherent sequence; Astra executes them together in order.
+- Keep each command response small enough to finish completely (roughly 6,000
+  characters). Do not write a large source file in one heredoc: create it in
+  small appended chunks, then verify it.
 - When completely finished, run this and NOTHING else:
   \`echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT\`
   After that command you cannot continue.`;
@@ -78,6 +81,10 @@ Respond again with exactly ONE bash code block containing ONE command, like:
 your_command_here
 \`\`\``;
 
+export const TRUNCATED_COMMAND = `Your previous response was cut off before its shell block closed, so no command was run.
+
+Do not repeat a large file write in one response. Continue with exactly one complete bash block containing a small command (under roughly 6,000 characters); append files in chunks or make a smaller focused change.`;
+
 export const OBSERVATION_TEMPLATE = `{{exception}}<returncode>{{returncode}}</returncode>
 <output>
 {{output}}

package/src/report.html

@@ -8,16 +8,15 @@
 <style>
   :root { --ink:#11131a; --muted:#69707d; --wash:#f5f6f8; --line:#e4e7eb; --lime:#a7ff63; --blue:#5285f7; --purple:#9160d8; --green:#39ae76; --orange:#f3a021; --red:#d85668; font-family:Satoshi,"Avenir Next",Arial,sans-serif; }
   * { box-sizing:border-box; } body { margin:0; background:#fff; color:var(--ink); font-size:14px; line-height:1.45; } .wrap { max-width:1240px; margin:auto; padding:28px 32px 68px; }
-  header.top { display:flex; justify-content:space-between; gap:20px; align-items:flex-start; padding-bottom:28px; border-bottom:1px solid var(--line); } header.top h1 { font-size:32px; letter-spacing:-.045em; line-height:1; margin:8px 0 0; font-weight:700; } .eyebrow { display:inline-block; padding:5px 14px; border-radius:99px; background:var(--lime); font-size:12px; font-weight:700; } header.top .meta { color:var(--muted); font-size:12px; text-align:right; }
+  header.top { display:flex; justify-content:space-between; gap:20px; align-items:flex-start; padding-bottom:28px; border-bottom:1px solid var(--line); } header.top h1 { font-size:32px; letter-spacing:-.045em; line-height:1; margin:8px 0 0; font-weight:700; } .eyebrow { display:inline-block; padding:5px 14px; border-radius:99px; background:var(--lime); font-size:12px; font-weight:700; } header.top .meta { color:var(--muted); font-size:12px; text-align:left; }
   .kpis { display:grid; grid-template-columns:repeat(5,1fr); margin:24px 0 52px; } .kpi { min-height:96px; padding:8px 18px; border-left:1px solid var(--line); } .kpi:first-child { border:0; padding-left:0; } .kpi .v { font-size:28px; letter-spacing:-.045em; font-weight:700; } .kpi .l { color:var(--muted); font-size:11px; font-weight:700; letter-spacing:.08em; text-transform:uppercase; margin-top:8px; }
-  section { margin:56px 0; } section > h2 { font-size:20px; letter-spacing:-.035em; margin:0 0 22px; } section > h2 .sub { color:var(--muted); font-size:13px; font-weight:400; letter-spacing:0; } .grid2 { display:grid; grid-template-columns:1.1fr 1fr; gap:44px; } .panel { padding:0; } .panel h3 { color:var(--muted); letter-spacing:.07em; text-transform:uppercase; font-size:12px; margin:0 0 16px; }
+  section { margin:56px 0; } section > h2 { font-size:20px; letter-spacing:-.035em; margin:0 0 22px; } .grid2 { display:grid; grid-template-columns:1.1fr 1fr; gap:44px; } .panel { padding:0; } .panel h3 { color:var(--muted); letter-spacing:.07em; text-transform:uppercase; font-size:12px; margin:0 0 16px; }
   table { width: 100%; border-collapse: collapse; font-size: 13px; }
   th, td { text-align: left; padding: 10px 8px; border-bottom: 1px solid var(--line); white-space: nowrap; }
   th { color: var(--muted); font-weight: 600; font-size: 11.5px; text-transform: uppercase; letter-spacing: .03em; cursor: pointer; user-select: none; }
   th:hover { color: var(--text); }
   th .arrow { opacity: .5; font-size: 10px; margin-left: 3px; }
   tbody tr:hover { background: rgba(255,255,255,0.02); }
-  tbody tr.run-row { cursor: pointer; }
   .pill { display: inline-block; padding: 1px 8px; border-radius: 100px; font-size: 11px; font-weight: 600; }
   .pill.ok { background: rgba(62,207,142,.15); color: var(--good); }
   .pill.bad { background: rgba(239,90,111,.15); color: var(--bad); }
@@ -33,17 +32,6 @@
   .legend .sw { display: inline-block; width: 9px; height: 9px; border-radius: 2px; margin-right: 5px; vertical-align: -1px; }
   .heat-table td { text-align: center; font-variant-numeric: tabular-nums; }
   .heat-cell { display: inline-block; min-width: 44px; padding: 3px 6px; border-radius: 5px; font-size: 12px; }
-  details.run-detail { margin: 0; }
-  details.run-detail > summary { display: none; }
-  .timeline { margin: 0 0 18px; background:var(--wash); padding:0 14px; overflow: hidden; }
-  .timeline .step { padding: 10px 14px; border-bottom: 1px solid var(--border); }
-  .timeline .step:last-child { border-bottom: none; }
-  .timeline .step .head { display: flex; justify-content: space-between; gap: 10px; align-items: baseline; margin-bottom: 4px; }
-  .timeline .step .n { color: var(--accent); font-weight: 700; font-size: 12px; }
-  .timeline .step .stats { color: var(--muted); font-size: 11.5px; }
-  .timeline .step .thought { color: var(--muted); font-size: 12.5px; margin-bottom: 6px; }
-  .timeline .step pre { background: var(--panel-2); border-radius: 6px; padding: 8px 10px; margin: 4px 0; font-size: 12px; overflow-x: auto; white-space: pre-wrap; word-break: break-word; }
-  .rc0 { color: var(--good); } .rcN { color: var(--bad); }
   .toolbar { display: flex; gap: 8px; margin-bottom: 12px; flex-wrap: wrap; align-items: center; }
   .toolbar select, .toolbar input { background: var(--wash); border:0; color:var(--ink); border-radius:4px; padding:7px 9px; font-size:12px; } .model-filter { align-items:end; } .model-filter > span { color:var(--muted); font-size:11px; } .model-menu { position:relative; } .model-menu summary { cursor:pointer; list-style:none; min-width:220px; padding:9px 32px 9px 11px; background:var(--wash); border-radius:5px; font-size:12px; position:relative; } .model-menu summary::-webkit-details-marker { display:none; } .model-menu summary::after { content:"⌄"; position:absolute; right:11px; color:var(--muted); } .model-options { position:absolute; z-index:2; top:calc(100% + 6px); left:0; width:260px; max-height:240px; overflow:auto; padding:6px; background:#fff; border:1px solid var(--line); box-shadow:0 10px 24px rgba(17,19,26,.1); } .model-option { display:flex; gap:9px; align-items:center; padding:8px; font-size:12px; cursor:pointer; } .model-option:hover { background:var(--wash); } .model-option input { accent-color:var(--blue); }
   .toolbar label { color: var(--muted); font-size: 12px; }
@@ -61,6 +49,17 @@
   .result-badge.passed { background:rgba(57,174,118,.14); color:#16734a; }
   .result-badge.failed { background:rgba(216,86,104,.14); color:#ad3042; }
   .result-badge.unscored { background:var(--wash); color:var(--muted); }
+  .score-chart-scroll { overflow-x:auto; padding:4px 0 12px; }
+  .score-chart { position:relative; height:280px; padding:18px 12px 68px 44px; display:flex; align-items:end; gap:4px; background:repeating-linear-gradient(to top, transparent 0, transparent 25%, rgba(228,231,235,.8) 25.2%, transparent 25.5%); }
+  .score-axis { position:absolute; left:0; color:var(--muted); font-size:10px; font-variant-numeric:tabular-nums; }
+  .score-axis.top { top:12px; } .score-axis.mid { top:50%; } .score-axis.zero { bottom:62px; }
+  .score-item { position:relative; height:100%; min-width:40px; flex:1 0 40px; display:flex; align-items:end; }
+  .score-bar { width:100%; min-height:2px; position:relative; border-radius:4px 4px 0 0; background:var(--blue); }
+  .score-bar.unscored { height:3px !important; }
+  .score-value { position:absolute; left:0; right:0; bottom:7px; color:#fff; text-align:center; font-size:10px; font-weight:700; font-variant-numeric:tabular-nums; }
+  .score-bar.unscored .score-value { color:var(--muted); bottom:7px; }
+  .score-label { position:absolute; top:calc(100% + 8px); left:0; width:40px; color:var(--muted); font-size:8px; line-height:1.2; overflow-wrap:anywhere; text-align:center; white-space:normal; transform:none; }
+  @media(max-width:800px){ .score-chart { height:260px; padding-bottom:64px; } .score-axis.zero { bottom:58px; } }
   footer { color: var(--muted); font-size: 11.5px; margin-top: 40px; text-align: center; }
   svg text { fill: var(--muted); font-size: 10.5px; }
   .axis line, .axis path { stroke: var(--line); }
@@ -70,15 +69,19 @@
 <body>
 <div class="wrap">
   <header class="top">
-    <div><h1>Benchmark report</h1><div class="meta" id="generated"></div><div class="meta" id="schema"></div></div>
+    <div><h1>Benchmark report</h1><div class="meta" id="generated"></div></div>
   </header>
 
   <div class="kpis" id="kpis"></div>
 
+  <section id="section-score-distribution">
+    <h2>Score distribution</h2>
+    <div class="score-chart-scroll"><div id="chart-score-distribution" role="img" aria-label="Verified score by model and reasoning configuration"></div></div>
+  </section>
+
   <section>
-    <h2>Run outcomes <span class="sub">generation completion versus verified correctness</span></h2>
+    <h2>Run outcomes</h2>
     <div class="toolbar model-filter">
-      <span>Models</span>
       <details class="model-menu">
         <summary id="model-filter-label">All models</summary>
         <div id="leaderboard-model-options" class="model-options"></div>
@@ -90,22 +93,15 @@
   </section>
 
   <section>
-    <h2>Test-case results <span class="sub">points earned by one selected model</span></h2>
+    <h2>Test-case results</h2>
     <div class="toolbar"><label for="verifier-model-select">Model</label><select id="verifier-model-select"></select></div>
     <div class="panel">
       <table id="tbl-verifier-matrix"></table>
     </div>
   </section>
 
-  <section>
-    <h2>Run details <span class="sub">generation and benchmarking status for each attempt</span></h2>
-    <div class="panel">
-      <table id="tbl-runs"></table>
-    </div>
-  </section>
-
   <section id="section-series">
-    <h2>Trajectory <span class="sub">where context and cost accumulate</span></h2>
+    <h2>Trajectory</h2>
     <div class="grid2">
       <div class="panel">
         <h3>Prompt tokens (context) vs. step</h3>
@@ -150,6 +146,21 @@
   }
 
   function fmt(n) { return Number(n || 0).toLocaleString("en-US"); }
+  function fmtDuration(seconds) {
+    var total = Math.max(0, Math.round(Number(seconds) || 0));
+    if (total < 60) return total + "s";
+    var minutes = Math.floor(total / 60);
+    var remainder = total % 60;
+    if (minutes < 60) return minutes + "m " + String(remainder).padStart(2, "0") + "s";
+    var hours = Math.floor(minutes / 60);
+    return hours + "h " + String(minutes % 60).padStart(2, "0") + "m";
+  }
+  function fmtTokens(tokens) {
+    var value = Math.max(0, Number(tokens) || 0);
+    if (value < 1000) return fmt(Math.round(value));
+    if (value < 1000000) return (value / 1000).toFixed(1).replace(/\.0$/, "") + "K";
+    return (value / 1000000).toFixed(1).replace(/\.0$/, "") + "M";
+  }
   function fmt1(n) { return (Number(n) || 0).toFixed(1); }
   function pct(n) { return Math.round((Number(n) || 0) * 100) + "%"; }
   function fmtUsd(n) {
@@ -179,19 +190,17 @@
 
   // ---------------------------------------------------------------- header
   document.getElementById("generated").textContent = DATA.generated_at
-    ? "Generated " + new Date(DATA.generated_at).toLocaleString("en-US", { dateStyle: "medium", timeStyle: "short" })
+    ? "Generated on " + new Date(DATA.generated_at).toLocaleString("en-US", { dateStyle: "medium", timeStyle: "short" })
     : "";
-  document.getElementById("schema").textContent = DATA.schema || "";
-
   // ---------------------------------------------------------------- KPIs
   (function renderKpis() {
     var k = DATA.kpis || {};
     var cards = [
-      ["Models Benchmarked", fmt(k.models)],
+      ["Model–reasoning configurations", fmt(k.models)],
       ["Average score", k.average_score == null ? "n/a" : fmt1(k.average_score) + "%"],
-      ["Total duration", fmt(k.elapsed_seconds) + "s"],
+      ["Average duration", fmtDuration(k.runs ? k.elapsed_seconds / k.runs : 0)],
       ["Total cost", fmtUsd(k.cost_usd) + (k.cost_source === "estimated" ? "~" : "")],
-      ["Tokens used", fmt(k.tokens)],
+      ["Tokens used", fmtTokens(k.tokens)],
     ];
     var host = document.getElementById("kpis");
     cards.forEach(function (c) {
@@ -199,6 +208,37 @@
     });
   })();
 
+  // ---------------------------------------------------------------- score distribution
+  (function renderScoreDistribution() {
+    var host = document.getElementById("chart-score-distribution");
+    if (!host) return;
+    var rows = (DATA.leaderboard || []).map(function (row) {
+      var verified = typeof row.average_score === "number";
+      return { slug: row.slug, score: verified ? Math.max(0, Math.min(100, row.average_score)) : null, verified: verified };
+    }).sort(function (a, b) {
+      if (a.verified !== b.verified) return a.verified ? -1 : 1;
+      return (b.score || 0) - (a.score || 0) || a.slug.localeCompare(b.slug);
+    });
+    if (!rows.length) {
+      document.getElementById("section-score-distribution").style.display = "none";
+      return;
+    }
+    host.className = "score-chart";
+    host.style.minWidth = Math.max(760, rows.length * 45) + "px";
+    [["top", "100"], ["mid", "50"], ["zero", "0"]].forEach(function (axis) {
+      host.appendChild(el("span", { class: "score-axis " + axis[0] }, axis[1]));
+    });
+    rows.forEach(function (row) {
+      var item = el("div", { class: "score-item" });
+      var bar = el("div", { class: "score-bar", title: row.slug + ": " + (row.verified ? fmt1(row.score) + "%" : "n/a") });
+      if (row.verified) bar.style.height = Math.max(2, row.score) + "%";
+      bar.appendChild(el("span", { class: "score-value" }, row.verified ? fmt1(row.score) + "%" : "n/a"));
+      item.appendChild(bar);
+      item.appendChild(el("span", { class: "score-label" }, esc(row.slug)));
+      host.appendChild(item);
+    });
+  })();
+
   // ---------------------------------------------------------------- sortable table helper
   function sortableTable(container, columns, rows, opts) {
     opts = opts || {};
@@ -206,6 +246,10 @@
     function draw() {
       var sorted = rows.slice().sort(function (a, b) {
         var va = a[state.key], vb = b[state.key];
+        if (va == null || vb == null) {
+          if (va == null && vb == null) return 0;
+          return va == null ? 1 : -1;
+        }
         if (typeof va === "string" || typeof vb === "string") {
           return state.dir * String(va).localeCompare(String(vb));
         }
@@ -291,13 +335,13 @@
       { key: "runs", label: "Runs", render: function (r) { return fmt(r.runs); } },
       { key: "average_score", label: "Score", render: function (r) { return r.average_score == null ? '<span class="n-a">n/a</span>' : fmt1(r.average_score) + "%"; } },
       { key: "sum_steps", label: "Total steps", render: function (r) { return fmt(r.sum_steps); } },
-      { key: "sum_elapsed_seconds", label: "Time taken", render: function (r) { return fmt(r.sum_elapsed_seconds) + "s"; } },
-      { key: "sum_tokens", label: "Tokens", render: function (r) { return fmt(r.sum_tokens); } },
+      { key: "sum_elapsed_seconds", label: "Time taken", render: function (r) { return fmtDuration(r.sum_elapsed_seconds); } },
+      { key: "sum_tokens", label: "Tokens", render: function (r) { return fmtTokens(r.sum_tokens); } },
       { key: "cost_usd", label: "Cost", render: function (r) { return fmtUsd(r.cost_usd) + (r.cost_source === "estimated" ? "~" : ""); } },
     ];
     (DATA.leaderboard || []).forEach(function (r) {
       var option = el("label", { class: "model-option" });
-      option.appendChild(el("input", { type: "checkbox", value: r.slug }));
+      option.appendChild(el("input", { type: "checkbox", value: r.slug, checked: "checked" }));
       option.appendChild(document.createTextNode(r.model + " · " + r.reasoning));
       options.appendChild(option);
     });
@@ -305,7 +349,7 @@
       var selected = Array.prototype.slice.call(options.querySelectorAll("input:checked")).map(function (o) { return o.value; });
       var rows = selected.length ? (DATA.leaderboard || []).filter(function (r) { return selected.indexOf(r.slug) !== -1; }) : DATA.leaderboard || [];
       filterLabel.textContent = selected.length ? selected.length + " model" + (selected.length === 1 ? "" : "s") + " selected" : "All models";
-      sortableTable(container, columns, rows, { defaultKey: "sum_tokens" });
+      sortableTable(container, columns, rows, { defaultKey: "average_score" });
     }
     options.addEventListener("change", draw);
     draw();
@@ -389,7 +433,7 @@
         if (w > 0) track.appendChild(el("div", { class: "bar-seg", style: "width:" + w + "%;background:" + colors[k] }));
       });
       row.appendChild(track);
-      row.appendChild(el("div", { class: "bar-val" }, fmt(total)));
+      row.appendChild(el("div", { class: "bar-val" }, fmtTokens(total)));
       host.appendChild(row);
     });
     keys.forEach(function (k) {
@@ -414,7 +458,7 @@
     [0, 0.25, 0.5, 0.75, 1].forEach(function (t) {
       var ty = y(maxDuration * t);
       var lab = svgEl("text", { x: pad.l - 8, y: ty + 3, "text-anchor": "end" });
-      lab.textContent = Math.round(maxDuration * t) + "s";
+      lab.textContent = fmtDuration(maxDuration * t);
       svg.appendChild(lab);
     });
     var lab2 = svgEl("text", { x: W - pad.r, y: H - pad.b + 20, "text-anchor": "end" });
@@ -510,7 +554,7 @@
       document.getElementById("section-series").style.display = "none";
       return;
     }
-    lineChart(document.getElementById("chart-line-tokens"), series, function (p) { return p.prompt_tokens.p50; }, { fmt: fmt });
+    lineChart(document.getElementById("chart-line-tokens"), series, function (p) { return p.prompt_tokens.p50; }, { fmt: fmtTokens });
     lineChart(document.getElementById("chart-line-cost"), series, function (p) { return p.cum_cost_usd.mean; }, { fmt: fmtUsd });
     var legend = document.getElementById("legend-series");
     series.forEach(function (s) {
@@ -553,99 +597,6 @@
     container.innerHTML = thead + tbody;
   })();
 
-  // ---------------------------------------------------------------- runs table + filters + expandable timeline
-  (function () {
-    var runs = DATA.runs || [];
-    var container = document.getElementById("tbl-runs");
-    var fModel = document.getElementById("filter-model");
-    var fTask = document.getElementById("filter-task");
-    var fOutcome = document.getElementById("filter-outcome");
-    if (!container) return;
-
-    function uniq(fn) {
-      var seen = {}, out = [];
-      runs.forEach(function (r) { var v = fn(r); if (v && !seen[v]) { seen[v] = 1; out.push(v); } });
-      return out.sort();
-    }
-    if (fModel) uniq(function (r) { return r.slug; }).forEach(function (v) { fModel.appendChild(el("option", { value: v }, esc(v))); });
-    if (fTask) uniq(function (r) { return r.task_id; }).forEach(function (v) { fTask.appendChild(el("option", { value: v }, esc(v))); });
-    if (fOutcome) uniq(function (r) { return r.status; }).forEach(function (v) { fOutcome.appendChild(el("option", { value: v }, esc(v))); });
-
-    var columns = [
-      { key: "slug", label: "Model", render: function (r) { return '<span class="mono">' + esc(r.slug) + "</span>"; } },
-      { key: "status", label: "generation_status", render: function (r) {
-          var cls = r.completed ? "ok" : (r.status === "request_error" ? "bad" : "warn");
-          return '<span class="pill ' + cls + '">' + esc(r.status || "Unknown") + "</span>";
-        } },
-      { key: "verifier_status", label: "benchmarking_status", render: function (r) {
-          var cls = r.verifier_status === "passed" ? "ok" : (r.verifier_status === "not_configured" ? "warn" : "bad");
-          return '<span class="pill ' + cls + '">' + esc(r.verifier_status || "NA") + "</span>";
-        } },
-      { key: "solved_score", label: "Solved", render: function (r) { return typeof r.solved_score === "number" ? fmt1(r.solved_score) + "%" : '<span class="n-a">NA</span>'; } },
-      { key: "steps", label: "Steps", render: function (r) { return fmt(r.steps); } },
-      { key: "elapsed_seconds", label: "Time", render: function (r) { return fmt(r.elapsed_seconds) + "s"; } },
-      { key: "tokens", label: "Tokens", render: function (r) { return fmt(r.tokens.total); } },
-      { key: "cost_usd", label: "Cost", render: function (r) { return fmtUsd(r.cost_usd) + (r.cost_source === "estimated" ? "~" : ""); } },
-      { key: "n_failed_commands", label: "Fails", render: function (r) { return fmt(r.n_failed_commands); } },
-    ];
-
-    function renderTimeline(run) {
-      var box = el("div", { class: "timeline" });
-      (run.timeline || []).forEach(function (s) {
-        var rcClass = s.returncode === 0 ? "rc0" : (s.returncode == null ? "muted" : "rcN");
-        var step = el("div", { class: "step" });
-        step.innerHTML =
-          '<div class="head"><span class="n">step ' + s.step + '</span>' +
-          '<span class="stats">' +
-            fmt(s.tokens.total) + " tok" +
-            (s.cost_usd != null ? " \u00b7 " + fmtUsd(s.cost_usd) : "") +
-            (s.returncode != null ? ' \u00b7 rc=<span class="' + rcClass + '">' + s.returncode + "</span>" : "") +
-          "</span></div>" +
-          (s.thought ? '<div class="thought">' + esc(s.thought) + "</div>" : "") +
-          (s.command ? "<pre>$ " + esc(s.command) + "</pre>" : "") +
-          (s.output_preview ? "<pre>" + esc(s.output_preview) + "</pre>" : "");
-        box.appendChild(step);
-      });
-      return box;
-    }
-
-    function draw() {
-      var filtered = runs.filter(function (r) {
-        if (fModel && fModel.value && r.slug !== fModel.value) return false;
-        if (fTask && fTask.value && r.task_id !== fTask.value) return false;
-        if (fOutcome && fOutcome.value && r.status !== fOutcome.value) return false;
-        return true;
-      });
-      sortableTable(container, columns, filtered, {
-        defaultKey: "slug",
-        defaultDir: 1,
-        rowAttrs: function (r, i) { return 'class="run-row" data-idx="' + i + '"'; },
-        afterDraw: function (containerEl, sorted) {
-          containerEl.querySelectorAll("tbody tr").forEach(function (tr) {
-            var idx = Number(tr.getAttribute("data-idx"));
-            var run = sorted[idx];
-            tr.addEventListener("click", function () {
-              var next = tr.nextElementSibling;
-              if (next && next.classList.contains("detail-row")) {
-                next.remove();
-                return;
-              }
-              containerEl.querySelectorAll(".detail-row").forEach(function (d) { d.remove(); });
-              var detailRow = document.createElement("tr");
-              detailRow.className = "detail-row";
-              var td = document.createElement("td");
-              td.colSpan = columns.length;
-              td.appendChild(renderTimeline(run));
-              detailRow.appendChild(td);
-              tr.parentNode.insertBefore(detailRow, tr.nextSibling);
-            });
-          });
-        },
-      });
-    }
-    [fModel, fTask, fOutcome].filter(Boolean).forEach(function (elm) { elm.addEventListener("change", draw); });
-    draw();
-  })();
 })();
 </script>
 </body>

package/src/report.js

@@ -148,7 +148,13 @@ export function buildSummary(runs) {
       avg_n_retries: avg(rs, (r) => r.n_retries),
     };
   });
-  leaderboard.sort((a, b) => b.completion_rate - a.completion_rate || a.avg_steps - b.avg_steps);
+  leaderboard.sort((a, b) => {
+    const aScored = a.average_score != null;
+    const bScored = b.average_score != null;
+    if (aScored !== bScored) return aScored ? -1 : 1;
+    return (b.average_score ?? 0) - (a.average_score ?? 0)
+      || a.slug.localeCompare(b.slug);
+  });
 
   const matrix = groupBy(runs, (r) => `${r.slug}\u0000${r.task_id}`).map(([, rs]) => {
     const completed = rs.filter((r) => r.completed).length;

package/src/verifier-runner.js

@@ -153,7 +153,7 @@ export async function runVerifier({ project, cell, runDir, candidateDir, config,
     if (!validation.ok) return { status: "error", failureOwner: "infrastructure", solvedScore: "NA", durationSeconds, error: validation.error, exitCode: commandResult.code };
     return {
       status: raw.status,
-      failureOwner: raw.status === "failed" ? "candidate" : raw.status === "passed" ? null : "infrastructure",
+      failureOwner: raw.status === "failed" || raw.status === "blocked" ? "candidate" : raw.status === "passed" ? null : "infrastructure",
       score: raw.score,
       solvedScore: raw.score?.percentage ?? "NA",
       criteria: raw.criteria,