@hackerrank/astra-cli 0.1.10 → 0.1.12

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hackerrank/astra-cli",
-  "version": "0.1.10",
+  "version": "0.1.12",
   "description": "Minimal zero-dependency AI coding agent for the HackerRank AI Gateway.",
   "type": "module",
   "bin": {

package/src/report.html

@@ -85,8 +85,8 @@
   </section>
 
   <section>
-    <h2>Test-case results <span class="sub">passed, failed, blocked, and errored criteria by model</span></h2>
-    <div id="verifier-model-pills" class="model-pills"></div>
+    <h2>Test-case results <span class="sub">points earned by one selected model</span></h2>
+    <div class="toolbar"><label for="verifier-model-select">Model</label><select id="verifier-model-select"></select></div>
     <div class="panel">
       <table id="tbl-verifier-matrix"></table>
     </div>
@@ -312,31 +312,21 @@
   // ---------------------------------------------------------------- verifier test-case matrix
   (function () {
     var container = document.getElementById("tbl-verifier-matrix");
-    var pills = document.getElementById("verifier-model-pills");
+    var selector = document.getElementById("verifier-model-select");
     if (!container) return;
     var matrix = DATA.verifier_matrix || { models: [], criteria: [] };
-    var active = matrix.models.slice();
-    function cellHtml(cell) {
-      if (!cell) return '<div class="criterion-cell na">NA</div>';
-      var rate = cell.total ? Math.round(cell.passed / cell.total * 100) : 0;
-      var cls = rate === 100 ? "pass" : (cell.failed || cell.blocked || cell.error ? "fail" : "na");
-      return '<div class="criterion-cell ' + cls + '"><span class="rate">' + rate + '%</span><span class="counts">' + cell.passed + ' pass · ' + (cell.failed + cell.blocked + cell.error) + ' fail</span></div>';
-    }
+    var selected = matrix.models[0] || "";
     function draw() {
-      var visible = matrix.models.filter(function (model) { return active.indexOf(model) !== -1; });
-      container.innerHTML = '<thead><tr><th>Test case</th>' + visible.map(function (model) { return '<th><span class="mono">' + esc(model) + '</span></th>'; }).join('') + '</tr></thead>' +
-        '<tbody>' + matrix.criteria.map(function (criterion) { return '<tr><td><div class="mono">' + esc(criterion.id) + '</div><div class="sub">' + esc(criterion.title || criterion.id) + '</div></td>' + visible.map(function (model) { return '<td>' + cellHtml(criterion.cells[model]) + '</td>'; }).join('') + '</tr>'; }).join('') + '</tbody>';
+      container.innerHTML = '<thead><tr><th>Test case</th><th>Available</th><th>Earned</th><th>Result</th></tr></thead><tbody>' + matrix.criteria.map(function (criterion) {
+        var cell = criterion.cells[selected];
+        if (!cell) return '<tr><td>' + esc(criterion.title || criterion.id) + '</td><td class="n-a">NA</td><td class="n-a">NA</td><td class="n-a">Unscored</td></tr>';
+        var max = cell.maxPoints || 0, earned = cell.earnedPoints || 0;
+        var status = cell.passed === cell.total ? 'Passed' : (cell.error ? 'Errored' : cell.blocked ? 'Blocked' : 'Failed');
+        return '<tr><td>' + esc(criterion.title || criterion.id) + '</td><td>' + fmt1(max) + '</td><td>' + fmt1(earned) + '</td><td>' + esc(status) + '</td></tr>';
+      }).join('') + '</tbody>';
     }
-    matrix.models.forEach(function (model, index) {
-      var pill = el("button", { class: "model-pill active", type: "button" }, esc(model));
-      pill.addEventListener("click", function () {
-        var pos = active.indexOf(model);
-        if (pos === -1) { active.push(model); pill.classList.add("active"); }
-        else if (active.length > 1) { active.splice(pos, 1); pill.classList.remove("active"); }
-        draw();
-      });
-      pills.appendChild(pill);
-    });
+    matrix.models.forEach(function (model) { selector.appendChild(el("option", { value: model }, esc(model))); });
+    selector.addEventListener("change", function () { selected = selector.value; draw(); });
     if (!matrix.criteria.length) {
       container.innerHTML = '<tbody><tr><td class="muted">No verifier criteria have produced results yet.</td></tr></tbody>';
       return;

package/src/report.js

@@ -20,7 +20,11 @@ export function refreshReport(root) {
   if (!fs.existsSync(rootDir)) {
     throw new Error(`bench root not found: ${rootDir}`);
   }
-  const runs = scanRuns(rootDir);
+  // Task versions are immutable benchmark definitions. A task output directory
+  // may retain old attempts for auditability, but combining versions changes
+  // the measured task and makes the dashboard misleading. Show the newest
+  // version for each task while preserving unversioned legacy runs.
+  const runs = latestTaskRuns(scanRuns(rootDir));
   const summary = buildSummary(runs);
 
   for (const run of runs) {
@@ -82,6 +86,21 @@ export function scanRuns(rootDir) {
   return runs;
 }
 
+function latestTaskRuns(runs) {
+  const latestVersionByTask = new Map();
+  for (const run of runs) {
+    const taskId = run.project?.id;
+    const version = Number(run.project?.version);
+    if (!taskId || !Number.isFinite(version)) continue;
+    latestVersionByTask.set(taskId, Math.max(latestVersionByTask.get(taskId) ?? version, version));
+  }
+  return runs.filter((run) => {
+    const taskId = run.project?.id;
+    const version = Number(run.project?.version);
+    return !taskId || !Number.isFinite(version) || latestVersionByTask.get(taskId) === version;
+  });
+}
+
 export function buildSummary(runs) {
   runs = runs.map((run) => ({ ...run, criteria: flattenCriteria(run.criteria) }));
   const verifierCriteria = summarizeCriteria(runs);
@@ -245,7 +264,10 @@ export function renderReportHtml(summary) {
   if (!tpl.includes("/*__ASTRA_DATA__*/")) {
     throw new Error("src/report.html is missing the /*__ASTRA_DATA__*/ injection marker");
   }
-  return tpl.replace("/*__ASTRA_DATA__*/", `window.__ASTRA_EMBEDDED_DATA__ = ${data};`);
+  // Use a callback replacement: a report payload can contain shell snippets
+  // such as `$'...'` or `$&`, which String.replace treats as replacement
+  // tokens when given a plain string and can therefore corrupt the HTML.
+  return tpl.replace("/*__ASTRA_DATA__*/", () => `window.__ASTRA_EMBEDDED_DATA__ = ${data};`);
 }
 
 function toRunDoc(run) {
@@ -521,9 +543,12 @@ function buildVerifierMatrix(runs) {
       if (!byCriterion.has(criterion.id)) byCriterion.set(criterion.id, { title: criterion.title || criterion.id, byModel: new Map() });
       const entry = byCriterion.get(criterion.id);
       const byModel = entry.byModel;
-      if (!byModel.has(run.slug)) byModel.set(run.slug, { passed: 0, failed: 0, blocked: 0, error: 0, total: 0 });
+      if (!byModel.has(run.slug)) byModel.set(run.slug, { passed: 0, failed: 0, blocked: 0, error: 0, total: 0, maxPoints: 0, earnedPoints: 0 });
       const cell = byModel.get(run.slug);
       cell.total += 1;
+      const weight = Math.max(0, Number(criterion.weight) || 0);
+      cell.maxPoints += weight;
+      if (criterion.status === "passed") cell.earnedPoints += weight;
       if (criterion.status === "passed") cell.passed += 1;
       else if (criterion.status === "blocked") cell.blocked += 1;
       else if (criterion.status === "error") cell.error += 1;