niceeval 0.9.0 → 0.10.0

package/src/shared/aggregate.test.ts

@@ -0,0 +1,52 @@
+// cases: docs/engineering/unit-tests/experiments-runner/cases.md
+import { describe, expect, it } from "vitest";
+import { matchExperimentSelector } from "./aggregate.ts";
+
+describe("matchExperimentSelector", () => {
+  const ids = [
+    "compare/bub-gpt-5.6-luna",
+    "compare/codex-gpt-5.6-luna",
+    "compare/codex-gpt-5.6-luna--agents-md",
+    "compare/codex-gpt-5.6-luna--mempal",
+    "dev-e2b/bub-e2b",
+    "dev/bub-gpt-5.4-mini",
+    "top-level",
+  ];
+
+  it("selects an entire group by exact directory match", () => {
+    expect(matchExperimentSelector(ids, "compare")).toEqual([
+      "compare/bub-gpt-5.6-luna",
+      "compare/codex-gpt-5.6-luna",
+      "compare/codex-gpt-5.6-luna--agents-md",
+      "compare/codex-gpt-5.6-luna--mempal",
+    ]);
+  });
+
+  it("an exact id match wins even when it is a prefix of sibling variants", () => {
+    expect(matchExperimentSelector(ids, "compare/codex-gpt-5.6-luna")).toEqual([
+      "compare/codex-gpt-5.6-luna",
+    ]);
+  });
+
+  it("falls back to a filename prefix within an exact-matched directory to select a family", () => {
+    expect(matchExperimentSelector(ids, "compare/codex")).toEqual([
+      "compare/codex-gpt-5.6-luna",
+      "compare/codex-gpt-5.6-luna--agents-md",
+      "compare/codex-gpt-5.6-luna--mempal",
+    ]);
+  });
+
+  it("requires an exact directory segment; a group-like prefix does not leak across groups", () => {
+    expect(matchExperimentSelector(ids, "dev")).toEqual(["dev/bub-gpt-5.4-mini"]);
+    expect(matchExperimentSelector(ids, "dev")).not.toContain("dev-e2b/bub-e2b");
+  });
+
+  it("matches a top-level (groupless) experiment by exact id", () => {
+    expect(matchExperimentSelector(ids, "top-level")).toEqual(["top-level"]);
+  });
+
+  it("returns empty when nothing matches", () => {
+    expect(matchExperimentSelector(ids, "dev-e3b")).toEqual([]);
+    expect(matchExperimentSelector(ids, "compare/nope")).toEqual([]);
+  });
+});

package/src/shared/aggregate.ts

@@ -47,7 +47,7 @@ export function experimentGroupOf(experimentId: string): string | undefined {
 /**
  * eval id 前缀过滤,同 CLI 位置参数语义(docs/feature/reports/show.md「打开与收窄」):
  * eval 位置参数是收窄过滤,按**裸前缀宽松匹配**——"algebra" 命中 "algebra"、"algebra/..."
- * 也命中 "algebra2",多命中正是它的用途(与 `--experiment` 的按路径段匹配有意不同)。
+ * 也命中 "algebra2",多命中正是它的用途(与 `--exp` 的按路径段匹配有意不同)。
  */
 export function evalPrefixPredicate(evals?: string | string[]): (id: string) => boolean {
   if (evals === undefined) return () => true;
@@ -55,6 +55,35 @@ export function evalPrefixPredicate(evals?: string | string[]): (id: string) =>
   return (id) => prefixes.some((prefix) => id.startsWith(prefix));
 }
 
+/**
+ * 实验选择器解析:`niceeval exp` 位置参数与 `--exp` 共用同一条规则(docs/feature/experiments/cli.md
+ * 「实验选择器怎样解析」)。按序:①精确 id 优先,即使它同时是同目录内其它文件名的前缀;
+ * ②目录段精确前缀(含更深层目录),选中该目录下全部实验;③以上都不命中且选择器形如
+ * `目录/文件名前缀` 时,目录段精确匹配后按文件名段前缀选中一族配置——目录段永远精确,不跨目录
+ * 误配,文件名段裸前缀,与 `evalPrefixPredicate` 同一条"共享前缀即家族"的逻辑。
+ */
+export function matchExperimentSelector(ids: readonly string[], selector: string): string[] {
+  const exact = ids.find((id) => id === selector);
+  if (exact !== undefined) return [exact];
+  const dirMatches = ids.filter((id) => id.startsWith(selector + "/"));
+  if (dirMatches.length > 0) return dirMatches;
+  const lastSlash = selector.lastIndexOf("/");
+  if (lastSlash === -1) return [];
+  const dir = selector.slice(0, lastSlash);
+  const namePrefix = selector.slice(lastSlash + 1);
+  if (namePrefix === "") return [];
+  return ids.filter((id) => id.startsWith(dir + "/") && id.slice(dir.length + 1).startsWith(namePrefix));
+}
+
+/**
+ * 零命中时展示的可浏览路径清单(docs/feature/experiments/cli.md「零命中」):按 id 的顶层目录段
+ * 去重、排序,带尾斜杠;没有目录段的顶层 experiment id 原样列出(它本身就是一个可选择的目标,
+ * 不是一段路径)。这是运行选择的展示投影,不是报告分组。
+ */
+export function browsableExperimentPaths(ids: readonly string[]): string[] {
+  return [...new Set(ids.map((id) => (id.includes("/") ? `${id.split("/")[0]}/` : id)))].sort();
+}
+
 /** 无 experimentId 时的兜底标签。 */
 export function fallbackExperimentLabel(result: {
   experimentId?: string;

package/src/shared/types.ts

@@ -30,7 +30,12 @@ export interface SourceArtifact {
   content: string;
 }
 
-/** 通用清理闭包(setup 返回值 / teardown 的形状),异步同步皆可,统一在 finally 里执行。 */
+/**
+ * 内部收尾闭包类型:供运行器内部的清理注册表使用(如 `postSetup` 钩子内部累积待收尾动作),
+ * 不出现在任何公开的生命周期钩子签名里——`SandboxHook`/`AgentSetup`/`AgentTeardown` 等公开
+ * 钩子一律 `void | Promise<void>`,setup 不返回值,收尾靠成对的 teardown(见
+ * docs/runner.md「环境预置不进运行器,但按顺序调它」)。
+ */
 export type Cleanup = () => Promise<void> | void;
 
 /** `ScopedFeedback.progress` 的入参:此刻正在做什么(短命状态,可被后续更新覆盖)。 */

package/src/show/command.test.ts

@@ -0,0 +1,34 @@
+// cases: docs/engineering/unit-tests/reports/cases.md
+// 「外壳、页面与 Tabs」——show 专属的索引命令拼装(showCommand)与其余页索引文案(otherPagesText)。
+
+import { describe, expect, it } from "vitest";
+import { showCommand } from "./command.ts";
+import { otherPagesText } from "./render.ts";
+
+describe("其余页索引与索引命令上下文", () => {
+  const pages = [
+    { id: "overview", title: { en: "Overview", "zh-CN": "总览" } },
+    { id: "exam", title: { en: "Exam", "zh-CN": "成绩单" } },
+  ];
+
+  it("只列未渲染的页,索引命令保留当前 --results / --report 与位置参数,复制即可复现下一层视图", () => {
+    // 渲染的是 overview,其余页索引只含 exam 一行——与「渲染初始页 + 尾部附其余页索引」
+    // 的行为一致(docs/feature/reports/show/reports.md Case 2)。
+    const text = otherPagesText({
+      otherPages: pages.filter((p) => p.id !== "overview"),
+      command: { patterns: [], results: "tmp/published-results", report: "reports/site.tsx" },
+      locale: "zh-CN",
+    });
+    expect(text).toContain("其余页：");
+    expect(text).toContain("niceeval show --results tmp/published-results --report reports/site.tsx --page exam");
+    expect(text).toContain("成绩单");
+    expect(text).not.toContain("总览");
+    expect(text).not.toContain("--page overview");
+  });
+
+  it("showCommand 按序携带位置参数与 --exp / --results / --report / --page", () => {
+    expect(
+      showCommand({ patterns: ["memory/swelancer"], experiment: "dev-e2b", report: "reports/site.tsx", page: "exam" }),
+    ).toBe("niceeval show memory/swelancer --exp dev-e2b --report reports/site.tsx --page exam");
+  });
+});

package/src/show/command.ts

@@ -0,0 +1,16 @@
+// show 专属的可复制命令拼装:页索引 / 组索引共用的携带规则(docs/feature/reports/show/reports.md
+// 「索引命令携带完整上下文」)。这不是两宿主共用的报告 runtime 部分——view 走网页路由,不生成
+// 终端命令——所以住在 show 自己这一侧,不进 src/report/runtime/host.ts。
+
+import type { HostCommandContext } from "../report/runtime/host.ts";
+
+/** 按上下文拼一条可复制的 show 命令(页索引 / 组索引共用的携带规则)。 */
+export function showCommand(ctx: HostCommandContext, extra: string[] = []): string {
+  const parts = ["niceeval show", ...ctx.patterns];
+  if (ctx.experiment !== undefined) parts.push(`--exp ${ctx.experiment}`);
+  if (ctx.results !== undefined) parts.push(`--results ${ctx.results}`);
+  if (ctx.report !== undefined) parts.push(`--report ${ctx.report}`);
+  if (ctx.page !== undefined) parts.push(`--page ${ctx.page}`);
+  parts.push(...extra);
+  return parts.join(" ");
+}

package/src/show/compose.ts

@@ -6,7 +6,7 @@
 // 现刻水位 Scope(两个宿主共用)住在 ../results/select.ts;本文件只留 show 独有的时间轴计算。
 // 数据只消费 niceeval/results 的读取面。
 
-import { attemptCostUSD } from "../report/metrics.ts";
+import { attemptCostUSD } from "../report/model/metrics.ts";
 import { compactAssertionSummary, primaryAssertionSummary, summaryText } from "../scoring/display.ts";
 import type { EvalResult, Verdict } from "../types.ts";
 import type { AttemptHandle, Experiment } from "../results/index.ts";

package/src/show/index.ts

@@ -4,12 +4,13 @@
 // 位置参数 = eval id 前缀,或 `@<locator>`(精确指名单个 attempt,见 results/locator.ts):
 //   裸跑 / 多 eval 前缀  内建报告(niceeval/report/built-in 默认导出)的 text 面(单 eval 前缀仍进入详情)
 //   恰好一个 eval     单 eval 详情(attempt / 断言明细,宿主本体)
-//   @<locator>        精确 attempt:无证据 flag → 紧凑全景;带 flag → 对应证据切面
+//   @<locator>        精确 attempt:无证据 flag → 当前 report 的 attempt-input page(内建 standard
+//                     或 --report 自定义);带 flag → 对应证据切面(宿主本体,不经报告管线)
 //   --source / --execution / --diff[=路径]   证据切面(宿主本体):出现即走证据室,不渲染报告槽
 //   --history        执行时间轴(逐 experimentId + evalId 分节),与 --report 互斥
-//   --report <文件>  整槽换成用户报告;位置前缀 / --results / --experiment 先收窄 Scope 再注入
+//   --report <文件>  整槽换成用户报告;位置前缀 / --results / --exp 先收窄 Scope 再注入
 //   --page <id>      多页报告选页;未命中列出可用页 id 按用法错误退出
-//   --results <目录>  结果根换成该目录;--experiment 让 Scope 只留该实验
+//   --results <目录>  结果根换成该目录;--exp 让 Scope 只留该实验
 //
 // 数据全部走 niceeval/results 的读取面(openResults + 合成 Scope + loadAttemptEvidence),
 // 不自己爬目录;证据可用性只由 loadAttemptEvidence 在单 Attempt 页面计算。
@@ -28,26 +29,25 @@ import {
 // against — `instanceof` is keyed by declaration site, so a raw src copy and the compiled
 // dist copy of "the same" class are two different types. The package-owned report runtime
 // ships as precompiled ESM (dist/report/**, built by `pnpm run build:report`); all report
-// loading/rendering goes through ./report-host.ts (the single contact surface with it).
-import { ReportLoadError } from "../../dist/report/load.js";
+// loading/rendering goes through ../report/runtime/host.ts (the shared contact surface).
+import { ReportLoadError } from "../../dist/report/runtime/load.js";
 import { detectLocale, t } from "../i18n/index.ts";
 import { foldEvalVerdict } from "../shared/verdict.ts";
 import { selectCurrentResults, filterExperiments } from "../results/select.ts";
 import { evalPrefixPredicate } from "../shared/aggregate.ts";
 import { attemptHistory } from "./compose.ts";
 import {
+  buildHostReportMeta,
   HostReportError,
   loadHostReport,
-  reportMetaFor,
   renderHostPageText,
   type HostCommandContext,
-} from "./report-host.ts";
+} from "../report/runtime/host.ts";
 import {
   attemptArtifactsPath,
   attemptEvidenceHeader,
   attemptHistoryText,
   attemptIndexLine,
-  attemptOverviewText,
   attemptsOfEval,
   diffText,
   evalDetailText,
@@ -96,8 +96,9 @@ function clampWidth(columns: number | undefined): number {
   return Math.max(40, Math.min(columns as number, 160));
 }
 
-// --report 的装载与规范化住在 ./report-host.ts(两个宿主共用的唯一联系面)。
-export { loadHostReport, normalizeHostReport, resolveReportTitle, localizeText } from "./report-host.ts";
+// --report 的装载住在 ../report/runtime/host.ts(两个宿主共用的中性联系面);规范化本身是
+// `defineReport` 自己的职责,不在宿主层重复。
+export { loadHostReport, localizeText } from "../report/runtime/host.ts";
 
 export async function runShow(
   cwd: string,
@@ -163,8 +164,7 @@ async function show(
   // `@<locator>` 位置参数:身份直达单个 attempt,与 eval id 前缀匹配完全不同的语义
   // (`@` 打头对 eval id 天然无歧义,见 locator.ts),必须在下面的前缀匹配逻辑之前分流掉,
   // 不然 "@1x7f3q" 会被当成一个谁都匹配不到的 eval id 前缀,报「no eval match」这种文不对题的
-  // 错误。这一步只解析并渲染出「当前 show 对单个已解析 attempt 能渲染的东西」(单 eval 详情 /
-  // 三个证据切面)——真正的 `--source`/`--execution`/`--diff` 统一 attempt 全景是后续阶段。
+  // 错误。
   const locatorArg = patterns.find((p) => p.startsWith(ATTEMPT_LOCATOR_PREFIX));
   if (locatorArg !== undefined) {
     if (patterns.length !== 1) {
@@ -199,7 +199,35 @@ async function show(
       io.out(blocks.join("\n\n") + "\n");
       return;
     }
-    io.out(attemptOverviewText(attemptEvidence, { header, artifactPath, width: io.width }) + "\n");
+    // 无证据 flag:选中当前 report definition 里唯一的 attempt-input page,注入这份 evidence,
+    // 走与其它 page 完全相同的 resolve → validate → render 管线(docs/feature/reports/show/attempt.md;
+    // docs/feature/reports/library/attempt-detail.md「在 show 与 view 怎样渲染」)。不带 --report
+    // 时装载内建 standard,其中就带这张 page;--report 指向的自定义报告没有声明 attempt-input page
+    // 时报完整用户反馈,不回退到内建详情(三条解决路径都在错误文案里给出)。
+    const report = await loadHostReport(cwd, flags.report);
+    const attemptPage = report.pages.find((p) => p.input === "attempt");
+    if (attemptPage === undefined) {
+      const sourceLabel = flags.report ?? "the built-in report";
+      throw new ShowError(
+        `error: ${sourceLabel} has no attempt-input page — "${locatorArg}" cannot be opened without one. ` +
+          `Add one: use \`extends: standard\` (inherits its attempt page), import { standardAttemptPage } from ` +
+          `"niceeval/report/built-in" and add it to your pages list, or declare your own \`input: "attempt"\` page.\n`,
+      );
+    }
+    const locale = detectLocale();
+    const selection = selectCurrentResults(results, {});
+    const meta = await buildHostReportMeta(report, selection);
+    const text = await renderHostPageText(
+      attemptPage,
+      {
+        scope: selection,
+        results,
+        report: meta,
+        page: { id: attemptPage.id, input: "attempt", locator: attempt.locator!, evidence: attemptEvidence },
+      },
+      { width: io.width, locale },
+    );
+    io.out(text + "\n");
     return;
   }
 
@@ -322,16 +350,23 @@ async function show(
   };
   const sourceLabel = flags.report ?? "the built-in report";
 
-  // 初始页 = --page 指定的页,缺省第一页(docs/feature/reports/show/reports.md Case 2);
-  // 本地宿主只 resolve 被打开的这一页——其余页只留 id / title,不触发取数(见 shell.md
-  // 「行为约束」「本地宿主只 resolve 被打开的页」)。
-  let page = report.pages[0];
+  // 初始页 = --page 指定的页,缺省第一张可导航页(docs/feature/reports/show/reports.md
+  // Case 2);本地宿主只 resolve 被打开的这一页——其余页只留 id / title,不触发取数(见
+  // shell.md「行为约束」「本地宿主只 resolve 被打开的页」)。navigation:false 的页(参数化
+  // attempt 详情)不参与缺省选择,也不能被 --page 直接打开——没有 locator 不能拿 Scope 强行
+  // resolve(architecture.md「Attempt 详情是一张参数化 page」)。
+  let page = report.pages.find((p) => p.navigation !== false) ?? report.pages[0];
   if (flags.page !== undefined) {
     const hit = report.pages.find((p) => p.id === flags.page);
     if (!hit) {
       // 用法错误:列出可用页 id(docs/feature/reports/show/reports.md Case 1/2 的报错样例)。
       throw new ShowError(
-        `error: page "${flags.page}" not found in ${sourceLabel}. Available pages: ${report.pages.map((p) => p.id).join(", ")}\n`,
+        `error: page "${flags.page}" not found in ${sourceLabel}. Available pages: ${report.pages.filter((p) => p.navigation !== false).map((p) => p.id).join(", ")}\n`,
+      );
+    }
+    if (hit.input === "attempt") {
+      throw new ShowError(
+        `error: page "${hit.id}" in ${sourceLabel} is an attempt-input page and needs a locator — it cannot be opened with --page directly. Use niceeval show @<locator> instead.\n`,
       );
     }
     page = hit;
@@ -339,10 +374,10 @@ async function show(
 
   // attemptCommand 留给渲染管线的默认值:AttemptLocator 已经是可直接 `niceeval show @<locator>`
   // 的真实 CLI 语法,不需要再反查 eval id 拼一条近似命令。
-  const meta = reportMetaFor(report, page.id, selection.snapshots);
+  const meta = await buildHostReportMeta(report, selection);
   const text = await renderHostPageText(
     page,
-    { scope: selection, results, report: meta },
+    { scope: selection, results, report: meta, page: { id: page.id, input: "scope" } },
     {
       width: io.width,
       locale,
@@ -350,8 +385,9 @@ async function show(
     },
   );
 
-  // 页数大于一时尾部附「其余页」索引(只列未渲染的页,不倾倒内容);单页定义没有这段。
-  const remaining = report.pages.filter((p) => p.id !== page.id);
+  // 页数大于一时尾部附「其余页」索引(只列未渲染、且可导航的页,不倾倒内容);单页定义
+  // 没有这段;隐藏的 attempt page 不出现在「其余页」里。
+  const remaining = report.pages.filter((p) => p.id !== page.id && p.navigation !== false);
   if (remaining.length === 0) {
     io.out(text + "\n");
     return;