niceeval 0.6.2 → 0.7.1

package/src/report/flag.ts

@@ -1,46 +1,110 @@
-// flag() / config():把 experiment 声明的变量当维度或轴(docs/feature/reports/library.md「维度与 flags」)。
+// flag() / runConfig() / numericFlag() / numericRunConfig():把 experiment 声明的变量当
+// 分组维度或数值轴(docs/feature/reports/library/metrics.md「维度与数值轴」)。
 // 变量来自配置,不来自命名 —— 报告不解析 experiment id 字符串抠变量。
 // flag() 只读 `ExperimentDef.flags` 里显式声明的 KV;model / reasoningEffort / budget / runs
-// 这类顶层运行配置不在 flags 里,用 config() 读快照的 ExperimentRunInfo 投影。
+// 这类顶层运行配置不在 flags 里,用 runConfig() 读快照的 ExperimentRunInfo 投影。
 
-import type { ConfigRef, FlagRef } from "./types.ts";
+import type { AttemptHandle } from "../results/types.ts";
+import type {
+  DimensionOptions,
+  DimensionRef,
+  NumericAxis,
+  NumericAxisOptions,
+  NumericRunConfigAxisOptions,
+  RunConfigKey,
+} from "./types.ts";
+
+/** experiment 声明的 flags(快照级投影优先;第三方落盘只拼在条目上时回退 result.experiment)。 */
+export function flagValueOf(attempt: AttemptHandle, name: string): unknown {
+  const info = attempt.snapshot?.experiment ?? attempt.result.experiment;
+  return info?.flags?.[name];
+}
 
 /**
- * 把 experiment 声明的一个 flag 当分组维度(series / rows / columns / points 槽)或数值轴
- * (MetricLine 的 x 槽)。只读 `ExperimentDef.flags` 里显式声明的 KV;未声明的 experiment
- * 分组归 `(unset)`,作轴不画点并报告缺失。
+ * runConfig() 的取值:读快照的 `ExperimentRunInfo` 投影,外加桥接到快照顶层权威字段的
+ * `model` / `agent` 两个键(与 "model" / "agent" 内置维度同一读法,不另造第二套口径)。
+ * 未投影 → undefined。
  */
-export function flag(
-  name: string,
-  opts?: {
-    /** 组标签 / 轴标签;函数形态把声明值折成组名(如 `(v) => \`${v} agents\``)。 */
-    label?: string | ((value: string | number | boolean) => string);
-    unit?: string;
-  },
-): FlagRef {
+export function runConfigValueOf(attempt: AttemptHandle, name: RunConfigKey): unknown {
+  if (name === "model") return attempt.result.model ?? attempt.snapshot?.model;
+  if (name === "agent") return attempt.result.agent || attempt.snapshot?.agent;
+  const info = attempt.snapshot?.experiment ?? attempt.result.experiment;
+  // ExperimentRunInfo 是穷尽可序列化投影;按字段名取值,新增投影字段无需改这里
+  return (info as Record<string, unknown> | undefined)?.[name];
+}
+
+function assertName(name: unknown, fn: string, hint: string): asserts name is string {
   if (typeof name !== "string" || name.length === 0) {
-    throw new Error("flag: name must be a non-empty string (the key declared in the experiment's flags).");
+    throw new Error(`${fn}: name must be a non-empty string (${hint}).`);
   }
-  return { kind: "flag", name, label: opts?.label, unit: opts?.unit };
 }
 
 /**
- * 把一项顶层运行配置当分组维度或数值轴,与 {@link flag} 同一套用法。读快照的
- * `ExperimentRunInfo` 投影(可用键是那张接口的字段全集),外加桥接到快照顶层权威字段的
- * `model` / `agent` 两个键。未投影的值分组归 `(unset)`,作轴不画点并报告缺失。
+ * 把 experiment 声明的一个 flag 当分组维度(rows / columns / points / series / by 槽)。
+ * 分组显示键按稳定 JSON 规则生成(字符串直接显示,其它值用键递归排序后的 JSON),
+ * 缺失值显示内置文案 `(missing)`;显示键冲突时计算报错并要求改用 CustomDimension。
  */
-export function config(
-  name: string,
-  opts?: {
-    /** 组标签 / 轴标签;函数形态把投影值折成组名(如 `(v) => \`effort ${v}\``)。 */
-    label?: string | ((value: string | number | boolean) => string);
-    unit?: string;
-  },
-): ConfigRef {
-  if (typeof name !== "string" || name.length === 0) {
-    throw new Error(
-      "config: name must be a non-empty string (an ExperimentRunInfo field, or the bridged \"model\" / \"agent\" keys).",
-    );
-  }
-  return { kind: "config", name, label: opts?.label, unit: opts?.unit };
+export function flag(name: string, options?: DimensionOptions): DimensionRef {
+  assertName(name, "flag", "the key declared in the experiment's flags");
+  return {
+    kind: "flag",
+    name,
+    ...(options?.label !== undefined ? { label: options.label } : {}),
+    ...(options?.unit !== undefined ? { unit: options.unit } : {}),
+  };
+}
+
+/**
+ * 把一项顶层运行配置当分组维度,与 {@link flag} 同一套用法。读快照的 `ExperimentRunInfo`
+ * 投影;可用键由 RunConfigKey 在类型层穷尽(那张接口的字段全集,外加桥接到快照顶层权威
+ * 字段的 `model` / `agent`),拼错键在编译期就被拒绝。
+ */
+export function runConfig(name: RunConfigKey, options?: DimensionOptions): DimensionRef {
+  assertName(name, "runConfig", 'an ExperimentRunInfo field, or the bridged "model" / "agent" keys');
+  return {
+    kind: "runConfig",
+    name,
+    ...(options?.label !== undefined ? { label: options.label } : {}),
+    ...(options?.unit !== undefined ? { unit: options.unit } : {}),
+  };
+}
+
+/**
+ * MetricLine 的 x 轴:只接受落盘值为 number 的 flag。未声明或非数值的 attempt 返回 null
+ * (折线不绘该点并报告缺失),不猜 `low < medium < high`。
+ */
+export function numericFlag(name: string, options?: NumericAxisOptions): NumericAxis {
+  assertName(name, "numericFlag", "the key declared in the experiment's flags");
+  return {
+    name,
+    ...(options?.label !== undefined ? { label: options.label } : {}),
+    ...(options?.unit !== undefined ? { unit: options.unit } : {}),
+    of(attempt) {
+      const value = flagValueOf(attempt, name);
+      return typeof value === "number" && Number.isFinite(value) ? value : null;
+    },
+  };
+}
+
+/**
+ * 顶层运行配置作数值轴:数值配置直接返回该值;字符串配置必须显式给 `map`
+ * (如 { low: 1, medium: 2, high: 3 })。未声明、未投影、非数值或未命中 map 的值返回 null。
+ */
+export function numericRunConfig(name: RunConfigKey, options?: NumericRunConfigAxisOptions): NumericAxis {
+  assertName(name, "numericRunConfig", 'an ExperimentRunInfo field, or the bridged "model" / "agent" keys');
+  const map = options?.map;
+  return {
+    name,
+    ...(options?.label !== undefined ? { label: options.label } : {}),
+    ...(options?.unit !== undefined ? { unit: options.unit } : {}),
+    of(attempt) {
+      const value = runConfigValueOf(attempt, name);
+      if (typeof value === "number" && Number.isFinite(value)) return value;
+      if (typeof value === "string" && map !== undefined) {
+        const mapped = map[value];
+        return typeof mapped === "number" && Number.isFinite(mapped) ? mapped : null;
+      }
+      return null;
+    },
+  };
 }

package/src/report/format.ts

@@ -2,8 +2,8 @@
 //   "%" → 87%    "ms" → 1.2s    "$" → $0.31    其余 → 1.2k 缩写(带 unit 后缀)
 // metric.display 可整体覆盖;这里只负责默认。
 
-import type { AssertionResult, Verdict } from "../types.ts";
-import { compactAssertionSummary, fitCompactAssertionSummary, primaryAssertionSummary } from "../scoring/display.ts";
+import type { Verdict } from "../types.ts";
+import { DISPLAY_LOCALES, type LocalizedText, type ReportLocale } from "./locale.ts";
 
 /**
  * experiment 行的显示名：给了父路径 `relativeTo` 且它确是前缀，就去掉 `relativeTo + "/"`，
@@ -57,6 +57,18 @@ export function formatMetricValue(value: number, unit?: string): string {
   return unit ? `${sign}${n} ${unit}` : `${sign}${n}`;
 }
 
+/**
+ * MetricCell.display 的生成:为官方生成面覆盖的每个 locale(DISPLAY_LOCALES)各生成一份;
+ * 全部相同(内置 unit 格式化都是 locale 无关的)时折叠成单个字符串,renderer 按
+ * LocalizedText 回退规则选择。`make` 抛错由调用方(computeCell)带 metric 上下文包装。
+ */
+export function localizedDisplay(make: (locale: ReportLocale) => string): LocalizedText {
+  const entries = DISPLAY_LOCALES.map((locale) => [locale, make(locale)] as const);
+  const first = entries[0]![1];
+  if (entries.every(([, text]) => text === first)) return first;
+  return Object.fromEntries(entries);
+}
+
 /** 无单位纯数字(scoreboard 总分等):一位小数,去尾零。 */
 export function formatPlainNumber(value: number): string {
   const sign = value < 0 ? "-" : "";
@@ -109,25 +121,9 @@ export function verdictMark(verdict: Verdict): string {
 }
 
 /**
- * Attempt 比较项的一层结果摘要；完整 assertions 只在 locator 详情里展开。
- * maxChars(可选)是渲染面的宽度收口预算(如两行单元格 = 2 × 列宽):断言摘要按
- * fitCompactAssertionSummary 的优先级让位,error 摘要折单行后尾截。
+ * `AttemptListItem.failureSummary` 的宽度收口:摘要已在计算侧按 Scoring display 契约折好,
+ * 渲染面只做尾截,不重算摘要。maxChars 是渲染面的宽度预算(如两行单元格 = 2 × 列宽)。
  */
-export function attemptItemReason(
-  item: {
-    verdict: Verdict;
-    error?: { message: string };
-    assertions: AssertionResult[];
-  },
-  maxChars?: number,
-): string | undefined {
-  if (item.error !== undefined) {
-    const message = item.error.message.replace(/\s+/g, " ").trim();
-    return maxChars !== undefined && message.length > maxChars
-      ? `${message.slice(0, Math.max(0, maxChars - 1))}…`
-      : message;
-  }
-  const summary = primaryAssertionSummary(item.assertions, item.verdict);
-  if (summary === undefined) return undefined;
-  return maxChars === undefined ? compactAssertionSummary(summary) : fitCompactAssertionSummary(summary, maxChars);
+export function fitFailureSummary(summary: string, maxChars: number): string {
+  return summary.length <= maxChars ? summary : `${summary.slice(0, Math.max(0, maxChars - 1))}…`;
 }

package/src/report/index.ts

@@ -1,44 +1,83 @@
 // niceeval/report —— 报告积木:指标 × 计算函数 × 双面组件 × defineReport。
-// 契约见 docs/feature/reports/README.md;公开叙事的准绳是 docs-site/zh/guides/custom-reports.mdx
-// 与 report-components.mdx。
+// 契约见 docs/feature/reports/README.md 与 docs/feature/reports/library/ 分篇。
 //
-// import 边界即运行时边界:计算函数(挂在组件上的 .data)会经句柄触碰文件系统
-// (懒加载 artifact),只能进服务端 / 脚本;组件的渲染面纯同步零 IO。text 宿主遍历渲染
-// 不需要 react-dom(renderReportToText);web 宿主的 renderReportToStaticHtml 在
-// ./web.ts,只有那一侧 import react-dom。写报告文件的项目要装 react(.tsx 编译产物
-// import react/jsx-runtime)。
+// import 边界即运行时边界:计算函数(*Data)会经句柄触碰文件系统(懒加载 artifact),
+// 只能进服务端 / 脚本;组件的渲染面纯同步零 IO。text 宿主遍历渲染不需要 react-dom
+// (renderReportToText);web 宿主的 renderReportToStaticHtml 在 ./web.ts,只有那一侧
+// import react-dom。写报告文件的项目要装 react(.tsx 编译产物 import react/jsx-runtime)。
 
-// 指标与维度读取器(flag / config)
+// 指标与维度读取器
 export {
+  assistantTurns,
+  costUSD,
   defineMetric,
-  taskPassRate,
-  executionReliability,
+  durationMs,
   endToEndPassRate,
   examScore,
-  durationMs,
+  executionReliability,
+  repeatedFailedCommands,
+  taskPassRate,
   tokens,
-  costUSD,
-  turns,
 } from "./metrics.ts";
-export { flag, config } from "./flag.ts";
+export { flag, numericFlag, numericRunConfig, runConfig } from "./flag.ts";
 
-// 报告基座与双面组件基座
-export { defineReport, isReportDefinition, renderReportToText } from "./report.ts";
-export type { ReportContext, ReportDefinition } from "./report.ts";
-export { defineComponent } from "./tree.ts";
+// 报告定义与组件基座
+export {
+  buildReportMeta,
+  defineReport,
+  isReportDefinition,
+  pickReportPage,
+  renderReportToText,
+  renderReportTreeToText,
+  reportTitleText,
+  resolveReportTitle,
+  ReportPageNotFoundError,
+  DEFAULT_PAGE_ID,
+} from "./report.ts";
+export type {
+  HostCommandContext,
+  NonEmptyArray,
+  RenderReportTextOptions,
+  RenderTreeTextOptions,
+  ReportTreeHostContext,
+  ReportAsset,
+  ReportDef,
+  ReportDefinition,
+  ReportHostContext,
+  ReportLink,
+  ReportMeta,
+  ReportPage,
+  ReportShell,
+} from "./report.ts";
+export { defineComponent, createTextContext, renderNodeToText, resolveReportTree, validateReportTree, ResolveMemo } from "./tree.ts";
 export type {
   ComponentFaces,
+  ComposeContext,
   ReportComponent,
   ReportElement,
   ReportNode,
+  ResolveContext,
+  ResolveEnv,
   TextContext,
   TextRenderOptions,
   WebContext,
 } from "./tree.ts";
 
-// 排版原语(六个内置双面组件)
-export { Row, Col, Section, Text, Style, Table } from "./primitives.tsx";
-export type { LayoutProps, SectionProps, StyleProps, TableColumn, TableProps, TableRow } from "./primitives.tsx";
+// 排版原语(八个内置双面组件)
+export { Col, Row, Section, Style, Tab, Table, Tabs, Text } from "./primitives.tsx";
+export type {
+  ColProps,
+  LayoutProps,
+  RowProps,
+  SectionProps,
+  StyleProps,
+  TabProps,
+  TableColumn,
+  TableProps,
+  TableRow,
+  TabsProps,
+  TextProps,
+} from "./primitives.tsx";
 
 // 文本排版工具箱:自定义组件的 text 面用的就是官方组件那把尺子。
 // 表格有 <Table> 承担,这里只给表以外的形态用 —— 尤其别拿 String.prototype.padEnd 对齐:
@@ -54,92 +93,108 @@ export {
 } from "./text/layout.ts";
 export type { ColumnAlign } from "./text/layout.ts";
 
-// 内置报告兼组合件(show / view 裸跑时报告槽的出厂填充;也可作组件整体引用,
-// `<ExperimentComparison data={await ExperimentComparison.data(selection)} />`),无 renderer 特权
-export { ExperimentComparison } from "./built-ins/index.ts";
-export type {
-  ExperimentComparisonData,
-  ExperimentComparisonGroupData,
-  ExperimentComparisonProps,
-} from "./built-ins/index.ts";
+// locale:官方组件 chrome 文案的语言(内置词典覆盖 en / zh-CN,其它 locale 走回退)
+export { DEFAULT_REPORT_LOCALE, localizedTextEquals, resolveLocalizedText, resolveMetricLabel } from "./locale.ts";
+export type { LocalizedText, ReportLocale } from "./locale.ts";
 
-// locale:官方组件 chrome 文案的语言(en / zh-CN);指标 label 可按 locale 给字典
-export { DEFAULT_REPORT_LOCALE, resolveMetricLabel } from "./locale.ts";
-export type { LocalizedLabel, ReportLocale } from "./locale.ts";
-
-// 官方双面组件(各自带 data 计算函数;MetricBars.data = MetricMatrix.data)
+// 官方双面组件(spec / data 双形态;配套 *Data 计算函数在下面成对导出)
 export {
   AttemptList,
   DeltaTable,
   EvalList,
+  ExperimentComparison,
   ExperimentList,
-  GroupSummary,
+  FailureList,
   MetricBars,
   MetricLine,
   MetricMatrix,
   MetricScatter,
   MetricTable,
-  RunOverview,
   Scoreboard,
+  ScopeSummary,
 } from "./components.tsx";
 export type {
   AttemptListProps,
+  DataProps,
   DeltaTableProps,
   EvalListProps,
+  ExperimentComparisonProps,
   ExperimentListProps,
-  GroupSummaryProps,
+  FailureListProps,
+  MetricBarsProps,
   MetricLineProps,
   MetricMatrixProps,
   MetricScatterProps,
   MetricTableProps,
-  RunOverviewProps,
   ScoreboardProps,
+  ScopeSummaryProps,
 } from "./components.tsx";
 
-// 计算函数的选项类型(函数本体挂在组件上,不做顶层导出)
+// 计算函数(组件解析面的具名形式,与组件成对;spec 形态下由管线代调,data 形态与
+// 嵌入场景下由作者手工调)
+export {
+  attemptListData,
+  deltaTableData,
+  evalListData,
+  experimentComparisonData,
+  experimentListData,
+  metricLineData,
+  metricMatrixData,
+  metricScatterData,
+  metricTableData,
+  pairsByFlag,
+  scopeSummaryData,
+  scoreboardData,
+} from "./compute.ts";
 export type {
-  AttemptListDataOptions,
-  DeltaDataOptions,
-  DeltaPair,
-  LineDataOptions,
-  MatrixDataOptions,
-  ScatterDataOptions,
-  ScoreboardDataOptions,
-  TableDataOptions,
+  DeltaTableOptions,
+  MetricLineOptions,
+  MetricMatrixOptions,
+  MetricScatterOptions,
+  MetricTableOptions,
+  ScoreboardOptions,
 } from "./compute.ts";
 
-// 数据契约(组件的 data props)
+// 数据契约(组件的 data)
 export type {
   Aggregator,
   AttemptListItem,
   AttemptLocator,
-  AxisInput,
-  ConfigRef,
+  BuiltInDimension,
+  CustomDimension,
   DeltaData,
-  Dimension,
+  DeltaPair,
   DimensionInput,
+  DimensionOptions,
+  DimensionRef,
+  EntityListDataOptions,
   EvalListItem,
+  ExperimentComparisonData,
+  ExperimentComparisonGroupData,
   ExperimentListEvalRow,
   ExperimentListItem,
-  GroupSummaryData,
-  LineAxis,
+  FlagPairs,
   LineData,
   MatrixData,
   Metric,
   MetricAggregate,
   MetricCell,
   MetricColumn,
-  OverviewData,
-  FlagRef,
+  NumericAxis,
+  NumericAxisOptions,
+  NumericRunConfigAxisOptions,
+  ReportInput,
+  RunConfigKey,
   ScatterData,
+  ScopeSummaryData,
+  ScopeWarning,
   ScoreboardData,
-  SelectionWarning,
   TableData,
-  TableRowMeta,
+  VerdictTally,
 } from "./types.ts";
 
 // 数据层输入的类型(家在 niceeval/results,这里 re-export 方便写指标 / 报告)
-export type { AttemptHandle, Results, Selection, Snapshot } from "../results/types.ts";
+export type { AttemptHandle, Results, Scope, Snapshot } from "../results/types.ts";
 
 // experiment id 的组键推导(id 的目录前缀,如 `compare/bub-low` 的 `compare`)。
 // 重新导出,让自定义报告能按同一份口径把 experiment 分组,不必自己重写这两行逻辑。

package/src/report/load.ts

@@ -28,7 +28,7 @@ export async function loadReportFile(
   const abs = resolve(cwd, path);
   if (!existsSync(abs)) {
     throw new ReportLoadError(
-      `Report file not found: ${abs}. Pass --report an explicit path to a module whose default export is defineReport(...).`,
+      `Report file not found: ${abs}. Pass --report an explicit path to a module whose default export is a defineReport(...) product.`,
     );
   }
   const plain = pathToFileURL(abs);
@@ -60,7 +60,8 @@ export async function loadReportFile(
   }
   if (!isReportDefinition(mod.default)) {
     throw new ReportLoadError(
-      `${path} does not default-export a report. Export default defineReport(async ({ selection, results }) => ...) from "niceeval/report".`,
+      `${path} does not default-export a report. Export the product of defineReport(...) from "niceeval/report" as the default export — ` +
+        `a tree (defineReport(<ExperimentComparison />)) or a config object (defineReport({ title, content | pages, … })).`,
     );
   }
   return mod.default;