niceeval 0.6.2 → 0.7.1

package/src/report/built-ins/experiment-comparison.tsx

@@ -1,179 +0,0 @@
-// ExperimentComparison:裸 niceeval show / view 首页渲染的内置默认报告,同时是可整体引用的
-// 官方组合件(docs/feature/reports/library.md「ExperimentComparison」)。
-//
-// 名字描述整份报告回答的问题,不绑定其中某一张图。它是组合件而非新的数据源——每组的
-// GroupSummary、成本 × 端到端成功率散点、固定列 ExperimentList 都消费各自公开 .data() 的结果。
-// 两种用法共享同一份口径:
-//   · 默认报告:宿主把它当普通 ReportDefinition(build 面),没有 renderer 私有通道;
-//   · 报告组件:`<ExperimentComparison data={await ExperimentComparison.data(selection)} />`,
-//     在自定义报告或用户自己的 React 页面里整体引用。
-
-import { Col, Section, Table } from "../primitives.tsx";
-import { ExperimentList, GroupSummary, MetricScatter } from "../components.tsx";
-import { costUSD, endToEndPassRate } from "../metrics.ts";
-import { defineReport, type ReportDefinition } from "../report.ts";
-import { defineComponent, type ReportComponent, type TextContext } from "../tree.ts";
-import { localeText, type ReportLocale } from "../locale.ts";
-import type { ExperimentListItem, GroupSummaryData, ScatterData } from "../types.ts";
-import type { ScatterDataOptions } from "../compute.ts";
-import { resolveInput, type SnapshotsInput } from "../aggregate.ts";
-import { ExperimentComparisonView } from "../react/ExperimentComparison.tsx";
-import { wrapDisplay } from "../text/layout.ts";
-import { formatUSD } from "../format.ts";
-
-// 每组散点的唯一口径：默认 definition 与公开 `.data()` 共用，不各写一份。
-const SCATTER_OPTIONS: ScatterDataOptions = {
-  points: "experiment",
-  series: "agent",
-  x: costUSD,
-  y: endToEndPassRate,
-};
-
-/**
- * 一个可比组的数据。三个子块都只消费本组快照，不能含其它父目录的引用。
- */
-export interface ExperimentComparisonGroupData {
-  /** experiment id 的完整父路径；根目录 experiment 使用自己的完整 id。 */
-  key: string;
-  summary: GroupSummaryData;
-  /** 成本 × 端到端成功率散点(`MetricScatter.data` 的口径:points=experiment、series=agent)。 */
-  scatter: ScatterData;
-  /** 固定列 experiment 比较表(`ExperimentList.data` 的口径,已按端到端成功率排序)。 */
-  experiments: ExperimentListItem[];
-}
-
-/** `ExperimentComparison.data(selection)` 的穷尽产物：按 key 排序的独立可比组。 */
-export interface ExperimentComparisonData {
-  groups: ExperimentComparisonGroupData[];
-}
-
-/** 完整父路径是组键；没有父路径的 experiment 不能互相比，自己形成单例组。 */
-export function experimentComparisonGroupKey(experimentId: string): string {
-  const slash = experimentId.lastIndexOf("/");
-  return slash === -1 ? experimentId : experimentId.slice(0, slash);
-}
-
-/** 组合件的数据计算：先分区，再让三个官方子块在各组内并行计算。 */
-async function experimentComparisonData(input: SnapshotsInput): Promise<ExperimentComparisonData> {
-  const { snapshots } = resolveInput(input);
-  const snapshotsByGroup = new Map<string, typeof snapshots>();
-  for (const snapshot of snapshots) {
-    const key = experimentComparisonGroupKey(snapshot.experimentId);
-    const group = snapshotsByGroup.get(key);
-    if (group) group.push(snapshot);
-    else snapshotsByGroup.set(key, [snapshot]);
-  }
-
-  const groups = await Promise.all(
-    [...snapshotsByGroup.entries()]
-      .sort(([a], [b]) => a.localeCompare(b))
-      .map(async ([key, groupSnapshots]): Promise<ExperimentComparisonGroupData> => {
-        const [summary, scatter, experiments] = await Promise.all([
-          GroupSummary.data(groupSnapshots),
-          MetricScatter.data(groupSnapshots, SCATTER_OPTIONS),
-          ExperimentList.data(groupSnapshots),
-        ]);
-        return { key, summary, scatter, experiments };
-      }),
-  );
-  return { groups };
-}
-
-export interface ExperimentComparisonProps {
-  data: ExperimentComparisonData;
-  /** chrome 文案 locale;省略时随宿主上下文(宿主外默认 "en")。 */
-  locale?: ReportLocale;
-  className?: string;
-}
-
-/** 单组详情：散点与实验列表全部来自同一个预分区结果。 */
-function groupComposition(group: ExperimentComparisonGroupData, locale?: ReportLocale, className?: string) {
-  return (
-    <Col className={className}>
-      <MetricScatter data={group.scatter} locale={locale} />
-      <ExperimentList items={group.experiments} filter locale={locale} relativeTo={group.key} />
-    </Col>
-  );
-}
-
-function shellQuote(value: string): string {
-  return /^[A-Za-z0-9._/-]+$/.test(value) ? value : `'${value.replaceAll("'", `'"'"'`)}'`;
-}
-
-function verdictsText(group: ExperimentComparisonGroupData, locale: ReportLocale): string {
-  const parts: string[] = [];
-  for (const verdict of ["passed", "failed", "errored", "skipped"] as const) {
-    const count = group.summary.verdicts[verdict];
-    if (count > 0) parts.push(`${count} ${localeText(locale, `verdict.${verdict}`)}`);
-  }
-  return parts.length > 0 ? parts.join(" / ") : "—";
-}
-
-function textFace(data: ExperimentComparisonData, className: string | undefined, ctx: TextContext): string {
-  if (data.groups.length === 0) return localeText(ctx.locale, "experimentComparison.empty");
-  if (data.groups.length === 1) {
-    const group = data.groups[0]!;
-    return ctx.render(
-      <Section title={group.key} className={className}>
-        {groupComposition(group)}
-      </Section>,
-    );
-  }
-
-  const table = ctx.render(
-    <Table
-      columns={[
-        { key: "group", header: localeText(ctx.locale, "experimentComparison.group") },
-        { key: "experiments", header: localeText(ctx.locale, "groupSummary.experiments"), align: "right" },
-        { key: "evals", header: localeText(ctx.locale, "overview.evals"), align: "right" },
-        { key: "passRate", header: localeText(ctx.locale, "overview.passRate"), align: "right" },
-        { key: "results", header: localeText(ctx.locale, "experimentComparison.results") },
-        { key: "cost", header: localeText(ctx.locale, "overview.totalCost"), align: "right" },
-        { key: "lastRun", header: localeText(ctx.locale, "experimentComparison.lastRun") },
-      ]}
-      rows={data.groups.map((group) => ({
-        key: group.key,
-        cells: {
-          group: group.key,
-          experiments: String(group.summary.experiments),
-          evals: String(group.summary.evals),
-          passRate: group.summary.passRate.display,
-          results: verdictsText(group, ctx.locale),
-          cost: group.summary.totalCostUSD === null ? null : formatUSD(group.summary.totalCostUSD),
-          lastRun: group.summary.lastRunAt ?? null,
-        },
-      }))}
-    />,
-  );
-  const commands = data.groups
-    .map((group) => {
-      const command = `niceeval show --experiment ${shellQuote(group.key)}`;
-      return wrapDisplay(localeText(ctx.locale, "experimentComparison.command", { command }), ctx.width).join("\n");
-    })
-    .join("\n");
-  return `${localeText(ctx.locale, "experimentComparison.groups")}\n\n${table}\n\n${commands}`;
-}
-
-/**
- * 内置默认报告兼官方组合件:作为组件时收 `data`(`.data()` 的产物),两个渲染面直接摆
- * 数据形态的子组件;作为默认报告时经 build 面走与用户 `--report` 完全相同的管线。
- */
-export const ExperimentComparison: ReportComponent<ExperimentComparisonProps> &
-  ReportDefinition & { data: typeof experimentComparisonData } = Object.assign(
-  defineComponent<ExperimentComparisonProps>({
-    web: (props, ctx) => (
-      <ExperimentComparisonView
-        data={props.data}
-        locale={props.locale ?? ctx.locale}
-        className={props.className}
-      />
-    ),
-    text: ({ data, className }, ctx) => textFace(data, className, ctx),
-  }),
-  defineReport(async ({ selection }) => {
-    const data = await experimentComparisonData(selection);
-    return <ExperimentComparison data={data} />;
-  }),
-  { data: experimentComparisonData },
-);
-ExperimentComparison.displayName = "ExperimentComparison";

package/src/report/built-ins/index.ts

@@ -1,7 +0,0 @@
-// 内置报告目录:一个文件一个按展示内容命名的组合件(组件兼默认报告),显式导出值,不建字符串 registry。
-export { ExperimentComparison } from "./experiment-comparison.tsx";
-export type {
-  ExperimentComparisonData,
-  ExperimentComparisonGroupData,
-  ExperimentComparisonProps,
-} from "./experiment-comparison.tsx";

package/src/report/react/GroupSummary.tsx

@@ -1,66 +0,0 @@
-// GroupSummary:一组 experiment(如自定义报告里同一 <Section> 内的全部 experiment)的
-// 紧凑摘要块——恢复旧 GroupSelector 卡片的信息密度(通过率、experiment/eval 数、failed、
-// errored、总成本、最后运行时间),但只渲染 GroupSummaryData 算好的 MetricCell / 计数,
-// 不现场重算比例(docs/feature/reports/library.md「GroupSummary」)。errored 为 0 时省略该片段(旧卡片同一姿态),
-// 但这只是渲染取舍——数据侧 verdicts.errored 字段本身永不省略。
-
-import type { ReactElement } from "react";
-import type { GroupSummaryData } from "../types.ts";
-import { DEFAULT_REPORT_LOCALE, localeText, type ReportLocale } from "../locale.ts";
-import { MetricCellView } from "./cell.tsx";
-import { cx, formatUSD } from "./format.ts";
-
-export function GroupSummary({
-  data,
-  className,
-  locale = DEFAULT_REPORT_LOCALE,
-}: {
-  data: GroupSummaryData;
-  className?: string;
-  locale?: ReportLocale;
-}): ReactElement {
-  const missing = <span className="nre-missing">{localeText(locale, "cell.missing")}</span>;
-  return (
-    <div className={cx("nre", "nre-group-summary", className)}>
-      <dl className="nre-group-kpis">
-        <div className="nre-group-kpi nre-group-kpi-rate">
-          <dt>{localeText(locale, "overview.passRate")}</dt>
-          <dd>
-            <MetricCellView cell={data.passRate} locale={locale} />
-          </dd>
-        </div>
-        <div className="nre-group-kpi">
-          <dt>{localeText(locale, "groupSummary.experiments")}</dt>
-          <dd>{data.experiments}</dd>
-        </div>
-        <div className="nre-group-kpi">
-          <dt>{localeText(locale, "overview.evals")}</dt>
-          <dd>{data.evals}</dd>
-        </div>
-        <div className="nre-group-kpi">
-          <dt>{localeText(locale, "overview.attempts")}</dt>
-          <dd>{data.attempts}</dd>
-        </div>
-        <div className="nre-group-kpi">
-          <dt>{localeText(locale, "verdict.failed")}</dt>
-          <dd className="nre-verdict-failed">{data.verdicts.failed}</dd>
-        </div>
-        {/* errored 为 0 省略这一片段(旧 GroupSelector 卡片同一姿态);数据字段本身不受影响 */}
-        {data.verdicts.errored > 0 && (
-          <div className="nre-group-kpi">
-            <dt>{localeText(locale, "verdict.errored")}</dt>
-            <dd className="nre-verdict-errored">{data.verdicts.errored}</dd>
-          </div>
-        )}
-        <div className="nre-group-kpi">
-          <dt>{localeText(locale, "overview.totalCost")}</dt>
-          {/* totalCostUSD 全缺 = null:显示缺数据,不编 $0 */}
-          <dd>{data.totalCostUSD === null ? missing : formatUSD(data.totalCostUSD)}</dd>
-        </div>
-      </dl>
-      {data.lastRunAt && (
-        <p className="nre-group-summary-time">{localeText(locale, "latestRun", { run: data.lastRunAt })}</p>
-      )}
-    </div>
-  );
-}

package/src/report/react/RunOverview.tsx

@@ -1,109 +0,0 @@
-// RunOverview:页头 KPI 条——「这批数据是什么」。
-// 数字下面标注数据来源(几个快照、何时跑的);warnings 有内容时直接显示在条内,
-// 诚实不靠使用者记得渲染(docs/feature/reports/library.md「RunOverview」行为清单)。
-
-import type { ReactElement } from "react";
-import type { OverviewData } from "../types.ts";
-import { DEFAULT_REPORT_LOCALE, localeText, type ReportLocale } from "../locale.ts";
-import { cx, formatDurationMs, formatUSD } from "./format.ts";
-
-export function RunOverview({
-  data,
-  className,
-  locale = DEFAULT_REPORT_LOCALE,
-}: {
-  data: OverviewData;
-  className?: string;
-  locale?: ReportLocale;
-}): ReactElement {
-  const { totals } = data;
-  const missing = <span className="nre-missing">{localeText(locale, "cell.missing")}</span>;
-  // 通过率只渲染 computeCell 算好的 MetricCell,不现场重算——覆盖率角标复用
-  // MetricCellView 的缺数据/coverage 语义(缺数据 = 显式文案,绝不画 0%)
-  const { passRate } = totals;
-  const passRateNode =
-    passRate.value === null ? (
-      missing
-    ) : (
-      <>
-        {passRate.display}
-        {passRate.samples < passRate.total && (
-          <sup
-            className="nre-coverage"
-            title={localeText(locale, "cell.coverageTitle", { samples: passRate.samples, total: passRate.total })}
-          >
-            {passRate.samples}/{passRate.total}
-          </sup>
-        )}
-      </>
-    );
-
-  return (
-    <header className={cx("nre", "nre-overview", className)}>
-      <dl className="nre-kpis">
-        <div className="nre-kpi">
-          <dt>{localeText(locale, "overview.snapshots")}</dt>
-          <dd>{data.snapshots.length}</dd>
-        </div>
-        <div className="nre-kpi">
-          <dt>{localeText(locale, "overview.evals")}</dt>
-          <dd>{totals.evals}</dd>
-        </div>
-        <div className="nre-kpi">
-          <dt>{localeText(locale, "overview.attempts")}</dt>
-          <dd>{totals.attempts}</dd>
-        </div>
-        <div className="nre-kpi">
-          <dt>{localeText(locale, "overview.passRate")}</dt>
-          <dd>{passRateNode}</dd>
-        </div>
-        <div className="nre-kpi">
-          <dt>{localeText(locale, "overview.totalCost")}</dt>
-          {/* costUSD 全缺 = null:显示缺数据,不编 $0 */}
-          <dd>{totals.costUSD === null ? missing : formatUSD(totals.costUSD)}</dd>
-        </div>
-        <div className="nre-kpi">
-          <dt>{localeText(locale, "overview.totalDuration")}</dt>
-          <dd>{formatDurationMs(totals.durationMs)}</dd>
-        </div>
-      </dl>
-
-      <p className="nre-verdicts">
-        <span className="nre-verdict nre-verdict-passed">
-          {localeText(locale, "verdict.passed")} {totals.passed}
-        </span>
-        <span className="nre-verdict nre-verdict-failed">
-          {localeText(locale, "verdict.failed")} {totals.failed}
-        </span>
-        <span className="nre-verdict nre-verdict-errored">
-          {localeText(locale, "verdict.errored")} {totals.errored}
-        </span>
-        <span className="nre-verdict nre-verdict-skipped">
-          {localeText(locale, "verdict.skipped")} {totals.skipped}
-        </span>
-      </p>
-
-      {/* 数据来源:哪些快照、何时跑的——报告的数字都从这里来 */}
-      <p className="nre-source">
-        {localeText(locale, "overview.source", { n: data.snapshots.length })}
-        {data.snapshots.map((s) => (
-          <span key={`${s.experimentId}@${s.startedAt}`} className="nre-source-snapshot">
-            {s.experimentId}({s.agent}
-            {s.model ? ` · ${s.model}` : ""})@ {s.startedAt}
-          </span>
-        ))}
-      </p>
-
-      {/* Selection 的警告(残缺快照等)直接渲染在条内,不静默;结构化字段供程序判断,这里打 message */}
-      {data.warnings.length > 0 && (
-        <ul className="nre-warnings">
-          {data.warnings.map((w, i) => (
-            <li key={i} className="nre-warning" data-kind={w.kind}>
-              {w.message}
-            </li>
-          ))}
-        </ul>
-      )}
-    </header>
-  );
-}