niceeval 0.1.0

package/src/util.test.ts

@@ -0,0 +1,31 @@
+import { describe, expect, it } from "vitest";
+import { formatThrown } from "./util.ts";
+
+describe("formatThrown", () => {
+  it("uses the stack trace when available, so the report can locate the throw site", () => {
+    function throwsFromHere(): never {
+      throw new TypeError("Cannot read properties of undefined (reading 'text')");
+    }
+    let caught: unknown;
+    try {
+      throwsFromHere();
+    } catch (e) {
+      caught = e;
+    }
+    const formatted = formatThrown(caught);
+    expect(formatted).toContain("TypeError: Cannot read properties of undefined (reading 'text')");
+    expect(formatted).toContain("throwsFromHere");
+    expect(formatted).toContain("util.test.ts");
+  });
+
+  it("falls back to name: message when the error has no stack", () => {
+    const e = new Error("boom");
+    delete (e as { stack?: string }).stack;
+    expect(formatThrown(e)).toBe("Error: boom");
+  });
+
+  it("stringifies non-Error thrown values", () => {
+    expect(formatThrown("just a string")).toBe("just a string");
+    expect(formatThrown(42)).toBe("42");
+  });
+});

package/src/util.ts

@@ -0,0 +1,50 @@
+// 小工具。
+
+import { t } from "./i18n/index.ts";
+
+/** 读必需的环境变量,缺了就清晰报错(agent 鉴权用)。 */
+export function requireEnv(name: string): string {
+  const v = process.env[name];
+  if (v === undefined || v === "") {
+    throw new Error(t("util.requiredEnv", { name }));
+  }
+  return v;
+}
+
+/** 取环境变量,缺了返回 undefined。 */
+export function getEnv(name: string): string | undefined {
+  const v = process.env[name];
+  return v === undefined || v === "" ? undefined : v;
+}
+
+/** 去掉 JS/TS 注释(块注释 + 行注释),好让断言只对真实代码生效,不被注释里的迁移说明误伤。 */
+export function stripComments(code: string): string {
+  return code.replace(/\/\*[\s\S]*?\*\//g, "").replace(/\/\/.*$/gm, "");
+}
+
+/**
+ * 把 catch 到的 e 转成报告用字符串。优先带 stack(定位到 eval 脚本抛错的具体 file:line),
+ * 只在没有 stack 时才退化到 `name: message`。EvalResult.error 走这个,别再手写
+ * `e instanceof Error ? e.message : String(e)`——那样用户永远看不出错误发生在哪一行。
+ */
+export function formatThrown(e: unknown): string {
+  if (e instanceof Error) return e.stack ?? `${e.name}: ${e.message}`;
+  return String(e);
+}
+
+/** 零填充到 4 位(数据集扇出的 id:sql/0000)。 */
+export function pad4(n: number): string {
+  return String(n).padStart(4, "0");
+}
+
+/** 把任意值安全地转成简短字符串(报告 / 日志用)。 */
+export function brief(value: unknown, max = 200): string {
+  let s: string;
+  try {
+    s = typeof value === "string" ? value : JSON.stringify(value);
+  } catch {
+    s = String(value);
+  }
+  if (s.length > max) return s.slice(0, max) + "…";
+  return s;
+}

package/src/view/app/App.tsx

@@ -0,0 +1,189 @@
+import { useCallback, useEffect, useMemo, useState } from "react";
+import { detectLocale, makeTranslator, persistLocale, setDocumentLocale } from "./i18n.ts";
+import type { MessageKey } from "./i18n.ts";
+import type { Locale, LocalizedText, SortKey, SortState, Tab, ViewData, ViewResult, ViewRow } from "./types.ts";
+import { buildGroupMap, compareRows, resultFromUrl } from "./lib/rows.ts";
+import { Metric } from "./components/primitives.tsx";
+import { GroupSelector } from "./components/GroupSelector.tsx";
+import { ExperimentTable } from "./components/ExperimentTable.tsx";
+import { CopyAllErrors } from "./components/CopyControls.tsx";
+import { AttemptModal } from "./components/AttemptModal.tsx";
+import { Tabs, TabsContent, TabsList, TabsTrigger } from "./components/ui/tabs.tsx";
+import { RunsView } from "./pages/RunsPage.tsx";
+import { TracesView } from "./pages/TracesPage.tsx";
+
+export const navItems: { id: Tab; label: MessageKey }[] = [
+  { id: "experiments", label: "nav.experiments" },
+  { id: "runs", label: "nav.runs" },
+  { id: "traces", label: "nav.traces" },
+];
+
+/** 把 config.name 解析成当前界面语言的一条文案:字符串原样返回,多语言映射按 locale 挑,挑不到回退 en / 第一条。 */
+export function localizedText(text: LocalizedText | undefined, locale: Locale): string | undefined {
+  if (!text) return undefined;
+  if (typeof text === "string") return text;
+  return text[locale] ?? text.en ?? Object.values(text)[0];
+}
+
+export function App({ data }: { data: ViewData }) {
+  const rows = data.rows ?? [];
+  const [locale, setLocale] = useState<Locale>(() => detectLocale());
+  const t = useMemo(() => makeTranslator(locale), [locale]);
+  const [tab, setTab] = useState<Tab>("experiments");
+  const [sort, setSort] = useState<SortState>({ key: "passRate", dir: -1 });
+  const [query, setQuery] = useState("");
+  const [openRows, setOpenRows] = useState<Set<string>>(() => new Set());
+  const [selectedGroup, setSelectedGroup] = useState(() => {
+    const groups = [...new Set(rows.map((r) => r.group).filter(Boolean))].sort();
+    return groups[0] ?? null;
+  });
+  const [modalResult, setModalResult] = useState<ViewResult | null>(() => resultFromUrl(rows));
+
+  useEffect(() => {
+    setDocumentLocale(locale);
+    persistLocale(locale);
+  }, [locale]);
+
+  const openModal = useCallback((result: ViewResult) => {
+    setModalResult(result);
+    const p = new URLSearchParams();
+    p.set("modal", result.id);
+    if (result.experimentId) p.set("exp", result.experimentId);
+    p.set("a", String(result.attempt));
+    history.replaceState(null, "", "?" + p.toString());
+  }, []);
+
+  const closeModal = useCallback(() => {
+    setModalResult(null);
+    history.replaceState(null, "", location.pathname);
+  }, []);
+
+  const groupMap = useMemo<Map<string, ViewRow[]>>(() => buildGroupMap(rows), [rows]);
+  const pool = selectedGroup ? groupMap.get(selectedGroup) ?? [] : rows;
+  const filtered = useMemo(() => {
+    const q = query.trim().toLowerCase();
+    return pool
+      .filter((row: ViewRow) => {
+        if (!q) return true;
+        return [
+          row.label,
+          row.group || "",
+          row.experimentId || "",
+          row.agent,
+          row.model || "",
+          ...(row.results ?? []).map((r: ViewResult) => r.id),
+        ]
+          .join(" ")
+          .toLowerCase()
+          .includes(q);
+      })
+      .sort((a: ViewRow, b: ViewRow) => compareRows(a, b, sort.key) * sort.dir);
+  }, [pool, query, sort]);
+
+  const setSortKey = (key: SortKey) => {
+    setSort((prev) =>
+      prev.key === key ? { key, dir: prev.dir === 1 ? -1 : 1 } : { key, dir: key === "experiment" || key === "agent" ? 1 : -1 },
+    );
+  };
+
+  const toggleRow = (key: string) => {
+    setOpenRows((prev) => {
+      const next = new Set(prev);
+      if (next.has(key)) next.delete(key);
+      else next.add(key);
+      return next;
+    });
+  };
+
+  return (
+    <Tabs value={tab} onValueChange={(v) => setTab(v as Tab)}>
+      <header className="topbar">
+        <a className="brand" href="https://github.com/CorrectRoadH/niceeval" target="_blank" rel="noreferrer">
+          <span className="mark" />
+          <span>NiceEval</span>
+        </a>
+        <TabsList aria-label={t("nav.label")}>
+          {navItems.map((item) => (
+            <TabsTrigger key={item.id} value={item.id}>
+              {t(item.label)}
+            </TabsTrigger>
+          ))}
+        </TabsList>
+        <div className="lang-switch" aria-label="Language">
+          {(["en", "zh-CN"] satisfies Locale[]).map((item) => (
+            <button
+              key={item}
+              className={locale === item ? "is-active" : ""}
+              type="button"
+              onClick={() => setLocale(item)}
+              aria-pressed={locale === item}
+            >
+              {item === "zh-CN" ? "中文" : "EN"}
+            </button>
+          ))}
+        </div>
+      </header>
+      <main>
+        <section className="hero">
+          <h1>{localizedText(data.name, locale) || t("hero.title")}</h1>
+          <div className="meta">
+            <span>
+              <b>{t("hero.lastRun")}</b> {data.lastRun}
+            </span>
+          </div>
+        </section>
+
+        <section className="summary" aria-label="Run summary">
+          <Metric label={t("metric.passRate")} value={data.passRate} />
+          <Metric label={t("metric.evalResults")} value={data.resultCount} />
+          <Metric label={t("metric.duration")} value={data.duration} />
+          <Metric label={t("metric.cost")} value={data.cost} />
+        </section>
+
+        <TabsContent value="experiments" id="tab-experiments">
+          <div className="section-head">
+            <h2>{t("section.experiments")}</h2>
+          </div>
+          <GroupSelector groupMap={groupMap} selectedGroup={selectedGroup} onSelect={setSelectedGroup} t={t} />
+          <div className="section-sub-head">
+            <span className="group-detail-label">{selectedGroup ?? ""}</span>
+            <div className="controls">
+              <input
+                className="search"
+                type="search"
+                placeholder={t("search.experiments")}
+                autoComplete="off"
+                value={query}
+                onChange={(e) => setQuery(e.target.value)}
+              />
+              <CopyAllErrors rows={filtered} t={t} />
+            </div>
+          </div>
+          {rows.length ? (
+            <ExperimentTable
+              rows={filtered}
+              sort={sort}
+              setSortKey={setSortKey}
+              openRows={openRows}
+              toggleRow={toggleRow}
+              openModal={openModal}
+              t={t}
+            />
+          ) : (
+            <div className="empty">
+              {t("empty.summary")}
+            </div>
+          )}
+        </TabsContent>
+
+        <TabsContent value="runs">
+          <RunsView rows={rows} t={t} />
+        </TabsContent>
+        <TabsContent value="traces">
+          <TracesView rows={rows} t={t} />
+        </TabsContent>
+      </main>
+      {modalResult && <AttemptModal result={modalResult} onClose={closeModal} t={t} />}
+    </Tabs>
+  );
+}

package/src/view/app/components/AttemptModal.tsx

@@ -0,0 +1,66 @@
+import { useEffect, useState } from "react";
+import type { ArtifactLoadState, T } from "../shared.ts";
+import type { ViewResult } from "../types.ts";
+import { asEvents, asSources } from "../lib/guards.ts";
+import { outcomeClass, outcomeLabel, outcomeOf } from "../lib/outcome.ts";
+import { CodeView, NoSourceBody } from "./CodeView.tsx";
+import { LazyArtifact } from "./LazyArtifact.tsx";
+import { Dialog, DialogClose, DialogContent, DialogTitle } from "./ui/dialog.tsx";
+import { Badge } from "./ui/badge.tsx";
+
+export function AttemptModal({ result, onClose, t }: { result: ViewResult; onClose: () => void; t: T }) {
+  const allAssertions = result.assertions || [];
+  const base = result.artifactBase;
+  const [data, setData] = useState<ArtifactLoadState>({ sources: null, events: null, status: "loading" });
+
+  // Esc / 焦点陷阱 / 背景滚动锁 / 点遮罩关闭 都交给 Radix Dialog;这里只保留工件拉取。
+  useEffect(() => {
+    if (!base) { setData({ sources: null, events: null, status: "none" }); return; }
+    let alive = true;
+    const grab = (name: string, has?: boolean): Promise<unknown> =>
+      has
+        ? fetch("/artifact?p=" + encodeURIComponent(`${base}/${name}`))
+            .then((r) => (r.ok ? r.json() : null))
+            .catch(() => null)
+        : Promise.resolve(null);
+    Promise.all([grab("sources.json", result.hasSources), grab("events.json", result.hasEvents)]).then(([sources, events]) => {
+      if (alive) setData({ sources: asSources(sources), events: asEvents(events), status: "ready" });
+    });
+    return () => { alive = false; };
+  }, [base, result.hasSources, result.hasEvents]);
+
+  const outcome = outcomeOf(result);
+  const hasCode = Boolean(data.sources?.length);
+
+  return (
+    <Dialog open onOpenChange={(o) => { if (!o) onClose(); }}>
+      <DialogContent aria-describedby={undefined}>
+        <div className="flex min-w-0 shrink-0 items-center justify-between gap-3 border-b border-line px-[18px] pb-[11px] pt-[13px]">
+          <div className="flex min-w-0 flex-col gap-[3px]">
+            <Badge tone={outcomeClass(outcome)}>{outcomeLabel(outcome, t)}</Badge>
+            <DialogTitle asChild>
+              <span className="truncate text-sm font-[640] text-text">{result.id}</span>
+            </DialogTitle>
+            {result.description ? <span className="truncate text-xs text-muted">{result.description}</span> : null}
+          </div>
+          <DialogClose
+            aria-label={t("action.close")}
+            className="grid h-7 w-7 shrink-0 place-items-center rounded-md border border-transparent text-sm text-muted transition-colors hover:border-line hover:bg-panel-2 hover:text-text"
+          >
+            x
+          </DialogClose>
+        </div>
+        <div className="flex-1 overflow-y-auto px-[18px] pb-[18px] pt-[14px]">
+          {result.error ? <div className="modal-error">{result.error}</div> : null}
+          {data.status === "loading" ? <div className="conv-loading">{t("trace.loading")}</div> : null}
+          {hasCode ? (
+            <CodeView sources={data.sources ?? []} events={data.events || []} assertions={allAssertions} t={t} />
+          ) : data.status !== "loading" ? (
+            <NoSourceBody assertions={allAssertions} events={data.events || []} t={t} />
+          ) : null}
+          {result.hasTrace && base ? <LazyArtifact type="trace" src={`${base}/trace.json`} t={t} /> : null}
+        </div>
+      </DialogContent>
+    </Dialog>
+  );
+}

package/src/view/app/components/CodeView.tsx

@@ -0,0 +1,272 @@
+import { useCallback, useMemo, useState } from "react";
+import { ChevronRight } from "lucide-react";
+import type { T } from "../shared.ts";
+import type { Assertion, CodeSource, SourceTurn, TranscriptEvent } from "../types.ts";
+import { TOOL_VERB, highlightTs, indexAsserts, indexTurns, locKey, resultBody, toolPrimaryArg } from "../lib/transcript-data.tsx";
+import { formatScore, previewText, truncate } from "../lib/format.ts";
+import { Transcript } from "./Transcript.tsx";
+
+/** soft 断言没过阈值不影响 outcome,颜色上跟 gate 失败(红)区分开,用 warn(黄)。 */
+function assertTone(a: Assertion): "good" | "warn" | "bad" {
+  if (a.passed) return "good";
+  return a.severity === "soft" ? "warn" : "bad";
+}
+
+export function CodeView({ sources, events, assertions, t }: { sources: CodeSource[]; events: TranscriptEvent[]; assertions: Assertion[]; t: T }) {
+  const turns = useMemo(() => indexTurns(events), [events]);
+  const asserts = useMemo(() => indexAsserts(assertions), [assertions]);
+  const [open, setOpen] = useState<Set<string>>(() => new Set());
+  const toggle = useCallback((k: string) => {
+    setOpen((prev) => {
+      const next = new Set(prev);
+      if (next.has(k)) next.delete(k); else next.add(k);
+      return next;
+    });
+  }, []);
+
+  // 哪些 loc 被源码行覆盖到了;没覆盖到的(读不到源码的文件)放底部兜底。
+  const sourceKeys = new Set<string>();
+  for (const f of sources) {
+    const n = f.content.split("\n").length;
+    for (let i = 1; i <= n; i++) sourceKeys.add(locKey(f.path, i));
+  }
+  const orphanAsserts = [...asserts.byKey.entries()]
+    .filter(([k]) => !sourceKeys.has(k))
+    .flatMap(([, v]) => v)
+    .concat(asserts.noloc);
+
+  return (
+    <div className="codeview">
+      {sources.map((file) => (
+        <CodeFile
+          key={file.path}
+          file={file}
+          turns={turns.byKey}
+          asserts={asserts.byKey}
+          open={open}
+          toggle={toggle}
+          t={t}
+        />
+      ))}
+      {orphanAsserts.length ? (
+        <div className="code-orphans">
+          <div className="code-orphans-head">{t("code.otherAssertions")}</div>
+          <AssertDetail asserts={orphanAsserts} t={t} />
+        </div>
+      ) : null}
+    </div>
+  );
+}
+
+export function CodeFile({
+  file,
+  turns,
+  asserts,
+  open,
+  toggle,
+  t,
+}: {
+  file: CodeSource;
+  turns: Map<string, SourceTurn>;
+  asserts: Map<string, Assertion[]>;
+  open: Set<string>;
+  toggle: (key: string) => void;
+  t: T;
+}) {
+  const lines = file.content.replace(/\n$/, "").split("\n");
+  return (
+    <div className="code-file">
+      <div className="code-file-head">{file.path}</div>
+      <div className="code-lines">
+        {lines.map((text: string, i: number) => {
+          const n = i + 1;
+          const k = locKey(file.path, n);
+          return (
+            <CodeLine
+              key={n}
+              n={n}
+              text={text}
+              turn={turns.get(k)}
+              asserts={asserts.get(k)}
+              isOpen={open.has(k)}
+              onToggle={() => toggle(k)}
+              t={t}
+            />
+          );
+        })}
+      </div>
+    </div>
+  );
+}
+
+export function CodeLine({
+  n,
+  text,
+  turn,
+  asserts,
+  isOpen,
+  onToggle,
+  t,
+}: {
+  n: number;
+  text: string;
+  turn?: SourceTurn;
+  asserts?: Assertion[];
+  isOpen: boolean;
+  onToggle: () => void;
+  t: T;
+}) {
+  const hasReply = !!turn;
+  const hasAsserts = !!(asserts && asserts.length);
+  // 只有 gate 断言没过才算这一行真的"fail";只剩 soft 断言没过阈值时是"warn"(不影响 outcome)。
+  const status = hasAsserts
+    ? asserts?.every((a: Assertion) => a.passed)
+      ? "pass"
+      : asserts?.some((a: Assertion) => !a.passed && a.severity === "gate")
+        ? "fail"
+        : "warn"
+    : null;
+  const clickable = hasReply || hasAsserts;
+  const rowCls =
+    "code-line" +
+    (status ? ` line-${status}` : "") +
+    (hasReply && !status ? " line-send" : "") +
+    (clickable ? " line-clickable" : "") +
+    (isOpen ? " is-open" : "");
+  return (
+    <>
+      <div
+        className={rowCls}
+        onClick={clickable ? onToggle : undefined}
+        role={clickable ? "button" : undefined}
+        tabIndex={clickable ? 0 : undefined}
+        onKeyDown={clickable ? (e) => { if (e.key === "Enter" || e.key === " ") { e.preventDefault(); onToggle(); } } : undefined}
+      >
+        <span className="ln">{n}</span>
+        <span className="gmark">
+          {hasAsserts ? (
+            <span className={`gstat ${status === "pass" ? "good" : status === "warn" ? "warn" : "bad"}`}>
+              {status === "pass" ? "✓" : status === "warn" ? "!" : "✗"}
+            </span>
+          ) : hasReply ? (
+            <ChevronRight className={`gchev${isOpen ? " is-open" : ""}`} aria-hidden="true" />
+          ) : null}
+        </span>
+        <code className="ctext">{highlightTs(text)}</code>
+        <span className="lbadges">
+          {hasAsserts ? asserts?.map((a: Assertion, i: number) => <AssertBadge key={i} a={a} />) : null}
+          {hasReply ? (
+            <span className="reply-hint">{isOpen ? t("code.hide") : t("code.reply")}</span>
+          ) : clickable ? (
+            <ChevronRight className={`line-chev${isOpen ? " is-open" : ""}`} aria-hidden="true" />
+          ) : null}
+        </span>
+      </div>
+      {isOpen && hasReply && turn ? <ReplyPanel turn={turn} t={t} /> : null}
+      {isOpen && hasAsserts && asserts ? <AssertDetail asserts={asserts} t={t} /> : null}
+    </>
+  );
+}
+
+/** 行尾分数徽章:judge / 带阈值的断言显示分数(过绿不过红);纯 gate 断言靠行色 + gutter 勾叉。 */
+export function AssertBadge({ a }: { a: Assertion }) {
+  const showPct = a.threshold !== undefined || (a.score > 0 && a.score < 1);
+  if (!showPct) return null;
+  return (
+    <span className={`abadge ${assertTone(a)}`}>
+      {formatScore(a.score)}
+      {a.threshold !== undefined ? <span className="abadge-th">/{formatScore(a.threshold)}</span> : null}
+    </span>
+  );
+}
+
+export function ReplyPanel({ turn, t }: { turn: SourceTurn; t: T }) {
+  if (!turn.replies.length) return <div className="line-detail reply-empty">{t("code.noReply")}</div>;
+  return (
+    <div className="line-detail reply-panel">
+      {turn.replies.map((r, j) => {
+        if (r.kind === "text")
+          return (
+            <div key={j} className="reply-assistant">
+              <span className="reply-role">{t("transcript.assistant")}</span>
+              <div className="reply-text">{r.text}</div>
+            </div>
+          );
+        if (r.kind === "thinking")
+          return (
+            <details key={j} className="reply-think">
+              <summary>{t("transcript.thinking")}</summary>
+              <div className="reply-think-text">{r.text}</div>
+            </details>
+          );
+        if (r.kind === "error")
+          return <div key={j} className="reply-err">! {r.text}</div>;
+        if (r.kind === "tool") {
+          const verb = (r.ev.tool ? TOOL_VERB[r.ev.tool] : undefined) || r.ev.name || r.ev.tool || "tool";
+          const arg = toolPrimaryArg(r.ev);
+          const out = r.result ? resultBody(r.result.output) : "";
+          return (
+            <div key={j} className="reply-tool">
+              <span className="reply-tool-name">{arg ? `${verb}(${truncate(arg, 80)})` : verb}</span>
+              {out ? <span className="reply-tool-out">→ {truncate(previewText(out), 100)}</span> : null}
+            </div>
+          );
+        }
+        return null;
+      })}
+    </div>
+  );
+}
+
+export function AssertDetail({ asserts, t }: { asserts: Assertion[]; t: T }) {
+  return (
+    <div className="line-detail assert-detail">
+      {asserts.map((a: Assertion, i: number) => (
+        <div key={i} className="assert-row">
+          <span className={`abadge ${assertTone(a)}`}>{a.passed ? t("assert.pass") : t("assert.fail")}</span>
+          <span className="assert-name">{a.name}</span>
+          {a.severity === "soft" ? <span className="assert-sev">{t("assert.soft")}</span> : null}
+          {a.threshold !== undefined ? (
+            <span className="assert-score">
+              {formatScore(a.score)} / {formatScore(a.threshold)}
+            </span>
+          ) : null}
+          {a.detail ? <div className="assert-reason">{a.detail}</div> : null}
+          {a.evidence ? (
+            <details className="assert-evidence">
+              <summary>{t("assert.evidence")}</summary>
+              <pre className="assert-evidence-pre">{a.evidence}</pre>
+            </details>
+          ) : null}
+        </div>
+      ))}
+    </div>
+  );
+}
+
+/**
+ * 没源码可叠时(此 run 早于 source-loc,或源码不可读:远程沙箱等)。不退回老的分组视图——
+ * 用代码视图同一套视觉语言:一句说明 + checks(绿过/红不过)+ 原始会话流。重跑即可看到代码视图。
+ */
+export function NoSourceBody({ assertions, events, t }: { assertions: Assertion[]; events: TranscriptEvent[]; t: T }) {
+  const checks = assertions || [];
+  return (
+    <div className="nosource">
+      <div className="nosource-note">
+        {t("code.noSource")}
+      </div>
+      {checks.length ? (
+        <div className="nosource-block">
+          <div className="nosource-head">{t("code.checks")}</div>
+          <AssertDetail asserts={checks} t={t} />
+        </div>
+      ) : null}
+      {events?.length ? (
+        <div className="nosource-block">
+          <div className="nosource-head">{t("code.conversation")}</div>
+          <Transcript events={events} t={t} />
+        </div>
+      ) : null}
+    </div>
+  );
+}