niceeval 0.5.4 → 0.6.0

package/src/scoring/scoped.ts

@@ -191,7 +191,19 @@ export function maxToolCalls(max: number): Spec {
 }
 
 export function loadedSkill(skill: string): Spec {
-  return calledTool("load_skill", { input: { skill } });
+  return {
+    name: `loadedSkill(${skill})`,
+    severity: "gate",
+    // 读 skill.loaded 一等事件,不按名字猜工具调用:各 agent 表达 Skill 加载的原生形态不同
+    // (Claude Code 是 Skill tool_use、eve 是 load-skill action kind),归一化的责任在 parser。
+    evaluate: (ctx) => {
+      const loaded = ctx.events.filter((e): e is Extract<StreamEvent, { type: "skill.loaded" }> => e.type === "skill.loaded");
+      const matched = loaded.filter((e) => e.skill === skill);
+      // 没命中时把实际加载过的 skill 列出来(常见失败是名字对不上,而不是一个都没加载)。
+      const shown = matched.length ? matched : loaded;
+      return { score: matched.length ? 1 : 0, evidence: shown.length ? shown.map((e) => e.skill).join(", ") : undefined };
+    },
+  };
 }
 
 export function noFailedActions(): Spec {

package/src/scoring/types.ts

@@ -1,9 +1,9 @@
-// scoring 域类型:值级断言(expect 匹配器)、断言记录与结果、评分上下文、judge 配置。
+// scoring 域类型:值断言(expect 匹配器)、断言记录与结果、评分上下文、judge 配置。
 
 import type { Severity, SourceLoc } from "../shared/types.ts";
 import type { DerivedFacts, StreamEvent, Usage } from "../o11y/types.ts";
 
-/** 值级断言(expect 匹配器)。纯函数 score + 可链式改严重级 / 阈值。 */
+/** 值断言(expect 匹配器)。纯函数 score + 可链式改严重度 / 阈值。 */
 export interface ValueAssertion {
   readonly name: string;
   readonly severity: Severity;
@@ -12,8 +12,8 @@ export interface ValueAssertion {
   /** 转成硬门槛断言:未达阈值(省略 threshold 则按 score > 0 判定)整条 eval 判为 failed。返回新实例,不改原对象。 */
   gate(threshold?: number): ValueAssertion;
   /**
-   * 转成软阈值断言:未达 threshold 时该条记为 failed,但默认不拖累整条 eval 的 outcome;
-   * `--strict` 运行下,软阈值失败也会把整条 eval 的 outcome 计为 failed。返回新实例,不改原对象。
+   * 转成软阈值断言:未达 threshold 时该条记为 failed,但默认不拖累整条 eval 的 verdict;
+   * `--strict` 运行下,软阈值失败也会把整条 eval 的 verdict 计为 failed。返回新实例,不改原对象。
    */
   atLeast(threshold: number): ValueAssertion;
 }
@@ -27,7 +27,7 @@ export interface AssertionSpec {
   evaluate(ctx: ScoringContext): Promise<number> | number;
 }
 
-/** 断言评估完的结果(进判决 / 报告)。 */
+/** 断言评估完的结果(进判定 / 报告)。 */
 export interface AssertionResult {
   name: string;
   severity: Severity;
@@ -39,7 +39,7 @@ export interface AssertionResult {
   evidence?: string;
   /** 所属分组(t.group 标题)。纯报告用,不影响 passed/score。 */
   group?: string;
-  /** 断言在 eval 源码里的调用点(栈回溯抠出);view 把判决叠回这一行。 */
+  /** 断言在 eval 源码里的调用点(栈回溯抠出);view 把判定叠回这一行。 */
   loc?: SourceLoc;
 }
 
@@ -71,7 +71,7 @@ export interface DiffData {
   deletedFiles: string[];
 }
 
-export type ResultOutcome = "passed" | "failed" | "errored" | "skipped";
+export type Verdict = "passed" | "failed" | "errored" | "skipped";
 
 export interface JudgeConfig {
   model: string;

package/src/scoring/verdict.ts

@@ -1,13 +1,13 @@
-// 判决:把执行结果 + 断言 + 跳过原因折叠成一个 Outcome(见 docs/scoring.md)。
+// 判定:把执行结果 + 断言 + 跳过原因折叠成一个 Verdict(见 docs/feature/scoring/README.md)。
 
-import type { AssertionResult, ResultOutcome } from "../types.ts";
+import type { AssertionResult, Verdict } from "../types.ts";
 
-export function computeOutcome(input: {
+export function computeVerdict(input: {
   error?: string;
   assertions: readonly AssertionResult[];
   skipReason?: string;
   strict?: boolean;
-}): ResultOutcome {
+}): Verdict {
   if (input.error !== undefined) return "errored";
 
   for (const a of input.assertions) {

package/src/shared/aggregate.ts

@@ -1,13 +1,13 @@
 // CLI 表格(runner/reporters/table.ts)与 view 榜单(view/aggregate.ts)共用的聚合小工具。
-// 实验标签推导、token/成本求和、outcome 排序各只有一份 —— 否则同一个实验在终端和网页上
+// 实验标签推导、token/成本求和、verdict 排序各只有一份 —— 否则同一个实验在终端和网页上
 // 会显示成两个名字 / 两组数。保持环境无关(纯函数,只 type import)。
 
 import type { Usage } from "../o11y/types.ts";
 import type { EvalResult, ExperimentRunInfo } from "../runner/types.ts";
-import type { ResultOutcome } from "../scoring/types.ts";
+import type { Verdict } from "../scoring/types.ts";
 
 /** 明细行排序:失败最靠前(failed > errored > skipped > passed 的紧急程度)。 */
-export const OUTCOME_ORDER: Record<ResultOutcome, number> = {
+export const VERDICT_ORDER: Record<Verdict, number> = {
   failed: 0,
   errored: 1,
   skipped: 2,
@@ -34,6 +34,26 @@ export function displayExperimentName(id: string | undefined): string | undefine
   return id.split("/").filter(Boolean).at(-1) ?? id;
 }
 
+/**
+ * 实验 id 的组推导:去掉末段的目录前缀("compare/bub-low" → "compare");
+ * 无 "/" 的顶层实验不属于任何组,返回 undefined。view 榜单分组与自定义报告
+ * 的分组用同一份,两边的「组」永远指同一个东西。
+ */
+export function experimentGroupOf(experimentId: string): string | undefined {
+  if (!experimentId.includes("/")) return undefined;
+  return experimentId.split("/").slice(0, -1).join("/");
+}
+
+/**
+ * eval id 前缀过滤,同 CLI 位置参数的分段语义(src/runner/discover.ts):
+ * "algebra" 匹配自身与 "algebra/..." 子级,不误配 "algebra2";允许 "algebra/" 尾斜杠写法,等价。
+ */
+export function evalPrefixPredicate(evals?: string | string[]): (id: string) => boolean {
+  if (evals === undefined) return () => true;
+  const prefixes = (Array.isArray(evals) ? evals : [evals]).map((p) => p.replace(/\/+$/, ""));
+  return (id) => prefixes.some((prefix) => id === prefix || id.startsWith(prefix + "/"));
+}
+
 /** 无 experimentId 时的兜底标签。 */
 export function fallbackExperimentLabel(result: {
   experiment?: ExperimentRunInfo;

package/src/shared/types.ts

@@ -1,4 +1,4 @@
-// 真正跨域的原子类型:序列化 / 严重级 / 源码位置 / 生命周期。
+// 真正跨域的原子类型:序列化 / 严重度 / 源码位置 / 生命周期。
 // 各域的类型住在各自目录的 types.ts(o11y / sandbox / agents / scoring / context / runner),
 // src/types.ts 是聚合 facade —— 模块代码统一从那里 import,不必记住每个类型的家。
 
@@ -11,7 +11,7 @@ export type JsonValue =
   | JsonValue[]
   | { [key: string]: JsonValue };
 
-/** 断言的严重级:"gate" 失败必判整轮 failed;"soft" 默认只记录不拦截,仅在 `--strict` 模式或显式设阈值未达标时才计入失败。 */
+/** 断言的严重度:"gate" 失败必判整轮 failed;"soft" 默认只记录不拦截,仅在 `--strict` 模式或显式设阈值未达标时才计入失败。 */
 export type Severity = "gate" | "soft";
 
 /**

package/src/shared/{outcome.ts → verdict.ts}

@@ -1,23 +1,23 @@
-// server(aggregate.ts)与前端(app/lib/outcome.ts)共用的判决折叠口径。
+// server(aggregate.ts)与前端(app/lib/verdict.ts)共用的判定折叠口径。
 // 必须保持环境无关且只依赖 type import,vite 前端会直接打包它。
 // 单独成模块的原因:折叠/计票口径两边必须逐字一致,否则折叠行状态会和 KPI / 成功率对不上。
 
-import type { ResultOutcome } from "../types.ts";
+import type { Verdict } from "../types.ts";
 
-/** 折叠/计票只需要 outcome 字段;server 传 EvalResult,前端传 ViewResult 都满足。 */
-export interface OutcomeLike {
-  outcome: ResultOutcome;
+/** 折叠/计票只需要 verdict 字段;server 传 EvalResult,前端传 ViewResult 都满足。 */
+export interface VerdictLike {
+  verdict: Verdict;
 }
 
 /**
- * 把同一个 eval 的多轮 attempt 折叠成单一判决:任一轮通过 → 该 eval 通过(对齐 earlyExit
+ * 把同一个 eval 的多轮 attempt 折叠成单一判定:任一轮通过 → 该 eval 通过(对齐 earlyExit
  * 「先过一次即停」语义),否则按 failed > errored > skipped 取最严重的一个。
  */
-export function foldEvalOutcome(attempts: OutcomeLike[]): ResultOutcome {
-  const outcomes = attempts.map((a) => a.outcome);
-  if (outcomes.some((o) => o === "passed")) return "passed";
-  if (outcomes.some((o) => o === "failed")) return "failed";
-  if (outcomes.some((o) => o === "errored")) return "errored";
+export function foldEvalVerdict(attempts: VerdictLike[]): Verdict {
+  const verdicts = attempts.map((a) => a.verdict);
+  if (verdicts.some((o) => o === "passed")) return "passed";
+  if (verdicts.some((o) => o === "failed")) return "failed";
+  if (verdicts.some((o) => o === "errored")) return "errored";
   return "skipped";
 }
 
@@ -33,16 +33,16 @@ export interface EvalLevelStats {
 
 /**
  * 通过率与 passed/failed 一律按 eval 计票,不按 attempt:每个 eval 不管跑几轮都只占一票,先把它
- * 的多轮折叠成单一判决再计数。否则 runs>1 时同一 eval 的 N 次 attempt 各算一票 —— 尤其 earlyExit
+ * 的多轮折叠成单一判定再计数。否则 runs>1 时同一 eval 的 N 次 attempt 各算一票 —— 尤其 earlyExit
  * 开时通过的 eval 只留 1 次、失败的 eval 跑满 N 次,失败 eval 被重复计入分母,把通过率拉低
- * (见 docs/runner.md、docs/scoring.md)。keyOf 决定「一个 eval」的粒度:单实验按 eval id,
+ * (见 docs/runner.md、docs/feature/scoring/README.md)。keyOf 决定「一个 eval」的粒度:单实验按 eval id,
  * 跨实验组按 experimentId|eval id。
  */
-export function evalLevelStats<T extends OutcomeLike>(results: T[], keyOf: (r: T) => string): EvalLevelStats {
+export function evalLevelStats<T extends VerdictLike>(results: T[], keyOf: (r: T) => string): EvalLevelStats {
   const byEval = new Map<string, T[]>();
   for (const r of results) byEval.set(keyOf(r), [...(byEval.get(keyOf(r)) ?? []), r]);
   const counts = { passed: 0, failed: 0, errored: 0, skipped: 0 };
-  for (const group of byEval.values()) counts[foldEvalOutcome(group)] += 1;
+  for (const group of byEval.values()) counts[foldEvalVerdict(group)] += 1;
   const ran = counts.passed + counts.failed + counts.errored; // skipped 不进分母
   return { evals: byEval.size, ...counts, passRate: ran ? counts.passed / ran : 0 };
 }

package/src/show/compose.ts

@@ -1,140 +1,20 @@
-// show 宿主的选集合成与时间轴口径(docs-site/zh/guides/viewing-results.mdx 是行为规范)。
+// show 专属的跨 run 时间轴口径(--history;docs-site/zh/guides/viewing-results.mdx 是行为规范)。
 //
-// 「现刻水位」= 每个 experiment × eval 取时间上最新的那份判决,跨 run 合成:
-// results.latest() 只挑「每实验最新快照」,带 eval 前缀的局部重跑会产出残缺快照;
-// 榜单承诺「不会因为一次局部重跑变残缺」,所以宿主在实验的全部历史快照上逐 eval
-// 向更早的 run 补齐,再把合成好的选集注入报告槽——内置默认报告与 --report 吃同一份,
-// 默认报告口径 = 宿主注入口径。本文件只消费 niceeval/results 的读取面。
+// 现刻水位 Selection(两个宿主共用)住在 ../results/select.ts 的 selectCurrentResults;
+// 本文件只留 show 独有的时间轴计算:每个快照 / 每次真实执行一行,resume 携带的复印件不占行。
+// 数据只消费 niceeval/results 的读取面。
 
-import { foldEvalOutcome } from "../shared/outcome.ts";
-import { evalPrefixPredicate } from "../report/aggregate.ts";
+import { foldEvalVerdict } from "../shared/verdict.ts";
 import { attemptCostUSD } from "../report/metrics.ts";
-import { makeSelection } from "../results/select.ts";
-import type { ResultOutcome } from "../types.ts";
-import type {
-  AttemptHandle,
-  Eval,
-  Experiment,
-  Results,
-  Selection,
-  SelectionWarning,
-  Snapshot,
-} from "../results/index.ts";
-
-export interface ComposeOptions {
-  /** experiment id 前缀(--experiment),与 latest({ experiments }) 同一分段匹配语义。 */
-  experiment?: string;
-  /** eval id 前缀(位置参数),收窄选集覆盖的 eval;覆盖警告分母 = 已知并集 ∩ 范围。 */
-  patterns?: string[];
-}
-
-/** --experiment 的实验过滤,同 results.latest({ experiments }) 的分段前缀语义。 */
-export function filterExperiments(experiments: Experiment[], prefix?: string): Experiment[] {
-  if (prefix === undefined) return experiments;
-  const p = prefix.replace(/\/+$/, "");
-  return experiments.filter((exp) => exp.id === p || exp.id.startsWith(p + "/"));
-}
-
-/**
- * 合成「现刻水位」选集:每个实验一份合成快照,快照里每道题的判决取该题最后一次
- * 出现的快照(--resume 携带的复印件身份与原判决相同,取到哪份内容都一致)。
- * 警告随选集重算:partial-coverage 的分母 = 已知并集 ∩ 范围;stale / synthetic
- * 与 results.latest() 同口径。
- */
-export function composeShowSelection(results: Results, opts: ComposeOptions = {}): Selection {
-  const match =
-    opts.patterns && opts.patterns.length > 0 ? evalPrefixPredicate(opts.patterns) : () => true;
-  const experiments = filterExperiments(results.experiments, opts.experiment);
-
-  const snapshots: Snapshot[] = [];
-  const warnings: SelectionWarning[] = [];
-
-  for (const exp of experiments) {
-    // 逐题取最新:快照按最新在前,首个出现即最新判决
-    const taken = new Map<string, { ev: Eval; snapshot: Snapshot }>();
-    for (const snapshot of exp.snapshots) {
-      for (const ev of snapshot.evals) {
-        if (!match(ev.id) || taken.has(ev.id)) continue;
-        taken.set(ev.id, { ev, snapshot });
-      }
-    }
-    if (taken.size === 0) continue;
-
-    const picks = [...taken.values()].sort((a, b) => a.ev.id.localeCompare(b.ev.id));
-    let startedAt = "";
-    let newest: Snapshot = picks[0].snapshot;
-    for (const pick of picks) {
-      if (pick.snapshot.startedAt > startedAt) {
-        startedAt = pick.snapshot.startedAt;
-        newest = pick.snapshot;
-      }
-    }
-    const evals = picks.map((p) => p.ev);
-    const base = exp.latest;
-    snapshots.push({
-      experimentId: exp.id,
-      startedAt,
-      agent: base.agent,
-      ...(base.model !== undefined ? { model: base.model } : {}),
-      ...(base.producer ? { producer: base.producer } : {}),
-      schemaVersion: base.schemaVersion,
-      evals,
-      attempts: evals.flatMap((ev) => ev.attempts),
-      runDir: newest.runDir,
-      ...(base.synthetic ? { synthetic: true } : {}),
-      ...(base.knownEvalIds ? { knownEvalIds: [...base.knownEvalIds] } : {}),
-    });
-
-    // 残缺检测:跨 run 补齐后仍缺,只可能是历史上见过(或 knownEvalIds 声明过)
-    // 却从未在可读落盘里出现的题 —— 分母收窄到范围内,不让范围外的缺口刷屏。
-    const total = exp.evalIds.filter(match).length;
-    if (evals.length < total) {
-      const hint = base.synthetic
-        ? "re-run the experiment for a full snapshot"
-        : `re-run \`niceeval exp ${exp.id}\` for a full snapshot`;
-      warnings.push({
-        kind: "partial-coverage",
-        experimentId: exp.id,
-        covered: evals.length,
-        total,
-        message: `verdicts cover ${evals.length} of ${total} evals seen in history; ${hint}`,
-      });
-    }
-  }
-
-  let latestStartedAt = "";
-  for (const snapshot of snapshots) {
-    if (snapshot.startedAt > latestStartedAt) latestStartedAt = snapshot.startedAt;
-  }
-  for (const snapshot of snapshots) {
-    if (snapshot.startedAt < latestStartedAt) {
-      warnings.push({
-        kind: "stale-snapshot",
-        experimentId: snapshot.experimentId,
-        startedAt: snapshot.startedAt,
-        latestStartedAt,
-        message: `verdicts for "${snapshot.experimentId}" were produced at ${snapshot.startedAt}, before the latest run in this selection (${latestStartedAt})`,
-      });
-    }
-    if (snapshot.synthetic) {
-      warnings.push({
-        kind: "synthetic-experiment-id",
-        experimentId: snapshot.experimentId,
-        runDir: snapshot.runDir.dir,
-        message: `run "${snapshot.runDir.dir}" has results without experimentId; grouped as "${snapshot.experimentId}" by agent/model`,
-      });
-    }
-  }
-
-  return makeSelection(snapshots, warnings);
-}
+import type { Verdict } from "../types.ts";
+import type { AttemptHandle, Experiment } from "../results/index.ts";
 
 // ───────────────────────── 时间轴(--history)─────────────────────────
 
 export interface EvalHistoryRow {
   /** 该次真实执行所在 run(快照)的时刻。 */
   startedAt: string;
-  outcome: ResultOutcome;
+  verdict: Verdict;
   attempts: number;
   costUSD: number | null;
   /** 最新一次 attempt 的第一条失败断言("gate calledTool(...)")。 */
@@ -151,7 +31,7 @@ function attemptKey(attempt: AttemptHandle): string | undefined {
 
 /**
  * 单 eval 的跨 run 时间轴:每次真实执行一行,新→旧。--resume 携带的复印件
- * (身份键与原判决相同的条目)不占行 —— 否则趋势会被复印件灌满假数据。
+ * (身份键与原判定相同的条目)不占行 —— 否则趋势会被复印件灌满假数据。
  */
 export function evalHistory(exp: Experiment, evalId: string): EvalHistoryRow[] {
   const rows: EvalHistoryRow[] = [];
@@ -171,7 +51,7 @@ export function evalHistory(exp: Experiment, evalId: string): EvalHistoryRow[] {
       seen.add(key);
       fresh.push(attempt);
     }
-    if (fresh.length === 0) continue; // 纯复印件:判决在更早的行里已经出现过
+    if (fresh.length === 0) continue; // 纯复印件:判定在更早的行里已经出现过
     let cost: number | null = null;
     for (const attempt of fresh) {
       const c = attemptCostUSD(attempt.result);
@@ -181,7 +61,7 @@ export function evalHistory(exp: Experiment, evalId: string): EvalHistoryRow[] {
     const failed = latest.result.assertions.find((a) => !a.passed);
     rows.push({
       startedAt: snapshot.startedAt,
-      outcome: foldEvalOutcome(fresh.map((a) => a.result)),
+      verdict: foldEvalVerdict(fresh.map((a) => a.result)),
       attempts: fresh.length,
       costUSD: cost,
       ...(failed ? { failedAssertion: `${failed.severity} ${failed.name}` } : {}),
@@ -203,7 +83,7 @@ export function experimentHistory(exp: Experiment): ExperimentHistoryRow[] {
   return exp.snapshots.map((snapshot) => {
     let passed = 0;
     for (const ev of snapshot.evals) {
-      if (foldEvalOutcome(ev.attempts.map((a) => a.result)) === "passed") passed += 1;
+      if (foldEvalVerdict(ev.attempts.map((a) => a.result)) === "passed") passed += 1;
     }
     let cost: number | null = null;
     for (const attempt of snapshot.attempts) {