niceeval 0.1.1 → 0.2.0

package/src/scoring/scoped.ts

@@ -49,8 +49,39 @@ function toolMatches(tc: ToolCall, name: string, match?: ToolMatch): boolean {
   return true;
 }
 
-function countMatches(toolCalls: readonly ToolCall[], name: string, match?: ToolMatch): number {
-  return toolCalls.filter((tc) => toolMatches(tc, name, match)).length;
+// ── evidence:把调用的出入参带回断言结果,view 展开可见,不用翻原始事件流 ──
+
+function briefJson(value: unknown, max = 800): string {
+  let s: string;
+  try {
+    s = JSON.stringify(value) ?? String(value);
+  } catch {
+    s = String(value);
+  }
+  return s.length > max ? s.slice(0, max) + "…" : s;
+}
+
+function describeCalls(calls: readonly ToolCall[]): string | undefined {
+  if (calls.length === 0) return undefined;
+  return calls
+    .map((tc) => {
+      const name = tc.originalName && tc.originalName !== tc.name ? tc.originalName : tc.name;
+      const lines = [`${name} [${tc.status}]`, `  input: ${briefJson(tc.input)}`];
+      if (tc.output !== undefined) lines.push(`  output: ${briefJson(tc.output)}`);
+      return lines.join("\n");
+    })
+    .join("\n");
+}
+
+function describeSubagents(calls: readonly SubagentCall[]): string | undefined {
+  if (calls.length === 0) return undefined;
+  return calls
+    .map((s) => {
+      const lines = [`${s.name} [${s.status}]${s.remoteUrl ? ` ${s.remoteUrl}` : ""}`];
+      if (s.output !== undefined) lines.push(`  output: ${briefJson(s.output)}`);
+      return lines.join("\n");
+    })
+    .join("\n");
 }
 
 function subagentMatches(call: SubagentCall, name: string, match?: SubagentMatch): boolean {
@@ -89,7 +120,8 @@ export function messageIncludes(token: string | RegExp): Spec {
         .map((e) => e.text)
         .join("\n");
       const ok = token instanceof RegExp ? token.test(text) : text.includes(token);
-      return ok ? 1 : 0;
+      // 失败时把实际被扫的助手文本带回来(和 t.check 的口径一致);命中时行色已说明一切。
+      return ok ? 1 : { score: 0, evidence: text ? (text.length > 4000 ? text.slice(0, 4000) + "…" : text) : undefined };
     },
   };
 }
@@ -99,9 +131,13 @@ export function calledTool(name: string, match?: ToolMatch): Spec {
     name: `calledTool(${name})`,
     severity: "gate",
     evaluate: (ctx) => {
-      const n = countMatches(ctx.facts.toolCalls, name, match);
-      if (match?.count !== undefined) return n === match.count ? 1 : 0;
-      return n >= 1 ? 1 : 0;
+      const matched = ctx.facts.toolCalls.filter((tc) => toolMatches(tc, name, match));
+      const n = matched.length;
+      const score = match?.count !== undefined ? (n === match.count ? 1 : 0) : n >= 1 ? 1 : 0;
+      // 命中给命中调用的出入参;没命中给同名调用(条件不满足的近失);再没有就列出实际调过的工具。
+      const sameName = ctx.facts.toolCalls.filter((tc) => tc.name === name || tc.originalName === name);
+      const shown = matched.length ? matched : sameName.length ? sameName : ctx.facts.toolCalls;
+      return { score, evidence: describeCalls(shown) };
     },
   };
 }
@@ -110,7 +146,10 @@ export function notCalledTool(name: string, match?: ToolMatch): Spec {
   return {
     name: `notCalledTool(${name})`,
     severity: "gate",
-    evaluate: (ctx) => (countMatches(ctx.facts.toolCalls, name, match) === 0 ? 1 : 0),
+    evaluate: (ctx) => {
+      const matched = ctx.facts.toolCalls.filter((tc) => toolMatches(tc, name, match));
+      return { score: matched.length === 0 ? 1 : 0, evidence: describeCalls(matched) };
+    },
   };
 }
 
@@ -123,7 +162,8 @@ export function toolOrder(names: string[]): Spec {
       for (const tc of ctx.facts.toolCalls) {
         if (i < names.length && (tc.name === names[i] || tc.originalName === names[i])) i++;
       }
-      return i === names.length ? 1 : 0;
+      const actual = ctx.facts.toolCalls.map((tc) => tc.originalName ?? tc.name).join(" → ");
+      return { score: i === names.length ? 1 : 0, evidence: actual || undefined };
     },
   };
 }
@@ -132,7 +172,10 @@ export function usedNoTools(): Spec {
   return {
     name: "usedNoTools",
     severity: "gate",
-    evaluate: (ctx) => (ctx.facts.toolCalls.length === 0 ? 1 : 0),
+    evaluate: (ctx) => ({
+      score: ctx.facts.toolCalls.length === 0 ? 1 : 0,
+      evidence: describeCalls(ctx.facts.toolCalls),
+    }),
   };
 }
 
@@ -140,7 +183,10 @@ export function maxToolCalls(max: number): Spec {
   return {
     name: `maxToolCalls(${max})`,
     severity: "gate",
-    evaluate: (ctx) => (ctx.facts.toolCalls.length <= max ? 1 : 0),
+    evaluate: (ctx) => ({
+      score: ctx.facts.toolCalls.length <= max ? 1 : 0,
+      evidence: describeCalls(ctx.facts.toolCalls),
+    }),
   };
 }
 
@@ -153,9 +199,10 @@ export function noFailedActions(): Spec {
     name: "noFailedActions",
     severity: "gate",
     evaluate: (ctx) => {
-      const toolFail = ctx.facts.toolCalls.some((tc) => tc.status === "failed");
-      const subFail = ctx.facts.subagentCalls.some((s) => s.status === "failed");
-      return toolFail || subFail ? 0 : 1;
+      const failedTools = ctx.facts.toolCalls.filter((tc) => tc.status === "failed");
+      const failedSubs = ctx.facts.subagentCalls.filter((s) => s.status === "failed");
+      const evidence = [describeCalls(failedTools), describeSubagents(failedSubs)].filter(Boolean).join("\n") || undefined;
+      return { score: failedTools.length || failedSubs.length ? 0 : 1, evidence };
     },
   };
 }
@@ -165,9 +212,10 @@ export function calledSubagent(name: string, match?: SubagentMatch): Spec {
     name: `calledSubagent(${name})`,
     severity: "gate",
     evaluate: (ctx) => {
-      const n = ctx.facts.subagentCalls.filter((call) => subagentMatches(call, name, match)).length;
-      if (match?.count !== undefined) return n === match.count ? 1 : 0;
-      return n >= 1 ? 1 : 0;
+      const matched = ctx.facts.subagentCalls.filter((call) => subagentMatches(call, name, match));
+      const n = matched.length;
+      const score = match?.count !== undefined ? (n === match.count ? 1 : 0) : n >= 1 ? 1 : 0;
+      return { score, evidence: describeSubagents(matched.length ? matched : ctx.facts.subagentCalls) };
     },
   };
 }
@@ -258,8 +306,8 @@ export function noFailedShellCommands(): Spec {
     name: "noFailedShellCommands",
     severity: "gate",
     evaluate: (ctx) => {
-      const failed = ctx.facts.toolCalls.some((tc) => tc.name === "shell" && tc.status === "failed");
-      return failed ? 0 : 1;
+      const failed = ctx.facts.toolCalls.filter((tc) => tc.name === "shell" && tc.status === "failed");
+      return { score: failed.length ? 0 : 1, evidence: describeCalls(failed) };
     },
   };
 }

package/src/scoring/types.ts

@@ -0,0 +1,76 @@
+// scoring 域类型:值级断言(expect 匹配器)、断言记录与结果、评分上下文、judge 配置。
+
+import type { Severity, SourceLoc } from "../shared/types.ts";
+import type { DerivedFacts, StreamEvent, Usage } from "../o11y/types.ts";
+
+/** 值级断言(expect 匹配器)。纯函数 score + 可链式改严重级 / 阈值。 */
+export interface ValueAssertion {
+  readonly name: string;
+  readonly severity: Severity;
+  readonly threshold?: number;
+  score(value: unknown): number | Promise<number>;
+  gate(threshold?: number): ValueAssertion;
+  atLeast(threshold: number): ValueAssertion;
+}
+
+/** 收集到 collector 里的一条断言记录(评估前)。 */
+export interface AssertionSpec {
+  name: string;
+  severity: Severity;
+  threshold?: number;
+  /** 延迟评估:final 时拿到完整运行结果再算分。 */
+  evaluate(ctx: ScoringContext): Promise<number> | number;
+}
+
+/** 断言评估完的结果(进判决 / 报告)。 */
+export interface AssertionResult {
+  name: string;
+  severity: Severity;
+  threshold?: number;
+  score: number;
+  passed: boolean;
+  detail?: string;
+  /** 这条分数是看着什么材料算出来的(judge 收到的输入,或 t.check 失败时实际被检查的值)。view 展开排查「为什么是这个分」,默认不展示。 */
+  evidence?: string;
+  /** 所属分组(t.group 标题)。纯报告用,不影响 passed/score。 */
+  group?: string;
+  /** 断言在 eval 源码里的调用点(栈回溯抠出);view 把判决叠回这一行。 */
+  loc?: SourceLoc;
+}
+
+/** eval 作者拿到的可链式句柄(t.judge.autoevals.closedQA(...).atLeast(0.7))。 */
+export interface AssertionHandle {
+  atLeast(threshold: number): AssertionHandle;
+  gate(threshold?: number): AssertionHandle;
+}
+
+/** scoped / judge 断言在 final 评估时拿到的运行结果。 */
+export interface ScoringContext {
+  readonly events: readonly StreamEvent[];
+  readonly facts: DerivedFacts;
+  readonly diff: DiffData;
+  readonly scripts: Record<string, ScriptResult>;
+  readonly usage: Usage;
+  readonly status: "completed" | "failed" | "waiting";
+  /** 读沙箱里某文件的最终内容(judge / file 断言用)。 */
+  readFile(path: string): Promise<string | undefined>;
+}
+
+export interface ScriptResult {
+  success: boolean;
+  output: string;
+}
+
+export interface DiffData {
+  generatedFiles: Record<string, string>;
+  deletedFiles: string[];
+}
+
+export type ResultOutcome = "passed" | "failed" | "errored" | "skipped";
+
+export interface JudgeConfig {
+  model: string;
+  /** OpenAI 兼容 base url + key 来源;省略则从 env 探测(见 scoring/judge.ts)。 */
+  baseUrl?: string;
+  apiKeyEnv?: string;
+}

package/src/shared/aggregate.ts

@@ -0,0 +1,48 @@
+// CLI 表格(runner/reporters/table.ts)与 view 榜单(view/aggregate.ts)共用的聚合小工具。
+// 实验标签推导、token/成本求和、outcome 排序各只有一份 —— 否则同一个实验在终端和网页上
+// 会显示成两个名字 / 两组数。保持环境无关(纯函数,只 type import)。
+
+import type { Usage } from "../o11y/types.ts";
+import type { EvalResult, ExperimentRunInfo } from "../runner/types.ts";
+import type { ResultOutcome } from "../scoring/types.ts";
+
+/** 明细行排序:失败最靠前(failed > errored > skipped > passed 的紧急程度)。 */
+export const OUTCOME_ORDER: Record<ResultOutcome, number> = {
+  failed: 0,
+  errored: 1,
+  skipped: 2,
+  passed: 3,
+};
+
+export function totalTokens(items: Array<Usage | undefined>): number {
+  return items.reduce((n, u) => n + (u?.inputTokens ?? 0) + (u?.outputTokens ?? 0), 0);
+}
+
+/** 求和,但「全都没报」时返回 undefined(区别于真 0 成本)。 */
+export function sumMaybe(items: Array<number | undefined>): number | undefined {
+  const known = items.filter((n): n is number => n !== undefined);
+  return known.length ? known.reduce((sum, n) => sum + n, 0) : undefined;
+}
+
+export function avg(items: number[]): number {
+  return items.length ? items.reduce((sum, n) => sum + n, 0) / items.length : 0;
+}
+
+/** 实验 id 的展示名:取路径最后一段(exp 分组用目录表达)。 */
+export function displayExperimentName(id: string | undefined): string | undefined {
+  if (!id) return undefined;
+  return id.split("/").filter(Boolean).at(-1) ?? id;
+}
+
+/** 无 experimentId 时的兜底标签。 */
+export function fallbackExperimentLabel(result: {
+  experiment?: ExperimentRunInfo;
+  agent: string;
+  model?: string;
+}): string {
+  if (result.experiment?.id) return displayExperimentName(result.experiment.id) ?? result.experiment.id;
+  if (result.model) return `${result.agent}/${result.model}`;
+  return result.agent || "ad hoc run";
+}
+
+export type { EvalResult };

package/src/shared/format.ts

@@ -0,0 +1,20 @@
+// server 与前端共用的数值格式化;两边展示口径(取整/单位切换阈值)必须一致,所以单独成模块。
+// 保持环境无关:不 import node/browser API。日期格式化不在这里 —— 它按前端当前 locale 做,
+// 留在 app/lib/format.ts(见 formatDateTime)。
+
+export function formatPercent(value?: number): string {
+  if (typeof value !== "number" || !Number.isFinite(value)) return "0%";
+  return Math.round(value * 100) + "%";
+}
+
+export function formatDuration(ms?: number): string {
+  if (typeof ms !== "number" || !Number.isFinite(ms) || ms <= 0) return "0ms";
+  if (ms >= 60_000) return (ms / 60_000).toFixed(1) + "m";
+  if (ms >= 1000) return (ms / 1000).toFixed(2) + "s";
+  return Math.round(ms) + "ms";
+}
+
+export function formatCost(value?: number): string {
+  if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) return "$0";
+  return "$" + value.toFixed(value < 1 ? 3 : 2);
+}

package/src/shared/outcome.ts

@@ -0,0 +1,48 @@
+// server(aggregate.ts)与前端(app/lib/outcome.ts)共用的判决折叠口径。
+// 必须保持环境无关且只依赖 type import,vite 前端会直接打包它。
+// 单独成模块的原因:折叠/计票口径两边必须逐字一致,否则折叠行状态会和 KPI / 成功率对不上。
+
+import type { ResultOutcome } from "../types.ts";
+
+/** 折叠/计票只需要 outcome 字段;server 传 EvalResult,前端传 ViewResult 都满足。 */
+export interface OutcomeLike {
+  outcome: ResultOutcome;
+}
+
+/**
+ * 把同一个 eval 的多轮 attempt 折叠成单一判决:任一轮通过 → 该 eval 通过(对齐 earlyExit
+ * 「先过一次即停」语义),否则按 failed > errored > skipped 取最严重的一个。
+ */
+export function foldEvalOutcome(attempts: OutcomeLike[]): ResultOutcome {
+  const outcomes = attempts.map((a) => a.outcome);
+  if (outcomes.some((o) => o === "passed")) return "passed";
+  if (outcomes.some((o) => o === "failed")) return "failed";
+  if (outcomes.some((o) => o === "errored")) return "errored";
+  return "skipped";
+}
+
+export interface EvalLevelStats {
+  /** 去重后的 eval 数(成功率分母的口径)。 */
+  evals: number;
+  passed: number;
+  failed: number;
+  errored: number;
+  skipped: number;
+  passRate: number;
+}
+
+/**
+ * 通过率与 passed/failed 一律按 eval 计票,不按 attempt:每个 eval 不管跑几轮都只占一票,先把它
+ * 的多轮折叠成单一判决再计数。否则 runs>1 时同一 eval 的 N 次 attempt 各算一票 —— 尤其 earlyExit
+ * 开时通过的 eval 只留 1 次、失败的 eval 跑满 N 次,失败 eval 被重复计入分母,把通过率拉低
+ * (见 docs/runner.md、docs/scoring.md)。keyOf 决定「一个 eval」的粒度:单实验按 eval id,
+ * 跨实验组按 experimentId|eval id。
+ */
+export function evalLevelStats<T extends OutcomeLike>(results: T[], keyOf: (r: T) => string): EvalLevelStats {
+  const byEval = new Map<string, T[]>();
+  for (const r of results) byEval.set(keyOf(r), [...(byEval.get(keyOf(r)) ?? []), r]);
+  const counts = { passed: 0, failed: 0, errored: 0, skipped: 0 };
+  for (const group of byEval.values()) counts[foldEvalOutcome(group)] += 1;
+  const ran = counts.passed + counts.failed + counts.errored; // skipped 不进分母
+  return { evals: byEval.size, ...counts, passRate: ran ? counts.passed / ran : 0 };
+}

package/src/shared/types.ts

@@ -0,0 +1,37 @@
+// 真正跨域的原子类型:序列化 / 严重级 / 源码位置 / 生命周期。
+// 各域的类型住在各自目录的 types.ts(o11y / sandbox / agents / scoring / context / runner),
+// src/types.ts 是聚合 facade —— 模块代码统一从那里 import,不必记住每个类型的家。
+
+export type JsonValue =
+  | string
+  | number
+  | boolean
+  | null
+  | JsonValue[]
+  | { [key: string]: JsonValue };
+
+export type Severity = "gate" | "soft";
+
+/**
+ * eval 源码里一次调用的位置(`t.send` / 各断言),运行期从栈回溯抠出来(见 src/source-loc.ts)。
+ * view 据此把运行结果叠回真实源码行(github-diff 式代码视图)。`file` 为相对项目根的路径。
+ */
+export interface SourceLoc {
+  file: string;
+  line: number;
+  column?: number;
+}
+
+/** 随结果回传的一份 eval 源码(相对项目根的路径 + 文本),供 view 渲染代码视图。 */
+export interface SourceArtifact {
+  path: string;
+  content: string;
+}
+
+export type Cleanup = () => Promise<void> | void;
+
+/**
+ * 可本地化文案:纯字符串,或按 locale 代码(如 "en"、"zh-CN")映射多语言。
+ * view 按当前界面语言挑一条,挑不到回退到 en / 第一条。
+ */
+export type LocalizedText = string | Record<string, string>;