niceeval 0.5.4 → 0.6.0

package/src/results/attempt-evidence.test.ts

@@ -0,0 +1,158 @@
+// loadAttemptEvidence 单测(定稿契约见 plan/attempt-evidence-feedback-loop.md「中性数据准备」、
+// src/results/attempt-evidence.ts 的头注)。用真实 createResultsWriter → openResults 的读写链路
+// 落一份最小 fixture(不手写 JSON 文件,理由同 loadAnnotatedEvalSource 的既有端到端测试:
+// 这条链路本身就是被测对象的一部分),覆盖四种 capability 组合与 identity 正确性。
+
+import { afterEach, describe, expect, it } from "vitest";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+  createResultsWriter,
+  loadAttemptEvidence,
+  openResults,
+  type AttemptArtifacts,
+  type AttemptEntry,
+  type AttemptHandle,
+  type DiffData,
+  type EvalResult,
+  type StreamEvent,
+  type TraceSpan,
+} from "./index.ts";
+
+const roots: string[] = [];
+async function makeRoot(): Promise<string> {
+  const root = await mkdtemp(join(tmpdir(), "niceeval-attempt-evidence-"));
+  roots.push(root);
+  return root;
+}
+afterEach(async () => {
+  await Promise.all(roots.splice(0).map((r) => rm(r, { recursive: true, force: true })));
+});
+
+const SOURCE_PATH = "evals/a.eval.ts";
+const SOURCE_CONTENT = 'import { defineEval } from "niceeval";\nexport default defineEval({\n  test() {},\n});\n';
+
+const ASSERTIONS: EvalResult["assertions"] = [
+  { name: "check-1", passed: true, severity: "gate", score: 1, loc: { file: SOURCE_PATH, line: 3 } },
+];
+
+const EVENTS: StreamEvent[] = [
+  { type: "message", role: "assistant", text: "looking at weather" },
+  { type: "action.called", callId: "c1", name: "get_weather", input: { city: "Brooklyn" } },
+  { type: "action.result", callId: "c1", output: { tempF: 72 }, status: "completed" },
+];
+
+const TRACE: TraceSpan[] = [
+  { traceId: "t1", spanId: "s1", name: "tool.get_weather", startMs: 0, endMs: 10, attributes: { call_id: "c1" } },
+];
+
+const NONEMPTY_DIFF: DiffData = { generatedFiles: { "a.txt": "hello" }, deletedFiles: [] };
+const EMPTY_DIFF: DiffData = { generatedFiles: {}, deletedFiles: [] };
+
+/** 起一个 writer,写一条 attempt,finish,再从头 openResults 读回它的 AttemptHandle。 */
+async function seedAttempt(
+  root: string,
+  entry: Partial<AttemptEntry> & { id: string },
+  artifacts?: AttemptArtifacts,
+): Promise<AttemptHandle> {
+  const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
+  const snap = await writer.snapshot({ experimentId: "compare/bub", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
+  await snap.writeAttempt(
+    { verdict: "passed", attempt: 0, durationMs: 1000, assertions: [], ...entry },
+    artifacts,
+  );
+  await writer.finish();
+  const results = await openResults(root);
+  return results.experiments[0]!.latest.evals.find((e) => e.id === entry.id)!.attempts[0]!;
+}
+
+describe("loadAttemptEvidence", () => {
+  it("四个 capability 全部具备:eval / execution / timing / diff", async () => {
+    const root = await makeRoot();
+    const attempt = await seedAttempt(
+      root,
+      { id: "weather/brooklyn", assertions: ASSERTIONS },
+      { events: EVENTS, trace: TRACE, diff: NONEMPTY_DIFF, sources: [{ path: SOURCE_PATH, content: SOURCE_CONTENT }] },
+    );
+
+    const evidence = await loadAttemptEvidence(attempt);
+
+    expect(evidence.evalSource).not.toBeNull();
+    expect(evidence.evalSource!.sourcePath).toBe(SOURCE_PATH);
+    expect(evidence.execution).not.toBeNull();
+    expect(evidence.execution!.timingAvailable).toBe(true);
+    // action 节点唯一关联上了 span(call_id 精确匹配),不是只挂了个 telemetry-only 节点。
+    const actionNode = evidence.execution!.nodes.find((n) => n.kind === "action");
+    expect(actionNode).toBeDefined();
+    expect((actionNode as { span?: TraceSpan }).span).toBeDefined();
+    expect(evidence.diff).toEqual(NONEMPTY_DIFF);
+
+    expect(evidence.capabilities).toEqual({ eval: true, execution: true, timing: true, diff: true });
+  });
+
+  it("四个 capability 全部缺失:没有 source / events / trace / diff,不崩溃", async () => {
+    const root = await makeRoot();
+    const attempt = await seedAttempt(root, { id: "weather/queens" });
+
+    const evidence = await loadAttemptEvidence(attempt);
+
+    expect(evidence.evalSource).toBeNull();
+    expect(evidence.execution).toBeNull();
+    expect(evidence.diff).toBeNull();
+    expect(evidence.capabilities).toEqual({ eval: false, execution: false, timing: false, diff: false });
+  });
+
+  it("有 events 没有 OTel spans:execution 为真、timing 为假", async () => {
+    const root = await makeRoot();
+    const attempt = await seedAttempt(root, { id: "weather/albany" }, { events: EVENTS });
+
+    const evidence = await loadAttemptEvidence(attempt);
+
+    expect(evidence.execution).not.toBeNull();
+    expect(evidence.execution!.timingAvailable).toBe(false);
+    expect(evidence.execution!.nodes.some((n) => n.kind === "action")).toBe(true);
+    expect(evidence.capabilities.execution).toBe(true);
+    expect(evidence.capabilities.timing).toBe(false);
+  });
+
+  it("diff 存在但两个数组都空:capabilities.diff 为假,不是真", async () => {
+    const root = await makeRoot();
+    const attempt = await seedAttempt(root, { id: "weather/rome" }, { diff: EMPTY_DIFF });
+
+    const evidence = await loadAttemptEvidence(attempt);
+
+    expect(evidence.diff).toEqual(EMPTY_DIFF);
+    expect(evidence.capabilities.diff).toBe(false);
+  });
+
+  it("identity 与源 attempt 的 experimentId / snapshotStartedAt / evalId / attempt 完全一致", async () => {
+    const root = await makeRoot();
+    const attempt = await seedAttempt(root, { id: "weather/dover", attempt: 0 });
+
+    const evidence = await loadAttemptEvidence(attempt);
+
+    expect(evidence.identity).toEqual({
+      experimentId: "compare/bub",
+      snapshotStartedAt: "2026-07-01T08:00:00.000Z",
+      evalId: "weather/dover",
+      attempt: 0,
+    });
+    expect(evidence.identity.experimentId).toBe(attempt.experimentId);
+    expect(evidence.identity.evalId).toBe(attempt.evalId);
+    expect(evidence.identity.attempt).toBe(attempt.result.attempt);
+    expect(evidence.identity.snapshotStartedAt).toBe(attempt.snapshot.startedAt);
+    // 真实读取路径:locator 恒有值,且与 attempt.locator 原样一致(不重算)。
+    expect(attempt.locator).toBeDefined();
+    expect(evidence.locator).toBe(attempt.locator);
+  });
+
+  it("artifactPaths.dir 是这个 attempt 落盘目录的绝对路径", async () => {
+    const root = await makeRoot();
+    const attempt = await seedAttempt(root, { id: "weather/salem" }, { events: EVENTS });
+
+    const evidence = await loadAttemptEvidence(attempt);
+
+    expect(evidence.artifactPaths.dir).toBe(join(attempt.snapshot.dir, attempt.ref.attempt));
+  });
+});

package/src/results/attempt-evidence.ts

@@ -0,0 +1,158 @@
+// AttemptEvidence:一次装配一个 Attempt 的全部证据(定稿见 plan/attempt-evidence-feedback-loop.md
+// 「中性数据准备」、docs/concepts.md「Attempt 证据」词条)。
+//
+// 这是 results/evidence 层的装配终点:locator + identity + EvalResult + AnnotatedEvalSource +
+// ExecutionTree + diff + artifact 路径 + capability 位,一次算好。show、view、静态导出、报告列表
+// 共用同一个 loadAttemptEvidence(attempt),不各自重读 artifact 或重算 capability——这正是
+// loadAnnotatedEvalSource(attempt-source.ts)文档里说的"后续阶段的真正 assembler",这里只组合
+// 已有的纯函数/薄壳,不重新实现 Eval 源码标注、ExecutionTree 合并或 diff 读取的任何一条逻辑。
+//
+// capability 位的门槛:每一位只在"数据真的存在且非空"时为 true,不是"artifact 文件存在"——
+// 一份缺内容但存在的 diff.json(两个数组都空)与压根没有 diff.json 都不能点亮 [D]:两者对
+// "有什么可看"这件事产生同一个答案(没有),只是原因不同,原因留给 unavailable 文案区分,
+// 不留给 capability 位区分(capability 位只回答"要不要显示这个证据切面")。
+//
+//   eval      —— evalSource 非空 AND (至少有一行源码 或 至少有一条断言);理论上
+//                buildAnnotatedEvalSource 对空文件也会产出一行 ""(split("\n") 的自然结果),
+//                所以这条件实践中约等于"evalSource 非空",按文档口径原样写出两个子句,
+//                不依赖这个实现细节。
+//   execution —— events() 非空且非空数组;ExecutionTree 的骨架完全来自 events,没有事件
+//                就没有骨架可看,不因为 trace 里有 span 就假装有一棵"只剩 telemetry 节点"的树
+//                (那种树不是骨架,是误导——见下面 execution 字段的 null 边界)。
+//   timing    —— execution 能力已经成立,且 ExecutionTree.timingAvailable 为真(spans.length > 0,
+//                语义见 execution-tree.ts 的同名字段注释:代表"这次运行接入过 OTel",不代表
+//                每个节点都关联上了 span——一个 agent 曾经接入过 OTel,人/agent 看到 [⏱] 才会去
+//                展开找时间,这与该字段"整体是否接入过"的语义正好对应)。
+//   diff      —— diff 非空 AND (generatedFiles 非空 或 deletedFiles 非空);空判定与
+//                show/render.ts::diffText 的既有口径(generated.length===0 && deleted.length===0
+//                时打印"no file changes recorded")完全一致,不另立一套空判定。
+
+import { join } from "node:path";
+import type { AttemptHandle } from "./types.ts";
+import { type AttemptIdentity, type AttemptLocator, encodeAttemptLocator } from "./locator.ts";
+import { loadAnnotatedEvalSource } from "./attempt-source.ts";
+import type { AnnotatedEvalSource } from "./annotated-source.ts";
+import { buildExecutionTree, type ExecutionTree } from "../o11y/execution-tree.ts";
+import type { DiffData, EvalResult } from "../types.ts";
+
+/**
+ * 一个 Attempt 的 artifact 落盘位置。单一目录足够:一个 attempt 的全部 artifact
+ * (events.json / trace.json / o11y.json / diff.json / sources.json,见 ARTIFACT_KINDS)
+ * 要么全在它自己的 attempt 目录,要么(--resume 携带条目——本轮没写任何新数据,
+ * 见 writer.ts 对携带分支的说明)全在 artifactBase 回退到的原快照 attempt 目录,
+ * 从不出现"这个 kind 在自己目录、那个 kind 在 artifactBase"的拆分场景,不需要
+ * 按 kind 逐个给路径。想要某一种 artifact 的具体文件,自己拼
+ * `join(artifactPaths.dir, "<kind>.json")`(kind 见 `ArtifactKind`)。
+ *
+ * 与 show/render.ts 的 `attemptArtifactsPath(attempt, cwd)` 是同一份"这个 attempt 的证据在哪"
+ * 知识的两种呈现:那边为 CLI 文本展示把它转成 cwd 相对的短路径,这里给 evidence 层的调用方
+ * (view / 静态导出 / 报告列表,它们没有统一的"cwd"概念)一份绝对路径,不做相对化。
+ */
+export interface EvidencePaths {
+  /** 绝对路径。 */
+  dir: string;
+}
+
+/**
+ * 每一位只在数据真的存在且非空时为 true——从不因为对应的 artifact 文件"存在"就点亮,
+ * 见本文件头注逐位的门槛说明。show/view/静态导出/报告列表读这四位来决定要不要显示
+ * 对应的证据切面,不自己重新判断"这个 attempt 有没有 X"。
+ */
+export interface AttemptEvidenceCapabilities {
+  /** evalSource 非空且有内容(源码行或断言二选一非空)。 */
+  eval: boolean;
+  /** events 非空数组——有骨架可看。 */
+  execution: boolean;
+  /** execution 成立且这次运行接入过 OTel(ExecutionTree.timingAvailable)。 */
+  timing: boolean;
+  /** diff 非空且至少改动/删除了一个文件。 */
+  diff: boolean;
+}
+
+/**
+ * 一个 Attempt 的完整证据聚合。除 `locator` 外每个字段要么是别的模块已经产出的纯数据类型,
+ * 要么是这四样拼出来的判定——本类型自己不发明新的证据形状。
+ */
+export interface AttemptEvidence {
+  locator: AttemptLocator;
+  /** locator 派生自的不可变身份元组,与 `locator` 编码的是同一份数据,供不想解码 locator
+   * 的调用方直接读结构化字段(如按 experimentId 分组、按时间排序)。 */
+  identity: AttemptIdentity;
+  result: EvalResult;
+  /** 运行时保存的 Eval 源码标注(`--eval` 切面);没有 sources() 就是 null,不伪造空文档。 */
+  evalSource: AnnotatedEvalSource | null;
+  /** 标准事件流 + OTel enrichment(`--execution` 切面);没有非空 events 就是 null,
+   * 不产出一棵只剩 telemetry-only 节点、没有真实骨架的误导性树(见头注 execution 门槛)。 */
+  execution: ExecutionTree | null;
+  /** 被测 Agent 对 Sandbox 工作区的文件变化(`--diff` 切面);原样透传 attempt.diff()。 */
+  diff: DiffData | null;
+  artifactPaths: EvidencePaths;
+  capabilities: AttemptEvidenceCapabilities;
+}
+
+/**
+ * 纯组合:一次性 await 好 attempt 的四类懒加载证据(events / trace / diff / 经
+ * loadAnnotatedEvalSource 解引用的 eval 源码),拼成一份 AttemptEvidence。不重新实现
+ * Eval 源码标注、ExecutionTree 合并或 diff 语义的任何一条规则——那些规则的家分别在
+ * annotated-source.ts / execution-tree.ts / attempt.diff() 自身,这里只调用与判定"够不够
+ * 亮起对应的证据切面"。
+ *
+ * `attempt.locator` 在真实读取路径(openResults() 产出的 handle)恒有值;缺失时按当前身份
+ * 元组按 encodeAttemptLocator 兜底算一份,同 open.ts 给 record.locator 做的兜底同一口径——
+ * 只服务手工构造 AttemptHandle 的测试场景,不改变真实读取路径的行为。
+ */
+export async function loadAttemptEvidence(attempt: AttemptHandle): Promise<AttemptEvidence> {
+  const identity: AttemptIdentity = {
+    experimentId: attempt.experimentId,
+    snapshotStartedAt: attempt.snapshot.startedAt,
+    evalId: attempt.evalId,
+    attempt: attempt.result.attempt,
+  };
+  const locator = attempt.locator ?? encodeAttemptLocator(identity);
+
+  const [evalSource, events, trace, diff] = await Promise.all([
+    loadAnnotatedEvalSource(attempt),
+    attempt.events(),
+    attempt.trace(),
+    attempt.diff(),
+  ]);
+
+  // execution 的 null 边界:没有非空事件骨架就是 null,即使 trace 里有 span(那些 span 在
+  // buildExecutionTree 里只会全部落进 telemetry-only 桶,产出一棵没有骨架、只有遥测的树——
+  // 那不是"这个 attempt 的执行记录",是噪音,不比 null 更有用)。
+  const hasEvents = events !== null && events.length > 0;
+  const execution = hasEvents ? buildExecutionTree(events, trace ?? []) : null;
+
+  const evalCapable = evalSource !== null && (evalSource.lines.length > 0 || evalSource.summary.totalAssertions > 0);
+  // 与 show/render.ts::diffText 的既有空判定同一口径:generatedFiles 与 deletedFiles 都空
+  // 才算"没有变化"。
+  const diffCapable = diff !== null && (Object.keys(diff.generatedFiles).length > 0 || diff.deletedFiles.length > 0);
+
+  return {
+    locator,
+    identity,
+    result: attempt.result,
+    evalSource,
+    execution,
+    diff,
+    artifactPaths: { dir: attemptArtifactDir(attempt) },
+    capabilities: {
+      eval: evalCapable,
+      execution: execution !== null,
+      timing: execution !== null && execution.timingAvailable,
+      diff: diffCapable,
+    },
+  };
+}
+
+/**
+ * attempt artifact 目录的绝对路径:与 show/render.ts::attemptArtifactsPath 同一条判定
+ * (本 attempt 目录,或 --resume 携带条目回退到的 artifactBase 目录),只是不做 cwd 相对化
+ * ——evidence 层没有统一的"当前工作目录"概念,相对化是 CLI 文本渲染层自己的事。
+ */
+function attemptArtifactDir(attempt: AttemptHandle): string {
+  const r = attempt.result;
+  return r.artifactBase
+    ? join(attempt.snapshot.dir, "..", "..", r.artifactBase)
+    : join(attempt.snapshot.dir, attempt.ref.attempt);
+}

package/src/results/attempt-source.ts

@@ -0,0 +1,48 @@
+// 薄组合层:给一个 AttemptHandle,把它的(已去重存储、经 sources() 解引用回来的)eval 源码
+// 与它的 EvalResult.assertions 叠成 AnnotatedEvalSource(annotated-source.ts 的纯函数)。
+//
+// 这不是 AttemptEvidence assembler(那是后续阶段——统一 locator + AnnotatedEvalSource +
+// ExecutionTree + diff 的中性证据聚合)。这里只证明一件事:discovery 时捕获 → 写入面按快照
+// 去重存储 → 读取面经 attempt.sources() 解引用取回 → buildAnnotatedEvalSource 标注,这条链路
+// 端到端能跑通,供后续阶段(text `--eval` renderer、web CodeView、真正的 assembler)直接调用
+// 或参照。
+
+import type { AttemptHandle } from "./types.ts";
+import { buildAnnotatedEvalSource, type AnnotatedEvalSource } from "./annotated-source.ts";
+
+/**
+ * 给定一个 attempt,取回它的 eval 源码(经 sources() 解引用,可能来自本快照或
+ * artifactBase 回退到的原快照)与它的断言,产出标注好的 AnnotatedEvalSource。
+ *
+ * 没有 sources()(eval 从没被 `loc` 引用到、或落盘里压根没有 sources.json/artifactBase
+ * 已失效)时返回 null——没有源码就没有「标注源码」这回事,不伪造一份空文档。
+ *
+ * 一个 attempt 的 sources() 理论上可以有多份文件(主 eval 文件 + 罕见的跨文件 loc);
+ * 这里按「哪份文件被最多断言的 loc 命中」选一份作为主文件喂给 buildAnnotatedEvalSource——
+ * 只服务这条打通链路的验证目的,不是最终选择策略(后续阶段如需要为每份引用到的文件
+ * 都出一份 AnnotatedEvalSource,在这基础上很容易扩成返回数组)。
+ */
+export async function loadAnnotatedEvalSource(attempt: AttemptHandle): Promise<AnnotatedEvalSource | null> {
+  const sources = await attempt.sources();
+  if (!sources || sources.length === 0) return null;
+
+  const assertions = attempt.result.assertions;
+  let primary = sources[0]!;
+  if (sources.length > 1) {
+    const hits = new Map<string, number>();
+    for (const a of assertions) {
+      if (!a.loc) continue;
+      hits.set(a.loc.file, (hits.get(a.loc.file) ?? 0) + 1);
+    }
+    let best = -1;
+    for (const s of sources) {
+      const count = hits.get(s.path) ?? 0;
+      if (count > best) {
+        best = count;
+        primary = s;
+      }
+    }
+  }
+
+  return buildAnnotatedEvalSource(primary, assertions);
+}