niceeval 0.10.3-canary.7 → 0.11.0

package/src/runner/attempt.ts

@@ -30,6 +30,11 @@ import { createChangeLedger, type ChangeLedger } from "./ledger.ts";
 import { deriveDiffData, emptyDiffData } from "../scoring/diff.ts";
 import { createRemoteSandbox, withEvalLocalPaths } from "./remote-sandbox.ts";
 import { recordFact, type FactValue } from "../shared/facts.ts";
+import {
+  attemptFailureInfo,
+  resolveAttemptFailureClass,
+  type FailureClass,
+} from "../shared/failure-class.ts";
 import type { CapturedEvalSource } from "./eval-source.ts";
 import type {
   AgentContext,
@@ -66,6 +71,18 @@ import type {
   RunOptions,
 } from "./types.ts";
 
+/**
+ * 一次终局失败的空间轴回执:止损闸的消费点(见 docs/feature/error-classification/architecture.md
+ * 「止损执行体」)。`class` 是分类链决议出的 `FailureClass`,`text` 与报错文案同源(作者的修复
+ * 提示),`phase` 是失败所在的生命周期阶段。回执是旁路:不改 verdict、不改 `AttemptError` 形状,
+ * 也不走错误通道传播(attempt fiber 的 `E` 仍恒为 `never`)。
+ */
+export interface AttemptFailureDeclaration {
+  readonly class: FailureClass;
+  readonly phase: LifecyclePhase;
+  readonly text: string;
+}
+
 export interface RunAttemptEffectOptions {
   /** 父级调度器的中断信号；测试直调时省略。 */
   parentSignal?: AbortSignal;
@@ -79,13 +96,19 @@ export interface RunAttemptEffectOptions {
    * 省略时(如测试直调)退避不释放槽位。
    */
   concurrencySlot?: ConcurrencySlot;
+  /**
+   * 终局失败的空间轴回执(每条终局失败各回调一次,含 per-attempt teardown 的失败——teardown
+   * 声明照常落闸、不改 verdict,见 architecture.md「生命周期边界」)。省略(如测试直调)时
+   * 分类照常决议、只是无人消费。回调必须不抛错:它跑在 attempt 的失败路径上,不得掩盖原始失败。
+   */
+  onFailureClass?: (declaration: AttemptFailureDeclaration) => void;
 }
 
 export function runAttemptEffect(
   a: Attempt,
   opts: RunOptions,
   sandboxSem: Effect.Semaphore,
-  { parentSignal, onPhase, concurrencySlot }: RunAttemptEffectOptions = {},
+  { parentSignal, onPhase, concurrencySlot, onFailureClass }: RunAttemptEffectOptions = {},
 ): Effect.Effect<EvalResult> {
   const config = opts.config;
   const { evalDef, run, attempt } = a;
@@ -155,6 +178,21 @@ export function runAttemptEffect(
     onPhase?.(phase);
     reportAttemptLifecycle({ type: "attempt:phase", at: Date.now(), identity, phase });
   };
+  /**
+   * 终局失败的分类与回执 —— **必须在把错误折成纯数据 `AttemptError` 之前调**:折完之后原错误
+   * 对象(连同它携带的 `_tag`/`class` 与 cause 链)就不再是任何人的引用,空间轴声明再也读不出来。
+   * 走生命周期链的三道决议(抛出点声明 → 实验分类器 → 缺省不可重试,见
+   * src/shared/failure-class.ts 的 `resolveAttemptFailureClass`);turn 失败的时间轴已由 send 重试
+   * 执行体在 context 层消费完,浮出到这里的一定是终局失败,这里只读空间轴。
+   * 缺省档(`scope` 省略或 `"attempt"`)不回执:那是「死因只属于本次执行」,没有闸可落。
+   */
+  const declareFailure = (phase: LifecyclePhase, e: unknown): void => {
+    if (!onFailureClass) return;
+    const info = attemptFailureInfo(phase, e);
+    const cls = resolveAttemptFailureClass(info, run.classifyFailure);
+    if (cls.scope !== "eval" && cls.scope !== "experiment") return;
+    onFailureClass({ class: cls, phase, text: info.text });
+  };
   // 本 attempt 累计的运行事实(与 verdict/diagnostics 独立):sandbox hook / eval.setup·teardown /
   // agent setup·send·teardown 经 ctx.fact() 上报的都落这里(同一 attempt 内后写覆盖先写),
   // 收尾时并入结果的 facts 字段(见 finally 末尾,与 diagnostics 同一种「累加器 + finally 并入」模式)。
@@ -188,12 +226,21 @@ export function runAttemptEffect(
     if (input.dedupeKey !== undefined) dedupeIndex.set(input.dedupeKey, record);
     diagnostics.push(record);
     // 同时进运行级永久事件流(human 撤下 dashboard 后追加、agent/ci 各追加一条,去重按 key)。
+    // `key` 只管折叠到多细(作者没给 dedupeKey 时折到「这一条 attempt 的这种诊断」,身份因此
+    // 编进去);对外稳定词法始终单独给 `code`,不让消费方从 key 反推(见 sink.ts 的
+    // DiagnosticInput.code、docs/feature/experiments/cli.md 的 WarningEvent)。
     reportDiagnostic({
       key: input.dedupeKey ?? `${input.code}:${encodeAttemptKey(identity)}`,
+      code: input.code,
       severity: input.level,
       message: input.message,
       identity,
-      data: input.data,
+      // phase 由运行器给,且**压过**作者 `data` 里的同名字段:`WarningEvent.phase` 是
+      // `LifecyclePhase` 闭集,取值只能由「这条诊断报上来时运行器正处在哪一步」决定。
+      // 作者的 `data` 是开放词表,让它盖住这个字段等于允许从 eval / adapter 代码里冒充一个
+      // 别的(甚至不在闭集里的)阶段,消费方按 phase 分支就此失效——与 ScopedFeedback
+      // 「两个方法都不接受 phase」是同一条纪律(见 src/shared/types.ts 的接口注释)。
+      data: { ...input.data, phase },
     });
   };
   // 作用域反馈:progress 走 attempt:progress(短命状态,归因由 runner 的当前阶段决定),
@@ -355,7 +402,12 @@ export function runAttemptEffect(
           const proto = run.agent.tracing?.protocol;
           log(t("runner.otlpReceiver", { endpoint, proto: proto ? ` (${proto})` : "" }));
         } else {
-          // 远程沙箱(e2b / vercel):在沙箱内起 collector,agent 往 localhost 端口发
+          // 远程沙箱(e2b / vercel):在沙箱内起 collector,agent 往 localhost 端口发。
+          // 这一步是「创建本次 tracing 出口」(telemetry.configure 的定义,见
+          // docs/feature/experiments/cli.md 的阶段表),而且是唯一一条要往沙箱里传脚本、
+          // 起进程、等端口的 tracing 出口——秒级耗时与失败都归它自己,不能顺着上一个还开着的
+          // 阶段记成 sandbox.create(那样 collector 起不来会伪装成「起沙箱失败」)。
+          enterPhase("telemetry.configure");
           receiver = yield* createInSandboxTraceReceiver(sandbox);
           const endpoint = receiver.endpoint("");
           const env = run.agent.tracing?.env?.(endpoint);
@@ -429,6 +481,7 @@ export function runAttemptEffect(
           facts,
           commands,
           concurrencySlot,
+          declareFailure,
           registerEvidence: (getEvents, getUsage) => {
             liveEvents = getEvents;
             liveUsage = getUsage;
@@ -543,11 +596,18 @@ export function runAttemptEffect(
     Effect.catchAllCause((cause) =>
       Cause.isInterrupted(cause)
         ? Effect.failCause(cause)
-        : Effect.succeed({
-            ...base,
-            durationMs: Date.now() - t0,
-            error: errorFromThrown(Cause.squash(cause), sendActive ? "agent.run" : lastPhase),
-            ...(commands.length > 0 ? { commands: [...commands] } : {}),
+        : Effect.suspend(() => {
+            // 资源获取 / Scope 层的意外(起沙箱失败、provisioning 的确定性配置死因)同样是终局
+            // 失败:先读空间轴回执,再折成纯数据 AttemptError(顺序不可换,见 declareFailure)。
+            const raw = Cause.squash(cause);
+            const phase = (sendActive ? "agent.run" : lastPhase) ?? "eval.run";
+            declareFailure(phase, raw);
+            return Effect.succeed({
+              ...base,
+              durationMs: Date.now() - t0,
+              error: errorFromThrown(raw, sendActive ? "agent.run" : lastPhase),
+              ...(commands.length > 0 ? { commands: [...commands] } : {}),
+            });
           }),
     ),
     // 结果封口在 Scope release 完成之后:sandbox.stop 已由 finalizer 写进 recorder,
@@ -624,6 +684,9 @@ interface AttemptResources {
   commands: FailedCommandEvidence[];
   /** turn 级重试退避期间释放/收回的全局并发槽位;透传给 createEvalContext。 */
   concurrencySlot?: ConcurrencySlot;
+  /** 终局失败的空间轴回执(runAttemptEffect 持有的同一个闭包):body 的失败路径与 finally 里的
+   *  per-attempt teardown 失败都经它上报,止损闸据此落闸(见 runAttemptEffect 的 declareFailure)。 */
+  declareFailure: (phase: LifecyclePhase, e: unknown) => void;
   /** SessionManager 一建好就登记事件/用量的读取句柄回外层(超时证据保全用,见
    *  runAttemptEffect 顶部 liveEvents/liveUsage 的注释与 docs/runner.md「超时:双层保护」)。 */
   registerEvidence: (getEvents: () => readonly StreamEvent[], getUsage: () => Usage) => void;
@@ -660,6 +723,7 @@ async function runAttemptBody(
     facts,
     commands,
     concurrencySlot,
+    declareFailure,
     registerEvidence,
     registerLedger,
   } = res;
@@ -826,6 +890,10 @@ async function runAttemptBody(
       feedback,
       fact,
       concurrencySlot,
+      // 实验分类器:turn 链上排在 adapter 的 classifyTurnError 之前(见
+      // docs/feature/error-classification/architecture.md「分类链」)。与本文件 declareFailure
+      // 走的生命周期链是同一个函数,两条链的决议序各自单源在 turn-errors.ts / failure-class.ts。
+      experimentClassifier: run.classifyFailure,
       // 题型:计分制下句柄上的 .gate() 是前置(就地求值 + 挂了中止 test()),见 collector。
       scoring: evalDef.scoring ?? "pass",
       // 前置断言就地求值要看当前已提交窗口的 agent diff(非沙箱型没有分类账,省略)。
@@ -876,10 +944,15 @@ async function runAttemptBody(
         // TurnFailed 是 eval 驱动 agent 时的可读失败(message 首行是一层摘要,后续行是
         // diagnose 的 output tail 等下钻证据,单行面各自取首行);稳定 code
         // `turn-failed`,不带控制流 stack(那指向 control-flow.ts,对定位无益)。
+        // 分类先读:turn 链决议出的 FailureClass 由 expectOk() 铸造 TurnFailed 时挂在错误上,
+        // 下一行折成纯数据后就读不到了(见 declareFailure)。
+        declareFailure(getPhase() ?? "eval.run", e);
         error = { code: "turn-failed", message: e.message, phase: getPhase() ?? "eval.run" };
       } else {
         // eval 脚本(比如引用了已改名/删掉的 API)抛出的 TypeError:message 是一层原因,完整 stack
         // 单独进 `error.stack`,niceeval show 展开时才看得到 eval 文件的 file:line。
+        // 作者从 test(t) 体内抛的 ExperimentFatalError / EvalFatalError 也走这条分支。
+        declareFailure(getPhase() ?? "eval.run", e);
         error = errorFromThrown(e, getPhase());
       }
     }
@@ -1043,6 +1116,9 @@ async function runAttemptBody(
     return value;
   } catch (e) {
     recorder.failCurrent();
+    // sandbox 钩子 / agent.setup / eval.setup / 评分链路抛出的终局失败:同样先读空间轴回执,
+    // 再折成纯数据 AttemptError(顺序不可换,见 declareFailure)。
+    declareFailure(getPhase() ?? "eval.run", e);
     const value: EvalResult = {
       ...base,
       durationMs: Date.now() - t0,
@@ -1069,6 +1145,9 @@ async function runAttemptBody(
             await withCleanupTimeout(() => evalDef.teardown!(withEvalLocalPaths(sandbox, evalDef.baseDir), hookCtx));
           } catch (e) {
             // 收尾失败只是 diagnostic,不改判定 —— 挂到 attempt.diagnostics(见 finally 末尾并入)。
+            // 但空间轴声明照常落闸:知识就是知识,兄弟 attempt 还在派发中(architecture.md
+            // 「生命周期边界」)。落闸不改 verdict,两件事互不影响。
+            declareFailure("eval.teardown", e);
             diagnostics.push(teardownDiagnostic("eval.teardown", e));
             throw e; // 让 measureClosing 把这段标 failed
           }
@@ -1082,6 +1161,7 @@ async function runAttemptBody(
           try {
             await withCleanupTimeout(() => run.agent.teardown!(sandbox, attemptCtx));
           } catch (e) {
+            declareFailure("agent.teardown", e);
             diagnostics.push(teardownDiagnostic("agent.teardown", e));
             throw e;
           }
@@ -1101,6 +1181,7 @@ async function runAttemptBody(
             try {
               await withCleanupTimeout(() => sandboxTeardownHooks[i](sandbox, hookCtx));
             } catch (e) {
+              declareFailure("sandbox.teardown", e);
               diagnostics.push(teardownDiagnostic("sandbox.teardown", e));
             }
           }

package/src/runner/discover.ts

@@ -12,6 +12,24 @@ import type { DiscoveredEval, DiscoveredExperiment, EvalDef, ExperimentDef } fro
 
 const SKIP_DIRS = new Set(["node_modules", ".git", ".niceeval", "dist", ".next"]);
 
+/**
+ * 发现阶段的动态 import 会执行被加载文件的**顶层代码**(配置文件里现拉 registry、读 .env、
+ * 连服务都很常见)。裸抛出去的话用户只看到一个不知从何而来的 `TypeError: fetch failed`——
+ * 发现要遍历整棵 `evals/` / `experiments/` 树,一个文件炸了并不会告诉你是哪一个。
+ * 这里把文件路径钉进 message,原错误挂 `cause`(`formatThrown` 会展开成 `caused by:` 链)。
+ */
+async function importDiscovered<T>(file: string, root: string, kind: "eval" | "experiment"): Promise<T> {
+  try {
+    return (await import(pathToFileURL(file).href)) as T;
+  } catch (e) {
+    throw new Error(
+      `Failed to load ${kind} file ${relative(root, file)}: its top-level code threw while being imported. ` +
+        `Fix the error below, or move the work into the ${kind} body so it only runs when this ${kind} is selected.`,
+      { cause: e },
+    );
+  }
+}
+
 async function walkFiles(dir: string, match: (name: string) => boolean): Promise<string[]> {
   const out: string[] = [];
   async function walk(current: string): Promise<void> {
@@ -40,9 +58,9 @@ export async function discoverEvals(root: string): Promise<DiscoveredEval[]> {
   const files = (await walkFiles(dir, (n) => n.endsWith(".eval.ts") || n.endsWith(".eval.tsx"))).sort();
   const out: DiscoveredEval[] = [];
   for (const file of files) {
-    const mod = (await import(pathToFileURL(file).href)) as {
+    const mod = await importDiscovered<{
       default?: EvalDef | EvalDef[] | Record<string, EvalDef>;
-    };
+    }>(file, root, "eval");
     const def = mod.default;
     if (!def) continue;
     const baseId = relative(dir, file).replace(/\.eval\.tsx?$/, "").split(sep).join("/");
@@ -107,7 +125,7 @@ export async function discoverExperiments(root: string): Promise<DiscoveredExper
   const files = (await walkFiles(dir, (n) => n.endsWith(".ts") && !n.endsWith(".d.ts"))).sort();
   const out: DiscoveredExperiment[] = [];
   for (const file of files) {
-    const mod = (await import(pathToFileURL(file).href)) as { default?: ExperimentDef };
+    const mod = await importDiscovered<{ default?: ExperimentDef }>(file, root, "experiment");
     const def = mod.default;
     if (!def || !def.agent) continue;
     const id = relative(dir, file)

package/src/runner/feedback/coordinator.ts

@@ -28,6 +28,7 @@ import {
   type ExperimentHookInput,
   type ExperimentProgressInput,
   type PrecheckInput,
+  type LockWaitInput,
   type FailureInput,
   type FeedbackSink,
   type KeptInput,
@@ -198,6 +199,7 @@ export function createFeedbackCoordinator(options: FeedbackCoordinatorOptions):
       type: "diagnostic",
       at: io.clock.now(),
       key: input.key,
+      ...(input.code !== undefined ? { code: input.code } : {}),
       severity: input.severity,
       message: input.message,
       identity: input.identity,
@@ -272,6 +274,22 @@ export function createFeedbackCoordinator(options: FeedbackCoordinatorOptions):
     });
   }
 
+  function lockWait(input: LockWaitInput): void {
+    emit({
+      type: "lock-wait",
+      at: io.clock.now(),
+      experimentId: input.experimentId,
+      evalId: input.evalId,
+      status: input.status,
+      ...(input.holderPid !== undefined ? { holderPid: input.holderPid } : {}),
+      ...(input.holderHost !== undefined ? { holderHost: input.holderHost } : {}),
+      ...(input.attempts !== undefined ? { attempts: input.attempts } : {}),
+      ...(input.carried !== undefined ? { carried: input.carried } : {}),
+      ...(input.dispatched !== undefined ? { dispatched: input.dispatched } : {}),
+      ...(input.waitedMs !== undefined ? { waitedMs: input.waitedMs } : {}),
+    });
+  }
+
   function reporterError(input: { reporter: string; required: boolean; message: string }): void {
     emit({
       type: "reporter-error",
@@ -313,6 +331,7 @@ export function createFeedbackCoordinator(options: FeedbackCoordinatorOptions):
       experimentHook,
       experimentProgress,
       precheck,
+      lockWait,
       lifecycle,
     });
     emit({ type: "plan", at: startedAtMs, plan });
@@ -378,6 +397,7 @@ export function createFeedbackCoordinator(options: FeedbackCoordinatorOptions):
     experimentHook,
     experimentProgress,
     precheck,
+    lockWait,
     lifecycle,
     stopDynamic,
     finish,
@@ -404,8 +424,10 @@ function fallbackTextFor(event: DurableFeedbackEvent): string | undefined {
     case "saved":
     case "experiment-hook":
     case "precheck":
-      // 钩子/预检起止不是失败证据:setup 失败的每条 attempt 另有 "failure" 事件兜底,
-      // 预检失败以既有错误路径中止运行,renderer 崩溃丢一行起止不丢数据。
+    case "lock-wait":
+      // 钩子/预检/锁等待起止都不是失败证据:setup 失败的每条 attempt 另有 "failure" 事件
+      // 兜底,预检失败以既有错误路径中止运行,锁等待解决与否不改变任何 verdict——
+      // renderer 崩溃丢一行起止不丢数据。
       return undefined;
   }
 }

package/src/runner/feedback/eval-conclusions.ts

@@ -40,12 +40,15 @@ export interface EvalConclusionRow {
   rate?: number;
 }
 
-/** `diagnostics` 里 `fail-fast:` 记录按 (experiment, eval) 求和的未派发次数;没有 `identity`
- *  的记录(理论上不应出现——fail-fast 诊断恒带 identity,见 run.ts)不计入任何 key。 */
+/** `diagnostics` 里 fail-fast 记录按 (experiment, eval) 求和的未派发次数;没有 `identity`
+ *  的记录(理论上不应出现——fail-fast 诊断恒带 identity,见 run.ts)不计入任何 key。
+ *  归类按稳定词法 `code`(省略时回落到 key 首段,缺省 key 恒是 `${code}:${identity}`),不按
+ *  `key` 前缀:key 里编着折叠身份,拿它匹配会在身份编码变动时静默失配,减不掉的那部分
+ *  fail-fast 份额会当成真实的 earlyExit 省略数报出去。 */
 function failFastByEval(diagnostics: readonly DiagnosticNotice[]): ReadonlyMap<string, number> {
   const result = new Map<string, number>();
   for (const d of diagnostics) {
-    if (!d.key.startsWith("fail-fast:") || !d.identity) continue;
+    if ((d.code ?? d.key.split(":", 1)[0]) !== "fail-fast" || !d.identity) continue;
     const key = evalConclusionKey(d.identity);
     result.set(key, (result.get(key) ?? 0) + d.count);
   }