niceeval 0.17.1 → 0.19.0

package/dist/runner/attempt.cjs

@@ -6,6 +6,7 @@ exports.scoreFactOutcomeForAttemptError = scoreFactOutcomeForAttemptError;
 exports.errorFromThrown = errorFromThrown;
 exports.executeSandboxAction = executeSandboxAction;
 exports.experimentRunInfo = experimentRunInfo;
+const attempt_notes_ts_1 = require("./attempt-notes.cjs");
 const attempt_cancellation_ts_1 = require("./attempt-cancellation.cjs");
 // 单个 attempt 的完整生命周期:资源(沙箱 / OTLP 接收器)经 Effect.Sample 的
 // acquireRelease 接管,无论 body 成功 / 抛错 / 被中断,stop() / close() 都保证执行。
@@ -36,7 +37,7 @@ const keep_registry_ts_1 = require("../sandbox/keep-registry.cjs");
 const project_state_database_ts_1 = require("../record/sqlite/project-state-database.cjs");
 const provisioner_ts_1 = require("../agents/provisioner.cjs");
 const callback_context_ts_1 = require("../agents/callback-context.cjs");
-const effect_runtime_ts_1 = require("../agents/effect-runtime.cjs");
+const agent_effects_ts_1 = require("../agents/agent-effects.cjs");
 const receiver_ts_1 = require("../o11y/otlp/receiver.cjs");
 const sandbox_receiver_ts_1 = require("../o11y/otlp/sandbox-receiver.cjs");
 const turn_otel_ts_1 = require("../o11y/otlp/turn-otel.cjs");
@@ -68,7 +69,7 @@ const redaction_ts_1 = require("../sandbox/redaction.cjs");
 const link_ts_1 = require("../sandbox/link.cjs");
 const config_identity_ts_1 = require("./config-identity.cjs");
 const setup_prefix_plan_ts_1 = require("./setup-prefix-plan.cjs");
-const contracts_ts_1 = require("../plugin/contracts.cjs");
+const link_ts_2 = require("../plugin/link.cjs");
 const source_loc_ts_1 = require("../source-loc.cjs");
 const source_producer_ts_1 = require("./source-producer.cjs");
 const sandbox_record_producer_ts_1 = require("./sandbox-record-producer.cjs");
@@ -126,7 +127,7 @@ function reportJudgeErrorAssistance(a, sealed) {
         }), identity: { experimentId: a.run.experimentId, evalId: a.evalDef.id, attempt: a.attempt } });
 }
 /**
- * 一次终局失败的空间轴决议:走三道链(抛出点声明 → 实验分类器 → 缺省不可重试,见
+ * 一次终局失败的空间轴决议:读取抛出点声明，否则缺省不可重试(见
  * src/shared/failure-class.ts 的 `resolveAttemptFailureClass`),缺省档(`scope` 省略或
  * `"attempt"`)返回 undefined —— 死因只属于本次执行,没有闸可落。
  *
@@ -135,9 +136,9 @@ function reportJudgeErrorAssistance(a, sealed) {
  * attempt 内的失败(本文件的 `declareFailure`)与 attempt 开跑前的资源获取失败
  * (run.ts 的复用池租借)共用这一份决议,两处不各写一遍链。
  */
-function attemptFailureDeclaration(classify, phase, e) {
+function attemptFailureDeclaration(phase, e) {
     const info = (0, failure_class_ts_2.attemptFailureInfo)(phase, e);
-    const cls = (0, failure_class_ts_2.resolveAttemptFailureClass)(info, classify);
+    const cls = (0, failure_class_ts_2.resolveAttemptFailureClass)(info);
     if (cls.scope !== "eval" && cls.scope !== "experiment")
         return undefined;
     return { class: cls, phase, text: info.text };
@@ -253,6 +254,7 @@ function runAttemptEffect(a, opts, sandboxSem, { attachmentScope, buildLocators,
         // Assert-first state becomes visible only after the real TestContext has
         // been constructed. Scope finalization owns the interruption seal so an
         // aborted Attempt cannot leave declared entries unsealed.
+        let liveNotes;
         let liveAssertions;
         let liveAssertionState;
         let adapterResources;
@@ -306,7 +308,7 @@ function runAttemptEffect(a, opts, sandboxSem, { attachmentScope, buildLocators,
         /**
          * 终局失败的分类与回执 —— **必须在把错误折成纯数据 `AttemptError` 之前调**:折完之后原错误
          * 对象(连同它携带的 `_tag`/`class` 与 cause 链)就不再是任何人的引用,空间轴声明再也读不出来。
-         * 走生命周期链的三道决议(抛出点声明 → 实验分类器 → 缺省不可重试,见
+         * 读取抛出点声明，否则缺省不可重试(见
          * src/shared/failure-class.ts 的 `resolveAttemptFailureClass`);send 失败的时间轴已由 send 重试
          * 执行体在 context 层消费完,浮出到这里的一定是终局失败,这里只读空间轴。
          * 缺省档(`scope` 省略或 `"attempt"`)不回执:那是「死因只属于本次执行」,没有闸可落。
@@ -314,7 +316,7 @@ function runAttemptEffect(a, opts, sandboxSem, { attachmentScope, buildLocators,
         const declareFailure = (phase, e) => {
             if (!onFailureClass)
                 return;
-            const declaration = attemptFailureDeclaration(run.classifyFailure, phase, e);
+            const declaration = attemptFailureDeclaration(phase, e);
             if (declaration) {
                 // 空间轴回执会先于 Attempt 结果封口进入 run.ts，并立即生成运行期反馈与 run.json 的
                 // dispatch-halted 诊断。它不能依赖下面的 EvalResult 最终扫描补救，否则同一条错误在
@@ -570,6 +572,21 @@ ${recentLogs.map((l) => `  · ${l}`).join("\n")}`;
             yield* effect_1.Effect.addFinalizer(() => effect_1.Effect.promise(() => judgeUsage.closeAndDrain()));
             const assertFirst = yield* (0, assert_first_bridge_ts_1.makeAssertFirstAttemptBridge)();
             if (adapter.kind === "custom") {
+                const pluginHooks = (0, link_ts_2.attemptPluginLifecycles)(evalDef, run);
+                let activated = 0;
+                const pluginContext = {
+                    experimentId: run.experimentId, evalId: evalDef.id, attempt: a.attempt,
+                    ...(evalDef.evalGroup === undefined ? {} : { evalGroup: { id: evalDef.evalGroup.id, definitionHash: evalDef.evalGroup.definitionHash } }),
+                    signal, progress: scopedFeedback.progress, diagnostic: scopedFeedback.diagnostic,
+                };
+                yield* effect_1.Effect.addFinalizer(() => effect_1.Effect.forEach(pluginHooks.slice(0, activated).reverse(), (entry) => entry.teardown === undefined ? effect_1.Effect.void : (0, cleanup_timeout_ts_1.cleanupCallback)((cleanupSignal) => entry.teardown({ ...pluginContext, signal: cleanupSignal })).pipe(effect_1.Effect.catchCause((cause) => effect_1.Effect.sync(() => scopedFeedback.diagnostic({
+                    code: "plugin-after-failed", level: "warning", message: String(effect_1.Cause.squash(cause)),
+                })))), { concurrency: 1, discard: true }));
+                for (const entry of pluginHooks) {
+                    activated += 1;
+                    if (entry.setup !== undefined)
+                        yield* effect_1.Effect.tryPromise({ try: () => Promise.resolve(entry.setup(pluginContext)), catch: (cause) => cause });
+                }
                 const resources = new attempt_resources_ts_1.AdapterAttemptResources(() => {
                     adapterUsageSnapshot = adapterUsage.close();
                     adapterExecutionTraces.close();
@@ -624,7 +641,8 @@ ${recentLogs.map((l) => `  · ${l}`).join("\n")}`;
                     feedback: scopedFeedback,
                     log,
                     declareFailure,
-                    registerAssertions: (runtime) => {
+                    registerAssertions: (runtime, state) => {
+                        liveNotes = state.notes;
                         if (liveAssertions !== undefined && liveAssertions !== runtime) {
                             throw new Error("Attempt tried to register a second Assertions runtime");
                         }
@@ -788,7 +806,8 @@ ${recentLogs.map((l) => `  · ${l}`).join("\n")}`;
             // forkIn() interrupts new children immediately even before the executor's own finalizer flips
             // its closed flag. Cleanup therefore uses the provider-owned resource facade; before/Agent/test
             // continue to use the interruptible author facade above.
-            const cleanupCommandTarget = (0, operations_ts_1.createSandboxCommandTarget)(reusedSandbox?.resourceSandbox ?? materializedCase?.sandbox ?? sandbox);
+            const cleanupSandbox = reusedSandbox?.resourceSandbox ?? materializedCase?.sandbox ?? sandbox;
+            const cleanupCommandTarget = (0, operations_ts_1.createSandboxCommandTarget)(cleanupSandbox);
             // ── tracing ──────────────────────────────────────────────────────────────────
             // sandbox.otlpHost:
             //   string → provider 承诺该 hostname 可访问宿主 receiver
@@ -996,10 +1015,15 @@ ${recentLogs.map((l) => `  · ${l}`).join("\n")}`;
                         }));
                         if (node)
                             recorder.pushParent(node);
-                        return (0, cleanup_timeout_ts_1.cleanupCallback)((cleanupSignal) => cleanup.command(cleanupCommandTarget, {
-                            ...cleanup.context,
-                            signal: cleanupSignal,
-                        })).pipe(effect_1.Effect.catch((error) => effect_1.Effect.sync(() => {
+                        return (0, cleanup_timeout_ts_1.cleanupCallback)((cleanupSignal) => {
+                            // The forward Attempt may have exhausted its deadline. Each cleanup
+                            // owns a fresh bounded command window, including on reused instances.
+                            (0, deadline_ts_1.applyCommandDeadline)(cleanupSandbox, clock.currentTimeMillisUnsafe() + cleanup_timeout_ts_1.CLEANUP_TIMEOUT_MS);
+                            return cleanup.command(cleanupCommandTarget, {
+                                ...cleanup.context,
+                                signal: cleanupSignal,
+                            });
+                        }).pipe(effect_1.Effect.catch((error) => effect_1.Effect.sync(() => {
                             if (node)
                                 node.failed = true;
                             declareFailure("sandbox.cleanup", error);
@@ -1084,6 +1108,7 @@ ${recentLogs.map((l) => `  · ${l}`).join("\n")}`;
                 },
                 assertFirst,
                 registerAssertions: (runtime, state) => {
+                    liveNotes = state.notes;
                     if (liveAssertions !== undefined && liveAssertions !== runtime) {
                         throw new Error("Attempt tried to register a second Assertions runtime");
                     }
@@ -1340,6 +1365,7 @@ ${recentLogs.map((l) => `  · ${l}`).join("\n")}`;
             if (capture !== undefined) {
                 (0, sandbox_record_producer_ts_1.retainRunnerAttemptFileChangesCapture)(finalResult, capture);
             }
+            (0, attempt_notes_ts_1.retainAttemptNotes)(finalResult, liveNotes?.snapshot() ?? []);
             (0, judge_usage_ts_2.retainJudgeUsage)(finalResult, judgeUsage.snapshot());
             if (adapterUsageSnapshot !== undefined)
                 (0, adapter_usage_ts_1.retainAdapterUsage)(finalResult, adapterUsageSnapshot);
@@ -1389,7 +1415,7 @@ function runAdapterAttemptBody(input) {
             judge: a.judge,
             executeStop: assertFirst.requestAssertion,
         });
-        registerAssertions(state.assertions);
+        registerAssertions(state.assertions, state);
         sourceCapture.attachAssertions(state.assertions);
         let error;
         let skipReason;
@@ -1921,15 +1947,15 @@ async function runAttemptBody(a, config, t0, base, res, parentSignal) {
     const managedProcess = managedProcessSupport?._tag === "Supported"
         ? managedProcessSupport.value
         : undefined;
-    /** agent.setup 时点已走到(未声明 setup 也置位)——agent.teardown 的触发条件(成对触发规则)。 */
+    /** agent.before 时点已走到(未声明 setup 也置位)——agent.after 的触发条件(成对触发规则)。 */
     let agentSetupReached = false;
-    const evalPluginLifecycles = (0, contracts_ts_1.linkPluginLifecycles)(evalDef.plugins, "eval");
+    const evalPluginLifecycles = (0, link_ts_2.attemptPluginLifecycles)(evalDef, run);
     let activatedEvalPlugins = 0;
     const evalPluginContext = {
         experimentId: run.experimentId,
         evalId: evalDef.id,
         attempt,
-        ...(evalDef.evalGroup === undefined ? {} : { evalGroupId: evalDef.evalGroup.id }),
+        ...(evalDef.evalGroup === undefined ? {} : { evalGroup: { id: evalDef.evalGroup.id, definitionHash: evalDef.evalGroup.definitionHash } }),
         signal,
         progress: feedback.progress,
         diagnostic: feedback.diagnostic,
@@ -1937,10 +1963,10 @@ async function runAttemptBody(a, config, t0, base, res, parentSignal) {
     // Only the Assert-first runtime is registered with the outer Scope. The
     // body requests its single seal after gathering non-authoring evidence.
     let assertionState;
-    /** adapter 在 agent.setup 里经 `ctx.reportSetup()` 交回的安装清单(宿主侧内存对象,不经沙箱磁盘;
+    /** adapter 在 agent.before 里经 `ctx.reportInstallation()` 交回的安装清单(宿主侧内存对象,不经沙箱磁盘;
      *  装了 Skill / plugin / MCP 的沙箱型 adapter 才有)。运行器只把它抬成 attempt artifact,
      *  不解释内容、不按 agent 名字分支;什么都没装的 adapter 不调用 → undefined,不生成空 artifact。 */
-    let agentSetup;
+    let agentInstallation;
     // 变更分类账(仅沙箱型;workspace.baseline 阶段建立)。
     let ledger;
     const linked = usesSandbox && a.plan._tag === "Sandbox" ? a.plan.pair : undefined;
@@ -2071,7 +2097,7 @@ async function runAttemptBody(a, config, t0, base, res, parentSignal) {
                 structurallyEligibleCount += 1;
             }
         }
-        if (run.sandboxSetupCache === "bypass") {
+        if (run.sandboxBeforeCache === "bypass") {
             await replayAllBefore(linked.before, "replay(bypass)");
             return;
         }
@@ -2443,30 +2469,16 @@ async function runAttemptBody(a, config, t0, base, res, parentSignal) {
             }
         }
         // agent 自己的 lifecycle:装 CLI、写 config(每个沙箱一次,不在每轮 send 里)。
-        // 时点在此走到(未声明 setup 也算)——agent.teardown 据此触发。
+        // 时点在此走到(未声明 setup 也算)——agent.after 据此触发。
         agentSetupReached = true;
-        if (agent.setup) {
+        if (agent.before) {
             enterPhase("agent.setup");
             log(`agent setup (install CLI / write config)...`);
-            const setupContext = agent.kind === "sandbox"
-                ? { ...sandboxAttemptCtx, reportSetup: (manifest) => (agentSetup = manifest) }
-                : attemptCtx;
-            let returned;
-            if (agent.kind === "sandbox") {
-                const setup = agent.setup;
-                const context = setupContext;
-                const native = (0, effect_runtime_ts_1.agentSetupEffect)(agent, sandbox, context);
-                returned = await assertFirst.requestEffect(native === undefined
-                    ? (0, callback_context_ts_1.withAgentCallbackContext)(context, (callbackContext) => (0, effect_runtime_ts_1.authorCallbackEffect)(() => setup(sandbox, callbackContext)))
-                    : (0, callback_context_ts_1.withAgentCallbackContext)(context, (callbackContext) => (0, effect_runtime_ts_1.agentSetupEffect)(agent, sandbox, callbackContext)));
-            }
-            else {
-                const setup = agent.setup;
-                const native = (0, effect_runtime_ts_1.agentSetupEffect)(agent, sandbox, attemptCtx);
-                returned = await assertFirst.requestEffect(native === undefined
-                    ? (0, callback_context_ts_1.withAgentCallbackContext)(attemptCtx, (callbackContext) => (0, effect_runtime_ts_1.authorCallbackEffect)(() => setup(callbackContext)))
-                    : (0, callback_context_ts_1.withAgentCallbackContext)(attemptCtx, (callbackContext) => (0, effect_runtime_ts_1.agentSetupEffect)(agent, sandbox, callbackContext)));
-            }
+            const context = {
+                ...sandboxAttemptCtx,
+                reportInstallation: (manifest) => { agentInstallation = manifest; },
+            };
+            const returned = await assertFirst.requestEffect((0, callback_context_ts_1.withAgentCallbackContext)(context, (callbackContext) => (0, agent_effects_ts_1.agentBeforeEffect)(agent, sandbox, callbackContext) ?? effect_1.Effect.void));
             if (typeof returned === "function") {
                 throw new Error(`${`Agent.setup (${agent.name})`} returned a function. setup does not carry cleanup and the returned value will not be executed — put the cleanup in the paired teardown of the same layer (${"Agent.teardown"}); see the experiments tutorial on docs-site or docs/runner.md.
 `.trimEnd());
@@ -2521,10 +2533,6 @@ async function runAttemptBody(a, config, t0, base, res, parentSignal) {
             otel,
             feedback,
             concurrencySlot,
-            // 实验分类器:send 链上排在 adapter 的 classifySendFailure 之前(见
-            // docs/feature/error-classification/architecture.md「分类链」)。与本文件 declareFailure
-            // 走的生命周期链是同一个函数,两条链的决议序各自单源在 send-failures.ts / failure-class.ts。
-            experimentClassifier: run.classifyFailure,
             sourceRegistry,
             resources: attemptResources,
             nextSourceOrder: sourceCapture.nextSourceOrder,
@@ -2867,7 +2875,7 @@ async function runAttemptBody(a, config, t0, base, res, parentSignal) {
             sources,
             o11y,
             trace,
-            agentSetup,
+            agentInstallation,
             ...(diffArtifactAvailable ? { diff: diffWindows } : {}),
             evidenceCoverage: state.manager.evidenceCoverage,
             // sandbox 归属不在这里拼:它是租借时刻就定死的调度事实,由 runAttemptEffect 统一挂到
@@ -2881,7 +2889,7 @@ async function runAttemptBody(a, config, t0, base, res, parentSignal) {
         if (isBodyInterrupted(signal, parentSignal, isDeadlineTimedOut()))
             throw e;
         recorder.failCurrent();
-        // SandboxLayer command / agent.setup / 评分链路抛出的终局失败:同样先读空间轴回执,
+        // SandboxLayer command / agent.before / 评分链路抛出的终局失败:同样先读空间轴回执,
         // 再折成纯数据 AttemptError(顺序不可换,见 declareFailure)。
         declareFailure(getPhase() ?? "eval.run", e);
         const error = errorFromThrown(e, getPhase(), res.attemptTimeout);
@@ -2914,45 +2922,25 @@ async function runAttemptBody(a, config, t0, base, res, parentSignal) {
             ...(sealed === undefined
                 ? (evalDef.evaluationKind === "score" ? { scoreResult: scoreFactOutcomeForAttemptError(error) } : {})
                 : legacyResultProjectionFromSealedAssertions(sealed, error)),
-            ...(agentSetup !== undefined ? { agentSetup } : {}),
+            ...(agentInstallation !== undefined ? { agentInstallation } : {}),
         };
         return value;
     }
     finally {
         // 收尾段一律在 finally 跑(主链成败都执行),不改判定,各自兜错(diagnostic)、各自计时
         // (不计入 durationMs 口径,见 docs/feature/record/architecture.md)。执行序与 LifecyclePhase
-        // 闭集声明一致:agent.teardown → sandbox.cleanup;各段可独立标 failed。
+        // 闭集声明一致:agent.after → sandbox.cleanup;各段可独立标 failed。
         // 沙箱 stop / 接收器 close 不在这里 —— 由 runAttemptEffect 的 Sample 在本函数返回后回收,
         // 并经 finalizer 计成 sandbox.stop。没有对应 teardown/cleanup 的段直接跳过，不产生空阶段。
         if (assertFirst.beginCleanup()) {
             try {
-                if (agentSetupReached && agent.teardown) {
+                if (agentSetupReached && agent.after) {
                     enterPhase("agent.teardown");
                     await recorder
                         .measureClosing("agent.teardown", async () => {
                         try {
-                            // 先按 kind 收窄 Agent 联合,再取 teardown —— 否则可选属性访问会把
-                            // AgentTeardown | DirectAgentTeardown 混成无法调用的签名。
-                            if (agent.kind === "sandbox") {
-                                const teardown = agent.teardown;
-                                if (teardown) {
-                                    const context = { ...sandboxAttemptCtx, signal };
-                                    const native = (0, effect_runtime_ts_1.agentTeardownEffect)(agent, sandbox, context);
-                                    await assertFirst.requestCleanup((0, cleanup_timeout_ts_1.withCleanupTimeout)(native === undefined
-                                        ? (0, callback_context_ts_1.withAgentCallbackContext)(context, (callbackContext) => (0, effect_runtime_ts_1.authorCallbackEffect)(() => teardown(sandbox, callbackContext)), { inheritAttemptSignal: false })
-                                        : (0, callback_context_ts_1.withAgentCallbackContext)(context, (callbackContext) => (0, effect_runtime_ts_1.agentTeardownEffect)(agent, sandbox, callbackContext), { inheritAttemptSignal: false })));
-                                }
-                            }
-                            else {
-                                const teardown = agent.teardown;
-                                if (teardown) {
-                                    const context = { ...attemptCtx, signal };
-                                    const native = (0, effect_runtime_ts_1.agentTeardownEffect)(agent, sandbox, context);
-                                    await assertFirst.requestCleanup((0, cleanup_timeout_ts_1.withCleanupTimeout)(native === undefined
-                                        ? (0, callback_context_ts_1.withAgentCallbackContext)(context, (callbackContext) => (0, effect_runtime_ts_1.authorCallbackEffect)(() => teardown(callbackContext)), { inheritAttemptSignal: false })
-                                        : (0, callback_context_ts_1.withAgentCallbackContext)(context, (callbackContext) => (0, effect_runtime_ts_1.agentTeardownEffect)(agent, sandbox, callbackContext), { inheritAttemptSignal: false })));
-                                }
-                            }
+                            const context = { ...sandboxAttemptCtx, signal };
+                            await assertFirst.requestCleanup((0, cleanup_timeout_ts_1.withCleanupTimeout)((0, callback_context_ts_1.withAgentCallbackContext)(context, (callbackContext) => (0, agent_effects_ts_1.agentAfterEffect)(agent, sandbox, callbackContext) ?? effect_1.Effect.void, { inheritAttemptSignal: false })));
                         }
                         catch (e) {
                             declareFailure("agent.teardown", e);
@@ -3095,7 +3083,7 @@ const EVAL_RESULT_REDACTED_EVIDENCE_KEYS = [
     "events",
     "o11y",
     "trace",
-    "agentSetup",
+    "agentInstallation",
     "evidenceCoverage",
     "diff",
     "commands",