niceeval 0.13.1 → 0.13.2

package/dist/runner/attempt.cjs

@@ -74,7 +74,7 @@ function attemptFailureDeclaration(classify, phase, e) {
         return undefined;
     return { class: cls, phase, text: info.text };
 }
-function runAttemptEffect(a, opts, sandboxSem, { buildLocators, runTiming, parentSignal, invocationSignal, onPhase, concurrencySlot, onFailureClass, onSealedEvaluation, reusedSandbox, }) {
+function runAttemptEffect(a, opts, sandboxSem, { buildLocators, runTiming, parentSignal, invocationSignal, onPhase, concurrencySlot, onFailureClass, onFreshSandboxCleanupFailure, onSealedEvaluation, reusedSandbox, }) {
     const config = opts.config;
     const { evalDef, run, attempt } = a;
     const coordinationRoot = opts.coordinationRoot ?? `${process.cwd()}/.niceeval`;
@@ -281,6 +281,40 @@ function runAttemptEffect(a, opts, sandboxSem, { buildLocators, runTiming, paren
         },
         diagnostic: recordDiagnostic,
     };
+    const recordFreshSandboxCleanupFailure = (stage, cause) => {
+        if (reusedSandbox !== undefined || onFreshSandboxCleanupFailure === undefined)
+            return;
+        const rawError = cause instanceof Error ? cause : new Error(String(cause));
+        // The ledger is later surfaced as an Experiment diagnostic, outside this
+        // Attempt's normal diagnostic sink. Preserve the real cleanup outcome, but
+        // cross that boundary only after applying this Attempt's evidence redaction.
+        const error = new Error((0, redaction_ts_1.redactSensitiveText)(rawError.message, sensitiveValues));
+        // This receiver runs inside the Scope finalizer path. It must not let a
+        // bookkeeping failure replace the provider or lifecycle cleanup outcome.
+        try {
+            onFreshSandboxCleanupFailure({ stage, error });
+        }
+        catch {
+            // The owning Experiment will still receive the original cleanup result.
+        }
+    };
+    /**
+     * Sandbox runtime deliberately keeps lifecycle cleanup diagnostic-only so it
+     * can continue other finalizers. Only this fresh-materialization wrapper
+     * observes those runtime-owned diagnostics; Eval/Agent diagnostics with the
+     * same public sink never enter the physical cleanup ledger.
+     */
+    const freshSandboxLifecycleFeedback = {
+        progress: scopedFeedback.progress,
+        diagnostic: (input) => {
+            scopedFeedback.diagnostic(input);
+            if (input.code === "sandbox-teardown-failed" ||
+                input.code === "sandbox-stop-failed" ||
+                input.code === "plugin-lifecycle-teardown-failed") {
+                recordFreshSandboxCleanupFailure("sandbox.lifecycle", new Error((0, redaction_ts_1.redactSensitiveText)(input.message, sensitiveValues)));
+            }
+        },
+    };
     const sealExecutionError = () => {
         const runtime = liveAssertions ?? (evalDef.evaluationKind === "score"
             ? (0, runtime_ts_2.createAssertionsRuntime)({ evaluationKind: "score" })
@@ -434,13 +468,13 @@ function runAttemptEffect(a, opts, sandboxSem, { buildLocators, runTiming, paren
                             plan: sandboxPlan,
                             evalId: evalDef.id,
                             deadline: runtimeDeadline,
-                            feedback: scopedFeedback,
+                            feedback: freshSandboxLifecycleFeedback,
                             signal,
                             hookContext: {
                                 experimentId: sandboxPlan.pair.experimentId,
                                 signal,
-                                progress: scopedFeedback.progress,
-                                diagnostic: scopedFeedback.diagnostic,
+                                progress: freshSandboxLifecycleFeedback.progress,
+                                diagnostic: freshSandboxLifecycleFeedback.diagnostic,
                             },
                             buildLocators,
                             agent: run.agent.kind === "sandbox" ? run.agent : undefined,
@@ -454,7 +488,11 @@ function runAttemptEffect(a, opts, sandboxSem, { buildLocators, runTiming, paren
                                 run: (owned) => effect_1.Effect.gen(function* () {
                                     const sb = owned.sandbox;
                                     if (disposition !== "keep") {
-                                        return yield* tryPromiseAsDefect(() => owned.group.stop());
+                                        return yield* tryPromiseAsDefect(() => owned.group.stop()).pipe(effect_1.Effect.onExit((exit) => effect_1.Exit.isFailure(exit)
+                                            ? effect_1.Effect.sync(() => {
+                                                recordFreshSandboxCleanupFailure("sandbox.stop", effect_1.Cause.squash(exit.cause));
+                                            })
+                                            : effect_1.Effect.void));
                                     }
                                     yield* effect_1.Effect.sync(() => (0, registry_ts_1.unregisterSandbox)(sb));
                                     const providerName = runtimeCapabilities.provider;
@@ -2149,6 +2187,7 @@ function experimentRunInfo(run, plan, sandboxPlansByEval, config, judge) {
         sandboxLayer: (0, link_ts_1.sandboxLayerIdentityFor)(plan.pair, "experiment"),
         sandboxPlansByEval: { ...sandboxPlansByEval },
         ...(run.sandboxReuse ? { sandboxReuse: true } : {}),
+        ...(run.sharedState === undefined ? {} : { sharedState: { key: run.sharedState.key } }),
         ...(judge
             ? { judge: { model: judge.model, baseUrl: judge.baseUrl, apiKeyEnv: judge.apiKeyEnv, timeoutMs: judge.timeoutMs } }
             : {}),