harnery 0.43.0 → 0.44.0

package/src/core/workflow/engine.ts

@@ -45,7 +45,16 @@ import { freezeWorkflowAttemptContext } from "./attempt-context.ts";
 import { type BillingProbe, probeBilling } from "./billing.ts";
 import { stableDigest } from "./durable-record.ts";
 import { evidencePreflightError } from "./evidence-preflight.ts";
-import { endWorkflowChildSessionV3, startWorkflowChildSessionV3 } from "./live-session-v3.ts";
+import {
+  canRetryFailedSpawn,
+  observeSpawnFailure,
+  resolveFailureEvidence,
+} from "./failure-evidence.ts";
+import {
+  endWorkflowChildSessionV3,
+  recordWorkflowChildTerminalFailureV3,
+  startWorkflowChildSessionV3,
+} from "./live-session-v3.ts";
 import {
   buildWorkflowProof,
   createEvidenceRecord,
@@ -404,7 +413,13 @@ async function executeWorkflow(
         durationMs,
         transcriptPath,
         before: resumeState.manifest.repository_before,
-        after: resumeState.manifest.repository_before,
+        after: {
+          cwd: frozenBinding.active_root,
+          observation: "unavailable",
+          observation_detail:
+            "Workspace reattachment failed; no trusted after-run snapshot was observed.",
+          dirty_paths: [],
+        },
         agents: [],
         evidence: [],
         diagnosticAdmission: initialDiagnosticAdmissionProof(diagnosticAdmissionConfig),
@@ -606,6 +621,7 @@ async function executeWorkflow(
   let policySeq = 0;
   const billingProbed = new Map<AdapterName, BillingProbe>();
   const agentProofs = new Map<string, WorkflowAgentProof>();
+  let terminalRecordingError: Error | undefined;
   const evidenceRecords: WorkflowEvidenceRecord[] = [];
   const policyDecisions = new Map<string, PolicyDecision>(
     history?.policyDecisions.map((decision) => [decision.id, decision]),
@@ -1022,6 +1038,49 @@ async function executeWorkflow(
     });
     agentProof.access = access;
 
+    let childGenerationId: string | undefined;
+    let recordAgentEnd: (() => void) | undefined;
+    const finalizeChild = (): void => {
+      try {
+        if (childGenerationId)
+          endWorkflowChildSessionV3({
+            coordRoot: opts.coordRoot,
+            instanceId: childInstanceId(runId, id),
+            runId,
+            agentId: id,
+            adapter,
+            cleanExit: agentProof.status === "succeeded",
+            expectedGenerationId: childGenerationId,
+          });
+        // Result publication is the resume-cache boundary, after canonical custody.
+        recordAgentEnd?.();
+      } catch (error) {
+        let reason = `V3 terminal recording failed: ${(error as Error).message}`;
+        try {
+          if (childGenerationId)
+            recordWorkflowChildTerminalFailureV3({
+              coordRoot: opts.coordRoot,
+              instanceId: childInstanceId(runId, id),
+              runId,
+              agentId: id,
+              adapter,
+              expectedGenerationId: childGenerationId,
+              reason,
+            });
+        } catch (visibilityError) {
+          reason += `; blocked lifecycle recording failed: ${(visibilityError as Error).message}`;
+        }
+        const failure = new Error(agentProof.error ? `${agentProof.error}; ${reason}` : reason, {
+          cause: error,
+        });
+        terminalRecordingError ??= failure;
+        agentProof.status = "failed";
+        agentProof.error = failure.message;
+        transcript("agent.failed", { id, key, error: failure.message });
+        log(`[${name}] ${failure.message}`);
+        throw failure;
+      }
+    };
     await acquire();
     try {
       await ensureDiagnosticAdmissionObserved();
@@ -1043,7 +1102,7 @@ async function executeWorkflow(
       // Headless children do not reliably fire adapter hooks. The engine owns
       // their canonical V3 start so every dispatched child has an auditable
       // generation before it begins work.
-      startWorkflowChildSessionV3({
+      const child = startWorkflowChildSessionV3({
         coordRoot: opts.coordRoot,
         instanceId: childInstanceId(runId, id),
         runId,
@@ -1052,10 +1111,13 @@ async function executeWorkflow(
         label,
         model: agentOpts.model,
       });
+      if (!child.v3_generation_id) throw new Error("workflow_child_v3_generation_missing");
+      childGenerationId = child.v3_generation_id;
 
       let attemptPrompt = dispatchPrompt;
       let last: SpawnResult | null = null;
       let agentCostUsd = 0;
+      let informativeAttempt: ReturnType<typeof observeSpawnFailure> | undefined;
       for (let attempt = 1; attempt <= maxAttempts; attempt++) {
         last = await spawner({
           prompt: attemptPrompt,
@@ -1078,33 +1140,51 @@ async function executeWorkflow(
         costUsd += last.costUsd ?? 0;
 
         if (!last.ok) {
-          transcript("agent.attempt_failed", { id, attempt, error: last.error });
-          // ADR 0046: an environment failure (the binary was absent) cannot be
-          // helped by retrying an unchanged environment, so stop the in-agent
-          // retry too — not just the outer attempt/replan budget. An upstream
-          // refusal keeps retrying here: the vendor may recover mid-loop.
-          if (last.class === "environment") break;
-          continue; // spawn-level failure: retry with the original prompt
+          const observed =
+            last.failureEvidence === undefined
+              ? observeSpawnFailure({}, last.text, "final_message")
+              : resolveFailureEvidence(last.failureEvidence);
+          agentProof.failure_evidence = informativeAttempt
+            ? {
+                ...observed,
+                effect: { state: "unknown" },
+                partial_output: informativeAttempt.partial_output,
+              }
+            : observed;
+          transcript("agent.attempt_failed", {
+            id,
+            attempt,
+            error: last.error,
+            failure_evidence: observed,
+            result_digest: digestResult(last.text, "text"),
+            text: last.text,
+          });
+          if (!canRetryFailedSpawn({ ...last, failureEvidence: agentProof.failure_evidence }))
+            break;
+          continue;
         }
+        informativeAttempt = observeSpawnFailure({}, last.text, "final_message");
         if (!agentOpts.schema) {
           agentProof.status = "succeeded";
           agentProof.cost_usd =
             agentCostUsd > 0 || last.costUsd !== undefined ? agentCostUsd : undefined;
           agentProof.session_id = last.sessionId;
           agentProof.result = digestResult(last.text, "text");
-          transcriptAgentEnd(
-            {
-              id,
-              key,
-              attempts: attempt,
-              cost_usd: last.costUsd,
-              total_cost_usd: agentCostUsd,
-              duration_ms: last.durationMs,
-              session_id: last.sessionId,
-            },
-            "text",
-            last.text,
-          );
+          const completed = last;
+          recordAgentEnd = () =>
+            transcriptAgentEnd(
+              {
+                id,
+                key,
+                attempts: attempt,
+                cost_usd: completed.costUsd,
+                total_cost_usd: agentCostUsd,
+                duration_ms: completed.durationMs,
+                session_id: completed.sessionId,
+              },
+              "text",
+              completed.text,
+            );
           return last.text;
         }
 
@@ -1119,23 +1199,32 @@ async function executeWorkflow(
             agentCostUsd > 0 || last.costUsd !== undefined ? agentCostUsd : undefined;
           agentProof.session_id = last.sessionId;
           agentProof.result = digestResult(parsed.value, "json");
-          transcriptAgentEnd(
-            {
-              id,
-              key,
-              attempts: attempt,
-              cost_usd: last.costUsd,
-              total_cost_usd: agentCostUsd,
-              duration_ms: last.durationMs,
-              session_id: last.sessionId,
-            },
-            "json",
-            parsed.value,
-          );
+          const completed = last;
+          recordAgentEnd = () =>
+            transcriptAgentEnd(
+              {
+                id,
+                key,
+                attempts: attempt,
+                cost_usd: completed.costUsd,
+                total_cost_usd: agentCostUsd,
+                duration_ms: completed.durationMs,
+                session_id: completed.sessionId,
+              },
+              "json",
+              parsed.value,
+            );
           return parsed.value;
         }
 
-        transcript("agent.schema_retry", { id, attempt, problems });
+        agentProof.failure_evidence = informativeAttempt;
+        transcript("agent.schema_retry", {
+          id,
+          attempt,
+          problems,
+          rejected_text: last.text,
+          result_digest: digestResult(last.text, "text"),
+        });
         // Keep the original cache identity while making the actual output
         // contract explicit. A full bounded copy of the rejected value lets the
         // child repair omissions instead of reconstructing an unseen object.
@@ -1168,20 +1257,12 @@ async function executeWorkflow(
       throw error;
     } finally {
       reservedCostUsd = Math.max(0, reservedCostUsd - reservedForDispatch);
-      // Record the canonical terminal before removing the disposable cache.
       try {
-        endWorkflowChildSessionV3({
-          coordRoot: opts.coordRoot,
-          instanceId: childInstanceId(runId, id),
-          runId,
-          agentId: id,
-          adapter,
-          cleanExit: agentProof.status === "succeeded",
-        });
-      } catch (error) {
-        log(`[${name}] V3 terminal recording failed: ${(error as Error).message}`);
+        finalizeChild();
+      } finally {
+        // A slot is disposable; unresolved custody remains visible and failed.
+        release();
       }
-      release();
     }
   };
 
@@ -1286,6 +1367,7 @@ async function executeWorkflow(
     if (initializationError) throw initializationError;
     if (!workflow) throw new Error("workflow module initialization did not produce an executable");
     const result = await workflow(ctx);
+    if (terminalRecordingError) throw terminalRecordingError;
     const endedAt = new Date().toISOString();
     const durationMs = Date.now() - t0;
     const workspaceAttestation = workspaceBinding

package/src/core/workflow/failure-evidence.ts

@@ -0,0 +1,141 @@
+import { createHash } from "node:crypto";
+import type { AdapterRawResult } from "../adapters/types.ts";
+import type { SpawnFailureEvidence, SpawnOutputSource, SpawnResult } from "./types.ts";
+
+export function unknownFailureEvidence(): SpawnFailureEvidence {
+  return {
+    effect: { state: "unknown" },
+    refusal: { state: "unknown" },
+    partial_output: { state: "unknown" },
+  };
+}
+
+/** Built-in subprocess adapters observe output and process-start failures.
+ * Their current vendor envelopes do not prove effects or native refusal.
+ * In particular, blank output and 429/5xx statuses leave both unknown. */
+export function observeSpawnFailure(
+  raw: Pick<AdapterRawResult, "spawnErrno" | "timedOut"> &
+    Partial<Pick<AdapterRawResult, "stdout" | "resultFileText">>,
+  text: string,
+  source: SpawnOutputSource,
+): SpawnFailureEvidence {
+  const bytes = Buffer.byteLength(text);
+  return {
+    effect:
+      raw.spawnErrno === "ENOENT" &&
+      !raw.timedOut &&
+      bytes === 0 &&
+      !raw.stdout?.trim() &&
+      !raw.resultFileText?.trim()
+        ? { state: "none", source: "spawn_errno", reference: "ENOENT" }
+        : { state: "unknown" },
+    refusal: { state: "unknown" },
+    partial_output:
+      bytes > 0
+        ? {
+            state: "present",
+            source,
+            bytes,
+            sha256: createHash("sha256").update(text).digest("hex"),
+          }
+        : { state: "absent", source },
+  };
+}
+
+function record(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function keys(value: Record<string, unknown>, expected: readonly string[]): boolean {
+  return (
+    Object.keys(value).length === expected.length &&
+    expected.every((key) => Object.hasOwn(value, key))
+  );
+}
+
+function reference(value: unknown): value is string {
+  return typeof value === "string" && value.trim().length > 0 && value.length <= 512;
+}
+
+function validEffect(value: unknown): boolean {
+  if (!record(value)) return false;
+  if (value.state === "unknown") return keys(value, ["state"]);
+  if (!keys(value, ["state", "source", "reference"])) return false;
+  if (value.state === "none" && value.source === "spawn_errno") {
+    return value.reference === "ENOENT";
+  }
+  return (
+    (value.state === "none" || value.state === "observed") &&
+    value.source === "adapter_evidence" &&
+    reference(value.reference)
+  );
+}
+
+function validRefusal(value: unknown): boolean {
+  if (!record(value)) return false;
+  if (value.state === "unknown") return keys(value, ["state"]);
+  return (
+    keys(value, ["state", "source", "reference"]) &&
+    value.state === "refused" &&
+    value.source === "vendor_envelope" &&
+    reference(value.reference)
+  );
+}
+
+function validOutput(value: unknown): boolean {
+  if (!record(value)) return false;
+  if (value.state === "unknown") return keys(value, ["state"]);
+  if (
+    value.source !== "final_message" &&
+    value.source !== "vendor_envelope" &&
+    value.source !== "stdout"
+  ) {
+    return false;
+  }
+  if (value.state === "absent") return keys(value, ["state", "source"]);
+  return (
+    value.state === "present" &&
+    keys(value, ["state", "source", "bytes", "sha256"]) &&
+    Number.isSafeInteger(value.bytes) &&
+    (value.bytes as number) > 0 &&
+    typeof value.sha256 === "string" &&
+    /^[a-f0-9]{64}$/.test(value.sha256)
+  );
+}
+
+export function isSpawnFailureEvidence(value: unknown): value is SpawnFailureEvidence {
+  return (
+    record(value) &&
+    keys(value, ["effect", "refusal", "partial_output"]) &&
+    validEffect(value.effect) &&
+    validRefusal(value.refusal) &&
+    validOutput(value.partial_output)
+  );
+}
+
+/** Missing observations are unknown; malformed positive claims are rejected.
+ * This is also used when reading persisted workflow proof. */
+export function resolveFailureEvidence(value: unknown): SpawnFailureEvidence {
+  if (value === undefined) return unknownFailureEvidence();
+  if (!isSpawnFailureEvidence(value)) throw new Error("invalid spawn failure evidence");
+  return value;
+}
+
+export function failureEvidenceAllowsUncharged(value: unknown): boolean {
+  const evidence = resolveFailureEvidence(value);
+  return (
+    evidence.effect.state === "none" &&
+    evidence.refusal.state !== "refused" &&
+    evidence.partial_output.state === "absent"
+  );
+}
+
+/** A provider retry requires positive no-effect evidence. A missing binary is
+ * known not to have started, but retrying that unchanged environment cannot help. */
+export function canRetryFailedSpawn(result: SpawnResult): boolean {
+  return (
+    !result.ok &&
+    result.class !== "environment" &&
+    failureEvidenceAllowsUncharged(result.failureEvidence)
+  );
+}

package/src/core/workflow/index.ts

@@ -166,7 +166,6 @@ export {
   isWorkspaceLifecycleState,
   listWorkflowWorkspaceInspections,
   prepareIntegration,
-  prepareReclaim,
   probeLocalGitWorktreeProvider,
   readWorkflowWorkspaceStatus,
   renderWorkflowWorkspaceStatus,

package/src/core/workflow/live-session-v3.ts

@@ -1,9 +1,16 @@
+import { randomUUID } from "node:crypto";
+import { existsSync, linkSync, readdirSync, readFileSync, renameSync, unlinkSync } from "node:fs";
+import { basename, dirname } from "node:path";
 import { type Adapter, adapterFromPlatform } from "../adapter.ts";
-import { recordLiveTaskChangeV3 } from "../agents/live-authority-v3.ts";
+import {
+  recordLiveLifecycleChangeV3,
+  recordLiveTaskChangeV3,
+} from "../agents/live-authority-v3.ts";
 import { endSessionExplicitV3 } from "../agents/session-finalizer-v3.ts";
+import { heartbeatPath } from "../agents/state/heartbeat-reader.ts";
 import {
-  clearCoordinationCache,
   type Heartbeat,
+  readHeartbeat,
   setIdentityCache,
 } from "../agents/state/heartbeat-writer.ts";
 import { readLiveCoordinationRow } from "../agents/state/live-coordination-view.ts";
@@ -12,9 +19,17 @@ import { assignName } from "../agents/state/names.ts";
 import { coordFreshnessSeconds } from "../config.ts";
 import { ensureEventLedgerV3 } from "../events/v3/bootstrap.ts";
 import {
+  readCoordinationViewV3,
+  requireAuthoritySafeCoordinationViewV3,
+} from "../events/v3/coordination-view.ts";
+import { withGenerationCustodyV3 } from "../events/v3/generation-custody.ts";
+import {
+  liveInstanceIdV3,
   recordLiveHookSignalV3,
   resolveLiveEventLedgerRouteV3,
 } from "../events/v3/live-routing.ts";
+import { readHookProducerStateV3 } from "../events/v3/producers/recorder.ts";
+import { fsyncParentDirectory } from "./durable-record.ts";
 import { stableScopeId } from "./scope-id.ts";
 
 export interface WorkflowChildSessionV3Input {
@@ -77,16 +92,45 @@ export function startWorkflowChildSessionV3(input: WorkflowChildSessionV3Input):
   }
   const projected = readLiveCoordinationRow(input.coordRoot, input.instanceId);
   if (!projected) throw new Error("workflow_child_v3_projection_missing");
+  const own = readHookProducerStateV3(input.coordRoot, adapter, sessionId);
+  if (!own || own.terminal || own.generation_id !== projected.v3_generation_id) {
+    throw new Error("workflow_child_v3_start_generation_changed");
+  }
   return projected;
 }
 
 /** Record the authoritative terminal before removing the disposable coordination cache. */
 export function endWorkflowChildSessionV3(
-  input: WorkflowChildSessionV3Input & { cleanExit: boolean },
+  input: WorkflowChildSessionV3Input & { cleanExit: boolean; expectedGenerationId: string },
+): void {
+  withGenerationCustodyV3(input.coordRoot, [liveInstanceIdV3(input.instanceId)], () =>
+    endExpectedChild(input),
+  );
+}
+
+function endExpectedChild(
+  input: WorkflowChildSessionV3Input & { cleanExit: boolean; expectedGenerationId: string },
 ): void {
   requireV3Route(input.coordRoot);
+  const view = requireAuthoritySafeCoordinationViewV3(readCoordinationViewV3(input.coordRoot));
+  const instanceId = liveInstanceIdV3(input.instanceId);
+  const current = view.instances[instanceId];
+  const terminal = view.terminal_generations[input.expectedGenerationId];
+  if (terminal?.instance_id === instanceId) recoverHeldCaches(input);
+  if (current && current.generation_id !== input.expectedGenerationId) {
+    throw new Error("workflow_child_v3_generation_changed");
+  }
+  if (!current) {
+    if (!terminal || terminal.instance_id !== instanceId)
+      throw new Error("workflow_child_v3_generation_missing");
+    if (terminal.terminal?.outcome !== (input.cleanExit ? "succeeded" : "failed")) {
+      throw new Error("workflow_child_v3_terminal_outcome_changed");
+    }
+    clearExpectedCache(input);
+    return;
+  }
   const row = readLiveCoordinationRow(input.coordRoot, input.instanceId);
-  if (!row?.v3_instance_id || !row.v3_generation_id) {
+  if (!row?.v3_instance_id || row.v3_generation_id !== input.expectedGenerationId) {
     throw new Error("workflow_child_v3_generation_missing");
   }
   const result = endSessionExplicitV3({
@@ -101,7 +145,104 @@ export function endWorkflowChildSessionV3(
       `workflow_child_v3_end_failed:${result.state}${"reason" in result ? `:${result.reason}` : ""}`,
     );
   }
-  clearCoordinationCache(input.coordRoot, input.instanceId);
+  clearExpectedCache(input);
+}
+
+/** Persist unresolved terminal custody only for the generation this dispatch owns. */
+export function recordWorkflowChildTerminalFailureV3(
+  input: WorkflowChildSessionV3Input & { expectedGenerationId: string; reason: string },
+): void {
+  withGenerationCustodyV3(input.coordRoot, [liveInstanceIdV3(input.instanceId)], () => {
+    const row = readLiveCoordinationRow(input.coordRoot, input.instanceId);
+    if (!row || row.v3_generation_id !== input.expectedGenerationId) return;
+    recordLiveLifecycleChangeV3({
+      coordRoot: input.coordRoot,
+      owner: input.instanceId,
+      nativeSessionId: input.sessionId ?? input.instanceId,
+      adapter: workflowAdapter(input.adapter),
+      expectedGenerationId: input.expectedGenerationId as `gen_${string}`,
+      state: "blocked",
+      reason: input.reason,
+    });
+  });
+}
+
+function heldPrefix(input: WorkflowChildSessionV3Input & { expectedGenerationId: string }): string {
+  if (!/^gen_[a-zA-Z0-9._-]{1,128}$/.test(input.expectedGenerationId))
+    throw new Error("workflow_child_v3_generation_invalid");
+  return `${basename(heartbeatPath(input.coordRoot, input.instanceId))}.terminal-${input.expectedGenerationId}-`;
+}
+
+function recoverHeldCaches(
+  input: WorkflowChildSessionV3Input & { expectedGenerationId: string },
+): void {
+  const path = heartbeatPath(input.coordRoot, input.instanceId);
+  if (!existsSync(dirname(path))) return;
+  const prefix = heldPrefix(input);
+  for (const name of readdirSync(dirname(path)).filter((value) => value.startsWith(prefix))) {
+    finishHeldCache(`${dirname(path)}/${name}`, path, input.expectedGenerationId);
+  }
+}
+
+function clearExpectedCache(
+  input: WorkflowChildSessionV3Input & { expectedGenerationId: string },
+): void {
+  recoverHeldCaches(input);
+  if (
+    readHeartbeat(input.coordRoot, input.instanceId)?.v3_generation_id !==
+    input.expectedGenerationId
+  )
+    return;
+  const path = heartbeatPath(input.coordRoot, input.instanceId);
+  const held = `${dirname(path)}/${heldPrefix(input)}${randomUUID()}`;
+  try {
+    // A unique held name pins the actual bytes atomically. Checking before
+    // unlink alone could consume a replacement published at this boundary.
+    renameSync(path, held);
+    fsyncParentDirectory(path);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
+    throw error;
+  }
+  finishHeldCache(held, path, input.expectedGenerationId);
+}
+
+function finishHeldCache(held: string, path: string, expectedGenerationId: string): void {
+  let removed = false;
+  let cleanupFailed = false;
+  let cleanupError: unknown;
+  try {
+    const cache = JSON.parse(readFileSync(held, "utf8")) as Heartbeat;
+    if (cache.v3_generation_id === expectedGenerationId) {
+      unlinkSync(held);
+      fsyncParentDirectory(held);
+      removed = true;
+    }
+  } catch (error) {
+    cleanupFailed = true;
+    cleanupError = error;
+  }
+  if (!removed) {
+    try {
+      // Never overwrite a later publication. Keep the held evidence when
+      // another cache already owns the live name.
+      linkSync(held, path);
+      fsyncParentDirectory(path);
+      unlinkSync(held);
+      fsyncParentDirectory(held);
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "EEXIST") {
+        if (cleanupFailed) {
+          throw new AggregateError(
+            [cleanupError, error],
+            `workflow_child_v3_cache_cleanup_and_restore_failed: ${String(cleanupError)}; restoration: ${String(error)}`,
+          );
+        }
+        throw error;
+      }
+    }
+  }
+  if (cleanupFailed) throw cleanupError;
 }
 
 function requireV3Route(coordRoot: string) {