@bastani/atomic 0.9.13-alpha.1 → 0.9.13-alpha.3

package/dist/builtin/workflows/src/durable/completed-catalog.ts

@@ -7,7 +7,9 @@ import {
 	boundaryOwner,
 	childRunIdFromDraft,
 	childRunStatus,
+	childScopedEvidenceExists,
 	compareDraftSourceOrder,
+	isSyntheticExitedChild,
 	mergeStageDraft,
 	mergeStageGroup,
 	retainReachableRunGroups,
@@ -18,9 +20,14 @@ import {
 	validStageGroup,
 	workflowChildFromDraft,
 } from "./completed-catalog-stage-groups.js";
+import { isDurableWorkflowResumable } from "./resume-eligibility.js";
 import { resolveDurableEntry } from "./resume-runtime.js";
 import { priorRunElapsedMs, RUN_TIMING_CHECKPOINT_NAME } from "./run-timing.js";
-import { groupByDurableStageKey, immutableStageGroupError } from "./stage-topology-validation.js";
+import {
+	directChildTopologyError,
+	groupByDurableStageKey,
+	immutableStageGroupError,
+} from "./stage-topology-validation.js";
 import { workflowToolOutcomeFromValue } from "./tool-outcome.js";
 import {
 	DURABLE_STAGE_TOPOLOGY_VERSION,
@@ -92,9 +99,13 @@ export function completedWorkflowRunSnapshots(
 	if (handle === undefined || (handle.status !== "completed" && handle.status !== "failed")) return [];
 	const checkpoints = backend.listCheckpoints(entry.workflowId);
 	if (checkpoints.length === 0) return [];
-	const runs = runSnapshotsFromCheckpoints(checkpoints, handle.workflowId, handle.name, handle.updatedAt).map(
-		(run) => ({ ...run, stages: run.stages.map(validatedStageTranscript) }),
-	);
+	const runs = runSnapshotsFromCheckpoints(
+		checkpoints,
+		handle.workflowId,
+		handle.name,
+		handle.updatedAt,
+		COMPLETED_RECONSTRUCTION,
+	).map((run) => ({ ...run, stages: run.stages.map(validatedStageTranscript) }));
 	const rootIndex = runs.findIndex((run) => run.id === handle.workflowId);
 	if (rootIndex < 0) return [];
 	const rootDuration =
@@ -107,6 +118,8 @@ export function completedWorkflowRunSnapshots(
 		endedAt: handle.updatedAt,
 		durationMs: rootDuration,
 		...(handle.error !== undefined ? { error: handle.error } : {}),
+		...(handle.exited !== undefined ? { exited: handle.exited } : {}),
+		...(handle.exitReason !== undefined ? { exitReason: handle.exitReason } : {}),
 		...(handle.failureKind !== undefined ? { failureKind: handle.failureKind } : {}),
 		...(handle.failureCode !== undefined ? { failureCode: handle.failureCode } : {}),
 		...(handle.failureRecoverability !== undefined ? { failureRecoverability: handle.failureRecoverability } : {}),
@@ -117,6 +130,59 @@ export function completedWorkflowRunSnapshots(
 	return [root, ...runs.filter((_, index) => index !== rootIndex)];
 }
 
+/** Rebuild one retained root, including in-progress nested runs, without relaxing topology validation. */
+export function durableWorkflowRunSnapshots(
+	backend: DurableWorkflowBackend,
+	handle: import("./types.js").DurableWorkflowHandle,
+): readonly RunSnapshot[] {
+	const checkpoints = backend.listCheckpoints(handle.workflowId);
+	if (checkpoints.length === 0) {
+		return [rootSnapshot(backend, handle, { stages: [], toolNodes: [] })];
+	}
+	const requiresTerminalTopology = handle.status === "completed" || handle.status === "failed";
+	const runs = runSnapshotsFromCheckpoints(
+		checkpoints,
+		handle.workflowId,
+		handle.name,
+		handle.updatedAt,
+		requiresTerminalTopology ? COMPLETED_RECONSTRUCTION : ACTIVE_INSPECTION_RECONSTRUCTION,
+	);
+	const rootIndex = runs.findIndex((run) => run.id === handle.workflowId);
+	if (rootIndex < 0) return [];
+	return [rootSnapshot(backend, handle, runs[rootIndex]!), ...runs.filter((_, index) => index !== rootIndex)];
+}
+
+function rootSnapshot(
+	backend: DurableWorkflowBackend,
+	handle: import("./types.js").DurableWorkflowHandle,
+	graph: Pick<RunSnapshot, "stages" | "toolNodes">,
+): RunSnapshot {
+	const terminal = handle.status === "completed" || handle.status === "failed" || handle.status === "cancelled";
+	const priorDurationMs = priorRunElapsedMs(backend, handle.workflowId);
+	const terminalDurationMs = priorDurationMs ?? Math.max(0, handle.updatedAt - handle.createdAt);
+	return {
+		id: handle.workflowId,
+		name: handle.name,
+		inputs: { ...handle.inputs } as WorkflowInputValues,
+		status: handle.status,
+		stages: graph.stages,
+		toolNodes: graph.toolNodes,
+		startedAt: handle.createdAt,
+		...(terminal
+			? { endedAt: handle.updatedAt, durationMs: terminalDurationMs }
+			: priorDurationMs === undefined
+				? {}
+				: { accumulatedDurationMs: priorDurationMs }),
+		...(handle.error !== undefined ? { error: handle.error } : {}),
+		...(handle.failureKind !== undefined ? { failureKind: handle.failureKind } : {}),
+		...(handle.failureCode !== undefined ? { failureCode: handle.failureCode } : {}),
+		...(handle.failureRecoverability !== undefined ? { failureRecoverability: handle.failureRecoverability } : {}),
+		...(handle.failureDisposition !== undefined ? { failureDisposition: handle.failureDisposition } : {}),
+		...(handle.failedToolNodeId !== undefined ? { failedToolNodeId: handle.failedToolNodeId } : {}),
+		resumable: isDurableWorkflowResumable(handle),
+	};
+}
+
 /** Rebuild completed nested runs while a paused root is replaying cached boundaries. */
 export function durableNestedRunSnapshots(
 	backend: DurableWorkflowBackend,
@@ -129,7 +195,7 @@ export function durableNestedRunSnapshots(
 		rootWorkflowId,
 		handle.name,
 		handle.updatedAt,
-		false,
+		LEGACY_ACTIVE_RECONSTRUCTION,
 	).filter((run) => run.id !== rootWorkflowId);
 }
 
@@ -220,14 +286,44 @@ function summarizeCompletedToolResult(value: WorkflowSerializableValue): string
 	return serialized.length <= 240 ? serialized : `${flattenTruncatedString(serialized.slice(0, 237))}...`;
 }
 
+interface ReconstructionPolicy {
+	readonly requireTerminal: boolean;
+	readonly failClosed: boolean;
+	readonly retainActive: boolean;
+	readonly failOnUnreachable: boolean;
+	readonly requireActiveBoundaryFingerprint: boolean;
+}
+
+const COMPLETED_RECONSTRUCTION: ReconstructionPolicy = {
+	requireTerminal: true,
+	failClosed: true,
+	retainActive: false,
+	failOnUnreachable: false,
+	requireActiveBoundaryFingerprint: false,
+};
+const ACTIVE_INSPECTION_RECONSTRUCTION: ReconstructionPolicy = {
+	requireTerminal: false,
+	failClosed: true,
+	retainActive: true,
+	failOnUnreachable: true,
+	requireActiveBoundaryFingerprint: true,
+};
+const LEGACY_ACTIVE_RECONSTRUCTION: ReconstructionPolicy = {
+	requireTerminal: false,
+	failClosed: false,
+	retainActive: true,
+	failOnUnreachable: false,
+	requireActiveBoundaryFingerprint: false,
+};
+
 function runSnapshotsFromCheckpoints(
 	checkpoints: readonly DurableCheckpoint[],
 	rootRunId: string,
 	rootRunName: string,
 	fallbackCompletedAt: number,
-	strict = true,
+	policy: ReconstructionPolicy = COMPLETED_RECONSTRUCTION,
 ): RunSnapshot[] {
-	if (!validBoundaryRecordSet(checkpoints, strict)) return [];
+	if (!validBoundaryRecordSet(checkpoints, policy.requireTerminal, policy.requireActiveBoundaryFingerprint)) return [];
 	const stageRecords = checkpoints.filter(
 		(checkpoint): checkpoint is DurableStageCheckpoint => checkpoint.kind === "stage",
 	);
@@ -244,7 +340,10 @@ function runSnapshotsFromCheckpoints(
 		.map((records) => mergeStageGroup(records, stageRecords, checkpoints))
 		.filter(
 			(draft) =>
-				!strict || draft.topology?.run === undefined || draft.endedAt !== undefined || draft.output !== undefined,
+				policy.retainActive ||
+				draft.topology?.run === undefined ||
+				draft.endedAt !== undefined ||
+				draft.output !== undefined,
 		)
 		.map(
 			(draft): StageDraft =>
@@ -263,14 +362,46 @@ function runSnapshotsFromCheckpoints(
 		.sort(compareDraftSourceOrder);
 	const drafts = checkpointDrafts(checkpoints);
 	const toolCheckpoints = drafts.tools;
+	if (policy.failClosed) {
+		for (const draft of candidates) {
+			const boundary = draft.topology?.boundary;
+			if (boundary === undefined || childRunIdFromDraft(draft) === undefined) continue;
+			if (boundary.replayScope !== draft.replayKey) return [];
+			const expectedChildRun = {
+				runId: boundary.child.runId,
+				runName: boundary.child.runName,
+				parentRunId: boundary.child.parentRunId,
+				parentStageId: boundary.child.parentStageId,
+				rootRunId: boundary.child.rootRunId,
+			};
+			const childToolIds = new Set(
+				toolCheckpoints.flatMap((checkpoint) =>
+					checkpoint.topology?.run?.runId === expectedChildRun.runId ? [checkpoint.topology.nodeId] : [],
+				),
+			);
+			if (
+				directChildTopologyError(stageRecords, boundary.replayScope, expectedChildRun, true, childToolIds) !==
+				undefined
+			)
+				return [];
+		}
+	}
 	if (candidates.length === 0 && toolCheckpoints.length === 0) {
-		return strict ? [] : [syntheticRun(rootRunId, rootRunName, checkpoints.length, fallbackCompletedAt)];
+		const supportedNonGraphState = checkpoints.every(
+			(checkpoint) =>
+				checkpoint.kind === "ui" ||
+				(checkpoint.kind === "tool" && checkpoint.argsHash === RUN_TIMING_CHECKPOINT_NAME),
+		);
+		if (policy.failClosed && supportedNonGraphState) {
+			return [emptyGraphRun(rootRunId, rootRunName, fallbackCompletedAt)];
+		}
+		return policy.failClosed ? [] : [syntheticRun(rootRunId, rootRunName, checkpoints.length, fallbackCompletedAt)];
 	}
 
 	const grouped = new Map<string, StageDraft[]>();
 	for (const draft of candidates) {
 		if (draft.topology?.version !== DURABLE_STAGE_TOPOLOGY_VERSION) {
-			if (strict) return [];
+			if (policy.failClosed) return [];
 			continue;
 		}
 		const runId = draft.topology.run?.runId ?? rootRunId;
@@ -283,19 +414,36 @@ function runSnapshotsFromCheckpoints(
 		if (!grouped.has(runId)) grouped.set(runId, []);
 	}
 	for (const group of grouped.values()) group.sort(compareDraftSourceOrder);
+	const groupedCount = grouped.size;
 	retainReachableRunGroups(grouped, rootRunId);
-	if (!validateRunGroups(grouped, rootRunId, toolCheckpoints)) return [];
+	if (policy.failOnUnreachable && grouped.size !== groupedCount) return [];
+	if (!validateRunGroups(grouped, rootRunId, toolCheckpoints, checkpoints)) return [];
+	const syntheticChildren: Array<{
+		readonly parentRunId: string;
+		readonly owner: StageDraft;
+		readonly child: NonNullable<StageSnapshot["workflowChild"]>;
+	}> = [];
+	for (const [parentRunId, runDrafts] of grouped) {
+		for (const draft of runDrafts) {
+			const child = workflowChildFromDraft(draft);
+			if (child?.exited !== true || grouped.has(child.runId)) continue;
+			if (
+				!isSyntheticExitedChild(draft, parentRunId, child.runId, rootRunId) ||
+				childScopedEvidenceExists(checkpoints, draft.replayKey, child.runId)
+			) {
+				if (policy.failClosed) return [];
+				continue;
+			}
+			syntheticChildren.push({ parentRunId, owner: draft, child });
+		}
+	}
 
 	const idMaps = new Map<string, Map<string, string>>();
 	for (const [runId, runDrafts] of grouped) {
-		const ownedToolIds = new Set(
-			toolCheckpoints.flatMap((checkpoint) =>
-				(checkpoint.topology?.run?.runId ?? rootRunId) === runId
-					? [checkpoint.topology?.nodeId ?? checkpoint.checkpointId]
-					: [],
-			),
+		const ownedTools = toolCheckpoints.filter(
+			(checkpoint) => (checkpoint.topology?.run?.runId ?? rootRunId) === runId,
 		);
-		if (!validStageGroup(runDrafts, runId, ownedToolIds)) return [];
+		if (!validStageGroup(runDrafts, runId, rootRunId, ownedTools)) return [];
 		const ids = new Map<string, string>();
 		let hasIdentityConflict = false;
 		runDrafts.forEach((draft) => {
@@ -308,7 +456,7 @@ function runSnapshotsFromCheckpoints(
 			}
 		});
 		if (hasIdentityConflict) {
-			if (strict) return [];
+			if (policy.failClosed) return [];
 			grouped.delete(runId);
 			continue;
 		}
@@ -321,6 +469,7 @@ function runSnapshotsFromCheckpoints(
 	}
 
 	const runs: RunSnapshot[] = [];
+	const emittedRunIds = new Set<string>();
 	for (const [runId, runDrafts] of grouped) {
 		const ids = idMaps.get(runId)!;
 		const ownedTools = toolCheckpoints.filter(
@@ -332,7 +481,7 @@ function runSnapshotsFromCheckpoints(
 				(checkpoint) => checkpoint.topology?.parentIds.some((parentId) => !ids.has(parentId)) === true,
 			);
 		if (hasUnknownParents) {
-			if (strict) return [];
+			if (policy.failClosed) return [];
 			continue;
 		}
 		const stages = runDrafts.map((draft) =>
@@ -353,6 +502,8 @@ function runSnapshotsFromCheckpoints(
 			...toolNodes.map((tool) => tool.endedAt ?? fallbackCompletedAt),
 		);
 		const owner = runId === rootRunId ? undefined : boundaryOwner(grouped, runId);
+		const status = owner === undefined ? "completed" : childRunStatus(owner);
+		const terminal = status !== "running" && status !== "pending" && status !== "paused" && status !== "blocked";
 		const parentRunId = run?.parentRunId ?? rootRunId;
 		const boundarySourceId = grouped.get(parentRunId)?.find((draft) => childRunIdFromDraft(draft) === runId)
 			?.topology?.stageId;
@@ -361,19 +512,60 @@ function runSnapshotsFromCheckpoints(
 		const parentStageId =
 			declaredParentStageId ??
 			(boundarySourceId === undefined ? undefined : idMaps.get(parentRunId)?.get(boundarySourceId));
+		const ownerChild = owner === undefined ? undefined : workflowChildFromDraft(owner);
 		runs.push({
 			id: runId,
 			name: run?.runName ?? rootRunName,
 			inputs: {},
-			status: owner === undefined ? "completed" : childRunStatus(owner),
+			status,
 			stages,
 			toolNodes,
 			startedAt,
-			endedAt,
-			durationMs: Math.max(0, endedAt - startedAt),
+			...(terminal ? { endedAt, durationMs: Math.max(0, endedAt - startedAt) } : {}),
 			...(run?.parentRunId !== undefined ? { parentRunId: run.parentRunId } : {}),
 			...(parentStageId !== undefined ? { parentStageId } : {}),
 			...(run?.rootRunId !== undefined ? { rootRunId: run.rootRunId } : {}),
+			...(ownerChild?.exited === true
+				? {
+						result: ownerChild.outputs,
+						exited: true,
+						...(ownerChild.exitReason !== undefined ? { exitReason: ownerChild.exitReason } : {}),
+					}
+				: {}),
+			resumable: false,
+		});
+		emittedRunIds.add(runId);
+	}
+	for (const { parentRunId, owner, child } of syntheticChildren) {
+		if (!emittedRunIds.has(parentRunId)) {
+			if (policy.failClosed) return [];
+			continue;
+		}
+		const topology = owner.topology!;
+		const boundary = topology.boundary!;
+		const parentStageId = idMaps.get(parentRunId)?.get(topology.stageId);
+		if (parentStageId === undefined) {
+			if (policy.failClosed) return [];
+			continue;
+		}
+		const startedAt = owner.startedAt ?? owner.firstCompletedAt;
+		const endedAt = owner.endedAt ?? owner.firstCompletedAt;
+		runs.push({
+			id: child.runId,
+			name: boundary.child.runName,
+			inputs: {},
+			status: child.status,
+			stages: [],
+			toolNodes: [],
+			startedAt,
+			endedAt,
+			durationMs: Math.max(0, endedAt - startedAt),
+			parentRunId,
+			parentStageId,
+			rootRunId: boundary.child.rootRunId,
+			result: child.outputs,
+			exited: true,
+			...(child.exitReason !== undefined ? { exitReason: child.exitReason } : {}),
 			resumable: false,
 		});
 	}
@@ -424,6 +616,21 @@ function stageResult(draft: StageDraft): string | undefined {
 	return typeof draft.output === "string" ? draft.output : JSON.stringify(draft.output);
 }
 
+function emptyGraphRun(runId: string, runName: string, completedAt: number): RunSnapshot {
+	return {
+		id: runId,
+		name: runName,
+		inputs: {},
+		status: "completed",
+		stages: [],
+		toolNodes: [],
+		startedAt: completedAt,
+		endedAt: completedAt,
+		durationMs: 0,
+		resumable: false,
+	};
+}
+
 function syntheticRun(runId: string, runName: string, checkpointCount: number, completedAt: number): RunSnapshot {
 	return {
 		id: runId,

package/dist/builtin/workflows/src/durable/completed-subtree.ts

@@ -1,6 +1,7 @@
 import type { RunSnapshot } from "../shared/store-types.js";
 import type { DurableWorkflowBackend } from "./backend.js";
 import { durableNestedRunSnapshots } from "./completed-catalog.js";
+import { parseWorkflowChildResult } from "./workflow-child-result.js";
 
 /** Return one cached boundary's exact subtree only when every selected node is fully terminal. */
 export function durableCompletedNestedRunSubtree(
@@ -10,6 +11,12 @@ export function durableCompletedNestedRunSubtree(
 ): readonly RunSnapshot[] | undefined {
 	const nested = durableNestedRunSnapshots(backend, rootWorkflowId);
 	if (!nested.some((run) => run.id === childRunId)) return undefined;
+	const intentionalExitStatuses = new Map<string, RunSnapshot["status"]>();
+	for (const checkpoint of backend.listCheckpoints(rootWorkflowId)) {
+		if (checkpoint.kind !== "stage") continue;
+		const child = parseWorkflowChildResult(checkpoint.output);
+		if (child?.exited === true) intentionalExitStatuses.set(child.runId, child.status);
+	}
 	const included = new Set([childRunId]);
 	let priorSize = -1;
 	while (priorSize !== included.size) {
@@ -20,16 +27,24 @@ export function durableCompletedNestedRunSubtree(
 	}
 	const subtree = nested.filter((run) => included.has(run.id));
 	const complete =
-		subtree.length === included.size &&
-		subtree.every(
-			(run) =>
-				run.status === "completed" &&
-				run.endedAt !== undefined &&
-				run.stages.every((stage) => isTerminalStage(stage.status) && stage.endedAt !== undefined),
-		);
+		subtree.length === included.size && subtree.every((run) => isCompleteCachedRun(run, intentionalExitStatuses));
 	return complete ? subtree : undefined;
 }
 
+function isCompleteCachedRun(
+	run: RunSnapshot,
+	intentionalExitStatuses: ReadonlyMap<string, RunSnapshot["status"]>,
+): boolean {
+	if (
+		run.endedAt === undefined ||
+		!run.stages.every((stage) => isTerminalStage(stage.status) && stage.endedAt !== undefined)
+	) {
+		return false;
+	}
+	if (run.status === "completed" || run.exited === true) return true;
+	return intentionalExitStatuses.get(run.id) === run.status;
+}
+
 function isTerminalStage(status: RunSnapshot["stages"][number]["status"]): boolean {
 	return status === "completed" || status === "failed" || status === "skipped";
 }

package/dist/builtin/workflows/src/durable/dbos-backend.ts

@@ -5,6 +5,7 @@ import {
 	type DurableInactiveDeleteResult,
 	type DurableWorkflowBackend,
 	type DurableWorkflowCatalogEntries,
+	type DurableWorkflowHydrationResult,
 	InMemoryDurableBackend,
 	type WorkflowRegistrationInput,
 } from "./backend.js";
@@ -439,15 +440,21 @@ export class DbosDurableBackend implements DurableWorkflowBackend {
 	}
 
 	async hydrateWorkflow(workflowId: string): Promise<void> {
-		if (this.locallyRegistered.has(workflowId)) return;
-		const info = await this.sdk.retrieveWorkflow(workflowId);
-		if (info !== undefined) {
-			await this.hydrateInfo(info);
-			return;
+		await this.hydrateWorkflowForInspection(workflowId);
+	}
+	async hydrateWorkflowForInspection(workflowId: string): Promise<DurableWorkflowHydrationResult> {
+		if (this.locallyRegistered.has(workflowId)) {
+			const handle = this.getLoadableWorkflow(workflowId);
+			return handle === undefined ? { kind: "malformed" } : { kind: "current", handle };
 		}
+		const info = await this.sdk.retrieveWorkflow(workflowId);
+		if (info !== undefined) return await this.hydrateInfo(info);
 		const records = await this.sdk.listStepRecords(workflowId);
 		const deletion = classifyDbosDeletionTombstone(records, workflowId);
-		if (deletion !== "absent") await this.suppressWorkflow(workflowId);
+		if (deletion === "absent" && records.length === 0) return { kind: "absent" };
+		await this.suppressWorkflow(workflowId);
+		if (deletion === "absent") return { kind: "malformed" };
+		return { kind: deletion === "current" ? "deleted" : "malformed" };
 	}
 	async hydrateResumableWorkflows(): Promise<void> {
 		const all = await this.sdk.listAllWorkflows();
@@ -457,12 +464,17 @@ export class DbosDurableBackend implements DurableWorkflowBackend {
 		}
 	}
 
-	private async hydrateInfo(info: DbosWorkflowInfo): Promise<void> {
+	private async hydrateInfo(info: DbosWorkflowInfo): Promise<DurableWorkflowHydrationResult> {
 		const records = await this.sdk.listStepRecords(info.workflowId);
+		const deletion = classifyDbosDeletionTombstone(records, info.workflowId);
+		if (deletion !== "absent") {
+			await this.suppressWorkflow(info.workflowId);
+			return { kind: deletion === "current" ? "deleted" : "malformed" };
+		}
 		const metadata = classifyLatestMetadata(records, info.workflowId);
 		if (metadata.kind !== "current") {
 			await this.suppressWorkflow(info.workflowId);
-			return;
+			return { kind: "malformed" };
 		}
 		const checkpoints: DurableCheckpoint[] = [];
 		for (const record of records) {
@@ -475,7 +487,7 @@ export class DbosDurableBackend implements DurableWorkflowBackend {
 			const classified = classifyCheckpointPayload(info.workflowId, record.stepName, record.output);
 			if (classified.kind === "unknown") {
 				await this.suppressWorkflow(info.workflowId);
-				return;
+				return { kind: "malformed" };
 			}
 			checkpoints.push(classified.checkpoint);
 		}
@@ -485,22 +497,28 @@ export class DbosDurableBackend implements DurableWorkflowBackend {
 		this.applyMetadata(info.workflowId, metadata.metadata);
 		for (const checkpoint of checkpoints) this.mem.recordCheckpoint(checkpoint);
 		const current = this.mem.getWorkflow(info.workflowId);
-		if (current !== undefined) {
-			const pendingPrompts = this.promptReservations.hydrate(
-				info.workflowId,
-				metadata.metadata.pendingPrompts,
-				records,
-				metadata.metadata.promptReservationEpoch,
-			);
-			// Re-register instead of setWorkflowStatus: hydration is a read-side
-			// reconstruction and must preserve the authoritative updatedAt, which
-			// doubles as the cross-session liveness heartbeat for running handles.
-			this.applyMetadata(info.workflowId, {
-				...metadata.metadata,
-				pendingPrompts,
-				completedCheckpoints: current.completedCheckpoints,
-			});
+		if (current === undefined) {
+			await this.suppressWorkflow(info.workflowId);
+			return { kind: "malformed" };
 		}
+		const pendingPrompts = this.promptReservations.hydrate(
+			info.workflowId,
+			metadata.metadata.pendingPrompts,
+			records,
+			metadata.metadata.promptReservationEpoch,
+		);
+		// Re-register instead of setWorkflowStatus: hydration is a read-side
+		// reconstruction and must preserve the authoritative updatedAt, which
+		// doubles as the cross-session liveness heartbeat for running handles.
+		this.applyMetadata(info.workflowId, {
+			...metadata.metadata,
+			pendingPrompts,
+			completedCheckpoints: current.completedCheckpoints,
+		});
+		const handle = this.getLoadableWorkflow(info.workflowId);
+		if (handle !== undefined) return { kind: "current", handle };
+		await this.suppressWorkflow(info.workflowId);
+		return { kind: "malformed" };
 	}
 
 	private async suppressWorkflow(workflowId: string): Promise<void> {

package/dist/builtin/workflows/src/durable/dbos-envelope.ts

@@ -477,7 +477,8 @@ function isModelUsage(value: WorkflowSerializableValue): boolean {
 	if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
 	const usage = value as Record<string, WorkflowSerializableValue>;
 	return ["input", "output", "cacheRead", "cacheWrite", "cost", "turns"].every(
-		(key) => usage[key] === undefined || (typeof usage[key] === "number" && Number.isFinite(usage[key])),
+		(key) =>
+			usage[key] === undefined || (typeof usage[key] === "number" && Number.isFinite(usage[key]) && usage[key] >= 0),
 	);
 }
 

package/dist/builtin/workflows/src/durable/dbos-metadata.ts

@@ -59,6 +59,8 @@ export function encodeMetadata(metadata: DurableWorkflowMetadata): WorkflowSeria
 			...(metadata.label !== undefined ? { label: metadata.label } : {}),
 			...(metadata.rootWorkflowId !== undefined ? { rootWorkflowId: metadata.rootWorkflowId } : {}),
 			...(metadata.resumable !== undefined ? { resumable: metadata.resumable } : {}),
+			...(metadata.exited !== undefined ? { exited: metadata.exited } : {}),
+			...(metadata.exitReason !== undefined ? { exitReason: metadata.exitReason } : {}),
 			...(metadata.error !== undefined ? { error: metadata.error } : {}),
 			...(metadata.failureKind !== undefined ? { failureKind: metadata.failureKind } : {}),
 			...(metadata.failureCode !== undefined ? { failureCode: metadata.failureCode } : {}),
@@ -156,6 +158,8 @@ function parseDurableWorkflowMetadata(
 		(metadata.label !== undefined && typeof metadata.label !== "string") ||
 		(metadata.rootWorkflowId !== undefined && typeof metadata.rootWorkflowId !== "string") ||
 		(metadata.resumable !== undefined && typeof metadata.resumable !== "boolean") ||
+		(metadata.exited !== undefined && typeof metadata.exited !== "boolean") ||
+		(metadata.exitReason !== undefined && typeof metadata.exitReason !== "string") ||
 		(metadata.error !== undefined && typeof metadata.error !== "string") ||
 		(metadata.failureKind !== undefined && !isWorkflowFailureKind(metadata.failureKind)) ||
 		(metadata.failureCode !== undefined && !isWorkflowFailureCode(metadata.failureCode)) ||

package/dist/builtin/workflows/src/durable/local-command.ts

@@ -2,6 +2,7 @@
 
 import { spawn } from "node:child_process";
 import { connect } from "node:net";
+import { createChildProcessEnvironment } from "@bastani/atomic";
 
 export interface LocalCommandResult {
 	readonly exitCode: number;
@@ -27,7 +28,7 @@ export function runLocalCommand(
 		const child = spawn(command, [...args], {
 			stdio: ["ignore", "pipe", "pipe"],
 			windowsHide: true,
-			...(options?.env !== undefined ? { env: { ...process.env, ...options.env } } : {}),
+			env: createChildProcessEnvironment(options?.env ? { ...options.env } : undefined),
 			...(options?.uid !== undefined ? { uid: options.uid } : {}),
 			...(options?.gid !== undefined ? { gid: options.gid } : {}),
 		});

package/dist/builtin/workflows/src/durable/run-timing.ts

@@ -56,16 +56,39 @@ export function recordRunTimingCheckpoint(
 	run: RunSnapshot,
 	options?: { readonly debounce?: boolean; readonly now?: number },
 ): boolean {
+	const checkpoint = runTimingCheckpoint(backend, run, options);
+	if (checkpoint === undefined) return false;
+	backend.recordCheckpoint(checkpoint);
+	return true;
+}
+
+/** Await the timing write so an active turn observes persistent storage faults at once. */
+export async function recordRunTimingCheckpointAsync(
+	backend: DurableWorkflowBackend,
+	run: RunSnapshot,
+	options?: { readonly debounce?: boolean; readonly now?: number },
+): Promise<boolean> {
+	const checkpoint = runTimingCheckpoint(backend, run, options);
+	if (checkpoint === undefined) return false;
+	await backend.recordCheckpointAsync(checkpoint);
+	return true;
+}
+
+function runTimingCheckpoint(
+	backend: DurableWorkflowBackend,
+	run: RunSnapshot,
+	options?: { readonly debounce?: boolean; readonly now?: number },
+): DurableToolCheckpoint | undefined {
 	const now = options?.now ?? Date.now();
 	const elapsedMs = elapsedRunMs(run, now);
-	if (elapsedMs <= 0) return false;
-	if (backend.listCheckpoints(run.id).length === 0) return false;
+	if (elapsedMs <= 0) return undefined;
+	if (backend.listCheckpoints(run.id).length === 0) return undefined;
 	const recorded = priorRunElapsedMs(backend, run.id);
 	if (recorded !== undefined) {
-		if (elapsedMs <= recorded) return false;
-		if (options?.debounce === true && timingBucket(elapsedMs) === timingBucket(recorded)) return false;
+		if (elapsedMs <= recorded) return undefined;
+		if (options?.debounce === true && timingBucket(elapsedMs) === timingBucket(recorded)) return undefined;
 	}
-	const checkpoint: DurableToolCheckpoint = {
+	return {
 		kind: "tool",
 		workflowId: run.id,
 		checkpointId: `run-timing:${elapsedMs}`,
@@ -74,8 +97,6 @@ export function recordRunTimingCheckpoint(
 		output: { elapsedMs },
 		completedAt: now,
 	};
-	backend.recordCheckpoint(checkpoint);
-	return true;
 }
 
 /**

package/dist/builtin/workflows/src/durable/scoped-backend.ts

@@ -24,6 +24,7 @@ import type {
 	DurableInactiveDeleteResult,
 	DurableWorkflowBackend,
 	DurableWorkflowCatalogEntries,
+	DurableWorkflowHydrationResult,
 	WorkflowRegistrationInput,
 } from "./backend.js";
 import {
@@ -210,6 +211,9 @@ export class ScopedDurableBackend implements DurableWorkflowBackend {
 	hydrateWorkflow(_workflowId: string): Promise<void> {
 		return Promise.resolve();
 	}
+	hydrateWorkflowForInspection(_workflowId: string): Promise<DurableWorkflowHydrationResult> {
+		return Promise.resolve({ kind: "absent" });
+	}
 
 	hydrateResumableWorkflows(): Promise<void> {
 		return Promise.resolve();