@bastani/atomic 0.9.13-alpha.2 → 0.9.13-alpha.4

package/dist/builtin/workflows/src/durable/completed-catalog.ts

@@ -20,9 +20,14 @@ import {
 	validStageGroup,
 	workflowChildFromDraft,
 } from "./completed-catalog-stage-groups.js";
+import { isDurableWorkflowResumable } from "./resume-eligibility.js";
 import { resolveDurableEntry } from "./resume-runtime.js";
 import { priorRunElapsedMs, RUN_TIMING_CHECKPOINT_NAME } from "./run-timing.js";
-import { groupByDurableStageKey, immutableStageGroupError } from "./stage-topology-validation.js";
+import {
+	directChildTopologyError,
+	groupByDurableStageKey,
+	immutableStageGroupError,
+} from "./stage-topology-validation.js";
 import { workflowToolOutcomeFromValue } from "./tool-outcome.js";
 import {
 	DURABLE_STAGE_TOPOLOGY_VERSION,
@@ -94,9 +99,13 @@ export function completedWorkflowRunSnapshots(
 	if (handle === undefined || (handle.status !== "completed" && handle.status !== "failed")) return [];
 	const checkpoints = backend.listCheckpoints(entry.workflowId);
 	if (checkpoints.length === 0) return [];
-	const runs = runSnapshotsFromCheckpoints(checkpoints, handle.workflowId, handle.name, handle.updatedAt).map(
-		(run) => ({ ...run, stages: run.stages.map(validatedStageTranscript) }),
-	);
+	const runs = runSnapshotsFromCheckpoints(
+		checkpoints,
+		handle.workflowId,
+		handle.name,
+		handle.updatedAt,
+		COMPLETED_RECONSTRUCTION,
+	).map((run) => ({ ...run, stages: run.stages.map(validatedStageTranscript) }));
 	const rootIndex = runs.findIndex((run) => run.id === handle.workflowId);
 	if (rootIndex < 0) return [];
 	const rootDuration =
@@ -121,6 +130,59 @@ export function completedWorkflowRunSnapshots(
 	return [root, ...runs.filter((_, index) => index !== rootIndex)];
 }
 
+/** Rebuild one retained root, including in-progress nested runs, without relaxing topology validation. */
+export function durableWorkflowRunSnapshots(
+	backend: DurableWorkflowBackend,
+	handle: import("./types.js").DurableWorkflowHandle,
+): readonly RunSnapshot[] {
+	const checkpoints = backend.listCheckpoints(handle.workflowId);
+	if (checkpoints.length === 0) {
+		return [rootSnapshot(backend, handle, { stages: [], toolNodes: [] })];
+	}
+	const requiresTerminalTopology = handle.status === "completed" || handle.status === "failed";
+	const runs = runSnapshotsFromCheckpoints(
+		checkpoints,
+		handle.workflowId,
+		handle.name,
+		handle.updatedAt,
+		requiresTerminalTopology ? COMPLETED_RECONSTRUCTION : ACTIVE_INSPECTION_RECONSTRUCTION,
+	);
+	const rootIndex = runs.findIndex((run) => run.id === handle.workflowId);
+	if (rootIndex < 0) return [];
+	return [rootSnapshot(backend, handle, runs[rootIndex]!), ...runs.filter((_, index) => index !== rootIndex)];
+}
+
+function rootSnapshot(
+	backend: DurableWorkflowBackend,
+	handle: import("./types.js").DurableWorkflowHandle,
+	graph: Pick<RunSnapshot, "stages" | "toolNodes">,
+): RunSnapshot {
+	const terminal = handle.status === "completed" || handle.status === "failed" || handle.status === "cancelled";
+	const priorDurationMs = priorRunElapsedMs(backend, handle.workflowId);
+	const terminalDurationMs = priorDurationMs ?? Math.max(0, handle.updatedAt - handle.createdAt);
+	return {
+		id: handle.workflowId,
+		name: handle.name,
+		inputs: { ...handle.inputs } as WorkflowInputValues,
+		status: handle.status,
+		stages: graph.stages,
+		toolNodes: graph.toolNodes,
+		startedAt: handle.createdAt,
+		...(terminal
+			? { endedAt: handle.updatedAt, durationMs: terminalDurationMs }
+			: priorDurationMs === undefined
+				? {}
+				: { accumulatedDurationMs: priorDurationMs }),
+		...(handle.error !== undefined ? { error: handle.error } : {}),
+		...(handle.failureKind !== undefined ? { failureKind: handle.failureKind } : {}),
+		...(handle.failureCode !== undefined ? { failureCode: handle.failureCode } : {}),
+		...(handle.failureRecoverability !== undefined ? { failureRecoverability: handle.failureRecoverability } : {}),
+		...(handle.failureDisposition !== undefined ? { failureDisposition: handle.failureDisposition } : {}),
+		...(handle.failedToolNodeId !== undefined ? { failedToolNodeId: handle.failedToolNodeId } : {}),
+		resumable: isDurableWorkflowResumable(handle),
+	};
+}
+
 /** Rebuild completed nested runs while a paused root is replaying cached boundaries. */
 export function durableNestedRunSnapshots(
 	backend: DurableWorkflowBackend,
@@ -133,7 +195,7 @@ export function durableNestedRunSnapshots(
 		rootWorkflowId,
 		handle.name,
 		handle.updatedAt,
-		false,
+		LEGACY_ACTIVE_RECONSTRUCTION,
 	).filter((run) => run.id !== rootWorkflowId);
 }
 
@@ -224,14 +286,44 @@ function summarizeCompletedToolResult(value: WorkflowSerializableValue): string
 	return serialized.length <= 240 ? serialized : `${flattenTruncatedString(serialized.slice(0, 237))}...`;
 }
 
+interface ReconstructionPolicy {
+	readonly requireTerminal: boolean;
+	readonly failClosed: boolean;
+	readonly retainActive: boolean;
+	readonly failOnUnreachable: boolean;
+	readonly requireActiveBoundaryFingerprint: boolean;
+}
+
+const COMPLETED_RECONSTRUCTION: ReconstructionPolicy = {
+	requireTerminal: true,
+	failClosed: true,
+	retainActive: false,
+	failOnUnreachable: false,
+	requireActiveBoundaryFingerprint: false,
+};
+const ACTIVE_INSPECTION_RECONSTRUCTION: ReconstructionPolicy = {
+	requireTerminal: false,
+	failClosed: true,
+	retainActive: true,
+	failOnUnreachable: true,
+	requireActiveBoundaryFingerprint: true,
+};
+const LEGACY_ACTIVE_RECONSTRUCTION: ReconstructionPolicy = {
+	requireTerminal: false,
+	failClosed: false,
+	retainActive: true,
+	failOnUnreachable: false,
+	requireActiveBoundaryFingerprint: false,
+};
+
 function runSnapshotsFromCheckpoints(
 	checkpoints: readonly DurableCheckpoint[],
 	rootRunId: string,
 	rootRunName: string,
 	fallbackCompletedAt: number,
-	strict = true,
+	policy: ReconstructionPolicy = COMPLETED_RECONSTRUCTION,
 ): RunSnapshot[] {
-	if (!validBoundaryRecordSet(checkpoints, strict)) return [];
+	if (!validBoundaryRecordSet(checkpoints, policy.requireTerminal, policy.requireActiveBoundaryFingerprint)) return [];
 	const stageRecords = checkpoints.filter(
 		(checkpoint): checkpoint is DurableStageCheckpoint => checkpoint.kind === "stage",
 	);
@@ -248,7 +340,10 @@ function runSnapshotsFromCheckpoints(
 		.map((records) => mergeStageGroup(records, stageRecords, checkpoints))
 		.filter(
 			(draft) =>
-				!strict || draft.topology?.run === undefined || draft.endedAt !== undefined || draft.output !== undefined,
+				policy.retainActive ||
+				draft.topology?.run === undefined ||
+				draft.endedAt !== undefined ||
+				draft.output !== undefined,
 		)
 		.map(
 			(draft): StageDraft =>
@@ -267,14 +362,46 @@ function runSnapshotsFromCheckpoints(
 		.sort(compareDraftSourceOrder);
 	const drafts = checkpointDrafts(checkpoints);
 	const toolCheckpoints = drafts.tools;
+	if (policy.failClosed) {
+		for (const draft of candidates) {
+			const boundary = draft.topology?.boundary;
+			if (boundary === undefined || childRunIdFromDraft(draft) === undefined) continue;
+			if (boundary.replayScope !== draft.replayKey) return [];
+			const expectedChildRun = {
+				runId: boundary.child.runId,
+				runName: boundary.child.runName,
+				parentRunId: boundary.child.parentRunId,
+				parentStageId: boundary.child.parentStageId,
+				rootRunId: boundary.child.rootRunId,
+			};
+			const childToolIds = new Set(
+				toolCheckpoints.flatMap((checkpoint) =>
+					checkpoint.topology?.run?.runId === expectedChildRun.runId ? [checkpoint.topology.nodeId] : [],
+				),
+			);
+			if (
+				directChildTopologyError(stageRecords, boundary.replayScope, expectedChildRun, true, childToolIds) !==
+				undefined
+			)
+				return [];
+		}
+	}
 	if (candidates.length === 0 && toolCheckpoints.length === 0) {
-		return strict ? [] : [syntheticRun(rootRunId, rootRunName, checkpoints.length, fallbackCompletedAt)];
+		const supportedNonGraphState = checkpoints.every(
+			(checkpoint) =>
+				checkpoint.kind === "ui" ||
+				(checkpoint.kind === "tool" && checkpoint.argsHash === RUN_TIMING_CHECKPOINT_NAME),
+		);
+		if (policy.failClosed && supportedNonGraphState) {
+			return [emptyGraphRun(rootRunId, rootRunName, fallbackCompletedAt)];
+		}
+		return policy.failClosed ? [] : [syntheticRun(rootRunId, rootRunName, checkpoints.length, fallbackCompletedAt)];
 	}
 
 	const grouped = new Map<string, StageDraft[]>();
 	for (const draft of candidates) {
 		if (draft.topology?.version !== DURABLE_STAGE_TOPOLOGY_VERSION) {
-			if (strict) return [];
+			if (policy.failClosed) return [];
 			continue;
 		}
 		const runId = draft.topology.run?.runId ?? rootRunId;
@@ -287,7 +414,9 @@ function runSnapshotsFromCheckpoints(
 		if (!grouped.has(runId)) grouped.set(runId, []);
 	}
 	for (const group of grouped.values()) group.sort(compareDraftSourceOrder);
+	const groupedCount = grouped.size;
 	retainReachableRunGroups(grouped, rootRunId);
+	if (policy.failOnUnreachable && grouped.size !== groupedCount) return [];
 	if (!validateRunGroups(grouped, rootRunId, toolCheckpoints, checkpoints)) return [];
 	const syntheticChildren: Array<{
 		readonly parentRunId: string;
@@ -302,7 +431,7 @@ function runSnapshotsFromCheckpoints(
 				!isSyntheticExitedChild(draft, parentRunId, child.runId, rootRunId) ||
 				childScopedEvidenceExists(checkpoints, draft.replayKey, child.runId)
 			) {
-				if (strict) return [];
+				if (policy.failClosed) return [];
 				continue;
 			}
 			syntheticChildren.push({ parentRunId, owner: draft, child });
@@ -311,14 +440,10 @@ function runSnapshotsFromCheckpoints(
 
 	const idMaps = new Map<string, Map<string, string>>();
 	for (const [runId, runDrafts] of grouped) {
-		const ownedToolIds = new Set(
-			toolCheckpoints.flatMap((checkpoint) =>
-				(checkpoint.topology?.run?.runId ?? rootRunId) === runId
-					? [checkpoint.topology?.nodeId ?? checkpoint.checkpointId]
-					: [],
-			),
+		const ownedTools = toolCheckpoints.filter(
+			(checkpoint) => (checkpoint.topology?.run?.runId ?? rootRunId) === runId,
 		);
-		if (!validStageGroup(runDrafts, runId, ownedToolIds)) return [];
+		if (!validStageGroup(runDrafts, runId, rootRunId, ownedTools)) return [];
 		const ids = new Map<string, string>();
 		let hasIdentityConflict = false;
 		runDrafts.forEach((draft) => {
@@ -331,7 +456,7 @@ function runSnapshotsFromCheckpoints(
 			}
 		});
 		if (hasIdentityConflict) {
-			if (strict) return [];
+			if (policy.failClosed) return [];
 			grouped.delete(runId);
 			continue;
 		}
@@ -356,7 +481,7 @@ function runSnapshotsFromCheckpoints(
 				(checkpoint) => checkpoint.topology?.parentIds.some((parentId) => !ids.has(parentId)) === true,
 			);
 		if (hasUnknownParents) {
-			if (strict) return [];
+			if (policy.failClosed) return [];
 			continue;
 		}
 		const stages = runDrafts.map((draft) =>
@@ -377,6 +502,8 @@ function runSnapshotsFromCheckpoints(
 			...toolNodes.map((tool) => tool.endedAt ?? fallbackCompletedAt),
 		);
 		const owner = runId === rootRunId ? undefined : boundaryOwner(grouped, runId);
+		const status = owner === undefined ? "completed" : childRunStatus(owner);
+		const terminal = status !== "running" && status !== "pending" && status !== "paused" && status !== "blocked";
 		const parentRunId = run?.parentRunId ?? rootRunId;
 		const boundarySourceId = grouped.get(parentRunId)?.find((draft) => childRunIdFromDraft(draft) === runId)
 			?.topology?.stageId;
@@ -390,12 +517,11 @@ function runSnapshotsFromCheckpoints(
 			id: runId,
 			name: run?.runName ?? rootRunName,
 			inputs: {},
-			status: owner === undefined ? "completed" : childRunStatus(owner),
+			status,
 			stages,
 			toolNodes,
 			startedAt,
-			endedAt,
-			durationMs: Math.max(0, endedAt - startedAt),
+			...(terminal ? { endedAt, durationMs: Math.max(0, endedAt - startedAt) } : {}),
 			...(run?.parentRunId !== undefined ? { parentRunId: run.parentRunId } : {}),
 			...(parentStageId !== undefined ? { parentStageId } : {}),
 			...(run?.rootRunId !== undefined ? { rootRunId: run.rootRunId } : {}),
@@ -412,14 +538,14 @@ function runSnapshotsFromCheckpoints(
 	}
 	for (const { parentRunId, owner, child } of syntheticChildren) {
 		if (!emittedRunIds.has(parentRunId)) {
-			if (strict) return [];
+			if (policy.failClosed) return [];
 			continue;
 		}
 		const topology = owner.topology!;
 		const boundary = topology.boundary!;
 		const parentStageId = idMaps.get(parentRunId)?.get(topology.stageId);
 		if (parentStageId === undefined) {
-			if (strict) return [];
+			if (policy.failClosed) return [];
 			continue;
 		}
 		const startedAt = owner.startedAt ?? owner.firstCompletedAt;
@@ -490,6 +616,21 @@ function stageResult(draft: StageDraft): string | undefined {
 	return typeof draft.output === "string" ? draft.output : JSON.stringify(draft.output);
 }
 
+function emptyGraphRun(runId: string, runName: string, completedAt: number): RunSnapshot {
+	return {
+		id: runId,
+		name: runName,
+		inputs: {},
+		status: "completed",
+		stages: [],
+		toolNodes: [],
+		startedAt: completedAt,
+		endedAt: completedAt,
+		durationMs: 0,
+		resumable: false,
+	};
+}
+
 function syntheticRun(runId: string, runName: string, checkpointCount: number, completedAt: number): RunSnapshot {
 	return {
 		id: runId,

package/dist/builtin/workflows/src/durable/dbos-backend.ts

@@ -5,6 +5,7 @@ import {
 	type DurableInactiveDeleteResult,
 	type DurableWorkflowBackend,
 	type DurableWorkflowCatalogEntries,
+	type DurableWorkflowHydrationResult,
 	InMemoryDurableBackend,
 	type WorkflowRegistrationInput,
 } from "./backend.js";
@@ -439,15 +440,21 @@ export class DbosDurableBackend implements DurableWorkflowBackend {
 	}
 
 	async hydrateWorkflow(workflowId: string): Promise<void> {
-		if (this.locallyRegistered.has(workflowId)) return;
-		const info = await this.sdk.retrieveWorkflow(workflowId);
-		if (info !== undefined) {
-			await this.hydrateInfo(info);
-			return;
+		await this.hydrateWorkflowForInspection(workflowId);
+	}
+	async hydrateWorkflowForInspection(workflowId: string): Promise<DurableWorkflowHydrationResult> {
+		if (this.locallyRegistered.has(workflowId)) {
+			const handle = this.getLoadableWorkflow(workflowId);
+			return handle === undefined ? { kind: "malformed" } : { kind: "current", handle };
 		}
+		const info = await this.sdk.retrieveWorkflow(workflowId);
+		if (info !== undefined) return await this.hydrateInfo(info);
 		const records = await this.sdk.listStepRecords(workflowId);
 		const deletion = classifyDbosDeletionTombstone(records, workflowId);
-		if (deletion !== "absent") await this.suppressWorkflow(workflowId);
+		if (deletion === "absent" && records.length === 0) return { kind: "absent" };
+		await this.suppressWorkflow(workflowId);
+		if (deletion === "absent") return { kind: "malformed" };
+		return { kind: deletion === "current" ? "deleted" : "malformed" };
 	}
 	async hydrateResumableWorkflows(): Promise<void> {
 		const all = await this.sdk.listAllWorkflows();
@@ -457,12 +464,17 @@ export class DbosDurableBackend implements DurableWorkflowBackend {
 		}
 	}
 
-	private async hydrateInfo(info: DbosWorkflowInfo): Promise<void> {
+	private async hydrateInfo(info: DbosWorkflowInfo): Promise<DurableWorkflowHydrationResult> {
 		const records = await this.sdk.listStepRecords(info.workflowId);
+		const deletion = classifyDbosDeletionTombstone(records, info.workflowId);
+		if (deletion !== "absent") {
+			await this.suppressWorkflow(info.workflowId);
+			return { kind: deletion === "current" ? "deleted" : "malformed" };
+		}
 		const metadata = classifyLatestMetadata(records, info.workflowId);
 		if (metadata.kind !== "current") {
 			await this.suppressWorkflow(info.workflowId);
-			return;
+			return { kind: "malformed" };
 		}
 		const checkpoints: DurableCheckpoint[] = [];
 		for (const record of records) {
@@ -475,7 +487,7 @@ export class DbosDurableBackend implements DurableWorkflowBackend {
 			const classified = classifyCheckpointPayload(info.workflowId, record.stepName, record.output);
 			if (classified.kind === "unknown") {
 				await this.suppressWorkflow(info.workflowId);
-				return;
+				return { kind: "malformed" };
 			}
 			checkpoints.push(classified.checkpoint);
 		}
@@ -485,22 +497,28 @@ export class DbosDurableBackend implements DurableWorkflowBackend {
 		this.applyMetadata(info.workflowId, metadata.metadata);
 		for (const checkpoint of checkpoints) this.mem.recordCheckpoint(checkpoint);
 		const current = this.mem.getWorkflow(info.workflowId);
-		if (current !== undefined) {
-			const pendingPrompts = this.promptReservations.hydrate(
-				info.workflowId,
-				metadata.metadata.pendingPrompts,
-				records,
-				metadata.metadata.promptReservationEpoch,
-			);
-			// Re-register instead of setWorkflowStatus: hydration is a read-side
-			// reconstruction and must preserve the authoritative updatedAt, which
-			// doubles as the cross-session liveness heartbeat for running handles.
-			this.applyMetadata(info.workflowId, {
-				...metadata.metadata,
-				pendingPrompts,
-				completedCheckpoints: current.completedCheckpoints,
-			});
+		if (current === undefined) {
+			await this.suppressWorkflow(info.workflowId);
+			return { kind: "malformed" };
 		}
+		const pendingPrompts = this.promptReservations.hydrate(
+			info.workflowId,
+			metadata.metadata.pendingPrompts,
+			records,
+			metadata.metadata.promptReservationEpoch,
+		);
+		// Re-register instead of setWorkflowStatus: hydration is a read-side
+		// reconstruction and must preserve the authoritative updatedAt, which
+		// doubles as the cross-session liveness heartbeat for running handles.
+		this.applyMetadata(info.workflowId, {
+			...metadata.metadata,
+			pendingPrompts,
+			completedCheckpoints: current.completedCheckpoints,
+		});
+		const handle = this.getLoadableWorkflow(info.workflowId);
+		if (handle !== undefined) return { kind: "current", handle };
+		await this.suppressWorkflow(info.workflowId);
+		return { kind: "malformed" };
 	}
 
 	private async suppressWorkflow(workflowId: string): Promise<void> {

package/dist/builtin/workflows/src/durable/dbos-envelope.ts

@@ -477,7 +477,8 @@ function isModelUsage(value: WorkflowSerializableValue): boolean {
 	if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
 	const usage = value as Record<string, WorkflowSerializableValue>;
 	return ["input", "output", "cacheRead", "cacheWrite", "cost", "turns"].every(
-		(key) => usage[key] === undefined || (typeof usage[key] === "number" && Number.isFinite(usage[key])),
+		(key) =>
+			usage[key] === undefined || (typeof usage[key] === "number" && Number.isFinite(usage[key]) && usage[key] >= 0),
 	);
 }
 

package/dist/builtin/workflows/src/durable/run-timing.ts

@@ -56,16 +56,39 @@ export function recordRunTimingCheckpoint(
 	run: RunSnapshot,
 	options?: { readonly debounce?: boolean; readonly now?: number },
 ): boolean {
+	const checkpoint = runTimingCheckpoint(backend, run, options);
+	if (checkpoint === undefined) return false;
+	backend.recordCheckpoint(checkpoint);
+	return true;
+}
+
+/** Await the timing write so an active turn observes persistent storage faults at once. */
+export async function recordRunTimingCheckpointAsync(
+	backend: DurableWorkflowBackend,
+	run: RunSnapshot,
+	options?: { readonly debounce?: boolean; readonly now?: number },
+): Promise<boolean> {
+	const checkpoint = runTimingCheckpoint(backend, run, options);
+	if (checkpoint === undefined) return false;
+	await backend.recordCheckpointAsync(checkpoint);
+	return true;
+}
+
+function runTimingCheckpoint(
+	backend: DurableWorkflowBackend,
+	run: RunSnapshot,
+	options?: { readonly debounce?: boolean; readonly now?: number },
+): DurableToolCheckpoint | undefined {
 	const now = options?.now ?? Date.now();
 	const elapsedMs = elapsedRunMs(run, now);
-	if (elapsedMs <= 0) return false;
-	if (backend.listCheckpoints(run.id).length === 0) return false;
+	if (elapsedMs <= 0) return undefined;
+	if (backend.listCheckpoints(run.id).length === 0) return undefined;
 	const recorded = priorRunElapsedMs(backend, run.id);
 	if (recorded !== undefined) {
-		if (elapsedMs <= recorded) return false;
-		if (options?.debounce === true && timingBucket(elapsedMs) === timingBucket(recorded)) return false;
+		if (elapsedMs <= recorded) return undefined;
+		if (options?.debounce === true && timingBucket(elapsedMs) === timingBucket(recorded)) return undefined;
 	}
-	const checkpoint: DurableToolCheckpoint = {
+	return {
 		kind: "tool",
 		workflowId: run.id,
 		checkpointId: `run-timing:${elapsedMs}`,
@@ -74,8 +97,6 @@ export function recordRunTimingCheckpoint(
 		output: { elapsedMs },
 		completedAt: now,
 	};
-	backend.recordCheckpoint(checkpoint);
-	return true;
 }
 
 /**

package/dist/builtin/workflows/src/durable/scoped-backend.ts

@@ -24,6 +24,7 @@ import type {
 	DurableInactiveDeleteResult,
 	DurableWorkflowBackend,
 	DurableWorkflowCatalogEntries,
+	DurableWorkflowHydrationResult,
 	WorkflowRegistrationInput,
 } from "./backend.js";
 import {
@@ -210,6 +211,9 @@ export class ScopedDurableBackend implements DurableWorkflowBackend {
 	hydrateWorkflow(_workflowId: string): Promise<void> {
 		return Promise.resolve();
 	}
+	hydrateWorkflowForInspection(_workflowId: string): Promise<DurableWorkflowHydrationResult> {
+		return Promise.resolve({ kind: "absent" });
+	}
 
 	hydrateResumableWorkflows(): Promise<void> {
 		return Promise.resolve();

package/dist/builtin/workflows/src/durable/targeted-inspection.ts

@@ -0,0 +1,110 @@
+import { inspectRun, type RunDetail } from "../runs/background/run-inspect.js";
+import { createStore, type Store } from "../shared/store.js";
+import type { RunSnapshot } from "../shared/store-types.js";
+import type { DurableWorkflowBackend } from "./backend.js";
+import { durableWorkflowRunSnapshots } from "./completed-catalog.js";
+import { getAtomicExecutorId } from "./dbos-sdk-handle.js";
+import { isForeignLiveWorkflow, isLiveRunningWorkflow } from "./resume-eligibility.js";
+
+export type TargetedDurableInspection =
+	| {
+			readonly kind: "found";
+			readonly detail: RunDetail;
+			readonly runs: readonly RunSnapshot[];
+			readonly store: Store;
+	  }
+	| { readonly kind: "absent" | "deleted" | "malformed"; readonly message: string };
+
+/**
+ * Hydrate and reconstruct one exact durable root for inspection only.
+ *
+ * This path never claims ownership, changes durable status, starts execution,
+ * or restores snapshots into the current session store.
+ */
+export async function inspectTargetedDurableWorkflow(
+	backend: DurableWorkflowBackend,
+	workflowId: string,
+	now: number = Date.now(),
+): Promise<TargetedDurableInspection> {
+	const hydrated = backend.hydrateWorkflowForInspection
+		? await backend.hydrateWorkflowForInspection(workflowId)
+		: await hydrateWithoutClassification(backend, workflowId);
+	if (hydrated.kind !== "current") return hydrationFailure(workflowId, hydrated.kind);
+	const handle = hydrated.handle;
+	if (handle.rootWorkflowId !== undefined && handle.rootWorkflowId !== handle.workflowId) {
+		return malformed(workflowId, "the requested id belongs to a nested workflow rather than a durable root");
+	}
+	const reconstructed = durableWorkflowRunSnapshots(backend, handle);
+	if (reconstructed.length === 0)
+		return malformed(workflowId, "durable checkpoint topology is malformed or incomplete");
+	const runs = structuredClone(reconstructed);
+
+	const inspectionStore = createStore();
+	for (const run of runs) inspectionStore.recordRunStart(structuredClone(run));
+	const inspected = inspectRun(workflowId, { store: inspectionStore });
+	if (!inspected.ok) return malformed(workflowId, "the durable checkpoint graph has no reciprocal root");
+
+	const foreignLive = isForeignLiveWorkflow(handle, getAtomicExecutorId(), now);
+	const live = isLiveRunningWorkflow(handle, now);
+	const crashed = handle.status === "running" && !live;
+	const resumeGuidance = crashed
+		? inspected.detail.resumable === true
+			? `This workflow appears to have crashed and is resumable. Resume it explicitly with /workflow resume ${workflowId}.`
+			: "This workflow appears to have crashed, but its retained state is not resumable."
+		: foreignLive
+			? "This workflow is actively running in another Atomic session. Inspect it here, but control it from its owner session."
+			: live
+				? "This workflow still has a fresh durable heartbeat. Inspect it here without starting another executor."
+				: terminalGuidance(handle.status, workflowId, inspected.detail.resumable === true);
+	return {
+		kind: "found",
+		runs,
+		store: inspectionStore,
+		detail: {
+			...inspected.detail,
+			...(crashed ? { status: "crashed" as const } : {}),
+			...(live ? { resumable: false } : {}),
+			...(foreignLive ? { ownerActiveElsewhere: true } : {}),
+			resumeGuidance,
+		},
+	};
+}
+
+function terminalGuidance(status: string, workflowId: string, resumable: boolean): string {
+	if (resumable)
+		return `This retained workflow is ${status} and resumable. Resume it explicitly with /workflow resume ${workflowId}.`;
+	return `This retained workflow is ${status} and available for read-only inspection.`;
+}
+
+async function hydrateWithoutClassification(
+	backend: DurableWorkflowBackend,
+	workflowId: string,
+): Promise<import("./backend.js").DurableWorkflowHydrationResult> {
+	await backend.hydrateWorkflow(workflowId);
+	const handle = backend.getLoadableWorkflow(workflowId);
+	return handle === undefined ? { kind: "absent" } : { kind: "current", handle };
+}
+
+function hydrationFailure(
+	workflowId: string,
+	kind: Exclude<TargetedDurableInspection["kind"], "found">,
+): TargetedDurableInspection {
+	switch (kind) {
+		case "absent":
+			return {
+				kind,
+				message: `Run not found: ${workflowId} (not present in the current session or durable store).`,
+			};
+		case "deleted":
+			return { kind, message: `Durable workflow was deleted: ${workflowId}` };
+		case "malformed":
+			return malformed(workflowId, "current DBOS metadata or checkpoints are malformed");
+	}
+}
+
+function malformed(workflowId: string, reason: string): TargetedDurableInspection {
+	return {
+		kind: "malformed",
+		message: `Durable workflow ${workflowId} cannot be inspected safely: ${reason}.`,
+	};
+}

package/dist/builtin/workflows/src/engine/run-durable-stage-session.ts

@@ -1,4 +1,4 @@
-import { recordRunTimingCheckpoint } from "../durable/run-timing.js";
+import { recordRunTimingCheckpointAsync } from "../durable/run-timing.js";
 import { type DurableStageDeps, recordStageSessionCheckpoint } from "../durable/stage-primitive.js";
 import type { StageSessionCheckpointOptions } from "../runs/foreground/executor-types.js";
 import type { RunSnapshot, StageSnapshot } from "../shared/store-types.js";
@@ -27,7 +27,7 @@ export function createDurableStageSessionRecorder(
 		if (stageRunId === input.runId) {
 			await recordStageSessionCheckpoint(input.deps, snapshot, { force: options?.forceDurable === true });
 			if (input.runSnapshot !== undefined) {
-				recordRunTimingCheckpoint(input.deps.backend, input.runSnapshot, {
+				await recordRunTimingCheckpointAsync(input.deps.backend, input.runSnapshot, {
 					debounce: options?.forceDurable !== true,
 				});
 			}

package/dist/builtin/workflows/src/engine/runtime.ts

@@ -180,8 +180,25 @@ export class EngineRuntime {
 		return {
 			apply: () => {
 				if (!this.childRunOptions.mcp || !hasScope) return;
-				if (depth === 0) this.childRunOptions.mcp.setScope(stageId, allow, deny);
-				depth += 1;
+				if (depth > 0) {
+					depth += 1;
+					return;
+				}
+				depth = 1;
+				try {
+					this.childRunOptions.mcp.setScope(stageId, allow, deny);
+				} catch (error) {
+					depth = 0;
+					try {
+						this.childRunOptions.mcp.clearScope(stageId);
+					} catch (cleanupError) {
+						throw new AggregateError(
+							[error, cleanupError],
+							"MCP scope setup failed and compensating cleanup failed",
+						);
+					}
+					throw error;
+				}
 			},
 			clear: () => {
 				if (!this.childRunOptions.mcp || !hasScope) return;

package/dist/builtin/workflows/src/extension/extension-runtime-state.ts

@@ -159,6 +159,9 @@ export function createWorkflowExtensionRuntimeState(
 		resumeDurableWorkflow(workflowId, options) {
 			return runtimeRef.current.resumeDurableWorkflow(workflowId, options);
 		},
+		inspectDurableWorkflow(workflowId) {
+			return runtimeRef.current.inspectDurableWorkflow(workflowId);
+		},
 		listDurableResumable() {
 			return runtimeRef.current.listDurableResumable();
 		},