@maxiaochao/pi-toolkit 0.10.20 → 0.10.22

package/packages/herdr-subagents/src/blocking.ts

@@ -1,6 +1,5 @@
 import { existsSync, readdirSync } from "node:fs";
-import { mkdir, readFile, rename, unlink, writeFile } from "node:fs/promises";
-import { randomUUID } from "node:crypto";
+import { mkdir, rename } from "node:fs/promises";
 import { join } from "node:path";
 import { CliHerdrAutomation, type HerdrAutomation } from "./herdr.ts";
 import { continueQueuedRun } from "./continue-run.ts";
@@ -8,20 +7,23 @@ import { liveAgentName, promptLiveAgent } from "./live-agent.ts";
 import { writeJsonAtomic, writeTextAtomic } from "./state.ts";
 import { cleanupRunTab } from "./tab-cleanup.ts";
 import type { OwnerIdentity } from "./ownership.ts";
-import { isProcessAlive, ownerRecord } from "./ownership.ts";
-import { aggregateTaskStatus, applyRunStatus } from "./run-status.ts";
+import { ownerRecord } from "./ownership.ts";
+import { applyRunStatus } from "./run-status.ts";
 import { applyReport, applyTaskStatus, isActiveStatus, persistFailureReport, projectTaskActivity, readReport, readRunRecord, readTaskRecord, readTurnRecord, settleAttemptFailure, type PersistedDocument, type PersistedReport, type RunRecord, type TaskActivity, type TaskRecord, type TaskStatus, type TurnRecord } from "./records.ts";
 import { isAbortError } from "./errors.ts";
 import { continuationSystemPrompt } from "./protocol.ts";
 import { RunPaneAllocator } from "./run-pane-allocator.ts";
 import { findRunDirectory } from "./run-locator.ts";
+import { acquireDirectoryLock, withRunStateLock } from "./run-state.ts";
+import { createAttemptId } from "./attempts.ts";
+import { aggregateTasksWithResume, claimResumeCleanup } from "./resume-state.ts";
 
-export interface RespondToBlockedTaskOptions { herdr?: HerdrAutomation; taskNumber?: number; signal?: AbortSignal; owner?: OwnerIdentity; onCleanupError?: (message: string) => void; onWarning?: (message: string) => void; onStalled?: (message: string) => void; onUpdate?: (result: ResumeBlockedResult) => void; }
+export interface RespondToBlockedTaskOptions { herdr?: HerdrAutomation; taskNumber: number; signal?: AbortSignal; owner?: OwnerIdentity; onCleanupError?: (message: string) => void; onWarning?: (message: string) => void; onStalled?: (message: string) => void; onUpdate?: (result: ResumeBlockedResult) => void; }
 export interface ResumeBlockedResult {
 	ok: boolean;
 	runId: string;
 	label: string;
-	status: "blocked" | "completed" | "partial" | "failed" | "cancelled";
+	status: "running" | "blocked" | "completed" | "partial" | "failed" | "cancelled";
 	summary: string;
 	question?: string;
 	documents: PersistedDocument[];
@@ -33,34 +35,18 @@ export interface ResumeBlockedResult {
 	tabId?: string;
 }
 
-interface ResponseLock { token?: string; processId?: unknown; sessionId?: unknown; createdAt?: unknown; }
-
-async function readResponseLock(lockFile: string): Promise<ResponseLock | undefined> {
-	try {
-		const lock = JSON.parse(await readFile(lockFile, "utf8")) as unknown;
-		return lock && typeof lock === "object" ? lock as ResponseLock : undefined;
-	} catch { return undefined; }
-}
-
-async function releaseResponseLock(lockFile: string, token: string): Promise<void> {
-	if ((await readResponseLock(lockFile))?.token !== token) return;
-	try { await unlink(lockFile); }
-	catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; }
+/**
+ * A blocked task still owns a live child waiting for an answer. A settled failed task can only
+ * continue from its saved Pi session.
+ */
+function isContinuable(task: TaskRecord, taskNumber: number): boolean {
+	if (task.order !== taskNumber) return false;
+	if (task.status === "blocked") return true;
+	return task.status === "failed" && existsSync(task.sessionFile ?? "");
 }
 
-async function acquireResponseLock(lockFile: string, runId: string, owner?: OwnerIdentity): Promise<string> {
-	const token = randomUUID();
-	try {
-		await writeFile(lockFile, `${JSON.stringify({ token, processId: process.pid, sessionId: owner?.sessionId, createdAt: new Date().toISOString() })}\n`, { flag: "wx", mode: 0o600 });
-		return token;
-	} catch (error) {
-		if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
-		const lock = await readResponseLock(lockFile);
-		if (lock && Number.isInteger(lock.processId) && !isProcessAlive(lock.processId)) {
-			throw new Error(`Subagent run '${runId}' has a stale blocked-response lock from process ${lock.processId}; remove ${lockFile} and retry.`);
-		}
-		throw new Error(`Subagent run '${runId}' already has a blocked response in progress (lock: ${lockFile}).`);
-	}
+function noContinuableTaskError(runId: string, taskNumber: number): Error {
+	return new Error(`Run '${runId}' task ${taskNumber} is neither blocked nor a failed task with a saved Pi session.`);
 }
 
 async function emitProgress(options: RespondToBlockedTaskOptions, runDirectory: string, run: RunRecord, taskDirectories: string[], summary: string): Promise<void> {
@@ -82,7 +68,26 @@ async function emitProgress(options: RespondToBlockedTaskOptions, runDirectory:
 }
 
 async function projectSettledRun(runDirectory: string, runFile: string, run: RunRecord, taskDirectories: string[], herdr: HerdrAutomation, onCleanupError?: (message: string) => void): Promise<ResumeBlockedResult> {
-	const finalTasks = await Promise.all(taskDirectories.map((directory) => readTaskRecord(join(directory, "task.json"))));
+	const settledState = await withRunStateLock(runFile, undefined, async (latest) => {
+		const tasks = await Promise.all(taskDirectories.map((directory) => readTaskRecord(join(directory, "task.json"))));
+		if (latest.resume) {
+			for (const task of tasks) {
+				const member = latest.resume.active[task.id];
+				if (!member || member.attemptId !== task.currentAttemptId) continue;
+				if (task.status === "blocked") member.phase = "blocked";
+				else if (!isActiveStatus(task.status)) delete latest.resume.active[task.id];
+			}
+		}
+		const aggregated = aggregateTasksWithResume(tasks, latest.resume?.active);
+		const blockedTask = tasks.find((task) => task.status === "blocked");
+		// Queued work aggregates as blocked; with no blocked task the run is simply still running.
+		const status = aggregated === "blocked" && !blockedTask ? "running" : aggregated;
+		applyRunStatus(latest, status, { question: blockedTask?.question });
+		const cleanup = status === "running" || status === "blocked" ? undefined : claimResumeCleanup(latest)
+			?? (!latest.resume && latest.tabId ? { tabId: latest.tabId, generation: undefined } : undefined);
+		return { run: latest, tasks, status, cleanup };
+	});
+	let finalTasks = settledState.tasks;
 	const finalReports = await Promise.all(taskDirectories.map(async (directory, index) => {
 		const reportFile = join(directory, "report.json");
 		if (!existsSync(reportFile) && (finalTasks[index].status === "queued" || finalTasks[index].status === "cancelled")) return { status: "failed" as const, summary: finalTasks[index].error ?? (finalTasks[index].status === "queued" ? "Task is still queued." : "Task was cancelled."), documents: [] };
@@ -95,15 +100,17 @@ async function projectSettledRun(runDirectory: string, runFile: string, run: Run
 			return failedReport;
 		}
 	}));
-	const aggregateStatus = aggregateTaskStatus(finalTasks.map((entry) => entry.status));
-	const runStatus = aggregateStatus === "running" ? "blocked" : aggregateStatus;
+	const runStatus = settledState.status;
 	const blockedIndex = finalTasks.findIndex((entry) => entry.status === "blocked");
 	const blockedReport = blockedIndex >= 0 ? finalReports[blockedIndex] : undefined;
-	run = await readRunRecord(runFile);
-	applyRunStatus(run, runStatus, { question: blockedReport?.question });
-	await writeJsonAtomic(runFile, run);
-	if (runStatus !== "blocked" && run.tabId) await cleanupRunTab({ herdr, tabId: run.tabId, runFile, runRecord: run, onError: onCleanupError });
-	const summary = runStatus === "completed" ? "All tasks completed after the answer." : runStatus === "blocked" ? `${finalTasks[blockedIndex]?.name ?? "Task"} needs input: ${blockedReport?.question ?? "Input required."}` : finalReports.map((entry, index) => `${finalTasks[index].name}: ${entry.summary}`).join("\n");
+	run = settledState.run;
+	if (settledState.cleanup) await cleanupRunTab({
+		herdr, tabId: settledState.cleanup.tabId, runFile, runRecord: run, onError: onCleanupError,
+		onClosed: (latest) => {
+			if (settledState.cleanup?.generation !== undefined && latest.resume?.generation === settledState.cleanup.generation && latest.resume.phase === "closing") delete latest.resume;
+		},
+	});
+	const summary = runStatus === "completed" ? "All tasks completed after the answer." : runStatus === "running" ? "Other resumed tasks are still running." : runStatus === "blocked" ? `${finalTasks[blockedIndex]?.name ?? "Task"} needs input: ${blockedReport?.question ?? "Input required."}` : finalReports.map((entry, index) => `${finalTasks[index].name}: ${entry.summary}`).join("\n");
 	const activity = finalTasks.map((entry, index) => projectTaskActivity(entry, index));
 	const tasks = finalTasks.map((entry, index) => ({ index, name: activity[index].name, status: entry.status, summary: String(finalReports[index].summary ?? "Task has no report."), documents: finalReports[index].documents ?? [], ...(entry.error ? { error: String(entry.error) } : {}), ...(entry.question ? { question: String(entry.question) } : {}), paneId: String(entry.paneId ?? ""), recordDirectory: taskDirectories[index], sessionFile: String(entry.sessionFile ?? "") }));
 	return { ok: runStatus === "completed", runId: run.id, label: String(run.label ?? "batch"), status: runStatus, summary, ...(blockedReport?.question ? { question: blockedReport.question } : {}), documents: finalReports.flatMap((entry) => entry.documents ?? []), recordDirectory: runDirectory, requestedConcurrency: Number(run.requestedConcurrency ?? run.effectiveConcurrency ?? 1), effectiveConcurrency: Number(run.effectiveConcurrency ?? 1), tasks, activity, ...(run.tabId ? { tabId: String(run.tabId) } : {}) };
@@ -113,10 +120,20 @@ export async function respondToBlockedTask(cwd: string, runId: string, answer: s
 	const runsDirectory = join(cwd, ".pi", "herdr-subagents", "runs");
 	const runDirectory = await findRunDirectory(runsDirectory, runId);
 	if (!runDirectory) throw new Error(`Unknown Subagent run '${runId}'.`);
-	const lockFile = join(runDirectory, ".respond.lock");
-	const lockToken = await acquireResponseLock(lockFile, runId, options.owner);
+	const taskDirectories = readdirSync(join(runDirectory, "tasks")).sort().map((entry) => join(runDirectory, "tasks", entry));
+	let blockedTaskDirectory: string | undefined;
+	for (const directory of taskDirectories) {
+		const task = await readTaskRecord(join(directory, "task.json"));
+		if (isContinuable(task, options.taskNumber)) { blockedTaskDirectory = directory; break; }
+	}
+	if (!blockedTaskDirectory) throw noContinuableTaskError(runId, options.taskNumber);
+	const taskLock = await acquireDirectoryLock(join(blockedTaskDirectory, ".resume.lock"), {
+		wait: false,
+		signal: options.signal,
+		contentionMessage: `Run '${runId}' already has a response or resume in progress for that task.`,
+	});
 	try { return await respondToBlockedTaskLocked(cwd, runId, answer, runDirectory, options); }
-	finally { await releaseResponseLock(lockFile, lockToken); }
+	finally { await taskLock.release(); }
 }
 
 async function respondToBlockedTaskLocked(cwd: string, runId: string, answer: string, runDirectory: string, options: RespondToBlockedTaskOptions): Promise<ResumeBlockedResult> {
@@ -129,28 +146,53 @@ async function respondToBlockedTaskLocked(cwd: string, runId: string, answer: st
 	let task: TaskRecord | undefined;
 	for (const directory of taskDirectories) {
 		const candidate = await readTaskRecord(join(directory, "task.json"));
-		if (candidate.status === "blocked" && (!taskNumber || candidate.order === taskNumber)) { taskDirectory = directory; task = candidate; break; }
+		if (isContinuable(candidate, taskNumber)) { taskDirectory = directory; task = candidate; break; }
 	}
-	if (!taskDirectory || !task) throw new Error(`Run '${runId}' has no blocked task.`);
+	if (!taskDirectory || !task) throw noContinuableTaskError(runId, taskNumber);
 	Object.assign(run, ownerRecord(options.owner));
+	const respondingToBlocked = task.status === "blocked";
+	const historicalResume = Boolean(run.resume?.active[task.id]);
+	const responseAttemptId = createAttemptId();
 	const previousReport = await readReport(join(taskDirectory, "report.json"));
 	const turnsDirectory = join(taskDirectory, "turns");
 	await mkdir(turnsDirectory, { recursive: true, mode: 0o700 });
 	const resumePrompt = join(taskDirectory, "resume-system-prompt.md");
 	await writeTextAtomic(resumePrompt, `${continuationSystemPrompt(task.systemPrompt, "Continue from the supplied answer.", "resumed")}\n`);
+	const taskFile = join(taskDirectory, "task.json");
+	const claimedFrom: TaskRecord = structuredClone(task);
+	// Claim the task before any pane work. A settling batch or a sibling continuation decides tab
+	// cleanup from the task records it reads under this lock, so an unclaimed continuation can have
+	// the tab it is about to use closed underneath it.
+	applyTaskStatus(task, "running");
+	await withRunStateLock(runFile, signal, async (latest) => {
+		await writeJsonAtomic(taskFile, task);
+		Object.assign(latest, ownerRecord(options.owner));
+		applyRunStatus(latest, "running");
+	});
 	let paneId = String(task.paneId ?? "");
 	let agentName = String(task.agentName ?? liveAgentName(task.id));
-	let live = Boolean(paneId) && await herdr.paneExists(paneId, signal) && await herdr.isTaskRunning(paneId, signal);
-	if (!live) {
-		const paneAllocator = new RunPaneAllocator({ herdr, run, runFile, signal, validateExistingPane: true, onCleanupError });
-		paneId = await paneAllocator.allocate(taskDirectory);
-		agentName = liveAgentName(task.id, Number(task.attempt ?? 1) + 1);
-		task.attempt = Number(task.attempt ?? 1) + 1;
-		await herdr.renamePane(paneId, `${String(task.order).padStart(2, "0")} · ${task.name ?? task.agent}`, signal);
+	let live = false;
+	try {
+		live = Boolean(paneId) && await herdr.paneExists(paneId, signal) && await herdr.isTaskRunning(paneId, signal);
+		if (!live) {
+			const paneAllocator = new RunPaneAllocator({ herdr, run, runFile, signal, validateExistingPane: true, onCleanupError });
+			paneId = await paneAllocator.allocate(taskDirectory);
+			agentName = liveAgentName(task.id, Number(task.attempt ?? 1) + 1);
+			task.attempt = Number(task.attempt ?? 1) + 1;
+			await herdr.renamePane(paneId, `${String(task.order).padStart(2, "0")} · ${task.name ?? task.agent}`, signal);
+		}
+	} catch (error) {
+		// Nothing was prompted, so release the claim exactly instead of inventing a new outcome.
+		await withRunStateLock(runFile, signal, async (latest) => {
+			await writeJsonAtomic(taskFile, claimedFrom);
+			const siblings = await Promise.all(taskDirectories.map((directory) => readTaskRecord(join(directory, "task.json"))));
+			applyRunStatus(latest, aggregateTasksWithResume(siblings, latest.resume?.active), { question: claimedFrom.question });
+		});
+		throw error;
 	}
 	const attempt = Number(task.attempt ?? 1);
 	const turnFile = live && task.currentTurnFile ? String(task.currentTurnFile) : join(turnsDirectory, `${String(attempt).padStart(2, "0")}.json`);
-	let turn: TurnRecord = { attempt, prompt: task.prompt, status: "blocked", summary: previousReport.summary, question: previousReport.question, documents: previousReport.documents ?? [], sessionFile: task.sessionFile, startedAt: task.startedAt };
+	let turn: TurnRecord = { attempt, prompt: task.prompt, status: task.status, summary: previousReport.summary, question: previousReport.question, documents: previousReport.documents ?? [], sessionFile: task.sessionFile, startedAt: task.startedAt };
 	if (existsSync(turnFile)) {
 		try { turn = await readTurnRecord(turnFile); }
 		catch (error) {
@@ -163,13 +205,24 @@ async function respondToBlockedTaskLocked(cwd: string, runId: string, answer: st
 	task.currentTurnFile = turnFile;
 	applyTaskStatus(task, "running");
 	Object.assign(task, { tabId: run.tabId, paneId, agentName });
-	await Promise.all([writeJsonAtomic(join(taskDirectory, "task.json"), task), writeJsonAtomic(runFile, run), writeJsonAtomic(turnFile, turn)]);
+	await Promise.all([
+		writeJsonAtomic(join(taskDirectory, "task.json"), task),
+		writeJsonAtomic(turnFile, turn),
+		withRunStateLock(runFile, signal, (latest) => {
+			Object.assign(latest, ownerRecord(options.owner));
+			const member = latest.resume?.active[task.id];
+			if (member) { member.attemptId = responseAttemptId; member.phase = "running"; }
+		}),
+	]);
 	await emitProgress(options, runDirectory, run, taskDirectories, `${task.name ?? task.agent} is continuing after input.`);
 	let report: PersistedReport;
 	let status: "blocked" | "completed" | "failed" | "cancelled";
 	let attemptError: unknown;
 	try {
-		const prompted = await promptLiveAgent({ herdr, task: { ...task, id: task.id, sessionFile: task.sessionFile }, paneId, prompt: answer, systemPromptFile: resumePrompt, agentName, start: !live, signal, stalledWarningMs: run.stalledWarningSeconds ? Number(run.stalledWarningSeconds) * 1000 : undefined, onStalled: () => options.onStalled?.(`${task.name ?? task.agent} appears stalled; it is still running.`) });
+		const prompted = await promptLiveAgent({ herdr, task: { ...task, id: task.id, sessionFile: task.sessionFile }, paneId, prompt: answer, systemPromptFile: resumePrompt, agentName, attemptId: responseAttemptId, start: !live, signal, stalledWarningMs: run.stalledWarningSeconds ? Number(run.stalledWarningSeconds) * 1000 : undefined, onStalled: () => options.onStalled?.(`${task.name ?? task.agent} appears stalled; it is still running.`), onRuntimeUpdate: async (updated) => {
+			Object.assign(task, updated);
+			await emitProgress(options, runDirectory, run, taskDirectories, `${task.name ?? task.agent} runtime: ${updated.runtimeState}.`);
+		} });
 		task.currentAttemptId = prompted.attemptId;
 		report = prompted.report;
 		status = applyReport(task, report);
@@ -186,29 +239,19 @@ async function respondToBlockedTaskLocked(cwd: string, runId: string, answer: st
 	}
 	if (attemptError && isAbortError(attemptError, signal)) {
 		const message = attemptError instanceof Error ? attemptError.message : String(attemptError);
-			applyTaskStatus(task, "cancelled", { error: message });
-			applyRunStatus(run, "cancelled", { error: message });
-			applyTaskStatus(turn, "cancelled", { error: message });
-			for (const siblingDirectory of taskDirectories) {
-				if (siblingDirectory === taskDirectory) continue;
-				const siblingFile = join(siblingDirectory, "task.json");
-				const sibling = await readTaskRecord(siblingFile);
-				if (!isActiveStatus(sibling.status)) continue;
-				applyTaskStatus(sibling, "cancelled", { error: "Parent Pi session closed." });
-				await writeJsonAtomic(siblingFile, sibling);
-			}
-			await Promise.all([writeJsonAtomic(join(taskDirectory, "task.json"), task), writeJsonAtomic(runFile, run), writeJsonAtomic(turnFile, turn)]);
-			if (run.tabId) await cleanupRunTab({ herdr, tabId: run.tabId, runFile, runRecord: run, signal, onError: onCleanupError });
-			return { ok: false, runId, label: String(run.label ?? "batch"), status: "cancelled", summary: "Subagent response cancelled.", documents: [], recordDirectory: runDirectory, requestedConcurrency: Number(run.requestedConcurrency ?? 1), effectiveConcurrency: Number(run.effectiveConcurrency ?? 1), ...(run.tabId ? { tabId: String(run.tabId) } : {}) };
+		applyTaskStatus(task, "cancelled", { error: message });
+		applyTaskStatus(turn, "cancelled", { error: message });
+		await Promise.all([writeJsonAtomic(join(taskDirectory, "task.json"), task), writeJsonAtomic(turnFile, turn)]);
+		return projectSettledRun(runDirectory, runFile, run, taskDirectories, herdr, onCleanupError);
 	}
 	turn.events.push({ type: "report", status, summary: report.summary, question: report.question, error: report.error, at: new Date().toISOString() });
 	Object.assign(turn, { summary: report.summary, documents: report.documents ?? [] });
 	applyReport(turn, report);
 	await Promise.all([writeJsonAtomic(join(taskDirectory, "task.json"), task), writeJsonAtomic(turnFile, turn)]);
 	await emitProgress(options, runDirectory, run, taskDirectories, `${task.name ?? task.agent} ${status}.`);
-	if (status !== "blocked") {
-		applyRunStatus(run, "queued");
-		await writeJsonAtomic(runFile, run);
+	// Only a blocked task halts its batch worker. Continuing a settled failure must not restart
+	// queued siblings that the still-running batch already owns.
+	if (status !== "blocked" && !historicalResume && respondingToBlocked) {
 		await continueQueuedRun(runDirectory, herdr, { cleanup: false, signal, onStalled: options.onStalled, onCleanupError, onUpdate: () => emitProgress(options, runDirectory, run, taskDirectories, "Queued tasks are continuing.") });
 	}
 	return projectSettledRun(runDirectory, runFile, run, taskDirectories, herdr, onCleanupError);

package/packages/herdr-subagents/src/continue-run.ts

@@ -7,6 +7,7 @@ import { cleanupRunTab } from "./tab-cleanup.ts";
 import { RunPaneAllocator } from "./run-pane-allocator.ts";
 import { aggregateTaskStatus, applyRunStatus } from "./run-status.ts";
 import { applyReport, applyTaskStatus, readRunRecord, readTaskRecord, settleAttemptFailure } from "./records.ts";
+import { acquireDirectoryLock, adoptRecord, queueLockPath, withRunStateLock, type DirectoryLock } from "./run-state.ts";
 import { isAbortError } from "./errors.ts";
 
 export interface ContinueQueuedRunOptions {
@@ -19,6 +20,20 @@ export interface ContinueQueuedRunOptions {
 
 export async function continueQueuedRun(runDirectory: string, herdr: HerdrAutomation, options: ContinueQueuedRunOptions = {}): Promise<void> {
 	const runFile = join(runDirectory, "run.json");
+	// A live batch keeps starting its own queued tasks even after one of its lanes blocks, so take the
+	// queue over only when nothing else owns it. Otherwise a task could be started twice.
+	const ownedElsewhere = `Run queue at ${runDirectory} is owned by its batch.`;
+	let queueLock: DirectoryLock;
+	try { queueLock = await acquireDirectoryLock(queueLockPath(runDirectory), { wait: false, signal: options.signal, contentionMessage: ownedElsewhere }); }
+	catch (error) {
+		if (error instanceof Error && error.message === ownedElsewhere) return;
+		throw error;
+	}
+	try { await continueQueuedTasks(runDirectory, runFile, herdr, options); }
+	finally { await queueLock.release(); }
+}
+
+async function continueQueuedTasks(runDirectory: string, runFile: string, herdr: HerdrAutomation, options: ContinueQueuedRunOptions): Promise<void> {
 	const run = await readRunRecord(runFile);
 	const taskDirectories = readdirSync(join(runDirectory, "tasks")).sort().map((entry) => join(runDirectory, "tasks", entry));
 	const queuedDirectories: string[] = [];
@@ -50,8 +65,7 @@ export async function continueQueuedRun(runDirectory: string, herdr: HerdrAutoma
 			await options.onUpdate?.();
 			const prompted = await promptLiveAgent({ herdr, task: { ...task, id: task.id, sessionFile: task.sessionFile }, paneId, prompt: task.prompt, systemPromptFile: join(directory, "system-prompt.md"), agentName, signal: options.signal, stalledWarningMs: run.stalledWarningSeconds ? Number(run.stalledWarningSeconds) * 1000 : undefined, onStalled: () => options.onStalled?.(`${task.name ?? task.agent} appears stalled; it is still running.`) });
 			task.currentAttemptId = prompted.attemptId;
-			const report = prompted.report;
-			applyReport(task, report);
+			applyReport(task, prompted.report);
 		} catch (error) {
 			await settleAttemptFailure(task, reportFile, {
 				cancelled: isAbortError(error, options.signal),
@@ -81,8 +95,11 @@ export async function continueQueuedRun(runDirectory: string, herdr: HerdrAutoma
 		}
 	}
 	const statuses = await Promise.all(taskDirectories.map(async (directory) => (await readTaskRecord(join(directory, "task.json"))).status));
-	applyRunStatus(run, aggregateTaskStatus(statuses));
-	await writeJsonAtomic(runFile, run);
+	// Settling the run must not be cancellable, so the terminal status is never written under the signal.
+	await withRunStateLock(runFile, undefined, (latest) => {
+		applyRunStatus(latest, aggregateTaskStatus(statuses));
+		adoptRecord(run, latest);
+	});
 	if (options.cleanup !== false && run.status !== "blocked" && run.tabId) {
 		await cleanupRunTab({ herdr, tabId: run.tabId, runFile, runRecord: run, signal: options.signal, onError: options.onCleanupError });
 	}

package/packages/herdr-subagents/src/dispatcher.ts

@@ -3,10 +3,19 @@ interface QueuedRun<T> {
 	execute: () => Promise<T>;
 	resolve: (value: T) => void;
 	reject: (error: unknown) => void;
+	signal?: AbortSignal;
+	onAbort?: () => void;
+	onRetained?: (retainedRunId: string) => T;
+}
+
+interface SubmitOptions<T> {
+	signal?: AbortSignal;
+	onRetained?: (retainedRunId: string) => T;
 }
 
 export interface DispatcherSnapshot {
 	activeRunId?: string;
+	retainedRunId?: string;
 	queuedRunIds: string[];
 }
 
@@ -19,15 +28,33 @@ export class RunDispatcher {
 
 	constructor(onChange?: (snapshot: DispatcherSnapshot) => void) { this.onChange = onChange; }
 
-	submit<T>(id: string, execute: () => Promise<T>): Promise<T> {
-		const promise = new Promise<T>((resolve, reject) => this.queue.push({ id, execute, resolve: resolve as (value: unknown) => void, reject }));
+	submit<T>(id: string, execute: () => Promise<T>, options: SubmitOptions<T> = {}): Promise<T> {
+		const { signal, onRetained } = options;
+		if (signal?.aborted) return Promise.reject(signal.reason ?? new Error(`Queued run '${id}' was cancelled.`));
+		if (this.retainedRunId && onRetained) return Promise.resolve(onRetained(this.retainedRunId));
+		const promise = new Promise<T>((resolve, reject) => {
+			const queued: QueuedRun<T> = { id, execute, resolve, reject, signal, onRetained };
+			queued.onAbort = () => {
+				const index = this.queue.indexOf(queued as QueuedRun<unknown>);
+				if (index < 0) return;
+				this.queue.splice(index, 1);
+				reject(signal?.reason ?? new Error(`Queued run '${id}' was cancelled.`));
+				this.changed();
+			};
+			signal?.addEventListener("abort", queued.onAbort, { once: true });
+			this.queue.push(queued as QueuedRun<unknown>);
+		});
 		this.changed();
 		void this.drain();
 		return promise;
 	}
 
 	snapshot(): DispatcherSnapshot {
-		return { ...(this.activeRunId ? { activeRunId: this.activeRunId } : {}), queuedRunIds: this.queue.map((run) => run.id) };
+		return {
+			...(this.activeRunId ? { activeRunId: this.activeRunId } : {}),
+			...(this.retainedRunId ? { retainedRunId: this.retainedRunId } : {}),
+			queuedRunIds: this.queue.map((run) => run.id),
+		};
 	}
 
 	pause(): void { this.paused = true; this.changed(); }
@@ -35,6 +62,14 @@ export class RunDispatcher {
 	retain(id: string): void {
 		if (this.activeRunId !== id) throw new Error(`Cannot retain inactive run '${id}'.`);
 		this.retainedRunId = id;
+		for (let index = this.queue.length - 1; index >= 0; index -= 1) {
+			const queued = this.queue[index];
+			if (!queued.onRetained) continue;
+			this.queue.splice(index, 1);
+			if (queued.onAbort) queued.signal?.removeEventListener("abort", queued.onAbort);
+			try { queued.resolve(queued.onRetained(id)); }
+			catch (error) { queued.reject(error); }
+		}
 		this.changed();
 	}
 	detach(id: string): boolean {
@@ -54,7 +89,10 @@ export class RunDispatcher {
 	}
 	cancelQueued(reason: Error): string[] {
 		const cancelled = this.queue.splice(0);
-		for (const run of cancelled) run.reject(reason);
+		for (const run of cancelled) {
+			if (run.onAbort) run.signal?.removeEventListener("abort", run.onAbort);
+			run.reject(reason);
+		}
 		this.changed();
 		return cancelled.map((run) => run.id);
 	}
@@ -62,6 +100,7 @@ export class RunDispatcher {
 		const index = this.queue.findIndex((run) => run.id === id);
 		if (index < 0) return false;
 		const [cancelled] = this.queue.splice(index, 1);
+		if (cancelled.onAbort) cancelled.signal?.removeEventListener("abort", cancelled.onAbort);
 		cancelled.reject(reason);
 		this.changed();
 		return true;
@@ -73,6 +112,7 @@ export class RunDispatcher {
 		if (this.activeRunId || this.paused) return;
 		const run = this.queue.shift();
 		if (!run) { this.changed(); return; }
+		if (run.onAbort) run.signal?.removeEventListener("abort", run.onAbort);
 		this.activeRunId = run.id;
 		this.changed();
 		try { run.resolve(await run.execute()); }

package/packages/herdr-subagents/src/fleet.ts

@@ -153,7 +153,8 @@ export function fleetLines(
 		const color = task.status === "completed" ? "success" : task.status === "running" ? "warning" : task.status === "failed" ? "error" : task.status === "blocked" ? "warning" : task.status === "cancelled" ? "dim" : "accent";
 		const marker = chosen ? theme.fg("accent", "›") : " ";
 		const persona = theme.fg("muted", task.agent || "worker");
-		const left = `${marker} ${theme.fg(color, circle)} ${persona}  ${task.name}  ${theme.fg("dim", task.status)}`;
+		const visibleStatus = task.runtimeState && task.status === "running" ? task.runtimeState : task.status;
+		const left = `${marker} ${theme.fg(color, circle)} ${persona}  ${task.name}  ${theme.fg("dim", visibleStatus)}`;
 		const duration = formatFleetDuration(task, options.now);
 		const tokens = task.status === "queued" || task.status === "starting" ? undefined : options.metrics?.get(task.index)?.tokens;
 		const right = [duration, tokens === undefined ? undefined : formatFleetTokens(tokens)].filter(Boolean).join("  ");

package/packages/herdr-subagents/src/herdr.ts

@@ -32,10 +32,13 @@ export interface SplitPaneRequest {
 	signal?: AbortSignal;
 }
 
+export type HerdrAgentState = "working" | "blocked" | "idle" | "done" | "unknown";
+
 export interface HerdrAutomation {
 	createTab(request: CreateTabRequest): Promise<{ tabId: string; paneId: string }>;
 	startAgent(request: StartAgentRequest): Promise<void>;
 	promptAgent(request: PromptAgentRequest): Promise<{ status?: string }>;
+	getAgentState(target: string, signal?: AbortSignal): Promise<HerdrAgentState>;
 	splitPane(request: SplitPaneRequest): Promise<{ paneId: string }>;
 	renamePane(paneId: string, label: string, signal?: AbortSignal): Promise<void>;
 	renameTab(tabId: string, label: string, signal?: AbortSignal): Promise<void>;
@@ -52,7 +55,7 @@ interface HerdrResponse {
 		tab?: { tab_id?: string; focused?: boolean };
 		root_pane?: { pane_id?: string };
 		pane?: { pane_id?: string };
-		agent?: { status?: string };
+		agent?: { status?: string; agent_status?: string };
 		process_info?: { foreground_processes?: Array<{ name?: string; argv?: string[] }> };
 	};
 }
@@ -156,6 +159,12 @@ export class CliHerdrAutomation implements HerdrAutomation {
 		return { status: response.result?.agent?.status };
 	}
 
+	async getAgentState(target: string, signal?: AbortSignal): Promise<HerdrAgentState> {
+		const response = parseResponse(await runCommand(this.binary, ["agent", "get", target], signal), "herdr agent get");
+		const state = response.result?.agent?.agent_status ?? response.result?.agent?.status;
+		return state === "working" || state === "blocked" || state === "idle" || state === "done" ? state : "unknown";
+	}
+
 	async splitPane(request: SplitPaneRequest): Promise<{ paneId: string }> {
 		const args = ["pane", "split", request.paneId, "--direction", request.direction, "--cwd", request.cwd, request.focus ? "--focus" : "--no-focus"];
 		for (const [key, value] of Object.entries(request.env ?? {}).sort(([left], [right]) => left.localeCompare(right))) args.push("--env", `${key}=${value}`);