@maxiaochao/pi-toolkit 0.10.21 → 0.10.22

package/packages/herdr-subagents/src/blocking.ts

@@ -1,6 +1,5 @@
 import { existsSync, readdirSync } from "node:fs";
-import { mkdir, readFile, rename, unlink, writeFile } from "node:fs/promises";
-import { randomUUID } from "node:crypto";
+import { mkdir, rename } from "node:fs/promises";
 import { join } from "node:path";
 import { CliHerdrAutomation, type HerdrAutomation } from "./herdr.ts";
 import { continueQueuedRun } from "./continue-run.ts";
@@ -8,20 +7,23 @@ import { liveAgentName, promptLiveAgent } from "./live-agent.ts";
 import { writeJsonAtomic, writeTextAtomic } from "./state.ts";
 import { cleanupRunTab } from "./tab-cleanup.ts";
 import type { OwnerIdentity } from "./ownership.ts";
-import { isProcessAlive, ownerRecord } from "./ownership.ts";
-import { aggregateTaskStatus, applyRunStatus } from "./run-status.ts";
+import { ownerRecord } from "./ownership.ts";
+import { applyRunStatus } from "./run-status.ts";
 import { applyReport, applyTaskStatus, isActiveStatus, persistFailureReport, projectTaskActivity, readReport, readRunRecord, readTaskRecord, readTurnRecord, settleAttemptFailure, type PersistedDocument, type PersistedReport, type RunRecord, type TaskActivity, type TaskRecord, type TaskStatus, type TurnRecord } from "./records.ts";
 import { isAbortError } from "./errors.ts";
 import { continuationSystemPrompt } from "./protocol.ts";
 import { RunPaneAllocator } from "./run-pane-allocator.ts";
 import { findRunDirectory } from "./run-locator.ts";
+import { acquireDirectoryLock, withRunStateLock } from "./run-state.ts";
+import { createAttemptId } from "./attempts.ts";
+import { aggregateTasksWithResume, claimResumeCleanup } from "./resume-state.ts";
 
-export interface RespondToBlockedTaskOptions { herdr?: HerdrAutomation; taskNumber?: number; signal?: AbortSignal; owner?: OwnerIdentity; onCleanupError?: (message: string) => void; onWarning?: (message: string) => void; onStalled?: (message: string) => void; onUpdate?: (result: ResumeBlockedResult) => void; }
+export interface RespondToBlockedTaskOptions { herdr?: HerdrAutomation; taskNumber: number; signal?: AbortSignal; owner?: OwnerIdentity; onCleanupError?: (message: string) => void; onWarning?: (message: string) => void; onStalled?: (message: string) => void; onUpdate?: (result: ResumeBlockedResult) => void; }
 export interface ResumeBlockedResult {
 	ok: boolean;
 	runId: string;
 	label: string;
-	status: "blocked" | "completed" | "partial" | "failed" | "cancelled";
+	status: "running" | "blocked" | "completed" | "partial" | "failed" | "cancelled";
 	summary: string;
 	question?: string;
 	documents: PersistedDocument[];
@@ -33,34 +35,18 @@ export interface ResumeBlockedResult {
 	tabId?: string;
 }
 
-interface ResponseLock { token?: string; processId?: unknown; sessionId?: unknown; createdAt?: unknown; }
-
-async function readResponseLock(lockFile: string): Promise<ResponseLock | undefined> {
-	try {
-		const lock = JSON.parse(await readFile(lockFile, "utf8")) as unknown;
-		return lock && typeof lock === "object" ? lock as ResponseLock : undefined;
-	} catch { return undefined; }
-}
-
-async function releaseResponseLock(lockFile: string, token: string): Promise<void> {
-	if ((await readResponseLock(lockFile))?.token !== token) return;
-	try { await unlink(lockFile); }
-	catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; }
+/**
+ * A blocked task still owns a live child waiting for an answer. A settled failed task can only
+ * continue from its saved Pi session.
+ */
+function isContinuable(task: TaskRecord, taskNumber: number): boolean {
+	if (task.order !== taskNumber) return false;
+	if (task.status === "blocked") return true;
+	return task.status === "failed" && existsSync(task.sessionFile ?? "");
 }
 
-async function acquireResponseLock(lockFile: string, runId: string, owner?: OwnerIdentity): Promise<string> {
-	const token = randomUUID();
-	try {
-		await writeFile(lockFile, `${JSON.stringify({ token, processId: process.pid, sessionId: owner?.sessionId, createdAt: new Date().toISOString() })}\n`, { flag: "wx", mode: 0o600 });
-		return token;
-	} catch (error) {
-		if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
-		const lock = await readResponseLock(lockFile);
-		if (lock && Number.isInteger(lock.processId) && !isProcessAlive(lock.processId)) {
-			throw new Error(`Subagent run '${runId}' has a stale blocked-response lock from process ${lock.processId}; remove ${lockFile} and retry.`);
-		}
-		throw new Error(`Subagent run '${runId}' already has a blocked response in progress (lock: ${lockFile}).`);
-	}
+function noContinuableTaskError(runId: string, taskNumber: number): Error {
+	return new Error(`Run '${runId}' task ${taskNumber} is neither blocked nor a failed task with a saved Pi session.`);
 }
 
 async function emitProgress(options: RespondToBlockedTaskOptions, runDirectory: string, run: RunRecord, taskDirectories: string[], summary: string): Promise<void> {
@@ -82,7 +68,26 @@ async function emitProgress(options: RespondToBlockedTaskOptions, runDirectory:
 }
 
 async function projectSettledRun(runDirectory: string, runFile: string, run: RunRecord, taskDirectories: string[], herdr: HerdrAutomation, onCleanupError?: (message: string) => void): Promise<ResumeBlockedResult> {
-	const finalTasks = await Promise.all(taskDirectories.map((directory) => readTaskRecord(join(directory, "task.json"))));
+	const settledState = await withRunStateLock(runFile, undefined, async (latest) => {
+		const tasks = await Promise.all(taskDirectories.map((directory) => readTaskRecord(join(directory, "task.json"))));
+		if (latest.resume) {
+			for (const task of tasks) {
+				const member = latest.resume.active[task.id];
+				if (!member || member.attemptId !== task.currentAttemptId) continue;
+				if (task.status === "blocked") member.phase = "blocked";
+				else if (!isActiveStatus(task.status)) delete latest.resume.active[task.id];
+			}
+		}
+		const aggregated = aggregateTasksWithResume(tasks, latest.resume?.active);
+		const blockedTask = tasks.find((task) => task.status === "blocked");
+		// Queued work aggregates as blocked; with no blocked task the run is simply still running.
+		const status = aggregated === "blocked" && !blockedTask ? "running" : aggregated;
+		applyRunStatus(latest, status, { question: blockedTask?.question });
+		const cleanup = status === "running" || status === "blocked" ? undefined : claimResumeCleanup(latest)
+			?? (!latest.resume && latest.tabId ? { tabId: latest.tabId, generation: undefined } : undefined);
+		return { run: latest, tasks, status, cleanup };
+	});
+	let finalTasks = settledState.tasks;
 	const finalReports = await Promise.all(taskDirectories.map(async (directory, index) => {
 		const reportFile = join(directory, "report.json");
 		if (!existsSync(reportFile) && (finalTasks[index].status === "queued" || finalTasks[index].status === "cancelled")) return { status: "failed" as const, summary: finalTasks[index].error ?? (finalTasks[index].status === "queued" ? "Task is still queued." : "Task was cancelled."), documents: [] };
@@ -95,15 +100,17 @@ async function projectSettledRun(runDirectory: string, runFile: string, run: Run
 			return failedReport;
 		}
 	}));
-	const aggregateStatus = aggregateTaskStatus(finalTasks.map((entry) => entry.status));
-	const runStatus = aggregateStatus === "running" ? "blocked" : aggregateStatus;
+	const runStatus = settledState.status;
 	const blockedIndex = finalTasks.findIndex((entry) => entry.status === "blocked");
 	const blockedReport = blockedIndex >= 0 ? finalReports[blockedIndex] : undefined;
-	run = await readRunRecord(runFile);
-	applyRunStatus(run, runStatus, { question: blockedReport?.question });
-	await writeJsonAtomic(runFile, run);
-	if (runStatus !== "blocked" && run.tabId) await cleanupRunTab({ herdr, tabId: run.tabId, runFile, runRecord: run, onError: onCleanupError });
-	const summary = runStatus === "completed" ? "All tasks completed after the answer." : runStatus === "blocked" ? `${finalTasks[blockedIndex]?.name ?? "Task"} needs input: ${blockedReport?.question ?? "Input required."}` : finalReports.map((entry, index) => `${finalTasks[index].name}: ${entry.summary}`).join("\n");
+	run = settledState.run;
+	if (settledState.cleanup) await cleanupRunTab({
+		herdr, tabId: settledState.cleanup.tabId, runFile, runRecord: run, onError: onCleanupError,
+		onClosed: (latest) => {
+			if (settledState.cleanup?.generation !== undefined && latest.resume?.generation === settledState.cleanup.generation && latest.resume.phase === "closing") delete latest.resume;
+		},
+	});
+	const summary = runStatus === "completed" ? "All tasks completed after the answer." : runStatus === "running" ? "Other resumed tasks are still running." : runStatus === "blocked" ? `${finalTasks[blockedIndex]?.name ?? "Task"} needs input: ${blockedReport?.question ?? "Input required."}` : finalReports.map((entry, index) => `${finalTasks[index].name}: ${entry.summary}`).join("\n");
 	const activity = finalTasks.map((entry, index) => projectTaskActivity(entry, index));
 	const tasks = finalTasks.map((entry, index) => ({ index, name: activity[index].name, status: entry.status, summary: String(finalReports[index].summary ?? "Task has no report."), documents: finalReports[index].documents ?? [], ...(entry.error ? { error: String(entry.error) } : {}), ...(entry.question ? { question: String(entry.question) } : {}), paneId: String(entry.paneId ?? ""), recordDirectory: taskDirectories[index], sessionFile: String(entry.sessionFile ?? "") }));
 	return { ok: runStatus === "completed", runId: run.id, label: String(run.label ?? "batch"), status: runStatus, summary, ...(blockedReport?.question ? { question: blockedReport.question } : {}), documents: finalReports.flatMap((entry) => entry.documents ?? []), recordDirectory: runDirectory, requestedConcurrency: Number(run.requestedConcurrency ?? run.effectiveConcurrency ?? 1), effectiveConcurrency: Number(run.effectiveConcurrency ?? 1), tasks, activity, ...(run.tabId ? { tabId: String(run.tabId) } : {}) };
@@ -113,10 +120,20 @@ export async function respondToBlockedTask(cwd: string, runId: string, answer: s
 	const runsDirectory = join(cwd, ".pi", "herdr-subagents", "runs");
 	const runDirectory = await findRunDirectory(runsDirectory, runId);
 	if (!runDirectory) throw new Error(`Unknown Subagent run '${runId}'.`);
-	const lockFile = join(runDirectory, ".respond.lock");
-	const lockToken = await acquireResponseLock(lockFile, runId, options.owner);
+	const taskDirectories = readdirSync(join(runDirectory, "tasks")).sort().map((entry) => join(runDirectory, "tasks", entry));
+	let blockedTaskDirectory: string | undefined;
+	for (const directory of taskDirectories) {
+		const task = await readTaskRecord(join(directory, "task.json"));
+		if (isContinuable(task, options.taskNumber)) { blockedTaskDirectory = directory; break; }
+	}
+	if (!blockedTaskDirectory) throw noContinuableTaskError(runId, options.taskNumber);
+	const taskLock = await acquireDirectoryLock(join(blockedTaskDirectory, ".resume.lock"), {
+		wait: false,
+		signal: options.signal,
+		contentionMessage: `Run '${runId}' already has a response or resume in progress for that task.`,
+	});
 	try { return await respondToBlockedTaskLocked(cwd, runId, answer, runDirectory, options); }
-	finally { await releaseResponseLock(lockFile, lockToken); }
+	finally { await taskLock.release(); }
 }
 
 async function respondToBlockedTaskLocked(cwd: string, runId: string, answer: string, runDirectory: string, options: RespondToBlockedTaskOptions): Promise<ResumeBlockedResult> {
@@ -129,28 +146,53 @@ async function respondToBlockedTaskLocked(cwd: string, runId: string, answer: st
 	let task: TaskRecord | undefined;
 	for (const directory of taskDirectories) {
 		const candidate = await readTaskRecord(join(directory, "task.json"));
-		if (candidate.status === "blocked" && (!taskNumber || candidate.order === taskNumber)) { taskDirectory = directory; task = candidate; break; }
+		if (isContinuable(candidate, taskNumber)) { taskDirectory = directory; task = candidate; break; }
 	}
-	if (!taskDirectory || !task) throw new Error(`Run '${runId}' has no blocked task.`);
+	if (!taskDirectory || !task) throw noContinuableTaskError(runId, taskNumber);
 	Object.assign(run, ownerRecord(options.owner));
+	const respondingToBlocked = task.status === "blocked";
+	const historicalResume = Boolean(run.resume?.active[task.id]);
+	const responseAttemptId = createAttemptId();
 	const previousReport = await readReport(join(taskDirectory, "report.json"));
 	const turnsDirectory = join(taskDirectory, "turns");
 	await mkdir(turnsDirectory, { recursive: true, mode: 0o700 });
 	const resumePrompt = join(taskDirectory, "resume-system-prompt.md");
 	await writeTextAtomic(resumePrompt, `${continuationSystemPrompt(task.systemPrompt, "Continue from the supplied answer.", "resumed")}\n`);
+	const taskFile = join(taskDirectory, "task.json");
+	const claimedFrom: TaskRecord = structuredClone(task);
+	// Claim the task before any pane work. A settling batch or a sibling continuation decides tab
+	// cleanup from the task records it reads under this lock, so an unclaimed continuation can have
+	// the tab it is about to use closed underneath it.
+	applyTaskStatus(task, "running");
+	await withRunStateLock(runFile, signal, async (latest) => {
+		await writeJsonAtomic(taskFile, task);
+		Object.assign(latest, ownerRecord(options.owner));
+		applyRunStatus(latest, "running");
+	});
 	let paneId = String(task.paneId ?? "");
 	let agentName = String(task.agentName ?? liveAgentName(task.id));
-	let live = Boolean(paneId) && await herdr.paneExists(paneId, signal) && await herdr.isTaskRunning(paneId, signal);
-	if (!live) {
-		const paneAllocator = new RunPaneAllocator({ herdr, run, runFile, signal, validateExistingPane: true, onCleanupError });
-		paneId = await paneAllocator.allocate(taskDirectory);
-		agentName = liveAgentName(task.id, Number(task.attempt ?? 1) + 1);
-		task.attempt = Number(task.attempt ?? 1) + 1;
-		await herdr.renamePane(paneId, `${String(task.order).padStart(2, "0")} · ${task.name ?? task.agent}`, signal);
+	let live = false;
+	try {
+		live = Boolean(paneId) && await herdr.paneExists(paneId, signal) && await herdr.isTaskRunning(paneId, signal);
+		if (!live) {
+			const paneAllocator = new RunPaneAllocator({ herdr, run, runFile, signal, validateExistingPane: true, onCleanupError });
+			paneId = await paneAllocator.allocate(taskDirectory);
+			agentName = liveAgentName(task.id, Number(task.attempt ?? 1) + 1);
+			task.attempt = Number(task.attempt ?? 1) + 1;
+			await herdr.renamePane(paneId, `${String(task.order).padStart(2, "0")} · ${task.name ?? task.agent}`, signal);
+		}
+	} catch (error) {
+		// Nothing was prompted, so release the claim exactly instead of inventing a new outcome.
+		await withRunStateLock(runFile, signal, async (latest) => {
+			await writeJsonAtomic(taskFile, claimedFrom);
+			const siblings = await Promise.all(taskDirectories.map((directory) => readTaskRecord(join(directory, "task.json"))));
+			applyRunStatus(latest, aggregateTasksWithResume(siblings, latest.resume?.active), { question: claimedFrom.question });
+		});
+		throw error;
 	}
 	const attempt = Number(task.attempt ?? 1);
 	const turnFile = live && task.currentTurnFile ? String(task.currentTurnFile) : join(turnsDirectory, `${String(attempt).padStart(2, "0")}.json`);
-	let turn: TurnRecord = { attempt, prompt: task.prompt, status: "blocked", summary: previousReport.summary, question: previousReport.question, documents: previousReport.documents ?? [], sessionFile: task.sessionFile, startedAt: task.startedAt };
+	let turn: TurnRecord = { attempt, prompt: task.prompt, status: task.status, summary: previousReport.summary, question: previousReport.question, documents: previousReport.documents ?? [], sessionFile: task.sessionFile, startedAt: task.startedAt };
 	if (existsSync(turnFile)) {
 		try { turn = await readTurnRecord(turnFile); }
 		catch (error) {
@@ -163,13 +205,24 @@ async function respondToBlockedTaskLocked(cwd: string, runId: string, answer: st
 	task.currentTurnFile = turnFile;
 	applyTaskStatus(task, "running");
 	Object.assign(task, { tabId: run.tabId, paneId, agentName });
-	await Promise.all([writeJsonAtomic(join(taskDirectory, "task.json"), task), writeJsonAtomic(runFile, run), writeJsonAtomic(turnFile, turn)]);
+	await Promise.all([
+		writeJsonAtomic(join(taskDirectory, "task.json"), task),
+		writeJsonAtomic(turnFile, turn),
+		withRunStateLock(runFile, signal, (latest) => {
+			Object.assign(latest, ownerRecord(options.owner));
+			const member = latest.resume?.active[task.id];
+			if (member) { member.attemptId = responseAttemptId; member.phase = "running"; }
+		}),
+	]);
 	await emitProgress(options, runDirectory, run, taskDirectories, `${task.name ?? task.agent} is continuing after input.`);
 	let report: PersistedReport;
 	let status: "blocked" | "completed" | "failed" | "cancelled";
 	let attemptError: unknown;
 	try {
-		const prompted = await promptLiveAgent({ herdr, task: { ...task, id: task.id, sessionFile: task.sessionFile }, paneId, prompt: answer, systemPromptFile: resumePrompt, agentName, start: !live, signal, stalledWarningMs: run.stalledWarningSeconds ? Number(run.stalledWarningSeconds) * 1000 : undefined, onStalled: () => options.onStalled?.(`${task.name ?? task.agent} appears stalled; it is still running.`) });
+		const prompted = await promptLiveAgent({ herdr, task: { ...task, id: task.id, sessionFile: task.sessionFile }, paneId, prompt: answer, systemPromptFile: resumePrompt, agentName, attemptId: responseAttemptId, start: !live, signal, stalledWarningMs: run.stalledWarningSeconds ? Number(run.stalledWarningSeconds) * 1000 : undefined, onStalled: () => options.onStalled?.(`${task.name ?? task.agent} appears stalled; it is still running.`), onRuntimeUpdate: async (updated) => {
+			Object.assign(task, updated);
+			await emitProgress(options, runDirectory, run, taskDirectories, `${task.name ?? task.agent} runtime: ${updated.runtimeState}.`);
+		} });
 		task.currentAttemptId = prompted.attemptId;
 		report = prompted.report;
 		status = applyReport(task, report);
@@ -186,29 +239,19 @@ async function respondToBlockedTaskLocked(cwd: string, runId: string, answer: st
 	}
 	if (attemptError && isAbortError(attemptError, signal)) {
 		const message = attemptError instanceof Error ? attemptError.message : String(attemptError);
-			applyTaskStatus(task, "cancelled", { error: message });
-			applyRunStatus(run, "cancelled", { error: message });
-			applyTaskStatus(turn, "cancelled", { error: message });
-			for (const siblingDirectory of taskDirectories) {
-				if (siblingDirectory === taskDirectory) continue;
-				const siblingFile = join(siblingDirectory, "task.json");
-				const sibling = await readTaskRecord(siblingFile);
-				if (!isActiveStatus(sibling.status)) continue;
-				applyTaskStatus(sibling, "cancelled", { error: "Parent Pi session closed." });
-				await writeJsonAtomic(siblingFile, sibling);
-			}
-			await Promise.all([writeJsonAtomic(join(taskDirectory, "task.json"), task), writeJsonAtomic(runFile, run), writeJsonAtomic(turnFile, turn)]);
-			if (run.tabId) await cleanupRunTab({ herdr, tabId: run.tabId, runFile, runRecord: run, signal, onError: onCleanupError });
-			return { ok: false, runId: run.id, label: String(run.label ?? "batch"), status: "cancelled", summary: "Subagent response cancelled.", documents: [], recordDirectory: runDirectory, requestedConcurrency: Number(run.requestedConcurrency ?? 1), effectiveConcurrency: Number(run.effectiveConcurrency ?? 1), ...(run.tabId ? { tabId: String(run.tabId) } : {}) };
+		applyTaskStatus(task, "cancelled", { error: message });
+		applyTaskStatus(turn, "cancelled", { error: message });
+		await Promise.all([writeJsonAtomic(join(taskDirectory, "task.json"), task), writeJsonAtomic(turnFile, turn)]);
+		return projectSettledRun(runDirectory, runFile, run, taskDirectories, herdr, onCleanupError);
 	}
 	turn.events.push({ type: "report", status, summary: report.summary, question: report.question, error: report.error, at: new Date().toISOString() });
 	Object.assign(turn, { summary: report.summary, documents: report.documents ?? [] });
 	applyReport(turn, report);
 	await Promise.all([writeJsonAtomic(join(taskDirectory, "task.json"), task), writeJsonAtomic(turnFile, turn)]);
 	await emitProgress(options, runDirectory, run, taskDirectories, `${task.name ?? task.agent} ${status}.`);
-	if (status !== "blocked") {
-		applyRunStatus(run, "queued");
-		await writeJsonAtomic(runFile, run);
+	// Only a blocked task halts its batch worker. Continuing a settled failure must not restart
+	// queued siblings that the still-running batch already owns.
+	if (status !== "blocked" && !historicalResume && respondingToBlocked) {
 		await continueQueuedRun(runDirectory, herdr, { cleanup: false, signal, onStalled: options.onStalled, onCleanupError, onUpdate: () => emitProgress(options, runDirectory, run, taskDirectories, "Queued tasks are continuing.") });
 	}
 	return projectSettledRun(runDirectory, runFile, run, taskDirectories, herdr, onCleanupError);

package/packages/herdr-subagents/src/continue-run.ts

@@ -6,9 +6,9 @@ import { writeJsonAtomic } from "./state.ts";
 import { cleanupRunTab } from "./tab-cleanup.ts";
 import { RunPaneAllocator } from "./run-pane-allocator.ts";
 import { aggregateTaskStatus, applyRunStatus } from "./run-status.ts";
-import { applyReport, applyTaskStatus, normalizeNeedsInputReport, readRunRecord, readTaskRecord, settleAttemptFailure } from "./records.ts";
+import { applyReport, applyTaskStatus, readRunRecord, readTaskRecord, settleAttemptFailure } from "./records.ts";
+import { acquireDirectoryLock, adoptRecord, queueLockPath, withRunStateLock, type DirectoryLock } from "./run-state.ts";
 import { isAbortError } from "./errors.ts";
-import { attemptReportPath } from "./attempts.ts";
 
 export interface ContinueQueuedRunOptions {
 	cleanup?: boolean;
@@ -20,6 +20,20 @@ export interface ContinueQueuedRunOptions {
 
 export async function continueQueuedRun(runDirectory: string, herdr: HerdrAutomation, options: ContinueQueuedRunOptions = {}): Promise<void> {
 	const runFile = join(runDirectory, "run.json");
+	// A live batch keeps starting its own queued tasks even after one of its lanes blocks, so take the
+	// queue over only when nothing else owns it. Otherwise a task could be started twice.
+	const ownedElsewhere = `Run queue at ${runDirectory} is owned by its batch.`;
+	let queueLock: DirectoryLock;
+	try { queueLock = await acquireDirectoryLock(queueLockPath(runDirectory), { wait: false, signal: options.signal, contentionMessage: ownedElsewhere }); }
+	catch (error) {
+		if (error instanceof Error && error.message === ownedElsewhere) return;
+		throw error;
+	}
+	try { await continueQueuedTasks(runDirectory, runFile, herdr, options); }
+	finally { await queueLock.release(); }
+}
+
+async function continueQueuedTasks(runDirectory: string, runFile: string, herdr: HerdrAutomation, options: ContinueQueuedRunOptions): Promise<void> {
 	const run = await readRunRecord(runFile);
 	const taskDirectories = readdirSync(join(runDirectory, "tasks")).sort().map((entry) => join(runDirectory, "tasks", entry));
 	const queuedDirectories: string[] = [];
@@ -51,9 +65,7 @@ export async function continueQueuedRun(runDirectory: string, herdr: HerdrAutoma
 			await options.onUpdate?.();
 			const prompted = await promptLiveAgent({ herdr, task: { ...task, id: task.id, sessionFile: task.sessionFile }, paneId, prompt: task.prompt, systemPromptFile: join(directory, "system-prompt.md"), agentName, signal: options.signal, stalledWarningMs: run.stalledWarningSeconds ? Number(run.stalledWarningSeconds) * 1000 : undefined, onStalled: () => options.onStalled?.(`${task.name ?? task.agent} appears stalled; it is still running.`) });
 			task.currentAttemptId = prompted.attemptId;
-			const report = normalizeNeedsInputReport(prompted.report, Number(run.effectiveConcurrency) || 1);
-			if (report !== prompted.report) await Promise.all([writeJsonAtomic(reportFile, report), writeJsonAtomic(attemptReportPath(directory, prompted.attemptId), report)]);
-			applyReport(task, report);
+			applyReport(task, prompted.report);
 		} catch (error) {
 			await settleAttemptFailure(task, reportFile, {
 				cancelled: isAbortError(error, options.signal),
@@ -83,8 +95,11 @@ export async function continueQueuedRun(runDirectory: string, herdr: HerdrAutoma
 		}
 	}
 	const statuses = await Promise.all(taskDirectories.map(async (directory) => (await readTaskRecord(join(directory, "task.json"))).status));
-	applyRunStatus(run, aggregateTaskStatus(statuses));
-	await writeJsonAtomic(runFile, run);
+	// Settling the run must not be cancellable, so the terminal status is never written under the signal.
+	await withRunStateLock(runFile, undefined, (latest) => {
+		applyRunStatus(latest, aggregateTaskStatus(statuses));
+		adoptRecord(run, latest);
+	});
 	if (options.cleanup !== false && run.status !== "blocked" && run.tabId) {
 		await cleanupRunTab({ herdr, tabId: run.tabId, runFile, runRecord: run, signal: options.signal, onError: options.onCleanupError });
 	}

package/packages/herdr-subagents/src/fleet.ts

@@ -153,7 +153,8 @@ export function fleetLines(
 		const color = task.status === "completed" ? "success" : task.status === "running" ? "warning" : task.status === "failed" ? "error" : task.status === "blocked" ? "warning" : task.status === "cancelled" ? "dim" : "accent";
 		const marker = chosen ? theme.fg("accent", "›") : " ";
 		const persona = theme.fg("muted", task.agent || "worker");
-		const left = `${marker} ${theme.fg(color, circle)} ${persona}  ${task.name}  ${theme.fg("dim", task.status)}`;
+		const visibleStatus = task.runtimeState && task.status === "running" ? task.runtimeState : task.status;
+		const left = `${marker} ${theme.fg(color, circle)} ${persona}  ${task.name}  ${theme.fg("dim", visibleStatus)}`;
 		const duration = formatFleetDuration(task, options.now);
 		const tokens = task.status === "queued" || task.status === "starting" ? undefined : options.metrics?.get(task.index)?.tokens;
 		const right = [duration, tokens === undefined ? undefined : formatFleetTokens(tokens)].filter(Boolean).join("  ");

package/packages/herdr-subagents/src/herdr.ts

@@ -32,10 +32,13 @@ export interface SplitPaneRequest {
 	signal?: AbortSignal;
 }
 
+export type HerdrAgentState = "working" | "blocked" | "idle" | "done" | "unknown";
+
 export interface HerdrAutomation {
 	createTab(request: CreateTabRequest): Promise<{ tabId: string; paneId: string }>;
 	startAgent(request: StartAgentRequest): Promise<void>;
 	promptAgent(request: PromptAgentRequest): Promise<{ status?: string }>;
+	getAgentState(target: string, signal?: AbortSignal): Promise<HerdrAgentState>;
 	splitPane(request: SplitPaneRequest): Promise<{ paneId: string }>;
 	renamePane(paneId: string, label: string, signal?: AbortSignal): Promise<void>;
 	renameTab(tabId: string, label: string, signal?: AbortSignal): Promise<void>;
@@ -52,7 +55,7 @@ interface HerdrResponse {
 		tab?: { tab_id?: string; focused?: boolean };
 		root_pane?: { pane_id?: string };
 		pane?: { pane_id?: string };
-		agent?: { status?: string };
+		agent?: { status?: string; agent_status?: string };
 		process_info?: { foreground_processes?: Array<{ name?: string; argv?: string[] }> };
 	};
 }
@@ -156,6 +159,12 @@ export class CliHerdrAutomation implements HerdrAutomation {
 		return { status: response.result?.agent?.status };
 	}
 
+	async getAgentState(target: string, signal?: AbortSignal): Promise<HerdrAgentState> {
+		const response = parseResponse(await runCommand(this.binary, ["agent", "get", target], signal), "herdr agent get");
+		const state = response.result?.agent?.agent_status ?? response.result?.agent?.status;
+		return state === "working" || state === "blocked" || state === "idle" || state === "done" ? state : "unknown";
+	}
+
 	async splitPane(request: SplitPaneRequest): Promise<{ paneId: string }> {
 		const args = ["pane", "split", request.paneId, "--direction", request.direction, "--cwd", request.cwd, request.focus ? "--focus" : "--no-focus"];
 		for (const [key, value] of Object.entries(request.env ?? {}).sort(([left], [right]) => left.localeCompare(right))) args.push("--env", `${key}=${value}`);

package/packages/herdr-subagents/src/historical-resume.ts

@@ -0,0 +1,184 @@
+import { randomUUID } from "node:crypto";
+import { existsSync } from "node:fs";
+import { mkdir } from "node:fs/promises";
+import { join } from "node:path";
+import { attemptReportPath, createAttemptId } from "./attempts.ts";
+import { isAbortError } from "./errors.ts";
+import { CliHerdrAutomation, type HerdrAutomation } from "./herdr.ts";
+import { childEnvironment, liveAgentName, promptLiveAgent } from "./live-agent.ts";
+import { ownerRecord } from "./ownership.ts";
+import { continuationSystemPrompt } from "./protocol.ts";
+import { applyRunStatus } from "./run-status.ts";
+import { acquireDirectoryLock, withRunStateLock } from "./run-state.ts";
+import { applyReport, applyTaskStatus, isActiveStatus, readReport, readRunRecord, readTaskRecord, settleAttemptFailure, projectTaskActivity, type PersistedReport, type ResumeMember, type TaskRecord } from "./records.ts";
+import { writeJsonAtomic, writeTextAtomic } from "./state.ts";
+import { cleanupRunTab } from "./tab-cleanup.ts";
+import { findRunDirectory } from "./run-locator.ts";
+import type { ResumeHistoricalTaskOptions, ResumeHistoricalTaskResult } from "./history.ts";
+import { aggregateTasksWithResume, claimResumeCleanup, readTaskEntries } from "./resume-state.ts";
+
+async function aggregateRun(runDirectory: string, active: Record<string, ResumeMember> = {}) {
+	return aggregateTasksWithResume((await readTaskEntries(runDirectory)).map(({ task }) => task), active);
+}
+
+export async function resumeHistoricalTaskConcurrent(cwd: string, requestedRunId: string, taskNumber: number, prompt: string, options: ResumeHistoricalTaskOptions = {}): Promise<ResumeHistoricalTaskResult> {
+	const runDirectory = await findRunDirectory(join(cwd, ".pi", "herdr-subagents", "runs"), requestedRunId);
+	if (!runDirectory) throw new Error(`Unknown Subagent run '${requestedRunId}'.`);
+	const runFile = join(runDirectory, "run.json");
+	const initialRun = await readRunRecord(runFile);
+	const entries = await readTaskEntries(runDirectory);
+	const selected = entries.find(({ task }) => task.order === taskNumber);
+	if (!selected) throw new Error(`Unknown historical task ${initialRun.id}#${taskNumber}.`);
+	const { directory: taskDirectory } = selected;
+	const taskFile = join(taskDirectory, "task.json");
+	const taskLock = await acquireDirectoryLock(join(taskDirectory, ".resume.lock"), {
+		wait: false,
+		signal: options.signal,
+		contentionMessage: `Historical task ${initialRun.id}#${taskNumber} already has a resume or response in progress.`,
+	});
+	try {
+		return await resumeLocked(cwd, runDirectory, runFile, taskDirectory, taskFile, prompt, options);
+	} finally {
+		await taskLock.release();
+	}
+}
+
+async function resumeLocked(cwd: string, runDirectory: string, runFile: string, taskDirectory: string, taskFile: string, prompt: string, options: ResumeHistoricalTaskOptions): Promise<ResumeHistoricalTaskResult> {
+	const herdr = options.herdr ?? new CliHerdrAutomation();
+	const task = await readTaskRecord(taskFile);
+	const initialRun = await readRunRecord(runFile);
+	if (!existsSync(task.sessionFile ?? "")) throw new Error(`Historical task ${initialRun.id}#${task.order} has no saved Pi session.`);
+	const attempt = Number(task.attempt ?? 1) + 1;
+	const attemptId = createAttemptId();
+	const ownerToken = randomUUID();
+	const agentName = liveAgentName(task.id, attempt);
+	const turnsDirectory = join(taskDirectory, "turns");
+	await mkdir(turnsDirectory, { recursive: true, mode: 0o700 });
+	const initialTurnFile = join(turnsDirectory, "01.json");
+	if (!existsSync(initialTurnFile)) {
+		let report: PersistedReport;
+		try { report = await readReport(join(taskDirectory, "report.json")); }
+		catch (error) { report = { status: "failed", summary: `Previous structured report was unavailable: ${error instanceof Error ? error.message : String(error)}`, documents: [] }; }
+		await writeJsonAtomic(initialTurnFile, { attempt: 1, prompt: task.prompt, status: task.status, summary: report.summary, question: report.question, error: report.error, documents: report.documents, sessionFile: task.sessionFile, startedAt: task.startedAt, completedAt: task.completedAt });
+	}
+	const turnFile = join(turnsDirectory, `${String(attempt).padStart(2, "0")}.json`);
+	const followupPrompt = join(taskDirectory, "followup-system-prompt.md");
+	await writeTextAtomic(followupPrompt, `${continuationSystemPrompt(task.systemPrompt, "Continue the requested historical work.", "followup")}\n`);
+	let allocation: { tabId: string; paneId: string; generation: number } | undefined;
+	let failed: { status: "cancelled" | "failed"; report: PersistedReport; error: unknown } | undefined;
+	const paneLabel = `${String(task.order).padStart(2, "0")} · ${task.name ?? task.agent ?? "worker"}`;
+	try {
+		allocation = await withRunStateLock(runFile, options.signal, async (run) => {
+			if (isActiveStatus(run.status) && (!run.resume || Object.keys(run.resume.active).length === 0)) {
+				throw new Error(`Subagent run '${run.id}' is still active. Use action 'respond' for blocked work or wait for it to settle before resuming history.`);
+			}
+			if (run.resume?.active[task.id]) throw new Error(`Historical task ${run.id}#${task.order} already has an active resume.`);
+			const env = childEnvironment(taskDirectory);
+			let generation: number;
+			let tabId: string;
+			let paneId: string;
+			let paneIds: string[];
+			if (run.resume?.phase === "active") {
+				generation = run.resume.generation;
+				tabId = run.resume.tabId;
+				const anchor = run.resume.paneIds.at(-1);
+				if (!anchor) throw new Error(`Resume generation ${generation} has no pane anchor.`);
+				paneId = (await herdr.splitPane({ paneId: anchor, cwd, direction: run.resume.paneIds.length % 2 ? "right" : "down", focus: false, env, signal: options.signal })).paneId;
+				paneIds = [...run.resume.paneIds, paneId];
+			} else {
+				generation = Number(run.resume?.generation ?? 0) + 1;
+				const created = await herdr.createTab({ workspaceId: process.env.HERDR_WORKSPACE_ID ?? "", cwd, label: `SA · ${run.label}`, env, focus: false, signal: options.signal });
+				tabId = created.tabId;
+				paneId = created.paneId;
+				paneIds = [paneId];
+			}
+			const member: ResumeMember = { paneId, generation, agentName, attemptId, ownerToken, phase: "allocating", ...ownerRecord(options.owner) };
+			run.resume = { generation, phase: "active", tabId, paneIds, active: { ...(run.resume?.phase === "active" ? run.resume.active : {}), [task.id]: member } };
+			applyRunStatus(run, "running");
+			Object.assign(run, { tabId, paneIds: [...new Set([...(run.paneIds ?? []), paneId])] });
+			return { tabId, paneId, generation };
+		});
+		await herdr.renamePane(allocation.paneId, paneLabel, options.signal);
+		applyTaskStatus(task, "running");
+		Object.assign(task, { attempt, currentAttemptId: attemptId, currentTurnFile: turnFile, tabId: allocation.tabId, paneId: allocation.paneId, paneLabel, agentName });
+		await Promise.all([
+			writeJsonAtomic(taskFile, task),
+			writeJsonAtomic(turnFile, { attempt, prompt, status: "running", sessionFile: task.sessionFile, tabId: allocation.tabId, paneId: allocation.paneId, agentName, startedAt: new Date().toISOString() }),
+		]);
+		await withRunStateLock(runFile, options.signal, (run) => {
+			const member = run.resume?.active[task.id];
+			if (member?.attemptId === attemptId && member.generation === allocation?.generation && member.phase === "allocating") member.phase = "running";
+		});
+		const runningEntries = await readTaskEntries(runDirectory);
+		options.onUpdate?.({ runId: initialRun.id, label: initialRun.label, status: "running", summary: `${task.name ?? task.agent} is continuing.`, documents: [], tabId: allocation.tabId, paneId: allocation.paneId, sessionFile: task.sessionFile as string, activity: runningEntries.map(({ task }, index) => projectTaskActivity(task, index)) });
+		let report: PersistedReport;
+		let status: "blocked" | "completed" | "failed";
+		try {
+			const prompted = await promptLiveAgent({ herdr, task: { ...task, sessionFile: task.sessionFile as string }, paneId: allocation.paneId, prompt, systemPromptFile: followupPrompt, agentName, attemptId, signal: options.signal, stalledWarningMs: options.stalledWarningSeconds ? options.stalledWarningSeconds * 1000 : undefined, onStalled: () => options.onStalled?.(`${task.name ?? task.agent} appears stalled; it is still running.`), onRuntimeUpdate: async (updated) => {
+				Object.assign(task, updated);
+				const liveEntries = await readTaskEntries(runDirectory);
+				options.onUpdate?.({ runId: initialRun.id, label: initialRun.label, status: "running", summary: `${task.name ?? task.agent} runtime: ${updated.runtimeState}.`, documents: [], tabId: allocation?.tabId ?? "", paneId: allocation?.paneId ?? "", sessionFile: task.sessionFile as string, activity: liveEntries.map(({ task }, index) => projectTaskActivity(task, index)) });
+			} });
+			report = prompted.report;
+			status = applyReport(task, report);
+		} catch (error) {
+			const settled = await settleAttemptFailure(task, join(taskDirectory, "report.json"), { cancelled: isAbortError(error, options.signal), error: error instanceof Error ? error.message : String(error), name: `${task.name ?? task.agent} follow-up`, replaceReport: true });
+			failed = { ...settled, error };
+			throw error;
+		}
+		await Promise.all([
+			writeJsonAtomic(taskFile, task),
+			writeJsonAtomic(turnFile, { attempt, prompt, status, summary: report.summary, documents: report.documents, sessionFile: task.sessionFile, tabId: allocation.tabId, paneId: allocation.paneId, agentName, completedAt: new Date().toISOString(), ...(report.question ? { question: report.question } : {}), ...(report.error ? { error: report.error } : {}) }),
+		]);
+		const cleanup = await settleMember(runDirectory, runFile, task.id, attemptId, allocation.generation, status, report.question, options);
+		if (cleanup) await closeClaimedTab(herdr, runFile, cleanup, options);
+		const finalEntries = await readTaskEntries(runDirectory);
+		return { runId: initialRun.id, label: initialRun.label, status, summary: report.summary, documents: report.documents, tabId: allocation.tabId, paneId: allocation.paneId, sessionFile: task.sessionFile as string, activity: finalEntries.map(({ task }, index) => projectTaskActivity(task, index)) };
+	} catch (error) {
+		const message = error instanceof Error ? error.message : String(error);
+		if (!failed) {
+			const settled = await settleAttemptFailure(task, join(taskDirectory, "report.json"), { cancelled: isAbortError(error, options.signal), error: message, name: `${task.name ?? task.agent} follow-up`, replaceReport: true });
+			failed = { ...settled, error };
+		}
+		task.attempt = attempt;
+		await Promise.all([
+			writeJsonAtomic(taskFile, task),
+			writeJsonAtomic(turnFile, { attempt, prompt, status: failed.status, summary: failed.report.summary, documents: failed.report.documents, error: message, sessionFile: task.sessionFile, ...(allocation ? { tabId: allocation.tabId, paneId: allocation.paneId } : {}), completedAt: new Date().toISOString() }),
+		]);
+		let cleanup: { tabId: string; generation: number } | undefined;
+		if (allocation) cleanup = await settleMember(runDirectory, runFile, task.id, attemptId, allocation.generation, failed.status, undefined, options);
+		else await withRunStateLock(runFile, options.signal, async (run) => applyRunStatus(run, await aggregateRun(runDirectory), { error: message }));
+		if (cleanup) await closeClaimedTab(herdr, runFile, cleanup, options);
+		const finalEntries = await readTaskEntries(runDirectory);
+		options.onUpdate?.({ runId: initialRun.id, label: initialRun.label, status: failed.status, summary: failed.report.summary, documents: failed.report.documents, tabId: allocation?.tabId ?? "", paneId: allocation?.paneId ?? "", sessionFile: task.sessionFile as string, activity: finalEntries.map(({ task }, index) => projectTaskActivity(task, index)) });
+		throw error;
+	}
+}
+
+async function settleMember(runDirectory: string, runFile: string, taskId: string, attemptId: string, generation: number, status: "blocked" | "completed" | "failed" | "cancelled", question: string | undefined, options: ResumeHistoricalTaskOptions): Promise<{ tabId: string; generation: number } | undefined> {
+	return withRunStateLock(runFile, options.signal, async (run) => {
+		const member = run.resume?.active[taskId];
+		if (!member || member.attemptId !== attemptId || member.generation !== generation) return undefined;
+		if (status === "blocked") member.phase = "blocked";
+		else if (run.resume) delete run.resume.active[taskId];
+		const aggregate = await aggregateRun(runDirectory, run.resume?.active);
+		applyRunStatus(run, aggregate, { question });
+		if (status === "blocked") return undefined;
+		return claimResumeCleanup(run);
+	});
+}
+
+async function closeClaimedTab(herdr: HerdrAutomation, runFile: string, claim: { tabId: string; generation: number }, options: ResumeHistoricalTaskOptions): Promise<void> {
+	const snapshot = await readRunRecord(runFile);
+	await cleanupRunTab({
+		herdr,
+		tabId: claim.tabId,
+		runFile,
+		runRecord: snapshot,
+		signal: options.signal,
+		onError: options.onCleanupError,
+		onClosed: (latest) => {
+			if (latest.resume?.generation === claim.generation && latest.resume.phase === "closing") delete latest.resume;
+		},
+	});
+}