@agwab/pi-workflow 0.2.1 → 0.4.0

package/src/subagent-backend.ts

@@ -1,3 +1,4 @@
+import { createHash } from "node:crypto";
 import { existsSync } from "node:fs";
 import {
 	copyFile,
@@ -24,6 +25,11 @@ import type {
 	CompiledTask,
 	CompiledToolProvider,
 	WorkflowRunRecord,
+	WorkflowTaskTimingAttemptRecord,
+	WorkflowTaskTimingRecord,
+	WorkflowTaskUsageAttemptRecord,
+	WorkflowTaskUsageRecord,
+	WorkflowTaskUsageValues,
 	WorkflowTaskRunRecord,
 } from "./types.js";
 import type { JsonSchema } from "./json-schema.js";
@@ -49,15 +55,26 @@ import {
 	parseWorkflowOutputForBundle,
 	writeWorkflowTaskArtifactBundle,
 } from "./workflow-output-artifacts.js";
+import { writeWorkflowPartialOutputLedgerFromFile } from "./workflow-partial-output.js";
 
 const DEFAULT_SUBAGENT_RUNS_ROOT = ".pi/workflow-subagents";
+const MAX_SUBAGENT_SESSION_ID_LENGTH = 64;
 const EXTRA_SUBAGENT_EXTENSIONS_ENV = "PI_WORKFLOW_SUBAGENT_EXTRA_EXTENSIONS";
 const FETCH_CONTENT_CACHE_ENV = "PI_WORKFLOW_FETCH_CONTENT_CACHE";
 const LEGACY_FETCH_CACHE_ENV = "PI_WORKFLOW_FETCH_CACHE";
+const FETCH_CONTENT_INLINE_CHARS_ENV = "PI_WORKFLOW_FETCH_CONTENT_INLINE_CHARS";
+const DEFAULT_WORKFLOW_FETCH_CONTENT_INLINE_CHARS = 12_000;
 const DEFAULT_TRANSIENT_MODEL_FAILURE_RETRIES = 5;
 const DEFAULT_ARTIFACT_OUTPUT_RETRIES = 2;
 const MAX_CONCURRENT_LAUNCHES_ENV = "PI_WORKFLOW_MAX_CONCURRENT_LAUNCHES";
+const LAUNCH_SLOT_RELEASE_DELAY_MS_ENV =
+	"PI_WORKFLOW_LAUNCH_SLOT_RELEASE_DELAY_MS";
+const PARENT_SUBAGENT_CWD_ENV = "PI_WORKFLOW_PARENT_SUBAGENT_CWD";
+const PARENT_SUBAGENT_RUNS_DIR_ENV = "PI_WORKFLOW_PARENT_SUBAGENT_RUNS_DIR";
+const PARENT_SUBAGENT_RUN_ID_ENV = "PI_WORKFLOW_PARENT_SUBAGENT_RUN_ID";
+const PARENT_SUBAGENT_ATTEMPT_ID_ENV = "PI_WORKFLOW_PARENT_SUBAGENT_ATTEMPT_ID";
 const DEFAULT_LAUNCH_SLOT_RELEASE_DELAY_MS = 3_000;
+const STALE_LAUNCH_CLAIM_GRACE_MS = 30_000;
 const MIN_TRANSIENT_RETRY_JITTER_MS = 1_000;
 const MAX_TRANSIENT_RETRY_JITTER_MS = 5_000;
 const MODULE_PATH = fileURLToPath(import.meta.url);
@@ -138,6 +155,7 @@ interface SubagentRunStatusSnapshot {
 	failureKind: string | null;
 	startedAt: string;
 	completedAt: string | null;
+	durationMs?: number | null;
 	logs: SubagentRunLogRef[];
 	metadata?: { contextLengthExceeded?: boolean; [key: string]: unknown };
 	completion?: unknown;
@@ -161,8 +179,29 @@ interface SubagentApi {
 	): Promise<SubagentRunStatusSnapshot | null>;
 	interruptSubagent(options: Record<string, unknown>): Promise<unknown>;
 	reconcileSubagentRun(options: Record<string, unknown>): Promise<unknown>;
+	recordSubagentChildEvent?(options: Record<string, unknown>): Promise<unknown>;
 }
 
+type ParentSubagentChildEvent =
+	| "started"
+	| "completed"
+	| "failed"
+	| "cancelled";
+
+interface ParentSubagentRef {
+	cwd: string;
+	runsDir: string;
+	runId: string;
+	attemptId?: string;
+}
+
+const GENERIC_TASK_STATUS_DETAILS = new Set([
+	"completed",
+	"failed",
+	"interrupted",
+	"running",
+]);
+
 const subagentApiSpecifier = "@agwab/pi-subagent/api";
 let cachedSubagentApi: Promise<SubagentApi> | undefined;
 let injectedSubagentApi: SubagentApi | undefined;
@@ -180,7 +219,86 @@ async function loadSubagentApi(): Promise<SubagentApi> {
 	return cachedSubagentApi;
 }
 
-let launchSlotReleaseDelayMs = DEFAULT_LAUNCH_SLOT_RELEASE_DELAY_MS;
+function nonEmptyEnv(
+	env: Record<string, string | undefined>,
+	key: string,
+): string | undefined {
+	const value = env[key]?.trim();
+	return value ? value : undefined;
+}
+
+function parentSubagentRefFromEnv(
+	env: Record<string, string | undefined> = process.env,
+): ParentSubagentRef | undefined {
+	const cwd = nonEmptyEnv(env, PARENT_SUBAGENT_CWD_ENV);
+	const runsDir = nonEmptyEnv(env, PARENT_SUBAGENT_RUNS_DIR_ENV);
+	const runId = nonEmptyEnv(env, PARENT_SUBAGENT_RUN_ID_ENV);
+	if (!cwd || !runsDir || !runId) return undefined;
+	const attemptId = nonEmptyEnv(env, PARENT_SUBAGENT_ATTEMPT_ID_ENV);
+	return { cwd, runsDir, runId, ...(attemptId ? { attemptId } : {}) };
+}
+
+function terminalChildEventForTaskStatus(
+	status: WorkflowTaskRunRecord["status"],
+): ParentSubagentChildEvent | undefined {
+	if (status === "completed") return "completed";
+	if (status === "failed") return "failed";
+	if (status === "interrupted") return "cancelled";
+	return undefined;
+}
+
+async function recordParentSubagentChildEvent(options: {
+	event: ParentSubagentChildEvent;
+	childRunId: string;
+	run: WorkflowRunRecord;
+	task: WorkflowTaskRunRecord;
+	failureKind?: string | null;
+	message?: string;
+}): Promise<void> {
+	const parent = parentSubagentRefFromEnv();
+	if (!parent) return;
+	const api = await loadSubagentApi().catch(() => undefined);
+	if (!api?.recordSubagentChildEvent) return;
+	await api
+		.recordSubagentChildEvent({
+			...parent,
+			event: options.event,
+			childRunId: options.childRunId,
+			workflowRunId: options.run.runId,
+			childTaskId: options.task.taskId,
+			...(options.failureKind === undefined
+				? {}
+				: { failureKind: options.failureKind }),
+			...(options.message === undefined ? {} : { message: options.message }),
+		})
+		.catch(() => undefined);
+}
+
+async function recordTerminalParentSubagentChildEvent(
+	run: WorkflowRunRecord,
+	task: WorkflowTaskRunRecord,
+	snapshot: SubagentRunStatusSnapshot,
+): Promise<void> {
+	const event = terminalChildEventForTaskStatus(task.status);
+	if (!event) return;
+	const taskFailureKind =
+		task.statusDetail && !GENERIC_TASK_STATUS_DETAILS.has(task.statusDetail)
+			? task.statusDetail
+			: undefined;
+	await recordParentSubagentChildEvent({
+		event,
+		childRunId: snapshot.runId,
+		run,
+		task,
+		failureKind:
+			event === "completed"
+				? undefined
+				: (snapshot.failureKind ?? taskFailureKind ?? task.statusDetail),
+		message: task.lastMessage,
+	});
+}
+
+let launchSlotReleaseDelayMsForTests: number | undefined;
 let transientRetryJitterForTests: (() => number) | undefined;
 const launchWaitQueue: Array<() => void> = [];
 let activeLaunchSlots = 0;
@@ -217,6 +335,18 @@ function releaseLaunchSlot(): void {
 	activeLaunchSlots = Math.max(0, activeLaunchSlots - 1);
 }
 
+function resolveLaunchSlotReleaseDelayMs(): number {
+	if (launchSlotReleaseDelayMsForTests !== undefined) {
+		return launchSlotReleaseDelayMsForTests;
+	}
+	const override = Number.parseInt(
+		process.env[LAUNCH_SLOT_RELEASE_DELAY_MS_ENV] ?? "",
+		10,
+	);
+	if (Number.isFinite(override)) return Math.max(0, Math.floor(override));
+	return DEFAULT_LAUNCH_SLOT_RELEASE_DELAY_MS;
+}
+
 function releaseLaunchSlotAfterDelay(
 	delayMs: number,
 	release: () => void,
@@ -225,12 +355,15 @@ function releaseLaunchSlotAfterDelay(
 		release();
 		return;
 	}
-	const timer = setTimeout(release, delayMs);
-	timer.unref?.();
+	setTimeout(release, delayMs);
 }
 
-async function runWithLaunchSlot<T>(action: () => Promise<T>): Promise<T> {
+async function runWithLaunchSlot<T>(
+	action: () => Promise<T>,
+	onAcquired?: () => void,
+): Promise<T> {
 	const release = await acquireLaunchSlot();
+	onAcquired?.();
 	let holdAfterReturn = false;
 	try {
 		const result = await action();
@@ -238,7 +371,7 @@ async function runWithLaunchSlot<T>(action: () => Promise<T>): Promise<T> {
 		return result;
 	} finally {
 		releaseLaunchSlotAfterDelay(
-			holdAfterReturn ? launchSlotReleaseDelayMs : 0,
+			holdAfterReturn ? resolveLaunchSlotReleaseDelayMs() : 0,
 			release,
 		);
 	}
@@ -259,13 +392,571 @@ function sleep(ms: number): Promise<void> {
 	return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+type UsageMetricKey = keyof WorkflowTaskUsageValues;
+const USAGE_METRIC_KEYS: UsageMetricKey[] = [
+	"inputTokens",
+	"outputTokens",
+	"totalTokens",
+	"cachedInputTokens",
+	"cacheCreationInputTokens",
+	"cacheReadInputTokens",
+	"reasoningTokens",
+	"costUsd",
+];
+const USAGE_FIELD_ALIASES: Record<
+	UsageMetricKey,
+	readonly (readonly string[])[]
+> = {
+	inputTokens: [
+		["inputTokens"],
+		["input_tokens"],
+		["input"],
+		["promptTokens"],
+		["prompt_tokens"],
+	],
+	outputTokens: [
+		["outputTokens"],
+		["output_tokens"],
+		["output"],
+		["completionTokens"],
+		["completion_tokens"],
+	],
+	totalTokens: [["totalTokens"], ["total_tokens"], ["tokens"], ["total"]],
+	cachedInputTokens: [
+		["cachedInputTokens"],
+		["cached_input_tokens"],
+		["prompt_tokens_details", "cached_tokens"],
+		["input_tokens_details", "cached_tokens"],
+	],
+	cacheCreationInputTokens: [
+		["cacheCreationInputTokens"],
+		["cacheCreationTokens"],
+		["cacheWriteTokens"],
+		["cache_creation_input_tokens"],
+		["cache_write_input_tokens"],
+		["cacheWrite"],
+		["cache_write"],
+	],
+	cacheReadInputTokens: [
+		["cacheReadInputTokens"],
+		["cacheReadTokens"],
+		["cache_read_input_tokens"],
+		["cacheRead"],
+		["cache_read"],
+	],
+	reasoningTokens: [
+		["reasoningTokens"],
+		["reasoning_tokens"],
+		["reasoning"],
+		["completion_tokens_details", "reasoning_tokens"],
+		["output_tokens_details", "reasoning_tokens"],
+	],
+	costUsd: [
+		["costUsd"],
+		["cost_usd"],
+		["totalCostUsd"],
+		["total_cost_usd"],
+		["estimatedCostUsd"],
+		["estimated_cost_usd"],
+		["cost", "total"],
+		["cost", "totalUsd"],
+		["cost", "total_usd"],
+	],
+};
+
+type TimingAggregateKey =
+	| "launchWaitMs"
+	| "launchDurationMs"
+	| "executionMs"
+	| "totalMs";
+const TIMING_AGGREGATE_KEYS: TimingAggregateKey[] = [
+	"launchWaitMs",
+	"launchDurationMs",
+	"executionMs",
+	"totalMs",
+];
+
+function isPlainRecord(value: unknown): value is Record<string, unknown> {
+	return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+
+function hasOwnValue(record: object, key: string): boolean {
+	return Object.hasOwn(record, key);
+}
+
+function valueAtPath(
+	record: Record<string, unknown>,
+	path: readonly string[],
+): { found: boolean; value: unknown } {
+	let current: unknown = record;
+	for (const part of path) {
+		if (!isPlainRecord(current) || !hasOwnValue(current, part)) {
+			return { found: false, value: undefined };
+		}
+		current = current[part];
+	}
+	return { found: true, value: current };
+}
+
+function usageNumberOrNull(value: unknown): number | null | undefined {
+	if (value === null) return null;
+	if (typeof value === "number" && Number.isFinite(value) && value >= 0) {
+		return value;
+	}
+	return undefined;
+}
+
+function normalizedUsageValues(raw: unknown): WorkflowTaskUsageValues {
+	const record = isPlainRecord(raw) ? raw : undefined;
+	const values: WorkflowTaskUsageValues = {};
+	if (!record) return values;
+	for (const key of USAGE_METRIC_KEYS) {
+		for (const path of USAGE_FIELD_ALIASES[key]) {
+			const candidate = valueAtPath(record, path);
+			if (!candidate.found) continue;
+			const value = usageNumberOrNull(candidate.value);
+			if (value === undefined) continue;
+			values[key] = value;
+			break;
+		}
+	}
+	return values;
+}
+
+function firstStringValue(
+	records: Array<Record<string, unknown> | undefined>,
+	keys: string[],
+): string | undefined {
+	for (const record of records) {
+		if (!record) continue;
+		for (const key of keys) {
+			const value = record[key];
+			if (typeof value === "string" && value.trim()) return value;
+		}
+	}
+	return undefined;
+}
+
+function metadataRecord(value: unknown): Record<string, unknown> | undefined {
+	if (!isPlainRecord(value)) return undefined;
+	return isPlainRecord(value.metadata) ? value.metadata : undefined;
+}
+
+function usageObservation(
+	subagentResult: Record<string, unknown> | undefined,
+	snapshot: SubagentRunStatusSnapshot,
+): { source: string; raw: unknown; present: true } | undefined {
+	const resultMetadata = metadataRecord(subagentResult);
+	if (resultMetadata && hasOwnValue(resultMetadata, "usage")) {
+		return {
+			source: "subagent-result-metadata",
+			raw: resultMetadata.usage,
+			present: true,
+		};
+	}
+	const snapshotMetadata = isPlainRecord(snapshot.metadata)
+		? snapshot.metadata
+		: undefined;
+	if (snapshotMetadata && hasOwnValue(snapshotMetadata, "usage")) {
+		return {
+			source: "subagent-snapshot-metadata",
+			raw: snapshotMetadata.usage,
+			present: true,
+		};
+	}
+	if (subagentResult && hasOwnValue(subagentResult, "usage")) {
+		return {
+			source: "subagent-result",
+			raw: subagentResult.usage,
+			present: true,
+		};
+	}
+	const snapshotRecord = snapshot as unknown as Record<string, unknown>;
+	if (hasOwnValue(snapshotRecord, "usage")) {
+		return {
+			source: "subagent-snapshot",
+			raw: snapshotRecord.usage,
+			present: true,
+		};
+	}
+	return undefined;
+}
+
+function buildTaskUsageAttempt(options: {
+	task: WorkflowTaskRunRecord;
+	snapshot: SubagentRunStatusSnapshot;
+	subagentResult?: Record<string, unknown>;
+	capturedAt: string;
+}): WorkflowTaskUsageAttemptRecord {
+	const resultMetadata = metadataRecord(options.subagentResult);
+	const snapshotMetadata = isPlainRecord(options.snapshot.metadata)
+		? options.snapshot.metadata
+		: undefined;
+	const resultRecord = options.subagentResult;
+	const snapshotRecord = options.snapshot as unknown as Record<string, unknown>;
+	const records = [
+		resultMetadata,
+		snapshotMetadata,
+		resultRecord,
+		snapshotRecord,
+	];
+	const observed = usageObservation(options.subagentResult, options.snapshot);
+	const raw = observed?.raw;
+	const unavailable = !observed || raw === null || raw === undefined;
+	const provider = firstStringValue(records, ["provider"]);
+	const model =
+		firstStringValue(records, ["model"]) ?? options.task.runtime.model;
+	const thinking =
+		firstStringValue(records, [
+			"thinking",
+			"thinkingLevel",
+			"reasoningLevel",
+		]) ??
+		options.task.runtime.thinkingResolution?.resolved ??
+		options.task.runtime.thinking;
+	return {
+		source: observed?.source ?? "subagent-usage-unavailable",
+		capturedAt: options.capturedAt,
+		backendRunId: options.snapshot.runId,
+		backendAttemptId: options.snapshot.attemptId,
+		...(provider === undefined ? {} : { provider }),
+		...(model === undefined ? {} : { model }),
+		...(thinking === undefined ? {} : { thinking }),
+		...(unavailable ? { unavailable: true as const } : {}),
+		...(observed?.present && raw !== undefined ? { raw } : {}),
+		...normalizedUsageValues(raw),
+	};
+}
+
+function usageAttemptKey(attempt: WorkflowTaskUsageAttemptRecord): string {
+	return `${attempt.backendRunId ?? ""}\0${attempt.backendAttemptId ?? ""}\0${attempt.source}`;
+}
+
+function upsertUsageAttempt(
+	attempts: WorkflowTaskUsageAttemptRecord[],
+	attempt: WorkflowTaskUsageAttemptRecord,
+): WorkflowTaskUsageAttemptRecord[] {
+	const key = usageAttemptKey(attempt);
+	const index = attempts.findIndex(
+		(candidate) => usageAttemptKey(candidate) === key,
+	);
+	if (index < 0) return [...attempts, attempt];
+	return attempts.map((candidate, candidateIndex) =>
+		candidateIndex === index ? attempt : candidate,
+	);
+}
+
+function aggregateUsageAttempts(attempts: WorkflowTaskUsageAttemptRecord[]): {
+	values: WorkflowTaskUsageValues;
+	incomplete: boolean;
+} {
+	const values: WorkflowTaskUsageValues = {};
+	let incomplete = attempts.some((attempt) => attempt.unavailable === true);
+	for (const key of USAGE_METRIC_KEYS) {
+		const anyPresent = attempts.some((attempt) => hasOwnValue(attempt, key));
+		if (!anyPresent) continue;
+		let total = 0;
+		let complete = true;
+		for (const attempt of attempts) {
+			if (!hasOwnValue(attempt, key)) {
+				complete = false;
+				break;
+			}
+			const value = attempt[key];
+			if (typeof value !== "number") {
+				complete = false;
+				break;
+			}
+			total += value;
+		}
+		values[key] = complete ? total : null;
+		if (!complete) incomplete = true;
+	}
+	return { values, incomplete };
+}
+
+function latestUsageString(
+	attempts: WorkflowTaskUsageAttemptRecord[],
+	key: "provider" | "model" | "thinking",
+): string | undefined {
+	for (let index = attempts.length - 1; index >= 0; index -= 1) {
+		const value = attempts[index]?.[key];
+		if (typeof value === "string" && value.trim()) return value;
+	}
+	return undefined;
+}
+
+function recordTaskUsageObservation(options: {
+	task: WorkflowTaskRunRecord;
+	snapshot: SubagentRunStatusSnapshot;
+	subagentResult?: Record<string, unknown>;
+	capturedAt: string;
+}): void {
+	const attempt = buildTaskUsageAttempt(options);
+	const attempts = upsertUsageAttempt(
+		options.task.usage?.attempts ?? [],
+		attempt,
+	);
+	const aggregate = aggregateUsageAttempts(attempts);
+	const usage: WorkflowTaskUsageRecord = {
+		source: "pi-subagent",
+		capturedAt: options.capturedAt,
+		...(latestUsageString(attempts, "provider") === undefined
+			? {}
+			: { provider: latestUsageString(attempts, "provider") }),
+		...(latestUsageString(attempts, "model") === undefined
+			? {}
+			: { model: latestUsageString(attempts, "model") }),
+		...(latestUsageString(attempts, "thinking") === undefined
+			? {}
+			: { thinking: latestUsageString(attempts, "thinking") }),
+		...(aggregate.incomplete ? { incomplete: true } : {}),
+		...aggregate.values,
+		aggregate: {
+			attempts: attempts.length,
+			...(aggregate.incomplete ? { incomplete: true } : {}),
+			...aggregate.values,
+		},
+		attempts,
+	};
+	options.task.usage = usage;
+}
+
+function isoTimestampMs(timestamp: string | undefined): number | undefined {
+	if (!timestamp) return undefined;
+	const parsed = Date.parse(timestamp);
+	return Number.isFinite(parsed) ? parsed : undefined;
+}
+
+function durationBetween(
+	startedAt: string | undefined,
+	completedAt: string | undefined,
+): number | undefined {
+	const startedAtMs = isoTimestampMs(startedAt);
+	const completedAtMs = isoTimestampMs(completedAt);
+	if (startedAtMs === undefined || completedAtMs === undefined)
+		return undefined;
+	return Math.max(0, completedAtMs - startedAtMs);
+}
+
+function durationNumber(value: unknown): number | null | undefined {
+	if (value === null) return null;
+	if (typeof value === "number" && Number.isFinite(value) && value >= 0) {
+		return value;
+	}
+	return undefined;
+}
+
+function recordTaskLaunchTiming(
+	task: WorkflowTaskRunRecord,
+	observation: {
+		launchQueuedAt: string;
+		launchStartedAt?: string;
+		launchCompletedAt?: string;
+	},
+): void {
+	const capturedAt = observation.launchCompletedAt ?? nowIso();
+	const launchWaitMs = durationBetween(
+		observation.launchQueuedAt,
+		observation.launchStartedAt,
+	);
+	const launchDurationMs = durationBetween(
+		observation.launchStartedAt,
+		observation.launchCompletedAt,
+	);
+	task.timing = {
+		source: "pi-workflow",
+		capturedAt,
+		launchQueuedAt: observation.launchQueuedAt,
+		...(observation.launchStartedAt === undefined
+			? {}
+			: { launchStartedAt: observation.launchStartedAt }),
+		...(observation.launchCompletedAt === undefined
+			? {}
+			: { launchCompletedAt: observation.launchCompletedAt }),
+		...(launchWaitMs === undefined ? {} : { launchWaitMs }),
+		...(launchDurationMs === undefined ? {} : { launchDurationMs }),
+		launchSlotReleaseDelayMs: resolveLaunchSlotReleaseDelayMs(),
+		...(task.timing?.aggregate === undefined
+			? {}
+			: { aggregate: task.timing.aggregate }),
+		...(task.timing?.attempts === undefined
+			? {}
+			: { attempts: task.timing.attempts }),
+	};
+}
+
+function buildTaskTimingAttempt(options: {
+	task: WorkflowTaskRunRecord;
+	snapshot: SubagentRunStatusSnapshot;
+	subagentResult?: Record<string, unknown>;
+	startedAt?: string;
+	completedAt?: string;
+	capturedAt: string;
+}): WorkflowTaskTimingAttemptRecord {
+	const resultDuration = options.subagentResult?.durationMs;
+	let executionMs = durationNumber(
+		resultDuration === undefined ? options.snapshot.durationMs : resultDuration,
+	);
+	if (executionMs === undefined || executionMs === null) {
+		executionMs =
+			durationBetween(options.startedAt, options.completedAt) ?? executionMs;
+	}
+	const totalMs = durationBetween(
+		options.task.startedAt ?? options.task.timing?.launchQueuedAt,
+		options.completedAt,
+	);
+	return {
+		source: "pi-subagent",
+		capturedAt: options.capturedAt,
+		backendRunId: options.snapshot.runId,
+		backendAttemptId: options.snapshot.attemptId,
+		...(options.task.timing?.launchQueuedAt === undefined
+			? {}
+			: { launchQueuedAt: options.task.timing.launchQueuedAt }),
+		...(options.task.timing?.launchStartedAt === undefined
+			? {}
+			: { launchStartedAt: options.task.timing.launchStartedAt }),
+		...(options.task.timing?.launchCompletedAt === undefined
+			? {}
+			: { launchCompletedAt: options.task.timing.launchCompletedAt }),
+		...(options.task.timing?.launchWaitMs === undefined
+			? {}
+			: { launchWaitMs: options.task.timing.launchWaitMs }),
+		...(options.task.timing?.launchDurationMs === undefined
+			? {}
+			: { launchDurationMs: options.task.timing.launchDurationMs }),
+		...(options.startedAt === undefined
+			? {}
+			: { executionStartedAt: options.startedAt }),
+		...(options.completedAt === undefined
+			? {}
+			: { executionCompletedAt: options.completedAt }),
+		...(executionMs === undefined ? {} : { executionMs }),
+		...(totalMs === undefined ? {} : { totalMs }),
+	};
+}
+
+function timingAttemptKey(attempt: WorkflowTaskTimingAttemptRecord): string {
+	return `${attempt.backendRunId ?? ""}\0${attempt.backendAttemptId ?? ""}`;
+}
+
+function upsertTimingAttempt(
+	attempts: WorkflowTaskTimingAttemptRecord[],
+	attempt: WorkflowTaskTimingAttemptRecord,
+): WorkflowTaskTimingAttemptRecord[] {
+	const key = timingAttemptKey(attempt);
+	const index = attempts.findIndex(
+		(candidate) => timingAttemptKey(candidate) === key,
+	);
+	if (index < 0) return [...attempts, attempt];
+	return attempts.map((candidate, candidateIndex) =>
+		candidateIndex === index ? attempt : candidate,
+	);
+}
+
+function aggregateTimingAttempts(
+	attempts: WorkflowTaskTimingAttemptRecord[],
+): NonNullable<WorkflowTaskTimingRecord["aggregate"]> {
+	const aggregate: NonNullable<WorkflowTaskTimingRecord["aggregate"]> = {
+		attempts: attempts.length,
+	};
+	let incomplete = false;
+	for (const key of TIMING_AGGREGATE_KEYS) {
+		const anyPresent = attempts.some((attempt) => hasOwnValue(attempt, key));
+		if (!anyPresent) continue;
+		let total = 0;
+		let complete = true;
+		for (const attempt of attempts) {
+			if (!hasOwnValue(attempt, key)) {
+				complete = false;
+				break;
+			}
+			const value = attempt[key];
+			if (typeof value !== "number") {
+				complete = false;
+				break;
+			}
+			total += value;
+		}
+		aggregate[key] = complete ? total : null;
+		if (!complete) incomplete = true;
+	}
+	if (incomplete) aggregate.incomplete = true;
+	return aggregate;
+}
+
+function recordTaskTerminalTiming(options: {
+	task: WorkflowTaskRunRecord;
+	snapshot: SubagentRunStatusSnapshot;
+	subagentResult?: Record<string, unknown>;
+	startedAt?: string;
+	completedAt?: string;
+	capturedAt: string;
+}): void {
+	const attempt = buildTaskTimingAttempt(options);
+	const attempts = upsertTimingAttempt(
+		options.task.timing?.attempts ?? [],
+		attempt,
+	);
+	options.task.timing = {
+		source: "pi-workflow",
+		capturedAt: options.capturedAt,
+		...(attempt.launchQueuedAt === undefined
+			? {}
+			: { launchQueuedAt: attempt.launchQueuedAt }),
+		...(attempt.launchStartedAt === undefined
+			? {}
+			: { launchStartedAt: attempt.launchStartedAt }),
+		...(attempt.launchCompletedAt === undefined
+			? {}
+			: { launchCompletedAt: attempt.launchCompletedAt }),
+		...(attempt.launchWaitMs === undefined
+			? {}
+			: { launchWaitMs: attempt.launchWaitMs }),
+		...(attempt.launchDurationMs === undefined
+			? {}
+			: { launchDurationMs: attempt.launchDurationMs }),
+		...(options.task.timing?.launchSlotReleaseDelayMs === undefined
+			? {}
+			: {
+					launchSlotReleaseDelayMs:
+						options.task.timing.launchSlotReleaseDelayMs,
+				}),
+		...(attempt.executionStartedAt === undefined
+			? {}
+			: { executionStartedAt: attempt.executionStartedAt }),
+		...(attempt.executionCompletedAt === undefined
+			? {}
+			: { executionCompletedAt: attempt.executionCompletedAt }),
+		...(attempt.executionMs === undefined
+			? {}
+			: { executionMs: attempt.executionMs }),
+		...(attempt.totalMs === undefined ? {} : { totalMs: attempt.totalMs }),
+		aggregate: aggregateTimingAttempts(attempts),
+		attempts,
+	};
+}
+
+function recordTerminalTaskObservability(options: {
+	task: WorkflowTaskRunRecord;
+	snapshot: SubagentRunStatusSnapshot;
+	subagentResult?: Record<string, unknown>;
+	startedAt?: string;
+	completedAt?: string;
+}): void {
+	const capturedAt = nowIso();
+	recordTaskUsageObservation({ ...options, capturedAt });
+	recordTaskTerminalTiming({ ...options, capturedAt });
+}
+
 export function setSubagentLaunchControlsForTests(options?: {
 	releaseDelayMs?: number;
 	retryJitterMs?: number | (() => number);
 }): void {
-	launchSlotReleaseDelayMs =
+	launchSlotReleaseDelayMsForTests =
 		options?.releaseDelayMs === undefined
-			? DEFAULT_LAUNCH_SLOT_RELEASE_DELAY_MS
+			? undefined
 			: Math.max(0, Math.floor(options.releaseDelayMs));
 	transientRetryJitterForTests =
 		options?.retryJitterMs === undefined
@@ -282,7 +973,6 @@ export async function cleanupSubagentRun(
 	run: WorkflowRunRecord,
 ): Promise<void> {
 	for (const task of run.tasks) {
-		if (isTerminalTaskStatus(task.status)) continue;
 		const handle = getSubagentHandle(task);
 		if (!handle) continue;
 		const api = await loadSubagentApi();
@@ -377,11 +1067,25 @@ export async function launchSubagentTask(
 		};
 		subagentOptions.extensions = extensions;
 		if (captureToolCallsEnabled()) subagentOptions.captureToolCalls = true;
+		const launchQueuedAt = nowIso();
+		let launchStartedAt: string | undefined;
+		recordTaskLaunchTiming(task, { launchQueuedAt });
 		if (isLaunchGateSaturated()) {
 			task.lastMessage = `waiting for pi-subagent launch slot (${resolveMaxConcurrentLaunches()} max)`;
 			await writeRunRecord(cwd, run).catch(() => undefined);
 		}
-		launched = await runWithLaunchSlot(() => api.runSubagent(subagentOptions));
+		launched = await runWithLaunchSlot(
+			() => api.runSubagent(subagentOptions),
+			() => {
+				launchStartedAt = nowIso();
+				recordTaskLaunchTiming(task, { launchQueuedAt, launchStartedAt });
+			},
+		);
+		recordTaskLaunchTiming(task, {
+			launchQueuedAt,
+			launchStartedAt,
+			launchCompletedAt: nowIso(),
+		});
 	} catch (error) {
 		task.status = "pending";
 		task.statusDetail = "pending";
@@ -409,6 +1113,13 @@ export async function launchSubagentTask(
 	task.statusDetail = "running";
 	task.lastMessage = "launched via pi-subagent/headless";
 	await writeRunRecord(cwd, run).catch(() => undefined);
+	await recordParentSubagentChildEvent({
+		event: "started",
+		childRunId: launched.runId,
+		run,
+		task,
+		message: task.lastMessage,
+	});
 	return { kind: "launched" };
 }
 
@@ -440,8 +1151,13 @@ export async function refreshRunFromSubagentArtifacts(
 			}
 		}
 		if (!handle) {
+			if (isStaleLaunchClaim(task)) {
+				resetStaleLaunchClaim(task);
+				changed = true;
+				continue;
+			}
 			if (isTaskTimedOut(task)) {
-				markTaskTimedOut(task);
+				markSubagentTaskTimedOut(task);
 				changed = true;
 			}
 			continue;
@@ -466,16 +1182,8 @@ export async function refreshRunFromSubagentArtifacts(
 
 		if (snapshot === null) {
 			if (isTaskTimedOut(task)) {
-				await api
-					.interruptSubagent({
-						cwd: handle.cwd,
-						runsDir: handle.runsDir,
-						runId: handle.runId,
-						attemptId: handle.attemptId,
-						reason: "workflow timeout",
-					})
-					.catch(() => undefined);
-				markTaskTimedOut(task);
+				await interruptTimedOutSubagent(api, handle);
+				markSubagentTaskTimedOut(task);
 				changed = true;
 			}
 			continue;
@@ -485,23 +1193,29 @@ export async function refreshRunFromSubagentArtifacts(
 			snapshot.attempts?.find(
 				(attempt) => attempt.attemptId === handle.attemptId,
 			) ?? snapshot.attempts?.at(-1);
-		task.pid = activeAttempt?.workerPid ?? activeAttempt?.pid ?? task.pid;
+		const nextPid = activeAttempt?.workerPid ?? activeAttempt?.pid ?? task.pid;
+		if (task.pid !== nextPid) {
+			task.pid = nextPid;
+			changed = true;
+		}
 		if (snapshot.status === "running" || snapshot.status === "pending") {
-			task.statusDetail = "running";
-			task.lastMessage = activeAttempt?.heartbeatAt
+			await refreshRunningArtifactGraphPartialOutput(cwd, task, snapshot).catch(
+				() => undefined,
+			);
+			if (task.statusDetail !== "running") {
+				task.statusDetail = "running";
+				changed = true;
+			}
+			const nextLastMessage = activeAttempt?.heartbeatAt
 				? `pi-subagent heartbeat ${activeAttempt.heartbeatAt}`
 				: "pi-subagent running";
+			if (task.lastMessage !== nextLastMessage) {
+				task.lastMessage = nextLastMessage;
+				changed = true;
+			}
 			if (isTaskTimedOut(task)) {
-				await api
-					.interruptSubagent({
-						cwd: handle.cwd,
-						runsDir: handle.runsDir,
-						runId: handle.runId,
-						attemptId: handle.attemptId,
-						reason: "workflow timeout",
-					})
-					.catch(() => undefined);
-				markTaskTimedOut(task);
+				await interruptTimedOutSubagent(api, handle);
+				markSubagentTaskTimedOut(task);
 				changed = true;
 			}
 			continue;
@@ -515,6 +1229,68 @@ export async function refreshRunFromSubagentArtifacts(
 	return run;
 }
 
+async function refreshRunningArtifactGraphPartialOutput(
+	cwd: string,
+	task: WorkflowTaskRunRecord,
+	snapshot: SubagentRunStatusSnapshot,
+): Promise<void> {
+	const partial = task.artifactGraph?.output.partial;
+	if (!partial || partial.paths.length === 0) return;
+	const outputRef = findLog(snapshot, "output");
+	const outputFile = fromProjectPath(cwd, task.files.output);
+	const artifactRoot = task.backendFiles?.runsDir
+		? fromProjectPath(task.cwd, task.backendFiles.runsDir)
+		: undefined;
+	await copyLogOrEmpty(snapshot, outputRef, outputFile, artifactRoot);
+	await writeWorkflowPartialOutputLedgerFromFile({
+		taskDir: dirname(fromProjectPath(cwd, task.files.result)),
+		outputFile,
+		allowedPaths: partial.paths,
+	});
+}
+
+async function interruptTimedOutSubagent(
+	api: Awaited<ReturnType<typeof loadSubagentApi>>,
+	handle: NonNullable<WorkflowTaskRunRecord["backendHandle"]>,
+): Promise<void> {
+	await api
+		.interruptSubagent({
+			cwd: handle.cwd,
+			runsDir: handle.runsDir,
+			runId: handle.runId,
+			attemptId: handle.attemptId,
+			reason: "workflow timeout",
+		})
+		.catch(() => undefined);
+}
+
+function markSubagentTaskTimedOut(task: WorkflowTaskRunRecord): void {
+	markTaskTimedOut(task);
+	task.backendHandle = undefined;
+	task.backendTaskId = task.taskId;
+	task.pid = undefined;
+}
+
+function isStaleLaunchClaim(task: WorkflowTaskRunRecord): boolean {
+	if (task.statusDetail !== "launching" || !task.startedAt) return false;
+	const startedAtMs = Date.parse(task.startedAt);
+	return (
+		Number.isFinite(startedAtMs) &&
+		Date.now() - startedAtMs > STALE_LAUNCH_CLAIM_GRACE_MS
+	);
+}
+
+function resetStaleLaunchClaim(task: WorkflowTaskRunRecord): void {
+	task.status = "pending";
+	task.statusDetail = "pending";
+	task.startedAt = undefined;
+	task.backendHandle = undefined;
+	task.backendFiles = undefined;
+	task.backendTaskId = task.taskId;
+	task.pid = undefined;
+	task.lastMessage = "stale pi-subagent launch claim reset";
+}
+
 async function materializeTerminalSubagentResult(
 	cwd: string,
 	run: WorkflowRunRecord,
@@ -592,16 +1368,30 @@ async function materializeTerminalSubagentResult(
 		(subagentResult?.metadata as any)?.contextLengthExceeded ??
 			snapshot.metadata?.contextLengthExceeded,
 	);
+	recordTerminalTaskObservability({
+		task,
+		snapshot,
+		subagentResult,
+		startedAt,
+		completedAt,
+	});
 	if (task.artifactGraph?.enabled && statusInfo.status === "completed") {
-		return await materializeTerminalArtifactGraphResult(cwd, run, task, {
-			outputFile,
-			stderrFile,
-			resultFile,
-			completedAt,
-			startedAt,
-			exitCode,
-			subagentResult,
-		});
+		const changed = await materializeTerminalArtifactGraphResult(
+			cwd,
+			run,
+			task,
+			{
+				outputFile,
+				stderrFile,
+				resultFile,
+				completedAt,
+				startedAt,
+				exitCode,
+				subagentResult,
+			},
+		);
+		await recordTerminalParentSubagentChildEvent(run, task, snapshot);
+		return changed;
 	}
 	if (
 		shouldAttemptArtifactGraphSalvage({
@@ -615,20 +1405,28 @@ async function materializeTerminalSubagentResult(
 			snapshot,
 		})
 	) {
-		return await materializeTerminalArtifactGraphResult(cwd, run, task, {
-			outputFile,
-			stderrFile,
-			resultFile,
-			completedAt,
-			startedAt,
-			exitCode,
-			subagentResult,
-			salvage: {
-				failureKind: statusInfo.failureKind ?? snapshot.failureKind ?? "model",
-				subagentStatus: snapshot.status,
-				subagentFailureKind: snapshot.failureKind,
+		const changed = await materializeTerminalArtifactGraphResult(
+			cwd,
+			run,
+			task,
+			{
+				outputFile,
+				stderrFile,
+				resultFile,
+				completedAt,
+				startedAt,
+				exitCode,
+				subagentResult,
+				salvage: {
+					failureKind:
+						statusInfo.failureKind ?? snapshot.failureKind ?? "model",
+					subagentStatus: snapshot.status,
+					subagentFailureKind: snapshot.failureKind,
+				},
 			},
-		});
+		);
+		await recordTerminalParentSubagentChildEvent(run, task, snapshot);
+		return changed;
 	}
 	const workflowResult = {
 		status: statusInfo.status,
@@ -664,10 +1462,12 @@ async function materializeTerminalSubagentResult(
 			),
 			workflowResult,
 		);
-		return retryOrFailTransientSubagentFailure(task, {
+		const changed = retryOrFailTransientSubagentFailure(task, {
 			reason: statusInfo.failureKind ?? "model",
 			message: errorMessage ?? "pi-subagent run failed before producing output",
 		});
+		await recordTerminalParentSubagentChildEvent(run, task, snapshot);
+		return changed;
 	}
 	await writeJson(resultFile, workflowResult);
 
@@ -682,6 +1482,7 @@ async function materializeTerminalSubagentResult(
 		delete task.backendHandle;
 		delete task.backendFiles;
 	}
+	await recordTerminalParentSubagentChildEvent(run, task, snapshot);
 	return changed;
 }
 
@@ -737,6 +1538,13 @@ async function materializeTerminalArtifactGraphResult(
 ): Promise<boolean> {
 	const rawOutput = await readFile(options.outputFile, "utf8").catch(() => "");
 	const artifactOptions = task.artifactGraph?.output;
+	if (artifactOptions?.partial && artifactOptions.partial.paths.length > 0) {
+		await writeWorkflowPartialOutputLedgerFromFile({
+			taskDir: dirname(options.resultFile),
+			outputFile: options.outputFile,
+			allowedPaths: artifactOptions.partial.paths,
+		}).catch(() => undefined);
+	}
 	let controlJsonSchema: JsonSchema | undefined;
 	try {
 		controlJsonSchema = await readTaskControlJsonSchema(task);
@@ -1432,6 +2240,7 @@ async function workflowTaskExtensions(
 					"source-cache",
 					"fetch-content",
 				),
+				maxInlineChars: fetchContentInlineCharsEnvValue(),
 			},
 		});
 		extensions = uniqueStrings([
@@ -1536,6 +2345,17 @@ function fetchContentCacheEnvValue(): string | undefined {
 	);
 }
 
+function fetchContentInlineCharsEnvValue(): number | undefined {
+	const raw = process.env[FETCH_CONTENT_INLINE_CHARS_ENV];
+	if (raw === undefined || raw.trim() === "")
+		return DEFAULT_WORKFLOW_FETCH_CONTENT_INLINE_CHARS;
+	if (isExplicitlyDisabled(raw)) return undefined;
+	const parsed = Number(raw);
+	if (!Number.isFinite(parsed))
+		return DEFAULT_WORKFLOW_FETCH_CONTENT_INLINE_CHARS;
+	return Math.max(1, Math.floor(parsed));
+}
+
 function isExplicitlyDisabled(value: string | undefined): boolean {
 	return typeof value === "string" && /^(0|false|no|off)$/i.test(value.trim());
 }
@@ -1674,6 +2494,7 @@ async function recoverSubagentHandle(
 	const runsDir = subagentRunsDir(run, task);
 	const absoluteRunsDir = resolve(task.cwd, runsDir);
 	const expectedCorrelationId = `${run.runId}:${task.taskId}`;
+	const claimStartedAtMs = timestampMs(task.startedAt);
 	const entries = await readdir(absoluteRunsDir, { withFileTypes: true }).catch(
 		() => [],
 	);
@@ -1688,6 +2509,7 @@ async function recoverSubagentHandle(
 			join(absoluteRunsDir, entry.name, "run.json"),
 		);
 		if (!record || record.correlationId !== expectedCorrelationId) continue;
+		if (isPreClaimSubagentRecord(record, claimStartedAtMs)) continue;
 		const attemptId =
 			record.activeAttemptId ??
 			record.latestAttemptId ??
@@ -1714,6 +2536,20 @@ async function recoverSubagentHandle(
 	return candidates[0]?.handle;
 }
 
+function isPreClaimSubagentRecord(
+	record: SubagentRunRecordLike,
+	claimStartedAtMs: number | undefined,
+): boolean {
+	if (claimStartedAtMs === undefined) return false;
+	const recordStartedAtMs =
+		timestampMs(record.startedAt) ??
+		timestampMs(record.attempts?.[0]?.startedAt) ??
+		timestampMs(record.updatedAt);
+	return (
+		recordStartedAtMs !== undefined && recordStartedAtMs < claimStartedAtMs
+	);
+}
+
 function timestampMs(value: string | undefined): number | undefined {
 	if (value === undefined) return undefined;
 	const time = Date.parse(value);
@@ -1774,17 +2610,24 @@ function subagentSessionId(
 	task: WorkflowTaskRunRecord,
 ): string | undefined {
 	if (!task.artifactGraph?.enabled) return undefined;
-	return task.outputRetry?.sessionId ?? baseSubagentSessionId(run, task);
+	const baseSessionId = baseSubagentSessionId(run, task);
+	if (task.outputRetry?.sessionId) return task.outputRetry.sessionId;
+	const launchAttempt = task.launchRetry?.attempts ?? 0;
+	if (launchAttempt > 0)
+		return boundedSubagentSessionId(
+			`${baseSessionId}.launch-retry-${launchAttempt}`,
+		);
+	const resumeAttempt = task.resumeEvents?.length ?? 0;
+	if (resumeAttempt > 0)
+		return boundedSubagentSessionId(`${baseSessionId}.resume-${resumeAttempt}`);
+	return baseSessionId;
 }
 
 function baseSubagentSessionId(
 	run: WorkflowRunRecord,
 	task: WorkflowTaskRunRecord,
 ): string {
-	return `pi-workflow.${run.runId}.${task.taskId}`.replace(
-		/[^A-Za-z0-9._-]/g,
-		"-",
-	);
+	return boundedSubagentSessionId(`pi-workflow.${run.runId}.${task.taskId}`);
 }
 
 function retrySubagentSessionId(
@@ -1792,7 +2635,23 @@ function retrySubagentSessionId(
 	task: WorkflowTaskRunRecord,
 	attempt: number,
 ): string {
-	return `${baseSubagentSessionId(run, task)}.retry-${attempt}`;
+	return boundedSubagentSessionId(
+		`${baseSubagentSessionId(run, task)}.retry-${attempt}`,
+	);
+}
+
+function boundedSubagentSessionId(value: string): string {
+	const sanitized = value.replace(/[^A-Za-z0-9._-]/g, "-");
+	if (sanitized.length <= MAX_SUBAGENT_SESSION_ID_LENGTH) return sanitized;
+	const digest = createHash("sha256")
+		.update(sanitized)
+		.digest("hex")
+		.slice(0, 16);
+	const suffix = sanitized.split(".").at(-1) || "session";
+	const prefix = `piwf.${digest}`;
+	const maxSuffixLength = MAX_SUBAGENT_SESSION_ID_LENGTH - prefix.length - 1;
+	const boundedSuffix = suffix.slice(-Math.max(1, maxSuffixLength));
+	return `${prefix}.${boundedSuffix}`;
 }
 
 function buildSystemPrompt(task: CompiledTask): string {