@matthewfl/pi-contemplator 0.1.1 → 0.1.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@matthewfl/pi-contemplator",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "description": "A Pi extension that keeps long-running agentic sessions on track with background memory, contemplation, and structural review.",
   "type": "module",
   "license": "MIT",
@@ -41,7 +41,7 @@
     "typecheck": "tsc --noEmit",
     "test": "npm run test:unit && npm run test:e2e",
     "test:unit": "vitest run",
-    "test:e2e": "node tests/e2e/rpc-contemplator.mjs && node tests/e2e/rpc-summarizer.mjs && node tests/e2e/rpc-delivery.mjs && node tests/e2e/rpc-restore-review.mjs && node tests/e2e/rpc-compaction.mjs && node tests/e2e/rpc-compaction-resilience.mjs && node tests/e2e/rpc-memory-edges.mjs && node tests/e2e/rpc-routing-isolation.mjs"
+    "test:e2e": "node tests/e2e/rpc-contemplator.mjs && node tests/e2e/rpc-summarizer.mjs && node tests/e2e/rpc-delivery.mjs && node tests/e2e/rpc-restore-review.mjs && node tests/e2e/rpc-compaction.mjs && node tests/e2e/rpc-compaction-resilience.mjs && node tests/e2e/rpc-memory-edges.mjs && node tests/e2e/rpc-observer-length.mjs && node tests/e2e/rpc-routing-isolation.mjs"
   },
   "peerDependencies": {
     "@earendil-works/pi-agent-core": "*",

package/src/agents/contemplator/agent.ts

@@ -5,7 +5,7 @@ import { streamSimple } from "@earendil-works/pi-ai/compat";
 import { generateSummaryWithUsage } from "@earendil-works/pi-coding-agent";
 import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
 import { Box, Text } from "@earendil-works/pi-tui";
-import { agentActiveTimeMs, assistantOutputTokens, assistantToolCallCount, fullProjection, isReviewRequestEntry, isReviewResultEntry, OM_AGENT_ACTIVITY, OM_REVIEWER_MESSAGE, OM_REVIEWER_NOTICE, OM_REVIEWER_STATE, OM_REVIEW_REQUEST, OM_REVIEW_RESULT, recallMemorySources, type Entry, type ReviewResult, type StructuralReviewRequest } from "../../session-ledger/index.js";
+import { agentActiveTimeMs, assistantOutputTokens, assistantToolCallCount, fullProjection, isReviewRequestEntry, isReviewResultEntry, OM_AGENT_ACTIVITY, OM_REVIEWER_MESSAGE, OM_REVIEWER_NOTICE, OM_REVIEWER_STATE, OM_REVIEW_REQUEST, OM_REVIEW_RESULT, rawTokensSinceObservationCoverage, recallMemorySources, type Entry, type ReviewResult, type StructuralReviewRequest } from "../../session-ledger/index.js";
 import { hashId } from "../../ids.js";
 import { createSearchMemoriesAgentTool } from "../../tools/search-memories.js";
 import { createRecallAgentTool } from "../../tools/recall-observation.js";
@@ -17,6 +17,7 @@ import { forceRequiredToolPayload, requiredToolChoice } from "../../required-too
 import { memoryReferenceIds } from "../../memory-citations.js";
 import { buildContemplatorSystemPrompt } from "./prompts.js";
 import { runStructuralReview } from "../reviewer/agent.js";
+import { createWorkerStallWatchdog } from "../../worker-watchdog.js";
 
 interface PendingUpdate {
 	observations: string[];
@@ -213,6 +214,10 @@ export class Contemplator {
 	private history: AgentMessage[] = [];
 	private pending: PendingUpdate | undefined;
 	private running = false;
+	/** Invalidates stale/hard-timed-out flush finalizers across session changes. */
+	private flushEpoch = 0;
+	/** Bounds retries of one poisoned memory update so future updates can run. */
+	private consecutiveFlushFailures = 0;
 	private seenObservationIds = new Set<string>();
 	private seenSummaryIds = new Set<string>();
 	private seenReviewIds = new Set<string>();
@@ -263,6 +268,9 @@ export class Contemplator {
 		});
 		this.pi.on("session_start", (event: any, ctx: ExtensionContext) => {
 			this.sessionGeneration++;
+			this.flushEpoch++;
+			this.running = false;
+			this.consecutiveFlushFailures = 0;
 			const generation = this.sessionGeneration;
 			this.agentActiveSince = undefined;
 			// AgentSession preserves its steering queue across extension reloads. An
@@ -278,12 +286,18 @@ export class Contemplator {
 		});
 		this.pi.on("session_tree", (_event: any, ctx: ExtensionContext) => {
 			this.sessionGeneration++;
+			this.flushEpoch++;
+			this.running = false;
+			this.consecutiveFlushFailures = 0;
 			this.agentActiveSince = undefined;
 			// Pending steering messages remain queued while navigating the tree.
 			this.restore(ctx, true, true);
 		});
 		this.pi.on("session_shutdown", () => {
 			this.sessionGeneration++;
+			this.flushEpoch++;
+			this.running = false;
+			this.consecutiveFlushFailures = 0;
 			this.agentActiveSince = undefined;
 			this.history = [];
 			this.pending = undefined;
@@ -555,6 +569,23 @@ export class Contemplator {
 			return;
 		}
 		const branchEntries = ctx.sessionManager.getBranch() as Entry[];
+		const observerBacklogTokens = rawTokensSinceObservationCoverage(branchEntries);
+		if (
+			observerBacklogTokens >= this.runtime.config.observeAfterTokens &&
+			(this.runtime.consolidationInFlight || this.runtime.lastObserverError === undefined)
+		) {
+			// A catch-up observer pipeline may append several partial batches while it
+			// drains old source. Do not let primary-agent checkpoints feed those
+			// fragments to the contemplator one at a time. The pipeline emits one
+			// memory update after the backlog falls below the observer trigger.
+			this.publishState("observer");
+			debugLog("contemplator.waiting", {
+				reason: "observer_backlog",
+				observerBacklogTokens,
+				observeAfterTokens: this.runtime.config.observeAfterTokens,
+			});
+			return;
+		}
 		const projection = fullProjection(branchEntries);
 		const observations = projection.observations.map((item) => `[${item.id}] ${item.content}`);
 		const summaries = projection.summaries.map((item) => `[${item.id}] ${item.content}`);
@@ -637,6 +668,7 @@ export class Contemplator {
 		this.pending = undefined;
 		const turnsBeforeRun = this.turnsSinceRun;
 		const sessionGeneration = this.sessionGeneration;
+		const flushEpoch = ++this.flushEpoch;
 		this.running = true;
 		this.turnsSinceRun = 0;
 		const startedAt = Date.now();
@@ -644,6 +676,7 @@ export class Contemplator {
 		let failureMessage: string | undefined;
 		let workerNotified = false;
 		let promptPersisted = false;
+		let workerWatchdog: ReturnType<typeof createWorkerStallWatchdog> | undefined;
 		this.publishState("running", { lastStartedAt: startedAt, lastError: undefined });
 		debugLog("contemplator.start", {
 			newObservationCount: update.observations.length,
@@ -664,16 +697,22 @@ export class Contemplator {
 				failureMessage = resolved.reason;
 				debugLog("contemplator.model_unavailable", { reason: resolved.reason });
 				if (sessionGeneration === this.sessionGeneration) {
-					const pending = this.pending as PendingUpdate | undefined;
-					this.pending = {
-						observations: mergeMemoryLines(pending?.observations ?? [], update.observations),
-						summaries: mergeMemoryLines(pending?.summaries ?? [], update.summaries),
-						reviews: mergeMemoryLines(pending?.reviews ?? [], update.reviews),
-						mainAgentOutputTokens: update.mainAgentOutputTokens,
-						mainAgentToolCalls: update.mainAgentToolCalls,
-						mainAgentActiveTimeMs: update.mainAgentActiveTimeMs,
-					};
-					this.turnsSinceRun = turnsBeforeRun;
+					this.consecutiveFlushFailures++;
+					if (this.consecutiveFlushFailures < 2) {
+						const pending = this.pending as PendingUpdate | undefined;
+						this.pending = {
+							observations: mergeMemoryLines(pending?.observations ?? [], update.observations),
+							summaries: mergeMemoryLines(pending?.summaries ?? [], update.summaries),
+							reviews: mergeMemoryLines(pending?.reviews ?? [], update.reviews),
+							mainAgentOutputTokens: update.mainAgentOutputTokens,
+							mainAgentToolCalls: update.mainAgentToolCalls,
+							mainAgentActiveTimeMs: update.mainAgentActiveTimeMs,
+						};
+					} else {
+						debugLog("contemplator.poisoned_update_released", { reason: resolved.reason, observationCount: update.observations.length, summaryCount: update.summaries.length, reviewCount: update.reviews.length });
+						this.consecutiveFlushFailures = 0;
+					}
+					this.turnsSinceRun = 0; // Back off until fresh primary responses arrive; never retry every checkpoint.
 				}
 				return;
 			}
@@ -687,6 +726,7 @@ export class Contemplator {
 				modelId: selectedModel.id,
 				contextWindow: selectedModel.contextWindow,
 			});
+			workerWatchdog = createWorkerStallWatchdog("contemplator");
 			if (this.runtime.config.showWorkerNotifications && ctx.hasUI) {
 				ctx.ui?.notify("pi-contemplator: contemplator running", "info");
 				workerNotified = true;
@@ -774,9 +814,15 @@ export class Contemplator {
 				// while individual provider APIs also support required/any. Preserve the
 				// runtime hint and final-payload enforcement without weakening base types.
 				if (invocation > 1) (invocationConfig as any).toolChoice = requiredToolChoice(api);
-				const stream = agentLoop([nextPrompt], context, invocationConfig, undefined, streamSimple);
-				for await (const event of stream) logAgentStreamError("contemplator", event);
-				const result = await stream.result();
+				workerWatchdog.progress();
+				const stream = agentLoop([nextPrompt], context, invocationConfig, workerWatchdog.signal, streamSimple);
+				const result = await workerWatchdog.race((async () => {
+					for await (const event of stream) {
+						workerWatchdog!.progress();
+						logAgentStreamError("contemplator", event);
+					}
+					return stream.result();
+				})());
 				// agentLoop returns its input prompt as the first new message. We already
 				// added nextPrompt above, so do not duplicate each update in the durable
 				// contemplator history or in a subsequent retry's context.
@@ -840,25 +886,37 @@ export class Contemplator {
 					});
 				}
 			}
-			if (sessionGeneration === this.sessionGeneration) await this.compactHistory(resolved.model as Model<any>, resolved.apiKey, resolved.headers, sessionGeneration);
+			if (sessionGeneration === this.sessionGeneration) {
+				workerWatchdog.progress();
+				await workerWatchdog.race(this.compactHistory(resolved.model as Model<any>, resolved.apiKey, resolved.headers, sessionGeneration, flushEpoch));
+			}
 		} catch (error) {
 			failed = true;
 			failureMessage = error instanceof Error ? error.message : String(error);
 			debugLog("contemplator.error", { errorMessage: failureMessage });
 			if (sessionGeneration === this.sessionGeneration && !promptPersisted) {
-				const pending = this.pending as PendingUpdate | undefined;
-				this.pending = {
-					observations: mergeMemoryLines(pending?.observations ?? [], update.observations),
-					summaries: mergeMemoryLines(pending?.summaries ?? [], update.summaries),
-					reviews: mergeMemoryLines(pending?.reviews ?? [], update.reviews),
-					mainAgentOutputTokens: update.mainAgentOutputTokens,
-					mainAgentToolCalls: update.mainAgentToolCalls,
-					mainAgentActiveTimeMs: update.mainAgentActiveTimeMs,
-				};
-				this.turnsSinceRun = turnsBeforeRun;
+				this.consecutiveFlushFailures++;
+				if (this.consecutiveFlushFailures < 2) {
+					const pending = this.pending as PendingUpdate | undefined;
+					this.pending = {
+						observations: mergeMemoryLines(pending?.observations ?? [], update.observations),
+						summaries: mergeMemoryLines(pending?.summaries ?? [], update.summaries),
+						reviews: mergeMemoryLines(pending?.reviews ?? [], update.reviews),
+						mainAgentOutputTokens: update.mainAgentOutputTokens,
+						mainAgentToolCalls: update.mainAgentToolCalls,
+						mainAgentActiveTimeMs: update.mainAgentActiveTimeMs,
+					};
+				} else {
+					debugLog("contemplator.poisoned_update_released", { reason: failureMessage, observationCount: update.observations.length, summaryCount: update.summaries.length, reviewCount: update.reviews.length });
+					this.consecutiveFlushFailures = 0;
+				}
+				this.turnsSinceRun = 0; // Back off until fresh primary responses arrive; never retry every checkpoint.
 			}
 		} finally {
+			workerWatchdog?.dispose();
+			if (flushEpoch !== this.flushEpoch) return;
 			this.running = false;
+			if (!failed) this.consecutiveFlushFailures = 0;
 			const pendingHasEnoughMemories = this.pending !== undefined && (
 				this.pending.reviews.length > 0 ||
 				this.pending.observations.length >= this.runtime.config.contemplatorMinNewObservations ||
@@ -951,15 +1009,20 @@ export class Contemplator {
 		const session = this.reviewerSessions.get(request.id) ?? { scope: request.scope, history: options.history ?? [], messageEntryIds: [], foldedEntryIds: new Set() };
 		this.reviewerSessions.set(request.id, session);
 		const task = this.runtime.launchReviewTask(ctx, async () => {
+			const watchdog = createWorkerStallWatchdog("structural reviewer");
+			let acceptsMessages = true;
 			try {
 				debugLog("reviewer.started", { reviewRequestId: request.id, scope: request.scope, resumed: session.history.length > 0 });
-				const result = await runStructuralReview({
+				const result = await watchdog.race(runStructuralReview({
 					request, model, apiKey, headers,
+					signal: watchdog.signal,
+					onProgress: watchdog.progress,
 					getBranch: () => ctx.sessionManager.getBranch() as Entry[],
 					recordUsage: (usage) => this.runtime.recordAgentUsage(usage),
 					history: session.history,
 					onMessages: (messages) => {
-						if (sessionGeneration !== this.sessionGeneration || !this.reviewIsPending(ctx, request.id)) return;
+						watchdog.progress();
+						if (!acceptsMessages || sessionGeneration !== this.sessionGeneration || !this.reviewIsPending(ctx, request.id)) return;
 						for (const message of messages) {
 							session.history.push(message);
 							const entryId = this.appendEntryWithId(ctx, OM_REVIEWER_MESSAGE, { version: 1, reviewRequestId: request.id, scope: request.scope, message }, request.id);
@@ -969,7 +1032,7 @@ export class Contemplator {
 							}
 						}
 					},
-				});
+				}));
 				if (sessionGeneration !== this.sessionGeneration) {
 					debugLog("reviewer.failed", { reviewRequestId: request.id, reason: "session_changed" });
 					return;
@@ -997,6 +1060,8 @@ export class Contemplator {
 				}
 				this.runtime.notifyMemoryUpdate(ctx);
 			} finally {
+				acceptsMessages = false;
+				watchdog.dispose();
 				this.inFlightReviewIds.delete(request.id);
 				if (key) this.inFlightReviewKeys.delete(key);
 			}
@@ -1080,7 +1145,7 @@ export class Contemplator {
 		return own?.id;
 	}
 
-	private async compactHistory(model: Model<any>, apiKey: string, headers: Record<string, string> | undefined, sessionGeneration: number): Promise<void> {
+	private async compactHistory(model: Model<any>, apiKey: string, headers: Record<string, string> | undefined, sessionGeneration: number, flushEpoch: number): Promise<void> {
 		const serializedLength = this.history.reduce((total, message) => total + JSON.stringify(message).length, 0);
 		if (this.history.length < 12 || serializedLength < 60_000) return;
 		const previousMessageCount = this.history.length;
@@ -1091,7 +1156,7 @@ export class Contemplator {
 		const history = this.history.slice();
 		const summaryWithUsage = await generateSummaryWithUsage(history as AgentMessage[], model, 4_000, apiKey, headers);
 		this.runtime.recordAgentUsage(summaryWithUsage.usage);
-		if (sessionGeneration !== this.sessionGeneration) {
+		if (sessionGeneration !== this.sessionGeneration || flushEpoch !== this.flushEpoch) {
 			debugLog("contemplator.compaction_stale", { reason: "session_or_branch_changed" });
 			return;
 		}

package/src/agents/observer/agent.ts

@@ -5,7 +5,7 @@ import { streamSimple } from "@earendil-works/pi-ai/compat";
 import type { Static } from "typebox";
 import { hashId } from "../../ids.js";
 import { logAgentStreamError } from "../stream-errors.js";
-import { AGENT_LOOP_MAX_TOKENS, boundedMaxTokens } from "../../model-budget.js";
+import { OBSERVER_AGENT_LOOP_MAX_TOKENS, boundedMaxTokens } from "../../model-budget.js";
 import { OBSERVER_SYSTEM } from "./prompts.js";
 import { nowTimestamp, truncateRecordContent } from "../../serialize.js";
 import type { Observation, Relevance, Retention } from "../../session-ledger/index.js";
@@ -25,6 +25,7 @@ interface RunObserverArgs {
 	maxTurns?: number;
 	thinkingLevel?: ModelThinkingLevel;
 	recordUsage?: (usage: LlmUsageInput) => void;
+	onProgress?: () => void;
 }
 
 const RelevanceSchema = Type.Union([
@@ -182,7 +183,11 @@ ${joinOrEmpty(priorObservations)}
 Compress the following new conversation chunk into observations by calling record_observations one or more times. Do not restate facts already present in current summaries or current observations. Prefer inline conversation timestamps when assigning times; fall back to the current local time above only if no message timestamp applies. When the chunk is fully covered, call done alone. If the chunk contains no useful new information, call done without calling record_observations.
 
 NEW CONVERSATION CHUNK:
-${conversation}`;
+${conversation}
+
+END NEW CONVERSATION CHUNK
+
+IMPORTANT: Now call record_observations to record the useful new observations from this conversation chunk. When the chunk is covered, call done; if there are no useful observations, call done without recording any.`;
 
 	const initialPrompt: Message = {
 		role: "user",
@@ -198,7 +203,7 @@ ${conversation}`;
 		model,
 		apiKey,
 		headers,
-		maxTokens: boundedMaxTokens(model, AGENT_LOOP_MAX_TOKENS),
+		maxTokens: boundedMaxTokens(model, OBSERVER_AGENT_LOOP_MAX_TOKENS),
 		convertToLlm: (msgs) => msgs as Message[],
 		toolExecution: "sequential",
 		shouldStopAfterTurn: () => {
@@ -211,14 +216,19 @@ ${conversation}`;
 	const loop = args.agentLoop ?? agentLoop;
 	const history: AgentMessage[] = [];
 	let terminalFailure: { stopReason: string; errorMessage?: string } | undefined;
-	const runInvocation = async (prompt: Message): Promise<void> => {
+	let lengthRetryAttempted = false;
+	const runInvocation = async (prompt: Message, afterLength = false): Promise<void> => {
 		const context: AgentContext = {
 			systemPrompt: OBSERVER_SYSTEM,
 			messages: history.slice(),
 			tools: [recordObservations as AgentTool<any>, doneTool],
 		};
-		const stream = loop([prompt], context, baseConfig, signal, streamSimple);
+		const invocationConfig: AgentLoopConfig = afterLength && reasoning
+			? { ...baseConfig, reasoning: "minimal" }
+			: baseConfig;
+		const stream = loop([prompt], context, invocationConfig, signal, streamSimple);
 		for await (const event of stream) {
+			args.onProgress?.();
 			logAgentStreamError("observer", event);
 			const message = (event as { message?: { role?: string; stopReason?: string; errorMessage?: string } }).message;
 			if (message?.role === "assistant" && ["error", "aborted", "length"].includes(message.stopReason ?? "")) {
@@ -237,6 +247,23 @@ ${conversation}`;
 	};
 
 	await runInvocation(initialPrompt);
+	if (accumulated.size === 0 && terminalFailure?.stopReason === "length") {
+		lengthRetryAttempted = true;
+		// A provider can impose a lower output ceiling than the advertised model
+		// maximum. agentLoop stops on `length` when no tool call was completed; it
+		// does not automatically send a continuation request. Preserve the partial
+		// response so the model can continue from work it already performed rather
+		// than paying to reproduce it, then append a short tool-focused instruction
+		// and reduce reasoning to minimal. A second length stop fails forward at the
+		// bounded-chunk level.
+		terminalFailure = undefined;
+		const retryPrompt: Message = {
+			role: "user",
+			content: [{ type: "text", text: "IMPORTANT: The previous response reached the provider output limit before recording anything. Continue from the work already above and call record_observations now instead of spending another response budget analyzing." }],
+			timestamp: Date.now(),
+		};
+		await runInvocation(retryPrompt, true);
+	}
 	if (accumulated.size === 0 && !doneCalled && !terminalFailure && rejectedTotal === 0) {
 		const reminder: Message = {
 			role: "user",
@@ -246,19 +273,19 @@ ${conversation}`;
 		await runInvocation(reminder);
 	}
 
-	// Accepted observations remain useful even if the model neglected the final
-	// confirmation. Zero-observation coverage is advanced only by an explicit
-	// done call; failures, truncation, malformed records, and repeated prose do
-	// not silently discard the source chunk.
+	// `done` is a behavioral aid and terminal shortcut, not a transaction gate.
+	// Accepted observations commit even if it was omitted. A second prose-only
+	// zero-observation stop is also a valid empty result after the reminder;
+	// actual stream failures, truncation, and malformed records still throw.
 	if (accumulated.size === 0 && terminalFailure) {
-		throw new ObserverStreamError(terminalFailure.stopReason, terminalFailure.errorMessage);
+		const detail = terminalFailure.stopReason === "length" && lengthRetryAttempted
+			? `provider reached the output limit twice without recording an observation (effective max output request: ${baseConfig.maxTokens} tokens)`
+			: terminalFailure.errorMessage;
+		throw new ObserverStreamError(terminalFailure.stopReason, detail);
 	}
 	if (accumulated.size === 0 && rejectedTotal > 0) {
 		throw new ObserverStreamError("invalid_observations", `${rejectedTotal} proposed observation${rejectedTotal === 1 ? " was" : "s were"} rejected`);
 	}
-	if (accumulated.size === 0 && !doneCalled) {
-		throw new ObserverStreamError("incomplete", "observer stopped twice without recording observations or calling done");
-	}
 	if (accumulated.size === 0) return undefined;
 	return Array.from(accumulated.values());
 }

package/src/agents/reviewer/agent.ts

@@ -30,6 +30,7 @@ export interface RunStructuralReviewArgs {
 	recordUsage?: (usage: LlmUsageInput) => void;
 	/** Receives the reviewer's assistant output for durable debug/view rendering. */
 	onMessages?: (messages: AgentMessage[]) => void;
+	onProgress?: () => void;
 	/** Previously persisted reviewer transcript; a non-empty history resumes work. */
 	history?: AgentMessage[];
 }
@@ -177,7 +178,10 @@ export async function runStructuralReview(args: RunStructuralReviewArgs): Promis
 		history.push(promptMessage);
 		args.onMessages?.([promptMessage]);
 		const stream = loop([prompt], context, config, args.signal, budgetedStreamSimple);
-		for await (const event of stream) logAgentStreamError("reviewer", event);
+		for await (const event of stream) {
+			args.onProgress?.();
+			logAgentStreamError("reviewer", event);
+		}
 		const newMessages = await stream.result();
 		history.push(...newMessages);
 		args.onMessages?.(newMessages);

package/src/commands/contemplator-view.ts

@@ -63,11 +63,13 @@ function liveStateLine(state: ContemplatorRunState): string {
 	const timing = `Last start: ${state.lastStartedAt === undefined ? "not run this launch" : new Date(state.lastStartedAt).toISOString()} · Last end: ${state.lastCompletedAt === undefined ? "not completed this launch" : new Date(state.lastCompletedAt).toISOString()}`;
 	const error = state.lastError ? `\nLast error: ${state.lastError}` : "";
 	if (state.running) return `LIVE · running for ${Math.max(0, Math.floor((Date.now() - (state.lastStartedAt ?? Date.now())) / 60_000))}m · ${pending}\n${timing}${error}`;
-	const reason = state.waitingFor === "memories"
-		? "waiting for memory threshold"
-		: state.waitingFor === "responses"
-			? "waiting for response spacing"
-			: state.waitingFor === "ready"
+	const reason = state.waitingFor === "observer"
+		? "waiting for observer backlog"
+		: state.waitingFor === "memories"
+			? "waiting for memory threshold"
+			: state.waitingFor === "responses"
+				? "waiting for response spacing"
+				: state.waitingFor === "ready"
 				? "ready to launch"
 				: state.waitingFor === "disabled"
 					? "disabled"

package/src/commands/status.ts

@@ -42,6 +42,7 @@ function formatRunAge(timestamp: number): string {
 
 function contemplatorWaitingLabel(waitingFor: Runtime["contemplatorState"]["waitingFor"]): string {
 	switch (waitingFor) {
+		case "observer": return "waiting for observer backlog";
 		case "memories": return "waiting for memory threshold";
 		case "responses": return "waiting for response spacing";
 		case "ready": return "ready to launch";

package/src/hooks/compaction-trigger.ts

@@ -9,6 +9,7 @@ import {
 } from "./compaction-resume.js";
 
 const COMPACTION_STATUS_KEY = "observational-memory-compaction";
+export const COMPACTION_CALLBACK_TIMEOUT_MS = 30 * 60_000;
 type CompactionOrigin = "agent-requested" | "length-stop" | "proactive";
 
 type TriggerOptions = {
@@ -37,6 +38,7 @@ export function registerCompactionTrigger(pi: ExtensionAPI, runtime: Runtime): v
 		const { origin, resume, threshold, shortContinuationPrompt } = options;
 		const hasUI = ctx.hasUI;
 		const ui = ctx.ui;
+		const generation = runtime.getContextGeneration?.() ?? 0;
 
 		runtime.compactInFlight = true;
 		runtime.compactOrigin = origin;
@@ -74,23 +76,39 @@ export function registerCompactionTrigger(pi: ExtensionAPI, runtime: Runtime): v
 					);
 				}
 				if (origin === "agent-requested") runtime.compactContinuationPrompt = undefined;
-				ctx.compact({
-					onComplete: () => {
-						runtime.compactInFlight = false;
-						runtime.compactOrigin = undefined;
-						if (hasUI && !resume) ui?.setStatus?.(COMPACTION_STATUS_KEY, undefined);
-						if (resume) resumeAfterCompaction(pi, runtime, { hasUI, ui }, false, shortContinuationPrompt);
-					},
-					onError: (error: { message: string }) => {
-						runtime.compactInFlight = false;
-						runtime.compactOrigin = undefined;
-						if (hasUI) ui?.setStatus?.(COMPACTION_STATUS_KEY, undefined);
-						if (error.message !== "Compaction cancelled" && hasUI) {
-							ui?.notify(`pi-contemplator: ${error.message}`, "error");
-						}
-						if (resume) resumeAfterCompaction(pi, runtime, { hasUI, ui }, true, shortContinuationPrompt);
-					},
-				});
+				let settled = false;
+				const callbackTimeout = setTimeout(() => {
+					if (settled || generation !== (runtime.getContextGeneration?.() ?? 0)) return;
+					settled = true;
+					runtime.compactInFlight = false;
+					runtime.compactOrigin = undefined;
+					if (hasUI) {
+						ui?.setStatus?.(COMPACTION_STATUS_KEY, undefined);
+						ui?.notify("pi-contemplator: compaction callback timed out; releasing the compaction lock", "error");
+					}
+					if (resume) resumeAfterCompaction(pi, runtime, { hasUI, ui }, true, shortContinuationPrompt);
+				}, COMPACTION_CALLBACK_TIMEOUT_MS);
+				(callbackTimeout as ReturnType<typeof setTimeout> & { unref?: () => void }).unref?.();
+				const settle = (failed: boolean, error?: { message: string }) => {
+					if (settled || generation !== (runtime.getContextGeneration?.() ?? 0)) return;
+					settled = true;
+					clearTimeout(callbackTimeout);
+					runtime.compactInFlight = false;
+					runtime.compactOrigin = undefined;
+					if (hasUI && (failed || !resume)) ui?.setStatus?.(COMPACTION_STATUS_KEY, undefined);
+					if (failed && error?.message !== "Compaction cancelled" && hasUI) ui?.notify(`pi-contemplator: ${error?.message ?? "compaction failed"}`, "error");
+					if (resume) resumeAfterCompaction(pi, runtime, { hasUI, ui }, failed, shortContinuationPrompt);
+				};
+				try {
+					ctx.compact({
+						onComplete: () => settle(false),
+						onError: (error: { message: string }) => settle(true, error),
+					});
+				} catch (error) {
+					settled = true;
+					clearTimeout(callbackTimeout);
+					throw error;
+				}
 			} catch (error) {
 				runtime.compactInFlight = false;
 				if (origin === "agent-requested") runtime.compactContinuationPrompt = undefined;

package/src/hooks/consolidation-trigger.ts

@@ -4,6 +4,8 @@ import { runObserver } from "../agents/observer/agent.js";
 import { debugLog, withDebugLogContext } from "../debug-log.js";
 import { resolveObserverChunkMaxTokens } from "../config.js";
 import type { ResolveResult, Runtime } from "../runtime.js";
+import { createWorkerStallWatchdog } from "../worker-watchdog.js";
+import { boundedMaxTokens, OBSERVER_AGENT_LOOP_MAX_TOKENS } from "../model-budget.js";
 import { serializeSourceAddressedBranchEntries } from "../serialize.js";
 import {
 	OM_SUMMARIZER_COMMIT,
@@ -43,28 +45,10 @@ export const SUMMARIZER_STALL_TIMEOUT_MS = 15 * 60_000;
 export function createSummarizerStallWatchdog(
 	timeoutMs: number,
 	onStall: (signal: AbortSignal) => void,
-): { signal: AbortSignal; progress: () => void; dispose: () => void } {
-	const controller = new AbortController();
-	let timer: ReturnType<typeof setTimeout> | undefined;
-	const progress = () => {
-		if (controller.signal.aborted) return;
-		if (timer !== undefined) clearTimeout(timer);
-		timer = setTimeout(() => {
-			timer = undefined;
-			controller.abort(new Error(`summarizer produced no progress for ${Math.round(timeoutMs / 60_000)} minutes`));
-			onStall(controller.signal);
-		}, timeoutMs);
-		(timer as ReturnType<typeof setTimeout> & { unref?: () => void }).unref?.();
-	};
-	progress();
-	return {
-		signal: controller.signal,
-		progress,
-		dispose: () => {
-			if (timer !== undefined) clearTimeout(timer);
-			timer = undefined;
-		},
-	};
+) {
+	let watchdog!: ReturnType<typeof createWorkerStallWatchdog>;
+	watchdog = createWorkerStallWatchdog("summarizer", timeoutMs, () => onStall(watchdog.signal));
+	return watchdog;
 }
 
 function sourceEntriesAfter(entries: Entry[], index: number): Entry[] {
@@ -96,6 +80,7 @@ function makeModelResolver(runtime: Runtime, ctx: ConsolidationCtx): (stage: "ob
 			runtime.resolveFailureNotified = false;
 			return cached;
 		}
+		runtime.lastObserverError = cached.reason;
 		debugLog(`${stage}.model_unavailable`, { reason: cached.reason });
 		if (!runtime.resolveFailureNotified && ctx.hasUI && ctx.ui) {
 			ctx.ui.notify(`pi-contemplator: ${stage} skipped — ${cached.reason}`, "warning");
@@ -109,6 +94,14 @@ export function registerConsolidationTrigger(pi: ExtensionAPI, runtime: Runtime)
 	const launch = (_event: unknown, ctx: ConsolidationCtx) => {
 		maybeLaunchConsolidation(pi, runtime, ctx);
 	};
+	pi.on("session_start", (event, ctx) => {
+		launch(event, ctx);
+		syncAndScheduleSummarizer(pi, runtime, ctx as ConsolidationCtx);
+	});
+	pi.on("session_tree", (event, ctx) => {
+		launch(event, ctx);
+		syncAndScheduleSummarizer(pi, runtime, ctx as ConsolidationCtx);
+	});
 	pi.on("agent_start", (event, ctx) => {
 		launch(event, ctx);
 		syncAndScheduleSummarizer(pi, runtime, ctx as ConsolidationCtx);
@@ -119,8 +112,10 @@ export function registerConsolidationTrigger(pi: ExtensionAPI, runtime: Runtime)
 	});
 	runtime.setAgentActivityListener((ctx) => {
 		runtime.ensureConfig(ctx.cwd);
-		// Token pools are re-evaluated at every primary-agent progress checkpoint;
-		// idle wall-clock time has no scheduling meaning.
+		// Long autonomous turns may run for hours without agent_start/turn_end.
+		// Re-evaluate both source coverage and memory pressure at every primary
+		// progress checkpoint so neither worker can silently stop for the turn.
+		maybeLaunchConsolidation(pi, runtime, ctx as ConsolidationCtx);
 		scheduleSummarizer(pi, runtime, ctx as ConsolidationCtx);
 	});
 	runtime.setSettingsUpdateListener((ctx, settings) => {
@@ -166,7 +161,7 @@ function maybeLaunchConsolidation(pi: ExtensionAPI, runtime: Runtime, ctx: Conso
 	};
 
 	const sessionMetadata = debugSessionMetadata(ctx);
-	void runtime.launchConsolidationTask(ctx, async () => withDebugLogContext({
+	const task = runtime.launchConsolidationTask(ctx, async () => withDebugLogContext({
 		enabled: runtime.config.debugLog === true,
 		cwd: ctx.cwd,
 		...sessionMetadata,
@@ -174,6 +169,13 @@ function maybeLaunchConsolidation(pi: ExtensionAPI, runtime: Runtime, ctx: Conso
 	}, async () => {
 		await runConsolidationPipeline(pi, runtime, consolidationCtx);
 	}));
+	// launchTrackedTask releases its lock before this continuation. If observer
+	// setup failed before coverage could move, wake the contemplator once in
+	// degraded mode rather than leaving all advisory work gated forever. Future
+	// primary activity still retries the observer.
+	void task.then(() => {
+		if (runtime.lastObserverError) runtime.notifyMemoryUpdate(ctx);
+	});
 }
 
 export function launchCompactionObserver(
@@ -314,7 +316,9 @@ export function scheduleSummarizer(pi: ExtensionAPI, runtime: Runtime, ctx: Cons
 	}, async () => {
 		let stalled = false;
 		let successfullyCompleted = false;
+		let modelRunAttempted = false;
 		let disposeStallWatchdog = () => {};
+		let acceptsSummarizerMessages = true;
 		const startedAt = Date.now();
 		runtime.lastSummarizerStartedAt = startedAt;
 		runtime.lastSummarizerRun = { startedAt, status: "running", messages: [] };
@@ -334,7 +338,8 @@ export function scheduleSummarizer(pi: ExtensionAPI, runtime: Runtime, ctx: Cons
 				if (shouldNotifyWorker(runtime, ctx)) ctx.ui?.notify(`pi-contemplator: ${reason}; cancelling and leaving the backlog eligible for retry`, "warning");
 			});
 			disposeStallWatchdog = watchdog.dispose;
-			const result = await runSummarizer({
+			modelRunAttempted = true;
+			const result = await watchdog.race(runSummarizer({
 				signal: watchdog.signal,
 				model: resolved.model as any,
 				apiKey: resolved.apiKey,
@@ -348,9 +353,9 @@ export function scheduleSummarizer(pi: ExtensionAPI, runtime: Runtime, ctx: Cons
 				recordUsage: (usage) => runtime.recordAgentUsage(usage),
 				onMessages: (messages) => {
 					watchdog.progress();
-					if (generation === runtime.getContextGeneration()) runtime.lastSummarizerRun = { startedAt, status: "running", messages: messages.slice() };
+					if (acceptsSummarizerMessages && generation === runtime.getContextGeneration()) runtime.lastSummarizerRun = { startedAt, status: "running", messages: messages.slice() };
 				},
-			});
+			}));
 			if (generation !== runtime.getContextGeneration()) return;
 			if (!result.completed) {
 				runtime.lastSummarizerRun = stalled
@@ -377,6 +382,7 @@ export function scheduleSummarizer(pi: ExtensionAPI, runtime: Runtime, ctx: Cons
 			}
 			throw error;
 		} finally {
+			acceptsSummarizerMessages = false;
 			disposeStallWatchdog();
 			if (generation === runtime.getContextGeneration()) {
 				const completedAt = Date.now();
@@ -385,7 +391,11 @@ export function scheduleSummarizer(pi: ExtensionAPI, runtime: Runtime, ctx: Cons
 				const postRunPools = currentMemoryPools(runtime, ctx.sessionManager.getBranch() as Entry[]);
 				const target = runtime.config.oldMemoryPoolTargetTokens;
 				runtime.summarizerNextTriggerTokens = summarizerTriggerAfterRun(
-					successfullyCompleted,
+					// A failed/no-progress model pass must not be retried with the
+					// identical pool at every primary-agent checkpoint. Require fresh
+					// old-memory growth before trying another (potentially different)
+					// sample. Model-resolution failures spend no tokens and stay eligible.
+					successfullyCompleted || modelRunAttempted,
 					runtime.summarizerNextTriggerTokens,
 					target,
 					postRunPools.oldTokens,
@@ -457,6 +467,9 @@ async function runObserverStage(
 	debugLog("observer.start", {
 		tokens,
 		chunkTokens,
+		requestedMaxOutputTokens: OBSERVER_AGENT_LOOP_MAX_TOKENS,
+		effectiveMaxOutputTokens: boundedMaxTokens(resolved.model as any, OBSERVER_AGENT_LOOP_MAX_TOKENS),
+		advertisedModelMaxTokens: (resolved.model as { maxTokens?: number }).maxTokens,
 		coversUpToId,
 		sourceEntryIds,
 		sourceEntryCount: sourceEntryIds.length,
@@ -464,18 +477,41 @@ async function runObserverStage(
 		priorObservations: priorObservations.length,
 	});
 
-	const observations = await runObserver({
-		model: resolved.model as any,
-		apiKey: resolved.apiKey,
-		headers: resolved.headers,
-		priorSummaries,
-		priorObservations,
-		chunk,
-		allowedSourceEntryIds: sourceEntryIds,
-		maxTurns: runtime.config.agentMaxTurns,
-		thinkingLevel: runtime.config.model?.thinking ?? "low",
-		recordUsage: (usage) => runtime.recordAgentUsage(usage),
-	});
+	let observations;
+	let failedMessage: string | undefined;
+	const observerWatchdog = createWorkerStallWatchdog("observer");
+	try {
+		observations = await observerWatchdog.race(runObserver({
+			model: resolved.model as any,
+			apiKey: resolved.apiKey,
+			headers: resolved.headers,
+			priorSummaries,
+			priorObservations,
+			chunk,
+			allowedSourceEntryIds: sourceEntryIds,
+			maxTurns: runtime.config.agentMaxTurns,
+			thinkingLevel: runtime.config.model?.thinking ?? "low",
+			recordUsage: (usage) => runtime.recordAgentUsage(usage),
+			onProgress: observerWatchdog.progress,
+			signal: observerWatchdog.signal,
+		}));
+	} catch (error) {
+		// A permanently pathological old chunk must not pin every newer source
+		// entry and block the contemplator forever. Accepted observations are
+		// already returned normally by runObserver; only a zero-progress failure
+		// reaches here. Record the failure visibly, then advance this bounded
+		// range with an empty coverage marker so catch-up can continue.
+		failedMessage = runtime.recordConsolidationStageError(ctx, "observer", error);
+		debugLog("observer.failed_chunk_advanced", {
+			failedMessage,
+			coversUpToId,
+			sourceEntryIds,
+			chunkTokens,
+		});
+		observations = undefined;
+	} finally {
+		observerWatchdog.dispose();
+	}
 	if (options.contextGeneration !== undefined && options.contextGeneration !== runtime.getContextGeneration()) {
 		debugLog("observer.stale", { reason: "session_or_branch_changed" });
 		return "abort";
@@ -497,18 +533,18 @@ async function runObserverStage(
 	const accepted = observations ?? [];
 	const data = buildObservationsRecordedData(accepted, effectiveCoversUpToId);
 	if (!data) return "continue";
-	debugLog(accepted.length > 0 ? "observer.records" : "observer.coverage_only", {
+	debugLog(failedMessage ? "observer.failed_coverage" : accepted.length > 0 ? "observer.records" : "observer.coverage_only", {
 		count: accepted.length,
 		observationTokens: accepted.reduce((sum, observation) => sum + observation.tokenCount, 0),
 		coversUpToId: effectiveCoversUpToId,
+		...(failedMessage ? { failedMessage } : {}),
 	});
-	// A clean zero-observation verdict is still successful coverage. Persist an
-	// empty batch so the next bounded pass starts after this chunk instead of
-	// retrying the same low-information source forever. Failures throw above and
-	// therefore never reach this coverage commit.
+	// A clean zero-observation verdict and a zero-progress failed chunk both use
+	// an empty coverage marker. This prevents low-information or pathological old
+	// source from pinning the entire observer/contemplator pipeline forever.
 	appendEntry(pi, OM_OBSERVATIONS_RECORDED, data);
 	debugLog("observer.appended", { count: accepted.length, coversUpToId: effectiveCoversUpToId });
-	if (shouldNotifyWorker(runtime, ctx)) ctx.ui?.notify(
+	if (!failedMessage && shouldNotifyWorker(runtime, ctx)) ctx.ui?.notify(
 		accepted.length > 0
 			? `pi-contemplator: ${accepted.length} observation${accepted.length === 1 ? "" : "s"} recorded`
 			: "pi-contemplator: observer found no new information; processed chunk marked covered",

package/src/model-budget.ts

@@ -2,6 +2,9 @@ import type { Model } from "@earendil-works/pi-ai";
 
 export const AGENT_LOOP_MAX_TOKENS = 32_000;
 
+/** Observer output allowance for difficult, high-volume source chunks. */
+export const OBSERVER_AGENT_LOOP_MAX_TOKENS = 160_000;
+
 /**
  * Lifetime output-token budget for one structural review request.
  * Persisted reviewer transcripts carry usage across keep-going iterations,

package/src/runtime.ts

@@ -72,7 +72,7 @@ export interface ContemplatorRunState {
 	pendingReviews: number;
 	/** Completed primary-model responses since the previous contemplator run. */
 	responsesSinceRun: number;
-	waitingFor: "disabled" | "passive" | "memories" | "responses" | "ready" | "running" | "idle";
+	waitingFor: "disabled" | "passive" | "observer" | "memories" | "responses" | "ready" | "running" | "idle";
 	lastStartedAt?: number;
 	lastCompletedAt?: number;
 	lastError?: string;
@@ -254,6 +254,16 @@ export class Runtime {
 		// against a different branch and would otherwise leak across sessions
 		// (e.g. a request made in session A compacting session B's branch, or a
 		// never-cleared compactInFlight bricking all future compactions).
+		// Detach stale background tasks immediately. Their promise finalizers are
+		// identity-guarded below, so an old session can never retain or later clear
+		// a lock owned by the new session.
+		this.consolidationInFlight = false;
+		this.consolidationPromise = null;
+		this.consolidationPhase = undefined;
+		this.summarizerInFlight = false;
+		this.summarizerPromise = null;
+		this.reviewInFlight = false;
+		this.reviewPromise = null;
 		this.compactInFlight = false;
 		this.compactRequested = false;
 		this.compactContinuationPrompt = undefined;
@@ -336,9 +346,10 @@ export class Runtime {
 		this.consolidationPhase = undefined;
 		this.lastObserverError = undefined;
 		const promise = this.launchTrackedTask(ctx, "consolidation", work, () => {
+			if (this.consolidationPromise !== promise) return;
 			this.consolidationInFlight = false;
 			this.consolidationPhase = undefined;
-			if (this.consolidationPromise === promise) this.consolidationPromise = null;
+			this.consolidationPromise = null;
 		});
 		this.consolidationPromise = promise;
 		return promise;
@@ -351,9 +362,10 @@ export class Runtime {
 		this.summarizerInFlight = true;
 		this.lastSummarizerError = undefined;
 		const promise = this.launchTrackedTask(ctx, "summarizer", work, (error) => {
+			if (this.summarizerPromise !== promise) return;
 			this.summarizerInFlight = false;
 			this.lastSummarizerError = error;
-			if (this.summarizerPromise === promise) this.summarizerPromise = null;
+			this.summarizerPromise = null;
 		});
 		this.summarizerPromise = promise;
 		return promise;
@@ -365,8 +377,9 @@ export class Runtime {
 		if (this.reviewInFlight) return undefined;
 		this.reviewInFlight = true;
 		const promise = this.launchTrackedTask(ctx, "structural review", work, () => {
+			if (this.reviewPromise !== promise) return;
 			this.reviewInFlight = false;
-			if (this.reviewPromise === promise) this.reviewPromise = null;
+			this.reviewPromise = null;
 		});
 		this.reviewPromise = promise;
 		return promise;

package/src/worker-watchdog.ts

@@ -0,0 +1,63 @@
+export const BACKGROUND_WORKER_STALL_TIMEOUT_MS = 15 * 60_000;
+export const BACKGROUND_WORKER_MAX_RUNTIME_MS = 6 * 60 * 60_000;
+
+/**
+ * No-progress watchdog for background model workers.
+ *
+ * Aborting the provider is the cooperative path. `race()` is the hard liveness
+ * boundary: even a provider that ignores AbortSignal cannot retain a worker's
+ * single-flight lock forever. The abandoned promise has handlers installed by
+ * Promise.race, so a later rejection is not unhandled.
+ */
+export function createWorkerStallWatchdog(
+	label: string,
+	timeoutMs = BACKGROUND_WORKER_STALL_TIMEOUT_MS,
+	onStall?: (error: Error) => void,
+	maxRuntimeMs = BACKGROUND_WORKER_MAX_RUNTIME_MS,
+): {
+	signal: AbortSignal;
+	progress: () => void;
+	race: <T>(work: Promise<T>) => Promise<T>;
+	dispose: () => void;
+} {
+	const controller = new AbortController();
+	let timer: ReturnType<typeof setTimeout> | undefined;
+	let maxRuntimeTimer: ReturnType<typeof setTimeout> | undefined;
+	let rejectStall!: (error: Error) => void;
+	let settled = false;
+	const stalled = new Promise<never>((_resolve, reject) => { rejectStall = reject; });
+	// Some callers use only the cooperative AbortSignal. Keep the hard-boundary
+	// promise observed even before/without a race() call.
+	void stalled.catch(() => {});
+	const trip = (error: Error) => {
+		if (settled || controller.signal.aborted) return;
+		controller.abort(error);
+		onStall?.(error);
+		rejectStall(error);
+	};
+	maxRuntimeTimer = setTimeout(() => trip(new Error(`${label} exceeded its ${Math.round(maxRuntimeMs / 3_600_000)}-hour runtime ceiling`)), maxRuntimeMs);
+	(maxRuntimeTimer as ReturnType<typeof setTimeout> & { unref?: () => void }).unref?.();
+	const progress = () => {
+		if (settled || controller.signal.aborted) return;
+		if (timer !== undefined) clearTimeout(timer);
+		timer = setTimeout(() => {
+			timer = undefined;
+			if (settled) return;
+			trip(new Error(`${label} produced no progress for ${Math.round(timeoutMs / 60_000)} minutes`));
+		}, timeoutMs);
+		(timer as ReturnType<typeof setTimeout> & { unref?: () => void }).unref?.();
+	};
+	progress();
+	return {
+		signal: controller.signal,
+		progress,
+		race: async <T>(work: Promise<T>) => Promise.race([work, stalled]),
+		dispose: () => {
+			settled = true;
+			if (timer !== undefined) clearTimeout(timer);
+			if (maxRuntimeTimer !== undefined) clearTimeout(maxRuntimeTimer);
+			timer = undefined;
+			maxRuntimeTimer = undefined;
+		},
+	};
+}