@matthewfl/pi-contemplator 0.1.4 → 0.1.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@matthewfl/pi-contemplator",
-  "version": "0.1.4",
+  "version": "0.1.6",
   "description": "A Pi extension that keeps long-running agentic sessions on track with background memory, contemplation, and structural review.",
   "type": "module",
   "license": "MIT",

package/src/agents/contemplator/agent.ts

@@ -229,9 +229,11 @@ export class Contemplator {
 	private deliveredProbeIds = new Set<string>();
 	/** Probe ids passed to pi.sendMessage by this live extension runtime. */
 	private queuedProbeIds = new Set<string>();
+	/** Probes whose provider-context delivery will establish the next response-spacing anchor. */
+	private probeCooldownPendingIds = new Set<string>();
 	private sessionGeneration = 0;
 	private latestCtx: MemoryUpdateCtx | undefined;
-	/** Completed primary-model responses since the previous contemplator run. */
+	/** Completed primary-model responses since the current completion/probe-delivery spacing anchor. */
 	private turnsSinceRun = 0;
 	/** Used to avoid counting the final turn_end after its assistant message_end. */
 	private assistantResponsesInCurrentTurn = 0;
@@ -311,6 +313,7 @@ export class Contemplator {
 			this.reviewerSessions.clear();
 			this.deliveredProbeIds.clear();
 			this.queuedProbeIds.clear();
+			this.probeCooldownPendingIds.clear();
 			this.latestCtx = undefined;
 			this.turnsSinceRun = 0;
 			this.assistantResponsesInCurrentTurn = 0;
@@ -359,9 +362,12 @@ export class Contemplator {
 		});
 		this.pi.on("context", (event: any, ctx: ExtensionContext) => {
 			const deliveredMessages = event.messages?.filter((message: any) => message?.role === "custom" && message.customType === CONTEMPLATOR_SUGGESTION && typeof message.details?.probeId === "string") ?? [];
+			let cooldownAnchored = false;
 			for (const delivered of deliveredMessages) {
 				if (this.deliveredProbeIds.has(delivered.details.probeId)) continue;
 				this.deliveredProbeIds.add(delivered.details.probeId);
+				this.probeCooldownPendingIds.delete(delivered.details.probeId);
+				cooldownAnchored = true;
 				// Once Pi includes the probe in a provider context it is no longer in
 				// either in-memory delivery queue. Keeping this id indefinitely caused
 				// later tree restores to suppress a genuinely needed requeue.
@@ -375,6 +381,12 @@ export class Contemplator {
 				this.markTipPersisted(ctx);
 				debugLog("contemplator.suggestion_delivered", { probeId: delivered.details.probeId });
 			}
+			if (cooldownAnchored) {
+				// Probe spacing begins only once Pi proves the probe reached an actual
+				// provider context. Responses generated before this point do not count.
+				this.turnsSinceRun = 0;
+				this.withDebugContext(ctx, () => this.observeTurn(ctx));
+			}
 		});
 		this.pi.on("turn_end", (_event: any, ctx: ExtensionContext) => {
 			this.persistAgentActivity(ctx);
@@ -437,6 +449,7 @@ export class Contemplator {
 		let resetProjection: ReturnType<typeof fullProjection> | undefined;
 		if (resetTracking) {
 			this.deliveredProbeIds.clear();
+			this.probeCooldownPendingIds.clear();
 			if (!retainQueuedIds) this.queuedProbeIds.clear();
 			this.inFlightReviewIds.clear();
 			this.resolvingReviewIds.clear();
@@ -545,6 +558,9 @@ export class Contemplator {
 		}
 		this.restoredTipId = tipId;
 		for (const [probeId, question] of undeliveredSuggestions) {
+			// An undelivered durable probe remains the cooldown anchor even when Pi's
+			// live queue survived an extension reload and must not be duplicated.
+			this.probeCooldownPendingIds.add(probeId);
 			// A durable custom_message proves only that Pi inserted the probe at some
 			// point; it does not prove an in-memory queue still owns it, and compaction
 			// may have removed it from active model context. Suppress requeue only for
@@ -570,18 +586,21 @@ export class Contemplator {
 		}
 		const branchEntries = ctx.sessionManager.getBranch() as Entry[];
 		const observerBacklogTokens = rawTokensSinceObservationCoverage(branchEntries);
-		if (
+		const waitingForCapturedObserverBacklog = this.runtime.observerBacklogBlocking || (
+			!this.runtime.consolidationInFlight &&
 			observerBacklogTokens >= this.runtime.config.observeAfterTokens &&
-			(this.runtime.consolidationInFlight || this.runtime.lastObserverError === undefined)
-		) {
+			this.runtime.lastObserverError === undefined
+		);
+		if (waitingForCapturedObserverBacklog) {
 			// A catch-up observer pipeline may append several partial batches while it
-			// drains old source. Do not let primary-agent checkpoints feed those
-			// fragments to the contemplator one at a time. The pipeline emits one
-			// memory update after the backlog falls below the observer trigger.
+			// drains the finite source snapshot captured at launch. Do not feed those
+			// fragments to the contemplator one at a time. Source appended concurrently
+			// belongs to the next snapshot and must not extend this waiting period.
 			this.publishState("observer");
 			debugLog("contemplator.waiting", {
 				reason: "observer_backlog",
 				observerBacklogTokens,
+				observerBacklogBlocking: this.runtime.observerBacklogBlocking,
 				observeAfterTokens: this.runtime.config.observeAfterTokens,
 			});
 			return;
@@ -620,7 +639,7 @@ export class Contemplator {
 			};
 		}
 		if (!this.pending) {
-			this.publishState(this.running ? "running" : "idle");
+			this.publishState(this.running ? "running" : this.probeCooldownPendingIds.size > 0 ? "probe" : "idle");
 			return;
 		}
 		// Activity values are cumulative send-time snapshots, not values frozen when
@@ -630,6 +649,11 @@ export class Contemplator {
 		this.pending.mainAgentToolCalls = assistantToolCallCount(branchEntries);
 		this.pending.mainAgentActiveTimeMs = agentActiveTimeMs(branchEntries);
 		const enoughMemories = this.pending.reviews.length > 0 || this.pending.observations.length >= this.runtime.config.contemplatorMinNewObservations || this.pending.summaries.length >= this.runtime.config.contemplatorMinNewSummaries;
+		if (this.probeCooldownPendingIds.size > 0) {
+			this.publishState("probe");
+			debugLog("contemplator.waiting", { reason: "probe_delivery", pendingProbeCount: this.probeCooldownPendingIds.size });
+			return;
+		}
 		if (!enoughMemories || this.turnsSinceRun < this.runtime.config.contemplatorMinTurns) {
 			this.publishState(!enoughMemories ? "memories" : "responses");
 			debugLog("contemplator.waiting", {
@@ -676,6 +700,7 @@ export class Contemplator {
 		let failureMessage: string | undefined;
 		let workerNotified = false;
 		let promptPersisted = false;
+		let emittedProbeId: string | undefined;
 		let workerWatchdog: ReturnType<typeof createWorkerStallWatchdog> | undefined;
 		this.publishState("running", { lastStartedAt: startedAt, lastError: undefined });
 		debugLog("contemplator.start", {
@@ -863,7 +888,7 @@ export class Contemplator {
 					this.markTipPersisted(ctx);
 				}
 			}
-			if (intervention?.kind === "probe" && sessionGeneration === this.sessionGeneration) this.queueProbe(ctx, intervention.question, "send_probe");
+			if (intervention?.kind === "probe" && sessionGeneration === this.sessionGeneration) emittedProbeId = this.queueProbe(ctx, intervention.question, "send_probe");
 			if (intervention?.kind === "review" && this.runtime.config.reviewerEnabled && sessionGeneration === this.sessionGeneration) {
 				const reviewerModel = await this.runtime.resolveModel({
 					model: ctx.model,
@@ -917,13 +942,21 @@ export class Contemplator {
 			if (flushEpoch !== this.flushEpoch) return;
 			this.running = false;
 			if (!failed) this.consecutiveFlushFailures = 0;
+			// Normal runs establish their spacing anchor at completion. A probe run
+			// instead anchors at provider-context delivery: if delivery already occurred
+			// during this run, retain responses counted since it; otherwise the pending
+			// probe gate blocks launches until the context event resets the counter.
+			if (emittedProbeId === undefined) this.turnsSinceRun = 0;
+			const waitingForProbe = this.probeCooldownPendingIds.size > 0;
 			const pendingHasEnoughMemories = this.pending !== undefined && (
 				this.pending.reviews.length > 0 ||
 				this.pending.observations.length >= this.runtime.config.contemplatorMinNewObservations ||
 				this.pending.summaries.length >= this.runtime.config.contemplatorMinNewSummaries
 			);
-			const waitingFor = !this.pending
-				? "idle"
+			const waitingFor = waitingForProbe
+				? "probe"
+				: !this.pending
+					? "idle"
 				: !pendingHasEnoughMemories
 					? "memories"
 					: this.turnsSinceRun < this.runtime.config.contemplatorMinTurns
@@ -948,7 +981,7 @@ export class Contemplator {
 		}
 	}
 
-	private queueProbe(ctx: MemoryUpdateCtx, question: string, source: "send_probe" | "restore", existingProbeId?: string): void {
+	private queueProbe(ctx: MemoryUpdateCtx, question: string, source: "send_probe" | "restore", existingProbeId?: string): string {
 		const probeId = existingProbeId ?? `${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 8)}`;
 		// Persist intent before touching Pi's in-memory queue. A crash in between
 		// leaves a recoverable pending probe rather than an invisible lost one.
@@ -967,6 +1000,7 @@ export class Contemplator {
 		// unrelated observer update or compaction callback cannot restore and enqueue
 		// a duplicate while the original idle steer is still pending.
 		this.queuedProbeIds.add(probeId);
+		this.probeCooldownPendingIds.add(probeId);
 		this.pi.sendMessage({
 			customType: CONTEMPLATOR_SUGGESTION,
 			content: `Background contemplator probe (advisory):\n${question}\n\nReferenced memories can be reviewed using the recall tool.`,
@@ -981,6 +1015,7 @@ export class Contemplator {
 			triggerTurn: "omitted",
 			source,
 		});
+		return probeId;
 	}
 
 	private queueStructuralReview(options: QueueStructuralReviewOptions): void {

package/src/commands/contemplator-view.ts

@@ -65,6 +65,8 @@ function liveStateLine(state: ContemplatorRunState): string {
 	if (state.running) return `LIVE · running for ${Math.max(0, Math.floor((Date.now() - (state.lastStartedAt ?? Date.now())) / 60_000))}m · ${pending}\n${timing}${error}`;
 	const reason = state.waitingFor === "observer"
 		? "waiting for observer backlog"
+		: state.waitingFor === "probe"
+			? "waiting for queued probe delivery"
 		: state.waitingFor === "memories"
 			? "waiting for memory threshold"
 			: state.waitingFor === "responses"
@@ -76,7 +78,7 @@ function liveStateLine(state: ContemplatorRunState): string {
 					: state.waitingFor === "passive"
 						? "passive mode"
 						: "idle";
-	return `LIVE · ${reason} · ${pending} · ${state.responsesSinceRun} primary responses since last run\n${timing}${error}`;
+	return `LIVE · ${reason} · ${pending} · ${state.responsesSinceRun} primary responses since cooldown anchor\n${timing}${error}`;
 }
 
 export function renderContemplator(entries: Entry[], state?: ContemplatorRunState): string {

package/src/commands/status.ts

@@ -43,6 +43,7 @@ function formatRunAge(timestamp: number): string {
 function contemplatorWaitingLabel(waitingFor: Runtime["contemplatorState"]["waitingFor"]): string {
 	switch (waitingFor) {
 		case "observer": return "waiting for observer backlog";
+		case "probe": return "waiting for queued probe delivery";
 		case "memories": return "waiting for memory threshold";
 		case "responses": return "waiting for response spacing";
 		case "ready": return "ready to launch";

package/src/config.ts

@@ -61,7 +61,7 @@ export interface Config {
 	reviewerModel?: ConfiguredModel;
 	contemplatorMinNewObservations: number;
 	contemplatorMinNewSummaries: number;
-	/** Minimum completed primary-model responses between contemplator runs. */
+	/** Minimum primary-model responses after contemplator completion, or after delivery of its probe, before the next run. */
 	contemplatorMinTurns: number;
 	/** Stateless loss-aware summarizer for the old memory pool. */
 	summarizerEnabled: boolean;

package/src/hooks/consolidation-trigger.ts

@@ -18,6 +18,7 @@ import {
 	observationToSummaryLine,
 	partitionMemoryPools,
 	rawTokensSinceObservationCoverage,
+	rawTokensSinceObservationCoverageThrough,
 	summaryToSummaryLine,
 	type Entry,
 } from "../session-ledger/index.js";
@@ -51,8 +52,12 @@ export function createSummarizerStallWatchdog(
 	return watchdog;
 }
 
-function sourceEntriesAfter(entries: Entry[], index: number): Entry[] {
-	return entries.slice(index + 1).filter(isSourceEntry);
+function sourceEntriesAfter(entries: Entry[], index: number, throughEntryId?: string): Entry[] {
+	const throughIndex = throughEntryId === undefined
+		? entries.length - 1
+		: entries.findIndex((entry) => entry.id === throughEntryId);
+	if (throughIndex < 0 || throughIndex <= index) return [];
+	return entries.slice(index + 1, throughIndex + 1).filter(isSourceEntry);
 }
 
 function appendEntry(pi: ExtensionAPI, customType: string, data: unknown): void {
@@ -161,6 +166,7 @@ function maybeLaunchConsolidation(pi: ExtensionAPI, runtime: Runtime, ctx: Conso
 	};
 
 	const sessionMetadata = debugSessionMetadata(ctx);
+	const launchGeneration = runtime.getContextGeneration();
 	const task = runtime.launchConsolidationTask(ctx, async () => withDebugLogContext({
 		enabled: runtime.config.debugLog === true,
 		cwd: ctx.cwd,
@@ -174,7 +180,15 @@ function maybeLaunchConsolidation(pi: ExtensionAPI, runtime: Runtime, ctx: Conso
 	// degraded mode rather than leaving all advisory work gated forever. Future
 	// primary activity still retries the observer.
 	void task.then(() => {
-		if (runtime.lastObserverError) runtime.notifyMemoryUpdate(ctx);
+		if (launchGeneration !== runtime.getContextGeneration()) return;
+		if (runtime.lastObserverError) {
+			runtime.notifyMemoryUpdate(ctx);
+			return;
+		}
+		// Source appended while the finite catch-up snapshot was running belongs
+		// to a later snapshot. Recheck after the lock is released; the completed
+		// pipeline has already given the contemplator its memory-update opportunity.
+		maybeLaunchConsolidation(pi, runtime, ctx);
 	});
 }
 
@@ -225,19 +239,25 @@ export async function runConsolidationPipeline(
 	const resolveModel = makeModelResolver(runtime, ctx);
 	const contextGeneration = runtime.getContextGeneration();
 
+	const pipelineEntries = options.observerEntries ?? (ctx.sessionManager.getBranch() as Entry[]);
+	const initialCoverage = latestCoverageIndex(pipelineEntries, OM_OBSERVATIONS_RECORDED);
+	const catchUpThroughId = sourceEntriesAfter(pipelineEntries, initialCoverage).at(-1)?.id;
 	const beforeFold = foldLedger(ctx.sessionManager.getBranch() as Entry[]);
 	runtime.consolidationPhase = "observer";
+	runtime.observerBacklogBlocking = catchUpThroughId !== undefined;
 	try {
-		// A large backlog is drained in bounded, oldest-first chunks. The normal
-		// trigger threshold controls when the batch stops; a static compaction
-		// snapshot is intentionally processed only once. Coverage must advance on
-		// every iteration, otherwise stop rather than spin on a failed chunk.
+		// Drain only the finite source snapshot captured at pipeline launch, using
+		// bounded oldest-first chunks. Concurrent source belongs to a later pipeline,
+		// so a fast primary agent cannot indefinitely extend this blocking backlog.
+		// A static compaction snapshot is intentionally processed only once. Coverage
+		// must advance on every iteration, otherwise stop rather than spin.
 		while (true) {
 			const beforeEntries = ctx.sessionManager.getBranch() as Entry[];
 			const beforeCoverage = latestCoverageIndex(beforeEntries, OM_OBSERVATIONS_RECORDED);
 			const observerOutcome = await runObserverStage(pi, runtime, ctx, resolveModel, {
 				force: options.forceObserver === true,
 				entries: options.observerEntries,
+				throughSourceEntryId: options.observerEntries ? undefined : catchUpThroughId,
 				contextGeneration,
 			});
 			if (observerOutcome === "abort") return;
@@ -245,13 +265,19 @@ export async function runConsolidationPipeline(
 
 			const afterEntries = ctx.sessionManager.getBranch() as Entry[];
 			const afterCoverage = latestCoverageIndex(afterEntries, OM_OBSERVATIONS_RECORDED);
-			const remainingTokens = rawTokensSinceObservationCoverage(afterEntries);
+			const remainingTokens = catchUpThroughId === undefined
+				? 0
+				: rawTokensSinceObservationCoverageThrough(afterEntries, catchUpThroughId);
 			if (afterCoverage <= beforeCoverage || remainingTokens < runtime.config.observeAfterTokens) break;
 			debugLog("observer.backlog_continue", { remainingTokens, afterCoverage });
 		}
 	} catch (error) {
 		debugLog("observer.error", { errorMessage: runtime.recordConsolidationStageError(ctx, "observer", error) });
 		return;
+	} finally {
+		// New source appended after catchUpThroughId was never part of this
+		// pipeline's blocking backlog. Clear before notifying the contemplator.
+		if (contextGeneration === runtime.getContextGeneration()) runtime.observerBacklogBlocking = false;
 	}
 	const afterFold = foldLedger(ctx.sessionManager.getBranch() as Entry[]);
 	const beforeIds = new Set(beforeFold.observations.map((item) => item.id));
@@ -408,10 +434,12 @@ async function runObserverStage(
 	runtime: Runtime,
 	ctx: ConsolidationCtx,
 	resolveModel: (stage: "observer") => Promise<ResolvedModel | undefined>,
-	options: { force?: boolean; entries?: Entry[]; contextGeneration?: number } = {},
+	options: { force?: boolean; entries?: Entry[]; throughSourceEntryId?: string; contextGeneration?: number } = {},
 ): Promise<StageOutcome> {
 	const entries = options.entries ?? (ctx.sessionManager.getBranch() as Entry[]);
-	const tokens = rawTokensSinceObservationCoverage(entries);
+	const tokens = options.throughSourceEntryId === undefined
+		? rawTokensSinceObservationCoverage(entries)
+		: rawTokensSinceObservationCoverageThrough(entries, options.throughSourceEntryId);
 	if (!options.force && tokens < runtime.config.observeAfterTokens) return "continue";
 
 	// Resolve the model before building the chunk: the default chunk cap
@@ -424,7 +452,7 @@ async function runObserverStage(
 	}
 
 	const lastCoverageIdx = latestCoverageIndex(entries, OM_OBSERVATIONS_RECORDED);
-	const backlogEntries = sourceEntriesAfter(entries, lastCoverageIdx);
+	const backlogEntries = sourceEntriesAfter(entries, lastCoverageIdx, options.throughSourceEntryId);
 
 	// Budget the text that is actually sent to the observer, including source
 	// labels and rendered message content. Complete entries are kept intact.

package/src/runtime.ts

@@ -82,9 +82,9 @@ export interface ContemplatorRunState {
 	pendingObservations: number;
 	pendingSummaries: number;
 	pendingReviews: number;
-	/** Completed primary-model responses since the previous contemplator run. */
+	/** Completed primary-model responses since the current completion/probe-delivery spacing anchor. */
 	responsesSinceRun: number;
-	waitingFor: "disabled" | "passive" | "observer" | "memories" | "responses" | "ready" | "running" | "idle";
+	waitingFor: "disabled" | "passive" | "observer" | "probe" | "memories" | "responses" | "ready" | "running" | "idle";
 	lastStartedAt?: number;
 	lastCompletedAt?: number;
 	lastError?: string;
@@ -182,6 +182,8 @@ export class Runtime {
 	private settingsUpdateListener: ((ctx: MemoryUpdateCtx, settings: SettingsUpdate) => void) | undefined;
 	private contextGeneration = 0;
 	consolidationPhase: ConsolidationPhase | undefined;
+	/** True only while the observer is draining the finite source snapshot captured at pipeline start. */
+	observerBacklogBlocking = false;
 	compactInFlight = false;
 	compactRequested = false;
 	/** Agent-authored instructions to deliver after an explicit compact_context request. */
@@ -274,6 +276,7 @@ export class Runtime {
 		this.consolidationInFlight = false;
 		this.consolidationPromise = null;
 		this.consolidationPhase = undefined;
+		this.observerBacklogBlocking = false;
 		this.summarizerInFlight = false;
 		this.summarizerPromise = null;
 		this.reviewInFlight = false;

package/src/session-ledger/progress.ts

@@ -138,6 +138,23 @@ export function rawTokensSinceObservationCoverage(entries: Entry[]): number {
 	return rawTokensSinceCoverage(entries, OM_OBSERVATIONS_RECORDED);
 }
 
+/**
+ * Sum uncovered observer source only through a fixed branch entry. Source
+ * appended after that boundary is intentionally excluded so one catch-up pass
+ * has a finite backlog even while the primary agent keeps producing output.
+ */
+export function rawTokensSinceObservationCoverageThrough(entries: Entry[], throughEntryId: string): number {
+	const throughIndex = entryIndexForId(entries, throughEntryId);
+	if (throughIndex < 0) return 0;
+	const coverageIndex = latestCoverageIndex(entries, OM_OBSERVATIONS_RECORDED);
+	if (coverageIndex >= throughIndex) return 0;
+	let total = 0;
+	for (let i = Math.max(0, coverageIndex + 1); i <= throughIndex; i++) {
+		if (isSourceEntry(entries[i])) total += estimateEntryTokens(entries[i]);
+	}
+	return total;
+}
+
 export function findLastCompactionIndex(entries: Entry[]): number {
 	for (let i = entries.length - 1; i >= 0; i--) {
 		if (entries[i].type === "compaction") return i;