@matthewfl/pi-contemplator 0.0.8 → 0.0.10

package/README.md

@@ -190,9 +190,15 @@ Model selection, trigger thresholds, passive mode (which stops all background wo
 ```bash
 npm install
 npm test
+npm run test:unit
+npm run test:e2e
 npm run typecheck
 ```
 
+`npm test` runs both the unit and RPC end-to-end suites. Use `test:unit` or `test:e2e` to run either suite independently.
+
+`test:e2e` starts a local OpenAI-compatible model server and launches isolated real Pi CLI processes in RPC mode while mocking only the external model-server boundary. The suites cover observer, reflector, dropper, contemplator, reviewer, primary-agent, memory-tool, and compaction flows. They exercise slow and parallel tools, probe races and feedback, hidden/idle delivery, cumulative activity time, process restore, reviewer transcript resume and budget exhaustion, accepted/rejected reviews, manual/failed compaction continuations and non-restarting proactive compaction, compaction observer sidecars, malformed memory records, huge-source bounding, id collisions, role-specific model routing, feature flags, session-tree forks, and concurrent session isolation.
+
 ## License
 
 MIT. See [LICENSE](LICENSE).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@matthewfl/pi-contemplator",
-  "version": "0.0.8",
+  "version": "0.0.10",
   "description": "A Pi extension that keeps long-running agentic sessions on track with background memory, contemplation, and structural review.",
   "type": "module",
   "license": "MIT",
@@ -39,7 +39,9 @@
   ],
   "scripts": {
     "typecheck": "tsc --noEmit",
-    "test": "vitest run"
+    "test": "npm run test:unit && npm run test:e2e",
+    "test:unit": "vitest run",
+    "test:e2e": "node tests/e2e/rpc-contemplator.mjs && node tests/e2e/rpc-delivery.mjs && node tests/e2e/rpc-restore-review.mjs && node tests/e2e/rpc-compaction.mjs && node tests/e2e/rpc-compaction-resilience.mjs && node tests/e2e/rpc-memory-edges.mjs && node tests/e2e/rpc-routing-isolation.mjs"
   },
   "peerDependencies": {
     "@earendil-works/pi-agent-core": "*",
@@ -48,10 +50,10 @@
     "@earendil-works/pi-tui": "*"
   },
   "devDependencies": {
-    "@earendil-works/pi-agent-core": "^0.81.0",
-    "@earendil-works/pi-ai": "^0.81.0",
-    "@earendil-works/pi-coding-agent": "^0.81.0",
-    "@earendil-works/pi-tui": "^0.81.0",
+    "@earendil-works/pi-agent-core": "^0.84.2",
+    "@earendil-works/pi-ai": "^0.84.2",
+    "@earendil-works/pi-coding-agent": "^0.84.2",
+    "@earendil-works/pi-tui": "^0.84.2",
     "@types/node": "^22.0.0",
     "typebox": "^1.1.38",
     "typescript": "^5.6.0",

package/src/agents/contemplator/agent.ts

@@ -670,20 +670,23 @@ export class Contemplator {
 		// designed for agents that run for hours; a probe must be injected after
 		// the current tool-call batch, before the very next model request. Never
 		// change this to nextTurn: that can postpone delivery until a user prompt.
-		// triggerTurn stays false so a probe never starts an extra agent turn.
+		// IMPORTANT: omit triggerTurn entirely. Pi 0.84+ interprets an explicit
+		// triggerTurn:false as "do not queue while streaming" and inserts directly
+		// into agent.state, outside the active run's context snapshot. Omitting it
+		// still does not start a turn while idle, but allows steer to work in-run.
 		if (this.agentActiveSince !== undefined) this.queuedProbeIds.add(probeId);
 		this.pi.sendMessage({
 			customType: CONTEMPLATOR_SUGGESTION,
 			content: `Background contemplator probe (advisory):\n${question}`,
 			display: this.runtime.config.showContemplatorMessages,
 			details: { version: 1, question, source, probeId },
-		}, { deliverAs: "steer", triggerTurn: false });
+		}, { deliverAs: "steer" });
 		debugLog("contemplator.suggestion_queued", {
 			probeId,
 			suggestionLength: question.length,
 			delivery: "pi.sendMessage",
 			deliverAs: "steer",
-			triggerTurn: false,
+			triggerTurn: "omitted",
 			source,
 		});
 	}
@@ -751,7 +754,9 @@ export class Contemplator {
 				debugLog(result.outcome === "proposal" ? "reviewer.proposal_created" : "reviewer.no_proposal", { reviewRequestId: request.id, reviewMemoryId: result.id, scope: result.scope });
 				if (result.outcome === "proposal") {
 					const notice = `BACKGROUND ${result.scope.toUpperCase()} REVIEW PROPOSAL [${result.id}]\n\n${result.summary}\n\nRecall memory [${result.id}] to read the full conceptual proposal when it is relevant.\n\nThis is advisory. Evaluate it against the actual environment and current work.`;
-					this.pi.sendMessage({ customType: REVIEW_PROPOSAL_MESSAGE, content: notice, display: this.runtime.config.showContemplatorMessages, details: { version: 1, reviewRequestId: request.id, reviewMemoryId: result.id, scope: result.scope } }, { deliverAs: "steer", triggerTurn: false });
+					// As with probes, triggerTurn must be omitted or Pi 0.84+ bypasses
+					// the streaming steer queue and the active run never sees this message.
+					this.pi.sendMessage({ customType: REVIEW_PROPOSAL_MESSAGE, content: notice, display: this.runtime.config.showContemplatorMessages, details: { version: 1, reviewRequestId: request.id, reviewMemoryId: result.id, scope: result.scope } }, { deliverAs: "steer" });
 					this.pi.appendEntry(OM_REVIEWER_NOTICE, { version: 1, reviewRequestId: request.id, reviewMemoryId: result.id, scope: result.scope, content: notice });
 					this.markTipPersisted(ctx);
 					debugLog("reviewer.primary_notice_queued", { reviewRequestId: request.id, reviewMemoryId: result.id });

package/src/hooks/compaction-hook.ts

@@ -29,10 +29,11 @@ export function registerCompactionHook(pi: ExtensionAPI, runtime: Runtime): void
 
 		const initiatedByOm = runtime.compactInFlight && event.reason === "manual";
 		const reason = initiatedByOm ? (runtime.compactOrigin ?? "proactive") : event.reason;
+		const omWillResume = initiatedByOm && (runtime.compactOrigin === "agent-requested" || runtime.compactOrigin === "length-stop");
 		if (ctx.hasUI) {
 			let pending = "";
 			if (event.willRetry) pending = ", retry pending";
-			else if (initiatedByOm) pending = ", resume pending";
+			else if (omWillResume) pending = ", resume pending";
 			ctx.ui.setStatus?.(COMPACTION_STATUS_KEY, `OM compaction: running (${reason}${pending})`);
 			if (!initiatedByOm) {
 				const continuation = event.willRetry ? "; the interrupted agent run will resume automatically" : "";
@@ -88,12 +89,13 @@ export function registerCompactionHook(pi: ExtensionAPI, runtime: Runtime): void
 	pi.on("session_compact", (event: any, ctx: any) => {
 		const initiatedByOm = runtime.compactInFlight && event.reason === "manual";
 		const reason = initiatedByOm ? (runtime.compactOrigin ?? "proactive") : event.reason;
+		const omWillResume = initiatedByOm && (runtime.compactOrigin === "agent-requested" || runtime.compactOrigin === "length-stop");
 		if (event.willRetry) watchForNativeCompactionResume(pi, runtime, ctx);
 		if (!ctx.hasUI) return;
 		ctx.ui.setStatus?.(COMPACTION_STATUS_KEY, undefined);
 		let continuation = "";
 		if (event.willRetry) continuation = "; resuming the interrupted agent run";
-		else if (initiatedByOm) continuation = "; resuming the agent run";
+		else if (omWillResume) continuation = "; resuming the agent run";
 		ctx.ui.notify(`Observational memory: compaction complete (${reason})${continuation}`, "info");
 	});
 }

package/src/hooks/compaction-trigger.ts

@@ -9,52 +9,22 @@ import {
 } from "./compaction-resume.js";
 
 const COMPACTION_STATUS_KEY = "observational-memory-compaction";
+type CompactionOrigin = "agent-requested" | "length-stop" | "proactive";
+
+type TriggerOptions = {
+	origin: CompactionOrigin;
+	resume: boolean;
+	threshold?: number;
+	shortContinuationPrompt?: string;
+};
 
 export function registerCompactionTrigger(pi: ExtensionAPI, runtime: Runtime): void {
 	registerCompactionResumeAcknowledgement(pi, runtime);
-	pi.on("agent_end", (event: any, ctx: any) => {
-		runtime.ensureConfig(ctx.cwd);
-		if (runtime.compactInFlight) return;
-
-		const agentRequested = runtime.compactRequested;
-		const shortContinuationPrompt = agentRequested ? runtime.compactContinuationPrompt : undefined;
-		if (agentRequested) runtime.compactRequested = false;
-		else if (runtime.config.passive === true) return;
 
-		const contextWindow = typeof ctx.model?.contextWindow === "number" ? ctx.model.contextWindow : 0;
-		let threshold: number | undefined;
-		if (!agentRequested) {
-			// agent_end fires before Pi decides whether to retry or compact-and-retry an
-			// interrupted request. Starting ctx.compact() here turns it into a manual
-			// compaction (willRetry=false) and can consume Pi's overflow recovery, leaving
-			// the agent idle. Let Pi handle every failed/aborted/overflow response; OM's
-			// session_before_compact hook still supplies the actual memory compaction.
-			const lastAssistant = [...event.messages].reverse().find(
-				(m): m is Extract<typeof m, { role: "assistant" }> => m.role === "assistant",
-			);
-			if (
-				lastAssistant
-				&& (
-					lastAssistant.stopReason === "error"
-						|| lastAssistant.stopReason === "aborted"
-						|| isContextOverflow(lastAssistant, contextWindow)
-				)
-			) return;
-
-			const entries = ctx.sessionManager.getBranch() as Entry[];
-			const tokens = rawTokensSinceLastCompaction(entries);
-			// Resolve the proactive-compaction threshold from the active model's context
-			// window when ratio mode is configured. ctx.model is the current session model
-			// (Model<any> | undefined per ExtensionContext).
-			threshold = resolveCompactAfterTokens(runtime.config, contextWindow > 0 ? contextWindow : undefined);
-			if (tokens < threshold) return;
-		}
-
-		// Capture ctx properties synchronously — the setTimeout + async work below
-		// may outlive the extension ctx (stale after session replacement/reload).
+	const triggerCompaction = (ctx: any, options: TriggerOptions): void => {
+		const { origin, resume, threshold, shortContinuationPrompt } = options;
 		const hasUI = ctx.hasUI;
 		const ui = ctx.ui;
-		const origin = agentRequested ? "agent-requested" : "proactive";
 
 		runtime.compactInFlight = true;
 		runtime.compactOrigin = origin;
@@ -63,7 +33,7 @@ export function registerCompactionTrigger(pi: ExtensionAPI, runtime: Runtime): v
 				if (!ctx.isIdle()) {
 					runtime.compactInFlight = false;
 					runtime.compactOrigin = undefined;
-					if (agentRequested) runtime.compactRequested = true;
+					if (origin === "agent-requested") runtime.compactRequested = true;
 					if (hasUI) ui?.notify(
 						"Observational memory: compaction deferred — agent became busy before compaction",
 						"info",
@@ -82,22 +52,22 @@ export function registerCompactionTrigger(pi: ExtensionAPI, runtime: Runtime): v
 					return;
 				}
 				if (hasUI) {
-					ui?.setStatus?.(COMPACTION_STATUS_KEY, `OM compaction: running (${origin}, resume pending)`);
-					const reason = agentRequested ? "agent-requested, " : "";
+					const pending = resume ? ", resume pending" : "";
+					ui?.setStatus?.(COMPACTION_STATUS_KEY, `OM compaction: running (${origin}${pending})`);
+					const reason = origin === "agent-requested" ? "agent-requested, " : origin === "length-stop" ? "length-stop, " : "";
+					const continuation = resume ? "; the interrupted agent run will resume automatically" : "";
 					ui?.notify(
-						`Observational memory: compaction started (${reason}~${currentTokens.toLocaleString()} tokens); the agent will resume automatically`,
+						`Observational memory: compaction started (${reason}~${currentTokens.toLocaleString()} tokens)${continuation}`,
 						"info",
 					);
 				}
-				if (agentRequested) runtime.compactContinuationPrompt = undefined;
+				if (origin === "agent-requested") runtime.compactContinuationPrompt = undefined;
 				ctx.compact({
 					onComplete: () => {
 						runtime.compactInFlight = false;
 						runtime.compactOrigin = undefined;
-						// Both explicit and proactive OM compactions are manual from Pi's
-						// perspective (willRetry=false), so Pi will not continue either one.
-						// Always enqueue a hidden continuation after OM finishes compacting.
-						resumeAfterCompaction(pi, runtime, { hasUI, ui }, false, shortContinuationPrompt);
+						if (hasUI && !resume) ui?.setStatus?.(COMPACTION_STATUS_KEY, undefined);
+						if (resume) resumeAfterCompaction(pi, runtime, { hasUI, ui }, false, shortContinuationPrompt);
 					},
 					onError: (error: { message: string }) => {
 						runtime.compactInFlight = false;
@@ -106,20 +76,68 @@ export function registerCompactionTrigger(pi: ExtensionAPI, runtime: Runtime): v
 						if (error.message !== "Compaction cancelled" && hasUI) {
 							ui?.notify(`Observational memory: ${error.message}`, "error");
 						}
-						resumeAfterCompaction(pi, runtime, { hasUI, ui }, true, shortContinuationPrompt);
+						if (resume) resumeAfterCompaction(pi, runtime, { hasUI, ui }, true, shortContinuationPrompt);
 					},
 				});
 			} catch (error) {
 				runtime.compactInFlight = false;
-				if (agentRequested) runtime.compactContinuationPrompt = undefined;
+				if (origin === "agent-requested") runtime.compactContinuationPrompt = undefined;
 				runtime.compactOrigin = undefined;
 				const msg = error instanceof Error ? error.message : String(error);
 				if (hasUI) {
 					ui?.setStatus?.(COMPACTION_STATUS_KEY, undefined);
 					ui?.notify(`Observational memory: compact threw: ${msg}`, "error");
 				}
-				resumeAfterCompaction(pi, runtime, { hasUI, ui }, true, shortContinuationPrompt);
+				if (resume) resumeAfterCompaction(pi, runtime, { hasUI, ui }, true, shortContinuationPrompt);
 			}
 		}, 0);
+	};
+
+	pi.on("agent_end", (event: any, ctx: any) => {
+		runtime.ensureConfig(ctx.cwd);
+		if (runtime.compactInFlight) return;
+
+		const agentRequested = runtime.compactRequested;
+		if (agentRequested) {
+			const shortContinuationPrompt = runtime.compactContinuationPrompt;
+			runtime.compactRequested = false;
+			triggerCompaction(ctx, { origin: "agent-requested", resume: true, shortContinuationPrompt });
+			return;
+		}
+		if (runtime.config.passive === true) return;
+
+		// Pi owns error, abort, and overflow retry policy. OM's session hook still
+		// supplies the compaction contents when Pi performs a native retry.
+		const lastAssistant = [...event.messages].reverse().find(
+			(m): m is Extract<typeof m, { role: "assistant" }> => m.role === "assistant",
+		);
+		const contextWindow = typeof ctx.model?.contextWindow === "number" ? ctx.model.contextWindow : 0;
+		if (
+			!lastAssistant
+			|| lastAssistant.stopReason === "error"
+			|| lastAssistant.stopReason === "aborted"
+			|| isContextOverflow(lastAssistant, contextWindow)
+		) return;
+
+		// A non-overflow length stop is interrupted work, not a normal completed
+		// turn. Preserve the older compact-and-resume behavior only for this case.
+		if (lastAssistant.stopReason !== "length") return;
+		const entries = ctx.sessionManager.getBranch() as Entry[];
+		const threshold = resolveCompactAfterTokens(runtime.config, contextWindow > 0 ? contextWindow : undefined);
+		if (rawTokensSinceLastCompaction(entries) < threshold) return;
+		triggerCompaction(ctx, { origin: "length-stop", resume: true, threshold });
+	});
+
+	// Proactive threshold compaction is maintenance after Pi has fully settled.
+	// It must not manufacture another agent turn: the previous run completed
+	// normally and there is no interrupted work to resume.
+	pi.on("agent_settled", (_event: any, ctx: any) => {
+		runtime.ensureConfig(ctx.cwd);
+		if (runtime.config.passive === true || runtime.compactInFlight || runtime.compactRequested) return;
+		const entries = ctx.sessionManager.getBranch() as Entry[];
+		const contextWindow = typeof ctx.model?.contextWindow === "number" ? ctx.model.contextWindow : undefined;
+		const threshold = resolveCompactAfterTokens(runtime.config, contextWindow);
+		if (rawTokensSinceLastCompaction(entries) < threshold) return;
+		triggerCompaction(ctx, { origin: "proactive", resume: false, threshold });
 	});
 }

package/src/runtime.ts

@@ -147,7 +147,7 @@ export class Runtime {
 	compactRequested = false;
 	/** Agent-authored instructions to deliver after an explicit compact_context request. */
 	compactContinuationPrompt: string | undefined;
-	compactOrigin: "proactive" | "agent-requested" | undefined;
+	compactOrigin: "proactive" | "agent-requested" | "length-stop" | undefined;
 	compactHookInFlight = false;
 	compactionResumePending = false;
 	compactionResumeGeneration = 0;