@matthewfl/pi-contemplator 0.1.14 → 0.1.16

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@matthewfl/pi-contemplator",
-  "version": "0.1.14",
+  "version": "0.1.16",
   "description": "A Pi extension that keeps long-running agentic sessions on track with background memory, contemplation, and structural review.",
   "type": "module",
   "license": "MIT",
@@ -50,10 +50,11 @@
     "@earendil-works/pi-tui": "*"
   },
   "devDependencies": {
-    "@earendil-works/pi-agent-core": "^0.84.4",
-    "@earendil-works/pi-ai": "^0.84.4",
-    "@earendil-works/pi-coding-agent": "^0.84.4",
-    "@earendil-works/pi-tui": "^0.84.4",
+    "@earendil-works/pi-agent-core": "^0.85.1",
+    "@earendil-works/pi-ai": "^0.85.1",
+    "@earendil-works/pi-coding-agent": "^0.85.1",
+    "@earendil-works/pi-server": "^0.85.1",
+    "@earendil-works/pi-tui": "^0.85.1",
     "@types/node": "^22.0.0",
     "typebox": "^1.1.38",
     "typescript": "^5.6.0",

package/src/agents/contemplator/agent.ts

@@ -506,6 +506,8 @@ export class Contemplator {
 		this.history = [];
 		this.historyEntryIds = [];
 		const historyMessagesByEntryId = new Map<string, AgentMessage>();
+		const checkpointCoveredObservationIds = new Set<string>();
+		const checkpointCoveredReviewIds = new Set<string>();
 		let resetProjection: ReturnType<typeof fullProjection> | undefined;
 		if (resetTracking) {
 			this.deliveredProbeIds.clear();
@@ -539,7 +541,7 @@ export class Contemplator {
 				}
 			}
 			if (entry.customType === CONTEMPLATOR_MESSAGE && entry.data && typeof entry.data === "object") {
-				const data = entry.data as { message?: unknown; compacted?: unknown; retainedMessageEntryIds?: unknown };
+				const data = entry.data as { message?: unknown; compacted?: unknown; retainedMessageEntryIds?: unknown; coveredObservationIds?: unknown; coveredReviewIds?: unknown };
 				const message = data.message;
 				if (message && typeof message === "object") {
 					const typedMessage = message as AgentMessage;
@@ -550,6 +552,8 @@ export class Contemplator {
 							: [];
 						this.history = [typedMessage, ...retainedIds.map((id) => historyMessagesByEntryId.get(id)!)];
 						this.historyEntryIds = [entry.id, ...retainedIds];
+						if (Array.isArray(data.coveredObservationIds)) for (const id of data.coveredObservationIds) if (typeof id === "string") checkpointCoveredObservationIds.add(id);
+						if (Array.isArray(data.coveredReviewIds)) for (const id of data.coveredReviewIds) if (typeof id === "string") checkpointCoveredReviewIds.add(id);
 					} else {
 						this.history.push(typedMessage);
 						this.historyEntryIds.push(entry.id);
@@ -607,10 +611,10 @@ export class Contemplator {
 			}
 		}
 		if (resetTracking && resetProjection) {
-			// Successful contemplator update prompts are the durable coverage record.
-			// Only memories present in those prompts are considered seen after reload;
-			// memories from a failed, unpersisted run remain pending and retryable.
-			const coveredIds = new Set<string>();
+			// Successful update prompts and private-history compaction checkpoints are
+			// the durable coverage record. Checkpoints carry ids from prompts replaced
+			// by their summary; failed, unpersisted runs remain pending and retryable.
+			const coveredIds = new Set<string>([...checkpointCoveredObservationIds, ...checkpointCoveredReviewIds]);
 			for (const message of this.history) {
 				if (message.role !== "user") continue;
 				const text = customMessageText(message.content);
@@ -1359,7 +1363,28 @@ export class Contemplator {
 			debugLog("contemplator.compaction_postponed", { reason: "retained history lacked durable entry ids", retainedMessageCount: retainedMessages.length, retainedReferenceCount: retainedMessageEntryIds.length });
 			return;
 		}
-		const checkpoint = { version: 2, compacted: true, message: summaryMessage, retainedMessageEntryIds };
+		// Record only coverage evidenced by the exact durable prompt prefix being
+		// replaced. `seen*Ids` may also contain observations appended concurrently
+		// while summary generation awaited; those remain pending and must not be
+		// declared covered before their own prompt is persisted. Coverage is a delta
+		// per checkpoint, and restore unions checkpoints, avoiding cumulative O(n²)
+		// id duplication across a very long session.
+		const prefixCoveredIds = new Set<string>();
+		for (const message of history.slice(0, prefixEnd)) {
+			if (message.role !== "user") continue;
+			const text = customMessageText(message.content);
+			if (!text.includes("NEW MEMORY UPDATE")) continue;
+			for (const id of memoryReferenceIds(text)) prefixCoveredIds.add(id);
+		}
+		const currentProjection = fullProjection(ctx.sessionManager.getBranch() as Entry[]);
+		const checkpoint = {
+			version: 2,
+			compacted: true,
+			message: summaryMessage,
+			retainedMessageEntryIds,
+			coveredObservationIds: currentProjection.observations.filter((item) => prefixCoveredIds.has(item.id)).map((item) => item.id),
+			coveredReviewIds: (currentProjection.reviews ?? []).filter((item) => prefixCoveredIds.has(item.id)).map((item) => item.id),
+		};
 		const checkpointEntryId = this.appendContemplatorHistoryEntry(ctx, checkpoint);
 		this.history = [summaryMessage, ...retainedMessages];
 		this.historyEntryIds = [checkpointEntryId, ...retainedMessageEntryIds];

package/src/agents/observer/agent.ts

@@ -4,6 +4,7 @@ import { Type } from "@earendil-works/pi-ai";
 import { streamSimple } from "@earendil-works/pi-ai/compat";
 import type { Static } from "typebox";
 import { hashId } from "../../ids.js";
+import { replayTruncatedThinkingAsText } from "../replay-truncated-thinking.js";
 import { logAgentStreamError } from "../stream-errors.js";
 import { OBSERVER_AGENT_LOOP_MAX_TOKENS, boundedMaxTokens } from "../../model-budget.js";
 import { OBSERVER_SYSTEM } from "./prompts.js";
@@ -205,7 +206,7 @@ IMPORTANT: Now call record_observations to record the useful new observations fr
 		apiKey,
 		headers,
 		maxTokens: boundedMaxTokens(model, OBSERVER_AGENT_LOOP_MAX_TOKENS),
-		convertToLlm: (msgs) => msgs as Message[],
+		convertToLlm: replayTruncatedThinkingAsText,
 		toolExecution: "sequential",
 		shouldStopAfterTurn: () => {
 			turnCount++;
@@ -270,9 +271,11 @@ IMPORTANT: Now call record_observations to record the useful new observations fr
 		// maximum. agentLoop stops on `length` when no tool call was completed; it
 		// does not automatically send a continuation request. Preserve the partial
 		// response so the model can continue from work it already performed rather
-		// than paying to reproduce it, then append a short tool-focused instruction
-		// and reduce reasoning to minimal. A second length stop fails forward at the
-		// bounded-chunk level.
+		// than paying to reproduce it. Plaintext thinking is replayed as ordinary
+		// assistant text at the LLM boundary because some provider templates strip
+		// historical reasoning; encrypted thinking retains its opaque structure.
+		// Then append a short tool-focused instruction and reduce reasoning to
+		// minimal. A second length stop fails forward at the bounded-chunk level.
 		terminalFailure = undefined;
 		const retryPrompt: Message = {
 			role: "user",

package/src/agents/replay-truncated-thinking.ts

@@ -0,0 +1,21 @@
+import type { AgentMessage } from "@earendil-works/pi-agent-core";
+import type { Message } from "@earendil-works/pi-ai";
+
+/**
+ * Many provider chat templates discard historical reasoning blocks. Preserve
+ * unfinished plaintext work after an output-length stop by replaying it as
+ * ordinary assistant text. Redacted/encrypted blocks remain structured so
+ * their opaque provider payload stays replayable. The original transcript is
+ * never mutated; this transformation is only applied at the LLM boundary.
+ */
+export function replayTruncatedThinkingAsText(messages: readonly AgentMessage[]): Message[] {
+	return messages.map((message) => {
+		if (message.role !== "assistant" || message.stopReason !== "length" || !message.content.some((part) => part.type === "thinking" && !part.redacted)) return message as Message;
+		return {
+			...message,
+			content: message.content.map((part) => part.type === "thinking" && !part.redacted
+				? { type: "text" as const, text: `[Incomplete analysis from the preceding truncated response]\n${part.thinking}` }
+				: part),
+		} as Message;
+	});
+}

package/src/agents/summarizer/agent.ts

@@ -26,6 +26,7 @@ import {
 import { estimateStringTokens } from "../../tokens.js";
 import { createRecallAgentTool } from "../../tools/recall-observation.js";
 import { createSearchMemoriesAgentTool } from "../../tools/search-memories.js";
+import { replayTruncatedThinkingAsText } from "../replay-truncated-thinking.js";
 import { logAgentStreamError } from "../stream-errors.js";
 import { summarizerContinue, SUMMARIZER_SYSTEM } from "./prompts.js";
 import {
@@ -106,27 +107,6 @@ function textResult(text: string, details: Record<string, unknown> = {}, termina
 	return { content: [{ type: "text" as const, text }], details, ...(terminate ? { terminate: true } : {}) };
 }
 
-/**
- * Many provider chat templates discard historical reasoning blocks. Preserve
- * unfinished plaintext work after an output-length stop by replaying it as
- * ordinary assistant text; unlike provider-specific thinking metadata, text
- * survives every supported conversation serializer. Redacted/encrypted blocks
- * must remain structured so their opaque provider payload stays replayable.
- * The durable/in-memory transcript remains unchanged—this transformation is
- * only applied at the LLM boundary.
- */
-export function replayTruncatedThinkingAsText(messages: readonly AgentMessage[]): Message[] {
-	return messages.map((message) => {
-		if (message.role !== "assistant" || message.stopReason !== "length" || !message.content.some((part) => part.type === "thinking" && !part.redacted)) return message as Message;
-		return {
-			...message,
-			content: message.content.map((part) => part.type === "thinking" && !part.redacted
-				? { type: "text" as const, text: `[Incomplete analysis from the preceding truncated response]\n${part.thinking}` }
-				: part),
-		} as Message;
-	});
-}
-
 function preview(content: string): string {
 	const compact = content.replace(/\s+/g, " ").trim();
 	return compact.length <= 100 ? compact : `${compact.slice(0, 100)}…`;

package/src/commands/status.ts

@@ -62,10 +62,6 @@ function truncateStatusText(value: string, limit = 1_000): string {
 	return value.length <= limit ? value : `${value.slice(0, limit - 1)}…`;
 }
 
-function tokenSum(items: { tokenCount: number }[]): number {
-	return items.reduce((sum, item) => sum + item.tokenCount, 0);
-}
-
 function addedSuffix(count: number): string | undefined {
 	return count > 0 ? `+${count.toLocaleString()}` : undefined;
 }
@@ -90,8 +86,6 @@ export function registerStatusCommand(pi: ExtensionAPI, runtime: Runtime): void
 			const full = fullProjection(entries);
 			const drift = diffProjection(visible, full);
 
-			const visibleObservationTokens = tokenSum(visible.observations);
-			const visibleSummaryTokens = tokenSum(visible.summaries);
 			const pools = partitionMemoryPools(folded.activeObservations, folded.activeSummaries, runtime.config.newMemoryPoolMaxTokens);
 			const observationLine = appendSuffixes(
 				`Observations: ${folded.observations.length} recorded / ${folded.activeObservations.length} active / ${visible.observations.length} visible`,
@@ -126,10 +120,9 @@ export function registerStatusCommand(pi: ExtensionAPI, runtime: Runtime): void
 				`Observer source backlog: ~${obsProgress.toLocaleString()} / ${runtime.config.observeAfterTokens.toLocaleString()} tokens (${pct(obsProgress, runtime.config.observeAfterTokens)}%)`,
 				`Summarizer trigger:      old pool ~${pools.oldTokens.toLocaleString()} / ${summarizerTrigger.toLocaleString()} tokens (${pct(pools.oldTokens, summarizerTrigger)}%)`,
 				`Automatic compaction source backlog: ~${compactionProgress.toLocaleString()} / ${compactThreshold.toLocaleString()} tokens (${pct(compactionProgress, compactThreshold)}%; injected memory excluded)`,
-				`Visible observation pool: ~${visibleObservationTokens.toLocaleString()} tokens`,
+				`Active memory total:     ~${pools.totalTokens.toLocaleString()} tokens (observations + summaries; split below)`,
 				`New memory pool:         ~${pools.newTokens.toLocaleString()} / ${runtime.config.newMemoryPoolMaxTokens.toLocaleString()} protection-budget tokens (${pct(pools.newTokens, runtime.config.newMemoryPoolMaxTokens)}%; newest memory always protected whole)`,
-				`Old memory pool:         ~${pools.oldTokens.toLocaleString()} / ${runtime.config.oldMemoryPoolTargetTokens.toLocaleString()} advisory target tokens (${pct(pools.oldTokens, runtime.config.oldMemoryPoolTargetTokens)}%)`,
-				`Summary pool:            ~${visibleSummaryTokens.toLocaleString()} visible tokens`,
+				`Old memory pool:         ~${pools.oldTokens.toLocaleString()} / ${runtime.config.oldMemoryPoolTargetTokens.toLocaleString()} advisory target tokens (${pct(pools.oldTokens, runtime.config.oldMemoryPoolTargetTokens)}%; observations + summaries)`,
 				`Summarizer:              ${runtime.config.summarizerEnabled === false ? "disabled" : "enabled"}; retrigger after +${runtime.config.summarizerRetriggerTokens.toLocaleString()} old-pool tokens / sample above ~${summarizerSamplingTokens.toLocaleString()} tokens`,
 				`Summarizer model:        ${configuredModelLabel(runtime.configuredMemoryWorkerModel("summarizer"))}`,
 				`Observer model:          ${configuredModelLabel(runtime.configuredMemoryWorkerModel("observer"))}`,