@matthewfl/pi-contemplator 0.1.8 → 0.1.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@matthewfl/pi-contemplator",
-  "version": "0.1.8",
+  "version": "0.1.10",
   "description": "A Pi extension that keeps long-running agentic sessions on track with background memory, contemplation, and structural review.",
   "type": "module",
   "license": "MIT",

package/src/agents/summarizer/agent.ts

@@ -27,7 +27,7 @@ import { estimateStringTokens } from "../../tokens.js";
 import { createRecallAgentTool } from "../../tools/recall-observation.js";
 import { createSearchMemoriesAgentTool } from "../../tools/search-memories.js";
 import { logAgentStreamError } from "../stream-errors.js";
-import { SUMMARIZER_CONTINUE, SUMMARIZER_SYSTEM } from "./prompts.js";
+import { summarizerContinue, SUMMARIZER_SYSTEM } from "./prompts.js";
 import {
 	renderSummarizerMemory,
 	sampleSummarizerMemories,
@@ -568,7 +568,7 @@ export async function runSummarizer(args: RunSummarizerArgs): Promise<Summarizer
 
 	try {
 		await runOnce("The preceding summarize call and receipt are an illustrative example only. Its placeholder ids are not real and it did not create a summary. Now inspect the actual records and use tools to register safe compression, or call done if none is warranted.", false);
-		for (let invocation = 1; !completedWithDone && invocation < SUMMARIZER_MAX_INVOCATIONS; invocation++) await runOnce(SUMMARIZER_CONTINUE, true);
+		for (let invocation = 1; !completedWithDone && invocation < SUMMARIZER_MAX_INVOCATIONS; invocation++) await runOnce(summarizerContinue(drafts.size, invocation), true);
 	} catch (error) {
 		debugLog("summarizer.error", { error: error instanceof Error ? error.message : String(error), acceptedSummaries: drafts.size });
 		if (drafts.size === 0) return { completed: false, reviewedUpToId: coversUpToId, sample };

package/src/agents/summarizer/prompts.ts

@@ -43,4 +43,8 @@ Tools:
 
 Prefer faithful useful compression over both distortion and indefinite accumulation. Under pressure, make progress on old low-value clusters first; treat durable valuable records and user intent as the last things to compress.`;
 
-export const SUMMARIZER_CONTINUE = "IMPORTANT!!!! CALL summarize TOOL NOW TO RECORD ANY SUMMARIES YOU HAVE DECIDED, OR CALL done IF NO SAFE SUMMARY IS WARRANTED. DO NOT DESCRIBE THE ACTION IN PROSE—USE A TOOL NOW.";
+export function summarizerContinue(recordedSummaries: number, reminderNumber: number): string {
+	const count = Math.max(0, Math.floor(recordedSummaries));
+	const thinkingMinutes = Math.max(1, Math.floor(reminderNumber)) * 20;
+	return `IMPORTANT!!!! YOU HAVE BEEN THINKING FOR ${thinkingMinutes} MINUTES. CALL A TOOL NOW. DO NOT WRITE SUMMARIES IN THE MAIN TEXT. THERE ${count === 1 ? "IS" : "ARE"} CURRENTLY ${count} RECORDED ${count === 1 ? "SUMMARY" : "SUMMARIES"}${count === 0 ? "; NOTHING HAS BEEN SUMMARIZED YET" : ""}. IF YOU WROTE SUMMARIES IN THE MAIN TEXT, RECORD THEM USING THE summarize TOOL NOW. IF NO SAFE SUMMARY IS WARRANTED, CALL done.`;
+}

package/src/hooks/consolidation-trigger.ts

@@ -242,7 +242,6 @@ export async function runConsolidationPipeline(
 	const pipelineEntries = options.observerEntries ?? (ctx.sessionManager.getBranch() as Entry[]);
 	const initialCoverage = latestCoverageIndex(pipelineEntries, OM_OBSERVATIONS_RECORDED);
 	const catchUpThroughId = sourceEntriesAfter(pipelineEntries, initialCoverage).at(-1)?.id;
-	const beforeFold = foldLedger(ctx.sessionManager.getBranch() as Entry[]);
 	runtime.consolidationPhase = "observer";
 	runtime.observerBacklogBlocking = catchUpThroughId !== undefined;
 	try {
@@ -254,6 +253,7 @@ export async function runConsolidationPipeline(
 		while (true) {
 			const beforeEntries = ctx.sessionManager.getBranch() as Entry[];
 			const beforeCoverage = latestCoverageIndex(beforeEntries, OM_OBSERVATIONS_RECORDED);
+			const beforeObservationIds = new Set(foldLedger(beforeEntries).observations.map((memory) => memory.id));
 			const observerOutcome = await runObserverStage(pi, runtime, ctx, resolveModel, {
 				force: options.forceObserver === true,
 				entries: options.observerEntries,
@@ -261,9 +261,18 @@ export async function runConsolidationPipeline(
 				contextGeneration,
 			});
 			if (observerOutcome === "abort") return;
-			if (options.observerEntries) break;
 
 			const afterEntries = ctx.sessionManager.getBranch() as Entry[];
+			if (shouldScheduleSummarizerFromObserver(options)) {
+				const passAddedObservations = foldLedger(afterEntries).observations.some((memory) => !beforeObservationIds.has(memory.id));
+				// A finite observer backlog can take hours to drain while the primary
+				// agent is idle. Let each completed chunk feed the independent
+				// summarizer; waiting for the entire observer pipeline can otherwise
+				// leave an oversized old pool untouched indefinitely.
+				if (passAddedObservations) scheduleSummarizer(pi, runtime, ctx);
+			}
+			if (options.observerEntries) break;
+
 			const afterCoverage = latestCoverageIndex(afterEntries, OM_OBSERVATIONS_RECORDED);
 			const remainingTokens = catchUpThroughId === undefined
 				? 0
@@ -279,10 +288,6 @@ export async function runConsolidationPipeline(
 		// pipeline's blocking backlog. Clear before notifying the contemplator.
 		if (contextGeneration === runtime.getContextGeneration()) runtime.observerBacklogBlocking = false;
 	}
-	const afterFold = foldLedger(ctx.sessionManager.getBranch() as Entry[]);
-	const beforeIds = new Set(beforeFold.observations.map((item) => item.id));
-	const added = afterFold.observations.filter((item) => !beforeIds.has(item.id));
-	if (added.length > 0 && shouldScheduleSummarizerFromObserver(options)) scheduleSummarizer(pi, runtime, ctx);
 	if (contextGeneration === runtime.getContextGeneration()) runtime.notifyMemoryUpdate?.(ctx);
 }
 
@@ -312,10 +317,14 @@ export function summarizerTriggerAfterRun(
 	targetTokens: number,
 	postRunOldTokens: number,
 	retriggerTokens: number,
+	failedAttemptStartOldTokens?: number,
 ): number | undefined {
-	return successfullyCompleted
-		? nextSummarizerTriggerTokens(targetTokens, postRunOldTokens, retriggerTokens)
-		: currentTriggerTokens;
+	if (successfullyCompleted) return nextSummarizerTriggerTokens(targetTokens, postRunOldTokens, retriggerTokens);
+	if (failedAttemptStartOldTokens === undefined) return currentTriggerTokens;
+	// Back off a model pass from the pool it actually received. Growth that
+	// arrived while it was running must count toward the retry rather than being
+	// swallowed by a threshold based on the larger post-run pool.
+	return Math.max(currentTriggerTokens ?? targetTokens, failedAttemptStartOldTokens + retriggerTokens);
 }
 
 function syncAndScheduleSummarizer(pi: ExtensionAPI, runtime: Runtime, ctx: ConsolidationCtx): void {
@@ -324,14 +333,27 @@ function syncAndScheduleSummarizer(pi: ExtensionAPI, runtime: Runtime, ctx: Cons
 }
 
 export function scheduleSummarizer(pi: ExtensionAPI, runtime: Runtime, ctx: ConsolidationCtx): void {
-	if (runtime.config.passive || !runtime.config.summarizerEnabled || runtime.summarizerInFlight) return;
+	if (runtime.config.passive || !runtime.config.summarizerEnabled) return;
+	if (runtime.summarizerInFlight) {
+		// Do not lose observer/activity checkpoints that arrive during a long run.
+		// The tracked task rechecks once its single-flight lock has been released.
+		runtime.summarizerRecheckPending = true;
+		return;
+	}
+	runtime.summarizerRecheckPending = false;
 	const entries = ctx.sessionManager.getBranch() as Entry[];
 	const pools = currentMemoryPools(runtime, entries);
-	if (pools.oldTokens <= summarizerTriggerTokens(runtime)) return;
+	const targetTokens = runtime.config.oldMemoryPoolTargetTokens;
+	const nextTriggerTokens = summarizerTriggerTokens(runtime);
+	// Initial/healthy eligibility strictly exceeds the advisory target. Once a
+	// growth backoff is installed, reaching that +N threshold is sufficient; it
+	// must not require an accidental extra token beyond the configured amount.
+	if (pools.oldTokens <= targetTokens || nextTriggerTokens > targetTokens && pools.oldTokens < nextTriggerTokens) return;
+	const runStartOldTokens = pools.oldTokens;
 	const generation = runtime.getContextGeneration();
 	const runId = `summarizer-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 8)}`;
 	const sessionMetadata = debugSessionMetadata(ctx);
-	void runtime.launchSummarizerTask(ctx, async () => withDebugLogContext({
+	const task = runtime.launchSummarizerTask(ctx, async () => withDebugLogContext({
 		enabled: runtime.config.debugLog === true,
 		cwd: ctx.cwd,
 		...sessionMetadata,
@@ -415,19 +437,28 @@ export function scheduleSummarizer(pi: ExtensionAPI, runtime: Runtime, ctx: Cons
 				const postRunPools = currentMemoryPools(runtime, ctx.sessionManager.getBranch() as Entry[]);
 				const target = runtime.config.oldMemoryPoolTargetTokens;
 				runtime.summarizerNextTriggerTokens = summarizerTriggerAfterRun(
-					// A failed/no-progress model pass must not be retried with the
-					// identical pool at every primary-agent checkpoint. Require fresh
-					// old-memory growth before trying another (potentially different)
-					// sample. Model-resolution failures spend no tokens and stay eligible.
-					successfullyCompleted || modelRunAttempted,
+					successfullyCompleted,
 					runtime.summarizerNextTriggerTokens,
 					target,
 					postRunPools.oldTokens,
 					runtime.config.summarizerRetriggerTokens,
+					// A failed/no-progress model pass should not retry an identical
+					// prompt at every checkpoint. Anchor the growth backoff to the pool
+					// seen at launch so concurrent growth is not accidentally erased.
+					modelRunAttempted ? runStartOldTokens : undefined,
 				);
 			}
 		}
 	}));
+	void task?.then(() => {
+		if (generation !== runtime.getContextGeneration()) return;
+		if (!runtime.summarizerRecheckPending) return;
+		// launchTrackedTask has released summarizerInFlight before resolving.
+		// Coalesce every checkpoint received during the old run into one fresh
+		// ledger/threshold evaluation.
+		runtime.summarizerRecheckPending = false;
+		scheduleSummarizer(pi, runtime, ctx);
+	});
 }
 
 async function runObserverStage(

package/src/runtime.ts

@@ -176,6 +176,8 @@ export class Runtime {
 	summarizerPromise: Promise<void> | null = null;
 	/** Old-pool token threshold for the next pass; undefined means configured target. */
 	summarizerNextTriggerTokens: number | undefined;
+	/** A scheduling checkpoint arrived while the single-flight summarizer lock was held. */
+	summarizerRecheckPending = false;
 	private memoryUpdateListener: ((ctx: MemoryUpdateCtx) => void) | undefined;
 	private agentActivityListener: ((ctx: MemoryUpdateCtx) => void) | undefined;
 	private settingsUpdateListener: ((ctx: MemoryUpdateCtx, settings: SettingsUpdate) => void) | undefined;
@@ -277,6 +279,7 @@ export class Runtime {
 		this.observerBacklogBlocking = false;
 		this.summarizerInFlight = false;
 		this.summarizerPromise = null;
+		this.summarizerRecheckPending = false;
 		this.reviewInFlight = false;
 		this.reviewPromise = null;
 		this.compactInFlight = false;