@matthewfl/pi-contemplator 0.0.1

package/src/hooks/compaction-trigger.ts

@@ -0,0 +1,122 @@
+import { isContextOverflow } from "@earendil-works/pi-ai/compat";
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { resolveCompactAfterTokens } from "../config.js";
+import { rawTokensSinceLastCompaction, type Entry } from "../session-ledger/index.js";
+import type { Runtime } from "../runtime.js";
+import {
+	registerCompactionResumeAcknowledgement,
+	resumeAfterCompaction,
+} from "./compaction-resume.js";
+
+const COMPACTION_STATUS_KEY = "observational-memory-compaction";
+
+export function registerCompactionTrigger(pi: ExtensionAPI, runtime: Runtime): void {
+	registerCompactionResumeAcknowledgement(pi, runtime);
+	pi.on("agent_end", (event: any, ctx: any) => {
+		runtime.ensureConfig(ctx.cwd);
+		if (runtime.compactInFlight) return;
+
+		const agentRequested = runtime.compactRequested;
+		if (agentRequested) runtime.compactRequested = false;
+		else if (runtime.config.passive === true) return;
+
+		const contextWindow = typeof ctx.model?.contextWindow === "number" ? ctx.model.contextWindow : 0;
+		let threshold: number | undefined;
+		if (!agentRequested) {
+			// agent_end fires before Pi decides whether to retry or compact-and-retry an
+			// interrupted request. Starting ctx.compact() here turns it into a manual
+			// compaction (willRetry=false) and can consume Pi's overflow recovery, leaving
+			// the agent idle. Let Pi handle every failed/aborted/overflow response; OM's
+			// session_before_compact hook still supplies the actual memory compaction.
+			const lastAssistant = [...event.messages].reverse().find(
+				(m): m is Extract<typeof m, { role: "assistant" }> => m.role === "assistant",
+			);
+			if (
+				lastAssistant
+				&& (
+					lastAssistant.stopReason === "error"
+						|| lastAssistant.stopReason === "aborted"
+						|| isContextOverflow(lastAssistant, contextWindow)
+				)
+			) return;
+
+			const entries = ctx.sessionManager.getBranch() as Entry[];
+			const tokens = rawTokensSinceLastCompaction(entries);
+			// Resolve the proactive-compaction threshold from the active model's context
+			// window when ratio mode is configured. ctx.model is the current session model
+			// (Model<any> | undefined per ExtensionContext).
+			threshold = resolveCompactAfterTokens(runtime.config, contextWindow > 0 ? contextWindow : undefined);
+			if (tokens < threshold) return;
+		}
+
+		// Capture ctx properties synchronously — the setTimeout + async work below
+		// may outlive the extension ctx (stale after session replacement/reload).
+		const hasUI = ctx.hasUI;
+		const ui = ctx.ui;
+		const origin = agentRequested ? "agent-requested" : "proactive";
+
+		runtime.compactInFlight = true;
+		runtime.compactOrigin = origin;
+		setTimeout(() => {
+			try {
+				if (!ctx.isIdle()) {
+					runtime.compactInFlight = false;
+					runtime.compactOrigin = undefined;
+					if (agentRequested) runtime.compactRequested = true;
+					if (hasUI) ui?.notify(
+						"Observational memory: compaction deferred — agent became busy before compaction",
+						"info",
+					);
+					return;
+				}
+				const currentEntries = ctx.sessionManager.getBranch() as Entry[];
+				const currentTokens = rawTokensSinceLastCompaction(currentEntries);
+				if (threshold !== undefined && currentTokens < threshold) {
+					runtime.compactInFlight = false;
+					runtime.compactOrigin = undefined;
+					if (hasUI) ui?.notify(
+						"Observational memory: compaction skipped — another compaction already ran before deferred compaction",
+						"info",
+					);
+					return;
+				}
+				if (hasUI) {
+					ui?.setStatus?.(COMPACTION_STATUS_KEY, `OM compaction: running (${origin}, resume pending)`);
+					const reason = agentRequested ? "agent-requested, " : "";
+					ui?.notify(
+						`Observational memory: compaction started (${reason}~${currentTokens.toLocaleString()} tokens); the agent will resume automatically`,
+						"info",
+					);
+				}
+				ctx.compact({
+					onComplete: () => {
+						runtime.compactInFlight = false;
+						runtime.compactOrigin = undefined;
+						// Both explicit and proactive OM compactions are manual from Pi's
+						// perspective (willRetry=false), so Pi will not continue either one.
+						// Always enqueue a hidden continuation after OM finishes compacting.
+						resumeAfterCompaction(pi, runtime, { hasUI, ui });
+					},
+					onError: (error: { message: string }) => {
+						runtime.compactInFlight = false;
+						runtime.compactOrigin = undefined;
+						if (hasUI) ui?.setStatus?.(COMPACTION_STATUS_KEY, undefined);
+						if (error.message !== "Compaction cancelled" && hasUI) {
+							ui?.notify(`Observational memory: ${error.message}`, "error");
+						}
+						resumeAfterCompaction(pi, runtime, { hasUI, ui }, true);
+					},
+				});
+			} catch (error) {
+				runtime.compactInFlight = false;
+				runtime.compactOrigin = undefined;
+				const msg = error instanceof Error ? error.message : String(error);
+				if (hasUI) {
+					ui?.setStatus?.(COMPACTION_STATUS_KEY, undefined);
+					ui?.notify(`Observational memory: compact threw: ${msg}`, "error");
+				}
+				resumeAfterCompaction(pi, runtime, { hasUI, ui }, true);
+			}
+		}, 0);
+	});
+}

package/src/hooks/consolidation-trigger.ts

@@ -0,0 +1,488 @@
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { runDropper } from "../agents/dropper/agent.js";
+import { observationPoolMetrics } from "../agents/dropper/pool.js";
+import { runObserver } from "../agents/observer/agent.js";
+import { runReflector } from "../agents/reflector/agent.js";
+import { debugLog, withDebugLogContext } from "../debug-log.js";
+import { resolveObserverChunkMaxTokens } from "../config.js";
+import type { ResolveResult, Runtime } from "../runtime.js";
+import { serializeSourceAddressedBranchEntries } from "../serialize.js";
+import {
+	OM_OBSERVATIONS_DROPPED,
+	OM_OBSERVATIONS_RECORDED,
+	OM_REFLECTIONS_RECORDED,
+	buildObservationsDroppedData,
+	buildObservationsRecordedData,
+	buildReflectionsRecordedData,
+	earlierCoverageMarkerId,
+	foldLedger,
+	fullProjection,
+	isSourceEntry,
+	latestCoverageIndex,
+	latestCoverageMarkerId,
+	observationToSummaryLine,
+	rawTokensSinceObservationCoverage,
+	rawTokensSinceReflectionCoverage,
+	reflectionToSummaryLine,
+	type Entry,
+	type Reflection,
+} from "../session-ledger/index.js";
+
+type ResolvedModel = Extract<ResolveResult, { ok: true }>;
+
+export type ConsolidationCtx = {
+	cwd: string;
+	hasUI: boolean;
+	ui?: { notify: (message: string, type?: "warning" | "info" | "error") => void };
+	model: unknown;
+	modelRegistry: any;
+	sessionManager: {
+		getBranch: () => readonly unknown[];
+		getSessionId?: () => string;
+		getSessionFile?: () => string | undefined;
+	};
+};
+
+type StageOutcome = "continue" | "abort";
+
+type ReflectorStageResult = {
+	outcome: StageOutcome;
+	sameRunReflections: Reflection[];
+	effectiveReflectionCoverageId?: string;
+};
+
+function sourceEntriesAfter(entries: Entry[], index: number): Entry[] {
+	return entries.slice(index + 1).filter(isSourceEntry);
+}
+
+function appendEntry(pi: ExtensionAPI, customType: string, data: unknown): void {
+	pi.appendEntry(customType, data);
+}
+
+function mergeReflections(existing: Reflection[], additional: Reflection[]): Reflection[] {
+	const seen = new Set(existing.map((reflection) => reflection.id));
+	const merged = [...existing];
+	for (const reflection of additional) {
+		if (seen.has(reflection.id)) continue;
+		seen.add(reflection.id);
+		merged.push(reflection);
+	}
+	return merged;
+}
+
+function anyStageDue(entries: Entry[], runtime: Runtime): boolean {
+	return rawTokensSinceObservationCoverage(entries) >= runtime.config.observeAfterTokens
+		|| rawTokensSinceReflectionCoverage(entries) >= runtime.config.reflectAfterTokens;
+}
+
+function shouldNotifyWorker(runtime: Runtime, ctx: ConsolidationCtx): boolean {
+	return runtime.config.showWorkerNotifications && ctx.hasUI;
+}
+
+function makeModelResolver(runtime: Runtime, ctx: ConsolidationCtx): (stage: "observer" | "reflector" | "dropper") => Promise<ResolvedModel | undefined> {
+	let cached: ResolveResult | undefined;
+	return async (stage) => {
+		cached ??= await runtime.resolveModel({
+			model: ctx.model,
+			modelRegistry: ctx.modelRegistry,
+			hasUI: ctx.hasUI,
+			ui: ctx.ui,
+		});
+		if (cached.ok) {
+			runtime.resolveFailureNotified = false;
+			return cached;
+		}
+		debugLog(`${stage}.model_unavailable`, { reason: cached.reason });
+		if (!runtime.resolveFailureNotified && ctx.hasUI && ctx.ui) {
+			ctx.ui.notify(`Observational memory: ${stage} skipped — ${cached.reason}`, "warning");
+			runtime.resolveFailureNotified = true;
+		}
+		return undefined;
+	};
+}
+
+export function registerConsolidationTrigger(pi: ExtensionAPI, runtime: Runtime): void {
+	const launch = (_event: unknown, ctx: ConsolidationCtx) => {
+		maybeLaunchConsolidation(pi, runtime, ctx);
+	};
+	pi.on("agent_start", launch);
+	pi.on("turn_end", launch);
+}
+
+function debugSessionMetadata(ctx: ConsolidationCtx): { sessionId?: string; sessionFile?: string } {
+	try {
+		return {
+			sessionId: ctx.sessionManager.getSessionId?.(),
+			sessionFile: ctx.sessionManager.getSessionFile?.(),
+		};
+	} catch {
+		return {};
+	}
+}
+
+function maybeLaunchConsolidation(pi: ExtensionAPI, runtime: Runtime, ctx: ConsolidationCtx): void {
+	runtime.ensureConfig(ctx.cwd);
+	if (runtime.config.passive === true) return;
+	if (runtime.consolidationInFlight) return;
+
+	const entries = ctx.sessionManager.getBranch() as Entry[];
+	if (!anyStageDue(entries, runtime)) return;
+
+	const runId = `consolidation-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 8)}`;
+	const consolidationCtx: ConsolidationCtx = {
+		cwd: ctx.cwd,
+		hasUI: ctx.hasUI,
+		ui: ctx.ui,
+		model: ctx.model,
+		modelRegistry: ctx.modelRegistry,
+		sessionManager: ctx.sessionManager,
+	};
+
+	const sessionMetadata = debugSessionMetadata(ctx);
+	void runtime.launchConsolidationTask(ctx, async () => withDebugLogContext({
+		enabled: runtime.config.debugLog === true,
+		cwd: ctx.cwd,
+		...sessionMetadata,
+		runId,
+	}, async () => {
+		await runConsolidationPipeline(pi, runtime, consolidationCtx);
+	}));
+}
+
+export function launchCompactionObserver(
+	pi: ExtensionAPI,
+	runtime: Runtime,
+	ctx: ConsolidationCtx,
+	branchEntries: Entry[],
+): void {
+	runtime.ensureConfig(ctx.cwd);
+	if (runtime.config.passive === true || runtime.consolidationInFlight) return;
+
+	const lastCoverageIdx = latestCoverageIndex(branchEntries, OM_OBSERVATIONS_RECORDED);
+	if (sourceEntriesAfter(branchEntries, lastCoverageIdx).length === 0) return;
+
+	const runId = `compaction-observer-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 8)}`;
+	const sessionMetadata = debugSessionMetadata(ctx);
+	void runtime.launchConsolidationTask(ctx, async () => withDebugLogContext({
+		enabled: runtime.config.debugLog === true,
+		cwd: ctx.cwd,
+		...sessionMetadata,
+		runId,
+	}, async () => {
+		await runConsolidationPipeline(pi, runtime, ctx, {
+			forceObserver: true,
+			observerEntries: branchEntries,
+			observerOnly: true,
+		});
+	}));
+}
+
+export type ConsolidationPipelineOptions = {
+	forceObserver?: boolean;
+	observerEntries?: Entry[];
+	observerOnly?: boolean;
+};
+
+export async function runConsolidationPipeline(
+	pi: ExtensionAPI,
+	runtime: Runtime,
+	ctx: ConsolidationCtx,
+	options: ConsolidationPipelineOptions = {},
+): Promise<void> {
+	const resolveModel = makeModelResolver(runtime, ctx);
+	const contextGeneration = runtime.getContextGeneration();
+
+	runtime.consolidationPhase = "observer";
+	try {
+		const observerOutcome = await runObserverStage(pi, runtime, ctx, resolveModel, {
+			force: options.forceObserver === true,
+			entries: options.observerEntries,
+			contextGeneration,
+		});
+		if (observerOutcome === "abort") return;
+	} catch (error) {
+		debugLog("observer.error", { errorMessage: runtime.recordConsolidationStageError(ctx, "observer", error) });
+		return;
+	}
+	if (options.observerOnly === true) {
+		runtime.notifyMemoryUpdate?.(ctx);
+		return;
+	}
+
+	runtime.consolidationPhase = "reflector";
+	let reflectorResult: ReflectorStageResult;
+	try {
+		reflectorResult = await runReflectorStage(pi, runtime, ctx, resolveModel, contextGeneration);
+		if (reflectorResult.outcome === "abort") return;
+	} catch (error) {
+		debugLog("reflector.error", { errorMessage: runtime.recordConsolidationStageError(ctx, "reflector", error) });
+		return;
+	}
+
+	runtime.consolidationPhase = "dropper";
+	try {
+		const dropperOutcome = await runDropperStage(pi, runtime, ctx, resolveModel, reflectorResult.sameRunReflections, reflectorResult.effectiveReflectionCoverageId, contextGeneration);
+		if (dropperOutcome === "abort") return;
+	} catch (error) {
+		debugLog("dropper.error", { errorMessage: runtime.recordConsolidationStageError(ctx, "dropper", error) });
+	}
+	if (contextGeneration === runtime.getContextGeneration()) runtime.notifyMemoryUpdate?.(ctx);
+}
+
+async function runObserverStage(
+	pi: ExtensionAPI,
+	runtime: Runtime,
+	ctx: ConsolidationCtx,
+	resolveModel: (stage: "observer") => Promise<ResolvedModel | undefined>,
+	options: { force?: boolean; entries?: Entry[]; contextGeneration?: number } = {},
+): Promise<StageOutcome> {
+	const entries = options.entries ?? (ctx.sessionManager.getBranch() as Entry[]);
+	const tokens = rawTokensSinceObservationCoverage(entries);
+	if (!options.force && tokens < runtime.config.observeAfterTokens) return "continue";
+
+	// Resolve the model before building the chunk: the default chunk cap
+	// derives from the resolved model's context window.
+	const resolved = await resolveModel("observer");
+	if (!resolved) return "abort";
+	if (options.contextGeneration !== undefined && options.contextGeneration !== runtime.getContextGeneration()) {
+		debugLog("observer.stale", { reason: "session_or_branch_changed" });
+		return "abort";
+	}
+
+	const lastCoverageIdx = latestCoverageIndex(entries, OM_OBSERVATIONS_RECORDED);
+	const backlogEntries = sourceEntriesAfter(entries, lastCoverageIdx);
+
+	// Budget the text that is actually sent to the observer, including source
+	// labels and rendered message content. Complete entries are kept intact.
+	// Only a first entry that cannot fit by itself is represented by a clearly
+	// marked head/tail excerpt; the original ledger entry remains untouched.
+	const contextWindow = (resolved.model as { contextWindow?: number }).contextWindow;
+	const maxChunkTokens = resolveObserverChunkMaxTokens(runtime.config, contextWindow);
+	const {
+		text: chunk,
+		sourceEntryIds,
+		estimatedTokens: chunkTokens,
+		truncatedSourceEntryIds,
+	} = serializeSourceAddressedBranchEntries(backlogEntries, { maxTokens: maxChunkTokens });
+	if (!chunk.trim() || sourceEntryIds.length === 0) return "continue";
+	const coversUpToId = sourceEntryIds.at(-1);
+	if (!coversUpToId) return "continue";
+
+	if (sourceEntryIds.length < backlogEntries.length || truncatedSourceEntryIds.length > 0) {
+		debugLog("observer.chunk_capped", {
+			maxChunkTokens,
+			backlogEntries: backlogEntries.length,
+			backlogTokens: tokens,
+			chunkEntries: sourceEntryIds.length,
+			chunkTokens,
+			truncatedSourceEntryIds,
+		});
+	}
+
+	const memory = fullProjection(entries);
+	const priorReflections = memory.reflections.map(reflectionToSummaryLine);
+	const priorObservations = memory.observations.map(observationToSummaryLine);
+
+	if (shouldNotifyWorker(runtime, ctx)) ctx.ui?.notify(
+		`Observational memory: observer running on ~${chunkTokens.toLocaleString()}-token chunk`,
+		"info",
+	);
+	debugLog("observer.start", {
+		tokens,
+		chunkTokens,
+		coversUpToId,
+		sourceEntryIds,
+		sourceEntryCount: sourceEntryIds.length,
+		priorReflections: priorReflections.length,
+		priorObservations: priorObservations.length,
+	});
+
+	const observations = await runObserver({
+		model: resolved.model as any,
+		apiKey: resolved.apiKey,
+		headers: resolved.headers,
+		priorReflections,
+		priorObservations,
+		chunk,
+		allowedSourceEntryIds: sourceEntryIds,
+		maxTurns: runtime.config.agentMaxTurns,
+		thinkingLevel: runtime.config.model?.thinking ?? "low",
+		recordUsage: (usage) => runtime.recordAgentUsage(usage),
+	});
+	if (options.contextGeneration !== undefined && options.contextGeneration !== runtime.getContextGeneration()) {
+		debugLog("observer.stale", { reason: "session_or_branch_changed" });
+		return "abort";
+	}
+	if (!observations || observations.length === 0) {
+		debugLog("observer.empty", { coversUpToId });
+		if (ctx.hasUI) ctx.ui?.notify(
+			"Observational memory: observer returned no observations",
+			"warning",
+		);
+		return "continue";
+	}
+
+	const currentEntries = ctx.sessionManager.getBranch() as Entry[];
+	let effectiveCoversUpToId = coversUpToId;
+	if (!currentEntries.some((entry) => entry.id === coversUpToId)) {
+		// A fire-and-forget compaction observer may finish after compaction has
+		// folded its source target away. Never append an unresolvable marker: it
+		// would make these observations invisible to projection and recall.
+		const compaction = [...currentEntries].reverse().find((entry) => entry.type === "compaction");
+		effectiveCoversUpToId = compaction?.id ?? currentEntries.at(-1)?.id ?? coversUpToId;
+		debugLog("observer.coverage_target_folded", {
+			requestedCoversUpToId: coversUpToId,
+			effectiveCoversUpToId,
+			compactionId: compaction?.id,
+		});
+	}
+	const data = buildObservationsRecordedData(observations, effectiveCoversUpToId);
+	if (!data) return "continue";
+	debugLog("observer.records", {
+		count: observations.length,
+		observationTokens: observations.reduce((sum, observation) => sum + observation.tokenCount, 0),
+		coversUpToId: effectiveCoversUpToId,
+	});
+	appendEntry(pi, OM_OBSERVATIONS_RECORDED, data);
+	debugLog("observer.appended", { count: observations.length, coversUpToId: effectiveCoversUpToId });
+	if (shouldNotifyWorker(runtime, ctx)) ctx.ui?.notify(
+		`Observational memory: ${observations.length} observation${observations.length === 1 ? "" : "s"} recorded`,
+		"info",
+	);
+	return "continue";
+}
+
+async function runReflectorStage(
+	pi: ExtensionAPI,
+	runtime: Runtime,
+	ctx: ConsolidationCtx,
+	resolveModel: (stage: "reflector") => Promise<ResolvedModel | undefined>,
+	contextGeneration: number,
+): Promise<ReflectorStageResult> {
+	const entries = ctx.sessionManager.getBranch() as Entry[];
+	const reflectionTokens = rawTokensSinceReflectionCoverage(entries);
+	if (reflectionTokens < runtime.config.reflectAfterTokens) return { outcome: "continue", sameRunReflections: [] };
+
+	const observationCoverageId = latestCoverageMarkerId(entries, OM_OBSERVATIONS_RECORDED);
+	if (!observationCoverageId) return { outcome: "continue", sameRunReflections: [] };
+
+	if (shouldNotifyWorker(runtime, ctx)) ctx.ui?.notify(
+		`Observational memory: reflector running (~${reflectionTokens.toLocaleString()} tokens)`,
+		"info",
+	);
+	const resolved = await resolveModel("reflector");
+	if (!resolved) return { outcome: "abort", sameRunReflections: [] };
+	if (contextGeneration !== runtime.getContextGeneration()) {
+		debugLog("reflector.stale", { reason: "session_or_branch_changed" });
+		return { outcome: "abort", sameRunReflections: [] };
+	}
+
+	const folded = foldLedger(entries);
+	const reflections = await runReflector({
+		model: resolved.model as any,
+		apiKey: resolved.apiKey,
+		headers: resolved.headers,
+		reflections: folded.reflections,
+		observations: folded.activeObservations,
+		maxTurns: runtime.config.agentMaxTurns,
+		thinkingLevel: runtime.config.model?.thinking ?? "low",
+		recordUsage: (usage) => runtime.recordAgentUsage(usage),
+	});
+	if (contextGeneration !== runtime.getContextGeneration()) {
+		debugLog("reflector.stale", { reason: "session_or_branch_changed" });
+		return { outcome: "abort", sameRunReflections: [] };
+	}
+	if (!reflections) return { outcome: "continue", sameRunReflections: [] };
+
+	const data = buildReflectionsRecordedData(reflections, observationCoverageId);
+	if (!data) return { outcome: "continue", sameRunReflections: [] };
+	appendEntry(pi, OM_REFLECTIONS_RECORDED, data);
+	return {
+		outcome: "continue",
+		sameRunReflections: reflections,
+		effectiveReflectionCoverageId: data.coversUpToId,
+	};
+}
+
+async function runDropperStage(
+	pi: ExtensionAPI,
+	runtime: Runtime,
+	ctx: ConsolidationCtx,
+	resolveModel: (stage: "dropper") => Promise<ResolvedModel | undefined>,
+	sameRunReflections: Reflection[],
+	sameRunReflectionCoverageId: string | undefined,
+	contextGeneration: number,
+): Promise<StageOutcome> {
+	if (!sameRunReflectionCoverageId || sameRunReflections.length === 0) {
+		debugLog("dropper.waiting_for_reflection", { sameRunReflections: sameRunReflections.length });
+		return "continue";
+	}
+
+	const entries = ctx.sessionManager.getBranch() as Entry[];
+	const observationCoverageId = latestCoverageMarkerId(entries, OM_OBSERVATIONS_RECORDED);
+	if (!observationCoverageId) return "continue";
+
+	const folded = foldLedger(entries);
+	const metrics = observationPoolMetrics(folded.activeObservations, runtime.config.observationsPoolTargetTokens);
+	if (!metrics.ready) {
+		debugLog("dropper.not_ready", {
+			observationTokens: metrics.observationTokens,
+			targetTokens: metrics.targetTokens,
+			tokensOverTarget: metrics.tokensOverTarget,
+			fullness: metrics.fullness,
+			activeObservationCount: metrics.activeObservationCount,
+			droppableCount: metrics.droppableCount,
+			maxDropsAllowed: metrics.maxDropsAllowed,
+		});
+		return "continue";
+	}
+	debugLog("dropper.stage_start", {
+		observationCoverageId,
+		sameRunReflectionCoverageId,
+		sameRunReflectionCount: sameRunReflections.length,
+		activeObservationCount: metrics.activeObservationCount,
+		observationTokens: metrics.observationTokens,
+		targetTokens: metrics.targetTokens,
+		tokensOverTarget: metrics.tokensOverTarget,
+		fullness: metrics.fullness,
+		maxDropsAllowed: metrics.maxDropsAllowed,
+	});
+
+	if (shouldNotifyWorker(runtime, ctx)) ctx.ui?.notify(
+		`Observational memory: dropper running after reflection — active observation pool ~${metrics.observationTokens.toLocaleString()} / ${metrics.targetTokens.toLocaleString()} target tokens (${Math.round(metrics.fullness * 100).toLocaleString()}%)`,
+		"info",
+	);
+	const resolved = await resolveModel("dropper");
+	if (!resolved) return "abort";
+	if (contextGeneration !== runtime.getContextGeneration()) {
+		debugLog("dropper.stale", { reason: "session_or_branch_changed" });
+		return "abort";
+	}
+
+	const reflectionsForDropper = mergeReflections(folded.reflections, sameRunReflections);
+	const droppedIds = await runDropper({
+		model: resolved.model as any,
+		apiKey: resolved.apiKey,
+		headers: resolved.headers,
+		reflections: reflectionsForDropper,
+		observations: folded.activeObservations,
+		targetTokens: runtime.config.observationsPoolTargetTokens,
+		maxTurns: runtime.config.agentMaxTurns,
+		thinkingLevel: runtime.config.model?.thinking ?? "low",
+		recordUsage: (usage) => runtime.recordAgentUsage(usage),
+	});
+	if (contextGeneration !== runtime.getContextGeneration()) {
+		debugLog("dropper.stale", { reason: "session_or_branch_changed" });
+		return "abort";
+	}
+	const coversUpToId = earlierCoverageMarkerId(entries, observationCoverageId, sameRunReflectionCoverageId);
+	const data = coversUpToId && droppedIds ? buildObservationsDroppedData(droppedIds, coversUpToId) : undefined;
+	debugLog("dropper.append", {
+		droppedIdsCount: droppedIds?.length ?? 0,
+		coversUpToId,
+		dataBuilt: data !== undefined,
+		appended: data !== undefined,
+	});
+	if (data) appendEntry(pi, OM_OBSERVATIONS_DROPPED, data);
+	return "continue";
+}

package/src/ids.ts

@@ -0,0 +1,5 @@
+import { createHash } from "node:crypto";
+
+export function hashId(content: string): string {
+	return createHash("sha256").update(content).digest("hex").slice(0, 12);
+}

package/src/index.ts

@@ -0,0 +1,32 @@
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { registerSettingsCommand } from "./commands/settings.js";
+import { registerStatusCommand } from "./commands/status.js";
+import { registerViewCommand } from "./commands/view.js";
+import { registerCompactionHook } from "./hooks/compaction-hook.js";
+import { registerCompactionTrigger } from "./hooks/compaction-trigger.js";
+import { registerConsolidationTrigger } from "./hooks/consolidation-trigger.js";
+import { Runtime } from "./runtime.js";
+import { registerCompactContextTool } from "./tools/compact-context.js";
+import { registerRecallTool } from "./tools/recall-observation.js";
+import { registerSearchMemoriesTool } from "./tools/search-memories.js";
+import { Contemplator } from "./agents/contemplator/agent.js";
+
+export default function observationalMemory(pi: ExtensionAPI) {
+	const runtime = new Runtime();
+	pi.on("session_start", () => runtime.advanceContextGeneration());
+	pi.on("session_tree", () => runtime.advanceContextGeneration());
+	pi.on("session_shutdown", () => runtime.advanceContextGeneration());
+
+	registerSettingsCommand(pi, runtime);
+	new Contemplator(pi, runtime).register();
+
+	registerConsolidationTrigger(pi, runtime);
+	registerCompactionTrigger(pi, runtime);
+	registerCompactionHook(pi, runtime);
+
+	registerStatusCommand(pi, runtime);
+	registerViewCommand(pi, runtime);
+	registerCompactContextTool(pi, runtime);
+	registerRecallTool(pi);
+	registerSearchMemoriesTool(pi);
+}

package/src/model-budget.ts

@@ -0,0 +1,16 @@
+import type { Model } from "@earendil-works/pi-ai";
+
+export const AGENT_LOOP_MAX_TOKENS = 32_000;
+
+/**
+ * Lifetime output-token budget for one structural review request.
+ * Persisted reviewer transcripts carry usage across keep-going iterations,
+ * internal tool turns, and later session/tree resumptions.
+ */
+export const REVIEWER_TOTAL_TOKEN_LIMIT = 1_000_000;
+
+export function boundedMaxTokens(model: Model<any>, requested: number = AGENT_LOOP_MAX_TOKENS): number {
+	return typeof model.maxTokens === "number" && model.maxTokens > 0
+		? Math.min(model.maxTokens, requested)
+		: requested;
+}