@matthewfl/pi-contemplator 0.1.2 → 0.1.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@matthewfl/pi-contemplator",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "description": "A Pi extension that keeps long-running agentic sessions on track with background memory, contemplation, and structural review.",
   "type": "module",
   "license": "MIT",
@@ -41,7 +41,7 @@
     "typecheck": "tsc --noEmit",
     "test": "npm run test:unit && npm run test:e2e",
     "test:unit": "vitest run",
-    "test:e2e": "node tests/e2e/rpc-contemplator.mjs && node tests/e2e/rpc-summarizer.mjs && node tests/e2e/rpc-delivery.mjs && node tests/e2e/rpc-restore-review.mjs && node tests/e2e/rpc-compaction.mjs && node tests/e2e/rpc-compaction-resilience.mjs && node tests/e2e/rpc-memory-edges.mjs && node tests/e2e/rpc-routing-isolation.mjs"
+    "test:e2e": "node tests/e2e/rpc-contemplator.mjs && node tests/e2e/rpc-summarizer.mjs && node tests/e2e/rpc-delivery.mjs && node tests/e2e/rpc-restore-review.mjs && node tests/e2e/rpc-compaction.mjs && node tests/e2e/rpc-compaction-resilience.mjs && node tests/e2e/rpc-memory-edges.mjs && node tests/e2e/rpc-observer-length.mjs && node tests/e2e/rpc-routing-isolation.mjs"
   },
   "peerDependencies": {
     "@earendil-works/pi-agent-core": "*",

package/src/agents/observer/agent.ts

@@ -26,6 +26,7 @@ interface RunObserverArgs {
 	thinkingLevel?: ModelThinkingLevel;
 	recordUsage?: (usage: LlmUsageInput) => void;
 	onProgress?: () => void;
+	onMessages?: (messages: readonly AgentMessage[]) => void;
 }
 
 const RelevanceSchema = Type.Union([
@@ -216,17 +217,35 @@ IMPORTANT: Now call record_observations to record the useful new observations fr
 	const loop = args.agentLoop ?? agentLoop;
 	const history: AgentMessage[] = [];
 	let terminalFailure: { stopReason: string; errorMessage?: string } | undefined;
-	const runInvocation = async (prompt: Message): Promise<void> => {
+	let lengthRetryAttempted = false;
+	const runInvocation = async (prompt: Message, afterLength = false): Promise<void> => {
 		const context: AgentContext = {
 			systemPrompt: OBSERVER_SYSTEM,
 			messages: history.slice(),
 			tools: [recordObservations as AgentTool<any>, doneTool],
 		};
-		const stream = loop([prompt], context, baseConfig, signal, streamSimple);
+		// Publish a live launch-local transcript for /om:view observer. Keep an
+		// invocation-local list because agentLoop owns its internal context copy.
+		let liveMessages: AgentMessage[] = [...history, prompt];
+		args.onMessages?.(liveMessages.slice());
+		const invocationConfig: AgentLoopConfig = afterLength && reasoning
+			? { ...baseConfig, reasoning: "minimal" }
+			: baseConfig;
+		const stream = loop([prompt], context, invocationConfig, signal, streamSimple);
 		for await (const event of stream) {
 			args.onProgress?.();
 			logAgentStreamError("observer", event);
-			const message = (event as { message?: { role?: string; stopReason?: string; errorMessage?: string } }).message;
+			const typedEvent = event as { type?: string; message?: AgentMessage & { role?: string; stopReason?: string; errorMessage?: string } };
+			const message = typedEvent.message;
+			if (message && message !== prompt && message.role !== "user") {
+				if (typedEvent.type === "message_start") liveMessages.push(message);
+				else if (typedEvent.type === "message_update" || typedEvent.type === "message_end") {
+					const index = liveMessages.map((item) => item.role).lastIndexOf(message.role);
+					if (index >= 0) liveMessages[index] = message;
+					else liveMessages.push(message);
+				}
+				args.onMessages?.(liveMessages.slice());
+			}
 			if (message?.role === "assistant" && ["error", "aborted", "length"].includes(message.stopReason ?? "")) {
 				terminalFailure = { stopReason: message.stopReason!, errorMessage: message.errorMessage };
 			}
@@ -234,6 +253,8 @@ IMPORTANT: Now call record_observations to record the useful new observations fr
 		const result = await stream.result();
 		if (!Array.isArray(result)) return;
 		history.push(...result);
+		liveMessages = history.slice();
+		args.onMessages?.(liveMessages);
 		for (const message of result) {
 			if (message.role === "assistant" && ["error", "aborted", "length"].includes(message.stopReason ?? "")) {
 				terminalFailure = { stopReason: message.stopReason, errorMessage: message.errorMessage };
@@ -243,6 +264,23 @@ IMPORTANT: Now call record_observations to record the useful new observations fr
 	};
 
 	await runInvocation(initialPrompt);
+	if (accumulated.size === 0 && terminalFailure?.stopReason === "length") {
+		lengthRetryAttempted = true;
+		// A provider can impose a lower output ceiling than the advertised model
+		// maximum. agentLoop stops on `length` when no tool call was completed; it
+		// does not automatically send a continuation request. Preserve the partial
+		// response so the model can continue from work it already performed rather
+		// than paying to reproduce it, then append a short tool-focused instruction
+		// and reduce reasoning to minimal. A second length stop fails forward at the
+		// bounded-chunk level.
+		terminalFailure = undefined;
+		const retryPrompt: Message = {
+			role: "user",
+			content: [{ type: "text", text: "IMPORTANT: The previous response reached the provider output limit before recording anything. Continue from the work already above and call record_observations now instead of spending another response budget analyzing." }],
+			timestamp: Date.now(),
+		};
+		await runInvocation(retryPrompt, true);
+	}
 	if (accumulated.size === 0 && !doneCalled && !terminalFailure && rejectedTotal === 0) {
 		const reminder: Message = {
 			role: "user",
@@ -257,7 +295,10 @@ IMPORTANT: Now call record_observations to record the useful new observations fr
 	// zero-observation stop is also a valid empty result after the reminder;
 	// actual stream failures, truncation, and malformed records still throw.
 	if (accumulated.size === 0 && terminalFailure) {
-		throw new ObserverStreamError(terminalFailure.stopReason, terminalFailure.errorMessage);
+		const detail = terminalFailure.stopReason === "length" && lengthRetryAttempted
+			? `provider reached the output limit twice without recording an observation (effective max output request: ${baseConfig.maxTokens} tokens)`
+			: terminalFailure.errorMessage;
+		throw new ObserverStreamError(terminalFailure.stopReason, detail);
 	}
 	if (accumulated.size === 0 && rejectedTotal > 0) {
 		throw new ObserverStreamError("invalid_observations", `${rejectedTotal} proposed observation${rejectedTotal === 1 ? " was" : "s were"} rejected`);

package/src/commands/observer-view.ts

@@ -0,0 +1,59 @@
+import type { ObserverRunView } from "../runtime.js";
+
+const DIM = "\x1b[2m";
+const RESET = "\x1b[0m";
+
+type StoredMessage = { role?: unknown; content?: unknown };
+type ContentPart = { type?: unknown; text?: unknown; thinking?: unknown; name?: unknown; arguments?: unknown; content?: unknown };
+
+function renderValue(value: unknown): string {
+	if (typeof value === "string") return value;
+	if (value === undefined || value === null) return "";
+	return JSON.stringify(value, null, 2);
+}
+
+function renderContent(content: unknown): string {
+	if (typeof content === "string") return content;
+	if (!Array.isArray(content)) return renderValue(content);
+	return content.map((part: ContentPart) => {
+		if (part.type === "text") return typeof part.text === "string" ? part.text : "";
+		if (part.type === "thinking") {
+			const thinking = typeof part.thinking === "string" ? part.thinking : typeof part.text === "string" ? part.text : renderValue(part);
+			return `[thinking]\n${thinking}`;
+		}
+		if (part.type === "toolCall" || part.type === "tool_use" || part.type === "toolUse") {
+			const name = typeof part.name === "string" ? part.name : "unknown tool";
+			return `[tool call: ${name}${part.arguments === undefined ? "" : ` ${renderValue(part.arguments)}`}]`;
+		}
+		if (part.type === "toolResult" || part.type === "tool_result") return `[tool result]\n${renderValue(part.content)}`;
+		return `[${String(part.type ?? "content")}] ${renderValue(part)}`;
+	}).filter(Boolean).join("\n");
+}
+
+function estimateTokens(value: unknown): number {
+	return Math.max(1, Math.ceil(JSON.stringify(value).length / 4));
+}
+
+/** Render the currently active observer chunk, or the most recently completed chunk. */
+export function renderObserver(run: ObserverRunView | undefined, now = Date.now()): string {
+	if (!run) return `${DIM}OBSERVER${RESET}\n\n${DIM}Observer has not run yet during this launch.${RESET}`;
+	const messages = run.messages.filter((message): message is StoredMessage => !!message && typeof message === "object");
+	const totalTokens = messages.reduce((sum, message) => sum + estimateTokens(message), 0);
+	const elapsedMs = Math.max(0, (run.completedAt ?? now) - run.startedAt);
+	const lines = [
+		`${DIM}OBSERVER · ${run.status} · ${messages.length} messages · ~${totalTokens.toLocaleString()} estimated transcript tokens${RESET}`,
+		`${DIM}Chunk ~${run.chunkTokens.toLocaleString()} tokens · backlog at start ~${run.backlogTokens.toLocaleString()} tokens · ${run.sourceEntryIds.length} source entr${run.sourceEntryIds.length === 1 ? "y" : "ies"}${RESET}`,
+		`${DIM}Started ${new Date(run.startedAt).toISOString()} · ${run.completedAt === undefined ? `running for ${Math.floor(elapsedMs / 1000)}s` : `ended ${new Date(run.completedAt).toISOString()} after ${Math.floor(elapsedMs / 1000)}s`}${RESET}`,
+		"",
+	];
+	if (messages.length === 0) lines.push(`${DIM}(no observer messages captured yet)${RESET}`);
+	for (const [index, message] of messages.entries()) {
+		if (index > 0) lines.push("");
+		const role = typeof message.role === "string" ? message.role : "unknown";
+		lines.push(`${DIM}── ${role} · ~${estimateTokens(message).toLocaleString()} tokens ──${RESET}`);
+		lines.push(renderContent(message.content) || `${DIM}(empty message)${RESET}`);
+	}
+	if (run.summary) lines.push("", `${DIM}── Completion summary ──${RESET}`, run.summary);
+	if (run.error) lines.push("", `${DIM}── Failure ──${RESET}`, run.error);
+	return lines.join("\n");
+}

package/src/commands/status.ts

@@ -142,8 +142,8 @@ export function registerStatusCommand(pi: ExtensionAPI, runtime: Runtime): void
 			}
 
 			lines.push("", "── Last worker runs ──");
-			lines.push(`Last observer start:     ${runtime.lastObserverStartedAt === undefined ? "not run this launch" : formatRunAge(runtime.lastObserverStartedAt)}`);
-			lines.push(`Last observer end:       ${runtime.lastObserverCompletedAt === undefined ? "not completed this launch" : formatRunAge(runtime.lastObserverCompletedAt)}`);
+			lines.push(`Observer chunk start:    ${runtime.lastObserverStartedAt === undefined ? "not run this launch" : formatRunAge(runtime.lastObserverStartedAt)}`);
+			lines.push(`Observer chunk end:      ${runtime.lastObserverCompletedAt === undefined ? runtime.lastObserverRun?.status === "running" ? "running" : "not completed this launch" : formatRunAge(runtime.lastObserverCompletedAt)}`);
 			lines.push(`Last summarizer start:   ${runtime.lastSummarizerStartedAt === undefined ? "not run this launch" : formatRunAge(runtime.lastSummarizerStartedAt)}`);
 			lines.push(`Last summarizer end:     ${runtime.lastSummarizerCompletedAt === undefined ? "not completed this launch" : formatRunAge(runtime.lastSummarizerCompletedAt)}`);
 			lines.push(`Last contemplator start: ${runtime.contemplatorState.lastStartedAt === undefined ? "not run this launch" : formatRunAge(runtime.contemplatorState.lastStartedAt)}`);

package/src/commands/view.ts

@@ -4,6 +4,7 @@ import { copyTextToClipboard } from "../clipboard.js";
 import { renderContemplator, stripAnsi } from "./contemplator-view.js";
 import { renderReviewer } from "./reviewer-view.js";
 import { renderSummarizer } from "./summarizer-view.js";
+import { renderObserver } from "./observer-view.js";
 import { executeRecall, formatRecallResultForTui } from "../tools/recall-observation.js";
 import {
 	chronologicalMemories,
@@ -76,7 +77,7 @@ export function registerViewCommand(
 
 	pi.registerCommand("om:view", {
 		description:
-			"Print and copy pi-contemplator memory content (visible, full, memory, contemplator, summarizer, reviewer, or reviews)",
+			"Print and copy pi-contemplator memory content (visible, full, memory, contemplator, observer, summarizer, reviewer, or reviews)",
 		handler: async (args, ctx) => {
 			runtime.ensureConfig(ctx.cwd);
 			const entries = ctx.sessionManager.getBranch() as Entry[];
@@ -116,6 +117,18 @@ export function registerViewCommand(
 				return;
 			}
 
+			if (mode === "observer") {
+				const output = renderObserver(runtime.lastObserverRun);
+				const copied = await copyToClipboard(stripAnsi(output)).catch(() => false);
+				ctx.ui.notify(
+					`${output}
+
+${copied ? "Copied /om:view observer output to clipboard." : "Warning: failed to copy /om:view observer output to clipboard."}`,
+					"info",
+				);
+				return;
+			}
+
 			if (mode === "summarizer") {
 				const output = renderSummarizer(runtime.lastSummarizerRun);
 				const copied = await copyToClipboard(stripAnsi(output)).catch(() => false);
@@ -154,7 +167,7 @@ export function registerViewCommand(
 			}
 
 			if (mode && mode !== "visible") {
-				ctx.ui.notify("Usage: /om:view [visible|full|memory <id>|contemplator|summarizer|reviewer|reviews]", "info");
+				ctx.ui.notify("Usage: /om:view [visible|full|memory <id>|contemplator|observer|summarizer|reviewer|reviews]", "info");
 				return;
 			}
 

package/src/hooks/consolidation-trigger.ts

@@ -5,6 +5,7 @@ import { debugLog, withDebugLogContext } from "../debug-log.js";
 import { resolveObserverChunkMaxTokens } from "../config.js";
 import type { ResolveResult, Runtime } from "../runtime.js";
 import { createWorkerStallWatchdog } from "../worker-watchdog.js";
+import { boundedMaxTokens, OBSERVER_AGENT_LOOP_MAX_TOKENS } from "../model-budget.js";
 import { serializeSourceAddressedBranchEntries } from "../serialize.js";
 import {
 	OM_SUMMARIZER_COMMIT,
@@ -226,7 +227,6 @@ export async function runConsolidationPipeline(
 
 	const beforeFold = foldLedger(ctx.sessionManager.getBranch() as Entry[]);
 	runtime.consolidationPhase = "observer";
-	runtime.lastObserverStartedAt = Date.now();
 	try {
 		// A large backlog is drained in bounded, oldest-first chunks. The normal
 		// trigger threshold controls when the batch stops; a static compaction
@@ -252,8 +252,6 @@ export async function runConsolidationPipeline(
 	} catch (error) {
 		debugLog("observer.error", { errorMessage: runtime.recordConsolidationStageError(ctx, "observer", error) });
 		return;
-	} finally {
-		runtime.lastObserverCompletedAt = Date.now();
 	}
 	const afterFold = foldLedger(ctx.sessionManager.getBranch() as Entry[]);
 	const beforeIds = new Set(beforeFold.observations.map((item) => item.id));
@@ -466,6 +464,9 @@ async function runObserverStage(
 	debugLog("observer.start", {
 		tokens,
 		chunkTokens,
+		requestedMaxOutputTokens: OBSERVER_AGENT_LOOP_MAX_TOKENS,
+		effectiveMaxOutputTokens: boundedMaxTokens(resolved.model as any, OBSERVER_AGENT_LOOP_MAX_TOKENS),
+		advertisedModelMaxTokens: (resolved.model as { maxTokens?: number }).maxTokens,
 		coversUpToId,
 		sourceEntryIds,
 		sourceEntryCount: sourceEntryIds.length,
@@ -473,6 +474,20 @@ async function runObserverStage(
 		priorObservations: priorObservations.length,
 	});
 
+	const observerStartedAt = Date.now();
+	runtime.lastObserverStartedAt = observerStartedAt;
+	// Clear the previous end when a new chunk starts so /om:status never pairs
+	// this chunk's start with the preceding chunk's completion.
+	runtime.lastObserverCompletedAt = undefined;
+	runtime.lastObserverRun = {
+		startedAt: observerStartedAt,
+		status: "running",
+		messages: [],
+		chunkTokens,
+		backlogTokens: tokens,
+		sourceEntryIds: sourceEntryIds.slice(),
+	};
+	let acceptsObserverMessages = true;
 	let observations;
 	let failedMessage: string | undefined;
 	const observerWatchdog = createWorkerStallWatchdog("observer");
@@ -489,6 +504,17 @@ async function runObserverStage(
 			thinkingLevel: runtime.config.model?.thinking ?? "low",
 			recordUsage: (usage) => runtime.recordAgentUsage(usage),
 			onProgress: observerWatchdog.progress,
+			onMessages: (messages) => {
+				if (!acceptsObserverMessages || options.contextGeneration !== undefined && options.contextGeneration !== runtime.getContextGeneration()) return;
+				runtime.lastObserverRun = {
+					startedAt: observerStartedAt,
+					status: "running",
+					messages: messages.slice(),
+					chunkTokens,
+					backlogTokens: tokens,
+					sourceEntryIds: sourceEntryIds.slice(),
+				};
+			},
 			signal: observerWatchdog.signal,
 		}));
 	} catch (error) {
@@ -505,8 +531,19 @@ async function runObserverStage(
 			chunkTokens,
 		});
 		observations = undefined;
+		if (runtime.lastObserverRun?.startedAt === observerStartedAt) {
+			runtime.lastObserverRun = { ...runtime.lastObserverRun, status: "failed", error: failedMessage };
+		}
 	} finally {
+		acceptsObserverMessages = false;
 		observerWatchdog.dispose();
+		if (options.contextGeneration === undefined || options.contextGeneration === runtime.getContextGeneration()) {
+			const completedAt = Date.now();
+			runtime.lastObserverCompletedAt = completedAt;
+			if (runtime.lastObserverRun?.startedAt === observerStartedAt) {
+				runtime.lastObserverRun = { ...runtime.lastObserverRun, completedAt };
+			}
+		}
 	}
 	if (options.contextGeneration !== undefined && options.contextGeneration !== runtime.getContextGeneration()) {
 		debugLog("observer.stale", { reason: "session_or_branch_changed" });
@@ -527,6 +564,15 @@ async function runObserverStage(
 		});
 	}
 	const accepted = observations ?? [];
+	if (!failedMessage && runtime.lastObserverRun?.startedAt === observerStartedAt) {
+		runtime.lastObserverRun = {
+			...runtime.lastObserverRun,
+			status: "completed",
+			summary: accepted.length > 0
+				? `${accepted.length} observation${accepted.length === 1 ? "" : "s"} recorded; chunk covered through ${effectiveCoversUpToId}.`
+				: `No observations recorded; chunk covered through ${effectiveCoversUpToId}.`,
+		};
+	}
 	const data = buildObservationsRecordedData(accepted, effectiveCoversUpToId);
 	if (!data) return "continue";
 	debugLog(failedMessage ? "observer.failed_coverage" : accepted.length > 0 ? "observer.records" : "observer.coverage_only", {

package/src/runtime.ts

@@ -56,6 +56,18 @@ export interface MemoryUpdateCtx extends LaunchCtx {
 
 export type SettingsUpdate = Partial<SessionSettings>;
 
+export interface ObserverRunView {
+	startedAt: number;
+	completedAt?: number;
+	status: "running" | "completed" | "failed";
+	messages: readonly unknown[];
+	chunkTokens: number;
+	backlogTokens: number;
+	sourceEntryIds: readonly string[];
+	summary?: string;
+	error?: string;
+}
+
 export interface SummarizerRunView {
 	startedAt: number;
 	completedAt?: number;
@@ -187,6 +199,8 @@ export class Runtime {
 	lastObserverCompletedAt: number | undefined;
 	lastSummarizerStartedAt: number | undefined;
 	lastSummarizerCompletedAt: number | undefined;
+	/** Current or most recent observer chunk transcript in this launch/session context. */
+	lastObserverRun: ObserverRunView | undefined;
 	/** Most recent summarizer transcript in this extension launch/session context. */
 	lastSummarizerRun: SummarizerRunView | undefined;
 	/** Launch-local liveness and trigger diagnostics published by the contemplator. */
@@ -277,6 +291,7 @@ export class Runtime {
 		this.lastObserverCompletedAt = undefined;
 		this.lastSummarizerStartedAt = undefined;
 		this.lastSummarizerCompletedAt = undefined;
+		this.lastObserverRun = undefined;
 		this.lastSummarizerRun = undefined;
 		this.contemplatorState = {
 			running: false,