@zihanw/pi-forge 0.2.0 → 0.3.0

package/src/compiler.ts

@@ -1,5 +1,8 @@
 import type { AgentMessage } from "@earendil-works/pi-agent-core";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
 import type {
+	CompileMessageSource,
 	CompileMessagesResult,
 	CompileSystemPromptResult,
 	PromptRuntime,
@@ -8,10 +11,12 @@ import type {
 	PromptStackItem,
 	PromptStackRole,
 	PromptStackSlotItem,
-	PromptStateScope,
-	PromptStateValue,
+	PromptStackSlotFormat,
+	PromptVariableValue,
 	PromptVariableStore,
 } from "./types.ts";
+import { applyResourcePolicy } from "./policy.ts";
+import { applyRegexRulesToMessages, applyRegexRulesToString } from "./regex.ts";
 import { SUPPORTED_SLOTS } from "./types.ts";
 
 const ZERO_USAGE = {
@@ -23,7 +28,11 @@ const ZERO_USAGE = {
 	cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
 };
 
-export function createPromptVariableStore(sessionVariables: Record<string, PromptStateValue> = {}): PromptVariableStore {
+type PromptVariableScope = "static" | "session" | "turn";
+
+const SUMMARY_ROLES = new Set(["branchSummary", "compactionSummary"]);
+
+export function createPromptVariableStore(sessionVariables: Record<string, PromptVariableValue> = {}): PromptVariableStore {
 	return { turn: {}, session: { ...sessionVariables }, sessionDirty: false };
 }
 
@@ -65,20 +74,20 @@ export function compileSystemPrompt(
 	const compiled = parts.join("\n\n");
 	const mode = stack.mode ?? "replace";
 
+	let systemPrompt: string;
 	if (!compiled) {
 		diagnostics.push({ level: "warning", message: "Compiled system prompt is empty; preserving base system prompt." });
-		return { systemPrompt: baseSystemPrompt, diagnostics };
-	}
-
-	if (mode === "append") {
-		return { systemPrompt: compiled ? `${baseSystemPrompt}\n\n${compiled}` : baseSystemPrompt, diagnostics };
-	}
-
-	if (mode === "prepend") {
-		return { systemPrompt: compiled ? `${compiled}\n\n${baseSystemPrompt}` : baseSystemPrompt, diagnostics };
+		systemPrompt = baseSystemPrompt;
+	} else if (mode === "append") {
+		systemPrompt = compiled ? `${baseSystemPrompt}\n\n${compiled}` : baseSystemPrompt;
+	} else if (mode === "prepend") {
+		systemPrompt = compiled ? `${compiled}\n\n${baseSystemPrompt}` : baseSystemPrompt;
+	} else {
+		systemPrompt = compiled;
 	}
 
-	return { systemPrompt: compiled, diagnostics };
+	systemPrompt = applyRegexRulesToString(stack, systemPrompt, "compiled", "system", diagnostics);
+	return { systemPrompt, diagnostics };
 }
 
 export function compileMessages(
@@ -87,7 +96,8 @@ export function compileMessages(
 	originalMessages: AgentMessage[],
 ): CompileMessagesResult {
 	const diagnostics: PromptStackDiagnostic[] = [];
-	const messages: AgentMessage[] = [];
+	let messages: AgentMessage[] = [];
+	let messageSources: CompileMessageSource[] = [];
 	let insertedHistory = false;
 
 	for (const item of enabledItems(stack)) {
@@ -100,7 +110,10 @@ export function compileMessages(
 				});
 				continue;
 			}
-			messages.push(...getChatHistoryMessages(originalMessages, item));
+			const historyMessages = getChatHistoryMessages(originalMessages, item, diagnostics);
+			const transformedHistory = applyRegexRulesToMessages(stack, historyMessages, "history", diagnostics);
+			messages.push(...transformedHistory);
+			messageSources.push(...chatHistoryMessageSources(transformedHistory, item));
 			insertedHistory = true;
 			continue;
 		}
@@ -112,14 +125,18 @@ export function compileMessages(
 			: renderSlotText(item, stack, runtime, diagnostics);
 
 		if (!content.trim()) continue;
-		messages.push(createSyntheticMessage(item.role, content, stack, runtime));
+		const message = createSyntheticMessage(item.role, content, stack, runtime);
+		messages.push(message);
+		messageSources.push(stackItemMessageSource(message, item));
 	}
 
 	if (!insertedHistory) {
 		messages.push(...originalMessages);
+		messageSources.push(...implicitHistoryMessageSources(originalMessages));
 	}
 
-	return { messages, diagnostics };
+	messages = applyRegexRulesToMessages(stack, messages, "compiled", diagnostics);
+	return { messages, messageSources, diagnostics };
 }
 
 export function getLatestUserMessage(messages: AgentMessage[]): string | undefined {
@@ -133,32 +150,68 @@ export function getLatestUserMessage(messages: AgentMessage[]): string | undefin
 	return undefined;
 }
 
-export function renderPreviewMessages(messages: AgentMessage[], maxChars = 8000): string {
-	let text = "";
-	for (const message of messages) {
-		const role = message.role;
-		text += `\n--- ${role} ---\n`;
-		text += contentToText((message as { content?: unknown; summary?: string; command?: string; output?: string }).content);
-		if (role === "bashExecution") {
-			const bash = message as { command?: string; output?: string };
-			text += `Ran ${bash.command ?? ""}\n${bash.output ?? ""}`;
-		}
-		if (role === "branchSummary" || role === "compactionSummary") {
-			text += (message as { summary?: string }).summary ?? "";
-		}
-		text += "\n";
-		if (text.length > maxChars) return `${text.slice(0, maxChars)}\n\n[preview truncated]`;
+export function agentMessageToPreviewText(message: AgentMessage): string {
+	let text = contentToText((message as { content?: unknown; summary?: string; command?: string; output?: string }).content);
+	const role = message.role;
+	if (role === "bashExecution") {
+		const bash = message as { command?: string; output?: string };
+		text += `Ran ${bash.command ?? ""}\n${bash.output ?? ""}`;
+	}
+	if (role === "branchSummary" || role === "compactionSummary") {
+		text += (message as { summary?: string }).summary ?? "";
 	}
-	return text.trimStart();
+	return text;
 }
 
-function getChatHistoryMessages(messages: AgentMessage[], item: PromptStackSlotItem): AgentMessage[] {
-	if (item.options?.includeLastUserMessage !== false) return messages;
+function getChatHistoryMessages(
+	messages: AgentMessage[],
+	item: PromptStackSlotItem,
+	diagnostics: PromptStackDiagnostic[],
+): AgentMessage[] {
+	let result = messages;
+	const options = item.options ?? {};
+	let shouldRepairToolPairs = false;
+
+	if (options.includeLastUserMessage === false) {
+		const lastUserIndex = findLastUserMessageIndex(result);
+		if (lastUserIndex !== -1) result = result.filter((_message, index) => index !== lastUserIndex);
+	}
+
+	if (options.includeSummaries === false) {
+		const next = result.filter((message) => !isSummaryMessage(message));
+		addHistoryFilterDiagnostic(diagnostics, item.id, "summary", result.length, next.length);
+		result = next;
+	}
+
+	if (isStringArray(options.roles) && options.roles.length > 0) {
+		const allowedRoles = new Set(options.roles);
+		const next = result.filter((message) => allowedRoles.has(messageRole(message)));
+		addHistoryFilterDiagnostic(diagnostics, item.id, "role", result.length, next.length);
+		result = next;
+		shouldRepairToolPairs = true;
+	}
+
+	if (options.toolMode === "drop") {
+		result = dropToolHistory(result, diagnostics, item.id);
+	} else if (options.toolMode !== undefined && options.toolMode !== "keep") {
+		diagnostics.push({ level: "warning", message: `Unsupported chat-history toolMode: ${String(options.toolMode)}.`, itemId: item.id });
+	}
 
-	const lastUserIndex = findLastUserMessageIndex(messages);
-	if (lastUserIndex === -1) return messages;
+	if (options.stripAssistantThinking === true) {
+		result = stripAssistantThinkingFromHistory(result, diagnostics, item.id);
+	}
 
-	return messages.filter((_message, index) => index !== lastUserIndex);
+	const limited = limitChatHistory(result, options, diagnostics, item.id);
+	if (limited !== result) {
+		result = limited;
+		shouldRepairToolPairs = true;
+	}
+
+	if (shouldRepairToolPairs && options.toolMode !== "drop") {
+		result = repairToolHistory(result, diagnostics, item.id);
+	}
+
+	return result;
 }
 
 function findLastUserMessageIndex(messages: AgentMessage[]): number {
@@ -168,6 +221,309 @@ function findLastUserMessageIndex(messages: AgentMessage[]): number {
 	return -1;
 }
 
+function limitChatHistory(
+	messages: AgentMessage[],
+	options: NonNullable<PromptStackSlotItem["options"]>,
+	diagnostics: PromptStackDiagnostic[],
+	itemId: string,
+): AgentMessage[] {
+	let result = messages;
+	const maxMessages = positiveIntegerOption(options.maxMessages);
+	const maxChars = positiveIntegerOption(options.maxChars);
+
+	if (maxMessages !== undefined && result.length > maxMessages) {
+		const next = result.slice(-maxMessages);
+		diagnostics.push({
+			level: "info",
+			message: `Trimmed chat history from ${result.length} to ${next.length} message(s) by maxMessages.`,
+			itemId,
+		});
+		result = next;
+	}
+
+	if (maxChars !== undefined) {
+		const next = takeRecentMessagesWithinChars(result, maxChars);
+		if (next.length < result.length) {
+			diagnostics.push({
+				level: "info",
+				message: `Trimmed chat history from ${result.length} to ${next.length} message(s) by maxChars.`,
+				itemId,
+			});
+			result = next;
+		}
+	}
+
+	return result;
+}
+
+function takeRecentMessagesWithinChars(messages: AgentMessage[], maxChars: number): AgentMessage[] {
+	const selected: AgentMessage[] = [];
+	let chars = 0;
+	for (let index = messages.length - 1; index >= 0; index--) {
+		const message = messages[index];
+		const messageChars = agentMessageToPreviewText(message).length;
+		if (selected.length > 0 && chars + messageChars > maxChars) break;
+		selected.push(message);
+		chars += messageChars;
+	}
+	return selected.reverse();
+}
+
+function addHistoryFilterDiagnostic(
+	diagnostics: PromptStackDiagnostic[],
+	itemId: string,
+	filter: string,
+	before: number,
+	after: number,
+): void {
+	if (before === after) return;
+	diagnostics.push({
+		level: "info",
+		message: `Filtered ${before - after} chat-history message(s) by ${filter}.`,
+		itemId,
+	});
+}
+
+function dropToolHistory(
+	messages: AgentMessage[],
+	diagnostics: PromptStackDiagnostic[],
+	itemId: string,
+): AgentMessage[] {
+	let removedToolCalls = 0;
+	let droppedToolResults = 0;
+	let droppedEmptyMessages = 0;
+	let changed = false;
+	const result: AgentMessage[] = [];
+
+	for (const message of messages) {
+		if (isToolResultMessage(message)) {
+			droppedToolResults++;
+			changed = true;
+			continue;
+		}
+		const stripped = stripToolCallParts(message, () => false);
+		removedToolCalls += stripped.removedCalls;
+		if (stripped.message !== message) changed = true;
+		if (!stripped.message) {
+			droppedEmptyMessages++;
+			continue;
+		}
+		result.push(stripped.message);
+	}
+
+	if (!changed) return messages;
+	diagnostics.push({
+		level: "info",
+		message: `Dropped tool history from chat-history: removed ${removedToolCalls} tool call(s), dropped ${droppedToolResults} tool result message(s)` +
+			(droppedEmptyMessages > 0 ? `, and dropped ${droppedEmptyMessages} empty message(s).` : "."),
+		itemId,
+	});
+	return result;
+}
+
+function repairToolHistory(
+	messages: AgentMessage[],
+	diagnostics: PromptStackDiagnostic[],
+	itemId: string,
+): AgentMessage[] {
+	const includedCallIds = new Set<string>();
+	const includedResultIds = new Set<string>();
+	for (const message of messages) {
+		for (const id of toolCallIdsForMessage(message)) includedCallIds.add(id);
+		const resultId = toolResultMessageId(message);
+		if (resultId) includedResultIds.add(resultId);
+	}
+
+	let removedToolCalls = 0;
+	let droppedToolResults = 0;
+	let droppedEmptyMessages = 0;
+	let changed = false;
+	const result: AgentMessage[] = [];
+
+	for (const message of messages) {
+		if (isToolResultMessage(message)) {
+			const resultId = toolResultMessageId(message);
+			if (!resultId || !includedCallIds.has(resultId)) {
+				droppedToolResults++;
+				changed = true;
+				continue;
+			}
+			result.push(message);
+			continue;
+		}
+
+		const stripped = stripToolCallParts(message, (id) => !!id && includedResultIds.has(id));
+		removedToolCalls += stripped.removedCalls;
+		if (stripped.message !== message) changed = true;
+		if (!stripped.message) {
+			droppedEmptyMessages++;
+			continue;
+		}
+		result.push(stripped.message);
+	}
+
+	if (!changed) return messages;
+	diagnostics.push({
+		level: "info",
+		message: `Repaired tool history after chat-history filtering: removed ${removedToolCalls} dangling tool call(s), dropped ${droppedToolResults} dangling tool result message(s)` +
+			(droppedEmptyMessages > 0 ? `, and dropped ${droppedEmptyMessages} empty message(s).` : "."),
+		itemId,
+	});
+	return result;
+}
+
+function stripToolCallParts(
+	message: AgentMessage,
+	keep: (id: string | undefined) => boolean,
+): { message?: AgentMessage; removedCalls: number } {
+	const content = (message as { content?: unknown }).content;
+	if (!Array.isArray(content)) return { message, removedCalls: 0 };
+
+	const nextContent = content.filter((part) => !isToolCallContent(part) || keep(toolCallPartId(part)));
+	const removedCalls = content.length - nextContent.length;
+	if (removedCalls === 0) return { message, removedCalls: 0 };
+	if (nextContent.length === 0) return { removedCalls };
+	return { message: { ...message, content: nextContent } as AgentMessage, removedCalls };
+}
+
+function toolCallIdsForMessage(message: AgentMessage): string[] {
+	const content = (message as { content?: unknown }).content;
+	if (!Array.isArray(content)) return [];
+	return content
+		.map((part) => toolCallPartId(part))
+		.filter((id): id is string => !!id);
+}
+
+function isToolCallContent(value: unknown): boolean {
+	return isPlainObject(value) && value.type === "toolCall";
+}
+
+function toolCallPartId(value: unknown): string | undefined {
+	if (!isPlainObject(value)) return undefined;
+	return firstString(value.id, value.toolCallId, value.callId);
+}
+
+function isToolResultMessage(message: AgentMessage): boolean {
+	return messageRole(message) === "toolResult";
+}
+
+function toolResultMessageId(message: AgentMessage): string | undefined {
+	if (!isToolResultMessage(message)) return undefined;
+	const raw = message as unknown as Record<string, unknown>;
+	return firstString(raw.toolCallId, raw.id, raw.callId);
+}
+
+function isSummaryMessage(message: AgentMessage): boolean {
+	return SUMMARY_ROLES.has(messageRole(message));
+}
+
+function messageRole(message: AgentMessage): string {
+	return String((message as { role?: unknown }).role ?? "");
+}
+
+function positiveIntegerOption(value: unknown): number | undefined {
+	return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : undefined;
+}
+
+function firstString(...values: unknown[]): string | undefined {
+	for (const value of values) {
+		if (typeof value === "string" && value) return value;
+	}
+	return undefined;
+}
+
+function isStringArray(value: unknown): value is string[] {
+	return Array.isArray(value) && value.every((item) => typeof item === "string");
+}
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+	return !!value && typeof value === "object" && !Array.isArray(value);
+}
+
+function chatHistoryMessageSources(messages: AgentMessage[], item: PromptStackSlotItem): CompileMessageSource[] {
+	return messages.map((message, index) => ({
+		kind: "chat-history",
+		itemId: item.id,
+		itemName: item.name,
+		slot: item.slot,
+		historyIndex: index + 1,
+		historyCount: messages.length,
+		role: String((message as { role?: unknown }).role ?? ""),
+	}));
+}
+
+function implicitHistoryMessageSources(messages: AgentMessage[]): CompileMessageSource[] {
+	return messages.map((message, index) => ({
+		kind: "implicit-history",
+		itemName: "Conversation history",
+		slot: "chat-history",
+		historyIndex: index + 1,
+		historyCount: messages.length,
+		role: String((message as { role?: unknown }).role ?? ""),
+	}));
+}
+
+function stackItemMessageSource(message: AgentMessage, item: PromptStackItem): CompileMessageSource {
+	return {
+		kind: "stack-item",
+		itemId: item.id,
+		itemName: item.name,
+		role: String((message as { role?: unknown }).role ?? ""),
+	};
+}
+
+function stripAssistantThinkingFromHistory(
+	messages: AgentMessage[],
+	diagnostics: PromptStackDiagnostic[],
+	itemId: string,
+): AgentMessage[] {
+	let strippedBlocks = 0;
+	let changedMessages = 0;
+	let droppedMessages = 0;
+	let changed = false;
+	const result: AgentMessage[] = [];
+
+	for (const message of messages) {
+		const stripped = stripAssistantThinkingFromMessage(message);
+		if (stripped.message !== message) {
+			changed = true;
+			strippedBlocks += stripped.removedBlocks;
+			changedMessages++;
+		}
+		if (!stripped.message) {
+			droppedMessages++;
+			continue;
+		}
+		result.push(stripped.message);
+	}
+
+	if (!changed) return messages;
+
+	diagnostics.push({
+		level: "info",
+		message: `Stripped ${strippedBlocks} assistant thinking block(s) from ${changedMessages} chat-history message(s)` +
+			(droppedMessages > 0 ? ` and dropped ${droppedMessages} empty assistant message(s).` : "."),
+		itemId,
+	});
+	return result;
+}
+
+function stripAssistantThinkingFromMessage(message: AgentMessage): { message?: AgentMessage; removedBlocks: number } {
+	if (message.role !== "assistant") return { message, removedBlocks: 0 };
+	const content = (message as { content?: unknown }).content;
+	if (!Array.isArray(content)) return { message, removedBlocks: 0 };
+
+	const nextContent = content.filter((part) => !isThinkingContent(part));
+	const removedBlocks = content.length - nextContent.length;
+	if (removedBlocks === 0) return { message, removedBlocks: 0 };
+	if (nextContent.length === 0) return { removedBlocks };
+	return { message: { ...message, content: nextContent } as AgentMessage, removedBlocks };
+}
+
+function isThinkingContent(value: unknown): boolean {
+	return !!value && typeof value === "object" && !Array.isArray(value) && (value as { type?: unknown }).type === "thinking";
+}
+
 function enabledItems(stack: PromptStack): PromptStackItem[] {
 	return stack.items.filter((item) => item.enabled !== false);
 }
@@ -187,13 +543,13 @@ function renderSlotText(
 
 	switch (slot) {
 		case "tools":
-			return renderTools(runtime);
+			return renderTools(item, stack, runtime);
 		case "tool-guidelines":
-			return renderToolGuidelines(runtime);
+			return renderToolGuidelines(item, stack, runtime);
 		case "skills":
-			return renderSkills(runtime);
+			return renderSkills(item, stack, runtime);
 		case "project-context":
-			return renderProjectContext(runtime);
+			return renderProjectContext(item, runtime);
 		case "append-system-prompt":
 			return runtime.options.appendSystemPrompt ?? "";
 		case "date":
@@ -217,9 +573,24 @@ function renderSlotText(
 	}
 }
 
-function renderTools(runtime: PromptRuntime): string {
-	const tools = runtime.options.selectedTools ?? [];
+function renderTools(item: PromptStackSlotItem, stack: PromptStack, runtime: PromptRuntime): string {
 	const snippets = runtime.options.toolSnippets ?? {};
+	const tools = item.options?.onlyWithSnippets === true
+		? scopedToolNames(stack, runtime).filter((name) => !!snippets[name])
+		: scopedToolNames(stack, runtime);
+
+	if (slotTextFormat(item) === "plain") {
+		const lines = ["Available tools:"];
+		if (tools.length === 0) {
+			lines.push(item.options?.onlyWithSnippets === true ? "(none)" : "- (none)");
+		} else {
+			for (const name of tools) {
+				lines.push(plainBullet(name, snippets[name] ?? "No prompt snippet provided."));
+			}
+		}
+		return lines.join("\n");
+	}
+
 	const lines = ["<available_tools>"];
 
 	if (tools.length === 0) {
@@ -235,8 +606,8 @@ function renderTools(runtime: PromptRuntime): string {
 	return lines.join("\n");
 }
 
-function renderToolGuidelines(runtime: PromptRuntime): string {
-	const tools = runtime.options.selectedTools ?? [];
+function renderToolGuidelines(item: PromptStackSlotItem, stack: PromptStack, runtime: PromptRuntime): string {
+	const tools = scopedToolNames(stack, runtime);
 	const guidelines: string[] = [];
 	const seen = new Set<string>();
 	const add = (line: string) => {
@@ -247,17 +618,31 @@ function renderToolGuidelines(runtime: PromptRuntime): string {
 	};
 
 	if (tools.includes("bash") && !tools.includes("grep") && !tools.includes("find") && !tools.includes("ls")) {
-		add("Use bash for file operations like ls, rg, find.");
+		add(item.options?.piStyle === true ? "Use bash for file operations like ls, rg, find" : "Use bash for file operations like ls, rg, find.");
 	}
 
 	for (const guideline of runtime.options.promptGuidelines ?? []) add(guideline);
+	if (item.options?.includePiDefaultGuidelines === true) {
+		add("Be concise in your responses");
+		add("Show file paths clearly when working with files");
+	}
 
 	if (guidelines.length === 0) return "";
+	if (slotTextFormat(item) === "plain") {
+		const heading = typeof item.options?.heading === "string" ? item.options.heading.trim() : "Tool guidelines:";
+		return [
+			...(heading ? [heading] : []),
+			...guidelines.map((line) => `- ${plainContinuation(line, "  ")}`),
+		].join("\n");
+	}
 	return ["<tool_guidelines>", ...guidelines.map((line) => `- ${line}`), "</tool_guidelines>"].join("\n");
 }
 
-function renderSkills(runtime: PromptRuntime): string {
-	const skills = (runtime.options.skills ?? []).filter((skill) => !skill.disableModelInvocation);
+function renderSkills(item: PromptStackSlotItem, stack: PromptStack, runtime: PromptRuntime): string {
+	if (item.options?.requireReadTool === true && !scopedToolNames(stack, runtime).includes("read")) return "";
+	const skills = (runtime.options.skills ?? [])
+		.filter((skill) => !skill.disableModelInvocation)
+		.filter((skill) => applyResourcePolicy([skill.name], stack.skills).length > 0);
 	if (skills.length === 0) return "";
 
 	const lines = [
@@ -265,9 +650,18 @@ function renderSkills(runtime: PromptRuntime): string {
 		"Use the read tool to load a skill's file when the task matches its description.",
 		"When a skill file references a relative path, resolve it against the skill directory (parent of SKILL.md / dirname of the path) and use that absolute path in tool commands.",
 		"",
-		"<available_skills>",
 	];
 
+	if (slotTextFormat(item) === "plain") {
+		lines.push("Available skills:");
+		for (const skill of skills) {
+			lines.push(plainBullet(skill.name, skill.description));
+			lines.push(`  Location: ${plainContinuation(skill.filePath, "  ")}`);
+		}
+		return lines.join("\n");
+	}
+
+	lines.push("<available_skills>");
 	for (const skill of skills) {
 		lines.push("  <skill>");
 		lines.push(`    <name>${escapeXml(skill.name)}</name>`);
@@ -280,10 +674,23 @@ function renderSkills(runtime: PromptRuntime): string {
 	return lines.join("\n");
 }
 
-function renderProjectContext(runtime: PromptRuntime): string {
+function scopedToolNames(stack: PromptStack, runtime: PromptRuntime): string[] {
+	return applyResourcePolicy(runtime.options.selectedTools ?? [], stack.tools);
+}
+
+function renderProjectContext(item: PromptStackSlotItem, runtime: PromptRuntime): string {
 	const contextFiles = runtime.options.contextFiles ?? [];
 	if (contextFiles.length === 0) return "";
 
+	if (slotTextFormat(item) === "plain") {
+		const lines = ["Project context:", "", "Project-specific instructions and guidelines:", ""];
+		for (const file of contextFiles) {
+			lines.push(`Path: ${file.path}`);
+			lines.push(indentPlainBlock(file.content, "  "), "");
+		}
+		return lines.join("\n").trimEnd();
+	}
+
 	const lines = ["<project_context>", "", "Project-specific instructions and guidelines:", ""];
 	for (const file of contextFiles) {
 		lines.push(`<project_instructions path=\"${escapeXml(file.path)}\">`);
@@ -294,14 +701,49 @@ function renderProjectContext(runtime: PromptRuntime): string {
 	return lines.join("\n");
 }
 
+function slotTextFormat(item: PromptStackSlotItem, options: { allowJson?: boolean } = {}): PromptStackSlotFormat {
+	const format = item.options?.format;
+	if (format === "plain") return "plain";
+	if (format === "json" && options.allowJson) return "json";
+	return "xml";
+}
+
 function renderPiDocsGuidance(): string {
+	const paths = piDocsPaths();
 	return [
-		"Pi documentation guidance:",
-		"- Read Pi documentation only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI.",
-		"- Resolve docs/... against Pi's installed documentation directory and examples/... against Pi's installed examples directory.",
+		"Pi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):",
+		`- Main documentation: ${paths.readme}`,
+		`- Additional docs: ${paths.docs}`,
+		`- Examples: ${paths.examples} (extensions, custom tools, SDK)`,
+		"- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory",
+		"- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)",
+		"- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing",
+		"- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)",
 	].join("\n");
 }
 
+function piDocsPaths(): { readme: string; docs: string; examples: string } {
+	const fallbackRoot = "@earendil-works/pi-coding-agent";
+	try {
+		const resolve = (import.meta as unknown as { resolve?: (specifier: string) => string }).resolve;
+		if (!resolve) throw new Error("import.meta.resolve unavailable");
+		const resolved = resolve("@earendil-works/pi-coding-agent");
+		if (!resolved.startsWith("file:")) throw new Error("non-file package resolution");
+		const packageRoot = dirname(dirname(fileURLToPath(resolved)));
+		return {
+			readme: join(packageRoot, "README.md"),
+			docs: join(packageRoot, "docs"),
+			examples: join(packageRoot, "examples"),
+		};
+	} catch {
+		return {
+			readme: `${fallbackRoot}/README.md`,
+			docs: `${fallbackRoot}/docs`,
+			examples: `${fallbackRoot}/examples`,
+		};
+	}
+}
+
 function renderVariables(
 	item: PromptStackSlotItem,
 	stack: PromptStack,
@@ -309,159 +751,88 @@ function renderVariables(
 ): string {
 	const options = item.options ?? {};
 	const scopes = selectedVariableScopes(options);
-	const includeMetadata = options.includeMetadata === true;
-	const format = options.format === "json" ? "json" : "xml";
-	const includeNamespaces = normalizeStringArray(options.includeNamespaces);
-	const excludeNamespaces = normalizeStringArray(options.excludeNamespaces);
-	const maxValueChars = typeof options.maxValueChars === "number" && Number.isFinite(options.maxValueChars) && options.maxValueChars > 0
-		? Math.floor(options.maxValueChars)
-		: undefined;
+	const format = slotTextFormat(item);
 	const store = runtime.variables;
-	const grouped: Record<PromptStateScope, Record<string, PromptStateValue>> = {
+	const grouped: Record<PromptVariableScope, Record<string, PromptVariableValue>> = {
 		static: {},
 		session: {},
 		turn: {},
 	};
 
-	if (scopes.has("static")) {
-		grouped.static = collectStaticVariables(stack);
-	}
-	if (scopes.has("session")) {
-		grouped.session = { ...collectDefaultVariables(stack, "session"), ...(store?.session ?? {}) };
-	}
-	if (scopes.has("turn")) {
-		grouped.turn = { ...collectDefaultVariables(stack, "turn"), ...(store?.turn ?? {}) };
-	}
+	if (scopes.includes("static")) grouped.static = collectStaticVariables(stack);
+	if (scopes.includes("session")) grouped.session = { ...(store?.session ?? {}) };
+	if (scopes.includes("turn")) grouped.turn = { ...(store?.turn ?? {}) };
 
-	for (const scope of ["static", "session", "turn"] as const) {
-		grouped[scope] = Object.fromEntries(
-			Object.entries(grouped[scope]).filter(([name]) => shouldRenderVariable(name, includeNamespaces, excludeNamespaces)),
-		);
-	}
-
-	if (!Object.values(grouped).some((values) => Object.keys(values).length > 0)) return "";
-
-	if (format === "json") {
-		const payload: Record<string, unknown> = {};
-		for (const scope of ["static", "session", "turn"] as const) {
-			if (Object.keys(grouped[scope]).length === 0) continue;
-			payload[scope] = includeMetadata
-				? Object.fromEntries(Object.entries(grouped[scope]).sort(([a], [b]) => a.localeCompare(b)).map(([name, value]) => [
-					name,
-					{
-						value,
-						...metadataForVariable(stack, name),
-					},
-				]))
-				: Object.fromEntries(Object.entries(grouped[scope]).sort(([a], [b]) => a.localeCompare(b)));
-		}
-		return `<prompt_state format=\"json\">\n${escapeXml(JSON.stringify(payload, null, 2))}\n</prompt_state>`;
+	const hasVariables = Object.values(grouped).some((values) => Object.keys(values).length > 0);
+	if (!hasVariables) return "";
+
+	if (format === "plain") {
+		return renderPlainVariables(grouped);
 	}
 
-	const parts: string[] = ["<prompt_state>"];
+	const parts: string[] = ["<variables>"];
 
-	for (const scope of ["static", "session", "turn"] as const) {
+	for (const scope of scopes) {
 		const entries = Object.entries(grouped[scope]).sort(([a], [b]) => a.localeCompare(b));
 		if (entries.length === 0) continue;
 		parts.push(`  <${scope}>`);
 		for (const [name, value] of entries) {
-			const metadata = includeMetadata ? metadataForVariable(stack, name) : {};
-			const attrs = [
-				`name=\"${escapeXml(name)}\"`,
-				`type=\"${escapeXml(metadata.type ?? inferStateValueType(value))}\"`,
-			];
-			if (metadata.description) attrs.push(`description=\"${escapeXml(metadata.description)}\"`);
-			if (metadata.agentWritable !== undefined) attrs.push(`agentWritable=\"${metadata.agentWritable ? "true" : "false"}\"`);
-			if (metadata.userWritable !== undefined) attrs.push(`userWritable=\"${metadata.userWritable ? "true" : "false"}\"`);
-			parts.push(`    <var ${attrs.join(" ")}>${escapeXml(truncateValue(stateValueToPromptText(value), maxValueChars))}</var>`);
+			parts.push(`    <var name=\"${escapeXml(name)}\">${escapeXml(variableValueToPromptText(value))}</var>`);
 		}
 		parts.push(`  </${scope}>`);
 	}
 
-	parts.push("</prompt_state>");
+	parts.push("</variables>");
 	return parts.join("\n");
 }
 
-function selectedVariableScopes(options: Record<string, unknown>): Set<PromptStateScope> {
-	const explicit = normalizeStringArray(options.includeScopes).filter(isStateScope);
-	if (explicit.length > 0) return new Set(explicit);
-
-	const scopes = new Set<PromptStateScope>();
-	if (options.includeStatic !== false) scopes.add("static");
-	if (options.includeSession !== false) scopes.add("session");
-	if (options.includeTurn !== false) scopes.add("turn");
-	return scopes;
-}
-
-function collectStaticVariables(stack: PromptStack): Record<string, PromptStateValue> {
-	return {
-		...collectDefaultVariables(stack, "static"),
-		...(stack.variables ?? {}),
-	};
-}
+function renderPlainVariables(
+	grouped: Record<PromptVariableScope, Record<string, PromptVariableValue>>,
+): string {
+	const parts: string[] = ["Variables:"];
 
-function collectDefaultVariables(stack: PromptStack, scope: PromptStateScope): Record<string, PromptStateValue> {
-	const values: Record<string, PromptStateValue> = {};
-	for (const [name, definition] of Object.entries(stack.state?.definitions ?? {})) {
-		if ((definition.scope ?? "session") !== scope) continue;
-		if (definition.default !== undefined) values[name] = definition.default;
+	for (const scope of ["static", "session", "turn"] as const) {
+		const entries = Object.entries(grouped[scope]).sort(([a], [b]) => a.localeCompare(b));
+		if (entries.length === 0) continue;
+		parts.push(`${scope}:`);
+		for (const [name, value] of entries) {
+			parts.push(plainBullet(name, variableValueToPromptText(value)));
+		}
 	}
-	return values;
-}
 
-function shouldRenderVariable(name: string, includePatterns: string[], excludePatterns: string[]): boolean {
-	if (includePatterns.length > 0 && !includePatterns.some((pattern) => matchesNamespace(name, pattern))) return false;
-	if (excludePatterns.some((pattern) => matchesNamespace(name, pattern))) return false;
-	return true;
-}
-
-function matchesNamespace(name: string, pattern: string): boolean {
-	if (!pattern) return false;
-	if (pattern.endsWith("*")) return name.startsWith(pattern.slice(0, -1));
-	return name === pattern;
+	return parts.join("\n");
 }
 
-function metadataForVariable(stack: PromptStack, name: string): {
-	type?: string;
-	description?: string;
-	agentWritable?: boolean;
-	userWritable?: boolean;
-} {
-	const definition = stack.state?.definitions?.[name];
-	if (!definition) return {};
-	return {
-		type: definition.type,
-		description: definition.description,
-		agentWritable: definition.agentWritable,
-		userWritable: definition.userWritable,
-	};
+function plainBullet(label: string, value: string): string {
+	return `- ${label}: ${plainContinuation(value, "  ")}`;
 }
 
-function normalizeStringArray(value: unknown): string[] {
-	return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : [];
+function plainContinuation(value: string, indent: string): string {
+	return value.split("\n").map((line, index) => index === 0 ? line : `${indent}${line}`).join("\n");
 }
 
-function isStateScope(value: string): value is PromptStateScope {
-	return value === "static" || value === "session" || value === "turn";
+function indentPlainBlock(value: string, indent: string): string {
+	return value.split("\n").map((line) => `${indent}${line}`).join("\n");
 }
 
-function inferStateValueType(value: PromptStateValue): string {
-	if (value === null) return "null";
-	if (Array.isArray(value)) return "array";
-	return typeof value;
+function selectedVariableScopes(options: Record<string, unknown>): PromptVariableScope[] {
+	const scopes: PromptVariableScope[] = [];
+	if (options.includeStatic !== false) scopes.push("static");
+	if (options.includeSession !== false) scopes.push("session");
+	if (options.includeTurn !== false) scopes.push("turn");
+	return scopes;
 }
 
-function truncateValue(value: string, maxChars: number | undefined): string {
-	if (!maxChars || value.length <= maxChars) return value;
-	return `${value.slice(0, maxChars)}\n[truncated]`;
+function collectStaticVariables(stack: PromptStack): Record<string, PromptVariableValue> {
+	return { ...(stack.variables ?? {}) };
 }
 
-function stateValueToPromptText(value: PromptStateValue): string {
+function variableValueToPromptText(value: PromptVariableValue): string {
 	if (typeof value === "string") return value;
 	return JSON.stringify(value, null, 2);
 }
 
-function stateValueToMacroText(value: PromptStateValue | undefined): string {
+function variableValueToMacroText(value: PromptVariableValue | undefined): string {
 	if (value === undefined) return "";
 	if (typeof value === "string") return value;
 	return JSON.stringify(value);
@@ -506,9 +877,9 @@ function expandMacros(
 				unknown.add(expression);
 				return full;
 			}
-			if (command === "getturnvar") return stateValueToMacroText(runtime.variables?.turn[name]);
-			if (command === "getsessionvar") return stateValueToMacroText(runtime.variables?.session[name]);
-			return stateValueToMacroText(getRuntimeVariable(runtime, stack, name));
+			if (command === "getturnvar") return variableValueToMacroText(runtime.variables?.turn[name]);
+			if (command === "getsessionvar") return variableValueToMacroText(runtime.variables?.session[name]);
+			return variableValueToMacroText(getRuntimeVariable(runtime, stack, name));
 		}
 
 		if (command === "clearvar" || command === "clearturnvar" || command === "clearsessionvar") {
@@ -523,11 +894,11 @@ function expandMacros(
 			return "";
 		}
 
-		const dynamicValue = getBuiltinMacro(command, runtime);
+		const dynamicValue = getBuiltinMacro(command, stack, runtime);
 		if (dynamicValue !== undefined) return dynamicValue;
 
 		const variableValue = getRuntimeVariable(runtime, stack, command);
-		if (variableValue !== undefined) return stateValueToMacroText(variableValue);
+		if (variableValue !== undefined) return variableValueToMacroText(variableValue);
 
 		unknown.add(expression);
 		return full;
@@ -545,7 +916,7 @@ function expandMacros(
 	return result;
 }
 
-function getBuiltinMacro(name: string, runtime: PromptRuntime): string | undefined {
+function getBuiltinMacro(name: string, stack: PromptStack, runtime: PromptRuntime): string | undefined {
 	switch (name) {
 		case "cwd":
 			return runtime.options.cwd;
@@ -557,7 +928,7 @@ function getBuiltinMacro(name: string, runtime: PromptRuntime): string | undefin
 			return runtime.latestUserMessage ?? "";
 		case "selectedTools":
 		case "tools":
-			return (runtime.options.selectedTools ?? []).join(", ");
+			return scopedToolNames(stack, runtime).join(", ");
 		case "activeModel": {
 			const model = runtime.ctx?.model;
 			return model ? `${model.provider}/${model.id}` : "";
@@ -567,7 +938,7 @@ function getBuiltinMacro(name: string, runtime: PromptRuntime): string | undefin
 	}
 }
 
-function getRuntimeVariable(runtime: PromptRuntime, stack: PromptStack, name: string): PromptStateValue | undefined {
+function getRuntimeVariable(runtime: PromptRuntime, stack: PromptStack, name: string): PromptVariableValue | undefined {
 	if (runtime.variables && Object.prototype.hasOwnProperty.call(runtime.variables.turn, name)) return runtime.variables.turn[name];
 	if (runtime.variables && Object.prototype.hasOwnProperty.call(runtime.variables.session, name)) return runtime.variables.session[name];
 	const staticVariables = collectStaticVariables(stack);