@f5xc-salesdemos/xcsh 2.2.0 → 14.0.2

package/src/session/compaction/index.ts

@@ -0,0 +1,7 @@
+/**
+ * Compaction and summarization utilities.
+ */
+
+export * from "./branch-summarization";
+export * from "./compaction";
+export * from "./utils";

package/src/session/compaction/pruning.ts

@@ -0,0 +1,91 @@
+/**
+ * Tool output pruning utilities for compaction.
+ */
+import type { AgentMessage } from "@f5xc-salesdemos/pi-agent-core";
+import type { ToolResultMessage } from "@f5xc-salesdemos/pi-ai";
+import type { SessionEntry, SessionMessageEntry } from "../session-manager";
+import { estimateTokens } from "./compaction";
+
+export interface PruneConfig {
+	/** Keep the most recent tool output tokens intact. */
+	protectTokens: number;
+	/** Only prune if total savings meets this threshold. */
+	minimumSavings: number;
+	/** Tool names that should never be pruned. */
+	protectedTools: string[];
+}
+
+export const DEFAULT_PRUNE_CONFIG: PruneConfig = {
+	protectTokens: 40_000,
+	minimumSavings: 20_000,
+	protectedTools: ["skill", "read"],
+};
+
+export interface PruneResult {
+	prunedCount: number;
+	tokensSaved: number;
+}
+
+function createPrunedNotice(tokens: number): string {
+	return `[Output truncated - ${tokens} tokens]`;
+}
+
+function getToolResultMessage(entry: SessionEntry): ToolResultMessage | undefined {
+	if (entry.type !== "message") return undefined;
+	const message = entry.message as AgentMessage;
+	if (message.role !== "toolResult") return undefined;
+	return message as ToolResultMessage;
+}
+
+function estimatePrunedSavings(tokens: number): number {
+	const noticeTokens = Math.ceil(createPrunedNotice(tokens).length / 4);
+	return Math.max(0, tokens - noticeTokens);
+}
+
+export function pruneToolOutputs(entries: SessionEntry[], config: PruneConfig = DEFAULT_PRUNE_CONFIG): PruneResult {
+	let accumulatedTokens = 0;
+	let tokensSaved = 0;
+	let prunedCount = 0;
+
+	const candidates: Array<{ entry: SessionMessageEntry; tokens: number }> = [];
+
+	for (let i = entries.length - 1; i >= 0; i--) {
+		const entry = entries[i];
+		const message = getToolResultMessage(entry);
+		if (!message) continue;
+
+		const tokens = estimateTokens(message as AgentMessage);
+		const isProtected = config.protectedTools.includes(message.toolName);
+
+		if (message.prunedAt !== undefined) {
+			accumulatedTokens += tokens;
+			continue;
+		}
+
+		if (accumulatedTokens < config.protectTokens || isProtected) {
+			accumulatedTokens += tokens;
+			continue;
+		}
+
+		candidates.push({ entry: entry as SessionMessageEntry, tokens });
+		accumulatedTokens += tokens;
+	}
+
+	for (const candidate of candidates) {
+		tokensSaved += estimatePrunedSavings(candidate.tokens);
+	}
+
+	if (tokensSaved < config.minimumSavings || candidates.length === 0) {
+		return { prunedCount: 0, tokensSaved: 0 };
+	}
+
+	const prunedAt = Date.now();
+	for (const candidate of candidates) {
+		const message = candidate.entry.message as ToolResultMessage;
+		message.content = [{ type: "text", text: createPrunedNotice(candidate.tokens) }];
+		message.prunedAt = prunedAt;
+		prunedCount++;
+	}
+
+	return { prunedCount, tokensSaved };
+}

package/src/session/compaction/utils.ts

@@ -0,0 +1,184 @@
+/**
+ * Shared utilities for compaction and branch summarization.
+ */
+import type { AgentMessage } from "@f5xc-salesdemos/pi-agent-core";
+import type { Message } from "@f5xc-salesdemos/pi-ai";
+import { prompt } from "@f5xc-salesdemos/pi-utils";
+import fileOperationsTemplate from "../../prompts/system/file-operations.md" with { type: "text" };
+import summarizationSystemPrompt from "../../prompts/system/summarization-system.md" with { type: "text" };
+
+// ============================================================================
+// File Operation Tracking
+// ============================================================================
+
+export interface FileOperations {
+	read: Set<string>;
+	written: Set<string>;
+	edited: Set<string>;
+}
+
+export function createFileOps(): FileOperations {
+	return {
+		read: new Set(),
+		written: new Set(),
+		edited: new Set(),
+	};
+}
+
+/**
+ * Extract file operations from tool calls in an assistant message.
+ */
+export function extractFileOpsFromMessage(message: AgentMessage, fileOps: FileOperations): void {
+	if (message.role !== "assistant") return;
+	if (!("content" in message) || !Array.isArray(message.content)) return;
+
+	for (const block of message.content) {
+		if (typeof block !== "object" || block === null) continue;
+		if (!("type" in block) || block.type !== "toolCall") continue;
+		if (!("arguments" in block) || !("name" in block)) continue;
+
+		const args = block.arguments as Record<string, unknown> | undefined;
+		if (!args) continue;
+
+		const path = typeof args.path === "string" ? args.path : undefined;
+		if (!path) continue;
+
+		switch (block.name) {
+			case "read":
+				fileOps.read.add(path);
+				break;
+			case "write":
+				fileOps.written.add(path);
+				break;
+			case "edit":
+				fileOps.edited.add(path);
+				break;
+		}
+	}
+}
+
+/**
+ * Compute final file lists from file operations.
+ * Returns readFiles (files only read, not modified) and modifiedFiles.
+ */
+export function computeFileLists(fileOps: FileOperations): { readFiles: string[]; modifiedFiles: string[] } {
+	const modified = new Set([...fileOps.edited, ...fileOps.written]);
+	const readOnly = [...fileOps.read].filter(f => !modified.has(f)).sort();
+	const modifiedFiles = [...modified].sort();
+	return { readFiles: readOnly, modifiedFiles };
+}
+
+/**
+ * Format file operations as XML tags for summary.
+ */
+const FILE_OPERATION_SUMMARY_LIMIT = 20;
+
+function truncateFileList(files: string[]): string[] {
+	if (files.length <= FILE_OPERATION_SUMMARY_LIMIT) return files;
+	const omitted = files.length - FILE_OPERATION_SUMMARY_LIMIT;
+	return [...files.slice(0, FILE_OPERATION_SUMMARY_LIMIT), `… (${omitted} more files omitted)`];
+}
+
+function stripFileOperationTags(summary: string): string {
+	const withoutReadFiles = summary.replace(/<read-files>[\s\S]*?<\/read-files>\s*/g, "");
+	const withoutModifiedFiles = withoutReadFiles.replace(/<modified-files>[\s\S]*?<\/modified-files>\s*/g, "");
+	return withoutModifiedFiles.trimEnd();
+}
+export function formatFileOperations(readFiles: string[], modifiedFiles: string[]): string {
+	if (readFiles.length === 0 && modifiedFiles.length === 0) return "";
+	return prompt.render(fileOperationsTemplate, {
+		readFiles: truncateFileList(readFiles),
+		modifiedFiles: truncateFileList(modifiedFiles),
+	});
+}
+
+export function upsertFileOperations(summary: string, readFiles: string[], modifiedFiles: string[]): string {
+	const baseSummary = stripFileOperationTags(summary);
+	const fileOperations = formatFileOperations(readFiles, modifiedFiles);
+	if (!fileOperations) return baseSummary;
+	if (!baseSummary) return fileOperations;
+	return `${baseSummary}\n\n${fileOperations}`;
+}
+
+// ============================================================================
+// Message Serialization
+// ============================================================================
+
+/** Maximum characters for a tool result in serialized summaries. */
+const TOOL_RESULT_MAX_CHARS = 2000;
+
+/**
+ * Truncate text to a maximum character length for summarization.
+ * Keeps the beginning and appends a truncation marker.
+ */
+function truncateForSummary(text: string, maxChars: number): string {
+	if (text.length <= maxChars) return text;
+	const truncatedChars = text.length - maxChars;
+	return `${text.slice(0, maxChars)}\n\n[... ${truncatedChars} more characters truncated]`;
+}
+
+/**
+ * Serialize LLM messages to text for summarization.
+ * This prevents the model from treating it as a conversation to continue.
+ * Call convertToLlm() first to handle custom message types.
+ */
+export function serializeConversation(messages: Message[]): string {
+	const parts: string[] = [];
+
+	for (const msg of messages) {
+		if (msg.role === "user") {
+			const content =
+				typeof msg.content === "string"
+					? msg.content
+					: msg.content
+							.filter((c): c is { type: "text"; text: string } => c.type === "text")
+							.map(c => c.text)
+							.join("");
+			if (content) parts.push(`[User]: ${content}`);
+		} else if (msg.role === "assistant") {
+			const textParts: string[] = [];
+			const thinkingParts: string[] = [];
+			const toolCalls: string[] = [];
+
+			for (const block of msg.content) {
+				if (block.type === "text") {
+					textParts.push(block.text);
+				} else if (block.type === "thinking") {
+					thinkingParts.push(block.thinking);
+				} else if (block.type === "toolCall") {
+					const args = block.arguments as Record<string, unknown>;
+					const argsStr = Object.entries(args)
+						.map(([k, v]) => `${k}=${JSON.stringify(v)}`)
+						.join(", ");
+					toolCalls.push(`${block.name}(${argsStr})`);
+				}
+			}
+
+			if (thinkingParts.length > 0) {
+				parts.push(`[Assistant thinking]: ${thinkingParts.join("\n")}`);
+			}
+			if (textParts.length > 0) {
+				parts.push(`[Assistant]: ${textParts.join("\n")}`);
+			}
+			if (toolCalls.length > 0) {
+				parts.push(`[Assistant tool calls]: ${toolCalls.join("; ")}`);
+			}
+		} else if (msg.role === "toolResult") {
+			const content = msg.content
+				.filter((c): c is { type: "text"; text: string } => c.type === "text")
+				.map(c => c.text)
+				.join("");
+			if (content) {
+				parts.push(`[Tool result]: ${truncateForSummary(content, TOOL_RESULT_MAX_CHARS)}`);
+			}
+		}
+	}
+
+	return parts.join("\n\n");
+}
+
+// ============================================================================
+// Summarization System Prompt
+// ============================================================================
+
+export const SUMMARIZATION_SYSTEM_PROMPT = prompt.render(summarizationSystemPrompt);

package/src/session/history-storage.ts

@@ -0,0 +1,253 @@
+import { Database, type Statement } from "bun:sqlite";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { getHistoryDbPath, logger } from "@f5xc-salesdemos/pi-utils";
+
+export interface HistoryEntry {
+	id: number;
+	prompt: string;
+	created_at: number;
+	cwd?: string;
+}
+
+type HistoryRow = {
+	id: number;
+	prompt: string;
+	created_at: number;
+	cwd: string | null;
+};
+
+const SQLITE_NOW_EPOCH = "CAST(strftime('%s','now') AS INTEGER)";
+
+class AsyncDrain<T> {
+	#queue?: T[];
+	#promise = Promise.resolve();
+
+	constructor(readonly delayMs: number = 0) {}
+
+	push(value: T, hnd: (values: T[]) => Promise<void> | void): Promise<void> {
+		let queue = this.#queue;
+		if (!queue) {
+			this.#queue = queue = [];
+			this.#promise = new Promise((resolve, reject) => {
+				const exec = () => {
+					try {
+						if (this.#queue === queue) {
+							this.#queue = undefined;
+						}
+						resolve(hnd(queue!));
+					} catch (error) {
+						reject(error);
+					}
+				};
+
+				if (this.delayMs > 0) {
+					setTimeout(exec, this.delayMs);
+				} else {
+					queueMicrotask(exec);
+				}
+			});
+		}
+		queue.push(value);
+		return this.#promise;
+	}
+}
+
+export class HistoryStorage {
+	#db: Database;
+	static #instance?: HistoryStorage;
+	#drain = new AsyncDrain<Pick<HistoryEntry, "prompt" | "cwd">>(100);
+
+	// Prepared statements
+	#insertRowStmt: Statement;
+	#recentStmt: Statement;
+	#searchStmt: Statement;
+	#lastPromptStmt: Statement;
+
+	// In-memory cache of last prompt to avoid sync DB reads on add
+	#lastPromptCache: string | null = null;
+
+	private constructor(dbPath: string) {
+		this.#ensureDir(dbPath);
+
+		this.#db = new Database(dbPath);
+
+		const hasFts = this.#db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='history_fts'").get();
+
+		this.#db.run(`
+PRAGMA journal_mode=WAL;
+PRAGMA synchronous=NORMAL;
+PRAGMA busy_timeout=5000;
+
+CREATE TABLE IF NOT EXISTS history (
+	id INTEGER PRIMARY KEY AUTOINCREMENT,
+	prompt TEXT NOT NULL,
+	created_at INTEGER NOT NULL DEFAULT (${SQLITE_NOW_EPOCH}),
+	cwd TEXT
+);
+CREATE INDEX IF NOT EXISTS idx_history_created_at ON history(created_at DESC);
+
+CREATE VIRTUAL TABLE IF NOT EXISTS history_fts USING fts5(prompt, content='history', content_rowid='id');
+
+CREATE TRIGGER IF NOT EXISTS history_ai AFTER INSERT ON history BEGIN
+	INSERT INTO history_fts(rowid, prompt) VALUES (new.id, new.prompt);
+	END;
+	`);
+
+		if (this.#historySchemaUsesUnixEpoch()) {
+			this.#migrateHistorySchema();
+		}
+
+		if (!hasFts) {
+			try {
+				this.#db.run("INSERT INTO history_fts(history_fts) VALUES('rebuild')");
+			} catch (error) {
+				logger.warn("HistoryStorage FTS rebuild failed", { error: String(error) });
+			}
+		}
+
+		this.#recentStmt = this.#db.prepare(
+			"SELECT id, prompt, created_at, cwd FROM history ORDER BY created_at DESC, id DESC LIMIT ?",
+		);
+		this.#searchStmt = this.#db.prepare(
+			"SELECT h.id, h.prompt, h.created_at, h.cwd FROM history_fts f JOIN history h ON h.id = f.rowid WHERE history_fts MATCH ? ORDER BY h.created_at DESC, h.id DESC LIMIT ?",
+		);
+		this.#lastPromptStmt = this.#db.prepare("SELECT prompt FROM history ORDER BY id DESC LIMIT 1");
+
+		this.#insertRowStmt = this.#db.prepare("INSERT INTO history (prompt, cwd) VALUES (?, ?)");
+
+		const last = this.#lastPromptStmt.get() as { prompt?: string } | undefined;
+		this.#lastPromptCache = last?.prompt ?? null;
+	}
+
+	static open(dbPath: string = getHistoryDbPath()): HistoryStorage {
+		if (!HistoryStorage.#instance) {
+			HistoryStorage.#instance = new HistoryStorage(dbPath);
+		}
+		return HistoryStorage.#instance;
+	}
+
+	/** @internal Reset the singleton — test-only. */
+	static resetInstance(): void {
+		HistoryStorage.#instance = undefined;
+	}
+
+	#insertBatch(rows: Array<Pick<HistoryEntry, "prompt" | "cwd">>): void {
+		this.#db.transaction((rows: Array<Pick<HistoryEntry, "prompt" | "cwd">>) => {
+			for (const row of rows) {
+				this.#insertRowStmt.run(row.prompt, row.cwd ?? null);
+			}
+		})(rows);
+	}
+
+	add(prompt: string, cwd?: string): Promise<void> {
+		const trimmed = prompt.trim();
+		if (!trimmed) return Promise.resolve();
+		if (this.#lastPromptCache === trimmed) return Promise.resolve();
+		this.#lastPromptCache = trimmed;
+		return this.#drain.push({ prompt: trimmed, cwd: cwd ?? undefined }, rows => {
+			this.#insertBatch(rows);
+		});
+	}
+
+	getRecent(limit: number): HistoryEntry[] {
+		const safeLimit = this.#normalizeLimit(limit);
+		if (safeLimit === 0) return [];
+
+		try {
+			const rows = this.#recentStmt.all(safeLimit) as HistoryRow[];
+			return rows.map(row => this.#toEntry(row));
+		} catch (error) {
+			logger.error("HistoryStorage getRecent failed", { error: String(error) });
+			return [];
+		}
+	}
+
+	search(query: string, limit: number): HistoryEntry[] {
+		const safeLimit = this.#normalizeLimit(limit);
+		if (safeLimit === 0) return [];
+
+		const ftsQuery = this.#buildFtsQuery(query);
+		if (!ftsQuery) return [];
+
+		try {
+			const rows = this.#searchStmt.all(ftsQuery, safeLimit) as HistoryRow[];
+			return rows.map(row => this.#toEntry(row));
+		} catch (error) {
+			logger.error("HistoryStorage search failed", { error: String(error) });
+			return [];
+		}
+	}
+
+	#ensureDir(dbPath: string): void {
+		const dir = path.dirname(dbPath);
+		fs.mkdirSync(dir, { recursive: true });
+	}
+
+	#historySchemaUsesUnixEpoch(): boolean {
+		const row = this.#db.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'history'").get() as
+			| { sql?: string | null }
+			| undefined;
+		return row?.sql?.includes("unixepoch(") ?? false;
+	}
+
+	#migrateHistorySchema(): void {
+		const migrate = this.#db.transaction(() => {
+			this.#db.run("ALTER TABLE history RENAME TO history_legacy");
+			this.#db.run("DROP INDEX IF EXISTS idx_history_created_at");
+			this.#db.run("DROP TRIGGER IF EXISTS history_ai");
+			this.#db.run("DROP TABLE IF EXISTS history_fts");
+			this.#db.run(`
+CREATE TABLE history (
+	id INTEGER PRIMARY KEY AUTOINCREMENT,
+	prompt TEXT NOT NULL,
+	created_at INTEGER NOT NULL DEFAULT (${SQLITE_NOW_EPOCH}),
+	cwd TEXT
+);
+CREATE INDEX IF NOT EXISTS idx_history_created_at ON history(created_at DESC);
+INSERT INTO history (id, prompt, created_at, cwd)
+SELECT id, prompt, created_at, cwd
+FROM history_legacy;
+DROP TABLE history_legacy;
+CREATE VIRTUAL TABLE history_fts USING fts5(prompt, content='history', content_rowid='id');
+CREATE TRIGGER history_ai AFTER INSERT ON history BEGIN
+	INSERT INTO history_fts(rowid, prompt) VALUES (new.id, new.prompt);
+END;
+			`);
+			this.#db.run("INSERT INTO history_fts(history_fts) VALUES('rebuild')");
+		});
+		migrate();
+	}
+
+	#normalizeLimit(limit: number): number {
+		if (!Number.isFinite(limit)) return 0;
+		const clamped = Math.max(0, Math.floor(limit));
+		return Math.min(clamped, 1000);
+	}
+
+	#buildFtsQuery(query: string): string | null {
+		const tokens = query
+			.trim()
+			.split(/\s+/)
+			.map(token => token.trim())
+			.filter(Boolean);
+
+		if (tokens.length === 0) return null;
+
+		return tokens
+			.map(token => {
+				const escaped = token.replace(/"/g, '""');
+				return `"${escaped}"*`;
+			})
+			.join(" ");
+	}
+
+	#toEntry(row: HistoryRow): HistoryEntry {
+		return {
+			id: row.id,
+			prompt: row.prompt,
+			created_at: row.created_at,
+			cwd: row.cwd ?? undefined,
+		};
+	}
+}