@xopcai/xopc 0.0.30 → 0.0.31

package/dist/src/session/store.js

@@ -4,12 +4,14 @@ import { createLogger } from "../utils/logger/index.js";
 import { init_logger } from "../utils/logger.js";
 import { FILENAMES, init_paths, resolveSessionsDir } from "../config/paths.js";
 import { init_write_file_atomic, writeTextAtomic } from "../infra/write-file-atomic.js";
+import { buildSessionContextForLlm, isTranscriptContextEntry, mergeLlmMessagesPreservingContextRows } from "./session-context-for-llm.js";
+import { buildTranscriptEnvelope, parseStoredTranscriptJson } from "./transcript-format.js";
 import { invalidateSessionSearchIndexCache } from "./search-index-cache.js";
 import { resolveSessionShardRelativePath } from "./shard-path.js";
 import "./types.js";
 import { SessionCompactor } from "../agent/memory/compaction.js";
 import { SlidingWindow } from "../agent/memory/window.js";
-import { cleanTrailingErrors, hasProblematicMessages } from "../agent/memory/message-sanitizer.js";
+import { normalizeCompactionCheckpointId } from "./compaction-checkpoints.js";
 import { basename, join } from "path";
 import { existsSync } from "fs";
 import { copyFile, mkdir, readFile, readdir, stat, unlink } from "fs/promises";
@@ -255,10 +257,18 @@ var SessionStore = class {
 		return out;
 	}
 	async scanSessionFile(key) {
-		const messages = await this.loadMessages(key);
-		if (messages.length === 0) return null;
 		const { jsonPath } = this.sessionPathsForKey(key);
+		let raw;
+		try {
+			raw = await readFile(jsonPath, "utf-8");
+		} catch {
+			return null;
+		}
+		const { messages, envelope } = parseStoredTranscriptJson(raw);
+		if (messages.length === 0) return null;
 		const stats = await stat(jsonPath);
+		const sessionStartedAt = envelope?.createdAt ?? stats.birthtime.toISOString();
+		const lastInteractionAt = envelope?.updatedAt ?? stats.mtime.toISOString();
 		const { channel, chatId } = this.parseSessionKey(key);
 		const routing = this.extractRoutingFromKey(key, channel);
 		const isCronSession = channel === "cron";
@@ -275,6 +285,9 @@ var SessionStore = class {
 			compactedCount: 0,
 			sourceChannel: channel,
 			sourceChatId: chatId,
+			...envelope?.id ? { transcriptId: envelope.id } : {},
+			sessionStartedAt,
+			lastInteractionAt,
 			routing,
 			...isCronSession ? {
 				sessionType: "cron",
@@ -352,15 +365,39 @@ var SessionStore = class {
 			hasMore: offset + limit < total
 		};
 	}
-	async get(key) {
+	async get(key, options) {
 		const metadata = await this.getMetadata(key);
 		if (!metadata) return null;
 		const messages = await this.loadMessages(key);
+		let transcriptSummary;
+		if (options?.includeTranscriptSummary) {
+			const env = await this.loadTranscriptDocument(key);
+			if (env) transcriptSummary = {
+				id: env.id,
+				version: env.version,
+				createdAt: env.createdAt,
+				updatedAt: env.updatedAt,
+				compactionCount: env.compactions?.length ?? 0
+			};
+		}
+		let transcriptRows;
+		if (options?.includeTranscriptRows) transcriptRows = await this.loadTranscriptRows(key);
 		return {
 			...metadata,
-			messages: this.convertMessages(messages)
+			messages: this.convertMessages(messages),
+			...transcriptSummary ? { transcriptSummary } : {},
+			...transcriptRows !== void 0 ? { transcriptRows } : {}
 		};
 	}
+	/** Full on-disk transcript rows (LLM messages + optional `kind: 'context'`). */
+	async loadTranscriptRows(key) {
+		const { jsonPath } = this.sessionPathsForKey(key);
+		try {
+			return parseStoredTranscriptJson(await readFile(jsonPath, "utf-8")).rows;
+		} catch {
+			return [];
+		}
+	}
 	async getMetadata(key) {
 		const metadata = (await this.loadIndex()).sessions.find((s) => s.key === key);
 		if (!metadata) {
@@ -458,17 +495,7 @@ var SessionStore = class {
 		const primary = this.sessionPathsForKey(key);
 		const readAndNormalize = async (path) => {
 			try {
-				const data = await readFile(path, "utf-8");
-				const messages = JSON.parse(data);
-				if (hasProblematicMessages(messages)) {
-					const cleaned = cleanTrailingErrors(messages);
-					if (cleaned.length !== messages.length) log.info({
-						key,
-						original: messages.length,
-						cleaned: cleaned.length
-					}, "Cleaned problematic messages on load");
-					return cleaned;
-				}
+				const { messages } = parseStoredTranscriptJson(await readFile(path, "utf-8"));
 				return messages;
 			} catch {
 				return null;
@@ -484,6 +511,17 @@ var SessionStore = class {
 		return [];
 	}
 	/**
+	* Load the versioned transcript document (stable id, compaction history), or null if missing/invalid.
+	*/
+	async loadTranscriptDocument(key) {
+		const { jsonPath } = this.sessionPathsForKey(key);
+		try {
+			return parseStoredTranscriptJson(await readFile(jsonPath, "utf-8")).envelope;
+		} catch {
+			return null;
+		}
+	}
+	/**
 	* Find the most recent archived session file for a given key.
 	* Archived files have format: {safeKey}.{timestamp}.json
 	*/
@@ -505,11 +543,22 @@ var SessionStore = class {
 	}
 	/**
 	* Persist transcript JSON + merge session row into the index. Caller must hold {@link runStoreMutation} (or be nested under it).
+	* Transcript is stored as a versioned document (pi-style header) with stable {@link XopcSessionTranscriptV1.id}.
 	*/
-	async writeSessionTranscriptAndUpdateIndex(key, messages) {
+	async writeSessionTranscriptFromStoredRows(key, storedRows, options) {
 		const { dir, jsonPath } = this.sessionPathsForKey(key);
 		await mkdir(dir, { recursive: true });
-		await writeTextAtomic(jsonPath, JSON.stringify(messages, null, 2));
+		let previous = null;
+		try {
+			previous = parseStoredTranscriptJson(await readFile(jsonPath, "utf-8")).envelope;
+		} catch {}
+		const doc = buildTranscriptEnvelope({
+			storedRows,
+			previous,
+			appendCompaction: options?.appendCompaction
+		});
+		await writeTextAtomic(jsonPath, JSON.stringify(doc, null, 2));
+		const llmMessages = buildSessionContextForLlm(storedRows);
 		const index = await this.loadIndex();
 		const existingIdx = index.sessions.findIndex((s) => s.key === key);
 		const now = (/* @__PURE__ */ new Date()).toISOString();
@@ -523,10 +572,13 @@ var SessionStore = class {
 				...prev,
 				sourceChannel: channel,
 				sourceChatId: chatId,
-				messageCount: messages.length,
-				estimatedTokens: this.estimateTokens(messages),
+				messageCount: llmMessages.length,
+				estimatedTokens: this.estimateTokens(llmMessages),
 				updatedAt: now,
 				lastAccessedAt: now,
+				transcriptId: doc.id,
+				sessionStartedAt: prev.sessionStartedAt ?? doc.createdAt,
+				lastInteractionAt: now,
 				routing: routing || prev.routing,
 				...isCronSession ? {
 					sessionType: "cron",
@@ -544,8 +596,8 @@ var SessionStore = class {
 				} : {},
 				stats: {
 					...prev.stats,
-					messageCount: messages.length,
-					tokenCount: this.estimateTokens(messages),
+					messageCount: llmMessages.length,
+					tokenCount: this.estimateTokens(llmMessages),
 					lastTurnAt: Date.now()
 				}
 			};
@@ -556,8 +608,11 @@ var SessionStore = class {
 			createdAt: now,
 			updatedAt: now,
 			lastAccessedAt: now,
-			messageCount: messages.length,
-			estimatedTokens: this.estimateTokens(messages),
+			transcriptId: doc.id,
+			sessionStartedAt: doc.createdAt,
+			lastInteractionAt: now,
+			messageCount: llmMessages.length,
+			estimatedTokens: this.estimateTokens(llmMessages),
 			compactedCount: 0,
 			sourceChannel: channel,
 			sourceChatId: chatId,
@@ -571,8 +626,8 @@ var SessionStore = class {
 				customData: { heartbeatTarget: chatId }
 			} : {},
 			stats: {
-				messageCount: messages.length,
-				tokenCount: this.estimateTokens(messages),
+				messageCount: llmMessages.length,
+				tokenCount: this.estimateTokens(llmMessages),
 				lastTurnAt: Date.now()
 			}
 		});
@@ -580,6 +635,37 @@ var SessionStore = class {
 		await this.saveIndex();
 		invalidateSessionSearchIndexCache();
 	}
+	async writeSessionTranscriptAndUpdateIndex(key, messages, options) {
+		const { jsonPath } = this.sessionPathsForKey(key);
+		let storedRows = messages;
+		try {
+			const parsed = parseStoredTranscriptJson(await readFile(jsonPath, "utf-8"));
+			if (parsed.rows.some((r) => isTranscriptContextEntry(r))) storedRows = mergeLlmMessagesPreservingContextRows(parsed.rows, messages);
+		} catch {}
+		await this.writeSessionTranscriptFromStoredRows(key, storedRows, options);
+	}
+	/**
+	* Append a persisted-only transcript row (`kind: 'context'`), visible on disk and in session search
+	* after stripping, but never returned from {@link loadMessages}.
+	*/
+	async appendTranscriptContextEntry(key, entry) {
+		return this.runStoreMutation(async () => {
+			this.invalidateIndexCache();
+			const { jsonPath } = this.sessionPathsForKey(key);
+			let rows = [];
+			try {
+				rows = parseStoredTranscriptJson(await readFile(jsonPath, "utf-8")).rows;
+			} catch {}
+			const row = {
+				kind: "context",
+				id: typeof entry.id === "string" ? entry.id : void 0,
+				text: typeof entry.text === "string" ? entry.text : void 0,
+				data: entry.data,
+				createdAt: entry.createdAt ?? (/* @__PURE__ */ new Date()).toISOString()
+			};
+			await this.writeSessionTranscriptFromStoredRows(key, [...rows, row], {});
+		});
+	}
 	async saveMessages(key, messages) {
 		return this.runStoreMutation(async () => {
 			this.invalidateIndexCache();
@@ -618,7 +704,13 @@ var SessionStore = class {
 			this.invalidateIndexCache();
 			const { jsonPath } = this.sessionPathsForKey(key);
 			await this.captureCompactionCheckpointIfExists(key, jsonPath);
-			await this.writeSessionTranscriptAndUpdateIndex(key, compacted);
+			await this.writeSessionTranscriptAndUpdateIndex(key, compacted, { appendCompaction: {
+				at: (/* @__PURE__ */ new Date()).toISOString(),
+				summary: result.summary,
+				firstKeptIndex: result.firstKeptIndex,
+				tokensBefore: result.tokensBefore,
+				tokensAfter: result.tokensAfter
+			} });
 			const metadata = await this.getMetadata(key);
 			if (metadata) await this.updateMetadata(key, { compactedCount: metadata.compactedCount + 1 });
 			log.info({
@@ -651,6 +743,81 @@ var SessionStore = class {
 			lastCompactionAt: void 0
 		};
 	}
+	/**
+	* List pre-compaction transcript snapshots for a session (newest first).
+	*/
+	async listCompactionCheckpoints(key) {
+		const safeKey = this.sanitizeKey(key);
+		const dir = join(this.sessionsDir, resolveSessionShardRelativePath(key));
+		const prefix = this.checkpointBasenamePrefix(safeKey);
+		let names;
+		try {
+			names = await readdir(dir);
+		} catch {
+			return [];
+		}
+		const files = names.filter((n) => n.startsWith(prefix) && n.endsWith(".json"));
+		const valid = (await Promise.all(files.map(async (name) => {
+			const p = join(dir, name);
+			try {
+				const s = await stat(p);
+				const id = name.slice(prefix.length, -5);
+				if (!normalizeCompactionCheckpointId(id)) return null;
+				return {
+					id: normalizeCompactionCheckpointId(id),
+					sizeBytes: s.size,
+					modifiedAt: new Date(s.mtimeMs).toISOString()
+				};
+			} catch {
+				return null;
+			}
+		}))).filter((r) => r !== null);
+		valid.sort((a, b) => b.modifiedAt.localeCompare(a.modifiedAt));
+		return valid;
+	}
+	/**
+	* Metadata for a single compaction checkpoint file.
+	*/
+	async getCompactionCheckpointDetail(key, checkpointId) {
+		const id = normalizeCompactionCheckpointId(checkpointId);
+		if (!id) return null;
+		const safeKey = this.sanitizeKey(key);
+		const cpPath = join(join(this.sessionsDir, resolveSessionShardRelativePath(key)), `${this.checkpointBasenamePrefix(safeKey)}${id}.json`);
+		if (!existsSync(cpPath)) return null;
+		try {
+			const { messages } = parseStoredTranscriptJson(await readFile(cpPath, "utf-8"));
+			const s = await stat(cpPath);
+			return {
+				id,
+				sizeBytes: s.size,
+				modifiedAt: new Date(s.mtimeMs).toISOString(),
+				messageCount: messages.length
+			};
+		} catch {
+			return null;
+		}
+	}
+	/**
+	* Restore main transcript from a pre-compaction snapshot (then re-wrap + index sync).
+	* Does not delete the checkpoint file.
+	*/
+	async restoreCompactionCheckpoint(key, checkpointId) {
+		const id = normalizeCompactionCheckpointId(checkpointId);
+		if (!id) throw new Error("Invalid checkpoint id");
+		return this.runStoreMutation(async () => {
+			const safeKey = this.sanitizeKey(key);
+			const cpPath = join(join(this.sessionsDir, resolveSessionShardRelativePath(key)), `${this.checkpointBasenamePrefix(safeKey)}${id}.json`);
+			if (!existsSync(cpPath)) throw new Error(`Checkpoint not found: ${id}`);
+			const { jsonPath } = this.sessionPathsForKey(key);
+			await copyFile(cpPath, jsonPath);
+			const messages = await this.loadMessages(key);
+			await this.writeSessionTranscriptAndUpdateIndex(key, messages);
+			log.info({
+				key,
+				checkpointId: id
+			}, "Session transcript restored from compaction checkpoint");
+		});
+	}
 	/** Alias for delete */
 	async deleteSession(key) {
 		return this.delete(key);
@@ -674,15 +841,20 @@ var SessionStore = class {
 			return this.extractTextContent(m.content).toLowerCase().includes(keywordLower);
 		}));
 	}
+	/**
+	* JSON export includes API-shaped `messages` (LLM-only) plus `transcriptRows` (full on-disk order).
+	*/
 	async exportSession(key, format) {
 		const detail = await this.get(key);
 		if (!detail) throw new Error(`Session not found: ${key}`);
 		if (format === "json") {
+			const transcriptRows = await this.loadTranscriptRows(key);
 			const exportData = {
 				version: INDEX_VERSION,
 				exportedAt: (/* @__PURE__ */ new Date()).toISOString(),
 				metadata: detail,
-				messages: detail.messages
+				messages: detail.messages,
+				transcriptRows
 			};
 			return JSON.stringify(exportData, null, 2);
 		} else {