@xopcai/xopc 0.0.170 → 0.0.171

package/dist/src/storage/sqlite/transcript-repository.js

@@ -1,10 +1,9 @@
 import { getSqliteDatabase, runSqliteWriteTransaction } from "./transaction.js";
-import { buildSessionContextForLlm } from "../../session/session-context-for-llm.js";
+import { buildSessionContextForLlm, isRuntimeOnlyTranscriptMessage } from "../../session/session-context-for-llm.js";
 import { classifyStoredRow, estimateTokensFromMessages, extractFtsContent, transcriptEntryRowToStoredRow } from "./row-mappers.js";
 import { getCurrentSessionId, readCurrentSessionId } from "./session-repository.js";
 import { randomUUID } from "node:crypto";
 //#region src/storage/sqlite/transcript-repository.ts
-const MAX_CHECKPOINTS_PER_TRANSCRIPT = 15;
 function nextSeq(db, sessionId) {
 	return (db.prepare(`SELECT COALESCE(MAX(seq), 0) AS max_seq FROM transcript_entries WHERE session_id = ?`).get(sessionId).max_seq ?? 0) + 1;
 }
@@ -34,6 +33,7 @@ function isUserMessageRow(row) {
 	return classified.entryKind === "message" && classified.role === "user";
 }
 function appendTranscriptEntry(sessionKey, row, opts) {
+	if (isRuntimeOnlyTranscriptMessage(row)) throw new Error("Runtime-only messages cannot be persisted in a session transcript");
 	return runSqliteWriteTransaction((db) => {
 		const sessionId = opts?.sessionId ?? readCurrentSessionId(db, sessionKey);
 		if (!sessionId) throw new Error(`Session not found: ${sessionKey}`);
@@ -58,6 +58,31 @@ function appendTranscriptEntry(sessionKey, row, opts) {
 		return inserted;
 	});
 }
+function appendCompactionBoundary(sessionKey, row) {
+	return runSqliteWriteTransaction((db) => {
+		const sessionId = readCurrentSessionId(db, sessionKey);
+		if (!sessionId) throw new Error(`Session not found: ${sessionKey}`);
+		const boundary = {
+			...row,
+			baseSeq: nextSeq(db, sessionId) - 1
+		};
+		const inserted = insertEntry(db, {
+			sessionId,
+			sessionKey,
+			row: boundary
+		});
+		const now = Date.now();
+		db.prepare(`UPDATE sessions SET
+        message_count = ?,
+        estimated_tokens = ?,
+        compacted_count = compacted_count + 1,
+        updated_at = ?,
+        last_accessed_at = ?,
+        last_interaction_at = ?
+       WHERE session_key = ?`).run(boundary.messages.length, boundary.tokensAfter, now, now, now, sessionKey);
+		return inserted;
+	});
+}
 function loadTranscriptRowsForSession(sessionKey) {
 	const sessionId = getCurrentSessionId(sessionKey);
 	if (!sessionId) return [];
@@ -79,25 +104,30 @@ function loadTranscriptRows(sessionId) {
        ORDER BY seq ASC`).all(sessionId).map(transcriptEntryRowToStoredRow);
 }
 function loadLlmMessagesForSession(sessionKey) {
-	return buildSessionContextForLlm(loadTranscriptRowsForSession(sessionKey));
+	const sessionId = getCurrentSessionId(sessionKey);
+	if (!sessionId) return [];
+	return buildSessionContextForLlm(getSqliteDatabase().prepare(`SELECT entry_id, session_id, seq, entry_kind, role, payload_json, created_at
+       FROM transcript_entries
+       WHERE session_id = ?
+         AND seq >= COALESCE(
+           (SELECT MAX(seq) FROM transcript_entries WHERE session_id = ? AND entry_kind = 'compaction'),
+           1
+         )
+       ORDER BY seq ASC`).all(sessionId, sessionId).map(transcriptEntryRowToStoredRow));
 }
-function replaceTranscriptRows(sessionKey, rows, opts) {
+function replaceTranscriptRows(sessionKey, rows) {
+	if (rows.some(isRuntimeOnlyTranscriptMessage)) throw new Error("Runtime-only messages cannot be persisted in a session transcript");
 	runSqliteWriteTransaction((db) => {
 		const sessionId = readCurrentSessionId(db, sessionKey);
 		if (!sessionId) throw new Error(`Session not found: ${sessionKey}`);
 		db.prepare(`DELETE FROM transcript_fts WHERE session_id = ?`).run(sessionId);
 		db.prepare(`DELETE FROM transcript_entries WHERE session_id = ?`).run(sessionId);
-		const toWrite = [...rows];
-		if (opts?.appendCompaction) toWrite.push({
-			type: "compaction",
-			...opts.appendCompaction
-		});
-		for (const row of toWrite) insertEntry(db, {
+		for (const row of rows) insertEntry(db, {
 			sessionId,
 			sessionKey,
 			row
 		});
-		const llm = buildSessionContextForLlm(toWrite);
+		const llm = buildSessionContextForLlm(rows);
 		const now = Date.now();
 		const hiddenUpdate = llm.some((message) => message.role === "user") ? `hidden_from_session_list = 0,` : "";
 		db.prepare(`UPDATE sessions SET
@@ -110,14 +140,6 @@ function replaceTranscriptRows(sessionKey, rows, opts) {
       WHERE session_key = ?`).run(llm.length, estimateTokensFromMessages(llm), now, now, now, sessionKey);
 	});
 }
-function loadCheckpointRows(sessionKey, checkpointId) {
-	const db = getSqliteDatabase();
-	if (!db.prepare(`SELECT checkpoint_id FROM compaction_checkpoints
-       WHERE session_key = ? AND checkpoint_id = ?`).get(sessionKey, checkpointId)) return [];
-	return db.prepare(`SELECT payload_json FROM checkpoint_entries
-       WHERE checkpoint_id = ?
-       ORDER BY seq ASC`).all(checkpointId).map((entry) => JSON.parse(entry.payload_json));
-}
 function paginateTranscriptMessages(sessionKey, options = {}) {
 	const sessionId = getCurrentSessionId(sessionKey);
 	if (!sessionId) return {
@@ -171,86 +193,70 @@ function paginateTranscriptMessages(sessionKey, options = {}) {
 		endSeq: rows[rows.length - 1]?.seq ?? 0
 	};
 }
-function captureCompactionCheckpoint(sessionKey) {
-	return runSqliteWriteTransaction((db) => {
-		const sessionId = readCurrentSessionId(db, sessionKey);
-		if (!sessionId) return null;
-		const entries = db.prepare(`SELECT entry_id, session_id, seq, entry_kind, role, payload_json, created_at
-         FROM transcript_entries WHERE session_id = ? ORDER BY seq ASC`).all(sessionId);
-		if (entries.length === 0) return null;
-		const checkpointId = randomUUID();
-		const now = Date.now();
-		const messageCount = entries.filter((e) => e.entry_kind === "message").length;
-		const sizeBytes = entries.reduce((sum, e) => sum + e.payload_json.length, 0);
-		db.prepare(`INSERT INTO compaction_checkpoints
-        (checkpoint_id, session_id, session_key, created_at, message_count, size_bytes)
-       VALUES (?, ?, ?, ?, ?, ?)`).run(checkpointId, sessionId, sessionKey, now, messageCount, sizeBytes);
-		const insertCheckpointEntry = db.prepare(`INSERT INTO checkpoint_entries (checkpoint_id, seq, entry_kind, role, payload_json)
-       VALUES (?, ?, ?, ?, ?)`);
-		for (const entry of entries) insertCheckpointEntry.run(checkpointId, entry.seq, entry.entry_kind, entry.role, entry.payload_json);
-		pruneCompactionCheckpoints(db, sessionId);
-		return checkpointId;
+function listCompactionBoundaries(sessionKey) {
+	const db = getSqliteDatabase();
+	const sessionId = getCurrentSessionId(sessionKey);
+	if (!sessionId) return [];
+	return db.prepare(`SELECT entry_id, seq, payload_json, created_at
+       FROM transcript_entries
+       WHERE session_id = ? AND entry_kind = 'compaction'
+       ORDER BY seq DESC`).all(sessionId).map((row) => {
+		const payload = JSON.parse(row.payload_json);
+		return {
+			id: row.entry_id,
+			seq: row.seq,
+			createdAt: new Date(row.created_at).toISOString(),
+			messageCount: payload.messages.length,
+			tokensBefore: payload.tokensBefore,
+			tokensAfter: payload.tokensAfter,
+			summaryPreview: payload.summary.slice(0, 500),
+			restoredFromCompactionId: payload.restoredFromCompactionId
+		};
 	});
 }
-function pruneCompactionCheckpoints(db, sessionId) {
-	const rows = db.prepare(`SELECT checkpoint_id FROM compaction_checkpoints
-       WHERE session_id = ?
-       ORDER BY created_at ASC`).all(sessionId);
-	if (rows.length <= MAX_CHECKPOINTS_PER_TRANSCRIPT) return;
-	const toDelete = rows.slice(0, rows.length - MAX_CHECKPOINTS_PER_TRANSCRIPT);
-	const del = db.prepare(`DELETE FROM compaction_checkpoints WHERE checkpoint_id = ?`);
-	for (const row of toDelete) del.run(row.checkpoint_id);
-}
-function listCompactionCheckpoints(sessionKey) {
-	return getSqliteDatabase().prepare(`SELECT checkpoint_id, created_at, message_count, size_bytes
-       FROM compaction_checkpoints
-       WHERE session_key = ?
-       ORDER BY created_at DESC`).all(sessionKey).map((row) => ({
-		id: row.checkpoint_id,
-		sizeBytes: row.size_bytes,
-		modifiedAt: new Date(row.created_at).toISOString()
-	}));
-}
-function getCompactionCheckpointDetail(sessionKey, checkpointId) {
-	const row = getSqliteDatabase().prepare(`SELECT checkpoint_id, created_at, message_count, size_bytes
-       FROM compaction_checkpoints
-       WHERE session_key = ? AND checkpoint_id = ?`).get(sessionKey, checkpointId);
-	if (!row) return null;
-	return {
-		id: row.checkpoint_id,
-		sizeBytes: row.size_bytes,
-		modifiedAt: new Date(row.created_at).toISOString(),
-		messageCount: row.message_count
-	};
-}
-function restoreCompactionCheckpoint(sessionKey, checkpointId) {
+function restoreBeforeCompactionBoundary(sessionKey, compactionId) {
 	runSqliteWriteTransaction((db) => {
 		const sessionId = readCurrentSessionId(db, sessionKey);
 		if (!sessionId) throw new Error(`Session not found: ${sessionKey}`);
-		if (!db.prepare(`SELECT checkpoint_id FROM compaction_checkpoints
-         WHERE session_key = ? AND checkpoint_id = ?`).get(sessionKey, checkpointId)) throw new Error(`Checkpoint not found: ${checkpointId}`);
-		const entries = db.prepare(`SELECT seq, entry_kind, role, payload_json
-         FROM checkpoint_entries
-         WHERE checkpoint_id = ?
-         ORDER BY seq ASC`).all(checkpointId);
-		db.prepare(`DELETE FROM transcript_fts WHERE session_id = ?`).run(sessionId);
-		db.prepare(`DELETE FROM transcript_entries WHERE session_id = ?`).run(sessionId);
-		for (const entry of entries) insertEntry(db, {
+		const boundary = db.prepare(`SELECT seq FROM transcript_entries
+         WHERE session_id = ? AND entry_id = ? AND entry_kind = 'compaction'`).get(sessionId, compactionId);
+		if (!boundary) throw new Error(`Compaction boundary not found: ${compactionId}`);
+		const restoredMessages = buildSessionContextForLlm(db.prepare(`SELECT entry_id, session_id, seq, entry_kind, role, payload_json, created_at
+         FROM transcript_entries
+         WHERE session_id = ? AND seq < ?
+         ORDER BY seq ASC`).all(sessionId, boundary.seq).map(transcriptEntryRowToStoredRow));
+		if (restoredMessages.length === 0) throw new Error(`No restorable context exists before compaction boundary: ${compactionId}`);
+		const current = db.prepare(`SELECT estimated_tokens FROM sessions WHERE session_key = ?`).get(sessionKey);
+		const tokensAfter = estimateTokensFromMessages(restoredMessages);
+		const baseSeq = nextSeq(db, sessionId) - 1;
+		insertEntry(db, {
 			sessionId,
 			sessionKey,
-			row: JSON.parse(entry.payload_json),
-			createdAt: Date.now()
+			row: {
+				type: "compaction",
+				at: (/* @__PURE__ */ new Date()).toISOString(),
+				baseSeq,
+				plannerVersion: 2,
+				summaryModelRef: "logical-restore",
+				qualityAudit: "disabled",
+				summary: `Restored the effective context that existed before compaction boundary ${compactionId}.`,
+				messages: restoredMessages,
+				firstKeptIndex: 0,
+				tokensBefore: current.estimated_tokens,
+				tokensAfter,
+				restoredFromCompactionId: compactionId
+			}
 		});
-		const llm = buildSessionContextForLlm(entries.map((e) => JSON.parse(e.payload_json)));
 		const now = Date.now();
 		db.prepare(`UPDATE sessions SET
         message_count = ?,
         estimated_tokens = ?,
+        compacted_count = compacted_count + 1,
         updated_at = ?,
         last_accessed_at = ?,
         last_interaction_at = ?
-      WHERE session_key = ?`).run(llm.length, estimateTokensFromMessages(llm), now, now, now, sessionKey);
+      WHERE session_key = ?`).run(restoredMessages.length, tokensAfter, now, now, now, sessionKey);
 	});
 }
 //#endregion
-export { appendTranscriptEntry, captureCompactionCheckpoint, getCompactionCheckpointDetail, listCompactionCheckpoints, loadCheckpointRows, loadLlmMessagesForSession, loadTranscriptHistoryRowsForSession, loadTranscriptRows, loadTranscriptRowsForSession, paginateTranscriptMessages, replaceTranscriptRows, restoreCompactionCheckpoint };
+export { appendCompactionBoundary, appendTranscriptEntry, listCompactionBoundaries, loadLlmMessagesForSession, loadTranscriptHistoryRowsForSession, loadTranscriptRows, loadTranscriptRowsForSession, paginateTranscriptMessages, replaceTranscriptRows, restoreBeforeCompactionBoundary };

package/dist/src/user-context/config.d.ts

@@ -5,6 +5,39 @@ export declare const UserMemoryModeSchema: z.ZodEnum<{
     readOnly: "readOnly";
     confirmWrite: "confirmWrite";
 }>;
+export declare const DEFAULT_CONTEXT_COMPACTION_POLICY: {
+    enabled: boolean;
+    triggerThreshold: number;
+    reserveTokens: number;
+    minMessagesBeforeCompact: number;
+    keepRecentTokens: number;
+    recentTurnsPreserve: number;
+    summaryMaxTokens: number;
+    summaryChunkTokens: number;
+    summaryTimeoutMs: number;
+    summaryRetries: number;
+    qualityGuard: boolean;
+    minToolResultKeepChars: number;
+    maxActiveTranscriptBytes: number;
+    postCompactionSections: string[];
+};
+export declare const ContextCompactionPolicySchema: z.ZodDefault<z.ZodObject<{
+    enabled: z.ZodDefault<z.ZodBoolean>;
+    triggerThreshold: z.ZodDefault<z.ZodNumber>;
+    reserveTokens: z.ZodDefault<z.ZodNumber>;
+    minMessagesBeforeCompact: z.ZodDefault<z.ZodNumber>;
+    keepRecentTokens: z.ZodDefault<z.ZodNumber>;
+    recentTurnsPreserve: z.ZodDefault<z.ZodNumber>;
+    summaryMaxTokens: z.ZodDefault<z.ZodNumber>;
+    summaryChunkTokens: z.ZodDefault<z.ZodNumber>;
+    summaryTimeoutMs: z.ZodDefault<z.ZodNumber>;
+    summaryRetries: z.ZodDefault<z.ZodNumber>;
+    qualityGuard: z.ZodDefault<z.ZodBoolean>;
+    model: z.ZodOptional<z.ZodString>;
+    minToolResultKeepChars: z.ZodDefault<z.ZodNumber>;
+    maxActiveTranscriptBytes: z.ZodDefault<z.ZodNumber>;
+    postCompactionSections: z.ZodDefault<z.ZodArray<z.ZodString>>;
+}, z.core.$strict>>;
 export declare const UserMemoryConfigSchema: z.ZodDefault<z.ZodObject<{
     mode: z.ZodDefault<z.ZodEnum<{
         off: "off";
@@ -43,7 +76,23 @@ export declare const UserMemoryConfigSchema: z.ZodDefault<z.ZodObject<{
         }>>;
     }, z.core.$strict>>;
     retention: z.ZodOptional<z.ZodObject<{
-        compaction: z.ZodDefault<z.ZodBoolean>;
+        compaction: z.ZodDefault<z.ZodObject<{
+            enabled: z.ZodDefault<z.ZodBoolean>;
+            triggerThreshold: z.ZodDefault<z.ZodNumber>;
+            reserveTokens: z.ZodDefault<z.ZodNumber>;
+            minMessagesBeforeCompact: z.ZodDefault<z.ZodNumber>;
+            keepRecentTokens: z.ZodDefault<z.ZodNumber>;
+            recentTurnsPreserve: z.ZodDefault<z.ZodNumber>;
+            summaryMaxTokens: z.ZodDefault<z.ZodNumber>;
+            summaryChunkTokens: z.ZodDefault<z.ZodNumber>;
+            summaryTimeoutMs: z.ZodDefault<z.ZodNumber>;
+            summaryRetries: z.ZodDefault<z.ZodNumber>;
+            qualityGuard: z.ZodDefault<z.ZodBoolean>;
+            model: z.ZodOptional<z.ZodString>;
+            minToolResultKeepChars: z.ZodDefault<z.ZodNumber>;
+            maxActiveTranscriptBytes: z.ZodDefault<z.ZodNumber>;
+            postCompactionSections: z.ZodDefault<z.ZodArray<z.ZodString>>;
+        }, z.core.$strict>>;
         maxAgeDays: z.ZodOptional<z.ZodNumber>;
         maxItems: z.ZodOptional<z.ZodNumber>;
         maxChars: z.ZodOptional<z.ZodNumber>;
@@ -153,7 +202,23 @@ export declare const UserContextConfigSchema: z.ZodDefault<z.ZodObject<{
             }>>;
         }, z.core.$strict>>;
         retention: z.ZodOptional<z.ZodObject<{
-            compaction: z.ZodDefault<z.ZodBoolean>;
+            compaction: z.ZodDefault<z.ZodObject<{
+                enabled: z.ZodDefault<z.ZodBoolean>;
+                triggerThreshold: z.ZodDefault<z.ZodNumber>;
+                reserveTokens: z.ZodDefault<z.ZodNumber>;
+                minMessagesBeforeCompact: z.ZodDefault<z.ZodNumber>;
+                keepRecentTokens: z.ZodDefault<z.ZodNumber>;
+                recentTurnsPreserve: z.ZodDefault<z.ZodNumber>;
+                summaryMaxTokens: z.ZodDefault<z.ZodNumber>;
+                summaryChunkTokens: z.ZodDefault<z.ZodNumber>;
+                summaryTimeoutMs: z.ZodDefault<z.ZodNumber>;
+                summaryRetries: z.ZodDefault<z.ZodNumber>;
+                qualityGuard: z.ZodDefault<z.ZodBoolean>;
+                model: z.ZodOptional<z.ZodString>;
+                minToolResultKeepChars: z.ZodDefault<z.ZodNumber>;
+                maxActiveTranscriptBytes: z.ZodDefault<z.ZodNumber>;
+                postCompactionSections: z.ZodDefault<z.ZodArray<z.ZodString>>;
+            }, z.core.$strict>>;
             maxAgeDays: z.ZodOptional<z.ZodNumber>;
             maxItems: z.ZodOptional<z.ZodNumber>;
             maxChars: z.ZodOptional<z.ZodNumber>;

package/dist/src/user-context/config.js

@@ -1,7 +1,7 @@
 import { __esmMin } from "../../_virtual/_rolldown/runtime.js";
 import { z } from "zod";
 //#region src/user-context/config.ts
-var UserMemoryModeSchema, UserMemoryConfigSchema, UserUnderstandingConfigSchema, UserContextPrivacySchema, UserContextProviderRoutingSchema, DreamingPhaseSchema, UserContextDreamingSchema, UserContextConfigSchema;
+var UserMemoryModeSchema, DEFAULT_CONTEXT_COMPACTION_POLICY, ContextCompactionPolicySchema, UserMemoryConfigSchema, UserUnderstandingConfigSchema, UserContextPrivacySchema, UserContextProviderRoutingSchema, DreamingPhaseSchema, UserContextDreamingSchema, UserContextConfigSchema;
 var init_config = __esmMin((() => {
 	UserMemoryModeSchema = z.enum([
 		"off",
@@ -9,6 +9,39 @@ var init_config = __esmMin((() => {
 		"confirmWrite",
 		"auto"
 	]);
+	DEFAULT_CONTEXT_COMPACTION_POLICY = {
+		enabled: true,
+		triggerThreshold: .8,
+		reserveTokens: 8192,
+		minMessagesBeforeCompact: 10,
+		keepRecentTokens: 2e4,
+		recentTurnsPreserve: 3,
+		summaryMaxTokens: 2e3,
+		summaryChunkTokens: 24e3,
+		summaryTimeoutMs: 18e4,
+		summaryRetries: 2,
+		qualityGuard: true,
+		minToolResultKeepChars: 1e3,
+		maxActiveTranscriptBytes: 2e6,
+		postCompactionSections: ["Session Startup", "Red Lines"]
+	};
+	ContextCompactionPolicySchema = z.object({
+		enabled: z.boolean().default(true),
+		triggerThreshold: z.number().min(.1).max(.98).default(.8),
+		reserveTokens: z.number().int().min(1024).default(8192),
+		minMessagesBeforeCompact: z.number().int().min(2).default(10),
+		keepRecentTokens: z.number().int().min(1e3).default(2e4),
+		recentTurnsPreserve: z.number().int().min(1).max(12).default(3),
+		summaryMaxTokens: z.number().int().min(256).default(2e3),
+		summaryChunkTokens: z.number().int().min(1e3).default(24e3),
+		summaryTimeoutMs: z.number().int().min(1e3).max(6e5).default(18e4),
+		summaryRetries: z.number().int().min(0).max(5).default(2),
+		qualityGuard: z.boolean().default(true),
+		model: z.string().min(1).optional(),
+		minToolResultKeepChars: z.number().int().min(200).default(1e3),
+		maxActiveTranscriptBytes: z.number().int().min(64e3).default(2e6),
+		postCompactionSections: z.array(z.string().min(1)).max(12).default(["Session Startup", "Red Lines"])
+	}).strict().default(DEFAULT_CONTEXT_COMPACTION_POLICY);
 	UserMemoryConfigSchema = z.object({
 		mode: UserMemoryModeSchema.default("off"),
 		sources: z.array(z.enum([
@@ -42,7 +75,7 @@ var init_config = __esmMin((() => {
 			]).optional()
 		}).strict().optional(),
 		retention: z.object({
-			compaction: z.boolean().default(true),
+			compaction: ContextCompactionPolicySchema,
 			maxAgeDays: z.number().int().positive().optional(),
 			maxItems: z.number().int().positive().optional(),
 			maxChars: z.number().int().positive().optional()
@@ -153,4 +186,4 @@ var init_config = __esmMin((() => {
 }));
 //#endregion
 init_config();
-export { UserContextConfigSchema, UserContextDreamingSchema, UserContextPrivacySchema, UserContextProviderRoutingSchema, UserMemoryConfigSchema, UserMemoryModeSchema, UserUnderstandingConfigSchema, init_config };
+export { ContextCompactionPolicySchema, DEFAULT_CONTEXT_COMPACTION_POLICY, UserContextConfigSchema, UserContextDreamingSchema, UserContextPrivacySchema, UserContextProviderRoutingSchema, UserMemoryConfigSchema, UserMemoryModeSchema, UserUnderstandingConfigSchema, init_config };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xopcai/xopc",
-  "version": "0.0.170",
+  "version": "0.0.171",
   "description": "Local-first AI system that remembers context, coordinates AI, and sustains long-term progress.",
   "type": "module",
   "main": "dist/src/index.js",