@dex-ai/context 0.7.16

package/src/store.ts

@@ -0,0 +1,678 @@
+/**
+ * ContentStore — In-process FTS5 knowledge base for context preservation.
+ *
+ * Inspired by context-mode's approach: instead of truncating/forgetting
+ * large tool results, we index them into a searchable FTS5 database.
+ * The original content is replaced with a compact pointer in context,
+ * and the agent can retrieve specific chunks on demand via search.
+ *
+ * Architecture:
+ * - Uses bun:sqlite for FTS5 full-text search
+ * - Chunks content by structure (markdown headings, paragraphs, JSON keys)
+ * - BM25-ranked search with fuzzy fallback
+ * - In-memory per-session (no persistence across sessions needed)
+ *
+ * Key principle: ZERO INFORMATION LOSS. Content leaves the context window
+ * but never leaves the process — it's always retrievable via search.
+ */
+
+import { Database } from "bun:sqlite";
+
+/* ── Types ─────────────────────────────────────────────── */
+
+export interface Chunk {
+	title: string;
+	content: string;
+	hasCode: boolean;
+}
+
+export interface IndexResult {
+	sourceId: number;
+	label: string;
+	totalChunks: number;
+	codeChunks: number;
+}
+
+export interface SearchResult {
+	title: string;
+	content: string;
+	source: string;
+	rank: number;
+	highlighted?: string;
+}
+
+export interface SourceInfo {
+	id: number;
+	label: string;
+	chunkCount: number;
+	indexedAt: string;
+}
+
+export interface StoreStats {
+	sources: number;
+	chunks: number;
+	totalBytes: number;
+}
+
+/* ── Constants ─────────────────────────────────────────── */
+
+/** Maximum chunk size in bytes. Oversized sections are split at paragraph boundaries. */
+const MAX_CHUNK_BYTES = 4096;
+
+/** Lines per chunk for plain-text (non-markdown) content. */
+const LINES_PER_CHUNK = 50;
+
+/** Stop words filtered from search queries for better BM25 ranking. */
+const STOPWORDS = new Set([
+	"the", "and", "for", "are", "but", "not", "you", "all", "can", "had",
+	"her", "was", "one", "our", "out", "has", "his", "how", "its", "may",
+	"new", "now", "old", "see", "way", "who", "did", "get", "got", "let",
+	"say", "she", "too", "use", "will", "with", "this", "that", "from",
+	"they", "been", "have", "many", "some", "them", "than", "each", "make",
+	"like", "just", "over", "such", "take", "into", "year", "your", "good",
+	"could", "would", "about", "which", "their", "there", "other", "after",
+	"should", "through", "also", "more", "most", "only", "very", "when",
+	"what", "then", "these", "those", "being", "does", "done", "both",
+	"same", "still", "while", "where", "here", "were", "much",
+	"update", "updates", "updated", "deps", "dev", "tests", "test",
+	"add", "added", "fix", "fixed", "run", "running", "using",
+]);
+
+/* ── ContentStore ──────────────────────────────────────── */
+
+export class ContentStore {
+	#db: Database;
+	#insertCount = 0;
+
+	constructor() {
+		// In-memory database — lives for the session only
+		this.#db = new Database(":memory:");
+		this.#db.exec("PRAGMA journal_mode = WAL");
+		this.#db.exec("PRAGMA synchronous = OFF");
+		this.#initSchema();
+	}
+
+	#initSchema(): void {
+		this.#db.exec(`
+			CREATE TABLE IF NOT EXISTS sources (
+				id INTEGER PRIMARY KEY AUTOINCREMENT,
+				label TEXT NOT NULL,
+				content_type TEXT DEFAULT 'prose',
+				indexed_at TEXT DEFAULT (datetime('now')),
+				total_bytes INTEGER DEFAULT 0
+			);
+
+			CREATE TABLE IF NOT EXISTS chunks (
+				id INTEGER PRIMARY KEY AUTOINCREMENT,
+				source_id INTEGER NOT NULL REFERENCES sources(id),
+				title TEXT NOT NULL,
+				content TEXT NOT NULL,
+				has_code INTEGER DEFAULT 0,
+				FOREIGN KEY (source_id) REFERENCES sources(id)
+			);
+
+			CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
+				title,
+				content,
+				content='chunks',
+				content_rowid='id',
+				tokenize='porter unicode61'
+			);
+
+			CREATE TRIGGER IF NOT EXISTS chunks_ai AFTER INSERT ON chunks BEGIN
+				INSERT INTO chunks_fts(rowid, title, content) VALUES (new.id, new.title, new.content);
+			END;
+
+			CREATE TRIGGER IF NOT EXISTS chunks_ad AFTER DELETE ON chunks BEGIN
+				INSERT INTO chunks_fts(chunks_fts, rowid, title, content) VALUES('delete', old.id, old.title, old.content);
+			END;
+		`);
+	}
+
+	/* ── Indexing ──────────────────────────────────────── */
+
+	/**
+	 * Index content into the store. Content is chunked by structure and
+	 * stored in FTS5 for BM25 search.
+	 *
+	 * Returns metadata about what was indexed — this becomes the "pointer"
+	 * that replaces the raw content in the context window.
+	 */
+	index(opts: {
+		content: string;
+		source: string;
+		contentType?: "code" | "prose" | "json";
+	}): IndexResult {
+		const { content, source, contentType } = opts;
+		const detectedType = contentType ?? detectContentType(content);
+
+		// Delete previous source with same label (re-indexing)
+		this.#deleteByLabel(source);
+
+		// Insert source
+		const insertSource = this.#db.prepare(
+			"INSERT INTO sources (label, content_type, total_bytes) VALUES (?, ?, ?)",
+		);
+		const result = insertSource.run(source, detectedType, Buffer.byteLength(content));
+		const sourceId = Number(result.lastInsertRowid);
+
+		// Chunk and insert
+		let chunks: Chunk[];
+		switch (detectedType) {
+			case "code":
+			case "prose":
+				chunks = this.#chunkMarkdown(content);
+				break;
+			case "json":
+				chunks = this.#chunkJSON(content);
+				break;
+			default:
+				chunks = this.#chunkPlainText(content);
+		}
+
+		// Fallback: if no chunks produced, treat as single chunk
+		if (chunks.length === 0) {
+			chunks = [{ title: source, content: content.slice(0, MAX_CHUNK_BYTES * 4), hasCode: false }];
+		}
+
+		const insertChunk = this.#db.prepare(
+			"INSERT INTO chunks (source_id, title, content, has_code) VALUES (?, ?, ?, ?)",
+		);
+		const insertAll = this.#db.transaction((chs: Chunk[]) => {
+			for (const chunk of chs) {
+				insertChunk.run(sourceId, chunk.title, chunk.content, chunk.hasCode ? 1 : 0);
+			}
+		});
+		insertAll(chunks);
+
+		this.#insertCount += chunks.length;
+		if (this.#insertCount > 50) {
+			this.#optimize();
+			this.#insertCount = 0;
+		}
+
+		const codeChunks = chunks.filter((c) => c.hasCode).length;
+		return { sourceId, label: source, totalChunks: chunks.length, codeChunks };
+	}
+
+	/* ── Search ───────────────────────────────────────── */
+
+	/**
+	 * Search indexed content using BM25 ranking.
+	 * Falls back to substring match if FTS5 finds nothing.
+	 */
+	search(query: string, limit = 5, source?: string): SearchResult[] {
+		// Try FTS5 first
+		const ftsQuery = sanitizeQuery(query);
+		let results = this.#searchFTS(ftsQuery, limit, source);
+
+		// Fallback: try OR mode if AND gave nothing
+		if (results.length === 0) {
+			const orQuery = sanitizeQuery(query, "OR");
+			results = this.#searchFTS(orQuery, limit, source);
+		}
+
+		// Final fallback: LIKE-based substring search
+		if (results.length === 0) {
+			results = this.#searchSubstring(query, limit, source);
+		}
+
+		return results;
+	}
+
+	#searchFTS(ftsQuery: string, limit: number, source?: string): SearchResult[] {
+		try {
+			const sql = source
+				? `SELECT c.title, c.content, s.label as source, rank
+				   FROM chunks_fts f
+				   JOIN chunks c ON c.id = f.rowid
+				   JOIN sources s ON s.id = c.source_id
+				   WHERE chunks_fts MATCH ?
+				   AND s.label LIKE ?
+				   ORDER BY rank
+				   LIMIT ?`
+				: `SELECT c.title, c.content, s.label as source, rank
+				   FROM chunks_fts f
+				   JOIN chunks c ON c.id = f.rowid
+				   JOIN sources s ON s.id = c.source_id
+				   WHERE chunks_fts MATCH ?
+				   ORDER BY rank
+				   LIMIT ?`;
+
+			const params = source
+				? [ftsQuery, `%${source}%`, limit]
+				: [ftsQuery, limit];
+
+			const rows = this.#db.prepare(sql).all(...params) as SearchResult[];
+			return rows;
+		} catch {
+			// FTS5 syntax error — fall through to substring
+			return [];
+		}
+	}
+
+	#searchSubstring(query: string, limit: number, source?: string): SearchResult[] {
+		const pattern = `%${query.replace(/[%_]/g, "\\$&")}%`;
+		const sql = source
+			? `SELECT c.title, c.content, s.label as source, 0 as rank
+			   FROM chunks c
+			   JOIN sources s ON s.id = c.source_id
+			   WHERE (c.content LIKE ? ESCAPE '\\' OR c.title LIKE ? ESCAPE '\\')
+			   AND s.label LIKE ?
+			   LIMIT ?`
+			: `SELECT c.title, c.content, s.label as source, 0 as rank
+			   FROM chunks c
+			   JOIN sources s ON s.id = c.source_id
+			   WHERE (c.content LIKE ? ESCAPE '\\' OR c.title LIKE ? ESCAPE '\\')
+			   LIMIT ?`;
+
+		const params = source
+			? [pattern, pattern, `%${source}%`, limit]
+			: [pattern, pattern, limit];
+
+		return this.#db.prepare(sql).all(...params) as SearchResult[];
+	}
+
+	/* ── Metadata ─────────────────────────────────────── */
+
+	listSources(): SourceInfo[] {
+		return this.#db.prepare(`
+			SELECT s.id, s.label, s.indexed_at as indexedAt,
+				   COUNT(c.id) as chunkCount
+			FROM sources s
+			LEFT JOIN chunks c ON c.source_id = s.id
+			GROUP BY s.id
+			ORDER BY s.indexed_at DESC
+		`).all() as SourceInfo[];
+	}
+
+	getStats(): StoreStats {
+		const row = this.#db.prepare(`
+			SELECT
+				(SELECT COUNT(*) FROM sources) as sources,
+				(SELECT COUNT(*) FROM chunks) as chunks,
+				(SELECT COALESCE(SUM(total_bytes), 0) FROM sources) as totalBytes
+		`).get() as StoreStats;
+		return row;
+	}
+
+	isEmpty(): boolean {
+		const row = this.#db.prepare("SELECT COUNT(*) as c FROM chunks").get() as { c: number };
+		return row.c === 0;
+	}
+
+	/* ── Cleanup ──────────────────────────────────────── */
+
+	#deleteByLabel(label: string): void {
+		const source = this.#db.prepare("SELECT id FROM sources WHERE label = ?").get(label) as { id: number } | undefined;
+		if (!source) return;
+
+		this.#db.prepare("DELETE FROM chunks WHERE source_id = ?").run(source.id);
+		this.#db.prepare("DELETE FROM sources WHERE id = ?").run(source.id);
+	}
+
+	#optimize(): void {
+		try {
+			this.#db.exec("INSERT INTO chunks_fts(chunks_fts) VALUES('optimize')");
+		} catch {
+			// Ignore FTS5 optimize errors
+		}
+	}
+
+	dispose(): void {
+		try {
+			this.#db.close();
+		} catch {
+			// Already closed
+		}
+	}
+
+	/* ── Chunking: Markdown ───────────────────────────── */
+
+	#chunkMarkdown(text: string): Chunk[] {
+		const chunks: Chunk[] = [];
+		const lines = text.split("\n");
+		const headingStack: Array<{ level: number; text: string }> = [];
+		let currentContent: string[] = [];
+		let currentHeading = "";
+
+		const flush = () => {
+			const joined = currentContent.join("\n").trim();
+			if (joined.length === 0) return;
+
+			const title = buildTitle(headingStack, currentHeading);
+			const hasCode = currentContent.some((l) => /^`{3,}/.test(l));
+
+			// Under cap: emit as single chunk
+			if (Buffer.byteLength(joined) <= MAX_CHUNK_BYTES) {
+				chunks.push({ title, content: joined, hasCode });
+				currentContent = [];
+				return;
+			}
+
+			// Split oversized chunk at paragraph boundaries
+			const paragraphs = joined.split(/\n\n+/);
+			let accumulator: string[] = [];
+			let partIndex = 1;
+
+			const flushAccumulator = () => {
+				if (accumulator.length === 0) return;
+				const part = accumulator.join("\n\n").trim();
+				if (!part) return;
+				const partTitle = paragraphs.length > 1 ? `${title} (${partIndex})` : title;
+				partIndex++;
+				chunks.push({
+					title: partTitle,
+					content: part,
+					hasCode: part.includes("```"),
+				});
+				accumulator = [];
+			};
+
+			for (const para of paragraphs) {
+				accumulator.push(para);
+				const candidate = accumulator.join("\n\n");
+				if (Buffer.byteLength(candidate) > MAX_CHUNK_BYTES && accumulator.length > 1) {
+					accumulator.pop();
+					flushAccumulator();
+					accumulator = [para];
+				}
+			}
+			flushAccumulator();
+			currentContent = [];
+		};
+
+		let i = 0;
+		while (i < lines.length) {
+			const line = lines[i]!;
+
+			// Horizontal rule
+			if (/^[-_*]{3,}\s*$/.test(line)) {
+				flush();
+				i++;
+				continue;
+			}
+
+			// Heading
+			const headingMatch = line.match(/^(#{1,4})\s+(.+)$/);
+			if (headingMatch) {
+				flush();
+				const level = headingMatch[1]!.length;
+				const heading = headingMatch[2]!.trim();
+
+				while (headingStack.length > 0 && headingStack[headingStack.length - 1]!.level >= level) {
+					headingStack.pop();
+				}
+				headingStack.push({ level, text: heading });
+				currentHeading = heading;
+				currentContent.push(line);
+				i++;
+				continue;
+			}
+
+			// Code block — collect entire block
+			const codeMatch = line.match(/^(`{3,})(.*)?$/);
+			if (codeMatch) {
+				const fence = codeMatch[1]!;
+				const codeLines: string[] = [line];
+				i++;
+				while (i < lines.length) {
+					codeLines.push(lines[i]!);
+					if (lines[i]!.startsWith(fence) && lines[i]!.trim() === fence) {
+						i++;
+						break;
+					}
+					i++;
+				}
+				currentContent.push(...codeLines);
+				continue;
+			}
+
+			// Regular line
+			currentContent.push(line);
+			i++;
+		}
+
+		flush();
+		return chunks;
+	}
+
+	/* ── Chunking: Plain Text ─────────────────────────── */
+
+	#chunkPlainText(text: string): Chunk[] {
+		// Try paragraph splitting first
+		const sections = text.split(/\n\s*\n/);
+		if (
+			sections.length >= 3 &&
+			sections.length <= 200 &&
+			sections.every((s) => Buffer.byteLength(s) < 5000)
+		) {
+			return sections
+				.map((section, i) => {
+					const trimmed = section.trim();
+					const firstLine = trimmed.split("\n")[0]!.slice(0, 80);
+					return {
+						title: firstLine || `Section ${i + 1}`,
+						content: trimmed,
+						hasCode: trimmed.includes("```") || /^\s*(function|class|const|let|var|import|export|def|fn)\s/.test(trimmed),
+					};
+				})
+				.filter((s) => s.content.length > 0);
+		}
+
+		// Fall back to line-group chunks
+		const lines = text.split("\n");
+		if (lines.length <= LINES_PER_CHUNK) {
+			return [{ title: "Output", content: text, hasCode: false }];
+		}
+
+		const chunks: Chunk[] = [];
+		const overlap = 2;
+		const step = Math.max(LINES_PER_CHUNK - overlap, 1);
+
+		for (let i = 0; i < lines.length; i += step) {
+			const slice = lines.slice(i, i + LINES_PER_CHUNK);
+			if (slice.length === 0) break;
+			const firstLine = slice[0]?.trim().slice(0, 80);
+			chunks.push({
+				title: firstLine || `Lines ${i + 1}-${i + slice.length}`,
+				content: slice.join("\n"),
+				hasCode: false,
+			});
+		}
+
+		return chunks;
+	}
+
+	/* ── Chunking: JSON ───────────────────────────────── */
+
+	#chunkJSON(text: string): Chunk[] {
+		try {
+			const parsed = JSON.parse(text);
+			const chunks: Chunk[] = [];
+			this.#walkJSON(parsed, [], chunks);
+			return chunks;
+		} catch {
+			// Not valid JSON — treat as plain text
+			return this.#chunkPlainText(text);
+		}
+	}
+
+	#walkJSON(value: unknown, path: string[], chunks: Chunk[]): void {
+		const title = path.length > 0 ? path.join(" > ") : "(root)";
+		const serialized = JSON.stringify(value, null, 2);
+
+		// Small enough — emit as single chunk
+		if (Buffer.byteLength(serialized) <= MAX_CHUNK_BYTES) {
+			// Recurse into nested objects for better searchability
+			const shouldRecurse =
+				typeof value === "object" &&
+				value !== null &&
+				!Array.isArray(value) &&
+				Object.values(value).some((v) => typeof v === "object" && v !== null);
+
+			if (!shouldRecurse) {
+				chunks.push({ title, content: serialized, hasCode: true });
+				return;
+			}
+		}
+
+		// Object — recurse
+		if (typeof value === "object" && value !== null && !Array.isArray(value)) {
+			const entries = Object.entries(value);
+			if (entries.length > 0) {
+				for (const [key, val] of entries) {
+					this.#walkJSON(val, [...path, key], chunks);
+				}
+				return;
+			}
+			chunks.push({ title, content: serialized, hasCode: true });
+			return;
+		}
+
+		// Array — batch by size
+		if (Array.isArray(value)) {
+			this.#chunkJSONArray(value, path, chunks);
+			return;
+		}
+
+		// Primitive
+		chunks.push({ title, content: serialized, hasCode: false });
+	}
+
+	#chunkJSONArray(arr: unknown[], path: string[], chunks: Chunk[]): void {
+		const prefix = path.length > 0 ? path.join(" > ") : "(root)";
+
+		// Find identity field for better titles
+		const identityField = findIdentityField(arr);
+
+		let batch: unknown[] = [];
+		let batchStart = 0;
+
+		const flushBatch = (batchEnd: number) => {
+			if (batch.length === 0) return;
+			const title = jsonBatchTitle(prefix, batchStart, batchEnd, batch, identityField);
+			chunks.push({
+				title,
+				content: JSON.stringify(batch, null, 2),
+				hasCode: true,
+			});
+		};
+
+		for (let i = 0; i < arr.length; i++) {
+			batch.push(arr[i]);
+			const candidate = JSON.stringify(batch, null, 2);
+
+			if (Buffer.byteLength(candidate) > MAX_CHUNK_BYTES && batch.length > 1) {
+				batch.pop();
+				flushBatch(i - 1);
+				batch = [arr[i]];
+				batchStart = i;
+			}
+		}
+
+		flushBatch(batchStart + batch.length - 1);
+	}
+}
+
+/* ── Helpers ───────────────────────────────────────────── */
+
+function buildTitle(
+	headingStack: Array<{ level: number; text: string }>,
+	currentHeading: string,
+): string {
+	if (headingStack.length === 0) {
+		return currentHeading || "Untitled";
+	}
+	return headingStack.map((h) => h.text).join(" > ");
+}
+
+function detectContentType(content: string): "code" | "prose" | "json" {
+	const trimmed = content.trimStart();
+	if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
+		try {
+			JSON.parse(content);
+			return "json";
+		} catch { /* not JSON */ }
+	}
+	// Check if markdown/code-heavy
+	const sample = content.slice(0, 2000);
+	const codeIndicators = (sample.match(/```|^(import|export|function|class|const|let|var|def|fn|pub|use)\s/gm) || []).length;
+	const headingIndicators = (sample.match(/^#{1,4}\s/gm) || []).length;
+	if (codeIndicators > 3 || headingIndicators > 2) return "code";
+	return "prose";
+}
+
+/**
+ * Sanitize a user query for FTS5 MATCH syntax.
+ * Removes special characters and joins terms with AND/OR.
+ */
+export function sanitizeQuery(query: string, mode: "AND" | "OR" = "AND"): string {
+	const words = query
+		.replace(/['"(){}[\]*:^~]/g, " ")
+		.split(/\s+/)
+		.filter(
+			(w) => w.length > 0 && !["AND", "OR", "NOT", "NEAR"].includes(w.toUpperCase()),
+		);
+
+	if (words.length === 0) return '""';
+
+	// Filter stopwords for better ranking
+	const meaningful = words.filter((w) => !STOPWORDS.has(w.toLowerCase()));
+	const final = meaningful.length > 0 ? meaningful : words;
+
+	// Deduplicate (case-insensitive)
+	const seen = new Set<string>();
+	const deduped: string[] = [];
+	for (const w of final) {
+		const key = w.toLowerCase();
+		if (!seen.has(key)) {
+			seen.add(key);
+			deduped.push(w);
+		}
+	}
+
+	return deduped.map((w) => `"${w}"`).join(mode === "OR" ? " OR " : " ");
+}
+
+function findIdentityField(arr: unknown[]): string | null {
+	if (arr.length === 0) return null;
+	const first = arr[0];
+	if (typeof first !== "object" || first === null || Array.isArray(first)) return null;
+
+	const candidates = ["id", "name", "title", "path", "slug", "key", "label"];
+	const obj = first as Record<string, unknown>;
+	for (const field of candidates) {
+		if (field in obj && (typeof obj[field] === "string" || typeof obj[field] === "number")) {
+			return field;
+		}
+	}
+	return null;
+}
+
+function jsonBatchTitle(
+	prefix: string,
+	startIdx: number,
+	endIdx: number,
+	batch: unknown[],
+	identityField: string | null,
+): string {
+	const sep = prefix ? `${prefix} > ` : "";
+
+	if (!identityField) {
+		return startIdx === endIdx
+			? `${sep}[${startIdx}]`
+			: `${sep}[${startIdx}-${endIdx}]`;
+	}
+
+	const getId = (item: unknown) =>
+		String((item as Record<string, unknown>)[identityField]);
+
+	if (batch.length === 1) {
+		return `${sep}${getId(batch[0])}`;
+	}
+	if (batch.length <= 3) {
+		return sep + batch.map(getId).join(", ");
+	}
+	return `${sep}${getId(batch[0])}…${getId(batch[batch.length - 1])}`;
+}