@operor/knowledge 0.1.0

package/dist/index.js

@@ -0,0 +1,1442 @@
+import { embed, embedMany, generateText } from "ai";
+import { createOpenAI } from "@ai-sdk/openai";
+import { createGoogleGenerativeAI } from "@ai-sdk/google";
+import { mistral } from "@ai-sdk/mistral";
+import { cohere } from "@ai-sdk/cohere";
+import { statSync } from "node:fs";
+import Database from "better-sqlite3";
+import * as sqliteVec from "sqlite-vec";
+import { MarkdownTextSplitter, RecursiveCharacterTextSplitter } from "@langchain/textsplitters";
+import { createHash, randomUUID } from "node:crypto";
+import { Readability } from "@mozilla/readability";
+import { parseHTML } from "linkedom";
+import { readFile } from "node:fs/promises";
+import { extname } from "node:path";
+
+//#region src/EmbeddingService.ts
+var EmbeddingService = class EmbeddingService {
+	config;
+	constructor(config) {
+		this.config = config;
+	}
+	getModel() {
+		const { provider, apiKey, baseURL, model } = this.config;
+		switch (provider) {
+			case "openai": return createOpenAI({
+				apiKey,
+				baseURL
+			}).embedding(model || "text-embedding-3-small", { dimensions: this.config.dimensions });
+			case "google": return createGoogleGenerativeAI({
+				apiKey,
+				baseURL
+			}).textEmbeddingModel(model || "text-embedding-004");
+			case "mistral": return mistral.embedding(model || "mistral-embed", { apiKey });
+			case "cohere": return cohere.embedding(model || "embed-english-v3.0", { apiKey });
+			case "ollama": return createOpenAI({
+				apiKey: apiKey || "ollama",
+				baseURL: baseURL || "http://localhost:11434/v1"
+			}).embedding(model || "nomic-embed-text");
+			default: throw new Error(`Unknown embedding provider: ${provider}`);
+		}
+	}
+	get provider() {
+		return this.config.provider;
+	}
+	get dimensions() {
+		if (this.config.dimensions) return this.config.dimensions;
+		return EmbeddingService.defaultDimensions(this.config.provider, this.config.model);
+	}
+	static defaultDimensions(provider, model) {
+		switch (provider) {
+			case "openai": return 1536;
+			case "google": return 768;
+			case "mistral": return 1024;
+			case "cohere": return 1024;
+			case "ollama": return 768;
+			default: return 1536;
+		}
+	}
+	async embed(text) {
+		return (await embed({
+			model: this.getModel(),
+			value: text
+		})).embedding;
+	}
+	async embedMany(texts) {
+		if (texts.length === 0) return [];
+		return (await embedMany({
+			model: this.getModel(),
+			values: texts
+		})).embeddings;
+	}
+};
+
+//#endregion
+//#region src/SQLiteKnowledgeStore.ts
+var SQLiteKnowledgeStore = class {
+	db;
+	dbPath;
+	dimensions;
+	dimensionWarned = false;
+	constructor(dbPath = "./knowledge.db", dimensions = 1536) {
+		this.db = new Database(dbPath);
+		this.dbPath = dbPath;
+		this.dimensions = dimensions;
+		this.db.pragma("journal_mode = WAL");
+		this.db.pragma("foreign_keys = ON");
+		sqliteVec.load(this.db);
+	}
+	getDimensions() {
+		return this.dimensions;
+	}
+	async initialize() {
+		this.db.exec(`
+      CREATE TABLE IF NOT EXISTS kb_documents (
+        id TEXT PRIMARY KEY,
+        source_type TEXT NOT NULL,
+        source_url TEXT,
+        file_name TEXT,
+        title TEXT,
+        content TEXT NOT NULL,
+        metadata TEXT,
+        created_at INTEGER NOT NULL,
+        updated_at INTEGER NOT NULL,
+        priority INTEGER DEFAULT 2,
+        content_hash TEXT
+      );
+
+      CREATE TABLE IF NOT EXISTS kb_chunks (
+        id TEXT PRIMARY KEY,
+        document_id TEXT NOT NULL,
+        content TEXT NOT NULL,
+        chunk_index INTEGER NOT NULL,
+        metadata TEXT,
+        FOREIGN KEY (document_id) REFERENCES kb_documents(id) ON DELETE CASCADE
+      );
+      CREATE INDEX IF NOT EXISTS idx_chunks_document ON kb_chunks(document_id);
+      CREATE INDEX IF NOT EXISTS idx_documents_source_url ON kb_documents(source_url);
+    `);
+		try {
+			this.db.exec("ALTER TABLE kb_documents ADD COLUMN priority INTEGER DEFAULT 2");
+		} catch {}
+		try {
+			this.db.exec("ALTER TABLE kb_documents ADD COLUMN content_hash TEXT");
+		} catch {}
+		try {
+			this.db.exec("CREATE INDEX IF NOT EXISTS idx_documents_source_url ON kb_documents(source_url)");
+		} catch {}
+		this.db.exec(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks USING vec0(
+        chunk_id TEXT PRIMARY KEY,
+        embedding float[${this.dimensions}]
+      );
+    `);
+		this.db.exec(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS fts_chunks USING fts5(
+        chunk_id UNINDEXED,
+        content,
+        tokenize='porter unicode61'
+      );
+    `);
+	}
+	async close() {
+		this.db.close();
+	}
+	async addDocument(doc) {
+		this.db.prepare(`
+      INSERT OR REPLACE INTO kb_documents (id, source_type, source_url, file_name, title, content, metadata, created_at, updated_at, priority, content_hash)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(doc.id, doc.sourceType, doc.sourceUrl || null, doc.fileName || null, doc.title || null, doc.content, doc.metadata ? JSON.stringify(doc.metadata) : null, doc.createdAt, doc.updatedAt, doc.priority ?? 2, doc.contentHash || null);
+	}
+	async getDocument(id) {
+		const row = this.db.prepare("SELECT * FROM kb_documents WHERE id = ?").get(id);
+		return row ? this.rowToDocument(row) : null;
+	}
+	async listDocuments() {
+		return this.db.prepare("SELECT * FROM kb_documents ORDER BY created_at DESC").all().map((r) => this.rowToDocument(r));
+	}
+	async deleteDocument(id) {
+		const chunks = this.db.prepare("SELECT id FROM kb_chunks WHERE document_id = ?").all(id);
+		for (const chunk of chunks) {
+			this.db.prepare("DELETE FROM vec_chunks WHERE chunk_id = ?").run(chunk.id);
+			this.db.prepare("DELETE FROM fts_chunks WHERE chunk_id = ?").run(chunk.id);
+		}
+		this.db.prepare("DELETE FROM kb_chunks WHERE document_id = ?").run(id);
+		this.db.prepare("DELETE FROM kb_documents WHERE id = ?").run(id);
+	}
+	async addChunks(chunks) {
+		const insertChunk = this.db.prepare(`
+      INSERT OR REPLACE INTO kb_chunks (id, document_id, content, chunk_index, metadata)
+      VALUES (?, ?, ?, ?, ?)
+    `);
+		const insertVec = this.db.prepare(`
+      INSERT OR REPLACE INTO vec_chunks (chunk_id, embedding)
+      VALUES (?, ?)
+    `);
+		const deleteFts = this.db.prepare(`
+      DELETE FROM fts_chunks WHERE chunk_id = ?
+    `);
+		const insertFts = this.db.prepare(`
+      INSERT INTO fts_chunks (chunk_id, content)
+      VALUES (?, ?)
+    `);
+		this.db.transaction((items) => {
+			for (const chunk of items) {
+				insertChunk.run(chunk.id, chunk.documentId, chunk.content, chunk.chunkIndex, chunk.metadata ? JSON.stringify(chunk.metadata) : null);
+				if (chunk.embedding) {
+					if (chunk.embedding.length !== this.dimensions && !this.dimensionWarned) {
+						this.dimensionWarned = true;
+						console.warn(`[KB] Dimension mismatch: store expects ${this.dimensions}d vectors but received ${chunk.embedding.length}d. This will cause search errors. Re-ingest your documents after switching embedding providers, or set the correct dimensions when creating the store.`);
+					}
+					insertVec.run(chunk.id, new Float32Array(chunk.embedding));
+				}
+				deleteFts.run(chunk.id);
+				insertFts.run(chunk.id, chunk.content);
+			}
+		})(chunks);
+	}
+	getChunkCount(documentId) {
+		return this.db.prepare("SELECT COUNT(*) as count FROM kb_chunks WHERE document_id = ?").get(documentId)?.count || 0;
+	}
+	async search(query, embedding, options) {
+		return this.searchByEmbedding(embedding, options);
+	}
+	async searchByEmbedding(embedding, options) {
+		const limit = options?.limit || 5;
+		const fetchLimit = options?.sourceTypes ? Math.min(limit * 10, 100) : limit;
+		const vecRows = this.db.prepare(`
+      SELECT chunk_id, distance
+      FROM vec_chunks
+      WHERE embedding MATCH ?
+      ORDER BY distance
+      LIMIT ?
+    `).all(new Float32Array(embedding), fetchLimit);
+		if (vecRows.length === 0) return [];
+		const results = [];
+		for (const vecRow of vecRows) {
+			const distance = vecRow.distance;
+			const score = 1 / (1 + distance);
+			if (options?.scoreThreshold && score < options.scoreThreshold) continue;
+			const chunk = this.db.prepare("SELECT * FROM kb_chunks WHERE id = ?").get(vecRow.chunk_id);
+			if (!chunk) continue;
+			const doc = this.db.prepare("SELECT * FROM kb_documents WHERE id = ?").get(chunk.document_id);
+			if (!doc) continue;
+			if (options?.sourceTypes && !options.sourceTypes.includes(doc.source_type)) continue;
+			results.push({
+				chunk: {
+					id: chunk.id,
+					documentId: chunk.document_id,
+					content: chunk.content,
+					chunkIndex: chunk.chunk_index,
+					metadata: chunk.metadata ? JSON.parse(chunk.metadata) : void 0
+				},
+				document: this.rowToDocument(doc),
+				score,
+				distance
+			});
+		}
+		return results.slice(0, limit);
+	}
+	async searchByKeyword(query, options) {
+		const limit = options?.limit || 5;
+		const safeQuery = query.replace(/"/g, "\"\"");
+		let ftsRows;
+		try {
+			ftsRows = this.db.prepare(`
+        SELECT chunk_id, rank
+        FROM fts_chunks
+        WHERE fts_chunks MATCH ?
+        ORDER BY rank
+        LIMIT ?
+      `).all(safeQuery, limit * 2);
+		} catch {
+			return [];
+		}
+		if (ftsRows.length === 0) return [];
+		const results = [];
+		for (const ftsRow of ftsRows) {
+			if (results.length >= limit) break;
+			const chunk = this.db.prepare("SELECT * FROM kb_chunks WHERE id = ?").get(ftsRow.chunk_id);
+			if (!chunk) continue;
+			const doc = this.db.prepare("SELECT * FROM kb_documents WHERE id = ?").get(chunk.document_id);
+			if (!doc) continue;
+			if (options?.sourceTypes && !options.sourceTypes.includes(doc.source_type)) continue;
+			const bm25Score = -ftsRow.rank;
+			if (options?.scoreThreshold && bm25Score < options.scoreThreshold) continue;
+			results.push({
+				chunk: {
+					id: chunk.id,
+					documentId: chunk.document_id,
+					content: chunk.content,
+					chunkIndex: chunk.chunk_index,
+					metadata: chunk.metadata ? JSON.parse(chunk.metadata) : void 0
+				},
+				document: this.rowToDocument(doc),
+				score: bm25Score,
+				distance: 0
+			});
+		}
+		return results;
+	}
+	/**
+	* Get all chunks from kb_chunks (text content only, no embeddings).
+	* Used by rebuild to re-embed all content.
+	*/
+	getAllChunks() {
+		return this.db.prepare("SELECT id, document_id AS documentId, content, chunk_index AS chunkIndex, metadata FROM kb_chunks ORDER BY document_id, chunk_index").all();
+	}
+	/**
+	* Drop and recreate the vec_chunks virtual table with new dimensions.
+	* Preserves kb_chunks, kb_documents, and fts_chunks — only vector data is affected.
+	*/
+	rebuildVecTable(newDimensions) {
+		this.db.exec("DROP TABLE IF EXISTS vec_chunks");
+		this.db.exec(`
+      CREATE VIRTUAL TABLE vec_chunks USING vec0(
+        chunk_id TEXT PRIMARY KEY,
+        embedding float[${newDimensions}]
+      );
+    `);
+		this.dimensions = newDimensions;
+		this.dimensionWarned = false;
+	}
+	/**
+	* Batch-insert embeddings into vec_chunks.
+	* Expects an array of { chunkId, embedding } pairs.
+	*/
+	batchInsertEmbeddings(items) {
+		const insert = this.db.prepare("INSERT OR REPLACE INTO vec_chunks (chunk_id, embedding) VALUES (?, ?)");
+		this.db.transaction((batch) => {
+			for (const item of batch) insert.run(item.chunkId, new Float32Array(item.embedding));
+		})(items);
+	}
+	async getStats() {
+		const docCount = this.db.prepare("SELECT COUNT(*) as count FROM kb_documents").get();
+		const chunkCount = this.db.prepare("SELECT COUNT(*) as count FROM kb_chunks").get();
+		let dbSizeBytes = 0;
+		try {
+			dbSizeBytes = statSync(this.dbPath).size;
+		} catch {}
+		return {
+			documentCount: docCount.count,
+			chunkCount: chunkCount.count,
+			embeddingDimensions: this.dimensions,
+			dbSizeBytes
+		};
+	}
+	async findBySourceUrl(url) {
+		const row = this.db.prepare("SELECT * FROM kb_documents WHERE source_url = ?").get(url);
+		return row ? this.rowToDocument(row) : null;
+	}
+	async findByContentHash(hash) {
+		const row = this.db.prepare("SELECT * FROM kb_documents WHERE content_hash = ?").get(hash);
+		return row ? this.rowToDocument(row) : null;
+	}
+	async updateDocument(id, updates) {
+		const sets = [];
+		const values = [];
+		if (updates.content !== void 0) {
+			sets.push("content = ?");
+			values.push(updates.content);
+		}
+		if (updates.title !== void 0) {
+			sets.push("title = ?");
+			values.push(updates.title);
+		}
+		if (updates.contentHash !== void 0) {
+			sets.push("content_hash = ?");
+			values.push(updates.contentHash);
+		}
+		if (updates.priority !== void 0) {
+			sets.push("priority = ?");
+			values.push(updates.priority);
+		}
+		if (updates.metadata !== void 0) {
+			sets.push("metadata = ?");
+			values.push(JSON.stringify(updates.metadata));
+		}
+		sets.push("updated_at = ?");
+		values.push(Date.now());
+		values.push(id);
+		this.db.prepare(`UPDATE kb_documents SET ${sets.join(", ")} WHERE id = ?`).run(...values);
+	}
+	async findSimilarFaq(embedding, threshold) {
+		const results = await this.searchByEmbedding(embedding, {
+			sourceTypes: ["faq"],
+			limit: 1
+		});
+		if (results.length > 0 && results[0].score >= threshold) return results[0];
+		return null;
+	}
+	rowToDocument(row) {
+		return {
+			id: row.id,
+			sourceType: row.source_type,
+			sourceUrl: row.source_url || void 0,
+			fileName: row.file_name || void 0,
+			title: row.title || void 0,
+			content: row.content,
+			metadata: row.metadata ? JSON.parse(row.metadata) : void 0,
+			createdAt: row.created_at,
+			updatedAt: row.updated_at,
+			priority: row.priority ?? 2,
+			contentHash: row.content_hash || void 0
+		};
+	}
+};
+
+//#endregion
+//#region src/TextChunker.ts
+var TextChunker = class {
+	defaultChunkSize;
+	defaultChunkOverlap;
+	constructor(options) {
+		this.defaultChunkSize = options?.chunkSize || 3200;
+		this.defaultChunkOverlap = options?.chunkOverlap || 200;
+	}
+	async chunk(text, options) {
+		return (await new RecursiveCharacterTextSplitter({
+			chunkSize: options?.chunkSize || this.defaultChunkSize,
+			chunkOverlap: options?.chunkOverlap || this.defaultChunkOverlap
+		}).createDocuments([text])).map((d) => d.pageContent);
+	}
+	async chunkMarkdown(markdown, options) {
+		return (await new MarkdownTextSplitter({
+			chunkSize: options?.chunkSize || this.defaultChunkSize,
+			chunkOverlap: options?.chunkOverlap || this.defaultChunkOverlap
+		}).createDocuments([markdown])).map((d) => d.pageContent);
+	}
+};
+
+//#endregion
+//#region src/QueryNormalizer.ts
+/**
+* Query normalization for improved KB retrieval.
+* Expands chat abbreviations and normalizes whitespace before embedding.
+*/
+const ABBREVIATIONS = [
+	[/\bu\b/gi, "you"],
+	[/\bur\b/gi, "your"],
+	[/\br\b/gi, "are"],
+	[/\bpls\b/gi, "please"],
+	[/\bplz\b/gi, "please"],
+	[/\bthx\b/gi, "thanks"],
+	[/\bthnx\b/gi, "thanks"],
+	[/\bty\b/gi, "thank you"],
+	[/\bwat\b/gi, "what"],
+	[/\bbc\b/gi, "because"],
+	[/\bcuz\b/gi, "because"],
+	[/\bgonna\b/gi, "going to"],
+	[/\bwanna\b/gi, "want to"],
+	[/\bgotta\b/gi, "got to"],
+	[/\blemme\b/gi, "let me"],
+	[/\bgimme\b/gi, "give me"],
+	[/\bdunno\b/gi, "do not know"],
+	[/\bhrs\b/gi, "hours"],
+	[/\bmins\b/gi, "minutes"],
+	[/\bmsg\b/gi, "message"],
+	[/\bmsgs\b/gi, "messages"],
+	[/\binfo\b/gi, "information"],
+	[/\btmr\b/gi, "tomorrow"],
+	[/\btmrw\b/gi, "tomorrow"],
+	[/\bw\/\b/gi, "with"],
+	[/\bw\/o\b/gi, "without"],
+	[/\bidk\b/gi, "I do not know"],
+	[/\bimo\b/gi, "in my opinion"],
+	[/\bbtw\b/gi, "by the way"],
+	[/\basap\b/gi, "as soon as possible"],
+	[/\b4\b/g, "for"],
+	[/\b2\b/g, "to"]
+];
+/**
+* Normalize a user query for better embedding similarity.
+* - Expands chat abbreviations with word-boundary awareness
+* - Lowercases
+* - Collapses whitespace
+*/
+function normalizeQuery(query) {
+	let normalized = query.toLowerCase();
+	for (const [pattern, replacement] of ABBREVIATIONS) normalized = normalized.replace(pattern, replacement);
+	normalized = normalized.replace(/\s+/g, " ").trim();
+	return normalized;
+}
+
+//#endregion
+//#region src/IngestionPipeline.ts
+var IngestionPipeline = class {
+	store;
+	embedder;
+	chunker;
+	llmProvider;
+	constructor(store, embedder, chunker, llmProvider) {
+		this.store = store;
+		this.embedder = embedder;
+		this.chunker = chunker;
+		this.llmProvider = llmProvider;
+	}
+	cleanContent(text) {
+		return text.replace(/!\[.*?\]\(.*?\)/g, "").replace(/[ \t]+/g, " ").replace(/(\n\s*){3,}/g, "\n\n").split("\n").filter((line, i, arr) => i === 0 || line !== arr[i - 1]).join("\n").trim().slice(0, 15e3);
+	}
+	async extractQAPairs(content, title) {
+		const cleaned = this.cleanContent(content);
+		const titleHint = title ? `\nPage title: "${title}"` : "";
+		const response = await this.llmProvider.complete([{
+			role: "system",
+			content: `You extract self-contained Q&A pairs from web page content. Each answer must include ALL relevant details (names, numbers, prices, dates) so it can be understood without the original page. Output ONLY a JSON array of {"question":"...","answer":"..."} objects. No markdown fences.`
+		}, {
+			role: "user",
+			content: `Extract Q&A pairs from this content.${titleHint}\n\n${cleaned}`
+		}]);
+		try {
+			const text = response.text.replace(/^```(?:json)?\s*/m, "").replace(/\s*```\s*$/m, "").trim();
+			const parsed = JSON.parse(text);
+			if (Array.isArray(parsed)) return parsed.filter((p) => p.question && p.answer);
+		} catch {
+			const match = response.text.match(/\[[\s\S]*\]/);
+			if (match) try {
+				const parsed = JSON.parse(match[0]);
+				if (Array.isArray(parsed)) return parsed.filter((p) => p.question && p.answer);
+			} catch {}
+		}
+		return [];
+	}
+	computeHash(content) {
+		return createHash("sha256").update(content).digest("hex");
+	}
+	async ingest(input) {
+		if (!input.content || input.content.trim().length === 0) throw new Error(`No content to ingest for "${input.title || input.sourceUrl || "unknown"}"`);
+		const sqliteStore = this.store;
+		const cleaned = this.cleanContent(input.content);
+		const contentHash = this.computeHash(cleaned);
+		const priority = input.priority ?? (input.sourceType === "faq" ? 1 : 2);
+		if (input.sourceUrl && sqliteStore.findBySourceUrl) {
+			const existing = await sqliteStore.findBySourceUrl(input.sourceUrl);
+			if (existing) {
+				await sqliteStore.updateDocument(existing.id, {
+					content: input.content,
+					title: input.title,
+					contentHash,
+					priority,
+					metadata: input.metadata
+				});
+				await this.store.deleteDocument(existing.id);
+			}
+		}
+		if (sqliteStore.findByContentHash) {
+			const existing = await sqliteStore.findByContentHash(contentHash);
+			if (existing) {
+				console.log(`[KB] Duplicate content detected (hash match), skipping: "${input.title || input.sourceUrl || "unknown"}"`);
+				return existing;
+			}
+		}
+		if (input.extractQA && this.llmProvider) {
+			const pairs = await this.extractQAPairs(input.content, input.title);
+			if (pairs.length > 0) {
+				for (const pair of pairs) await this.ingestFaq(pair.question, pair.answer, { sourceUrl: input.sourceUrl });
+				const now = Date.now();
+				const parentDoc = {
+					id: randomUUID(),
+					sourceType: input.sourceType,
+					sourceUrl: input.sourceUrl,
+					fileName: input.fileName,
+					title: input.title,
+					content: `Extracted ${pairs.length} Q&A pairs`,
+					metadata: {
+						...input.metadata,
+						faqCount: pairs.length
+					},
+					createdAt: now,
+					updatedAt: now,
+					priority,
+					contentHash
+				};
+				await this.store.addDocument(parentDoc);
+				return parentDoc;
+			}
+		}
+		const now = Date.now();
+		const doc = {
+			id: randomUUID(),
+			sourceType: input.sourceType,
+			sourceUrl: input.sourceUrl,
+			fileName: input.fileName,
+			title: input.title,
+			content: input.content,
+			metadata: input.metadata,
+			createdAt: now,
+			updatedAt: now,
+			priority,
+			contentHash
+		};
+		await this.store.addDocument(doc);
+		const texts = input.isMarkdown || input.sourceType === "url" ? await this.chunker.chunkMarkdown(input.content) : await this.chunker.chunk(input.content);
+		const embeddings = await this.embedder.embedMany(texts);
+		const chunks = texts.map((text, i) => ({
+			id: randomUUID(),
+			documentId: doc.id,
+			content: text,
+			chunkIndex: i,
+			embedding: embeddings[i],
+			metadata: input.metadata
+		}));
+		await this.store.addChunks(chunks);
+		if (this.store.getChunkCount) {
+			if (this.store.getChunkCount(doc.id) === 0) console.warn(`[KB] WARNING: Document "${input.title || doc.id}" was saved but NO vector embeddings were stored.`);
+		}
+		return doc;
+	}
+	async ingestFaq(question, answer, metadata) {
+		const embedding = await this.embedder.embed(normalizeQuery(question));
+		const sqliteStore = this.store;
+		if (sqliteStore.findSimilarFaq && !metadata?.forceReplace) {
+			const match = await sqliteStore.findSimilarFaq(embedding, .9);
+			if (match) {
+				const existingQ = match.chunk.metadata?.question || match.document.title;
+				const existingA = match.chunk.metadata?.answer;
+				const now = Date.now();
+				const content = `Q: ${question}\nA: ${answer}`;
+				return {
+					id: randomUUID(),
+					sourceType: "faq",
+					sourceUrl: metadata?.sourceUrl,
+					title: question,
+					content,
+					metadata: {
+						...metadata,
+						question,
+						answer
+					},
+					priority: 1,
+					createdAt: now,
+					updatedAt: now,
+					existingMatch: {
+						id: match.document.id,
+						question: existingQ,
+						answer: existingA,
+						score: match.score
+					}
+				};
+			}
+		}
+		if (metadata?.forceReplace && metadata?.replaceId) await this.store.deleteDocument(metadata.replaceId);
+		const now = Date.now();
+		const content = `Q: ${question}\nA: ${answer}`;
+		const doc = {
+			id: randomUUID(),
+			sourceType: "faq",
+			sourceUrl: metadata?.sourceUrl,
+			title: question,
+			content,
+			metadata: {
+				...metadata,
+				question,
+				answer
+			},
+			priority: 1,
+			createdAt: now,
+			updatedAt: now
+		};
+		await this.store.addDocument(doc);
+		const chunk = {
+			id: randomUUID(),
+			documentId: doc.id,
+			content,
+			chunkIndex: 0,
+			embedding,
+			metadata: {
+				question,
+				answer
+			}
+		};
+		await this.store.addChunks([chunk]);
+		return doc;
+	}
+	/**
+	* Rebuild all vector embeddings using the current embedding provider.
+	* Preserves all document content, chunks, and FTS data — only replaces vectors.
+	*
+	* Requires the store to be a SQLiteKnowledgeStore (uses rebuild-specific methods).
+	*/
+	async rebuild(onProgress) {
+		const sqliteStore = this.store;
+		if (!sqliteStore.getAllChunks || !sqliteStore.rebuildVecTable || !sqliteStore.batchInsertEmbeddings) throw new Error("Rebuild requires a SQLiteKnowledgeStore with rebuild methods.");
+		const oldDimensions = sqliteStore.getDimensions();
+		const newDimensions = this.embedder.dimensions;
+		const documents = await this.store.listDocuments();
+		const docMap = new Map(documents.map((d) => [d.id, d]));
+		const allChunks = sqliteStore.getAllChunks();
+		if (allChunks.length === 0) return {
+			documentsRebuilt: 0,
+			chunksRebuilt: 0,
+			oldDimensions,
+			newDimensions
+		};
+		sqliteStore.rebuildVecTable(newDimensions);
+		const chunksByDoc = /* @__PURE__ */ new Map();
+		for (const chunk of allChunks) {
+			const list = chunksByDoc.get(chunk.documentId) || [];
+			list.push(chunk);
+			chunksByDoc.set(chunk.documentId, list);
+		}
+		let processedDocs = 0;
+		const totalDocs = chunksByDoc.size;
+		let totalChunksRebuilt = 0;
+		for (const [docId, chunks] of chunksByDoc) {
+			const doc = docMap.get(docId);
+			const docTitle = doc?.title || docId.slice(0, 8);
+			onProgress?.(processedDocs, totalDocs, docTitle);
+			const textsToEmbed = [];
+			for (const chunk of chunks) if (doc?.sourceType === "faq") {
+				const question = (chunk.metadata ? JSON.parse(chunk.metadata) : null)?.question || doc.title || chunk.content;
+				textsToEmbed.push(normalizeQuery(question));
+			} else textsToEmbed.push(chunk.content);
+			const embeddings = await this.embedder.embedMany(textsToEmbed);
+			const items = chunks.map((chunk, i) => ({
+				chunkId: chunk.id,
+				embedding: embeddings[i]
+			}));
+			sqliteStore.batchInsertEmbeddings(items);
+			totalChunksRebuilt += chunks.length;
+			processedDocs++;
+		}
+		onProgress?.(totalDocs, totalDocs, "done");
+		return {
+			documentsRebuilt: totalDocs,
+			chunksRebuilt: totalChunksRebuilt,
+			oldDimensions,
+			newDimensions
+		};
+	}
+};
+
+//#endregion
+//#region src/RankFusion.ts
+/**
+* Reciprocal Rank Fusion (RRF) for combining multiple ranked result sets.
+* Standard technique for hybrid search (vector + keyword).
+*/
+/**
+* Fuse multiple ranked result sets using Reciprocal Rank Fusion.
+*
+* @param resultSets - Array of Maps where key = item ID, value = rank (0-based)
+* @param k - Smoothing constant (default 60, industry standard)
+* @returns Map of item ID → fused RRF score, sorted descending by score
+*/
+function reciprocalRankFusion(resultSets, k = 60) {
+	const scores = /* @__PURE__ */ new Map();
+	for (const rankMap of resultSets) for (const [id, rank] of rankMap) {
+		const prev = scores.get(id) ?? 0;
+		scores.set(id, prev + 1 / (k + rank));
+	}
+	return new Map([...scores.entries()].sort((a, b) => b[1] - a[1]));
+}
+/**
+* Weighted Score Fusion: combine vector and keyword scores using weighted average.
+* BM25 scores are min-max normalized to 0-1 before combining.
+*
+* @returns Map of item ID → fused score, sorted descending
+*/
+function weightedScoreFusion(vectorResults, keywordResults, vectorWeight = .7, keywordWeight = .3) {
+	const bm25Scores = /* @__PURE__ */ new Map();
+	if (keywordResults.length > 0) {
+		const scores = keywordResults.map((r) => r.score);
+		const min = Math.min(...scores);
+		const range = Math.max(...scores) - min || 1;
+		for (const r of keywordResults) bm25Scores.set(r.id, (r.score - min) / range);
+	}
+	const vecScores = /* @__PURE__ */ new Map();
+	for (const r of vectorResults) vecScores.set(r.id, r.score);
+	const allIds = new Set([...vecScores.keys(), ...bm25Scores.keys()]);
+	const fused = /* @__PURE__ */ new Map();
+	for (const id of allIds) {
+		const vs = vecScores.get(id) ?? 0;
+		const ks = bm25Scores.get(id) ?? 0;
+		fused.set(id, vectorWeight * vs + keywordWeight * ks);
+	}
+	return new Map([...fused.entries()].sort((a, b) => b[1] - a[1]));
+}
+
+//#endregion
+//#region src/RetrievalPipeline.ts
+/**
+* Heuristic splitter for compound questions. No LLM call — zero latency cost.
+* Splits on "?" followed by more text, or " and " when both sides are >3 chars.
+* Returns the original query in a single-element array if no split detected.
+* Capped at 4 sub-queries max.
+*/
+function splitCompoundQuery(query) {
+	const qParts = query.split(/\?\s*/).filter((p) => p.trim().length > 3);
+	if (qParts.length > 1) return qParts.slice(0, 4).map((p) => p.trim());
+	const andParts = query.split(/\s+and\s+/i).filter((p) => p.trim().length > 3);
+	if (andParts.length > 1) return andParts.slice(0, 4).map((p) => p.trim());
+	return [query];
+}
+var RetrievalPipeline = class {
+	store;
+	embedder;
+	faqThreshold;
+	faqLowThreshold;
+	faqScoreGap;
+	useHybridSearch;
+	queryRewriter;
+	rewriteHighThreshold;
+	rewriteLowThreshold;
+	fusionStrategy;
+	constructor(store, embedder, thresholdOrOptions) {
+		this.store = store;
+		this.embedder = embedder;
+		if (typeof thresholdOrOptions === "number") {
+			this.faqThreshold = thresholdOrOptions;
+			this.faqLowThreshold = .7;
+			this.faqScoreGap = .15;
+			this.useHybridSearch = true;
+			this.rewriteHighThreshold = .7;
+			this.rewriteLowThreshold = .5;
+			this.fusionStrategy = "rrf";
+		} else {
+			const opts = thresholdOrOptions ?? {};
+			this.faqThreshold = opts.faqThreshold ?? .85;
+			this.faqLowThreshold = opts.faqLowThreshold ?? .7;
+			this.faqScoreGap = opts.faqScoreGap ?? .15;
+			this.useHybridSearch = opts.useHybridSearch ?? true;
+			this.queryRewriter = opts.queryRewriter;
+			this.rewriteHighThreshold = opts.rewriteHighThreshold ?? .7;
+			this.rewriteLowThreshold = opts.rewriteLowThreshold ?? .5;
+			this.fusionStrategy = opts.fusionStrategy ?? "rrf";
+		}
+	}
+	async retrieve(query, options) {
+		const subQueries = splitCompoundQuery(query);
+		if (subQueries.length <= 1) return this.retrieveSingle(query, options);
+		const subResults = await Promise.all(subQueries.map((sq) => this.retrieveSingle(sq, options)));
+		const seen = /* @__PURE__ */ new Set();
+		const faqMatches = [];
+		for (const sr of subResults) if (sr.isFaqMatch && sr.faqAnswer && sr.faqQuestion) {
+			const docId = sr.results[0]?.document?.id;
+			if (docId && !seen.has(docId)) {
+				seen.add(docId);
+				faqMatches.push({
+					faqQuestion: sr.faqQuestion,
+					faqAnswer: sr.faqAnswer,
+					score: sr.results[0]?.score ?? 0
+				});
+			}
+		}
+		if (faqMatches.length >= 2) {
+			const allResults = subResults.flatMap((sr) => sr.results);
+			const seenChunks = /* @__PURE__ */ new Set();
+			const dedupedResults = allResults.filter((r) => {
+				if (seenChunks.has(r.chunk.id)) return false;
+				seenChunks.add(r.chunk.id);
+				return true;
+			});
+			return {
+				results: dedupedResults,
+				context: this.formatContext(dedupedResults),
+				isFaqMatch: true,
+				faqMatches,
+				faqAnswer: faqMatches[0].faqAnswer,
+				faqQuestion: faqMatches[0].faqQuestion
+			};
+		}
+		return this.retrieveSingle(query, options);
+	}
+	async retrieveSingle(query, options) {
+		const normalized = normalizeQuery(query);
+		const embedding = await this.embedder.embed(normalized);
+		const faqResults = await this.store.searchByEmbedding(embedding, {
+			...options,
+			sourceTypes: ["faq"],
+			limit: 2
+		});
+		if (faqResults.length > 0) {
+			let top = faqResults[0];
+			if (faqResults.length > 1) {
+				if (top.score - faqResults[1].score <= .02 && (faqResults[1].document.updatedAt ?? 0) > (top.document.updatedAt ?? 0)) top = faqResults[1];
+			}
+			const faqAnswer = top.chunk.metadata?.answer || top.document.metadata?.answer;
+			const faqQuestion = top.chunk.metadata?.question || top.document.metadata?.question;
+			if (top.score >= this.faqThreshold) return {
+				results: [top],
+				context: this.formatContext([top]),
+				isFaqMatch: true,
+				faqAnswer,
+				faqQuestion
+			};
+			if (top.score >= this.faqLowThreshold) {
+				if ((faqResults.length > 1 ? top.score - faqResults[1].score : 1) > this.faqScoreGap) return {
+					results: [top],
+					context: this.formatContext([top]),
+					isFaqMatch: true,
+					faqAnswer,
+					faqQuestion
+				};
+			}
+		}
+		const results = await this.hybridSearch(normalized, embedding, options);
+		const topScore = results.length > 0 ? results[0].score : 0;
+		if (this.queryRewriter && topScore >= this.rewriteLowThreshold && topScore < this.rewriteHighThreshold) try {
+			const rewriteResult = await this.queryRewriter.rewrite(normalized);
+			const rewrittenEmbedding = await this.embedder.embed(rewriteResult.rewritten);
+			const rewrittenFaqResults = await this.store.searchByEmbedding(rewrittenEmbedding, {
+				...options,
+				sourceTypes: ["faq"],
+				limit: 2
+			});
+			if (rewrittenFaqResults.length > 0 && rewrittenFaqResults[0].score >= this.faqLowThreshold) {
+				const top = rewrittenFaqResults[0];
+				const gap = rewrittenFaqResults.length > 1 ? top.score - rewrittenFaqResults[1].score : 1;
+				if (top.score >= this.faqThreshold || gap > this.faqScoreGap) {
+					const faqAnswer = top.chunk.metadata?.answer || top.document.metadata?.answer;
+					const faqQuestion = top.chunk.metadata?.question || top.document.metadata?.question;
+					return {
+						results: [top],
+						context: this.formatContext([top]),
+						isFaqMatch: true,
+						rewritten: rewriteResult.rewritten,
+						faqAnswer,
+						faqQuestion
+					};
+				}
+			}
+			const rewrittenResults = await this.hybridSearch(rewriteResult.rewritten, rewrittenEmbedding, options);
+			if (rewrittenResults.length > 0 && rewrittenResults[0].score > topScore) return {
+				results: rewrittenResults,
+				context: this.formatContext(rewrittenResults),
+				isFaqMatch: false,
+				rewritten: rewriteResult.rewritten
+			};
+		} catch {}
+		return {
+			results,
+			context: this.formatContext(results),
+			isFaqMatch: false
+		};
+	}
+	async hybridSearch(query, embedding, options) {
+		const limit = options?.limit || 5;
+		if (!this.useHybridSearch || !this.store.searchByKeyword) {
+			const vecResults = await this.store.searchByEmbedding(embedding, {
+				...options,
+				limit
+			});
+			return this.applyBoosts(vecResults.slice(0, limit));
+		}
+		const searchOpts = {
+			...options,
+			limit: limit * 2
+		};
+		const [vecResults, ftsResults] = await Promise.all([this.store.searchByEmbedding(embedding, searchOpts), this.store.searchByKeyword(query, searchOpts)]);
+		if (ftsResults.length === 0) return this.applyBoosts(vecResults.slice(0, limit));
+		let fusedResults;
+		if (this.fusionStrategy === "weighted") {
+			const fused = weightedScoreFusion(vecResults.map((r) => ({
+				id: r.chunk.id,
+				score: r.score
+			})), ftsResults.map((r) => ({
+				id: r.chunk.id,
+				score: r.score
+			})));
+			const resultMap = /* @__PURE__ */ new Map();
+			for (const r of vecResults) resultMap.set(r.chunk.id, r);
+			for (const r of ftsResults) if (!resultMap.has(r.chunk.id)) resultMap.set(r.chunk.id, r);
+			fusedResults = [];
+			for (const [chunkId, fusedScore] of fused) {
+				if (fusedResults.length >= limit) break;
+				const result = resultMap.get(chunkId);
+				if (result) fusedResults.push({
+					...result,
+					score: fusedScore
+				});
+			}
+		} else {
+			const vecRanks = /* @__PURE__ */ new Map();
+			vecResults.forEach((r, i) => vecRanks.set(r.chunk.id, i));
+			const ftsRanks = /* @__PURE__ */ new Map();
+			ftsResults.forEach((r, i) => ftsRanks.set(r.chunk.id, i));
+			const fused = reciprocalRankFusion([vecRanks, ftsRanks]);
+			const resultMap = /* @__PURE__ */ new Map();
+			for (const r of vecResults) resultMap.set(r.chunk.id, r);
+			for (const r of ftsResults) if (!resultMap.has(r.chunk.id)) resultMap.set(r.chunk.id, r);
+			fusedResults = [];
+			for (const [chunkId, _rrfScore] of fused) {
+				if (fusedResults.length >= limit) break;
+				const result = resultMap.get(chunkId);
+				if (result) fusedResults.push(result);
+			}
+		}
+		return this.applyBoosts(fusedResults);
+	}
+	/**
+	* Apply freshness and priority boosts to search results, then re-sort.
+	*/
+	applyBoosts(results) {
+		if (results.length === 0) return results;
+		const thirtyDaysAgo = Date.now() - 720 * 60 * 60 * 1e3;
+		const boosted = results.map((r) => {
+			let score = r.score;
+			if (r.document.updatedAt && r.document.updatedAt > thirtyDaysAgo) score += .05;
+			const priority = r.document.priority ?? 2;
+			if (priority === 1) score += .03;
+			else if (priority === 3) score -= .02;
+			return {
+				...r,
+				score
+			};
+		});
+		boosted.sort((a, b) => b.score - a.score);
+		return boosted;
+	}
+	formatContext(results) {
+		if (results.length === 0) return "";
+		return `## Knowledge Base Context\n\n${results.map((r, i) => {
+			const source = r.document.title || r.document.sourceUrl || r.document.fileName || "Unknown";
+			return `### Source ${i + 1}: ${source} (score: ${r.score.toFixed(2)})\n${r.chunk.content}`;
+		}).join("\n\n")}`;
+	}
+};
+
+//#endregion
+//#region src/QueryRewriter.ts
+const SYSTEM_PROMPT = "You are a query normalizer. Rewrite the following informal/casual query into a clear, well-formed question. Only output the rewritten question, nothing else.";
+const MAX_CACHE_SIZE = 1e3;
+var QueryRewriter = class {
+	model;
+	cache;
+	maxCacheSize;
+	constructor(options) {
+		this.model = options.model;
+		this.cache = /* @__PURE__ */ new Map();
+		this.maxCacheSize = options.maxCacheSize ?? MAX_CACHE_SIZE;
+	}
+	async rewrite(query) {
+		const cacheKey = query.toLowerCase().trim();
+		const cached = this.cache.get(cacheKey);
+		if (cached) return {
+			original: query,
+			rewritten: cached,
+			cached: true
+		};
+		const { text, usage } = await generateText({
+			model: this.model,
+			system: SYSTEM_PROMPT,
+			prompt: query
+		});
+		const rewritten = text.trim();
+		if (this.cache.size >= this.maxCacheSize) {
+			const oldest = this.cache.keys().next().value;
+			this.cache.delete(oldest);
+		}
+		this.cache.set(cacheKey, rewritten);
+		return {
+			original: query,
+			rewritten,
+			cached: false,
+			tokenUsage: usage ? {
+				prompt: usage.promptTokens,
+				completion: usage.completionTokens
+			} : void 0
+		};
+	}
+	get cacheSize() {
+		return this.cache.size;
+	}
+	clearCache() {
+		this.cache.clear();
+	}
+};
+
+//#endregion
+//#region src/ingestors/content-fetcher.ts
+let crawl4aiHealthy = null;
+let crawl4aiHealthCheckedAt = 0;
+const HEALTH_CACHE_MS = 300 * 1e3;
+/** Reset the Crawl4AI health check cache. Used in tests. */
+function resetCrawl4aiHealthCache() {
+	crawl4aiHealthy = null;
+	crawl4aiHealthCheckedAt = 0;
+}
+/**
+* Fetch raw HTML from a URL. Used for link extraction, sitemaps, etc.
+*/
+async function fetchHtml(url) {
+	const response = await fetch(url, { headers: { "User-Agent": "Operor-KB/1.0" } });
+	if (!response.ok) throw new Error(`Failed to fetch ${url}: ${response.status}`);
+	return response.text();
+}
+/**
+* Smart content fetch: tries Crawl4AI first (if configured), falls back to Readability.
+*/
+async function fetchContent(url, options) {
+	if (options?.crawl4aiUrl) try {
+		if (await isCrawl4aiHealthy(options.crawl4aiUrl)) return await fetchViaCrawl4AI(url, options.crawl4aiUrl);
+	} catch {}
+	const { title, content } = extractFromHtml(await fetchHtml(url), url);
+	return {
+		title,
+		content,
+		isMarkdown: false
+	};
+}
+/**
+* Extract readable content from pre-fetched HTML using @mozilla/readability.
+*/
+function extractFromHtml(html, url) {
+	const { document } = parseHTML(html);
+	const article = new Readability(document, { url }).parse();
+	return {
+		title: article?.title || "",
+		content: article?.textContent?.trim() || ""
+	};
+}
+/**
+* Extract same-domain links from HTML.
+*/
+function extractLinks(html, baseUrl) {
+	const { document } = parseHTML(html);
+	const links = [];
+	const base = new URL(baseUrl);
+	for (const a of document.querySelectorAll("a[href]")) try {
+		const href = a.getAttribute("href");
+		if (!href) continue;
+		const resolved = new URL(href, baseUrl);
+		if (resolved.hostname === base.hostname && resolved.protocol.startsWith("http")) links.push(resolved.href.split("#")[0]);
+	} catch {}
+	return [...new Set(links)];
+}
+async function isCrawl4aiHealthy(baseUrl) {
+	if (crawl4aiHealthy !== null && Date.now() - crawl4aiHealthCheckedAt < HEALTH_CACHE_MS) return crawl4aiHealthy;
+	try {
+		crawl4aiHealthy = (await fetch(`${baseUrl}/health`, { signal: AbortSignal.timeout(2e3) })).ok;
+	} catch {
+		crawl4aiHealthy = false;
+	}
+	crawl4aiHealthCheckedAt = Date.now();
+	return crawl4aiHealthy;
+}
+async function fetchViaCrawl4AI(url, baseUrl) {
+	const res = await fetch(`${baseUrl}/crawl`, {
+		method: "POST",
+		headers: { "Content-Type": "application/json" },
+		body: JSON.stringify({
+			urls: [url],
+			browser_config: {
+				type: "BrowserConfig",
+				params: { headless: true }
+			},
+			crawler_config: {
+				type: "CrawlerRunConfig",
+				params: {
+					cache_mode: "bypass",
+					markdown_generator: {
+						type: "DefaultMarkdownGenerator",
+						params: { content_filter: {
+							type: "PruningContentFilter",
+							params: { threshold: .48 }
+						} }
+					}
+				}
+			}
+		}),
+		signal: AbortSignal.timeout(3e4)
+	});
+	if (!res.ok) throw new Error(`Crawl4AI error: ${res.status}`);
+	const result = (await res.json()).results?.[0];
+	if (!result) throw new Error("Crawl4AI returned no results");
+	const markdown = result.markdown?.fit_markdown || result.markdown?.raw_markdown || "";
+	if (!markdown) throw new Error("Crawl4AI returned empty markdown");
+	return {
+		title: markdown.match(/^#\s+(.+)$/m)?.[1] || "",
+		content: markdown,
+		isMarkdown: true
+	};
+}
+
+//#endregion
+//#region src/ingestors/UrlIngestor.ts
+var UrlIngestor = class {
+	pipeline;
+	fetchOptions;
+	constructor(pipeline, options) {
+		this.pipeline = pipeline;
+		this.fetchOptions = { crawl4aiUrl: options?.crawl4aiUrl };
+	}
+	async ingestUrl(url, options) {
+		const { title, content, isMarkdown } = await fetchContent(url, this.fetchOptions);
+		return this.pipeline.ingest({
+			sourceType: "url",
+			sourceUrl: url,
+			title,
+			content,
+			isMarkdown,
+			priority: options?.priority,
+			extractQA: options?.extractQA
+		});
+	}
+	async ingestSitemap(sitemapUrl, options) {
+		const maxPages = options?.maxPages || 50;
+		const xml = await fetchHtml(sitemapUrl);
+		const urls = this.parseSitemapUrls(xml).slice(0, maxPages);
+		const docs = [];
+		for (const url of urls) try {
+			const doc = await this.ingestUrl(url);
+			docs.push(doc);
+		} catch {}
+		return docs;
+	}
+	async crawl(startUrl, options) {
+		const maxPages = options?.maxPages || 20;
+		const maxDepth = options?.maxDepth || 2;
+		const visited = /* @__PURE__ */ new Set();
+		const docs = [];
+		const queue = [{
+			url: startUrl,
+			depth: 0
+		}];
+		while (queue.length > 0 && docs.length < maxPages) {
+			const item = queue.shift();
+			if (visited.has(item.url) || item.depth > maxDepth) continue;
+			visited.add(item.url);
+			try {
+				const html = await fetchHtml(item.url);
+				const { title, content, isMarkdown } = await fetchContent(item.url, this.fetchOptions);
+				const doc = await this.pipeline.ingest({
+					sourceType: "url",
+					sourceUrl: item.url,
+					title,
+					content,
+					isMarkdown
+				});
+				docs.push(doc);
+				if (item.depth < maxDepth) {
+					const links = extractLinks(html, item.url);
+					for (const link of links) if (!visited.has(link)) queue.push({
+						url: link,
+						depth: item.depth + 1
+					});
+				}
+			} catch {}
+		}
+		return docs;
+	}
+	parseSitemapUrls(xml) {
+		const urls = [];
+		const locRegex = /<loc>(.*?)<\/loc>/g;
+		let match;
+		while ((match = locRegex.exec(xml)) !== null) urls.push(match[1]);
+		return urls.filter((u) => !u.endsWith(".xml"));
+	}
+};
+
+//#endregion
+//#region src/ingestors/FileIngestor.ts
+var FileIngestor = class {
+	pipeline;
+	constructor(pipeline) {
+		this.pipeline = pipeline;
+	}
+	async ingestFile(filePath, title, options) {
+		const ext = extname(filePath).toLowerCase();
+		const content = await this.extractContent(filePath, ext);
+		const fileName = filePath.split("/").pop() || filePath;
+		return this.pipeline.ingest({
+			sourceType: "file",
+			fileName,
+			title: title || fileName,
+			content,
+			priority: options?.priority
+		});
+	}
+	async extractContent(filePath, ext) {
+		switch (ext) {
+			case ".pdf": return this.extractPdf(filePath);
+			case ".docx": return this.extractDocx(filePath);
+			case ".xlsx":
+			case ".xls": return this.extractXlsx(filePath);
+			case ".csv":
+			case ".txt":
+			case ".md": return readFile(filePath, "utf-8");
+			case ".html":
+			case ".htm": return this.extractHtml(filePath);
+			default: return readFile(filePath, "utf-8");
+		}
+	}
+	async extractPdf(filePath) {
+		const { getDocumentProxy, extractText } = await import("unpdf");
+		const buffer = await readFile(filePath);
+		const { text } = await extractText(await getDocumentProxy(new Uint8Array(buffer)), { mergePages: true });
+		return text;
+	}
+	async extractDocx(filePath) {
+		const mammoth = await import("mammoth");
+		const buffer = await readFile(filePath);
+		return (await mammoth.extractRawText({ buffer })).value;
+	}
+	async extractXlsx(filePath) {
+		const XLSX = await import("xlsx");
+		const buffer = await readFile(filePath);
+		const workbook = XLSX.read(buffer, { type: "buffer" });
+		const lines = [];
+		for (const sheetName of workbook.SheetNames) {
+			const sheet = workbook.Sheets[sheetName];
+			const csv = XLSX.utils.sheet_to_csv(sheet);
+			lines.push(`## ${sheetName}\n${csv}`);
+		}
+		return lines.join("\n\n");
+	}
+	async extractHtml(filePath) {
+		const { parseHTML } = await import("linkedom");
+		const { Readability } = await import("@mozilla/readability");
+		const html = await readFile(filePath, "utf-8");
+		const { document } = parseHTML(html);
+		return new Readability(document).parse()?.textContent?.trim() || html;
+	}
+};
+
+//#endregion
+//#region src/ingestors/SiteCrawler.ts
+var SiteCrawler = class {
+	pipeline;
+	fetchOptions;
+	constructor(pipeline, options) {
+		this.pipeline = pipeline;
+		this.fetchOptions = { crawl4aiUrl: options?.crawl4aiUrl };
+	}
+	async crawlSite(startUrl, options = {}) {
+		const { maxDepth = 2, maxPages = 50, useSitemap = true, delayMs = 500, onProgress } = options;
+		const docs = [];
+		const visited = /* @__PURE__ */ new Set();
+		if (useSitemap) {
+			const sitemapUrls = await this.tryFetchSitemap(startUrl);
+			if (sitemapUrls.length > 0) {
+				const urlsToIngest = sitemapUrls.slice(0, maxPages);
+				for (const url of urlsToIngest) {
+					if (visited.has(url)) continue;
+					visited.add(url);
+					try {
+						onProgress?.(docs.length + 1, urlsToIngest.length, url);
+						const doc = await this.ingestPage(url);
+						docs.push(doc);
+						if (delayMs > 0) await this.delay(delayMs);
+					} catch {}
+					if (docs.length >= maxPages) break;
+				}
+				return docs;
+			}
+		}
+		const queue = [{
+			url: startUrl,
+			depth: 0
+		}];
+		while (queue.length > 0 && docs.length < maxPages) {
+			const item = queue.shift();
+			if (visited.has(item.url) || item.depth > maxDepth) continue;
+			visited.add(item.url);
+			try {
+				onProgress?.(docs.length + 1, docs.length + queue.length + 1, item.url);
+				const html = await fetchHtml(item.url);
+				const { title, content, isMarkdown } = await fetchContent(item.url, this.fetchOptions);
+				const doc = await this.pipeline.ingest({
+					sourceType: "url",
+					sourceUrl: item.url,
+					title,
+					content,
+					isMarkdown
+				});
+				docs.push(doc);
+				if (delayMs > 0) await this.delay(delayMs);
+				if (item.depth < maxDepth) {
+					for (const link of extractLinks(html, item.url)) if (!visited.has(link)) queue.push({
+						url: link,
+						depth: item.depth + 1
+					});
+				}
+			} catch {}
+		}
+		return docs;
+	}
+	async tryFetchSitemap(baseUrl) {
+		try {
+			const url = new URL(baseUrl);
+			const xml = await fetchHtml(`${url.protocol}//${url.hostname}/sitemap.xml`);
+			return await this.parseSitemapUrls(xml);
+		} catch {
+			return [];
+		}
+	}
+	async parseSitemapUrls(xml) {
+		const urls = [];
+		const locRegex = /<loc>(.*?)<\/loc>/g;
+		let match;
+		while ((match = locRegex.exec(xml)) !== null) urls.push(match[1]);
+		if (xml.includes("<sitemapindex") || urls.every((u) => u.endsWith(".xml"))) {
+			const pageUrls = [];
+			for (const childSitemapUrl of urls) try {
+				const childXml = await fetchHtml(childSitemapUrl);
+				const childUrls = [];
+				const childRegex = /<loc>(.*?)<\/loc>/g;
+				let childMatch;
+				while ((childMatch = childRegex.exec(childXml)) !== null) childUrls.push(childMatch[1]);
+				pageUrls.push(...childUrls.filter((u) => !u.endsWith(".xml")));
+			} catch {}
+			return pageUrls;
+		}
+		return urls.filter((u) => !u.endsWith(".xml"));
+	}
+	async ingestPage(url) {
+		const { title, content, isMarkdown } = await fetchContent(url, this.fetchOptions);
+		return this.pipeline.ingest({
+			sourceType: "url",
+			sourceUrl: url,
+			title,
+			content,
+			isMarkdown
+		});
+	}
+	delay(ms) {
+		return new Promise((resolve) => setTimeout(resolve, ms));
+	}
+};
+
+//#endregion
+//#region src/ingestors/WatiFaqSync.ts
+var WatiFaqSync = class {
+	pipeline;
+	llmExtract;
+	constructor(pipeline, llmExtract) {
+		this.pipeline = pipeline;
+		this.llmExtract = llmExtract;
+	}
+	async syncFromConversations(conversations, options) {
+		const minLen = options?.minAnswerLength || 20;
+		const maxPairs = options?.maxPairs || 100;
+		if (!this.llmExtract) throw new Error("LLM extract function required for FAQ extraction");
+		const allPairs = [];
+		for (const convo of conversations) {
+			const pairs = await this.llmExtract(convo);
+			allPairs.push(...pairs);
+			if (allPairs.length >= maxPairs) break;
+		}
+		const filtered = allPairs.filter((p) => p.answer.length >= minLen && p.question.trim().length > 0).slice(0, maxPairs);
+		const docs = [];
+		for (const pair of filtered) {
+			const doc = await this.pipeline.ingestFaq(pair.question, pair.answer, { source: "wati-sync" });
+			docs.push(doc);
+		}
+		return docs;
+	}
+	async syncFromPairs(pairs, options) {
+		const minLen = options?.minAnswerLength || 20;
+		const filtered = pairs.filter((p) => p.answer.length >= minLen && p.question.trim().length > 0);
+		const docs = [];
+		for (const pair of filtered) {
+			const doc = await this.pipeline.ingestFaq(pair.question, pair.answer, { source: "wati-sync" });
+			docs.push(doc);
+		}
+		return docs;
+	}
+};
+
+//#endregion
+export { EmbeddingService, FileIngestor, IngestionPipeline, QueryRewriter, RetrievalPipeline, SQLiteKnowledgeStore, SiteCrawler, TextChunker, UrlIngestor, WatiFaqSync, extractFromHtml, extractLinks, fetchContent, fetchHtml, normalizeQuery, reciprocalRankFusion, resetCrawl4aiHealthCache, splitCompoundQuery, weightedScoreFusion };
+//# sourceMappingURL=index.js.map