@tekmidian/pai 0.1.0

package/dist/indexer-B20bPHL-.mjs

@@ -0,0 +1,677 @@
+import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
+import { homedir } from "node:os";
+import { basename, join, normalize, relative } from "node:path";
+import { createHash } from "node:crypto";
+
+//#region src/memory/chunker.ts
+/**
+* Markdown text chunker for the PAI memory engine.
+*
+* Splits markdown files into overlapping text segments suitable for BM25
+* full-text indexing.  Respects heading boundaries where possible, falling
+* back to paragraph and sentence splitting when sections are large.
+*/
+const DEFAULT_MAX_TOKENS = 400;
+const DEFAULT_OVERLAP = 80;
+/**
+* Approximate token count using a words * 1.3 heuristic.
+* Matches the OpenClaw estimate approach.
+*/
+function estimateTokens(text) {
+	const wordCount = text.split(/\s+/).filter(Boolean).length;
+	return Math.ceil(wordCount * 1.3);
+}
+/**
+* Compute SHA-256 hash of a string, returning a hex string.
+*/
+function sha256(text) {
+	return createHash("sha256").update(text).digest("hex");
+}
+/**
+* Split content into sections delimited by ## or ### headings.
+* Each section starts at its heading line (or at line 1 for a preamble).
+*/
+function splitBySections(lines) {
+	const sections = [];
+	let current = [];
+	for (const line of lines) {
+		if (/^#{1,3}\s/.test(line.text) && current.length > 0) {
+			const text = current.map((l) => l.text).join("\n");
+			sections.push({
+				lines: current,
+				tokens: estimateTokens(text)
+			});
+			current = [];
+		}
+		current.push(line);
+	}
+	if (current.length > 0) {
+		const text = current.map((l) => l.text).join("\n");
+		sections.push({
+			lines: current,
+			tokens: estimateTokens(text)
+		});
+	}
+	return sections;
+}
+/**
+* Split a LineBlock by double-newline paragraph boundaries.
+*/
+function splitByParagraphs(block) {
+	const paragraphs = [];
+	let current = [];
+	for (const line of block.lines) if (line.text.trim() === "" && current.length > 0) {
+		const text = current.map((l) => l.text).join("\n");
+		paragraphs.push({
+			lines: [...current],
+			tokens: estimateTokens(text)
+		});
+		current = [];
+	} else current.push(line);
+	if (current.length > 0) {
+		const text = current.map((l) => l.text).join("\n");
+		paragraphs.push({
+			lines: current,
+			tokens: estimateTokens(text)
+		});
+	}
+	return paragraphs.length > 0 ? paragraphs : [block];
+}
+/**
+* Split a LineBlock by sentence boundaries (. ! ?) when even paragraphs are
+* too large.  Works character-by-character within joined lines.
+*/
+function splitBySentences(block, maxTokens) {
+	const sentences = block.lines.map((l) => l.text).join(" ").split(/(?<=[.!?])\s+(?=[A-Z"'])/g);
+	const result = [];
+	let accText = "";
+	const startLine = block.lines[0]?.lineNo ?? 1;
+	const endLine = block.lines[block.lines.length - 1]?.lineNo ?? startLine;
+	const totalLines = endLine - startLine + 1;
+	const linesPerSentence = Math.max(1, Math.floor(totalLines / Math.max(1, sentences.length)));
+	let sentenceIdx = 0;
+	let approxLine = startLine;
+	const flush = () => {
+		if (!accText.trim()) return;
+		const endApprox = Math.min(approxLine + linesPerSentence - 1, endLine);
+		result.push({
+			lines: [{
+				text: accText.trim(),
+				lineNo: approxLine
+			}],
+			tokens: estimateTokens(accText)
+		});
+		approxLine = endApprox + 1;
+		accText = "";
+	};
+	for (const sentence of sentences) {
+		sentenceIdx++;
+		const candidateText = accText ? accText + " " + sentence : sentence;
+		if (estimateTokens(candidateText) > maxTokens && accText) {
+			flush();
+			accText = sentence;
+		} else accText = candidateText;
+	}
+	flush();
+	return result.length > 0 ? result : [block];
+}
+/**
+* Extract the last `overlapTokens` worth of text from a list of previously
+* emitted chunks to prepend to the next chunk.
+*/
+function buildOverlapPrefix(chunks, overlapTokens) {
+	if (overlapTokens <= 0 || chunks.length === 0) return [];
+	const lastChunk = chunks[chunks.length - 1];
+	if (!lastChunk) return [];
+	const lines = lastChunk.text.split("\n");
+	const kept = [];
+	let acc = 0;
+	for (let i = lines.length - 1; i >= 0; i--) {
+		const lineTokens = estimateTokens(lines[i] ?? "");
+		acc += lineTokens;
+		kept.unshift(lines[i] ?? "");
+		if (acc >= overlapTokens) break;
+	}
+	const startLine = lastChunk.endLine - kept.length + 1;
+	return kept.map((text, idx) => ({
+		text,
+		lineNo: Math.max(lastChunk.startLine, startLine + idx)
+	}));
+}
+/**
+* Chunk a markdown file into overlapping segments for BM25 indexing.
+*
+* Strategy:
+*  1. Split by headings (##, ###) as natural boundaries.
+*  2. If a section exceeds maxTokens, split by paragraphs.
+*  3. If a paragraph still exceeds maxTokens, split by sentences.
+*  4. Apply overlap: each chunk includes the last `overlap` tokens from the
+*     previous chunk.
+*/
+function chunkMarkdown(content, opts) {
+	const maxTokens = opts?.maxTokens ?? DEFAULT_MAX_TOKENS;
+	const overlapTokens = opts?.overlap ?? DEFAULT_OVERLAP;
+	if (!content.trim()) return [];
+	const sections = splitBySections(content.split("\n").map((text, idx) => ({
+		text,
+		lineNo: idx + 1
+	})));
+	const finalBlocks = [];
+	for (const section of sections) {
+		if (section.tokens <= maxTokens) {
+			finalBlocks.push(section);
+			continue;
+		}
+		const paras = splitByParagraphs(section);
+		for (const para of paras) {
+			if (para.tokens <= maxTokens) {
+				finalBlocks.push(para);
+				continue;
+			}
+			const sentences = splitBySentences(para, maxTokens);
+			finalBlocks.push(...sentences);
+		}
+	}
+	const chunks = [];
+	for (const block of finalBlocks) {
+		if (block.lines.length === 0) continue;
+		const text = [...buildOverlapPrefix(chunks, overlapTokens), ...block.lines].map((l) => l.text).join("\n").trim();
+		if (!text) continue;
+		const startLine = block.lines[0]?.lineNo ?? 1;
+		const endLine = block.lines[block.lines.length - 1]?.lineNo ?? startLine;
+		chunks.push({
+			text,
+			startLine,
+			endLine,
+			hash: sha256(text)
+		});
+	}
+	return chunks;
+}
+
+//#endregion
+//#region src/memory/indexer.ts
+/**
+* File indexer for the PAI federation memory engine.
+*
+* Scans project memory/ and Notes/ directories, chunks markdown files, and
+* inserts the resulting chunks into federation.db for BM25 search.
+*
+* Change detection: files whose SHA-256 hash has not changed since the last
+* index run are skipped, keeping incremental re-indexing fast.
+*
+* Phase 2.5: adds embedChunks() for generating vector embeddings on indexed
+* chunks that do not yet have an embedding stored.
+*/
+/**
+* Classify a relative file path into one of the four memory tiers.
+*
+* Rules (in priority order):
+*  - MEMORY.md anywhere in memory/  → 'evergreen'
+*  - YYYY-MM-DD.md in memory/       → 'daily'
+*  - anything else in memory/       → 'topic'
+*  - anything in Notes/             → 'session'
+*/
+function detectTier(relativePath) {
+	const p = relativePath.replace(/\\/g, "/").replace(/^\.\//, "");
+	if (p.startsWith("Notes/") || p === "Notes") return "session";
+	const fileName = basename(p);
+	if (fileName === "MEMORY.md") return "evergreen";
+	if (/^\d{4}-\d{2}-\d{2}\.md$/.test(fileName)) return "daily";
+	return "topic";
+}
+/**
+* Generate a deterministic chunk ID from its coordinates.
+* Format: sha256("projectId:path:chunkIndex:startLine:endLine")
+*
+* The chunkIndex (0-based position within the file) is included so that
+* chunks with approximated line numbers (e.g. from splitBySentences) never
+* produce colliding IDs even when multiple chunks share the same startLine/endLine.
+*/
+function chunkId(projectId, path, chunkIndex, startLine, endLine) {
+	return createHash("sha256").update(`${projectId}:${path}:${chunkIndex}:${startLine}:${endLine}`).digest("hex");
+}
+function sha256File(content) {
+	return createHash("sha256").update(content).digest("hex");
+}
+/**
+* Index a single file into the federation database.
+*
+* @returns true if the file was re-indexed (changed or new), false if skipped.
+*/
+function indexFile(db, projectId, rootPath, relativePath, source, tier) {
+	const absPath = join(rootPath, relativePath);
+	let content;
+	let stat;
+	try {
+		content = readFileSync(absPath, "utf8");
+		stat = statSync(absPath);
+	} catch {
+		return false;
+	}
+	const hash = sha256File(content);
+	const mtime = Math.floor(stat.mtimeMs);
+	const size = stat.size;
+	if (db.prepare("SELECT hash FROM memory_files WHERE project_id = ? AND path = ?").get(projectId, relativePath)?.hash === hash) return false;
+	const oldChunkIds = db.prepare("SELECT id FROM memory_chunks WHERE project_id = ? AND path = ?").all(projectId, relativePath);
+	const deleteFts = db.prepare("DELETE FROM memory_fts WHERE id = ?");
+	const deleteChunk = db.prepare("DELETE FROM memory_chunks WHERE project_id = ? AND path = ?");
+	db.transaction(() => {
+		for (const row of oldChunkIds) deleteFts.run(row.id);
+		deleteChunk.run(projectId, relativePath);
+	})();
+	const chunks = chunkMarkdown(content);
+	const insertChunk = db.prepare(`
+    INSERT INTO memory_chunks (id, project_id, source, tier, path, start_line, end_line, hash, text, updated_at)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `);
+	const insertFts = db.prepare(`
+    INSERT INTO memory_fts (text, id, project_id, path, source, tier, start_line, end_line)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+  `);
+	const upsertFile = db.prepare(`
+    INSERT INTO memory_files (project_id, path, source, tier, hash, mtime, size)
+    VALUES (?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(project_id, path) DO UPDATE SET
+      source = excluded.source,
+      tier   = excluded.tier,
+      hash   = excluded.hash,
+      mtime  = excluded.mtime,
+      size   = excluded.size
+  `);
+	const updatedAt = Date.now();
+	db.transaction(() => {
+		for (let i = 0; i < chunks.length; i++) {
+			const chunk = chunks[i];
+			const id = chunkId(projectId, relativePath, i, chunk.startLine, chunk.endLine);
+			insertChunk.run(id, projectId, source, tier, relativePath, chunk.startLine, chunk.endLine, chunk.hash, chunk.text, updatedAt);
+			insertFts.run(chunk.text, id, projectId, relativePath, source, tier, chunk.startLine, chunk.endLine);
+		}
+		upsertFile.run(projectId, relativePath, source, tier, hash, mtime, size);
+	})();
+	return true;
+}
+/**
+* Safety cap: maximum number of .md files collected per project scan.
+* Prevents runaway scans on huge root paths (e.g. home directory).
+* Projects with more files than this are scanned up to the cap only.
+*/
+const MAX_FILES_PER_PROJECT = 5e3;
+/**
+* Maximum recursion depth for directory walks.
+* Prevents deep traversal of large directory trees (e.g. development repos).
+* Depth 0 = the given directory itself (no recursion).
+* Value 6 allows: root → subdirs → sub-subdirs → ... up to 6 levels.
+* Sufficient for memory/, Notes/, and typical docs structures.
+*/
+const MAX_WALK_DEPTH = 6;
+/**
+* Recursively collect all .md files under a directory.
+* Returns absolute paths. Stops early if the accumulated count hits the cap
+* or if the recursion depth exceeds MAX_WALK_DEPTH.
+*
+* @param dir    Directory to scan.
+* @param acc    Shared accumulator array (mutated in place for early exit).
+* @param cap    Maximum number of files to collect (across all recursive calls).
+* @param depth  Current recursion depth (0 = the initial call).
+*/
+function walkMdFiles(dir, acc, cap = MAX_FILES_PER_PROJECT, depth = 0) {
+	const results = acc ?? [];
+	if (!existsSync(dir)) return results;
+	if (results.length >= cap) return results;
+	if (depth > MAX_WALK_DEPTH) return results;
+	try {
+		for (const entry of readdirSync(dir, { withFileTypes: true })) {
+			if (results.length >= cap) break;
+			if (entry.isSymbolicLink()) continue;
+			if (ALWAYS_SKIP_DIRS.has(entry.name)) continue;
+			const full = join(dir, entry.name);
+			if (entry.isDirectory()) walkMdFiles(full, results, cap, depth + 1);
+			else if (entry.isFile() && entry.name.endsWith(".md")) results.push(full);
+		}
+	} catch {}
+	return results;
+}
+/**
+* Directories to ALWAYS skip, at any depth, during any directory walk.
+* These are build artifacts, dependency trees, and VCS internals that
+* should never be indexed regardless of where they appear in the tree.
+*/
+const ALWAYS_SKIP_DIRS = new Set([
+	".git",
+	"node_modules",
+	"vendor",
+	"Pods",
+	"dist",
+	"build",
+	"out",
+	"DerivedData",
+	".next",
+	".venv",
+	"venv",
+	"__pycache__",
+	".cache",
+	".bun"
+]);
+/**
+* Directories to skip when doing a root-level content scan.
+* These are either already handled by dedicated scans or should never be indexed.
+*/
+const ROOT_SCAN_SKIP_DIRS = new Set([
+	"memory",
+	"Notes",
+	".claude",
+	".DS_Store",
+	...ALWAYS_SKIP_DIRS
+]);
+/**
+* Additional directories to skip at the content-scan level (first level below root).
+* These are common macOS/Linux home-directory or repo noise directories that are
+* never meaningful as project content.
+*/
+const CONTENT_SCAN_SKIP_DIRS = new Set([
+	"Library",
+	"Applications",
+	"Music",
+	"Movies",
+	"Pictures",
+	"Desktop",
+	"Downloads",
+	"Public",
+	"coverage",
+	...ALWAYS_SKIP_DIRS
+]);
+/**
+* Recursively collect all .md files under rootPath, excluding directories
+* that are already covered by dedicated scans (memory/, Notes/) and
+* common noise directories (.git, node_modules, etc.).
+*
+* Returns absolute paths for files NOT already handled by the specific scanners.
+* Stops collecting once MAX_FILES_PER_PROJECT is reached.
+*/
+function walkContentFiles(rootPath) {
+	if (!existsSync(rootPath)) return [];
+	const results = [];
+	try {
+		for (const entry of readdirSync(rootPath, { withFileTypes: true })) {
+			if (results.length >= MAX_FILES_PER_PROJECT) break;
+			if (entry.isSymbolicLink()) continue;
+			if (ROOT_SCAN_SKIP_DIRS.has(entry.name)) continue;
+			if (CONTENT_SCAN_SKIP_DIRS.has(entry.name)) continue;
+			const full = join(rootPath, entry.name);
+			if (entry.isDirectory()) walkMdFiles(full, results, MAX_FILES_PER_PROJECT);
+			else if (entry.isFile() && entry.name.endsWith(".md")) {
+				if (entry.name !== "MEMORY.md") results.push(full);
+			}
+		}
+	} catch {}
+	return results;
+}
+/**
+* Index all memory, Notes, and content files for a single registered project.
+*
+* Scans:
+*  - {rootPath}/MEMORY.md    → source='memory', tier='evergreen'
+*  - {rootPath}/memory/      → source='memory', tier from detectTier()
+*  - {rootPath}/Notes/       → source='notes',  tier='session'
+*  - {rootPath}/**\/\*.md    → source='content', tier='topic'  (all other .md files, recursive)
+*  - {claudeNotesDir}/       → source='notes',  tier='session'  (if set and different)
+*
+* The content scan covers projects like job-discussions where markdown files
+* live in date/topic subdirectories rather than a memory/ folder.  The
+* memory/, Notes/, .git/, and node_modules/ directories are excluded from
+* the content scan to avoid double-indexing.
+*
+* The claudeNotesDir parameter points to ~/.claude/projects/{encoded}/Notes/
+* where Claude Code writes session notes for a given working directory.
+* It is stored on the project row as claude_notes_dir after a registry scan.
+*/
+/**
+* Number of files to process before yielding to the event loop inside
+* indexProject. Keeps IPC responsive even while indexing large projects.
+* Lower = more responsive but more overhead. 10 is a good balance.
+*/
+const INDEX_YIELD_EVERY = 10;
+/**
+* Returns true if rootPath should skip the recursive content scan.
+*
+* Skips content scanning for:
+*  - The home directory itself or any ancestor (too broad — millions of files)
+*  - Git repositories (code repos — index memory/ and Notes/ only, not all .md files)
+*
+* The content scan is still useful for Obsidian vaults, Notes folders, and
+* other doc-centric project trees where ALL markdown files are meaningful.
+*
+* The memory/, Notes/, and claude_notes_dir scans always run regardless.
+*/
+function isPathTooBroadForContentScan(rootPath) {
+	const normalized = normalize(rootPath);
+	const home = homedir();
+	if (home.startsWith(normalized) || normalized === "/") return true;
+	if (normalized.startsWith(home)) {
+		const rel = normalized.slice(home.length).replace(/^\//, "");
+		if ((rel ? rel.split("/").length : 0) === 0) return true;
+	}
+	if (existsSync(join(normalized, ".git"))) return true;
+	return false;
+}
+async function indexProject(db, projectId, rootPath, claudeNotesDir) {
+	const result = {
+		filesProcessed: 0,
+		chunksCreated: 0,
+		filesSkipped: 0
+	};
+	const filesToIndex = [];
+	const rootMemoryMd = join(rootPath, "MEMORY.md");
+	if (existsSync(rootMemoryMd)) filesToIndex.push({
+		absPath: rootMemoryMd,
+		rootBase: rootPath,
+		source: "memory",
+		tier: "evergreen"
+	});
+	const memoryDir = join(rootPath, "memory");
+	for (const absPath of walkMdFiles(memoryDir)) {
+		const tier = detectTier(relative(rootPath, absPath));
+		filesToIndex.push({
+			absPath,
+			rootBase: rootPath,
+			source: "memory",
+			tier
+		});
+	}
+	const notesDir = join(rootPath, "Notes");
+	for (const absPath of walkMdFiles(notesDir)) filesToIndex.push({
+		absPath,
+		rootBase: rootPath,
+		source: "notes",
+		tier: "session"
+	});
+	{
+		const SESSION_TITLE_RE = /^(\d{4})\s*-\s*(\d{4}-\d{2}-\d{2})\s*-\s*(.+)\.md$/;
+		const titleInsertChunk = db.prepare(`
+      INSERT OR IGNORE INTO memory_chunks (id, project_id, source, tier, path, start_line, end_line, hash, text, updated_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+		const titleInsertFts = db.prepare(`
+      INSERT OR IGNORE INTO memory_fts (text, id, project_id, path, source, tier, start_line, end_line)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+		const updatedAt = Date.now();
+		for (const absPath of walkMdFiles(notesDir)) {
+			const fileName = basename(absPath);
+			const m = SESSION_TITLE_RE.exec(fileName);
+			if (!m) continue;
+			const [, num, date, title] = m;
+			const text = `Session #${num} ${date}: ${title}`;
+			const syntheticPath = `${relative(rootPath, absPath)}::title`;
+			const id = chunkId(projectId, syntheticPath, 0, 0, 0);
+			const hash = sha256File(text);
+			db.transaction(() => {
+				titleInsertChunk.run(id, projectId, "notes", "session", syntheticPath, 0, 0, hash, text, updatedAt);
+				titleInsertFts.run(text, id, projectId, syntheticPath, "notes", "session", 0, 0);
+			})();
+		}
+	}
+	if (!isPathTooBroadForContentScan(rootPath)) for (const absPath of walkContentFiles(rootPath)) filesToIndex.push({
+		absPath,
+		rootBase: rootPath,
+		source: "content",
+		tier: "topic"
+	});
+	if (claudeNotesDir && claudeNotesDir !== notesDir) {
+		for (const absPath of walkMdFiles(claudeNotesDir)) filesToIndex.push({
+			absPath,
+			rootBase: claudeNotesDir,
+			source: "notes",
+			tier: "session"
+		});
+		{
+			const SESSION_TITLE_RE_CLAUDE = /^(\d{4})\s*-\s*(\d{4}-\d{2}-\d{2})\s*-\s*(.+)\.md$/;
+			const updatedAt = Date.now();
+			const titleInsertChunk2 = db.prepare(`
+        INSERT OR IGNORE INTO memory_chunks (id, project_id, source, tier, path, start_line, end_line, hash, text, updated_at)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `);
+			const titleInsertFts2 = db.prepare(`
+        INSERT OR IGNORE INTO memory_fts (text, id, project_id, path, source, tier, start_line, end_line)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+      `);
+			for (const absPath of walkMdFiles(claudeNotesDir)) {
+				const fileName = basename(absPath);
+				const m = SESSION_TITLE_RE_CLAUDE.exec(fileName);
+				if (!m) continue;
+				const [, num, date, title] = m;
+				const text = `Session #${num} ${date}: ${title}`;
+				const syntheticPath = `${relative(claudeNotesDir, absPath)}::title`;
+				const id = chunkId(projectId, syntheticPath, 0, 0, 0);
+				const hash = sha256File(text);
+				db.transaction(() => {
+					titleInsertChunk2.run(id, projectId, "notes", "session", syntheticPath, 0, 0, hash, text, updatedAt);
+					titleInsertFts2.run(text, id, projectId, syntheticPath, "notes", "session", 0, 0);
+				})();
+			}
+		}
+		if (claudeNotesDir.endsWith("/Notes")) {
+			const claudeProjectDir = claudeNotesDir.slice(0, -6);
+			const claudeMemoryDir = join(claudeProjectDir, "memory");
+			const claudeMemoryMd = join(claudeProjectDir, "MEMORY.md");
+			if (existsSync(claudeMemoryMd)) filesToIndex.push({
+				absPath: claudeMemoryMd,
+				rootBase: claudeProjectDir,
+				source: "memory",
+				tier: "evergreen"
+			});
+			for (const absPath of walkMdFiles(claudeMemoryDir)) {
+				const tier = detectTier(relative(claudeProjectDir, absPath));
+				filesToIndex.push({
+					absPath,
+					rootBase: claudeProjectDir,
+					source: "memory",
+					tier
+				});
+			}
+		}
+	}
+	await yieldToEventLoop();
+	let filesSinceYield = 0;
+	for (const { absPath, rootBase, source, tier } of filesToIndex) {
+		if (filesSinceYield >= INDEX_YIELD_EVERY) {
+			await yieldToEventLoop();
+			filesSinceYield = 0;
+		}
+		filesSinceYield++;
+		const relPath = relative(rootBase, absPath);
+		if (indexFile(db, projectId, rootBase, relPath, source, tier)) {
+			const count = db.prepare("SELECT COUNT(*) as n FROM memory_chunks WHERE project_id = ? AND path = ?").get(projectId, relPath);
+			result.filesProcessed++;
+			result.chunksCreated += count.n;
+		} else result.filesSkipped++;
+	}
+	return result;
+}
+/**
+* Yield to the Node.js event loop between projects so the IPC server
+* remains responsive during long index runs.
+*/
+function yieldToEventLoop() {
+	return new Promise((resolve) => setImmediate(resolve));
+}
+/**
+* Index all active projects registered in the registry DB.
+*
+* Async: yields to the event loop between each project so that the daemon's
+* Unix socket server can process IPC requests (e.g. status) while indexing.
+*/
+async function indexAll(db, registryDb) {
+	const projects = registryDb.prepare("SELECT id, root_path, claude_notes_dir FROM projects WHERE status = 'active'").all();
+	const totals = {
+		filesProcessed: 0,
+		chunksCreated: 0,
+		filesSkipped: 0
+	};
+	for (const project of projects) {
+		await yieldToEventLoop();
+		const r = await indexProject(db, project.id, project.root_path, project.claude_notes_dir);
+		totals.filesProcessed += r.filesProcessed;
+		totals.chunksCreated += r.chunksCreated;
+		totals.filesSkipped += r.filesSkipped;
+	}
+	return {
+		projects: projects.length,
+		result: totals
+	};
+}
+/**
+* Generate and store embeddings for chunks that do not yet have one.
+*
+* Because better-sqlite3 is synchronous but the embedding pipeline is async,
+* we fetch all unembedded chunk texts first, generate embeddings in batches,
+* and then write them back in a transaction.
+*
+* @param db         Open federation database.
+* @param projectId  Optional — restrict to a specific project.
+* @param batchSize  Number of chunks to embed per round. Default 50.
+* @param onProgress Optional callback called after each batch with running totals.
+*/
+async function embedChunks(db, projectId, batchSize = 50, onProgress) {
+	const { generateEmbedding, serializeEmbedding } = await import("./embeddings-mfqv-jFu.mjs").then((n) => n.i);
+	const conditions = ["embedding IS NULL"];
+	const params = [];
+	if (projectId !== void 0) {
+		conditions.push("project_id = ?");
+		params.push(projectId);
+	}
+	const where = "WHERE " + conditions.join(" AND ");
+	const rows = db.prepare(`SELECT id, text FROM memory_chunks ${where} ORDER BY id`).all(...params);
+	if (rows.length === 0) return {
+		chunksEmbedded: 0,
+		chunksSkipped: 0
+	};
+	const updateStmt = db.prepare("UPDATE memory_chunks SET embedding = ? WHERE id = ?");
+	let embedded = 0;
+	const total = rows.length;
+	for (let i = 0; i < rows.length; i += batchSize) {
+		const batch = rows.slice(i, i + batchSize);
+		const embeddings = [];
+		for (const row of batch) {
+			const blob = serializeEmbedding(await generateEmbedding(row.text));
+			embeddings.push({
+				id: row.id,
+				blob
+			});
+		}
+		db.transaction(() => {
+			for (const { id, blob } of embeddings) updateStmt.run(blob, id);
+		})();
+		embedded += embeddings.length;
+		onProgress?.(embedded, total);
+	}
+	return {
+		chunksEmbedded: embedded,
+		chunksSkipped: 0
+	};
+}
+
+//#endregion
+export { indexProject as a, indexFile as i, embedChunks as n, chunkMarkdown as o, indexAll as r, estimateTokens as s, detectTier as t };
+//# sourceMappingURL=indexer-B20bPHL-.mjs.map