mdcontext 0.0.1 → 0.2.0

package/dist/chunk-QNN4TT23.js

@@ -0,0 +1,1430 @@
+// src/utils/tokens.ts
+import { Effect } from "effect";
+
+var encoder = null;
+var getEncoder = Effect.gen(function* () {
+	if (encoder === null) {
+		const { get_encoding } = yield* Effect.promise(() => import("tiktoken"));
+		encoder = get_encoding("cl100k_base");
+	}
+	return encoder;
+});
+var countTokens = (text) =>
+	Effect.gen(function* () {
+		const enc = yield* getEncoder;
+		const tokens = enc.encode(text);
+		return tokens.length;
+	});
+var countTokensApprox = (text) => {
+	if (text.length === 0) return 0;
+	const cjkPattern =
+		/[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af\u3400-\u4dbf]/g;
+	const cjkMatches = text.match(cjkPattern) || [];
+	const cjkCount = cjkMatches.length;
+	const emojiPattern =
+		/[\u{1F300}-\u{1F9FF}\u{2600}-\u{26FF}\u{2700}-\u{27BF}\u{1F600}-\u{1F64F}\u{1F680}-\u{1F6FF}\u{2300}-\u{23FF}\u{2190}-\u{21FF}\u{25A0}-\u{25FF}\u{2B00}-\u{2BFF}]/gu;
+	const emojiMatches = text.match(emojiPattern) || [];
+	const emojiCount = emojiMatches.length;
+	const variationSelectorPattern = /[\uFE0E\uFE0F]/g;
+	const variationMatches = text.match(variationSelectorPattern) || [];
+	const variationCount = variationMatches.length;
+	let workingText = text;
+	const codeBlockMatches = text.match(/```[\s\S]*?```/g) || [];
+	let codeBlockTokens = 0;
+	for (const block of codeBlockMatches) {
+		const hasLang = /^```\w+/.test(block);
+		const overhead = hasLang ? 6 : 4;
+		const content = block.replace(/^```\w*\n?/, "").replace(/\n?```$/, "");
+		const contentNewlines = (content.match(/\n/g) || []).length;
+		const contentTokens = content.length > 0 ? content.length / 2.5 : 0;
+		codeBlockTokens += Math.max(
+			overhead,
+			overhead + contentNewlines + contentTokens,
+		);
+		workingText = workingText.replace(block, "");
+	}
+	const inlineCodeMatches = workingText.match(/`[^`]+`/g) || [];
+	let inlineCodeTokens = 0;
+	for (const match of inlineCodeMatches) {
+		const content = match.slice(1, -1);
+		inlineCodeTokens += 2 + content.length / 2.5;
+		workingText = workingText.replace(match, "");
+	}
+	const pathMatches = workingText.match(/(?:\/[\w.-]+)+/g) || [];
+	let pathTokens = 0;
+	for (const match of pathMatches) {
+		const slashCount = (match.match(/\//g) || []).length;
+		const contentLength = match.length - slashCount;
+		pathTokens += slashCount + contentLength / 3.5;
+		workingText = workingText.replace(match, "");
+	}
+	const punctuationMatches =
+		workingText.match(/[!?,.:;'"()[\]{}@#$%^&*+=|\\<>~\-/]/g) || [];
+	const punctuationCount = punctuationMatches.length;
+	const proseNewlines = (workingText.match(/\n/g) || []).length;
+	const proseLength = Math.max(
+		0,
+		workingText.length -
+			proseNewlines -
+			cjkCount -
+			emojiCount -
+			variationCount -
+			punctuationCount,
+	);
+	const proseTokens = proseLength / 3.5;
+	const proseNewlineTokens = proseNewlines * 1;
+	const punctuationBonus = punctuationCount * 0.8;
+	const cjkTokens = cjkCount * 1.2;
+	const emojiTokens = emojiCount * 2.5;
+	const variationTokens = variationCount * 1;
+	const estimate =
+		proseTokens +
+		proseNewlineTokens +
+		codeBlockTokens +
+		inlineCodeTokens +
+		pathTokens +
+		punctuationBonus +
+		cjkTokens +
+		emojiTokens +
+		variationTokens;
+	return Math.ceil(estimate * 1.1);
+};
+var countWords = (text) => {
+	const trimmed = text.trim();
+	if (trimmed.length === 0) return 0;
+	return trimmed.split(/\s+/).length;
+};
+var freeEncoder = () => {
+	if (encoder !== null) {
+		encoder.free();
+		encoder = null;
+	}
+};
+
+// src/parser/parser.ts
+import * as crypto from "crypto";
+// src/errors/index.ts
+import { Data, Effect as Effect2 } from "effect";
+import matter from "gray-matter";
+import remarkGfm from "remark-gfm";
+import remarkParse from "remark-parse";
+import { unified } from "unified";
+import { visit } from "unist-util-visit";
+
+var ErrorCode = {
+	// File system errors (E1xx)
+	FILE_READ: "E100",
+	FILE_WRITE: "E101",
+	DIRECTORY_CREATE: "E102",
+	DIRECTORY_WALK: "E103",
+	// Parse errors (E2xx)
+	PARSE: "E200",
+	// API/authentication errors (E3xx)
+	API_KEY_MISSING: "E300",
+	API_KEY_INVALID: "E301",
+	EMBEDDING_RATE_LIMIT: "E310",
+	EMBEDDING_QUOTA: "E311",
+	EMBEDDING_NETWORK: "E312",
+	EMBEDDING_MODEL: "E313",
+	EMBEDDING_UNKNOWN: "E319",
+	// Index errors (E4xx)
+	INDEX_NOT_FOUND: "E400",
+	INDEX_CORRUPTED: "E401",
+	INDEX_BUILD: "E402",
+	// Search errors (E5xx)
+	DOCUMENT_NOT_FOUND: "E500",
+	EMBEDDINGS_NOT_FOUND: "E501",
+	// Vector store errors (E6xx)
+	VECTOR_STORE: "E600",
+	// Config errors (E7xx)
+	CONFIG: "E700",
+	// Watch errors (E8xx)
+	WATCH: "E800",
+	// CLI errors (E9xx)
+	CLI_VALIDATION: "E900",
+};
+var FileReadError = class extends Data.TaggedError("FileReadError") {
+	get code() {
+		return ErrorCode.FILE_READ;
+	}
+};
+var FileWriteError = class extends Data.TaggedError("FileWriteError") {
+	get code() {
+		return ErrorCode.FILE_WRITE;
+	}
+};
+var DirectoryCreateError = class extends Data.TaggedError(
+	"DirectoryCreateError",
+) {
+	get code() {
+		return ErrorCode.DIRECTORY_CREATE;
+	}
+};
+var DirectoryWalkError = class extends Data.TaggedError("DirectoryWalkError") {
+	get code() {
+		return ErrorCode.DIRECTORY_WALK;
+	}
+};
+var ParseError = class extends Data.TaggedError("ParseError") {
+	get code() {
+		return ErrorCode.PARSE;
+	}
+};
+var ApiKeyMissingError = class extends Data.TaggedError("ApiKeyMissingError") {
+	get code() {
+		return ErrorCode.API_KEY_MISSING;
+	}
+	get message() {
+		return `${this.envVar} not set`;
+	}
+};
+var ApiKeyInvalidError = class extends Data.TaggedError("ApiKeyInvalidError") {
+	get code() {
+		return ErrorCode.API_KEY_INVALID;
+	}
+	get message() {
+		return this.details ?? `Invalid API key for ${this.provider}`;
+	}
+};
+var EmbeddingError = class extends Data.TaggedError("EmbeddingError") {
+	get code() {
+		switch (this.reason) {
+			case "RateLimit":
+				return ErrorCode.EMBEDDING_RATE_LIMIT;
+			case "QuotaExceeded":
+				return ErrorCode.EMBEDDING_QUOTA;
+			case "Network":
+				return ErrorCode.EMBEDDING_NETWORK;
+			case "ModelError":
+				return ErrorCode.EMBEDDING_MODEL;
+			default:
+				return ErrorCode.EMBEDDING_UNKNOWN;
+		}
+	}
+};
+var IndexNotFoundError = class extends Data.TaggedError("IndexNotFoundError") {
+	get code() {
+		return ErrorCode.INDEX_NOT_FOUND;
+	}
+	get message() {
+		return `Index not found at ${this.path}`;
+	}
+};
+var IndexCorruptedError = class extends Data.TaggedError(
+	"IndexCorruptedError",
+) {
+	get code() {
+		return ErrorCode.INDEX_CORRUPTED;
+	}
+	get message() {
+		return `Index corrupted at ${this.path}: ${this.reason}`;
+	}
+};
+var IndexBuildError = class extends Data.TaggedError("IndexBuildError") {
+	get code() {
+		return ErrorCode.INDEX_BUILD;
+	}
+};
+var DocumentNotFoundError = class extends Data.TaggedError(
+	"DocumentNotFoundError",
+) {
+	get code() {
+		return ErrorCode.DOCUMENT_NOT_FOUND;
+	}
+	get message() {
+		return `Document not found in index: ${this.path}`;
+	}
+};
+var ConfigError = class extends Data.TaggedError("ConfigError") {
+	get code() {
+		return ErrorCode.CONFIG;
+	}
+};
+var VectorStoreError = class extends Data.TaggedError("VectorStoreError") {
+	get code() {
+		return ErrorCode.VECTOR_STORE;
+	}
+};
+var EmbeddingsNotFoundError = class extends Data.TaggedError(
+	"EmbeddingsNotFoundError",
+) {
+	get code() {
+		return ErrorCode.EMBEDDINGS_NOT_FOUND;
+	}
+	get message() {
+		return `Embeddings not found at ${this.path}. Run 'mdcontext index --embed' first.`;
+	}
+};
+var WatchError = class extends Data.TaggedError("WatchError") {
+	get code() {
+		return ErrorCode.WATCH;
+	}
+};
+var CliValidationError = class extends Data.TaggedError("CliValidationError") {
+	get code() {
+		return ErrorCode.CLI_VALIDATION;
+	}
+};
+
+// src/parser/parser.ts
+var processor = unified().use(remarkParse).use(remarkGfm);
+var generateId = (input) => {
+	return crypto.createHash("md5").update(input).digest("hex").slice(0, 12);
+};
+var slugify = (text) => {
+	return text
+		.toLowerCase()
+		.replace(/[^\w\s-]/g, "")
+		.replace(/\s+/g, "-")
+		.replace(/-+/g, "-")
+		.trim();
+};
+var isInternalLink = (href) => {
+	if (href.startsWith("http://") || href.startsWith("https://")) return false;
+	if (href.startsWith("mailto:")) return false;
+	if (href.startsWith("#")) return true;
+	if (href.endsWith(".md") || href.includes(".md#")) return true;
+	return !href.includes("://");
+};
+var extractPlainText = (node) => {
+	const texts = [];
+	visit(node, "text", (textNode) => {
+		texts.push(textNode.value);
+	});
+	return texts.join(" ");
+};
+var getNodeEndLine = (node) => {
+	return node?.position?.end?.line ?? 0;
+};
+var getNodeStartLine = (node) => {
+	return node?.position?.start?.line ?? 0;
+};
+var extractRawSections = (tree) => {
+	const sections = [];
+	const headings = [];
+	tree.children.forEach((node, index) => {
+		if (node.type === "heading") {
+			const heading = node;
+			headings.push({
+				heading: extractPlainText(heading),
+				level: heading.depth,
+				line: getNodeStartLine(node),
+				index,
+			});
+		}
+	});
+	headings.forEach((h, i) => {
+		const nextHeading = headings[i + 1];
+		const endIndex = nextHeading ? nextHeading.index : tree.children.length;
+		const contentNodes = tree.children.slice(h.index + 1, endIndex);
+		const lastContentNode = contentNodes[contentNodes.length - 1];
+		const endLine = lastContentNode ? getNodeEndLine(lastContentNode) : h.line;
+		sections.push({
+			heading: h.heading,
+			level: h.level,
+			startLine: h.line,
+			endLine,
+			contentStartLine: h.line + 1,
+			contentNodes,
+		});
+	});
+	return sections;
+};
+var buildSectionHierarchy = (rawSections, docId, lines) => {
+	const result = [];
+	const stack = [];
+	for (const raw of rawSections) {
+		const contentLines = lines.slice(raw.startLine - 1, raw.endLine);
+		const content = contentLines.join("\n");
+		const plainText = extractSectionPlainText(raw.contentNodes);
+		const hasCode = raw.contentNodes.some((n) => n.type === "code");
+		const hasList = raw.contentNodes.some((n) => n.type === "list");
+		const hasTable = raw.contentNodes.some((n) => n.type === "table");
+		const section = {
+			id: `${docId}-${slugify(raw.heading)}`,
+			heading: raw.heading,
+			level: raw.level,
+			content,
+			plainText,
+			startLine: raw.startLine,
+			endLine: raw.endLine,
+			children: [],
+			metadata: {
+				wordCount: countWords(plainText),
+				tokenCount: countTokensApprox(content),
+				hasCode,
+				hasList,
+				hasTable,
+			},
+		};
+		while (stack.length > 0 && stack[stack.length - 1].level >= raw.level) {
+			stack.pop();
+		}
+		if (stack.length === 0) {
+			result.push(section);
+		} else {
+			const parent = stack[stack.length - 1];
+			parent.section.children.push(section);
+		}
+		stack.push({ section, level: raw.level });
+	}
+	return result;
+};
+var extractSectionPlainText = (nodes) => {
+	const texts = [];
+	for (const node of nodes) {
+		if ("value" in node && typeof node.value === "string") {
+			texts.push(node.value);
+		} else if ("children" in node) {
+			texts.push(extractPlainText(node));
+		}
+	}
+	return texts.join(" ");
+};
+var countAllSections = (sections) => {
+	let count = 0;
+	for (const section of sections) {
+		count += 1;
+		count += countAllSections(section.children);
+	}
+	return count;
+};
+var extractLinks = (tree, docId) => {
+	const links = [];
+	let currentSectionId = docId;
+	visit(tree, (node) => {
+		if (node.type === "heading") {
+			currentSectionId = `${docId}-${slugify(extractPlainText(node))}`;
+		}
+		if (node.type === "link") {
+			const link = node;
+			const internal = isInternalLink(link.url);
+			links.push({
+				type: internal ? "internal" : "external",
+				href: link.url,
+				text: extractPlainText(link),
+				sectionId: currentSectionId,
+				line: getNodeStartLine(node),
+			});
+		}
+		if (node.type === "image") {
+			const img = node;
+			links.push({
+				type: "image",
+				href: img.url,
+				text: img.alt ?? "",
+				sectionId: currentSectionId,
+				line: getNodeStartLine(node),
+			});
+		}
+	});
+	return links;
+};
+var extractCodeBlocks = (tree, docId) => {
+	const codeBlocks = [];
+	let currentSectionId = docId;
+	visit(tree, (node) => {
+		if (node.type === "heading") {
+			currentSectionId = `${docId}-${slugify(extractPlainText(node))}`;
+		}
+		if (node.type === "code") {
+			const code = node;
+			codeBlocks.push({
+				language: code.lang ?? null,
+				content: code.value,
+				sectionId: currentSectionId,
+				startLine: getNodeStartLine(node),
+				endLine: getNodeEndLine(node),
+			});
+		}
+	});
+	return codeBlocks;
+};
+var parse = (content, options = {}) =>
+	Effect2.gen(function* () {
+		const path5 = options.path ?? "unknown";
+		const docId = generateId(path5);
+		const now = /* @__PURE__ */ new Date();
+		let frontmatter = {};
+		let markdownContent = content;
+		try {
+			const parsed = matter(content);
+			frontmatter = parsed.data;
+			markdownContent = parsed.content;
+		} catch (error) {
+			const msg = error instanceof Error ? error.message : String(error);
+			console.warn(
+				`Warning: Malformed frontmatter in ${path5}, skipping: ${msg.split("\n")[0]}`,
+			);
+		}
+		const tree = processor.parse(markdownContent);
+		const lines = markdownContent.split("\n");
+		const rawSections = extractRawSections(tree);
+		const sections = buildSectionHierarchy(rawSections, docId, lines);
+		const links = extractLinks(tree, docId);
+		const codeBlocks = extractCodeBlocks(tree, docId);
+		const firstH1 = sections.find((s) => s.level === 1);
+		const title =
+			firstH1?.heading ??
+			(typeof frontmatter.title === "string" ? frontmatter.title : null) ??
+			path5.split("/").pop()?.replace(/\.md$/, "") ??
+			"Untitled";
+		const totalContent = sections.map((s) => s.content).join("\n");
+		const metadata = {
+			wordCount: countWords(totalContent),
+			tokenCount: countTokensApprox(content),
+			headingCount: countAllSections(sections),
+			linkCount: links.length,
+			codeBlockCount: codeBlocks.length,
+			lastModified: options.lastModified ?? now,
+			indexedAt: now,
+		};
+		const document = {
+			id: docId,
+			path: path5,
+			title,
+			frontmatter,
+			sections,
+			links,
+			codeBlocks,
+			metadata,
+		};
+		return document;
+	});
+var parseFile = (filePath) =>
+	Effect2.gen(function* () {
+		const fs5 = yield* Effect2.promise(() => import("fs/promises"));
+		const [content, stats] = yield* Effect2.tryPromise({
+			try: () =>
+				Promise.all([fs5.readFile(filePath, "utf-8"), fs5.stat(filePath)]),
+			catch: (error) =>
+				new FileReadError({
+					path: filePath,
+					message: error instanceof Error ? error.message : "Unknown error",
+					cause: error,
+				}),
+		});
+		return yield* parse(content, {
+			path: filePath,
+			lastModified: stats.mtime,
+		});
+	});
+
+// src/index/types.ts
+var INDEX_DIR = ".mdcontext";
+var INDEX_VERSION = 1;
+var getIndexPaths = (rootPath) => ({
+	root: `${rootPath}/${INDEX_DIR}`,
+	config: `${rootPath}/${INDEX_DIR}/config.json`,
+	documents: `${rootPath}/${INDEX_DIR}/indexes/documents.json`,
+	sections: `${rootPath}/${INDEX_DIR}/indexes/sections.json`,
+	links: `${rootPath}/${INDEX_DIR}/indexes/links.json`,
+	cache: `${rootPath}/${INDEX_DIR}/cache`,
+	parsed: `${rootPath}/${INDEX_DIR}/cache/parsed`,
+});
+
+// src/index/storage.ts
+import * as crypto2 from "crypto";
+import { Effect as Effect3 } from "effect";
+import * as fs from "fs/promises";
+import * as path from "path";
+
+var ensureDir = (dirPath) =>
+	Effect3.tryPromise({
+		try: () => fs.mkdir(dirPath, { recursive: true }),
+		catch: (e) =>
+			new DirectoryCreateError({
+				path: dirPath,
+				message: e instanceof Error ? e.message : String(e),
+				cause: e,
+			}),
+	}).pipe(Effect3.map(() => void 0));
+var readJsonFile = (filePath) =>
+	Effect3.gen(function* () {
+		const contentResult = yield* Effect3.tryPromise({
+			try: () => fs.readFile(filePath, "utf-8"),
+			catch: (e) => {
+				if (e && typeof e === "object" && "code" in e && e.code === "ENOENT") {
+					return { notFound: true };
+				}
+				return new FileReadError({
+					path: filePath,
+					message: e instanceof Error ? e.message : String(e),
+					cause: e,
+				});
+			},
+		}).pipe(
+			Effect3.map((content) =>
+				typeof content === "string" ? { content } : content,
+			),
+			// Note: catchAll here filters out "file not found" as expected case (returns null),
+			// while other errors are re-thrown to propagate as typed FileReadError
+			Effect3.catchAll((e) =>
+				e && "notFound" in e
+					? Effect3.succeed({ notFound: true })
+					: Effect3.fail(e),
+			),
+		);
+		if ("notFound" in contentResult) {
+			return null;
+		}
+		return yield* Effect3.try({
+			try: () => JSON.parse(contentResult.content),
+			catch: (e) =>
+				new IndexCorruptedError({
+					path: filePath,
+					reason: "InvalidJson",
+					details: e instanceof Error ? e.message : String(e),
+				}),
+		});
+	});
+var writeJsonFile = (filePath, data) =>
+	Effect3.gen(function* () {
+		const dir = path.dirname(filePath);
+		yield* ensureDir(dir);
+		yield* Effect3.tryPromise({
+			try: () => fs.writeFile(filePath, JSON.stringify(data, null, 2)),
+			catch: (e) =>
+				new FileWriteError({
+					path: filePath,
+					message: e instanceof Error ? e.message : String(e),
+					cause: e,
+				}),
+		});
+	});
+var computeHash = (content) => {
+	return crypto2
+		.createHash("sha256")
+		.update(content)
+		.digest("hex")
+		.slice(0, 16);
+};
+var createStorage = (rootPath) => ({
+	rootPath: path.resolve(rootPath),
+	paths: getIndexPaths(path.resolve(rootPath)),
+});
+var initializeIndex = (storage) =>
+	Effect3.gen(function* () {
+		yield* ensureDir(storage.paths.root);
+		yield* ensureDir(storage.paths.parsed);
+		yield* ensureDir(path.dirname(storage.paths.documents));
+		const existingConfig = yield* loadConfig(storage);
+		if (!existingConfig) {
+			const config = {
+				version: INDEX_VERSION,
+				rootPath: storage.rootPath,
+				include: ["**/*.md", "**/*.mdx"],
+				exclude: ["**/node_modules/**", "**/.*/**"],
+				createdAt: /* @__PURE__ */ new Date().toISOString(),
+				updatedAt: /* @__PURE__ */ new Date().toISOString(),
+			};
+			yield* saveConfig(storage, config);
+		}
+	});
+var loadConfig = (storage) => readJsonFile(storage.paths.config);
+var saveConfig = (storage, config) =>
+	writeJsonFile(storage.paths.config, {
+		...config,
+		updatedAt: /* @__PURE__ */ new Date().toISOString(),
+	});
+var loadDocumentIndex = (storage) => readJsonFile(storage.paths.documents);
+var saveDocumentIndex = (storage, index) =>
+	writeJsonFile(storage.paths.documents, index);
+var createEmptyDocumentIndex = (rootPath) => ({
+	version: INDEX_VERSION,
+	rootPath,
+	documents: {},
+});
+var loadSectionIndex = (storage) => readJsonFile(storage.paths.sections);
+var saveSectionIndex = (storage, index) =>
+	writeJsonFile(storage.paths.sections, index);
+var createEmptySectionIndex = () => ({
+	version: INDEX_VERSION,
+	sections: {},
+	byHeading: {},
+	byDocument: {},
+});
+var loadLinkIndex = (storage) => readJsonFile(storage.paths.links);
+var saveLinkIndex = (storage, index) =>
+	writeJsonFile(storage.paths.links, index);
+var createEmptyLinkIndex = () => ({
+	version: INDEX_VERSION,
+	forward: {},
+	backward: {},
+	broken: [],
+});
+var indexExists = (storage) =>
+	Effect3.tryPromise({
+		try: async () => {
+			try {
+				await fs.access(storage.paths.config);
+				return true;
+			} catch {
+				return false;
+			}
+		},
+		catch: (e) =>
+			new FileReadError({
+				path: storage.paths.config,
+				message: e instanceof Error ? e.message : String(e),
+				cause: e,
+			}),
+	});
+
+import { Effect as Effect4, Effect as Effect6 } from "effect";
+// src/index/indexer.ts
+import * as fs4 from "fs/promises";
+
+// src/index/ignore-patterns.ts
+import * as fs2 from "fs/promises";
+import ignore from "ignore";
+import * as path4 from "path";
+import * as path2 from "path";
+
+var DEFAULT_IGNORE_PATTERNS = ["node_modules", ".git", "dist", "build"];
+var tryReadIgnoreFile = (filePath) =>
+	Effect4.tryPromise({
+		try: () => fs2.readFile(filePath, "utf-8"),
+		catch: () => "",
+	}).pipe(Effect4.catchAll(() => Effect4.succeed("")));
+var countPatterns = (content) => {
+	if (!content.trim()) return 0;
+	return content.split("\n").filter((line) => {
+		const trimmed = line.trim();
+		return trimmed.length > 0 && !trimmed.startsWith("#");
+	}).length;
+};
+var createIgnoreFilter = (options) =>
+	Effect4.gen(function* () {
+		const {
+			rootPath,
+			cliPatterns = [],
+			honorGitignore = true,
+			honorMdcontextignore = true,
+		} = options;
+		const ig = ignore();
+		const sources = [];
+		let patternCount = 0;
+		ig.add(DEFAULT_IGNORE_PATTERNS);
+		patternCount += DEFAULT_IGNORE_PATTERNS.length;
+		if (honorGitignore) {
+			const gitignorePath = path2.join(rootPath, ".gitignore");
+			const gitignoreContent = yield* tryReadIgnoreFile(gitignorePath);
+			if (gitignoreContent.trim()) {
+				ig.add(gitignoreContent);
+				const count = countPatterns(gitignoreContent);
+				patternCount += count;
+				sources.push(".gitignore");
+			}
+		}
+		if (honorMdcontextignore) {
+			const mdcontextignorePath = path2.join(rootPath, ".mdcontextignore");
+			const mdcontextignoreContent =
+				yield* tryReadIgnoreFile(mdcontextignorePath);
+			if (mdcontextignoreContent.trim()) {
+				ig.add(mdcontextignoreContent);
+				const count = countPatterns(mdcontextignoreContent);
+				patternCount += count;
+				sources.push(".mdcontextignore");
+			}
+		}
+		if (cliPatterns.length > 0) {
+			ig.add(cliPatterns);
+			patternCount += cliPatterns.length;
+			sources.push("CLI/config");
+		}
+		return {
+			filter: ig,
+			sources,
+			patternCount,
+		};
+	});
+var shouldIgnore = (relativePath, filter) => {
+	const normalized = relativePath.replace(/^\//, "");
+	return filter.ignores(normalized);
+};
+var getChokidarIgnorePatterns = (options) =>
+	Effect4.gen(function* () {
+		const {
+			rootPath,
+			cliPatterns = [],
+			honorGitignore = true,
+			honorMdcontextignore = true,
+		} = options;
+		const patterns = [];
+		patterns.push(/(^|[/\\])\./.source);
+		for (const p of DEFAULT_IGNORE_PATTERNS) {
+			patterns.push(`**/${p}/**`);
+		}
+		if (honorGitignore) {
+			const gitignorePath = path2.join(rootPath, ".gitignore");
+			const content = yield* tryReadIgnoreFile(gitignorePath);
+			if (content.trim()) {
+				const parsed = parseIgnoreFile(content);
+				for (const p of parsed) {
+					patterns.push(convertToGlob(p));
+				}
+			}
+		}
+		if (honorMdcontextignore) {
+			const mdcontextignorePath = path2.join(rootPath, ".mdcontextignore");
+			const content = yield* tryReadIgnoreFile(mdcontextignorePath);
+			if (content.trim()) {
+				const parsed = parseIgnoreFile(content);
+				for (const p of parsed) {
+					patterns.push(convertToGlob(p));
+				}
+			}
+		}
+		for (const p of cliPatterns) {
+			patterns.push(convertToGlob(p));
+		}
+		return patterns;
+	});
+var parseIgnoreFile = (content) => {
+	return content
+		.split("\n")
+		.map((line) => line.trim())
+		.filter((line) => line.length > 0 && !line.startsWith("#"));
+};
+var convertToGlob = (pattern) => {
+	if (pattern.startsWith("!")) {
+		return pattern;
+	}
+	if (pattern.includes("*") || pattern.includes("/")) {
+		return pattern.startsWith("/") ? pattern.slice(1) : `**/${pattern}`;
+	}
+	return `**/${pattern}/**`;
+};
+
+import { Effect as Effect5 } from "effect";
+// src/search/bm25-store.ts
+import * as fs3 from "fs/promises";
+import * as path3 from "path";
+import bm25 from "wink-bm25-text-search";
+
+var tokenize = (text) => {
+	return text
+		.toLowerCase()
+		.split(/\W+/)
+		.filter((token) => token.length > 2);
+};
+var createBM25Store = (rootPath) => {
+	const resolvedRoot = path3.resolve(rootPath);
+	const indexPath = path3.join(resolvedRoot, INDEX_DIR, "bm25.json");
+	const metadataPath = path3.join(resolvedRoot, INDEX_DIR, "bm25.meta.json");
+	const sectionMap = /* @__PURE__ */ new Map();
+	let documentCount = 0;
+	let consolidated = false;
+	let lastUpdated = /* @__PURE__ */ new Date().toISOString();
+	let engine = bm25();
+	engine.defineConfig({
+		fldWeights: {
+			heading: 2,
+			content: 1,
+		},
+	});
+	engine.definePrepTasks([tokenize]);
+	return {
+		add(docs) {
+			return Effect5.sync(() => {
+				for (const doc of docs) {
+					const idx = documentCount++;
+					sectionMap.set(idx, {
+						sectionId: doc.sectionId,
+						documentPath: doc.documentPath,
+						heading: doc.heading,
+					});
+					engine.addDoc(
+						{
+							heading: doc.heading,
+							content: doc.content,
+						},
+						idx,
+					);
+				}
+				consolidated = false;
+				lastUpdated = /* @__PURE__ */ new Date().toISOString();
+			});
+		},
+		consolidate() {
+			return Effect5.sync(() => {
+				if (!consolidated && documentCount > 0) {
+					engine.consolidate();
+					consolidated = true;
+				}
+			});
+		},
+		search(query, limit = 10) {
+			return Effect5.sync(() => {
+				if (!consolidated || documentCount === 0) {
+					return [];
+				}
+				const results = engine.search(query, limit);
+				return results.map(([idx, score], rank) => {
+					const info = sectionMap.get(idx);
+					return {
+						sectionId: info?.sectionId ?? "",
+						documentPath: info?.documentPath ?? "",
+						heading: info?.heading ?? "",
+						score,
+						rank: rank + 1,
+					};
+				});
+			});
+		},
+		save() {
+			return Effect5.gen(function* () {
+				const jsonModel = engine.exportJSON();
+				const sectionMapArray = Array.from(sectionMap.entries());
+				const data = {
+					engine: jsonModel,
+					sectionMap: sectionMapArray,
+				};
+				const metadata = {
+					version: 1,
+					count: documentCount,
+					lastUpdated,
+				};
+				yield* Effect5.tryPromise({
+					try: async () => {
+						await fs3.writeFile(indexPath, JSON.stringify(data), "utf-8");
+						await fs3.writeFile(
+							metadataPath,
+							JSON.stringify(metadata, null, 2),
+							"utf-8",
+						);
+					},
+					catch: (e) =>
+						new FileWriteError({
+							path: indexPath,
+							message: `Failed to save BM25 index: ${e instanceof Error ? e.message : String(e)}`,
+						}),
+				});
+			});
+		},
+		load() {
+			return Effect5.gen(function* () {
+				const exists = yield* Effect5.promise(async () => {
+					try {
+						await fs3.access(indexPath);
+						return true;
+					} catch {
+						return false;
+					}
+				});
+				if (!exists) {
+					return false;
+				}
+				const [dataStr, metaStr] = yield* Effect5.tryPromise({
+					try: async () => {
+						const data2 = await fs3.readFile(indexPath, "utf-8");
+						const meta = await fs3.readFile(metadataPath, "utf-8");
+						return [data2, meta];
+					},
+					catch: (e) =>
+						new FileReadError({
+							path: indexPath,
+							message: `Failed to load BM25 index: ${e instanceof Error ? e.message : String(e)}`,
+						}),
+				});
+				const data = JSON.parse(dataStr);
+				const metadata = JSON.parse(metaStr);
+				engine = bm25();
+				engine.importJSON(data.engine);
+				engine.definePrepTasks([tokenize]);
+				sectionMap.clear();
+				for (const [idx, info] of data.sectionMap) {
+					sectionMap.set(idx, info);
+				}
+				documentCount = metadata.count;
+				lastUpdated = metadata.lastUpdated;
+				consolidated = true;
+				return true;
+			});
+		},
+		getStats() {
+			return {
+				count: documentCount,
+				lastUpdated,
+			};
+		},
+		isConsolidated() {
+			return consolidated;
+		},
+		clear() {
+			engine = bm25();
+			engine.defineConfig({
+				fldWeights: {
+					heading: 2,
+					content: 1,
+				},
+			});
+			engine.definePrepTasks([tokenize]);
+			sectionMap.clear();
+			documentCount = 0;
+			consolidated = false;
+			lastUpdated = /* @__PURE__ */ new Date().toISOString();
+		},
+	};
+};
+var bm25Search = (rootPath, query, limit = 10) =>
+	Effect5.gen(function* () {
+		const store = createBM25Store(rootPath);
+		const loaded = yield* store.load();
+		if (!loaded) {
+			return [];
+		}
+		return yield* store.search(query, limit);
+	});
+var bm25IndexExists = (rootPath) =>
+	Effect5.promise(async () => {
+		const resolvedRoot = path3.resolve(rootPath);
+		const indexPath = path3.join(resolvedRoot, INDEX_DIR, "bm25.json");
+		try {
+			await fs3.access(indexPath);
+			return true;
+		} catch {
+			return false;
+		}
+	});
+
+// src/index/indexer.ts
+var isMarkdownFile = (filename) =>
+	filename.endsWith(".md") || filename.endsWith(".mdx");
+var walkDirectory = async (dir, rootPath, filter) => {
+	const files = [];
+	let hiddenCount = 0;
+	let excludedCount = 0;
+	const entries = await fs4.readdir(dir, { withFileTypes: true });
+	for (const entry of entries) {
+		const fullPath = path4.join(dir, entry.name);
+		const relativePath = path4.relative(rootPath, fullPath);
+		if (entry.name.startsWith(".")) {
+			if (entry.isDirectory()) {
+				hiddenCount++;
+			}
+			continue;
+		}
+		if (shouldIgnore(relativePath, filter)) {
+			if (entry.isDirectory()) {
+				excludedCount++;
+			} else {
+				excludedCount++;
+			}
+			continue;
+		}
+		if (entry.isDirectory()) {
+			const subResult = await walkDirectory(fullPath, rootPath, filter);
+			files.push(...subResult.files);
+			hiddenCount += subResult.skipped.hidden;
+			excludedCount += subResult.skipped.excluded;
+		} else if (entry.isFile() && isMarkdownFile(entry.name)) {
+			files.push(fullPath);
+		}
+	}
+	return { files, skipped: { hidden: hiddenCount, excluded: excludedCount } };
+};
+var flattenSections = (sections, docId, docPath) => {
+	const result = [];
+	const traverse = (section) => {
+		result.push({
+			id: section.id,
+			documentId: docId,
+			documentPath: docPath,
+			heading: section.heading,
+			level: section.level,
+			startLine: section.startLine,
+			endLine: section.endLine,
+			tokenCount: section.metadata.tokenCount,
+			hasCode: section.metadata.hasCode,
+			hasList: section.metadata.hasList,
+			hasTable: section.metadata.hasTable,
+		});
+		for (const child of section.children) {
+			traverse(child);
+		}
+	};
+	for (const section of sections) {
+		traverse(section);
+	}
+	return result;
+};
+var resolveInternalLink = (href, fromPath, rootPath) => {
+	if (href.startsWith("#")) {
+		return fromPath;
+	}
+	if (href.startsWith("http://") || href.startsWith("https://")) {
+		return null;
+	}
+	const linkPath = href.split("#")[0] ?? "";
+	if (!linkPath) return null;
+	const fromDir = path4.dirname(fromPath);
+	const resolved = path4.resolve(fromDir, linkPath);
+	if (!resolved.startsWith(rootPath)) {
+		return null;
+	}
+	return path4.relative(rootPath, resolved);
+};
+var buildIndex = (rootPath, options = {}) =>
+	Effect6.gen(function* () {
+		const startTime = Date.now();
+		const storage = createStorage(rootPath);
+		const errors = [];
+		yield* initializeIndex(storage);
+		const existingDocIndex = yield* loadDocumentIndex(storage);
+		const docIndex =
+			options.force || !existingDocIndex
+				? createEmptyDocumentIndex(storage.rootPath)
+				: existingDocIndex;
+		const existingSectionIndex = yield* loadSectionIndex(storage);
+		const existingLinkIndex = yield* loadLinkIndex(storage);
+		const sectionIndex = existingSectionIndex ?? createEmptySectionIndex();
+		const linkIndex = existingLinkIndex ?? createEmptyLinkIndex();
+		const ignoreResult = yield* createIgnoreFilter({
+			rootPath: storage.rootPath,
+			cliPatterns: options.exclude,
+			honorGitignore: options.honorGitignore ?? true,
+			honorMdcontextignore: options.honorMdcontextignore ?? true,
+		});
+		const walkResult = yield* Effect6.tryPromise({
+			try: () =>
+				walkDirectory(storage.rootPath, storage.rootPath, ignoreResult.filter),
+			catch: (e) =>
+				new DirectoryWalkError({
+					path: storage.rootPath,
+					message: `Failed to traverse directory: ${e instanceof Error ? e.message : String(e)}`,
+					cause: e,
+				}),
+		});
+		const { files, skipped: walkSkipped } = walkResult;
+		let documentsIndexed = 0;
+		let sectionsIndexed = 0;
+		let linksIndexed = 0;
+		let unchangedCount = 0;
+		const mutableDocuments = {
+			...docIndex.documents,
+		};
+		const mutableSections = {
+			...sectionIndex.sections,
+		};
+		const mutableByHeading = Object.fromEntries(
+			Object.entries(sectionIndex.byHeading).map(([k, v]) => [k, [...v]]),
+		);
+		const mutableByDocument = Object.fromEntries(
+			Object.entries(sectionIndex.byDocument).map(([k, v]) => [k, [...v]]),
+		);
+		const mutableForward = Object.fromEntries(
+			Object.entries(linkIndex.forward).map(([k, v]) => [k, [...v]]),
+		);
+		const mutableBackward = Object.fromEntries(
+			Object.entries(linkIndex.backward).map(([k, v]) => [k, [...v]]),
+		);
+		const brokenLinks = [...linkIndex.broken];
+		for (const filePath of files) {
+			const relativePath = path4.relative(storage.rootPath, filePath);
+			const processFile = Effect6.gen(function* () {
+				const [content, stats] = yield* Effect6.promise(() =>
+					Promise.all([fs4.readFile(filePath, "utf-8"), fs4.stat(filePath)]),
+				);
+				const hash = computeHash(content);
+				const existingEntry = mutableDocuments[relativePath];
+				if (
+					!options.force &&
+					existingEntry &&
+					existingEntry.hash === hash &&
+					existingEntry.mtime === stats.mtime.getTime()
+				) {
+					unchangedCount++;
+					return;
+				}
+				const doc = yield* parse(content, {
+					path: relativePath,
+					lastModified: stats.mtime,
+				}).pipe(
+					Effect6.mapError(
+						(e) =>
+							new ParseError({
+								message: e.message,
+								path: relativePath,
+								...(e.line !== void 0 && { line: e.line }),
+								...(e.column !== void 0 && { column: e.column }),
+							}),
+					),
+				);
+				if (existingEntry) {
+					const oldSectionIds = mutableByDocument[existingEntry.id] ?? [];
+					for (const sectionId of oldSectionIds) {
+						const oldSection = mutableSections[sectionId];
+						if (oldSection) {
+							const headingKey = oldSection.heading.toLowerCase();
+							const headingList = mutableByHeading[headingKey];
+							if (headingList) {
+								const idx = headingList.indexOf(sectionId);
+								if (idx !== -1) headingList.splice(idx, 1);
+							}
+						}
+						delete mutableSections[sectionId];
+					}
+					delete mutableByDocument[existingEntry.id];
+					delete mutableForward[relativePath];
+				}
+				mutableDocuments[relativePath] = {
+					id: doc.id,
+					path: relativePath,
+					title: doc.title,
+					mtime: stats.mtime.getTime(),
+					hash,
+					tokenCount: doc.metadata.tokenCount,
+					sectionCount: doc.metadata.headingCount,
+				};
+				documentsIndexed++;
+				const sections = flattenSections(doc.sections, doc.id, relativePath);
+				mutableByDocument[doc.id] = [];
+				for (const section of sections) {
+					mutableSections[section.id] = section;
+					mutableByDocument[doc.id]?.push(section.id);
+					const headingKey = section.heading.toLowerCase();
+					if (!mutableByHeading[headingKey]) {
+						mutableByHeading[headingKey] = [];
+					}
+					mutableByHeading[headingKey]?.push(section.id);
+					sectionsIndexed++;
+				}
+				const internalLinks = doc.links.filter((l) => l.type === "internal");
+				const outgoingLinks = [];
+				for (const link of internalLinks) {
+					const target = resolveInternalLink(
+						link.href,
+						filePath,
+						storage.rootPath,
+					);
+					if (target) {
+						outgoingLinks.push(target);
+						if (!mutableBackward[target]) {
+							mutableBackward[target] = [];
+						}
+						if (!mutableBackward[target]?.includes(relativePath)) {
+							mutableBackward[target]?.push(relativePath);
+						}
+						linksIndexed++;
+					}
+				}
+				mutableForward[relativePath] = outgoingLinks;
+			}).pipe(
+				// Note: catchAll is intentional for batch file processing.
+				// Individual file failures should be collected in errors array
+				// rather than stopping the entire index build operation.
+				Effect6.catchAll((error) => {
+					const message =
+						"message" in error && typeof error.message === "string"
+							? error.message
+							: String(error);
+					errors.push({
+						path: relativePath,
+						message,
+					});
+					return Effect6.void;
+				}),
+			);
+			yield* processFile;
+		}
+		for (const [_from, targets] of Object.entries(mutableForward)) {
+			for (const target of targets) {
+				if (!mutableDocuments[target] && !brokenLinks.includes(target)) {
+					brokenLinks.push(target);
+				}
+			}
+		}
+		yield* saveDocumentIndex(storage, {
+			version: docIndex.version,
+			rootPath: storage.rootPath,
+			documents: mutableDocuments,
+		});
+		yield* saveSectionIndex(storage, {
+			version: sectionIndex.version,
+			sections: mutableSections,
+			byHeading: mutableByHeading,
+			byDocument: mutableByDocument,
+		});
+		yield* saveLinkIndex(storage, {
+			version: linkIndex.version,
+			forward: mutableForward,
+			backward: mutableBackward,
+			broken: brokenLinks,
+		});
+		const duration = Date.now() - startTime;
+		const totalLinks = Object.values(mutableForward).reduce(
+			(sum, links) => sum + links.length,
+			0,
+		);
+		const skipped = {
+			unchanged: unchangedCount,
+			excluded: walkSkipped.excluded,
+			hidden: walkSkipped.hidden,
+			total: unchangedCount + walkSkipped.excluded + walkSkipped.hidden,
+		};
+		return {
+			documentsIndexed,
+			sectionsIndexed,
+			linksIndexed,
+			totalDocuments: Object.keys(mutableDocuments).length,
+			totalSections: Object.keys(mutableSections).length,
+			totalLinks,
+			duration,
+			errors,
+			skipped,
+		};
+	});
+var getOutgoingLinks = (rootPath, filePath) =>
+	Effect6.gen(function* () {
+		const storage = createStorage(rootPath);
+		const linkIndex = yield* loadLinkIndex(storage);
+		if (!linkIndex) {
+			return [];
+		}
+		const relativePath = path4.relative(
+			storage.rootPath,
+			path4.resolve(filePath),
+		);
+		return linkIndex.forward[relativePath] ?? [];
+	});
+var getIncomingLinks = (rootPath, filePath) =>
+	Effect6.gen(function* () {
+		const storage = createStorage(rootPath);
+		const linkIndex = yield* loadLinkIndex(storage);
+		if (!linkIndex) {
+			return [];
+		}
+		const relativePath = path4.relative(
+			storage.rootPath,
+			path4.resolve(filePath),
+		);
+		return linkIndex.backward[relativePath] ?? [];
+	});
+var getBrokenLinks = (rootPath) =>
+	Effect6.gen(function* () {
+		const storage = createStorage(rootPath);
+		const linkIndex = yield* loadLinkIndex(storage);
+		if (!linkIndex) {
+			return [];
+		}
+		return linkIndex.broken;
+	});
+var buildBM25Index = (rootPath, options = {}) =>
+	Effect6.gen(function* () {
+		const startTime = Date.now();
+		const storage = createStorage(rootPath);
+		const docIndex = yield* loadDocumentIndex(storage);
+		const sectionIndex = yield* loadSectionIndex(storage);
+		if (!docIndex || !sectionIndex) {
+			return { sectionsIndexed: 0, duration: 0 };
+		}
+		const bm25Store = createBM25Store(storage.rootPath);
+		if (!options.force) {
+			const loaded = yield* bm25Store.load();
+			if (loaded) {
+				const stats = bm25Store.getStats();
+				if (stats.count > 0) {
+					return { sectionsIndexed: 0, duration: Date.now() - startTime };
+				}
+			}
+		}
+		bm25Store.clear();
+		const sectionsByDoc = /* @__PURE__ */ new Map();
+		for (const section of Object.values(sectionIndex.sections)) {
+			if (section.tokenCount < 10) continue;
+			const existing = sectionsByDoc.get(section.documentPath);
+			if (existing) {
+				existing.push(section);
+			} else {
+				sectionsByDoc.set(section.documentPath, [section]);
+			}
+		}
+		const totalDocs = sectionsByDoc.size;
+		let processedDocs = 0;
+		let sectionsIndexed = 0;
+		for (const [docPath, sections] of sectionsByDoc) {
+			const filePath = path4.join(storage.rootPath, docPath);
+			const fileContentResult = yield* Effect6.promise(() =>
+				fs4.readFile(filePath, "utf-8"),
+			).pipe(
+				Effect6.map((content) => ({ ok: true, content })),
+				Effect6.catchAll(() => Effect6.succeed({ ok: false, content: "" })),
+			);
+			if (!fileContentResult.ok) continue;
+			const lines = fileContentResult.content.split("\n");
+			const docs = [];
+			for (const section of sections) {
+				const content = lines
+					.slice(section.startLine - 1, section.endLine)
+					.join("\n");
+				docs.push({
+					id: section.id,
+					sectionId: section.id,
+					documentPath: section.documentPath,
+					heading: section.heading,
+					content,
+				});
+				sectionsIndexed++;
+			}
+			yield* bm25Store.add(docs);
+			processedDocs++;
+			if (options.onProgress) {
+				options.onProgress({ current: processedDocs, total: totalDocs });
+			}
+		}
+		yield* bm25Store.consolidate();
+		yield* bm25Store.save();
+		return {
+			sectionsIndexed,
+			duration: Date.now() - startTime,
+		};
+	});
+
+export {
+	FileReadError,
+	DirectoryWalkError,
+	ParseError,
+	ApiKeyMissingError,
+	ApiKeyInvalidError,
+	EmbeddingError,
+	IndexNotFoundError,
+	ConfigError,
+	VectorStoreError,
+	EmbeddingsNotFoundError,
+	WatchError,
+	CliValidationError,
+	countTokens,
+	countTokensApprox,
+	countWords,
+	freeEncoder,
+	parse,
+	parseFile,
+	getChokidarIgnorePatterns,
+	INDEX_DIR,
+	INDEX_VERSION,
+	getIndexPaths,
+	computeHash,
+	createStorage,
+	initializeIndex,
+	loadConfig,
+	saveConfig,
+	loadDocumentIndex,
+	saveDocumentIndex,
+	createEmptyDocumentIndex,
+	loadSectionIndex,
+	saveSectionIndex,
+	createEmptySectionIndex,
+	loadLinkIndex,
+	saveLinkIndex,
+	createEmptyLinkIndex,
+	indexExists,
+	bm25Search,
+	bm25IndexExists,
+	buildIndex,
+	getOutgoingLinks,
+	getIncomingLinks,
+	getBrokenLinks,
+	buildBM25Index,
+};