mdcontext 0.1.0 → 0.2.0

package/src/index/indexer.ts

@@ -5,8 +5,18 @@
 import * as fs from 'node:fs/promises'
 import * as path from 'node:path'
 import { Effect } from 'effect'
+import type { Ignore } from 'ignore'
 import type { MdSection } from '../core/types.js'
+import {
+  type DirectoryCreateError,
+  DirectoryWalkError,
+  type FileReadError,
+  type FileWriteError,
+  type IndexCorruptedError,
+  ParseError,
+} from '../errors/index.js'
 import { parse } from '../parser/parser.js'
+import { createIgnoreFilter, shouldIgnore } from './ignore-patterns.js'
 import {
   computeHash,
   createEmptyDocumentIndex,
@@ -24,9 +34,10 @@ import {
 import type {
   DocumentEntry,
   DocumentIndex,
-  IndexBuildError,
+  FileProcessingError,
   IndexResult,
   SectionEntry,
+  SkipSummary,
 } from './types.js'
 
 // ============================================================================
@@ -36,52 +47,68 @@ import type {
 const isMarkdownFile = (filename: string): boolean =>
   filename.endsWith('.md') || filename.endsWith('.mdx')
 
-const shouldExclude = (
-  filePath: string,
-  exclude: readonly string[],
-): boolean => {
-  const normalized = filePath.toLowerCase()
-  for (const pattern of exclude) {
-    if (
-      pattern.includes('node_modules') &&
-      normalized.includes('node_modules')
-    ) {
-      return true
-    }
-    if (pattern.startsWith('**/.*') && normalized.includes('/.')) {
-      return true
-    }
+/**
+ * Result of directory walk including tracked skip counts
+ */
+interface WalkResult {
+  readonly files: string[]
+  readonly skipped: {
+    hidden: number
+    excluded: number
   }
-  return false
 }
 
+/**
+ * Walk directory using ignore filter for pattern matching.
+ *
+ * @param dir - Directory to walk
+ * @param rootPath - Root path for computing relative paths
+ * @param filter - Ignore filter instance
+ * @returns Walk result with files and skip counts
+ */
 const walkDirectory = async (
   dir: string,
-  exclude: readonly string[],
-): Promise<string[]> => {
+  rootPath: string,
+  filter: Ignore,
+): Promise<WalkResult> => {
   const files: string[] = []
+  let hiddenCount = 0
+  let excludedCount = 0
   const entries = await fs.readdir(dir, { withFileTypes: true })
 
   for (const entry of entries) {
     const fullPath = path.join(dir, entry.name)
+    const relativePath = path.relative(rootPath, fullPath)
 
-    if (entry.name.startsWith('.') || entry.name === 'node_modules') {
+    // Skip hidden files/directories (starting with .)
+    if (entry.name.startsWith('.')) {
+      if (entry.isDirectory()) {
+        hiddenCount++
+      }
       continue
     }
 
-    if (shouldExclude(fullPath, exclude)) {
+    // Check ignore filter for both files and directories
+    if (shouldIgnore(relativePath, filter)) {
+      if (entry.isDirectory()) {
+        excludedCount++
+      } else {
+        excludedCount++
+      }
       continue
     }
 
     if (entry.isDirectory()) {
-      const subFiles = await walkDirectory(fullPath, exclude)
-      files.push(...subFiles)
+      const subResult = await walkDirectory(fullPath, rootPath, filter)
+      files.push(...subResult.files)
+      hiddenCount += subResult.skipped.hidden
+      excludedCount += subResult.skipped.excluded
     } else if (entry.isFile() && isMarkdownFile(entry.name)) {
       files.push(fullPath)
     }
   }
 
-  return files
+  return { files, skipped: { hidden: hiddenCount, excluded: excludedCount } }
 }
 
 // ============================================================================
@@ -156,19 +183,39 @@ const resolveInternalLink = (
 // Index Building
 // ============================================================================
 
+export interface IndexProgress {
+  readonly current: number
+  readonly total: number
+  readonly filePath: string
+}
+
 export interface IndexOptions {
-  readonly force?: boolean
-  readonly exclude?: readonly string[]
+  readonly force?: boolean | undefined
+  /** CLI/config exclude patterns (overrides ignore files) */
+  readonly exclude?: readonly string[] | undefined
+  /** Whether to honor .gitignore (default: true) */
+  readonly honorGitignore?: boolean | undefined
+  /** Whether to honor .mdcontextignore (default: true) */
+  readonly honorMdcontextignore?: boolean | undefined
+  /** Callback for progress updates during file indexing */
+  readonly onProgress?: ((progress: IndexProgress) => void) | undefined
 }
 
 export const buildIndex = (
   rootPath: string,
   options: IndexOptions = {},
-): Effect.Effect<IndexResult, Error> =>
+): Effect.Effect<
+  IndexResult,
+  | DirectoryWalkError
+  | DirectoryCreateError
+  | FileReadError
+  | FileWriteError
+  | IndexCorruptedError
+> =>
   Effect.gen(function* () {
     const startTime = Date.now()
     const storage = createStorage(rootPath)
-    const errors: IndexBuildError[] = []
+    const errors: FileProcessingError[] = []
 
     // Initialize storage
     yield* initializeIndex(storage)
@@ -186,42 +233,82 @@ export const buildIndex = (
     const sectionIndex = existingSectionIndex ?? createEmptySectionIndex()
     const linkIndex = existingLinkIndex ?? createEmptyLinkIndex()
 
-    // Discover files
-    const exclude = options.exclude ?? ['**/node_modules/**', '**/.*/**']
-    const files = yield* Effect.tryPromise({
-      try: () => walkDirectory(storage.rootPath, exclude),
-      catch: (e) => new Error(`Failed to walk directory: ${e}`),
+    // Build ignore filter with proper precedence:
+    // CLI/config patterns > .mdcontextignore > .gitignore > defaults
+    const ignoreResult = yield* createIgnoreFilter({
+      rootPath: storage.rootPath,
+      cliPatterns: options.exclude,
+      honorGitignore: options.honorGitignore ?? true,
+      honorMdcontextignore: options.honorMdcontextignore ?? true,
+    })
+
+    // Discover files using the ignore filter
+    const walkResult = yield* Effect.tryPromise({
+      try: () =>
+        walkDirectory(storage.rootPath, storage.rootPath, ignoreResult.filter),
+      catch: (e) =>
+        new DirectoryWalkError({
+          path: storage.rootPath,
+          message: `Failed to traverse directory: ${e instanceof Error ? e.message : String(e)}`,
+          cause: e,
+        }),
     })
 
+    const { files, skipped: walkSkipped } = walkResult
+
     // Process each file
     let documentsIndexed = 0
     let sectionsIndexed = 0
     let linksIndexed = 0
+    let unchangedCount = 0
 
     const mutableDocuments: Record<string, DocumentEntry> = {
       ...docIndex.documents,
     }
-    // Initialize with existing data to preserve sections/links for unchanged files
     const mutableSections: Record<string, SectionEntry> = {
       ...sectionIndex.sections,
     }
-    const mutableByHeading: Record<string, string[]> = Object.fromEntries(
-      Object.entries(sectionIndex.byHeading).map(([k, v]) => [k, [...v]]),
+
+    const mutableByHeading: Record<string, string[]> = Object.assign(
+      Object.create(null),
+      Object.fromEntries(
+        Object.entries(sectionIndex.byHeading).map(([k, v]) => [k, [...v]]),
+      ),
     )
-    const mutableByDocument: Record<string, string[]> = Object.fromEntries(
-      Object.entries(sectionIndex.byDocument).map(([k, v]) => [k, [...v]]),
+    const mutableByDocument: Record<string, string[]> = Object.assign(
+      Object.create(null),
+      Object.fromEntries(
+        Object.entries(sectionIndex.byDocument).map(([k, v]) => [k, [...v]]),
+      ),
     )
-    const mutableForward: Record<string, string[]> = Object.fromEntries(
-      Object.entries(linkIndex.forward).map(([k, v]) => [k, [...v]]),
+    const mutableForward: Record<string, string[]> = Object.assign(
+      Object.create(null),
+      Object.fromEntries(
+        Object.entries(linkIndex.forward).map(([k, v]) => [k, [...v]]),
+      ),
     )
-    const mutableBackward: Record<string, string[]> = Object.fromEntries(
-      Object.entries(linkIndex.backward).map(([k, v]) => [k, [...v]]),
+    const mutableBackward: Record<string, string[]> = Object.assign(
+      Object.create(null),
+      Object.fromEntries(
+        Object.entries(linkIndex.backward).map(([k, v]) => [k, [...v]]),
+      ),
     )
     const brokenLinks: string[] = [...linkIndex.broken]
+    const totalFiles = files.length
 
-    for (const filePath of files) {
+    for (let fileIndex = 0; fileIndex < files.length; fileIndex++) {
+      const filePath = files[fileIndex]!
       const relativePath = path.relative(storage.rootPath, filePath)
 
+      // Report progress
+      if (options.onProgress) {
+        options.onProgress({
+          current: fileIndex + 1,
+          total: totalFiles,
+          filePath: relativePath,
+        })
+      }
+
       // Process each file, collecting errors instead of failing
       const processFile = Effect.gen(function* () {
         // Read file content and stats
@@ -239,6 +326,7 @@ export const buildIndex = (
           existingEntry.hash === hash &&
           existingEntry.mtime === stats.mtime.getTime()
         ) {
+          unchangedCount++
           return // File unchanged, skip processing
         }
 
@@ -248,7 +336,13 @@ export const buildIndex = (
           lastModified: stats.mtime,
         }).pipe(
           Effect.mapError(
-            (e) => new Error(`Parse error in ${relativePath}: ${e.message}`),
+            (e) =>
+              new ParseError({
+                message: e.message,
+                path: relativePath,
+                ...(e.line !== undefined && { line: e.line }),
+                ...(e.column !== undefined && { column: e.column }),
+              }),
           ),
         )
 
@@ -333,10 +427,18 @@ export const buildIndex = (
 
         mutableForward[relativePath] = outgoingLinks
       }).pipe(
+        // Note: catchAll is intentional for batch file processing.
+        // Individual file failures should be collected in errors array
+        // rather than stopping the entire index build operation.
         Effect.catchAll((error) => {
+          // Extract message from typed errors or generic errors
+          const message =
+            'message' in error && typeof error.message === 'string'
+              ? error.message
+              : String(error)
           errors.push({
             path: relativePath,
-            message: error instanceof Error ? error.message : String(error),
+            message,
           })
           return Effect.void
         }),
@@ -383,6 +485,14 @@ export const buildIndex = (
       0,
     )
 
+    // Build skip summary
+    const skipped: SkipSummary = {
+      unchanged: unchangedCount,
+      excluded: walkSkipped.excluded,
+      hidden: walkSkipped.hidden,
+      total: unchangedCount + walkSkipped.excluded + walkSkipped.hidden,
+    }
+
     return {
       documentsIndexed,
       sectionsIndexed,
@@ -392,6 +502,7 @@ export const buildIndex = (
       totalLinks,
       duration,
       errors,
+      skipped,
     }
   })
 
@@ -402,7 +513,7 @@ export const buildIndex = (
 export const getOutgoingLinks = (
   rootPath: string,
   filePath: string,
-): Effect.Effect<readonly string[], Error> =>
+): Effect.Effect<readonly string[], FileReadError | IndexCorruptedError> =>
   Effect.gen(function* () {
     const storage = createStorage(rootPath)
     const linkIndex = yield* loadLinkIndex(storage)
@@ -418,7 +529,7 @@ export const getOutgoingLinks = (
 export const getIncomingLinks = (
   rootPath: string,
   filePath: string,
-): Effect.Effect<readonly string[], Error> =>
+): Effect.Effect<readonly string[], FileReadError | IndexCorruptedError> =>
   Effect.gen(function* () {
     const storage = createStorage(rootPath)
     const linkIndex = yield* loadLinkIndex(storage)
@@ -433,7 +544,7 @@ export const getIncomingLinks = (
 
 export const getBrokenLinks = (
   rootPath: string,
-): Effect.Effect<readonly string[], Error> =>
+): Effect.Effect<readonly string[], FileReadError | IndexCorruptedError> =>
   Effect.gen(function* () {
     const storage = createStorage(rootPath)
     const linkIndex = yield* loadLinkIndex(storage)
@@ -444,3 +555,130 @@ export const getBrokenLinks = (
 
     return linkIndex.broken
   })
+
+// ============================================================================
+// BM25 Index Building
+// ============================================================================
+
+import { type BM25Document, createBM25Store } from '../search/bm25-store.js'
+
+export interface BuildBM25Options {
+  readonly force?: boolean
+  readonly onProgress?: (progress: { current: number; total: number }) => void
+}
+
+export interface BuildBM25Result {
+  readonly sectionsIndexed: number
+  readonly duration: number
+}
+
+/**
+ * Build BM25 keyword index for all sections.
+ *
+ * @param rootPath - Root directory containing indexed markdown files
+ * @param options - Build options (force rebuild, progress callback)
+ * @returns Result with section count and timing
+ */
+export const buildBM25Index = (
+  rootPath: string,
+  options: BuildBM25Options = {},
+): Effect.Effect<
+  BuildBM25Result,
+  FileReadError | IndexCorruptedError | FileWriteError
+> =>
+  Effect.gen(function* () {
+    const startTime = Date.now()
+    const storage = createStorage(rootPath)
+
+    // Load section index
+    const docIndex = yield* loadDocumentIndex(storage)
+    const sectionIndex = yield* loadSectionIndex(storage)
+
+    if (!docIndex || !sectionIndex) {
+      return { sectionsIndexed: 0, duration: 0 }
+    }
+
+    // Create BM25 store
+    const bm25Store = createBM25Store(storage.rootPath)
+
+    // Check if we can skip
+    if (!options.force) {
+      const loaded = yield* bm25Store.load()
+      if (loaded) {
+        const stats = bm25Store.getStats()
+        if (stats.count > 0) {
+          return { sectionsIndexed: 0, duration: Date.now() - startTime }
+        }
+      }
+    }
+
+    // Clear and rebuild
+    bm25Store.clear()
+
+    // Group sections by document for efficient file reading
+    const sectionsByDoc: Map<string, SectionEntry[]> = new Map()
+    for (const section of Object.values(sectionIndex.sections)) {
+      if (section.tokenCount < 10) continue
+      const existing = sectionsByDoc.get(section.documentPath)
+      if (existing) {
+        existing.push(section)
+      } else {
+        sectionsByDoc.set(section.documentPath, [section])
+      }
+    }
+
+    const totalDocs = sectionsByDoc.size
+    let processedDocs = 0
+    let sectionsIndexed = 0
+
+    // Process each document
+    for (const [docPath, sections] of sectionsByDoc) {
+      const filePath = path.join(storage.rootPath, docPath)
+
+      // Read file content
+      const fileContentResult = yield* Effect.promise(() =>
+        fs.readFile(filePath, 'utf-8'),
+      ).pipe(
+        Effect.map((content) => ({ ok: true as const, content })),
+        Effect.catchAll(() =>
+          Effect.succeed({ ok: false as const, content: '' }),
+        ),
+      )
+
+      if (!fileContentResult.ok) continue
+
+      const lines = fileContentResult.content.split('\n')
+      const docs: BM25Document[] = []
+
+      for (const section of sections) {
+        const content = lines
+          .slice(section.startLine - 1, section.endLine)
+          .join('\n')
+
+        docs.push({
+          id: section.id,
+          sectionId: section.id,
+          documentPath: section.documentPath,
+          heading: section.heading,
+          content,
+        })
+        sectionsIndexed++
+      }
+
+      yield* bm25Store.add(docs)
+
+      processedDocs++
+      if (options.onProgress) {
+        options.onProgress({ current: processedDocs, total: totalDocs })
+      }
+    }
+
+    // Consolidate and save
+    yield* bm25Store.consolidate()
+    yield* bm25Store.save()
+
+    return {
+      sectionsIndexed,
+      duration: Date.now() - startTime,
+    }
+  })