mdcontext 0.1.0 → 0.2.0

package/src/search/searcher.ts

@@ -6,12 +6,25 @@ import * as fs from 'node:fs/promises'
 import * as path from 'node:path'
 import { Effect } from 'effect'
 
+import {
+  DocumentNotFoundError,
+  type FileReadError,
+  type IndexCorruptedError,
+  IndexNotFoundError,
+} from '../errors/index.js'
 import {
   createStorage,
   loadDocumentIndex,
   loadSectionIndex,
 } from '../index/storage.js'
 import type { DocumentEntry, SectionEntry } from '../index/types.js'
+import {
+  buildFuzzyHighlightPattern,
+  findMatchesInLine,
+  type MatchOptions,
+  matchesWithOptions,
+} from './fuzzy-search.js'
+import { matchPath } from './path-matcher.js'
 import {
   buildHighlightPattern,
   evaluateQuery,
@@ -47,6 +60,12 @@ export interface SearchOptions {
   readonly contextBefore?: number | undefined
   /** Lines of context after matches */
   readonly contextAfter?: number | undefined
+  /** Enable fuzzy matching with typo tolerance */
+  readonly fuzzy?: boolean | undefined
+  /** Max edit distance for fuzzy matching (default: 2) */
+  readonly fuzzyDistance?: number | undefined
+  /** Enable word stemming (fail matches failure, failed, etc.) */
+  readonly stem?: boolean | undefined
 }
 
 export interface ContentMatch {
@@ -77,29 +96,27 @@ export interface SearchResult {
   readonly matches?: readonly ContentMatch[]
 }
 
-// ============================================================================
-// Path Matching
-// ============================================================================
-
-const matchPath = (filePath: string, pattern: string): boolean => {
-  // Simple glob-like matching
-  const regexPattern = pattern
-    .replace(/\./g, '\\.')
-    .replace(/\*/g, '.*')
-    .replace(/\?/g, '.')
-
-  const regex = new RegExp(`^${regexPattern}$`, 'i')
-  return regex.test(filePath)
-}
-
 // ============================================================================
 // Search Implementation
 // ============================================================================
 
+/**
+ * Search for sections by metadata (heading, path, content flags).
+ *
+ * @param rootPath - Root directory containing indexed markdown files
+ * @param options - Search filters (heading, path pattern, code/list/table flags)
+ * @returns Matching sections
+ *
+ * @throws FileReadError - Cannot read index files
+ * @throws IndexCorruptedError - Index files are corrupted
+ */
 export const search = (
   rootPath: string,
   options: SearchOptions = {},
-): Effect.Effect<readonly SearchResult[], Error> =>
+): Effect.Effect<
+  readonly SearchResult[],
+  FileReadError | IndexCorruptedError
+> =>
   Effect.gen(function* () {
     const storage = createStorage(rootPath)
 
@@ -184,11 +201,21 @@ export const search = (
  * Search within section content.
  * Supports boolean operators (AND, OR, NOT) and quoted phrases.
  * Falls back to regex for simple patterns.
+ *
+ * @param rootPath - Root directory containing indexed markdown files
+ * @param options - Search options including content pattern
+ * @returns Matching sections with match highlights
+ *
+ * @throws FileReadError - Cannot read index or source files
+ * @throws IndexCorruptedError - Index files are corrupted
  */
 export const searchContent = (
   rootPath: string,
   options: SearchOptions = {},
-): Effect.Effect<readonly SearchResult[], Error> =>
+): Effect.Effect<
+  readonly SearchResult[],
+  FileReadError | IndexCorruptedError
+> =>
   Effect.gen(function* () {
     const storage = createStorage(rootPath)
 
@@ -204,16 +231,38 @@ export const searchContent = (
     let contentRegex: RegExp | null = null
     let highlightRegex: RegExp | null = null
 
+    // Configure fuzzy/stem matching options
+    const matchOptions: MatchOptions = {
+      stem: options.stem,
+      fuzzyDistance: options.fuzzy ? (options.fuzzyDistance ?? 2) : undefined,
+    }
+    const useFuzzyOrStem = options.fuzzy || options.stem
+
     if (options.content) {
       if (isAdvancedQuery(options.content)) {
         parsedQuery = parseQuery(options.content)
         if (parsedQuery) {
-          highlightRegex = buildHighlightPattern(parsedQuery)
+          if (useFuzzyOrStem) {
+            highlightRegex = buildFuzzyHighlightPattern(
+              options.content,
+              matchOptions,
+            )
+          } else {
+            highlightRegex = buildHighlightPattern(parsedQuery)
+          }
         }
       } else {
-        // Simple search - use as regex
-        contentRegex = new RegExp(options.content, 'gi')
-        highlightRegex = contentRegex
+        // Simple search - use regex for exact match, or fuzzy/stem matching
+        if (!useFuzzyOrStem) {
+          contentRegex = new RegExp(options.content, 'gi')
+          highlightRegex = contentRegex
+        } else {
+          // For fuzzy/stem mode, build a highlight pattern
+          highlightRegex = buildFuzzyHighlightPattern(
+            options.content,
+            matchOptions,
+          )
+        }
       }
     }
 
@@ -248,7 +297,11 @@ export const searchContent = (
       let fileContent: string | null = null
       let fileLines: string[] = []
 
-      if (parsedQuery || contentRegex) {
+      // Need to load file if we have any content matching to do:
+      // - parsedQuery: boolean query evaluation
+      // - contentRegex: regex matching
+      // - useFuzzyOrStem: fuzzy/stem matching
+      if (parsedQuery || contentRegex || (useFuzzyOrStem && options.content)) {
         const filePath = path.join(storage.rootPath, docPath)
         try {
           fileContent = yield* Effect.promise(() =>
@@ -299,7 +352,7 @@ export const searchContent = (
         }
 
         // Content search
-        if ((parsedQuery || contentRegex) && fileContent) {
+        if ((parsedQuery || contentRegex || useFuzzyOrStem) && fileContent) {
           const sectionLines = fileLines.slice(
             section.startLine - 1,
             section.endLine,
@@ -313,6 +366,15 @@ export const searchContent = (
             }
           }
 
+          // For fuzzy/stem mode without boolean query, check section content
+          if (useFuzzyOrStem && !parsedQuery && options.content) {
+            if (
+              !matchesWithOptions(options.content, sectionContent, matchOptions)
+            ) {
+              continue // Section doesn't match with fuzzy/stem
+            }
+          }
+
           // Find individual line matches for highlighting
           const matches: ContentMatch[] = []
           const searchRegex = contentRegex || highlightRegex
@@ -321,47 +383,72 @@ export const searchContent = (
           const contextBefore = options.contextBefore ?? 1
           const contextAfter = options.contextAfter ?? 1
 
-          if (searchRegex) {
-            for (let i = 0; i < sectionLines.length; i++) {
-              const line = sectionLines[i]
-              if (line && searchRegex.test(line)) {
-                // Reset regex lastIndex for next test
-                searchRegex.lastIndex = 0
-
-                const absoluteLineNum = section.startLine + i
-
-                // Create snippet with configurable context
-                const snippetStart = Math.max(0, i - contextBefore)
-                const snippetEnd = Math.min(
-                  sectionLines.length,
-                  i + contextAfter + 1,
-                )
-                const snippetLines = sectionLines.slice(
-                  snippetStart,
-                  snippetEnd,
-                )
-                const snippet = snippetLines.join('\n')
-
-                // Build context lines array for JSON output
-                const contextLines: ContextLine[] = []
-                for (let j = snippetStart; j < snippetEnd; j++) {
-                  const ctxLine = sectionLines[j]
-                  if (ctxLine !== undefined) {
-                    contextLines.push({
-                      lineNumber: section.startLine + j,
-                      line: ctxLine,
-                      isMatch: j === i,
-                    })
-                  }
-                }
+          // Get query words for fuzzy/stem matching
+          const queryWords = options.content
+            ? options.content
+                .toLowerCase()
+                .split(/\W+/)
+                .filter((w) => w.length > 0)
+            : []
+
+          for (let i = 0; i < sectionLines.length; i++) {
+            const line = sectionLines[i]
+            if (!line) continue
+
+            let isMatch = false
 
-                matches.push({
-                  lineNumber: absoluteLineNum,
-                  line: line,
-                  snippet,
-                  contextLines,
-                })
+            // Check with regex for exact match mode
+            if (searchRegex) {
+              if (searchRegex.test(line)) {
+                isMatch = true
               }
+              // Reset regex lastIndex for next test
+              searchRegex.lastIndex = 0
+            }
+
+            // Check with fuzzy/stem matching
+            if (!isMatch && useFuzzyOrStem && queryWords.length > 0) {
+              const lineMatches = findMatchesInLine(
+                queryWords,
+                line,
+                matchOptions,
+              )
+              if (lineMatches.length > 0) {
+                isMatch = true
+              }
+            }
+
+            if (isMatch) {
+              const absoluteLineNum = section.startLine + i
+
+              // Create snippet with configurable context
+              const snippetStart = Math.max(0, i - contextBefore)
+              const snippetEnd = Math.min(
+                sectionLines.length,
+                i + contextAfter + 1,
+              )
+              const snippetLines = sectionLines.slice(snippetStart, snippetEnd)
+              const snippet = snippetLines.join('\n')
+
+              // Build context lines array for JSON output
+              const contextLines: ContextLine[] = []
+              for (let j = snippetStart; j < snippetEnd; j++) {
+                const ctxLine = sectionLines[j]
+                if (ctxLine !== undefined) {
+                  contextLines.push({
+                    lineNumber: section.startLine + j,
+                    line: ctxLine,
+                    isMatch: j === i,
+                  })
+                }
+              }
+
+              matches.push({
+                lineNumber: absoluteLineNum,
+                line: line,
+                snippet,
+                contextLines,
+              })
             }
           }
 
@@ -386,7 +473,7 @@ export const searchContent = (
               return results
             }
           }
-        } else if (!parsedQuery && !contentRegex) {
+        } else if (!parsedQuery && !contentRegex && !useFuzzyOrStem) {
           // No content search, heading-only search
           results.push({ section, document })
 
@@ -404,10 +491,23 @@ export const searchContent = (
 // Search with Content (legacy, uses heading-only search)
 // ============================================================================
 
+/**
+ * Search for sections by metadata and include section content.
+ *
+ * @param rootPath - Root directory containing indexed markdown files
+ * @param options - Search filters
+ * @returns Matching sections with content
+ *
+ * @throws FileReadError - Cannot read index or source files
+ * @throws IndexCorruptedError - Index files are corrupted
+ */
 export const searchWithContent = (
   rootPath: string,
   options: SearchOptions = {},
-): Effect.Effect<readonly SearchResult[], Error> =>
+): Effect.Effect<
+  readonly SearchResult[],
+  FileReadError | IndexCorruptedError
+> =>
   Effect.gen(function* () {
     const storage = createStorage(rootPath)
     const results = yield* search(rootPath, options)
@@ -471,11 +571,30 @@ export interface SectionContext {
   readonly hasTable: boolean
 }
 
+/**
+ * Get context information for a document.
+ *
+ * @param rootPath - Root directory containing indexed markdown files
+ * @param filePath - Path to the document
+ * @param options - Context options (max tokens, include content)
+ * @returns Document context with sections
+ *
+ * @throws IndexNotFoundError - Index doesn't exist
+ * @throws DocumentNotFoundError - Document not in index
+ * @throws FileReadError - Cannot read index or source files
+ * @throws IndexCorruptedError - Index files are corrupted
+ */
 export const getContext = (
   rootPath: string,
   filePath: string,
   options: ContextOptions = {},
-): Effect.Effect<DocumentContext, Error> =>
+): Effect.Effect<
+  DocumentContext,
+  | IndexNotFoundError
+  | DocumentNotFoundError
+  | FileReadError
+  | IndexCorruptedError
+> =>
   Effect.gen(function* () {
     const storage = createStorage(rootPath)
     const resolvedFile = path.resolve(filePath)
@@ -486,14 +605,17 @@ export const getContext = (
 
     if (!docIndex || !sectionIndex) {
       return yield* Effect.fail(
-        new Error("Index not found. Run 'mdcontext index' first."),
+        new IndexNotFoundError({ path: storage.rootPath }),
       )
     }
 
     const document = docIndex.documents[relativePath]
     if (!document) {
       return yield* Effect.fail(
-        new Error(`Document not found in index: ${relativePath}`),
+        new DocumentNotFoundError({
+          path: relativePath,
+          indexPath: storage.rootPath,
+        }),
       )
     }
 

package/src/search/wink-bm25.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Type declarations for wink-bm25-text-search
+ */
+
+declare module 'wink-bm25-text-search' {
+  interface BM25Config {
+    fldWeights?: Record<string, number>
+    bm25Params?: {
+      k1?: number
+      b?: number
+    }
+    ovFldNames?: string[]
+  }
+
+  type PrepTask = (text: string) => string[]
+
+  interface BM25Engine {
+    defineConfig(config: BM25Config): void
+    definePrepTasks(tasks: PrepTask[]): void
+    addDoc(doc: Record<string, string>, id: number): void
+    consolidate(): void
+    search(query: string, limit?: number): [number, number][]
+    exportJSON(): string
+    importJSON(json: string): void
+    reset(): void
+  }
+
+  function bm25(): BM25Engine
+  export default bm25
+}

package/src/summarization/cli-providers/claude.ts

@@ -0,0 +1,202 @@
+/**
+ * Claude CLI Summarizer
+ *
+ * Uses Claude Code CLI for AI summarization.
+ * FREE for users with Claude Code subscriptions.
+ *
+ * SECURITY: Uses spawn() with argument arrays - NEVER exec() with string interpolation.
+ */
+
+import { spawn } from 'node:child_process'
+import type {
+  StreamingSummarizer,
+  StreamOptions,
+  SummaryResult,
+} from '../types.js'
+import { SummarizationError as SummarizationErrorClass } from '../types.js'
+
+/**
+ * Claude CLI provider for summarization.
+ *
+ * Uses the `claude` CLI tool in non-interactive mode with text output.
+ * Requires Claude Code installation and authentication.
+ *
+ * @security Uses spawn() with argument arrays to prevent shell injection.
+ *           User input is passed as array elements, never interpolated.
+ *
+ * @cost Free (uses existing Claude subscription)
+ *
+ * @example
+ * ```typescript
+ * const summarizer = new ClaudeCLISummarizer()
+ *
+ * // Check availability
+ * if (await summarizer.isAvailable()) {
+ *   const result = await summarizer.summarize(searchResults, prompt)
+ *   console.log(result.summary)
+ *   // result.estimatedCost is always 0 (free)
+ * }
+ *
+ * // Streaming output
+ * await summarizer.summarizeStream(searchResults, prompt, {
+ *   onChunk: (chunk) => process.stdout.write(chunk),
+ *   onComplete: (result) => console.log(`Done in ${result.durationMs}ms`),
+ * })
+ * ```
+ */
+export class ClaudeCLISummarizer implements StreamingSummarizer {
+  private readonly command = 'claude'
+
+  async summarize(input: string, prompt: string): Promise<SummaryResult> {
+    const startTime = Date.now()
+    const fullPrompt = `${prompt}\n\n${input}`
+
+    return new Promise((resolve, reject) => {
+      // SECURITY: spawn() with argument array - safe from shell injection
+      const proc = spawn(
+        this.command,
+        ['-p', fullPrompt, '--output-format', 'text'],
+        {
+          stdio: ['ignore', 'pipe', 'pipe'],
+        },
+      )
+
+      let stdout = ''
+      let stderr = ''
+
+      proc.stdout.on('data', (data: Buffer) => {
+        stdout += data.toString()
+      })
+
+      proc.stderr.on('data', (data: Buffer) => {
+        stderr += data.toString()
+      })
+
+      proc.on('close', (code: number | null) => {
+        const durationMs = Date.now() - startTime
+
+        if (code !== 0) {
+          reject(
+            new SummarizationErrorClass(
+              `Claude CLI exited with code ${code}: ${stderr}`,
+              'CLI_EXECUTION_FAILED',
+              'claude',
+            ),
+          )
+          return
+        }
+
+        resolve({
+          summary: stdout.trim(),
+          provider: 'claude',
+          mode: 'cli',
+          estimatedCost: 0,
+          durationMs,
+        })
+      })
+
+      proc.on('error', (error: Error) => {
+        reject(
+          new SummarizationErrorClass(
+            `Failed to spawn Claude CLI: ${error.message}`,
+            'CLI_EXECUTION_FAILED',
+            'claude',
+            error,
+          ),
+        )
+      })
+    })
+  }
+
+  async summarizeStream(
+    input: string,
+    prompt: string,
+    options: StreamOptions,
+  ): Promise<void> {
+    const startTime = Date.now()
+    const fullPrompt = `${prompt}\n\n${input}`
+
+    return new Promise((resolve, reject) => {
+      // SECURITY: spawn() with argument array - safe from shell injection
+      const proc = spawn(
+        this.command,
+        ['-p', fullPrompt, '--output-format', 'text'],
+        {
+          stdio: ['ignore', 'pipe', 'pipe'],
+        },
+      )
+
+      let fullOutput = ''
+      let stderr = ''
+
+      proc.stdout.on('data', (data: Buffer) => {
+        const chunk = data.toString()
+        fullOutput += chunk
+        options.onChunk(chunk)
+      })
+
+      proc.stderr.on('data', (data: Buffer) => {
+        stderr += data.toString()
+      })
+
+      proc.on('close', (code: number | null) => {
+        const durationMs = Date.now() - startTime
+
+        if (code !== 0) {
+          const error = new SummarizationErrorClass(
+            `Claude CLI exited with code ${code}: ${stderr}`,
+            'CLI_EXECUTION_FAILED',
+            'claude',
+          )
+          options.onError?.(error)
+          reject(error)
+          return
+        }
+
+        const result: SummaryResult = {
+          summary: fullOutput.trim(),
+          provider: 'claude',
+          mode: 'cli',
+          estimatedCost: 0,
+          durationMs,
+        }
+
+        options.onComplete?.(result)
+        resolve()
+      })
+
+      proc.on('error', (error: Error) => {
+        const sumError = new SummarizationErrorClass(
+          `Failed to spawn Claude CLI: ${error.message}`,
+          'CLI_EXECUTION_FAILED',
+          'claude',
+          error,
+        )
+        options.onError?.(sumError)
+        reject(sumError)
+      })
+    })
+  }
+
+  estimateCost(_inputTokens: number): number {
+    // CLI providers are free with subscription
+    return 0
+  }
+
+  async isAvailable(): Promise<boolean> {
+    return new Promise((resolve) => {
+      const checkCommand = process.platform === 'win32' ? 'where' : 'which'
+      const proc = spawn(checkCommand, [this.command], {
+        stdio: ['ignore', 'ignore', 'ignore'],
+      })
+
+      proc.on('close', (code) => {
+        resolve(code === 0)
+      })
+
+      proc.on('error', () => {
+        resolve(false)
+      })
+    })
+  }
+}