mdcontext 0.0.1 → 0.1.0

package/src/search/searcher.ts

@@ -0,0 +1,602 @@
+/**
+ * Keyword search for mdcontext
+ */
+
+import * as fs from 'node:fs/promises'
+import * as path from 'node:path'
+import { Effect } from 'effect'
+
+import {
+  createStorage,
+  loadDocumentIndex,
+  loadSectionIndex,
+} from '../index/storage.js'
+import type { DocumentEntry, SectionEntry } from '../index/types.js'
+import {
+  buildHighlightPattern,
+  evaluateQuery,
+  isAdvancedQuery,
+  type ParsedQuery,
+  parseQuery,
+} from './query-parser.js'
+
+// ============================================================================
+// Search Options
+// ============================================================================
+
+export interface SearchOptions {
+  /** Filter by heading pattern (regex) */
+  readonly heading?: string | undefined
+  /** Search within section content (regex) */
+  readonly content?: string | undefined
+  /** Filter by file path pattern (glob-like) */
+  readonly pathPattern?: string | undefined
+  /** Only sections with code blocks */
+  readonly hasCode?: boolean | undefined
+  /** Only sections with lists */
+  readonly hasList?: boolean | undefined
+  /** Only sections with tables */
+  readonly hasTable?: boolean | undefined
+  /** Minimum heading level */
+  readonly minLevel?: number | undefined
+  /** Maximum heading level */
+  readonly maxLevel?: number | undefined
+  /** Maximum results */
+  readonly limit?: number | undefined
+  /** Lines of context before matches */
+  readonly contextBefore?: number | undefined
+  /** Lines of context after matches */
+  readonly contextAfter?: number | undefined
+}
+
+export interface ContentMatch {
+  /** The line number where match was found (1-based) */
+  readonly lineNumber: number
+  /** The matching line text */
+  readonly line: string
+  /** Snippet showing match context (lines before and after) */
+  readonly snippet: string
+  /** Context lines with their line numbers (for JSON output) */
+  readonly contextLines?: readonly ContextLine[]
+}
+
+export interface ContextLine {
+  /** The line number (1-based) */
+  readonly lineNumber: number
+  /** The line text */
+  readonly line: string
+  /** Whether this is the matching line */
+  readonly isMatch: boolean
+}
+
+export interface SearchResult {
+  readonly section: SectionEntry
+  readonly document: DocumentEntry
+  readonly sectionContent?: string
+  /** Matches found within the content (when content search is used) */
+  readonly matches?: readonly ContentMatch[]
+}
+
+// ============================================================================
+// Path Matching
+// ============================================================================
+
+const matchPath = (filePath: string, pattern: string): boolean => {
+  // Simple glob-like matching
+  const regexPattern = pattern
+    .replace(/\./g, '\\.')
+    .replace(/\*/g, '.*')
+    .replace(/\?/g, '.')
+
+  const regex = new RegExp(`^${regexPattern}$`, 'i')
+  return regex.test(filePath)
+}
+
+// ============================================================================
+// Search Implementation
+// ============================================================================
+
+export const search = (
+  rootPath: string,
+  options: SearchOptions = {},
+): Effect.Effect<readonly SearchResult[], Error> =>
+  Effect.gen(function* () {
+    const storage = createStorage(rootPath)
+
+    const docIndex = yield* loadDocumentIndex(storage)
+    const sectionIndex = yield* loadSectionIndex(storage)
+
+    if (!docIndex || !sectionIndex) {
+      return []
+    }
+
+    const results: SearchResult[] = []
+    const headingRegex = options.heading
+      ? new RegExp(options.heading, 'i')
+      : null
+
+    for (const section of Object.values(sectionIndex.sections)) {
+      // Filter by heading pattern
+      if (headingRegex && !headingRegex.test(section.heading)) {
+        continue
+      }
+
+      // Filter by path pattern
+      if (
+        options.pathPattern &&
+        !matchPath(section.documentPath, options.pathPattern)
+      ) {
+        continue
+      }
+
+      // Filter by code blocks
+      if (
+        options.hasCode !== undefined &&
+        section.hasCode !== options.hasCode
+      ) {
+        continue
+      }
+
+      // Filter by lists
+      if (
+        options.hasList !== undefined &&
+        section.hasList !== options.hasList
+      ) {
+        continue
+      }
+
+      // Filter by tables
+      if (
+        options.hasTable !== undefined &&
+        section.hasTable !== options.hasTable
+      ) {
+        continue
+      }
+
+      // Filter by level range
+      if (options.minLevel !== undefined && section.level < options.minLevel) {
+        continue
+      }
+
+      if (options.maxLevel !== undefined && section.level > options.maxLevel) {
+        continue
+      }
+
+      const document = docIndex.documents[section.documentPath]
+      if (document) {
+        results.push({ section, document })
+      }
+
+      // Check limit
+      if (options.limit !== undefined && results.length >= options.limit) {
+        break
+      }
+    }
+
+    return results
+  })
+
+// ============================================================================
+// Content Search Implementation
+// ============================================================================
+
+/**
+ * Search within section content.
+ * Supports boolean operators (AND, OR, NOT) and quoted phrases.
+ * Falls back to regex for simple patterns.
+ */
+export const searchContent = (
+  rootPath: string,
+  options: SearchOptions = {},
+): Effect.Effect<readonly SearchResult[], Error> =>
+  Effect.gen(function* () {
+    const storage = createStorage(rootPath)
+
+    const docIndex = yield* loadDocumentIndex(storage)
+    const sectionIndex = yield* loadSectionIndex(storage)
+
+    if (!docIndex || !sectionIndex) {
+      return []
+    }
+
+    // Parse content query - use boolean parser if advanced, else regex
+    let parsedQuery: ParsedQuery | null = null
+    let contentRegex: RegExp | null = null
+    let highlightRegex: RegExp | null = null
+
+    if (options.content) {
+      if (isAdvancedQuery(options.content)) {
+        parsedQuery = parseQuery(options.content)
+        if (parsedQuery) {
+          highlightRegex = buildHighlightPattern(parsedQuery)
+        }
+      } else {
+        // Simple search - use as regex
+        contentRegex = new RegExp(options.content, 'gi')
+        highlightRegex = contentRegex
+      }
+    }
+
+    const headingRegex = options.heading
+      ? new RegExp(options.heading, 'i')
+      : null
+
+    const results: SearchResult[] = []
+
+    // Group sections by document for efficient file reading
+    const sectionsByDoc: Record<string, SectionEntry[]> = {}
+    for (const section of Object.values(sectionIndex.sections)) {
+      const docSections = sectionsByDoc[section.documentPath]
+      if (docSections) {
+        docSections.push(section)
+      } else {
+        sectionsByDoc[section.documentPath] = [section]
+      }
+    }
+
+    // Process each document
+    for (const [docPath, sections] of Object.entries(sectionsByDoc)) {
+      // Apply path filter early
+      if (options.pathPattern && !matchPath(docPath, options.pathPattern)) {
+        continue
+      }
+
+      const document = docIndex.documents[docPath]
+      if (!document) continue
+
+      // Load file content for content search
+      let fileContent: string | null = null
+      let fileLines: string[] = []
+
+      if (parsedQuery || contentRegex) {
+        const filePath = path.join(storage.rootPath, docPath)
+        try {
+          fileContent = yield* Effect.promise(() =>
+            fs.readFile(filePath, 'utf-8'),
+          )
+          fileLines = fileContent.split('\n')
+        } catch {
+          continue // Skip files that can't be read
+        }
+      }
+
+      for (const section of sections) {
+        // Apply heading filter
+        if (headingRegex && !headingRegex.test(section.heading)) {
+          continue
+        }
+
+        // Apply other filters
+        if (
+          options.hasCode !== undefined &&
+          section.hasCode !== options.hasCode
+        ) {
+          continue
+        }
+        if (
+          options.hasList !== undefined &&
+          section.hasList !== options.hasList
+        ) {
+          continue
+        }
+        if (
+          options.hasTable !== undefined &&
+          section.hasTable !== options.hasTable
+        ) {
+          continue
+        }
+        if (
+          options.minLevel !== undefined &&
+          section.level < options.minLevel
+        ) {
+          continue
+        }
+        if (
+          options.maxLevel !== undefined &&
+          section.level > options.maxLevel
+        ) {
+          continue
+        }
+
+        // Content search
+        if ((parsedQuery || contentRegex) && fileContent) {
+          const sectionLines = fileLines.slice(
+            section.startLine - 1,
+            section.endLine,
+          )
+          const sectionContent = sectionLines.join('\n')
+
+          // For boolean queries, evaluate against entire section content
+          if (parsedQuery) {
+            if (!evaluateQuery(parsedQuery.ast, sectionContent)) {
+              continue // Section doesn't match query
+            }
+          }
+
+          // Find individual line matches for highlighting
+          const matches: ContentMatch[] = []
+          const searchRegex = contentRegex || highlightRegex
+
+          // Use configurable context lines (default to 1 if not specified)
+          const contextBefore = options.contextBefore ?? 1
+          const contextAfter = options.contextAfter ?? 1
+
+          if (searchRegex) {
+            for (let i = 0; i < sectionLines.length; i++) {
+              const line = sectionLines[i]
+              if (line && searchRegex.test(line)) {
+                // Reset regex lastIndex for next test
+                searchRegex.lastIndex = 0
+
+                const absoluteLineNum = section.startLine + i
+
+                // Create snippet with configurable context
+                const snippetStart = Math.max(0, i - contextBefore)
+                const snippetEnd = Math.min(
+                  sectionLines.length,
+                  i + contextAfter + 1,
+                )
+                const snippetLines = sectionLines.slice(
+                  snippetStart,
+                  snippetEnd,
+                )
+                const snippet = snippetLines.join('\n')
+
+                // Build context lines array for JSON output
+                const contextLines: ContextLine[] = []
+                for (let j = snippetStart; j < snippetEnd; j++) {
+                  const ctxLine = sectionLines[j]
+                  if (ctxLine !== undefined) {
+                    contextLines.push({
+                      lineNumber: section.startLine + j,
+                      line: ctxLine,
+                      isMatch: j === i,
+                    })
+                  }
+                }
+
+                matches.push({
+                  lineNumber: absoluteLineNum,
+                  line: line,
+                  snippet,
+                  contextLines,
+                })
+              }
+            }
+          }
+
+          // For boolean queries, include section even without line-level matches
+          // (the section matched as a whole)
+          if (parsedQuery || matches.length > 0) {
+            const result: SearchResult = {
+              section,
+              document,
+              sectionContent,
+            }
+            if (matches.length > 0) {
+              results.push({ ...result, matches })
+            } else {
+              results.push(result)
+            }
+
+            if (
+              options.limit !== undefined &&
+              results.length >= options.limit
+            ) {
+              return results
+            }
+          }
+        } else if (!parsedQuery && !contentRegex) {
+          // No content search, heading-only search
+          results.push({ section, document })
+
+          if (options.limit !== undefined && results.length >= options.limit) {
+            return results
+          }
+        }
+      }
+    }
+
+    return results
+  })
+
+// ============================================================================
+// Search with Content (legacy, uses heading-only search)
+// ============================================================================
+
+export const searchWithContent = (
+  rootPath: string,
+  options: SearchOptions = {},
+): Effect.Effect<readonly SearchResult[], Error> =>
+  Effect.gen(function* () {
+    const storage = createStorage(rootPath)
+    const results = yield* search(rootPath, options)
+
+    const resultsWithContent: SearchResult[] = []
+
+    for (const result of results) {
+      const filePath = path.join(storage.rootPath, result.section.documentPath)
+
+      try {
+        const fileContent = yield* Effect.promise(() =>
+          fs.readFile(filePath, 'utf-8'),
+        )
+
+        const lines = fileContent.split('\n')
+        const sectionContent = lines
+          .slice(result.section.startLine - 1, result.section.endLine)
+          .join('\n')
+
+        resultsWithContent.push({
+          ...result,
+          sectionContent,
+        })
+      } catch {
+        // If file can't be read, include result without content
+        resultsWithContent.push(result)
+      }
+    }
+
+    return resultsWithContent
+  })
+
+// ============================================================================
+// Context Generation
+// ============================================================================
+
+export interface ContextOptions {
+  /** Maximum tokens to include */
+  readonly maxTokens?: number | undefined
+  /** Include section content */
+  readonly includeContent?: boolean | undefined
+  /** Compression level: brief, summary, full */
+  readonly level?: 'brief' | 'summary' | 'full' | undefined
+}
+
+export interface DocumentContext {
+  readonly path: string
+  readonly title: string
+  readonly totalTokens: number
+  readonly includedTokens: number
+  readonly sections: readonly SectionContext[]
+}
+
+export interface SectionContext {
+  readonly heading: string
+  readonly level: number
+  readonly tokens: number
+  readonly content?: string | undefined
+  readonly hasCode: boolean
+  readonly hasList: boolean
+  readonly hasTable: boolean
+}
+
+export const getContext = (
+  rootPath: string,
+  filePath: string,
+  options: ContextOptions = {},
+): Effect.Effect<DocumentContext, Error> =>
+  Effect.gen(function* () {
+    const storage = createStorage(rootPath)
+    const resolvedFile = path.resolve(filePath)
+    const relativePath = path.relative(storage.rootPath, resolvedFile)
+
+    const docIndex = yield* loadDocumentIndex(storage)
+    const sectionIndex = yield* loadSectionIndex(storage)
+
+    if (!docIndex || !sectionIndex) {
+      return yield* Effect.fail(
+        new Error("Index not found. Run 'mdcontext index' first."),
+      )
+    }
+
+    const document = docIndex.documents[relativePath]
+    if (!document) {
+      return yield* Effect.fail(
+        new Error(`Document not found in index: ${relativePath}`),
+      )
+    }
+
+    // Get sections for this document
+    const sectionIds = sectionIndex.byDocument[document.id] ?? []
+    const sections: SectionContext[] = []
+    let includedTokens = 0
+    const maxTokens = options.maxTokens ?? Infinity
+    const includeContent = options.includeContent ?? options.level === 'full'
+
+    // Read file content if needed
+    let fileContent: string | null = null
+    if (includeContent) {
+      try {
+        fileContent = yield* Effect.promise(() =>
+          fs.readFile(resolvedFile, 'utf-8'),
+        )
+      } catch {
+        // Continue without content
+      }
+    }
+
+    const fileLines = fileContent?.split('\n') ?? []
+
+    for (const sectionId of sectionIds) {
+      const section = sectionIndex.sections[sectionId]
+      if (!section) continue
+
+      // Check token budget
+      if (includedTokens + section.tokenCount > maxTokens) {
+        // Include brief info only if we're over budget
+        if (options.level === 'brief') continue
+
+        sections.push({
+          heading: section.heading,
+          level: section.level,
+          tokens: section.tokenCount,
+          hasCode: section.hasCode,
+          hasList: section.hasList,
+          hasTable: section.hasTable,
+        })
+        continue
+      }
+
+      includedTokens += section.tokenCount
+
+      let content: string | undefined
+      if (includeContent && fileContent) {
+        content = fileLines
+          .slice(section.startLine - 1, section.endLine)
+          .join('\n')
+      }
+
+      sections.push({
+        heading: section.heading,
+        level: section.level,
+        tokens: section.tokenCount,
+        content,
+        hasCode: section.hasCode,
+        hasList: section.hasList,
+        hasTable: section.hasTable,
+      })
+    }
+
+    return {
+      path: relativePath,
+      title: document.title,
+      totalTokens: document.tokenCount,
+      includedTokens,
+      sections,
+    }
+  })
+
+// ============================================================================
+// LLM-Ready Output
+// ============================================================================
+
+export const formatContextForLLM = (context: DocumentContext): string => {
+  const lines: string[] = []
+
+  lines.push(`# ${context.title}`)
+  lines.push(`Path: ${context.path}`)
+  lines.push(`Tokens: ${context.includedTokens}/${context.totalTokens}`)
+  lines.push('')
+
+  for (const section of context.sections) {
+    const prefix = '#'.repeat(section.level)
+    const meta: string[] = []
+    if (section.hasCode) meta.push('code')
+    if (section.hasList) meta.push('list')
+    if (section.hasTable) meta.push('table')
+
+    const metaStr = meta.length > 0 ? ` [${meta.join(', ')}]` : ''
+    lines.push(
+      `${prefix} ${section.heading}${metaStr} (${section.tokens} tokens)`,
+    )
+
+    if (section.content) {
+      lines.push('')
+      lines.push(section.content)
+      lines.push('')
+    }
+  }
+
+  return lines.join('\n')
+}