mdcontext 0.0.1 → 0.1.0

package/src/parser/section-filter.ts

@@ -0,0 +1,270 @@
+/**
+ * Section filtering utilities for extracting specific sections from markdown documents
+ */
+
+import type { MdDocument, MdSection } from '../core/types.js'
+
+// ============================================================================
+// Simple Glob Matching
+// ============================================================================
+
+/**
+ * Simple glob pattern matching (supports * and ?)
+ */
+const globMatch = (text: string, pattern: string): boolean => {
+  // Convert glob pattern to regex
+  const regexPattern = pattern
+    .replace(/[.+^${}()|[\]\\]/g, '\\$&') // Escape regex special chars except * and ?
+    .replace(/\*/g, '.*')
+    .replace(/\?/g, '.')
+
+  const regex = new RegExp(`^${regexPattern}$`, 'i')
+  return regex.test(text)
+}
+
+// ============================================================================
+// Types
+// ============================================================================
+
+export interface SectionListItem {
+  readonly number: string
+  readonly heading: string
+  readonly level: number
+  readonly tokenCount: number
+}
+
+export interface SectionFilterOptions {
+  /** If true, don't include nested subsections */
+  readonly shallow?: boolean
+}
+
+// ============================================================================
+// Section Map Building
+// ============================================================================
+
+/**
+ * Build a flat list of all sections with their hierarchical numbers
+ * e.g., "1", "1.1", "1.2", "2", "2.1", etc.
+ */
+export const buildSectionList = (document: MdDocument): SectionListItem[] => {
+  const result: SectionListItem[] = []
+
+  const processSection = (
+    section: MdSection,
+    prefix: string,
+    index: number,
+  ): void => {
+    const number = prefix ? `${prefix}.${index + 1}` : `${index + 1}`
+
+    result.push({
+      number,
+      heading: section.heading,
+      level: section.level,
+      tokenCount: section.metadata.tokenCount,
+    })
+
+    // Process children
+    section.children.forEach((child, i) => {
+      processSection(child, number, i)
+    })
+  }
+
+  document.sections.forEach((section, i) => {
+    processSection(section, '', i)
+  })
+
+  return result
+}
+
+/**
+ * Format section list for display
+ */
+export const formatSectionList = (sections: SectionListItem[]): string => {
+  const lines: string[] = []
+
+  for (const section of sections) {
+    // Indent based on dots in number
+    const depth = (section.number.match(/\./g) || []).length
+    const indent = '   '.repeat(depth)
+    lines.push(
+      `${indent}${section.number}. ${section.heading} (${section.tokenCount} tokens)`,
+    )
+  }
+
+  return lines.join('\n')
+}
+
+// ============================================================================
+// Section Matching
+// ============================================================================
+
+/**
+ * Check if a section matches a selector (by number, exact name, or glob pattern)
+ */
+const matchesSelector = (
+  section: SectionListItem,
+  selector: string,
+): boolean => {
+  // Check if it's a number match (e.g., "5.3")
+  if (/^[\d.]+$/.test(selector)) {
+    // Exact number match
+    return section.number === selector
+  }
+
+  // Check for exact heading match (case-insensitive)
+  if (section.heading.toLowerCase() === selector.toLowerCase()) {
+    return true
+  }
+
+  // Check for glob pattern match
+  if (selector.includes('*') || selector.includes('?')) {
+    return globMatch(section.heading, selector)
+  }
+
+  // Partial match (contains)
+  return section.heading.toLowerCase().includes(selector.toLowerCase())
+}
+
+/**
+ * Find all sections matching a selector
+ */
+export const findMatchingSections = (
+  sectionList: SectionListItem[],
+  selector: string,
+): SectionListItem[] => {
+  return sectionList.filter((s) => matchesSelector(s, selector))
+}
+
+/**
+ * Get all descendant section numbers for a given section number
+ */
+const getDescendantNumbers = (
+  sectionList: SectionListItem[],
+  parentNumber: string,
+): Set<string> => {
+  const result = new Set<string>()
+  const prefix = `${parentNumber}.`
+
+  for (const section of sectionList) {
+    if (section.number.startsWith(prefix)) {
+      result.add(section.number)
+    }
+  }
+
+  return result
+}
+
+// ============================================================================
+// Section Content Extraction
+// ============================================================================
+
+/**
+ * Extract content for specific sections from a document
+ */
+export const extractSectionContent = (
+  document: MdDocument,
+  selector: string,
+  options: SectionFilterOptions = {},
+): {
+  sections: MdSection[]
+  matchedNumbers: string[]
+} => {
+  const sectionList = buildSectionList(document)
+  const matchedSections = findMatchingSections(sectionList, selector)
+
+  if (matchedSections.length === 0) {
+    return { sections: [], matchedNumbers: [] }
+  }
+
+  // Get all section numbers to include
+  const numbersToInclude = new Set<string>()
+  const matchedNumbers: string[] = []
+
+  for (const matched of matchedSections) {
+    numbersToInclude.add(matched.number)
+    matchedNumbers.push(matched.number)
+
+    if (!options.shallow) {
+      // Include all descendants
+      const descendants = getDescendantNumbers(sectionList, matched.number)
+      for (const desc of descendants) {
+        numbersToInclude.add(desc)
+      }
+    }
+  }
+
+  // Build a map from section number to section for efficient lookup
+  const numberToSection = new Map<string, MdSection>()
+
+  const mapSections = (
+    sections: readonly MdSection[],
+    prefix: string,
+  ): void => {
+    sections.forEach((section, i) => {
+      const number = prefix ? `${prefix}.${i + 1}` : `${i + 1}`
+      numberToSection.set(number, section)
+      mapSections(section.children, number)
+    })
+  }
+
+  mapSections(document.sections, '')
+
+  // Extract matching sections
+  const extractedSections: MdSection[] = []
+
+  for (const number of matchedNumbers) {
+    const section = numberToSection.get(number)
+    if (section) {
+      if (options.shallow) {
+        // Clone without children for shallow mode
+        extractedSections.push({
+          ...section,
+          children: [],
+        })
+      } else {
+        extractedSections.push(section)
+      }
+    }
+  }
+
+  return { sections: extractedSections, matchedNumbers }
+}
+
+/**
+ * Format extracted sections as markdown content
+ */
+export const formatExtractedSections = (sections: MdSection[]): string => {
+  const formatSection = (
+    section: MdSection,
+    includeChildren: boolean,
+  ): string => {
+    const lines: string[] = []
+
+    // Add heading
+    const headingPrefix = '#'.repeat(section.level)
+    lines.push(`${headingPrefix} ${section.heading}`)
+    lines.push('')
+
+    // Add content (strip the heading line if it starts with #)
+    const contentLines = section.content.split('\n')
+    const contentWithoutHeading = contentLines
+      .filter((line, i) => i > 0 || !line.startsWith('#'))
+      .join('\n')
+      .trim()
+
+    if (contentWithoutHeading) {
+      lines.push(contentWithoutHeading)
+    }
+
+    if (includeChildren) {
+      for (const child of section.children) {
+        lines.push('')
+        lines.push(formatSection(child, true))
+      }
+    }
+
+    return lines.join('\n')
+  }
+
+  return sections.map((s) => formatSection(s, true)).join('\n\n')
+}

package/src/search/query-parser.test.ts

@@ -0,0 +1,260 @@
+/**
+ * Tests for query parser
+ */
+
+import { describe, expect, it } from 'vitest'
+import {
+  buildHighlightPattern,
+  evaluateQuery,
+  isAdvancedQuery,
+  parseQuery,
+  type QueryNode,
+} from './query-parser.js'
+
+describe('query-parser', () => {
+  describe('parseQuery', () => {
+    it('should parse a single term', () => {
+      const result = parseQuery('auth')
+      expect(result).not.toBeNull()
+      expect(result!.ast).toEqual({ type: 'term', value: 'auth' })
+      expect(result!.terms).toEqual(['auth'])
+    })
+
+    it('should parse a quoted phrase', () => {
+      const result = parseQuery('"context resumption"')
+      expect(result).not.toBeNull()
+      expect(result!.ast).toEqual({
+        type: 'phrase',
+        value: 'context resumption',
+      })
+      expect(result!.phrases).toEqual(['context resumption'])
+    })
+
+    it('should parse AND operator', () => {
+      const result = parseQuery('auth AND criticism')
+      expect(result).not.toBeNull()
+      expect(result!.ast).toEqual({
+        type: 'and',
+        left: { type: 'term', value: 'auth' },
+        right: { type: 'term', value: 'criticism' },
+      })
+    })
+
+    it('should parse OR operator', () => {
+      const result = parseQuery('checkpoint OR gate')
+      expect(result).not.toBeNull()
+      expect(result!.ast).toEqual({
+        type: 'or',
+        left: { type: 'term', value: 'checkpoint' },
+        right: { type: 'term', value: 'gate' },
+      })
+    })
+
+    it('should parse NOT operator', () => {
+      const result = parseQuery('implementation NOT example')
+      expect(result).not.toBeNull()
+      // "implementation NOT example" is parsed as: implementation AND (NOT example)
+      expect(result!.ast).toEqual({
+        type: 'and',
+        left: { type: 'term', value: 'implementation' },
+        right: { type: 'not', operand: { type: 'term', value: 'example' } },
+      })
+    })
+
+    it('should parse grouped expressions', () => {
+      const result = parseQuery('auth AND (error OR bug)')
+      expect(result).not.toBeNull()
+      expect(result!.ast).toEqual({
+        type: 'and',
+        left: { type: 'term', value: 'auth' },
+        right: {
+          type: 'or',
+          left: { type: 'term', value: 'error' },
+          right: { type: 'term', value: 'bug' },
+        },
+      })
+    })
+
+    it('should handle case-insensitive operators', () => {
+      const result1 = parseQuery('auth and criticism')
+      const result2 = parseQuery('auth And criticism')
+      expect(result1!.ast).toEqual(result2!.ast)
+    })
+
+    it('should parse phrase combined with boolean', () => {
+      const result = parseQuery('"context resumption" AND drift')
+      expect(result).not.toBeNull()
+      expect(result!.ast).toEqual({
+        type: 'and',
+        left: { type: 'phrase', value: 'context resumption' },
+        right: { type: 'term', value: 'drift' },
+      })
+    })
+
+    it('should parse implicit AND between terms', () => {
+      const result = parseQuery('auth error')
+      expect(result).not.toBeNull()
+      expect(result!.ast).toEqual({
+        type: 'and',
+        left: { type: 'term', value: 'auth' },
+        right: { type: 'term', value: 'error' },
+      })
+    })
+
+    it('should respect operator precedence (NOT > AND > OR)', () => {
+      // "a OR b AND NOT c" should parse as "a OR (b AND (NOT c))"
+      const result = parseQuery('a OR b AND NOT c')
+      expect(result).not.toBeNull()
+      expect(result!.ast).toEqual({
+        type: 'or',
+        left: { type: 'term', value: 'a' },
+        right: {
+          type: 'and',
+          left: { type: 'term', value: 'b' },
+          right: { type: 'not', operand: { type: 'term', value: 'c' } },
+        },
+      })
+    })
+
+    it('should return null for empty query', () => {
+      expect(parseQuery('')).toBeNull()
+      expect(parseQuery('   ')).toBeNull()
+    })
+  })
+
+  describe('isAdvancedQuery', () => {
+    it('should return false for simple terms', () => {
+      expect(isAdvancedQuery('auth')).toBe(false)
+      expect(isAdvancedQuery('some term')).toBe(false)
+    })
+
+    it('should return true for boolean operators', () => {
+      expect(isAdvancedQuery('auth AND error')).toBe(true)
+      expect(isAdvancedQuery('auth OR error')).toBe(true)
+      expect(isAdvancedQuery('auth NOT error')).toBe(true)
+    })
+
+    it('should return true for phrases', () => {
+      expect(isAdvancedQuery('"exact phrase"')).toBe(true)
+    })
+
+    it('should return true for grouped expressions', () => {
+      expect(isAdvancedQuery('(a OR b)')).toBe(true)
+    })
+  })
+
+  describe('evaluateQuery', () => {
+    const text = 'This is about authentication errors and bug fixes'
+
+    it('should match single term', () => {
+      const ast: QueryNode = { type: 'term', value: 'authentication' }
+      expect(evaluateQuery(ast, text)).toBe(true)
+    })
+
+    it('should not match missing term', () => {
+      const ast: QueryNode = { type: 'term', value: 'security' }
+      expect(evaluateQuery(ast, text)).toBe(false)
+    })
+
+    it('should match phrase', () => {
+      const ast: QueryNode = { type: 'phrase', value: 'authentication errors' }
+      expect(evaluateQuery(ast, text)).toBe(true)
+    })
+
+    it('should not match partial phrase', () => {
+      const ast: QueryNode = { type: 'phrase', value: 'authentication bug' }
+      expect(evaluateQuery(ast, text)).toBe(false)
+    })
+
+    it('should evaluate AND correctly', () => {
+      const ast: QueryNode = {
+        type: 'and',
+        left: { type: 'term', value: 'authentication' },
+        right: { type: 'term', value: 'bug' },
+      }
+      expect(evaluateQuery(ast, text)).toBe(true)
+
+      const ast2: QueryNode = {
+        type: 'and',
+        left: { type: 'term', value: 'authentication' },
+        right: { type: 'term', value: 'security' },
+      }
+      expect(evaluateQuery(ast2, text)).toBe(false)
+    })
+
+    it('should evaluate OR correctly', () => {
+      const ast: QueryNode = {
+        type: 'or',
+        left: { type: 'term', value: 'authentication' },
+        right: { type: 'term', value: 'security' },
+      }
+      expect(evaluateQuery(ast, text)).toBe(true)
+
+      const ast2: QueryNode = {
+        type: 'or',
+        left: { type: 'term', value: 'crypto' },
+        right: { type: 'term', value: 'security' },
+      }
+      expect(evaluateQuery(ast2, text)).toBe(false)
+    })
+
+    it('should evaluate NOT correctly', () => {
+      const ast: QueryNode = {
+        type: 'not',
+        operand: { type: 'term', value: 'security' },
+      }
+      expect(evaluateQuery(ast, text)).toBe(true)
+
+      const ast2: QueryNode = {
+        type: 'not',
+        operand: { type: 'term', value: 'authentication' },
+      }
+      expect(evaluateQuery(ast2, text)).toBe(false)
+    })
+
+    it('should be case-insensitive', () => {
+      const ast: QueryNode = { type: 'term', value: 'AUTHENTICATION' }
+      expect(evaluateQuery(ast, text)).toBe(true)
+    })
+
+    it('should handle complex nested expressions', () => {
+      // (auth AND bug) OR (error AND NOT fixes)
+      const ast: QueryNode = {
+        type: 'or',
+        left: {
+          type: 'and',
+          left: { type: 'term', value: 'auth' },
+          right: { type: 'term', value: 'bug' },
+        },
+        right: {
+          type: 'and',
+          left: { type: 'term', value: 'error' },
+          right: { type: 'not', operand: { type: 'term', value: 'fixes' } },
+        },
+      }
+      expect(evaluateQuery(ast, text)).toBe(true)
+    })
+  })
+
+  describe('buildHighlightPattern', () => {
+    it('should create pattern from terms', () => {
+      const parsed = parseQuery('auth error')!
+      const pattern = buildHighlightPattern(parsed)
+      expect(pattern.test('authentication error')).toBe(true)
+      expect(pattern.test('no match here')).toBe(false)
+    })
+
+    it('should create pattern from phrases', () => {
+      const parsed = parseQuery('"exact phrase"')!
+      const pattern = buildHighlightPattern(parsed)
+      expect(pattern.test('this is the exact phrase here')).toBe(true)
+    })
+
+    it('should escape special regex characters', () => {
+      const parsed = parseQuery('"test.value"')!
+      const pattern = buildHighlightPattern(parsed)
+      expect(pattern.test('has test.value inside')).toBe(true)
+      expect(pattern.test('has testXvalue inside')).toBe(false) // . should not match any char
+    })
+  })
+})