mdcontext 0.0.1 → 0.2.0

package/src/index/ignore-patterns.test.ts

@@ -0,0 +1,354 @@
+/**
+ * Unit tests for ignore patterns module
+ *
+ * Tests verify:
+ * - Pattern matching (globs, negation, comments, directory-only)
+ * - Precedence (CLI > .mdcontextignore > .gitignore > defaults)
+ * - Edge cases (missing files, empty files, invalid patterns)
+ */
+
+import * as fs from 'node:fs/promises'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import { Effect } from 'effect'
+import { afterEach, beforeEach, describe, expect, it } from 'vitest'
+import {
+  createFilterFunction,
+  createIgnoreFilter,
+  DEFAULT_IGNORE_PATTERNS,
+  getChokidarIgnorePatterns,
+  shouldIgnore,
+} from './ignore-patterns.js'
+
+describe('Ignore Patterns Module', () => {
+  let testDir: string
+
+  beforeEach(async () => {
+    testDir = await fs.mkdtemp(path.join(os.tmpdir(), 'mdcontext-ignore-test-'))
+  })
+
+  afterEach(async () => {
+    await fs.rm(testDir, { recursive: true, force: true })
+  })
+
+  // ==========================================================================
+  // Pattern Matching
+  // ==========================================================================
+
+  describe('Pattern Matching', () => {
+    it('matches glob patterns (*.log)', async () => {
+      await fs.writeFile(path.join(testDir, '.gitignore'), '*.log\n')
+
+      const result = await Effect.runPromise(
+        createIgnoreFilter({ rootPath: testDir }),
+      )
+
+      expect(shouldIgnore('debug.log', result.filter)).toBe(true)
+      expect(shouldIgnore('app.log', result.filter)).toBe(true)
+      expect(shouldIgnore('app.txt', result.filter)).toBe(false)
+    })
+
+    it('matches deep glob patterns (**/*.tmp)', async () => {
+      await fs.writeFile(path.join(testDir, '.gitignore'), '**/*.tmp\n')
+
+      const result = await Effect.runPromise(
+        createIgnoreFilter({ rootPath: testDir }),
+      )
+
+      expect(shouldIgnore('file.tmp', result.filter)).toBe(true)
+      expect(shouldIgnore('src/file.tmp', result.filter)).toBe(true)
+      expect(shouldIgnore('src/deep/file.tmp', result.filter)).toBe(true)
+      expect(shouldIgnore('src/file.md', result.filter)).toBe(false)
+    })
+
+    it('handles negation patterns (!important.log)', async () => {
+      await fs.writeFile(
+        path.join(testDir, '.gitignore'),
+        '*.log\n!important.log\n',
+      )
+
+      const result = await Effect.runPromise(
+        createIgnoreFilter({ rootPath: testDir }),
+      )
+
+      expect(shouldIgnore('debug.log', result.filter)).toBe(true)
+      expect(shouldIgnore('important.log', result.filter)).toBe(false)
+    })
+
+    it('ignores comments (# ignore this)', async () => {
+      await fs.writeFile(
+        path.join(testDir, '.gitignore'),
+        '# This is a comment\n*.log\n# Another comment\n',
+      )
+
+      const result = await Effect.runPromise(
+        createIgnoreFilter({ rootPath: testDir }),
+      )
+
+      // Comments should not affect pattern matching
+      expect(shouldIgnore('debug.log', result.filter)).toBe(true)
+      expect(shouldIgnore('# This is a comment', result.filter)).toBe(false)
+    })
+
+    it('matches directory-only patterns (node_modules/)', async () => {
+      await fs.writeFile(path.join(testDir, '.gitignore'), 'build/\n')
+
+      const result = await Effect.runPromise(
+        createIgnoreFilter({ rootPath: testDir }),
+      )
+
+      expect(shouldIgnore('build/output.js', result.filter)).toBe(true)
+      expect(shouldIgnore('build', result.filter)).toBe(true)
+    })
+  })
+
+  // ==========================================================================
+  // Precedence
+  // ==========================================================================
+
+  describe('Precedence', () => {
+    it('CLI patterns override .mdcontextignore', async () => {
+      await fs.writeFile(
+        path.join(testDir, '.mdcontextignore'),
+        '*.md\n!important.md\n',
+      )
+
+      const result = await Effect.runPromise(
+        createIgnoreFilter({
+          rootPath: testDir,
+          cliPatterns: ['important.md'], // CLI says ignore it
+        }),
+      )
+
+      // CLI takes precedence - important.md should be ignored
+      expect(shouldIgnore('important.md', result.filter)).toBe(true)
+    })
+
+    it('.mdcontextignore overrides .gitignore', async () => {
+      await fs.writeFile(path.join(testDir, '.gitignore'), '*.md\n')
+      await fs.writeFile(
+        path.join(testDir, '.mdcontextignore'),
+        '!README.md\n', // Allow README.md
+      )
+
+      const result = await Effect.runPromise(
+        createIgnoreFilter({ rootPath: testDir }),
+      )
+
+      // .mdcontextignore negation should override .gitignore
+      expect(shouldIgnore('README.md', result.filter)).toBe(false)
+      expect(shouldIgnore('other.md', result.filter)).toBe(true)
+    })
+
+    it('later rules override earlier (gitignore behavior)', async () => {
+      await fs.writeFile(
+        path.join(testDir, '.gitignore'),
+        '*.md\n!important.md\nimportant.md\n', // Re-ignore important.md
+      )
+
+      const result = await Effect.runPromise(
+        createIgnoreFilter({ rootPath: testDir }),
+      )
+
+      // Last rule wins - important.md should be ignored again
+      expect(shouldIgnore('important.md', result.filter)).toBe(true)
+    })
+  })
+
+  // ==========================================================================
+  // Edge Cases
+  // ==========================================================================
+
+  describe('Edge Cases', () => {
+    it('handles missing .gitignore gracefully', async () => {
+      // No .gitignore file exists
+      const result = await Effect.runPromise(
+        createIgnoreFilter({ rootPath: testDir }),
+      )
+
+      // Should still work with defaults
+      expect(result.filter).toBeDefined()
+      expect(result.sources).not.toContain('.gitignore')
+      expect(shouldIgnore('node_modules/pkg/file.js', result.filter)).toBe(true)
+    })
+
+    it('handles missing .mdcontextignore gracefully', async () => {
+      await fs.writeFile(path.join(testDir, '.gitignore'), '*.log\n')
+      // No .mdcontextignore file exists
+
+      const result = await Effect.runPromise(
+        createIgnoreFilter({ rootPath: testDir }),
+      )
+
+      expect(result.sources).toContain('.gitignore')
+      expect(result.sources).not.toContain('.mdcontextignore')
+    })
+
+    it('handles empty files', async () => {
+      await fs.writeFile(path.join(testDir, '.gitignore'), '')
+      await fs.writeFile(path.join(testDir, '.mdcontextignore'), '   \n\n')
+
+      const result = await Effect.runPromise(
+        createIgnoreFilter({ rootPath: testDir }),
+      )
+
+      // Should work with just defaults
+      expect(result.filter).toBeDefined()
+      expect(result.sources).not.toContain('.gitignore')
+      expect(result.sources).not.toContain('.mdcontextignore')
+    })
+
+    it('handles whitespace-only patterns (skipped)', async () => {
+      await fs.writeFile(
+        path.join(testDir, '.gitignore'),
+        '   \n\n*.log\n   \n',
+      )
+
+      const result = await Effect.runPromise(
+        createIgnoreFilter({ rootPath: testDir }),
+      )
+
+      // Should skip whitespace lines
+      expect(shouldIgnore('debug.log', result.filter)).toBe(true)
+    })
+  })
+
+  // ==========================================================================
+  // Default Patterns
+  // ==========================================================================
+
+  describe('Default Patterns', () => {
+    it('includes node_modules in defaults', () => {
+      expect(DEFAULT_IGNORE_PATTERNS).toContain('node_modules')
+    })
+
+    it('includes .git in defaults', () => {
+      expect(DEFAULT_IGNORE_PATTERNS).toContain('.git')
+    })
+
+    it('includes dist in defaults', () => {
+      expect(DEFAULT_IGNORE_PATTERNS).toContain('dist')
+    })
+
+    it('includes build in defaults', () => {
+      expect(DEFAULT_IGNORE_PATTERNS).toContain('build')
+    })
+
+    it('applies defaults without any ignore files', async () => {
+      const result = await Effect.runPromise(
+        createIgnoreFilter({ rootPath: testDir }),
+      )
+
+      expect(shouldIgnore('node_modules/pkg/index.js', result.filter)).toBe(
+        true,
+      )
+      expect(shouldIgnore('dist/bundle.js', result.filter)).toBe(true)
+      expect(shouldIgnore('build/output.js', result.filter)).toBe(true)
+      expect(shouldIgnore('.git/config', result.filter)).toBe(true)
+    })
+  })
+
+  // ==========================================================================
+  // Honor Flags
+  // ==========================================================================
+
+  describe('Honor Flags', () => {
+    it('respects honorGitignore=false', async () => {
+      await fs.writeFile(path.join(testDir, '.gitignore'), '*.secret\n')
+
+      const result = await Effect.runPromise(
+        createIgnoreFilter({
+          rootPath: testDir,
+          honorGitignore: false,
+        }),
+      )
+
+      // .gitignore patterns should not be applied
+      expect(shouldIgnore('password.secret', result.filter)).toBe(false)
+      expect(result.sources).not.toContain('.gitignore')
+    })
+
+    it('respects honorMdcontextignore=false', async () => {
+      await fs.writeFile(path.join(testDir, '.mdcontextignore'), 'drafts/\n')
+
+      const result = await Effect.runPromise(
+        createIgnoreFilter({
+          rootPath: testDir,
+          honorMdcontextignore: false,
+        }),
+      )
+
+      // .mdcontextignore patterns should not be applied
+      expect(shouldIgnore('drafts/doc.md', result.filter)).toBe(false)
+      expect(result.sources).not.toContain('.mdcontextignore')
+    })
+  })
+
+  // ==========================================================================
+  // Filter Function
+  // ==========================================================================
+
+  describe('createFilterFunction', () => {
+    it('creates a function suitable for Array.filter', async () => {
+      await fs.writeFile(path.join(testDir, '.gitignore'), '*.log\n')
+
+      const result = await Effect.runPromise(
+        createIgnoreFilter({ rootPath: testDir }),
+      )
+
+      const filterFn = createFilterFunction(result.filter)
+      const files = ['app.md', 'debug.log', 'src/index.ts', 'error.log']
+      const filtered = files.filter(filterFn)
+
+      expect(filtered).toEqual(['app.md', 'src/index.ts'])
+    })
+  })
+
+  // ==========================================================================
+  // Chokidar Patterns
+  // ==========================================================================
+
+  describe('getChokidarIgnorePatterns', () => {
+    it('returns patterns suitable for chokidar', async () => {
+      await fs.writeFile(path.join(testDir, '.gitignore'), '*.log\n')
+
+      const patterns = await Effect.runPromise(
+        getChokidarIgnorePatterns({ rootPath: testDir }),
+      )
+
+      expect(patterns.length).toBeGreaterThan(0)
+      // Should include a regex string for dotfiles
+      expect(patterns.some((p) => typeof p === 'string')).toBe(true)
+    })
+
+    it('includes default patterns', async () => {
+      const patterns = await Effect.runPromise(
+        getChokidarIgnorePatterns({ rootPath: testDir }),
+      )
+
+      // Should have patterns for node_modules, etc.
+      expect(patterns.some((p) => p.includes('node_modules'))).toBe(true)
+    })
+  })
+
+  // ==========================================================================
+  // Pattern Count
+  // ==========================================================================
+
+  describe('Pattern Count', () => {
+    it('counts patterns from all sources', async () => {
+      await fs.writeFile(path.join(testDir, '.gitignore'), '*.log\n*.tmp\n')
+      await fs.writeFile(path.join(testDir, '.mdcontextignore'), 'drafts/\n')
+
+      const result = await Effect.runPromise(
+        createIgnoreFilter({
+          rootPath: testDir,
+          cliPatterns: ['*.bak'],
+        }),
+      )
+
+      // 4 defaults + 2 gitignore + 1 mdcontextignore + 1 CLI = 8
+      expect(result.patternCount).toBe(8)
+    })
+  })
+})

package/src/index/ignore-patterns.ts

@@ -0,0 +1,305 @@
+/**
+ * Ignore Pattern Support Module
+ *
+ * Provides .gitignore and .mdcontextignore support using the battle-tested `ignore` npm package.
+ * Implements the following precedence (highest to lowest):
+ *
+ * 1. CLI --exclude flag
+ * 2. MDCONTEXT_INDEX_EXCLUDEPATTERNS env var
+ * 3. Config file excludePatterns
+ * 4. .mdcontextignore file
+ * 5. .gitignore file
+ * 6. Built-in defaults: ['node_modules', '.git', 'dist', 'build']
+ */
+
+import * as fs from 'node:fs/promises'
+import * as path from 'node:path'
+import { Effect } from 'effect'
+import ignore, { type Ignore } from 'ignore'
+
+// ============================================================================
+// Types
+// ============================================================================
+
+/**
+ * Options for building the ignore filter
+ */
+export interface IgnoreOptions {
+  /** Root directory to search for ignore files */
+  readonly rootPath: string
+  /** CLI/config exclude patterns (highest priority) */
+  readonly cliPatterns?: readonly string[] | undefined
+  /** Whether to honor .gitignore (default: true) */
+  readonly honorGitignore?: boolean | undefined
+  /** Whether to honor .mdcontextignore (default: true) */
+  readonly honorMdcontextignore?: boolean | undefined
+}
+
+/**
+ * Result of loading ignore patterns
+ */
+export interface IgnoreFilterResult {
+  /** The ignore filter instance */
+  readonly filter: Ignore
+  /** Source files that were loaded */
+  readonly sources: readonly string[]
+  /** Total number of patterns loaded */
+  readonly patternCount: number
+}
+
+// ============================================================================
+// Constants
+// ============================================================================
+
+/**
+ * Default patterns always applied (lowest priority)
+ */
+export const DEFAULT_IGNORE_PATTERNS: readonly string[] = [
+  'node_modules',
+  '.git',
+  'dist',
+  'build',
+]
+
+// ============================================================================
+// File Loading
+// ============================================================================
+
+/**
+ * Try to read an ignore file, returning empty string if it doesn't exist or is unreadable.
+ */
+const tryReadIgnoreFile = (filePath: string): Effect.Effect<string, never> =>
+  Effect.tryPromise({
+    try: () => fs.readFile(filePath, 'utf-8'),
+    catch: () => '',
+  }).pipe(Effect.catchAll(() => Effect.succeed('')))
+
+/**
+ * Parse ignore file contents, filtering out empty lines and comments.
+ * Returns the number of valid patterns found.
+ */
+const countPatterns = (content: string): number => {
+  if (!content.trim()) return 0
+  return content.split('\n').filter((line) => {
+    const trimmed = line.trim()
+    return trimmed.length > 0 && !trimmed.startsWith('#')
+  }).length
+}
+
+// ============================================================================
+// Main API
+// ============================================================================
+
+/**
+ * Create an ignore filter with proper precedence.
+ *
+ * Loads patterns from (in order, lower priority first):
+ * 1. Built-in defaults
+ * 2. .gitignore (if exists and honorGitignore is true)
+ * 3. .mdcontextignore (if exists and honorMdcontextignore is true)
+ * 4. CLI/config patterns (highest priority)
+ *
+ * @example
+ * ```typescript
+ * const result = yield* createIgnoreFilter({
+ *   rootPath: '/my/project',
+ *   cliPatterns: ['*.log', 'temp/'],
+ * })
+ *
+ * // Check if a file should be ignored
+ * if (result.filter.ignores('node_modules/package/file.md')) {
+ *   // Skip this file
+ * }
+ *
+ * // Or filter an array of paths
+ * const includedFiles = files.filter(result.filter.createFilter())
+ * ```
+ */
+export const createIgnoreFilter = (
+  options: IgnoreOptions,
+): Effect.Effect<IgnoreFilterResult, never> =>
+  Effect.gen(function* () {
+    const {
+      rootPath,
+      cliPatterns = [],
+      honorGitignore = true,
+      honorMdcontextignore = true,
+    } = options
+
+    const ig = ignore()
+    const sources: string[] = []
+    let patternCount = 0
+
+    // 1. Add defaults (lowest priority)
+    ig.add(DEFAULT_IGNORE_PATTERNS as string[])
+    patternCount += DEFAULT_IGNORE_PATTERNS.length
+
+    // 2. Load .gitignore if enabled
+    if (honorGitignore) {
+      const gitignorePath = path.join(rootPath, '.gitignore')
+      const gitignoreContent = yield* tryReadIgnoreFile(gitignorePath)
+      if (gitignoreContent.trim()) {
+        ig.add(gitignoreContent)
+        const count = countPatterns(gitignoreContent)
+        patternCount += count
+        sources.push('.gitignore')
+      }
+    }
+
+    // 3. Load .mdcontextignore if enabled
+    if (honorMdcontextignore) {
+      const mdcontextignorePath = path.join(rootPath, '.mdcontextignore')
+      const mdcontextignoreContent =
+        yield* tryReadIgnoreFile(mdcontextignorePath)
+      if (mdcontextignoreContent.trim()) {
+        ig.add(mdcontextignoreContent)
+        const count = countPatterns(mdcontextignoreContent)
+        patternCount += count
+        sources.push('.mdcontextignore')
+      }
+    }
+
+    // 4. Add CLI/config patterns (highest priority)
+    if (cliPatterns.length > 0) {
+      ig.add(cliPatterns as string[])
+      patternCount += cliPatterns.length
+      sources.push('CLI/config')
+    }
+
+    return {
+      filter: ig,
+      sources,
+      patternCount,
+    }
+  })
+
+/**
+ * Check if a path should be ignored.
+ *
+ * This is a convenience wrapper around createIgnoreFilter for single-path checks.
+ * For checking multiple paths, prefer creating the filter once and reusing it.
+ *
+ * @param relativePath - Path relative to root (e.g., 'src/foo/bar.md')
+ * @param filter - The ignore filter instance
+ * @returns true if the path should be ignored
+ */
+export const shouldIgnore = (relativePath: string, filter: Ignore): boolean => {
+  // The ignore package requires paths without leading slash
+  const normalized = relativePath.replace(/^\//, '')
+  return filter.ignores(normalized)
+}
+
+/**
+ * Create a filter function suitable for Array.filter().
+ *
+ * @example
+ * ```typescript
+ * const result = yield* createIgnoreFilter({ rootPath })
+ * const filterFn = createFilterFunction(result.filter)
+ * const includedFiles = files.filter(filterFn)
+ * ```
+ */
+export const createFilterFunction = (
+  filter: Ignore,
+): ((relativePath: string) => boolean) => {
+  const innerFilter = filter.createFilter()
+  return (relativePath: string) => {
+    // The ignore package's createFilter returns true for non-ignored files
+    const normalized = relativePath.replace(/^\//, '')
+    return innerFilter(normalized)
+  }
+}
+
+/**
+ * Get ignore patterns as an array of strings for chokidar.
+ *
+ * Chokidar uses anymatch which accepts globs, so we convert
+ * the ignore patterns to glob format.
+ *
+ * @param options - Ignore options
+ * @returns Array of patterns suitable for chokidar's `ignored` option
+ */
+export const getChokidarIgnorePatterns = (
+  options: IgnoreOptions,
+): Effect.Effect<string[], never> =>
+  Effect.gen(function* () {
+    const {
+      rootPath,
+      cliPatterns = [],
+      honorGitignore = true,
+      honorMdcontextignore = true,
+    } = options
+
+    const patterns: string[] = []
+
+    // Always ignore dotfiles (chokidar regex format)
+    patterns.push(/(^|[/\\])\./.source)
+
+    // Add defaults
+    for (const p of DEFAULT_IGNORE_PATTERNS) {
+      patterns.push(`**/${p}/**`)
+    }
+
+    // Load .gitignore patterns
+    if (honorGitignore) {
+      const gitignorePath = path.join(rootPath, '.gitignore')
+      const content = yield* tryReadIgnoreFile(gitignorePath)
+      if (content.trim()) {
+        const parsed = parseIgnoreFile(content)
+        for (const p of parsed) {
+          patterns.push(convertToGlob(p))
+        }
+      }
+    }
+
+    // Load .mdcontextignore patterns
+    if (honorMdcontextignore) {
+      const mdcontextignorePath = path.join(rootPath, '.mdcontextignore')
+      const content = yield* tryReadIgnoreFile(mdcontextignorePath)
+      if (content.trim()) {
+        const parsed = parseIgnoreFile(content)
+        for (const p of parsed) {
+          patterns.push(convertToGlob(p))
+        }
+      }
+    }
+
+    // Add CLI patterns
+    for (const p of cliPatterns) {
+      patterns.push(convertToGlob(p))
+    }
+
+    return patterns
+  })
+
+// ============================================================================
+// Helpers
+// ============================================================================
+
+/**
+ * Parse ignore file content into individual patterns
+ */
+const parseIgnoreFile = (content: string): string[] => {
+  return content
+    .split('\n')
+    .map((line) => line.trim())
+    .filter((line) => line.length > 0 && !line.startsWith('#'))
+}
+
+/**
+ * Convert a gitignore pattern to glob format for chokidar
+ */
+const convertToGlob = (pattern: string): string => {
+  // Negation patterns - keep as is for now (chokidar handles them differently)
+  if (pattern.startsWith('!')) {
+    return pattern
+  }
+
+  // Already a glob pattern
+  if (pattern.includes('*') || pattern.includes('/')) {
+    return pattern.startsWith('/') ? pattern.slice(1) : `**/${pattern}`
+  }
+
+  // Simple name - match anywhere
+  return `**/${pattern}/**`
+}

package/src/index/index.ts

@@ -0,0 +1,4 @@
+export * from './indexer.js'
+export * from './storage.js'
+export * from './types.js'
+export * from './watcher.js'