mdcontext 0.0.1 → 0.2.0

package/src/index/storage.ts

@@ -0,0 +1,260 @@
+/**
+ * Index storage operations
+ */
+
+import * as crypto from 'node:crypto'
+import * as fs from 'node:fs/promises'
+import * as path from 'node:path'
+import { Effect } from 'effect'
+
+import {
+  DirectoryCreateError,
+  FileReadError,
+  FileWriteError,
+  IndexCorruptedError,
+} from '../errors/index.js'
+import type {
+  DocumentIndex,
+  IndexConfig,
+  LinkIndex,
+  SectionIndex,
+} from './types.js'
+import { getIndexPaths, INDEX_VERSION } from './types.js'
+
+// ============================================================================
+// File System Helpers
+// ============================================================================
+
+const ensureDir = (
+  dirPath: string,
+): Effect.Effect<void, DirectoryCreateError> =>
+  Effect.tryPromise({
+    try: () => fs.mkdir(dirPath, { recursive: true }),
+    catch: (e) =>
+      new DirectoryCreateError({
+        path: dirPath,
+        message: e instanceof Error ? e.message : String(e),
+        cause: e,
+      }),
+  }).pipe(Effect.map(() => undefined))
+
+const readJsonFile = <T>(
+  filePath: string,
+): Effect.Effect<T | null, FileReadError | IndexCorruptedError> =>
+  Effect.gen(function* () {
+    // Try to read file content
+    const contentResult = yield* Effect.tryPromise({
+      try: () => fs.readFile(filePath, 'utf-8'),
+      catch: (e) => {
+        // File not found is not an error - return null
+        if (e && typeof e === 'object' && 'code' in e && e.code === 'ENOENT') {
+          return { notFound: true as const }
+        }
+        return new FileReadError({
+          path: filePath,
+          message: e instanceof Error ? e.message : String(e),
+          cause: e,
+        })
+      },
+    }).pipe(
+      Effect.map((content) =>
+        typeof content === 'string' ? { content } : content,
+      ),
+      // Note: catchAll here filters out "file not found" as expected case (returns null),
+      // while other errors are re-thrown to propagate as typed FileReadError
+      Effect.catchAll((e) =>
+        e && 'notFound' in e
+          ? Effect.succeed({ notFound: true as const })
+          : Effect.fail(e),
+      ),
+    )
+
+    // Handle not found
+    if ('notFound' in contentResult) {
+      return null
+    }
+
+    // Parse JSON - corrupted files should fail with IndexCorruptedError
+    return yield* Effect.try({
+      try: () => JSON.parse(contentResult.content) as T,
+      catch: (e) =>
+        new IndexCorruptedError({
+          path: filePath,
+          reason: 'InvalidJson',
+          details: e instanceof Error ? e.message : String(e),
+        }),
+    })
+  })
+
+const writeJsonFile = <T>(
+  filePath: string,
+  data: T,
+): Effect.Effect<void, DirectoryCreateError | FileWriteError> =>
+  Effect.gen(function* () {
+    const dir = path.dirname(filePath)
+    yield* ensureDir(dir)
+    yield* Effect.tryPromise({
+      try: () => fs.writeFile(filePath, JSON.stringify(data, null, 2)),
+      catch: (e) =>
+        new FileWriteError({
+          path: filePath,
+          message: e instanceof Error ? e.message : String(e),
+          cause: e,
+        }),
+    })
+  })
+
+// ============================================================================
+// Hash Computation
+// ============================================================================
+
+export const computeHash = (content: string): string => {
+  return crypto.createHash('sha256').update(content).digest('hex').slice(0, 16)
+}
+
+// ============================================================================
+// Index Storage Operations
+// ============================================================================
+
+export interface IndexStorage {
+  readonly rootPath: string
+  readonly paths: ReturnType<typeof getIndexPaths>
+}
+
+export const createStorage = (rootPath: string): IndexStorage => ({
+  rootPath: path.resolve(rootPath),
+  paths: getIndexPaths(path.resolve(rootPath)),
+})
+
+export const initializeIndex = (
+  storage: IndexStorage,
+): Effect.Effect<
+  void,
+  DirectoryCreateError | FileReadError | FileWriteError | IndexCorruptedError
+> =>
+  Effect.gen(function* () {
+    yield* ensureDir(storage.paths.root)
+    yield* ensureDir(storage.paths.parsed)
+    yield* ensureDir(path.dirname(storage.paths.documents))
+
+    // Create default config if it doesn't exist
+    const existingConfig = yield* loadConfig(storage)
+    if (!existingConfig) {
+      const config: IndexConfig = {
+        version: INDEX_VERSION,
+        rootPath: storage.rootPath,
+        include: ['**/*.md', '**/*.mdx'],
+        exclude: ['**/node_modules/**', '**/.*/**'],
+        createdAt: new Date().toISOString(),
+        updatedAt: new Date().toISOString(),
+      }
+      yield* saveConfig(storage, config)
+    }
+  })
+
+// ============================================================================
+// Config Operations
+// ============================================================================
+
+export const loadConfig = (
+  storage: IndexStorage,
+): Effect.Effect<IndexConfig | null, FileReadError | IndexCorruptedError> =>
+  readJsonFile<IndexConfig>(storage.paths.config)
+
+export const saveConfig = (
+  storage: IndexStorage,
+  config: IndexConfig,
+): Effect.Effect<void, DirectoryCreateError | FileWriteError> =>
+  writeJsonFile(storage.paths.config, {
+    ...config,
+    updatedAt: new Date().toISOString(),
+  })
+
+// ============================================================================
+// Document Index Operations
+// ============================================================================
+
+export const loadDocumentIndex = (
+  storage: IndexStorage,
+): Effect.Effect<DocumentIndex | null, FileReadError | IndexCorruptedError> =>
+  readJsonFile<DocumentIndex>(storage.paths.documents)
+
+export const saveDocumentIndex = (
+  storage: IndexStorage,
+  index: DocumentIndex,
+): Effect.Effect<void, DirectoryCreateError | FileWriteError> =>
+  writeJsonFile(storage.paths.documents, index)
+
+export const createEmptyDocumentIndex = (rootPath: string): DocumentIndex => ({
+  version: INDEX_VERSION,
+  rootPath,
+  documents: {},
+})
+
+// ============================================================================
+// Section Index Operations
+// ============================================================================
+
+export const loadSectionIndex = (
+  storage: IndexStorage,
+): Effect.Effect<SectionIndex | null, FileReadError | IndexCorruptedError> =>
+  readJsonFile<SectionIndex>(storage.paths.sections)
+
+export const saveSectionIndex = (
+  storage: IndexStorage,
+  index: SectionIndex,
+): Effect.Effect<void, DirectoryCreateError | FileWriteError> =>
+  writeJsonFile(storage.paths.sections, index)
+
+export const createEmptySectionIndex = (): SectionIndex => ({
+  version: INDEX_VERSION,
+  sections: {},
+  byHeading: Object.create(null),
+  byDocument: Object.create(null),
+})
+
+// ============================================================================
+// Link Index Operations
+// ============================================================================
+
+export const loadLinkIndex = (
+  storage: IndexStorage,
+): Effect.Effect<LinkIndex | null, FileReadError | IndexCorruptedError> =>
+  readJsonFile<LinkIndex>(storage.paths.links)
+
+export const saveLinkIndex = (
+  storage: IndexStorage,
+  index: LinkIndex,
+): Effect.Effect<void, DirectoryCreateError | FileWriteError> =>
+  writeJsonFile(storage.paths.links, index)
+
+export const createEmptyLinkIndex = (): LinkIndex => ({
+  version: INDEX_VERSION,
+  forward: Object.create(null),
+  backward: Object.create(null),
+  broken: [],
+})
+
+// ============================================================================
+// Index Existence Check
+// ============================================================================
+
+export const indexExists = (
+  storage: IndexStorage,
+): Effect.Effect<boolean, FileReadError> =>
+  Effect.tryPromise({
+    try: async () => {
+      try {
+        await fs.access(storage.paths.config)
+        return true
+      } catch {
+        return false
+      }
+    },
+    catch: (e) =>
+      new FileReadError({
+        path: storage.paths.config,
+        message: e instanceof Error ? e.message : String(e),
+        cause: e,
+      }),
+  })

package/src/index/types.ts

@@ -0,0 +1,147 @@
+/**
+ * Index data types for mdcontext
+ */
+
+// ============================================================================
+// Configuration
+// ============================================================================
+
+export interface IndexConfig {
+  readonly version: number
+  readonly rootPath: string
+  readonly include: readonly string[]
+  readonly exclude: readonly string[]
+  readonly createdAt: string
+  readonly updatedAt: string
+}
+
+// ============================================================================
+// Document Index
+// ============================================================================
+
+export interface DocumentIndex {
+  readonly version: number
+  readonly rootPath: string
+  readonly documents: Record<string, DocumentEntry>
+}
+
+export interface DocumentEntry {
+  readonly id: string
+  readonly path: string
+  readonly title: string
+  readonly mtime: number
+  readonly hash: string
+  readonly tokenCount: number
+  readonly sectionCount: number
+}
+
+// ============================================================================
+// Section Index
+// ============================================================================
+
+export interface SectionIndex {
+  readonly version: number
+  readonly sections: Record<string, SectionEntry>
+  readonly byHeading: Record<string, readonly string[]>
+  readonly byDocument: Record<string, readonly string[]>
+}
+
+export interface SectionEntry {
+  readonly id: string
+  readonly documentId: string
+  readonly documentPath: string
+  readonly heading: string
+  readonly level: number
+  readonly startLine: number
+  readonly endLine: number
+  readonly tokenCount: number
+  readonly hasCode: boolean
+  readonly hasList: boolean
+  readonly hasTable: boolean
+}
+
+// ============================================================================
+// Link Index
+// ============================================================================
+
+export interface LinkIndex {
+  readonly version: number
+  readonly forward: Record<string, readonly string[]>
+  readonly backward: Record<string, readonly string[]>
+  readonly broken: readonly string[]
+}
+
+// ============================================================================
+// Index Result
+// ============================================================================
+
+/**
+ * Reason why a file was skipped during indexing
+ */
+export type SkipReason =
+  | 'unchanged' // File hash and mtime unchanged
+  | 'excluded' // Matches exclude pattern
+  | 'hidden' // Hidden file or directory
+  | 'not-markdown' // Not a markdown file
+  | 'binary' // Binary file detected
+  | 'oversized' // File too large
+
+/**
+ * Information about a skipped file
+ */
+export interface SkippedFile {
+  readonly path: string
+  readonly reason: SkipReason
+}
+
+/**
+ * Summary of skipped files by reason
+ */
+export interface SkipSummary {
+  readonly unchanged: number
+  readonly excluded: number
+  readonly hidden: number
+  readonly total: number
+}
+
+export interface IndexResult {
+  readonly documentsIndexed: number
+  readonly sectionsIndexed: number
+  readonly linksIndexed: number
+  readonly totalDocuments: number
+  readonly totalSections: number
+  readonly totalLinks: number
+  readonly duration: number
+  /** Non-fatal file processing errors (files that couldn't be indexed) */
+  readonly errors: readonly FileProcessingError[]
+  readonly skipped: SkipSummary
+}
+
+/**
+ * Non-fatal error during file processing in index build.
+ * These are collected and reported but don't stop the build.
+ *
+ * Note: This is distinct from IndexBuildError in errors/index.ts,
+ * which is a TaggedError for fatal build failures.
+ */
+export interface FileProcessingError {
+  readonly path: string
+  readonly message: string
+}
+
+// ============================================================================
+// Index Paths
+// ============================================================================
+
+export const INDEX_DIR = '.mdcontext'
+export const INDEX_VERSION = 1
+
+export const getIndexPaths = (rootPath: string) => ({
+  root: `${rootPath}/${INDEX_DIR}`,
+  config: `${rootPath}/${INDEX_DIR}/config.json`,
+  documents: `${rootPath}/${INDEX_DIR}/indexes/documents.json`,
+  sections: `${rootPath}/${INDEX_DIR}/indexes/sections.json`,
+  links: `${rootPath}/${INDEX_DIR}/indexes/links.json`,
+  cache: `${rootPath}/${INDEX_DIR}/cache`,
+  parsed: `${rootPath}/${INDEX_DIR}/cache/parsed`,
+})

package/src/index/watcher.ts

@@ -0,0 +1,189 @@
+/**
+ * File watcher for automatic re-indexing
+ *
+ * ## Why Not Effect Streams?
+ *
+ * We evaluated using Effect Streams (ALP-101) but decided the current approach is better:
+ *
+ * 1. **chokidar is battle-tested** - Handles OS-specific quirks (FSEvents on macOS,
+ *    inotify on Linux, ReadDirectoryChangesW on Windows)
+ *
+ * 2. **Debouncing handles backpressure** - The 300ms debounce already batches rapid
+ *    changes, so Stream backpressure isn't needed
+ *
+ * 3. **Simple use case** - File change → rebuild index. No complex transformations
+ *    or compositions that would benefit from Stream operators
+ *
+ * 4. **Already Effect-based** - The setup/teardown is wrapped in Effect for proper
+ *    error handling, and we use typed errors (WatchError, IndexBuildError)
+ *
+ * If future requirements need more sophisticated event processing (filtering by
+ * content type, incremental updates, event replay), reconsider Streams then.
+ */
+
+import * as path from 'node:path'
+import { watch } from 'chokidar'
+import { Effect } from 'effect'
+
+import {
+  type DirectoryCreateError,
+  type DirectoryWalkError,
+  type FileReadError,
+  type FileWriteError,
+  type IndexCorruptedError,
+  WatchError,
+} from '../errors/index.js'
+import { getChokidarIgnorePatterns } from './ignore-patterns.js'
+import { buildIndex, type IndexOptions } from './indexer.js'
+import { createStorage, indexExists } from './storage.js'
+
+/**
+ * Union of errors that can occur during watch operations
+ */
+export type WatchDirectoryError =
+  | WatchError
+  | DirectoryWalkError
+  | DirectoryCreateError
+  | FileReadError
+  | FileWriteError
+  | IndexCorruptedError
+
+// ============================================================================
+// Watcher Types
+// ============================================================================
+
+export interface WatcherOptions extends IndexOptions {
+  readonly debounceMs?: number
+  readonly onIndex?: (result: {
+    documentsIndexed: number
+    duration: number
+  }) => void
+  readonly onError?: (error: WatchError) => void
+  /** Whether to honor .gitignore for file watching (default: true) */
+  readonly honorGitignore?: boolean
+  /** Whether to honor .mdcontextignore for file watching (default: true) */
+  readonly honorMdcontextignore?: boolean
+}
+
+export interface Watcher {
+  readonly stop: () => void
+}
+
+// ============================================================================
+// Watcher Implementation
+// ============================================================================
+
+const isMarkdownFile = (filePath: string): boolean =>
+  filePath.endsWith('.md') || filePath.endsWith('.mdx')
+
+export const watchDirectory = (
+  rootPath: string,
+  options: WatcherOptions = {},
+): Effect.Effect<Watcher, WatchDirectoryError> =>
+  Effect.gen(function* () {
+    const resolvedRoot = path.resolve(rootPath)
+    const storage = createStorage(resolvedRoot)
+    const debounceMs = options.debounceMs ?? 300
+
+    // Ensure index exists
+    const exists = yield* indexExists(storage)
+    if (!exists) {
+      // Build initial index
+      const result = yield* buildIndex(resolvedRoot, options)
+      options.onIndex?.({
+        documentsIndexed: result.documentsIndexed,
+        duration: result.duration,
+      })
+    }
+
+    // Create a debounce queue
+    const pendingPaths = new Set<string>()
+    let debounceTimer: NodeJS.Timeout | null = null
+
+    const scheduleReindex = () => {
+      if (debounceTimer) {
+        clearTimeout(debounceTimer)
+      }
+
+      debounceTimer = setTimeout(async () => {
+        if (pendingPaths.size === 0) return
+
+        pendingPaths.clear()
+
+        try {
+          const result = await Effect.runPromise(
+            buildIndex(resolvedRoot, options),
+          )
+          options.onIndex?.({
+            documentsIndexed: result.documentsIndexed,
+            duration: result.duration,
+          })
+        } catch (error) {
+          options.onError?.(
+            new WatchError({
+              path: resolvedRoot,
+              message:
+                error instanceof Error ? error.message : 'Index rebuild failed',
+              cause: error,
+            }),
+          )
+        }
+      }, debounceMs)
+    }
+
+    // Build ignore patterns for chokidar
+    const ignorePatterns = yield* getChokidarIgnorePatterns({
+      rootPath: resolvedRoot,
+      cliPatterns: options.exclude,
+      honorGitignore: options.honorGitignore ?? true,
+      honorMdcontextignore: options.honorMdcontextignore ?? true,
+    })
+
+    // Set up chokidar watcher with dynamic ignore patterns
+    const watcher = watch(resolvedRoot, {
+      ignored: ignorePatterns,
+      persistent: true,
+      ignoreInitial: true,
+    })
+
+    watcher.on('add', (filePath) => {
+      if (isMarkdownFile(filePath)) {
+        pendingPaths.add(filePath)
+        scheduleReindex()
+      }
+    })
+
+    watcher.on('change', (filePath) => {
+      if (isMarkdownFile(filePath)) {
+        pendingPaths.add(filePath)
+        scheduleReindex()
+      }
+    })
+
+    watcher.on('unlink', (filePath) => {
+      if (isMarkdownFile(filePath)) {
+        pendingPaths.add(filePath)
+        scheduleReindex()
+      }
+    })
+
+    watcher.on('error', (error: unknown) => {
+      options.onError?.(
+        new WatchError({
+          path: resolvedRoot,
+          message:
+            error instanceof Error ? error.message : 'File watcher error',
+          cause: error,
+        }),
+      )
+    })
+
+    return {
+      stop: () => {
+        if (debounceTimer) {
+          clearTimeout(debounceTimer)
+        }
+        watcher.close()
+      },
+    }
+  })

package/src/index.ts

@@ -0,0 +1,30 @@
+/**
+ * mdcontext - Token-efficient markdown analysis for LLMs
+ */
+
+// Config utilities for user config files
+export type { PartialMdContextConfig } from './config/service.js'
+export * from './core/index.js'
+export * from './index/index.js'
+export * from './parser/index.js'
+export * from './utils/index.js'
+
+/**
+ * Type-safe configuration helper for mdcontext.config.ts files.
+ *
+ * @example
+ * ```typescript
+ * import { defineConfig } from 'mdcontext'
+ *
+ * export default defineConfig({
+ *   index: {
+ *     maxDepth: 5,
+ *   },
+ * })
+ * ```
+ */
+export const defineConfig = <
+  T extends import('./config/service.js').PartialMdContextConfig,
+>(
+  config: T,
+): T => config