mdcontext 0.0.1 → 0.1.0

package/src/index/types.ts

@@ -0,0 +1,109 @@
+/**
+ * Index data types for mdcontext
+ */
+
+// ============================================================================
+// Configuration
+// ============================================================================
+
+export interface IndexConfig {
+  readonly version: number
+  readonly rootPath: string
+  readonly include: readonly string[]
+  readonly exclude: readonly string[]
+  readonly createdAt: string
+  readonly updatedAt: string
+}
+
+// ============================================================================
+// Document Index
+// ============================================================================
+
+export interface DocumentIndex {
+  readonly version: number
+  readonly rootPath: string
+  readonly documents: Record<string, DocumentEntry>
+}
+
+export interface DocumentEntry {
+  readonly id: string
+  readonly path: string
+  readonly title: string
+  readonly mtime: number
+  readonly hash: string
+  readonly tokenCount: number
+  readonly sectionCount: number
+}
+
+// ============================================================================
+// Section Index
+// ============================================================================
+
+export interface SectionIndex {
+  readonly version: number
+  readonly sections: Record<string, SectionEntry>
+  readonly byHeading: Record<string, readonly string[]>
+  readonly byDocument: Record<string, readonly string[]>
+}
+
+export interface SectionEntry {
+  readonly id: string
+  readonly documentId: string
+  readonly documentPath: string
+  readonly heading: string
+  readonly level: number
+  readonly startLine: number
+  readonly endLine: number
+  readonly tokenCount: number
+  readonly hasCode: boolean
+  readonly hasList: boolean
+  readonly hasTable: boolean
+}
+
+// ============================================================================
+// Link Index
+// ============================================================================
+
+export interface LinkIndex {
+  readonly version: number
+  readonly forward: Record<string, readonly string[]>
+  readonly backward: Record<string, readonly string[]>
+  readonly broken: readonly string[]
+}
+
+// ============================================================================
+// Index Result
+// ============================================================================
+
+export interface IndexResult {
+  readonly documentsIndexed: number
+  readonly sectionsIndexed: number
+  readonly linksIndexed: number
+  readonly totalDocuments: number
+  readonly totalSections: number
+  readonly totalLinks: number
+  readonly duration: number
+  readonly errors: readonly IndexBuildError[]
+}
+
+export interface IndexBuildError {
+  readonly path: string
+  readonly message: string
+}
+
+// ============================================================================
+// Index Paths
+// ============================================================================
+
+export const INDEX_DIR = '.mdcontext'
+export const INDEX_VERSION = 1
+
+export const getIndexPaths = (rootPath: string) => ({
+  root: `${rootPath}/${INDEX_DIR}`,
+  config: `${rootPath}/${INDEX_DIR}/config.json`,
+  documents: `${rootPath}/${INDEX_DIR}/indexes/documents.json`,
+  sections: `${rootPath}/${INDEX_DIR}/indexes/sections.json`,
+  links: `${rootPath}/${INDEX_DIR}/indexes/links.json`,
+  cache: `${rootPath}/${INDEX_DIR}/cache`,
+  parsed: `${rootPath}/${INDEX_DIR}/cache/parsed`,
+})

package/src/index/watcher.ts

@@ -0,0 +1,131 @@
+/**
+ * File watcher for automatic re-indexing
+ */
+
+import * as path from 'node:path'
+import { watch } from 'chokidar'
+import { Effect } from 'effect'
+
+import { buildIndex, type IndexOptions } from './indexer.js'
+import { createStorage, indexExists } from './storage.js'
+
+// ============================================================================
+// Watcher Types
+// ============================================================================
+
+export interface WatcherOptions extends IndexOptions {
+  readonly debounceMs?: number
+  readonly onIndex?: (result: {
+    documentsIndexed: number
+    duration: number
+  }) => void
+  readonly onError?: (error: Error) => void
+}
+
+export interface Watcher {
+  readonly stop: () => void
+}
+
+// ============================================================================
+// Watcher Implementation
+// ============================================================================
+
+const isMarkdownFile = (filePath: string): boolean =>
+  filePath.endsWith('.md') || filePath.endsWith('.mdx')
+
+export const watchDirectory = (
+  rootPath: string,
+  options: WatcherOptions = {},
+): Effect.Effect<Watcher, Error> =>
+  Effect.gen(function* () {
+    const resolvedRoot = path.resolve(rootPath)
+    const storage = createStorage(resolvedRoot)
+    const debounceMs = options.debounceMs ?? 300
+
+    // Ensure index exists
+    const exists = yield* indexExists(storage)
+    if (!exists) {
+      // Build initial index
+      const result = yield* buildIndex(resolvedRoot, options)
+      options.onIndex?.({
+        documentsIndexed: result.documentsIndexed,
+        duration: result.duration,
+      })
+    }
+
+    // Create a debounce queue
+    const pendingPaths = new Set<string>()
+    let debounceTimer: NodeJS.Timeout | null = null
+
+    const scheduleReindex = () => {
+      if (debounceTimer) {
+        clearTimeout(debounceTimer)
+      }
+
+      debounceTimer = setTimeout(async () => {
+        if (pendingPaths.size === 0) return
+
+        pendingPaths.clear()
+
+        try {
+          const result = await Effect.runPromise(
+            buildIndex(resolvedRoot, options),
+          )
+          options.onIndex?.({
+            documentsIndexed: result.documentsIndexed,
+            duration: result.duration,
+          })
+        } catch (error) {
+          options.onError?.(
+            error instanceof Error ? error : new Error(String(error)),
+          )
+        }
+      }, debounceMs)
+    }
+
+    // Set up chokidar watcher
+    const watcher = watch(resolvedRoot, {
+      ignored: [
+        /(^|[/\\])\../, // Ignore dotfiles
+        '**/node_modules/**',
+      ],
+      persistent: true,
+      ignoreInitial: true,
+    })
+
+    watcher.on('add', (filePath) => {
+      if (isMarkdownFile(filePath)) {
+        pendingPaths.add(filePath)
+        scheduleReindex()
+      }
+    })
+
+    watcher.on('change', (filePath) => {
+      if (isMarkdownFile(filePath)) {
+        pendingPaths.add(filePath)
+        scheduleReindex()
+      }
+    })
+
+    watcher.on('unlink', (filePath) => {
+      if (isMarkdownFile(filePath)) {
+        pendingPaths.add(filePath)
+        scheduleReindex()
+      }
+    })
+
+    watcher.on('error', (error: unknown) => {
+      options.onError?.(
+        error instanceof Error ? error : new Error(String(error)),
+      )
+    })
+
+    return {
+      stop: () => {
+        if (debounceTimer) {
+          clearTimeout(debounceTimer)
+        }
+        watcher.close()
+      },
+    }
+  })

package/src/index.ts

@@ -0,0 +1,8 @@
+/**
+ * mdcontext - Token-efficient markdown analysis for LLMs
+ */
+
+export * from './core/index.js'
+export * from './index/index.js'
+export * from './parser/index.js'
+export * from './utils/index.js'