mdcontext 0.0.1 → 0.1.0

package/src/cli/main.ts

@@ -0,0 +1,145 @@
+#!/usr/bin/env node
+
+/**
+ * mdcontext CLI - Token-efficient markdown analysis
+ *
+ * CORE COMMANDS
+ *   mdcontext index [path]           Index markdown files (default: .)
+ *   mdcontext search <query> [path]  Search by meaning or structure
+ *   mdcontext context <files...>     Get LLM-ready summary
+ *   mdcontext tree [path|file]       Show files or document outline
+ *
+ * LINK ANALYSIS
+ *   mdcontext links <file>           What does this link to?
+ *   mdcontext backlinks <file>       What links to this?
+ *
+ * INSPECTION
+ *   mdcontext stats [path]           Index statistics
+ */
+
+import { CliConfig, Command } from '@effect/cli'
+import { NodeContext, NodeRuntime } from '@effect/platform-node'
+import { Effect, Layer } from 'effect'
+import { preprocessArgv } from './argv-preprocessor.js'
+import {
+  backlinksCommand,
+  contextCommand,
+  indexCommand,
+  linksCommand,
+  searchCommand,
+  statsCommand,
+  treeCommand,
+} from './commands/index.js'
+import {
+  checkSubcommandHelp,
+  shouldShowMainHelp,
+  showMainHelp,
+} from './help.js'
+
+// ============================================================================
+// Main CLI
+// ============================================================================
+
+const mainCommand = Command.make('mdcontext').pipe(
+  Command.withDescription('Token-efficient markdown analysis for LLMs'),
+  Command.withSubcommands([
+    indexCommand,
+    searchCommand,
+    contextCommand,
+    treeCommand,
+    linksCommand,
+    backlinksCommand,
+    statsCommand,
+  ]),
+)
+
+const cli = Command.run(mainCommand, {
+  name: 'mdcontext',
+  version: '0.1.0',
+})
+
+// Clean CLI config: hide built-in options from help
+const cliConfigLayer = CliConfig.layer({
+  showBuiltIns: false,
+})
+
+// ============================================================================
+// Error Handling
+// ============================================================================
+
+// Custom error formatter
+const formatCliError = (error: unknown): string => {
+  if (error && typeof error === 'object') {
+    // Handle Effect CLI validation errors
+    const err = error as Record<string, unknown>
+    if (err._tag === 'ValidationError' && err.error) {
+      const validationError = err.error as Record<string, unknown>
+      // Extract the actual error message
+      if (validationError._tag === 'Paragraph' && validationError.value) {
+        const paragraph = validationError.value as Record<string, unknown>
+        if (paragraph._tag === 'Text' && typeof paragraph.value === 'string') {
+          return paragraph.value
+        }
+      }
+    }
+    // Handle MissingValue errors
+    if (err._tag === 'MissingValue' && err.error) {
+      const missingError = err.error as Record<string, unknown>
+      if (missingError._tag === 'Paragraph' && missingError.value) {
+        const paragraph = missingError.value as Record<string, unknown>
+        if (paragraph._tag === 'Text' && typeof paragraph.value === 'string') {
+          return paragraph.value
+        }
+      }
+    }
+  }
+  return String(error)
+}
+
+// Check if error is a CLI validation error (should show friendly message)
+const isValidationError = (error: unknown): boolean => {
+  if (error && typeof error === 'object') {
+    const err = error as Record<string, unknown>
+    return (
+      err._tag === 'ValidationError' ||
+      err._tag === 'MissingValue' ||
+      err._tag === 'InvalidValue'
+    )
+  }
+  return false
+}
+
+// ============================================================================
+// Custom Help Handling
+// ============================================================================
+
+// Check for subcommand help before anything else
+checkSubcommandHelp()
+
+// Check if we should show main help
+if (shouldShowMainHelp()) {
+  showMainHelp()
+  process.exit(0)
+}
+
+// Preprocess argv to allow flexible flag positioning
+const processedArgv = preprocessArgv(process.argv)
+
+// Run with clean config and friendly errors
+Effect.suspend(() => cli(processedArgv)).pipe(
+  Effect.provide(Layer.merge(NodeContext.layer, cliConfigLayer)),
+  Effect.catchAll((error) =>
+    Effect.sync(() => {
+      // Only show friendly error for validation errors
+      if (isValidationError(error)) {
+        const message = formatCliError(error)
+        console.error(`\nError: ${message}`)
+        console.error('\nRun "mdcontext --help" for usage information.')
+        process.exit(1)
+      }
+      // Re-throw other errors to be handled normally
+      throw error
+    }),
+  ),
+  NodeRuntime.runMain,
+)

package/src/cli/options.ts

@@ -0,0 +1,31 @@
+/**
+ * Shared CLI Options
+ *
+ * Common options used across multiple commands.
+ */
+
+import { Options } from '@effect/cli'
+
+/**
+ * Output as JSON
+ */
+export const jsonOption = Options.boolean('json').pipe(
+  Options.withDescription('Output as JSON'),
+  Options.withDefault(false),
+)
+
+/**
+ * Pretty-print JSON output
+ */
+export const prettyOption = Options.boolean('pretty').pipe(
+  Options.withDescription('Pretty-print JSON output'),
+  Options.withDefault(true),
+)
+
+/**
+ * Force full rebuild
+ */
+export const forceOption = Options.boolean('force').pipe(
+  Options.withDescription('Force full rebuild, ignoring cache'),
+  Options.withDefault(false),
+)

package/src/cli/typo-suggester.test.ts

@@ -0,0 +1,105 @@
+/**
+ * Unit tests for typo-suggester
+ */
+
+import { describe, expect, it } from 'vitest'
+import type { CommandSchema } from './flag-schemas.js'
+import {
+  formatValidFlags,
+  levenshteinDistance,
+  suggestFlag,
+} from './typo-suggester.js'
+
+describe('levenshteinDistance', () => {
+  it('returns 0 for identical strings', () => {
+    expect(levenshteinDistance('test', 'test')).toBe(0)
+  })
+
+  it('returns string length for empty comparison', () => {
+    expect(levenshteinDistance('test', '')).toBe(4)
+    expect(levenshteinDistance('', 'test')).toBe(4)
+  })
+
+  it('calculates single character difference', () => {
+    expect(levenshteinDistance('test', 'tset')).toBe(2) // transposition
+    expect(levenshteinDistance('test', 'tests')).toBe(1) // insertion
+    expect(levenshteinDistance('test', 'tes')).toBe(1) // deletion
+    expect(levenshteinDistance('test', 'tast')).toBe(1) // substitution
+  })
+
+  it('calculates distance for common typos', () => {
+    expect(levenshteinDistance('json', 'jsno')).toBe(2)
+    expect(levenshteinDistance('limit', 'limt')).toBe(1)
+    expect(levenshteinDistance('tokens', 'toekns')).toBe(2)
+  })
+})
+
+describe('suggestFlag', () => {
+  const mockSchema: CommandSchema = {
+    name: 'test',
+    flags: [
+      { name: 'json', type: 'boolean', description: 'Output JSON' },
+      { name: 'limit', type: 'string', alias: 'n', description: 'Max results' },
+      { name: 'threshold', type: 'string', description: 'Threshold' },
+    ],
+  }
+
+  it('suggests correct flag for typo', () => {
+    const result = suggestFlag('--jsno', mockSchema)
+    expect(result).toBeDefined()
+    expect(result?.flag).toBe('--json')
+  })
+
+  it('suggests correct flag for missing letter', () => {
+    const result = suggestFlag('--limt', mockSchema)
+    expect(result).toBeDefined()
+    expect(result?.flag).toBe('--limit')
+  })
+
+  it('returns undefined for no close match', () => {
+    const result = suggestFlag('--foobar', mockSchema)
+    expect(result).toBeUndefined()
+  })
+
+  it('handles prefix matches', () => {
+    const result = suggestFlag('--js', mockSchema)
+    expect(result).toBeDefined()
+    expect(result?.flag).toBe('--json')
+  })
+
+  it('handles short flag typos', () => {
+    const result = suggestFlag('-m', mockSchema)
+    expect(result).toBeDefined()
+    expect(result?.flag).toBe('--limit') // -m is close to -n
+  })
+
+  it('respects maxDistance parameter', () => {
+    const result = suggestFlag('--jsno', mockSchema, 1)
+    expect(result).toBeUndefined() // distance is 2, exceeds max
+  })
+})
+
+describe('formatValidFlags', () => {
+  const mockSchema: CommandSchema = {
+    name: 'test',
+    flags: [
+      { name: 'json', type: 'boolean', description: 'Output JSON' },
+      { name: 'limit', type: 'string', alias: 'n', description: 'Max results' },
+      { name: 'threshold', type: 'string' },
+    ],
+  }
+
+  it('formats flags with descriptions', () => {
+    const output = formatValidFlags(mockSchema)
+    expect(output).toContain('--json')
+    expect(output).toContain('Output JSON')
+    expect(output).toContain('--limit, -n')
+    expect(output).toContain('Max results')
+    expect(output).toContain('--threshold')
+  })
+
+  it('includes alias when present', () => {
+    const output = formatValidFlags(mockSchema)
+    expect(output).toContain(', -n')
+  })
+})

package/src/cli/typo-suggester.ts

@@ -0,0 +1,130 @@
+/**
+ * Typo Suggester
+ *
+ * Uses Levenshtein distance to suggest correct flags when users mistype.
+ */
+
+import type { CommandSchema } from './flag-schemas.js'
+
+/**
+ * Calculate Levenshtein distance between two strings
+ */
+export const levenshteinDistance = (a: string, b: string): number => {
+  const matrix: number[][] = []
+
+  // Initialize first column
+  for (let i = 0; i <= a.length; i++) {
+    matrix[i] = [i]
+  }
+
+  // Initialize first row
+  for (let j = 0; j <= b.length; j++) {
+    matrix[0]![j] = j
+  }
+
+  // Fill in the rest
+  for (let i = 1; i <= a.length; i++) {
+    for (let j = 1; j <= b.length; j++) {
+      const cost = a[i - 1] === b[j - 1] ? 0 : 1
+      matrix[i]![j] = Math.min(
+        matrix[i - 1]![j]! + 1, // deletion
+        matrix[i]![j - 1]! + 1, // insertion
+        matrix[i - 1]![j - 1]! + cost, // substitution
+      )
+    }
+  }
+
+  return matrix[a.length]![b.length]!
+}
+
+/**
+ * Suggestion result
+ */
+export interface Suggestion {
+  flag: string
+  distance: number
+  description: string | undefined
+}
+
+/**
+ * Find the best flag suggestion for a typo
+ *
+ * @param typo - The mistyped flag (e.g., '--jsno')
+ * @param schema - The command schema to search
+ * @param maxDistance - Maximum Levenshtein distance to consider (default: 2)
+ * @returns Best matching flag or undefined
+ */
+export const suggestFlag = (
+  typo: string,
+  schema: CommandSchema,
+  maxDistance: number = 2,
+): Suggestion | undefined => {
+  // Normalize the typo (remove leading dashes for comparison)
+  const normalizedTypo = typo.replace(/^-+/, '')
+
+  let bestMatch: Suggestion | undefined
+  let bestDistance = Infinity
+
+  for (const spec of schema.flags) {
+    // Check against full flag name
+    const flagName = spec.name
+    const distance = levenshteinDistance(normalizedTypo, flagName)
+
+    if (distance <= maxDistance && distance < bestDistance) {
+      bestDistance = distance
+      bestMatch = {
+        flag: `--${spec.name}`,
+        distance,
+        description: spec.description,
+      }
+    }
+
+    // Also check against alias if present
+    if (spec.alias) {
+      const aliasDistance = levenshteinDistance(normalizedTypo, spec.alias)
+      if (aliasDistance <= maxDistance && aliasDistance < bestDistance) {
+        bestDistance = aliasDistance
+        bestMatch = {
+          flag: `--${spec.name}`,
+          distance: aliasDistance,
+          description: spec.description,
+        }
+      }
+    }
+  }
+
+  // Prefer exact prefix matches (e.g., '--js' should suggest '--json')
+  if (!bestMatch || bestDistance > 0) {
+    for (const spec of schema.flags) {
+      if (spec.name.startsWith(normalizedTypo)) {
+        // Prefix match - this is likely what they meant
+        const prefixDistance = spec.name.length - normalizedTypo.length
+        if (prefixDistance <= maxDistance && prefixDistance < bestDistance) {
+          bestDistance = prefixDistance
+          bestMatch = {
+            flag: `--${spec.name}`,
+            distance: prefixDistance,
+            description: spec.description,
+          }
+        }
+      }
+    }
+  }
+
+  return bestMatch
+}
+
+/**
+ * Format a list of valid flags for a command
+ */
+export const formatValidFlags = (schema: CommandSchema): string => {
+  const lines: string[] = []
+
+  for (const spec of schema.flags) {
+    const alias = spec.alias ? `, -${spec.alias}` : ''
+    const desc = spec.description ? `  ${spec.description}` : ''
+    lines.push(`  --${spec.name}${alias}${desc}`)
+  }
+
+  return lines.join('\n')
+}

package/src/cli/utils.ts

@@ -0,0 +1,126 @@
+/**
+ * CLI Utility Functions
+ *
+ * Shared helper functions used across CLI commands.
+ */
+
+import * as fsPromises from 'node:fs/promises'
+import * as path from 'node:path'
+
+/**
+ * Format object as JSON string
+ */
+export const formatJson = (obj: unknown, pretty: boolean): string => {
+  return pretty ? JSON.stringify(obj, null, 2) : JSON.stringify(obj)
+}
+
+/**
+ * Check if filename is a markdown file
+ */
+export const isMarkdownFile = (filename: string): boolean => {
+  return filename.endsWith('.md') || filename.endsWith('.mdx')
+}
+
+/**
+ * Recursively walk directory and collect markdown files
+ */
+export const walkDir = async (dir: string): Promise<string[]> => {
+  const files: string[] = []
+  const entries = await fsPromises.readdir(dir, { withFileTypes: true })
+
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name)
+
+    // Skip hidden directories and node_modules
+    if (entry.name.startsWith('.') || entry.name === 'node_modules') {
+      continue
+    }
+
+    if (entry.isDirectory()) {
+      const subFiles = await walkDir(fullPath)
+      files.push(...subFiles)
+    } else if (entry.isFile() && isMarkdownFile(entry.name)) {
+      files.push(fullPath)
+    }
+  }
+
+  return files
+}
+
+/**
+ * Check if a query looks like a regex pattern
+ */
+export const isRegexPattern = (query: string): boolean => {
+  // Has regex special characters (excluding simple spaces and common punctuation)
+  return /[.*+?^${}()|[\]\\]/.test(query)
+}
+
+/**
+ * Check if embeddings exist for a directory
+ */
+export const hasEmbeddings = async (dir: string): Promise<boolean> => {
+  const vectorsPath = path.join(dir, '.mdcontext', 'vectors.bin')
+  try {
+    await fsPromises.access(vectorsPath)
+    return true
+  } catch {
+    return false
+  }
+}
+
+/**
+ * Get index information for display
+ */
+export interface IndexInfo {
+  exists: boolean
+  lastUpdated?: string | undefined
+  sectionCount?: number | undefined
+  embeddingsExist: boolean
+  vectorCount?: number | undefined
+}
+
+export const getIndexInfo = async (dir: string): Promise<IndexInfo> => {
+  const sectionsPath = path.join(dir, '.mdcontext', 'indexes', 'sections.json')
+  const vectorsMetaPath = path.join(dir, '.mdcontext', 'vectors.meta.json')
+
+  let exists = false
+  let lastUpdated: string | undefined
+  let sectionCount: number | undefined
+  let embeddingsExist = false
+  let vectorCount: number | undefined
+
+  // Check sections index
+  try {
+    const stat = await fsPromises.stat(sectionsPath)
+    exists = true
+    lastUpdated = stat.mtime.toISOString()
+
+    const content = await fsPromises.readFile(sectionsPath, 'utf-8')
+    const sections = JSON.parse(content)
+    sectionCount = Object.keys(sections.sections || {}).length
+  } catch {
+    // Index doesn't exist
+  }
+
+  // Check vectors metadata
+  try {
+    const content = await fsPromises.readFile(vectorsMetaPath, 'utf-8')
+    const meta = JSON.parse(content)
+    embeddingsExist = true
+    vectorCount = Object.keys(meta.entries || {}).length
+    // Use vector meta updatedAt if available
+    if (meta.updatedAt) {
+      lastUpdated = meta.updatedAt
+    }
+  } catch {
+    // Embeddings don't exist
+  }
+
+  return {
+    exists,
+    lastUpdated,
+    sectionCount,
+    embeddingsExist,
+    vectorCount,
+  }
+}

package/src/core/index.ts

@@ -0,0 +1 @@
+export * from './types.js'

package/src/core/types.ts

@@ -0,0 +1,140 @@
+/**
+ * Core data types for mdcontext
+ * Based on DESIGN.md specifications
+ */
+
+// ============================================================================
+// Document Types
+// ============================================================================
+
+export interface MdDocument {
+  readonly id: string
+  readonly path: string
+  readonly title: string
+  readonly frontmatter: Record<string, unknown>
+  readonly sections: readonly MdSection[]
+  readonly links: readonly MdLink[]
+  readonly codeBlocks: readonly MdCodeBlock[]
+  readonly metadata: DocumentMetadata
+}
+
+export interface DocumentMetadata {
+  readonly wordCount: number
+  readonly tokenCount: number
+  readonly headingCount: number
+  readonly linkCount: number
+  readonly codeBlockCount: number
+  readonly lastModified: Date
+  readonly indexedAt: Date
+}
+
+// ============================================================================
+// Section Types
+// ============================================================================
+
+export type HeadingLevel = 1 | 2 | 3 | 4 | 5 | 6
+
+export interface MdSection {
+  readonly id: string
+  readonly heading: string
+  readonly level: HeadingLevel
+  readonly content: string
+  readonly plainText: string
+  readonly startLine: number
+  readonly endLine: number
+  readonly children: readonly MdSection[]
+  readonly metadata: SectionMetadata
+}
+
+export interface SectionMetadata {
+  readonly wordCount: number
+  readonly tokenCount: number
+  readonly hasCode: boolean
+  readonly hasList: boolean
+  readonly hasTable: boolean
+}
+
+// ============================================================================
+// Link Types
+// ============================================================================
+
+export type LinkType = 'internal' | 'external' | 'image'
+
+export interface MdLink {
+  readonly type: LinkType
+  readonly href: string
+  readonly text: string
+  readonly sectionId: string
+  readonly line: number
+}
+
+// ============================================================================
+// Code Block Types
+// ============================================================================
+
+export interface MdCodeBlock {
+  readonly language: string | null
+  readonly content: string
+  readonly sectionId: string
+  readonly startLine: number
+  readonly endLine: number
+}
+
+// ============================================================================
+// Error Types
+// ============================================================================
+
+export interface ParseError {
+  readonly _tag: 'ParseError'
+  readonly message: string
+  readonly line?: number | undefined
+  readonly column?: number | undefined
+}
+
+export interface IoError {
+  readonly _tag: 'IoError'
+  readonly message: string
+  readonly path: string
+  readonly cause?: unknown
+}
+
+export interface IndexError {
+  readonly _tag: 'IndexError'
+  readonly cause: 'DiskFull' | 'Permission' | 'Corrupted' | 'Unknown'
+  readonly message: string
+}
+
+// ============================================================================
+// Constructor Functions
+// ============================================================================
+
+export const ParseError = (
+  message: string,
+  line?: number,
+  column?: number,
+): ParseError => ({
+  _tag: 'ParseError',
+  message,
+  line,
+  column,
+})
+
+export const IoError = (
+  message: string,
+  path: string,
+  cause?: unknown,
+): IoError => ({
+  _tag: 'IoError',
+  message,
+  path,
+  cause,
+})
+
+export const IndexError = (
+  cause: IndexError['cause'],
+  message: string,
+): IndexError => ({
+  _tag: 'IndexError',
+  cause,
+  message,
+})

package/src/embeddings/index.ts

@@ -0,0 +1,8 @@
+/**
+ * Embeddings module exports
+ */
+
+export * from './openai-provider.js'
+export * from './semantic-search.js'
+export * from './types.js'
+export * from './vector-store.js'