mdcontext 0.0.1 → 0.2.0

package/src/cli/commands/stats.ts

@@ -0,0 +1,149 @@
+/**
+ * STATS Command
+ *
+ * Show index statistics.
+ */
+
+import * as path from 'node:path'
+import { Args, Command } from '@effect/cli'
+import { Console, Effect } from 'effect'
+import { getEmbeddingStats } from '../../embeddings/semantic-search.js'
+import {
+  createStorage,
+  loadDocumentIndex,
+  loadSectionIndex,
+} from '../../index/storage.js'
+import { jsonOption, prettyOption } from '../options.js'
+import { formatJson } from '../utils.js'
+
+interface IndexStats {
+  documentCount: number
+  totalTokens: number
+  avgTokensPerDoc: number
+  totalSections: number
+  sectionsByLevel: Record<number, number>
+  tokenDistribution: {
+    min: number
+    max: number
+    median: number
+  }
+}
+
+export const statsCommand = Command.make(
+  'stats',
+  {
+    path: Args.directory({ name: 'path' }).pipe(
+      Args.withDescription('Directory to show stats for'),
+      Args.withDefault('.'),
+    ),
+    json: jsonOption,
+    pretty: prettyOption,
+  },
+  ({ path: dirPath, json, pretty }) =>
+    Effect.gen(function* () {
+      const resolvedRoot = path.resolve(dirPath)
+      const storage = createStorage(resolvedRoot)
+
+      // Load document and section indexes
+      const docIndex = yield* loadDocumentIndex(storage)
+      const sectionIndex = yield* loadSectionIndex(storage)
+
+      // Handle case where index doesn't exist
+      if (!docIndex || !sectionIndex) {
+        if (json) {
+          yield* Console.log(formatJson({ error: 'No index found' }, pretty))
+        } else {
+          yield* Console.log('No index found.')
+          yield* Console.log("Run 'mdcontext index <path>' to create an index.")
+        }
+        return
+      }
+
+      // Calculate index stats
+      const docs = Object.values(docIndex.documents)
+      const sections = Object.values(sectionIndex.sections)
+
+      const tokenCounts = docs.map((d) => d.tokenCount).sort((a, b) => a - b)
+      const totalTokens = tokenCounts.reduce((sum, t) => sum + t, 0)
+
+      // Count sections by level
+      const sectionsByLevel: Record<number, number> = {}
+      for (const section of sections) {
+        sectionsByLevel[section.level] =
+          (sectionsByLevel[section.level] || 0) + 1
+      }
+
+      const indexStats: IndexStats = {
+        documentCount: docs.length,
+        totalTokens,
+        avgTokensPerDoc:
+          docs.length > 0 ? Math.round(totalTokens / docs.length) : 0,
+        totalSections: sections.length,
+        sectionsByLevel,
+        tokenDistribution: {
+          min: tokenCounts[0] || 0,
+          max: tokenCounts[tokenCounts.length - 1] || 0,
+          median: tokenCounts[Math.floor(tokenCounts.length / 2)] || 0,
+        },
+      }
+
+      // Get embedding stats
+      const embeddingStats = yield* getEmbeddingStats(resolvedRoot)
+
+      if (json) {
+        yield* Console.log(
+          formatJson({ ...indexStats, embeddings: embeddingStats }, pretty),
+        )
+      } else {
+        yield* Console.log('Index statistics:')
+        yield* Console.log('')
+        yield* Console.log('  Documents')
+        yield* Console.log(`    Count:       ${indexStats.documentCount}`)
+        yield* Console.log(
+          `    Tokens:      ${indexStats.totalTokens.toLocaleString()}`,
+        )
+        yield* Console.log(`    Avg/doc:     ${indexStats.avgTokensPerDoc}`)
+        yield* Console.log('')
+        yield* Console.log('  Token distribution')
+        yield* Console.log(
+          `    Min:         ${indexStats.tokenDistribution.min}`,
+        )
+        yield* Console.log(
+          `    Median:      ${indexStats.tokenDistribution.median}`,
+        )
+        yield* Console.log(
+          `    Max:         ${indexStats.tokenDistribution.max}`,
+        )
+        yield* Console.log('')
+        yield* Console.log('  Sections')
+        yield* Console.log(`    Total:       ${indexStats.totalSections}`)
+        // Show section depth breakdown
+        const levels = Object.keys(sectionsByLevel)
+          .map(Number)
+          .sort((a, b) => a - b)
+        for (const level of levels) {
+          yield* Console.log(
+            `    h${level}:          ${sectionsByLevel[level]}`,
+          )
+        }
+        yield* Console.log('')
+        yield* Console.log('  Embeddings')
+        if (embeddingStats.hasEmbeddings) {
+          yield* Console.log(`    Vectors:     ${embeddingStats.count}`)
+          yield* Console.log(`    Provider:    ${embeddingStats.provider}`)
+          if (embeddingStats.model) {
+            yield* Console.log(`    Model:       ${embeddingStats.model}`)
+          }
+          yield* Console.log(`    Dimensions:  ${embeddingStats.dimensions}`)
+          yield* Console.log(
+            `    Cost:        $${embeddingStats.totalCost.toFixed(6)}`,
+          )
+        } else {
+          yield* Console.log('    Not enabled')
+          yield* Console.log(
+            "    Run 'mdcontext index --embed' to build embeddings.",
+          )
+        }
+      }
+    }),
+).pipe(Command.withDescription('Index statistics'))

package/src/cli/commands/tree.ts

@@ -0,0 +1,128 @@
+/**
+ * TREE Command
+ *
+ * Show file tree or document outline.
+ */
+
+import * as fs from 'node:fs'
+import * as path from 'node:path'
+import { Args, Command } from '@effect/cli'
+import { Console, Effect } from 'effect'
+import type { MdSection } from '../../core/types.js'
+import { FileReadError, ParseError } from '../../errors/index.js'
+import { parseFile } from '../../parser/parser.js'
+import { jsonOption, prettyOption } from '../options.js'
+import { formatJson, walkDirEffect } from '../utils.js'
+
+export const treeCommand = Command.make(
+  'tree',
+  {
+    pathArg: Args.text({ name: 'path' }).pipe(
+      Args.withDescription('Directory (shows files) or file (shows outline)'),
+      Args.withDefault('.'),
+    ),
+    json: jsonOption,
+    pretty: prettyOption,
+  },
+  ({ pathArg, json, pretty }) =>
+    Effect.gen(function* () {
+      const resolvedPath = path.resolve(pathArg)
+
+      // Auto-detect: file or directory
+      const stat = yield* Effect.try({
+        try: () => fs.statSync(resolvedPath),
+        catch: (e) =>
+          new FileReadError({
+            path: resolvedPath,
+            message: `Cannot access path: ${e instanceof Error ? e.message : String(e)}`,
+            cause: e,
+          }),
+      })
+
+      if (stat.isFile()) {
+        // Show document outline
+        const result = yield* parseFile(resolvedPath).pipe(
+          Effect.mapError((e) =>
+            e._tag === 'ParseError'
+              ? new ParseError({
+                  message: e.message,
+                  path: resolvedPath,
+                  ...(e.line !== undefined && { line: e.line }),
+                  ...(e.column !== undefined && { column: e.column }),
+                })
+              : new FileReadError({
+                  path: e.path,
+                  message: e.message,
+                }),
+          ),
+        )
+
+        const extractStructure = (
+          section: MdSection,
+        ): {
+          heading: string
+          level: number
+          tokens: number
+          children: unknown[]
+        } => ({
+          heading: section.heading,
+          level: section.level,
+          tokens: section.metadata.tokenCount,
+          children: section.children.map(extractStructure),
+        })
+
+        if (json) {
+          const structure = {
+            title: result.title,
+            path: result.path,
+            totalTokens: result.metadata.tokenCount,
+            sections: result.sections.map(extractStructure),
+          }
+          yield* Console.log(formatJson(structure, pretty))
+        } else {
+          yield* Console.log(`# ${result.title}`)
+          yield* Console.log(`Total tokens: ${result.metadata.tokenCount}`)
+          yield* Console.log('')
+
+          const printOutline = (
+            section: MdSection,
+            depth: number = 0,
+          ): Effect.Effect<void> =>
+            Effect.gen(function* () {
+              const indent = '  '.repeat(depth)
+              const marker = '#'.repeat(section.level)
+              yield* Console.log(
+                `${indent}${marker} ${section.heading} [${section.metadata.tokenCount} tokens]`,
+              )
+              for (const child of section.children) {
+                yield* printOutline(child, depth + 1)
+              }
+            })
+
+          for (const section of result.sections) {
+            yield* printOutline(section)
+          }
+        }
+      } else {
+        // Show file list
+        const files = yield* walkDirEffect(resolvedPath)
+
+        const tree = [...files].sort().map((f) => ({
+          path: f,
+          relativePath: path.relative(resolvedPath, f),
+        }))
+
+        if (json) {
+          yield* Console.log(formatJson(tree, pretty))
+        } else {
+          yield* Console.log(`Markdown files in ${resolvedPath}:`)
+          yield* Console.log('')
+          for (const file of tree) {
+            yield* Console.log(`  ${file.relativePath}`)
+          }
+          yield* Console.log('')
+          yield* Console.log(`Total: ${tree.length} files`)
+        }
+      }
+    }),
+).pipe(Command.withDescription('Show files or document outline'))

package/src/cli/config-layer.ts

@@ -0,0 +1,176 @@
+/**
+ * CLI Configuration Layer
+ *
+ * Creates a configuration layer for use in CLI commands.
+ * Loads config with precedence: CLI flags > Environment > Config file > Defaults
+ */
+
+import { Effect, Layer } from 'effect'
+import {
+  type ConfigProviderOptions,
+  ConfigService,
+  ConfigServiceDefault,
+  createConfigProvider,
+  type MdContextConfig,
+} from '../config/index.js'
+
+/**
+ * Create a ConfigService layer from options.
+ *
+ * This loads configuration with the standard precedence chain:
+ * 1. CLI flags (highest priority)
+ * 2. Environment variables (MDCONTEXT_*)
+ * 3. Config file (if found)
+ * 4. Defaults
+ *
+ * @param options - Configuration provider options
+ * @returns A Layer that provides ConfigService
+ */
+export const makeCliConfigLayer = (
+  options: ConfigProviderOptions = {},
+): Effect.Effect<Layer.Layer<ConfigService, never, never>, never, never> =>
+  Effect.gen(function* () {
+    // Create the config provider with precedence chain
+    const providerResult = yield* createConfigProvider(options).pipe(
+      Effect.catchAll(() =>
+        // If config loading fails, use empty provider (defaults will apply)
+        Effect.succeed(null),
+      ),
+    )
+
+    if (!providerResult) {
+      // Fall back to default config if provider creation failed
+      return ConfigServiceDefault
+    }
+
+    // Load the config using the provider
+    const configResult = yield* Effect.gen(function* () {
+      // Import the schema to load config
+      const { MdContextConfig: MdContextConfigSchema } = yield* Effect.promise(
+        async () => import('../config/schema.js'),
+      )
+      return yield* MdContextConfigSchema
+    }).pipe(
+      Effect.withConfigProvider(providerResult),
+      Effect.catchAll(() => Effect.succeed(null)),
+    )
+
+    if (!configResult) {
+      // Fall back to default config if loading failed
+      return ConfigServiceDefault
+    }
+
+    // Create a layer with the loaded config
+    return Layer.succeed(ConfigService, configResult)
+  })
+
+/**
+ * Create the default CLI configuration layer.
+ *
+ * This loads configuration from:
+ * - Environment variables (MDCONTEXT_*)
+ * - Config file (mdcontext.config.ts/json)
+ * - Built-in defaults
+ *
+ * No CLI flags are applied at this level - commands handle their own flag overrides.
+ */
+export const defaultCliConfigLayer: Effect.Effect<
+  Layer.Layer<ConfigService, never, never>,
+  never,
+  never
+> = makeCliConfigLayer({
+  workingDir: process.cwd(),
+})
+
+/**
+ * Synchronously create a default config layer.
+ *
+ * For use in cases where async config loading isn't possible.
+ * Uses only environment variables and defaults.
+ */
+export const defaultCliConfigLayerSync: Layer.Layer<
+  ConfigService,
+  never,
+  never
+> = ConfigServiceDefault
+
+/**
+ * Get config value with CLI flag override.
+ *
+ * Helper function for commands to get a config value, preferring
+ * an explicit CLI flag value if provided.
+ *
+ * @param cliValue - Value from CLI flag (may be undefined)
+ * @param configValue - Value from config
+ * @returns The CLI value if provided, otherwise the config value
+ */
+export const withCliOverride = <T>(
+  cliValue: T | undefined,
+  configValue: T,
+): T => {
+  return cliValue !== undefined ? cliValue : configValue
+}
+
+/**
+ * Extract relevant config sections for a command.
+ */
+export type SearchConfigValues = {
+  defaultLimit: number
+  maxLimit: number
+  minSimilarity: number
+  includeSnippets: boolean
+  snippetLength: number
+}
+
+export type IndexConfigValues = {
+  maxDepth: number
+  excludePatterns: readonly string[]
+  fileExtensions: readonly string[]
+  followSymlinks: boolean
+  indexDir: string
+}
+
+export type OutputConfigValues = {
+  format: 'text' | 'json'
+  color: boolean
+  prettyJson: boolean
+  verbose: boolean
+  debug: boolean
+}
+
+/**
+ * Extract search config from full config.
+ */
+export const getSearchConfig = (
+  config: MdContextConfig,
+): SearchConfigValues => ({
+  defaultLimit: config.search.defaultLimit,
+  maxLimit: config.search.maxLimit,
+  minSimilarity: config.search.minSimilarity,
+  includeSnippets: config.search.includeSnippets,
+  snippetLength: config.search.snippetLength,
+})
+
+/**
+ * Extract index config from full config.
+ */
+export const getIndexConfig = (config: MdContextConfig): IndexConfigValues => ({
+  maxDepth: config.index.maxDepth,
+  excludePatterns: config.index.excludePatterns,
+  fileExtensions: config.index.fileExtensions,
+  followSymlinks: config.index.followSymlinks,
+  indexDir: config.index.indexDir,
+})
+
+/**
+ * Extract output config from full config.
+ */
+export const getOutputConfig = (
+  config: MdContextConfig,
+): OutputConfigValues => ({
+  format: config.output.format,
+  color: config.output.color,
+  prettyJson: config.output.prettyJson,
+  verbose: config.output.verbose,
+  debug: config.output.debug,
+})

package/src/cli/error-handler.test.ts

@@ -0,0 +1,235 @@
+/**
+ * Unit tests for CLI error handler
+ */
+
+import { afterEach, beforeEach, describe, expect, it } from 'vitest'
+import { formatEffectCliError } from './error-handler.js'
+
+describe('formatEffectCliError', () => {
+  let originalArgv: string[]
+
+  beforeEach(() => {
+    originalArgv = process.argv
+  })
+
+  afterEach(() => {
+    process.argv = originalArgv
+  })
+
+  describe('mode validation errors with suggestions', () => {
+    it('suggests "semantic" for "semantics"', () => {
+      process.argv = ['node', 'cli', 'search', '--mode', 'semantics', 'test']
+
+      const error = {
+        _tag: 'ValidationError',
+        error: {
+          _tag: 'Paragraph',
+          value: {
+            _tag: 'Text',
+            value:
+              'Expected one of the following cases: hybrid, semantic, keyword',
+          },
+        },
+      }
+
+      const result = formatEffectCliError(error)
+      expect(result).toContain("Did you mean '--mode semantic'?")
+    })
+
+    it('suggests "keyword" for "keywords"', () => {
+      process.argv = ['node', 'cli', 'search', '--mode', 'keywords', 'test']
+
+      const error = {
+        _tag: 'ValidationError',
+        error: {
+          _tag: 'Paragraph',
+          value: {
+            _tag: 'Text',
+            value:
+              'Expected one of the following cases: hybrid, semantic, keyword',
+          },
+        },
+      }
+
+      const result = formatEffectCliError(error)
+      expect(result).toContain("Did you mean '--mode keyword'?")
+    })
+
+    it('suggests "hybrid" for "hybrit"', () => {
+      process.argv = ['node', 'cli', 'search', '--mode', 'hybrit', 'test']
+
+      const error = {
+        _tag: 'ValidationError',
+        error: {
+          _tag: 'Paragraph',
+          value: {
+            _tag: 'Text',
+            value:
+              'Expected one of the following cases: hybrid, semantic, keyword',
+          },
+        },
+      }
+
+      const result = formatEffectCliError(error)
+      expect(result).toContain("Did you mean '--mode hybrid'?")
+    })
+
+    it('suggests "semantic" for "semant"', () => {
+      process.argv = ['node', 'cli', 'search', '--mode', 'semant', 'test']
+
+      const error = {
+        _tag: 'ValidationError',
+        error: {
+          _tag: 'Paragraph',
+          value: {
+            _tag: 'Text',
+            value:
+              'Expected one of the following cases: hybrid, semantic, keyword',
+          },
+        },
+      }
+
+      const result = formatEffectCliError(error)
+      expect(result).toContain("Did you mean '--mode semantic'?")
+    })
+
+    it('suggests "keyword" for "keywordd"', () => {
+      process.argv = ['node', 'cli', 'search', '--mode', 'keywordd', 'test']
+
+      const error = {
+        _tag: 'ValidationError',
+        error: {
+          _tag: 'Paragraph',
+          value: {
+            _tag: 'Text',
+            value:
+              'Expected one of the following cases: hybrid, semantic, keyword',
+          },
+        },
+      }
+
+      const result = formatEffectCliError(error)
+      expect(result).toContain("Did you mean '--mode keyword'?")
+    })
+
+    it('does not suggest for typos too far off', () => {
+      process.argv = ['node', 'cli', 'search', '--mode', 'xyz', 'test']
+
+      const error = {
+        _tag: 'ValidationError',
+        error: {
+          _tag: 'Paragraph',
+          value: {
+            _tag: 'Text',
+            value:
+              'Expected one of the following cases: hybrid, semantic, keyword',
+          },
+        },
+      }
+
+      const result = formatEffectCliError(error)
+      expect(result).not.toContain('Did you mean')
+    })
+
+    it('handles --mode=value syntax', () => {
+      process.argv = ['node', 'cli', 'search', '--mode=semantics', 'test']
+
+      const error = {
+        _tag: 'ValidationError',
+        error: {
+          _tag: 'Paragraph',
+          value: {
+            _tag: 'Text',
+            value:
+              'Expected one of the following cases: hybrid, semantic, keyword',
+          },
+        },
+      }
+
+      const result = formatEffectCliError(error)
+      expect(result).toContain("Did you mean '--mode semantic'?")
+    })
+
+    it('handles -m short flag', () => {
+      process.argv = ['node', 'cli', 'search', '-m', 'semantics', 'test']
+
+      const error = {
+        _tag: 'ValidationError',
+        error: {
+          _tag: 'Paragraph',
+          value: {
+            _tag: 'Text',
+            value:
+              'Expected one of the following cases: hybrid, semantic, keyword',
+          },
+        },
+      }
+
+      const result = formatEffectCliError(error)
+      expect(result).toContain("Did you mean '--mode semantic'?")
+    })
+
+    it('does not suggest for non-mode validation errors', () => {
+      process.argv = ['node', 'cli', 'search', 'test']
+
+      const error = {
+        _tag: 'ValidationError',
+        error: {
+          _tag: 'Paragraph',
+          value: {
+            _tag: 'Text',
+            value: 'Some other validation error',
+          },
+        },
+      }
+
+      const result = formatEffectCliError(error)
+      expect(result).toBe('Some other validation error')
+      expect(result).not.toContain('Did you mean')
+    })
+  })
+
+  describe('other error types', () => {
+    it('handles MissingValue errors', () => {
+      process.argv = ['node', 'cli', 'search']
+
+      const error = {
+        _tag: 'MissingValue',
+        error: {
+          _tag: 'Paragraph',
+          value: {
+            _tag: 'Text',
+            value: 'Missing required argument',
+          },
+        },
+      }
+
+      const result = formatEffectCliError(error)
+      expect(result).toBe('Missing required argument')
+    })
+
+    it('handles InvalidValue errors', () => {
+      process.argv = ['node', 'cli', 'search']
+
+      const error = {
+        _tag: 'InvalidValue',
+        error: {
+          _tag: 'Paragraph',
+          value: {
+            _tag: 'Text',
+            value: 'Invalid value provided',
+          },
+        },
+      }
+
+      const result = formatEffectCliError(error)
+      expect(result).toBe('Invalid value provided')
+    })
+
+    it('handles unknown error types', () => {
+      const error = { message: 'Unknown error' }
+      const result = formatEffectCliError(error)
+      expect(result).toBe('[object Object]')
+    })
+  })
+})