mdcontext 0.1.0 → 0.2.0

package/src/cli/commands/index-cmd.ts

@@ -7,15 +7,24 @@
 import * as path from 'node:path'
 import * as readline from 'node:readline'
 import { Args, Command, Options } from '@effect/cli'
-import { Console, Effect } from 'effect'
-import { handleApiKeyError } from '../../embeddings/openai-provider.js'
+import { Console, Effect, Option } from 'effect'
+import type {
+  BuildEmbeddingsResult,
+  EmbeddingEstimate,
+} from '../../embeddings/semantic-search.js'
 import {
   buildEmbeddings,
+  checkPricingFreshness,
   estimateEmbeddingCost,
+  getPricingDate,
 } from '../../embeddings/semantic-search.js'
 import { buildIndex } from '../../index/indexer.js'
 import { watchDirectory } from '../../index/watcher.js'
 import { forceOption, jsonOption, prettyOption } from '../options.js'
+import {
+  createCostEstimateErrorHandler,
+  createEmbeddingErrorHandler,
+} from '../shared-error-handling.js'
 import { formatJson, hasEmbeddings } from '../utils.js'
 
 const promptUser = (message: string): Promise<string> => {
@@ -50,7 +59,50 @@ export const indexCommand = Command.make(
     exclude: Options.text('exclude').pipe(
       Options.withAlias('x'),
       Options.withDescription(
-        'Exclude files matching patterns (comma-separated globs)',
+        'Additional patterns to exclude (comma-separated). Patterns from .gitignore and .mdcontextignore are honored automatically.',
+      ),
+      Options.optional,
+    ),
+    noGitignore: Options.boolean('no-gitignore').pipe(
+      Options.withDescription('Ignore .gitignore file'),
+      Options.withDefault(false),
+    ),
+    provider: Options.choice('provider', [
+      'openai',
+      'ollama',
+      'lm-studio',
+      'openrouter',
+      'voyage',
+    ]).pipe(
+      Options.withDescription(
+        'Embedding provider: openai, ollama, lm-studio, openrouter, or voyage',
+      ),
+      Options.optional,
+    ),
+    providerBaseUrl: Options.text('provider-base-url').pipe(
+      Options.withDescription('Custom provider API base URL'),
+      Options.optional,
+    ),
+    providerModel: Options.text('provider-model').pipe(
+      Options.withDescription('Embedding model to use'),
+      Options.optional,
+    ),
+    hnswM: Options.integer('hnsw-m').pipe(
+      Options.withDescription(
+        'HNSW M parameter: max connections per node. Higher = better recall, larger index. Recommended: 12 (speed), 16 (balanced, default), 24 (quality)',
+      ),
+      Options.optional,
+    ),
+    hnswEfConstruction: Options.integer('hnsw-ef-construction').pipe(
+      Options.withDescription(
+        'HNSW efConstruction: construction-time search width. Higher = better quality, slower builds. Recommended: 128 (speed), 200 (balanced, default), 256 (quality)',
+      ),
+      Options.optional,
+    ),
+    timeout: Options.integer('timeout').pipe(
+      Options.withAlias('t'),
+      Options.withDescription(
+        'Request timeout in milliseconds for embedding API calls (default: 30000)',
       ),
       Options.optional,
     ),
@@ -68,6 +120,13 @@ export const indexCommand = Command.make(
     embed,
     noEmbed,
     exclude,
+    noGitignore,
+    provider,
+    providerBaseUrl,
+    providerModel,
+    hnswM,
+    hnswEfConstruction,
+    timeout,
     watch: watchMode,
     force,
     json,
@@ -76,8 +135,9 @@ export const indexCommand = Command.make(
     Effect.gen(function* () {
       const resolvedDir = path.resolve(dirPath)
 
-      // Parse exclude patterns
-      const excludePatterns =
+      // Parse exclude patterns - CLI adds to ignore files
+      // Note: buildIndex now honors .gitignore and .mdcontextignore by default
+      const cliExcludePatterns =
         exclude._tag === 'Some'
           ? exclude.value.split(',').map((p) => p.trim())
           : undefined
@@ -89,6 +149,8 @@ export const indexCommand = Command.make(
 
         const watcher = yield* watchDirectory(resolvedDir, {
           force,
+          exclude: cliExcludePatterns,
+          honorGitignore: !noGitignore,
           onIndex: (result) => {
             if (json) {
               console.log(formatJson(result, pretty))
@@ -113,7 +175,22 @@ export const indexCommand = Command.make(
       } else {
         yield* Console.log(`Indexing ${resolvedDir}...`)
 
-        const result = yield* buildIndex(resolvedDir, { force })
+        const result = yield* buildIndex(resolvedDir, {
+          force,
+          exclude: cliExcludePatterns,
+          honorGitignore: !noGitignore,
+          onProgress: (progress) => {
+            if (!json) {
+              const progressMsg = `  [${progress.current}/${progress.total}] ${progress.filePath}`
+              process.stdout.write(`\x1b[2K\r${progressMsg}`)
+            }
+          },
+        })
+
+        // Clear the progress line after indexing completes
+        if (!json) {
+          process.stdout.write('\x1b[2K\r')
+        }
 
         if (!json) {
           yield* Console.log('')
@@ -129,6 +206,21 @@ export const indexCommand = Command.make(
           yield* Console.log(`  Links: ${result.totalLinks}`)
           yield* Console.log(`  Duration: ${result.duration}ms`)
 
+          // Show skip summary if any files were skipped
+          if (result.skipped.total > 0) {
+            const skipParts: string[] = []
+            if (result.skipped.unchanged > 0) {
+              skipParts.push(`${result.skipped.unchanged} unchanged`)
+            }
+            if (result.skipped.hidden > 0) {
+              skipParts.push(`${result.skipped.hidden} hidden`)
+            }
+            if (result.skipped.excluded > 0) {
+              skipParts.push(`${result.skipped.excluded} excluded`)
+            }
+            yield* Console.log(`  Skipped: ${skipParts.join(', ')}`)
+          }
+
           if (result.errors.length > 0) {
             yield* Console.log('')
             yield* Console.log(`Errors (${result.errors.length}):`)
@@ -147,10 +239,10 @@ export const indexCommand = Command.make(
         if (embed) {
           yield* Console.log('')
 
-          // Show cost estimate first
+          // Show cost estimate first - errors propagate to CLI boundary
           const estimate = yield* estimateEmbeddingCost(resolvedDir, {
-            excludePatterns,
-          }).pipe(handleApiKeyError)
+            excludePatterns: cliExcludePatterns,
+          })
 
           if (!json) {
             yield* Console.log(`Found ${estimate.totalFiles} files to embed:`)
@@ -165,8 +257,14 @@ export const indexCommand = Command.make(
             }
             yield* Console.log('')
             yield* Console.log(
-              `Total: ~${estimate.totalTokens.toLocaleString()} tokens, ~$${estimate.totalCost.toFixed(4)}, ~${estimate.estimatedTimeSeconds}s`,
+              `Total: ~${estimate.totalTokens.toLocaleString()} tokens, ~$${estimate.totalCost.toFixed(4)} (pricing as of ${getPricingDate()}), ~${estimate.estimatedTimeSeconds}s`,
             )
+
+            // Check for stale pricing data
+            const stalenessWarning = checkPricingFreshness()
+            if (stalenessWarning) {
+              yield* Console.log(`  Warning: ${stalenessWarning}`)
+            }
             yield* Console.log('')
           }
 
@@ -176,21 +274,56 @@ export const indexCommand = Command.make(
             yield* Console.log('Rebuilding embeddings (--force specified)...')
           }
 
+          // Build provider config from CLI flags if specified
+          const cliTimeout = Option.getOrUndefined(timeout)
+          const providerConfig = Option.isSome(provider)
+            ? {
+                provider: provider.value as
+                  | 'openai'
+                  | 'ollama'
+                  | 'lm-studio'
+                  | 'openrouter'
+                  | 'voyage',
+                baseURL: Option.getOrUndefined(providerBaseUrl),
+                model: Option.getOrUndefined(providerModel),
+                timeout: cliTimeout,
+              }
+            : cliTimeout !== undefined
+              ? { provider: 'openai' as const, timeout: cliTimeout }
+              : undefined
+
+          // Build HNSW options from CLI flags if specified
+          const hnswOptions =
+            Option.isSome(hnswM) || Option.isSome(hnswEfConstruction)
+              ? {
+                  m: Option.getOrUndefined(hnswM),
+                  efConstruction: Option.getOrUndefined(hnswEfConstruction),
+                }
+              : undefined
+
+          // Build embeddings - errors propagate to CLI boundary
           const embedResult = yield* buildEmbeddings(resolvedDir, {
             force,
-            excludePatterns,
+            excludePatterns: cliExcludePatterns,
+            providerConfig,
+            hnswOptions,
             onFileProgress: (progress) => {
               if (!json) {
-                process.stdout.write(
-                  `\r  [${progress.fileIndex}/${progress.totalFiles}] ${progress.filePath} (${progress.sectionCount} sections)...`,
-                )
+                const progressMsg = `  [${progress.fileIndex}/${progress.totalFiles}] ${progress.filePath} (${progress.sectionCount} sections)...`
+                process.stdout.write(`\x1b[2K\r${progressMsg}`)
               }
             },
-          }).pipe(handleApiKeyError)
+            onBatchProgress: (progress) => {
+              if (!json) {
+                const progressMsg = `  Embedding [${progress.processedSections}/${progress.totalSections}] sections (batch ${progress.batchIndex}/${progress.totalBatches})...`
+                process.stdout.write(`\x1b[2K\r${progressMsg}`)
+              }
+            },
+          })
 
           if (!json) {
-            // Clear the progress line
-            process.stdout.write(`\r${' '.repeat(80)}\r`)
+            // Clear the progress line completely
+            process.stdout.write(`\r${' '.repeat(120)}\r`)
             yield* Console.log('')
 
             if (embedResult.cacheHit) {
@@ -228,8 +361,11 @@ export const indexCommand = Command.make(
           yield* Console.log('')
 
           // Get cost estimate for the prompt
+          // Note: We gracefully handle errors here since this is optional information
+          // for the user prompt. IndexNotFoundError is expected if index doesn't exist.
           const estimate = yield* estimateEmbeddingCost(resolvedDir).pipe(
-            Effect.catchAll(() => Effect.succeed(null)),
+            Effect.map((r): EmbeddingEstimate | null => r),
+            Effect.catchTags(createCostEstimateErrorHandler()),
           )
 
           if (estimate) {
@@ -237,7 +373,21 @@ export const indexCommand = Command.make(
               `Cost: ~$${estimate.totalCost.toFixed(4)} for this corpus (~${estimate.estimatedTimeSeconds}s)`,
             )
           }
-          yield* Console.log('Requires: OPENAI_API_KEY environment variable')
+          yield* Console.log('Requires an embedding provider. Options:')
+          yield* Console.log(
+            '  - OpenAI (cloud): Set OPENAI_API_KEY environment variable',
+          )
+          yield* Console.log(
+            '  - Ollama (free, local): Run "ollama serve" - no API key needed',
+          )
+          yield* Console.log(
+            '  - LM Studio (free, local): Start the server - no API key needed',
+          )
+          yield* Console.log(
+            '  - OpenRouter (cloud): Set OPENROUTER_API_KEY environment variable',
+          )
+          yield* Console.log('')
+          yield* Console.log('See CONFIG.md for detailed setup instructions.')
           yield* Console.log('')
 
           const answer = yield* Effect.promise(() =>
@@ -245,36 +395,66 @@ export const indexCommand = Command.make(
           )
 
           if (answer === 'y' || answer === 'yes') {
-            // Check for API key
+            // Check for API key (only required for cloud providers)
+            // Note: When no provider is configured, we default to OpenAI which needs a key
+            // Local providers (Ollama, LM Studio) don't need API keys
             if (!process.env.OPENAI_API_KEY) {
               yield* Console.log('')
-              yield* Console.log('OPENAI_API_KEY not set.')
+              yield* Console.log('No embedding provider configured.')
+              yield* Console.log('')
+              yield* Console.log('Choose a provider:')
               yield* Console.log('')
+              yield* Console.log('  Cloud (requires API key):')
+              yield* Console.log('    export OPENAI_API_KEY=sk-...')
+              yield* Console.log('    export OPENROUTER_API_KEY=sk-...')
+              yield* Console.log('')
+              yield* Console.log('  Local (free, no API key needed):')
               yield* Console.log(
-                'To enable semantic search, set your OpenAI API key:',
+                '    Ollama: ollama serve && ollama pull nomic-embed-text',
               )
-              yield* Console.log('  export OPENAI_API_KEY=sk-...')
+              yield* Console.log('    LM Studio: Start the server GUI')
               yield* Console.log('')
-              yield* Console.log('Or add to .env file in project root.')
+              yield* Console.log(
+                'Then run: mdcontext index --embed [--provider <name>]',
+              )
+              yield* Console.log('See CONFIG.md for detailed setup.')
             } else {
               yield* Console.log('')
               yield* Console.log('Building embeddings...')
 
+              // Build HNSW options from CLI flags if specified
+              const hnswOptionsPrompt =
+                Option.isSome(hnswM) || Option.isSome(hnswEfConstruction)
+                  ? {
+                      m: Option.getOrUndefined(hnswM),
+                      efConstruction: Option.getOrUndefined(hnswEfConstruction),
+                    }
+                  : undefined
+
+              // Build provider config if timeout specified
+              const promptTimeout = Option.getOrUndefined(timeout)
+              const providerConfigPrompt =
+                promptTimeout !== undefined
+                  ? { provider: 'openai' as const, timeout: promptTimeout }
+                  : undefined
+
+              // Note: We gracefully handle errors here since embedding failure
+              // shouldn't block the main index operation. Errors are logged for debugging.
               const embedResult = yield* buildEmbeddings(resolvedDir, {
                 force: false,
+                hnswOptions: hnswOptionsPrompt,
+                providerConfig: providerConfigPrompt,
                 onFileProgress: (progress) => {
-                  process.stdout.write(
-                    `\r  [${progress.fileIndex}/${progress.totalFiles}] ${progress.filePath} (${progress.sectionCount} sections)...`,
+                  console.log(
+                    `  [${progress.fileIndex}/${progress.totalFiles}] ${progress.filePath}`,
                   )
                 },
               }).pipe(
-                handleApiKeyError,
-                Effect.catchAll(() => Effect.succeed(null)),
+                Effect.map((r): BuildEmbeddingsResult | null => r),
+                Effect.catchTags(createEmbeddingErrorHandler()),
               )
 
               if (embedResult) {
-                // Clear the progress line
-                process.stdout.write(`\r${' '.repeat(80)}\r`)
                 yield* Console.log('')
                 yield* Console.log(
                   `Completed in ${(embedResult.duration / 1000).toFixed(1)}s`,

package/src/cli/commands/index.ts

@@ -5,7 +5,10 @@
  */
 
 export { backlinksCommand } from './backlinks.js'
+export { configCommand } from './config-cmd.js'
 export { contextCommand } from './context.js'
+export { duplicatesCommand } from './duplicates.js'
+export { embeddingsCommand } from './embeddings.js'
 export { indexCommand } from './index-cmd.js'
 export { linksCommand } from './links.js'
 export { searchCommand } from './search.js'