mdcontext 0.1.0 → 0.2.0

package/src/cli/main.ts

@@ -17,20 +17,42 @@
  *   mdcontext stats [path]           Index statistics
  */
 
+import * as fs from 'node:fs'
+import { createRequire } from 'node:module'
+import * as path from 'node:path'
+import * as util from 'node:util'
+
+// Read version from package.json using createRequire for ESM compatibility
+const require = createRequire(import.meta.url)
+const packageJson = require('../../package.json') as { version: string }
+const CLI_VERSION: string = packageJson.version
+
 import { CliConfig, Command } from '@effect/cli'
 import { NodeContext, NodeRuntime } from '@effect/platform-node'
 import { Effect, Layer } from 'effect'
+import { ConfigService, createConfigProviderSync } from '../config/index.js'
+import { MdContextConfig } from '../config/schema.js'
+import type { PartialMdContextConfig } from '../config/service.js'
 import { preprocessArgv } from './argv-preprocessor.js'
 import {
   backlinksCommand,
+  configCommand,
   contextCommand,
+  duplicatesCommand,
+  embeddingsCommand,
   indexCommand,
   linksCommand,
   searchCommand,
   statsCommand,
   treeCommand,
 } from './commands/index.js'
+import { defaultCliConfigLayerSync } from './config-layer.js'
 import {
+  formatEffectCliError,
+  isEffectCliValidationError,
+} from './error-handler.js'
+import {
+  checkBareSubcommandHelp,
   checkSubcommandHelp,
   shouldShowMainHelp,
   showMainHelp,
@@ -49,13 +71,16 @@ const mainCommand = Command.make('mdcontext').pipe(
     treeCommand,
     linksCommand,
     backlinksCommand,
+    duplicatesCommand,
     statsCommand,
+    configCommand,
+    embeddingsCommand,
   ]),
 )
 
 const cli = Command.run(mainCommand, {
   name: 'mdcontext',
-  version: '0.1.0',
+  version: CLI_VERSION,
 })
 
 // Clean CLI config: hide built-in options from help
@@ -67,47 +92,7 @@ const cliConfigLayer = CliConfig.layer({
 // Error Handling
 // ============================================================================
 
-// Custom error formatter
-const formatCliError = (error: unknown): string => {
-  if (error && typeof error === 'object') {
-    // Handle Effect CLI validation errors
-    const err = error as Record<string, unknown>
-    if (err._tag === 'ValidationError' && err.error) {
-      const validationError = err.error as Record<string, unknown>
-      // Extract the actual error message
-      if (validationError._tag === 'Paragraph' && validationError.value) {
-        const paragraph = validationError.value as Record<string, unknown>
-        if (paragraph._tag === 'Text' && typeof paragraph.value === 'string') {
-          return paragraph.value
-        }
-      }
-    }
-    // Handle MissingValue errors
-    if (err._tag === 'MissingValue' && err.error) {
-      const missingError = err.error as Record<string, unknown>
-      if (missingError._tag === 'Paragraph' && missingError.value) {
-        const paragraph = missingError.value as Record<string, unknown>
-        if (paragraph._tag === 'Text' && typeof paragraph.value === 'string') {
-          return paragraph.value
-        }
-      }
-    }
-  }
-  return String(error)
-}
-
-// Check if error is a CLI validation error (should show friendly message)
-const isValidationError = (error: unknown): boolean => {
-  if (error && typeof error === 'object') {
-    const err = error as Record<string, unknown>
-    return (
-      err._tag === 'ValidationError' ||
-      err._tag === 'MissingValue' ||
-      err._tag === 'InvalidValue'
-    )
-  }
-  return false
-}
+// Note: Error formatting and validation checking moved to error-handler.ts
 
 // ============================================================================
 // Custom Help Handling
@@ -116,6 +101,9 @@ const isValidationError = (error: unknown): boolean => {
 // Check for subcommand help before anything else
 checkSubcommandHelp()
 
+// Check for bare subcommand that has nested subcommands (e.g., "config")
+checkBareSubcommandHelp()
+
 // Check if we should show main help
 if (shouldShowMainHelp()) {
   showMainHelp()
@@ -125,21 +113,323 @@ if (shouldShowMainHelp()) {
 // Preprocess argv to allow flexible flag positioning
 const processedArgv = preprocessArgv(process.argv)
 
-// Run with clean config and friendly errors
-Effect.suspend(() => cli(processedArgv)).pipe(
-  Effect.provide(Layer.merge(NodeContext.layer, cliConfigLayer)),
-  Effect.catchAll((error) =>
-    Effect.sync(() => {
-      // Only show friendly error for validation errors
-      if (isValidationError(error)) {
-        const message = formatCliError(error)
-        console.error(`\nError: ${message}`)
-        console.error('\nRun "mdcontext --help" for usage information.')
+// ============================================================================
+// Global --config Flag Handling
+// ============================================================================
+
+/**
+ * Extract --config or -c flag from argv before CLI parsing.
+ * This allows loading custom config files before the CLI runs.
+ */
+const extractConfigPath = (
+  argv: string[],
+): { configPath: string | undefined; filteredArgv: string[] } => {
+  const filteredArgv: string[] = []
+  let configPath: string | undefined
+
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i]
+    if (arg === undefined) continue
+
+    // --config=path or -c=path
+    if (arg.startsWith('--config=')) {
+      const value = arg.slice('--config='.length)
+      if (value.length === 0) {
+        console.error('\nError: --config requires a path')
+        console.error('  Usage: --config=path/to/config.js')
         process.exit(1)
       }
-      // Re-throw other errors to be handled normally
-      throw error
-    }),
-  ),
-  NodeRuntime.runMain,
-)
+      configPath = value
+      continue
+    }
+    if (arg.startsWith('-c=')) {
+      const value = arg.slice('-c='.length)
+      if (value.length === 0) {
+        console.error('\nError: -c requires a path')
+        console.error('  Usage: -c=path/to/config.js')
+        process.exit(1)
+      }
+      configPath = value
+      continue
+    }
+
+    // --config path or -c path
+    if (arg === '--config' || arg === '-c') {
+      const nextArg = argv[i + 1]
+      if (!nextArg || nextArg.startsWith('-')) {
+        console.error('\nError: --config requires a path')
+        console.error('  Usage: --config path/to/config.js')
+        process.exit(1)
+      }
+      if (nextArg.length === 0) {
+        console.error('\nError: --config path cannot be empty')
+        process.exit(1)
+      }
+      configPath = nextArg
+      i++ // Skip the path argument
+      continue
+    }
+
+    filteredArgv.push(arg)
+  }
+
+  return { configPath, filteredArgv }
+}
+
+// Extract config path from processed argv
+const { configPath: customConfigPath, filteredArgv } =
+  extractConfigPath(processedArgv)
+
+// ============================================================================
+// Config Loading Utilities (shared between async and sync paths)
+// ============================================================================
+
+/**
+ * Validate config file exists and exit with error if not found.
+ */
+function validateConfigFileExists(resolvedPath: string): void {
+  if (!fs.existsSync(resolvedPath)) {
+    console.error(`\nError: Config file not found: ${resolvedPath}`)
+    process.exit(1)
+  }
+}
+
+/**
+ * Handle config loading error with consistent formatting.
+ */
+const handleConfigLoadError = (error: unknown, resolvedPath: string): never => {
+  console.error(`\nError: Failed to load config file: ${resolvedPath}`)
+  if (error instanceof Error) {
+    console.error(`  ${error.message}`)
+  }
+  process.exit(1)
+}
+
+/**
+ * Valid top-level config keys that the config system recognizes.
+ */
+const VALID_CONFIG_KEYS = [
+  'index',
+  'search',
+  'embeddings',
+  'summarization',
+  'output',
+  'paths',
+] as const
+
+/**
+ * Validate that a loaded config is a valid object (not null, not array).
+ * Also validates that if it has keys, at least one is a recognized config key.
+ * Uses assertion function to narrow type for TypeScript.
+ */
+function validateConfigObject(
+  config: unknown,
+  resolvedPath: string,
+): asserts config is PartialMdContextConfig {
+  // Check it's a non-null, non-array object
+  if (!config || typeof config !== 'object' || Array.isArray(config)) {
+    console.error(
+      `\nError: Config file must export a default object or named "config" export`,
+    )
+    console.error(`  File: ${resolvedPath}`)
+    process.exit(1)
+  }
+
+  // Validate structure - if there are keys, at least one should be recognized
+  const configKeys = Object.keys(config)
+  const hasValidKey = configKeys.some((key) =>
+    VALID_CONFIG_KEYS.includes(key as (typeof VALID_CONFIG_KEYS)[number]),
+  )
+
+  if (configKeys.length > 0 && !hasValidKey) {
+    console.error(`\nError: Config file has no recognized configuration keys`)
+    console.error(`  File: ${resolvedPath}`)
+    console.error(`  Found keys: ${configKeys.join(', ')}`)
+    console.error(`  Expected at least one of: ${VALID_CONFIG_KEYS.join(', ')}`)
+    process.exit(1)
+  }
+}
+
+/**
+ * Create a ConfigService Layer from a validated config object.
+ */
+const createConfigLayerFromConfig = (
+  fileConfig: PartialMdContextConfig,
+): Layer.Layer<ConfigService, never, never> => {
+  const provider = createConfigProviderSync({
+    fileConfig,
+    skipEnv: false,
+  })
+  const configResult = Effect.runSync(
+    MdContextConfig.pipe(Effect.withConfigProvider(provider)),
+  )
+  return Layer.succeed(ConfigService, configResult)
+}
+
+/**
+ * Load a TS/JS/MJS config file asynchronously using dynamic import.
+ * Returns a promise that resolves to a ConfigService Layer.
+ */
+async function loadConfigAsync(
+  configPath: string,
+): Promise<Layer.Layer<ConfigService, never, never>> {
+  const resolvedPath = path.resolve(configPath)
+  validateConfigFileExists(resolvedPath)
+
+  try {
+    // Use dynamic import to load TS/JS/MJS files
+    const fileUrl = `file://${resolvedPath}`
+    const module = (await import(fileUrl)) as {
+      default?: PartialMdContextConfig
+      config?: PartialMdContextConfig
+    }
+    const fileConfig = module.default ?? module.config
+
+    validateConfigObject(fileConfig, resolvedPath)
+    return createConfigLayerFromConfig(fileConfig)
+  } catch (error) {
+    // handleConfigLoadError calls process.exit(1) and never returns
+    // TypeScript needs explicit return for type checking - this is unreachable
+    return handleConfigLoadError(error, resolvedPath)
+  }
+}
+
+/**
+ * Determine if we need async loading (for TS/JS config files).
+ * All non-JSON config files need async loading via dynamic import.
+ */
+const needsAsyncLoading = (configPath: string | undefined): boolean => {
+  if (!configPath) return false
+  const ext = path.extname(configPath).toLowerCase()
+  // Async load for all JS/TS variants, sync for JSON only
+  return ext !== '.json'
+}
+
+/**
+ * Create config layer synchronously (for JSON or no custom config).
+ */
+function createConfigLayerSync(): Layer.Layer<ConfigService, never, never> {
+  if (!customConfigPath) {
+    return defaultCliConfigLayerSync
+  }
+
+  const resolvedPath = path.resolve(customConfigPath)
+  validateConfigFileExists(resolvedPath)
+
+  try {
+    const content = fs.readFileSync(resolvedPath, 'utf-8')
+
+    // Parse JSON with proper validation
+    let parsed: unknown
+    try {
+      parsed = JSON.parse(content)
+    } catch (parseError) {
+      console.error(`\nError: Invalid JSON in config file: ${resolvedPath}`)
+      console.error(
+        `  ${parseError instanceof Error ? parseError.message : String(parseError)}`,
+      )
+      process.exit(1)
+    }
+
+    // Validate structure before using
+    validateConfigObject(parsed, resolvedPath)
+    return createConfigLayerFromConfig(parsed)
+  } catch (error) {
+    // handleConfigLoadError calls process.exit(1) and never returns
+    return handleConfigLoadError(error, resolvedPath)
+  }
+}
+
+/**
+ * Run the CLI with error handling.
+ */
+const runCli = (
+  configLayer: Layer.Layer<ConfigService, never, never>,
+): void => {
+  const appLayers = Layer.mergeAll(
+    NodeContext.layer,
+    cliConfigLayer,
+    configLayer,
+  )
+
+  Effect.suspend(() => cli(filteredArgv)).pipe(
+    Effect.provide(appLayers),
+    Effect.tap(() =>
+      Effect.sync(() => {
+        // Force exit after successful completion to prevent hanging
+        // This is necessary because some dependencies (like OpenAI SDK)
+        // may keep the event loop alive with HTTP keep-alive connections
+        setImmediate(() => process.exit(0))
+      }),
+    ),
+    Effect.catchAll((error) =>
+      Effect.sync(() => {
+        if (isEffectCliValidationError(error)) {
+          const message = formatEffectCliError(error)
+          console.error(`\nError: ${message}`)
+          console.error('\nRun "mdcontext --help" for usage information.')
+          process.exit(1)
+        }
+        // Handle all other unexpected errors instead of rethrowing
+        console.error('\nUnexpected error:')
+        if (error instanceof Error) {
+          console.error(`  ${error.message}`)
+          if (error.stack) {
+            console.error(`\nStack trace:`)
+            console.error(error.stack)
+          }
+        } else {
+          console.error(util.inspect(error, { depth: null }))
+        }
+        process.exit(2)
+      }),
+    ),
+    NodeRuntime.runMain,
+  )
+}
+
+// Handle async vs sync config loading based on file type
+if (needsAsyncLoading(customConfigPath)) {
+  // Async path for TS/JS/MJS config files using async/await with proper error handling
+  ;(async () => {
+    // Runtime check for config path - TypeScript can't verify needsAsyncLoading's guard
+    if (!customConfigPath) {
+      console.error('\nError: Config path is required for async loading')
+      process.exit(1)
+    }
+
+    try {
+      const configLayer = await loadConfigAsync(customConfigPath)
+      runCli(configLayer)
+    } catch (error) {
+      // This catches errors from runCli, not loadConfigAsync
+      // (loadConfigAsync has its own error handling that calls process.exit)
+      console.error(`\nError: Failed to initialize CLI`)
+      if (error instanceof Error) {
+        console.error(`  ${error.message}`)
+        if (error.stack) {
+          console.error(`\nStack trace:`)
+          console.error(error.stack)
+        }
+      }
+      process.exit(1)
+    }
+  })().catch((error) => {
+    // Catch any errors that escape the try-catch (e.g., errors before try block)
+    console.error('\nUnexpected error during initialization')
+    if (error instanceof Error) {
+      console.error(`  ${error.message}`)
+      if (error.stack) {
+        console.error(`\nStack trace:`)
+        console.error(error.stack)
+      }
+    } else {
+      console.error(util.inspect(error, { depth: null }))
+    }
+    process.exit(1)
+  })
+} else {
+  // Sync path for JSON configs or no custom config
+  const configLayer = createConfigLayerSync()
+  runCli(configLayer)
+}

package/src/cli/options.ts

@@ -6,6 +6,16 @@
 
 import { Options } from '@effect/cli'
 
+/**
+ * Global config file path override
+ * Allows specifying a custom config file instead of auto-detection.
+ */
+export const configOption = Options.file('config').pipe(
+  Options.withAlias('c'),
+  Options.withDescription('Path to config file'),
+  Options.optional,
+)
+
 /**
  * Output as JSON
  */

package/src/cli/shared-error-handling.ts

@@ -0,0 +1,199 @@
+/**
+ * Shared Error Handling Utilities
+ *
+ * This module provides reusable error handling patterns for CLI commands.
+ * It eliminates duplication in catchTags blocks across index-cmd.ts and search.ts.
+ *
+ * ## Design Principles
+ *
+ * 1. **Use proper Effect composition** - Never use Effect.runSync inside error handlers.
+ *    Instead, return Effect values that compose properly in the Effect pipeline.
+ *
+ * 2. **Graceful degradation** - For optional operations, return null on failure
+ *    and let the caller decide what to do next.
+ *
+ * 3. **Consistent logging** - Use Effect.logWarning for debugging info and
+ *    Console.error for user-facing errors. Support silent mode for JSON output.
+ *
+ * ## When to Catch vs Propagate
+ *
+ * **CATCH errors when:**
+ * - The operation is optional (e.g., cost estimate for user prompt)
+ * - Failure should fall back gracefully (e.g., auto-index attempt)
+ *
+ * **PROPAGATE errors when:**
+ * - The operation is required for the command to succeed
+ * - The centralized error handler (error-handler.ts) should format the message
+ *
+ * ## Usage
+ * ```typescript
+ * const result = yield* someOperation.pipe(
+ *   Effect.map((r): SomeType | null => r),
+ *   Effect.catchTags(createEmbeddingErrorHandler({ silent: json }))
+ * )
+ * ```
+ */
+
+import { Console, Effect } from 'effect'
+import type { BuildEmbeddingsResult } from '../embeddings/semantic-search.js'
+import type {
+  ApiKeyInvalidError,
+  ApiKeyMissingError,
+  EmbeddingError,
+  FileReadError,
+  IndexCorruptedError,
+  IndexNotFoundError,
+  VectorStoreError,
+} from '../errors/index.js'
+
+// ============================================================================
+// Types
+// ============================================================================
+
+/**
+ * Options for error handlers.
+ */
+export interface ErrorHandlerOptions {
+  /**
+   * When true, suppress error output (for JSON mode).
+   */
+  readonly silent?: boolean
+}
+
+/**
+ * A result type that can be null when operation fails gracefully.
+ */
+export type NullableResult<T> = T | null
+
+// ============================================================================
+// Logging Helpers
+// ============================================================================
+
+/**
+ * Log an error message if not in silent mode.
+ * Returns an Effect that can be composed properly.
+ */
+const logErrorUnlessSilent = (
+  message: string,
+  silent: boolean,
+): Effect.Effect<void, never, never> =>
+  silent ? Effect.void : Console.error(message)
+
+/**
+ * Log a warning message if not in silent mode.
+ * Returns an Effect that can be composed properly.
+ */
+const logWarningUnlessSilent = (
+  message: string,
+  silent: boolean,
+): Effect.Effect<void, never, never> =>
+  silent ? Effect.void : Effect.logWarning(message)
+
+// ============================================================================
+// Index/File Error Handlers
+// ============================================================================
+
+/**
+ * Create a handler for index-related errors that returns null on failure.
+ * Use this for operations where index errors should gracefully degrade.
+ */
+export const createIndexErrorHandler = <T>(options: ErrorHandlerOptions = {}) =>
+  ({
+    IndexNotFoundError: (_e: IndexNotFoundError) =>
+      Effect.succeed(null as NullableResult<T>),
+    FileReadError: (e: FileReadError) =>
+      logWarningUnlessSilent(
+        `Could not read index files: ${e.message}`,
+        options.silent ?? false,
+      ).pipe(Effect.map(() => null as NullableResult<T>)),
+    IndexCorruptedError: (e: IndexCorruptedError) =>
+      logWarningUnlessSilent(
+        `Index is corrupted: ${e.details ?? e.reason}`,
+        options.silent ?? false,
+      ).pipe(Effect.map(() => null as NullableResult<T>)),
+  }) as const
+
+// ============================================================================
+// Embedding Error Handlers
+// ============================================================================
+
+/**
+ * Create a comprehensive handler for embedding-related errors.
+ * Handles API key errors, index errors, embedding errors, and vector store errors.
+ * Returns null on failure for graceful degradation.
+ *
+ * Use this for:
+ * - Optional embedding operations (e.g., user prompt in index command)
+ * - Auto-index attempts in search command
+ * - Any embedding operation that should fall back gracefully
+ */
+export const createEmbeddingErrorHandler = (
+  options: ErrorHandlerOptions = {},
+) => {
+  const silent = options.silent ?? false
+
+  return {
+    // API key errors - user needs to set up API key
+    ApiKeyMissingError: (e: ApiKeyMissingError) =>
+      logErrorUnlessSilent(`\n${e.message}`, silent).pipe(
+        Effect.map(() => null as BuildEmbeddingsResult | null),
+      ),
+    ApiKeyInvalidError: (e: ApiKeyInvalidError) =>
+      logErrorUnlessSilent(`\n${e.message}`, silent).pipe(
+        Effect.map(() => null as BuildEmbeddingsResult | null),
+      ),
+    // Index not found - shouldn't happen after buildIndex but handle gracefully
+    IndexNotFoundError: (_e: IndexNotFoundError) =>
+      Effect.succeed(null as BuildEmbeddingsResult | null),
+    // File system errors
+    FileReadError: (e: FileReadError) =>
+      logErrorUnlessSilent(
+        `\nCannot read index files: ${e.message}`,
+        silent,
+      ).pipe(Effect.map(() => null as BuildEmbeddingsResult | null)),
+    IndexCorruptedError: (e: IndexCorruptedError) =>
+      logErrorUnlessSilent(
+        `\nIndex is corrupted: ${e.details ?? e.reason}`,
+        silent,
+      ).pipe(Effect.map(() => null as BuildEmbeddingsResult | null)),
+    // Embedding errors - network, rate limit, etc
+    EmbeddingError: (e: EmbeddingError) =>
+      logErrorUnlessSilent(`\nEmbedding failed: ${e.message}`, silent).pipe(
+        Effect.map(() => null as BuildEmbeddingsResult | null),
+      ),
+    // Vector store errors
+    VectorStoreError: (e: VectorStoreError) =>
+      logErrorUnlessSilent(`\nVector store error: ${e.message}`, silent).pipe(
+        Effect.map(() => null as BuildEmbeddingsResult | null),
+      ),
+  } as const
+}
+
+/**
+ * Create a handler for cost estimation errors.
+ * Returns null on failure for graceful degradation.
+ *
+ * NOTE: Use with Effect.catchTags after Effect.map to preserve type:
+ * ```typescript
+ * const result = yield* operation.pipe(
+ *   Effect.map((r): TargetType | null => r),
+ *   Effect.catchTags(createCostEstimateErrorHandler())
+ * )
+ * ```
+ */
+export const createCostEstimateErrorHandler = (
+  options: ErrorHandlerOptions = {},
+) =>
+  ({
+    IndexNotFoundError: (_e: IndexNotFoundError) => Effect.succeed(null),
+    FileReadError: (e: FileReadError) =>
+      logWarningUnlessSilent(
+        `Could not read index files: ${e.message}`,
+        options.silent ?? false,
+      ).pipe(Effect.map(() => null)),
+    IndexCorruptedError: (e: IndexCorruptedError) =>
+      logWarningUnlessSilent(
+        `Index is corrupted: ${e.details ?? e.reason}`,
+        options.silent ?? false,
+      ).pipe(Effect.map(() => null)),
+  }) as const