mdcontext 0.1.0 → 0.2.0

package/src/embeddings/semantic-search.ts

@@ -5,20 +5,83 @@
 import * as fs from 'node:fs/promises'
 import * as path from 'node:path'
 import { Effect } from 'effect'
+import {
+  type ApiKeyInvalidError,
+  type ApiKeyMissingError,
+  DimensionMismatchError,
+  EmbeddingError,
+  EmbeddingsNotFoundError,
+  type FileReadError,
+  type IndexCorruptedError,
+  IndexNotFoundError,
+  type VectorStoreError,
+} from '../errors/index.js'
 import {
   createStorage,
   loadDocumentIndex,
   loadSectionIndex,
 } from '../index/storage.js'
 import type { SectionEntry } from '../index/types.js'
-import { createOpenAIProvider, InvalidApiKeyError } from './openai-provider.js'
-import type {
-  EmbeddingProvider,
-  SemanticSearchOptions,
-  SemanticSearchResult,
-  VectorEntry,
+import {
+  type ActiveProvider,
+  generateNamespace,
+  getActiveNamespace,
+  writeActiveProvider,
+} from './embedding-namespace.js'
+import { generateHypotheticalDocument, type HydeResult } from './hyde.js'
+import {
+  checkPricingFreshness,
+  getPricingDate,
+  PRICING_DATA,
+  wrapEmbedding,
+} from './openai-provider.js'
+import {
+  createEmbeddingProviderDirect,
+  type ProviderFactoryConfig,
+} from './provider-factory.js'
+import {
+  calculateFileImportanceBoost,
+  calculateHeadingBoost,
+  type EmbeddingProvider,
+  hasProviderMetadata,
+  preprocessQuery,
+  QUALITY_EF_SEARCH,
+  type SemanticSearchOptions,
+  type SemanticSearchResult,
+  type SemanticSearchResultWithStats,
+  type VectorEntry,
 } from './types.js'
-import { createVectorStore, type HnswVectorStore } from './vector-store.js'
+import {
+  createNamespacedVectorStore,
+  type HnswBuildOptions,
+  type HnswMismatchWarning,
+  type HnswVectorStore,
+  type VectorSearchResult,
+  type VectorStoreLoadResult,
+} from './vector-store.js'
+
+// ============================================================================
+// HNSW Parameter Warning
+// ============================================================================
+
+/**
+ * Check for HNSW parameter mismatch and log a warning if found.
+ * This helps users understand when their config doesn't match the stored index.
+ */
+const checkHnswMismatch = (
+  mismatch: HnswMismatchWarning | undefined,
+): Effect.Effect<void, never, never> => {
+  if (!mismatch) {
+    return Effect.void
+  }
+
+  const { configParams, indexParams } = mismatch
+  return Effect.logWarning(
+    `HNSW parameter mismatch: Index was built with M=${indexParams.m}, efConstruction=${indexParams.efConstruction}, ` +
+      `but config specifies M=${configParams.m}, efConstruction=${configParams.efConstruction}. ` +
+      `HNSW parameters only affect index construction. Run 'mdcontext index --embed --force' to rebuild with new parameters.`,
+  )
+}
 
 // ============================================================================
 // Embedding Text Generation
@@ -47,8 +110,12 @@ const generateEmbeddingText = (
 // Cost Estimation
 // ============================================================================
 
-// Price per 1M tokens for text-embedding-3-small
-const EMBEDDING_PRICE_PER_MILLION = 0.02
+// Price per 1M tokens for text-embedding-3-small (from PRICING_DATA)
+const EMBEDDING_PRICE_PER_MILLION =
+  PRICING_DATA.prices['text-embedding-3-small'] ?? 0.02
+
+// Re-export pricing utilities for CLI use
+export { checkPricingFreshness, getPricingDate }
 
 export interface DirectoryEstimate {
   readonly directory: string
@@ -67,10 +134,24 @@ export interface EmbeddingEstimate {
   readonly byDirectory: readonly DirectoryEstimate[]
 }
 
+/**
+ * Estimate the cost of generating embeddings for a directory.
+ *
+ * @param rootPath - Root directory containing indexed markdown files
+ * @param options - Optional exclude patterns
+ * @returns Estimate with token counts and costs
+ *
+ * @throws IndexNotFoundError - Index doesn't exist at path
+ * @throws FileReadError - Cannot read index files
+ * @throws IndexCorruptedError - Index files are corrupted
+ */
 export const estimateEmbeddingCost = (
   rootPath: string,
   options: { excludePatterns?: readonly string[] | undefined } = {},
-): Effect.Effect<EmbeddingEstimate, Error> =>
+): Effect.Effect<
+  EmbeddingEstimate,
+  IndexNotFoundError | FileReadError | IndexCorruptedError
+> =>
   Effect.gen(function* () {
     const resolvedRoot = path.resolve(rootPath)
     const storage = createStorage(resolvedRoot)
@@ -79,9 +160,7 @@ export const estimateEmbeddingCost = (
     const sectionIndex = yield* loadSectionIndex(storage)
 
     if (!docIndex || !sectionIndex) {
-      return yield* Effect.fail(
-        new Error("Index not found. Run 'mdcontext index' first."),
-      )
+      return yield* Effect.fail(new IndexNotFoundError({ path: resolvedRoot }))
     }
 
     // Group by directory
@@ -160,11 +239,25 @@ export interface FileProgress {
   readonly sectionCount: number
 }
 
+export interface EmbeddingBatchProgress {
+  readonly batchIndex: number
+  readonly totalBatches: number
+  readonly processedSections: number
+  readonly totalSections: number
+}
+
 export interface BuildEmbeddingsOptions {
   readonly force?: boolean | undefined
   readonly provider?: EmbeddingProvider | undefined
+  readonly providerConfig?: ProviderFactoryConfig | undefined
   readonly excludePatterns?: readonly string[] | undefined
   readonly onFileProgress?: ((progress: FileProgress) => void) | undefined
+  /** Callback for batch progress during embedding API calls */
+  readonly onBatchProgress?:
+    | ((progress: EmbeddingBatchProgress) => void)
+    | undefined
+  /** HNSW build parameters for vector index construction */
+  readonly hnswOptions?: HnswBuildOptions | undefined
 }
 
 export interface BuildEmbeddingsResult {
@@ -178,10 +271,36 @@ export interface BuildEmbeddingsResult {
   readonly estimatedSavings?: number | undefined
 }
 
+/**
+ * Build embeddings for all indexed sections in a directory.
+ *
+ * @param rootPath - Root directory containing indexed markdown files
+ * @param options - Build options (force rebuild, progress callbacks)
+ * @returns Result with embedding counts, costs, and timing
+ *
+ * @throws IndexNotFoundError - Index doesn't exist at path
+ * @throws FileReadError - Cannot read index or source files
+ * @throws IndexCorruptedError - Index files are corrupted
+ * @throws ApiKeyMissingError - API key not set (check provider config)
+ * @throws ApiKeyInvalidError - API key rejected by provider
+ * @throws EmbeddingError - Embedding API failure (rate limit, quota, network)
+ * @throws VectorStoreError - Cannot save vector index
+ * @throws DimensionMismatchError - Existing embeddings have different dimensions
+ */
 export const buildEmbeddings = (
   rootPath: string,
   options: BuildEmbeddingsOptions = {},
-): Effect.Effect<BuildEmbeddingsResult, Error> =>
+): Effect.Effect<
+  BuildEmbeddingsResult,
+  | IndexNotFoundError
+  | FileReadError
+  | IndexCorruptedError
+  | ApiKeyMissingError
+  | ApiKeyInvalidError
+  | EmbeddingError
+  | VectorStoreError
+  | DimensionMismatchError
+> =>
   Effect.gen(function* () {
     const startTime = Date.now()
     const resolvedRoot = path.resolve(rootPath)
@@ -192,31 +311,52 @@ export const buildEmbeddings = (
     const sectionIndex = yield* loadSectionIndex(storage)
 
     if (!docIndex || !sectionIndex) {
-      return yield* Effect.fail(
-        new Error("Index not found. Run 'mdcontext index' first."),
-      )
+      return yield* Effect.fail(new IndexNotFoundError({ path: resolvedRoot }))
     }
 
-    // Get or create provider (wrap in Effect.try to catch MissingApiKeyError)
+    // Get or create provider - use factory for config-driven provider selection
+    // Priority: explicit provider > providerConfig > default (openai)
+    const providerConfig = options.providerConfig ?? { provider: 'openai' }
     const provider =
-      options.provider ??
-      (yield* Effect.try({
-        try: () => createOpenAIProvider(),
-        catch: (e) => e as Error,
-      }))
+      options.provider ?? (yield* createEmbeddingProviderDirect(providerConfig))
     const dimensions = provider.dimensions
 
-    // Create vector store
-    const vectorStore = createVectorStore(
+    // Extract provider info for namespacing from the actual provider instance
+    // This ensures we use the correct values even when options.provider is explicitly set
+    let providerName: string
+    let providerModel: string
+
+    if (hasProviderMetadata(provider)) {
+      // Provider has metadata - extract provider name from provider.name (format: "provider:model")
+      const nameParts = provider.name.split(':')
+      providerName = nameParts[0] || 'openai'
+      providerModel = provider.model
+    } else {
+      // Fallback to config values for providers without metadata
+      providerName = providerConfig.provider ?? 'openai'
+      providerModel = providerConfig.model ?? 'text-embedding-3-small'
+    }
+
+    // Create namespaced vector store for this provider/model/dimensions combination
+    const vectorStore = createNamespacedVectorStore(
       resolvedRoot,
+      providerName,
+      providerModel,
       dimensions,
+      options.hnswOptions,
     ) as HnswVectorStore
-    vectorStore.setProvider(provider.name)
+
+    // Set provider metadata
+    if (hasProviderMetadata(provider)) {
+      vectorStore.setProvider(provider.name, provider.model, provider.baseURL)
+    } else {
+      vectorStore.setProvider(providerName, providerModel, undefined)
+    }
 
     // Load existing if not forcing
     if (!options.force) {
-      const loaded = yield* vectorStore.load()
-      if (loaded) {
+      const loadResult = yield* vectorStore.load()
+      if (loadResult.loaded) {
         const stats = vectorStore.getStats()
         // Skip if any embeddings exist
         if (stats.count > 0) {
@@ -321,18 +461,26 @@ export const buildEmbeddings = (
       }
 
       const filePath = path.join(resolvedRoot, docPath)
-      let fileContent: string
-      try {
-        fileContent = yield* Effect.promise(() =>
-          fs.readFile(filePath, 'utf-8'),
-        )
-      } catch {
-        // Skip files that can't be read
+
+      // Note: catchAll is intentional - file read failures during embedding
+      // should skip the file with a warning rather than abort the entire operation.
+      // A warning is logged below when the read fails.
+      const fileContentResult = yield* Effect.promise(() =>
+        fs.readFile(filePath, 'utf-8'),
+      ).pipe(
+        Effect.map((content) => ({ ok: true as const, content })),
+        Effect.catchAll(() =>
+          Effect.succeed({ ok: false as const, content: '' }),
+        ),
+      )
+
+      if (!fileContentResult.ok) {
+        yield* Effect.logWarning(`Skipping file (cannot read): ${docPath}`)
         continue
       }
 
       filesProcessed++
-      const lines = fileContent.split('\n')
+      const lines = fileContentResult.content.split('\n')
 
       for (const { section, parentHeading } of sections) {
         // Extract section content from file
@@ -363,16 +511,20 @@ export const buildEmbeddings = (
 
     // Generate embeddings
     const texts = sectionsToEmbed.map((s) => s.text)
-    const result = yield* Effect.tryPromise({
-      try: () => provider.embed(texts),
-      catch: (e) => {
-        // Preserve InvalidApiKeyError so handleApiKeyError can catch it
-        if (e instanceof InvalidApiKeyError) return e
-        return new Error(
-          `Embedding failed: ${e instanceof Error ? e.message : String(e)}`,
-        )
-      },
-    })
+    const result = yield* wrapEmbedding(
+      provider.embed(texts, {
+        onBatchProgress: options.onBatchProgress
+          ? (p) =>
+              options.onBatchProgress?.({
+                batchIndex: p.batchIndex,
+                totalBatches: p.totalBatches,
+                processedSections: p.processedTexts,
+                totalSections: p.totalTexts,
+              })
+          : undefined,
+      }),
+      providerConfig.provider ?? 'openai',
+    )
 
     // Create vector entries
     const entries: VectorEntry[] = []
@@ -397,6 +549,22 @@ export const buildEmbeddings = (
     // Save
     yield* vectorStore.save()
 
+    // Set this namespace as the active provider
+    const namespace = generateNamespace(providerName, providerModel, dimensions)
+    yield* writeActiveProvider(resolvedRoot, {
+      namespace,
+      provider: providerName,
+      model: providerModel,
+      dimensions,
+      activatedAt: new Date().toISOString(),
+    }).pipe(
+      Effect.catchAll((e) => {
+        // Don't fail the build if we can't write the active provider file
+        console.warn(`Warning: Could not set active provider: ${e.message}`)
+        return Effect.succeed(undefined)
+      }),
+    )
+
     const duration = Date.now() - startTime
 
     return {
@@ -408,57 +576,239 @@ export const buildEmbeddings = (
     }
   })
 
+// ============================================================================
+// Context Lines Helper
+// ============================================================================
+
+/**
+ * Add context lines to search results by loading section content from files.
+ * This helper is used by both semanticSearch and semanticSearchWithStats to avoid code duplication.
+ */
+const addContextLinesToResults = (
+  limitedResults: readonly VectorSearchResult[],
+  sectionIndex: { sections: Record<string, SectionEntry> },
+  resolvedRoot: string,
+  options: {
+    contextBefore?: number | undefined
+    contextAfter?: number | undefined
+  },
+): Effect.Effect<readonly SemanticSearchResult[], FileReadError, never> =>
+  Effect.gen(function* () {
+    const contextBefore = options.contextBefore ?? 0
+    const contextAfter = options.contextAfter ?? 0
+
+    const resultsWithContext: SemanticSearchResult[] = []
+    const fileCache = new Map<string, string>()
+
+    for (const r of limitedResults) {
+      const section = sectionIndex.sections[r.sectionId]
+      if (!section) {
+        resultsWithContext.push({
+          sectionId: r.sectionId,
+          documentPath: r.documentPath,
+          heading: r.heading,
+          similarity: r.similarity,
+        })
+        continue
+      }
+
+      let fileContent = fileCache.get(r.documentPath)
+      if (!fileContent) {
+        const filePath = path.join(resolvedRoot, r.documentPath)
+        const contentResult = yield* Effect.promise(() =>
+          fs.readFile(filePath, 'utf-8'),
+        ).pipe(
+          Effect.map((content) => content),
+          Effect.catchAll(() => Effect.succeed(null as string | null)),
+        )
+
+        if (contentResult) {
+          fileContent = contentResult
+          fileCache.set(r.documentPath, fileContent)
+        }
+      }
+
+      if (fileContent) {
+        const lines = fileContent.split('\n')
+        const startIdx = Math.max(0, section.startLine - 1 - contextBefore)
+        const endIdx = Math.min(lines.length, section.endLine + contextAfter)
+
+        const contextLines: {
+          lineNumber: number
+          line: string
+          isMatch: boolean
+        }[] = []
+        for (let i = startIdx; i < endIdx; i++) {
+          const line = lines[i]
+          if (line !== undefined) {
+            contextLines.push({
+              lineNumber: i + 1,
+              line,
+              isMatch: i >= section.startLine - 1 && i < section.endLine,
+            })
+          }
+        }
+
+        resultsWithContext.push({
+          sectionId: r.sectionId,
+          documentPath: r.documentPath,
+          heading: r.heading,
+          similarity: r.similarity,
+          contextLines,
+        })
+      } else {
+        resultsWithContext.push({
+          sectionId: r.sectionId,
+          documentPath: r.documentPath,
+          heading: r.heading,
+          similarity: r.similarity,
+        })
+      }
+    }
+
+    return resultsWithContext
+  })
+
 // ============================================================================
 // Semantic Search
 // ============================================================================
 
+/**
+ * Perform semantic search over embedded sections.
+ *
+ * @param rootPath - Root directory containing embeddings
+ * @param query - Natural language search query
+ * @param options - Search options (limit, threshold, path filter)
+ * @returns Ranked list of matching sections by similarity
+ *
+ * @throws EmbeddingsNotFoundError - No embeddings exist (run index --embed first)
+ * @throws ApiKeyMissingError - API key not set (check provider config)
+ * @throws ApiKeyInvalidError - API key rejected by provider
+ * @throws EmbeddingError - Embedding API failure (rate limit, quota, network)
+ * @throws VectorStoreError - Cannot load or search vector index
+ * @throws DimensionMismatchError - Corpus has different dimensions than current provider
+ */
 export const semanticSearch = (
   rootPath: string,
   query: string,
   options: SemanticSearchOptions = {},
-): Effect.Effect<readonly SemanticSearchResult[], Error> =>
+): Effect.Effect<
+  readonly SemanticSearchResult[],
+  | EmbeddingsNotFoundError
+  | FileReadError
+  | IndexCorruptedError
+  | ApiKeyMissingError
+  | ApiKeyInvalidError
+  | EmbeddingError
+  | VectorStoreError
+  | DimensionMismatchError
+> =>
   Effect.gen(function* () {
     const resolvedRoot = path.resolve(rootPath)
 
-    // Get provider for query embedding (wrap in Effect.try to catch MissingApiKeyError)
-    const provider = yield* Effect.try({
-      try: () => createOpenAIProvider(),
-      catch: (e) => e as Error,
-    })
+    // Get active namespace to determine which embedding index to use
+    const activeProvider = yield* getActiveNamespace(resolvedRoot).pipe(
+      Effect.catchAll(() => Effect.succeed(null as ActiveProvider | null)),
+    )
+
+    if (!activeProvider) {
+      return yield* Effect.fail(
+        new EmbeddingsNotFoundError({ path: resolvedRoot }),
+      )
+    }
+
+    // Create provider for query embedding
+    const provider = yield* createEmbeddingProviderDirect(
+      options.providerConfig ?? { provider: 'openai' },
+    )
     const dimensions = provider.dimensions
 
-    // Load vector store
-    const vectorStore = createVectorStore(resolvedRoot, dimensions)
-    const loaded = yield* vectorStore.load()
+    // Get current provider name for error messages
+    const currentProviderName = options.providerConfig?.provider ?? 'openai'
 
-    if (!loaded) {
+    // Verify dimensions match the active namespace
+    if (dimensions !== activeProvider.dimensions) {
       return yield* Effect.fail(
-        new Error("Embeddings not found. Run 'mdcontext embed' first."),
+        new DimensionMismatchError({
+          corpusDimensions: activeProvider.dimensions,
+          providerDimensions: dimensions,
+          corpusProvider: `${activeProvider.provider}:${activeProvider.model}`,
+          currentProvider: currentProviderName,
+          path: resolvedRoot,
+        }),
       )
     }
 
-    // Embed the query
-    const queryResult = yield* Effect.tryPromise({
-      try: () => provider.embed([query]),
-      catch: (e) =>
-        new Error(
-          `Query embedding failed: ${e instanceof Error ? e.message : String(e)}`,
-        ),
-    })
+    // Load vector store from the active namespace
+    const vectorStore = createNamespacedVectorStore(
+      resolvedRoot,
+      activeProvider.provider,
+      activeProvider.model,
+      activeProvider.dimensions,
+    )
+    const loadResult = yield* vectorStore.load()
+
+    if (!loadResult.loaded) {
+      return yield* Effect.fail(
+        new EmbeddingsNotFoundError({ path: resolvedRoot }),
+      )
+    }
+
+    // Check for HNSW parameter mismatch
+    yield* checkHnswMismatch(loadResult.hnswMismatch)
+
+    // Determine the text to embed
+    // If HyDE is enabled, generate a hypothetical document first
+    let textToEmbed: string
+    let hydeResult: HydeResult | undefined
+
+    if (options.hyde) {
+      // Generate hypothetical document using LLM
+      hydeResult = yield* generateHypotheticalDocument(query, {
+        model: options.hydeOptions?.model,
+        maxTokens: options.hydeOptions?.maxTokens,
+        temperature: options.hydeOptions?.temperature,
+      })
+      textToEmbed = hydeResult.hypotheticalDocument
+      yield* Effect.logDebug(
+        `HyDE generated ${hydeResult.tokensUsed} tokens ($${hydeResult.cost.toFixed(6)})`,
+      )
+    } else {
+      // Preprocess query for better recall (unless disabled)
+      textToEmbed = options.skipPreprocessing ? query : preprocessQuery(query)
+    }
+
+    // Embed the query (or hypothetical document)
+    const queryResult = yield* wrapEmbedding(
+      provider.embed([textToEmbed]),
+      currentProviderName,
+    )
 
     const queryVector = queryResult.embeddings[0]
     if (!queryVector) {
-      return yield* Effect.fail(new Error('Failed to generate query embedding'))
+      return yield* Effect.fail(
+        new EmbeddingError({
+          reason: 'Unknown',
+          message: 'Failed to generate query embedding',
+          provider: currentProviderName,
+        }),
+      )
     }
 
     // Search
     const limit = options.limit ?? 10
     const threshold = options.threshold ?? 0
 
+    // Convert quality mode to efSearch value
+    const efSearch = options.quality
+      ? QUALITY_EF_SEARCH[options.quality]
+      : undefined
+
     const searchResults = yield* vectorStore.search(
       queryVector,
       limit * 2,
       threshold,
+      { efSearch },
     )
 
     // Apply path filter if specified
@@ -471,28 +821,313 @@ export const semanticSearch = (
       filteredResults = searchResults.filter((r) => regex.test(r.documentPath))
     }
 
-    // Convert to SemanticSearchResult
-    const results: SemanticSearchResult[] = filteredResults
-      .slice(0, limit)
-      .map((r) => ({
+    // Apply ranking boost (heading + file importance, enabled by default)
+    const applyBoost = options.headingBoost !== false
+    const boostedResults = applyBoost
+      ? filteredResults.map((r) => ({
+          ...r,
+          similarity: Math.min(
+            1,
+            r.similarity +
+              calculateHeadingBoost(r.heading, query) +
+              calculateFileImportanceBoost(r.documentPath),
+          ),
+        }))
+      : filteredResults
+
+    // Re-sort by boosted similarity
+    const sortedResults = boostedResults.sort(
+      (a, b) => b.similarity - a.similarity,
+    )
+    const limitedResults = sortedResults.slice(0, limit)
+
+    // If context lines are requested, load section content
+    let results: readonly SemanticSearchResult[]
+    if (
+      options.contextBefore !== undefined ||
+      options.contextAfter !== undefined
+    ) {
+      const storage = createStorage(resolvedRoot)
+      const sectionIndex = yield* loadSectionIndex(storage)
+
+      if (sectionIndex) {
+        results = yield* addContextLinesToResults(
+          limitedResults,
+          sectionIndex,
+          resolvedRoot,
+          options,
+        )
+      } else {
+        results = limitedResults.map((r) => ({
+          sectionId: r.sectionId,
+          documentPath: r.documentPath,
+          heading: r.heading,
+          similarity: r.similarity,
+        }))
+      }
+    } else {
+      results = limitedResults.map((r) => ({
         sectionId: r.sectionId,
         documentPath: r.documentPath,
         heading: r.heading,
         similarity: r.similarity,
       }))
+    }
 
     return results
   })
 
+/**
+ * Perform semantic search with stats about below-threshold results.
+ * Use this when you want to provide feedback to users about results that
+ * didn't meet the threshold.
+ *
+ * @param rootPath - Root directory containing embeddings
+ * @param query - Natural language search query
+ * @param options - Search options (limit, threshold, path filter)
+ * @returns Results with optional below-threshold stats
+ *
+ * @throws EmbeddingsNotFoundError - No embeddings exist (run index --embed first)
+ * @throws ApiKeyMissingError - API key not set (check provider config)
+ * @throws ApiKeyInvalidError - API key rejected by provider
+ * @throws EmbeddingError - Embedding API failure (rate limit, quota, network)
+ * @throws VectorStoreError - Cannot load or search vector index
+ * @throws DimensionMismatchError - Corpus has different dimensions than current provider
+ */
+export const semanticSearchWithStats = (
+  rootPath: string,
+  query: string,
+  options: SemanticSearchOptions = {},
+): Effect.Effect<
+  SemanticSearchResultWithStats,
+  | EmbeddingsNotFoundError
+  | FileReadError
+  | IndexCorruptedError
+  | ApiKeyMissingError
+  | ApiKeyInvalidError
+  | EmbeddingError
+  | VectorStoreError
+  | DimensionMismatchError
+> =>
+  Effect.gen(function* () {
+    const resolvedRoot = path.resolve(rootPath)
+
+    // Get active namespace to determine which embedding index to use
+    const activeProvider = yield* getActiveNamespace(resolvedRoot).pipe(
+      Effect.catchAll(() => Effect.succeed(null as ActiveProvider | null)),
+    )
+
+    if (!activeProvider) {
+      return yield* Effect.fail(
+        new EmbeddingsNotFoundError({ path: resolvedRoot }),
+      )
+    }
+
+    // Create provider for query embedding
+    const provider = yield* createEmbeddingProviderDirect(
+      options.providerConfig ?? { provider: 'openai' },
+    )
+    const dimensions = provider.dimensions
+
+    // Get current provider name for error messages
+    const currentProviderName = options.providerConfig?.provider ?? 'openai'
+
+    // Verify dimensions match the active namespace
+    if (dimensions !== activeProvider.dimensions) {
+      return yield* Effect.fail(
+        new DimensionMismatchError({
+          corpusDimensions: activeProvider.dimensions,
+          providerDimensions: dimensions,
+          corpusProvider: `${activeProvider.provider}:${activeProvider.model}`,
+          currentProvider: currentProviderName,
+          path: resolvedRoot,
+        }),
+      )
+    }
+
+    // Load vector store from the active namespace
+    const vectorStore = createNamespacedVectorStore(
+      resolvedRoot,
+      activeProvider.provider,
+      activeProvider.model,
+      activeProvider.dimensions,
+    )
+    const loadResult = yield* vectorStore.load()
+
+    if (!loadResult.loaded) {
+      return yield* Effect.fail(
+        new EmbeddingsNotFoundError({ path: resolvedRoot }),
+      )
+    }
+
+    // Check for HNSW parameter mismatch
+    yield* checkHnswMismatch(loadResult.hnswMismatch)
+
+    // Determine the text to embed
+    // If HyDE is enabled, generate a hypothetical document first
+    let textToEmbed: string
+    let hydeResult: HydeResult | undefined
+
+    if (options.hyde) {
+      // Generate hypothetical document using LLM
+      hydeResult = yield* generateHypotheticalDocument(query, {
+        model: options.hydeOptions?.model,
+        maxTokens: options.hydeOptions?.maxTokens,
+        temperature: options.hydeOptions?.temperature,
+      })
+      textToEmbed = hydeResult.hypotheticalDocument
+      yield* Effect.logDebug(
+        `HyDE generated ${hydeResult.tokensUsed} tokens ($${hydeResult.cost.toFixed(6)})`,
+      )
+    } else {
+      // Preprocess query for better recall (unless disabled)
+      textToEmbed = options.skipPreprocessing ? query : preprocessQuery(query)
+    }
+
+    // Embed the query (or hypothetical document)
+    const queryResult = yield* wrapEmbedding(
+      provider.embed([textToEmbed]),
+      currentProviderName,
+    )
+
+    const queryVector = queryResult.embeddings[0]
+    if (!queryVector) {
+      return yield* Effect.fail(
+        new EmbeddingError({
+          reason: 'Unknown',
+          message: 'Failed to generate query embedding',
+          provider: currentProviderName,
+        }),
+      )
+    }
+
+    // Search with stats
+    const limit = options.limit ?? 10
+    const threshold = options.threshold ?? 0
+
+    // Convert quality mode to efSearch value
+    const efSearch = options.quality
+      ? QUALITY_EF_SEARCH[options.quality]
+      : undefined
+
+    const searchResultWithStats = yield* vectorStore.searchWithStats(
+      queryVector,
+      limit * 2,
+      threshold,
+      { efSearch },
+    )
+
+    // Apply path filter if specified
+    let filteredResults = searchResultWithStats.results
+    if (options.pathPattern) {
+      const pattern = options.pathPattern
+        .replace(/\./g, '\\.')
+        .replace(/\*/g, '.*')
+      const regex = new RegExp(`^${pattern}$`, 'i')
+      filteredResults = searchResultWithStats.results.filter((r) =>
+        regex.test(r.documentPath),
+      )
+    }
+
+    // Apply ranking boost (heading + file importance, enabled by default)
+    const applyBoost = options.headingBoost !== false
+    const boostedResults = applyBoost
+      ? filteredResults.map((r) => ({
+          ...r,
+          similarity: Math.min(
+            1,
+            r.similarity +
+              calculateHeadingBoost(r.heading, query) +
+              calculateFileImportanceBoost(r.documentPath),
+          ),
+        }))
+      : filteredResults
+
+    // Re-sort by boosted similarity and convert to SemanticSearchResult
+    const sortedResults = boostedResults.sort(
+      (a, b) => b.similarity - a.similarity,
+    )
+    const totalAvailable = sortedResults.length
+    const limitedResults = sortedResults.slice(0, limit)
+
+    // If context lines are requested, load section content
+    let results: readonly SemanticSearchResult[]
+    if (
+      options.contextBefore !== undefined ||
+      options.contextAfter !== undefined
+    ) {
+      const storage = createStorage(resolvedRoot)
+      const sectionIndex = yield* loadSectionIndex(storage)
+
+      if (sectionIndex) {
+        results = yield* addContextLinesToResults(
+          limitedResults,
+          sectionIndex,
+          resolvedRoot,
+          options,
+        )
+      } else {
+        results = limitedResults.map((r) => ({
+          sectionId: r.sectionId,
+          documentPath: r.documentPath,
+          heading: r.heading,
+          similarity: r.similarity,
+        }))
+      }
+    } else {
+      results = limitedResults.map((r) => ({
+        sectionId: r.sectionId,
+        documentPath: r.documentPath,
+        heading: r.heading,
+        similarity: r.similarity,
+      }))
+    }
+
+    return {
+      results,
+      belowThresholdCount: searchResultWithStats.belowThresholdCount,
+      belowThresholdHighest:
+        searchResultWithStats.belowThresholdHighest ?? undefined,
+      totalAvailable,
+    }
+  })
+
 // ============================================================================
 // Search with Content
 // ============================================================================
 
+/**
+ * Perform semantic search and include section content in results.
+ *
+ * @param rootPath - Root directory containing embeddings
+ * @param query - Natural language search query
+ * @param options - Search options (limit, threshold, path filter)
+ * @returns Ranked list of matching sections with content
+ *
+ * @throws EmbeddingsNotFoundError - No embeddings exist (run index --embed first)
+ * @throws FileReadError - Cannot read index files
+ * @throws IndexCorruptedError - Index files are corrupted
+ * @throws ApiKeyMissingError - API key not set (check provider config)
+ * @throws ApiKeyInvalidError - API key rejected by provider
+ * @throws EmbeddingError - Embedding API failure (rate limit, quota, network)
+ * @throws VectorStoreError - Cannot load or search vector index
+ * @throws DimensionMismatchError - Corpus has different dimensions than current provider
+ */
 export const semanticSearchWithContent = (
   rootPath: string,
   query: string,
   options: SemanticSearchOptions = {},
-): Effect.Effect<readonly SemanticSearchResult[], Error> =>
+): Effect.Effect<
+  readonly SemanticSearchResult[],
+  | EmbeddingsNotFoundError
+  | FileReadError
+  | IndexCorruptedError
+  | ApiKeyMissingError
+  | ApiKeyInvalidError
+  | EmbeddingError
+  | VectorStoreError
+  | DimensionMismatchError
+> =>
   Effect.gen(function* () {
     const resolvedRoot = path.resolve(rootPath)
     const results = yield* semanticSearch(resolvedRoot, query, options)
@@ -515,23 +1150,35 @@ export const semanticSearchWithContent = (
 
       const filePath = path.join(resolvedRoot, result.documentPath)
 
-      try {
-        const fileContent = yield* Effect.promise(() =>
-          fs.readFile(filePath, 'utf-8'),
-        )
-
-        const lines = fileContent.split('\n')
-        const content = lines
-          .slice(section.startLine - 1, section.endLine)
-          .join('\n')
+      // Note: catchAll is intentional - file read failures during search result
+      // enrichment should skip content loading with a warning, not fail the search.
+      // Results are still returned without content when files can't be read.
+      const fileContentResult = yield* Effect.promise(() =>
+        fs.readFile(filePath, 'utf-8'),
+      ).pipe(
+        Effect.map((content) => ({ ok: true as const, content })),
+        Effect.catchAll(() =>
+          Effect.succeed({ ok: false as const, content: '' }),
+        ),
+      )
 
-        resultsWithContent.push({
-          ...result,
-          content,
-        })
-      } catch {
+      if (!fileContentResult.ok) {
+        yield* Effect.logWarning(
+          `Skipping content load (cannot read): ${result.documentPath}`,
+        )
         resultsWithContent.push(result)
+        continue
       }
+
+      const lines = fileContentResult.content.split('\n')
+      const content = lines
+        .slice(section.startLine - 1, section.endLine)
+        .join('\n')
+
+      resultsWithContent.push({
+        ...result,
+        content,
+      })
     }
 
     return resultsWithContent
@@ -545,22 +1192,60 @@ export interface EmbeddingStats {
   readonly hasEmbeddings: boolean
   readonly count: number
   readonly provider: string
+  readonly model?: string | undefined
   readonly dimensions: number
   readonly totalCost: number
   readonly totalTokens: number
 }
 
+/**
+ * Get statistics about stored embeddings.
+ * Uses the active namespace to find the current embedding index.
+ *
+ * @param rootPath - Root directory containing embeddings
+ * @returns Embedding statistics (count, provider, costs)
+ *
+ * @throws VectorStoreError - Cannot load vector index metadata
+ */
 export const getEmbeddingStats = (
   rootPath: string,
-): Effect.Effect<EmbeddingStats, Error> =>
+): Effect.Effect<EmbeddingStats, VectorStoreError> =>
   Effect.gen(function* () {
     const resolvedRoot = path.resolve(rootPath)
 
-    // Try to load with default dimensions
-    const vectorStore = createVectorStore(resolvedRoot, 1536)
-    const loaded = yield* vectorStore.load()
+    // Get the active namespace to find where embeddings are stored
+    const activeProvider = yield* getActiveNamespace(resolvedRoot).pipe(
+      Effect.catchAll(() => Effect.succeed(null as ActiveProvider | null)),
+    )
 
-    if (!loaded) {
+    if (!activeProvider) {
+      return {
+        hasEmbeddings: false,
+        count: 0,
+        provider: 'none',
+        dimensions: 0,
+        totalCost: 0,
+        totalTokens: 0,
+      }
+    }
+
+    // Load the namespaced vector store to get stats
+    const vectorStore = createNamespacedVectorStore(
+      resolvedRoot,
+      activeProvider.provider,
+      activeProvider.model,
+      activeProvider.dimensions,
+    )
+
+    const loadResult = yield* vectorStore
+      .load()
+      .pipe(
+        Effect.catchAll(() =>
+          Effect.succeed({ loaded: false } as VectorStoreLoadResult),
+        ),
+      )
+
+    if (!loadResult.loaded) {
       return {
         hasEmbeddings: false,
         count: 0,
@@ -572,12 +1257,14 @@ export const getEmbeddingStats = (
     }
 
     const stats = vectorStore.getStats()
+
     return {
       hasEmbeddings: true,
       count: stats.count,
-      provider: stats.provider,
+      provider: stats.provider || 'openai',
+      model: stats.providerModel,
       dimensions: stats.dimensions,
-      totalCost: stats.totalCost,
-      totalTokens: stats.totalTokens,
+      totalCost: stats.totalCost || 0,
+      totalTokens: stats.totalTokens || 0,
     }
   })