@open-mercato/search 0.4.2-canary-c02407ff85

package/src/queue/vector-indexing.ts

@@ -0,0 +1,66 @@
+import { createQueue } from '@open-mercato/queue'
+import type { Queue } from '@open-mercato/queue'
+
+/**
+ * Job types for vector indexing queue
+ */
+export type VectorIndexJobType = 'index' | 'delete' | 'batch-index'
+
+/**
+ * Record reference for batch indexing
+ */
+export type VectorBatchRecord = {
+  entityId: string
+  recordId: string
+}
+
+/**
+ * Payload for vector indexing jobs
+ */
+export type VectorIndexJobPayload = {
+  jobType: VectorIndexJobType
+  tenantId: string
+  organizationId: string | null
+  // For single record jobs (index/delete)
+  entityType?: string
+  recordId?: string
+  // For batch-index jobs
+  records?: VectorBatchRecord[]
+}
+
+/**
+ * Queue name for vector indexing
+ */
+export const VECTOR_INDEXING_QUEUE_NAME = 'vector-indexing'
+
+/**
+ * Creates a vector indexing queue instance.
+ *
+ * @param strategy - Queue strategy: 'local' for file-based, 'async' for BullMQ/Redis
+ * @param options - Strategy-specific options
+ * @returns Queue instance for vector indexing jobs
+ *
+ * @example
+ * ```typescript
+ * // Local queue for development
+ * const queue = createVectorIndexingQueue('local')
+ *
+ * // Async queue for production
+ * const queue = createVectorIndexingQueue('async', {
+ *   connection: { url: process.env.REDIS_URL }
+ * })
+ * ```
+ */
+export function createVectorIndexingQueue(
+  strategy: 'local' | 'async' = 'local',
+  options?: {
+    connection?: { url?: string; host?: string; port?: number }
+  },
+): Queue<VectorIndexJobPayload> {
+  if (strategy === 'async') {
+    return createQueue<VectorIndexJobPayload>(VECTOR_INDEXING_QUEUE_NAME, 'async', {
+      connection: options?.connection,
+    })
+  }
+  return createQueue<VectorIndexJobPayload>(VECTOR_INDEXING_QUEUE_NAME, 'local')
+}

package/src/service.ts

@@ -0,0 +1,397 @@
+import type {
+  SearchStrategy,
+  SearchStrategyId,
+  SearchOptions,
+  SearchResult,
+  SearchServiceOptions,
+  ResultMergeConfig,
+  IndexableRecord,
+  PresenterEnricherFn,
+} from './types'
+import { mergeAndRankResults } from './lib/merger'
+import { searchError } from './lib/debug'
+
+/**
+ * Default merge configuration.
+ */
+const DEFAULT_MERGE_CONFIG: ResultMergeConfig = {
+  duplicateHandling: 'highest_score',
+}
+
+/**
+ * SearchService orchestrates multiple search strategies, executing searches in parallel
+ * and merging results using the RRF algorithm.
+ *
+ * Features:
+ * - Parallel strategy execution for optimal performance
+ * - Graceful degradation when strategies fail
+ * - Result merging with configurable weights
+ * - Strategy availability checking
+ *
+ * @example
+ * ```typescript
+ * const service = new SearchService({
+ *   strategies: [tokenStrategy, vectorStrategy, fulltextStrategy],
+ *   defaultStrategies: ['fulltext', 'vector', 'tokens'],
+ *   mergeConfig: {
+ *     duplicateHandling: 'highest_score',
+ *     strategyWeights: { fulltext: 1.2, vector: 1.0, tokens: 0.8 },
+ *   },
+ * })
+ *
+ * const results = await service.search('john doe', { tenantId: 'tenant-123' })
+ * ```
+ */
+export class SearchService {
+  private readonly strategies: Map<SearchStrategyId, SearchStrategy>
+  private readonly defaultStrategies: SearchStrategyId[]
+  private readonly fallbackStrategy: SearchStrategyId | undefined
+  private readonly mergeConfig: ResultMergeConfig
+  private readonly presenterEnricher?: PresenterEnricherFn
+
+  constructor(options: SearchServiceOptions = {}) {
+    this.strategies = new Map()
+    for (const strategy of options.strategies ?? []) {
+      this.strategies.set(strategy.id, strategy)
+    }
+    this.defaultStrategies = options.defaultStrategies ?? ['tokens']
+    this.fallbackStrategy = options.fallbackStrategy
+    this.mergeConfig = options.mergeConfig ?? DEFAULT_MERGE_CONFIG
+    this.presenterEnricher = options.presenterEnricher
+  }
+
+  /**
+   * Get all registered strategies.
+   */
+  getStrategies(): SearchStrategy[] {
+    return Array.from(this.strategies.values())
+  }
+
+  /**
+   * Execute a search query across configured strategies.
+   *
+   * @param query - Search query string
+   * @param options - Search options with tenant, filters, etc.
+   * @returns Merged and ranked search results
+   */
+  async search(query: string, options: SearchOptions): Promise<SearchResult[]> {
+    const strategyIds = options.strategies ?? this.defaultStrategies
+    const activeStrategies = await this.getAvailableStrategies(strategyIds)
+
+    if (activeStrategies.length === 0) {
+      // Try fallback strategy if defined
+      if (this.fallbackStrategy) {
+        const fallback = await this.getAvailableStrategies([this.fallbackStrategy])
+        if (fallback.length > 0) {
+          activeStrategies.push(...fallback)
+        }
+      }
+    }
+
+    if (activeStrategies.length === 0) {
+      return []
+    }
+
+    // Execute searches in parallel with graceful degradation
+    const results = await Promise.allSettled(
+      activeStrategies.map((strategy) => this.executeStrategySearch(strategy, query, options)),
+    )
+
+    // Collect successful results, log failures
+    const allResults: SearchResult[] = []
+    for (let i = 0; i < results.length; i++) {
+      const result = results[i]
+      if (result.status === 'fulfilled') {
+        allResults.push(...result.value)
+      } else {
+        const strategy = activeStrategies[i]
+        searchError('SearchService', 'Strategy search failed', {
+          strategyId: strategy?.id,
+          error: result.reason instanceof Error ? result.reason.message : result.reason,
+        })
+      }
+    }
+
+    // Merge and rank results
+    const merged = mergeAndRankResults(allResults, this.mergeConfig)
+
+    // Enrich results missing presenter data
+    return this.enrichResultsWithPresenter(merged, options.tenantId, options.organizationId)
+  }
+
+  /**
+   * Enrich results that are missing presenter data using the configured enricher.
+   * This ensures token-only results get proper titles/subtitles for display.
+   */
+  private async enrichResultsWithPresenter(
+    results: SearchResult[],
+    tenantId: string,
+    organizationId?: string | null,
+  ): Promise<SearchResult[]> {
+    // If no enricher configured, return as-is
+    if (!this.presenterEnricher) return results
+
+    // Check if any results need enrichment (missing or encrypted presenter)
+    const needsEnrichment = (r: SearchResult) => {
+      if (!r.presenter?.title) return true
+      // Also enrich if presenter looks encrypted (format: iv:ciphertext:authTag:v1)
+      const title = r.presenter.title
+      if (typeof title === 'string' && title.includes(':')) {
+        const parts = title.split(':')
+        if (parts.length >= 3 && parts[parts.length - 1] === 'v1') return true
+      }
+      return false
+    }
+    const hasMissing = results.some(needsEnrichment)
+    if (!hasMissing) return results
+
+    // Use the configured presenter enricher
+    try {
+      return await this.presenterEnricher(results, tenantId, organizationId)
+    } catch {
+      // Enrichment failed, return results as-is
+      return results
+    }
+  }
+
+  /**
+   * Index a record across all available strategies.
+   *
+   * @param record - Record to index
+   */
+  async index(record: IndexableRecord): Promise<void> {
+    const strategies = await this.getAvailableStrategies()
+
+    if (strategies.length === 0) {
+      return
+    }
+
+    const results = await Promise.allSettled(
+      strategies.map((strategy) => this.executeStrategyIndex(strategy, record)),
+    )
+
+    // Log any failures
+    for (let i = 0; i < results.length; i++) {
+      const result = results[i]
+      if (result.status === 'rejected') {
+        const strategy = strategies[i]
+        searchError('SearchService', 'Strategy index failed', {
+          strategyId: strategy?.id,
+          entityId: record.entityId,
+          recordId: record.recordId,
+          error: result.reason instanceof Error ? result.reason.message : result.reason,
+        })
+      }
+    }
+  }
+
+  /**
+   * Delete a record from all strategies.
+   *
+   * @param entityId - Entity type identifier
+   * @param recordId - Record primary key
+   * @param tenantId - Tenant for isolation
+   */
+  async delete(entityId: string, recordId: string, tenantId: string): Promise<void> {
+    const strategies = await this.getAvailableStrategies()
+
+    const results = await Promise.allSettled(
+      strategies.map((strategy) => this.executeStrategyDelete(strategy, entityId, recordId, tenantId)),
+    )
+
+    // Log any failures
+    for (let i = 0; i < results.length; i++) {
+      const result = results[i]
+      if (result.status === 'rejected') {
+        const strategy = strategies[i]
+        searchError('SearchService', 'Strategy delete failed', {
+          strategyId: strategy?.id,
+          entityId,
+          recordId,
+          error: result.reason instanceof Error ? result.reason.message : result.reason,
+        })
+      }
+    }
+  }
+
+  /**
+   * Bulk index multiple records.
+   *
+   * @param records - Records to index
+   */
+  async bulkIndex(records: IndexableRecord[]): Promise<void> {
+    if (records.length === 0) return
+
+    const strategies = await this.getAvailableStrategies()
+
+    const results = await Promise.allSettled(
+      strategies.map((strategy) => {
+        if (strategy.bulkIndex) {
+          return strategy.bulkIndex(records)
+        }
+        // Fallback to individual indexing
+        return Promise.all(records.map((record) => this.executeStrategyIndex(strategy, record)))
+      }),
+    )
+
+    // Log any failures
+    for (let i = 0; i < results.length; i++) {
+      const result = results[i]
+      if (result.status === 'rejected') {
+        const strategy = strategies[i]
+        searchError('SearchService', 'Strategy bulkIndex failed', {
+          strategyId: strategy?.id,
+          recordCount: records.length,
+          error: result.reason instanceof Error ? result.reason.message : result.reason,
+        })
+      }
+    }
+  }
+
+  /**
+   * Purge all records for an entity type.
+   *
+   * @param entityId - Entity type to purge
+   * @param tenantId - Tenant for isolation
+   */
+  async purge(entityId: string, tenantId: string): Promise<void> {
+    const strategies = await this.getAvailableStrategies()
+
+    const results = await Promise.allSettled(
+      strategies.map((strategy) => {
+        if (strategy.purge) {
+          return strategy.purge(entityId, tenantId)
+        }
+        return Promise.resolve()
+      }),
+    )
+
+    // Log any failures
+    for (let i = 0; i < results.length; i++) {
+      const result = results[i]
+      if (result.status === 'rejected') {
+        const strategy = strategies[i]
+        searchError('SearchService', 'Strategy purge failed', {
+          strategyId: strategy?.id,
+          entityId,
+          error: result.reason instanceof Error ? result.reason.message : result.reason,
+        })
+      }
+    }
+  }
+
+  /**
+   * Register a new strategy at runtime.
+   *
+   * @param strategy - Strategy to register
+   */
+  registerStrategy(strategy: SearchStrategy): void {
+    this.strategies.set(strategy.id, strategy)
+  }
+
+  /**
+   * Unregister a strategy.
+   *
+   * @param strategyId - Strategy ID to remove
+   */
+  unregisterStrategy(strategyId: SearchStrategyId): void {
+    this.strategies.delete(strategyId)
+  }
+
+  /**
+   * Get all registered strategy IDs.
+   */
+  getRegisteredStrategies(): SearchStrategyId[] {
+    return Array.from(this.strategies.keys())
+  }
+
+  /**
+   * Get a specific strategy by ID.
+   *
+   * @param strategyId - Strategy ID to retrieve
+   * @returns The strategy if registered, undefined otherwise
+   */
+  getStrategy(strategyId: SearchStrategyId): SearchStrategy | undefined {
+    return this.strategies.get(strategyId)
+  }
+
+  /**
+   * Get the default strategies list.
+   */
+  getDefaultStrategies(): SearchStrategyId[] {
+    return [...this.defaultStrategies]
+  }
+
+  /**
+   * Check if a specific strategy is available.
+   *
+   * @param strategyId - Strategy ID to check
+   */
+  async isStrategyAvailable(strategyId: SearchStrategyId): Promise<boolean> {
+    const strategy = this.strategies.get(strategyId)
+    if (!strategy) return false
+    return strategy.isAvailable()
+  }
+
+  /**
+   * Get available strategies from the requested list.
+   * Filters out strategies that are not registered or not available.
+   */
+  private async getAvailableStrategies(ids?: SearchStrategyId[]): Promise<SearchStrategy[]> {
+    const targetIds = ids ?? Array.from(this.strategies.keys())
+    const available: SearchStrategy[] = []
+
+    for (const id of targetIds) {
+      const strategy = this.strategies.get(id)
+      if (strategy) {
+        try {
+          const isAvailable = await strategy.isAvailable()
+          if (isAvailable) {
+            available.push(strategy)
+          }
+        } catch {
+          // Strategy availability check failed, skip it
+        }
+      }
+    }
+
+    // Sort by priority (higher priority first)
+    return available.sort((a, b) => b.priority - a.priority)
+  }
+
+  /**
+   * Execute search on a single strategy with error handling.
+   */
+  private async executeStrategySearch(
+    strategy: SearchStrategy,
+    query: string,
+    options: SearchOptions,
+  ): Promise<SearchResult[]> {
+    await strategy.ensureReady()
+    return strategy.search(query, options)
+  }
+
+  /**
+   * Execute index on a single strategy with error handling.
+   */
+  private async executeStrategyIndex(
+    strategy: SearchStrategy,
+    record: IndexableRecord,
+  ): Promise<void> {
+    await strategy.ensureReady()
+    return strategy.index(record)
+  }
+
+  /**
+   * Execute delete on a single strategy with error handling.
+   */
+  private async executeStrategyDelete(
+    strategy: SearchStrategy,
+    entityId: string,
+    recordId: string,
+    tenantId: string,
+  ): Promise<void> {
+    await strategy.ensureReady()
+    return strategy.delete(entityId, recordId, tenantId)
+  }
+}

package/src/strategies/fulltext.strategy.ts

@@ -0,0 +1,155 @@
+import type {
+  SearchStrategy,
+  SearchStrategyId,
+  SearchOptions,
+  SearchResult,
+  IndexableRecord,
+} from '../types'
+import type { EntityId } from '@open-mercato/shared/modules/entities'
+import type {
+  FullTextSearchDriver,
+  FullTextSearchDocument,
+  FullTextSearchHit,
+  DocumentLookupKey,
+  IndexStats,
+} from '../fulltext/types'
+
+/**
+ * FullTextSearchStrategy provides full-text fuzzy search using a pluggable driver.
+ * Default driver is Meilisearch, but can be swapped for Algolia, Elasticsearch, etc.
+ */
+export class FullTextSearchStrategy implements SearchStrategy {
+  readonly id: SearchStrategyId = 'fulltext'
+  readonly name = 'Full-Text Search'
+  readonly priority = 30 // Highest priority when available
+
+  constructor(private readonly driver: FullTextSearchDriver) {}
+
+  async isAvailable(): Promise<boolean> {
+    return this.driver.isHealthy()
+  }
+
+  async ensureReady(): Promise<void> {
+    return this.driver.ensureReady()
+  }
+
+  async search(query: string, options: SearchOptions): Promise<SearchResult[]> {
+    const hits = await this.driver.search(query, {
+      tenantId: options.tenantId,
+      organizationId: options.organizationId,
+      entityTypes: options.entityTypes,
+      limit: options.limit,
+      offset: options.offset,
+    })
+
+    return hits.map((hit) => this.mapHitToResult(hit))
+  }
+
+  async index(record: IndexableRecord): Promise<void> {
+    const doc = this.mapRecordToDocument(record)
+    await this.driver.index(doc)
+  }
+
+  async delete(entityId: EntityId, recordId: string, tenantId: string): Promise<void> {
+    return this.driver.delete(recordId, tenantId)
+  }
+
+  async bulkIndex(records: IndexableRecord[]): Promise<void> {
+    if (!this.driver.bulkIndex) {
+      // Fallback to sequential indexing
+      for (const record of records) {
+        await this.index(record)
+      }
+      return
+    }
+
+    const docs = records.map((record) => this.mapRecordToDocument(record))
+    return this.driver.bulkIndex(docs)
+  }
+
+  async purge(entityId: EntityId, tenantId: string): Promise<void> {
+    if (!this.driver.purge) {
+      return
+    }
+    return this.driver.purge(entityId, tenantId)
+  }
+
+  // Additional methods exposed for enrichment and admin purposes
+  // These delegate to optional driver methods
+
+  async clearIndex(tenantId: string): Promise<void> {
+    if (!this.driver.clearIndex) {
+      return
+    }
+    return this.driver.clearIndex(tenantId)
+  }
+
+  async recreateIndex(tenantId: string): Promise<void> {
+    if (!this.driver.recreateIndex) {
+      return
+    }
+    return this.driver.recreateIndex(tenantId)
+  }
+
+  async getDocuments(
+    ids: DocumentLookupKey[],
+    tenantId: string
+  ): Promise<Map<string, SearchResult>> {
+    if (!this.driver.getDocuments) {
+      return new Map()
+    }
+
+    const hits = await this.driver.getDocuments(ids, tenantId)
+    const result = new Map<string, SearchResult>()
+
+    for (const [key, hit] of hits) {
+      result.set(key, this.mapHitToResult(hit))
+    }
+
+    return result
+  }
+
+  async getIndexStats(tenantId: string): Promise<IndexStats | null> {
+    if (!this.driver.getIndexStats) {
+      return null
+    }
+    return this.driver.getIndexStats(tenantId)
+  }
+
+  async getEntityCounts(tenantId: string): Promise<Record<string, number> | null> {
+    if (!this.driver.getEntityCounts) {
+      return null
+    }
+    return this.driver.getEntityCounts(tenantId)
+  }
+
+  get driverId(): string {
+    return this.driver.id
+  }
+
+  private mapHitToResult(hit: FullTextSearchHit): SearchResult {
+    return {
+      entityId: hit.entityId,
+      recordId: hit.recordId,
+      score: hit.score,
+      source: this.id,
+      presenter: hit.presenter,
+      url: hit.url,
+      links: hit.links,
+      metadata: hit.metadata,
+    }
+  }
+
+  private mapRecordToDocument(record: IndexableRecord): FullTextSearchDocument {
+    return {
+      recordId: record.recordId,
+      entityId: record.entityId,
+      tenantId: record.tenantId,
+      organizationId: record.organizationId,
+      fields: record.fields,
+      presenter: record.presenter,
+      url: record.url,
+      links: record.links,
+    }
+  }
+}

package/src/strategies/index.ts

@@ -0,0 +1,17 @@
+export { TokenSearchStrategy, type TokenStrategyConfig } from './token.strategy'
+export { VectorSearchStrategy, type VectorStrategyConfig, type EmbeddingService } from './vector.strategy'
+export { FullTextSearchStrategy } from './fulltext.strategy'
+
+// Re-export fulltext driver types for convenience
+export type {
+  FullTextSearchDriver,
+  FullTextSearchDriverId,
+  FullTextSearchDocument,
+  FullTextSearchQuery,
+  FullTextSearchHit,
+  FullTextSearchDriverConfig,
+  DocumentLookupKey,
+  IndexStats,
+} from '../fulltext/types'
+export { createMeilisearchDriver, createFulltextDriver } from '../fulltext/drivers'
+export type { MeilisearchDriverOptions } from '../fulltext/drivers/meilisearch'