@open-mercato/search 0.4.2-canary-c02407ff85

package/src/indexer/subscribers/delete.ts

@@ -0,0 +1,49 @@
+import type { SearchIndexer } from '../search-indexer'
+import type { EntityId } from '@open-mercato/shared/modules/entities'
+import type { SearchDeletePayload } from '@open-mercato/shared/modules/search'
+import { searchDebugWarn, searchError } from '../../lib/debug'
+
+/**
+ * Event subscriber metadata.
+ */
+export const metadata = {
+  event: 'search.delete_record',
+  persistent: false,
+}
+
+/**
+ * Factory to create the search delete subscriber handler.
+ */
+export function createSearchDeleteSubscriber(indexer: SearchIndexer) {
+  return async function handle(payload: SearchDeletePayload): Promise<void> {
+    const entityId = String(payload?.entityId || '') as EntityId
+    const recordId = String(payload?.recordId || '')
+    const tenantId = String(payload?.tenantId || '')
+
+    if (!entityId || !recordId || !tenantId) {
+      searchDebugWarn('search.delete_record', 'Missing required fields', {
+        entityId,
+        recordId,
+        tenantId,
+      })
+      return
+    }
+
+    try {
+      await indexer.deleteRecord({
+        entityId,
+        recordId,
+        tenantId,
+      })
+    } catch (error) {
+      searchError('search.delete_record', 'Failed to delete record', {
+        entityId,
+        recordId,
+        error: error instanceof Error ? error.message : error,
+      })
+      throw error
+    }
+  }
+}
+
+export default createSearchDeleteSubscriber

package/src/lib/debug.ts

@@ -0,0 +1,46 @@
+/**
+ * Debug utilities for search module.
+ *
+ * Set OM_SEARCH_DEBUG=true to enable debug logging.
+ */
+
+export function isSearchDebugEnabled(): boolean {
+  const raw = (process.env.OM_SEARCH_DEBUG ?? '').toLowerCase()
+  return raw === '1' || raw === 'true' || raw === 'yes' || raw === 'on'
+}
+
+/**
+ * Log a debug message if OM_SEARCH_DEBUG is enabled.
+ */
+export function searchDebug(prefix: string, message: string, data?: Record<string, unknown>): void {
+  if (!isSearchDebugEnabled()) return
+  if (data) {
+    console.log(`[${prefix}] ${message}`, data)
+  } else {
+    console.log(`[${prefix}] ${message}`)
+  }
+}
+
+/**
+ * Log a warning message if OM_SEARCH_DEBUG is enabled.
+ */
+export function searchDebugWarn(prefix: string, message: string, data?: Record<string, unknown>): void {
+  if (!isSearchDebugEnabled()) return
+  if (data) {
+    console.warn(`[${prefix}] ${message}`, data)
+  } else {
+    console.warn(`[${prefix}] ${message}`)
+  }
+}
+
+/**
+ * Log an error message (always logs, not gated by debug flag).
+ * Errors should always be visible for troubleshooting.
+ */
+export function searchError(prefix: string, message: string, data?: Record<string, unknown>): void {
+  if (data) {
+    console.error(`[${prefix}] ${message}`, data)
+  } else {
+    console.error(`[${prefix}] ${message}`)
+  }
+}

package/src/lib/fallback-presenter.ts

@@ -0,0 +1,106 @@
+import type { SearchResultPresenter } from '@open-mercato/shared/modules/search'
+
+// Fields to check for title, in priority order
+const TITLE_FIELDS = [
+  'display_name', 'displayName',
+  'name', 'title', 'label',
+  'full_name', 'fullName',
+  'brand_name', 'brandName',
+  'legal_name', 'legalName',
+  'first_name', 'firstName',
+  'last_name', 'lastName',
+  'preferred_name', 'preferredName',
+  'email', 'primary_email', 'primaryEmail',
+  'code', 'sku', 'reference',
+  'identifier', 'slug',
+]
+
+// Fields to check for subtitle
+const SUBTITLE_FIELDS = [
+  'description', 'summary', 'notes',
+  'email', 'primary_email', 'primaryEmail',
+  'phone', 'primary_phone', 'primaryPhone',
+  'status', 'type', 'kind', 'category',
+]
+
+function findFirstValue(doc: Record<string, unknown>, fields: string[]): string | null {
+  for (const field of fields) {
+    const value = doc[field]
+    if (value != null && String(value).trim().length > 0) {
+      return String(value).trim()
+    }
+  }
+  return null
+}
+
+function findAnyStringValue(doc: Record<string, unknown>, excludeFields: Set<string>): string | null {
+  // Skip these fields as they're not meaningful for display
+  const skipFields = new Set([
+    'id', 'tenant_id', 'tenantId', 'organization_id', 'organizationId',
+    'created_at', 'createdAt', 'updated_at', 'updatedAt', 'deleted_at', 'deletedAt',
+    ...excludeFields,
+  ])
+
+  for (const [key, value] of Object.entries(doc)) {
+    if (skipFields.has(key)) continue
+    if (key.startsWith('cf:') || key.startsWith('cf_')) continue
+    if (typeof value === 'string' && value.trim().length > 0 && value.length < 200) {
+      return value.trim()
+    }
+  }
+  return null
+}
+
+function formatEntityLabel(entityId: string): string {
+  const entityName = entityId.split(':')[1] ?? entityId
+  return entityName
+    .replace(/_/g, ' ')
+    .replace(/\b\w/g, (c) => c.toUpperCase())
+}
+
+/**
+ * Extract a presenter from doc fields when no search.ts config exists.
+ *
+ * TODO: This is a basic implementation. Future improvements could include:
+ * - Entity-type specific field mappings
+ * - Smarter field combination (e.g., first_name + last_name)
+ * - Custom field (cf:*) inspection for user-defined display fields
+ * - Configuration for default presenter fields per entity type
+ */
+export function extractFallbackPresenter(
+  doc: Record<string, unknown>,
+  entityId: string,
+  recordId: string,
+): SearchResultPresenter {
+  const entityLabel = formatEntityLabel(entityId)
+
+  // 1. Try common title fields
+  let title = findFirstValue(doc, TITLE_FIELDS)
+
+  // 2. If no title found, try any string field
+  if (!title) {
+    title = findAnyStringValue(doc, new Set(SUBTITLE_FIELDS))
+  }
+
+  // 3. Last resort: use entity label + truncated record ID
+  if (!title) {
+    const shortId = recordId.length > 8 ? recordId.slice(0, 8) + '...' : recordId
+    title = `${entityLabel} ${shortId}`
+  }
+
+  // Build subtitle from multiple relevant fields to show more context
+  const subtitleParts: string[] = []
+  for (const field of SUBTITLE_FIELDS) {
+    const value = doc[field]
+    if (value != null && String(value).trim().length > 0 && String(value) !== title) {
+      subtitleParts.push(String(value).trim())
+      if (subtitleParts.length >= 3) break // Limit to 3 parts
+    }
+  }
+
+  return {
+    title,
+    subtitle: subtitleParts.length > 0 ? subtitleParts.join(' · ').slice(0, 120) : undefined,
+    badge: entityLabel,
+  }
+}

package/src/lib/field-policy.ts

@@ -0,0 +1,169 @@
+import type { SearchFieldPolicy } from '../types'
+
+/**
+ * Encryption map entry as stored in the database.
+ * Matches the structure from entities/data/entities EncryptionMap.
+ */
+export type EncryptionMapEntry = {
+  field: string
+  hashField?: string | null
+}
+
+/**
+ * Configuration for field extraction.
+ */
+export type FieldExtractionConfig = {
+  /** Encryption map entries from the database */
+  encryptedFields?: EncryptionMapEntry[]
+  /** Additional field policy from entity search config */
+  fieldPolicy?: SearchFieldPolicy
+}
+
+/**
+ * Extract only searchable (non-sensitive) fields from a record.
+ * This ensures encrypted and sensitive fields are never sent to external search providers.
+ *
+ * Field filtering logic:
+ * 1. Exclude fields in encryption map (they contain encrypted data)
+ * 2. Exclude fields in fieldPolicy.excluded
+ * 3. Exclude fields in fieldPolicy.hashOnly (should only use hash-based search)
+ * 4. If fieldPolicy.searchable is defined, only include those fields (whitelist mode)
+ *
+ * @param fields - All fields from the record
+ * @param config - Extraction configuration with encryption map and field policy
+ * @returns Object containing only safe-to-index fields
+ */
+export function extractSearchableFields(
+  fields: Record<string, unknown>,
+  config?: FieldExtractionConfig,
+): Record<string, unknown> {
+  const encryptedFieldSet = new Set<string>(
+    config?.encryptedFields?.map((e) => e.field) ?? [],
+  )
+
+  const policy = config?.fieldPolicy
+  const searchableWhitelist = policy?.searchable ? new Set(policy.searchable) : null
+  const excludedBlacklist = new Set([
+    ...(policy?.excluded ?? []),
+    ...(policy?.hashOnly ?? []),
+  ])
+
+  const result: Record<string, unknown> = {}
+
+  for (const [field, value] of Object.entries(fields)) {
+    // Skip null/undefined values
+    if (value == null) continue
+
+    // Skip encrypted fields
+    if (encryptedFieldSet.has(field)) continue
+
+    // Skip explicitly excluded fields
+    if (excludedBlacklist.has(field)) continue
+
+    // If whitelist is defined, only include whitelisted fields
+    if (searchableWhitelist && !searchableWhitelist.has(field)) continue
+
+    result[field] = value
+  }
+
+  return result
+}
+
+/**
+ * Extract fields that should use hash-based search only.
+ * These are typically encrypted fields that have corresponding hash columns.
+ *
+ * @param fields - All fields from the record
+ * @param config - Extraction configuration with encryption map and field policy
+ * @returns Object containing field values for hash-based search
+ */
+export function extractHashOnlyFields(
+  fields: Record<string, unknown>,
+  config?: FieldExtractionConfig,
+): Record<string, unknown> {
+  const hashOnlyFromPolicy = new Set(config?.fieldPolicy?.hashOnly ?? [])
+
+  // Fields with hashField in encryption map are also hash-searchable
+  const hashFieldsFromEncryption = new Set<string>(
+    config?.encryptedFields
+      ?.filter((e) => e.hashField)
+      .map((e) => e.field) ?? [],
+  )
+
+  const result: Record<string, unknown> = {}
+
+  for (const [field, value] of Object.entries(fields)) {
+    if (value == null) continue
+
+    if (hashOnlyFromPolicy.has(field) || hashFieldsFromEncryption.has(field)) {
+      result[field] = value
+    }
+  }
+
+  return result
+}
+
+/**
+ * Build a complete field classification for a record.
+ * Useful for debugging and understanding how fields will be indexed.
+ *
+ * @param fields - All fields from the record
+ * @param config - Extraction configuration
+ * @returns Classification of each field
+ */
+export function classifyFields(
+  fields: Record<string, unknown>,
+  config?: FieldExtractionConfig,
+): {
+  searchable: string[]
+  hashOnly: string[]
+  excluded: string[]
+} {
+  const searchable: string[] = []
+  const hashOnly: string[] = []
+  const excluded: string[] = []
+
+  const encryptedFieldSet = new Set<string>(
+    config?.encryptedFields?.map((e) => e.field) ?? [],
+  )
+  const hashFieldsFromEncryption = new Set<string>(
+    config?.encryptedFields
+      ?.filter((e) => e.hashField)
+      .map((e) => e.field) ?? [],
+  )
+
+  const policy = config?.fieldPolicy
+  const searchableWhitelist = policy?.searchable ? new Set(policy.searchable) : null
+  const hashOnlyFromPolicy = new Set(policy?.hashOnly ?? [])
+  const excludedFromPolicy = new Set(policy?.excluded ?? [])
+
+  for (const field of Object.keys(fields)) {
+    // Check explicit exclusions
+    if (excludedFromPolicy.has(field)) {
+      excluded.push(field)
+      continue
+    }
+
+    // Check hash-only
+    if (hashOnlyFromPolicy.has(field) || hashFieldsFromEncryption.has(field)) {
+      hashOnly.push(field)
+      continue
+    }
+
+    // Check encrypted (without hash)
+    if (encryptedFieldSet.has(field) && !hashFieldsFromEncryption.has(field)) {
+      excluded.push(field)
+      continue
+    }
+
+    // Check whitelist if defined
+    if (searchableWhitelist && !searchableWhitelist.has(field)) {
+      excluded.push(field)
+      continue
+    }
+
+    searchable.push(field)
+  }
+
+  return { searchable, hashOnly, excluded }
+}

package/src/lib/index.ts

@@ -0,0 +1,13 @@
+export {
+  mergeAndRankResults,
+  deduplicateResults,
+  normalizeScores,
+} from './merger'
+
+export {
+  extractSearchableFields,
+  extractHashOnlyFields,
+  classifyFields,
+  type EncryptionMapEntry,
+  type FieldExtractionConfig,
+} from './field-policy'

package/src/lib/merger.ts

@@ -0,0 +1,159 @@
+import type { SearchResult, ResultMergeConfig, SearchStrategyId } from '../types'
+
+/**
+ * Default RRF constant (k=60 is standard in literature).
+ * Higher values reduce the influence of ranking position.
+ */
+const RRF_K = 60
+
+/**
+ * Reciprocal Rank Fusion (RRF) algorithm for combining results from multiple search strategies.
+ *
+ * RRF is a simple but effective method for combining ranked lists. For each result,
+ * it computes: score = sum(weight / (k + rank)) across all lists containing that result.
+ *
+ * Reference: Cormack, G.V., Clarke, C.L.A., & Buettcher, S. (2009).
+ * "Reciprocal rank fusion outperforms condorcet and individual rank learning methods"
+ * https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf
+ *
+ * @param results - Array of search results from multiple strategies
+ * @param config - Merge configuration with weights and thresholds
+ * @returns Merged and ranked results
+ */
+export function mergeAndRankResults(
+  results: SearchResult[],
+  config: ResultMergeConfig,
+): SearchResult[] {
+  if (results.length === 0) return []
+
+  // Group results by source strategy for rank calculation
+  const bySource = new Map<SearchStrategyId, SearchResult[]>()
+  for (const result of results) {
+    const list = bySource.get(result.source) ?? []
+    list.push(result)
+    bySource.set(result.source, list)
+  }
+
+  // Track seen results with their RRF scores
+  // bestContribution tracks the highest single RRF contribution for the kept result object
+  const seen = new Map<string, { result: SearchResult; rrf: number; sources: Set<SearchStrategyId>; bestContribution: number }>()
+
+  // Calculate RRF score for each result
+  for (const [source, sourceResults] of bySource) {
+    const weight = config.strategyWeights?.[source] ?? 1.0
+
+    for (let rank = 0; rank < sourceResults.length; rank++) {
+      const result = sourceResults[rank]
+      const key = `${result.entityId}:${result.recordId}`
+      const rrfScore = weight / (RRF_K + rank + 1)
+
+      const existing = seen.get(key)
+      if (existing) {
+        // Combine RRF scores for duplicates found in multiple strategies
+        existing.rrf += rrfScore
+        existing.sources.add(source)
+
+        // Merge presenter data - prefer result that has it
+        // This ensures token results get enriched with presenter from meilisearch/vector
+        const hasExistingPresenter = existing.result.presenter?.title != null
+        const hasNewPresenter = result.presenter?.title != null
+
+        if (!hasExistingPresenter && hasNewPresenter) {
+          // Current result has no presenter, new one does - take new one's presenter
+          existing.result = {
+            ...existing.result,
+            presenter: result.presenter,
+            url: existing.result.url ?? result.url,
+            links: existing.result.links ?? result.links,
+          }
+          existing.bestContribution = Math.max(existing.bestContribution, rrfScore)
+        } else if (hasExistingPresenter && hasNewPresenter && rrfScore > existing.bestContribution) {
+          // Both have presenter, keep the one with better RRF contribution (not raw score)
+          existing.result = { ...result }
+          existing.bestContribution = rrfScore
+        } else if (!hasExistingPresenter && !hasNewPresenter && rrfScore > existing.bestContribution) {
+          // Neither has presenter, keep result with better RRF contribution
+          existing.result = { ...result }
+          existing.bestContribution = rrfScore
+        }
+        // If existing has presenter and new doesn't, keep existing (do nothing)
+      } else {
+        seen.set(key, {
+          result: { ...result },
+          rrf: rrfScore,
+          sources: new Set([source]),
+          bestContribution: rrfScore,
+        })
+      }
+    }
+  }
+
+  // Convert to array with final RRF scores
+  let merged = Array.from(seen.values()).map(({ result, rrf, sources }) => ({
+    ...result,
+    score: rrf,
+    metadata: {
+      ...result.metadata,
+      _sources: Array.from(sources),
+      _rrfScore: rrf,
+    },
+  }))
+
+  // Apply minimum score threshold
+  if (config.minScore != null) {
+    merged = merged.filter((r) => r.score >= config.minScore!)
+  }
+
+  // Sort by RRF score descending
+  merged.sort((a, b) => b.score - a.score)
+
+  return merged
+}
+
+/**
+ * Simple deduplication without RRF scoring.
+ * Keeps the highest-scored result for each entity+record pair.
+ *
+ * @param results - Array of search results
+ * @returns Deduplicated results sorted by score
+ */
+export function deduplicateResults(results: SearchResult[]): SearchResult[] {
+  const seen = new Map<string, SearchResult>()
+
+  for (const result of results) {
+    const key = `${result.entityId}:${result.recordId}`
+    const existing = seen.get(key)
+
+    if (!existing || result.score > existing.score) {
+      seen.set(key, result)
+    }
+  }
+
+  return Array.from(seen.values()).sort((a, b) => b.score - a.score)
+}
+
+/**
+ * Normalize scores to 0-1 range using min-max normalization.
+ * Useful when combining strategies with different score scales.
+ *
+ * @param results - Array of search results
+ * @returns Results with normalized scores
+ */
+export function normalizeScores(results: SearchResult[]): SearchResult[] {
+  if (results.length === 0) return []
+
+  const scores = results.map((r) => r.score)
+  const minScore = Math.min(...scores)
+  const maxScore = Math.max(...scores)
+  const range = maxScore - minScore
+
+  if (range === 0) {
+    // All scores are the same, normalize to 1.0
+    return results.map((r) => ({ ...r, score: 1.0 }))
+  }
+
+  return results.map((r) => ({
+    ...r,
+    score: (r.score - minScore) / range,
+  }))
+}