@open-mercato/search 0.4.2-canary-c02407ff85

package/src/strategies/token.strategy.ts

@@ -0,0 +1,153 @@
+import type { Knex } from 'knex'
+import type {
+  SearchStrategy,
+  SearchStrategyId,
+  SearchOptions,
+  SearchResult,
+  IndexableRecord,
+} from '../types'
+import type { EntityId } from '@open-mercato/shared/modules/entities'
+
+/**
+ * Configuration for TokenSearchStrategy.
+ */
+export type TokenStrategyConfig = {
+  /** Minimum number of query tokens that must match (0-1 ratio, default 0.5) */
+  minMatchRatio?: number
+  /** Default limit for search results */
+  defaultLimit?: number
+}
+
+/**
+ * TokenSearchStrategy provides hash-based search using the existing search_tokens table.
+ * This strategy is always available and serves as a fallback when other strategies fail.
+ *
+ * It tokenizes queries into hashes and matches against pre-indexed token hashes,
+ * enabling search on encrypted fields without exposing plaintext to external services.
+ */
+export class TokenSearchStrategy implements SearchStrategy {
+  readonly id: SearchStrategyId = 'tokens'
+  readonly name = 'Token Search'
+  readonly priority = 10 // Lowest priority, always available as fallback
+
+  private readonly minMatchRatio: number
+  private readonly defaultLimit: number
+
+  constructor(
+    private readonly knex: Knex,
+    config?: TokenStrategyConfig,
+  ) {
+    this.minMatchRatio = config?.minMatchRatio ?? 0.5
+    this.defaultLimit = config?.defaultLimit ?? 50
+  }
+
+  async isAvailable(): Promise<boolean> {
+    return true // Always available
+  }
+
+  async ensureReady(): Promise<void> {
+    // No initialization needed
+  }
+
+  async search(query: string, options: SearchOptions): Promise<SearchResult[]> {
+    // Dynamically import tokenization to avoid circular dependencies
+    const { tokenizeText } = await import('@open-mercato/shared/lib/search/tokenize')
+    const { resolveSearchConfig } = await import('@open-mercato/shared/lib/search/config')
+
+    const config = resolveSearchConfig()
+    if (!config.enabled) return []
+
+    const { hashes } = tokenizeText(query, config)
+    if (hashes.length === 0) return []
+
+    const minMatches = Math.max(1, Math.ceil(hashes.length * this.minMatchRatio))
+    const limit = options.limit ?? this.defaultLimit
+
+    let queryBuilder = this.knex('search_tokens')
+      .select('entity_type', 'entity_id')
+      .count('* as match_count')
+      .whereIn('token_hash', hashes)
+      .where('tenant_id', options.tenantId)
+      .groupBy('entity_type', 'entity_id')
+      .havingRaw('COUNT(DISTINCT token_hash) >= ?', [minMatches])
+      .orderByRaw('COUNT(DISTINCT token_hash) DESC')
+      .limit(limit)
+
+    if (options.organizationId) {
+      queryBuilder = queryBuilder.where('organization_id', options.organizationId)
+    }
+
+    if (options.entityTypes?.length) {
+      queryBuilder = queryBuilder.whereIn('entity_type', options.entityTypes)
+    }
+
+    const rows = await queryBuilder as Array<{ entity_type: string; entity_id: string; match_count: string | number }>
+
+    return rows.map((row) => {
+      const matchCount = typeof row.match_count === 'string'
+        ? parseInt(row.match_count, 10)
+        : row.match_count
+      // Calculate score based on match ratio
+      const score = matchCount / hashes.length
+
+      return {
+        entityId: row.entity_type as EntityId,
+        recordId: row.entity_id,
+        score,
+        source: this.id,
+      }
+    })
+  }
+
+  async index(record: IndexableRecord): Promise<void> {
+    // Dynamically import to avoid circular dependencies
+    const { replaceSearchTokensForRecord } = await import(
+      '@open-mercato/core/modules/query_index/lib/search-tokens'
+    )
+
+    await replaceSearchTokensForRecord(this.knex, {
+      entityType: record.entityId,
+      recordId: record.recordId,
+      tenantId: record.tenantId,
+      organizationId: record.organizationId,
+      doc: record.fields,
+    })
+  }
+
+  async delete(entityId: EntityId, recordId: string, tenantId: string): Promise<void> {
+    // Dynamically import to avoid circular dependencies
+    const { deleteSearchTokensForRecord } = await import(
+      '@open-mercato/core/modules/query_index/lib/search-tokens'
+    )
+
+    await deleteSearchTokensForRecord(this.knex, {
+      entityType: entityId,
+      recordId,
+      tenantId,
+    })
+  }
+
+  async bulkIndex(records: IndexableRecord[]): Promise<void> {
+    if (records.length === 0) return
+
+    const { replaceSearchTokensForBatch } = await import(
+      '@open-mercato/core/modules/query_index/lib/search-tokens'
+    )
+
+    const payloads = records.map((record) => ({
+      entityType: record.entityId,
+      recordId: record.recordId,
+      tenantId: record.tenantId,
+      organizationId: record.organizationId,
+      doc: record.fields as Record<string, unknown>,
+    }))
+
+    await replaceSearchTokensForBatch(this.knex, payloads)
+  }
+
+  async purge(entityId: EntityId, tenantId: string): Promise<void> {
+    await this.knex('search_tokens')
+      .where({ entity_type: entityId, tenant_id: tenantId })
+      .del()
+  }
+}

package/src/strategies/vector.strategy.ts

@@ -0,0 +1,234 @@
+import { createHash } from 'crypto'
+import type {
+  SearchStrategy,
+  SearchStrategyId,
+  SearchOptions,
+  SearchResult,
+  IndexableRecord,
+} from '../types'
+import type { EntityId } from '@open-mercato/shared/modules/entities'
+import type { VectorDriver, VectorDriverDocument } from '../vector/types'
+import { searchDebugWarn } from '../lib/debug'
+
+/**
+ * Embedding service interface - minimal subset needed by VectorSearchStrategy.
+ */
+export interface EmbeddingService {
+  createEmbedding(text: string): Promise<number[]>
+  available: boolean
+}
+
+/**
+ * Configuration for VectorSearchStrategy.
+ */
+export type VectorStrategyConfig = {
+  /** Default limit for search results */
+  defaultLimit?: number
+}
+
+/**
+ * VectorSearchStrategy provides semantic search using embeddings.
+ * It wraps the existing vector module infrastructure.
+ */
+export class VectorSearchStrategy implements SearchStrategy {
+  readonly id: SearchStrategyId = 'vector'
+  readonly name = 'Vector Search'
+  readonly priority = 20 // Medium priority
+
+  private readonly defaultLimit: number
+  private ready = false
+  private readyPromise: Promise<void> | null = null
+
+  constructor(
+    private readonly embeddingService: EmbeddingService,
+    private readonly vectorDriver: VectorDriver,
+    config?: VectorStrategyConfig,
+  ) {
+    this.defaultLimit = config?.defaultLimit ?? 20
+  }
+
+  async isAvailable(): Promise<boolean> {
+    return this.embeddingService.available
+  }
+
+  async ensureReady(): Promise<void> {
+    if (this.ready) return
+    if (!this.readyPromise) {
+      this.readyPromise = this.vectorDriver.ensureReady().then(() => {
+        this.ready = true
+      })
+    }
+    return this.readyPromise
+  }
+
+  async search(query: string, options: SearchOptions): Promise<SearchResult[]> {
+    await this.ensureReady()
+    const embedding = await this.embeddingService.createEmbedding(query)
+
+    // Build filter - only include organizationId if it's a real value
+    // The pgvector driver treats null as "only records with null org_id",
+    // but we want null/undefined to mean "no organization filter"
+    const filter: {
+      tenantId: string
+      organizationId?: string | null
+      entityIds?: EntityId[]
+    } = {
+      tenantId: options.tenantId,
+      entityIds: options.entityTypes as EntityId[],
+    }
+
+    // Only add organizationId filter if it's a real org ID
+    if (options.organizationId) {
+      filter.organizationId = options.organizationId
+    }
+
+    const results = await this.vectorDriver.query({
+      vector: embedding,
+      limit: options.limit ?? this.defaultLimit,
+      filter,
+    })
+
+    return results.map((hit) => ({
+      entityId: hit.entityId,
+      recordId: hit.recordId,
+      score: hit.score,
+      source: this.id,
+      presenter: hit.presenter ?? undefined,
+      url: hit.primaryLinkHref ?? hit.url ?? undefined,
+      links: hit.links?.map((link) => ({
+        href: link.href,
+        label: link.label ?? '',
+        kind: link.kind,
+      })),
+      metadata: hit.payload ?? undefined,
+    }))
+  }
+
+  async index(record: IndexableRecord): Promise<void> {
+    await this.ensureReady()
+    // Use text from buildSource if available, otherwise fall back to generic extraction
+    const textContent = record.text
+      ? (Array.isArray(record.text) ? record.text.join('\n') : record.text)
+      : this.buildTextContent(record)
+    if (!textContent) return
+
+    const embedding = await this.embeddingService.createEmbedding(textContent)
+
+    const doc: VectorDriverDocument = {
+      entityId: record.entityId as EntityId,
+      recordId: record.recordId,
+      tenantId: record.tenantId,
+      organizationId: record.organizationId,
+      checksum: this.computeChecksum(record),
+      embedding,
+      url: record.url,
+      presenter: record.presenter,
+      links: record.links,
+      driverId: this.vectorDriver.id,
+      resultTitle: record.presenter?.title ?? record.recordId,
+      resultSubtitle: record.presenter?.subtitle,
+      resultIcon: record.presenter?.icon,
+      resultBadge: record.presenter?.badge,
+    }
+
+    await this.vectorDriver.upsert(doc)
+  }
+
+  async delete(entityId: EntityId, recordId: string, tenantId: string): Promise<void> {
+    await this.ensureReady()
+    await this.vectorDriver.delete(entityId, recordId, tenantId)
+  }
+
+  async purge(entityId: EntityId, tenantId: string): Promise<void> {
+    await this.ensureReady()
+    if (this.vectorDriver.purge) {
+      await this.vectorDriver.purge(entityId, tenantId)
+    }
+  }
+
+  /**
+   * Build text content from record fields for embedding.
+   */
+  private buildTextContent(record: IndexableRecord): string {
+    const parts: string[] = []
+
+    // Add presenter info
+    if (record.presenter?.title) {
+      parts.push(record.presenter.title)
+    }
+    if (record.presenter?.subtitle) {
+      parts.push(record.presenter.subtitle)
+    }
+
+    // Add string fields from record
+    for (const [, value] of Object.entries(record.fields)) {
+      if (typeof value === 'string' && value.trim()) {
+        parts.push(value)
+      }
+    }
+
+    return parts.join(' ').trim()
+  }
+
+  /**
+   * Compute a checksum for change detection using SHA-256.
+   * Uses checksumSource from buildSource if available, otherwise uses fields/presenter/url.
+   */
+  private computeChecksum(record: IndexableRecord): string {
+    const source = record.checksumSource !== undefined
+      ? record.checksumSource
+      : {
+          fields: record.fields,
+          presenter: record.presenter,
+          url: record.url,
+        }
+    const content = JSON.stringify(source)
+    return createHash('sha256').update(content).digest('hex').slice(0, 16)
+  }
+
+  /**
+   * List entries in the vector index (for admin/debugging).
+   */
+  async listEntries(options: {
+    tenantId: string
+    organizationId?: string | null
+    entityId?: string
+    limit?: number
+    offset?: number
+  }): Promise<Array<{
+    entityId: string
+    recordId: string
+    tenantId: string
+    organizationId: string | null
+    presenter?: unknown
+    url?: string
+  }>> {
+    await this.ensureReady()
+    // Delegate to vector driver's list method if available
+    const listMethod = (this.vectorDriver as unknown as {
+      list?: (options: {
+        tenantId: string
+        organizationId?: string | null
+        entityId?: string
+        limit?: number
+        offset?: number
+      }) => Promise<unknown[]>
+    }).list
+
+    if (typeof listMethod === 'function') {
+      const entries = await listMethod.call(this.vectorDriver, options)
+      return entries as Array<{
+        entityId: string
+        recordId: string
+        tenantId: string
+        organizationId: string | null
+        presenter?: unknown
+        url?: string
+      }>
+    }
+
+    // Fallback: return empty array if driver doesn't support listing
+    searchDebugWarn('VectorSearchStrategy', 'Vector driver does not support listing entries')
+    return []
+  }
+}

package/src/types.ts

@@ -0,0 +1,38 @@
+/**
+ * Re-export all search types from shared package.
+ * This allows consumers to import from '@open-mercato/search/types' directly.
+ */
+export type {
+  // Strategy identifiers
+  SearchStrategyId,
+
+  // Result types
+  SearchResult,
+  SearchResultPresenter,
+  SearchResultLink,
+
+  // Search options
+  SearchOptions,
+
+  // Indexable record
+  IndexableRecord,
+
+  // Strategy interface
+  SearchStrategy,
+
+  // Service configuration
+  ResultMergeConfig,
+  SearchServiceOptions,
+  PresenterEnricherFn,
+
+  // Module configuration
+  SearchBuildContext,
+  SearchIndexSource,
+  SearchFieldPolicy,
+  SearchEntityConfig,
+  SearchModuleConfig,
+
+  // Event payloads
+  SearchIndexPayload,
+  SearchDeletePayload,
+} from '@open-mercato/shared/modules/search'

package/src/vector/drivers/chromadb/index.ts

@@ -0,0 +1,49 @@
+import type {
+  VectorDriver,
+  VectorDriverDocument,
+  VectorDriverQuery,
+  VectorDriverQueryResult,
+  VectorDriverCountParams,
+} from '../../types'
+
+function notImplemented(method: string): never {
+  throw new Error(`[vector.chromadb] ${method} not implemented yet`)
+}
+
+export function createChromaDbDriver(): VectorDriver {
+  return {
+    id: 'chromadb',
+    async ensureReady() {
+      notImplemented('ensureReady')
+    },
+    async upsert(doc: VectorDriverDocument) {
+      void doc
+      notImplemented('upsert')
+    },
+    async delete(entityId: string, recordId: string, tenantId: string) {
+      void entityId
+      void recordId
+      void tenantId
+      notImplemented('delete')
+    },
+    async getChecksum(entityId: string, recordId: string, tenantId: string) {
+      void entityId
+      void recordId
+      void tenantId
+      notImplemented('getChecksum')
+    },
+    async query(input: VectorDriverQuery): Promise<VectorDriverQueryResult[]> {
+      void input
+      notImplemented('query')
+    },
+    async purge(entityId: string, tenantId: string) {
+      void entityId
+      void tenantId
+      notImplemented('purge')
+    },
+    async count(params: VectorDriverCountParams) {
+      void params
+      notImplemented('count')
+    },
+  }
+}

package/src/vector/drivers/index.ts

@@ -0,0 +1,4 @@
+export { createPgVectorDriver } from './pgvector'
+export { createQdrantDriver } from './qdrant'
+export { createChromaDbDriver } from './chromadb'
+export type { VectorDriver } from '../types'