@open-mercato/search 0.4.2-canary-c02407ff85

package/src/modules/search/workers/fulltext-index.worker.ts

@@ -0,0 +1,318 @@
+import type { QueuedJob, JobContext, WorkerMeta } from '@open-mercato/queue'
+import { FULLTEXT_INDEXING_QUEUE_NAME, type FulltextIndexJobPayload } from '../../../queue/fulltext-indexing'
+import type { FullTextSearchStrategy } from '../../../strategies/fulltext.strategy'
+import type { SearchIndexer } from '../../../indexer/search-indexer'
+import type { EntityManager } from '@mikro-orm/postgresql'
+import type { Knex } from 'knex'
+import type { EntityId } from '@open-mercato/shared/modules/entities'
+import { recordIndexerLog } from '@open-mercato/shared/lib/indexers/status-log'
+import { recordIndexerError } from '@open-mercato/shared/lib/indexers/error-log'
+import { searchDebug, searchDebugWarn, searchError } from '../../../lib/debug'
+import { updateReindexProgress } from '../lib/reindex-lock'
+
+// Worker metadata for auto-discovery
+const DEFAULT_CONCURRENCY = 2
+const envConcurrency = process.env.WORKERS_FULLTEXT_INDEXING_CONCURRENCY
+
+export const metadata: WorkerMeta = {
+  queue: FULLTEXT_INDEXING_QUEUE_NAME,
+  concurrency: envConcurrency ? parseInt(envConcurrency, 10) : DEFAULT_CONCURRENCY,
+}
+
+type HandlerContext = { resolve: <T = unknown>(name: string) => T }
+
+/**
+ * Process a fulltext indexing job.
+ *
+ * This handler processes single record indexing, batch indexing, deletion, and purge
+ * operations for the fulltext search strategy.
+ *
+ * All indexing operations (single and batch) use searchIndexer.indexRecordById() to load
+ * fresh data, ensuring consistency with the vector worker pattern.
+ *
+ * @param job - The queued job containing payload
+ * @param jobCtx - Queue job context with job ID and attempt info
+ * @param ctx - DI container context for resolving services
+ */
+export async function handleFulltextIndexJob(
+  job: QueuedJob<FulltextIndexJobPayload>,
+  jobCtx: JobContext,
+  ctx: HandlerContext,
+): Promise<void> {
+  const { jobType, tenantId } = job.payload
+
+  if (!tenantId) {
+    searchDebugWarn('fulltext-index.worker', 'Skipping job with missing tenantId', {
+      jobId: jobCtx.jobId,
+      jobType,
+    })
+    return
+  }
+
+  // Resolve EntityManager for logging and knex for database queries
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  let em: any | null = null
+  let knex: Knex | null = null
+  try {
+    em = ctx.resolve('em') as EntityManager
+    knex = (em.getConnection() as unknown as { getKnex: () => Knex }).getKnex()
+  } catch {
+    em = null
+    knex = null
+  }
+
+  // Resolve searchIndexer for loading fresh data
+  let searchIndexer: SearchIndexer | undefined
+  try {
+    searchIndexer = ctx.resolve<SearchIndexer>('searchIndexer')
+  } catch {
+    searchDebugWarn('fulltext-index.worker', 'searchIndexer not available')
+  }
+
+  // Resolve fulltext strategy
+  let fulltextStrategy: FullTextSearchStrategy | undefined
+  try {
+    const searchStrategies = ctx.resolve<unknown[]>('searchStrategies')
+    fulltextStrategy = searchStrategies?.find(
+      (s: unknown) => (s as { id?: string })?.id === 'fulltext',
+    ) as FullTextSearchStrategy | undefined
+  } catch {
+    searchDebugWarn('fulltext-index.worker', 'searchStrategies not available')
+    return
+  }
+
+  if (!fulltextStrategy) {
+    searchDebugWarn('fulltext-index.worker', 'Fulltext strategy not configured')
+    return
+  }
+
+  // Check if fulltext is available
+  const isAvailable = await fulltextStrategy.isAvailable()
+  if (!isAvailable) {
+    throw new Error('Fulltext search is not available') // Will trigger retry
+  }
+
+  try {
+    // ========== SINGLE INDEX: Use searchIndexer.indexRecordById() for fresh data ==========
+    if (jobType === 'index') {
+      const { entityType, recordId, organizationId } = job.payload as {
+        entityType: string
+        recordId: string
+        organizationId?: string | null
+      }
+
+      if (!entityType || !recordId) {
+        searchDebugWarn('fulltext-index.worker', 'Skipping index with missing fields', {
+          jobId: jobCtx.jobId,
+          entityType,
+          recordId,
+        })
+        return
+      }
+
+      if (!searchIndexer) {
+        throw new Error('searchIndexer not available for single-record index')
+      }
+
+      const result = await searchIndexer.indexRecordById({
+        entityId: entityType as EntityId,
+        recordId,
+        tenantId,
+        organizationId,
+      })
+
+      searchDebug('fulltext-index.worker', 'Indexed single record to fulltext', {
+        jobId: jobCtx.jobId,
+        tenantId,
+        entityType,
+        recordId,
+        action: result.action,
+      })
+
+      await recordIndexerLog(
+        { em: em ?? undefined },
+        {
+          source: 'fulltext',
+          handler: 'worker:fulltext:index',
+          message: `Indexed record to fulltext (${result.action})`,
+          entityType,
+          recordId,
+          tenantId,
+          details: { jobId: jobCtx.jobId },
+        },
+      )
+      return
+    }
+
+    // ========== BATCH-INDEX: Use searchIndexer.indexRecordById() for fresh data ==========
+    if (jobType === 'batch-index') {
+      const { records, organizationId } = job.payload
+      if (!records || records.length === 0) {
+        searchDebugWarn('fulltext-index.worker', 'Skipping batch-index with no records', {
+          jobId: jobCtx.jobId,
+        })
+        return
+      }
+
+      if (!searchIndexer) {
+        throw new Error('searchIndexer not available for batch indexing')
+      }
+
+      // Process each record using indexRecordById (same pattern as vector worker)
+      let successCount = 0
+      let failCount = 0
+
+      for (const { entityId, recordId } of records) {
+        try {
+          const result = await searchIndexer.indexRecordById({
+            entityId: entityId as EntityId,
+            recordId,
+            tenantId,
+            organizationId,
+          })
+          if (result.action === 'indexed') {
+            successCount++
+          }
+        } catch (error) {
+          failCount++
+          searchDebugWarn('fulltext-index.worker', 'Failed to index record in batch', {
+            entityId,
+            recordId,
+            error: error instanceof Error ? error.message : error,
+          })
+        }
+      }
+
+      // Update heartbeat to signal worker is still processing
+      if (knex && successCount > 0) {
+        await updateReindexProgress(knex, tenantId, 'fulltext', successCount, organizationId ?? null)
+      }
+
+      searchDebug('fulltext-index.worker', 'Batch indexed to fulltext', {
+        jobId: jobCtx.jobId,
+        tenantId,
+        requestedCount: records.length,
+        successCount,
+        failCount,
+      })
+
+      await recordIndexerLog(
+        { em: em ?? undefined },
+        {
+          source: 'fulltext',
+          handler: 'worker:fulltext:batch-index',
+          message: `Indexed ${successCount}/${records.length} records to fulltext`,
+          tenantId,
+          details: { jobId: jobCtx.jobId, requestedCount: records.length, successCount, failCount },
+        },
+      )
+      return
+    }
+
+    // ========== DELETE ==========
+    if (jobType === 'delete') {
+      const { entityId, recordId } = job.payload
+      if (!entityId || !recordId) {
+        searchDebugWarn('fulltext-index.worker', 'Skipping delete with missing fields', {
+          jobId: jobCtx.jobId,
+          entityId,
+          recordId,
+        })
+        return
+      }
+
+      await fulltextStrategy.delete(entityId, recordId, tenantId)
+
+      searchDebug('fulltext-index.worker', 'Deleted from fulltext', {
+        jobId: jobCtx.jobId,
+        tenantId,
+        entityId,
+        recordId,
+      })
+
+      await recordIndexerLog(
+        { em: em ?? undefined },
+        {
+          source: 'fulltext',
+          handler: 'worker:fulltext:delete',
+          message: `Deleted record from fulltext`,
+          entityType: entityId,
+          recordId,
+          tenantId,
+          details: { jobId: jobCtx.jobId },
+        },
+      )
+      return
+    }
+
+    // ========== PURGE ==========
+    if (jobType === 'purge') {
+      const { entityId } = job.payload
+      if (!entityId) {
+        searchDebugWarn('fulltext-index.worker', 'Skipping purge with missing entityId', {
+          jobId: jobCtx.jobId,
+        })
+        return
+      }
+
+      await fulltextStrategy.purge(entityId, tenantId)
+
+      searchDebug('fulltext-index.worker', 'Purged entity from fulltext', {
+        jobId: jobCtx.jobId,
+        tenantId,
+        entityId,
+      })
+
+      await recordIndexerLog(
+        { em: em ?? undefined },
+        {
+          source: 'fulltext',
+          handler: 'worker:fulltext:purge',
+          message: `Purged entity from fulltext`,
+          entityType: entityId,
+          tenantId,
+          details: { jobId: jobCtx.jobId },
+        },
+      )
+      return
+    }
+  } catch (error) {
+    searchError('fulltext-index.worker', `Failed to ${jobType}`, {
+      jobId: jobCtx.jobId,
+      tenantId,
+      error: error instanceof Error ? error.message : error,
+      attemptNumber: jobCtx.attemptNumber,
+    })
+
+    const entityId = 'entityId' in job.payload ? job.payload.entityId :
+                     'entityType' in job.payload ? (job.payload as { entityType?: string }).entityType : undefined
+    const recordId = 'recordId' in job.payload ? job.payload.recordId : undefined
+
+    await recordIndexerError(
+      { em: em ?? undefined },
+      {
+        source: 'fulltext',
+        handler: `worker:fulltext:${jobType}`,
+        error,
+        entityType: entityId,
+        recordId,
+        tenantId,
+        payload: job.payload,
+      },
+    )
+
+    // Re-throw to let the queue handle retry logic
+    throw error
+  }
+}
+
+/**
+ * Default export for worker auto-discovery.
+ * Wraps handleFulltextIndexJob to match the expected handler signature.
+ */
+export default async function handle(
+  job: QueuedJob<FulltextIndexJobPayload>,
+  ctx: JobContext & HandlerContext
+): Promise<void> {
+  return handleFulltextIndexJob(job, ctx, ctx)
+}

package/src/modules/search/workers/vector-index.worker.ts

@@ -0,0 +1,292 @@
+import type { QueuedJob, JobContext, WorkerMeta } from '@open-mercato/queue'
+import { VECTOR_INDEXING_QUEUE_NAME, type VectorIndexJobPayload } from '../../../queue/vector-indexing'
+import type { SearchIndexer } from '../../../indexer/search-indexer'
+import type { EmbeddingService } from '../../../vector'
+import type { EntityManager } from '@mikro-orm/postgresql'
+import type { Knex } from 'knex'
+import { recordIndexerError } from '@open-mercato/shared/lib/indexers/error-log'
+import { applyCoverageAdjustments, createCoverageAdjustments } from '@open-mercato/core/modules/query_index/lib/coverage'
+import { logVectorOperation } from '../../../vector/lib/vector-logs'
+import { resolveAutoIndexingEnabled } from '../lib/auto-indexing'
+import { resolveEmbeddingConfig } from '../lib/embedding-config'
+import { searchDebugWarn } from '../../../lib/debug'
+import { updateReindexProgress } from '../lib/reindex-lock'
+
+// Worker metadata for auto-discovery
+const DEFAULT_CONCURRENCY = 2
+const envConcurrency = process.env.WORKERS_VECTOR_INDEXING_CONCURRENCY
+
+export const metadata: WorkerMeta = {
+  queue: VECTOR_INDEXING_QUEUE_NAME,
+  concurrency: envConcurrency ? parseInt(envConcurrency, 10) : DEFAULT_CONCURRENCY,
+}
+
+type HandlerContext = { resolve: <T = unknown>(name: string) => T }
+
+/**
+ * Process a vector index job.
+ *
+ * This handler is called by the queue worker to process indexing and deletion jobs.
+ * It uses SearchIndexer to load records and index them via SearchService.
+ *
+ * @param job - The queued job containing payload
+ * @param jobCtx - Queue job context with job ID and attempt info
+ * @param ctx - DI container context for resolving services
+ */
+export async function handleVectorIndexJob(
+  job: QueuedJob<VectorIndexJobPayload>,
+  jobCtx: JobContext,
+  ctx: HandlerContext,
+): Promise<void> {
+  const { jobType, entityType, recordId, tenantId, organizationId, records } = job.payload
+
+  // Handle batch-index jobs (from reindex operations)
+  if (jobType === 'batch-index') {
+    if (!records?.length || !tenantId) {
+      searchDebugWarn('vector-index.worker', 'Skipping batch-index job with missing required fields', {
+        jobId: jobCtx.jobId,
+        recordCount: records?.length ?? 0,
+        tenantId,
+      })
+      return
+    }
+
+    let searchIndexer: SearchIndexer
+    try {
+      searchIndexer = ctx.resolve<SearchIndexer>('searchIndexer')
+    } catch {
+      searchDebugWarn('vector-index.worker', 'searchIndexer not available')
+      return
+    }
+
+    // Get knex for heartbeat updates
+    let knex: Knex | null = null
+    try {
+      const em = ctx.resolve('em') as EntityManager
+      knex = (em.getConnection() as unknown as { getKnex: () => Knex }).getKnex()
+    } catch {
+      knex = null
+    }
+
+    // Load saved embedding config to use the correct provider/model
+    try {
+      const embeddingConfig = await resolveEmbeddingConfig(ctx, { defaultValue: null })
+      if (embeddingConfig) {
+        const embeddingService = ctx.resolve<EmbeddingService>('vectorEmbeddingService')
+        embeddingService.updateConfig(embeddingConfig)
+      }
+    } catch (configErr) {
+      searchDebugWarn('vector-index.worker', 'Failed to load embedding config for batch, using defaults', {
+        error: configErr instanceof Error ? configErr.message : configErr,
+      })
+    }
+
+    // Process each record in the batch
+    let successCount = 0
+    let failCount = 0
+    for (const { entityId, recordId: recId } of records) {
+      try {
+        const result = await searchIndexer.indexRecordById({
+          entityId: entityId as Parameters<typeof searchIndexer.indexRecordById>[0]['entityId'],
+          recordId: recId,
+          tenantId,
+          organizationId,
+        })
+        if (result.action === 'indexed') {
+          successCount++
+        }
+      } catch (error) {
+        failCount++
+        searchDebugWarn('vector-index.worker', 'Failed to index record in batch', {
+          entityId,
+          recordId: recId,
+          error: error instanceof Error ? error.message : error,
+        })
+      }
+    }
+
+    // Update heartbeat to signal worker is still processing
+    if (knex && successCount > 0) {
+      await updateReindexProgress(knex, tenantId, 'vector', successCount, organizationId ?? null)
+    }
+
+    searchDebugWarn('vector-index.worker', 'Batch-index job completed', {
+      jobId: jobCtx.jobId,
+      totalRecords: records.length,
+      successCount,
+      failCount,
+    })
+    return
+  }
+
+  // Handle single record jobs (index/delete)
+  if (!entityType || !recordId || !tenantId) {
+    searchDebugWarn('vector-index.worker', 'Skipping job with missing required fields', {
+      jobId: jobCtx.jobId,
+      entityType,
+      recordId,
+      tenantId,
+    })
+    return
+  }
+
+  const autoIndexingEnabled = await resolveAutoIndexingEnabled(ctx, { defaultValue: true })
+  if (!autoIndexingEnabled) {
+    return
+  }
+
+  let searchIndexer: SearchIndexer
+  try {
+    searchIndexer = ctx.resolve<SearchIndexer>('searchIndexer')
+  } catch {
+    searchDebugWarn('vector-index.worker', 'searchIndexer not available')
+    return
+  }
+
+  // Load saved embedding config to use the correct provider/model
+  try {
+    const embeddingConfig = await resolveEmbeddingConfig(ctx, { defaultValue: null })
+    if (embeddingConfig) {
+      const embeddingService = ctx.resolve<EmbeddingService>('vectorEmbeddingService')
+      embeddingService.updateConfig(embeddingConfig)
+    }
+  } catch (configErr) {
+    // Delete operations don't require embedding, only warn for index operations
+    if (jobType === 'index') {
+      searchDebugWarn('vector-index.worker', 'Failed to load embedding config, using defaults', {
+        error: configErr instanceof Error ? configErr.message : configErr,
+      })
+    }
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  let em: any | null = null
+  try {
+    em = ctx.resolve('em')
+  } catch {
+    em = null
+  }
+
+  let eventBus: { emitEvent(event: string, payload: unknown, options?: unknown): Promise<void> } | null = null
+  try {
+    eventBus = ctx.resolve('eventBus')
+  } catch {
+    eventBus = null
+  }
+
+  const handlerName = jobType === 'delete'
+    ? 'worker:vector-indexing:delete'
+    : 'worker:vector-indexing:index'
+
+  try {
+    let action: 'indexed' | 'deleted' | 'skipped' = 'skipped'
+    let delta = 0
+
+    if (jobType === 'delete') {
+      await searchIndexer.deleteRecord({
+        entityId: entityType,
+        recordId,
+        tenantId,
+      })
+      action = 'deleted'
+      delta = -1
+    } else {
+      const result = await searchIndexer.indexRecordById({
+        entityId: entityType,
+        recordId,
+        tenantId,
+        organizationId,
+      })
+      action = result.action
+      if (result.action === 'indexed') {
+        delta = 1
+      }
+    }
+
+    if (delta !== 0) {
+      let adjustmentsApplied = false
+      if (em) {
+        try {
+          const adjustments = createCoverageAdjustments({
+            entityType,
+            tenantId,
+            organizationId,
+            baseDelta: 0,
+            indexDelta: 0,
+            vectorDelta: delta,
+          })
+          if (adjustments.length) {
+            await applyCoverageAdjustments(em, adjustments)
+            adjustmentsApplied = true
+          }
+        } catch (coverageError) {
+          searchDebugWarn('vector-index.worker', 'Failed to adjust vector coverage', {
+            error: coverageError instanceof Error ? coverageError.message : coverageError,
+          })
+        }
+      }
+
+      if (!adjustmentsApplied && eventBus) {
+        try {
+          await eventBus.emitEvent('query_index.coverage.refresh', {
+            entityType,
+            tenantId,
+            organizationId,
+            withDeleted: false,
+            delayMs: 1000,
+          })
+        } catch (emitError) {
+          searchDebugWarn('vector-index.worker', 'Failed to enqueue coverage refresh', {
+            error: emitError instanceof Error ? emitError.message : emitError,
+          })
+        }
+      }
+    }
+
+    await logVectorOperation({
+      em,
+      handler: handlerName,
+      entityType,
+      recordId,
+      result: {
+        action,
+        tenantId,
+        organizationId: organizationId ?? null,
+        created: action === 'indexed',
+        existed: action === 'deleted',
+      },
+    })
+  } catch (error) {
+    searchDebugWarn('vector-index.worker', `Failed to ${jobType} vector index`, {
+      entityType,
+      recordId,
+      error: error instanceof Error ? error.message : error,
+    })
+    await recordIndexerError(
+      { em: em ?? undefined },
+      {
+        source: 'vector',
+        handler: handlerName,
+        error,
+        entityType,
+        recordId,
+        tenantId,
+        organizationId,
+        payload: job.payload,
+      },
+    )
+    // Re-throw to let the queue handle retry logic
+    throw error
+  }
+}
+
+/**
+ * Default export for worker auto-discovery.
+ * Wraps handleVectorIndexJob to match the expected handler signature.
+ */
+export default async function handle(
+  job: QueuedJob<VectorIndexJobPayload>,
+  ctx: JobContext & HandlerContext
+): Promise<void> {
+  return handleVectorIndexJob(job, ctx, ctx)
+}

package/src/queue/fulltext-indexing.ts

@@ -0,0 +1,87 @@
+import { createQueue, type Queue } from '@open-mercato/queue'
+
+/**
+ * Job types for fulltext indexing queue.
+ */
+export type FulltextIndexJobType = 'index' | 'batch-index' | 'delete' | 'purge'
+
+/**
+ * Minimal record reference for batch indexing.
+ * Only contains identifiers - actual data is loaded fresh via searchIndexer.indexRecordById().
+ * This keeps queue payloads small and ensures fresh data is indexed.
+ */
+export type FulltextBatchRecord = {
+  entityId: string
+  recordId: string
+}
+
+/**
+ * Payload for single record indexing jobs.
+ * Worker loads fresh data via searchIndexer.indexRecordById().
+ */
+export type FulltextIndexPayload = {
+  jobType: 'index'
+  tenantId: string
+  organizationId?: string | null
+  entityType: string
+  recordId: string
+}
+
+/**
+ * Payload for batch indexing jobs.
+ * Worker loads fresh data via searchIndexer.indexRecordById() for each record.
+ */
+export type FulltextBatchIndexPayload = {
+  jobType: 'batch-index'
+  tenantId: string
+  organizationId?: string | null
+  records: FulltextBatchRecord[]
+}
+
+/**
+ * Payload for delete jobs.
+ */
+export type FulltextDeletePayload = {
+  jobType: 'delete'
+  tenantId: string
+  entityId: string
+  recordId: string
+}
+
+/**
+ * Payload for purge jobs (delete all records of an entity type).
+ */
+export type FulltextPurgePayload = {
+  jobType: 'purge'
+  tenantId: string
+  entityId: string
+}
+
+/**
+ * Union type for all fulltext indexing job payloads.
+ */
+export type FulltextIndexJobPayload =
+  | FulltextIndexPayload
+  | FulltextBatchIndexPayload
+  | FulltextDeletePayload
+  | FulltextPurgePayload
+
+export const FULLTEXT_INDEXING_QUEUE_NAME = 'fulltext-indexing'
+
+/**
+ * Create a fulltext indexing queue.
+ *
+ * @param strategy - Queue strategy ('local' for development, 'async' for production with Redis)
+ * @param options - Optional connection configuration for async strategy
+ */
+export function createFulltextIndexingQueue(
+  strategy: 'local' | 'async' = 'local',
+  options?: { connection?: { url?: string; host?: string; port?: number } },
+): Queue<FulltextIndexJobPayload> {
+  if (strategy === 'async') {
+    return createQueue<FulltextIndexJobPayload>(FULLTEXT_INDEXING_QUEUE_NAME, 'async', {
+      connection: options?.connection,
+    })
+  }
+  return createQueue<FulltextIndexJobPayload>(FULLTEXT_INDEXING_QUEUE_NAME, 'local')
+}

package/src/queue/index.ts

@@ -0,0 +1,2 @@
+export * from './vector-indexing'
+export * from './fulltext-indexing'