@open-mercato/search 0.6.8-develop.7096.1.97319f09f6 → 0.6.8-develop.7100.1.fbf66fca35

package/src/modules/search/workers/fulltext-index.worker.ts

@@ -78,8 +78,9 @@ async function advanceFulltextReindexProgress(params: {
  * This handler processes single record indexing, batch indexing, deletion, and purge
  * operations for the fulltext search strategy.
  *
- * All indexing operations (single and batch) use searchIndexer.indexRecordById() to load
- * fresh data, ensuring consistency with the vector worker pattern.
+ * Single-record jobs load fresh data via searchIndexer.indexRecordById(). Batch jobs load
+ * fresh data per record via searchIndexer.indexRecordsById(), which flushes the whole batch
+ * through a single bulk write instead of one write per record.
  *
  * @param job - The queued job containing payload
  * @param jobCtx - Queue job context with job ID and attempt info
@@ -202,7 +203,7 @@ export async function handleFulltextIndexJob(
       return
     }
 
-    // ========== BATCH-INDEX: Use searchIndexer.indexRecordById() for fresh data ==========
+    // ========== BATCH-INDEX: Load fresh data, write the whole batch in one call ==========
     if (jobType === 'batch-index') {
       const { records, organizationId } = job.payload
       if (!records || records.length === 0) {
@@ -216,30 +217,13 @@ export async function handleFulltextIndexJob(
         throw new Error('searchIndexer not available for batch indexing')
       }
 
-      // Process each record using indexRecordById (same pattern as vector worker)
-      let successCount = 0
-      let failCount = 0
-
-      for (const { entityId, recordId } of records) {
-        try {
-          const result = await searchIndexer.indexRecordById({
-            entityId: entityId as EntityId,
-            recordId,
-            tenantId,
-            organizationId,
-          })
-          if (result.action === 'indexed') {
-            successCount++
-          }
-        } catch (error) {
-          failCount++
-          searchDebugWarn('fulltext-index.worker', 'Failed to index record in batch', {
-            entityId,
-            recordId,
-            error: error instanceof Error ? error.message : error,
-          })
-        }
-      }
+      // Load and index the whole batch through a single bulk write instead of
+      // one indexRecordById() call per record.
+      const { indexed: successCount, skipped: skippedCount } = await searchIndexer.indexRecordsById({
+        items: records.map(({ entityId, recordId }) => ({ entityId: entityId as EntityId, recordId })),
+        tenantId,
+        organizationId,
+      })
 
       await advanceFulltextReindexProgress({
         db,
@@ -255,7 +239,7 @@ export async function handleFulltextIndexJob(
         tenantId,
         requestedCount: records.length,
         successCount,
-        failCount,
+        skippedCount,
       })
 
       await recordIndexerLog(
@@ -265,7 +249,7 @@ export async function handleFulltextIndexJob(
           handler: 'worker:fulltext:batch-index',
           message: `Indexed ${successCount}/${records.length} records to fulltext`,
           tenantId,
-          details: { jobId: jobCtx.jobId, requestedCount: records.length, successCount, failCount },
+          details: { jobId: jobCtx.jobId, requestedCount: records.length, successCount, skippedCount },
         },
       )
       return

package/src/service.ts

@@ -25,6 +25,38 @@ const DEFAULT_MERGE_CONFIG: ResultMergeConfig = {
  */
 const STRATEGY_AVAILABILITY_CACHE_TTL_MS = 2_000
 
+/**
+ * Maximum records indexed at once when bulkIndex falls back to per-record writes
+ * for a strategy that has no bulkIndex implementation (currently the vector
+ * strategy, whose index() performs an embedding-provider round trip per record).
+ * A whole reindex page arrives in one bulkIndex call, so an unbounded fan-out
+ * would burst hundreds of concurrent provider requests from a single job.
+ */
+const BULK_INDEX_FALLBACK_CONCURRENCY = 4
+
+/**
+ * Map items through an async worker with a fixed number of in-flight calls.
+ * Rejects with the first error, matching Promise.all semantics.
+ */
+async function mapWithConcurrency<T>(
+  items: T[],
+  limit: number,
+  worker: (item: T) => Promise<void>,
+): Promise<void> {
+  if (items.length === 0) return
+
+  let nextIndex = 0
+  const runners = Array.from({ length: Math.min(limit, items.length) }, async () => {
+    for (;;) {
+      const currentIndex = nextIndex++
+      if (currentIndex >= items.length) return
+      await worker(items[currentIndex])
+    }
+  })
+
+  await Promise.all(runners)
+}
+
 function normalizeOrganizationFilter(options: SearchOptions): string[] | null {
   const single = typeof options.organizationId === 'string' ? options.organizationId.trim() : ''
   if (single) return [single]
@@ -235,8 +267,11 @@ export class SearchService {
         if (strategy.bulkIndex) {
           return strategy.bulkIndex(records)
         }
-        // Fallback to individual indexing
-        return Promise.all(records.map((record) => this.executeStrategyIndex(strategy, record)))
+        // Fallback to individual indexing, bounded so a strategy without a batch
+        // implementation cannot turn one batch job into hundreds of concurrent writes.
+        return mapWithConcurrency(records, BULK_INDEX_FALLBACK_CONCURRENCY, (record) =>
+          this.executeStrategyIndex(strategy, record),
+        )
       }),
     )