@open-mercato/search 0.4.6-develop-6953d75a91 → 0.4.6-develop-90c3eb0e8a
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/modules/search/__integration__/TC-SEARCH-001.spec.js +64 -0
- package/dist/modules/search/__integration__/TC-SEARCH-001.spec.js.map +7 -0
- package/dist/modules/search/api/embeddings/reindex/cancel/route.js +10 -0
- package/dist/modules/search/api/embeddings/reindex/cancel/route.js.map +2 -2
- package/dist/modules/search/api/embeddings/reindex/route.js +23 -0
- package/dist/modules/search/api/embeddings/reindex/route.js.map +2 -2
- package/dist/modules/search/api/reindex/cancel/route.js +10 -0
- package/dist/modules/search/api/reindex/cancel/route.js.map +2 -2
- package/dist/modules/search/api/reindex/route.js +39 -0
- package/dist/modules/search/api/reindex/route.js.map +2 -2
- package/dist/modules/search/frontend/components/SearchSettingsPageClient.js +13 -26
- package/dist/modules/search/frontend/components/SearchSettingsPageClient.js.map +2 -2
- package/dist/modules/search/frontend/components/sections/FulltextSearchSection.js +10 -6
- package/dist/modules/search/frontend/components/sections/FulltextSearchSection.js.map +2 -2
- package/dist/modules/search/frontend/components/sections/VectorSearchSection.js +10 -6
- package/dist/modules/search/frontend/components/sections/VectorSearchSection.js.map +2 -2
- package/dist/modules/search/lib/reindex-progress.js +151 -0
- package/dist/modules/search/lib/reindex-progress.js.map +7 -0
- package/dist/modules/search/workers/fulltext-index.worker.js +22 -2
- package/dist/modules/search/workers/fulltext-index.worker.js.map +2 -2
- package/dist/modules/search/workers/vector-index.worker.js +25 -3
- package/dist/modules/search/workers/vector-index.worker.js.map +2 -2
- package/package.json +4 -4
- package/src/modules/search/README.md +13 -0
- package/src/modules/search/__integration__/TC-SEARCH-001.spec.ts +80 -0
- package/src/modules/search/api/embeddings/reindex/cancel/route.ts +11 -0
- package/src/modules/search/api/embeddings/reindex/route.ts +27 -0
- package/src/modules/search/api/reindex/cancel/route.ts +11 -0
- package/src/modules/search/api/reindex/route.ts +44 -0
- package/src/modules/search/frontend/components/SearchSettingsPageClient.tsx +13 -33
- package/src/modules/search/frontend/components/sections/FulltextSearchSection.tsx +12 -7
- package/src/modules/search/frontend/components/sections/VectorSearchSection.tsx +12 -7
- package/src/modules/search/lib/reindex-progress.ts +202 -0
- package/src/modules/search/workers/fulltext-index.worker.ts +24 -2
- package/src/modules/search/workers/vector-index.worker.ts +27 -3
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../../src/modules/search/workers/fulltext-index.worker.ts"],
|
|
4
|
-
"sourcesContent": ["import type { QueuedJob, JobContext, WorkerMeta } from '@open-mercato/queue'\nimport { FULLTEXT_INDEXING_QUEUE_NAME, type FulltextIndexJobPayload } from '../../../queue/fulltext-indexing'\nimport type { FullTextSearchStrategy } from '../../../strategies/fulltext.strategy'\nimport type { SearchIndexer } from '../../../indexer/search-indexer'\nimport type { EntityManager } from '@mikro-orm/postgresql'\nimport type { Knex } from 'knex'\nimport type { EntityId } from '@open-mercato/shared/modules/entities'\nimport { recordIndexerLog } from '@open-mercato/shared/lib/indexers/status-log'\nimport { recordIndexerError } from '@open-mercato/shared/lib/indexers/error-log'\nimport { searchDebug, searchDebugWarn, searchError } from '../../../lib/debug'\nimport { updateReindexProgress } from '../lib/reindex-lock'\n\n// Worker metadata for auto-discovery\nconst DEFAULT_CONCURRENCY = 2\nconst envConcurrency = process.env.WORKERS_FULLTEXT_INDEXING_CONCURRENCY\n\nexport const metadata: WorkerMeta = {\n queue: FULLTEXT_INDEXING_QUEUE_NAME,\n concurrency: envConcurrency ? parseInt(envConcurrency, 10) : DEFAULT_CONCURRENCY,\n}\n\ntype HandlerContext = { resolve: <T = unknown>(name: string) => T }\n\n/**\n * Process a fulltext indexing job.\n *\n * This handler processes single record indexing, batch indexing, deletion, and purge\n * operations for the fulltext search strategy.\n *\n * All indexing operations (single and batch) use searchIndexer.indexRecordById() to load\n * fresh data, ensuring consistency with the vector worker pattern.\n *\n * @param job - The queued job containing payload\n * @param jobCtx - Queue job context with job ID and attempt info\n * @param ctx - DI container context for resolving services\n */\nexport async function handleFulltextIndexJob(\n job: QueuedJob<FulltextIndexJobPayload>,\n jobCtx: JobContext,\n ctx: HandlerContext,\n): Promise<void> {\n const { jobType, tenantId } = job.payload\n\n if (!tenantId) {\n searchDebugWarn('fulltext-index.worker', 'Skipping job with missing tenantId', {\n jobId: jobCtx.jobId,\n jobType,\n })\n return\n }\n\n // Resolve EntityManager for logging and knex for database queries\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n let em: any | null = null\n let knex: Knex | null = null\n try {\n em = ctx.resolve('em') as EntityManager\n knex = (em.getConnection() as unknown as { getKnex: () => Knex }).getKnex()\n } catch {\n em = null\n knex = null\n }\n\n // Resolve searchIndexer for loading fresh data\n let searchIndexer: SearchIndexer | undefined\n try {\n searchIndexer = ctx.resolve<SearchIndexer>('searchIndexer')\n } catch {\n searchDebugWarn('fulltext-index.worker', 'searchIndexer not available')\n }\n\n // Resolve fulltext strategy\n let fulltextStrategy: FullTextSearchStrategy | undefined\n try {\n const searchStrategies = ctx.resolve<unknown[]>('searchStrategies')\n fulltextStrategy = searchStrategies?.find(\n (s: unknown) => (s as { id?: string })?.id === 'fulltext',\n ) as FullTextSearchStrategy | undefined\n } catch {\n searchDebugWarn('fulltext-index.worker', 'searchStrategies not available')\n return\n }\n\n if (!fulltextStrategy) {\n searchDebugWarn('fulltext-index.worker', 'Fulltext strategy not configured')\n return\n }\n\n // Check if fulltext is available\n const isAvailable = await fulltextStrategy.isAvailable()\n if (!isAvailable) {\n throw new Error('Fulltext search is not available') // Will trigger retry\n }\n\n try {\n // ========== SINGLE INDEX: Use searchIndexer.indexRecordById() for fresh data ==========\n if (jobType === 'index') {\n const { entityType, recordId, organizationId } = job.payload as {\n entityType: string\n recordId: string\n organizationId?: string | null\n }\n\n if (!entityType || !recordId) {\n searchDebugWarn('fulltext-index.worker', 'Skipping index with missing fields', {\n jobId: jobCtx.jobId,\n entityType,\n recordId,\n })\n return\n }\n\n if (!searchIndexer) {\n throw new Error('searchIndexer not available for single-record index')\n }\n\n const result = await searchIndexer.indexRecordById({\n entityId: entityType as EntityId,\n recordId,\n tenantId,\n organizationId,\n })\n\n searchDebug('fulltext-index.worker', 'Indexed single record to fulltext', {\n jobId: jobCtx.jobId,\n tenantId,\n entityType,\n recordId,\n action: result.action,\n })\n\n await recordIndexerLog(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: 'worker:fulltext:index',\n message: `Indexed record to fulltext (${result.action})`,\n entityType,\n recordId,\n tenantId,\n details: { jobId: jobCtx.jobId },\n },\n )\n return\n }\n\n // ========== BATCH-INDEX: Use searchIndexer.indexRecordById() for fresh data ==========\n if (jobType === 'batch-index') {\n const { records, organizationId } = job.payload\n if (!records || records.length === 0) {\n searchDebugWarn('fulltext-index.worker', 'Skipping batch-index with no records', {\n jobId: jobCtx.jobId,\n })\n return\n }\n\n if (!searchIndexer) {\n throw new Error('searchIndexer not available for batch indexing')\n }\n\n // Process each record using indexRecordById (same pattern as vector worker)\n let successCount = 0\n let failCount = 0\n\n for (const { entityId, recordId } of records) {\n try {\n const result = await searchIndexer.indexRecordById({\n entityId: entityId as EntityId,\n recordId,\n tenantId,\n organizationId,\n })\n if (result.action === 'indexed') {\n successCount++\n }\n } catch (error) {\n failCount++\n searchDebugWarn('fulltext-index.worker', 'Failed to index record in batch', {\n entityId,\n recordId,\n error: error instanceof Error ? error.message : error,\n })\n }\n }\n\n // Update heartbeat to signal worker is still processing\n if (knex && successCount > 0) {\n await updateReindexProgress(knex, tenantId, 'fulltext', successCount, organizationId ?? null)\n }\n\n searchDebug('fulltext-index.worker', 'Batch indexed to fulltext', {\n jobId: jobCtx.jobId,\n tenantId,\n requestedCount: records.length,\n successCount,\n failCount,\n })\n\n await recordIndexerLog(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: 'worker:fulltext:batch-index',\n message: `Indexed ${successCount}/${records.length} records to fulltext`,\n tenantId,\n details: { jobId: jobCtx.jobId, requestedCount: records.length, successCount, failCount },\n },\n )\n return\n }\n\n // ========== DELETE ==========\n if (jobType === 'delete') {\n const { entityId, recordId } = job.payload\n if (!entityId || !recordId) {\n searchDebugWarn('fulltext-index.worker', 'Skipping delete with missing fields', {\n jobId: jobCtx.jobId,\n entityId,\n recordId,\n })\n return\n }\n\n await fulltextStrategy.delete(entityId, recordId, tenantId)\n\n searchDebug('fulltext-index.worker', 'Deleted from fulltext', {\n jobId: jobCtx.jobId,\n tenantId,\n entityId,\n recordId,\n })\n\n await recordIndexerLog(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: 'worker:fulltext:delete',\n message: `Deleted record from fulltext`,\n entityType: entityId,\n recordId,\n tenantId,\n details: { jobId: jobCtx.jobId },\n },\n )\n return\n }\n\n // ========== PURGE ==========\n if (jobType === 'purge') {\n const { entityId } = job.payload\n if (!entityId) {\n searchDebugWarn('fulltext-index.worker', 'Skipping purge with missing entityId', {\n jobId: jobCtx.jobId,\n })\n return\n }\n\n await fulltextStrategy.purge(entityId, tenantId)\n\n searchDebug('fulltext-index.worker', 'Purged entity from fulltext', {\n jobId: jobCtx.jobId,\n tenantId,\n entityId,\n })\n\n await recordIndexerLog(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: 'worker:fulltext:purge',\n message: `Purged entity from fulltext`,\n entityType: entityId,\n tenantId,\n details: { jobId: jobCtx.jobId },\n },\n )\n return\n }\n } catch (error) {\n searchError('fulltext-index.worker', `Failed to ${jobType}`, {\n jobId: jobCtx.jobId,\n tenantId,\n error: error instanceof Error ? error.message : error,\n attemptNumber: jobCtx.attemptNumber,\n })\n\n const entityId = 'entityId' in job.payload ? job.payload.entityId :\n 'entityType' in job.payload ? (job.payload as { entityType?: string }).entityType : undefined\n const recordId = 'recordId' in job.payload ? job.payload.recordId : undefined\n\n await recordIndexerError(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: `worker:fulltext:${jobType}`,\n error,\n entityType: entityId,\n recordId,\n tenantId,\n payload: job.payload,\n },\n )\n\n // Re-throw to let the queue handle retry logic\n throw error\n }\n}\n\n/**\n * Default export for worker auto-discovery.\n * Wraps handleFulltextIndexJob to match the expected handler signature.\n */\nexport default async function handle(\n job: QueuedJob<FulltextIndexJobPayload>,\n ctx: JobContext & HandlerContext\n): Promise<void> {\n return handleFulltextIndexJob(job, ctx, ctx)\n}\n"],
|
|
5
|
-
"mappings": "AACA,SAAS,oCAAkE;AAM3E,SAAS,wBAAwB;AACjC,SAAS,0BAA0B;
|
|
4
|
+
"sourcesContent": ["import type { QueuedJob, JobContext, WorkerMeta } from '@open-mercato/queue'\nimport { FULLTEXT_INDEXING_QUEUE_NAME, type FulltextIndexJobPayload } from '../../../queue/fulltext-indexing'\nimport type { FullTextSearchStrategy } from '../../../strategies/fulltext.strategy'\nimport type { SearchIndexer } from '../../../indexer/search-indexer'\nimport type { EntityManager } from '@mikro-orm/postgresql'\nimport type { Knex } from 'knex'\nimport type { EntityId } from '@open-mercato/shared/modules/entities'\nimport { recordIndexerLog } from '@open-mercato/shared/lib/indexers/status-log'\nimport { recordIndexerError } from '@open-mercato/shared/lib/indexers/error-log'\nimport type { ProgressService } from '@open-mercato/core/modules/progress/lib/progressService'\nimport { searchDebug, searchDebugWarn, searchError } from '../../../lib/debug'\nimport { clearReindexLock, updateReindexProgress } from '../lib/reindex-lock'\nimport { incrementReindexProgress } from '../lib/reindex-progress'\n\n// Worker metadata for auto-discovery\nconst DEFAULT_CONCURRENCY = 2\nconst envConcurrency = process.env.WORKERS_FULLTEXT_INDEXING_CONCURRENCY\n\nexport const metadata: WorkerMeta = {\n queue: FULLTEXT_INDEXING_QUEUE_NAME,\n concurrency: envConcurrency ? parseInt(envConcurrency, 10) : DEFAULT_CONCURRENCY,\n}\n\ntype HandlerContext = { resolve: <T = unknown>(name: string) => T }\n\n/**\n * Process a fulltext indexing job.\n *\n * This handler processes single record indexing, batch indexing, deletion, and purge\n * operations for the fulltext search strategy.\n *\n * All indexing operations (single and batch) use searchIndexer.indexRecordById() to load\n * fresh data, ensuring consistency with the vector worker pattern.\n *\n * @param job - The queued job containing payload\n * @param jobCtx - Queue job context with job ID and attempt info\n * @param ctx - DI container context for resolving services\n */\nexport async function handleFulltextIndexJob(\n job: QueuedJob<FulltextIndexJobPayload>,\n jobCtx: JobContext,\n ctx: HandlerContext,\n): Promise<void> {\n const { jobType, tenantId } = job.payload\n\n if (!tenantId) {\n searchDebugWarn('fulltext-index.worker', 'Skipping job with missing tenantId', {\n jobId: jobCtx.jobId,\n jobType,\n })\n return\n }\n\n // Resolve EntityManager for logging and knex for database queries\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n let em: any | null = null\n let knex: Knex | null = null\n try {\n em = ctx.resolve('em') as EntityManager\n knex = (em.getConnection() as unknown as { getKnex: () => Knex }).getKnex()\n } catch {\n em = null\n knex = null\n }\n\n // Resolve searchIndexer for loading fresh data\n let searchIndexer: SearchIndexer | undefined\n try {\n searchIndexer = ctx.resolve<SearchIndexer>('searchIndexer')\n } catch {\n searchDebugWarn('fulltext-index.worker', 'searchIndexer not available')\n }\n\n // Resolve fulltext strategy\n let fulltextStrategy: FullTextSearchStrategy | undefined\n try {\n const searchStrategies = ctx.resolve<unknown[]>('searchStrategies')\n fulltextStrategy = searchStrategies?.find(\n (s: unknown) => (s as { id?: string })?.id === 'fulltext',\n ) as FullTextSearchStrategy | undefined\n } catch {\n searchDebugWarn('fulltext-index.worker', 'searchStrategies not available')\n return\n }\n\n if (!fulltextStrategy) {\n searchDebugWarn('fulltext-index.worker', 'Fulltext strategy not configured')\n return\n }\n\n // Check if fulltext is available\n const isAvailable = await fulltextStrategy.isAvailable()\n if (!isAvailable) {\n throw new Error('Fulltext search is not available') // Will trigger retry\n }\n\n try {\n let progressService: ProgressService | null = null\n try {\n progressService = ctx.resolve<ProgressService>('progressService')\n } catch {\n progressService = null\n }\n\n // ========== SINGLE INDEX: Use searchIndexer.indexRecordById() for fresh data ==========\n if (jobType === 'index') {\n const { entityType, recordId, organizationId } = job.payload as {\n entityType: string\n recordId: string\n organizationId?: string | null\n }\n\n if (!entityType || !recordId) {\n searchDebugWarn('fulltext-index.worker', 'Skipping index with missing fields', {\n jobId: jobCtx.jobId,\n entityType,\n recordId,\n })\n return\n }\n\n if (!searchIndexer) {\n throw new Error('searchIndexer not available for single-record index')\n }\n\n const result = await searchIndexer.indexRecordById({\n entityId: entityType as EntityId,\n recordId,\n tenantId,\n organizationId,\n })\n\n searchDebug('fulltext-index.worker', 'Indexed single record to fulltext', {\n jobId: jobCtx.jobId,\n tenantId,\n entityType,\n recordId,\n action: result.action,\n })\n\n await recordIndexerLog(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: 'worker:fulltext:index',\n message: `Indexed record to fulltext (${result.action})`,\n entityType,\n recordId,\n tenantId,\n details: { jobId: jobCtx.jobId },\n },\n )\n return\n }\n\n // ========== BATCH-INDEX: Use searchIndexer.indexRecordById() for fresh data ==========\n if (jobType === 'batch-index') {\n const { records, organizationId } = job.payload\n if (!records || records.length === 0) {\n searchDebugWarn('fulltext-index.worker', 'Skipping batch-index with no records', {\n jobId: jobCtx.jobId,\n })\n return\n }\n\n if (!searchIndexer) {\n throw new Error('searchIndexer not available for batch indexing')\n }\n\n // Process each record using indexRecordById (same pattern as vector worker)\n let successCount = 0\n let failCount = 0\n\n for (const { entityId, recordId } of records) {\n try {\n const result = await searchIndexer.indexRecordById({\n entityId: entityId as EntityId,\n recordId,\n tenantId,\n organizationId,\n })\n if (result.action === 'indexed') {\n successCount++\n }\n } catch (error) {\n failCount++\n searchDebugWarn('fulltext-index.worker', 'Failed to index record in batch', {\n entityId,\n recordId,\n error: error instanceof Error ? error.message : error,\n })\n }\n }\n\n // Update heartbeat to signal worker is still processing\n if (knex && records.length > 0) {\n await updateReindexProgress(knex, tenantId, 'fulltext', successCount, organizationId ?? null)\n }\n if (progressService && em && records.length > 0) {\n const completed = await incrementReindexProgress({\n em,\n progressService,\n type: 'fulltext',\n tenantId,\n organizationId: organizationId ?? null,\n delta: successCount,\n })\n if (completed && knex) {\n await clearReindexLock(knex, tenantId, 'fulltext', organizationId ?? null)\n }\n }\n\n searchDebug('fulltext-index.worker', 'Batch indexed to fulltext', {\n jobId: jobCtx.jobId,\n tenantId,\n requestedCount: records.length,\n successCount,\n failCount,\n })\n\n await recordIndexerLog(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: 'worker:fulltext:batch-index',\n message: `Indexed ${successCount}/${records.length} records to fulltext`,\n tenantId,\n details: { jobId: jobCtx.jobId, requestedCount: records.length, successCount, failCount },\n },\n )\n return\n }\n\n // ========== DELETE ==========\n if (jobType === 'delete') {\n const { entityId, recordId } = job.payload\n if (!entityId || !recordId) {\n searchDebugWarn('fulltext-index.worker', 'Skipping delete with missing fields', {\n jobId: jobCtx.jobId,\n entityId,\n recordId,\n })\n return\n }\n\n await fulltextStrategy.delete(entityId, recordId, tenantId)\n\n searchDebug('fulltext-index.worker', 'Deleted from fulltext', {\n jobId: jobCtx.jobId,\n tenantId,\n entityId,\n recordId,\n })\n\n await recordIndexerLog(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: 'worker:fulltext:delete',\n message: `Deleted record from fulltext`,\n entityType: entityId,\n recordId,\n tenantId,\n details: { jobId: jobCtx.jobId },\n },\n )\n return\n }\n\n // ========== PURGE ==========\n if (jobType === 'purge') {\n const { entityId } = job.payload\n if (!entityId) {\n searchDebugWarn('fulltext-index.worker', 'Skipping purge with missing entityId', {\n jobId: jobCtx.jobId,\n })\n return\n }\n\n await fulltextStrategy.purge(entityId, tenantId)\n\n searchDebug('fulltext-index.worker', 'Purged entity from fulltext', {\n jobId: jobCtx.jobId,\n tenantId,\n entityId,\n })\n\n await recordIndexerLog(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: 'worker:fulltext:purge',\n message: `Purged entity from fulltext`,\n entityType: entityId,\n tenantId,\n details: { jobId: jobCtx.jobId },\n },\n )\n return\n }\n } catch (error) {\n searchError('fulltext-index.worker', `Failed to ${jobType}`, {\n jobId: jobCtx.jobId,\n tenantId,\n error: error instanceof Error ? error.message : error,\n attemptNumber: jobCtx.attemptNumber,\n })\n\n const entityId = 'entityId' in job.payload ? job.payload.entityId :\n 'entityType' in job.payload ? (job.payload as { entityType?: string }).entityType : undefined\n const recordId = 'recordId' in job.payload ? job.payload.recordId : undefined\n\n await recordIndexerError(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: `worker:fulltext:${jobType}`,\n error,\n entityType: entityId,\n recordId,\n tenantId,\n payload: job.payload,\n },\n )\n\n // Re-throw to let the queue handle retry logic\n throw error\n }\n}\n\n/**\n * Default export for worker auto-discovery.\n * Wraps handleFulltextIndexJob to match the expected handler signature.\n */\nexport default async function handle(\n job: QueuedJob<FulltextIndexJobPayload>,\n ctx: JobContext & HandlerContext\n): Promise<void> {\n return handleFulltextIndexJob(job, ctx, ctx)\n}\n"],
|
|
5
|
+
"mappings": "AACA,SAAS,oCAAkE;AAM3E,SAAS,wBAAwB;AACjC,SAAS,0BAA0B;AAEnC,SAAS,aAAa,iBAAiB,mBAAmB;AAC1D,SAAS,kBAAkB,6BAA6B;AACxD,SAAS,gCAAgC;AAGzC,MAAM,sBAAsB;AAC5B,MAAM,iBAAiB,QAAQ,IAAI;AAE5B,MAAM,WAAuB;AAAA,EAClC,OAAO;AAAA,EACP,aAAa,iBAAiB,SAAS,gBAAgB,EAAE,IAAI;AAC/D;AAiBA,eAAsB,uBACpB,KACA,QACA,KACe;AACf,QAAM,EAAE,SAAS,SAAS,IAAI,IAAI;AAElC,MAAI,CAAC,UAAU;AACb,oBAAgB,yBAAyB,sCAAsC;AAAA,MAC7E,OAAO,OAAO;AAAA,MACd;AAAA,IACF,CAAC;AACD;AAAA,EACF;AAIA,MAAI,KAAiB;AACrB,MAAI,OAAoB;AACxB,MAAI;AACF,SAAK,IAAI,QAAQ,IAAI;AACrB,WAAQ,GAAG,cAAc,EAAyC,QAAQ;AAAA,EAC5E,QAAQ;AACN,SAAK;AACL,WAAO;AAAA,EACT;AAGA,MAAI;AACJ,MAAI;AACF,oBAAgB,IAAI,QAAuB,eAAe;AAAA,EAC5D,QAAQ;AACN,oBAAgB,yBAAyB,6BAA6B;AAAA,EACxE;AAGA,MAAI;AACJ,MAAI;AACF,UAAM,mBAAmB,IAAI,QAAmB,kBAAkB;AAClE,uBAAmB,kBAAkB;AAAA,MACnC,CAAC,MAAgB,GAAuB,OAAO;AAAA,IACjD;AAAA,EACF,QAAQ;AACN,oBAAgB,yBAAyB,gCAAgC;AACzE;AAAA,EACF;AAEA,MAAI,CAAC,kBAAkB;AACrB,oBAAgB,yBAAyB,kCAAkC;AAC3E;AAAA,EACF;AAGA,QAAM,cAAc,MAAM,iBAAiB,YAAY;AACvD,MAAI,CAAC,aAAa;AAChB,UAAM,IAAI,MAAM,kCAAkC;AAAA,EACpD;AAEA,MAAI;AACF,QAAI,kBAA0C;AAC9C,QAAI;AACF,wBAAkB,IAAI,QAAyB,iBAAiB;AAAA,IAClE,QAAQ;AACN,wBAAkB;AAAA,IACpB;AAGA,QAAI,YAAY,SAAS;AACvB,YAAM,EAAE,YAAY,UAAU,eAAe,IAAI,IAAI;AAMrD,UAAI,CAAC,cAAc,CAAC,UAAU;AAC5B,wBAAgB,yBAAyB,sCAAsC;AAAA,UAC7E,OAAO,OAAO;AAAA,UACd;AAAA,UACA;AAAA,QACF,CAAC;AACD;AAAA,MACF;AAEA,UAAI,CAAC,eAAe;AAClB,cAAM,IAAI,MAAM,qDAAqD;AAAA,MACvE;AAEA,YAAM,SAAS,MAAM,cAAc,gBAAgB;AAAA,QACjD,UAAU;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AAED,kBAAY,yBAAyB,qCAAqC;AAAA,QACxE,OAAO,OAAO;AAAA,QACd;AAAA,QACA;AAAA,QACA;AAAA,QACA,QAAQ,OAAO;AAAA,MACjB,CAAC;AAED,YAAM;AAAA,QACJ,EAAE,IAAI,MAAM,OAAU;AAAA,QACtB;AAAA,UACE,QAAQ;AAAA,UACR,SAAS;AAAA,UACT,SAAS,+BAA+B,OAAO,MAAM;AAAA,UACrD;AAAA,UACA;AAAA,UACA;AAAA,UACA,SAAS,EAAE,OAAO,OAAO,MAAM;AAAA,QACjC;AAAA,MACF;AACA;AAAA,IACF;AAGA,QAAI,YAAY,eAAe;AAC7B,YAAM,EAAE,SAAS,eAAe,IAAI,IAAI;AACxC,UAAI,CAAC,WAAW,QAAQ,WAAW,GAAG;AACpC,wBAAgB,yBAAyB,wCAAwC;AAAA,UAC/E,OAAO,OAAO;AAAA,QAChB,CAAC;AACD;AAAA,MACF;AAEA,UAAI,CAAC,eAAe;AAClB,cAAM,IAAI,MAAM,gDAAgD;AAAA,MAClE;AAGA,UAAI,eAAe;AACnB,UAAI,YAAY;AAEhB,iBAAW,EAAE,UAAU,SAAS,KAAK,SAAS;AAC5C,YAAI;AACF,gBAAM,SAAS,MAAM,cAAc,gBAAgB;AAAA,YACjD;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,UACF,CAAC;AACD,cAAI,OAAO,WAAW,WAAW;AAC/B;AAAA,UACF;AAAA,QACF,SAAS,OAAO;AACd;AACA,0BAAgB,yBAAyB,mCAAmC;AAAA,YAC1E;AAAA,YACA;AAAA,YACA,OAAO,iBAAiB,QAAQ,MAAM,UAAU;AAAA,UAClD,CAAC;AAAA,QACH;AAAA,MACF;AAGA,UAAI,QAAQ,QAAQ,SAAS,GAAG;AAC9B,cAAM,sBAAsB,MAAM,UAAU,YAAY,cAAc,kBAAkB,IAAI;AAAA,MAC9F;AACA,UAAI,mBAAmB,MAAM,QAAQ,SAAS,GAAG;AAC/C,cAAM,YAAY,MAAM,yBAAyB;AAAA,UAC/C;AAAA,UACA;AAAA,UACA,MAAM;AAAA,UACN;AAAA,UACA,gBAAgB,kBAAkB;AAAA,UAClC,OAAO;AAAA,QACT,CAAC;AACD,YAAI,aAAa,MAAM;AACrB,gBAAM,iBAAiB,MAAM,UAAU,YAAY,kBAAkB,IAAI;AAAA,QAC3E;AAAA,MACF;AAEA,kBAAY,yBAAyB,6BAA6B;AAAA,QAChE,OAAO,OAAO;AAAA,QACd;AAAA,QACA,gBAAgB,QAAQ;AAAA,QACxB;AAAA,QACA;AAAA,MACF,CAAC;AAED,YAAM;AAAA,QACJ,EAAE,IAAI,MAAM,OAAU;AAAA,QACtB;AAAA,UACE,QAAQ;AAAA,UACR,SAAS;AAAA,UACT,SAAS,WAAW,YAAY,IAAI,QAAQ,MAAM;AAAA,UAClD;AAAA,UACA,SAAS,EAAE,OAAO,OAAO,OAAO,gBAAgB,QAAQ,QAAQ,cAAc,UAAU;AAAA,QAC1F;AAAA,MACF;AACA;AAAA,IACF;AAGA,QAAI,YAAY,UAAU;AACxB,YAAM,EAAE,UAAU,SAAS,IAAI,IAAI;AACnC,UAAI,CAAC,YAAY,CAAC,UAAU;AAC1B,wBAAgB,yBAAyB,uCAAuC;AAAA,UAC9E,OAAO,OAAO;AAAA,UACd;AAAA,UACA;AAAA,QACF,CAAC;AACD;AAAA,MACF;AAEA,YAAM,iBAAiB,OAAO,UAAU,UAAU,QAAQ;AAE1D,kBAAY,yBAAyB,yBAAyB;AAAA,QAC5D,OAAO,OAAO;AAAA,QACd;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AAED,YAAM;AAAA,QACJ,EAAE,IAAI,MAAM,OAAU;AAAA,QACtB;AAAA,UACE,QAAQ;AAAA,UACR,SAAS;AAAA,UACT,SAAS;AAAA,UACT,YAAY;AAAA,UACZ;AAAA,UACA;AAAA,UACA,SAAS,EAAE,OAAO,OAAO,MAAM;AAAA,QACjC;AAAA,MACF;AACA;AAAA,IACF;AAGA,QAAI,YAAY,SAAS;AACvB,YAAM,EAAE,SAAS,IAAI,IAAI;AACzB,UAAI,CAAC,UAAU;AACb,wBAAgB,yBAAyB,wCAAwC;AAAA,UAC/E,OAAO,OAAO;AAAA,QAChB,CAAC;AACD;AAAA,MACF;AAEA,YAAM,iBAAiB,MAAM,UAAU,QAAQ;AAE/C,kBAAY,yBAAyB,+BAA+B;AAAA,QAClE,OAAO,OAAO;AAAA,QACd;AAAA,QACA;AAAA,MACF,CAAC;AAED,YAAM;AAAA,QACJ,EAAE,IAAI,MAAM,OAAU;AAAA,QACtB;AAAA,UACE,QAAQ;AAAA,UACR,SAAS;AAAA,UACT,SAAS;AAAA,UACT,YAAY;AAAA,UACZ;AAAA,UACA,SAAS,EAAE,OAAO,OAAO,MAAM;AAAA,QACjC;AAAA,MACF;AACA;AAAA,IACF;AAAA,EACF,SAAS,OAAO;AACd,gBAAY,yBAAyB,aAAa,OAAO,IAAI;AAAA,MAC3D,OAAO,OAAO;AAAA,MACd;AAAA,MACA,OAAO,iBAAiB,QAAQ,MAAM,UAAU;AAAA,MAChD,eAAe,OAAO;AAAA,IACxB,CAAC;AAED,UAAM,WAAW,cAAc,IAAI,UAAU,IAAI,QAAQ,WACxC,gBAAgB,IAAI,UAAW,IAAI,QAAoC,aAAa;AACrG,UAAM,WAAW,cAAc,IAAI,UAAU,IAAI,QAAQ,WAAW;AAEpE,UAAM;AAAA,MACJ,EAAE,IAAI,MAAM,OAAU;AAAA,MACtB;AAAA,QACE,QAAQ;AAAA,QACR,SAAS,mBAAmB,OAAO;AAAA,QACnC;AAAA,QACA,YAAY;AAAA,QACZ;AAAA,QACA;AAAA,QACA,SAAS,IAAI;AAAA,MACf;AAAA,IACF;AAGA,UAAM;AAAA,EACR;AACF;AAMA,eAAO,OACL,KACA,KACe;AACf,SAAO,uBAAuB,KAAK,KAAK,GAAG;AAC7C;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
|
@@ -5,7 +5,8 @@ import { logVectorOperation } from "../../../vector/lib/vector-logs.js";
|
|
|
5
5
|
import { resolveAutoIndexingEnabled } from "../lib/auto-indexing.js";
|
|
6
6
|
import { resolveEmbeddingConfig } from "../lib/embedding-config.js";
|
|
7
7
|
import { searchDebugWarn } from "../../../lib/debug.js";
|
|
8
|
-
import { updateReindexProgress } from "../lib/reindex-lock.js";
|
|
8
|
+
import { clearReindexLock, updateReindexProgress } from "../lib/reindex-lock.js";
|
|
9
|
+
import { incrementReindexProgress } from "../lib/reindex-progress.js";
|
|
9
10
|
const DEFAULT_CONCURRENCY = 2;
|
|
10
11
|
const envConcurrency = process.env.WORKERS_VECTOR_INDEXING_CONCURRENCY;
|
|
11
12
|
const metadata = {
|
|
@@ -31,11 +32,19 @@ async function handleVectorIndexJob(job, jobCtx, ctx) {
|
|
|
31
32
|
return;
|
|
32
33
|
}
|
|
33
34
|
let knex = null;
|
|
35
|
+
let em2 = null;
|
|
34
36
|
try {
|
|
35
|
-
|
|
37
|
+
em2 = ctx.resolve("em");
|
|
36
38
|
knex = em2.getConnection().getKnex();
|
|
37
39
|
} catch {
|
|
38
40
|
knex = null;
|
|
41
|
+
em2 = null;
|
|
42
|
+
}
|
|
43
|
+
let progressService = null;
|
|
44
|
+
try {
|
|
45
|
+
progressService = ctx.resolve("progressService");
|
|
46
|
+
} catch {
|
|
47
|
+
progressService = null;
|
|
39
48
|
}
|
|
40
49
|
try {
|
|
41
50
|
const embeddingConfig = await resolveEmbeddingConfig(ctx, { defaultValue: null });
|
|
@@ -70,9 +79,22 @@ async function handleVectorIndexJob(job, jobCtx, ctx) {
|
|
|
70
79
|
});
|
|
71
80
|
}
|
|
72
81
|
}
|
|
73
|
-
if (knex &&
|
|
82
|
+
if (knex && records.length > 0) {
|
|
74
83
|
await updateReindexProgress(knex, tenantId, "vector", successCount, organizationId ?? null);
|
|
75
84
|
}
|
|
85
|
+
if (progressService && em2 && records.length > 0) {
|
|
86
|
+
const completed = await incrementReindexProgress({
|
|
87
|
+
em: em2,
|
|
88
|
+
progressService,
|
|
89
|
+
type: "vector",
|
|
90
|
+
tenantId,
|
|
91
|
+
organizationId: organizationId ?? null,
|
|
92
|
+
delta: successCount
|
|
93
|
+
});
|
|
94
|
+
if (completed && knex) {
|
|
95
|
+
await clearReindexLock(knex, tenantId, "vector", organizationId ?? null);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
76
98
|
searchDebugWarn("vector-index.worker", "Batch-index job completed", {
|
|
77
99
|
jobId: jobCtx.jobId,
|
|
78
100
|
totalRecords: records.length,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../../src/modules/search/workers/vector-index.worker.ts"],
|
|
4
|
-
"sourcesContent": ["import type { QueuedJob, JobContext, WorkerMeta } from '@open-mercato/queue'\nimport { VECTOR_INDEXING_QUEUE_NAME, type VectorIndexJobPayload } from '../../../queue/vector-indexing'\nimport type { SearchIndexer } from '../../../indexer/search-indexer'\nimport type { EmbeddingService } from '../../../vector'\nimport type { EntityManager } from '@mikro-orm/postgresql'\nimport type { Knex } from 'knex'\nimport { recordIndexerError } from '@open-mercato/shared/lib/indexers/error-log'\nimport { applyCoverageAdjustments, createCoverageAdjustments } from '@open-mercato/core/modules/query_index/lib/coverage'\nimport { logVectorOperation } from '../../../vector/lib/vector-logs'\nimport { resolveAutoIndexingEnabled } from '../lib/auto-indexing'\nimport { resolveEmbeddingConfig } from '../lib/embedding-config'\nimport { searchDebugWarn } from '../../../lib/debug'\nimport { updateReindexProgress } from '../lib/reindex-lock'\n\n// Worker metadata for auto-discovery\nconst DEFAULT_CONCURRENCY = 2\nconst envConcurrency = process.env.WORKERS_VECTOR_INDEXING_CONCURRENCY\n\nexport const metadata: WorkerMeta = {\n queue: VECTOR_INDEXING_QUEUE_NAME,\n concurrency: envConcurrency ? parseInt(envConcurrency, 10) : DEFAULT_CONCURRENCY,\n}\n\ntype HandlerContext = { resolve: <T = unknown>(name: string) => T }\n\n/**\n * Process a vector index job.\n *\n * This handler is called by the queue worker to process indexing and deletion jobs.\n * It uses SearchIndexer to load records and index them via SearchService.\n *\n * @param job - The queued job containing payload\n * @param jobCtx - Queue job context with job ID and attempt info\n * @param ctx - DI container context for resolving services\n */\nexport async function handleVectorIndexJob(\n job: QueuedJob<VectorIndexJobPayload>,\n jobCtx: JobContext,\n ctx: HandlerContext,\n): Promise<void> {\n const { jobType, entityType, recordId, tenantId, organizationId, records } = job.payload\n\n // Handle batch-index jobs (from reindex operations)\n if (jobType === 'batch-index') {\n if (!records?.length || !tenantId) {\n searchDebugWarn('vector-index.worker', 'Skipping batch-index job with missing required fields', {\n jobId: jobCtx.jobId,\n recordCount: records?.length ?? 0,\n tenantId,\n })\n return\n }\n\n let searchIndexer: SearchIndexer\n try {\n searchIndexer = ctx.resolve<SearchIndexer>('searchIndexer')\n } catch {\n searchDebugWarn('vector-index.worker', 'searchIndexer not available')\n return\n }\n\n // Get knex for heartbeat updates\n let knex: Knex | null = null\n try {\n const em = ctx.resolve('em') as EntityManager\n knex = (em.getConnection() as unknown as { getKnex: () => Knex }).getKnex()\n } catch {\n knex = null\n }\n\n // Load saved embedding config to use the correct provider/model\n try {\n const embeddingConfig = await resolveEmbeddingConfig(ctx, { defaultValue: null })\n if (embeddingConfig) {\n const embeddingService = ctx.resolve<EmbeddingService>('vectorEmbeddingService')\n embeddingService.updateConfig(embeddingConfig)\n }\n } catch (configErr) {\n searchDebugWarn('vector-index.worker', 'Failed to load embedding config for batch, using defaults', {\n error: configErr instanceof Error ? configErr.message : configErr,\n })\n }\n\n // Process each record in the batch\n let successCount = 0\n let failCount = 0\n for (const { entityId, recordId: recId } of records) {\n try {\n const result = await searchIndexer.indexRecordById({\n entityId: entityId as Parameters<typeof searchIndexer.indexRecordById>[0]['entityId'],\n recordId: recId,\n tenantId,\n organizationId,\n })\n if (result.action === 'indexed') {\n successCount++\n }\n } catch (error) {\n failCount++\n searchDebugWarn('vector-index.worker', 'Failed to index record in batch', {\n entityId,\n recordId: recId,\n error: error instanceof Error ? error.message : error,\n })\n }\n }\n\n // Update heartbeat to signal worker is still processing\n if (knex && successCount > 0) {\n await updateReindexProgress(knex, tenantId, 'vector', successCount, organizationId ?? null)\n }\n\n searchDebugWarn('vector-index.worker', 'Batch-index job completed', {\n jobId: jobCtx.jobId,\n totalRecords: records.length,\n successCount,\n failCount,\n })\n return\n }\n\n // Handle single record jobs (index/delete)\n if (!entityType || !recordId || !tenantId) {\n searchDebugWarn('vector-index.worker', 'Skipping job with missing required fields', {\n jobId: jobCtx.jobId,\n entityType,\n recordId,\n tenantId,\n })\n return\n }\n\n const autoIndexingEnabled = await resolveAutoIndexingEnabled(ctx, { defaultValue: true })\n if (!autoIndexingEnabled) {\n return\n }\n\n let searchIndexer: SearchIndexer\n try {\n searchIndexer = ctx.resolve<SearchIndexer>('searchIndexer')\n } catch {\n searchDebugWarn('vector-index.worker', 'searchIndexer not available')\n return\n }\n\n // Load saved embedding config to use the correct provider/model\n try {\n const embeddingConfig = await resolveEmbeddingConfig(ctx, { defaultValue: null })\n if (embeddingConfig) {\n const embeddingService = ctx.resolve<EmbeddingService>('vectorEmbeddingService')\n embeddingService.updateConfig(embeddingConfig)\n }\n } catch (configErr) {\n // Delete operations don't require embedding, only warn for index operations\n if (jobType === 'index') {\n searchDebugWarn('vector-index.worker', 'Failed to load embedding config, using defaults', {\n error: configErr instanceof Error ? configErr.message : configErr,\n })\n }\n }\n\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n let em: any | null = null\n try {\n em = ctx.resolve('em')\n } catch {\n em = null\n }\n\n let eventBus: { emitEvent(event: string, payload: unknown, options?: unknown): Promise<void> } | null = null\n try {\n eventBus = ctx.resolve('eventBus')\n } catch {\n eventBus = null\n }\n\n const handlerName = jobType === 'delete'\n ? 'worker:vector-indexing:delete'\n : 'worker:vector-indexing:index'\n\n try {\n let action: 'indexed' | 'deleted' | 'skipped' = 'skipped'\n let delta = 0\n\n if (jobType === 'delete') {\n await searchIndexer.deleteRecord({\n entityId: entityType,\n recordId,\n tenantId,\n })\n action = 'deleted'\n delta = -1\n } else {\n const result = await searchIndexer.indexRecordById({\n entityId: entityType,\n recordId,\n tenantId,\n organizationId,\n })\n action = result.action\n if (result.action === 'indexed') {\n delta = 1\n }\n }\n\n if (delta !== 0) {\n let adjustmentsApplied = false\n if (em) {\n try {\n const adjustments = createCoverageAdjustments({\n entityType,\n tenantId,\n organizationId,\n baseDelta: 0,\n indexDelta: 0,\n vectorDelta: delta,\n })\n if (adjustments.length) {\n await applyCoverageAdjustments(em, adjustments)\n adjustmentsApplied = true\n }\n } catch (coverageError) {\n searchDebugWarn('vector-index.worker', 'Failed to adjust vector coverage', {\n error: coverageError instanceof Error ? coverageError.message : coverageError,\n })\n }\n }\n\n if (!adjustmentsApplied && eventBus) {\n try {\n await eventBus.emitEvent('query_index.coverage.refresh', {\n entityType,\n tenantId,\n organizationId,\n withDeleted: false,\n delayMs: 1000,\n })\n } catch (emitError) {\n searchDebugWarn('vector-index.worker', 'Failed to enqueue coverage refresh', {\n error: emitError instanceof Error ? emitError.message : emitError,\n })\n }\n }\n }\n\n await logVectorOperation({\n em,\n handler: handlerName,\n entityType,\n recordId,\n result: {\n action,\n tenantId,\n organizationId: organizationId ?? null,\n created: action === 'indexed',\n existed: action === 'deleted',\n },\n })\n } catch (error) {\n searchDebugWarn('vector-index.worker', `Failed to ${jobType} vector index`, {\n entityType,\n recordId,\n error: error instanceof Error ? error.message : error,\n })\n await recordIndexerError(\n { em: em ?? undefined },\n {\n source: 'vector',\n handler: handlerName,\n error,\n entityType,\n recordId,\n tenantId,\n organizationId,\n payload: job.payload,\n },\n )\n // Re-throw to let the queue handle retry logic\n throw error\n }\n}\n\n/**\n * Default export for worker auto-discovery.\n * Wraps handleVectorIndexJob to match the expected handler signature.\n */\nexport default async function handle(\n job: QueuedJob<VectorIndexJobPayload>,\n ctx: JobContext & HandlerContext\n): Promise<void> {\n return handleVectorIndexJob(job, ctx, ctx)\n}\n"],
|
|
5
|
-
"mappings": "AACA,SAAS,kCAA8D;
|
|
4
|
+
"sourcesContent": ["import type { QueuedJob, JobContext, WorkerMeta } from '@open-mercato/queue'\nimport { VECTOR_INDEXING_QUEUE_NAME, type VectorIndexJobPayload } from '../../../queue/vector-indexing'\nimport type { SearchIndexer } from '../../../indexer/search-indexer'\nimport type { EmbeddingService } from '../../../vector'\nimport type { EntityManager } from '@mikro-orm/postgresql'\nimport type { Knex } from 'knex'\nimport type { ProgressService } from '@open-mercato/core/modules/progress/lib/progressService'\nimport { recordIndexerError } from '@open-mercato/shared/lib/indexers/error-log'\nimport { applyCoverageAdjustments, createCoverageAdjustments } from '@open-mercato/core/modules/query_index/lib/coverage'\nimport { logVectorOperation } from '../../../vector/lib/vector-logs'\nimport { resolveAutoIndexingEnabled } from '../lib/auto-indexing'\nimport { resolveEmbeddingConfig } from '../lib/embedding-config'\nimport { searchDebugWarn } from '../../../lib/debug'\nimport { clearReindexLock, updateReindexProgress } from '../lib/reindex-lock'\nimport { incrementReindexProgress } from '../lib/reindex-progress'\n\n// Worker metadata for auto-discovery\nconst DEFAULT_CONCURRENCY = 2\nconst envConcurrency = process.env.WORKERS_VECTOR_INDEXING_CONCURRENCY\n\nexport const metadata: WorkerMeta = {\n queue: VECTOR_INDEXING_QUEUE_NAME,\n concurrency: envConcurrency ? parseInt(envConcurrency, 10) : DEFAULT_CONCURRENCY,\n}\n\ntype HandlerContext = { resolve: <T = unknown>(name: string) => T }\n\n/**\n * Process a vector index job.\n *\n * This handler is called by the queue worker to process indexing and deletion jobs.\n * It uses SearchIndexer to load records and index them via SearchService.\n *\n * @param job - The queued job containing payload\n * @param jobCtx - Queue job context with job ID and attempt info\n * @param ctx - DI container context for resolving services\n */\nexport async function handleVectorIndexJob(\n job: QueuedJob<VectorIndexJobPayload>,\n jobCtx: JobContext,\n ctx: HandlerContext,\n): Promise<void> {\n const { jobType, entityType, recordId, tenantId, organizationId, records } = job.payload\n\n // Handle batch-index jobs (from reindex operations)\n if (jobType === 'batch-index') {\n if (!records?.length || !tenantId) {\n searchDebugWarn('vector-index.worker', 'Skipping batch-index job with missing required fields', {\n jobId: jobCtx.jobId,\n recordCount: records?.length ?? 0,\n tenantId,\n })\n return\n }\n\n let searchIndexer: SearchIndexer\n try {\n searchIndexer = ctx.resolve<SearchIndexer>('searchIndexer')\n } catch {\n searchDebugWarn('vector-index.worker', 'searchIndexer not available')\n return\n }\n\n // Get knex for heartbeat updates\n let knex: Knex | null = null\n let em: EntityManager | null = null\n try {\n em = ctx.resolve('em') as EntityManager\n knex = (em.getConnection() as unknown as { getKnex: () => Knex }).getKnex()\n } catch {\n knex = null\n em = null\n }\n\n let progressService: ProgressService | null = null\n try {\n progressService = ctx.resolve<ProgressService>('progressService')\n } catch {\n progressService = null\n }\n\n // Load saved embedding config to use the correct provider/model\n try {\n const embeddingConfig = await resolveEmbeddingConfig(ctx, { defaultValue: null })\n if (embeddingConfig) {\n const embeddingService = ctx.resolve<EmbeddingService>('vectorEmbeddingService')\n embeddingService.updateConfig(embeddingConfig)\n }\n } catch (configErr) {\n searchDebugWarn('vector-index.worker', 'Failed to load embedding config for batch, using defaults', {\n error: configErr instanceof Error ? configErr.message : configErr,\n })\n }\n\n // Process each record in the batch\n let successCount = 0\n let failCount = 0\n for (const { entityId, recordId: recId } of records) {\n try {\n const result = await searchIndexer.indexRecordById({\n entityId: entityId as Parameters<typeof searchIndexer.indexRecordById>[0]['entityId'],\n recordId: recId,\n tenantId,\n organizationId,\n })\n if (result.action === 'indexed') {\n successCount++\n }\n } catch (error) {\n failCount++\n searchDebugWarn('vector-index.worker', 'Failed to index record in batch', {\n entityId,\n recordId: recId,\n error: error instanceof Error ? error.message : error,\n })\n }\n }\n\n // Update heartbeat to signal worker is still processing\n if (knex && records.length > 0) {\n await updateReindexProgress(knex, tenantId, 'vector', successCount, organizationId ?? null)\n }\n if (progressService && em && records.length > 0) {\n const completed = await incrementReindexProgress({\n em,\n progressService,\n type: 'vector',\n tenantId,\n organizationId: organizationId ?? null,\n delta: successCount,\n })\n if (completed && knex) {\n await clearReindexLock(knex, tenantId, 'vector', organizationId ?? null)\n }\n }\n\n searchDebugWarn('vector-index.worker', 'Batch-index job completed', {\n jobId: jobCtx.jobId,\n totalRecords: records.length,\n successCount,\n failCount,\n })\n return\n }\n\n // Handle single record jobs (index/delete)\n if (!entityType || !recordId || !tenantId) {\n searchDebugWarn('vector-index.worker', 'Skipping job with missing required fields', {\n jobId: jobCtx.jobId,\n entityType,\n recordId,\n tenantId,\n })\n return\n }\n\n const autoIndexingEnabled = await resolveAutoIndexingEnabled(ctx, { defaultValue: true })\n if (!autoIndexingEnabled) {\n return\n }\n\n let searchIndexer: SearchIndexer\n try {\n searchIndexer = ctx.resolve<SearchIndexer>('searchIndexer')\n } catch {\n searchDebugWarn('vector-index.worker', 'searchIndexer not available')\n return\n }\n\n // Load saved embedding config to use the correct provider/model\n try {\n const embeddingConfig = await resolveEmbeddingConfig(ctx, { defaultValue: null })\n if (embeddingConfig) {\n const embeddingService = ctx.resolve<EmbeddingService>('vectorEmbeddingService')\n embeddingService.updateConfig(embeddingConfig)\n }\n } catch (configErr) {\n // Delete operations don't require embedding, only warn for index operations\n if (jobType === 'index') {\n searchDebugWarn('vector-index.worker', 'Failed to load embedding config, using defaults', {\n error: configErr instanceof Error ? configErr.message : configErr,\n })\n }\n }\n\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n let em: any | null = null\n try {\n em = ctx.resolve('em')\n } catch {\n em = null\n }\n\n let eventBus: { emitEvent(event: string, payload: unknown, options?: unknown): Promise<void> } | null = null\n try {\n eventBus = ctx.resolve('eventBus')\n } catch {\n eventBus = null\n }\n\n const handlerName = jobType === 'delete'\n ? 'worker:vector-indexing:delete'\n : 'worker:vector-indexing:index'\n\n try {\n let action: 'indexed' | 'deleted' | 'skipped' = 'skipped'\n let delta = 0\n\n if (jobType === 'delete') {\n await searchIndexer.deleteRecord({\n entityId: entityType,\n recordId,\n tenantId,\n })\n action = 'deleted'\n delta = -1\n } else {\n const result = await searchIndexer.indexRecordById({\n entityId: entityType,\n recordId,\n tenantId,\n organizationId,\n })\n action = result.action\n if (result.action === 'indexed') {\n delta = 1\n }\n }\n\n if (delta !== 0) {\n let adjustmentsApplied = false\n if (em) {\n try {\n const adjustments = createCoverageAdjustments({\n entityType,\n tenantId,\n organizationId,\n baseDelta: 0,\n indexDelta: 0,\n vectorDelta: delta,\n })\n if (adjustments.length) {\n await applyCoverageAdjustments(em, adjustments)\n adjustmentsApplied = true\n }\n } catch (coverageError) {\n searchDebugWarn('vector-index.worker', 'Failed to adjust vector coverage', {\n error: coverageError instanceof Error ? coverageError.message : coverageError,\n })\n }\n }\n\n if (!adjustmentsApplied && eventBus) {\n try {\n await eventBus.emitEvent('query_index.coverage.refresh', {\n entityType,\n tenantId,\n organizationId,\n withDeleted: false,\n delayMs: 1000,\n })\n } catch (emitError) {\n searchDebugWarn('vector-index.worker', 'Failed to enqueue coverage refresh', {\n error: emitError instanceof Error ? emitError.message : emitError,\n })\n }\n }\n }\n\n await logVectorOperation({\n em,\n handler: handlerName,\n entityType,\n recordId,\n result: {\n action,\n tenantId,\n organizationId: organizationId ?? null,\n created: action === 'indexed',\n existed: action === 'deleted',\n },\n })\n } catch (error) {\n searchDebugWarn('vector-index.worker', `Failed to ${jobType} vector index`, {\n entityType,\n recordId,\n error: error instanceof Error ? error.message : error,\n })\n await recordIndexerError(\n { em: em ?? undefined },\n {\n source: 'vector',\n handler: handlerName,\n error,\n entityType,\n recordId,\n tenantId,\n organizationId,\n payload: job.payload,\n },\n )\n // Re-throw to let the queue handle retry logic\n throw error\n }\n}\n\n/**\n * Default export for worker auto-discovery.\n * Wraps handleVectorIndexJob to match the expected handler signature.\n */\nexport default async function handle(\n job: QueuedJob<VectorIndexJobPayload>,\n ctx: JobContext & HandlerContext\n): Promise<void> {\n return handleVectorIndexJob(job, ctx, ctx)\n}\n"],
|
|
5
|
+
"mappings": "AACA,SAAS,kCAA8D;AAMvE,SAAS,0BAA0B;AACnC,SAAS,0BAA0B,iCAAiC;AACpE,SAAS,0BAA0B;AACnC,SAAS,kCAAkC;AAC3C,SAAS,8BAA8B;AACvC,SAAS,uBAAuB;AAChC,SAAS,kBAAkB,6BAA6B;AACxD,SAAS,gCAAgC;AAGzC,MAAM,sBAAsB;AAC5B,MAAM,iBAAiB,QAAQ,IAAI;AAE5B,MAAM,WAAuB;AAAA,EAClC,OAAO;AAAA,EACP,aAAa,iBAAiB,SAAS,gBAAgB,EAAE,IAAI;AAC/D;AAcA,eAAsB,qBACpB,KACA,QACA,KACe;AACf,QAAM,EAAE,SAAS,YAAY,UAAU,UAAU,gBAAgB,QAAQ,IAAI,IAAI;AAGjF,MAAI,YAAY,eAAe;AAC7B,QAAI,CAAC,SAAS,UAAU,CAAC,UAAU;AACjC,sBAAgB,uBAAuB,yDAAyD;AAAA,QAC9F,OAAO,OAAO;AAAA,QACd,aAAa,SAAS,UAAU;AAAA,QAChC;AAAA,MACF,CAAC;AACD;AAAA,IACF;AAEA,QAAIA;AACJ,QAAI;AACF,MAAAA,iBAAgB,IAAI,QAAuB,eAAe;AAAA,IAC5D,QAAQ;AACN,sBAAgB,uBAAuB,6BAA6B;AACpE;AAAA,IACF;AAGA,QAAI,OAAoB;AACxB,QAAIC,MAA2B;AAC/B,QAAI;AACF,MAAAA,MAAK,IAAI,QAAQ,IAAI;AACrB,aAAQA,IAAG,cAAc,EAAyC,QAAQ;AAAA,IAC5E,QAAQ;AACN,aAAO;AACP,MAAAA,MAAK;AAAA,IACP;AAEA,QAAI,kBAA0C;AAC9C,QAAI;AACF,wBAAkB,IAAI,QAAyB,iBAAiB;AAAA,IAClE,QAAQ;AACN,wBAAkB;AAAA,IACpB;AAGA,QAAI;AACF,YAAM,kBAAkB,MAAM,uBAAuB,KAAK,EAAE,cAAc,KAAK,CAAC;AAChF,UAAI,iBAAiB;AACnB,cAAM,mBAAmB,IAAI,QAA0B,wBAAwB;AAC/E,yBAAiB,aAAa,eAAe;AAAA,MAC/C;AAAA,IACF,SAAS,WAAW;AAClB,sBAAgB,uBAAuB,6DAA6D;AAAA,QAClG,OAAO,qBAAqB,QAAQ,UAAU,UAAU;AAAA,MAC1D,CAAC;AAAA,IACH;AAGA,QAAI,eAAe;AACnB,QAAI,YAAY;AAChB,eAAW,EAAE,UAAU,UAAU,MAAM,KAAK,SAAS;AACnD,UAAI;AACF,cAAM,SAAS,MAAMD,eAAc,gBAAgB;AAAA,UACjD;AAAA,UACA,UAAU;AAAA,UACV;AAAA,UACA;AAAA,QACF,CAAC;AACD,YAAI,OAAO,WAAW,WAAW;AAC/B;AAAA,QACF;AAAA,MACF,SAAS,OAAO;AACd;AACA,wBAAgB,uBAAuB,mCAAmC;AAAA,UACxE;AAAA,UACA,UAAU;AAAA,UACV,OAAO,iBAAiB,QAAQ,MAAM,UAAU;AAAA,QAClD,CAAC;AAAA,MACH;AAAA,IACF;AAGA,QAAI,QAAQ,QAAQ,SAAS,GAAG;AAC9B,YAAM,sBAAsB,MAAM,UAAU,UAAU,cAAc,kBAAkB,IAAI;AAAA,IAC5F;AACA,QAAI,mBAAmBC,OAAM,QAAQ,SAAS,GAAG;AAC/C,YAAM,YAAY,MAAM,yBAAyB;AAAA,QAC/C,IAAAA;AAAA,QACA;AAAA,QACA,MAAM;AAAA,QACN;AAAA,QACA,gBAAgB,kBAAkB;AAAA,QAClC,OAAO;AAAA,MACT,CAAC;AACD,UAAI,aAAa,MAAM;AACrB,cAAM,iBAAiB,MAAM,UAAU,UAAU,kBAAkB,IAAI;AAAA,MACzE;AAAA,IACF;AAEA,oBAAgB,uBAAuB,6BAA6B;AAAA,MAClE,OAAO,OAAO;AAAA,MACd,cAAc,QAAQ;AAAA,MACtB;AAAA,MACA;AAAA,IACF,CAAC;AACD;AAAA,EACF;AAGA,MAAI,CAAC,cAAc,CAAC,YAAY,CAAC,UAAU;AACzC,oBAAgB,uBAAuB,6CAA6C;AAAA,MAClF,OAAO,OAAO;AAAA,MACd;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AACD;AAAA,EACF;AAEA,QAAM,sBAAsB,MAAM,2BAA2B,KAAK,EAAE,cAAc,KAAK,CAAC;AACxF,MAAI,CAAC,qBAAqB;AACxB;AAAA,EACF;AAEA,MAAI;AACJ,MAAI;AACF,oBAAgB,IAAI,QAAuB,eAAe;AAAA,EAC5D,QAAQ;AACN,oBAAgB,uBAAuB,6BAA6B;AACpE;AAAA,EACF;AAGA,MAAI;AACF,UAAM,kBAAkB,MAAM,uBAAuB,KAAK,EAAE,cAAc,KAAK,CAAC;AAChF,QAAI,iBAAiB;AACnB,YAAM,mBAAmB,IAAI,QAA0B,wBAAwB;AAC/E,uBAAiB,aAAa,eAAe;AAAA,IAC/C;AAAA,EACF,SAAS,WAAW;AAElB,QAAI,YAAY,SAAS;AACvB,sBAAgB,uBAAuB,mDAAmD;AAAA,QACxF,OAAO,qBAAqB,QAAQ,UAAU,UAAU;AAAA,MAC1D,CAAC;AAAA,IACH;AAAA,EACF;AAGA,MAAI,KAAiB;AACrB,MAAI;AACF,SAAK,IAAI,QAAQ,IAAI;AAAA,EACvB,QAAQ;AACN,SAAK;AAAA,EACP;AAEA,MAAI,WAAoG;AACxG,MAAI;AACF,eAAW,IAAI,QAAQ,UAAU;AAAA,EACnC,QAAQ;AACN,eAAW;AAAA,EACb;AAEA,QAAM,cAAc,YAAY,WAC5B,kCACA;AAEJ,MAAI;AACF,QAAI,SAA4C;AAChD,QAAI,QAAQ;AAEZ,QAAI,YAAY,UAAU;AACxB,YAAM,cAAc,aAAa;AAAA,QAC/B,UAAU;AAAA,QACV;AAAA,QACA;AAAA,MACF,CAAC;AACD,eAAS;AACT,cAAQ;AAAA,IACV,OAAO;AACL,YAAM,SAAS,MAAM,cAAc,gBAAgB;AAAA,QACjD,UAAU;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AACD,eAAS,OAAO;AAChB,UAAI,OAAO,WAAW,WAAW;AAC/B,gBAAQ;AAAA,MACV;AAAA,IACF;AAEA,QAAI,UAAU,GAAG;AACf,UAAI,qBAAqB;AACzB,UAAI,IAAI;AACN,YAAI;AACF,gBAAM,cAAc,0BAA0B;AAAA,YAC5C;AAAA,YACA;AAAA,YACA;AAAA,YACA,WAAW;AAAA,YACX,YAAY;AAAA,YACZ,aAAa;AAAA,UACf,CAAC;AACD,cAAI,YAAY,QAAQ;AACtB,kBAAM,yBAAyB,IAAI,WAAW;AAC9C,iCAAqB;AAAA,UACvB;AAAA,QACF,SAAS,eAAe;AACtB,0BAAgB,uBAAuB,oCAAoC;AAAA,YACzE,OAAO,yBAAyB,QAAQ,cAAc,UAAU;AAAA,UAClE,CAAC;AAAA,QACH;AAAA,MACF;AAEA,UAAI,CAAC,sBAAsB,UAAU;AACnC,YAAI;AACF,gBAAM,SAAS,UAAU,gCAAgC;AAAA,YACvD;AAAA,YACA;AAAA,YACA;AAAA,YACA,aAAa;AAAA,YACb,SAAS;AAAA,UACX,CAAC;AAAA,QACH,SAAS,WAAW;AAClB,0BAAgB,uBAAuB,sCAAsC;AAAA,YAC3E,OAAO,qBAAqB,QAAQ,UAAU,UAAU;AAAA,UAC1D,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAEA,UAAM,mBAAmB;AAAA,MACvB;AAAA,MACA,SAAS;AAAA,MACT;AAAA,MACA;AAAA,MACA,QAAQ;AAAA,QACN;AAAA,QACA;AAAA,QACA,gBAAgB,kBAAkB;AAAA,QAClC,SAAS,WAAW;AAAA,QACpB,SAAS,WAAW;AAAA,MACtB;AAAA,IACF,CAAC;AAAA,EACH,SAAS,OAAO;AACd,oBAAgB,uBAAuB,aAAa,OAAO,iBAAiB;AAAA,MAC1E;AAAA,MACA;AAAA,MACA,OAAO,iBAAiB,QAAQ,MAAM,UAAU;AAAA,IAClD,CAAC;AACD,UAAM;AAAA,MACJ,EAAE,IAAI,MAAM,OAAU;AAAA,MACtB;AAAA,QACE,QAAQ;AAAA,QACR,SAAS;AAAA,QACT;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,SAAS,IAAI;AAAA,MACf;AAAA,IACF;AAEA,UAAM;AAAA,EACR;AACF;AAMA,eAAO,OACL,KACA,KACe;AACf,SAAO,qBAAqB,KAAK,KAAK,GAAG;AAC3C;",
|
|
6
6
|
"names": ["searchIndexer", "em"]
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@open-mercato/search",
|
|
3
|
-
"version": "0.4.6-develop-
|
|
3
|
+
"version": "0.4.6-develop-90c3eb0e8a",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"exports": {
|
|
@@ -126,9 +126,9 @@
|
|
|
126
126
|
"zod": "^4.0.0"
|
|
127
127
|
},
|
|
128
128
|
"peerDependencies": {
|
|
129
|
-
"@open-mercato/core": "0.4.6-develop-
|
|
130
|
-
"@open-mercato/queue": "0.4.6-develop-
|
|
131
|
-
"@open-mercato/shared": "0.4.6-develop-
|
|
129
|
+
"@open-mercato/core": "0.4.6-develop-90c3eb0e8a",
|
|
130
|
+
"@open-mercato/queue": "0.4.6-develop-90c3eb0e8a",
|
|
131
|
+
"@open-mercato/shared": "0.4.6-develop-90c3eb0e8a"
|
|
132
132
|
},
|
|
133
133
|
"devDependencies": {
|
|
134
134
|
"@types/jest": "^30.0.0",
|
|
@@ -7,10 +7,23 @@ The search module provides unified search capabilities across all entities in Op
|
|
|
7
7
|
- **Multi-strategy search**: Combines full-text search (Meilisearch), vector-based semantic search, and token matching
|
|
8
8
|
- **Automatic indexing**: Subscribes to entity events for real-time index updates
|
|
9
9
|
- **Queue-based processing**: Supports async batch processing via Redis/BullMQ for high-volume indexing
|
|
10
|
+
- **Topbar progress integration**: Fulltext and vector reindex operations create `progress_jobs` entries and stream updates to the global progress bar
|
|
10
11
|
- **Configurable embeddings**: Supports OpenAI, Ollama, and other embedding providers
|
|
11
12
|
- **Tenant-scoped**: All indexes are scoped by tenant and optionally by organization
|
|
12
13
|
- **Admin-configurable**: Global search (Cmd+K) strategies can be configured per-tenant
|
|
13
14
|
|
|
15
|
+
## Reindex Progress and SSE
|
|
16
|
+
|
|
17
|
+
Search reindex endpoints now integrate with the Progress module:
|
|
18
|
+
|
|
19
|
+
- `POST /api/search/reindex` (fulltext) creates/updates a progress job with `jobType = search.reindex.fulltext`
|
|
20
|
+
- `POST /api/search/embeddings/reindex` (vector) creates/updates a progress job with `jobType = search.reindex.vector`
|
|
21
|
+
- Queue workers increment progress as batches are processed and complete jobs when totals are reached
|
|
22
|
+
- The backend topbar (`ProgressTopBar`) receives real-time updates via SSE (`progress.job.*` events), so no periodic polling is required for progress updates
|
|
23
|
+
- Search settings sections (`FulltextSearchSection`, `VectorSearchSection`) now refresh logs/settings on `progress.job.*` and `om:bridge:reconnected` events instead of 5s polling loops
|
|
24
|
+
|
|
25
|
+
Cancellation endpoints (`/api/search/reindex/cancel`, `/api/search/embeddings/reindex/cancel`) now also cancel the corresponding progress job.
|
|
26
|
+
|
|
14
27
|
## Global Search Settings
|
|
15
28
|
|
|
16
29
|
The global search dialog (Cmd+K) can be configured by administrators to control which search strategies are used. This configuration is stored per-tenant.
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { expect, test } from '@playwright/test'
|
|
2
|
+
import { login } from '@open-mercato/core/modules/core/__integration__/helpers/auth'
|
|
3
|
+
|
|
4
|
+
type EventPayload = {
|
|
5
|
+
id: string
|
|
6
|
+
payload: Record<string, unknown>
|
|
7
|
+
timestamp: number
|
|
8
|
+
organizationId: string
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
test.describe('TC-SEARCH-001: search reindex progress SSE refresh behavior', () => {
|
|
12
|
+
test('refreshes search settings from progress SSE events and reconnect without periodic polling', async ({ page }) => {
|
|
13
|
+
const requestCounts = {
|
|
14
|
+
settings: 0,
|
|
15
|
+
embeddings: 0,
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
const onRequest = (rawRequest: { url: () => string; method: () => string }) => {
|
|
19
|
+
if (rawRequest.method() !== 'GET') return
|
|
20
|
+
const url = rawRequest.url()
|
|
21
|
+
if (url.includes('/api/search/settings')) {
|
|
22
|
+
requestCounts.settings += 1
|
|
23
|
+
}
|
|
24
|
+
if (url.includes('/api/search/embeddings')) {
|
|
25
|
+
requestCounts.embeddings += 1
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const emitAppEvent = async (detail: EventPayload) => {
|
|
30
|
+
await page.evaluate((eventDetail) => {
|
|
31
|
+
window.dispatchEvent(new CustomEvent('om:event', { detail: eventDetail }))
|
|
32
|
+
}, detail)
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
try {
|
|
36
|
+
await login(page, 'superadmin')
|
|
37
|
+
page.on('request', onRequest)
|
|
38
|
+
await page.goto('/backend/config/search')
|
|
39
|
+
await page.waitForLoadState('domcontentloaded')
|
|
40
|
+
await expect(page.getByRole('heading', { name: 'Search Settings' })).toBeVisible()
|
|
41
|
+
|
|
42
|
+
await page.waitForTimeout(2_000)
|
|
43
|
+
const baseline = { ...requestCounts }
|
|
44
|
+
|
|
45
|
+
await emitAppEvent({
|
|
46
|
+
id: 'progress.job.updated',
|
|
47
|
+
payload: {
|
|
48
|
+
jobId: 'job-fulltext',
|
|
49
|
+
jobType: 'search.reindex.fulltext',
|
|
50
|
+
status: 'running',
|
|
51
|
+
progressPercent: 10,
|
|
52
|
+
processedCount: 10,
|
|
53
|
+
totalCount: 100,
|
|
54
|
+
},
|
|
55
|
+
timestamp: Date.now(),
|
|
56
|
+
organizationId: 'org',
|
|
57
|
+
})
|
|
58
|
+
|
|
59
|
+
await expect.poll(() => requestCounts.settings, { timeout: 15_000 }).toBeGreaterThan(baseline.settings)
|
|
60
|
+
await expect.poll(() => requestCounts.embeddings, { timeout: 15_000 }).toBeGreaterThan(baseline.embeddings)
|
|
61
|
+
const afterProgressEvent = { ...requestCounts }
|
|
62
|
+
|
|
63
|
+
await page.waitForTimeout(6_000)
|
|
64
|
+
expect(requestCounts.settings).toBe(afterProgressEvent.settings)
|
|
65
|
+
expect(requestCounts.embeddings).toBe(afterProgressEvent.embeddings)
|
|
66
|
+
|
|
67
|
+
await emitAppEvent({
|
|
68
|
+
id: 'om:bridge:reconnected',
|
|
69
|
+
payload: {},
|
|
70
|
+
timestamp: Date.now(),
|
|
71
|
+
organizationId: 'org',
|
|
72
|
+
})
|
|
73
|
+
|
|
74
|
+
await expect.poll(() => requestCounts.settings, { timeout: 15_000 }).toBeGreaterThan(afterProgressEvent.settings)
|
|
75
|
+
await expect.poll(() => requestCounts.embeddings, { timeout: 15_000 }).toBeGreaterThan(afterProgressEvent.embeddings)
|
|
76
|
+
} finally {
|
|
77
|
+
page.off('request', onRequest)
|
|
78
|
+
}
|
|
79
|
+
})
|
|
80
|
+
})
|
|
@@ -4,7 +4,9 @@ import { getAuthFromRequest } from '@open-mercato/shared/lib/auth/server'
|
|
|
4
4
|
import type { Queue } from '@open-mercato/queue'
|
|
5
5
|
import type { Knex } from 'knex'
|
|
6
6
|
import type { EntityManager } from '@mikro-orm/postgresql'
|
|
7
|
+
import type { ProgressService } from '@open-mercato/core/modules/progress/lib/progressService'
|
|
7
8
|
import { clearReindexLock } from '../../../../lib/reindex-lock'
|
|
9
|
+
import { cancelReindexProgress } from '../../../../lib/reindex-progress'
|
|
8
10
|
import { resolveTranslations } from '@open-mercato/shared/lib/i18n/server'
|
|
9
11
|
import { recordIndexerLog } from '@open-mercato/shared/lib/indexers/status-log'
|
|
10
12
|
import { embeddingsReindexCancelOpenApi } from '../../../openapi'
|
|
@@ -22,6 +24,7 @@ export async function POST(req: Request) {
|
|
|
22
24
|
|
|
23
25
|
const container = await createRequestContainer()
|
|
24
26
|
const em = container.resolve('em') as EntityManager
|
|
27
|
+
const progressService = container.resolve('progressService') as ProgressService
|
|
25
28
|
const knex = (em.getConnection() as unknown as { getKnex: () => Knex }).getKnex()
|
|
26
29
|
|
|
27
30
|
let queue: Queue | undefined
|
|
@@ -43,6 +46,14 @@ export async function POST(req: Request) {
|
|
|
43
46
|
}
|
|
44
47
|
|
|
45
48
|
await clearReindexLock(knex, auth.tenantId, 'vector', auth.orgId ?? null)
|
|
49
|
+
await cancelReindexProgress({
|
|
50
|
+
em,
|
|
51
|
+
progressService,
|
|
52
|
+
type: 'vector',
|
|
53
|
+
tenantId: auth.tenantId,
|
|
54
|
+
organizationId: auth.orgId ?? null,
|
|
55
|
+
userId: auth.sub ?? null,
|
|
56
|
+
})
|
|
46
57
|
|
|
47
58
|
// Log the cancellation
|
|
48
59
|
try {
|
|
@@ -3,6 +3,7 @@ import { createRequestContainer } from '@open-mercato/shared/lib/di/container'
|
|
|
3
3
|
import { getAuthFromRequest } from '@open-mercato/shared/lib/auth/server'
|
|
4
4
|
import type { SearchIndexer } from '../../../../../indexer/search-indexer'
|
|
5
5
|
import type { EmbeddingService } from '../../../../../vector'
|
|
6
|
+
import type { ProgressService } from '@open-mercato/core/modules/progress/lib/progressService'
|
|
6
7
|
import type { Knex } from 'knex'
|
|
7
8
|
import type { EntityManager } from '@mikro-orm/postgresql'
|
|
8
9
|
import { recordIndexerLog } from '@open-mercato/shared/lib/indexers/status-log'
|
|
@@ -11,6 +12,10 @@ import { resolveEmbeddingConfig } from '../../../lib/embedding-config'
|
|
|
11
12
|
import type { EntityId } from '@open-mercato/shared/modules/entities'
|
|
12
13
|
import { searchDebug, searchDebugWarn, searchError } from '../../../../../lib/debug'
|
|
13
14
|
import { acquireReindexLock, clearReindexLock, getReindexLockStatus } from '../../../lib/reindex-lock'
|
|
15
|
+
import {
|
|
16
|
+
ensureReindexProgressJob,
|
|
17
|
+
failReindexProgress,
|
|
18
|
+
} from '../../../lib/reindex-progress'
|
|
14
19
|
import { embeddingsReindexOpenApi } from '../../openapi'
|
|
15
20
|
|
|
16
21
|
export const metadata = {
|
|
@@ -36,6 +41,7 @@ export async function POST(req: Request) {
|
|
|
36
41
|
|
|
37
42
|
const container = await createRequestContainer()
|
|
38
43
|
const em = container.resolve('em') as EntityManager
|
|
44
|
+
const progressService = container.resolve('progressService') as ProgressService
|
|
39
45
|
const knex = (em.getConnection() as unknown as { getKnex: () => Knex }).getKnex()
|
|
40
46
|
|
|
41
47
|
// Check if another vector reindex operation is already in progress
|
|
@@ -145,6 +151,19 @@ export async function POST(req: Request) {
|
|
|
145
151
|
})
|
|
146
152
|
}
|
|
147
153
|
|
|
154
|
+
await ensureReindexProgressJob({
|
|
155
|
+
em,
|
|
156
|
+
progressService,
|
|
157
|
+
type: 'vector',
|
|
158
|
+
tenantId: auth.tenantId,
|
|
159
|
+
organizationId: auth.orgId ?? null,
|
|
160
|
+
userId: auth.sub ?? null,
|
|
161
|
+
totalCount: result.recordsIndexed,
|
|
162
|
+
description: entityId
|
|
163
|
+
? `Vector reindex ${entityId} (queued)`
|
|
164
|
+
: 'Vector reindex all entities (queued)',
|
|
165
|
+
})
|
|
166
|
+
|
|
148
167
|
await recordIndexerLog(
|
|
149
168
|
{ em: em ?? undefined },
|
|
150
169
|
{
|
|
@@ -182,6 +201,14 @@ export async function POST(req: Request) {
|
|
|
182
201
|
stack: error instanceof Error ? error.stack : undefined,
|
|
183
202
|
status,
|
|
184
203
|
})
|
|
204
|
+
await failReindexProgress({
|
|
205
|
+
em,
|
|
206
|
+
progressService,
|
|
207
|
+
type: 'vector',
|
|
208
|
+
tenantId: auth.tenantId,
|
|
209
|
+
organizationId: auth.orgId ?? null,
|
|
210
|
+
errorMessage: error instanceof Error ? error.message : 'Vector reindex failed',
|
|
211
|
+
})
|
|
185
212
|
return NextResponse.json(
|
|
186
213
|
{ error: t('search.api.errors.reindexFailed', 'Vector reindex failed. Please try again or contact support.') },
|
|
187
214
|
{ status: status >= 400 ? status : 500 }
|
|
@@ -4,7 +4,9 @@ import { getAuthFromRequest } from '@open-mercato/shared/lib/auth/server'
|
|
|
4
4
|
import type { Queue } from '@open-mercato/queue'
|
|
5
5
|
import type { Knex } from 'knex'
|
|
6
6
|
import type { EntityManager } from '@mikro-orm/postgresql'
|
|
7
|
+
import type { ProgressService } from '@open-mercato/core/modules/progress/lib/progressService'
|
|
7
8
|
import { clearReindexLock } from '../../../lib/reindex-lock'
|
|
9
|
+
import { cancelReindexProgress } from '../../../lib/reindex-progress'
|
|
8
10
|
import { resolveTranslations } from '@open-mercato/shared/lib/i18n/server'
|
|
9
11
|
import { recordIndexerLog } from '@open-mercato/shared/lib/indexers/status-log'
|
|
10
12
|
import { reindexCancelOpenApi } from '../../openapi'
|
|
@@ -22,6 +24,7 @@ export async function POST(req: Request) {
|
|
|
22
24
|
|
|
23
25
|
const container = await createRequestContainer()
|
|
24
26
|
const em = container.resolve('em') as EntityManager
|
|
27
|
+
const progressService = container.resolve('progressService') as ProgressService
|
|
25
28
|
const knex = (em.getConnection() as unknown as { getKnex: () => Knex }).getKnex()
|
|
26
29
|
|
|
27
30
|
let queue: Queue | undefined
|
|
@@ -43,6 +46,14 @@ export async function POST(req: Request) {
|
|
|
43
46
|
}
|
|
44
47
|
|
|
45
48
|
await clearReindexLock(knex, auth.tenantId, 'fulltext', auth.orgId ?? null)
|
|
49
|
+
await cancelReindexProgress({
|
|
50
|
+
em,
|
|
51
|
+
progressService,
|
|
52
|
+
type: 'fulltext',
|
|
53
|
+
tenantId: auth.tenantId,
|
|
54
|
+
organizationId: auth.orgId ?? null,
|
|
55
|
+
userId: auth.sub ?? null,
|
|
56
|
+
})
|
|
46
57
|
|
|
47
58
|
// Log the cancellation
|
|
48
59
|
try {
|
|
@@ -7,6 +7,7 @@ import type { SearchIndexer } from '@open-mercato/search/indexer'
|
|
|
7
7
|
import type { EntityId } from '@open-mercato/shared/modules/entities'
|
|
8
8
|
import { recordIndexerLog } from '@open-mercato/shared/lib/indexers/status-log'
|
|
9
9
|
import { recordIndexerError } from '@open-mercato/shared/lib/indexers/error-log'
|
|
10
|
+
import type { ProgressService } from '@open-mercato/core/modules/progress/lib/progressService'
|
|
10
11
|
import type { EntityManager } from '@mikro-orm/postgresql'
|
|
11
12
|
import type { Knex } from 'knex'
|
|
12
13
|
import { searchDebug, searchError } from '../../../../lib/debug'
|
|
@@ -15,6 +16,11 @@ import {
|
|
|
15
16
|
clearReindexLock,
|
|
16
17
|
getReindexLockStatus,
|
|
17
18
|
} from '../../lib/reindex-lock'
|
|
19
|
+
import {
|
|
20
|
+
completeReindexProgress,
|
|
21
|
+
ensureReindexProgressJob,
|
|
22
|
+
failReindexProgress,
|
|
23
|
+
} from '../../lib/reindex-progress'
|
|
18
24
|
import { reindexOpenApi } from '../openapi'
|
|
19
25
|
|
|
20
26
|
/** Strategy with optional stats support */
|
|
@@ -84,6 +90,7 @@ export async function POST(req: Request) {
|
|
|
84
90
|
|
|
85
91
|
const container = await createRequestContainer()
|
|
86
92
|
const em = container.resolve('em') as EntityManager
|
|
93
|
+
const progressService = container.resolve('progressService') as ProgressService
|
|
87
94
|
const knex = (em.getConnection() as unknown as { getKnex: () => Knex }).getKnex()
|
|
88
95
|
|
|
89
96
|
// Check if another fulltext reindex operation is already in progress
|
|
@@ -301,6 +308,34 @@ export async function POST(req: Request) {
|
|
|
301
308
|
}
|
|
302
309
|
}
|
|
303
310
|
|
|
311
|
+
await ensureReindexProgressJob({
|
|
312
|
+
em,
|
|
313
|
+
progressService,
|
|
314
|
+
type: 'fulltext',
|
|
315
|
+
tenantId,
|
|
316
|
+
organizationId: auth.orgId ?? null,
|
|
317
|
+
userId: auth.sub ?? null,
|
|
318
|
+
totalCount: result.recordsIndexed,
|
|
319
|
+
description: entityId
|
|
320
|
+
? `Reindex ${entityId} (${useQueue ? 'queued' : 'sync'})`
|
|
321
|
+
: `Reindex all entities (${useQueue ? 'queued' : 'sync'})`,
|
|
322
|
+
})
|
|
323
|
+
if (!useQueue) {
|
|
324
|
+
await completeReindexProgress({
|
|
325
|
+
em,
|
|
326
|
+
progressService,
|
|
327
|
+
type: 'fulltext',
|
|
328
|
+
tenantId,
|
|
329
|
+
organizationId: auth.orgId ?? null,
|
|
330
|
+
resultSummary: {
|
|
331
|
+
entitiesProcessed: result.entitiesProcessed,
|
|
332
|
+
recordsIndexed: result.recordsIndexed,
|
|
333
|
+
jobsEnqueued: result.jobsEnqueued ?? 0,
|
|
334
|
+
errors: result.errors.length,
|
|
335
|
+
},
|
|
336
|
+
})
|
|
337
|
+
}
|
|
338
|
+
|
|
304
339
|
// Get updated stats from all strategies
|
|
305
340
|
const stats = await collectStrategyStats(searchStrategies, tenantId)
|
|
306
341
|
|
|
@@ -400,6 +435,15 @@ export async function POST(req: Request) {
|
|
|
400
435
|
},
|
|
401
436
|
)
|
|
402
437
|
|
|
438
|
+
await failReindexProgress({
|
|
439
|
+
em,
|
|
440
|
+
progressService,
|
|
441
|
+
type: 'fulltext',
|
|
442
|
+
tenantId,
|
|
443
|
+
organizationId: auth.orgId ?? null,
|
|
444
|
+
errorMessage: error instanceof Error ? error.message : 'Fulltext reindex failed',
|
|
445
|
+
})
|
|
446
|
+
|
|
403
447
|
// Return generic message to client - don't expose internal error details
|
|
404
448
|
return toJson(
|
|
405
449
|
{ error: t('search.api.errors.reindexFailed', 'Reindex operation failed. Please try again or contact support.') },
|
|
@@ -6,6 +6,7 @@ import { readApiResultOrThrow } from '@open-mercato/ui/backend/utils/apiCall'
|
|
|
6
6
|
import { flash } from '@open-mercato/ui/backend/FlashMessages'
|
|
7
7
|
import { Button } from '@open-mercato/ui/primitives/button'
|
|
8
8
|
import { Spinner } from '@open-mercato/ui/primitives/spinner'
|
|
9
|
+
import { useAppEvent } from '@open-mercato/ui/backend/injection/useAppEvent'
|
|
9
10
|
import { GlobalSearchSection } from './sections/GlobalSearchSection'
|
|
10
11
|
import { FulltextSearchSection } from './sections/FulltextSearchSection'
|
|
11
12
|
import { VectorSearchSection } from './sections/VectorSearchSection'
|
|
@@ -228,41 +229,20 @@ export function SearchSettingsPageClient() {
|
|
|
228
229
|
}
|
|
229
230
|
}, [])
|
|
230
231
|
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
232
|
+
useAppEvent('progress.job.updated', () => {
|
|
233
|
+
void refreshStatsOnly()
|
|
234
|
+
void refreshEmbeddingStatsOnly()
|
|
235
|
+
}, [refreshStatsOnly, refreshEmbeddingStatsOnly])
|
|
234
236
|
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
const shouldPoll = hasFulltextLock || hasVectorLock ||
|
|
240
|
-
(wasPollingRef.current && pollCountAfterClearRef.current < 3)
|
|
241
|
-
|
|
242
|
-
if (!shouldPoll) {
|
|
243
|
-
wasPollingRef.current = false
|
|
244
|
-
pollCountAfterClearRef.current = 0
|
|
245
|
-
return
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
if (hasFulltextLock || hasVectorLock) {
|
|
249
|
-
wasPollingRef.current = true
|
|
250
|
-
pollCountAfterClearRef.current = 0
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
const pollInterval = setInterval(() => {
|
|
254
|
-
if (!hasFulltextLock && !hasVectorLock) {
|
|
255
|
-
pollCountAfterClearRef.current += 1
|
|
256
|
-
}
|
|
257
|
-
|
|
258
|
-
refreshStatsOnly()
|
|
259
|
-
if (hasVectorLock) {
|
|
260
|
-
refreshEmbeddingStatsOnly()
|
|
261
|
-
}
|
|
262
|
-
}, 3000)
|
|
237
|
+
useAppEvent('progress.job.completed', () => {
|
|
238
|
+
void refreshStatsOnly()
|
|
239
|
+
void refreshEmbeddingStatsOnly()
|
|
240
|
+
}, [refreshStatsOnly, refreshEmbeddingStatsOnly])
|
|
263
241
|
|
|
264
|
-
|
|
265
|
-
|
|
242
|
+
useAppEvent('om:bridge:reconnected', () => {
|
|
243
|
+
void refreshStatsOnly()
|
|
244
|
+
void refreshEmbeddingStatsOnly()
|
|
245
|
+
}, [refreshStatsOnly, refreshEmbeddingStatsOnly])
|
|
266
246
|
|
|
267
247
|
// Fetch embedding settings
|
|
268
248
|
const fetchEmbeddingSettings = React.useCallback(async () => {
|
|
@@ -4,6 +4,7 @@ import * as React from 'react'
|
|
|
4
4
|
import { useT } from '@open-mercato/shared/lib/i18n/context'
|
|
5
5
|
import { readApiResultOrThrow } from '@open-mercato/ui/backend/utils/apiCall'
|
|
6
6
|
import { flash } from '@open-mercato/ui/backend/FlashMessages'
|
|
7
|
+
import { useAppEvent } from '@open-mercato/ui/backend/injection/useAppEvent'
|
|
7
8
|
import { Button } from '@open-mercato/ui/primitives/button'
|
|
8
9
|
import { Spinner } from '@open-mercato/ui/primitives/spinner'
|
|
9
10
|
import { Tabs, TabsList, TabsTrigger, TabsContent } from '@open-mercato/ui/primitives/tabs'
|
|
@@ -144,13 +145,17 @@ export function FulltextSearchSection({
|
|
|
144
145
|
fetchActivityLogs()
|
|
145
146
|
}, [fetchActivityLogs])
|
|
146
147
|
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
}, [
|
|
148
|
+
useAppEvent('progress.job.updated', () => {
|
|
149
|
+
void fetchActivityLogs()
|
|
150
|
+
}, [fetchActivityLogs])
|
|
151
|
+
|
|
152
|
+
useAppEvent('progress.job.completed', () => {
|
|
153
|
+
void fetchActivityLogs()
|
|
154
|
+
}, [fetchActivityLogs])
|
|
155
|
+
|
|
156
|
+
useAppEvent('om:bridge:reconnected', () => {
|
|
157
|
+
void fetchActivityLogs()
|
|
158
|
+
}, [fetchActivityLogs])
|
|
154
159
|
|
|
155
160
|
const handleReindexClick = (action: ReindexAction) => {
|
|
156
161
|
setShowReindexDialog(action)
|