@open-mercato/search 0.10.1-develop.7941.1.2d89db94ca → 0.10.1-develop.7954.1.48da4a7d13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -47,14 +47,17 @@ async function advanceFulltextReindexProgress(params) {
|
|
|
47
47
|
await updateReindexProgress(params.db, params.tenantId, "fulltext", params.delta, params.organizationId ?? null);
|
|
48
48
|
}
|
|
49
49
|
}
|
|
50
|
-
const
|
|
50
|
+
const SKIPPED_WITHOUT_INDEXING_COOLDOWN_MS = 5 * 60 * 1e3;
|
|
51
|
+
const reportedSkipKeys = /* @__PURE__ */ new Map();
|
|
51
52
|
function __resetSkippedWithoutIndexingMemo() {
|
|
52
53
|
reportedSkipKeys.clear();
|
|
53
54
|
}
|
|
54
55
|
async function reportSkippedWithoutIndexing(params) {
|
|
55
56
|
const memoKey = `${params.tenantId}:${params.jobType}:${params.reason}`;
|
|
56
|
-
|
|
57
|
-
reportedSkipKeys.
|
|
57
|
+
const now = Date.now();
|
|
58
|
+
const lastReportedAt = reportedSkipKeys.get(memoKey);
|
|
59
|
+
if (lastReportedAt !== void 0 && now - lastReportedAt < SKIPPED_WITHOUT_INDEXING_COOLDOWN_MS) return;
|
|
60
|
+
reportedSkipKeys.set(memoKey, now);
|
|
58
61
|
const message = `${params.reason}; job skipped without indexing`;
|
|
59
62
|
searchWarn("fulltext-index.worker", message, {
|
|
60
63
|
jobId: params.jobId,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../../src/modules/search/workers/fulltext-index.worker.ts"],
|
|
4
|
-
"sourcesContent": ["import type { QueuedJob, JobContext, WorkerMeta } from '@open-mercato/queue'\nimport type { Kysely } from 'kysely'\nimport {\n FULLTEXT_INDEXING_QUEUE_NAME,\n type FulltextIndexJobPayload,\n type FulltextIndexJobType,\n} from '../../../queue/fulltext-indexing'\nimport type { FullTextSearchStrategy } from '../../../strategies/fulltext.strategy'\nimport type { SearchIndexer } from '../../../indexer/search-indexer'\nimport type { EntityManager } from '@mikro-orm/postgresql'\n\nimport type { EntityId } from '@open-mercato/shared/modules/entities'\nimport { recordIndexerLog } from '@open-mercato/shared/lib/indexers/status-log'\nimport { recordIndexerError } from '@open-mercato/shared/lib/indexers/error-log'\nimport type { ProgressService } from '@open-mercato/core/modules/progress/lib/progressService'\nimport { searchDebug, searchDebugWarn, searchError, searchWarn } from '../../../lib/debug'\nimport { clearReindexLock, updateReindexProgress } from '../lib/reindex-lock'\nimport { hasActiveReindexProgress, incrementReindexProgress } from '../lib/reindex-progress'\n\n// Worker metadata for auto-discovery\nconst DEFAULT_CONCURRENCY = 2\nconst envConcurrency = process.env.WORKERS_FULLTEXT_INDEXING_CONCURRENCY\n\nexport const metadata: WorkerMeta = {\n queue: FULLTEXT_INDEXING_QUEUE_NAME,\n concurrency: envConcurrency ? parseInt(envConcurrency, 10) : DEFAULT_CONCURRENCY,\n}\n\ntype HandlerContext = { resolve: <T = unknown>(name: string) => T }\n\nasync function advanceFulltextReindexProgress(params: {\n db: Kysely<any> | null\n em: EntityManager | null\n progressService: ProgressService | null\n tenantId: string\n organizationId?: string | null\n delta: number\n}): Promise<void> {\n if (!Number.isFinite(params.delta) || params.delta <= 0) return\n\n if (params.progressService && params.em) {\n const hasActiveProgress = await hasActiveReindexProgress({\n em: params.em,\n type: 'fulltext',\n tenantId: params.tenantId,\n organizationId: params.organizationId ?? null,\n })\n\n if (!hasActiveProgress) {\n if (params.db) {\n await clearReindexLock(params.db, params.tenantId, 'fulltext', params.organizationId ?? null)\n }\n return\n }\n\n if (params.db) {\n await updateReindexProgress(params.db, params.tenantId, 'fulltext', params.delta, params.organizationId ?? null)\n }\n\n const completed = await incrementReindexProgress({\n em: params.em,\n progressService: params.progressService,\n type: 'fulltext',\n tenantId: params.tenantId,\n organizationId: params.organizationId ?? null,\n delta: params.delta,\n })\n if (completed && params.db) {\n await clearReindexLock(params.db, params.tenantId, 'fulltext', params.organizationId ?? null)\n }\n return\n }\n\n if (params.db) {\n await updateReindexProgress(params.db, params.tenantId, 'fulltext', params.delta, params.organizationId ?? null)\n }\n}\n\n/**\n * Report a job that returned without indexing anything.\n *\n * A fulltext strategy that was never registered (no `MEILISEARCH_HOST`, so\n * `createFulltextDriver()` returns null and `di.ts` pushes no strategy) makes\n * every job a no-op. The bare `return` that follows leaves BullMQ reporting\n * `completed` with zero failures and writes no `indexer_status_logs` row, so\n * \"fulltext is not configured\" and \"everything indexed fine\" are\n * indistinguishable from both signals an operator has.\n *\n * Deliberately does NOT throw. A deployment may legitimately run without\n * Meilisearch, and throwing would retry these jobs forever, turning a\n * configuration choice into a queue backlog. The `isAvailable()` check further\n * down throws precisely because a registered-but-unreachable Meili is expected\n * to recover; an unregistered strategy is not.\n *\n * Reported ONCE per (tenant, job type, reason) per process, deliberately.\n * `searchStrategies` and `searchIndexer` are registered together in one\n * `container.register` call, from a `strategies` array that only receives a\n * `FullTextSearchStrategy` when the driver exists - so the outcome of this guard\n * is static for the lifetime of the process and rows 2..N carry exactly the\n * information of row 1. The surface an operator reads is the newest 100\n * `indexer_status_logs` rows across ALL sources, so an unthrottled row per job\n * would saturate that window under ordinary write traffic and push out every\n * other source's rows - trading invisibility for a panel showing one signal a\n * hundred times. Do not \"fix\" this back to per-job.\n */\nconst reportedSkipKeys = new Set<string>()\n\n/** Test seam: the memo above is process-scoped, so a suite must be able to clear it. */\nexport function __resetSkippedWithoutIndexingMemo(): void {\n reportedSkipKeys.clear()\n}\n\nasync function reportSkippedWithoutIndexing(params: {\n em: EntityManager | null\n jobType: FulltextIndexJobType\n tenantId: string\n organizationId: string | null\n jobId: string\n reason: string\n}): Promise<void> {\n const memoKey = `${params.tenantId}:${params.jobType}:${params.reason}`\n if (reportedSkipKeys.has(memoKey)) return\n reportedSkipKeys.add(memoKey)\n\n const message = `${params.reason}; job skipped without indexing`\n // `searchWarn`, not `searchError`: this is an operational warning about a\n // configuration the guard above calls legitimate, it is the level the sibling\n // vector worker uses for the same skip, and it agrees with the `level: 'warn'`\n // of the row written beside it. It is un-gated by OM_SEARCH_DEBUG either way,\n // which is the property this fix needs.\n searchWarn('fulltext-index.worker', message, {\n jobId: params.jobId,\n tenantId: params.tenantId,\n jobType: params.jobType,\n })\n await recordIndexerLog(\n { em: params.em ?? undefined },\n {\n source: 'fulltext',\n handler: `worker:fulltext:${params.jobType}`,\n level: 'warn',\n message,\n tenantId: params.tenantId,\n // No `organizationId`: every other recordIndexerLog call in this handler\n // omits it, so all existing `source: 'fulltext'` rows persist with\n // organization_id = NULL - and the unrestricted indexer-status view filters\n // on `organization_id IS NULL` (query_index/api/status.ts). A non-null org\n // here would hide this row in exactly the view an operator lands on when\n // asking \"did indexing run?\". The value stays in `details` for triage.\n details: {\n jobId: params.jobId,\n jobType: params.jobType,\n organizationId: params.organizationId,\n skippedWithoutIndexing: true,\n },\n },\n )\n}\n\n/**\n * Process a fulltext indexing job.\n *\n * This handler processes single record indexing, batch indexing, deletion, and purge\n * operations for the fulltext search strategy.\n *\n * Single-record jobs load fresh data via searchIndexer.indexRecordById(). Batch jobs load\n * fresh data per record via searchIndexer.indexRecordsById(), which flushes the whole batch\n * through a single bulk write instead of one write per record.\n *\n * @param job - The queued job containing payload\n * @param jobCtx - Queue job context with job ID and attempt info\n * @param ctx - DI container context for resolving services\n */\nexport async function handleFulltextIndexJob(\n job: QueuedJob<FulltextIndexJobPayload>,\n jobCtx: JobContext,\n ctx: HandlerContext,\n): Promise<void> {\n const { jobType, tenantId } = job.payload\n const payloadOrganizationId =\n 'organizationId' in job.payload ? job.payload.organizationId ?? null : null\n\n if (!tenantId) {\n searchDebugWarn('fulltext-index.worker', 'Skipping job with missing tenantId', {\n jobId: jobCtx.jobId,\n jobType,\n })\n return\n }\n\n // Resolve EntityManager for logging and Kysely for database queries\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n let em: any | null = null\n let db: Kysely<any> | null = null\n try {\n em = ctx.resolve('em') as EntityManager\n db = (em as unknown as { getKysely: () => Kysely<any> }).getKysely()\n } catch {\n em = null\n db = null\n }\n\n // Resolve searchIndexer for loading fresh data\n let searchIndexer: SearchIndexer | undefined\n try {\n searchIndexer = ctx.resolve<SearchIndexer>('searchIndexer')\n } catch {\n // Left at debug level to match vector-index.worker.ts for the identical\n // condition. It is also close to unreachable: `searchIndexer` and\n // `searchStrategies` are registered in the same `container.register` call, so\n // the only way this throws is that search DI never ran - in which case the\n // next guard reports it anyway.\n searchDebugWarn('fulltext-index.worker', 'searchIndexer not available')\n }\n\n // Resolve fulltext strategy\n let fulltextStrategy: FullTextSearchStrategy | undefined\n try {\n const searchStrategies = ctx.resolve<unknown[]>('searchStrategies')\n fulltextStrategy = searchStrategies?.find(\n (s: unknown) => (s as { id?: string })?.id === 'fulltext',\n ) as FullTextSearchStrategy | undefined\n } catch {\n await reportSkippedWithoutIndexing({\n em,\n jobType,\n tenantId,\n organizationId: payloadOrganizationId,\n jobId: jobCtx.jobId,\n reason: 'searchStrategies not available',\n })\n return\n }\n\n if (!fulltextStrategy) {\n await reportSkippedWithoutIndexing({\n em,\n jobType,\n tenantId,\n organizationId: payloadOrganizationId,\n jobId: jobCtx.jobId,\n reason: 'Fulltext strategy not configured (MEILISEARCH_HOST unset?)',\n })\n return\n }\n\n // Check if fulltext is available\n const isAvailable = await fulltextStrategy.isAvailable()\n if (!isAvailable) {\n throw new Error('Fulltext search is not available') // Will trigger retry\n }\n\n try {\n let progressService: ProgressService | null = null\n try {\n progressService = ctx.resolve<ProgressService>('progressService')\n } catch {\n progressService = null\n }\n\n // ========== SINGLE INDEX: Use searchIndexer.indexRecordById() for fresh data ==========\n if (jobType === 'index') {\n const { entityType, recordId, organizationId } = job.payload as {\n entityType: string\n recordId: string\n organizationId?: string | null\n }\n\n if (!entityType || !recordId) {\n searchDebugWarn('fulltext-index.worker', 'Skipping index with missing fields', {\n jobId: jobCtx.jobId,\n entityType,\n recordId,\n })\n return\n }\n\n if (!searchIndexer) {\n throw new Error('searchIndexer not available for single-record index')\n }\n\n const result = await searchIndexer.indexRecordById({\n entityId: entityType as EntityId,\n recordId,\n tenantId,\n organizationId,\n })\n\n searchDebug('fulltext-index.worker', 'Indexed single record to fulltext', {\n jobId: jobCtx.jobId,\n tenantId,\n entityType,\n recordId,\n action: result.action,\n })\n\n await recordIndexerLog(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: 'worker:fulltext:index',\n message: `Indexed record to fulltext (${result.action})`,\n entityType,\n recordId,\n tenantId,\n details: { jobId: jobCtx.jobId },\n },\n )\n return\n }\n\n // ========== BATCH-INDEX: Load fresh data, write the whole batch in one call ==========\n if (jobType === 'batch-index') {\n const { records, organizationId } = job.payload\n if (!records || records.length === 0) {\n searchDebugWarn('fulltext-index.worker', 'Skipping batch-index with no records', {\n jobId: jobCtx.jobId,\n })\n return\n }\n\n if (!searchIndexer) {\n throw new Error('searchIndexer not available for batch indexing')\n }\n\n // Load and index the whole batch through a single bulk write instead of\n // one indexRecordById() call per record.\n const { indexed: successCount, skipped: skippedCount } = await searchIndexer.indexRecordsById({\n items: records.map(({ entityId, recordId }) => ({ entityId: entityId as EntityId, recordId })),\n tenantId,\n organizationId,\n })\n\n await advanceFulltextReindexProgress({\n db,\n em,\n progressService,\n tenantId,\n organizationId: organizationId ?? null,\n delta: records.length,\n })\n\n searchDebug('fulltext-index.worker', 'Batch indexed to fulltext', {\n jobId: jobCtx.jobId,\n tenantId,\n requestedCount: records.length,\n successCount,\n skippedCount,\n })\n\n await recordIndexerLog(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: 'worker:fulltext:batch-index',\n message: `Indexed ${successCount}/${records.length} records to fulltext`,\n tenantId,\n details: { jobId: jobCtx.jobId, requestedCount: records.length, successCount, skippedCount },\n },\n )\n return\n }\n\n // ========== DELETE ==========\n if (jobType === 'delete') {\n const { entityId, recordId } = job.payload\n if (!entityId || !recordId) {\n searchDebugWarn('fulltext-index.worker', 'Skipping delete with missing fields', {\n jobId: jobCtx.jobId,\n entityId,\n recordId,\n })\n return\n }\n\n await fulltextStrategy.delete(entityId, recordId, tenantId)\n\n searchDebug('fulltext-index.worker', 'Deleted from fulltext', {\n jobId: jobCtx.jobId,\n tenantId,\n entityId,\n recordId,\n })\n\n await recordIndexerLog(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: 'worker:fulltext:delete',\n message: `Deleted record from fulltext`,\n entityType: entityId,\n recordId,\n tenantId,\n details: { jobId: jobCtx.jobId },\n },\n )\n return\n }\n\n // ========== PURGE ==========\n if (jobType === 'purge') {\n const { entityId } = job.payload\n if (!entityId) {\n searchDebugWarn('fulltext-index.worker', 'Skipping purge with missing entityId', {\n jobId: jobCtx.jobId,\n })\n return\n }\n\n await fulltextStrategy.purge(entityId, tenantId)\n\n searchDebug('fulltext-index.worker', 'Purged entity from fulltext', {\n jobId: jobCtx.jobId,\n tenantId,\n entityId,\n })\n\n await recordIndexerLog(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: 'worker:fulltext:purge',\n message: `Purged entity from fulltext`,\n entityType: entityId,\n tenantId,\n details: { jobId: jobCtx.jobId },\n },\n )\n return\n }\n } catch (error) {\n searchError('fulltext-index.worker', `Failed to ${jobType}`, {\n jobId: jobCtx.jobId,\n tenantId,\n error: error instanceof Error ? error.message : error,\n attemptNumber: jobCtx.attemptNumber,\n })\n\n const entityId = 'entityId' in job.payload ? job.payload.entityId :\n 'entityType' in job.payload ? (job.payload as { entityType?: string }).entityType : undefined\n const recordId = 'recordId' in job.payload ? job.payload.recordId : undefined\n\n await recordIndexerError(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: `worker:fulltext:${jobType}`,\n error,\n entityType: entityId,\n recordId,\n tenantId,\n payload: job.payload,\n },\n )\n\n // Re-throw to let the queue handle retry logic\n throw error\n }\n}\n\n/**\n * Default export for worker auto-discovery.\n * Wraps handleFulltextIndexJob to match the expected handler signature.\n */\nexport default async function handle(\n job: QueuedJob<FulltextIndexJobPayload>,\n ctx: JobContext & HandlerContext\n): Promise<void> {\n return handleFulltextIndexJob(job, ctx, ctx)\n}\n"],
|
|
5
|
-
"mappings": "AAEA;AAAA,EACE;AAAA,OAGK;AAMP,SAAS,wBAAwB;AACjC,SAAS,0BAA0B;AAEnC,SAAS,aAAa,iBAAiB,aAAa,kBAAkB;AACtE,SAAS,kBAAkB,6BAA6B;AACxD,SAAS,0BAA0B,gCAAgC;AAGnE,MAAM,sBAAsB;AAC5B,MAAM,iBAAiB,QAAQ,IAAI;AAE5B,MAAM,WAAuB;AAAA,EAClC,OAAO;AAAA,EACP,aAAa,iBAAiB,SAAS,gBAAgB,EAAE,IAAI;AAC/D;AAIA,eAAe,+BAA+B,QAO5B;AAChB,MAAI,CAAC,OAAO,SAAS,OAAO,KAAK,KAAK,OAAO,SAAS,EAAG;AAEzD,MAAI,OAAO,mBAAmB,OAAO,IAAI;AACvC,UAAM,oBAAoB,MAAM,yBAAyB;AAAA,MACvD,IAAI,OAAO;AAAA,MACX,MAAM;AAAA,MACN,UAAU,OAAO;AAAA,MACjB,gBAAgB,OAAO,kBAAkB;AAAA,IAC3C,CAAC;AAED,QAAI,CAAC,mBAAmB;AACtB,UAAI,OAAO,IAAI;AACb,cAAM,iBAAiB,OAAO,IAAI,OAAO,UAAU,YAAY,OAAO,kBAAkB,IAAI;AAAA,MAC9F;AACA;AAAA,IACF;AAEA,QAAI,OAAO,IAAI;AACb,YAAM,sBAAsB,OAAO,IAAI,OAAO,UAAU,YAAY,OAAO,OAAO,OAAO,kBAAkB,IAAI;AAAA,IACjH;AAEA,UAAM,YAAY,MAAM,yBAAyB;AAAA,MAC/C,IAAI,OAAO;AAAA,MACX,iBAAiB,OAAO;AAAA,MACxB,MAAM;AAAA,MACN,UAAU,OAAO;AAAA,MACjB,gBAAgB,OAAO,kBAAkB;AAAA,MACzC,OAAO,OAAO;AAAA,IAChB,CAAC;AACD,QAAI,aAAa,OAAO,IAAI;AAC1B,YAAM,iBAAiB,OAAO,IAAI,OAAO,UAAU,YAAY,OAAO,kBAAkB,IAAI;AAAA,IAC9F;AACA;AAAA,EACF;AAEA,MAAI,OAAO,IAAI;AACb,UAAM,sBAAsB,OAAO,IAAI,OAAO,UAAU,YAAY,OAAO,OAAO,OAAO,kBAAkB,IAAI;AAAA,EACjH;AACF;
|
|
4
|
+
"sourcesContent": ["import type { QueuedJob, JobContext, WorkerMeta } from '@open-mercato/queue'\nimport type { Kysely } from 'kysely'\nimport {\n FULLTEXT_INDEXING_QUEUE_NAME,\n type FulltextIndexJobPayload,\n type FulltextIndexJobType,\n} from '../../../queue/fulltext-indexing'\nimport type { FullTextSearchStrategy } from '../../../strategies/fulltext.strategy'\nimport type { SearchIndexer } from '../../../indexer/search-indexer'\nimport type { EntityManager } from '@mikro-orm/postgresql'\n\nimport type { EntityId } from '@open-mercato/shared/modules/entities'\nimport { recordIndexerLog } from '@open-mercato/shared/lib/indexers/status-log'\nimport { recordIndexerError } from '@open-mercato/shared/lib/indexers/error-log'\nimport type { ProgressService } from '@open-mercato/core/modules/progress/lib/progressService'\nimport { searchDebug, searchDebugWarn, searchError, searchWarn } from '../../../lib/debug'\nimport { clearReindexLock, updateReindexProgress } from '../lib/reindex-lock'\nimport { hasActiveReindexProgress, incrementReindexProgress } from '../lib/reindex-progress'\n\n// Worker metadata for auto-discovery\nconst DEFAULT_CONCURRENCY = 2\nconst envConcurrency = process.env.WORKERS_FULLTEXT_INDEXING_CONCURRENCY\n\nexport const metadata: WorkerMeta = {\n queue: FULLTEXT_INDEXING_QUEUE_NAME,\n concurrency: envConcurrency ? parseInt(envConcurrency, 10) : DEFAULT_CONCURRENCY,\n}\n\ntype HandlerContext = { resolve: <T = unknown>(name: string) => T }\n\nasync function advanceFulltextReindexProgress(params: {\n db: Kysely<any> | null\n em: EntityManager | null\n progressService: ProgressService | null\n tenantId: string\n organizationId?: string | null\n delta: number\n}): Promise<void> {\n if (!Number.isFinite(params.delta) || params.delta <= 0) return\n\n if (params.progressService && params.em) {\n const hasActiveProgress = await hasActiveReindexProgress({\n em: params.em,\n type: 'fulltext',\n tenantId: params.tenantId,\n organizationId: params.organizationId ?? null,\n })\n\n if (!hasActiveProgress) {\n if (params.db) {\n await clearReindexLock(params.db, params.tenantId, 'fulltext', params.organizationId ?? null)\n }\n return\n }\n\n if (params.db) {\n await updateReindexProgress(params.db, params.tenantId, 'fulltext', params.delta, params.organizationId ?? null)\n }\n\n const completed = await incrementReindexProgress({\n em: params.em,\n progressService: params.progressService,\n type: 'fulltext',\n tenantId: params.tenantId,\n organizationId: params.organizationId ?? null,\n delta: params.delta,\n })\n if (completed && params.db) {\n await clearReindexLock(params.db, params.tenantId, 'fulltext', params.organizationId ?? null)\n }\n return\n }\n\n if (params.db) {\n await updateReindexProgress(params.db, params.tenantId, 'fulltext', params.delta, params.organizationId ?? null)\n }\n}\n\n/**\n * Report a job that returned without indexing anything.\n *\n * A fulltext strategy that was never registered (no `MEILISEARCH_HOST`, so\n * `createFulltextDriver()` returns null and `di.ts` pushes no strategy) makes\n * every job a no-op. The bare `return` that follows leaves BullMQ reporting\n * `completed` with zero failures and writes no `indexer_status_logs` row, so\n * \"fulltext is not configured\" and \"everything indexed fine\" are\n * indistinguishable from both signals an operator has.\n *\n * Deliberately does NOT throw. A deployment may legitimately run without\n * Meilisearch, and throwing would retry these jobs forever, turning a\n * configuration choice into a queue backlog. The `isAvailable()` check further\n * down throws precisely because a registered-but-unreachable Meili is expected\n * to recover; an unregistered strategy is not.\n *\n * Report at most once per (tenant, job type, reason) every five minutes per\n * process. Reserve the cooldown before the best-effort write so concurrent jobs\n * and failed writes cannot flood the newest 100 status rows. Later skipped jobs\n * renew diagnostics that failed to persist or aged out of the reader's window.\n */\nconst SKIPPED_WITHOUT_INDEXING_COOLDOWN_MS = 5 * 60 * 1000\nconst reportedSkipKeys = new Map<string, number>()\n\n/** Test seam: the memo above is process-scoped, so a suite must be able to clear it. */\nexport function __resetSkippedWithoutIndexingMemo(): void {\n reportedSkipKeys.clear()\n}\n\nasync function reportSkippedWithoutIndexing(params: {\n em: EntityManager | null\n jobType: FulltextIndexJobType\n tenantId: string\n organizationId: string | null\n jobId: string\n reason: string\n}): Promise<void> {\n const memoKey = `${params.tenantId}:${params.jobType}:${params.reason}`\n const now = Date.now()\n const lastReportedAt = reportedSkipKeys.get(memoKey)\n if (lastReportedAt !== undefined && now - lastReportedAt < SKIPPED_WITHOUT_INDEXING_COOLDOWN_MS) return\n reportedSkipKeys.set(memoKey, now)\n\n const message = `${params.reason}; job skipped without indexing`\n // `searchWarn`, not `searchError`: this is an operational warning about a\n // configuration the guard above calls legitimate, it is the level the sibling\n // vector worker uses for the same skip, and it agrees with the `level: 'warn'`\n // of the row written beside it. It is un-gated by OM_SEARCH_DEBUG either way,\n // which is the property this fix needs.\n searchWarn('fulltext-index.worker', message, {\n jobId: params.jobId,\n tenantId: params.tenantId,\n jobType: params.jobType,\n })\n await recordIndexerLog(\n { em: params.em ?? undefined },\n {\n source: 'fulltext',\n handler: `worker:fulltext:${params.jobType}`,\n level: 'warn',\n message,\n tenantId: params.tenantId,\n // No `organizationId`: every other recordIndexerLog call in this handler\n // omits it, so all existing `source: 'fulltext'` rows persist with\n // organization_id = NULL - and the unrestricted indexer-status view filters\n // on `organization_id IS NULL` (query_index/api/status.ts). A non-null org\n // here would hide this row in exactly the view an operator lands on when\n // asking \"did indexing run?\". The value stays in `details` for triage.\n details: {\n jobId: params.jobId,\n jobType: params.jobType,\n organizationId: params.organizationId,\n skippedWithoutIndexing: true,\n },\n },\n )\n}\n\n/**\n * Process a fulltext indexing job.\n *\n * This handler processes single record indexing, batch indexing, deletion, and purge\n * operations for the fulltext search strategy.\n *\n * Single-record jobs load fresh data via searchIndexer.indexRecordById(). Batch jobs load\n * fresh data per record via searchIndexer.indexRecordsById(), which flushes the whole batch\n * through a single bulk write instead of one write per record.\n *\n * @param job - The queued job containing payload\n * @param jobCtx - Queue job context with job ID and attempt info\n * @param ctx - DI container context for resolving services\n */\nexport async function handleFulltextIndexJob(\n job: QueuedJob<FulltextIndexJobPayload>,\n jobCtx: JobContext,\n ctx: HandlerContext,\n): Promise<void> {\n const { jobType, tenantId } = job.payload\n const payloadOrganizationId =\n 'organizationId' in job.payload ? job.payload.organizationId ?? null : null\n\n if (!tenantId) {\n searchDebugWarn('fulltext-index.worker', 'Skipping job with missing tenantId', {\n jobId: jobCtx.jobId,\n jobType,\n })\n return\n }\n\n // Resolve EntityManager for logging and Kysely for database queries\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n let em: any | null = null\n let db: Kysely<any> | null = null\n try {\n em = ctx.resolve('em') as EntityManager\n db = (em as unknown as { getKysely: () => Kysely<any> }).getKysely()\n } catch {\n em = null\n db = null\n }\n\n // Resolve searchIndexer for loading fresh data\n let searchIndexer: SearchIndexer | undefined\n try {\n searchIndexer = ctx.resolve<SearchIndexer>('searchIndexer')\n } catch {\n // Left at debug level to match vector-index.worker.ts for the identical\n // condition. It is also close to unreachable: `searchIndexer` and\n // `searchStrategies` are registered in the same `container.register` call, so\n // the only way this throws is that search DI never ran - in which case the\n // next guard reports it anyway.\n searchDebugWarn('fulltext-index.worker', 'searchIndexer not available')\n }\n\n // Resolve fulltext strategy\n let fulltextStrategy: FullTextSearchStrategy | undefined\n try {\n const searchStrategies = ctx.resolve<unknown[]>('searchStrategies')\n fulltextStrategy = searchStrategies?.find(\n (s: unknown) => (s as { id?: string })?.id === 'fulltext',\n ) as FullTextSearchStrategy | undefined\n } catch {\n await reportSkippedWithoutIndexing({\n em,\n jobType,\n tenantId,\n organizationId: payloadOrganizationId,\n jobId: jobCtx.jobId,\n reason: 'searchStrategies not available',\n })\n return\n }\n\n if (!fulltextStrategy) {\n await reportSkippedWithoutIndexing({\n em,\n jobType,\n tenantId,\n organizationId: payloadOrganizationId,\n jobId: jobCtx.jobId,\n reason: 'Fulltext strategy not configured (MEILISEARCH_HOST unset?)',\n })\n return\n }\n\n // Check if fulltext is available\n const isAvailable = await fulltextStrategy.isAvailable()\n if (!isAvailable) {\n throw new Error('Fulltext search is not available') // Will trigger retry\n }\n\n try {\n let progressService: ProgressService | null = null\n try {\n progressService = ctx.resolve<ProgressService>('progressService')\n } catch {\n progressService = null\n }\n\n // ========== SINGLE INDEX: Use searchIndexer.indexRecordById() for fresh data ==========\n if (jobType === 'index') {\n const { entityType, recordId, organizationId } = job.payload as {\n entityType: string\n recordId: string\n organizationId?: string | null\n }\n\n if (!entityType || !recordId) {\n searchDebugWarn('fulltext-index.worker', 'Skipping index with missing fields', {\n jobId: jobCtx.jobId,\n entityType,\n recordId,\n })\n return\n }\n\n if (!searchIndexer) {\n throw new Error('searchIndexer not available for single-record index')\n }\n\n const result = await searchIndexer.indexRecordById({\n entityId: entityType as EntityId,\n recordId,\n tenantId,\n organizationId,\n })\n\n searchDebug('fulltext-index.worker', 'Indexed single record to fulltext', {\n jobId: jobCtx.jobId,\n tenantId,\n entityType,\n recordId,\n action: result.action,\n })\n\n await recordIndexerLog(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: 'worker:fulltext:index',\n message: `Indexed record to fulltext (${result.action})`,\n entityType,\n recordId,\n tenantId,\n details: { jobId: jobCtx.jobId },\n },\n )\n return\n }\n\n // ========== BATCH-INDEX: Load fresh data, write the whole batch in one call ==========\n if (jobType === 'batch-index') {\n const { records, organizationId } = job.payload\n if (!records || records.length === 0) {\n searchDebugWarn('fulltext-index.worker', 'Skipping batch-index with no records', {\n jobId: jobCtx.jobId,\n })\n return\n }\n\n if (!searchIndexer) {\n throw new Error('searchIndexer not available for batch indexing')\n }\n\n // Load and index the whole batch through a single bulk write instead of\n // one indexRecordById() call per record.\n const { indexed: successCount, skipped: skippedCount } = await searchIndexer.indexRecordsById({\n items: records.map(({ entityId, recordId }) => ({ entityId: entityId as EntityId, recordId })),\n tenantId,\n organizationId,\n })\n\n await advanceFulltextReindexProgress({\n db,\n em,\n progressService,\n tenantId,\n organizationId: organizationId ?? null,\n delta: records.length,\n })\n\n searchDebug('fulltext-index.worker', 'Batch indexed to fulltext', {\n jobId: jobCtx.jobId,\n tenantId,\n requestedCount: records.length,\n successCount,\n skippedCount,\n })\n\n await recordIndexerLog(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: 'worker:fulltext:batch-index',\n message: `Indexed ${successCount}/${records.length} records to fulltext`,\n tenantId,\n details: { jobId: jobCtx.jobId, requestedCount: records.length, successCount, skippedCount },\n },\n )\n return\n }\n\n // ========== DELETE ==========\n if (jobType === 'delete') {\n const { entityId, recordId } = job.payload\n if (!entityId || !recordId) {\n searchDebugWarn('fulltext-index.worker', 'Skipping delete with missing fields', {\n jobId: jobCtx.jobId,\n entityId,\n recordId,\n })\n return\n }\n\n await fulltextStrategy.delete(entityId, recordId, tenantId)\n\n searchDebug('fulltext-index.worker', 'Deleted from fulltext', {\n jobId: jobCtx.jobId,\n tenantId,\n entityId,\n recordId,\n })\n\n await recordIndexerLog(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: 'worker:fulltext:delete',\n message: `Deleted record from fulltext`,\n entityType: entityId,\n recordId,\n tenantId,\n details: { jobId: jobCtx.jobId },\n },\n )\n return\n }\n\n // ========== PURGE ==========\n if (jobType === 'purge') {\n const { entityId } = job.payload\n if (!entityId) {\n searchDebugWarn('fulltext-index.worker', 'Skipping purge with missing entityId', {\n jobId: jobCtx.jobId,\n })\n return\n }\n\n await fulltextStrategy.purge(entityId, tenantId)\n\n searchDebug('fulltext-index.worker', 'Purged entity from fulltext', {\n jobId: jobCtx.jobId,\n tenantId,\n entityId,\n })\n\n await recordIndexerLog(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: 'worker:fulltext:purge',\n message: `Purged entity from fulltext`,\n entityType: entityId,\n tenantId,\n details: { jobId: jobCtx.jobId },\n },\n )\n return\n }\n } catch (error) {\n searchError('fulltext-index.worker', `Failed to ${jobType}`, {\n jobId: jobCtx.jobId,\n tenantId,\n error: error instanceof Error ? error.message : error,\n attemptNumber: jobCtx.attemptNumber,\n })\n\n const entityId = 'entityId' in job.payload ? job.payload.entityId :\n 'entityType' in job.payload ? (job.payload as { entityType?: string }).entityType : undefined\n const recordId = 'recordId' in job.payload ? job.payload.recordId : undefined\n\n await recordIndexerError(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: `worker:fulltext:${jobType}`,\n error,\n entityType: entityId,\n recordId,\n tenantId,\n payload: job.payload,\n },\n )\n\n // Re-throw to let the queue handle retry logic\n throw error\n }\n}\n\n/**\n * Default export for worker auto-discovery.\n * Wraps handleFulltextIndexJob to match the expected handler signature.\n */\nexport default async function handle(\n job: QueuedJob<FulltextIndexJobPayload>,\n ctx: JobContext & HandlerContext\n): Promise<void> {\n return handleFulltextIndexJob(job, ctx, ctx)\n}\n"],
|
|
5
|
+
"mappings": "AAEA;AAAA,EACE;AAAA,OAGK;AAMP,SAAS,wBAAwB;AACjC,SAAS,0BAA0B;AAEnC,SAAS,aAAa,iBAAiB,aAAa,kBAAkB;AACtE,SAAS,kBAAkB,6BAA6B;AACxD,SAAS,0BAA0B,gCAAgC;AAGnE,MAAM,sBAAsB;AAC5B,MAAM,iBAAiB,QAAQ,IAAI;AAE5B,MAAM,WAAuB;AAAA,EAClC,OAAO;AAAA,EACP,aAAa,iBAAiB,SAAS,gBAAgB,EAAE,IAAI;AAC/D;AAIA,eAAe,+BAA+B,QAO5B;AAChB,MAAI,CAAC,OAAO,SAAS,OAAO,KAAK,KAAK,OAAO,SAAS,EAAG;AAEzD,MAAI,OAAO,mBAAmB,OAAO,IAAI;AACvC,UAAM,oBAAoB,MAAM,yBAAyB;AAAA,MACvD,IAAI,OAAO;AAAA,MACX,MAAM;AAAA,MACN,UAAU,OAAO;AAAA,MACjB,gBAAgB,OAAO,kBAAkB;AAAA,IAC3C,CAAC;AAED,QAAI,CAAC,mBAAmB;AACtB,UAAI,OAAO,IAAI;AACb,cAAM,iBAAiB,OAAO,IAAI,OAAO,UAAU,YAAY,OAAO,kBAAkB,IAAI;AAAA,MAC9F;AACA;AAAA,IACF;AAEA,QAAI,OAAO,IAAI;AACb,YAAM,sBAAsB,OAAO,IAAI,OAAO,UAAU,YAAY,OAAO,OAAO,OAAO,kBAAkB,IAAI;AAAA,IACjH;AAEA,UAAM,YAAY,MAAM,yBAAyB;AAAA,MAC/C,IAAI,OAAO;AAAA,MACX,iBAAiB,OAAO;AAAA,MACxB,MAAM;AAAA,MACN,UAAU,OAAO;AAAA,MACjB,gBAAgB,OAAO,kBAAkB;AAAA,MACzC,OAAO,OAAO;AAAA,IAChB,CAAC;AACD,QAAI,aAAa,OAAO,IAAI;AAC1B,YAAM,iBAAiB,OAAO,IAAI,OAAO,UAAU,YAAY,OAAO,kBAAkB,IAAI;AAAA,IAC9F;AACA;AAAA,EACF;AAEA,MAAI,OAAO,IAAI;AACb,UAAM,sBAAsB,OAAO,IAAI,OAAO,UAAU,YAAY,OAAO,OAAO,OAAO,kBAAkB,IAAI;AAAA,EACjH;AACF;AAuBA,MAAM,uCAAuC,IAAI,KAAK;AACtD,MAAM,mBAAmB,oBAAI,IAAoB;AAG1C,SAAS,oCAA0C;AACxD,mBAAiB,MAAM;AACzB;AAEA,eAAe,6BAA6B,QAO1B;AAChB,QAAM,UAAU,GAAG,OAAO,QAAQ,IAAI,OAAO,OAAO,IAAI,OAAO,MAAM;AACrE,QAAM,MAAM,KAAK,IAAI;AACrB,QAAM,iBAAiB,iBAAiB,IAAI,OAAO;AACnD,MAAI,mBAAmB,UAAa,MAAM,iBAAiB,qCAAsC;AACjG,mBAAiB,IAAI,SAAS,GAAG;AAEjC,QAAM,UAAU,GAAG,OAAO,MAAM;AAMhC,aAAW,yBAAyB,SAAS;AAAA,IAC3C,OAAO,OAAO;AAAA,IACd,UAAU,OAAO;AAAA,IACjB,SAAS,OAAO;AAAA,EAClB,CAAC;AACD,QAAM;AAAA,IACJ,EAAE,IAAI,OAAO,MAAM,OAAU;AAAA,IAC7B;AAAA,MACE,QAAQ;AAAA,MACR,SAAS,mBAAmB,OAAO,OAAO;AAAA,MAC1C,OAAO;AAAA,MACP;AAAA,MACA,UAAU,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAOjB,SAAS;AAAA,QACP,OAAO,OAAO;AAAA,QACd,SAAS,OAAO;AAAA,QAChB,gBAAgB,OAAO;AAAA,QACvB,wBAAwB;AAAA,MAC1B;AAAA,IACF;AAAA,EACF;AACF;AAgBA,eAAsB,uBACpB,KACA,QACA,KACe;AACf,QAAM,EAAE,SAAS,SAAS,IAAI,IAAI;AAClC,QAAM,wBACJ,oBAAoB,IAAI,UAAU,IAAI,QAAQ,kBAAkB,OAAO;AAEzE,MAAI,CAAC,UAAU;AACb,oBAAgB,yBAAyB,sCAAsC;AAAA,MAC7E,OAAO,OAAO;AAAA,MACd;AAAA,IACF,CAAC;AACD;AAAA,EACF;AAIA,MAAI,KAAiB;AACrB,MAAI,KAAyB;AAC7B,MAAI;AACF,SAAK,IAAI,QAAQ,IAAI;AACrB,SAAM,GAAmD,UAAU;AAAA,EACrE,QAAQ;AACN,SAAK;AACL,SAAK;AAAA,EACP;AAGA,MAAI;AACJ,MAAI;AACF,oBAAgB,IAAI,QAAuB,eAAe;AAAA,EAC5D,QAAQ;AAMN,oBAAgB,yBAAyB,6BAA6B;AAAA,EACxE;AAGA,MAAI;AACJ,MAAI;AACF,UAAM,mBAAmB,IAAI,QAAmB,kBAAkB;AAClE,uBAAmB,kBAAkB;AAAA,MACnC,CAAC,MAAgB,GAAuB,OAAO;AAAA,IACjD;AAAA,EACF,QAAQ;AACN,UAAM,6BAA6B;AAAA,MACjC;AAAA,MACA;AAAA,MACA;AAAA,MACA,gBAAgB;AAAA,MAChB,OAAO,OAAO;AAAA,MACd,QAAQ;AAAA,IACV,CAAC;AACD;AAAA,EACF;AAEA,MAAI,CAAC,kBAAkB;AACrB,UAAM,6BAA6B;AAAA,MACjC;AAAA,MACA;AAAA,MACA;AAAA,MACA,gBAAgB;AAAA,MAChB,OAAO,OAAO;AAAA,MACd,QAAQ;AAAA,IACV,CAAC;AACD;AAAA,EACF;AAGA,QAAM,cAAc,MAAM,iBAAiB,YAAY;AACvD,MAAI,CAAC,aAAa;AAChB,UAAM,IAAI,MAAM,kCAAkC;AAAA,EACpD;AAEA,MAAI;AACF,QAAI,kBAA0C;AAC9C,QAAI;AACF,wBAAkB,IAAI,QAAyB,iBAAiB;AAAA,IAClE,QAAQ;AACN,wBAAkB;AAAA,IACpB;AAGA,QAAI,YAAY,SAAS;AACvB,YAAM,EAAE,YAAY,UAAU,eAAe,IAAI,IAAI;AAMrD,UAAI,CAAC,cAAc,CAAC,UAAU;AAC5B,wBAAgB,yBAAyB,sCAAsC;AAAA,UAC7E,OAAO,OAAO;AAAA,UACd;AAAA,UACA;AAAA,QACF,CAAC;AACD;AAAA,MACF;AAEA,UAAI,CAAC,eAAe;AAClB,cAAM,IAAI,MAAM,qDAAqD;AAAA,MACvE;AAEA,YAAM,SAAS,MAAM,cAAc,gBAAgB;AAAA,QACjD,UAAU;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AAED,kBAAY,yBAAyB,qCAAqC;AAAA,QACxE,OAAO,OAAO;AAAA,QACd;AAAA,QACA;AAAA,QACA;AAAA,QACA,QAAQ,OAAO;AAAA,MACjB,CAAC;AAED,YAAM;AAAA,QACJ,EAAE,IAAI,MAAM,OAAU;AAAA,QACtB;AAAA,UACE,QAAQ;AAAA,UACR,SAAS;AAAA,UACT,SAAS,+BAA+B,OAAO,MAAM;AAAA,UACrD;AAAA,UACA;AAAA,UACA;AAAA,UACA,SAAS,EAAE,OAAO,OAAO,MAAM;AAAA,QACjC;AAAA,MACF;AACA;AAAA,IACF;AAGA,QAAI,YAAY,eAAe;AAC7B,YAAM,EAAE,SAAS,eAAe,IAAI,IAAI;AACxC,UAAI,CAAC,WAAW,QAAQ,WAAW,GAAG;AACpC,wBAAgB,yBAAyB,wCAAwC;AAAA,UAC/E,OAAO,OAAO;AAAA,QAChB,CAAC;AACD;AAAA,MACF;AAEA,UAAI,CAAC,eAAe;AAClB,cAAM,IAAI,MAAM,gDAAgD;AAAA,MAClE;AAIA,YAAM,EAAE,SAAS,cAAc,SAAS,aAAa,IAAI,MAAM,cAAc,iBAAiB;AAAA,QAC5F,OAAO,QAAQ,IAAI,CAAC,EAAE,UAAU,SAAS,OAAO,EAAE,UAAgC,SAAS,EAAE;AAAA,QAC7F;AAAA,QACA;AAAA,MACF,CAAC;AAED,YAAM,+BAA+B;AAAA,QACnC;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,gBAAgB,kBAAkB;AAAA,QAClC,OAAO,QAAQ;AAAA,MACjB,CAAC;AAED,kBAAY,yBAAyB,6BAA6B;AAAA,QAChE,OAAO,OAAO;AAAA,QACd;AAAA,QACA,gBAAgB,QAAQ;AAAA,QACxB;AAAA,QACA;AAAA,MACF,CAAC;AAED,YAAM;AAAA,QACJ,EAAE,IAAI,MAAM,OAAU;AAAA,QACtB;AAAA,UACE,QAAQ;AAAA,UACR,SAAS;AAAA,UACT,SAAS,WAAW,YAAY,IAAI,QAAQ,MAAM;AAAA,UAClD;AAAA,UACA,SAAS,EAAE,OAAO,OAAO,OAAO,gBAAgB,QAAQ,QAAQ,cAAc,aAAa;AAAA,QAC7F;AAAA,MACF;AACA;AAAA,IACF;AAGA,QAAI,YAAY,UAAU;AACxB,YAAM,EAAE,UAAU,SAAS,IAAI,IAAI;AACnC,UAAI,CAAC,YAAY,CAAC,UAAU;AAC1B,wBAAgB,yBAAyB,uCAAuC;AAAA,UAC9E,OAAO,OAAO;AAAA,UACd;AAAA,UACA;AAAA,QACF,CAAC;AACD;AAAA,MACF;AAEA,YAAM,iBAAiB,OAAO,UAAU,UAAU,QAAQ;AAE1D,kBAAY,yBAAyB,yBAAyB;AAAA,QAC5D,OAAO,OAAO;AAAA,QACd;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AAED,YAAM;AAAA,QACJ,EAAE,IAAI,MAAM,OAAU;AAAA,QACtB;AAAA,UACE,QAAQ;AAAA,UACR,SAAS;AAAA,UACT,SAAS;AAAA,UACT,YAAY;AAAA,UACZ;AAAA,UACA;AAAA,UACA,SAAS,EAAE,OAAO,OAAO,MAAM;AAAA,QACjC;AAAA,MACF;AACA;AAAA,IACF;AAGA,QAAI,YAAY,SAAS;AACvB,YAAM,EAAE,SAAS,IAAI,IAAI;AACzB,UAAI,CAAC,UAAU;AACb,wBAAgB,yBAAyB,wCAAwC;AAAA,UAC/E,OAAO,OAAO;AAAA,QAChB,CAAC;AACD;AAAA,MACF;AAEA,YAAM,iBAAiB,MAAM,UAAU,QAAQ;AAE/C,kBAAY,yBAAyB,+BAA+B;AAAA,QAClE,OAAO,OAAO;AAAA,QACd;AAAA,QACA;AAAA,MACF,CAAC;AAED,YAAM;AAAA,QACJ,EAAE,IAAI,MAAM,OAAU;AAAA,QACtB;AAAA,UACE,QAAQ;AAAA,UACR,SAAS;AAAA,UACT,SAAS;AAAA,UACT,YAAY;AAAA,UACZ;AAAA,UACA,SAAS,EAAE,OAAO,OAAO,MAAM;AAAA,QACjC;AAAA,MACF;AACA;AAAA,IACF;AAAA,EACF,SAAS,OAAO;AACd,gBAAY,yBAAyB,aAAa,OAAO,IAAI;AAAA,MAC3D,OAAO,OAAO;AAAA,MACd;AAAA,MACA,OAAO,iBAAiB,QAAQ,MAAM,UAAU;AAAA,MAChD,eAAe,OAAO;AAAA,IACxB,CAAC;AAED,UAAM,WAAW,cAAc,IAAI,UAAU,IAAI,QAAQ,WACxC,gBAAgB,IAAI,UAAW,IAAI,QAAoC,aAAa;AACrG,UAAM,WAAW,cAAc,IAAI,UAAU,IAAI,QAAQ,WAAW;AAEpE,UAAM;AAAA,MACJ,EAAE,IAAI,MAAM,OAAU;AAAA,MACtB;AAAA,QACE,QAAQ;AAAA,QACR,SAAS,mBAAmB,OAAO;AAAA,QACnC;AAAA,QACA,YAAY;AAAA,QACZ;AAAA,QACA;AAAA,QACA,SAAS,IAAI;AAAA,MACf;AAAA,IACF;AAGA,UAAM;AAAA,EACR;AACF;AAMA,eAAO,OACL,KACA,KACe;AACf,SAAO,uBAAuB,KAAK,KAAK,GAAG;AAC7C;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@open-mercato/search",
|
|
3
|
-
"version": "0.10.1-develop.
|
|
3
|
+
"version": "0.10.1-develop.7954.1.48da4a7d13",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -127,9 +127,9 @@
|
|
|
127
127
|
"zod": "^4.6.6"
|
|
128
128
|
},
|
|
129
129
|
"peerDependencies": {
|
|
130
|
-
"@open-mercato/core": "0.10.1-develop.
|
|
131
|
-
"@open-mercato/queue": "0.10.1-develop.
|
|
132
|
-
"@open-mercato/shared": "0.10.1-develop.
|
|
130
|
+
"@open-mercato/core": "0.10.1-develop.7954.1.48da4a7d13",
|
|
131
|
+
"@open-mercato/queue": "0.10.1-develop.7954.1.48da4a7d13",
|
|
132
|
+
"@open-mercato/shared": "0.10.1-develop.7954.1.48da4a7d13"
|
|
133
133
|
},
|
|
134
134
|
"devDependencies": {
|
|
135
135
|
"@types/jest": "^30.0.0",
|
|
@@ -441,6 +441,9 @@ describe('Fulltext Index Worker', () => {
|
|
|
441
441
|
select: jest.fn(() => chain),
|
|
442
442
|
selectAll: jest.fn(() => chain),
|
|
443
443
|
from: jest.fn(() => chain),
|
|
444
|
+
orderBy: jest.fn(() => chain),
|
|
445
|
+
offset: jest.fn(() => chain),
|
|
446
|
+
limit: jest.fn(() => chain),
|
|
444
447
|
execute: jest.fn().mockResolvedValue([]),
|
|
445
448
|
executeTakeFirst: jest.fn().mockResolvedValue(undefined),
|
|
446
449
|
}
|
|
@@ -800,11 +803,7 @@ describe('Fulltext Index Worker', () => {
|
|
|
800
803
|
)
|
|
801
804
|
})
|
|
802
805
|
|
|
803
|
-
|
|
804
|
-
// searchIndexer are registered in one call - so rows 2..N carry exactly the information
|
|
805
|
-
// of row 1. The operator surface is the newest 100 rows across ALL sources, so a row per
|
|
806
|
-
// job would saturate it under ordinary write traffic.
|
|
807
|
-
it('reports the same skip once per process, not once per job', async () => {
|
|
806
|
+
it('reports the same skip once inside the cooldown, not once per job', async () => {
|
|
808
807
|
const containerWithoutStrategy: HandlerContext = {
|
|
809
808
|
resolve: jest.fn((name: string) => {
|
|
810
809
|
if (name === 'searchStrategies') return []
|
|
@@ -861,6 +860,150 @@ describe('Fulltext Index Worker', () => {
|
|
|
861
860
|
expect(recordIndexerLog).toHaveBeenCalledTimes(3)
|
|
862
861
|
})
|
|
863
862
|
|
|
863
|
+
describe('skipped diagnostic cooldown', () => {
|
|
864
|
+
const cooldownMs = 5 * 60 * 1000
|
|
865
|
+
const job = createMockJob<FulltextIndexJobPayload>({
|
|
866
|
+
jobType: 'index',
|
|
867
|
+
tenantId: 'tenant-a',
|
|
868
|
+
organizationId: null,
|
|
869
|
+
entityType: 'test:entity',
|
|
870
|
+
recordId: 'rec-1',
|
|
871
|
+
})
|
|
872
|
+
const skipContainer = (options: { missingStrategies?: boolean; missingDb?: boolean } = {}): HandlerContext => ({
|
|
873
|
+
resolve: jest.fn((name: string) => {
|
|
874
|
+
if (name === 'searchStrategies' && !options.missingStrategies) return []
|
|
875
|
+
if (name === 'em' && !options.missingDb) return mockEm
|
|
876
|
+
if (name === 'searchIndexer') return mockSearchIndexer
|
|
877
|
+
throw new Error(`Unknown service: ${name}`)
|
|
878
|
+
}) as HandlerContext['resolve'],
|
|
879
|
+
})
|
|
880
|
+
const runSkippedJob = (container: HandlerContext = skipContainer()) =>
|
|
881
|
+
handleFulltextIndexJob(job, createMockJobContext(), container)
|
|
882
|
+
|
|
883
|
+
beforeEach(() => {
|
|
884
|
+
jest.useFakeTimers({ now: 0 })
|
|
885
|
+
jest.mocked(recordIndexerLog).mockResolvedValue(undefined)
|
|
886
|
+
})
|
|
887
|
+
|
|
888
|
+
afterEach(() => {
|
|
889
|
+
jest.useRealTimers()
|
|
890
|
+
jest.mocked(recordIndexerLog).mockReset().mockResolvedValue(undefined)
|
|
891
|
+
})
|
|
892
|
+
|
|
893
|
+
it.each([false, true])('renews at each boundary without extending suppression (missingStrategies: %s)', async (missingStrategies) => {
|
|
894
|
+
const container = skipContainer({ missingStrategies })
|
|
895
|
+
|
|
896
|
+
await expect(runSkippedJob(container)).resolves.toBeUndefined()
|
|
897
|
+
jest.setSystemTime(cooldownMs - 1)
|
|
898
|
+
await expect(runSkippedJob(container)).resolves.toBeUndefined()
|
|
899
|
+
expect(recordIndexerLog).toHaveBeenCalledTimes(1)
|
|
900
|
+
expect(searchLoggerWarn).toHaveBeenCalledTimes(1)
|
|
901
|
+
|
|
902
|
+
jest.setSystemTime(cooldownMs)
|
|
903
|
+
await expect(runSkippedJob(container)).resolves.toBeUndefined()
|
|
904
|
+
expect(recordIndexerLog).toHaveBeenCalledTimes(2)
|
|
905
|
+
jest.setSystemTime(2 * cooldownMs - 1)
|
|
906
|
+
await expect(runSkippedJob(container)).resolves.toBeUndefined()
|
|
907
|
+
expect(recordIndexerLog).toHaveBeenCalledTimes(2)
|
|
908
|
+
jest.setSystemTime(2 * cooldownMs)
|
|
909
|
+
await expect(runSkippedJob(container)).resolves.toBeUndefined()
|
|
910
|
+
expect(recordIndexerLog).toHaveBeenCalledTimes(3)
|
|
911
|
+
expect(searchLoggerWarn).toHaveBeenCalledTimes(3)
|
|
912
|
+
expect(mockSearchIndexer.indexRecordById).not.toHaveBeenCalled()
|
|
913
|
+
})
|
|
914
|
+
|
|
915
|
+
it('keeps reasons independent for the same tenant and job type', async () => {
|
|
916
|
+
const configuredContainer = skipContainer()
|
|
917
|
+
const missingContainer = skipContainer({ missingStrategies: true })
|
|
918
|
+
for (const container of [configuredContainer, missingContainer, configuredContainer, missingContainer]) {
|
|
919
|
+
await runSkippedJob(container)
|
|
920
|
+
}
|
|
921
|
+
expect(recordIndexerLog).toHaveBeenCalledTimes(2)
|
|
922
|
+
expect(jest.mocked(recordIndexerLog).mock.calls.map(([, input]) => input.message)).toEqual([
|
|
923
|
+
'Fulltext strategy not configured (MEILISEARCH_HOST unset?); job skipped without indexing',
|
|
924
|
+
'searchStrategies not available; job skipped without indexing',
|
|
925
|
+
])
|
|
926
|
+
})
|
|
927
|
+
|
|
928
|
+
it('does not renew other tenant or job-type keys prematurely', async () => {
|
|
929
|
+
const container = skipContainer()
|
|
930
|
+
const otherTenantJob = createMockJob<FulltextIndexJobPayload>({ ...job.payload, tenantId: 'tenant-b' })
|
|
931
|
+
const otherTypeJob = createMockJob<FulltextIndexJobPayload>({
|
|
932
|
+
jobType: 'purge',
|
|
933
|
+
tenantId: 'tenant-a',
|
|
934
|
+
entityId: 'test:entity',
|
|
935
|
+
})
|
|
936
|
+
await runSkippedJob(container)
|
|
937
|
+
jest.setSystemTime(cooldownMs - 1)
|
|
938
|
+
await handleFulltextIndexJob(otherTenantJob, createMockJobContext(), container)
|
|
939
|
+
await handleFulltextIndexJob(otherTypeJob, createMockJobContext(), container)
|
|
940
|
+
expect(recordIndexerLog).toHaveBeenCalledTimes(3)
|
|
941
|
+
|
|
942
|
+
jest.setSystemTime(cooldownMs)
|
|
943
|
+
await runSkippedJob(container)
|
|
944
|
+
await handleFulltextIndexJob(otherTenantJob, createMockJobContext(), container)
|
|
945
|
+
await handleFulltextIndexJob(otherTypeJob, createMockJobContext(), container)
|
|
946
|
+
expect(recordIndexerLog).toHaveBeenCalledTimes(4)
|
|
947
|
+
})
|
|
948
|
+
|
|
949
|
+
it('reserves the cooldown before a concurrent status write completes', async () => {
|
|
950
|
+
let releaseWrite: () => void = () => undefined
|
|
951
|
+
jest.mocked(recordIndexerLog).mockImplementationOnce(() => new Promise<void>((resolve) => {
|
|
952
|
+
releaseWrite = resolve
|
|
953
|
+
}))
|
|
954
|
+
const firstJob = runSkippedJob()
|
|
955
|
+
try {
|
|
956
|
+
await expect(runSkippedJob()).resolves.toBeUndefined()
|
|
957
|
+
expect(recordIndexerLog).toHaveBeenCalledTimes(1)
|
|
958
|
+
} finally {
|
|
959
|
+
releaseWrite()
|
|
960
|
+
await firstJob
|
|
961
|
+
}
|
|
962
|
+
})
|
|
963
|
+
|
|
964
|
+
it('renews after a failed status write without rejecting or retrying the skipped job', async () => {
|
|
965
|
+
const actualStatusLog = jest.requireActual<typeof import('@open-mercato/shared/lib/indexers/status-log')>(
|
|
966
|
+
'@open-mercato/shared/lib/indexers/status-log',
|
|
967
|
+
)
|
|
968
|
+
jest.mocked(recordIndexerLog).mockImplementation(actualStatusLog.recordIndexerLog)
|
|
969
|
+
const execute = jest.fn().mockRejectedValueOnce(new Error('database unavailable')).mockResolvedValue([])
|
|
970
|
+
const values = jest.fn().mockReturnValue({ execute })
|
|
971
|
+
mockDb.insertInto.mockReturnValueOnce({ ...createKyselyChain(), values })
|
|
972
|
+
|
|
973
|
+
await expect(runSkippedJob()).resolves.toBeUndefined()
|
|
974
|
+
expect(execute).toHaveBeenCalledTimes(1)
|
|
975
|
+
jest.setSystemTime(cooldownMs - 1)
|
|
976
|
+
await expect(runSkippedJob()).resolves.toBeUndefined()
|
|
977
|
+
expect(recordIndexerLog).toHaveBeenCalledTimes(1)
|
|
978
|
+
|
|
979
|
+
jest.setSystemTime(cooldownMs)
|
|
980
|
+
await expect(runSkippedJob()).resolves.toBeUndefined()
|
|
981
|
+
expect(recordIndexerLog).toHaveBeenCalledTimes(2)
|
|
982
|
+
expect(mockDb.insertInto).toHaveBeenCalledTimes(2)
|
|
983
|
+
expect(mockSearchIndexer.indexRecordById).not.toHaveBeenCalled()
|
|
984
|
+
})
|
|
985
|
+
|
|
986
|
+
it('renews after a missing DB becomes available without rejecting the skipped job', async () => {
|
|
987
|
+
const actualStatusLog = jest.requireActual<typeof import('@open-mercato/shared/lib/indexers/status-log')>(
|
|
988
|
+
'@open-mercato/shared/lib/indexers/status-log',
|
|
989
|
+
)
|
|
990
|
+
jest.mocked(recordIndexerLog).mockImplementation(actualStatusLog.recordIndexerLog)
|
|
991
|
+
await expect(runSkippedJob(skipContainer({ missingDb: true }))).resolves.toBeUndefined()
|
|
992
|
+
expect(mockDb.insertInto).not.toHaveBeenCalled()
|
|
993
|
+
|
|
994
|
+
jest.setSystemTime(cooldownMs - 1)
|
|
995
|
+
await expect(runSkippedJob()).resolves.toBeUndefined()
|
|
996
|
+
expect(recordIndexerLog).toHaveBeenCalledTimes(1)
|
|
997
|
+
expect(mockDb.insertInto).not.toHaveBeenCalled()
|
|
998
|
+
|
|
999
|
+
jest.setSystemTime(cooldownMs)
|
|
1000
|
+
await expect(runSkippedJob()).resolves.toBeUndefined()
|
|
1001
|
+
expect(recordIndexerLog).toHaveBeenCalledTimes(2)
|
|
1002
|
+
expect(mockDb.insertInto).toHaveBeenCalledTimes(1)
|
|
1003
|
+
expect(mockSearchIndexer.indexRecordById).not.toHaveBeenCalled()
|
|
1004
|
+
})
|
|
1005
|
+
})
|
|
1006
|
+
|
|
864
1007
|
it('should throw when fulltext search is not available', async () => {
|
|
865
1008
|
mockFulltextStrategy.isAvailable.mockResolvedValueOnce(false)
|
|
866
1009
|
const job = createMockJob<FulltextIndexJobPayload>({
|
|
@@ -92,18 +92,13 @@ async function advanceFulltextReindexProgress(params: {
|
|
|
92
92
|
* down throws precisely because a registered-but-unreachable Meili is expected
|
|
93
93
|
* to recover; an unregistered strategy is not.
|
|
94
94
|
*
|
|
95
|
-
*
|
|
96
|
-
*
|
|
97
|
-
*
|
|
98
|
-
*
|
|
99
|
-
* is static for the lifetime of the process and rows 2..N carry exactly the
|
|
100
|
-
* information of row 1. The surface an operator reads is the newest 100
|
|
101
|
-
* `indexer_status_logs` rows across ALL sources, so an unthrottled row per job
|
|
102
|
-
* would saturate that window under ordinary write traffic and push out every
|
|
103
|
-
* other source's rows - trading invisibility for a panel showing one signal a
|
|
104
|
-
* hundred times. Do not "fix" this back to per-job.
|
|
95
|
+
* Report at most once per (tenant, job type, reason) every five minutes per
|
|
96
|
+
* process. Reserve the cooldown before the best-effort write so concurrent jobs
|
|
97
|
+
* and failed writes cannot flood the newest 100 status rows. Later skipped jobs
|
|
98
|
+
* renew diagnostics that failed to persist or aged out of the reader's window.
|
|
105
99
|
*/
|
|
106
|
-
const
|
|
100
|
+
const SKIPPED_WITHOUT_INDEXING_COOLDOWN_MS = 5 * 60 * 1000
|
|
101
|
+
const reportedSkipKeys = new Map<string, number>()
|
|
107
102
|
|
|
108
103
|
/** Test seam: the memo above is process-scoped, so a suite must be able to clear it. */
|
|
109
104
|
export function __resetSkippedWithoutIndexingMemo(): void {
|
|
@@ -119,8 +114,10 @@ async function reportSkippedWithoutIndexing(params: {
|
|
|
119
114
|
reason: string
|
|
120
115
|
}): Promise<void> {
|
|
121
116
|
const memoKey = `${params.tenantId}:${params.jobType}:${params.reason}`
|
|
122
|
-
|
|
123
|
-
reportedSkipKeys.
|
|
117
|
+
const now = Date.now()
|
|
118
|
+
const lastReportedAt = reportedSkipKeys.get(memoKey)
|
|
119
|
+
if (lastReportedAt !== undefined && now - lastReportedAt < SKIPPED_WITHOUT_INDEXING_COOLDOWN_MS) return
|
|
120
|
+
reportedSkipKeys.set(memoKey, now)
|
|
124
121
|
|
|
125
122
|
const message = `${params.reason}; job skipped without indexing`
|
|
126
123
|
// `searchWarn`, not `searchError`: this is an operational warning about a
|