@open-mercato/search 0.10.1-develop.7857.1.259928eb85 → 0.10.1-develop.7863.1.335fb1fe52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
FULLTEXT_INDEXING_QUEUE_NAME
|
|
3
|
+
} from "../../../queue/fulltext-indexing.js";
|
|
2
4
|
import { recordIndexerLog } from "@open-mercato/shared/lib/indexers/status-log";
|
|
3
5
|
import { recordIndexerError } from "@open-mercato/shared/lib/indexers/error-log";
|
|
4
|
-
import { searchDebug, searchDebugWarn, searchError } from "../../../lib/debug.js";
|
|
6
|
+
import { searchDebug, searchDebugWarn, searchError, searchWarn } from "../../../lib/debug.js";
|
|
5
7
|
import { clearReindexLock, updateReindexProgress } from "../lib/reindex-lock.js";
|
|
6
8
|
import { hasActiveReindexProgress, incrementReindexProgress } from "../lib/reindex-progress.js";
|
|
7
9
|
const DEFAULT_CONCURRENCY = 2;
|
|
@@ -45,8 +47,46 @@ async function advanceFulltextReindexProgress(params) {
|
|
|
45
47
|
await updateReindexProgress(params.db, params.tenantId, "fulltext", params.delta, params.organizationId ?? null);
|
|
46
48
|
}
|
|
47
49
|
}
|
|
50
|
+
const reportedSkipKeys = /* @__PURE__ */ new Set();
|
|
51
|
+
function __resetSkippedWithoutIndexingMemo() {
|
|
52
|
+
reportedSkipKeys.clear();
|
|
53
|
+
}
|
|
54
|
+
async function reportSkippedWithoutIndexing(params) {
|
|
55
|
+
const memoKey = `${params.tenantId}:${params.jobType}:${params.reason}`;
|
|
56
|
+
if (reportedSkipKeys.has(memoKey)) return;
|
|
57
|
+
reportedSkipKeys.add(memoKey);
|
|
58
|
+
const message = `${params.reason}; job skipped without indexing`;
|
|
59
|
+
searchWarn("fulltext-index.worker", message, {
|
|
60
|
+
jobId: params.jobId,
|
|
61
|
+
tenantId: params.tenantId,
|
|
62
|
+
jobType: params.jobType
|
|
63
|
+
});
|
|
64
|
+
await recordIndexerLog(
|
|
65
|
+
{ em: params.em ?? void 0 },
|
|
66
|
+
{
|
|
67
|
+
source: "fulltext",
|
|
68
|
+
handler: `worker:fulltext:${params.jobType}`,
|
|
69
|
+
level: "warn",
|
|
70
|
+
message,
|
|
71
|
+
tenantId: params.tenantId,
|
|
72
|
+
// No `organizationId`: every other recordIndexerLog call in this handler
|
|
73
|
+
// omits it, so all existing `source: 'fulltext'` rows persist with
|
|
74
|
+
// organization_id = NULL - and the unrestricted indexer-status view filters
|
|
75
|
+
// on `organization_id IS NULL` (query_index/api/status.ts). A non-null org
|
|
76
|
+
// here would hide this row in exactly the view an operator lands on when
|
|
77
|
+
// asking "did indexing run?". The value stays in `details` for triage.
|
|
78
|
+
details: {
|
|
79
|
+
jobId: params.jobId,
|
|
80
|
+
jobType: params.jobType,
|
|
81
|
+
organizationId: params.organizationId,
|
|
82
|
+
skippedWithoutIndexing: true
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
);
|
|
86
|
+
}
|
|
48
87
|
async function handleFulltextIndexJob(job, jobCtx, ctx) {
|
|
49
88
|
const { jobType, tenantId } = job.payload;
|
|
89
|
+
const payloadOrganizationId = "organizationId" in job.payload ? job.payload.organizationId ?? null : null;
|
|
50
90
|
if (!tenantId) {
|
|
51
91
|
searchDebugWarn("fulltext-index.worker", "Skipping job with missing tenantId", {
|
|
52
92
|
jobId: jobCtx.jobId,
|
|
@@ -76,11 +116,25 @@ async function handleFulltextIndexJob(job, jobCtx, ctx) {
|
|
|
76
116
|
(s) => s?.id === "fulltext"
|
|
77
117
|
);
|
|
78
118
|
} catch {
|
|
79
|
-
|
|
119
|
+
await reportSkippedWithoutIndexing({
|
|
120
|
+
em,
|
|
121
|
+
jobType,
|
|
122
|
+
tenantId,
|
|
123
|
+
organizationId: payloadOrganizationId,
|
|
124
|
+
jobId: jobCtx.jobId,
|
|
125
|
+
reason: "searchStrategies not available"
|
|
126
|
+
});
|
|
80
127
|
return;
|
|
81
128
|
}
|
|
82
129
|
if (!fulltextStrategy) {
|
|
83
|
-
|
|
130
|
+
await reportSkippedWithoutIndexing({
|
|
131
|
+
em,
|
|
132
|
+
jobType,
|
|
133
|
+
tenantId,
|
|
134
|
+
organizationId: payloadOrganizationId,
|
|
135
|
+
jobId: jobCtx.jobId,
|
|
136
|
+
reason: "Fulltext strategy not configured (MEILISEARCH_HOST unset?)"
|
|
137
|
+
});
|
|
84
138
|
return;
|
|
85
139
|
}
|
|
86
140
|
const isAvailable = await fulltextStrategy.isAvailable();
|
|
@@ -263,6 +317,7 @@ async function handle(job, ctx) {
|
|
|
263
317
|
return handleFulltextIndexJob(job, ctx, ctx);
|
|
264
318
|
}
|
|
265
319
|
export {
|
|
320
|
+
__resetSkippedWithoutIndexingMemo,
|
|
266
321
|
handle as default,
|
|
267
322
|
handleFulltextIndexJob,
|
|
268
323
|
metadata
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../../src/modules/search/workers/fulltext-index.worker.ts"],
|
|
4
|
-
"sourcesContent": ["import type { QueuedJob, JobContext, WorkerMeta } from '@open-mercato/queue'\nimport type { Kysely } from 'kysely'\nimport { FULLTEXT_INDEXING_QUEUE_NAME, type FulltextIndexJobPayload } from '../../../queue/fulltext-indexing'\nimport type { FullTextSearchStrategy } from '../../../strategies/fulltext.strategy'\nimport type { SearchIndexer } from '../../../indexer/search-indexer'\nimport type { EntityManager } from '@mikro-orm/postgresql'\n\nimport type { EntityId } from '@open-mercato/shared/modules/entities'\nimport { recordIndexerLog } from '@open-mercato/shared/lib/indexers/status-log'\nimport { recordIndexerError } from '@open-mercato/shared/lib/indexers/error-log'\nimport type { ProgressService } from '@open-mercato/core/modules/progress/lib/progressService'\nimport { searchDebug, searchDebugWarn, searchError } from '../../../lib/debug'\nimport { clearReindexLock, updateReindexProgress } from '../lib/reindex-lock'\nimport { hasActiveReindexProgress, incrementReindexProgress } from '../lib/reindex-progress'\n\n// Worker metadata for auto-discovery\nconst DEFAULT_CONCURRENCY = 2\nconst envConcurrency = process.env.WORKERS_FULLTEXT_INDEXING_CONCURRENCY\n\nexport const metadata: WorkerMeta = {\n queue: FULLTEXT_INDEXING_QUEUE_NAME,\n concurrency: envConcurrency ? parseInt(envConcurrency, 10) : DEFAULT_CONCURRENCY,\n}\n\ntype HandlerContext = { resolve: <T = unknown>(name: string) => T }\n\nasync function advanceFulltextReindexProgress(params: {\n db: Kysely<any> | null\n em: EntityManager | null\n progressService: ProgressService | null\n tenantId: string\n organizationId?: string | null\n delta: number\n}): Promise<void> {\n if (!Number.isFinite(params.delta) || params.delta <= 0) return\n\n if (params.progressService && params.em) {\n const hasActiveProgress = await hasActiveReindexProgress({\n em: params.em,\n type: 'fulltext',\n tenantId: params.tenantId,\n organizationId: params.organizationId ?? null,\n })\n\n if (!hasActiveProgress) {\n if (params.db) {\n await clearReindexLock(params.db, params.tenantId, 'fulltext', params.organizationId ?? null)\n }\n return\n }\n\n if (params.db) {\n await updateReindexProgress(params.db, params.tenantId, 'fulltext', params.delta, params.organizationId ?? null)\n }\n\n const completed = await incrementReindexProgress({\n em: params.em,\n progressService: params.progressService,\n type: 'fulltext',\n tenantId: params.tenantId,\n organizationId: params.organizationId ?? null,\n delta: params.delta,\n })\n if (completed && params.db) {\n await clearReindexLock(params.db, params.tenantId, 'fulltext', params.organizationId ?? null)\n }\n return\n }\n\n if (params.db) {\n await updateReindexProgress(params.db, params.tenantId, 'fulltext', params.delta, params.organizationId ?? null)\n }\n}\n\n/**\n * Process a fulltext indexing job.\n *\n * This handler processes single record indexing, batch indexing, deletion, and purge\n * operations for the fulltext search strategy.\n *\n * Single-record jobs load fresh data via searchIndexer.indexRecordById(). Batch jobs load\n * fresh data per record via searchIndexer.indexRecordsById(), which flushes the whole batch\n * through a single bulk write instead of one write per record.\n *\n * @param job - The queued job containing payload\n * @param jobCtx - Queue job context with job ID and attempt info\n * @param ctx - DI container context for resolving services\n */\nexport async function handleFulltextIndexJob(\n job: QueuedJob<FulltextIndexJobPayload>,\n jobCtx: JobContext,\n ctx: HandlerContext,\n): Promise<void> {\n const { jobType, tenantId } = job.payload\n\n if (!tenantId) {\n searchDebugWarn('fulltext-index.worker', 'Skipping job with missing tenantId', {\n jobId: jobCtx.jobId,\n jobType,\n })\n return\n }\n\n // Resolve EntityManager for logging and Kysely for database queries\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n let em: any | null = null\n let db: Kysely<any> | null = null\n try {\n em = ctx.resolve('em') as EntityManager\n db = (em as unknown as { getKysely: () => Kysely<any> }).getKysely()\n } catch {\n em = null\n db = null\n }\n\n // Resolve searchIndexer for loading fresh data\n let searchIndexer: SearchIndexer | undefined\n try {\n searchIndexer = ctx.resolve<SearchIndexer>('searchIndexer')\n } catch {\n searchDebugWarn('fulltext-index.worker', 'searchIndexer not available')\n }\n\n // Resolve fulltext strategy\n let fulltextStrategy: FullTextSearchStrategy | undefined\n try {\n const searchStrategies = ctx.resolve<unknown[]>('searchStrategies')\n fulltextStrategy = searchStrategies?.find(\n (s: unknown) => (s as { id?: string })?.id === 'fulltext',\n ) as FullTextSearchStrategy | undefined\n } catch {\n searchDebugWarn('fulltext-index.worker', 'searchStrategies not available')\n return\n }\n\n if (!fulltextStrategy) {\n searchDebugWarn('fulltext-index.worker', 'Fulltext strategy not configured')\n return\n }\n\n // Check if fulltext is available\n const isAvailable = await fulltextStrategy.isAvailable()\n if (!isAvailable) {\n throw new Error('Fulltext search is not available') // Will trigger retry\n }\n\n try {\n let progressService: ProgressService | null = null\n try {\n progressService = ctx.resolve<ProgressService>('progressService')\n } catch {\n progressService = null\n }\n\n // ========== SINGLE INDEX: Use searchIndexer.indexRecordById() for fresh data ==========\n if (jobType === 'index') {\n const { entityType, recordId, organizationId } = job.payload as {\n entityType: string\n recordId: string\n organizationId?: string | null\n }\n\n if (!entityType || !recordId) {\n searchDebugWarn('fulltext-index.worker', 'Skipping index with missing fields', {\n jobId: jobCtx.jobId,\n entityType,\n recordId,\n })\n return\n }\n\n if (!searchIndexer) {\n throw new Error('searchIndexer not available for single-record index')\n }\n\n const result = await searchIndexer.indexRecordById({\n entityId: entityType as EntityId,\n recordId,\n tenantId,\n organizationId,\n })\n\n searchDebug('fulltext-index.worker', 'Indexed single record to fulltext', {\n jobId: jobCtx.jobId,\n tenantId,\n entityType,\n recordId,\n action: result.action,\n })\n\n await recordIndexerLog(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: 'worker:fulltext:index',\n message: `Indexed record to fulltext (${result.action})`,\n entityType,\n recordId,\n tenantId,\n details: { jobId: jobCtx.jobId },\n },\n )\n return\n }\n\n // ========== BATCH-INDEX: Load fresh data, write the whole batch in one call ==========\n if (jobType === 'batch-index') {\n const { records, organizationId } = job.payload\n if (!records || records.length === 0) {\n searchDebugWarn('fulltext-index.worker', 'Skipping batch-index with no records', {\n jobId: jobCtx.jobId,\n })\n return\n }\n\n if (!searchIndexer) {\n throw new Error('searchIndexer not available for batch indexing')\n }\n\n // Load and index the whole batch through a single bulk write instead of\n // one indexRecordById() call per record.\n const { indexed: successCount, skipped: skippedCount } = await searchIndexer.indexRecordsById({\n items: records.map(({ entityId, recordId }) => ({ entityId: entityId as EntityId, recordId })),\n tenantId,\n organizationId,\n })\n\n await advanceFulltextReindexProgress({\n db,\n em,\n progressService,\n tenantId,\n organizationId: organizationId ?? null,\n delta: records.length,\n })\n\n searchDebug('fulltext-index.worker', 'Batch indexed to fulltext', {\n jobId: jobCtx.jobId,\n tenantId,\n requestedCount: records.length,\n successCount,\n skippedCount,\n })\n\n await recordIndexerLog(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: 'worker:fulltext:batch-index',\n message: `Indexed ${successCount}/${records.length} records to fulltext`,\n tenantId,\n details: { jobId: jobCtx.jobId, requestedCount: records.length, successCount, skippedCount },\n },\n )\n return\n }\n\n // ========== DELETE ==========\n if (jobType === 'delete') {\n const { entityId, recordId } = job.payload\n if (!entityId || !recordId) {\n searchDebugWarn('fulltext-index.worker', 'Skipping delete with missing fields', {\n jobId: jobCtx.jobId,\n entityId,\n recordId,\n })\n return\n }\n\n await fulltextStrategy.delete(entityId, recordId, tenantId)\n\n searchDebug('fulltext-index.worker', 'Deleted from fulltext', {\n jobId: jobCtx.jobId,\n tenantId,\n entityId,\n recordId,\n })\n\n await recordIndexerLog(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: 'worker:fulltext:delete',\n message: `Deleted record from fulltext`,\n entityType: entityId,\n recordId,\n tenantId,\n details: { jobId: jobCtx.jobId },\n },\n )\n return\n }\n\n // ========== PURGE ==========\n if (jobType === 'purge') {\n const { entityId } = job.payload\n if (!entityId) {\n searchDebugWarn('fulltext-index.worker', 'Skipping purge with missing entityId', {\n jobId: jobCtx.jobId,\n })\n return\n }\n\n await fulltextStrategy.purge(entityId, tenantId)\n\n searchDebug('fulltext-index.worker', 'Purged entity from fulltext', {\n jobId: jobCtx.jobId,\n tenantId,\n entityId,\n })\n\n await recordIndexerLog(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: 'worker:fulltext:purge',\n message: `Purged entity from fulltext`,\n entityType: entityId,\n tenantId,\n details: { jobId: jobCtx.jobId },\n },\n )\n return\n }\n } catch (error) {\n searchError('fulltext-index.worker', `Failed to ${jobType}`, {\n jobId: jobCtx.jobId,\n tenantId,\n error: error instanceof Error ? error.message : error,\n attemptNumber: jobCtx.attemptNumber,\n })\n\n const entityId = 'entityId' in job.payload ? job.payload.entityId :\n 'entityType' in job.payload ? (job.payload as { entityType?: string }).entityType : undefined\n const recordId = 'recordId' in job.payload ? job.payload.recordId : undefined\n\n await recordIndexerError(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: `worker:fulltext:${jobType}`,\n error,\n entityType: entityId,\n recordId,\n tenantId,\n payload: job.payload,\n },\n )\n\n // Re-throw to let the queue handle retry logic\n throw error\n }\n}\n\n/**\n * Default export for worker auto-discovery.\n * Wraps handleFulltextIndexJob to match the expected handler signature.\n */\nexport default async function handle(\n job: QueuedJob<FulltextIndexJobPayload>,\n ctx: JobContext & HandlerContext\n): Promise<void> {\n return handleFulltextIndexJob(job, ctx, ctx)\n}\n"],
|
|
5
|
-
"mappings": "AAEA,
|
|
4
|
+
"sourcesContent": ["import type { QueuedJob, JobContext, WorkerMeta } from '@open-mercato/queue'\nimport type { Kysely } from 'kysely'\nimport {\n FULLTEXT_INDEXING_QUEUE_NAME,\n type FulltextIndexJobPayload,\n type FulltextIndexJobType,\n} from '../../../queue/fulltext-indexing'\nimport type { FullTextSearchStrategy } from '../../../strategies/fulltext.strategy'\nimport type { SearchIndexer } from '../../../indexer/search-indexer'\nimport type { EntityManager } from '@mikro-orm/postgresql'\n\nimport type { EntityId } from '@open-mercato/shared/modules/entities'\nimport { recordIndexerLog } from '@open-mercato/shared/lib/indexers/status-log'\nimport { recordIndexerError } from '@open-mercato/shared/lib/indexers/error-log'\nimport type { ProgressService } from '@open-mercato/core/modules/progress/lib/progressService'\nimport { searchDebug, searchDebugWarn, searchError, searchWarn } from '../../../lib/debug'\nimport { clearReindexLock, updateReindexProgress } from '../lib/reindex-lock'\nimport { hasActiveReindexProgress, incrementReindexProgress } from '../lib/reindex-progress'\n\n// Worker metadata for auto-discovery\nconst DEFAULT_CONCURRENCY = 2\nconst envConcurrency = process.env.WORKERS_FULLTEXT_INDEXING_CONCURRENCY\n\nexport const metadata: WorkerMeta = {\n queue: FULLTEXT_INDEXING_QUEUE_NAME,\n concurrency: envConcurrency ? parseInt(envConcurrency, 10) : DEFAULT_CONCURRENCY,\n}\n\ntype HandlerContext = { resolve: <T = unknown>(name: string) => T }\n\nasync function advanceFulltextReindexProgress(params: {\n db: Kysely<any> | null\n em: EntityManager | null\n progressService: ProgressService | null\n tenantId: string\n organizationId?: string | null\n delta: number\n}): Promise<void> {\n if (!Number.isFinite(params.delta) || params.delta <= 0) return\n\n if (params.progressService && params.em) {\n const hasActiveProgress = await hasActiveReindexProgress({\n em: params.em,\n type: 'fulltext',\n tenantId: params.tenantId,\n organizationId: params.organizationId ?? null,\n })\n\n if (!hasActiveProgress) {\n if (params.db) {\n await clearReindexLock(params.db, params.tenantId, 'fulltext', params.organizationId ?? null)\n }\n return\n }\n\n if (params.db) {\n await updateReindexProgress(params.db, params.tenantId, 'fulltext', params.delta, params.organizationId ?? null)\n }\n\n const completed = await incrementReindexProgress({\n em: params.em,\n progressService: params.progressService,\n type: 'fulltext',\n tenantId: params.tenantId,\n organizationId: params.organizationId ?? null,\n delta: params.delta,\n })\n if (completed && params.db) {\n await clearReindexLock(params.db, params.tenantId, 'fulltext', params.organizationId ?? null)\n }\n return\n }\n\n if (params.db) {\n await updateReindexProgress(params.db, params.tenantId, 'fulltext', params.delta, params.organizationId ?? null)\n }\n}\n\n/**\n * Report a job that returned without indexing anything.\n *\n * A fulltext strategy that was never registered (no `MEILISEARCH_HOST`, so\n * `createFulltextDriver()` returns null and `di.ts` pushes no strategy) makes\n * every job a no-op. The bare `return` that follows leaves BullMQ reporting\n * `completed` with zero failures and writes no `indexer_status_logs` row, so\n * \"fulltext is not configured\" and \"everything indexed fine\" are\n * indistinguishable from both signals an operator has.\n *\n * Deliberately does NOT throw. A deployment may legitimately run without\n * Meilisearch, and throwing would retry these jobs forever, turning a\n * configuration choice into a queue backlog. The `isAvailable()` check further\n * down throws precisely because a registered-but-unreachable Meili is expected\n * to recover; an unregistered strategy is not.\n *\n * Reported ONCE per (tenant, job type, reason) per process, deliberately.\n * `searchStrategies` and `searchIndexer` are registered together in one\n * `container.register` call, from a `strategies` array that only receives a\n * `FullTextSearchStrategy` when the driver exists - so the outcome of this guard\n * is static for the lifetime of the process and rows 2..N carry exactly the\n * information of row 1. The surface an operator reads is the newest 100\n * `indexer_status_logs` rows across ALL sources, so an unthrottled row per job\n * would saturate that window under ordinary write traffic and push out every\n * other source's rows - trading invisibility for a panel showing one signal a\n * hundred times. Do not \"fix\" this back to per-job.\n */\nconst reportedSkipKeys = new Set<string>()\n\n/** Test seam: the memo above is process-scoped, so a suite must be able to clear it. */\nexport function __resetSkippedWithoutIndexingMemo(): void {\n reportedSkipKeys.clear()\n}\n\nasync function reportSkippedWithoutIndexing(params: {\n em: EntityManager | null\n jobType: FulltextIndexJobType\n tenantId: string\n organizationId: string | null\n jobId: string\n reason: string\n}): Promise<void> {\n const memoKey = `${params.tenantId}:${params.jobType}:${params.reason}`\n if (reportedSkipKeys.has(memoKey)) return\n reportedSkipKeys.add(memoKey)\n\n const message = `${params.reason}; job skipped without indexing`\n // `searchWarn`, not `searchError`: this is an operational warning about a\n // configuration the guard above calls legitimate, it is the level the sibling\n // vector worker uses for the same skip, and it agrees with the `level: 'warn'`\n // of the row written beside it. It is un-gated by OM_SEARCH_DEBUG either way,\n // which is the property this fix needs.\n searchWarn('fulltext-index.worker', message, {\n jobId: params.jobId,\n tenantId: params.tenantId,\n jobType: params.jobType,\n })\n await recordIndexerLog(\n { em: params.em ?? undefined },\n {\n source: 'fulltext',\n handler: `worker:fulltext:${params.jobType}`,\n level: 'warn',\n message,\n tenantId: params.tenantId,\n // No `organizationId`: every other recordIndexerLog call in this handler\n // omits it, so all existing `source: 'fulltext'` rows persist with\n // organization_id = NULL - and the unrestricted indexer-status view filters\n // on `organization_id IS NULL` (query_index/api/status.ts). A non-null org\n // here would hide this row in exactly the view an operator lands on when\n // asking \"did indexing run?\". The value stays in `details` for triage.\n details: {\n jobId: params.jobId,\n jobType: params.jobType,\n organizationId: params.organizationId,\n skippedWithoutIndexing: true,\n },\n },\n )\n}\n\n/**\n * Process a fulltext indexing job.\n *\n * This handler processes single record indexing, batch indexing, deletion, and purge\n * operations for the fulltext search strategy.\n *\n * Single-record jobs load fresh data via searchIndexer.indexRecordById(). Batch jobs load\n * fresh data per record via searchIndexer.indexRecordsById(), which flushes the whole batch\n * through a single bulk write instead of one write per record.\n *\n * @param job - The queued job containing payload\n * @param jobCtx - Queue job context with job ID and attempt info\n * @param ctx - DI container context for resolving services\n */\nexport async function handleFulltextIndexJob(\n job: QueuedJob<FulltextIndexJobPayload>,\n jobCtx: JobContext,\n ctx: HandlerContext,\n): Promise<void> {\n const { jobType, tenantId } = job.payload\n const payloadOrganizationId =\n 'organizationId' in job.payload ? job.payload.organizationId ?? null : null\n\n if (!tenantId) {\n searchDebugWarn('fulltext-index.worker', 'Skipping job with missing tenantId', {\n jobId: jobCtx.jobId,\n jobType,\n })\n return\n }\n\n // Resolve EntityManager for logging and Kysely for database queries\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n let em: any | null = null\n let db: Kysely<any> | null = null\n try {\n em = ctx.resolve('em') as EntityManager\n db = (em as unknown as { getKysely: () => Kysely<any> }).getKysely()\n } catch {\n em = null\n db = null\n }\n\n // Resolve searchIndexer for loading fresh data\n let searchIndexer: SearchIndexer | undefined\n try {\n searchIndexer = ctx.resolve<SearchIndexer>('searchIndexer')\n } catch {\n // Left at debug level to match vector-index.worker.ts for the identical\n // condition. It is also close to unreachable: `searchIndexer` and\n // `searchStrategies` are registered in the same `container.register` call, so\n // the only way this throws is that search DI never ran - in which case the\n // next guard reports it anyway.\n searchDebugWarn('fulltext-index.worker', 'searchIndexer not available')\n }\n\n // Resolve fulltext strategy\n let fulltextStrategy: FullTextSearchStrategy | undefined\n try {\n const searchStrategies = ctx.resolve<unknown[]>('searchStrategies')\n fulltextStrategy = searchStrategies?.find(\n (s: unknown) => (s as { id?: string })?.id === 'fulltext',\n ) as FullTextSearchStrategy | undefined\n } catch {\n await reportSkippedWithoutIndexing({\n em,\n jobType,\n tenantId,\n organizationId: payloadOrganizationId,\n jobId: jobCtx.jobId,\n reason: 'searchStrategies not available',\n })\n return\n }\n\n if (!fulltextStrategy) {\n await reportSkippedWithoutIndexing({\n em,\n jobType,\n tenantId,\n organizationId: payloadOrganizationId,\n jobId: jobCtx.jobId,\n reason: 'Fulltext strategy not configured (MEILISEARCH_HOST unset?)',\n })\n return\n }\n\n // Check if fulltext is available\n const isAvailable = await fulltextStrategy.isAvailable()\n if (!isAvailable) {\n throw new Error('Fulltext search is not available') // Will trigger retry\n }\n\n try {\n let progressService: ProgressService | null = null\n try {\n progressService = ctx.resolve<ProgressService>('progressService')\n } catch {\n progressService = null\n }\n\n // ========== SINGLE INDEX: Use searchIndexer.indexRecordById() for fresh data ==========\n if (jobType === 'index') {\n const { entityType, recordId, organizationId } = job.payload as {\n entityType: string\n recordId: string\n organizationId?: string | null\n }\n\n if (!entityType || !recordId) {\n searchDebugWarn('fulltext-index.worker', 'Skipping index with missing fields', {\n jobId: jobCtx.jobId,\n entityType,\n recordId,\n })\n return\n }\n\n if (!searchIndexer) {\n throw new Error('searchIndexer not available for single-record index')\n }\n\n const result = await searchIndexer.indexRecordById({\n entityId: entityType as EntityId,\n recordId,\n tenantId,\n organizationId,\n })\n\n searchDebug('fulltext-index.worker', 'Indexed single record to fulltext', {\n jobId: jobCtx.jobId,\n tenantId,\n entityType,\n recordId,\n action: result.action,\n })\n\n await recordIndexerLog(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: 'worker:fulltext:index',\n message: `Indexed record to fulltext (${result.action})`,\n entityType,\n recordId,\n tenantId,\n details: { jobId: jobCtx.jobId },\n },\n )\n return\n }\n\n // ========== BATCH-INDEX: Load fresh data, write the whole batch in one call ==========\n if (jobType === 'batch-index') {\n const { records, organizationId } = job.payload\n if (!records || records.length === 0) {\n searchDebugWarn('fulltext-index.worker', 'Skipping batch-index with no records', {\n jobId: jobCtx.jobId,\n })\n return\n }\n\n if (!searchIndexer) {\n throw new Error('searchIndexer not available for batch indexing')\n }\n\n // Load and index the whole batch through a single bulk write instead of\n // one indexRecordById() call per record.\n const { indexed: successCount, skipped: skippedCount } = await searchIndexer.indexRecordsById({\n items: records.map(({ entityId, recordId }) => ({ entityId: entityId as EntityId, recordId })),\n tenantId,\n organizationId,\n })\n\n await advanceFulltextReindexProgress({\n db,\n em,\n progressService,\n tenantId,\n organizationId: organizationId ?? null,\n delta: records.length,\n })\n\n searchDebug('fulltext-index.worker', 'Batch indexed to fulltext', {\n jobId: jobCtx.jobId,\n tenantId,\n requestedCount: records.length,\n successCount,\n skippedCount,\n })\n\n await recordIndexerLog(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: 'worker:fulltext:batch-index',\n message: `Indexed ${successCount}/${records.length} records to fulltext`,\n tenantId,\n details: { jobId: jobCtx.jobId, requestedCount: records.length, successCount, skippedCount },\n },\n )\n return\n }\n\n // ========== DELETE ==========\n if (jobType === 'delete') {\n const { entityId, recordId } = job.payload\n if (!entityId || !recordId) {\n searchDebugWarn('fulltext-index.worker', 'Skipping delete with missing fields', {\n jobId: jobCtx.jobId,\n entityId,\n recordId,\n })\n return\n }\n\n await fulltextStrategy.delete(entityId, recordId, tenantId)\n\n searchDebug('fulltext-index.worker', 'Deleted from fulltext', {\n jobId: jobCtx.jobId,\n tenantId,\n entityId,\n recordId,\n })\n\n await recordIndexerLog(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: 'worker:fulltext:delete',\n message: `Deleted record from fulltext`,\n entityType: entityId,\n recordId,\n tenantId,\n details: { jobId: jobCtx.jobId },\n },\n )\n return\n }\n\n // ========== PURGE ==========\n if (jobType === 'purge') {\n const { entityId } = job.payload\n if (!entityId) {\n searchDebugWarn('fulltext-index.worker', 'Skipping purge with missing entityId', {\n jobId: jobCtx.jobId,\n })\n return\n }\n\n await fulltextStrategy.purge(entityId, tenantId)\n\n searchDebug('fulltext-index.worker', 'Purged entity from fulltext', {\n jobId: jobCtx.jobId,\n tenantId,\n entityId,\n })\n\n await recordIndexerLog(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: 'worker:fulltext:purge',\n message: `Purged entity from fulltext`,\n entityType: entityId,\n tenantId,\n details: { jobId: jobCtx.jobId },\n },\n )\n return\n }\n } catch (error) {\n searchError('fulltext-index.worker', `Failed to ${jobType}`, {\n jobId: jobCtx.jobId,\n tenantId,\n error: error instanceof Error ? error.message : error,\n attemptNumber: jobCtx.attemptNumber,\n })\n\n const entityId = 'entityId' in job.payload ? job.payload.entityId :\n 'entityType' in job.payload ? (job.payload as { entityType?: string }).entityType : undefined\n const recordId = 'recordId' in job.payload ? job.payload.recordId : undefined\n\n await recordIndexerError(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: `worker:fulltext:${jobType}`,\n error,\n entityType: entityId,\n recordId,\n tenantId,\n payload: job.payload,\n },\n )\n\n // Re-throw to let the queue handle retry logic\n throw error\n }\n}\n\n/**\n * Default export for worker auto-discovery.\n * Wraps handleFulltextIndexJob to match the expected handler signature.\n */\nexport default async function handle(\n job: QueuedJob<FulltextIndexJobPayload>,\n ctx: JobContext & HandlerContext\n): Promise<void> {\n return handleFulltextIndexJob(job, ctx, ctx)\n}\n"],
|
|
5
|
+
"mappings": "AAEA;AAAA,EACE;AAAA,OAGK;AAMP,SAAS,wBAAwB;AACjC,SAAS,0BAA0B;AAEnC,SAAS,aAAa,iBAAiB,aAAa,kBAAkB;AACtE,SAAS,kBAAkB,6BAA6B;AACxD,SAAS,0BAA0B,gCAAgC;AAGnE,MAAM,sBAAsB;AAC5B,MAAM,iBAAiB,QAAQ,IAAI;AAE5B,MAAM,WAAuB;AAAA,EAClC,OAAO;AAAA,EACP,aAAa,iBAAiB,SAAS,gBAAgB,EAAE,IAAI;AAC/D;AAIA,eAAe,+BAA+B,QAO5B;AAChB,MAAI,CAAC,OAAO,SAAS,OAAO,KAAK,KAAK,OAAO,SAAS,EAAG;AAEzD,MAAI,OAAO,mBAAmB,OAAO,IAAI;AACvC,UAAM,oBAAoB,MAAM,yBAAyB;AAAA,MACvD,IAAI,OAAO;AAAA,MACX,MAAM;AAAA,MACN,UAAU,OAAO;AAAA,MACjB,gBAAgB,OAAO,kBAAkB;AAAA,IAC3C,CAAC;AAED,QAAI,CAAC,mBAAmB;AACtB,UAAI,OAAO,IAAI;AACb,cAAM,iBAAiB,OAAO,IAAI,OAAO,UAAU,YAAY,OAAO,kBAAkB,IAAI;AAAA,MAC9F;AACA;AAAA,IACF;AAEA,QAAI,OAAO,IAAI;AACb,YAAM,sBAAsB,OAAO,IAAI,OAAO,UAAU,YAAY,OAAO,OAAO,OAAO,kBAAkB,IAAI;AAAA,IACjH;AAEA,UAAM,YAAY,MAAM,yBAAyB;AAAA,MAC/C,IAAI,OAAO;AAAA,MACX,iBAAiB,OAAO;AAAA,MACxB,MAAM;AAAA,MACN,UAAU,OAAO;AAAA,MACjB,gBAAgB,OAAO,kBAAkB;AAAA,MACzC,OAAO,OAAO;AAAA,IAChB,CAAC;AACD,QAAI,aAAa,OAAO,IAAI;AAC1B,YAAM,iBAAiB,OAAO,IAAI,OAAO,UAAU,YAAY,OAAO,kBAAkB,IAAI;AAAA,IAC9F;AACA;AAAA,EACF;AAEA,MAAI,OAAO,IAAI;AACb,UAAM,sBAAsB,OAAO,IAAI,OAAO,UAAU,YAAY,OAAO,OAAO,OAAO,kBAAkB,IAAI;AAAA,EACjH;AACF;AA6BA,MAAM,mBAAmB,oBAAI,IAAY;AAGlC,SAAS,oCAA0C;AACxD,mBAAiB,MAAM;AACzB;AAEA,eAAe,6BAA6B,QAO1B;AAChB,QAAM,UAAU,GAAG,OAAO,QAAQ,IAAI,OAAO,OAAO,IAAI,OAAO,MAAM;AACrE,MAAI,iBAAiB,IAAI,OAAO,EAAG;AACnC,mBAAiB,IAAI,OAAO;AAE5B,QAAM,UAAU,GAAG,OAAO,MAAM;AAMhC,aAAW,yBAAyB,SAAS;AAAA,IAC3C,OAAO,OAAO;AAAA,IACd,UAAU,OAAO;AAAA,IACjB,SAAS,OAAO;AAAA,EAClB,CAAC;AACD,QAAM;AAAA,IACJ,EAAE,IAAI,OAAO,MAAM,OAAU;AAAA,IAC7B;AAAA,MACE,QAAQ;AAAA,MACR,SAAS,mBAAmB,OAAO,OAAO;AAAA,MAC1C,OAAO;AAAA,MACP;AAAA,MACA,UAAU,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAOjB,SAAS;AAAA,QACP,OAAO,OAAO;AAAA,QACd,SAAS,OAAO;AAAA,QAChB,gBAAgB,OAAO;AAAA,QACvB,wBAAwB;AAAA,MAC1B;AAAA,IACF;AAAA,EACF;AACF;AAgBA,eAAsB,uBACpB,KACA,QACA,KACe;AACf,QAAM,EAAE,SAAS,SAAS,IAAI,IAAI;AAClC,QAAM,wBACJ,oBAAoB,IAAI,UAAU,IAAI,QAAQ,kBAAkB,OAAO;AAEzE,MAAI,CAAC,UAAU;AACb,oBAAgB,yBAAyB,sCAAsC;AAAA,MAC7E,OAAO,OAAO;AAAA,MACd;AAAA,IACF,CAAC;AACD;AAAA,EACF;AAIA,MAAI,KAAiB;AACrB,MAAI,KAAyB;AAC7B,MAAI;AACF,SAAK,IAAI,QAAQ,IAAI;AACrB,SAAM,GAAmD,UAAU;AAAA,EACrE,QAAQ;AACN,SAAK;AACL,SAAK;AAAA,EACP;AAGA,MAAI;AACJ,MAAI;AACF,oBAAgB,IAAI,QAAuB,eAAe;AAAA,EAC5D,QAAQ;AAMN,oBAAgB,yBAAyB,6BAA6B;AAAA,EACxE;AAGA,MAAI;AACJ,MAAI;AACF,UAAM,mBAAmB,IAAI,QAAmB,kBAAkB;AAClE,uBAAmB,kBAAkB;AAAA,MACnC,CAAC,MAAgB,GAAuB,OAAO;AAAA,IACjD;AAAA,EACF,QAAQ;AACN,UAAM,6BAA6B;AAAA,MACjC;AAAA,MACA;AAAA,MACA;AAAA,MACA,gBAAgB;AAAA,MAChB,OAAO,OAAO;AAAA,MACd,QAAQ;AAAA,IACV,CAAC;AACD;AAAA,EACF;AAEA,MAAI,CAAC,kBAAkB;AACrB,UAAM,6BAA6B;AAAA,MACjC;AAAA,MACA;AAAA,MACA;AAAA,MACA,gBAAgB;AAAA,MAChB,OAAO,OAAO;AAAA,MACd,QAAQ;AAAA,IACV,CAAC;AACD;AAAA,EACF;AAGA,QAAM,cAAc,MAAM,iBAAiB,YAAY;AACvD,MAAI,CAAC,aAAa;AAChB,UAAM,IAAI,MAAM,kCAAkC;AAAA,EACpD;AAEA,MAAI;AACF,QAAI,kBAA0C;AAC9C,QAAI;AACF,wBAAkB,IAAI,QAAyB,iBAAiB;AAAA,IAClE,QAAQ;AACN,wBAAkB;AAAA,IACpB;AAGA,QAAI,YAAY,SAAS;AACvB,YAAM,EAAE,YAAY,UAAU,eAAe,IAAI,IAAI;AAMrD,UAAI,CAAC,cAAc,CAAC,UAAU;AAC5B,wBAAgB,yBAAyB,sCAAsC;AAAA,UAC7E,OAAO,OAAO;AAAA,UACd;AAAA,UACA;AAAA,QACF,CAAC;AACD;AAAA,MACF;AAEA,UAAI,CAAC,eAAe;AAClB,cAAM,IAAI,MAAM,qDAAqD;AAAA,MACvE;AAEA,YAAM,SAAS,MAAM,cAAc,gBAAgB;AAAA,QACjD,UAAU;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AAED,kBAAY,yBAAyB,qCAAqC;AAAA,QACxE,OAAO,OAAO;AAAA,QACd;AAAA,QACA;AAAA,QACA;AAAA,QACA,QAAQ,OAAO;AAAA,MACjB,CAAC;AAED,YAAM;AAAA,QACJ,EAAE,IAAI,MAAM,OAAU;AAAA,QACtB;AAAA,UACE,QAAQ;AAAA,UACR,SAAS;AAAA,UACT,SAAS,+BAA+B,OAAO,MAAM;AAAA,UACrD;AAAA,UACA;AAAA,UACA;AAAA,UACA,SAAS,EAAE,OAAO,OAAO,MAAM;AAAA,QACjC;AAAA,MACF;AACA;AAAA,IACF;AAGA,QAAI,YAAY,eAAe;AAC7B,YAAM,EAAE,SAAS,eAAe,IAAI,IAAI;AACxC,UAAI,CAAC,WAAW,QAAQ,WAAW,GAAG;AACpC,wBAAgB,yBAAyB,wCAAwC;AAAA,UAC/E,OAAO,OAAO;AAAA,QAChB,CAAC;AACD;AAAA,MACF;AAEA,UAAI,CAAC,eAAe;AAClB,cAAM,IAAI,MAAM,gDAAgD;AAAA,MAClE;AAIA,YAAM,EAAE,SAAS,cAAc,SAAS,aAAa,IAAI,MAAM,cAAc,iBAAiB;AAAA,QAC5F,OAAO,QAAQ,IAAI,CAAC,EAAE,UAAU,SAAS,OAAO,EAAE,UAAgC,SAAS,EAAE;AAAA,QAC7F;AAAA,QACA;AAAA,MACF,CAAC;AAED,YAAM,+BAA+B;AAAA,QACnC;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,gBAAgB,kBAAkB;AAAA,QAClC,OAAO,QAAQ;AAAA,MACjB,CAAC;AAED,kBAAY,yBAAyB,6BAA6B;AAAA,QAChE,OAAO,OAAO;AAAA,QACd;AAAA,QACA,gBAAgB,QAAQ;AAAA,QACxB;AAAA,QACA;AAAA,MACF,CAAC;AAED,YAAM;AAAA,QACJ,EAAE,IAAI,MAAM,OAAU;AAAA,QACtB;AAAA,UACE,QAAQ;AAAA,UACR,SAAS;AAAA,UACT,SAAS,WAAW,YAAY,IAAI,QAAQ,MAAM;AAAA,UAClD;AAAA,UACA,SAAS,EAAE,OAAO,OAAO,OAAO,gBAAgB,QAAQ,QAAQ,cAAc,aAAa;AAAA,QAC7F;AAAA,MACF;AACA;AAAA,IACF;AAGA,QAAI,YAAY,UAAU;AACxB,YAAM,EAAE,UAAU,SAAS,IAAI,IAAI;AACnC,UAAI,CAAC,YAAY,CAAC,UAAU;AAC1B,wBAAgB,yBAAyB,uCAAuC;AAAA,UAC9E,OAAO,OAAO;AAAA,UACd;AAAA,UACA;AAAA,QACF,CAAC;AACD;AAAA,MACF;AAEA,YAAM,iBAAiB,OAAO,UAAU,UAAU,QAAQ;AAE1D,kBAAY,yBAAyB,yBAAyB;AAAA,QAC5D,OAAO,OAAO;AAAA,QACd;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AAED,YAAM;AAAA,QACJ,EAAE,IAAI,MAAM,OAAU;AAAA,QACtB;AAAA,UACE,QAAQ;AAAA,UACR,SAAS;AAAA,UACT,SAAS;AAAA,UACT,YAAY;AAAA,UACZ;AAAA,UACA;AAAA,UACA,SAAS,EAAE,OAAO,OAAO,MAAM;AAAA,QACjC;AAAA,MACF;AACA;AAAA,IACF;AAGA,QAAI,YAAY,SAAS;AACvB,YAAM,EAAE,SAAS,IAAI,IAAI;AACzB,UAAI,CAAC,UAAU;AACb,wBAAgB,yBAAyB,wCAAwC;AAAA,UAC/E,OAAO,OAAO;AAAA,QAChB,CAAC;AACD;AAAA,MACF;AAEA,YAAM,iBAAiB,MAAM,UAAU,QAAQ;AAE/C,kBAAY,yBAAyB,+BAA+B;AAAA,QAClE,OAAO,OAAO;AAAA,QACd;AAAA,QACA;AAAA,MACF,CAAC;AAED,YAAM;AAAA,QACJ,EAAE,IAAI,MAAM,OAAU;AAAA,QACtB;AAAA,UACE,QAAQ;AAAA,UACR,SAAS;AAAA,UACT,SAAS;AAAA,UACT,YAAY;AAAA,UACZ;AAAA,UACA,SAAS,EAAE,OAAO,OAAO,MAAM;AAAA,QACjC;AAAA,MACF;AACA;AAAA,IACF;AAAA,EACF,SAAS,OAAO;AACd,gBAAY,yBAAyB,aAAa,OAAO,IAAI;AAAA,MAC3D,OAAO,OAAO;AAAA,MACd;AAAA,MACA,OAAO,iBAAiB,QAAQ,MAAM,UAAU;AAAA,MAChD,eAAe,OAAO;AAAA,IACxB,CAAC;AAED,UAAM,WAAW,cAAc,IAAI,UAAU,IAAI,QAAQ,WACxC,gBAAgB,IAAI,UAAW,IAAI,QAAoC,aAAa;AACrG,UAAM,WAAW,cAAc,IAAI,UAAU,IAAI,QAAQ,WAAW;AAEpE,UAAM;AAAA,MACJ,EAAE,IAAI,MAAM,OAAU;AAAA,MACtB;AAAA,QACE,QAAQ;AAAA,QACR,SAAS,mBAAmB,OAAO;AAAA,QACnC;AAAA,QACA,YAAY;AAAA,QACZ;AAAA,QACA;AAAA,QACA,SAAS,IAAI;AAAA,MACf;AAAA,IACF;AAGA,UAAM;AAAA,EACR;AACF;AAMA,eAAO,OACL,KACA,KACe;AACf,SAAO,uBAAuB,KAAK,KAAK,GAAG;AAC7C;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@open-mercato/search",
|
|
3
|
-
"version": "0.10.1-develop.
|
|
3
|
+
"version": "0.10.1-develop.7863.1.335fb1fe52",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -127,9 +127,9 @@
|
|
|
127
127
|
"zod": "^4.6.6"
|
|
128
128
|
},
|
|
129
129
|
"peerDependencies": {
|
|
130
|
-
"@open-mercato/core": "0.10.1-develop.
|
|
131
|
-
"@open-mercato/queue": "0.10.1-develop.
|
|
132
|
-
"@open-mercato/shared": "0.10.1-develop.
|
|
130
|
+
"@open-mercato/core": "0.10.1-develop.7863.1.335fb1fe52",
|
|
131
|
+
"@open-mercato/queue": "0.10.1-develop.7863.1.335fb1fe52",
|
|
132
|
+
"@open-mercato/shared": "0.10.1-develop.7863.1.335fb1fe52"
|
|
133
133
|
},
|
|
134
134
|
"devDependencies": {
|
|
135
135
|
"@types/jest": "^30.0.0",
|
|
@@ -19,6 +19,7 @@ jest.mock('@open-mercato/shared/lib/logger', () => {
|
|
|
19
19
|
|
|
20
20
|
|
|
21
21
|
const searchLoggerWarn = createLogger('search').warn as jest.Mock
|
|
22
|
+
const searchLoggerError = createLogger('search').error as jest.Mock
|
|
22
23
|
|
|
23
24
|
// Mock dependencies before importing workers
|
|
24
25
|
jest.mock('@open-mercato/shared/lib/indexers/error-log', () => ({
|
|
@@ -56,10 +57,14 @@ jest.mock('../modules/search/lib/reindex-progress', () => ({
|
|
|
56
57
|
}))
|
|
57
58
|
|
|
58
59
|
import { handleVectorIndexJob } from '../modules/search/workers/vector-index.worker'
|
|
59
|
-
import {
|
|
60
|
+
import {
|
|
61
|
+
__resetSkippedWithoutIndexingMemo,
|
|
62
|
+
handleFulltextIndexJob,
|
|
63
|
+
} from '../modules/search/workers/fulltext-index.worker'
|
|
60
64
|
import { updateReindexProgress, clearReindexLock } from '../modules/search/lib/reindex-lock'
|
|
61
65
|
import { hasActiveReindexProgress, incrementReindexProgress } from '../modules/search/lib/reindex-progress'
|
|
62
66
|
import { refreshCoverageSnapshot } from '@open-mercato/core/modules/query_index/lib/coverage'
|
|
67
|
+
import { recordIndexerLog } from '@open-mercato/shared/lib/indexers/status-log'
|
|
63
68
|
|
|
64
69
|
/**
|
|
65
70
|
* Create a mock job context
|
|
@@ -469,6 +474,8 @@ describe('Fulltext Index Worker', () => {
|
|
|
469
474
|
|
|
470
475
|
beforeEach(() => {
|
|
471
476
|
jest.clearAllMocks()
|
|
477
|
+
// The skip report is memoised per process, so it must be cleared between cases.
|
|
478
|
+
__resetSkippedWithoutIndexingMemo()
|
|
472
479
|
;(hasActiveReindexProgress as jest.Mock).mockResolvedValue(true)
|
|
473
480
|
mockFulltextStrategy.isAvailable.mockResolvedValue(true)
|
|
474
481
|
mockSearchIndexer.indexRecordById.mockResolvedValue({ action: 'indexed', created: true })
|
|
@@ -663,6 +670,197 @@ describe('Fulltext Index Worker', () => {
|
|
|
663
670
|
expect(mockFulltextStrategy.bulkIndex).not.toHaveBeenCalled()
|
|
664
671
|
})
|
|
665
672
|
|
|
673
|
+
// A job that indexes nothing used to report `completed` with no trace anywhere, so
|
|
674
|
+
// "fulltext is not configured" and "everything indexed fine" were indistinguishable.
|
|
675
|
+
it('records a status-log row when the fulltext strategy is unregistered, and still completes', async () => {
|
|
676
|
+
const containerWithoutStrategy: HandlerContext = {
|
|
677
|
+
resolve: jest.fn((name: string) => {
|
|
678
|
+
if (name === 'searchStrategies') return []
|
|
679
|
+
if (name === 'em') return mockEm
|
|
680
|
+
if (name === 'searchIndexer') return mockSearchIndexer
|
|
681
|
+
throw new Error(`Unknown service: ${name}`)
|
|
682
|
+
}) as HandlerContext['resolve'],
|
|
683
|
+
}
|
|
684
|
+
const job = createMockJob<FulltextIndexJobPayload>({
|
|
685
|
+
jobType: 'batch-index',
|
|
686
|
+
tenantId: 'tenant-123',
|
|
687
|
+
organizationId: 'org-456',
|
|
688
|
+
records: [{ entityId: 'test:entity', recordId: 'rec-1' }],
|
|
689
|
+
})
|
|
690
|
+
|
|
691
|
+
// Must NOT throw: retrying forever would turn a config choice into a queue backlog.
|
|
692
|
+
await expect(
|
|
693
|
+
handleFulltextIndexJob(job, createMockJobContext(), containerWithoutStrategy),
|
|
694
|
+
).resolves.toBeUndefined()
|
|
695
|
+
expect(mockSearchIndexer.indexRecordsById).not.toHaveBeenCalled()
|
|
696
|
+
|
|
697
|
+
expect(recordIndexerLog).toHaveBeenCalledTimes(1)
|
|
698
|
+
expect(recordIndexerLog).toHaveBeenCalledWith(
|
|
699
|
+
{ em: mockEm },
|
|
700
|
+
expect.objectContaining({
|
|
701
|
+
source: 'fulltext',
|
|
702
|
+
handler: 'worker:fulltext:batch-index',
|
|
703
|
+
level: 'warn',
|
|
704
|
+
message: expect.stringContaining('job skipped without indexing'),
|
|
705
|
+
tenantId: 'tenant-123',
|
|
706
|
+
details: expect.objectContaining({
|
|
707
|
+
jobType: 'batch-index',
|
|
708
|
+
organizationId: 'org-456',
|
|
709
|
+
skippedWithoutIndexing: true,
|
|
710
|
+
}),
|
|
711
|
+
}),
|
|
712
|
+
)
|
|
713
|
+
|
|
714
|
+
// The row must NOT carry organizationId. The unrestricted indexer-status view
|
|
715
|
+
// filters on `organization_id IS NULL` (query_index/api/status.ts), which is where
|
|
716
|
+
// every other fulltext row lands - a non-null org would hide this row in exactly
|
|
717
|
+
// the view an operator opens to ask "did indexing run?".
|
|
718
|
+
expect(recordIndexerLog).not.toHaveBeenCalledWith(
|
|
719
|
+
expect.anything(),
|
|
720
|
+
expect.objectContaining({ organizationId: expect.anything() }),
|
|
721
|
+
)
|
|
722
|
+
|
|
723
|
+
// ...and the log line must be readable without OM_SEARCH_DEBUG=1, at the level the
|
|
724
|
+
// sibling vector worker uses for the same skip and the row uses beside it.
|
|
725
|
+
expect(searchLoggerWarn).toHaveBeenCalledTimes(1)
|
|
726
|
+
expect(String(searchLoggerWarn.mock.calls[0][0])).toContain(
|
|
727
|
+
'Fulltext strategy not configured',
|
|
728
|
+
)
|
|
729
|
+
expect(searchLoggerError).not.toHaveBeenCalled()
|
|
730
|
+
})
|
|
731
|
+
|
|
732
|
+
it('records a status-log row when searchStrategies cannot be resolved, and still completes', async () => {
|
|
733
|
+
const containerWithoutStrategies: HandlerContext = {
|
|
734
|
+
resolve: jest.fn((name: string) => {
|
|
735
|
+
if (name === 'em') return mockEm
|
|
736
|
+
if (name === 'searchIndexer') return mockSearchIndexer
|
|
737
|
+
throw new Error(`Unknown service: ${name}`)
|
|
738
|
+
}) as HandlerContext['resolve'],
|
|
739
|
+
}
|
|
740
|
+
const job = createMockJob<FulltextIndexJobPayload>({
|
|
741
|
+
jobType: 'index',
|
|
742
|
+
tenantId: 'tenant-123',
|
|
743
|
+
organizationId: null,
|
|
744
|
+
entityType: 'test:entity',
|
|
745
|
+
recordId: 'rec-1',
|
|
746
|
+
})
|
|
747
|
+
|
|
748
|
+
await expect(
|
|
749
|
+
handleFulltextIndexJob(job, createMockJobContext(), containerWithoutStrategies),
|
|
750
|
+
).resolves.toBeUndefined()
|
|
751
|
+
|
|
752
|
+
expect(mockSearchIndexer.indexRecordById).not.toHaveBeenCalled()
|
|
753
|
+
expect(recordIndexerLog).toHaveBeenCalledTimes(1)
|
|
754
|
+
expect(recordIndexerLog).toHaveBeenCalledWith(
|
|
755
|
+
{ em: mockEm },
|
|
756
|
+
expect.objectContaining({
|
|
757
|
+
source: 'fulltext',
|
|
758
|
+
handler: 'worker:fulltext:index',
|
|
759
|
+
level: 'warn',
|
|
760
|
+
message: expect.stringContaining('searchStrategies not available'),
|
|
761
|
+
tenantId: 'tenant-123',
|
|
762
|
+
details: expect.objectContaining({ jobType: 'index', organizationId: null }),
|
|
763
|
+
}),
|
|
764
|
+
)
|
|
765
|
+
expect(searchLoggerWarn).toHaveBeenCalledTimes(1)
|
|
766
|
+
expect(searchLoggerError).not.toHaveBeenCalled()
|
|
767
|
+
})
|
|
768
|
+
|
|
769
|
+
// `payloadOrganizationId` guards with `'organizationId' in job.payload` because
|
|
770
|
+
// FulltextDeletePayload and FulltextPurgePayload have no such field at all. The two
|
|
771
|
+
// cases above cover key-present-truthy and key-present-null; this is the branch the
|
|
772
|
+
// `in` operator exists for.
|
|
773
|
+
it('handles a skip on a payload shape that has no organizationId field', async () => {
|
|
774
|
+
const containerWithoutStrategy: HandlerContext = {
|
|
775
|
+
resolve: jest.fn((name: string) => {
|
|
776
|
+
if (name === 'searchStrategies') return []
|
|
777
|
+
if (name === 'em') return mockEm
|
|
778
|
+
if (name === 'searchIndexer') return mockSearchIndexer
|
|
779
|
+
throw new Error(`Unknown service: ${name}`)
|
|
780
|
+
}) as HandlerContext['resolve'],
|
|
781
|
+
}
|
|
782
|
+
const job = createMockJob<FulltextIndexJobPayload>({
|
|
783
|
+
jobType: 'delete',
|
|
784
|
+
tenantId: 'tenant-123',
|
|
785
|
+
entityType: 'test:entity',
|
|
786
|
+
recordId: 'rec-1',
|
|
787
|
+
} as unknown as FulltextIndexJobPayload)
|
|
788
|
+
|
|
789
|
+
await expect(
|
|
790
|
+
handleFulltextIndexJob(job, createMockJobContext(), containerWithoutStrategy),
|
|
791
|
+
).resolves.toBeUndefined()
|
|
792
|
+
|
|
793
|
+
expect(recordIndexerLog).toHaveBeenCalledTimes(1)
|
|
794
|
+
expect(recordIndexerLog).toHaveBeenCalledWith(
|
|
795
|
+
{ em: mockEm },
|
|
796
|
+
expect.objectContaining({
|
|
797
|
+
handler: 'worker:fulltext:delete',
|
|
798
|
+
details: expect.objectContaining({ jobType: 'delete', organizationId: null }),
|
|
799
|
+
}),
|
|
800
|
+
)
|
|
801
|
+
})
|
|
802
|
+
|
|
803
|
+
// The guard's outcome is static for the lifetime of the process - searchStrategies and
|
|
804
|
+
// searchIndexer are registered in one call - so rows 2..N carry exactly the information
|
|
805
|
+
// of row 1. The operator surface is the newest 100 rows across ALL sources, so a row per
|
|
806
|
+
// job would saturate it under ordinary write traffic.
|
|
807
|
+
it('reports the same skip once per process, not once per job', async () => {
|
|
808
|
+
const containerWithoutStrategy: HandlerContext = {
|
|
809
|
+
resolve: jest.fn((name: string) => {
|
|
810
|
+
if (name === 'searchStrategies') return []
|
|
811
|
+
if (name === 'em') return mockEm
|
|
812
|
+
if (name === 'searchIndexer') return mockSearchIndexer
|
|
813
|
+
throw new Error(`Unknown service: ${name}`)
|
|
814
|
+
}) as HandlerContext['resolve'],
|
|
815
|
+
}
|
|
816
|
+
const makeJob = (recordId: string) =>
|
|
817
|
+
createMockJob<FulltextIndexJobPayload>({
|
|
818
|
+
jobType: 'index',
|
|
819
|
+
tenantId: 'tenant-123',
|
|
820
|
+
organizationId: 'org-456',
|
|
821
|
+
entityType: 'test:entity',
|
|
822
|
+
recordId,
|
|
823
|
+
})
|
|
824
|
+
|
|
825
|
+
for (const recordId of ['rec-1', 'rec-2', 'rec-3']) {
|
|
826
|
+
await handleFulltextIndexJob(makeJob(recordId), createMockJobContext(), containerWithoutStrategy)
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
expect(recordIndexerLog).toHaveBeenCalledTimes(1)
|
|
830
|
+
expect(searchLoggerWarn).toHaveBeenCalledTimes(1)
|
|
831
|
+
})
|
|
832
|
+
|
|
833
|
+
it('still reports a second tenant and a second job type', async () => {
|
|
834
|
+
const containerWithoutStrategy: HandlerContext = {
|
|
835
|
+
resolve: jest.fn((name: string) => {
|
|
836
|
+
if (name === 'searchStrategies') return []
|
|
837
|
+
if (name === 'em') return mockEm
|
|
838
|
+
if (name === 'searchIndexer') return mockSearchIndexer
|
|
839
|
+
throw new Error(`Unknown service: ${name}`)
|
|
840
|
+
}) as HandlerContext['resolve'],
|
|
841
|
+
}
|
|
842
|
+
const jobs = [
|
|
843
|
+
{ jobType: 'index' as const, tenantId: 'tenant-a' },
|
|
844
|
+
{ jobType: 'index' as const, tenantId: 'tenant-b' },
|
|
845
|
+
{ jobType: 'batch-index' as const, tenantId: 'tenant-a' },
|
|
846
|
+
{ jobType: 'index' as const, tenantId: 'tenant-a' },
|
|
847
|
+
]
|
|
848
|
+
for (const { jobType, tenantId } of jobs) {
|
|
849
|
+
const job = createMockJob<FulltextIndexJobPayload>({
|
|
850
|
+
jobType,
|
|
851
|
+
tenantId,
|
|
852
|
+
organizationId: null,
|
|
853
|
+
entityType: 'test:entity',
|
|
854
|
+
recordId: 'rec-1',
|
|
855
|
+
records: [{ entityId: 'test:entity', recordId: 'rec-1' }],
|
|
856
|
+
})
|
|
857
|
+
await handleFulltextIndexJob(job, createMockJobContext(), containerWithoutStrategy)
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
// Three distinct (tenant, jobType) pairs; the repeat of the first is suppressed.
|
|
861
|
+
expect(recordIndexerLog).toHaveBeenCalledTimes(3)
|
|
862
|
+
})
|
|
863
|
+
|
|
666
864
|
it('should throw when fulltext search is not available', async () => {
|
|
667
865
|
mockFulltextStrategy.isAvailable.mockResolvedValueOnce(false)
|
|
668
866
|
const job = createMockJob<FulltextIndexJobPayload>({
|
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import type { QueuedJob, JobContext, WorkerMeta } from '@open-mercato/queue'
|
|
2
2
|
import type { Kysely } from 'kysely'
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
FULLTEXT_INDEXING_QUEUE_NAME,
|
|
5
|
+
type FulltextIndexJobPayload,
|
|
6
|
+
type FulltextIndexJobType,
|
|
7
|
+
} from '../../../queue/fulltext-indexing'
|
|
4
8
|
import type { FullTextSearchStrategy } from '../../../strategies/fulltext.strategy'
|
|
5
9
|
import type { SearchIndexer } from '../../../indexer/search-indexer'
|
|
6
10
|
import type { EntityManager } from '@mikro-orm/postgresql'
|
|
@@ -9,7 +13,7 @@ import type { EntityId } from '@open-mercato/shared/modules/entities'
|
|
|
9
13
|
import { recordIndexerLog } from '@open-mercato/shared/lib/indexers/status-log'
|
|
10
14
|
import { recordIndexerError } from '@open-mercato/shared/lib/indexers/error-log'
|
|
11
15
|
import type { ProgressService } from '@open-mercato/core/modules/progress/lib/progressService'
|
|
12
|
-
import { searchDebug, searchDebugWarn, searchError } from '../../../lib/debug'
|
|
16
|
+
import { searchDebug, searchDebugWarn, searchError, searchWarn } from '../../../lib/debug'
|
|
13
17
|
import { clearReindexLock, updateReindexProgress } from '../lib/reindex-lock'
|
|
14
18
|
import { hasActiveReindexProgress, incrementReindexProgress } from '../lib/reindex-progress'
|
|
15
19
|
|
|
@@ -72,6 +76,87 @@ async function advanceFulltextReindexProgress(params: {
|
|
|
72
76
|
}
|
|
73
77
|
}
|
|
74
78
|
|
|
79
|
+
/**
|
|
80
|
+
* Report a job that returned without indexing anything.
|
|
81
|
+
*
|
|
82
|
+
* A fulltext strategy that was never registered (no `MEILISEARCH_HOST`, so
|
|
83
|
+
* `createFulltextDriver()` returns null and `di.ts` pushes no strategy) makes
|
|
84
|
+
* every job a no-op. The bare `return` that follows leaves BullMQ reporting
|
|
85
|
+
* `completed` with zero failures and writes no `indexer_status_logs` row, so
|
|
86
|
+
* "fulltext is not configured" and "everything indexed fine" are
|
|
87
|
+
* indistinguishable from both signals an operator has.
|
|
88
|
+
*
|
|
89
|
+
* Deliberately does NOT throw. A deployment may legitimately run without
|
|
90
|
+
* Meilisearch, and throwing would retry these jobs forever, turning a
|
|
91
|
+
* configuration choice into a queue backlog. The `isAvailable()` check further
|
|
92
|
+
* down throws precisely because a registered-but-unreachable Meili is expected
|
|
93
|
+
* to recover; an unregistered strategy is not.
|
|
94
|
+
*
|
|
95
|
+
* Reported ONCE per (tenant, job type, reason) per process, deliberately.
|
|
96
|
+
* `searchStrategies` and `searchIndexer` are registered together in one
|
|
97
|
+
* `container.register` call, from a `strategies` array that only receives a
|
|
98
|
+
* `FullTextSearchStrategy` when the driver exists - so the outcome of this guard
|
|
99
|
+
* is static for the lifetime of the process and rows 2..N carry exactly the
|
|
100
|
+
* information of row 1. The surface an operator reads is the newest 100
|
|
101
|
+
* `indexer_status_logs` rows across ALL sources, so an unthrottled row per job
|
|
102
|
+
* would saturate that window under ordinary write traffic and push out every
|
|
103
|
+
* other source's rows - trading invisibility for a panel showing one signal a
|
|
104
|
+
* hundred times. Do not "fix" this back to per-job.
|
|
105
|
+
*/
|
|
106
|
+
const reportedSkipKeys = new Set<string>()
|
|
107
|
+
|
|
108
|
+
/** Test seam: the memo above is process-scoped, so a suite must be able to clear it. */
|
|
109
|
+
export function __resetSkippedWithoutIndexingMemo(): void {
|
|
110
|
+
reportedSkipKeys.clear()
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
async function reportSkippedWithoutIndexing(params: {
|
|
114
|
+
em: EntityManager | null
|
|
115
|
+
jobType: FulltextIndexJobType
|
|
116
|
+
tenantId: string
|
|
117
|
+
organizationId: string | null
|
|
118
|
+
jobId: string
|
|
119
|
+
reason: string
|
|
120
|
+
}): Promise<void> {
|
|
121
|
+
const memoKey = `${params.tenantId}:${params.jobType}:${params.reason}`
|
|
122
|
+
if (reportedSkipKeys.has(memoKey)) return
|
|
123
|
+
reportedSkipKeys.add(memoKey)
|
|
124
|
+
|
|
125
|
+
const message = `${params.reason}; job skipped without indexing`
|
|
126
|
+
// `searchWarn`, not `searchError`: this is an operational warning about a
|
|
127
|
+
// configuration the guard above calls legitimate, it is the level the sibling
|
|
128
|
+
// vector worker uses for the same skip, and it agrees with the `level: 'warn'`
|
|
129
|
+
// of the row written beside it. It is un-gated by OM_SEARCH_DEBUG either way,
|
|
130
|
+
// which is the property this fix needs.
|
|
131
|
+
searchWarn('fulltext-index.worker', message, {
|
|
132
|
+
jobId: params.jobId,
|
|
133
|
+
tenantId: params.tenantId,
|
|
134
|
+
jobType: params.jobType,
|
|
135
|
+
})
|
|
136
|
+
await recordIndexerLog(
|
|
137
|
+
{ em: params.em ?? undefined },
|
|
138
|
+
{
|
|
139
|
+
source: 'fulltext',
|
|
140
|
+
handler: `worker:fulltext:${params.jobType}`,
|
|
141
|
+
level: 'warn',
|
|
142
|
+
message,
|
|
143
|
+
tenantId: params.tenantId,
|
|
144
|
+
// No `organizationId`: every other recordIndexerLog call in this handler
|
|
145
|
+
// omits it, so all existing `source: 'fulltext'` rows persist with
|
|
146
|
+
// organization_id = NULL - and the unrestricted indexer-status view filters
|
|
147
|
+
// on `organization_id IS NULL` (query_index/api/status.ts). A non-null org
|
|
148
|
+
// here would hide this row in exactly the view an operator lands on when
|
|
149
|
+
// asking "did indexing run?". The value stays in `details` for triage.
|
|
150
|
+
details: {
|
|
151
|
+
jobId: params.jobId,
|
|
152
|
+
jobType: params.jobType,
|
|
153
|
+
organizationId: params.organizationId,
|
|
154
|
+
skippedWithoutIndexing: true,
|
|
155
|
+
},
|
|
156
|
+
},
|
|
157
|
+
)
|
|
158
|
+
}
|
|
159
|
+
|
|
75
160
|
/**
|
|
76
161
|
* Process a fulltext indexing job.
|
|
77
162
|
*
|
|
@@ -92,6 +177,8 @@ export async function handleFulltextIndexJob(
|
|
|
92
177
|
ctx: HandlerContext,
|
|
93
178
|
): Promise<void> {
|
|
94
179
|
const { jobType, tenantId } = job.payload
|
|
180
|
+
const payloadOrganizationId =
|
|
181
|
+
'organizationId' in job.payload ? job.payload.organizationId ?? null : null
|
|
95
182
|
|
|
96
183
|
if (!tenantId) {
|
|
97
184
|
searchDebugWarn('fulltext-index.worker', 'Skipping job with missing tenantId', {
|
|
@@ -118,6 +205,11 @@ export async function handleFulltextIndexJob(
|
|
|
118
205
|
try {
|
|
119
206
|
searchIndexer = ctx.resolve<SearchIndexer>('searchIndexer')
|
|
120
207
|
} catch {
|
|
208
|
+
// Left at debug level to match vector-index.worker.ts for the identical
|
|
209
|
+
// condition. It is also close to unreachable: `searchIndexer` and
|
|
210
|
+
// `searchStrategies` are registered in the same `container.register` call, so
|
|
211
|
+
// the only way this throws is that search DI never ran - in which case the
|
|
212
|
+
// next guard reports it anyway.
|
|
121
213
|
searchDebugWarn('fulltext-index.worker', 'searchIndexer not available')
|
|
122
214
|
}
|
|
123
215
|
|
|
@@ -129,12 +221,26 @@ export async function handleFulltextIndexJob(
|
|
|
129
221
|
(s: unknown) => (s as { id?: string })?.id === 'fulltext',
|
|
130
222
|
) as FullTextSearchStrategy | undefined
|
|
131
223
|
} catch {
|
|
132
|
-
|
|
224
|
+
await reportSkippedWithoutIndexing({
|
|
225
|
+
em,
|
|
226
|
+
jobType,
|
|
227
|
+
tenantId,
|
|
228
|
+
organizationId: payloadOrganizationId,
|
|
229
|
+
jobId: jobCtx.jobId,
|
|
230
|
+
reason: 'searchStrategies not available',
|
|
231
|
+
})
|
|
133
232
|
return
|
|
134
233
|
}
|
|
135
234
|
|
|
136
235
|
if (!fulltextStrategy) {
|
|
137
|
-
|
|
236
|
+
await reportSkippedWithoutIndexing({
|
|
237
|
+
em,
|
|
238
|
+
jobType,
|
|
239
|
+
tenantId,
|
|
240
|
+
organizationId: payloadOrganizationId,
|
|
241
|
+
jobId: jobCtx.jobId,
|
|
242
|
+
reason: 'Fulltext strategy not configured (MEILISEARCH_HOST unset?)',
|
|
243
|
+
})
|
|
138
244
|
return
|
|
139
245
|
}
|
|
140
246
|
|