@open-mercato/search 0.4.2-canary-c02407ff85
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +678 -0
- package/build.mjs +92 -0
- package/dist/di.js +157 -0
- package/dist/di.js.map +7 -0
- package/dist/fulltext/drivers/index.js +21 -0
- package/dist/fulltext/drivers/index.js.map +7 -0
- package/dist/fulltext/drivers/meilisearch/index.js +320 -0
- package/dist/fulltext/drivers/meilisearch/index.js.map +7 -0
- package/dist/fulltext/index.js +7 -0
- package/dist/fulltext/index.js.map +7 -0
- package/dist/fulltext/types.js +1 -0
- package/dist/fulltext/types.js.map +7 -0
- package/dist/index.js +12 -0
- package/dist/index.js.map +7 -0
- package/dist/indexer/index.js +8 -0
- package/dist/indexer/index.js.map +7 -0
- package/dist/indexer/search-indexer.js +848 -0
- package/dist/indexer/search-indexer.js.map +7 -0
- package/dist/indexer/subscribers/delete.js +41 -0
- package/dist/indexer/subscribers/delete.js.map +7 -0
- package/dist/lib/debug.js +34 -0
- package/dist/lib/debug.js.map +7 -0
- package/dist/lib/fallback-presenter.js +107 -0
- package/dist/lib/fallback-presenter.js.map +7 -0
- package/dist/lib/field-policy.js +75 -0
- package/dist/lib/field-policy.js.map +7 -0
- package/dist/lib/index.js +19 -0
- package/dist/lib/index.js.map +7 -0
- package/dist/lib/merger.js +93 -0
- package/dist/lib/merger.js.map +7 -0
- package/dist/lib/presenter-enricher.js +192 -0
- package/dist/lib/presenter-enricher.js.map +7 -0
- package/dist/modules/search/acl.js +14 -0
- package/dist/modules/search/acl.js.map +7 -0
- package/dist/modules/search/ai-tools.js +284 -0
- package/dist/modules/search/ai-tools.js.map +7 -0
- package/dist/modules/search/api/embeddings/reindex/cancel/route.js +65 -0
- package/dist/modules/search/api/embeddings/reindex/cancel/route.js.map +7 -0
- package/dist/modules/search/api/embeddings/reindex/route.js +165 -0
- package/dist/modules/search/api/embeddings/reindex/route.js.map +7 -0
- package/dist/modules/search/api/embeddings/route.js +246 -0
- package/dist/modules/search/api/embeddings/route.js.map +7 -0
- package/dist/modules/search/api/index/route.js +245 -0
- package/dist/modules/search/api/index/route.js.map +7 -0
- package/dist/modules/search/api/reindex/cancel/route.js +65 -0
- package/dist/modules/search/api/reindex/cancel/route.js.map +7 -0
- package/dist/modules/search/api/reindex/route.js +332 -0
- package/dist/modules/search/api/reindex/route.js.map +7 -0
- package/dist/modules/search/api/search/global/route.js +100 -0
- package/dist/modules/search/api/search/global/route.js.map +7 -0
- package/dist/modules/search/api/search/route.js +101 -0
- package/dist/modules/search/api/search/route.js.map +7 -0
- package/dist/modules/search/api/settings/fulltext/route.js +55 -0
- package/dist/modules/search/api/settings/fulltext/route.js.map +7 -0
- package/dist/modules/search/api/settings/global-search/route.js +80 -0
- package/dist/modules/search/api/settings/global-search/route.js.map +7 -0
- package/dist/modules/search/api/settings/route.js +118 -0
- package/dist/modules/search/api/settings/route.js.map +7 -0
- package/dist/modules/search/api/settings/vector-store/route.js +77 -0
- package/dist/modules/search/api/settings/vector-store/route.js.map +7 -0
- package/dist/modules/search/backend/config/search/page.js +10 -0
- package/dist/modules/search/backend/config/search/page.js.map +7 -0
- package/dist/modules/search/backend/config/search/page.meta.js +24 -0
- package/dist/modules/search/backend/config/search/page.meta.js.map +7 -0
- package/dist/modules/search/cli.js +698 -0
- package/dist/modules/search/cli.js.map +7 -0
- package/dist/modules/search/di.js +32 -0
- package/dist/modules/search/di.js.map +7 -0
- package/dist/modules/search/frontend/components/GlobalSearchDialog.js +357 -0
- package/dist/modules/search/frontend/components/GlobalSearchDialog.js.map +7 -0
- package/dist/modules/search/frontend/components/HybridSearchTable.js +343 -0
- package/dist/modules/search/frontend/components/HybridSearchTable.js.map +7 -0
- package/dist/modules/search/frontend/components/SearchSettingsPageClient.js +303 -0
- package/dist/modules/search/frontend/components/SearchSettingsPageClient.js.map +7 -0
- package/dist/modules/search/frontend/components/sections/FulltextSearchSection.js +360 -0
- package/dist/modules/search/frontend/components/sections/FulltextSearchSection.js.map +7 -0
- package/dist/modules/search/frontend/components/sections/GlobalSearchSection.js +101 -0
- package/dist/modules/search/frontend/components/sections/GlobalSearchSection.js.map +7 -0
- package/dist/modules/search/frontend/components/sections/VectorSearchSection.js +608 -0
- package/dist/modules/search/frontend/components/sections/VectorSearchSection.js.map +7 -0
- package/dist/modules/search/frontend/index.js +9 -0
- package/dist/modules/search/frontend/index.js.map +7 -0
- package/dist/modules/search/frontend/utils.js +41 -0
- package/dist/modules/search/frontend/utils.js.map +7 -0
- package/dist/modules/search/i18n/de.json +61 -0
- package/dist/modules/search/i18n/en.json +72 -0
- package/dist/modules/search/i18n/es.json +61 -0
- package/dist/modules/search/i18n/pl.json +61 -0
- package/dist/modules/search/index.js +11 -0
- package/dist/modules/search/index.js.map +7 -0
- package/dist/modules/search/lib/auto-indexing.js +29 -0
- package/dist/modules/search/lib/auto-indexing.js.map +7 -0
- package/dist/modules/search/lib/embedding-config.js +131 -0
- package/dist/modules/search/lib/embedding-config.js.map +7 -0
- package/dist/modules/search/lib/global-search-config.js +45 -0
- package/dist/modules/search/lib/global-search-config.js.map +7 -0
- package/dist/modules/search/lib/reindex-lock.js +99 -0
- package/dist/modules/search/lib/reindex-lock.js.map +7 -0
- package/dist/modules/search/subscribers/fulltext_upsert.js +64 -0
- package/dist/modules/search/subscribers/fulltext_upsert.js.map +7 -0
- package/dist/modules/search/subscribers/vector_delete.js +58 -0
- package/dist/modules/search/subscribers/vector_delete.js.map +7 -0
- package/dist/modules/search/subscribers/vector_purge.js +142 -0
- package/dist/modules/search/subscribers/vector_purge.js.map +7 -0
- package/dist/modules/search/subscribers/vector_upsert.js +58 -0
- package/dist/modules/search/subscribers/vector_upsert.js.map +7 -0
- package/dist/modules/search/workers/fulltext-index.worker.js +240 -0
- package/dist/modules/search/workers/fulltext-index.worker.js.map +7 -0
- package/dist/modules/search/workers/vector-index.worker.js +234 -0
- package/dist/modules/search/workers/vector-index.worker.js.map +7 -0
- package/dist/queue/fulltext-indexing.js +15 -0
- package/dist/queue/fulltext-indexing.js.map +7 -0
- package/dist/queue/index.js +3 -0
- package/dist/queue/index.js.map +7 -0
- package/dist/queue/vector-indexing.js +15 -0
- package/dist/queue/vector-indexing.js.map +7 -0
- package/dist/service.js +286 -0
- package/dist/service.js.map +7 -0
- package/dist/strategies/fulltext.strategy.js +116 -0
- package/dist/strategies/fulltext.strategy.js.map +7 -0
- package/dist/strategies/index.js +12 -0
- package/dist/strategies/index.js.map +7 -0
- package/dist/strategies/token.strategy.js +80 -0
- package/dist/strategies/token.strategy.js.map +7 -0
- package/dist/strategies/vector.strategy.js +137 -0
- package/dist/strategies/vector.strategy.js.map +7 -0
- package/dist/types.js +1 -0
- package/dist/types.js.map +7 -0
- package/dist/vector/drivers/chromadb/index.js +44 -0
- package/dist/vector/drivers/chromadb/index.js.map +7 -0
- package/dist/vector/drivers/index.js +9 -0
- package/dist/vector/drivers/index.js.map +7 -0
- package/dist/vector/drivers/pgvector/index.js +509 -0
- package/dist/vector/drivers/pgvector/index.js.map +7 -0
- package/dist/vector/drivers/qdrant/index.js +44 -0
- package/dist/vector/drivers/qdrant/index.js.map +7 -0
- package/dist/vector/index.js +4 -0
- package/dist/vector/index.js.map +7 -0
- package/dist/vector/lib/vector-logs.js +33 -0
- package/dist/vector/lib/vector-logs.js.map +7 -0
- package/dist/vector/services/checksum.js +20 -0
- package/dist/vector/services/checksum.js.map +7 -0
- package/dist/vector/services/embedding.js +222 -0
- package/dist/vector/services/embedding.js.map +7 -0
- package/dist/vector/services/index.js +4 -0
- package/dist/vector/services/index.js.map +7 -0
- package/dist/vector/services/vector-index.service.js +960 -0
- package/dist/vector/services/vector-index.service.js.map +7 -0
- package/dist/vector/types/pg.d.js +1 -0
- package/dist/vector/types/pg.d.js.map +7 -0
- package/dist/vector/types.js +75 -0
- package/dist/vector/types.js.map +7 -0
- package/jest.config.cjs +19 -0
- package/package.json +142 -0
- package/src/__tests__/queue.test.ts +148 -0
- package/src/__tests__/service.test.ts +345 -0
- package/src/__tests__/workers.test.ts +319 -0
- package/src/di.ts +291 -0
- package/src/fulltext/drivers/index.ts +41 -0
- package/src/fulltext/drivers/meilisearch/index.ts +410 -0
- package/src/fulltext/index.ts +13 -0
- package/src/fulltext/types.ts +115 -0
- package/src/index.ts +36 -0
- package/src/indexer/index.ts +13 -0
- package/src/indexer/search-indexer.ts +1141 -0
- package/src/indexer/subscribers/delete.ts +49 -0
- package/src/lib/debug.ts +46 -0
- package/src/lib/fallback-presenter.ts +106 -0
- package/src/lib/field-policy.ts +169 -0
- package/src/lib/index.ts +13 -0
- package/src/lib/merger.ts +159 -0
- package/src/lib/presenter-enricher.ts +323 -0
- package/src/modules/search/README.md +694 -0
- package/src/modules/search/acl.ts +10 -0
- package/src/modules/search/ai-tools.ts +467 -0
- package/src/modules/search/api/embeddings/reindex/cancel/route.ts +77 -0
- package/src/modules/search/api/embeddings/reindex/route.ts +197 -0
- package/src/modules/search/api/embeddings/route.ts +304 -0
- package/src/modules/search/api/index/route.ts +297 -0
- package/src/modules/search/api/reindex/cancel/route.ts +77 -0
- package/src/modules/search/api/reindex/route.ts +419 -0
- package/src/modules/search/api/search/global/route.ts +120 -0
- package/src/modules/search/api/search/route.ts +121 -0
- package/src/modules/search/api/settings/fulltext/route.ts +82 -0
- package/src/modules/search/api/settings/global-search/route.ts +91 -0
- package/src/modules/search/api/settings/route.ts +187 -0
- package/src/modules/search/api/settings/vector-store/route.ts +105 -0
- package/src/modules/search/backend/config/search/page.meta.ts +22 -0
- package/src/modules/search/backend/config/search/page.tsx +12 -0
- package/src/modules/search/cli.ts +818 -0
- package/src/modules/search/di.ts +50 -0
- package/src/modules/search/frontend/components/GlobalSearchDialog.tsx +436 -0
- package/src/modules/search/frontend/components/HybridSearchTable.tsx +418 -0
- package/src/modules/search/frontend/components/SearchSettingsPageClient.tsx +476 -0
- package/src/modules/search/frontend/components/sections/FulltextSearchSection.tsx +624 -0
- package/src/modules/search/frontend/components/sections/GlobalSearchSection.tsx +124 -0
- package/src/modules/search/frontend/components/sections/VectorSearchSection.tsx +943 -0
- package/src/modules/search/frontend/index.ts +3 -0
- package/src/modules/search/frontend/utils.ts +82 -0
- package/src/modules/search/i18n/de.json +61 -0
- package/src/modules/search/i18n/en.json +72 -0
- package/src/modules/search/i18n/es.json +61 -0
- package/src/modules/search/i18n/pl.json +61 -0
- package/src/modules/search/index.ts +9 -0
- package/src/modules/search/lib/auto-indexing.ts +35 -0
- package/src/modules/search/lib/embedding-config.ts +161 -0
- package/src/modules/search/lib/global-search-config.ts +69 -0
- package/src/modules/search/lib/reindex-lock.ts +201 -0
- package/src/modules/search/subscribers/fulltext_upsert.ts +83 -0
- package/src/modules/search/subscribers/vector_delete.ts +75 -0
- package/src/modules/search/subscribers/vector_purge.ts +161 -0
- package/src/modules/search/subscribers/vector_upsert.ts +75 -0
- package/src/modules/search/workers/fulltext-index.worker.ts +318 -0
- package/src/modules/search/workers/vector-index.worker.ts +292 -0
- package/src/queue/fulltext-indexing.ts +87 -0
- package/src/queue/index.ts +2 -0
- package/src/queue/vector-indexing.ts +66 -0
- package/src/service.ts +397 -0
- package/src/strategies/fulltext.strategy.ts +155 -0
- package/src/strategies/index.ts +17 -0
- package/src/strategies/token.strategy.ts +153 -0
- package/src/strategies/vector.strategy.ts +234 -0
- package/src/types.ts +38 -0
- package/src/vector/drivers/chromadb/index.ts +49 -0
- package/src/vector/drivers/index.ts +4 -0
- package/src/vector/drivers/pgvector/index.ts +627 -0
- package/src/vector/drivers/qdrant/index.ts +49 -0
- package/src/vector/index.ts +3 -0
- package/src/vector/lib/vector-logs.ts +46 -0
- package/src/vector/services/checksum.ts +18 -0
- package/src/vector/services/embedding.ts +275 -0
- package/src/vector/services/index.ts +3 -0
- package/src/vector/services/vector-index.service.ts +1234 -0
- package/src/vector/types/pg.d.ts +1 -0
- package/src/vector/types.ts +220 -0
- package/tsconfig.json +9 -0
- package/watch.mjs +6 -0
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { resolveEntityTableName } from "@open-mercato/shared/lib/query/engine";
|
|
2
|
+
import { resolveAutoIndexingEnabled } from "../lib/auto-indexing.js";
|
|
3
|
+
import { searchDebugWarn, searchError } from "../../../lib/debug.js";
|
|
4
|
+
const metadata = { event: "query_index.vectorize_one", persistent: false };
|
|
5
|
+
async function handle(payload, ctx) {
|
|
6
|
+
const entityType = String(payload?.entityType ?? "");
|
|
7
|
+
const recordId = String(payload?.recordId ?? "");
|
|
8
|
+
if (!entityType || !recordId) return;
|
|
9
|
+
let organizationId = payload?.organizationId ?? null;
|
|
10
|
+
let tenantId = payload?.tenantId ?? null;
|
|
11
|
+
let em = null;
|
|
12
|
+
try {
|
|
13
|
+
em = ctx.resolve("em");
|
|
14
|
+
} catch {
|
|
15
|
+
em = null;
|
|
16
|
+
}
|
|
17
|
+
if ((organizationId == null || tenantId == null) && em) {
|
|
18
|
+
try {
|
|
19
|
+
const knex = em.getConnection().getKnex();
|
|
20
|
+
const table = resolveEntityTableName(em, entityType);
|
|
21
|
+
const row = await knex(table).select(["organization_id", "tenant_id"]).where({ id: recordId }).first();
|
|
22
|
+
if (organizationId == null) organizationId = row?.organization_id ?? organizationId;
|
|
23
|
+
if (tenantId == null) tenantId = row?.tenant_id ?? tenantId;
|
|
24
|
+
} catch {
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
if (!tenantId) return;
|
|
28
|
+
const autoIndexingEnabled = await resolveAutoIndexingEnabled(ctx, { defaultValue: true });
|
|
29
|
+
if (!autoIndexingEnabled) return;
|
|
30
|
+
let queue;
|
|
31
|
+
try {
|
|
32
|
+
queue = ctx.resolve("vectorIndexQueue");
|
|
33
|
+
} catch {
|
|
34
|
+
searchDebugWarn("search.vector", "vectorIndexQueue not available, skipping vector indexing");
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
try {
|
|
38
|
+
await queue.enqueue({
|
|
39
|
+
jobType: "index",
|
|
40
|
+
entityType,
|
|
41
|
+
recordId,
|
|
42
|
+
tenantId: String(tenantId),
|
|
43
|
+
organizationId: organizationId ? String(organizationId) : null
|
|
44
|
+
});
|
|
45
|
+
} catch (error) {
|
|
46
|
+
searchError("search.vector", "Failed to enqueue vector index job", {
|
|
47
|
+
entityType,
|
|
48
|
+
recordId,
|
|
49
|
+
error: error instanceof Error ? error.message : error
|
|
50
|
+
});
|
|
51
|
+
throw error;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
export {
|
|
55
|
+
handle as default,
|
|
56
|
+
metadata
|
|
57
|
+
};
|
|
58
|
+
//# sourceMappingURL=vector_upsert.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../../../src/modules/search/subscribers/vector_upsert.ts"],
|
|
4
|
+
"sourcesContent": ["import { resolveEntityTableName } from '@open-mercato/shared/lib/query/engine'\nimport type { Queue } from '@open-mercato/queue'\nimport type { VectorIndexJobPayload } from '../../../queue/vector-indexing'\nimport { resolveAutoIndexingEnabled } from '../lib/auto-indexing'\nimport { searchDebugWarn, searchError } from '../../../lib/debug'\n\nexport const metadata = { event: 'query_index.vectorize_one', persistent: false }\n\ntype Payload = {\n entityType?: string\n recordId?: string\n organizationId?: string | null\n tenantId?: string | null\n}\n\ntype HandlerContext = { resolve: <T = unknown>(name: string) => T }\n\nexport default async function handle(payload: Payload, ctx: HandlerContext) {\n const entityType = String(payload?.entityType ?? '')\n const recordId = String(payload?.recordId ?? '')\n if (!entityType || !recordId) return\n\n let organizationId = payload?.organizationId ?? null\n let tenantId = payload?.tenantId ?? null\n\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n let em: any | null = null\n try {\n em = ctx.resolve('em')\n } catch {\n em = null\n }\n\n if ((organizationId == null || tenantId == null) && em) {\n try {\n const knex = em.getConnection().getKnex()\n const table = resolveEntityTableName(em, entityType)\n const row = await knex(table).select(['organization_id', 'tenant_id']).where({ id: recordId }).first()\n if (organizationId == null) organizationId = row?.organization_id ?? organizationId\n if (tenantId == null) tenantId = row?.tenant_id ?? tenantId\n } catch {\n // Ignore lookup errors\n }\n }\n\n if (!tenantId) return\n\n const autoIndexingEnabled = await resolveAutoIndexingEnabled(ctx, { defaultValue: true })\n if (!autoIndexingEnabled) return\n\n let queue: Queue<VectorIndexJobPayload>\n try {\n queue = ctx.resolve<Queue<VectorIndexJobPayload>>('vectorIndexQueue')\n } catch {\n searchDebugWarn('search.vector', 'vectorIndexQueue not available, skipping vector indexing')\n return\n }\n\n try {\n await queue.enqueue({\n jobType: 'index',\n entityType,\n recordId,\n tenantId: String(tenantId),\n organizationId: organizationId ? String(organizationId) : null,\n })\n } catch (error) {\n searchError('search.vector', 'Failed to enqueue vector index job', {\n entityType,\n recordId,\n error: error instanceof Error ? error.message : error,\n })\n throw error // Propagate to caller so failure is visible\n }\n}\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,8BAA8B;AAGvC,SAAS,kCAAkC;AAC3C,SAAS,iBAAiB,mBAAmB;AAEtC,MAAM,WAAW,EAAE,OAAO,6BAA6B,YAAY,MAAM;AAWhF,eAAO,OAA8B,SAAkB,KAAqB;AAC1E,QAAM,aAAa,OAAO,SAAS,cAAc,EAAE;AACnD,QAAM,WAAW,OAAO,SAAS,YAAY,EAAE;AAC/C,MAAI,CAAC,cAAc,CAAC,SAAU;AAE9B,MAAI,iBAAiB,SAAS,kBAAkB;AAChD,MAAI,WAAW,SAAS,YAAY;AAGpC,MAAI,KAAiB;AACrB,MAAI;AACF,SAAK,IAAI,QAAQ,IAAI;AAAA,EACvB,QAAQ;AACN,SAAK;AAAA,EACP;AAEA,OAAK,kBAAkB,QAAQ,YAAY,SAAS,IAAI;AACtD,QAAI;AACF,YAAM,OAAO,GAAG,cAAc,EAAE,QAAQ;AACxC,YAAM,QAAQ,uBAAuB,IAAI,UAAU;AACnD,YAAM,MAAM,MAAM,KAAK,KAAK,EAAE,OAAO,CAAC,mBAAmB,WAAW,CAAC,EAAE,MAAM,EAAE,IAAI,SAAS,CAAC,EAAE,MAAM;AACrG,UAAI,kBAAkB,KAAM,kBAAiB,KAAK,mBAAmB;AACrE,UAAI,YAAY,KAAM,YAAW,KAAK,aAAa;AAAA,IACrD,QAAQ;AAAA,IAER;AAAA,EACF;AAEA,MAAI,CAAC,SAAU;AAEf,QAAM,sBAAsB,MAAM,2BAA2B,KAAK,EAAE,cAAc,KAAK,CAAC;AACxF,MAAI,CAAC,oBAAqB;AAE1B,MAAI;AACJ,MAAI;AACF,YAAQ,IAAI,QAAsC,kBAAkB;AAAA,EACtE,QAAQ;AACN,oBAAgB,iBAAiB,0DAA0D;AAC3F;AAAA,EACF;AAEA,MAAI;AACF,UAAM,MAAM,QAAQ;AAAA,MAClB,SAAS;AAAA,MACT;AAAA,MACA;AAAA,MACA,UAAU,OAAO,QAAQ;AAAA,MACzB,gBAAgB,iBAAiB,OAAO,cAAc,IAAI;AAAA,IAC5D,CAAC;AAAA,EACH,SAAS,OAAO;AACd,gBAAY,iBAAiB,sCAAsC;AAAA,MACjE;AAAA,MACA;AAAA,MACA,OAAO,iBAAiB,QAAQ,MAAM,UAAU;AAAA,IAClD,CAAC;AACD,UAAM;AAAA,EACR;AACF;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
import { FULLTEXT_INDEXING_QUEUE_NAME } from "../../../queue/fulltext-indexing.js";
|
|
2
|
+
import { recordIndexerLog } from "@open-mercato/shared/lib/indexers/status-log";
|
|
3
|
+
import { recordIndexerError } from "@open-mercato/shared/lib/indexers/error-log";
|
|
4
|
+
import { searchDebug, searchDebugWarn, searchError } from "../../../lib/debug.js";
|
|
5
|
+
import { updateReindexProgress } from "../lib/reindex-lock.js";
|
|
6
|
+
const DEFAULT_CONCURRENCY = 2;
|
|
7
|
+
const envConcurrency = process.env.WORKERS_FULLTEXT_INDEXING_CONCURRENCY;
|
|
8
|
+
const metadata = {
|
|
9
|
+
queue: FULLTEXT_INDEXING_QUEUE_NAME,
|
|
10
|
+
concurrency: envConcurrency ? parseInt(envConcurrency, 10) : DEFAULT_CONCURRENCY
|
|
11
|
+
};
|
|
12
|
+
async function handleFulltextIndexJob(job, jobCtx, ctx) {
|
|
13
|
+
const { jobType, tenantId } = job.payload;
|
|
14
|
+
if (!tenantId) {
|
|
15
|
+
searchDebugWarn("fulltext-index.worker", "Skipping job with missing tenantId", {
|
|
16
|
+
jobId: jobCtx.jobId,
|
|
17
|
+
jobType
|
|
18
|
+
});
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
21
|
+
let em = null;
|
|
22
|
+
let knex = null;
|
|
23
|
+
try {
|
|
24
|
+
em = ctx.resolve("em");
|
|
25
|
+
knex = em.getConnection().getKnex();
|
|
26
|
+
} catch {
|
|
27
|
+
em = null;
|
|
28
|
+
knex = null;
|
|
29
|
+
}
|
|
30
|
+
let searchIndexer;
|
|
31
|
+
try {
|
|
32
|
+
searchIndexer = ctx.resolve("searchIndexer");
|
|
33
|
+
} catch {
|
|
34
|
+
searchDebugWarn("fulltext-index.worker", "searchIndexer not available");
|
|
35
|
+
}
|
|
36
|
+
let fulltextStrategy;
|
|
37
|
+
try {
|
|
38
|
+
const searchStrategies = ctx.resolve("searchStrategies");
|
|
39
|
+
fulltextStrategy = searchStrategies?.find(
|
|
40
|
+
(s) => s?.id === "fulltext"
|
|
41
|
+
);
|
|
42
|
+
} catch {
|
|
43
|
+
searchDebugWarn("fulltext-index.worker", "searchStrategies not available");
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
if (!fulltextStrategy) {
|
|
47
|
+
searchDebugWarn("fulltext-index.worker", "Fulltext strategy not configured");
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
50
|
+
const isAvailable = await fulltextStrategy.isAvailable();
|
|
51
|
+
if (!isAvailable) {
|
|
52
|
+
throw new Error("Fulltext search is not available");
|
|
53
|
+
}
|
|
54
|
+
try {
|
|
55
|
+
if (jobType === "index") {
|
|
56
|
+
const { entityType, recordId, organizationId } = job.payload;
|
|
57
|
+
if (!entityType || !recordId) {
|
|
58
|
+
searchDebugWarn("fulltext-index.worker", "Skipping index with missing fields", {
|
|
59
|
+
jobId: jobCtx.jobId,
|
|
60
|
+
entityType,
|
|
61
|
+
recordId
|
|
62
|
+
});
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
65
|
+
if (!searchIndexer) {
|
|
66
|
+
throw new Error("searchIndexer not available for single-record index");
|
|
67
|
+
}
|
|
68
|
+
const result = await searchIndexer.indexRecordById({
|
|
69
|
+
entityId: entityType,
|
|
70
|
+
recordId,
|
|
71
|
+
tenantId,
|
|
72
|
+
organizationId
|
|
73
|
+
});
|
|
74
|
+
searchDebug("fulltext-index.worker", "Indexed single record to fulltext", {
|
|
75
|
+
jobId: jobCtx.jobId,
|
|
76
|
+
tenantId,
|
|
77
|
+
entityType,
|
|
78
|
+
recordId,
|
|
79
|
+
action: result.action
|
|
80
|
+
});
|
|
81
|
+
await recordIndexerLog(
|
|
82
|
+
{ em: em ?? void 0 },
|
|
83
|
+
{
|
|
84
|
+
source: "fulltext",
|
|
85
|
+
handler: "worker:fulltext:index",
|
|
86
|
+
message: `Indexed record to fulltext (${result.action})`,
|
|
87
|
+
entityType,
|
|
88
|
+
recordId,
|
|
89
|
+
tenantId,
|
|
90
|
+
details: { jobId: jobCtx.jobId }
|
|
91
|
+
}
|
|
92
|
+
);
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
if (jobType === "batch-index") {
|
|
96
|
+
const { records, organizationId } = job.payload;
|
|
97
|
+
if (!records || records.length === 0) {
|
|
98
|
+
searchDebugWarn("fulltext-index.worker", "Skipping batch-index with no records", {
|
|
99
|
+
jobId: jobCtx.jobId
|
|
100
|
+
});
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
103
|
+
if (!searchIndexer) {
|
|
104
|
+
throw new Error("searchIndexer not available for batch indexing");
|
|
105
|
+
}
|
|
106
|
+
let successCount = 0;
|
|
107
|
+
let failCount = 0;
|
|
108
|
+
for (const { entityId, recordId } of records) {
|
|
109
|
+
try {
|
|
110
|
+
const result = await searchIndexer.indexRecordById({
|
|
111
|
+
entityId,
|
|
112
|
+
recordId,
|
|
113
|
+
tenantId,
|
|
114
|
+
organizationId
|
|
115
|
+
});
|
|
116
|
+
if (result.action === "indexed") {
|
|
117
|
+
successCount++;
|
|
118
|
+
}
|
|
119
|
+
} catch (error) {
|
|
120
|
+
failCount++;
|
|
121
|
+
searchDebugWarn("fulltext-index.worker", "Failed to index record in batch", {
|
|
122
|
+
entityId,
|
|
123
|
+
recordId,
|
|
124
|
+
error: error instanceof Error ? error.message : error
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
if (knex && successCount > 0) {
|
|
129
|
+
await updateReindexProgress(knex, tenantId, "fulltext", successCount, organizationId ?? null);
|
|
130
|
+
}
|
|
131
|
+
searchDebug("fulltext-index.worker", "Batch indexed to fulltext", {
|
|
132
|
+
jobId: jobCtx.jobId,
|
|
133
|
+
tenantId,
|
|
134
|
+
requestedCount: records.length,
|
|
135
|
+
successCount,
|
|
136
|
+
failCount
|
|
137
|
+
});
|
|
138
|
+
await recordIndexerLog(
|
|
139
|
+
{ em: em ?? void 0 },
|
|
140
|
+
{
|
|
141
|
+
source: "fulltext",
|
|
142
|
+
handler: "worker:fulltext:batch-index",
|
|
143
|
+
message: `Indexed ${successCount}/${records.length} records to fulltext`,
|
|
144
|
+
tenantId,
|
|
145
|
+
details: { jobId: jobCtx.jobId, requestedCount: records.length, successCount, failCount }
|
|
146
|
+
}
|
|
147
|
+
);
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
if (jobType === "delete") {
|
|
151
|
+
const { entityId, recordId } = job.payload;
|
|
152
|
+
if (!entityId || !recordId) {
|
|
153
|
+
searchDebugWarn("fulltext-index.worker", "Skipping delete with missing fields", {
|
|
154
|
+
jobId: jobCtx.jobId,
|
|
155
|
+
entityId,
|
|
156
|
+
recordId
|
|
157
|
+
});
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
await fulltextStrategy.delete(entityId, recordId, tenantId);
|
|
161
|
+
searchDebug("fulltext-index.worker", "Deleted from fulltext", {
|
|
162
|
+
jobId: jobCtx.jobId,
|
|
163
|
+
tenantId,
|
|
164
|
+
entityId,
|
|
165
|
+
recordId
|
|
166
|
+
});
|
|
167
|
+
await recordIndexerLog(
|
|
168
|
+
{ em: em ?? void 0 },
|
|
169
|
+
{
|
|
170
|
+
source: "fulltext",
|
|
171
|
+
handler: "worker:fulltext:delete",
|
|
172
|
+
message: `Deleted record from fulltext`,
|
|
173
|
+
entityType: entityId,
|
|
174
|
+
recordId,
|
|
175
|
+
tenantId,
|
|
176
|
+
details: { jobId: jobCtx.jobId }
|
|
177
|
+
}
|
|
178
|
+
);
|
|
179
|
+
return;
|
|
180
|
+
}
|
|
181
|
+
if (jobType === "purge") {
|
|
182
|
+
const { entityId } = job.payload;
|
|
183
|
+
if (!entityId) {
|
|
184
|
+
searchDebugWarn("fulltext-index.worker", "Skipping purge with missing entityId", {
|
|
185
|
+
jobId: jobCtx.jobId
|
|
186
|
+
});
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
await fulltextStrategy.purge(entityId, tenantId);
|
|
190
|
+
searchDebug("fulltext-index.worker", "Purged entity from fulltext", {
|
|
191
|
+
jobId: jobCtx.jobId,
|
|
192
|
+
tenantId,
|
|
193
|
+
entityId
|
|
194
|
+
});
|
|
195
|
+
await recordIndexerLog(
|
|
196
|
+
{ em: em ?? void 0 },
|
|
197
|
+
{
|
|
198
|
+
source: "fulltext",
|
|
199
|
+
handler: "worker:fulltext:purge",
|
|
200
|
+
message: `Purged entity from fulltext`,
|
|
201
|
+
entityType: entityId,
|
|
202
|
+
tenantId,
|
|
203
|
+
details: { jobId: jobCtx.jobId }
|
|
204
|
+
}
|
|
205
|
+
);
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
208
|
+
} catch (error) {
|
|
209
|
+
searchError("fulltext-index.worker", `Failed to ${jobType}`, {
|
|
210
|
+
jobId: jobCtx.jobId,
|
|
211
|
+
tenantId,
|
|
212
|
+
error: error instanceof Error ? error.message : error,
|
|
213
|
+
attemptNumber: jobCtx.attemptNumber
|
|
214
|
+
});
|
|
215
|
+
const entityId = "entityId" in job.payload ? job.payload.entityId : "entityType" in job.payload ? job.payload.entityType : void 0;
|
|
216
|
+
const recordId = "recordId" in job.payload ? job.payload.recordId : void 0;
|
|
217
|
+
await recordIndexerError(
|
|
218
|
+
{ em: em ?? void 0 },
|
|
219
|
+
{
|
|
220
|
+
source: "fulltext",
|
|
221
|
+
handler: `worker:fulltext:${jobType}`,
|
|
222
|
+
error,
|
|
223
|
+
entityType: entityId,
|
|
224
|
+
recordId,
|
|
225
|
+
tenantId,
|
|
226
|
+
payload: job.payload
|
|
227
|
+
}
|
|
228
|
+
);
|
|
229
|
+
throw error;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
async function handle(job, ctx) {
|
|
233
|
+
return handleFulltextIndexJob(job, ctx, ctx);
|
|
234
|
+
}
|
|
235
|
+
export {
|
|
236
|
+
handle as default,
|
|
237
|
+
handleFulltextIndexJob,
|
|
238
|
+
metadata
|
|
239
|
+
};
|
|
240
|
+
//# sourceMappingURL=fulltext-index.worker.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../../../src/modules/search/workers/fulltext-index.worker.ts"],
|
|
4
|
+
"sourcesContent": ["import type { QueuedJob, JobContext, WorkerMeta } from '@open-mercato/queue'\nimport { FULLTEXT_INDEXING_QUEUE_NAME, type FulltextIndexJobPayload } from '../../../queue/fulltext-indexing'\nimport type { FullTextSearchStrategy } from '../../../strategies/fulltext.strategy'\nimport type { SearchIndexer } from '../../../indexer/search-indexer'\nimport type { EntityManager } from '@mikro-orm/postgresql'\nimport type { Knex } from 'knex'\nimport type { EntityId } from '@open-mercato/shared/modules/entities'\nimport { recordIndexerLog } from '@open-mercato/shared/lib/indexers/status-log'\nimport { recordIndexerError } from '@open-mercato/shared/lib/indexers/error-log'\nimport { searchDebug, searchDebugWarn, searchError } from '../../../lib/debug'\nimport { updateReindexProgress } from '../lib/reindex-lock'\n\n// Worker metadata for auto-discovery\nconst DEFAULT_CONCURRENCY = 2\nconst envConcurrency = process.env.WORKERS_FULLTEXT_INDEXING_CONCURRENCY\n\nexport const metadata: WorkerMeta = {\n queue: FULLTEXT_INDEXING_QUEUE_NAME,\n concurrency: envConcurrency ? parseInt(envConcurrency, 10) : DEFAULT_CONCURRENCY,\n}\n\ntype HandlerContext = { resolve: <T = unknown>(name: string) => T }\n\n/**\n * Process a fulltext indexing job.\n *\n * This handler processes single record indexing, batch indexing, deletion, and purge\n * operations for the fulltext search strategy.\n *\n * All indexing operations (single and batch) use searchIndexer.indexRecordById() to load\n * fresh data, ensuring consistency with the vector worker pattern.\n *\n * @param job - The queued job containing payload\n * @param jobCtx - Queue job context with job ID and attempt info\n * @param ctx - DI container context for resolving services\n */\nexport async function handleFulltextIndexJob(\n job: QueuedJob<FulltextIndexJobPayload>,\n jobCtx: JobContext,\n ctx: HandlerContext,\n): Promise<void> {\n const { jobType, tenantId } = job.payload\n\n if (!tenantId) {\n searchDebugWarn('fulltext-index.worker', 'Skipping job with missing tenantId', {\n jobId: jobCtx.jobId,\n jobType,\n })\n return\n }\n\n // Resolve EntityManager for logging and knex for database queries\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n let em: any | null = null\n let knex: Knex | null = null\n try {\n em = ctx.resolve('em') as EntityManager\n knex = (em.getConnection() as unknown as { getKnex: () => Knex }).getKnex()\n } catch {\n em = null\n knex = null\n }\n\n // Resolve searchIndexer for loading fresh data\n let searchIndexer: SearchIndexer | undefined\n try {\n searchIndexer = ctx.resolve<SearchIndexer>('searchIndexer')\n } catch {\n searchDebugWarn('fulltext-index.worker', 'searchIndexer not available')\n }\n\n // Resolve fulltext strategy\n let fulltextStrategy: FullTextSearchStrategy | undefined\n try {\n const searchStrategies = ctx.resolve<unknown[]>('searchStrategies')\n fulltextStrategy = searchStrategies?.find(\n (s: unknown) => (s as { id?: string })?.id === 'fulltext',\n ) as FullTextSearchStrategy | undefined\n } catch {\n searchDebugWarn('fulltext-index.worker', 'searchStrategies not available')\n return\n }\n\n if (!fulltextStrategy) {\n searchDebugWarn('fulltext-index.worker', 'Fulltext strategy not configured')\n return\n }\n\n // Check if fulltext is available\n const isAvailable = await fulltextStrategy.isAvailable()\n if (!isAvailable) {\n throw new Error('Fulltext search is not available') // Will trigger retry\n }\n\n try {\n // ========== SINGLE INDEX: Use searchIndexer.indexRecordById() for fresh data ==========\n if (jobType === 'index') {\n const { entityType, recordId, organizationId } = job.payload as {\n entityType: string\n recordId: string\n organizationId?: string | null\n }\n\n if (!entityType || !recordId) {\n searchDebugWarn('fulltext-index.worker', 'Skipping index with missing fields', {\n jobId: jobCtx.jobId,\n entityType,\n recordId,\n })\n return\n }\n\n if (!searchIndexer) {\n throw new Error('searchIndexer not available for single-record index')\n }\n\n const result = await searchIndexer.indexRecordById({\n entityId: entityType as EntityId,\n recordId,\n tenantId,\n organizationId,\n })\n\n searchDebug('fulltext-index.worker', 'Indexed single record to fulltext', {\n jobId: jobCtx.jobId,\n tenantId,\n entityType,\n recordId,\n action: result.action,\n })\n\n await recordIndexerLog(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: 'worker:fulltext:index',\n message: `Indexed record to fulltext (${result.action})`,\n entityType,\n recordId,\n tenantId,\n details: { jobId: jobCtx.jobId },\n },\n )\n return\n }\n\n // ========== BATCH-INDEX: Use searchIndexer.indexRecordById() for fresh data ==========\n if (jobType === 'batch-index') {\n const { records, organizationId } = job.payload\n if (!records || records.length === 0) {\n searchDebugWarn('fulltext-index.worker', 'Skipping batch-index with no records', {\n jobId: jobCtx.jobId,\n })\n return\n }\n\n if (!searchIndexer) {\n throw new Error('searchIndexer not available for batch indexing')\n }\n\n // Process each record using indexRecordById (same pattern as vector worker)\n let successCount = 0\n let failCount = 0\n\n for (const { entityId, recordId } of records) {\n try {\n const result = await searchIndexer.indexRecordById({\n entityId: entityId as EntityId,\n recordId,\n tenantId,\n organizationId,\n })\n if (result.action === 'indexed') {\n successCount++\n }\n } catch (error) {\n failCount++\n searchDebugWarn('fulltext-index.worker', 'Failed to index record in batch', {\n entityId,\n recordId,\n error: error instanceof Error ? error.message : error,\n })\n }\n }\n\n // Update heartbeat to signal worker is still processing\n if (knex && successCount > 0) {\n await updateReindexProgress(knex, tenantId, 'fulltext', successCount, organizationId ?? null)\n }\n\n searchDebug('fulltext-index.worker', 'Batch indexed to fulltext', {\n jobId: jobCtx.jobId,\n tenantId,\n requestedCount: records.length,\n successCount,\n failCount,\n })\n\n await recordIndexerLog(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: 'worker:fulltext:batch-index',\n message: `Indexed ${successCount}/${records.length} records to fulltext`,\n tenantId,\n details: { jobId: jobCtx.jobId, requestedCount: records.length, successCount, failCount },\n },\n )\n return\n }\n\n // ========== DELETE ==========\n if (jobType === 'delete') {\n const { entityId, recordId } = job.payload\n if (!entityId || !recordId) {\n searchDebugWarn('fulltext-index.worker', 'Skipping delete with missing fields', {\n jobId: jobCtx.jobId,\n entityId,\n recordId,\n })\n return\n }\n\n await fulltextStrategy.delete(entityId, recordId, tenantId)\n\n searchDebug('fulltext-index.worker', 'Deleted from fulltext', {\n jobId: jobCtx.jobId,\n tenantId,\n entityId,\n recordId,\n })\n\n await recordIndexerLog(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: 'worker:fulltext:delete',\n message: `Deleted record from fulltext`,\n entityType: entityId,\n recordId,\n tenantId,\n details: { jobId: jobCtx.jobId },\n },\n )\n return\n }\n\n // ========== PURGE ==========\n if (jobType === 'purge') {\n const { entityId } = job.payload\n if (!entityId) {\n searchDebugWarn('fulltext-index.worker', 'Skipping purge with missing entityId', {\n jobId: jobCtx.jobId,\n })\n return\n }\n\n await fulltextStrategy.purge(entityId, tenantId)\n\n searchDebug('fulltext-index.worker', 'Purged entity from fulltext', {\n jobId: jobCtx.jobId,\n tenantId,\n entityId,\n })\n\n await recordIndexerLog(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: 'worker:fulltext:purge',\n message: `Purged entity from fulltext`,\n entityType: entityId,\n tenantId,\n details: { jobId: jobCtx.jobId },\n },\n )\n return\n }\n } catch (error) {\n searchError('fulltext-index.worker', `Failed to ${jobType}`, {\n jobId: jobCtx.jobId,\n tenantId,\n error: error instanceof Error ? error.message : error,\n attemptNumber: jobCtx.attemptNumber,\n })\n\n const entityId = 'entityId' in job.payload ? job.payload.entityId :\n 'entityType' in job.payload ? (job.payload as { entityType?: string }).entityType : undefined\n const recordId = 'recordId' in job.payload ? job.payload.recordId : undefined\n\n await recordIndexerError(\n { em: em ?? undefined },\n {\n source: 'fulltext',\n handler: `worker:fulltext:${jobType}`,\n error,\n entityType: entityId,\n recordId,\n tenantId,\n payload: job.payload,\n },\n )\n\n // Re-throw to let the queue handle retry logic\n throw error\n }\n}\n\n/**\n * Default export for worker auto-discovery.\n * Wraps handleFulltextIndexJob to match the expected handler signature.\n */\nexport default async function handle(\n job: QueuedJob<FulltextIndexJobPayload>,\n ctx: JobContext & HandlerContext\n): Promise<void> {\n return handleFulltextIndexJob(job, ctx, ctx)\n}\n"],
|
|
5
|
+
"mappings": "AACA,SAAS,oCAAkE;AAM3E,SAAS,wBAAwB;AACjC,SAAS,0BAA0B;AACnC,SAAS,aAAa,iBAAiB,mBAAmB;AAC1D,SAAS,6BAA6B;AAGtC,MAAM,sBAAsB;AAC5B,MAAM,iBAAiB,QAAQ,IAAI;AAE5B,MAAM,WAAuB;AAAA,EAClC,OAAO;AAAA,EACP,aAAa,iBAAiB,SAAS,gBAAgB,EAAE,IAAI;AAC/D;AAiBA,eAAsB,uBACpB,KACA,QACA,KACe;AACf,QAAM,EAAE,SAAS,SAAS,IAAI,IAAI;AAElC,MAAI,CAAC,UAAU;AACb,oBAAgB,yBAAyB,sCAAsC;AAAA,MAC7E,OAAO,OAAO;AAAA,MACd;AAAA,IACF,CAAC;AACD;AAAA,EACF;AAIA,MAAI,KAAiB;AACrB,MAAI,OAAoB;AACxB,MAAI;AACF,SAAK,IAAI,QAAQ,IAAI;AACrB,WAAQ,GAAG,cAAc,EAAyC,QAAQ;AAAA,EAC5E,QAAQ;AACN,SAAK;AACL,WAAO;AAAA,EACT;AAGA,MAAI;AACJ,MAAI;AACF,oBAAgB,IAAI,QAAuB,eAAe;AAAA,EAC5D,QAAQ;AACN,oBAAgB,yBAAyB,6BAA6B;AAAA,EACxE;AAGA,MAAI;AACJ,MAAI;AACF,UAAM,mBAAmB,IAAI,QAAmB,kBAAkB;AAClE,uBAAmB,kBAAkB;AAAA,MACnC,CAAC,MAAgB,GAAuB,OAAO;AAAA,IACjD;AAAA,EACF,QAAQ;AACN,oBAAgB,yBAAyB,gCAAgC;AACzE;AAAA,EACF;AAEA,MAAI,CAAC,kBAAkB;AACrB,oBAAgB,yBAAyB,kCAAkC;AAC3E;AAAA,EACF;AAGA,QAAM,cAAc,MAAM,iBAAiB,YAAY;AACvD,MAAI,CAAC,aAAa;AAChB,UAAM,IAAI,MAAM,kCAAkC;AAAA,EACpD;AAEA,MAAI;AAEF,QAAI,YAAY,SAAS;AACvB,YAAM,EAAE,YAAY,UAAU,eAAe,IAAI,IAAI;AAMrD,UAAI,CAAC,cAAc,CAAC,UAAU;AAC5B,wBAAgB,yBAAyB,sCAAsC;AAAA,UAC7E,OAAO,OAAO;AAAA,UACd;AAAA,UACA;AAAA,QACF,CAAC;AACD;AAAA,MACF;AAEA,UAAI,CAAC,eAAe;AAClB,cAAM,IAAI,MAAM,qDAAqD;AAAA,MACvE;AAEA,YAAM,SAAS,MAAM,cAAc,gBAAgB;AAAA,QACjD,UAAU;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AAED,kBAAY,yBAAyB,qCAAqC;AAAA,QACxE,OAAO,OAAO;AAAA,QACd;AAAA,QACA;AAAA,QACA;AAAA,QACA,QAAQ,OAAO;AAAA,MACjB,CAAC;AAED,YAAM;AAAA,QACJ,EAAE,IAAI,MAAM,OAAU;AAAA,QACtB;AAAA,UACE,QAAQ;AAAA,UACR,SAAS;AAAA,UACT,SAAS,+BAA+B,OAAO,MAAM;AAAA,UACrD;AAAA,UACA;AAAA,UACA;AAAA,UACA,SAAS,EAAE,OAAO,OAAO,MAAM;AAAA,QACjC;AAAA,MACF;AACA;AAAA,IACF;AAGA,QAAI,YAAY,eAAe;AAC7B,YAAM,EAAE,SAAS,eAAe,IAAI,IAAI;AACxC,UAAI,CAAC,WAAW,QAAQ,WAAW,GAAG;AACpC,wBAAgB,yBAAyB,wCAAwC;AAAA,UAC/E,OAAO,OAAO;AAAA,QAChB,CAAC;AACD;AAAA,MACF;AAEA,UAAI,CAAC,eAAe;AAClB,cAAM,IAAI,MAAM,gDAAgD;AAAA,MAClE;AAGA,UAAI,eAAe;AACnB,UAAI,YAAY;AAEhB,iBAAW,EAAE,UAAU,SAAS,KAAK,SAAS;AAC5C,YAAI;AACF,gBAAM,SAAS,MAAM,cAAc,gBAAgB;AAAA,YACjD;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,UACF,CAAC;AACD,cAAI,OAAO,WAAW,WAAW;AAC/B;AAAA,UACF;AAAA,QACF,SAAS,OAAO;AACd;AACA,0BAAgB,yBAAyB,mCAAmC;AAAA,YAC1E;AAAA,YACA;AAAA,YACA,OAAO,iBAAiB,QAAQ,MAAM,UAAU;AAAA,UAClD,CAAC;AAAA,QACH;AAAA,MACF;AAGA,UAAI,QAAQ,eAAe,GAAG;AAC5B,cAAM,sBAAsB,MAAM,UAAU,YAAY,cAAc,kBAAkB,IAAI;AAAA,MAC9F;AAEA,kBAAY,yBAAyB,6BAA6B;AAAA,QAChE,OAAO,OAAO;AAAA,QACd;AAAA,QACA,gBAAgB,QAAQ;AAAA,QACxB;AAAA,QACA;AAAA,MACF,CAAC;AAED,YAAM;AAAA,QACJ,EAAE,IAAI,MAAM,OAAU;AAAA,QACtB;AAAA,UACE,QAAQ;AAAA,UACR,SAAS;AAAA,UACT,SAAS,WAAW,YAAY,IAAI,QAAQ,MAAM;AAAA,UAClD;AAAA,UACA,SAAS,EAAE,OAAO,OAAO,OAAO,gBAAgB,QAAQ,QAAQ,cAAc,UAAU;AAAA,QAC1F;AAAA,MACF;AACA;AAAA,IACF;AAGA,QAAI,YAAY,UAAU;AACxB,YAAM,EAAE,UAAU,SAAS,IAAI,IAAI;AACnC,UAAI,CAAC,YAAY,CAAC,UAAU;AAC1B,wBAAgB,yBAAyB,uCAAuC;AAAA,UAC9E,OAAO,OAAO;AAAA,UACd;AAAA,UACA;AAAA,QACF,CAAC;AACD;AAAA,MACF;AAEA,YAAM,iBAAiB,OAAO,UAAU,UAAU,QAAQ;AAE1D,kBAAY,yBAAyB,yBAAyB;AAAA,QAC5D,OAAO,OAAO;AAAA,QACd;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AAED,YAAM;AAAA,QACJ,EAAE,IAAI,MAAM,OAAU;AAAA,QACtB;AAAA,UACE,QAAQ;AAAA,UACR,SAAS;AAAA,UACT,SAAS;AAAA,UACT,YAAY;AAAA,UACZ;AAAA,UACA;AAAA,UACA,SAAS,EAAE,OAAO,OAAO,MAAM;AAAA,QACjC;AAAA,MACF;AACA;AAAA,IACF;AAGA,QAAI,YAAY,SAAS;AACvB,YAAM,EAAE,SAAS,IAAI,IAAI;AACzB,UAAI,CAAC,UAAU;AACb,wBAAgB,yBAAyB,wCAAwC;AAAA,UAC/E,OAAO,OAAO;AAAA,QAChB,CAAC;AACD;AAAA,MACF;AAEA,YAAM,iBAAiB,MAAM,UAAU,QAAQ;AAE/C,kBAAY,yBAAyB,+BAA+B;AAAA,QAClE,OAAO,OAAO;AAAA,QACd;AAAA,QACA;AAAA,MACF,CAAC;AAED,YAAM;AAAA,QACJ,EAAE,IAAI,MAAM,OAAU;AAAA,QACtB;AAAA,UACE,QAAQ;AAAA,UACR,SAAS;AAAA,UACT,SAAS;AAAA,UACT,YAAY;AAAA,UACZ;AAAA,UACA,SAAS,EAAE,OAAO,OAAO,MAAM;AAAA,QACjC;AAAA,MACF;AACA;AAAA,IACF;AAAA,EACF,SAAS,OAAO;AACd,gBAAY,yBAAyB,aAAa,OAAO,IAAI;AAAA,MAC3D,OAAO,OAAO;AAAA,MACd;AAAA,MACA,OAAO,iBAAiB,QAAQ,MAAM,UAAU;AAAA,MAChD,eAAe,OAAO;AAAA,IACxB,CAAC;AAED,UAAM,WAAW,cAAc,IAAI,UAAU,IAAI,QAAQ,WACxC,gBAAgB,IAAI,UAAW,IAAI,QAAoC,aAAa;AACrG,UAAM,WAAW,cAAc,IAAI,UAAU,IAAI,QAAQ,WAAW;AAEpE,UAAM;AAAA,MACJ,EAAE,IAAI,MAAM,OAAU;AAAA,MACtB;AAAA,QACE,QAAQ;AAAA,QACR,SAAS,mBAAmB,OAAO;AAAA,QACnC;AAAA,QACA,YAAY;AAAA,QACZ;AAAA,QACA;AAAA,QACA,SAAS,IAAI;AAAA,MACf;AAAA,IACF;AAGA,UAAM;AAAA,EACR;AACF;AAMA,eAAO,OACL,KACA,KACe;AACf,SAAO,uBAAuB,KAAK,KAAK,GAAG;AAC7C;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
import { VECTOR_INDEXING_QUEUE_NAME } from "../../../queue/vector-indexing.js";
|
|
2
|
+
import { recordIndexerError } from "@open-mercato/shared/lib/indexers/error-log";
|
|
3
|
+
import { applyCoverageAdjustments, createCoverageAdjustments } from "@open-mercato/core/modules/query_index/lib/coverage";
|
|
4
|
+
import { logVectorOperation } from "../../../vector/lib/vector-logs.js";
|
|
5
|
+
import { resolveAutoIndexingEnabled } from "../lib/auto-indexing.js";
|
|
6
|
+
import { resolveEmbeddingConfig } from "../lib/embedding-config.js";
|
|
7
|
+
import { searchDebugWarn } from "../../../lib/debug.js";
|
|
8
|
+
import { updateReindexProgress } from "../lib/reindex-lock.js";
|
|
9
|
+
const DEFAULT_CONCURRENCY = 2;
|
|
10
|
+
const envConcurrency = process.env.WORKERS_VECTOR_INDEXING_CONCURRENCY;
|
|
11
|
+
const metadata = {
|
|
12
|
+
queue: VECTOR_INDEXING_QUEUE_NAME,
|
|
13
|
+
concurrency: envConcurrency ? parseInt(envConcurrency, 10) : DEFAULT_CONCURRENCY
|
|
14
|
+
};
|
|
15
|
+
async function handleVectorIndexJob(job, jobCtx, ctx) {
|
|
16
|
+
const { jobType, entityType, recordId, tenantId, organizationId, records } = job.payload;
|
|
17
|
+
if (jobType === "batch-index") {
|
|
18
|
+
if (!records?.length || !tenantId) {
|
|
19
|
+
searchDebugWarn("vector-index.worker", "Skipping batch-index job with missing required fields", {
|
|
20
|
+
jobId: jobCtx.jobId,
|
|
21
|
+
recordCount: records?.length ?? 0,
|
|
22
|
+
tenantId
|
|
23
|
+
});
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
26
|
+
let searchIndexer2;
|
|
27
|
+
try {
|
|
28
|
+
searchIndexer2 = ctx.resolve("searchIndexer");
|
|
29
|
+
} catch {
|
|
30
|
+
searchDebugWarn("vector-index.worker", "searchIndexer not available");
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
let knex = null;
|
|
34
|
+
try {
|
|
35
|
+
const em2 = ctx.resolve("em");
|
|
36
|
+
knex = em2.getConnection().getKnex();
|
|
37
|
+
} catch {
|
|
38
|
+
knex = null;
|
|
39
|
+
}
|
|
40
|
+
try {
|
|
41
|
+
const embeddingConfig = await resolveEmbeddingConfig(ctx, { defaultValue: null });
|
|
42
|
+
if (embeddingConfig) {
|
|
43
|
+
const embeddingService = ctx.resolve("vectorEmbeddingService");
|
|
44
|
+
embeddingService.updateConfig(embeddingConfig);
|
|
45
|
+
}
|
|
46
|
+
} catch (configErr) {
|
|
47
|
+
searchDebugWarn("vector-index.worker", "Failed to load embedding config for batch, using defaults", {
|
|
48
|
+
error: configErr instanceof Error ? configErr.message : configErr
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
let successCount = 0;
|
|
52
|
+
let failCount = 0;
|
|
53
|
+
for (const { entityId, recordId: recId } of records) {
|
|
54
|
+
try {
|
|
55
|
+
const result = await searchIndexer2.indexRecordById({
|
|
56
|
+
entityId,
|
|
57
|
+
recordId: recId,
|
|
58
|
+
tenantId,
|
|
59
|
+
organizationId
|
|
60
|
+
});
|
|
61
|
+
if (result.action === "indexed") {
|
|
62
|
+
successCount++;
|
|
63
|
+
}
|
|
64
|
+
} catch (error) {
|
|
65
|
+
failCount++;
|
|
66
|
+
searchDebugWarn("vector-index.worker", "Failed to index record in batch", {
|
|
67
|
+
entityId,
|
|
68
|
+
recordId: recId,
|
|
69
|
+
error: error instanceof Error ? error.message : error
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
if (knex && successCount > 0) {
|
|
74
|
+
await updateReindexProgress(knex, tenantId, "vector", successCount, organizationId ?? null);
|
|
75
|
+
}
|
|
76
|
+
searchDebugWarn("vector-index.worker", "Batch-index job completed", {
|
|
77
|
+
jobId: jobCtx.jobId,
|
|
78
|
+
totalRecords: records.length,
|
|
79
|
+
successCount,
|
|
80
|
+
failCount
|
|
81
|
+
});
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
if (!entityType || !recordId || !tenantId) {
|
|
85
|
+
searchDebugWarn("vector-index.worker", "Skipping job with missing required fields", {
|
|
86
|
+
jobId: jobCtx.jobId,
|
|
87
|
+
entityType,
|
|
88
|
+
recordId,
|
|
89
|
+
tenantId
|
|
90
|
+
});
|
|
91
|
+
return;
|
|
92
|
+
}
|
|
93
|
+
const autoIndexingEnabled = await resolveAutoIndexingEnabled(ctx, { defaultValue: true });
|
|
94
|
+
if (!autoIndexingEnabled) {
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
let searchIndexer;
|
|
98
|
+
try {
|
|
99
|
+
searchIndexer = ctx.resolve("searchIndexer");
|
|
100
|
+
} catch {
|
|
101
|
+
searchDebugWarn("vector-index.worker", "searchIndexer not available");
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
try {
|
|
105
|
+
const embeddingConfig = await resolveEmbeddingConfig(ctx, { defaultValue: null });
|
|
106
|
+
if (embeddingConfig) {
|
|
107
|
+
const embeddingService = ctx.resolve("vectorEmbeddingService");
|
|
108
|
+
embeddingService.updateConfig(embeddingConfig);
|
|
109
|
+
}
|
|
110
|
+
} catch (configErr) {
|
|
111
|
+
if (jobType === "index") {
|
|
112
|
+
searchDebugWarn("vector-index.worker", "Failed to load embedding config, using defaults", {
|
|
113
|
+
error: configErr instanceof Error ? configErr.message : configErr
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
let em = null;
|
|
118
|
+
try {
|
|
119
|
+
em = ctx.resolve("em");
|
|
120
|
+
} catch {
|
|
121
|
+
em = null;
|
|
122
|
+
}
|
|
123
|
+
let eventBus = null;
|
|
124
|
+
try {
|
|
125
|
+
eventBus = ctx.resolve("eventBus");
|
|
126
|
+
} catch {
|
|
127
|
+
eventBus = null;
|
|
128
|
+
}
|
|
129
|
+
const handlerName = jobType === "delete" ? "worker:vector-indexing:delete" : "worker:vector-indexing:index";
|
|
130
|
+
try {
|
|
131
|
+
let action = "skipped";
|
|
132
|
+
let delta = 0;
|
|
133
|
+
if (jobType === "delete") {
|
|
134
|
+
await searchIndexer.deleteRecord({
|
|
135
|
+
entityId: entityType,
|
|
136
|
+
recordId,
|
|
137
|
+
tenantId
|
|
138
|
+
});
|
|
139
|
+
action = "deleted";
|
|
140
|
+
delta = -1;
|
|
141
|
+
} else {
|
|
142
|
+
const result = await searchIndexer.indexRecordById({
|
|
143
|
+
entityId: entityType,
|
|
144
|
+
recordId,
|
|
145
|
+
tenantId,
|
|
146
|
+
organizationId
|
|
147
|
+
});
|
|
148
|
+
action = result.action;
|
|
149
|
+
if (result.action === "indexed") {
|
|
150
|
+
delta = 1;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
if (delta !== 0) {
|
|
154
|
+
let adjustmentsApplied = false;
|
|
155
|
+
if (em) {
|
|
156
|
+
try {
|
|
157
|
+
const adjustments = createCoverageAdjustments({
|
|
158
|
+
entityType,
|
|
159
|
+
tenantId,
|
|
160
|
+
organizationId,
|
|
161
|
+
baseDelta: 0,
|
|
162
|
+
indexDelta: 0,
|
|
163
|
+
vectorDelta: delta
|
|
164
|
+
});
|
|
165
|
+
if (adjustments.length) {
|
|
166
|
+
await applyCoverageAdjustments(em, adjustments);
|
|
167
|
+
adjustmentsApplied = true;
|
|
168
|
+
}
|
|
169
|
+
} catch (coverageError) {
|
|
170
|
+
searchDebugWarn("vector-index.worker", "Failed to adjust vector coverage", {
|
|
171
|
+
error: coverageError instanceof Error ? coverageError.message : coverageError
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
if (!adjustmentsApplied && eventBus) {
|
|
176
|
+
try {
|
|
177
|
+
await eventBus.emitEvent("query_index.coverage.refresh", {
|
|
178
|
+
entityType,
|
|
179
|
+
tenantId,
|
|
180
|
+
organizationId,
|
|
181
|
+
withDeleted: false,
|
|
182
|
+
delayMs: 1e3
|
|
183
|
+
});
|
|
184
|
+
} catch (emitError) {
|
|
185
|
+
searchDebugWarn("vector-index.worker", "Failed to enqueue coverage refresh", {
|
|
186
|
+
error: emitError instanceof Error ? emitError.message : emitError
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
await logVectorOperation({
|
|
192
|
+
em,
|
|
193
|
+
handler: handlerName,
|
|
194
|
+
entityType,
|
|
195
|
+
recordId,
|
|
196
|
+
result: {
|
|
197
|
+
action,
|
|
198
|
+
tenantId,
|
|
199
|
+
organizationId: organizationId ?? null,
|
|
200
|
+
created: action === "indexed",
|
|
201
|
+
existed: action === "deleted"
|
|
202
|
+
}
|
|
203
|
+
});
|
|
204
|
+
} catch (error) {
|
|
205
|
+
searchDebugWarn("vector-index.worker", `Failed to ${jobType} vector index`, {
|
|
206
|
+
entityType,
|
|
207
|
+
recordId,
|
|
208
|
+
error: error instanceof Error ? error.message : error
|
|
209
|
+
});
|
|
210
|
+
await recordIndexerError(
|
|
211
|
+
{ em: em ?? void 0 },
|
|
212
|
+
{
|
|
213
|
+
source: "vector",
|
|
214
|
+
handler: handlerName,
|
|
215
|
+
error,
|
|
216
|
+
entityType,
|
|
217
|
+
recordId,
|
|
218
|
+
tenantId,
|
|
219
|
+
organizationId,
|
|
220
|
+
payload: job.payload
|
|
221
|
+
}
|
|
222
|
+
);
|
|
223
|
+
throw error;
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
async function handle(job, ctx) {
|
|
227
|
+
return handleVectorIndexJob(job, ctx, ctx);
|
|
228
|
+
}
|
|
229
|
+
export {
|
|
230
|
+
handle as default,
|
|
231
|
+
handleVectorIndexJob,
|
|
232
|
+
metadata
|
|
233
|
+
};
|
|
234
|
+
//# sourceMappingURL=vector-index.worker.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../../../src/modules/search/workers/vector-index.worker.ts"],
|
|
4
|
+
"sourcesContent": ["import type { QueuedJob, JobContext, WorkerMeta } from '@open-mercato/queue'\nimport { VECTOR_INDEXING_QUEUE_NAME, type VectorIndexJobPayload } from '../../../queue/vector-indexing'\nimport type { SearchIndexer } from '../../../indexer/search-indexer'\nimport type { EmbeddingService } from '../../../vector'\nimport type { EntityManager } from '@mikro-orm/postgresql'\nimport type { Knex } from 'knex'\nimport { recordIndexerError } from '@open-mercato/shared/lib/indexers/error-log'\nimport { applyCoverageAdjustments, createCoverageAdjustments } from '@open-mercato/core/modules/query_index/lib/coverage'\nimport { logVectorOperation } from '../../../vector/lib/vector-logs'\nimport { resolveAutoIndexingEnabled } from '../lib/auto-indexing'\nimport { resolveEmbeddingConfig } from '../lib/embedding-config'\nimport { searchDebugWarn } from '../../../lib/debug'\nimport { updateReindexProgress } from '../lib/reindex-lock'\n\n// Worker metadata for auto-discovery\nconst DEFAULT_CONCURRENCY = 2\nconst envConcurrency = process.env.WORKERS_VECTOR_INDEXING_CONCURRENCY\n\nexport const metadata: WorkerMeta = {\n queue: VECTOR_INDEXING_QUEUE_NAME,\n concurrency: envConcurrency ? parseInt(envConcurrency, 10) : DEFAULT_CONCURRENCY,\n}\n\ntype HandlerContext = { resolve: <T = unknown>(name: string) => T }\n\n/**\n * Process a vector index job.\n *\n * This handler is called by the queue worker to process indexing and deletion jobs.\n * It uses SearchIndexer to load records and index them via SearchService.\n *\n * @param job - The queued job containing payload\n * @param jobCtx - Queue job context with job ID and attempt info\n * @param ctx - DI container context for resolving services\n */\nexport async function handleVectorIndexJob(\n job: QueuedJob<VectorIndexJobPayload>,\n jobCtx: JobContext,\n ctx: HandlerContext,\n): Promise<void> {\n const { jobType, entityType, recordId, tenantId, organizationId, records } = job.payload\n\n // Handle batch-index jobs (from reindex operations)\n if (jobType === 'batch-index') {\n if (!records?.length || !tenantId) {\n searchDebugWarn('vector-index.worker', 'Skipping batch-index job with missing required fields', {\n jobId: jobCtx.jobId,\n recordCount: records?.length ?? 0,\n tenantId,\n })\n return\n }\n\n let searchIndexer: SearchIndexer\n try {\n searchIndexer = ctx.resolve<SearchIndexer>('searchIndexer')\n } catch {\n searchDebugWarn('vector-index.worker', 'searchIndexer not available')\n return\n }\n\n // Get knex for heartbeat updates\n let knex: Knex | null = null\n try {\n const em = ctx.resolve('em') as EntityManager\n knex = (em.getConnection() as unknown as { getKnex: () => Knex }).getKnex()\n } catch {\n knex = null\n }\n\n // Load saved embedding config to use the correct provider/model\n try {\n const embeddingConfig = await resolveEmbeddingConfig(ctx, { defaultValue: null })\n if (embeddingConfig) {\n const embeddingService = ctx.resolve<EmbeddingService>('vectorEmbeddingService')\n embeddingService.updateConfig(embeddingConfig)\n }\n } catch (configErr) {\n searchDebugWarn('vector-index.worker', 'Failed to load embedding config for batch, using defaults', {\n error: configErr instanceof Error ? configErr.message : configErr,\n })\n }\n\n // Process each record in the batch\n let successCount = 0\n let failCount = 0\n for (const { entityId, recordId: recId } of records) {\n try {\n const result = await searchIndexer.indexRecordById({\n entityId: entityId as Parameters<typeof searchIndexer.indexRecordById>[0]['entityId'],\n recordId: recId,\n tenantId,\n organizationId,\n })\n if (result.action === 'indexed') {\n successCount++\n }\n } catch (error) {\n failCount++\n searchDebugWarn('vector-index.worker', 'Failed to index record in batch', {\n entityId,\n recordId: recId,\n error: error instanceof Error ? error.message : error,\n })\n }\n }\n\n // Update heartbeat to signal worker is still processing\n if (knex && successCount > 0) {\n await updateReindexProgress(knex, tenantId, 'vector', successCount, organizationId ?? null)\n }\n\n searchDebugWarn('vector-index.worker', 'Batch-index job completed', {\n jobId: jobCtx.jobId,\n totalRecords: records.length,\n successCount,\n failCount,\n })\n return\n }\n\n // Handle single record jobs (index/delete)\n if (!entityType || !recordId || !tenantId) {\n searchDebugWarn('vector-index.worker', 'Skipping job with missing required fields', {\n jobId: jobCtx.jobId,\n entityType,\n recordId,\n tenantId,\n })\n return\n }\n\n const autoIndexingEnabled = await resolveAutoIndexingEnabled(ctx, { defaultValue: true })\n if (!autoIndexingEnabled) {\n return\n }\n\n let searchIndexer: SearchIndexer\n try {\n searchIndexer = ctx.resolve<SearchIndexer>('searchIndexer')\n } catch {\n searchDebugWarn('vector-index.worker', 'searchIndexer not available')\n return\n }\n\n // Load saved embedding config to use the correct provider/model\n try {\n const embeddingConfig = await resolveEmbeddingConfig(ctx, { defaultValue: null })\n if (embeddingConfig) {\n const embeddingService = ctx.resolve<EmbeddingService>('vectorEmbeddingService')\n embeddingService.updateConfig(embeddingConfig)\n }\n } catch (configErr) {\n // Delete operations don't require embedding, only warn for index operations\n if (jobType === 'index') {\n searchDebugWarn('vector-index.worker', 'Failed to load embedding config, using defaults', {\n error: configErr instanceof Error ? configErr.message : configErr,\n })\n }\n }\n\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n let em: any | null = null\n try {\n em = ctx.resolve('em')\n } catch {\n em = null\n }\n\n let eventBus: { emitEvent(event: string, payload: unknown, options?: unknown): Promise<void> } | null = null\n try {\n eventBus = ctx.resolve('eventBus')\n } catch {\n eventBus = null\n }\n\n const handlerName = jobType === 'delete'\n ? 'worker:vector-indexing:delete'\n : 'worker:vector-indexing:index'\n\n try {\n let action: 'indexed' | 'deleted' | 'skipped' = 'skipped'\n let delta = 0\n\n if (jobType === 'delete') {\n await searchIndexer.deleteRecord({\n entityId: entityType,\n recordId,\n tenantId,\n })\n action = 'deleted'\n delta = -1\n } else {\n const result = await searchIndexer.indexRecordById({\n entityId: entityType,\n recordId,\n tenantId,\n organizationId,\n })\n action = result.action\n if (result.action === 'indexed') {\n delta = 1\n }\n }\n\n if (delta !== 0) {\n let adjustmentsApplied = false\n if (em) {\n try {\n const adjustments = createCoverageAdjustments({\n entityType,\n tenantId,\n organizationId,\n baseDelta: 0,\n indexDelta: 0,\n vectorDelta: delta,\n })\n if (adjustments.length) {\n await applyCoverageAdjustments(em, adjustments)\n adjustmentsApplied = true\n }\n } catch (coverageError) {\n searchDebugWarn('vector-index.worker', 'Failed to adjust vector coverage', {\n error: coverageError instanceof Error ? coverageError.message : coverageError,\n })\n }\n }\n\n if (!adjustmentsApplied && eventBus) {\n try {\n await eventBus.emitEvent('query_index.coverage.refresh', {\n entityType,\n tenantId,\n organizationId,\n withDeleted: false,\n delayMs: 1000,\n })\n } catch (emitError) {\n searchDebugWarn('vector-index.worker', 'Failed to enqueue coverage refresh', {\n error: emitError instanceof Error ? emitError.message : emitError,\n })\n }\n }\n }\n\n await logVectorOperation({\n em,\n handler: handlerName,\n entityType,\n recordId,\n result: {\n action,\n tenantId,\n organizationId: organizationId ?? null,\n created: action === 'indexed',\n existed: action === 'deleted',\n },\n })\n } catch (error) {\n searchDebugWarn('vector-index.worker', `Failed to ${jobType} vector index`, {\n entityType,\n recordId,\n error: error instanceof Error ? error.message : error,\n })\n await recordIndexerError(\n { em: em ?? undefined },\n {\n source: 'vector',\n handler: handlerName,\n error,\n entityType,\n recordId,\n tenantId,\n organizationId,\n payload: job.payload,\n },\n )\n // Re-throw to let the queue handle retry logic\n throw error\n }\n}\n\n/**\n * Default export for worker auto-discovery.\n * Wraps handleVectorIndexJob to match the expected handler signature.\n */\nexport default async function handle(\n job: QueuedJob<VectorIndexJobPayload>,\n ctx: JobContext & HandlerContext\n): Promise<void> {\n return handleVectorIndexJob(job, ctx, ctx)\n}\n"],
|
|
5
|
+
"mappings": "AACA,SAAS,kCAA8D;AAKvE,SAAS,0BAA0B;AACnC,SAAS,0BAA0B,iCAAiC;AACpE,SAAS,0BAA0B;AACnC,SAAS,kCAAkC;AAC3C,SAAS,8BAA8B;AACvC,SAAS,uBAAuB;AAChC,SAAS,6BAA6B;AAGtC,MAAM,sBAAsB;AAC5B,MAAM,iBAAiB,QAAQ,IAAI;AAE5B,MAAM,WAAuB;AAAA,EAClC,OAAO;AAAA,EACP,aAAa,iBAAiB,SAAS,gBAAgB,EAAE,IAAI;AAC/D;AAcA,eAAsB,qBACpB,KACA,QACA,KACe;AACf,QAAM,EAAE,SAAS,YAAY,UAAU,UAAU,gBAAgB,QAAQ,IAAI,IAAI;AAGjF,MAAI,YAAY,eAAe;AAC7B,QAAI,CAAC,SAAS,UAAU,CAAC,UAAU;AACjC,sBAAgB,uBAAuB,yDAAyD;AAAA,QAC9F,OAAO,OAAO;AAAA,QACd,aAAa,SAAS,UAAU;AAAA,QAChC;AAAA,MACF,CAAC;AACD;AAAA,IACF;AAEA,QAAIA;AACJ,QAAI;AACF,MAAAA,iBAAgB,IAAI,QAAuB,eAAe;AAAA,IAC5D,QAAQ;AACN,sBAAgB,uBAAuB,6BAA6B;AACpE;AAAA,IACF;AAGA,QAAI,OAAoB;AACxB,QAAI;AACF,YAAMC,MAAK,IAAI,QAAQ,IAAI;AAC3B,aAAQA,IAAG,cAAc,EAAyC,QAAQ;AAAA,IAC5E,QAAQ;AACN,aAAO;AAAA,IACT;AAGA,QAAI;AACF,YAAM,kBAAkB,MAAM,uBAAuB,KAAK,EAAE,cAAc,KAAK,CAAC;AAChF,UAAI,iBAAiB;AACnB,cAAM,mBAAmB,IAAI,QAA0B,wBAAwB;AAC/E,yBAAiB,aAAa,eAAe;AAAA,MAC/C;AAAA,IACF,SAAS,WAAW;AAClB,sBAAgB,uBAAuB,6DAA6D;AAAA,QAClG,OAAO,qBAAqB,QAAQ,UAAU,UAAU;AAAA,MAC1D,CAAC;AAAA,IACH;AAGA,QAAI,eAAe;AACnB,QAAI,YAAY;AAChB,eAAW,EAAE,UAAU,UAAU,MAAM,KAAK,SAAS;AACnD,UAAI;AACF,cAAM,SAAS,MAAMD,eAAc,gBAAgB;AAAA,UACjD;AAAA,UACA,UAAU;AAAA,UACV;AAAA,UACA;AAAA,QACF,CAAC;AACD,YAAI,OAAO,WAAW,WAAW;AAC/B;AAAA,QACF;AAAA,MACF,SAAS,OAAO;AACd;AACA,wBAAgB,uBAAuB,mCAAmC;AAAA,UACxE;AAAA,UACA,UAAU;AAAA,UACV,OAAO,iBAAiB,QAAQ,MAAM,UAAU;AAAA,QAClD,CAAC;AAAA,MACH;AAAA,IACF;AAGA,QAAI,QAAQ,eAAe,GAAG;AAC5B,YAAM,sBAAsB,MAAM,UAAU,UAAU,cAAc,kBAAkB,IAAI;AAAA,IAC5F;AAEA,oBAAgB,uBAAuB,6BAA6B;AAAA,MAClE,OAAO,OAAO;AAAA,MACd,cAAc,QAAQ;AAAA,MACtB;AAAA,MACA;AAAA,IACF,CAAC;AACD;AAAA,EACF;AAGA,MAAI,CAAC,cAAc,CAAC,YAAY,CAAC,UAAU;AACzC,oBAAgB,uBAAuB,6CAA6C;AAAA,MAClF,OAAO,OAAO;AAAA,MACd;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AACD;AAAA,EACF;AAEA,QAAM,sBAAsB,MAAM,2BAA2B,KAAK,EAAE,cAAc,KAAK,CAAC;AACxF,MAAI,CAAC,qBAAqB;AACxB;AAAA,EACF;AAEA,MAAI;AACJ,MAAI;AACF,oBAAgB,IAAI,QAAuB,eAAe;AAAA,EAC5D,QAAQ;AACN,oBAAgB,uBAAuB,6BAA6B;AACpE;AAAA,EACF;AAGA,MAAI;AACF,UAAM,kBAAkB,MAAM,uBAAuB,KAAK,EAAE,cAAc,KAAK,CAAC;AAChF,QAAI,iBAAiB;AACnB,YAAM,mBAAmB,IAAI,QAA0B,wBAAwB;AAC/E,uBAAiB,aAAa,eAAe;AAAA,IAC/C;AAAA,EACF,SAAS,WAAW;AAElB,QAAI,YAAY,SAAS;AACvB,sBAAgB,uBAAuB,mDAAmD;AAAA,QACxF,OAAO,qBAAqB,QAAQ,UAAU,UAAU;AAAA,MAC1D,CAAC;AAAA,IACH;AAAA,EACF;AAGA,MAAI,KAAiB;AACrB,MAAI;AACF,SAAK,IAAI,QAAQ,IAAI;AAAA,EACvB,QAAQ;AACN,SAAK;AAAA,EACP;AAEA,MAAI,WAAoG;AACxG,MAAI;AACF,eAAW,IAAI,QAAQ,UAAU;AAAA,EACnC,QAAQ;AACN,eAAW;AAAA,EACb;AAEA,QAAM,cAAc,YAAY,WAC5B,kCACA;AAEJ,MAAI;AACF,QAAI,SAA4C;AAChD,QAAI,QAAQ;AAEZ,QAAI,YAAY,UAAU;AACxB,YAAM,cAAc,aAAa;AAAA,QAC/B,UAAU;AAAA,QACV;AAAA,QACA;AAAA,MACF,CAAC;AACD,eAAS;AACT,cAAQ;AAAA,IACV,OAAO;AACL,YAAM,SAAS,MAAM,cAAc,gBAAgB;AAAA,QACjD,UAAU;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AACD,eAAS,OAAO;AAChB,UAAI,OAAO,WAAW,WAAW;AAC/B,gBAAQ;AAAA,MACV;AAAA,IACF;AAEA,QAAI,UAAU,GAAG;AACf,UAAI,qBAAqB;AACzB,UAAI,IAAI;AACN,YAAI;AACF,gBAAM,cAAc,0BAA0B;AAAA,YAC5C;AAAA,YACA;AAAA,YACA;AAAA,YACA,WAAW;AAAA,YACX,YAAY;AAAA,YACZ,aAAa;AAAA,UACf,CAAC;AACD,cAAI,YAAY,QAAQ;AACtB,kBAAM,yBAAyB,IAAI,WAAW;AAC9C,iCAAqB;AAAA,UACvB;AAAA,QACF,SAAS,eAAe;AACtB,0BAAgB,uBAAuB,oCAAoC;AAAA,YACzE,OAAO,yBAAyB,QAAQ,cAAc,UAAU;AAAA,UAClE,CAAC;AAAA,QACH;AAAA,MACF;AAEA,UAAI,CAAC,sBAAsB,UAAU;AACnC,YAAI;AACF,gBAAM,SAAS,UAAU,gCAAgC;AAAA,YACvD;AAAA,YACA;AAAA,YACA;AAAA,YACA,aAAa;AAAA,YACb,SAAS;AAAA,UACX,CAAC;AAAA,QACH,SAAS,WAAW;AAClB,0BAAgB,uBAAuB,sCAAsC;AAAA,YAC3E,OAAO,qBAAqB,QAAQ,UAAU,UAAU;AAAA,UAC1D,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAEA,UAAM,mBAAmB;AAAA,MACvB;AAAA,MACA,SAAS;AAAA,MACT;AAAA,MACA;AAAA,MACA,QAAQ;AAAA,QACN;AAAA,QACA;AAAA,QACA,gBAAgB,kBAAkB;AAAA,QAClC,SAAS,WAAW;AAAA,QACpB,SAAS,WAAW;AAAA,MACtB;AAAA,IACF,CAAC;AAAA,EACH,SAAS,OAAO;AACd,oBAAgB,uBAAuB,aAAa,OAAO,iBAAiB;AAAA,MAC1E;AAAA,MACA;AAAA,MACA,OAAO,iBAAiB,QAAQ,MAAM,UAAU;AAAA,IAClD,CAAC;AACD,UAAM;AAAA,MACJ,EAAE,IAAI,MAAM,OAAU;AAAA,MACtB;AAAA,QACE,QAAQ;AAAA,QACR,SAAS;AAAA,QACT;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,SAAS,IAAI;AAAA,MACf;AAAA,IACF;AAEA,UAAM;AAAA,EACR;AACF;AAMA,eAAO,OACL,KACA,KACe;AACf,SAAO,qBAAqB,KAAK,KAAK,GAAG;AAC3C;",
|
|
6
|
+
"names": ["searchIndexer", "em"]
|
|
7
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { createQueue } from "@open-mercato/queue";
|
|
2
|
+
const FULLTEXT_INDEXING_QUEUE_NAME = "fulltext-indexing";
|
|
3
|
+
function createFulltextIndexingQueue(strategy = "local", options) {
|
|
4
|
+
if (strategy === "async") {
|
|
5
|
+
return createQueue(FULLTEXT_INDEXING_QUEUE_NAME, "async", {
|
|
6
|
+
connection: options?.connection
|
|
7
|
+
});
|
|
8
|
+
}
|
|
9
|
+
return createQueue(FULLTEXT_INDEXING_QUEUE_NAME, "local");
|
|
10
|
+
}
|
|
11
|
+
export {
|
|
12
|
+
FULLTEXT_INDEXING_QUEUE_NAME,
|
|
13
|
+
createFulltextIndexingQueue
|
|
14
|
+
};
|
|
15
|
+
//# sourceMappingURL=fulltext-indexing.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../src/queue/fulltext-indexing.ts"],
|
|
4
|
+
"sourcesContent": ["import { createQueue, type Queue } from '@open-mercato/queue'\n\n/**\n * Job types for fulltext indexing queue.\n */\nexport type FulltextIndexJobType = 'index' | 'batch-index' | 'delete' | 'purge'\n\n/**\n * Minimal record reference for batch indexing.\n * Only contains identifiers - actual data is loaded fresh via searchIndexer.indexRecordById().\n * This keeps queue payloads small and ensures fresh data is indexed.\n */\nexport type FulltextBatchRecord = {\n entityId: string\n recordId: string\n}\n\n/**\n * Payload for single record indexing jobs.\n * Worker loads fresh data via searchIndexer.indexRecordById().\n */\nexport type FulltextIndexPayload = {\n jobType: 'index'\n tenantId: string\n organizationId?: string | null\n entityType: string\n recordId: string\n}\n\n/**\n * Payload for batch indexing jobs.\n * Worker loads fresh data via searchIndexer.indexRecordById() for each record.\n */\nexport type FulltextBatchIndexPayload = {\n jobType: 'batch-index'\n tenantId: string\n organizationId?: string | null\n records: FulltextBatchRecord[]\n}\n\n/**\n * Payload for delete jobs.\n */\nexport type FulltextDeletePayload = {\n jobType: 'delete'\n tenantId: string\n entityId: string\n recordId: string\n}\n\n/**\n * Payload for purge jobs (delete all records of an entity type).\n */\nexport type FulltextPurgePayload = {\n jobType: 'purge'\n tenantId: string\n entityId: string\n}\n\n/**\n * Union type for all fulltext indexing job payloads.\n */\nexport type FulltextIndexJobPayload =\n | FulltextIndexPayload\n | FulltextBatchIndexPayload\n | FulltextDeletePayload\n | FulltextPurgePayload\n\nexport const FULLTEXT_INDEXING_QUEUE_NAME = 'fulltext-indexing'\n\n/**\n * Create a fulltext indexing queue.\n *\n * @param strategy - Queue strategy ('local' for development, 'async' for production with Redis)\n * @param options - Optional connection configuration for async strategy\n */\nexport function createFulltextIndexingQueue(\n strategy: 'local' | 'async' = 'local',\n options?: { connection?: { url?: string; host?: string; port?: number } },\n): Queue<FulltextIndexJobPayload> {\n if (strategy === 'async') {\n return createQueue<FulltextIndexJobPayload>(FULLTEXT_INDEXING_QUEUE_NAME, 'async', {\n connection: options?.connection,\n })\n }\n return createQueue<FulltextIndexJobPayload>(FULLTEXT_INDEXING_QUEUE_NAME, 'local')\n}\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,mBAA+B;AAoEjC,MAAM,+BAA+B;AAQrC,SAAS,4BACd,WAA8B,SAC9B,SACgC;AAChC,MAAI,aAAa,SAAS;AACxB,WAAO,YAAqC,8BAA8B,SAAS;AAAA,MACjF,YAAY,SAAS;AAAA,IACvB,CAAC;AAAA,EACH;AACA,SAAO,YAAqC,8BAA8B,OAAO;AACnF;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|