@open-mercato/search 0.4.2-canary-c02407ff85
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +678 -0
- package/build.mjs +92 -0
- package/dist/di.js +157 -0
- package/dist/di.js.map +7 -0
- package/dist/fulltext/drivers/index.js +21 -0
- package/dist/fulltext/drivers/index.js.map +7 -0
- package/dist/fulltext/drivers/meilisearch/index.js +320 -0
- package/dist/fulltext/drivers/meilisearch/index.js.map +7 -0
- package/dist/fulltext/index.js +7 -0
- package/dist/fulltext/index.js.map +7 -0
- package/dist/fulltext/types.js +1 -0
- package/dist/fulltext/types.js.map +7 -0
- package/dist/index.js +12 -0
- package/dist/index.js.map +7 -0
- package/dist/indexer/index.js +8 -0
- package/dist/indexer/index.js.map +7 -0
- package/dist/indexer/search-indexer.js +848 -0
- package/dist/indexer/search-indexer.js.map +7 -0
- package/dist/indexer/subscribers/delete.js +41 -0
- package/dist/indexer/subscribers/delete.js.map +7 -0
- package/dist/lib/debug.js +34 -0
- package/dist/lib/debug.js.map +7 -0
- package/dist/lib/fallback-presenter.js +107 -0
- package/dist/lib/fallback-presenter.js.map +7 -0
- package/dist/lib/field-policy.js +75 -0
- package/dist/lib/field-policy.js.map +7 -0
- package/dist/lib/index.js +19 -0
- package/dist/lib/index.js.map +7 -0
- package/dist/lib/merger.js +93 -0
- package/dist/lib/merger.js.map +7 -0
- package/dist/lib/presenter-enricher.js +192 -0
- package/dist/lib/presenter-enricher.js.map +7 -0
- package/dist/modules/search/acl.js +14 -0
- package/dist/modules/search/acl.js.map +7 -0
- package/dist/modules/search/ai-tools.js +284 -0
- package/dist/modules/search/ai-tools.js.map +7 -0
- package/dist/modules/search/api/embeddings/reindex/cancel/route.js +65 -0
- package/dist/modules/search/api/embeddings/reindex/cancel/route.js.map +7 -0
- package/dist/modules/search/api/embeddings/reindex/route.js +165 -0
- package/dist/modules/search/api/embeddings/reindex/route.js.map +7 -0
- package/dist/modules/search/api/embeddings/route.js +246 -0
- package/dist/modules/search/api/embeddings/route.js.map +7 -0
- package/dist/modules/search/api/index/route.js +245 -0
- package/dist/modules/search/api/index/route.js.map +7 -0
- package/dist/modules/search/api/reindex/cancel/route.js +65 -0
- package/dist/modules/search/api/reindex/cancel/route.js.map +7 -0
- package/dist/modules/search/api/reindex/route.js +332 -0
- package/dist/modules/search/api/reindex/route.js.map +7 -0
- package/dist/modules/search/api/search/global/route.js +100 -0
- package/dist/modules/search/api/search/global/route.js.map +7 -0
- package/dist/modules/search/api/search/route.js +101 -0
- package/dist/modules/search/api/search/route.js.map +7 -0
- package/dist/modules/search/api/settings/fulltext/route.js +55 -0
- package/dist/modules/search/api/settings/fulltext/route.js.map +7 -0
- package/dist/modules/search/api/settings/global-search/route.js +80 -0
- package/dist/modules/search/api/settings/global-search/route.js.map +7 -0
- package/dist/modules/search/api/settings/route.js +118 -0
- package/dist/modules/search/api/settings/route.js.map +7 -0
- package/dist/modules/search/api/settings/vector-store/route.js +77 -0
- package/dist/modules/search/api/settings/vector-store/route.js.map +7 -0
- package/dist/modules/search/backend/config/search/page.js +10 -0
- package/dist/modules/search/backend/config/search/page.js.map +7 -0
- package/dist/modules/search/backend/config/search/page.meta.js +24 -0
- package/dist/modules/search/backend/config/search/page.meta.js.map +7 -0
- package/dist/modules/search/cli.js +698 -0
- package/dist/modules/search/cli.js.map +7 -0
- package/dist/modules/search/di.js +32 -0
- package/dist/modules/search/di.js.map +7 -0
- package/dist/modules/search/frontend/components/GlobalSearchDialog.js +357 -0
- package/dist/modules/search/frontend/components/GlobalSearchDialog.js.map +7 -0
- package/dist/modules/search/frontend/components/HybridSearchTable.js +343 -0
- package/dist/modules/search/frontend/components/HybridSearchTable.js.map +7 -0
- package/dist/modules/search/frontend/components/SearchSettingsPageClient.js +303 -0
- package/dist/modules/search/frontend/components/SearchSettingsPageClient.js.map +7 -0
- package/dist/modules/search/frontend/components/sections/FulltextSearchSection.js +360 -0
- package/dist/modules/search/frontend/components/sections/FulltextSearchSection.js.map +7 -0
- package/dist/modules/search/frontend/components/sections/GlobalSearchSection.js +101 -0
- package/dist/modules/search/frontend/components/sections/GlobalSearchSection.js.map +7 -0
- package/dist/modules/search/frontend/components/sections/VectorSearchSection.js +608 -0
- package/dist/modules/search/frontend/components/sections/VectorSearchSection.js.map +7 -0
- package/dist/modules/search/frontend/index.js +9 -0
- package/dist/modules/search/frontend/index.js.map +7 -0
- package/dist/modules/search/frontend/utils.js +41 -0
- package/dist/modules/search/frontend/utils.js.map +7 -0
- package/dist/modules/search/i18n/de.json +61 -0
- package/dist/modules/search/i18n/en.json +72 -0
- package/dist/modules/search/i18n/es.json +61 -0
- package/dist/modules/search/i18n/pl.json +61 -0
- package/dist/modules/search/index.js +11 -0
- package/dist/modules/search/index.js.map +7 -0
- package/dist/modules/search/lib/auto-indexing.js +29 -0
- package/dist/modules/search/lib/auto-indexing.js.map +7 -0
- package/dist/modules/search/lib/embedding-config.js +131 -0
- package/dist/modules/search/lib/embedding-config.js.map +7 -0
- package/dist/modules/search/lib/global-search-config.js +45 -0
- package/dist/modules/search/lib/global-search-config.js.map +7 -0
- package/dist/modules/search/lib/reindex-lock.js +99 -0
- package/dist/modules/search/lib/reindex-lock.js.map +7 -0
- package/dist/modules/search/subscribers/fulltext_upsert.js +64 -0
- package/dist/modules/search/subscribers/fulltext_upsert.js.map +7 -0
- package/dist/modules/search/subscribers/vector_delete.js +58 -0
- package/dist/modules/search/subscribers/vector_delete.js.map +7 -0
- package/dist/modules/search/subscribers/vector_purge.js +142 -0
- package/dist/modules/search/subscribers/vector_purge.js.map +7 -0
- package/dist/modules/search/subscribers/vector_upsert.js +58 -0
- package/dist/modules/search/subscribers/vector_upsert.js.map +7 -0
- package/dist/modules/search/workers/fulltext-index.worker.js +240 -0
- package/dist/modules/search/workers/fulltext-index.worker.js.map +7 -0
- package/dist/modules/search/workers/vector-index.worker.js +234 -0
- package/dist/modules/search/workers/vector-index.worker.js.map +7 -0
- package/dist/queue/fulltext-indexing.js +15 -0
- package/dist/queue/fulltext-indexing.js.map +7 -0
- package/dist/queue/index.js +3 -0
- package/dist/queue/index.js.map +7 -0
- package/dist/queue/vector-indexing.js +15 -0
- package/dist/queue/vector-indexing.js.map +7 -0
- package/dist/service.js +286 -0
- package/dist/service.js.map +7 -0
- package/dist/strategies/fulltext.strategy.js +116 -0
- package/dist/strategies/fulltext.strategy.js.map +7 -0
- package/dist/strategies/index.js +12 -0
- package/dist/strategies/index.js.map +7 -0
- package/dist/strategies/token.strategy.js +80 -0
- package/dist/strategies/token.strategy.js.map +7 -0
- package/dist/strategies/vector.strategy.js +137 -0
- package/dist/strategies/vector.strategy.js.map +7 -0
- package/dist/types.js +1 -0
- package/dist/types.js.map +7 -0
- package/dist/vector/drivers/chromadb/index.js +44 -0
- package/dist/vector/drivers/chromadb/index.js.map +7 -0
- package/dist/vector/drivers/index.js +9 -0
- package/dist/vector/drivers/index.js.map +7 -0
- package/dist/vector/drivers/pgvector/index.js +509 -0
- package/dist/vector/drivers/pgvector/index.js.map +7 -0
- package/dist/vector/drivers/qdrant/index.js +44 -0
- package/dist/vector/drivers/qdrant/index.js.map +7 -0
- package/dist/vector/index.js +4 -0
- package/dist/vector/index.js.map +7 -0
- package/dist/vector/lib/vector-logs.js +33 -0
- package/dist/vector/lib/vector-logs.js.map +7 -0
- package/dist/vector/services/checksum.js +20 -0
- package/dist/vector/services/checksum.js.map +7 -0
- package/dist/vector/services/embedding.js +222 -0
- package/dist/vector/services/embedding.js.map +7 -0
- package/dist/vector/services/index.js +4 -0
- package/dist/vector/services/index.js.map +7 -0
- package/dist/vector/services/vector-index.service.js +960 -0
- package/dist/vector/services/vector-index.service.js.map +7 -0
- package/dist/vector/types/pg.d.js +1 -0
- package/dist/vector/types/pg.d.js.map +7 -0
- package/dist/vector/types.js +75 -0
- package/dist/vector/types.js.map +7 -0
- package/jest.config.cjs +19 -0
- package/package.json +142 -0
- package/src/__tests__/queue.test.ts +148 -0
- package/src/__tests__/service.test.ts +345 -0
- package/src/__tests__/workers.test.ts +319 -0
- package/src/di.ts +291 -0
- package/src/fulltext/drivers/index.ts +41 -0
- package/src/fulltext/drivers/meilisearch/index.ts +410 -0
- package/src/fulltext/index.ts +13 -0
- package/src/fulltext/types.ts +115 -0
- package/src/index.ts +36 -0
- package/src/indexer/index.ts +13 -0
- package/src/indexer/search-indexer.ts +1141 -0
- package/src/indexer/subscribers/delete.ts +49 -0
- package/src/lib/debug.ts +46 -0
- package/src/lib/fallback-presenter.ts +106 -0
- package/src/lib/field-policy.ts +169 -0
- package/src/lib/index.ts +13 -0
- package/src/lib/merger.ts +159 -0
- package/src/lib/presenter-enricher.ts +323 -0
- package/src/modules/search/README.md +694 -0
- package/src/modules/search/acl.ts +10 -0
- package/src/modules/search/ai-tools.ts +467 -0
- package/src/modules/search/api/embeddings/reindex/cancel/route.ts +77 -0
- package/src/modules/search/api/embeddings/reindex/route.ts +197 -0
- package/src/modules/search/api/embeddings/route.ts +304 -0
- package/src/modules/search/api/index/route.ts +297 -0
- package/src/modules/search/api/reindex/cancel/route.ts +77 -0
- package/src/modules/search/api/reindex/route.ts +419 -0
- package/src/modules/search/api/search/global/route.ts +120 -0
- package/src/modules/search/api/search/route.ts +121 -0
- package/src/modules/search/api/settings/fulltext/route.ts +82 -0
- package/src/modules/search/api/settings/global-search/route.ts +91 -0
- package/src/modules/search/api/settings/route.ts +187 -0
- package/src/modules/search/api/settings/vector-store/route.ts +105 -0
- package/src/modules/search/backend/config/search/page.meta.ts +22 -0
- package/src/modules/search/backend/config/search/page.tsx +12 -0
- package/src/modules/search/cli.ts +818 -0
- package/src/modules/search/di.ts +50 -0
- package/src/modules/search/frontend/components/GlobalSearchDialog.tsx +436 -0
- package/src/modules/search/frontend/components/HybridSearchTable.tsx +418 -0
- package/src/modules/search/frontend/components/SearchSettingsPageClient.tsx +476 -0
- package/src/modules/search/frontend/components/sections/FulltextSearchSection.tsx +624 -0
- package/src/modules/search/frontend/components/sections/GlobalSearchSection.tsx +124 -0
- package/src/modules/search/frontend/components/sections/VectorSearchSection.tsx +943 -0
- package/src/modules/search/frontend/index.ts +3 -0
- package/src/modules/search/frontend/utils.ts +82 -0
- package/src/modules/search/i18n/de.json +61 -0
- package/src/modules/search/i18n/en.json +72 -0
- package/src/modules/search/i18n/es.json +61 -0
- package/src/modules/search/i18n/pl.json +61 -0
- package/src/modules/search/index.ts +9 -0
- package/src/modules/search/lib/auto-indexing.ts +35 -0
- package/src/modules/search/lib/embedding-config.ts +161 -0
- package/src/modules/search/lib/global-search-config.ts +69 -0
- package/src/modules/search/lib/reindex-lock.ts +201 -0
- package/src/modules/search/subscribers/fulltext_upsert.ts +83 -0
- package/src/modules/search/subscribers/vector_delete.ts +75 -0
- package/src/modules/search/subscribers/vector_purge.ts +161 -0
- package/src/modules/search/subscribers/vector_upsert.ts +75 -0
- package/src/modules/search/workers/fulltext-index.worker.ts +318 -0
- package/src/modules/search/workers/vector-index.worker.ts +292 -0
- package/src/queue/fulltext-indexing.ts +87 -0
- package/src/queue/index.ts +2 -0
- package/src/queue/vector-indexing.ts +66 -0
- package/src/service.ts +397 -0
- package/src/strategies/fulltext.strategy.ts +155 -0
- package/src/strategies/index.ts +17 -0
- package/src/strategies/token.strategy.ts +153 -0
- package/src/strategies/vector.strategy.ts +234 -0
- package/src/types.ts +38 -0
- package/src/vector/drivers/chromadb/index.ts +49 -0
- package/src/vector/drivers/index.ts +4 -0
- package/src/vector/drivers/pgvector/index.ts +627 -0
- package/src/vector/drivers/qdrant/index.ts +49 -0
- package/src/vector/index.ts +3 -0
- package/src/vector/lib/vector-logs.ts +46 -0
- package/src/vector/services/checksum.ts +18 -0
- package/src/vector/services/embedding.ts +275 -0
- package/src/vector/services/index.ts +3 -0
- package/src/vector/services/vector-index.service.ts +1234 -0
- package/src/vector/types/pg.d.ts +1 -0
- package/src/vector/types.ts +220 -0
- package/tsconfig.json +9 -0
- package/watch.mjs +6 -0
|
@@ -0,0 +1,848 @@
|
|
|
1
|
+
import { searchDebug, searchDebugWarn, searchError } from "../lib/debug.js";
|
|
2
|
+
const MAX_PAGES = 1e4;
|
|
3
|
+
class SearchIndexer {
|
|
4
|
+
constructor(searchService, moduleConfigs, options) {
|
|
5
|
+
this.searchService = searchService;
|
|
6
|
+
this.moduleConfigs = moduleConfigs;
|
|
7
|
+
this.entityConfigMap = /* @__PURE__ */ new Map();
|
|
8
|
+
this.queryEngine = options?.queryEngine;
|
|
9
|
+
this.fulltextQueue = options?.fulltextQueue;
|
|
10
|
+
this.vectorQueue = options?.vectorQueue;
|
|
11
|
+
for (const moduleConfig of moduleConfigs) {
|
|
12
|
+
for (const entityConfig of moduleConfig.entities) {
|
|
13
|
+
if (entityConfig.enabled !== false) {
|
|
14
|
+
this.entityConfigMap.set(entityConfig.entityId, entityConfig);
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Get the entity config for a given entity ID.
|
|
21
|
+
*/
|
|
22
|
+
getEntityConfig(entityId) {
|
|
23
|
+
return this.entityConfigMap.get(entityId);
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Get all configured entity configs.
|
|
27
|
+
*/
|
|
28
|
+
getAllEntityConfigs() {
|
|
29
|
+
return Array.from(this.entityConfigMap.values());
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Check if an entity is configured for search indexing.
|
|
33
|
+
*/
|
|
34
|
+
isEntityEnabled(entityId) {
|
|
35
|
+
const config = this.entityConfigMap.get(entityId);
|
|
36
|
+
return config?.enabled !== false;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Index a record in the search service.
|
|
40
|
+
*/
|
|
41
|
+
async indexRecord(params) {
|
|
42
|
+
const config = this.entityConfigMap.get(params.entityId);
|
|
43
|
+
if (!config || config.enabled === false) {
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
const buildContext = {
|
|
47
|
+
record: params.record,
|
|
48
|
+
customFields: params.customFields ?? {},
|
|
49
|
+
organizationId: params.organizationId,
|
|
50
|
+
tenantId: params.tenantId,
|
|
51
|
+
queryEngine: this.queryEngine
|
|
52
|
+
};
|
|
53
|
+
let text;
|
|
54
|
+
let presenter;
|
|
55
|
+
let url;
|
|
56
|
+
let links;
|
|
57
|
+
let checksumSource;
|
|
58
|
+
if (config.buildSource) {
|
|
59
|
+
try {
|
|
60
|
+
const source = await config.buildSource(buildContext);
|
|
61
|
+
if (source) {
|
|
62
|
+
text = source.text;
|
|
63
|
+
if (source.presenter) presenter = source.presenter;
|
|
64
|
+
if (source.links) links = source.links;
|
|
65
|
+
if (source.checksumSource !== void 0) checksumSource = source.checksumSource;
|
|
66
|
+
}
|
|
67
|
+
} catch (error) {
|
|
68
|
+
searchDebugWarn("SearchIndexer", "buildSource failed", {
|
|
69
|
+
entityId: params.entityId,
|
|
70
|
+
recordId: params.recordId,
|
|
71
|
+
error: error instanceof Error ? error.message : error
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
if (!presenter && config.formatResult) {
|
|
76
|
+
try {
|
|
77
|
+
const result = await config.formatResult(buildContext);
|
|
78
|
+
if (result) presenter = result;
|
|
79
|
+
} catch (error) {
|
|
80
|
+
searchDebugWarn("SearchIndexer", "formatResult failed", {
|
|
81
|
+
entityId: params.entityId,
|
|
82
|
+
recordId: params.recordId,
|
|
83
|
+
error: error instanceof Error ? error.message : error
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
if (!url && config.resolveUrl) {
|
|
88
|
+
try {
|
|
89
|
+
const result = await config.resolveUrl(buildContext);
|
|
90
|
+
if (result) url = result;
|
|
91
|
+
} catch (error) {
|
|
92
|
+
searchDebugWarn("SearchIndexer", "resolveUrl failed", {
|
|
93
|
+
entityId: params.entityId,
|
|
94
|
+
recordId: params.recordId,
|
|
95
|
+
error: error instanceof Error ? error.message : error
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
if (!links && config.resolveLinks) {
|
|
100
|
+
try {
|
|
101
|
+
const result = await config.resolveLinks(buildContext);
|
|
102
|
+
if (result) links = result;
|
|
103
|
+
} catch (error) {
|
|
104
|
+
searchDebugWarn("SearchIndexer", "resolveLinks failed", {
|
|
105
|
+
entityId: params.entityId,
|
|
106
|
+
recordId: params.recordId,
|
|
107
|
+
error: error instanceof Error ? error.message : error
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
const indexableRecord = {
|
|
112
|
+
entityId: params.entityId,
|
|
113
|
+
recordId: params.recordId,
|
|
114
|
+
tenantId: params.tenantId,
|
|
115
|
+
organizationId: params.organizationId,
|
|
116
|
+
fields: params.record,
|
|
117
|
+
presenter,
|
|
118
|
+
url,
|
|
119
|
+
links,
|
|
120
|
+
text,
|
|
121
|
+
checksumSource
|
|
122
|
+
};
|
|
123
|
+
await this.searchService.index(indexableRecord);
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Index a record by ID (loads the record from database first).
|
|
127
|
+
* Used by workers that only have record identifiers.
|
|
128
|
+
*/
|
|
129
|
+
async indexRecordById(params) {
|
|
130
|
+
if (!this.queryEngine) {
|
|
131
|
+
return { action: "skipped", reason: "queryEngine not available" };
|
|
132
|
+
}
|
|
133
|
+
const config = this.entityConfigMap.get(params.entityId);
|
|
134
|
+
if (!config || config.enabled === false) {
|
|
135
|
+
return { action: "skipped", reason: "entity not configured" };
|
|
136
|
+
}
|
|
137
|
+
try {
|
|
138
|
+
const result = await this.queryEngine.query(params.entityId, {
|
|
139
|
+
tenantId: params.tenantId,
|
|
140
|
+
organizationId: params.organizationId ?? void 0,
|
|
141
|
+
filters: { id: params.recordId },
|
|
142
|
+
includeCustomFields: true,
|
|
143
|
+
page: { page: 1, pageSize: 1 }
|
|
144
|
+
});
|
|
145
|
+
const record = result.items[0];
|
|
146
|
+
if (!record) {
|
|
147
|
+
return { action: "skipped", reason: "record not found" };
|
|
148
|
+
}
|
|
149
|
+
const customFields = {};
|
|
150
|
+
for (const [key, value] of Object.entries(record)) {
|
|
151
|
+
if (key.startsWith("cf:") || key.startsWith("cf_")) {
|
|
152
|
+
customFields[key.slice(3)] = value;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
await this.indexRecord({
|
|
156
|
+
entityId: params.entityId,
|
|
157
|
+
recordId: params.recordId,
|
|
158
|
+
tenantId: params.tenantId,
|
|
159
|
+
organizationId: params.organizationId,
|
|
160
|
+
record,
|
|
161
|
+
customFields
|
|
162
|
+
});
|
|
163
|
+
return { action: "indexed" };
|
|
164
|
+
} catch (error) {
|
|
165
|
+
searchError("SearchIndexer", "Failed to load record for indexing", {
|
|
166
|
+
entityId: params.entityId,
|
|
167
|
+
recordId: params.recordId,
|
|
168
|
+
error: error instanceof Error ? error.message : error
|
|
169
|
+
});
|
|
170
|
+
throw error;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* Delete a record from the search index.
|
|
175
|
+
*/
|
|
176
|
+
async deleteRecord(params) {
|
|
177
|
+
await this.searchService.delete(params.entityId, params.recordId, params.tenantId);
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* Purge all records of an entity type from the search index.
|
|
181
|
+
*/
|
|
182
|
+
async purgeEntity(params) {
|
|
183
|
+
await this.searchService.purge(params.entityId, params.tenantId);
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* Reindex an entity via all configured strategies (including vector).
|
|
187
|
+
* This is the general reindex method that works with all search strategies.
|
|
188
|
+
*/
|
|
189
|
+
async reindexEntity(params) {
|
|
190
|
+
if (!this.queryEngine) {
|
|
191
|
+
return {
|
|
192
|
+
success: false,
|
|
193
|
+
entitiesProcessed: 0,
|
|
194
|
+
recordsIndexed: 0,
|
|
195
|
+
errors: [{ entityId: params.entityId, error: "Query engine not available" }]
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
const config = this.entityConfigMap.get(params.entityId);
|
|
199
|
+
if (!config || config.enabled === false) {
|
|
200
|
+
return {
|
|
201
|
+
success: false,
|
|
202
|
+
entitiesProcessed: 0,
|
|
203
|
+
recordsIndexed: 0,
|
|
204
|
+
errors: [{ entityId: params.entityId, error: "Entity not configured for search" }]
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
const result = {
|
|
208
|
+
success: true,
|
|
209
|
+
entitiesProcessed: 1,
|
|
210
|
+
recordsIndexed: 0,
|
|
211
|
+
errors: []
|
|
212
|
+
};
|
|
213
|
+
if (params.purgeFirst) {
|
|
214
|
+
try {
|
|
215
|
+
await this.searchService.purge(params.entityId, params.tenantId);
|
|
216
|
+
} catch (error) {
|
|
217
|
+
searchDebugWarn("SearchIndexer", "Failed to purge entity before reindex", {
|
|
218
|
+
entityId: params.entityId,
|
|
219
|
+
error: error instanceof Error ? error.message : error
|
|
220
|
+
});
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
let page = 1;
|
|
224
|
+
const pageSize = 200;
|
|
225
|
+
let hasMore = true;
|
|
226
|
+
while (hasMore && page <= MAX_PAGES) {
|
|
227
|
+
try {
|
|
228
|
+
const queryResult = await this.queryEngine.query(params.entityId, {
|
|
229
|
+
tenantId: params.tenantId,
|
|
230
|
+
organizationId: params.organizationId ?? void 0,
|
|
231
|
+
includeCustomFields: true,
|
|
232
|
+
page: { page, pageSize }
|
|
233
|
+
});
|
|
234
|
+
const items = queryResult.items;
|
|
235
|
+
if (items.length === 0) {
|
|
236
|
+
hasMore = false;
|
|
237
|
+
break;
|
|
238
|
+
}
|
|
239
|
+
const { records } = await this.buildIndexableRecords(
|
|
240
|
+
params.entityId,
|
|
241
|
+
params.tenantId,
|
|
242
|
+
params.organizationId ?? null,
|
|
243
|
+
items,
|
|
244
|
+
config
|
|
245
|
+
);
|
|
246
|
+
for (const record of records) {
|
|
247
|
+
try {
|
|
248
|
+
await this.searchService.index(record);
|
|
249
|
+
result.recordsIndexed++;
|
|
250
|
+
} catch (error) {
|
|
251
|
+
searchDebugWarn("SearchIndexer", "Failed to index record", {
|
|
252
|
+
entityId: params.entityId,
|
|
253
|
+
recordId: record.recordId,
|
|
254
|
+
error: error instanceof Error ? error.message : error
|
|
255
|
+
});
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
page++;
|
|
259
|
+
hasMore = items.length === pageSize;
|
|
260
|
+
} catch (error) {
|
|
261
|
+
result.success = false;
|
|
262
|
+
result.errors.push({
|
|
263
|
+
entityId: params.entityId,
|
|
264
|
+
error: error instanceof Error ? error.message : String(error)
|
|
265
|
+
});
|
|
266
|
+
break;
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
return result;
|
|
270
|
+
}
|
|
271
|
+
/**
|
|
272
|
+
* Reindex all enabled entities via all configured strategies.
|
|
273
|
+
*/
|
|
274
|
+
async reindexAll(params) {
|
|
275
|
+
const result = {
|
|
276
|
+
success: true,
|
|
277
|
+
entitiesProcessed: 0,
|
|
278
|
+
recordsIndexed: 0,
|
|
279
|
+
errors: []
|
|
280
|
+
};
|
|
281
|
+
const enabledEntities = this.listEnabledEntities();
|
|
282
|
+
for (const entityId of enabledEntities) {
|
|
283
|
+
const entityResult = await this.reindexEntity({
|
|
284
|
+
entityId,
|
|
285
|
+
tenantId: params.tenantId,
|
|
286
|
+
organizationId: params.organizationId,
|
|
287
|
+
purgeFirst: params.purgeFirst
|
|
288
|
+
});
|
|
289
|
+
result.entitiesProcessed++;
|
|
290
|
+
result.recordsIndexed += entityResult.recordsIndexed;
|
|
291
|
+
result.errors.push(...entityResult.errors);
|
|
292
|
+
if (!entityResult.success) {
|
|
293
|
+
result.success = false;
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
return result;
|
|
297
|
+
}
|
|
298
|
+
/**
|
|
299
|
+
* Bulk index multiple records.
|
|
300
|
+
*/
|
|
301
|
+
async bulkIndexRecords(params) {
|
|
302
|
+
const indexableRecords = [];
|
|
303
|
+
for (const param of params) {
|
|
304
|
+
const config = this.entityConfigMap.get(param.entityId);
|
|
305
|
+
if (!config || config.enabled === false) {
|
|
306
|
+
continue;
|
|
307
|
+
}
|
|
308
|
+
const buildContext = {
|
|
309
|
+
record: param.record,
|
|
310
|
+
customFields: param.customFields ?? {},
|
|
311
|
+
organizationId: param.organizationId,
|
|
312
|
+
tenantId: param.tenantId
|
|
313
|
+
};
|
|
314
|
+
let presenter;
|
|
315
|
+
if (config.formatResult) {
|
|
316
|
+
try {
|
|
317
|
+
const result = await config.formatResult(buildContext);
|
|
318
|
+
if (result) presenter = result;
|
|
319
|
+
} catch {
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
let url;
|
|
323
|
+
if (config.resolveUrl) {
|
|
324
|
+
try {
|
|
325
|
+
const result = await config.resolveUrl(buildContext);
|
|
326
|
+
if (result) url = result;
|
|
327
|
+
} catch {
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
let links;
|
|
331
|
+
if (config.resolveLinks) {
|
|
332
|
+
try {
|
|
333
|
+
const result = await config.resolveLinks(buildContext);
|
|
334
|
+
if (result) links = result;
|
|
335
|
+
} catch {
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
indexableRecords.push({
|
|
339
|
+
entityId: param.entityId,
|
|
340
|
+
recordId: param.recordId,
|
|
341
|
+
tenantId: param.tenantId,
|
|
342
|
+
organizationId: param.organizationId,
|
|
343
|
+
fields: param.record,
|
|
344
|
+
presenter,
|
|
345
|
+
url,
|
|
346
|
+
links
|
|
347
|
+
});
|
|
348
|
+
}
|
|
349
|
+
if (indexableRecords.length > 0) {
|
|
350
|
+
await this.searchService.bulkIndex(indexableRecords);
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
/**
|
|
354
|
+
* List all enabled entity IDs from the module configurations.
|
|
355
|
+
*/
|
|
356
|
+
listEnabledEntities() {
|
|
357
|
+
return Array.from(this.entityConfigMap.keys());
|
|
358
|
+
}
|
|
359
|
+
/**
|
|
360
|
+
* Get the fulltext strategy from the search service.
|
|
361
|
+
*/
|
|
362
|
+
getFulltextStrategy() {
|
|
363
|
+
const strategy = this.searchService.getStrategy("fulltext");
|
|
364
|
+
if (!strategy) return void 0;
|
|
365
|
+
return strategy;
|
|
366
|
+
}
|
|
367
|
+
/**
|
|
368
|
+
* Reindex a single entity type to fulltext search.
|
|
369
|
+
* This fetches all records from the database and re-indexes them to fulltext only.
|
|
370
|
+
*
|
|
371
|
+
* When `useQueue` is true, batches are enqueued for background processing by workers.
|
|
372
|
+
* When `useQueue` is false (default), batches are indexed directly (blocking).
|
|
373
|
+
*/
|
|
374
|
+
async reindexEntityToFulltext(params) {
|
|
375
|
+
const result = {
|
|
376
|
+
success: true,
|
|
377
|
+
entitiesProcessed: 0,
|
|
378
|
+
recordsIndexed: 0,
|
|
379
|
+
recordsDropped: 0,
|
|
380
|
+
jobsEnqueued: 0,
|
|
381
|
+
errors: []
|
|
382
|
+
};
|
|
383
|
+
const fulltext = this.getFulltextStrategy();
|
|
384
|
+
if (!fulltext) {
|
|
385
|
+
result.success = false;
|
|
386
|
+
result.errors.push({ entityId: params.entityId, error: "Fulltext strategy not available" });
|
|
387
|
+
return result;
|
|
388
|
+
}
|
|
389
|
+
if (params.useQueue && !this.fulltextQueue) {
|
|
390
|
+
result.success = false;
|
|
391
|
+
result.errors.push({ entityId: params.entityId, error: "Fulltext queue not configured for queue-based reindexing" });
|
|
392
|
+
return result;
|
|
393
|
+
}
|
|
394
|
+
if (!this.queryEngine) {
|
|
395
|
+
result.success = false;
|
|
396
|
+
result.errors.push({ entityId: params.entityId, error: "QueryEngine not available for reindexing" });
|
|
397
|
+
return result;
|
|
398
|
+
}
|
|
399
|
+
const config = this.entityConfigMap.get(params.entityId);
|
|
400
|
+
if (!config) {
|
|
401
|
+
result.success = false;
|
|
402
|
+
result.errors.push({ entityId: params.entityId, error: "Entity not configured for search" });
|
|
403
|
+
return result;
|
|
404
|
+
}
|
|
405
|
+
try {
|
|
406
|
+
params.onProgress?.({
|
|
407
|
+
entityId: params.entityId,
|
|
408
|
+
phase: "starting",
|
|
409
|
+
processed: 0
|
|
410
|
+
});
|
|
411
|
+
if (params.recreateIndex !== false) {
|
|
412
|
+
await fulltext.recreateIndex(params.tenantId);
|
|
413
|
+
}
|
|
414
|
+
const pageSize = 200;
|
|
415
|
+
let page = 1;
|
|
416
|
+
let totalProcessed = 0;
|
|
417
|
+
let jobsEnqueued = 0;
|
|
418
|
+
for (; ; ) {
|
|
419
|
+
params.onProgress?.({
|
|
420
|
+
entityId: params.entityId,
|
|
421
|
+
phase: "fetching",
|
|
422
|
+
processed: totalProcessed
|
|
423
|
+
});
|
|
424
|
+
try {
|
|
425
|
+
const queryResult = await this.queryEngine.query(params.entityId, {
|
|
426
|
+
tenantId: params.tenantId,
|
|
427
|
+
organizationId: params.organizationId ?? void 0,
|
|
428
|
+
page: { page, pageSize }
|
|
429
|
+
});
|
|
430
|
+
if (!queryResult.items.length) {
|
|
431
|
+
break;
|
|
432
|
+
}
|
|
433
|
+
params.onProgress?.({
|
|
434
|
+
entityId: params.entityId,
|
|
435
|
+
phase: "indexing",
|
|
436
|
+
processed: totalProcessed,
|
|
437
|
+
total: queryResult.total
|
|
438
|
+
});
|
|
439
|
+
const { records: indexableRecords, dropped } = await this.buildIndexableRecords(
|
|
440
|
+
params.entityId,
|
|
441
|
+
params.tenantId,
|
|
442
|
+
params.organizationId ?? null,
|
|
443
|
+
queryResult.items,
|
|
444
|
+
config
|
|
445
|
+
);
|
|
446
|
+
result.recordsDropped = (result.recordsDropped ?? 0) + dropped;
|
|
447
|
+
if (indexableRecords.length > 0) {
|
|
448
|
+
if (params.useQueue && this.fulltextQueue) {
|
|
449
|
+
await this.fulltextQueue.enqueue({
|
|
450
|
+
jobType: "batch-index",
|
|
451
|
+
tenantId: params.tenantId,
|
|
452
|
+
organizationId: params.organizationId,
|
|
453
|
+
records: indexableRecords.map((r) => ({ entityId: r.entityId, recordId: r.recordId }))
|
|
454
|
+
});
|
|
455
|
+
jobsEnqueued += 1;
|
|
456
|
+
totalProcessed += indexableRecords.length;
|
|
457
|
+
} else {
|
|
458
|
+
try {
|
|
459
|
+
await fulltext.bulkIndex(indexableRecords);
|
|
460
|
+
totalProcessed += indexableRecords.length;
|
|
461
|
+
} catch (indexError) {
|
|
462
|
+
const errorMsg = indexError instanceof Error ? indexError.message : String(indexError);
|
|
463
|
+
result.errors.push({
|
|
464
|
+
entityId: params.entityId,
|
|
465
|
+
error: `Batch ${page} failed: ${errorMsg}`
|
|
466
|
+
});
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
if (queryResult.items.length < pageSize) {
|
|
471
|
+
break;
|
|
472
|
+
}
|
|
473
|
+
page += 1;
|
|
474
|
+
if (page > MAX_PAGES) {
|
|
475
|
+
break;
|
|
476
|
+
}
|
|
477
|
+
} catch (queryError) {
|
|
478
|
+
const errorMsg = queryError instanceof Error ? queryError.message : String(queryError);
|
|
479
|
+
result.errors.push({
|
|
480
|
+
entityId: params.entityId,
|
|
481
|
+
error: `Query failed: ${errorMsg}`
|
|
482
|
+
});
|
|
483
|
+
break;
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
result.entitiesProcessed = 1;
|
|
487
|
+
result.recordsIndexed = totalProcessed;
|
|
488
|
+
result.jobsEnqueued = jobsEnqueued;
|
|
489
|
+
params.onProgress?.({
|
|
490
|
+
entityId: params.entityId,
|
|
491
|
+
phase: "complete",
|
|
492
|
+
processed: totalProcessed,
|
|
493
|
+
total: totalProcessed
|
|
494
|
+
});
|
|
495
|
+
} catch (error) {
|
|
496
|
+
result.success = false;
|
|
497
|
+
result.errors.push({
|
|
498
|
+
entityId: params.entityId,
|
|
499
|
+
error: error instanceof Error ? error.message : String(error)
|
|
500
|
+
});
|
|
501
|
+
}
|
|
502
|
+
return result;
|
|
503
|
+
}
|
|
504
|
+
/**
|
|
505
|
+
* Reindex all enabled entities to fulltext search.
|
|
506
|
+
*
|
|
507
|
+
* When `useQueue` is true, batches are enqueued for background processing by workers.
|
|
508
|
+
* When `useQueue` is false (default), batches are indexed directly (blocking).
|
|
509
|
+
*/
|
|
510
|
+
async reindexAllToFulltext(params) {
|
|
511
|
+
const result = {
|
|
512
|
+
success: true,
|
|
513
|
+
entitiesProcessed: 0,
|
|
514
|
+
recordsIndexed: 0,
|
|
515
|
+
recordsDropped: 0,
|
|
516
|
+
jobsEnqueued: 0,
|
|
517
|
+
errors: []
|
|
518
|
+
};
|
|
519
|
+
const fulltext = this.getFulltextStrategy();
|
|
520
|
+
if (!fulltext) {
|
|
521
|
+
result.success = false;
|
|
522
|
+
result.errors.push({ entityId: "all", error: "Fulltext strategy not available" });
|
|
523
|
+
return result;
|
|
524
|
+
}
|
|
525
|
+
if (params.recreateIndex !== false) {
|
|
526
|
+
await fulltext.recreateIndex(params.tenantId);
|
|
527
|
+
}
|
|
528
|
+
const entities = this.listEnabledEntities();
|
|
529
|
+
for (const entityId of entities) {
|
|
530
|
+
const entityResult = await this.reindexEntityToFulltext({
|
|
531
|
+
entityId,
|
|
532
|
+
tenantId: params.tenantId,
|
|
533
|
+
organizationId: params.organizationId,
|
|
534
|
+
recreateIndex: false,
|
|
535
|
+
// Already recreated above
|
|
536
|
+
onProgress: params.onProgress,
|
|
537
|
+
useQueue: params.useQueue
|
|
538
|
+
});
|
|
539
|
+
result.entitiesProcessed += entityResult.entitiesProcessed;
|
|
540
|
+
result.recordsIndexed += entityResult.recordsIndexed;
|
|
541
|
+
result.recordsDropped = (result.recordsDropped ?? 0) + (entityResult.recordsDropped ?? 0);
|
|
542
|
+
result.jobsEnqueued = (result.jobsEnqueued ?? 0) + (entityResult.jobsEnqueued ?? 0);
|
|
543
|
+
result.errors.push(...entityResult.errors);
|
|
544
|
+
if (!entityResult.success) {
|
|
545
|
+
result.success = false;
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
return result;
|
|
549
|
+
}
|
|
550
|
+
/**
|
|
551
|
+
* Reindex a single entity type to vector search.
|
|
552
|
+
* This fetches all records from the database and enqueues them for vector indexing.
|
|
553
|
+
*
|
|
554
|
+
* When `useQueue` is true (default), record IDs are enqueued for background processing by workers.
|
|
555
|
+
* When `useQueue` is false, records are indexed directly (blocking).
|
|
556
|
+
*/
|
|
557
|
+
async reindexEntityToVector(params) {
|
|
558
|
+
searchDebug("SearchIndexer", "reindexEntityToVector called", {
|
|
559
|
+
entityId: params.entityId,
|
|
560
|
+
tenantId: params.tenantId,
|
|
561
|
+
organizationId: params.organizationId,
|
|
562
|
+
useQueue: params.useQueue,
|
|
563
|
+
purgeFirst: params.purgeFirst
|
|
564
|
+
});
|
|
565
|
+
const result = {
|
|
566
|
+
success: true,
|
|
567
|
+
entitiesProcessed: 0,
|
|
568
|
+
recordsIndexed: 0,
|
|
569
|
+
recordsDropped: 0,
|
|
570
|
+
jobsEnqueued: 0,
|
|
571
|
+
errors: []
|
|
572
|
+
};
|
|
573
|
+
if (params.useQueue !== false && !this.vectorQueue) {
|
|
574
|
+
result.success = false;
|
|
575
|
+
result.errors.push({ entityId: params.entityId, error: "Vector queue not configured for queue-based reindexing" });
|
|
576
|
+
return result;
|
|
577
|
+
}
|
|
578
|
+
if (!this.queryEngine) {
|
|
579
|
+
result.success = false;
|
|
580
|
+
result.errors.push({ entityId: params.entityId, error: "QueryEngine not available for reindexing" });
|
|
581
|
+
return result;
|
|
582
|
+
}
|
|
583
|
+
const config = this.entityConfigMap.get(params.entityId);
|
|
584
|
+
if (!config) {
|
|
585
|
+
result.success = false;
|
|
586
|
+
result.errors.push({ entityId: params.entityId, error: "Entity not configured for search" });
|
|
587
|
+
return result;
|
|
588
|
+
}
|
|
589
|
+
try {
|
|
590
|
+
params.onProgress?.({
|
|
591
|
+
entityId: params.entityId,
|
|
592
|
+
phase: "starting",
|
|
593
|
+
processed: 0
|
|
594
|
+
});
|
|
595
|
+
if (params.purgeFirst) {
|
|
596
|
+
try {
|
|
597
|
+
await this.searchService.purge(params.entityId, params.tenantId);
|
|
598
|
+
} catch (error) {
|
|
599
|
+
searchDebugWarn("SearchIndexer", "Failed to purge entity before vector reindex", {
|
|
600
|
+
entityId: params.entityId,
|
|
601
|
+
error: error instanceof Error ? error.message : error
|
|
602
|
+
});
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
const pageSize = 200;
|
|
606
|
+
let page = 1;
|
|
607
|
+
let totalProcessed = 0;
|
|
608
|
+
let jobsEnqueued = 0;
|
|
609
|
+
for (; ; ) {
|
|
610
|
+
params.onProgress?.({
|
|
611
|
+
entityId: params.entityId,
|
|
612
|
+
phase: "fetching",
|
|
613
|
+
processed: totalProcessed
|
|
614
|
+
});
|
|
615
|
+
const queryResult = await this.queryEngine.query(params.entityId, {
|
|
616
|
+
tenantId: params.tenantId,
|
|
617
|
+
organizationId: params.organizationId ?? void 0,
|
|
618
|
+
page: { page, pageSize }
|
|
619
|
+
});
|
|
620
|
+
if (!queryResult.items.length) break;
|
|
621
|
+
params.onProgress?.({
|
|
622
|
+
entityId: params.entityId,
|
|
623
|
+
phase: "indexing",
|
|
624
|
+
processed: totalProcessed,
|
|
625
|
+
total: queryResult.total
|
|
626
|
+
});
|
|
627
|
+
const batchRecords = [];
|
|
628
|
+
for (const item of queryResult.items) {
|
|
629
|
+
const recordId = String(item.id ?? "");
|
|
630
|
+
if (!recordId) {
|
|
631
|
+
result.recordsDropped = (result.recordsDropped ?? 0) + 1;
|
|
632
|
+
continue;
|
|
633
|
+
}
|
|
634
|
+
batchRecords.push({
|
|
635
|
+
entityId: params.entityId,
|
|
636
|
+
recordId
|
|
637
|
+
});
|
|
638
|
+
}
|
|
639
|
+
if (batchRecords.length > 0) {
|
|
640
|
+
if (params.useQueue !== false && this.vectorQueue) {
|
|
641
|
+
await this.vectorQueue.enqueue({
|
|
642
|
+
jobType: "batch-index",
|
|
643
|
+
tenantId: params.tenantId,
|
|
644
|
+
organizationId: params.organizationId ?? null,
|
|
645
|
+
records: batchRecords
|
|
646
|
+
});
|
|
647
|
+
jobsEnqueued += 1;
|
|
648
|
+
totalProcessed += batchRecords.length;
|
|
649
|
+
searchDebug("SearchIndexer", "Enqueued batch for vector indexing", {
|
|
650
|
+
entityId: params.entityId,
|
|
651
|
+
batchSize: batchRecords.length,
|
|
652
|
+
jobsEnqueued,
|
|
653
|
+
totalProcessed
|
|
654
|
+
});
|
|
655
|
+
} else {
|
|
656
|
+
for (const { entityId, recordId } of batchRecords) {
|
|
657
|
+
try {
|
|
658
|
+
await this.indexRecordById({
|
|
659
|
+
entityId,
|
|
660
|
+
recordId,
|
|
661
|
+
tenantId: params.tenantId,
|
|
662
|
+
organizationId: params.organizationId
|
|
663
|
+
});
|
|
664
|
+
totalProcessed++;
|
|
665
|
+
} catch (error) {
|
|
666
|
+
searchDebugWarn("SearchIndexer", "Failed to index record to vector", {
|
|
667
|
+
entityId,
|
|
668
|
+
recordId,
|
|
669
|
+
error: error instanceof Error ? error.message : error
|
|
670
|
+
});
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
if (queryResult.items.length < pageSize) break;
|
|
676
|
+
page += 1;
|
|
677
|
+
if (page > MAX_PAGES) {
|
|
678
|
+
searchDebugWarn("SearchIndexer", "Reached MAX_PAGES limit, stopping pagination", {
|
|
679
|
+
entityId: params.entityId,
|
|
680
|
+
maxPages: MAX_PAGES,
|
|
681
|
+
totalProcessed
|
|
682
|
+
});
|
|
683
|
+
break;
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
result.entitiesProcessed = 1;
|
|
687
|
+
result.recordsIndexed = totalProcessed;
|
|
688
|
+
result.jobsEnqueued = jobsEnqueued;
|
|
689
|
+
params.onProgress?.({
|
|
690
|
+
entityId: params.entityId,
|
|
691
|
+
phase: "complete",
|
|
692
|
+
processed: totalProcessed,
|
|
693
|
+
total: totalProcessed
|
|
694
|
+
});
|
|
695
|
+
} catch (error) {
|
|
696
|
+
result.success = false;
|
|
697
|
+
result.errors.push({
|
|
698
|
+
entityId: params.entityId,
|
|
699
|
+
error: error instanceof Error ? error.message : String(error)
|
|
700
|
+
});
|
|
701
|
+
}
|
|
702
|
+
return result;
|
|
703
|
+
}
|
|
704
|
+
/**
|
|
705
|
+
* Reindex all enabled entities to vector search.
|
|
706
|
+
*
|
|
707
|
+
* When `useQueue` is true (default), batches are enqueued for background processing by workers.
|
|
708
|
+
* When `useQueue` is false, batches are indexed directly (blocking).
|
|
709
|
+
*/
|
|
710
|
+
async reindexAllToVector(params) {
|
|
711
|
+
const result = {
|
|
712
|
+
success: true,
|
|
713
|
+
entitiesProcessed: 0,
|
|
714
|
+
recordsIndexed: 0,
|
|
715
|
+
recordsDropped: 0,
|
|
716
|
+
jobsEnqueued: 0,
|
|
717
|
+
errors: []
|
|
718
|
+
};
|
|
719
|
+
const entities = this.listEnabledEntities();
|
|
720
|
+
for (const entityId of entities) {
|
|
721
|
+
const entityResult = await this.reindexEntityToVector({
|
|
722
|
+
entityId,
|
|
723
|
+
tenantId: params.tenantId,
|
|
724
|
+
organizationId: params.organizationId,
|
|
725
|
+
onProgress: params.onProgress,
|
|
726
|
+
useQueue: params.useQueue,
|
|
727
|
+
purgeFirst: params.purgeFirst
|
|
728
|
+
});
|
|
729
|
+
result.entitiesProcessed += entityResult.entitiesProcessed;
|
|
730
|
+
result.recordsIndexed += entityResult.recordsIndexed;
|
|
731
|
+
result.recordsDropped = (result.recordsDropped ?? 0) + (entityResult.recordsDropped ?? 0);
|
|
732
|
+
result.jobsEnqueued = (result.jobsEnqueued ?? 0) + (entityResult.jobsEnqueued ?? 0);
|
|
733
|
+
result.errors.push(...entityResult.errors);
|
|
734
|
+
if (!entityResult.success) {
|
|
735
|
+
result.success = false;
|
|
736
|
+
}
|
|
737
|
+
}
|
|
738
|
+
return result;
|
|
739
|
+
}
|
|
740
|
+
/**
|
|
741
|
+
* Build IndexableRecords from raw query results.
|
|
742
|
+
* Returns records and count of dropped items (missing id or other validation failures).
|
|
743
|
+
*/
|
|
744
|
+
async buildIndexableRecords(entityId, tenantId, organizationId, items, config) {
|
|
745
|
+
const records = [];
|
|
746
|
+
let dropped = 0;
|
|
747
|
+
if (items.length > 0) {
|
|
748
|
+
searchDebug("SearchIndexer", "Sample item structure", {
|
|
749
|
+
entityId,
|
|
750
|
+
sampleKeys: Object.keys(items[0]),
|
|
751
|
+
sampleId: items[0].id,
|
|
752
|
+
hasId: "id" in items[0],
|
|
753
|
+
firstName: items[0].first_name,
|
|
754
|
+
lastName: items[0].last_name,
|
|
755
|
+
preferredName: items[0].preferred_name,
|
|
756
|
+
sampleItem: JSON.stringify(items[0]).slice(0, 500)
|
|
757
|
+
});
|
|
758
|
+
}
|
|
759
|
+
for (const item of items) {
|
|
760
|
+
const recordId = String(item.id ?? "");
|
|
761
|
+
if (!recordId) {
|
|
762
|
+
searchDebugWarn("SearchIndexer", "Skipping item without id", { entityId, itemKeys: Object.keys(item) });
|
|
763
|
+
dropped++;
|
|
764
|
+
continue;
|
|
765
|
+
}
|
|
766
|
+
const customFields = {};
|
|
767
|
+
for (const [key, value] of Object.entries(item)) {
|
|
768
|
+
if (key.startsWith("cf:") || key.startsWith("cf_")) {
|
|
769
|
+
const cfKey = key.slice(3);
|
|
770
|
+
customFields[cfKey] = value;
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
const buildContext = {
|
|
774
|
+
record: item,
|
|
775
|
+
customFields,
|
|
776
|
+
organizationId,
|
|
777
|
+
tenantId,
|
|
778
|
+
queryEngine: this.queryEngine
|
|
779
|
+
};
|
|
780
|
+
let text;
|
|
781
|
+
let presenter;
|
|
782
|
+
let url;
|
|
783
|
+
let links;
|
|
784
|
+
let checksumSource;
|
|
785
|
+
if (config.buildSource) {
|
|
786
|
+
try {
|
|
787
|
+
const source = await config.buildSource(buildContext);
|
|
788
|
+
if (source) {
|
|
789
|
+
text = source.text;
|
|
790
|
+
if (source.presenter) presenter = source.presenter;
|
|
791
|
+
if (source.links) links = source.links;
|
|
792
|
+
if (source.checksumSource !== void 0) checksumSource = source.checksumSource;
|
|
793
|
+
}
|
|
794
|
+
} catch (err) {
|
|
795
|
+
searchDebugWarn("SearchIndexer", "buildSource failed", {
|
|
796
|
+
entityId,
|
|
797
|
+
recordId,
|
|
798
|
+
error: err instanceof Error ? err.message : err
|
|
799
|
+
});
|
|
800
|
+
}
|
|
801
|
+
}
|
|
802
|
+
if (!presenter && config.formatResult) {
|
|
803
|
+
try {
|
|
804
|
+
const result = await config.formatResult(buildContext);
|
|
805
|
+
if (result) presenter = result;
|
|
806
|
+
} catch {
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
if (!url && config.resolveUrl) {
|
|
810
|
+
try {
|
|
811
|
+
const result = await config.resolveUrl(buildContext);
|
|
812
|
+
if (result) url = result;
|
|
813
|
+
} catch {
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
if (!links && config.resolveLinks) {
|
|
817
|
+
try {
|
|
818
|
+
const result = await config.resolveLinks(buildContext);
|
|
819
|
+
if (result) links = result;
|
|
820
|
+
} catch {
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
records.push({
|
|
824
|
+
entityId,
|
|
825
|
+
recordId,
|
|
826
|
+
tenantId,
|
|
827
|
+
organizationId,
|
|
828
|
+
fields: item,
|
|
829
|
+
presenter,
|
|
830
|
+
url,
|
|
831
|
+
links,
|
|
832
|
+
text,
|
|
833
|
+
checksumSource
|
|
834
|
+
});
|
|
835
|
+
}
|
|
836
|
+
searchDebug("SearchIndexer", "Finished building records", {
|
|
837
|
+
entityId,
|
|
838
|
+
inputCount: items.length,
|
|
839
|
+
outputCount: records.length,
|
|
840
|
+
dropped
|
|
841
|
+
});
|
|
842
|
+
return { records, dropped };
|
|
843
|
+
}
|
|
844
|
+
}
|
|
845
|
+
export {
|
|
846
|
+
SearchIndexer
|
|
847
|
+
};
|
|
848
|
+
//# sourceMappingURL=search-indexer.js.map
|