@open-mercato/search 0.4.2-canary-c02407ff85

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. package/AGENTS.md +678 -0
  2. package/build.mjs +92 -0
  3. package/dist/di.js +157 -0
  4. package/dist/di.js.map +7 -0
  5. package/dist/fulltext/drivers/index.js +21 -0
  6. package/dist/fulltext/drivers/index.js.map +7 -0
  7. package/dist/fulltext/drivers/meilisearch/index.js +320 -0
  8. package/dist/fulltext/drivers/meilisearch/index.js.map +7 -0
  9. package/dist/fulltext/index.js +7 -0
  10. package/dist/fulltext/index.js.map +7 -0
  11. package/dist/fulltext/types.js +1 -0
  12. package/dist/fulltext/types.js.map +7 -0
  13. package/dist/index.js +12 -0
  14. package/dist/index.js.map +7 -0
  15. package/dist/indexer/index.js +8 -0
  16. package/dist/indexer/index.js.map +7 -0
  17. package/dist/indexer/search-indexer.js +848 -0
  18. package/dist/indexer/search-indexer.js.map +7 -0
  19. package/dist/indexer/subscribers/delete.js +41 -0
  20. package/dist/indexer/subscribers/delete.js.map +7 -0
  21. package/dist/lib/debug.js +34 -0
  22. package/dist/lib/debug.js.map +7 -0
  23. package/dist/lib/fallback-presenter.js +107 -0
  24. package/dist/lib/fallback-presenter.js.map +7 -0
  25. package/dist/lib/field-policy.js +75 -0
  26. package/dist/lib/field-policy.js.map +7 -0
  27. package/dist/lib/index.js +19 -0
  28. package/dist/lib/index.js.map +7 -0
  29. package/dist/lib/merger.js +93 -0
  30. package/dist/lib/merger.js.map +7 -0
  31. package/dist/lib/presenter-enricher.js +192 -0
  32. package/dist/lib/presenter-enricher.js.map +7 -0
  33. package/dist/modules/search/acl.js +14 -0
  34. package/dist/modules/search/acl.js.map +7 -0
  35. package/dist/modules/search/ai-tools.js +284 -0
  36. package/dist/modules/search/ai-tools.js.map +7 -0
  37. package/dist/modules/search/api/embeddings/reindex/cancel/route.js +65 -0
  38. package/dist/modules/search/api/embeddings/reindex/cancel/route.js.map +7 -0
  39. package/dist/modules/search/api/embeddings/reindex/route.js +165 -0
  40. package/dist/modules/search/api/embeddings/reindex/route.js.map +7 -0
  41. package/dist/modules/search/api/embeddings/route.js +246 -0
  42. package/dist/modules/search/api/embeddings/route.js.map +7 -0
  43. package/dist/modules/search/api/index/route.js +245 -0
  44. package/dist/modules/search/api/index/route.js.map +7 -0
  45. package/dist/modules/search/api/reindex/cancel/route.js +65 -0
  46. package/dist/modules/search/api/reindex/cancel/route.js.map +7 -0
  47. package/dist/modules/search/api/reindex/route.js +332 -0
  48. package/dist/modules/search/api/reindex/route.js.map +7 -0
  49. package/dist/modules/search/api/search/global/route.js +100 -0
  50. package/dist/modules/search/api/search/global/route.js.map +7 -0
  51. package/dist/modules/search/api/search/route.js +101 -0
  52. package/dist/modules/search/api/search/route.js.map +7 -0
  53. package/dist/modules/search/api/settings/fulltext/route.js +55 -0
  54. package/dist/modules/search/api/settings/fulltext/route.js.map +7 -0
  55. package/dist/modules/search/api/settings/global-search/route.js +80 -0
  56. package/dist/modules/search/api/settings/global-search/route.js.map +7 -0
  57. package/dist/modules/search/api/settings/route.js +118 -0
  58. package/dist/modules/search/api/settings/route.js.map +7 -0
  59. package/dist/modules/search/api/settings/vector-store/route.js +77 -0
  60. package/dist/modules/search/api/settings/vector-store/route.js.map +7 -0
  61. package/dist/modules/search/backend/config/search/page.js +10 -0
  62. package/dist/modules/search/backend/config/search/page.js.map +7 -0
  63. package/dist/modules/search/backend/config/search/page.meta.js +24 -0
  64. package/dist/modules/search/backend/config/search/page.meta.js.map +7 -0
  65. package/dist/modules/search/cli.js +698 -0
  66. package/dist/modules/search/cli.js.map +7 -0
  67. package/dist/modules/search/di.js +32 -0
  68. package/dist/modules/search/di.js.map +7 -0
  69. package/dist/modules/search/frontend/components/GlobalSearchDialog.js +357 -0
  70. package/dist/modules/search/frontend/components/GlobalSearchDialog.js.map +7 -0
  71. package/dist/modules/search/frontend/components/HybridSearchTable.js +343 -0
  72. package/dist/modules/search/frontend/components/HybridSearchTable.js.map +7 -0
  73. package/dist/modules/search/frontend/components/SearchSettingsPageClient.js +303 -0
  74. package/dist/modules/search/frontend/components/SearchSettingsPageClient.js.map +7 -0
  75. package/dist/modules/search/frontend/components/sections/FulltextSearchSection.js +360 -0
  76. package/dist/modules/search/frontend/components/sections/FulltextSearchSection.js.map +7 -0
  77. package/dist/modules/search/frontend/components/sections/GlobalSearchSection.js +101 -0
  78. package/dist/modules/search/frontend/components/sections/GlobalSearchSection.js.map +7 -0
  79. package/dist/modules/search/frontend/components/sections/VectorSearchSection.js +608 -0
  80. package/dist/modules/search/frontend/components/sections/VectorSearchSection.js.map +7 -0
  81. package/dist/modules/search/frontend/index.js +9 -0
  82. package/dist/modules/search/frontend/index.js.map +7 -0
  83. package/dist/modules/search/frontend/utils.js +41 -0
  84. package/dist/modules/search/frontend/utils.js.map +7 -0
  85. package/dist/modules/search/i18n/de.json +61 -0
  86. package/dist/modules/search/i18n/en.json +72 -0
  87. package/dist/modules/search/i18n/es.json +61 -0
  88. package/dist/modules/search/i18n/pl.json +61 -0
  89. package/dist/modules/search/index.js +11 -0
  90. package/dist/modules/search/index.js.map +7 -0
  91. package/dist/modules/search/lib/auto-indexing.js +29 -0
  92. package/dist/modules/search/lib/auto-indexing.js.map +7 -0
  93. package/dist/modules/search/lib/embedding-config.js +131 -0
  94. package/dist/modules/search/lib/embedding-config.js.map +7 -0
  95. package/dist/modules/search/lib/global-search-config.js +45 -0
  96. package/dist/modules/search/lib/global-search-config.js.map +7 -0
  97. package/dist/modules/search/lib/reindex-lock.js +99 -0
  98. package/dist/modules/search/lib/reindex-lock.js.map +7 -0
  99. package/dist/modules/search/subscribers/fulltext_upsert.js +64 -0
  100. package/dist/modules/search/subscribers/fulltext_upsert.js.map +7 -0
  101. package/dist/modules/search/subscribers/vector_delete.js +58 -0
  102. package/dist/modules/search/subscribers/vector_delete.js.map +7 -0
  103. package/dist/modules/search/subscribers/vector_purge.js +142 -0
  104. package/dist/modules/search/subscribers/vector_purge.js.map +7 -0
  105. package/dist/modules/search/subscribers/vector_upsert.js +58 -0
  106. package/dist/modules/search/subscribers/vector_upsert.js.map +7 -0
  107. package/dist/modules/search/workers/fulltext-index.worker.js +240 -0
  108. package/dist/modules/search/workers/fulltext-index.worker.js.map +7 -0
  109. package/dist/modules/search/workers/vector-index.worker.js +234 -0
  110. package/dist/modules/search/workers/vector-index.worker.js.map +7 -0
  111. package/dist/queue/fulltext-indexing.js +15 -0
  112. package/dist/queue/fulltext-indexing.js.map +7 -0
  113. package/dist/queue/index.js +3 -0
  114. package/dist/queue/index.js.map +7 -0
  115. package/dist/queue/vector-indexing.js +15 -0
  116. package/dist/queue/vector-indexing.js.map +7 -0
  117. package/dist/service.js +286 -0
  118. package/dist/service.js.map +7 -0
  119. package/dist/strategies/fulltext.strategy.js +116 -0
  120. package/dist/strategies/fulltext.strategy.js.map +7 -0
  121. package/dist/strategies/index.js +12 -0
  122. package/dist/strategies/index.js.map +7 -0
  123. package/dist/strategies/token.strategy.js +80 -0
  124. package/dist/strategies/token.strategy.js.map +7 -0
  125. package/dist/strategies/vector.strategy.js +137 -0
  126. package/dist/strategies/vector.strategy.js.map +7 -0
  127. package/dist/types.js +1 -0
  128. package/dist/types.js.map +7 -0
  129. package/dist/vector/drivers/chromadb/index.js +44 -0
  130. package/dist/vector/drivers/chromadb/index.js.map +7 -0
  131. package/dist/vector/drivers/index.js +9 -0
  132. package/dist/vector/drivers/index.js.map +7 -0
  133. package/dist/vector/drivers/pgvector/index.js +509 -0
  134. package/dist/vector/drivers/pgvector/index.js.map +7 -0
  135. package/dist/vector/drivers/qdrant/index.js +44 -0
  136. package/dist/vector/drivers/qdrant/index.js.map +7 -0
  137. package/dist/vector/index.js +4 -0
  138. package/dist/vector/index.js.map +7 -0
  139. package/dist/vector/lib/vector-logs.js +33 -0
  140. package/dist/vector/lib/vector-logs.js.map +7 -0
  141. package/dist/vector/services/checksum.js +20 -0
  142. package/dist/vector/services/checksum.js.map +7 -0
  143. package/dist/vector/services/embedding.js +222 -0
  144. package/dist/vector/services/embedding.js.map +7 -0
  145. package/dist/vector/services/index.js +4 -0
  146. package/dist/vector/services/index.js.map +7 -0
  147. package/dist/vector/services/vector-index.service.js +960 -0
  148. package/dist/vector/services/vector-index.service.js.map +7 -0
  149. package/dist/vector/types/pg.d.js +1 -0
  150. package/dist/vector/types/pg.d.js.map +7 -0
  151. package/dist/vector/types.js +75 -0
  152. package/dist/vector/types.js.map +7 -0
  153. package/jest.config.cjs +19 -0
  154. package/package.json +142 -0
  155. package/src/__tests__/queue.test.ts +148 -0
  156. package/src/__tests__/service.test.ts +345 -0
  157. package/src/__tests__/workers.test.ts +319 -0
  158. package/src/di.ts +291 -0
  159. package/src/fulltext/drivers/index.ts +41 -0
  160. package/src/fulltext/drivers/meilisearch/index.ts +410 -0
  161. package/src/fulltext/index.ts +13 -0
  162. package/src/fulltext/types.ts +115 -0
  163. package/src/index.ts +36 -0
  164. package/src/indexer/index.ts +13 -0
  165. package/src/indexer/search-indexer.ts +1141 -0
  166. package/src/indexer/subscribers/delete.ts +49 -0
  167. package/src/lib/debug.ts +46 -0
  168. package/src/lib/fallback-presenter.ts +106 -0
  169. package/src/lib/field-policy.ts +169 -0
  170. package/src/lib/index.ts +13 -0
  171. package/src/lib/merger.ts +159 -0
  172. package/src/lib/presenter-enricher.ts +323 -0
  173. package/src/modules/search/README.md +694 -0
  174. package/src/modules/search/acl.ts +10 -0
  175. package/src/modules/search/ai-tools.ts +467 -0
  176. package/src/modules/search/api/embeddings/reindex/cancel/route.ts +77 -0
  177. package/src/modules/search/api/embeddings/reindex/route.ts +197 -0
  178. package/src/modules/search/api/embeddings/route.ts +304 -0
  179. package/src/modules/search/api/index/route.ts +297 -0
  180. package/src/modules/search/api/reindex/cancel/route.ts +77 -0
  181. package/src/modules/search/api/reindex/route.ts +419 -0
  182. package/src/modules/search/api/search/global/route.ts +120 -0
  183. package/src/modules/search/api/search/route.ts +121 -0
  184. package/src/modules/search/api/settings/fulltext/route.ts +82 -0
  185. package/src/modules/search/api/settings/global-search/route.ts +91 -0
  186. package/src/modules/search/api/settings/route.ts +187 -0
  187. package/src/modules/search/api/settings/vector-store/route.ts +105 -0
  188. package/src/modules/search/backend/config/search/page.meta.ts +22 -0
  189. package/src/modules/search/backend/config/search/page.tsx +12 -0
  190. package/src/modules/search/cli.ts +818 -0
  191. package/src/modules/search/di.ts +50 -0
  192. package/src/modules/search/frontend/components/GlobalSearchDialog.tsx +436 -0
  193. package/src/modules/search/frontend/components/HybridSearchTable.tsx +418 -0
  194. package/src/modules/search/frontend/components/SearchSettingsPageClient.tsx +476 -0
  195. package/src/modules/search/frontend/components/sections/FulltextSearchSection.tsx +624 -0
  196. package/src/modules/search/frontend/components/sections/GlobalSearchSection.tsx +124 -0
  197. package/src/modules/search/frontend/components/sections/VectorSearchSection.tsx +943 -0
  198. package/src/modules/search/frontend/index.ts +3 -0
  199. package/src/modules/search/frontend/utils.ts +82 -0
  200. package/src/modules/search/i18n/de.json +61 -0
  201. package/src/modules/search/i18n/en.json +72 -0
  202. package/src/modules/search/i18n/es.json +61 -0
  203. package/src/modules/search/i18n/pl.json +61 -0
  204. package/src/modules/search/index.ts +9 -0
  205. package/src/modules/search/lib/auto-indexing.ts +35 -0
  206. package/src/modules/search/lib/embedding-config.ts +161 -0
  207. package/src/modules/search/lib/global-search-config.ts +69 -0
  208. package/src/modules/search/lib/reindex-lock.ts +201 -0
  209. package/src/modules/search/subscribers/fulltext_upsert.ts +83 -0
  210. package/src/modules/search/subscribers/vector_delete.ts +75 -0
  211. package/src/modules/search/subscribers/vector_purge.ts +161 -0
  212. package/src/modules/search/subscribers/vector_upsert.ts +75 -0
  213. package/src/modules/search/workers/fulltext-index.worker.ts +318 -0
  214. package/src/modules/search/workers/vector-index.worker.ts +292 -0
  215. package/src/queue/fulltext-indexing.ts +87 -0
  216. package/src/queue/index.ts +2 -0
  217. package/src/queue/vector-indexing.ts +66 -0
  218. package/src/service.ts +397 -0
  219. package/src/strategies/fulltext.strategy.ts +155 -0
  220. package/src/strategies/index.ts +17 -0
  221. package/src/strategies/token.strategy.ts +153 -0
  222. package/src/strategies/vector.strategy.ts +234 -0
  223. package/src/types.ts +38 -0
  224. package/src/vector/drivers/chromadb/index.ts +49 -0
  225. package/src/vector/drivers/index.ts +4 -0
  226. package/src/vector/drivers/pgvector/index.ts +627 -0
  227. package/src/vector/drivers/qdrant/index.ts +49 -0
  228. package/src/vector/index.ts +3 -0
  229. package/src/vector/lib/vector-logs.ts +46 -0
  230. package/src/vector/services/checksum.ts +18 -0
  231. package/src/vector/services/embedding.ts +275 -0
  232. package/src/vector/services/index.ts +3 -0
  233. package/src/vector/services/vector-index.service.ts +1234 -0
  234. package/src/vector/types/pg.d.ts +1 -0
  235. package/src/vector/types.ts +220 -0
  236. package/tsconfig.json +9 -0
  237. package/watch.mjs +6 -0
@@ -0,0 +1,848 @@
1
+ import { searchDebug, searchDebugWarn, searchError } from "../lib/debug.js";
2
+ const MAX_PAGES = 1e4;
3
+ class SearchIndexer {
4
+ constructor(searchService, moduleConfigs, options) {
5
+ this.searchService = searchService;
6
+ this.moduleConfigs = moduleConfigs;
7
+ this.entityConfigMap = /* @__PURE__ */ new Map();
8
+ this.queryEngine = options?.queryEngine;
9
+ this.fulltextQueue = options?.fulltextQueue;
10
+ this.vectorQueue = options?.vectorQueue;
11
+ for (const moduleConfig of moduleConfigs) {
12
+ for (const entityConfig of moduleConfig.entities) {
13
+ if (entityConfig.enabled !== false) {
14
+ this.entityConfigMap.set(entityConfig.entityId, entityConfig);
15
+ }
16
+ }
17
+ }
18
+ }
19
+ /**
20
+ * Get the entity config for a given entity ID.
21
+ */
22
+ getEntityConfig(entityId) {
23
+ return this.entityConfigMap.get(entityId);
24
+ }
25
+ /**
26
+ * Get all configured entity configs.
27
+ */
28
+ getAllEntityConfigs() {
29
+ return Array.from(this.entityConfigMap.values());
30
+ }
31
+ /**
32
+ * Check if an entity is configured for search indexing.
33
+ */
34
+ isEntityEnabled(entityId) {
35
+ const config = this.entityConfigMap.get(entityId);
36
+ return config?.enabled !== false;
37
+ }
38
+ /**
39
+ * Index a record in the search service.
40
+ */
41
+ async indexRecord(params) {
42
+ const config = this.entityConfigMap.get(params.entityId);
43
+ if (!config || config.enabled === false) {
44
+ return;
45
+ }
46
+ const buildContext = {
47
+ record: params.record,
48
+ customFields: params.customFields ?? {},
49
+ organizationId: params.organizationId,
50
+ tenantId: params.tenantId,
51
+ queryEngine: this.queryEngine
52
+ };
53
+ let text;
54
+ let presenter;
55
+ let url;
56
+ let links;
57
+ let checksumSource;
58
+ if (config.buildSource) {
59
+ try {
60
+ const source = await config.buildSource(buildContext);
61
+ if (source) {
62
+ text = source.text;
63
+ if (source.presenter) presenter = source.presenter;
64
+ if (source.links) links = source.links;
65
+ if (source.checksumSource !== void 0) checksumSource = source.checksumSource;
66
+ }
67
+ } catch (error) {
68
+ searchDebugWarn("SearchIndexer", "buildSource failed", {
69
+ entityId: params.entityId,
70
+ recordId: params.recordId,
71
+ error: error instanceof Error ? error.message : error
72
+ });
73
+ }
74
+ }
75
+ if (!presenter && config.formatResult) {
76
+ try {
77
+ const result = await config.formatResult(buildContext);
78
+ if (result) presenter = result;
79
+ } catch (error) {
80
+ searchDebugWarn("SearchIndexer", "formatResult failed", {
81
+ entityId: params.entityId,
82
+ recordId: params.recordId,
83
+ error: error instanceof Error ? error.message : error
84
+ });
85
+ }
86
+ }
87
+ if (!url && config.resolveUrl) {
88
+ try {
89
+ const result = await config.resolveUrl(buildContext);
90
+ if (result) url = result;
91
+ } catch (error) {
92
+ searchDebugWarn("SearchIndexer", "resolveUrl failed", {
93
+ entityId: params.entityId,
94
+ recordId: params.recordId,
95
+ error: error instanceof Error ? error.message : error
96
+ });
97
+ }
98
+ }
99
+ if (!links && config.resolveLinks) {
100
+ try {
101
+ const result = await config.resolveLinks(buildContext);
102
+ if (result) links = result;
103
+ } catch (error) {
104
+ searchDebugWarn("SearchIndexer", "resolveLinks failed", {
105
+ entityId: params.entityId,
106
+ recordId: params.recordId,
107
+ error: error instanceof Error ? error.message : error
108
+ });
109
+ }
110
+ }
111
+ const indexableRecord = {
112
+ entityId: params.entityId,
113
+ recordId: params.recordId,
114
+ tenantId: params.tenantId,
115
+ organizationId: params.organizationId,
116
+ fields: params.record,
117
+ presenter,
118
+ url,
119
+ links,
120
+ text,
121
+ checksumSource
122
+ };
123
+ await this.searchService.index(indexableRecord);
124
+ }
125
+ /**
126
+ * Index a record by ID (loads the record from database first).
127
+ * Used by workers that only have record identifiers.
128
+ */
129
+ async indexRecordById(params) {
130
+ if (!this.queryEngine) {
131
+ return { action: "skipped", reason: "queryEngine not available" };
132
+ }
133
+ const config = this.entityConfigMap.get(params.entityId);
134
+ if (!config || config.enabled === false) {
135
+ return { action: "skipped", reason: "entity not configured" };
136
+ }
137
+ try {
138
+ const result = await this.queryEngine.query(params.entityId, {
139
+ tenantId: params.tenantId,
140
+ organizationId: params.organizationId ?? void 0,
141
+ filters: { id: params.recordId },
142
+ includeCustomFields: true,
143
+ page: { page: 1, pageSize: 1 }
144
+ });
145
+ const record = result.items[0];
146
+ if (!record) {
147
+ return { action: "skipped", reason: "record not found" };
148
+ }
149
+ const customFields = {};
150
+ for (const [key, value] of Object.entries(record)) {
151
+ if (key.startsWith("cf:") || key.startsWith("cf_")) {
152
+ customFields[key.slice(3)] = value;
153
+ }
154
+ }
155
+ await this.indexRecord({
156
+ entityId: params.entityId,
157
+ recordId: params.recordId,
158
+ tenantId: params.tenantId,
159
+ organizationId: params.organizationId,
160
+ record,
161
+ customFields
162
+ });
163
+ return { action: "indexed" };
164
+ } catch (error) {
165
+ searchError("SearchIndexer", "Failed to load record for indexing", {
166
+ entityId: params.entityId,
167
+ recordId: params.recordId,
168
+ error: error instanceof Error ? error.message : error
169
+ });
170
+ throw error;
171
+ }
172
+ }
173
+ /**
174
+ * Delete a record from the search index.
175
+ */
176
+ async deleteRecord(params) {
177
+ await this.searchService.delete(params.entityId, params.recordId, params.tenantId);
178
+ }
179
+ /**
180
+ * Purge all records of an entity type from the search index.
181
+ */
182
+ async purgeEntity(params) {
183
+ await this.searchService.purge(params.entityId, params.tenantId);
184
+ }
185
+ /**
186
+ * Reindex an entity via all configured strategies (including vector).
187
+ * This is the general reindex method that works with all search strategies.
188
+ */
189
+ async reindexEntity(params) {
190
+ if (!this.queryEngine) {
191
+ return {
192
+ success: false,
193
+ entitiesProcessed: 0,
194
+ recordsIndexed: 0,
195
+ errors: [{ entityId: params.entityId, error: "Query engine not available" }]
196
+ };
197
+ }
198
+ const config = this.entityConfigMap.get(params.entityId);
199
+ if (!config || config.enabled === false) {
200
+ return {
201
+ success: false,
202
+ entitiesProcessed: 0,
203
+ recordsIndexed: 0,
204
+ errors: [{ entityId: params.entityId, error: "Entity not configured for search" }]
205
+ };
206
+ }
207
+ const result = {
208
+ success: true,
209
+ entitiesProcessed: 1,
210
+ recordsIndexed: 0,
211
+ errors: []
212
+ };
213
+ if (params.purgeFirst) {
214
+ try {
215
+ await this.searchService.purge(params.entityId, params.tenantId);
216
+ } catch (error) {
217
+ searchDebugWarn("SearchIndexer", "Failed to purge entity before reindex", {
218
+ entityId: params.entityId,
219
+ error: error instanceof Error ? error.message : error
220
+ });
221
+ }
222
+ }
223
+ let page = 1;
224
+ const pageSize = 200;
225
+ let hasMore = true;
226
+ while (hasMore && page <= MAX_PAGES) {
227
+ try {
228
+ const queryResult = await this.queryEngine.query(params.entityId, {
229
+ tenantId: params.tenantId,
230
+ organizationId: params.organizationId ?? void 0,
231
+ includeCustomFields: true,
232
+ page: { page, pageSize }
233
+ });
234
+ const items = queryResult.items;
235
+ if (items.length === 0) {
236
+ hasMore = false;
237
+ break;
238
+ }
239
+ const { records } = await this.buildIndexableRecords(
240
+ params.entityId,
241
+ params.tenantId,
242
+ params.organizationId ?? null,
243
+ items,
244
+ config
245
+ );
246
+ for (const record of records) {
247
+ try {
248
+ await this.searchService.index(record);
249
+ result.recordsIndexed++;
250
+ } catch (error) {
251
+ searchDebugWarn("SearchIndexer", "Failed to index record", {
252
+ entityId: params.entityId,
253
+ recordId: record.recordId,
254
+ error: error instanceof Error ? error.message : error
255
+ });
256
+ }
257
+ }
258
+ page++;
259
+ hasMore = items.length === pageSize;
260
+ } catch (error) {
261
+ result.success = false;
262
+ result.errors.push({
263
+ entityId: params.entityId,
264
+ error: error instanceof Error ? error.message : String(error)
265
+ });
266
+ break;
267
+ }
268
+ }
269
+ return result;
270
+ }
271
+ /**
272
+ * Reindex all enabled entities via all configured strategies.
273
+ */
274
+ async reindexAll(params) {
275
+ const result = {
276
+ success: true,
277
+ entitiesProcessed: 0,
278
+ recordsIndexed: 0,
279
+ errors: []
280
+ };
281
+ const enabledEntities = this.listEnabledEntities();
282
+ for (const entityId of enabledEntities) {
283
+ const entityResult = await this.reindexEntity({
284
+ entityId,
285
+ tenantId: params.tenantId,
286
+ organizationId: params.organizationId,
287
+ purgeFirst: params.purgeFirst
288
+ });
289
+ result.entitiesProcessed++;
290
+ result.recordsIndexed += entityResult.recordsIndexed;
291
+ result.errors.push(...entityResult.errors);
292
+ if (!entityResult.success) {
293
+ result.success = false;
294
+ }
295
+ }
296
+ return result;
297
+ }
298
+ /**
299
+ * Bulk index multiple records.
300
+ */
301
+ async bulkIndexRecords(params) {
302
+ const indexableRecords = [];
303
+ for (const param of params) {
304
+ const config = this.entityConfigMap.get(param.entityId);
305
+ if (!config || config.enabled === false) {
306
+ continue;
307
+ }
308
+ const buildContext = {
309
+ record: param.record,
310
+ customFields: param.customFields ?? {},
311
+ organizationId: param.organizationId,
312
+ tenantId: param.tenantId
313
+ };
314
+ let presenter;
315
+ if (config.formatResult) {
316
+ try {
317
+ const result = await config.formatResult(buildContext);
318
+ if (result) presenter = result;
319
+ } catch {
320
+ }
321
+ }
322
+ let url;
323
+ if (config.resolveUrl) {
324
+ try {
325
+ const result = await config.resolveUrl(buildContext);
326
+ if (result) url = result;
327
+ } catch {
328
+ }
329
+ }
330
+ let links;
331
+ if (config.resolveLinks) {
332
+ try {
333
+ const result = await config.resolveLinks(buildContext);
334
+ if (result) links = result;
335
+ } catch {
336
+ }
337
+ }
338
+ indexableRecords.push({
339
+ entityId: param.entityId,
340
+ recordId: param.recordId,
341
+ tenantId: param.tenantId,
342
+ organizationId: param.organizationId,
343
+ fields: param.record,
344
+ presenter,
345
+ url,
346
+ links
347
+ });
348
+ }
349
+ if (indexableRecords.length > 0) {
350
+ await this.searchService.bulkIndex(indexableRecords);
351
+ }
352
+ }
353
+ /**
354
+ * List all enabled entity IDs from the module configurations.
355
+ */
356
+ listEnabledEntities() {
357
+ return Array.from(this.entityConfigMap.keys());
358
+ }
359
+ /**
360
+ * Get the fulltext strategy from the search service.
361
+ */
362
+ getFulltextStrategy() {
363
+ const strategy = this.searchService.getStrategy("fulltext");
364
+ if (!strategy) return void 0;
365
+ return strategy;
366
+ }
367
+ /**
368
+ * Reindex a single entity type to fulltext search.
369
+ * This fetches all records from the database and re-indexes them to fulltext only.
370
+ *
371
+ * When `useQueue` is true, batches are enqueued for background processing by workers.
372
+ * When `useQueue` is false (default), batches are indexed directly (blocking).
373
+ */
374
+ async reindexEntityToFulltext(params) {
375
+ const result = {
376
+ success: true,
377
+ entitiesProcessed: 0,
378
+ recordsIndexed: 0,
379
+ recordsDropped: 0,
380
+ jobsEnqueued: 0,
381
+ errors: []
382
+ };
383
+ const fulltext = this.getFulltextStrategy();
384
+ if (!fulltext) {
385
+ result.success = false;
386
+ result.errors.push({ entityId: params.entityId, error: "Fulltext strategy not available" });
387
+ return result;
388
+ }
389
+ if (params.useQueue && !this.fulltextQueue) {
390
+ result.success = false;
391
+ result.errors.push({ entityId: params.entityId, error: "Fulltext queue not configured for queue-based reindexing" });
392
+ return result;
393
+ }
394
+ if (!this.queryEngine) {
395
+ result.success = false;
396
+ result.errors.push({ entityId: params.entityId, error: "QueryEngine not available for reindexing" });
397
+ return result;
398
+ }
399
+ const config = this.entityConfigMap.get(params.entityId);
400
+ if (!config) {
401
+ result.success = false;
402
+ result.errors.push({ entityId: params.entityId, error: "Entity not configured for search" });
403
+ return result;
404
+ }
405
+ try {
406
+ params.onProgress?.({
407
+ entityId: params.entityId,
408
+ phase: "starting",
409
+ processed: 0
410
+ });
411
+ if (params.recreateIndex !== false) {
412
+ await fulltext.recreateIndex(params.tenantId);
413
+ }
414
+ const pageSize = 200;
415
+ let page = 1;
416
+ let totalProcessed = 0;
417
+ let jobsEnqueued = 0;
418
+ for (; ; ) {
419
+ params.onProgress?.({
420
+ entityId: params.entityId,
421
+ phase: "fetching",
422
+ processed: totalProcessed
423
+ });
424
+ try {
425
+ const queryResult = await this.queryEngine.query(params.entityId, {
426
+ tenantId: params.tenantId,
427
+ organizationId: params.organizationId ?? void 0,
428
+ page: { page, pageSize }
429
+ });
430
+ if (!queryResult.items.length) {
431
+ break;
432
+ }
433
+ params.onProgress?.({
434
+ entityId: params.entityId,
435
+ phase: "indexing",
436
+ processed: totalProcessed,
437
+ total: queryResult.total
438
+ });
439
+ const { records: indexableRecords, dropped } = await this.buildIndexableRecords(
440
+ params.entityId,
441
+ params.tenantId,
442
+ params.organizationId ?? null,
443
+ queryResult.items,
444
+ config
445
+ );
446
+ result.recordsDropped = (result.recordsDropped ?? 0) + dropped;
447
+ if (indexableRecords.length > 0) {
448
+ if (params.useQueue && this.fulltextQueue) {
449
+ await this.fulltextQueue.enqueue({
450
+ jobType: "batch-index",
451
+ tenantId: params.tenantId,
452
+ organizationId: params.organizationId,
453
+ records: indexableRecords.map((r) => ({ entityId: r.entityId, recordId: r.recordId }))
454
+ });
455
+ jobsEnqueued += 1;
456
+ totalProcessed += indexableRecords.length;
457
+ } else {
458
+ try {
459
+ await fulltext.bulkIndex(indexableRecords);
460
+ totalProcessed += indexableRecords.length;
461
+ } catch (indexError) {
462
+ const errorMsg = indexError instanceof Error ? indexError.message : String(indexError);
463
+ result.errors.push({
464
+ entityId: params.entityId,
465
+ error: `Batch ${page} failed: ${errorMsg}`
466
+ });
467
+ }
468
+ }
469
+ }
470
+ if (queryResult.items.length < pageSize) {
471
+ break;
472
+ }
473
+ page += 1;
474
+ if (page > MAX_PAGES) {
475
+ break;
476
+ }
477
+ } catch (queryError) {
478
+ const errorMsg = queryError instanceof Error ? queryError.message : String(queryError);
479
+ result.errors.push({
480
+ entityId: params.entityId,
481
+ error: `Query failed: ${errorMsg}`
482
+ });
483
+ break;
484
+ }
485
+ }
486
+ result.entitiesProcessed = 1;
487
+ result.recordsIndexed = totalProcessed;
488
+ result.jobsEnqueued = jobsEnqueued;
489
+ params.onProgress?.({
490
+ entityId: params.entityId,
491
+ phase: "complete",
492
+ processed: totalProcessed,
493
+ total: totalProcessed
494
+ });
495
+ } catch (error) {
496
+ result.success = false;
497
+ result.errors.push({
498
+ entityId: params.entityId,
499
+ error: error instanceof Error ? error.message : String(error)
500
+ });
501
+ }
502
+ return result;
503
+ }
504
+ /**
505
+ * Reindex all enabled entities to fulltext search.
506
+ *
507
+ * When `useQueue` is true, batches are enqueued for background processing by workers.
508
+ * When `useQueue` is false (default), batches are indexed directly (blocking).
509
+ */
510
+ async reindexAllToFulltext(params) {
511
+ const result = {
512
+ success: true,
513
+ entitiesProcessed: 0,
514
+ recordsIndexed: 0,
515
+ recordsDropped: 0,
516
+ jobsEnqueued: 0,
517
+ errors: []
518
+ };
519
+ const fulltext = this.getFulltextStrategy();
520
+ if (!fulltext) {
521
+ result.success = false;
522
+ result.errors.push({ entityId: "all", error: "Fulltext strategy not available" });
523
+ return result;
524
+ }
525
+ if (params.recreateIndex !== false) {
526
+ await fulltext.recreateIndex(params.tenantId);
527
+ }
528
+ const entities = this.listEnabledEntities();
529
+ for (const entityId of entities) {
530
+ const entityResult = await this.reindexEntityToFulltext({
531
+ entityId,
532
+ tenantId: params.tenantId,
533
+ organizationId: params.organizationId,
534
+ recreateIndex: false,
535
+ // Already recreated above
536
+ onProgress: params.onProgress,
537
+ useQueue: params.useQueue
538
+ });
539
+ result.entitiesProcessed += entityResult.entitiesProcessed;
540
+ result.recordsIndexed += entityResult.recordsIndexed;
541
+ result.recordsDropped = (result.recordsDropped ?? 0) + (entityResult.recordsDropped ?? 0);
542
+ result.jobsEnqueued = (result.jobsEnqueued ?? 0) + (entityResult.jobsEnqueued ?? 0);
543
+ result.errors.push(...entityResult.errors);
544
+ if (!entityResult.success) {
545
+ result.success = false;
546
+ }
547
+ }
548
+ return result;
549
+ }
550
+ /**
551
+ * Reindex a single entity type to vector search.
552
+ * This fetches all records from the database and enqueues them for vector indexing.
553
+ *
554
+ * When `useQueue` is true (default), record IDs are enqueued for background processing by workers.
555
+ * When `useQueue` is false, records are indexed directly (blocking).
556
+ */
557
+ async reindexEntityToVector(params) {
558
+ searchDebug("SearchIndexer", "reindexEntityToVector called", {
559
+ entityId: params.entityId,
560
+ tenantId: params.tenantId,
561
+ organizationId: params.organizationId,
562
+ useQueue: params.useQueue,
563
+ purgeFirst: params.purgeFirst
564
+ });
565
+ const result = {
566
+ success: true,
567
+ entitiesProcessed: 0,
568
+ recordsIndexed: 0,
569
+ recordsDropped: 0,
570
+ jobsEnqueued: 0,
571
+ errors: []
572
+ };
573
+ if (params.useQueue !== false && !this.vectorQueue) {
574
+ result.success = false;
575
+ result.errors.push({ entityId: params.entityId, error: "Vector queue not configured for queue-based reindexing" });
576
+ return result;
577
+ }
578
+ if (!this.queryEngine) {
579
+ result.success = false;
580
+ result.errors.push({ entityId: params.entityId, error: "QueryEngine not available for reindexing" });
581
+ return result;
582
+ }
583
+ const config = this.entityConfigMap.get(params.entityId);
584
+ if (!config) {
585
+ result.success = false;
586
+ result.errors.push({ entityId: params.entityId, error: "Entity not configured for search" });
587
+ return result;
588
+ }
589
+ try {
590
+ params.onProgress?.({
591
+ entityId: params.entityId,
592
+ phase: "starting",
593
+ processed: 0
594
+ });
595
+ if (params.purgeFirst) {
596
+ try {
597
+ await this.searchService.purge(params.entityId, params.tenantId);
598
+ } catch (error) {
599
+ searchDebugWarn("SearchIndexer", "Failed to purge entity before vector reindex", {
600
+ entityId: params.entityId,
601
+ error: error instanceof Error ? error.message : error
602
+ });
603
+ }
604
+ }
605
+ const pageSize = 200;
606
+ let page = 1;
607
+ let totalProcessed = 0;
608
+ let jobsEnqueued = 0;
609
+ for (; ; ) {
610
+ params.onProgress?.({
611
+ entityId: params.entityId,
612
+ phase: "fetching",
613
+ processed: totalProcessed
614
+ });
615
+ const queryResult = await this.queryEngine.query(params.entityId, {
616
+ tenantId: params.tenantId,
617
+ organizationId: params.organizationId ?? void 0,
618
+ page: { page, pageSize }
619
+ });
620
+ if (!queryResult.items.length) break;
621
+ params.onProgress?.({
622
+ entityId: params.entityId,
623
+ phase: "indexing",
624
+ processed: totalProcessed,
625
+ total: queryResult.total
626
+ });
627
+ const batchRecords = [];
628
+ for (const item of queryResult.items) {
629
+ const recordId = String(item.id ?? "");
630
+ if (!recordId) {
631
+ result.recordsDropped = (result.recordsDropped ?? 0) + 1;
632
+ continue;
633
+ }
634
+ batchRecords.push({
635
+ entityId: params.entityId,
636
+ recordId
637
+ });
638
+ }
639
+ if (batchRecords.length > 0) {
640
+ if (params.useQueue !== false && this.vectorQueue) {
641
+ await this.vectorQueue.enqueue({
642
+ jobType: "batch-index",
643
+ tenantId: params.tenantId,
644
+ organizationId: params.organizationId ?? null,
645
+ records: batchRecords
646
+ });
647
+ jobsEnqueued += 1;
648
+ totalProcessed += batchRecords.length;
649
+ searchDebug("SearchIndexer", "Enqueued batch for vector indexing", {
650
+ entityId: params.entityId,
651
+ batchSize: batchRecords.length,
652
+ jobsEnqueued,
653
+ totalProcessed
654
+ });
655
+ } else {
656
+ for (const { entityId, recordId } of batchRecords) {
657
+ try {
658
+ await this.indexRecordById({
659
+ entityId,
660
+ recordId,
661
+ tenantId: params.tenantId,
662
+ organizationId: params.organizationId
663
+ });
664
+ totalProcessed++;
665
+ } catch (error) {
666
+ searchDebugWarn("SearchIndexer", "Failed to index record to vector", {
667
+ entityId,
668
+ recordId,
669
+ error: error instanceof Error ? error.message : error
670
+ });
671
+ }
672
+ }
673
+ }
674
+ }
675
+ if (queryResult.items.length < pageSize) break;
676
+ page += 1;
677
+ if (page > MAX_PAGES) {
678
+ searchDebugWarn("SearchIndexer", "Reached MAX_PAGES limit, stopping pagination", {
679
+ entityId: params.entityId,
680
+ maxPages: MAX_PAGES,
681
+ totalProcessed
682
+ });
683
+ break;
684
+ }
685
+ }
686
+ result.entitiesProcessed = 1;
687
+ result.recordsIndexed = totalProcessed;
688
+ result.jobsEnqueued = jobsEnqueued;
689
+ params.onProgress?.({
690
+ entityId: params.entityId,
691
+ phase: "complete",
692
+ processed: totalProcessed,
693
+ total: totalProcessed
694
+ });
695
+ } catch (error) {
696
+ result.success = false;
697
+ result.errors.push({
698
+ entityId: params.entityId,
699
+ error: error instanceof Error ? error.message : String(error)
700
+ });
701
+ }
702
+ return result;
703
+ }
704
+ /**
705
+ * Reindex all enabled entities to vector search.
706
+ *
707
+ * When `useQueue` is true (default), batches are enqueued for background processing by workers.
708
+ * When `useQueue` is false, batches are indexed directly (blocking).
709
+ */
710
+ async reindexAllToVector(params) {
711
+ const result = {
712
+ success: true,
713
+ entitiesProcessed: 0,
714
+ recordsIndexed: 0,
715
+ recordsDropped: 0,
716
+ jobsEnqueued: 0,
717
+ errors: []
718
+ };
719
+ const entities = this.listEnabledEntities();
720
+ for (const entityId of entities) {
721
+ const entityResult = await this.reindexEntityToVector({
722
+ entityId,
723
+ tenantId: params.tenantId,
724
+ organizationId: params.organizationId,
725
+ onProgress: params.onProgress,
726
+ useQueue: params.useQueue,
727
+ purgeFirst: params.purgeFirst
728
+ });
729
+ result.entitiesProcessed += entityResult.entitiesProcessed;
730
+ result.recordsIndexed += entityResult.recordsIndexed;
731
+ result.recordsDropped = (result.recordsDropped ?? 0) + (entityResult.recordsDropped ?? 0);
732
+ result.jobsEnqueued = (result.jobsEnqueued ?? 0) + (entityResult.jobsEnqueued ?? 0);
733
+ result.errors.push(...entityResult.errors);
734
+ if (!entityResult.success) {
735
+ result.success = false;
736
+ }
737
+ }
738
+ return result;
739
+ }
740
+ /**
741
+ * Build IndexableRecords from raw query results.
742
+ * Returns records and count of dropped items (missing id or other validation failures).
743
+ */
744
+ async buildIndexableRecords(entityId, tenantId, organizationId, items, config) {
745
+ const records = [];
746
+ let dropped = 0;
747
+ if (items.length > 0) {
748
+ searchDebug("SearchIndexer", "Sample item structure", {
749
+ entityId,
750
+ sampleKeys: Object.keys(items[0]),
751
+ sampleId: items[0].id,
752
+ hasId: "id" in items[0],
753
+ firstName: items[0].first_name,
754
+ lastName: items[0].last_name,
755
+ preferredName: items[0].preferred_name,
756
+ sampleItem: JSON.stringify(items[0]).slice(0, 500)
757
+ });
758
+ }
759
+ for (const item of items) {
760
+ const recordId = String(item.id ?? "");
761
+ if (!recordId) {
762
+ searchDebugWarn("SearchIndexer", "Skipping item without id", { entityId, itemKeys: Object.keys(item) });
763
+ dropped++;
764
+ continue;
765
+ }
766
+ const customFields = {};
767
+ for (const [key, value] of Object.entries(item)) {
768
+ if (key.startsWith("cf:") || key.startsWith("cf_")) {
769
+ const cfKey = key.slice(3);
770
+ customFields[cfKey] = value;
771
+ }
772
+ }
773
+ const buildContext = {
774
+ record: item,
775
+ customFields,
776
+ organizationId,
777
+ tenantId,
778
+ queryEngine: this.queryEngine
779
+ };
780
+ let text;
781
+ let presenter;
782
+ let url;
783
+ let links;
784
+ let checksumSource;
785
+ if (config.buildSource) {
786
+ try {
787
+ const source = await config.buildSource(buildContext);
788
+ if (source) {
789
+ text = source.text;
790
+ if (source.presenter) presenter = source.presenter;
791
+ if (source.links) links = source.links;
792
+ if (source.checksumSource !== void 0) checksumSource = source.checksumSource;
793
+ }
794
+ } catch (err) {
795
+ searchDebugWarn("SearchIndexer", "buildSource failed", {
796
+ entityId,
797
+ recordId,
798
+ error: err instanceof Error ? err.message : err
799
+ });
800
+ }
801
+ }
802
+ if (!presenter && config.formatResult) {
803
+ try {
804
+ const result = await config.formatResult(buildContext);
805
+ if (result) presenter = result;
806
+ } catch {
807
+ }
808
+ }
809
+ if (!url && config.resolveUrl) {
810
+ try {
811
+ const result = await config.resolveUrl(buildContext);
812
+ if (result) url = result;
813
+ } catch {
814
+ }
815
+ }
816
+ if (!links && config.resolveLinks) {
817
+ try {
818
+ const result = await config.resolveLinks(buildContext);
819
+ if (result) links = result;
820
+ } catch {
821
+ }
822
+ }
823
+ records.push({
824
+ entityId,
825
+ recordId,
826
+ tenantId,
827
+ organizationId,
828
+ fields: item,
829
+ presenter,
830
+ url,
831
+ links,
832
+ text,
833
+ checksumSource
834
+ });
835
+ }
836
+ searchDebug("SearchIndexer", "Finished building records", {
837
+ entityId,
838
+ inputCount: items.length,
839
+ outputCount: records.length,
840
+ dropped
841
+ });
842
+ return { records, dropped };
843
+ }
844
+ }
845
+ export {
846
+ SearchIndexer
847
+ };
848
+ //# sourceMappingURL=search-indexer.js.map