@open-mercato/search 0.10.1-develop.8231.1.ae76b1d2c5 → 0.10.1-develop.8258.1.62a94b74a2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/dist/indexer/search-indexer.js +21 -4
- package/dist/indexer/search-indexer.js.map +2 -2
- package/dist/modules/search/__integration__/TC-SEARCH-016.spec.js +187 -0
- package/dist/modules/search/__integration__/TC-SEARCH-016.spec.js.map +7 -0
- package/dist/modules/search/__integration__/TC-SEARCH-017.spec.js +94 -0
- package/dist/modules/search/__integration__/TC-SEARCH-017.spec.js.map +7 -0
- package/dist/modules/search/__integration__/TC-SEARCH-018.spec.js +74 -0
- package/dist/modules/search/__integration__/TC-SEARCH-018.spec.js.map +7 -0
- package/dist/modules/search/ai-tools.js +3 -1
- package/dist/modules/search/ai-tools.js.map +2 -2
- package/dist/modules/search/api/openapi.js +1 -0
- package/dist/modules/search/api/openapi.js.map +2 -2
- package/dist/modules/search/api/reindex/route.js +35 -16
- package/dist/modules/search/api/reindex/route.js.map +2 -2
- package/dist/modules/search/cli.js +2 -16
- package/dist/modules/search/cli.js.map +2 -2
- package/dist/modules/search/frontend/components/sections/FulltextSearchSection.js +1 -1
- package/dist/modules/search/frontend/components/sections/FulltextSearchSection.js.map +2 -2
- package/dist/modules/search/i18n/de.json +2 -0
- package/dist/modules/search/i18n/en.json +2 -0
- package/dist/modules/search/i18n/es.json +2 -0
- package/dist/modules/search/i18n/ko.json +2 -0
- package/dist/modules/search/i18n/pl.json +2 -0
- package/dist/modules/search/lib/reindex-lock.js +29 -6
- package/dist/modules/search/lib/reindex-lock.js.map +2 -2
- package/dist/modules/search/lib/reindex-progress.js +28 -3
- package/dist/modules/search/lib/reindex-progress.js.map +2 -2
- package/dist/modules/search/workers/fulltext-index.worker.js +19 -6
- package/dist/modules/search/workers/fulltext-index.worker.js.map +2 -2
- package/dist/strategies/fulltext.strategy.js +33 -2
- package/dist/strategies/fulltext.strategy.js.map +2 -2
- package/package.json +4 -4
- package/src/__tests__/reindex-strategy-policy.test.ts +2 -0
- package/src/__tests__/search-indexer-fulltext-reindex-scope.test.ts +435 -0
- package/src/__tests__/workers.test.ts +46 -1
- package/src/indexer/search-indexer.ts +31 -7
- package/src/modules/search/__integration__/TC-SEARCH-016.spec.ts +247 -0
- package/src/modules/search/__integration__/TC-SEARCH-017.spec.ts +141 -0
- package/src/modules/search/__integration__/TC-SEARCH-018.spec.ts +114 -0
- package/src/modules/search/ai-tools.ts +3 -1
- package/src/modules/search/api/__tests__/reindex.routes.test.ts +222 -6
- package/src/modules/search/api/openapi.ts +1 -0
- package/src/modules/search/api/reindex/route.ts +39 -16
- package/src/modules/search/cli.ts +2 -16
- package/src/modules/search/frontend/components/sections/FulltextSearchSection.tsx +1 -1
- package/src/modules/search/frontend/components/sections/__tests__/FulltextSearchSection.test.tsx +112 -3
- package/src/modules/search/i18n/de.json +2 -0
- package/src/modules/search/i18n/en.json +2 -0
- package/src/modules/search/i18n/es.json +2 -0
- package/src/modules/search/i18n/ko.json +2 -0
- package/src/modules/search/i18n/pl.json +2 -0
- package/src/modules/search/lib/__tests__/reindex-progress.test.ts +157 -0
- package/src/modules/search/lib/reindex-lock.ts +66 -8
- package/src/modules/search/lib/reindex-progress.ts +55 -2
- package/src/modules/search/workers/fulltext-index.worker.ts +21 -6
- package/src/strategies/fulltext.strategy.ts +39 -2
|
@@ -0,0 +1,435 @@
|
|
|
1
|
+
import { SearchIndexer } from '../indexer/search-indexer'
|
|
2
|
+
import { SearchService } from '../service'
|
|
3
|
+
import { FullTextSearchStrategy } from '../strategies/fulltext.strategy'
|
|
4
|
+
import type { FullTextSearchDocument, FullTextSearchDriver, ListDocumentIdsOptions } from '../fulltext/types'
|
|
5
|
+
import type { SearchModuleConfig } from '../types'
|
|
6
|
+
import type { SearchIndexerOptions } from '../indexer/search-indexer'
|
|
7
|
+
import type { QueryEngine, QueryResult } from '@open-mercato/shared/lib/query/types'
|
|
8
|
+
|
|
9
|
+
const TENANT = 'tenant-1'
|
|
10
|
+
const ORG_A = 'org-a'
|
|
11
|
+
const ORG_B = 'org-b'
|
|
12
|
+
|
|
13
|
+
function createTenantIndexDriver() {
|
|
14
|
+
const indexes = new Map<string, Map<string, FullTextSearchDocument>>()
|
|
15
|
+
const tenantIndex = (tenantId: string) => {
|
|
16
|
+
const existing = indexes.get(tenantId)
|
|
17
|
+
if (existing) return existing
|
|
18
|
+
const created = new Map<string, FullTextSearchDocument>()
|
|
19
|
+
indexes.set(tenantId, created)
|
|
20
|
+
return created
|
|
21
|
+
}
|
|
22
|
+
const upsert = async (doc: FullTextSearchDocument) => {
|
|
23
|
+
tenantIndex(doc.tenantId).set(`${doc.entityId}:${doc.recordId}`, doc)
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const driver: FullTextSearchDriver = {
|
|
27
|
+
id: 'meilisearch',
|
|
28
|
+
ensureReady: jest.fn(async () => {}),
|
|
29
|
+
isHealthy: jest.fn(async () => true),
|
|
30
|
+
search: jest.fn(async () => []),
|
|
31
|
+
index: jest.fn(upsert),
|
|
32
|
+
delete: jest.fn(async () => {}),
|
|
33
|
+
bulkIndex: jest.fn(async (docs: FullTextSearchDocument[]) => {
|
|
34
|
+
for (const doc of docs) await upsert(doc)
|
|
35
|
+
}),
|
|
36
|
+
purge: jest.fn(async (entityId: string, tenantId: string, organizationId?: string | null) => {
|
|
37
|
+
const index = tenantIndex(tenantId)
|
|
38
|
+
for (const [key, doc] of index) {
|
|
39
|
+
if (doc.entityId !== entityId) continue
|
|
40
|
+
if (organizationId && doc.organizationId !== organizationId) continue
|
|
41
|
+
index.delete(key)
|
|
42
|
+
}
|
|
43
|
+
}),
|
|
44
|
+
recreateIndex: jest.fn(async (tenantId: string) => {
|
|
45
|
+
indexes.set(tenantId, new Map())
|
|
46
|
+
}),
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const keys = (tenantId: string) => [...tenantIndex(tenantId).keys()].sort()
|
|
50
|
+
return { driver, upsert, keys, tenantIndex }
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function withoutPurge(fake: ReturnType<typeof createTenantIndexDriver>, options: { listDocumentIds: boolean }) {
|
|
54
|
+
const { purge: _purge, ...rest } = fake.driver
|
|
55
|
+
const driver: FullTextSearchDriver = {
|
|
56
|
+
...rest,
|
|
57
|
+
delete: jest.fn(async (recordId: string, tenantId: string) => {
|
|
58
|
+
const index = fake.tenantIndex(tenantId)
|
|
59
|
+
for (const [key, doc] of index) {
|
|
60
|
+
if (doc.recordId === recordId) index.delete(key)
|
|
61
|
+
}
|
|
62
|
+
}),
|
|
63
|
+
}
|
|
64
|
+
if (options.listDocumentIds) {
|
|
65
|
+
driver.listDocumentIds = jest.fn(async (entityId: string, tenantId: string, listOptions?: ListDocumentIdsOptions) => {
|
|
66
|
+
const ids = [...fake.tenantIndex(tenantId).values()]
|
|
67
|
+
.filter((doc) => doc.entityId === entityId)
|
|
68
|
+
.filter((doc) => !listOptions?.organizationId || doc.organizationId === listOptions.organizationId)
|
|
69
|
+
.map((doc) => doc.recordId)
|
|
70
|
+
const offset = listOptions?.offset ?? 0
|
|
71
|
+
return ids.slice(offset, offset + (listOptions?.limit ?? 20))
|
|
72
|
+
})
|
|
73
|
+
}
|
|
74
|
+
return driver
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function seedDoc(entityId: string, recordId: string, organizationId: string | null): FullTextSearchDocument {
|
|
78
|
+
return { entityId, recordId, tenantId: TENANT, organizationId, fields: { name: recordId } }
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const moduleConfigs: SearchModuleConfig[] = [
|
|
82
|
+
{
|
|
83
|
+
entities: [
|
|
84
|
+
{ entityId: 'test:product', enabled: true },
|
|
85
|
+
{ entityId: 'test:customer', enabled: true },
|
|
86
|
+
],
|
|
87
|
+
},
|
|
88
|
+
]
|
|
89
|
+
|
|
90
|
+
function makeQueryEngine(rows: Record<string, Array<{ id: string; organization_id: string }>>): QueryEngine {
|
|
91
|
+
return {
|
|
92
|
+
query: jest.fn(async (entity, opts) => {
|
|
93
|
+
const all = rows[entity as string] ?? []
|
|
94
|
+
const items = opts?.organizationId ? all.filter((row) => row.organization_id === opts.organizationId) : all
|
|
95
|
+
return { items, total: items.length } as QueryResult
|
|
96
|
+
}),
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
async function setup(
|
|
101
|
+
fulltextQueue?: SearchIndexerOptions['fulltextQueue'],
|
|
102
|
+
buildDriver: (fake: ReturnType<typeof createTenantIndexDriver>) => FullTextSearchDriver = (fake) => fake.driver,
|
|
103
|
+
) {
|
|
104
|
+
const fake = createTenantIndexDriver()
|
|
105
|
+
const driver = buildDriver(fake)
|
|
106
|
+
const searchService = new SearchService({ strategies: [new FullTextSearchStrategy(driver)] })
|
|
107
|
+
await fake.upsert(seedDoc('test:product', 'p-a-stale', ORG_A))
|
|
108
|
+
await fake.upsert(seedDoc('test:product', 'p-b-1', ORG_B))
|
|
109
|
+
await fake.upsert(seedDoc('test:customer', 'c-a-1', ORG_A))
|
|
110
|
+
await fake.upsert(seedDoc('test:customer', 'c-b-1', ORG_B))
|
|
111
|
+
const indexer = new SearchIndexer(searchService, moduleConfigs, {
|
|
112
|
+
fulltextQueue,
|
|
113
|
+
queryEngine: makeQueryEngine({
|
|
114
|
+
'test:product': [
|
|
115
|
+
{ id: 'p-a-1', organization_id: ORG_A },
|
|
116
|
+
{ id: 'p-b-1', organization_id: ORG_B },
|
|
117
|
+
],
|
|
118
|
+
'test:customer': [
|
|
119
|
+
{ id: 'c-a-1', organization_id: ORG_A },
|
|
120
|
+
{ id: 'c-b-1', organization_id: ORG_B },
|
|
121
|
+
],
|
|
122
|
+
}),
|
|
123
|
+
})
|
|
124
|
+
return { ...fake, driver, indexer }
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
describe('SearchIndexer fulltext reindex keeps other entities and organizations in the shared tenant index', () => {
|
|
128
|
+
it('reindexEntityToFulltext keeps other entity types and other organizations searchable', async () => {
|
|
129
|
+
const { indexer, driver, keys } = await setup()
|
|
130
|
+
|
|
131
|
+
const result = await indexer.reindexEntityToFulltext({
|
|
132
|
+
entityId: 'test:product',
|
|
133
|
+
tenantId: TENANT,
|
|
134
|
+
organizationId: ORG_A,
|
|
135
|
+
})
|
|
136
|
+
|
|
137
|
+
expect(result.success).toBe(true)
|
|
138
|
+
expect(keys(TENANT)).toEqual([
|
|
139
|
+
'test:customer:c-a-1',
|
|
140
|
+
'test:customer:c-b-1',
|
|
141
|
+
'test:product:p-a-1',
|
|
142
|
+
'test:product:p-b-1',
|
|
143
|
+
])
|
|
144
|
+
expect(driver.recreateIndex).not.toHaveBeenCalled()
|
|
145
|
+
expect(driver.purge).toHaveBeenCalledWith('test:product', TENANT, ORG_A)
|
|
146
|
+
})
|
|
147
|
+
|
|
148
|
+
it('reindexEntityToFulltext without an organization clears only the requested entity type', async () => {
|
|
149
|
+
const { indexer, driver, keys } = await setup()
|
|
150
|
+
|
|
151
|
+
const result = await indexer.reindexEntityToFulltext({ entityId: 'test:product', tenantId: TENANT })
|
|
152
|
+
|
|
153
|
+
expect(result.success).toBe(true)
|
|
154
|
+
expect(keys(TENANT)).toEqual([
|
|
155
|
+
'test:customer:c-a-1',
|
|
156
|
+
'test:customer:c-b-1',
|
|
157
|
+
'test:product:p-a-1',
|
|
158
|
+
'test:product:p-b-1',
|
|
159
|
+
])
|
|
160
|
+
expect(driver.recreateIndex).not.toHaveBeenCalled()
|
|
161
|
+
expect(driver.purge).toHaveBeenCalledWith('test:product', TENANT, null)
|
|
162
|
+
})
|
|
163
|
+
|
|
164
|
+
it('reindexEntityToFulltext with recreateIndex false clears nothing', async () => {
|
|
165
|
+
const { indexer, driver, keys } = await setup()
|
|
166
|
+
|
|
167
|
+
await indexer.reindexEntityToFulltext({
|
|
168
|
+
entityId: 'test:product',
|
|
169
|
+
tenantId: TENANT,
|
|
170
|
+
organizationId: ORG_A,
|
|
171
|
+
recreateIndex: false,
|
|
172
|
+
})
|
|
173
|
+
|
|
174
|
+
expect(driver.recreateIndex).not.toHaveBeenCalled()
|
|
175
|
+
expect(driver.purge).not.toHaveBeenCalled()
|
|
176
|
+
expect(keys(TENANT)).toContain('test:product:p-a-stale')
|
|
177
|
+
})
|
|
178
|
+
|
|
179
|
+
it('reindexAllToFulltext scoped to an organization keeps other organizations searchable', async () => {
|
|
180
|
+
const { indexer, driver, keys } = await setup()
|
|
181
|
+
|
|
182
|
+
const result = await indexer.reindexAllToFulltext({ tenantId: TENANT, organizationId: ORG_A })
|
|
183
|
+
|
|
184
|
+
expect(result.success).toBe(true)
|
|
185
|
+
expect(keys(TENANT)).toEqual([
|
|
186
|
+
'test:customer:c-a-1',
|
|
187
|
+
'test:customer:c-b-1',
|
|
188
|
+
'test:product:p-a-1',
|
|
189
|
+
'test:product:p-b-1',
|
|
190
|
+
])
|
|
191
|
+
expect(driver.recreateIndex).not.toHaveBeenCalled()
|
|
192
|
+
expect(driver.purge).toHaveBeenCalledWith('test:product', TENANT, ORG_A)
|
|
193
|
+
expect(driver.purge).toHaveBeenCalledWith('test:customer', TENANT, ORG_A)
|
|
194
|
+
})
|
|
195
|
+
|
|
196
|
+
it('reindexAllToFulltext without an organization still recreates the tenant index once', async () => {
|
|
197
|
+
const { indexer, driver, keys } = await setup()
|
|
198
|
+
|
|
199
|
+
const result = await indexer.reindexAllToFulltext({ tenantId: TENANT })
|
|
200
|
+
|
|
201
|
+
expect(result.success).toBe(true)
|
|
202
|
+
expect(keys(TENANT)).toEqual([
|
|
203
|
+
'test:customer:c-a-1',
|
|
204
|
+
'test:customer:c-b-1',
|
|
205
|
+
'test:product:p-a-1',
|
|
206
|
+
'test:product:p-b-1',
|
|
207
|
+
])
|
|
208
|
+
expect(driver.recreateIndex).toHaveBeenCalledTimes(1)
|
|
209
|
+
expect(driver.recreateIndex).toHaveBeenCalledWith(TENANT)
|
|
210
|
+
expect(driver.purge).not.toHaveBeenCalled()
|
|
211
|
+
})
|
|
212
|
+
|
|
213
|
+
it('reindexEntityToFulltext in queue mode clears the scoped documents before enqueueing and indexes nothing directly', async () => {
|
|
214
|
+
const order: string[] = []
|
|
215
|
+
const enqueue = jest.fn(async () => {
|
|
216
|
+
order.push('enqueue')
|
|
217
|
+
return 'job-1'
|
|
218
|
+
})
|
|
219
|
+
const { indexer, driver, keys } = await setup({ enqueue } as unknown as SearchIndexerOptions['fulltextQueue'])
|
|
220
|
+
const purge = driver.purge as jest.Mock
|
|
221
|
+
const purgeImpl = purge.getMockImplementation()!
|
|
222
|
+
purge.mockImplementation(async (...args: Parameters<typeof purgeImpl>) => {
|
|
223
|
+
order.push('purge')
|
|
224
|
+
return purgeImpl(...args)
|
|
225
|
+
})
|
|
226
|
+
|
|
227
|
+
const result = await indexer.reindexEntityToFulltext({
|
|
228
|
+
entityId: 'test:product',
|
|
229
|
+
tenantId: TENANT,
|
|
230
|
+
organizationId: ORG_A,
|
|
231
|
+
useQueue: true,
|
|
232
|
+
})
|
|
233
|
+
|
|
234
|
+
expect(result.success).toBe(true)
|
|
235
|
+
expect(order).toEqual(['purge', 'enqueue'])
|
|
236
|
+
expect(driver.recreateIndex).not.toHaveBeenCalled()
|
|
237
|
+
expect(driver.bulkIndex).not.toHaveBeenCalled()
|
|
238
|
+
expect(enqueue).toHaveBeenCalledWith(
|
|
239
|
+
expect.objectContaining({
|
|
240
|
+
jobType: 'batch-index',
|
|
241
|
+
tenantId: TENANT,
|
|
242
|
+
organizationId: ORG_A,
|
|
243
|
+
records: [{ entityId: 'test:product', recordId: 'p-a-1' }],
|
|
244
|
+
}),
|
|
245
|
+
)
|
|
246
|
+
expect(keys(TENANT)).toEqual(['test:customer:c-a-1', 'test:customer:c-b-1', 'test:product:p-b-1'])
|
|
247
|
+
})
|
|
248
|
+
|
|
249
|
+
it('reindexAllToFulltext scoped to an organization with recreateIndex false clears nothing', async () => {
|
|
250
|
+
const { indexer, driver, keys } = await setup()
|
|
251
|
+
|
|
252
|
+
const result = await indexer.reindexAllToFulltext({ tenantId: TENANT, organizationId: ORG_A, recreateIndex: false })
|
|
253
|
+
|
|
254
|
+
expect(result.success).toBe(true)
|
|
255
|
+
expect(driver.recreateIndex).not.toHaveBeenCalled()
|
|
256
|
+
expect(driver.purge).not.toHaveBeenCalled()
|
|
257
|
+
expect(keys(TENANT)).toContain('test:product:p-a-stale')
|
|
258
|
+
})
|
|
259
|
+
|
|
260
|
+
it('reindexEntityToFulltext reports a failed clear as an error without indexing', async () => {
|
|
261
|
+
const { indexer, driver } = await setup()
|
|
262
|
+
;(driver.purge as jest.Mock).mockRejectedValueOnce(new Error('purge unavailable'))
|
|
263
|
+
|
|
264
|
+
const result = await indexer.reindexEntityToFulltext({
|
|
265
|
+
entityId: 'test:product',
|
|
266
|
+
tenantId: TENANT,
|
|
267
|
+
organizationId: ORG_A,
|
|
268
|
+
})
|
|
269
|
+
|
|
270
|
+
expect(result.success).toBe(false)
|
|
271
|
+
expect(result.errors).toEqual([{ entityId: 'test:product', error: 'purge unavailable' }])
|
|
272
|
+
expect(driver.bulkIndex).not.toHaveBeenCalled()
|
|
273
|
+
})
|
|
274
|
+
|
|
275
|
+
it('reindexAllToFulltext scoped to an organization reports a failed clear per entity and keeps indexing the rest', async () => {
|
|
276
|
+
const { indexer, driver, keys } = await setup()
|
|
277
|
+
;(driver.purge as jest.Mock).mockRejectedValueOnce(new Error('purge unavailable'))
|
|
278
|
+
|
|
279
|
+
const result = await indexer.reindexAllToFulltext({ tenantId: TENANT, organizationId: ORG_A })
|
|
280
|
+
|
|
281
|
+
expect(result.success).toBe(false)
|
|
282
|
+
expect(result.errors).toEqual([{ entityId: 'test:product', error: 'purge unavailable' }])
|
|
283
|
+
expect(driver.recreateIndex).not.toHaveBeenCalled()
|
|
284
|
+
expect(keys(TENANT)).toEqual([
|
|
285
|
+
'test:customer:c-a-1',
|
|
286
|
+
'test:customer:c-b-1',
|
|
287
|
+
'test:product:p-a-stale',
|
|
288
|
+
'test:product:p-b-1',
|
|
289
|
+
])
|
|
290
|
+
})
|
|
291
|
+
})
|
|
292
|
+
|
|
293
|
+
describe('SearchIndexer fulltext reindex with a driver that does not implement purge', () => {
|
|
294
|
+
it('reindexEntityToFulltext clears the scoped documents through listDocumentIds and delete', async () => {
|
|
295
|
+
const { indexer, driver, keys } = await setup(undefined, (fake) => withoutPurge(fake, { listDocumentIds: true }))
|
|
296
|
+
|
|
297
|
+
const result = await indexer.reindexEntityToFulltext({
|
|
298
|
+
entityId: 'test:product',
|
|
299
|
+
tenantId: TENANT,
|
|
300
|
+
organizationId: ORG_A,
|
|
301
|
+
})
|
|
302
|
+
|
|
303
|
+
expect(result.success).toBe(true)
|
|
304
|
+
expect(keys(TENANT)).toEqual([
|
|
305
|
+
'test:customer:c-a-1',
|
|
306
|
+
'test:customer:c-b-1',
|
|
307
|
+
'test:product:p-a-1',
|
|
308
|
+
'test:product:p-b-1',
|
|
309
|
+
])
|
|
310
|
+
expect(driver.recreateIndex).not.toHaveBeenCalled()
|
|
311
|
+
expect(driver.listDocumentIds).toHaveBeenCalledWith('test:product', TENANT, { offset: 0, limit: 1000, organizationId: ORG_A })
|
|
312
|
+
expect(driver.delete).toHaveBeenCalledTimes(1)
|
|
313
|
+
expect(driver.delete).toHaveBeenCalledWith('p-a-stale', TENANT)
|
|
314
|
+
})
|
|
315
|
+
|
|
316
|
+
it('reindexAllToFulltext scoped to an organization clears only that organization through listDocumentIds', async () => {
|
|
317
|
+
const { indexer, driver, keys } = await setup(undefined, (fake) => withoutPurge(fake, { listDocumentIds: true }))
|
|
318
|
+
|
|
319
|
+
const result = await indexer.reindexAllToFulltext({ tenantId: TENANT, organizationId: ORG_A })
|
|
320
|
+
|
|
321
|
+
expect(result.success).toBe(true)
|
|
322
|
+
expect(keys(TENANT)).toEqual([
|
|
323
|
+
'test:customer:c-a-1',
|
|
324
|
+
'test:customer:c-b-1',
|
|
325
|
+
'test:product:p-a-1',
|
|
326
|
+
'test:product:p-b-1',
|
|
327
|
+
])
|
|
328
|
+
expect(driver.recreateIndex).not.toHaveBeenCalled()
|
|
329
|
+
})
|
|
330
|
+
|
|
331
|
+
it('reindexEntityToFulltext fails without indexing or recreating the tenant index when the driver cannot clear per entity', async () => {
|
|
332
|
+
const { indexer, driver, keys } = await setup(undefined, (fake) => withoutPurge(fake, { listDocumentIds: false }))
|
|
333
|
+
|
|
334
|
+
const result = await indexer.reindexEntityToFulltext({
|
|
335
|
+
entityId: 'test:product',
|
|
336
|
+
tenantId: TENANT,
|
|
337
|
+
organizationId: ORG_A,
|
|
338
|
+
})
|
|
339
|
+
|
|
340
|
+
expect(result.success).toBe(false)
|
|
341
|
+
expect(result.errors).toEqual([
|
|
342
|
+
{ entityId: 'test:product', error: expect.stringContaining('implement purge or listDocumentIds') },
|
|
343
|
+
])
|
|
344
|
+
expect(result.recordsIndexed).toBe(0)
|
|
345
|
+
expect(driver.recreateIndex).not.toHaveBeenCalled()
|
|
346
|
+
expect(driver.bulkIndex).not.toHaveBeenCalled()
|
|
347
|
+
expect(keys(TENANT)).toContain('test:product:p-b-1')
|
|
348
|
+
})
|
|
349
|
+
|
|
350
|
+
it('reindexAllToFulltext scoped to an organization fails once, up front, when the driver cannot clear per entity', async () => {
|
|
351
|
+
const enqueue = jest.fn(async () => 'job-1')
|
|
352
|
+
const { indexer, driver, keys } = await setup(
|
|
353
|
+
{ enqueue } as unknown as SearchIndexerOptions['fulltextQueue'],
|
|
354
|
+
(fake) => withoutPurge(fake, { listDocumentIds: false }),
|
|
355
|
+
)
|
|
356
|
+
|
|
357
|
+
const result = await indexer.reindexAllToFulltext({ tenantId: TENANT, organizationId: ORG_A, useQueue: true })
|
|
358
|
+
|
|
359
|
+
expect(result.success).toBe(false)
|
|
360
|
+
expect(result.errors).toEqual([
|
|
361
|
+
{ entityId: 'all', error: expect.stringContaining('implement purge or listDocumentIds') },
|
|
362
|
+
])
|
|
363
|
+
expect(result.entitiesProcessed).toBe(0)
|
|
364
|
+
expect(driver.recreateIndex).not.toHaveBeenCalled()
|
|
365
|
+
expect(driver.bulkIndex).not.toHaveBeenCalled()
|
|
366
|
+
expect(enqueue).not.toHaveBeenCalled()
|
|
367
|
+
expect(keys(TENANT)).toContain('test:product:p-b-1')
|
|
368
|
+
})
|
|
369
|
+
|
|
370
|
+
it('reindexAllToFulltext without an organization still recreates the tenant index when the driver has no purge', async () => {
|
|
371
|
+
const { indexer, driver } = await setup(undefined, (fake) => withoutPurge(fake, { listDocumentIds: false }))
|
|
372
|
+
|
|
373
|
+
const result = await indexer.reindexAllToFulltext({ tenantId: TENANT })
|
|
374
|
+
|
|
375
|
+
expect(result.success).toBe(true)
|
|
376
|
+
expect(driver.recreateIndex).toHaveBeenCalledTimes(1)
|
|
377
|
+
})
|
|
378
|
+
})
|
|
379
|
+
|
|
380
|
+
describe('FullTextSearchStrategy.purge fallback', () => {
|
|
381
|
+
it('pages through every listed document id before deleting', async () => {
|
|
382
|
+
const recordIds = Array.from({ length: 1005 }, (_, index) => `rec-${index}`)
|
|
383
|
+
const driver: FullTextSearchDriver = {
|
|
384
|
+
id: 'meilisearch',
|
|
385
|
+
ensureReady: jest.fn(async () => {}),
|
|
386
|
+
isHealthy: jest.fn(async () => true),
|
|
387
|
+
search: jest.fn(async () => []),
|
|
388
|
+
index: jest.fn(async () => {}),
|
|
389
|
+
delete: jest.fn(async () => {}),
|
|
390
|
+
listDocumentIds: jest.fn(async (_entityId: string, _tenantId: string, options?: ListDocumentIdsOptions) => {
|
|
391
|
+
const offset = options?.offset ?? 0
|
|
392
|
+
return recordIds.slice(offset, offset + (options?.limit ?? 20))
|
|
393
|
+
}),
|
|
394
|
+
}
|
|
395
|
+
const strategy = new FullTextSearchStrategy(driver)
|
|
396
|
+
|
|
397
|
+
expect(strategy.canPurge()).toBe(true)
|
|
398
|
+
await strategy.purge('test:product', TENANT, ORG_A)
|
|
399
|
+
|
|
400
|
+
expect(driver.listDocumentIds).toHaveBeenCalledTimes(2)
|
|
401
|
+
expect(driver.delete).toHaveBeenCalledTimes(1005)
|
|
402
|
+
})
|
|
403
|
+
|
|
404
|
+
it('stops paging when a driver ignores the offset', async () => {
|
|
405
|
+
const page = Array.from({ length: 1000 }, (_, index) => `rec-${index}`)
|
|
406
|
+
const driver: FullTextSearchDriver = {
|
|
407
|
+
id: 'meilisearch',
|
|
408
|
+
ensureReady: jest.fn(async () => {}),
|
|
409
|
+
isHealthy: jest.fn(async () => true),
|
|
410
|
+
search: jest.fn(async () => []),
|
|
411
|
+
index: jest.fn(async () => {}),
|
|
412
|
+
delete: jest.fn(async () => {}),
|
|
413
|
+
listDocumentIds: jest.fn(async () => page),
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
await new FullTextSearchStrategy(driver).purge('test:product', TENANT, null)
|
|
417
|
+
|
|
418
|
+
expect(driver.listDocumentIds).toHaveBeenCalledTimes(2)
|
|
419
|
+
expect(driver.delete).toHaveBeenCalledTimes(1000)
|
|
420
|
+
})
|
|
421
|
+
|
|
422
|
+
it('reports that a driver with neither purge nor listDocumentIds cannot purge', () => {
|
|
423
|
+
const driver: FullTextSearchDriver = {
|
|
424
|
+
id: 'meilisearch',
|
|
425
|
+
ensureReady: jest.fn(async () => {}),
|
|
426
|
+
isHealthy: jest.fn(async () => true),
|
|
427
|
+
search: jest.fn(async () => []),
|
|
428
|
+
index: jest.fn(async () => {}),
|
|
429
|
+
delete: jest.fn(async () => {}),
|
|
430
|
+
recreateIndex: jest.fn(async () => {}),
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
expect(new FullTextSearchStrategy(driver).canPurge()).toBe(false)
|
|
434
|
+
})
|
|
435
|
+
})
|
|
@@ -49,11 +49,13 @@ jest.mock('../modules/search/lib/embedding-config', () => ({
|
|
|
49
49
|
jest.mock('../modules/search/lib/reindex-lock', () => ({
|
|
50
50
|
updateReindexProgress: jest.fn().mockResolvedValue(undefined),
|
|
51
51
|
clearReindexLock: jest.fn().mockResolvedValue(undefined),
|
|
52
|
+
isReindexLockDrained: jest.requireActual('../modules/search/lib/reindex-lock').isReindexLockDrained,
|
|
52
53
|
}))
|
|
53
54
|
|
|
54
55
|
jest.mock('../modules/search/lib/reindex-progress', () => ({
|
|
55
56
|
hasActiveReindexProgress: jest.fn().mockResolvedValue(true),
|
|
56
57
|
incrementReindexProgress: jest.fn().mockResolvedValue(true),
|
|
58
|
+
completeReindexProgress: jest.fn().mockResolvedValue(undefined),
|
|
57
59
|
}))
|
|
58
60
|
|
|
59
61
|
import { handleVectorIndexJob } from '../modules/search/workers/vector-index.worker'
|
|
@@ -62,7 +64,11 @@ import {
|
|
|
62
64
|
handleFulltextIndexJob,
|
|
63
65
|
} from '../modules/search/workers/fulltext-index.worker'
|
|
64
66
|
import { updateReindexProgress, clearReindexLock } from '../modules/search/lib/reindex-lock'
|
|
65
|
-
import {
|
|
67
|
+
import {
|
|
68
|
+
completeReindexProgress,
|
|
69
|
+
hasActiveReindexProgress,
|
|
70
|
+
incrementReindexProgress,
|
|
71
|
+
} from '../modules/search/lib/reindex-progress'
|
|
66
72
|
import { refreshCoverageSnapshot } from '@open-mercato/core/modules/query_index/lib/coverage'
|
|
67
73
|
import { recordIndexerLog } from '@open-mercato/shared/lib/indexers/status-log'
|
|
68
74
|
|
|
@@ -560,6 +566,45 @@ describe('Fulltext Index Worker', () => {
|
|
|
560
566
|
expect(clearReindexLock).toHaveBeenCalledWith(mockDb, 'tenant-123', 'fulltext', 'org-456')
|
|
561
567
|
})
|
|
562
568
|
|
|
569
|
+
it.each([
|
|
570
|
+
{ label: 'drained', lockCounts: { processedCount: 3, totalCount: 3 }, settles: true },
|
|
571
|
+
{ label: 'still draining', lockCounts: { processedCount: 2, totalCount: 3 }, settles: false },
|
|
572
|
+
{ label: 'total not recorded yet', lockCounts: { processedCount: 2, totalCount: null }, settles: false },
|
|
573
|
+
])('settles a fulltext reindex from the lock counts only when the lock is $label', async ({ lockCounts, settles }) => {
|
|
574
|
+
;(incrementReindexProgress as jest.Mock).mockResolvedValueOnce(false)
|
|
575
|
+
;(updateReindexProgress as jest.Mock).mockResolvedValueOnce(lockCounts)
|
|
576
|
+
const containerWithProgress: HandlerContext = {
|
|
577
|
+
resolve: jest.fn((name: string) => {
|
|
578
|
+
if (name === 'searchStrategies') return [mockFulltextStrategy]
|
|
579
|
+
if (name === 'em') return mockEm
|
|
580
|
+
if (name === 'searchIndexer') return mockSearchIndexer
|
|
581
|
+
if (name === 'progressService') return { id: 'progress' }
|
|
582
|
+
throw new Error(`Unknown service: ${name}`)
|
|
583
|
+
}) as HandlerContext['resolve'],
|
|
584
|
+
}
|
|
585
|
+
const job = createMockJob<FulltextIndexJobPayload>({
|
|
586
|
+
jobType: 'batch-index',
|
|
587
|
+
tenantId: 'tenant-123',
|
|
588
|
+
organizationId: 'org-456',
|
|
589
|
+
records: [{ entityId: 'test:entity', recordId: 'rec-1' }],
|
|
590
|
+
})
|
|
591
|
+
|
|
592
|
+
await handleFulltextIndexJob(job, createMockJobContext(), containerWithProgress)
|
|
593
|
+
|
|
594
|
+
if (settles) {
|
|
595
|
+
expect(completeReindexProgress).toHaveBeenCalledWith(expect.objectContaining({
|
|
596
|
+
type: 'fulltext',
|
|
597
|
+
tenantId: 'tenant-123',
|
|
598
|
+
organizationId: 'org-456',
|
|
599
|
+
resultSummary: lockCounts,
|
|
600
|
+
}))
|
|
601
|
+
expect(clearReindexLock).toHaveBeenCalledWith(mockDb, 'tenant-123', 'fulltext', 'org-456')
|
|
602
|
+
} else {
|
|
603
|
+
expect(completeReindexProgress).not.toHaveBeenCalled()
|
|
604
|
+
expect(clearReindexLock).not.toHaveBeenCalled()
|
|
605
|
+
}
|
|
606
|
+
})
|
|
607
|
+
|
|
563
608
|
it('re-throws a failed fulltext batch write without advancing reindex progress so the queue retries it', async () => {
|
|
564
609
|
mockSearchIndexer.indexRecordsById.mockRejectedValueOnce(new Error('meilisearch unavailable'))
|
|
565
610
|
const records = [
|
|
@@ -107,6 +107,20 @@ function withoutExcludedCustomFields(
|
|
|
107
107
|
return filtered
|
|
108
108
|
}
|
|
109
109
|
|
|
110
|
+
/**
|
|
111
|
+
* Scoped reindexes clear only the reindexed entity's (or organization's) documents. A driver
|
|
112
|
+
* that can do neither `purge` nor `listDocumentIds` would keep deleted records, and recreating
|
|
113
|
+
* the whole tenant index instead would drop every other entity and organization, so such a
|
|
114
|
+
* reindex fails up front.
|
|
115
|
+
*/
|
|
116
|
+
function canScopedPurge(fulltext: FullTextSearchStrategy): boolean {
|
|
117
|
+
return typeof fulltext.canPurge !== 'function' || fulltext.canPurge()
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function scopedPurgeUnsupportedError(fulltext: FullTextSearchStrategy): string {
|
|
121
|
+
return `Fulltext driver "${fulltext.driverId}" cannot clear documents per entity (implement purge or listDocumentIds); reindex aborted to avoid keeping stale documents`
|
|
122
|
+
}
|
|
123
|
+
|
|
110
124
|
/**
|
|
111
125
|
* Parameters for indexing a record.
|
|
112
126
|
*/
|
|
@@ -160,7 +174,7 @@ export type ReindexEntityParams = {
|
|
|
160
174
|
entityId: EntityId
|
|
161
175
|
tenantId: string
|
|
162
176
|
organizationId?: string | null
|
|
163
|
-
/** Whether to
|
|
177
|
+
/** Whether to clear this entity's documents (scoped to `organizationId` when set) first (default: true) */
|
|
164
178
|
recreateIndex?: boolean
|
|
165
179
|
/** Callback for progress tracking */
|
|
166
180
|
onProgress?: (progress: ReindexProgress) => void
|
|
@@ -174,7 +188,7 @@ export type ReindexEntityParams = {
|
|
|
174
188
|
export type ReindexAllParams = {
|
|
175
189
|
tenantId: string
|
|
176
190
|
organizationId?: string | null
|
|
177
|
-
/** Whether to
|
|
191
|
+
/** Whether to clear indexed documents first (default: true): the whole tenant index, or only `organizationId`'s documents when set */
|
|
178
192
|
recreateIndex?: boolean
|
|
179
193
|
/** Callback for progress tracking */
|
|
180
194
|
onProgress?: (progress: ReindexProgress) => void
|
|
@@ -865,9 +879,13 @@ export class SearchIndexer {
|
|
|
865
879
|
processed: 0,
|
|
866
880
|
})
|
|
867
881
|
|
|
868
|
-
// Recreate index if requested (default: true)
|
|
869
882
|
if (params.recreateIndex !== false) {
|
|
870
|
-
|
|
883
|
+
if (!canScopedPurge(fulltext)) {
|
|
884
|
+
result.success = false
|
|
885
|
+
result.errors.push({ entityId: params.entityId, error: scopedPurgeUnsupportedError(fulltext) })
|
|
886
|
+
return result
|
|
887
|
+
}
|
|
888
|
+
await fulltext.purge(params.entityId, params.tenantId, params.organizationId ?? null)
|
|
871
889
|
}
|
|
872
890
|
|
|
873
891
|
// Fetch and index records with pagination
|
|
@@ -1006,10 +1024,16 @@ export class SearchIndexer {
|
|
|
1006
1024
|
return result
|
|
1007
1025
|
}
|
|
1008
1026
|
|
|
1009
|
-
|
|
1010
|
-
|
|
1027
|
+
const clearIndex = params.recreateIndex !== false
|
|
1028
|
+
const organizationScoped = typeof params.organizationId === 'string' && params.organizationId.trim().length > 0
|
|
1029
|
+
if (clearIndex && !organizationScoped) {
|
|
1011
1030
|
await fulltext.recreateIndex(params.tenantId)
|
|
1012
1031
|
}
|
|
1032
|
+
if (clearIndex && organizationScoped && !canScopedPurge(fulltext)) {
|
|
1033
|
+
result.success = false
|
|
1034
|
+
result.errors.push({ entityId: 'all' as EntityId, error: scopedPurgeUnsupportedError(fulltext) })
|
|
1035
|
+
return result
|
|
1036
|
+
}
|
|
1013
1037
|
|
|
1014
1038
|
const entities = this.listEnabledEntities()
|
|
1015
1039
|
|
|
@@ -1018,7 +1042,7 @@ export class SearchIndexer {
|
|
|
1018
1042
|
entityId,
|
|
1019
1043
|
tenantId: params.tenantId,
|
|
1020
1044
|
organizationId: params.organizationId,
|
|
1021
|
-
recreateIndex:
|
|
1045
|
+
recreateIndex: clearIndex && organizationScoped,
|
|
1022
1046
|
onProgress: params.onProgress,
|
|
1023
1047
|
useQueue: params.useQueue,
|
|
1024
1048
|
})
|