@open-mercato/search 0.10.1-develop.8231.1.ae76b1d2c5 → 0.10.1-develop.8258.1.62a94b74a2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/dist/indexer/search-indexer.js +21 -4
- package/dist/indexer/search-indexer.js.map +2 -2
- package/dist/modules/search/__integration__/TC-SEARCH-016.spec.js +187 -0
- package/dist/modules/search/__integration__/TC-SEARCH-016.spec.js.map +7 -0
- package/dist/modules/search/__integration__/TC-SEARCH-017.spec.js +94 -0
- package/dist/modules/search/__integration__/TC-SEARCH-017.spec.js.map +7 -0
- package/dist/modules/search/__integration__/TC-SEARCH-018.spec.js +74 -0
- package/dist/modules/search/__integration__/TC-SEARCH-018.spec.js.map +7 -0
- package/dist/modules/search/ai-tools.js +3 -1
- package/dist/modules/search/ai-tools.js.map +2 -2
- package/dist/modules/search/api/openapi.js +1 -0
- package/dist/modules/search/api/openapi.js.map +2 -2
- package/dist/modules/search/api/reindex/route.js +35 -16
- package/dist/modules/search/api/reindex/route.js.map +2 -2
- package/dist/modules/search/cli.js +2 -16
- package/dist/modules/search/cli.js.map +2 -2
- package/dist/modules/search/frontend/components/sections/FulltextSearchSection.js +1 -1
- package/dist/modules/search/frontend/components/sections/FulltextSearchSection.js.map +2 -2
- package/dist/modules/search/i18n/de.json +2 -0
- package/dist/modules/search/i18n/en.json +2 -0
- package/dist/modules/search/i18n/es.json +2 -0
- package/dist/modules/search/i18n/ko.json +2 -0
- package/dist/modules/search/i18n/pl.json +2 -0
- package/dist/modules/search/lib/reindex-lock.js +29 -6
- package/dist/modules/search/lib/reindex-lock.js.map +2 -2
- package/dist/modules/search/lib/reindex-progress.js +28 -3
- package/dist/modules/search/lib/reindex-progress.js.map +2 -2
- package/dist/modules/search/workers/fulltext-index.worker.js +19 -6
- package/dist/modules/search/workers/fulltext-index.worker.js.map +2 -2
- package/dist/strategies/fulltext.strategy.js +33 -2
- package/dist/strategies/fulltext.strategy.js.map +2 -2
- package/package.json +4 -4
- package/src/__tests__/reindex-strategy-policy.test.ts +2 -0
- package/src/__tests__/search-indexer-fulltext-reindex-scope.test.ts +435 -0
- package/src/__tests__/workers.test.ts +46 -1
- package/src/indexer/search-indexer.ts +31 -7
- package/src/modules/search/__integration__/TC-SEARCH-016.spec.ts +247 -0
- package/src/modules/search/__integration__/TC-SEARCH-017.spec.ts +141 -0
- package/src/modules/search/__integration__/TC-SEARCH-018.spec.ts +114 -0
- package/src/modules/search/ai-tools.ts +3 -1
- package/src/modules/search/api/__tests__/reindex.routes.test.ts +222 -6
- package/src/modules/search/api/openapi.ts +1 -0
- package/src/modules/search/api/reindex/route.ts +39 -16
- package/src/modules/search/cli.ts +2 -16
- package/src/modules/search/frontend/components/sections/FulltextSearchSection.tsx +1 -1
- package/src/modules/search/frontend/components/sections/__tests__/FulltextSearchSection.test.tsx +112 -3
- package/src/modules/search/i18n/de.json +2 -0
- package/src/modules/search/i18n/en.json +2 -0
- package/src/modules/search/i18n/es.json +2 -0
- package/src/modules/search/i18n/ko.json +2 -0
- package/src/modules/search/i18n/pl.json +2 -0
- package/src/modules/search/lib/__tests__/reindex-progress.test.ts +157 -0
- package/src/modules/search/lib/reindex-lock.ts +66 -8
- package/src/modules/search/lib/reindex-progress.ts +55 -2
- package/src/modules/search/workers/fulltext-index.worker.ts +21 -6
- package/src/strategies/fulltext.strategy.ts +39 -2
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
import type { EntityManager } from '@mikro-orm/postgresql'
|
|
2
|
+
import type { ProgressJob } from '@open-mercato/core/modules/progress/data/entities'
|
|
3
|
+
import type { ProgressService } from '@open-mercato/core/modules/progress/lib/progressService'
|
|
4
|
+
import { startReindexProgressJob } from '../reindex-progress'
|
|
5
|
+
|
|
6
|
+
type JobStatus = 'pending' | 'running' | 'completed' | 'failed' | 'cancelled'
|
|
7
|
+
|
|
8
|
+
type FakeJob = {
|
|
9
|
+
id: string
|
|
10
|
+
status: JobStatus
|
|
11
|
+
errorMessage?: string
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
const TENANT_ID = '11111111-1111-4111-8111-111111111111'
|
|
15
|
+
const ORG_ID = '22222222-2222-4222-8222-222222222222'
|
|
16
|
+
const USER_ID = '33333333-3333-4333-8333-333333333333'
|
|
17
|
+
const SUPERSEDED = 'Superseded by a newer reindex.'
|
|
18
|
+
|
|
19
|
+
function createHarness(initialJobs: FakeJob[], options: { stuckJobIds?: string[] } = {}) {
|
|
20
|
+
const jobs = [...initialJobs]
|
|
21
|
+
const stuck = new Set(options.stuckJobIds ?? [])
|
|
22
|
+
let created = 0
|
|
23
|
+
|
|
24
|
+
const em = {
|
|
25
|
+
findOne: jest.fn(async () => {
|
|
26
|
+
const active = jobs.filter((job) => job.status === 'pending' || job.status === 'running')
|
|
27
|
+
return (active[active.length - 1] ?? null) as unknown as ProgressJob | null
|
|
28
|
+
}),
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const progressService = {
|
|
32
|
+
failJob: jest.fn(async (jobId: string, input: { errorMessage: string }) => {
|
|
33
|
+
const job = jobs.find((entry) => entry.id === jobId)
|
|
34
|
+
if (!job) throw new Error(`[internal] job ${jobId} not found`)
|
|
35
|
+
if (!stuck.has(jobId)) {
|
|
36
|
+
job.status = 'failed'
|
|
37
|
+
job.errorMessage = input.errorMessage
|
|
38
|
+
}
|
|
39
|
+
return { ...job } as unknown as ProgressJob
|
|
40
|
+
}),
|
|
41
|
+
createJob: jest.fn(async () => {
|
|
42
|
+
created += 1
|
|
43
|
+
const job: FakeJob = { id: `new-job-${created}`, status: 'pending' }
|
|
44
|
+
jobs.push(job)
|
|
45
|
+
return { ...job } as unknown as ProgressJob
|
|
46
|
+
}),
|
|
47
|
+
startJob: jest.fn(async (jobId: string) => {
|
|
48
|
+
const job = jobs.find((entry) => entry.id === jobId)
|
|
49
|
+
if (job) job.status = 'running'
|
|
50
|
+
return { ...job } as unknown as ProgressJob
|
|
51
|
+
}),
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const start = () =>
|
|
55
|
+
startReindexProgressJob({
|
|
56
|
+
em: em as unknown as EntityManager,
|
|
57
|
+
progressService: progressService as unknown as ProgressService,
|
|
58
|
+
type: 'fulltext',
|
|
59
|
+
tenantId: TENANT_ID,
|
|
60
|
+
organizationId: ORG_ID,
|
|
61
|
+
userId: USER_ID,
|
|
62
|
+
description: 'Reindex all entities (queued)',
|
|
63
|
+
supersededMessage: SUPERSEDED,
|
|
64
|
+
})
|
|
65
|
+
|
|
66
|
+
return { jobs, em, progressService, start }
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
describe('startReindexProgressJob', () => {
|
|
70
|
+
it('fails an active job left by a stale run and creates a new running job', async () => {
|
|
71
|
+
const harness = createHarness([{ id: 'orphan-job', status: 'running' }])
|
|
72
|
+
|
|
73
|
+
const jobId = await harness.start()
|
|
74
|
+
|
|
75
|
+
expect(harness.progressService.failJob).toHaveBeenCalledTimes(1)
|
|
76
|
+
expect(harness.progressService.failJob).toHaveBeenCalledWith(
|
|
77
|
+
'orphan-job',
|
|
78
|
+
{ errorMessage: SUPERSEDED },
|
|
79
|
+
{ tenantId: TENANT_ID, organizationId: ORG_ID, userId: USER_ID },
|
|
80
|
+
)
|
|
81
|
+
expect(harness.jobs.find((job) => job.id === 'orphan-job')).toMatchObject({
|
|
82
|
+
status: 'failed',
|
|
83
|
+
errorMessage: SUPERSEDED,
|
|
84
|
+
})
|
|
85
|
+
expect(jobId).toBe('new-job-1')
|
|
86
|
+
expect(harness.progressService.createJob).toHaveBeenCalledTimes(1)
|
|
87
|
+
expect(harness.progressService.createJob).toHaveBeenCalledWith(
|
|
88
|
+
expect.objectContaining({ jobType: 'search.reindex.fulltext', cancellable: true }),
|
|
89
|
+
{ tenantId: TENANT_ID, organizationId: ORG_ID, userId: USER_ID },
|
|
90
|
+
)
|
|
91
|
+
expect(harness.progressService.startJob).toHaveBeenCalledWith('new-job-1', {
|
|
92
|
+
tenantId: TENANT_ID,
|
|
93
|
+
organizationId: ORG_ID,
|
|
94
|
+
userId: USER_ID,
|
|
95
|
+
})
|
|
96
|
+
expect(harness.jobs.filter((job) => job.status === 'running').map((job) => job.id)).toEqual(['new-job-1'])
|
|
97
|
+
})
|
|
98
|
+
|
|
99
|
+
it('fails every active job in the scope before creating the new one', async () => {
|
|
100
|
+
const harness = createHarness([
|
|
101
|
+
{ id: 'older-orphan', status: 'pending' },
|
|
102
|
+
{ id: 'newer-orphan', status: 'running' },
|
|
103
|
+
])
|
|
104
|
+
|
|
105
|
+
const jobId = await harness.start()
|
|
106
|
+
|
|
107
|
+
expect(harness.progressService.failJob.mock.calls.map(([id]) => id)).toEqual(['newer-orphan', 'older-orphan'])
|
|
108
|
+
expect(jobId).toBe('new-job-1')
|
|
109
|
+
expect(harness.progressService.createJob).toHaveBeenCalledTimes(1)
|
|
110
|
+
})
|
|
111
|
+
|
|
112
|
+
it('does not fail anything when no job is active', async () => {
|
|
113
|
+
const harness = createHarness([{ id: 'finished-job', status: 'completed' }])
|
|
114
|
+
|
|
115
|
+
const jobId = await harness.start()
|
|
116
|
+
|
|
117
|
+
expect(harness.progressService.failJob).not.toHaveBeenCalled()
|
|
118
|
+
expect(jobId).toBe('new-job-1')
|
|
119
|
+
expect(harness.progressService.createJob).toHaveBeenCalledTimes(1)
|
|
120
|
+
expect(harness.progressService.startJob).toHaveBeenCalledTimes(1)
|
|
121
|
+
})
|
|
122
|
+
|
|
123
|
+
it('stops retrying a job that cannot transition and creates exactly one new job', async () => {
|
|
124
|
+
const harness = createHarness([{ id: 'stuck-job', status: 'running' }], { stuckJobIds: ['stuck-job'] })
|
|
125
|
+
|
|
126
|
+
const jobId = await harness.start()
|
|
127
|
+
|
|
128
|
+
expect(harness.progressService.failJob).toHaveBeenCalledTimes(1)
|
|
129
|
+
expect(harness.progressService.failJob).toHaveBeenCalledWith(
|
|
130
|
+
'stuck-job',
|
|
131
|
+
{ errorMessage: SUPERSEDED },
|
|
132
|
+
{ tenantId: TENANT_ID, organizationId: ORG_ID, userId: USER_ID },
|
|
133
|
+
)
|
|
134
|
+
expect(harness.progressService.createJob).toHaveBeenCalledTimes(1)
|
|
135
|
+
expect(harness.progressService.startJob).toHaveBeenCalledTimes(1)
|
|
136
|
+
expect(jobId).toBe('new-job-1')
|
|
137
|
+
expect(harness.jobs.filter((job) => job.id.startsWith('new-job-'))).toHaveLength(1)
|
|
138
|
+
})
|
|
139
|
+
|
|
140
|
+
it('returns null without reading progress jobs when the progress module is disabled', async () => {
|
|
141
|
+
const harness = createHarness([{ id: 'orphan-job', status: 'running' }])
|
|
142
|
+
|
|
143
|
+
const jobId = await startReindexProgressJob({
|
|
144
|
+
em: harness.em as unknown as EntityManager,
|
|
145
|
+
progressService: null,
|
|
146
|
+
type: 'fulltext',
|
|
147
|
+
tenantId: TENANT_ID,
|
|
148
|
+
organizationId: ORG_ID,
|
|
149
|
+
userId: USER_ID,
|
|
150
|
+
supersededMessage: SUPERSEDED,
|
|
151
|
+
})
|
|
152
|
+
|
|
153
|
+
expect(jobId).toBeNull()
|
|
154
|
+
expect(harness.em.findOne).not.toHaveBeenCalled()
|
|
155
|
+
expect(harness.jobs.find((job) => job.id === 'orphan-job')?.status).toBe('running')
|
|
156
|
+
})
|
|
157
|
+
})
|
|
@@ -29,6 +29,23 @@ export type ReindexLockStatus = {
|
|
|
29
29
|
totalCount?: number | null
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
+
export type ReindexLockCounts = {
|
|
33
|
+
processedCount: number
|
|
34
|
+
totalCount: number | null
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
type LockCountRow = {
|
|
38
|
+
processed_count?: number | string | null
|
|
39
|
+
total_count?: number | string | null
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function toLockCounts(row: LockCountRow): ReindexLockCounts {
|
|
43
|
+
return {
|
|
44
|
+
processedCount: Number(row.processed_count ?? 0),
|
|
45
|
+
totalCount: row.total_count == null ? null : Number(row.total_count),
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
32
49
|
function buildScope(
|
|
33
50
|
type: ReindexLockType,
|
|
34
51
|
tenantId: string,
|
|
@@ -187,14 +204,14 @@ export async function updateReindexProgress(
|
|
|
187
204
|
type: ReindexLockType,
|
|
188
205
|
processedDelta: number,
|
|
189
206
|
organizationId?: string | null,
|
|
190
|
-
): Promise<
|
|
207
|
+
): Promise<ReindexLockCounts | null> {
|
|
191
208
|
try {
|
|
192
209
|
const scope = buildScope(type, tenantId, organizationId)
|
|
193
210
|
const entityType = LOCK_ENTITY_TYPES[type]
|
|
194
211
|
const delta = Math.max(0, processedDelta)
|
|
195
212
|
|
|
196
213
|
// Try to update existing active job first
|
|
197
|
-
const
|
|
214
|
+
const rows = await db
|
|
198
215
|
.updateTable('entity_index_jobs' as any)
|
|
199
216
|
.set({
|
|
200
217
|
processed_count: sql`coalesce(processed_count, 0) + ${delta}`,
|
|
@@ -204,16 +221,57 @@ export async function updateReindexProgress(
|
|
|
204
221
|
.where(sql<boolean>`tenant_id is not distinct from ${tenantId}`)
|
|
205
222
|
.where(sql<boolean>`organization_id is not distinct from ${organizationId ?? null}`)
|
|
206
223
|
.where('finished_at' as any, 'is', null)
|
|
207
|
-
.
|
|
224
|
+
.returning(['processed_count' as any, 'total_count' as any])
|
|
225
|
+
.execute() as LockCountRow[]
|
|
208
226
|
|
|
209
|
-
|
|
210
|
-
const updated = Number(result?.numUpdatedRows ?? 0)
|
|
227
|
+
if (rows.length > 0) return toLockCounts(rows[0])
|
|
211
228
|
|
|
212
229
|
// If no active lock exists, recreate it
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
}
|
|
230
|
+
await prepareJob(db, scope, 'reindexing')
|
|
231
|
+
return null
|
|
216
232
|
} catch {
|
|
217
233
|
// Ignore errors when updating progress
|
|
234
|
+
return null
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* Record the number of records a queued reindex enqueued on its active lock and return
|
|
240
|
+
* the counts workers have already reported. Workers can finish batches before the
|
|
241
|
+
* enqueueing request knows the total, so the caller uses the returned counts to settle
|
|
242
|
+
* a run whose batches all drained first. Returns null when the lock is no longer active
|
|
243
|
+
* (for example cancelled).
|
|
244
|
+
*/
|
|
245
|
+
export async function setReindexLockTotal(
|
|
246
|
+
db: Kysely<any>,
|
|
247
|
+
tenantId: string,
|
|
248
|
+
type: ReindexLockType,
|
|
249
|
+
totalCount: number,
|
|
250
|
+
organizationId?: string | null,
|
|
251
|
+
): Promise<ReindexLockCounts | null> {
|
|
252
|
+
try {
|
|
253
|
+
const rows = await db
|
|
254
|
+
.updateTable('entity_index_jobs' as any)
|
|
255
|
+
.set({
|
|
256
|
+
total_count: Math.max(0, totalCount),
|
|
257
|
+
heartbeat_at: sql`now()`,
|
|
258
|
+
} as any)
|
|
259
|
+
.where('entity_type' as any, '=', LOCK_ENTITY_TYPES[type])
|
|
260
|
+
.where(sql<boolean>`tenant_id is not distinct from ${tenantId}`)
|
|
261
|
+
.where(sql<boolean>`organization_id is not distinct from ${organizationId ?? null}`)
|
|
262
|
+
.where('finished_at' as any, 'is', null)
|
|
263
|
+
.returning(['processed_count' as any, 'total_count' as any])
|
|
264
|
+
.execute() as LockCountRow[]
|
|
265
|
+
return rows.length > 0 ? toLockCounts(rows[0]) : null
|
|
266
|
+
} catch {
|
|
267
|
+
return null
|
|
218
268
|
}
|
|
219
269
|
}
|
|
270
|
+
|
|
271
|
+
/**
|
|
272
|
+
* Whether the counts on a reindex lock show that every enqueued record was processed.
|
|
273
|
+
*/
|
|
274
|
+
export function isReindexLockDrained(counts: ReindexLockCounts | null | undefined): boolean {
|
|
275
|
+
if (!counts || counts.totalCount == null) return false
|
|
276
|
+
return counts.processedCount >= counts.totalCount
|
|
277
|
+
}
|
|
@@ -56,11 +56,12 @@ export async function ensureReindexProgressJob(params: {
|
|
|
56
56
|
totalCount?: number | null
|
|
57
57
|
description?: string | null
|
|
58
58
|
}): Promise<string | null> {
|
|
59
|
-
|
|
59
|
+
const progressService = params.progressService
|
|
60
|
+
if (!progressService) return null
|
|
60
61
|
const current = await findActiveJob(params.em, params.type, params.tenantId, params.organizationId)
|
|
61
62
|
if (current) {
|
|
62
63
|
if (typeof params.totalCount === 'number') {
|
|
63
|
-
await
|
|
64
|
+
await progressService.updateProgress(
|
|
64
65
|
current.id,
|
|
65
66
|
{ totalCount: params.totalCount },
|
|
66
67
|
{
|
|
@@ -73,6 +74,58 @@ export async function ensureReindexProgressJob(params: {
|
|
|
73
74
|
return current.id
|
|
74
75
|
}
|
|
75
76
|
|
|
77
|
+
return createRunningJob({ ...params, progressService })
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const MAX_SUPERSEDED_JOBS = 20
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Start a fresh progress job for a reindex run that has just acquired its lock.
|
|
84
|
+
*
|
|
85
|
+
* Holding the lock means no other run in this scope is alive, so any job still marked
|
|
86
|
+
* active belongs to a run whose lock went stale; it is failed instead of adopted, so its
|
|
87
|
+
* processed count cannot leak into the new run's totals. Call this before enqueueing
|
|
88
|
+
* batches: workers drop the lock when they find no active job. `supersededMessage` is
|
|
89
|
+
* stored as the failed job's error message, so pass it already translated.
|
|
90
|
+
*/
|
|
91
|
+
export async function startReindexProgressJob(params: {
|
|
92
|
+
em: EntityManager
|
|
93
|
+
progressService: ProgressService | null
|
|
94
|
+
type: ReindexProgressType
|
|
95
|
+
tenantId: string
|
|
96
|
+
organizationId?: string | null
|
|
97
|
+
userId?: string | null
|
|
98
|
+
description?: string | null
|
|
99
|
+
supersededMessage: string
|
|
100
|
+
}): Promise<string | null> {
|
|
101
|
+
const progressService = params.progressService
|
|
102
|
+
if (!progressService) return null
|
|
103
|
+
for (let attempt = 0; attempt < MAX_SUPERSEDED_JOBS; attempt += 1) {
|
|
104
|
+
const orphan = await findActiveJob(params.em, params.type, params.tenantId, params.organizationId)
|
|
105
|
+
if (!orphan) break
|
|
106
|
+
const settled = await progressService.failJob(
|
|
107
|
+
orphan.id,
|
|
108
|
+
{ errorMessage: params.supersededMessage },
|
|
109
|
+
{
|
|
110
|
+
tenantId: params.tenantId,
|
|
111
|
+
organizationId: params.organizationId ?? null,
|
|
112
|
+
userId: params.userId ?? null,
|
|
113
|
+
},
|
|
114
|
+
)
|
|
115
|
+
if (settled.status === 'pending' || settled.status === 'running') break
|
|
116
|
+
}
|
|
117
|
+
return createRunningJob({ ...params, progressService })
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
async function createRunningJob(params: {
|
|
121
|
+
progressService: ProgressService
|
|
122
|
+
type: ReindexProgressType
|
|
123
|
+
tenantId: string
|
|
124
|
+
organizationId?: string | null
|
|
125
|
+
userId?: string | null
|
|
126
|
+
totalCount?: number | null
|
|
127
|
+
description?: string | null
|
|
128
|
+
}): Promise<string> {
|
|
76
129
|
const config = REINDEX_JOB_CONFIG[params.type]
|
|
77
130
|
const created = await params.progressService.createJob(
|
|
78
131
|
{
|
|
@@ -14,8 +14,12 @@ import { recordIndexerLog } from '@open-mercato/shared/lib/indexers/status-log'
|
|
|
14
14
|
import { recordIndexerError } from '@open-mercato/shared/lib/indexers/error-log'
|
|
15
15
|
import type { ProgressService } from '@open-mercato/core/modules/progress/lib/progressService'
|
|
16
16
|
import { searchDebug, searchDebugWarn, searchError, searchWarn } from '../../../lib/debug'
|
|
17
|
-
import { clearReindexLock, updateReindexProgress } from '../lib/reindex-lock'
|
|
18
|
-
import {
|
|
17
|
+
import { clearReindexLock, isReindexLockDrained, updateReindexProgress } from '../lib/reindex-lock'
|
|
18
|
+
import {
|
|
19
|
+
completeReindexProgress,
|
|
20
|
+
hasActiveReindexProgress,
|
|
21
|
+
incrementReindexProgress,
|
|
22
|
+
} from '../lib/reindex-progress'
|
|
19
23
|
|
|
20
24
|
// Worker metadata for auto-discovery
|
|
21
25
|
const DEFAULT_CONCURRENCY = 2
|
|
@@ -53,11 +57,11 @@ async function advanceFulltextReindexProgress(params: {
|
|
|
53
57
|
return
|
|
54
58
|
}
|
|
55
59
|
|
|
56
|
-
|
|
57
|
-
await updateReindexProgress(params.db, params.tenantId, 'fulltext', params.delta, params.organizationId ?? null)
|
|
58
|
-
|
|
60
|
+
const lockCounts = params.db
|
|
61
|
+
? await updateReindexProgress(params.db, params.tenantId, 'fulltext', params.delta, params.organizationId ?? null)
|
|
62
|
+
: null
|
|
59
63
|
|
|
60
|
-
|
|
64
|
+
let completed = await incrementReindexProgress({
|
|
61
65
|
em: params.em,
|
|
62
66
|
progressService: params.progressService,
|
|
63
67
|
type: 'fulltext',
|
|
@@ -65,6 +69,17 @@ async function advanceFulltextReindexProgress(params: {
|
|
|
65
69
|
organizationId: params.organizationId ?? null,
|
|
66
70
|
delta: params.delta,
|
|
67
71
|
})
|
|
72
|
+
if (!completed && lockCounts && isReindexLockDrained(lockCounts)) {
|
|
73
|
+
await completeReindexProgress({
|
|
74
|
+
em: params.em,
|
|
75
|
+
progressService: params.progressService,
|
|
76
|
+
type: 'fulltext',
|
|
77
|
+
tenantId: params.tenantId,
|
|
78
|
+
organizationId: params.organizationId ?? null,
|
|
79
|
+
resultSummary: { ...lockCounts },
|
|
80
|
+
})
|
|
81
|
+
completed = true
|
|
82
|
+
}
|
|
68
83
|
if (completed && params.db) {
|
|
69
84
|
await clearReindexLock(params.db, params.tenantId, 'fulltext', params.organizationId ?? null)
|
|
70
85
|
}
|
|
@@ -78,11 +78,48 @@ export class FullTextSearchStrategy implements SearchStrategy {
|
|
|
78
78
|
return this.driver.bulkIndex(docs)
|
|
79
79
|
}
|
|
80
80
|
|
|
81
|
+
/**
|
|
82
|
+
* Whether one entity's documents (optionally one organization's) can be removed without
|
|
83
|
+
* touching the rest of the tenant index: natively through `driver.purge`, or by enumerating
|
|
84
|
+
* them with `driver.listDocumentIds` and deleting them one by one.
|
|
85
|
+
*/
|
|
86
|
+
canPurge(): boolean {
|
|
87
|
+
return typeof this.driver.purge === 'function' || typeof this.driver.listDocumentIds === 'function'
|
|
88
|
+
}
|
|
89
|
+
|
|
81
90
|
async purge(entityId: EntityId, tenantId: string, organizationId?: string | null): Promise<void> {
|
|
82
|
-
if (
|
|
91
|
+
if (this.driver.purge) {
|
|
92
|
+
return this.driver.purge(entityId, tenantId, organizationId)
|
|
93
|
+
}
|
|
94
|
+
if (!this.driver.listDocumentIds) {
|
|
83
95
|
return
|
|
84
96
|
}
|
|
85
|
-
|
|
97
|
+
const recordIds = await this.listAllDocumentIds(entityId, tenantId, organizationId ?? null)
|
|
98
|
+
for (const recordId of recordIds) {
|
|
99
|
+
await this.driver.delete(recordId, tenantId)
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
private async listAllDocumentIds(
|
|
104
|
+
entityId: EntityId,
|
|
105
|
+
tenantId: string,
|
|
106
|
+
organizationId: string | null,
|
|
107
|
+
): Promise<string[]> {
|
|
108
|
+
const listDocumentIds = this.driver.listDocumentIds
|
|
109
|
+
if (!listDocumentIds) return []
|
|
110
|
+
const pageSize = 1000
|
|
111
|
+
const recordIds = new Set<string>()
|
|
112
|
+
for (let offset = 0; ; offset += pageSize) {
|
|
113
|
+
const page = await listDocumentIds.call(this.driver, entityId, tenantId, {
|
|
114
|
+
offset,
|
|
115
|
+
limit: pageSize,
|
|
116
|
+
organizationId,
|
|
117
|
+
})
|
|
118
|
+
const knownCount = recordIds.size
|
|
119
|
+
for (const recordId of page) recordIds.add(recordId)
|
|
120
|
+
if (page.length < pageSize || recordIds.size === knownCount) break
|
|
121
|
+
}
|
|
122
|
+
return [...recordIds]
|
|
86
123
|
}
|
|
87
124
|
|
|
88
125
|
// Additional methods exposed for enrichment and admin purposes
|