@open-mercato/search 0.10.1-develop.8231.1.ae76b1d2c5 → 0.10.1-develop.8258.1.62a94b74a2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/.turbo/turbo-build.log +1 -1
  2. package/dist/indexer/search-indexer.js +21 -4
  3. package/dist/indexer/search-indexer.js.map +2 -2
  4. package/dist/modules/search/__integration__/TC-SEARCH-016.spec.js +187 -0
  5. package/dist/modules/search/__integration__/TC-SEARCH-016.spec.js.map +7 -0
  6. package/dist/modules/search/__integration__/TC-SEARCH-017.spec.js +94 -0
  7. package/dist/modules/search/__integration__/TC-SEARCH-017.spec.js.map +7 -0
  8. package/dist/modules/search/__integration__/TC-SEARCH-018.spec.js +74 -0
  9. package/dist/modules/search/__integration__/TC-SEARCH-018.spec.js.map +7 -0
  10. package/dist/modules/search/ai-tools.js +3 -1
  11. package/dist/modules/search/ai-tools.js.map +2 -2
  12. package/dist/modules/search/api/openapi.js +1 -0
  13. package/dist/modules/search/api/openapi.js.map +2 -2
  14. package/dist/modules/search/api/reindex/route.js +35 -16
  15. package/dist/modules/search/api/reindex/route.js.map +2 -2
  16. package/dist/modules/search/cli.js +2 -16
  17. package/dist/modules/search/cli.js.map +2 -2
  18. package/dist/modules/search/frontend/components/sections/FulltextSearchSection.js +1 -1
  19. package/dist/modules/search/frontend/components/sections/FulltextSearchSection.js.map +2 -2
  20. package/dist/modules/search/i18n/de.json +2 -0
  21. package/dist/modules/search/i18n/en.json +2 -0
  22. package/dist/modules/search/i18n/es.json +2 -0
  23. package/dist/modules/search/i18n/ko.json +2 -0
  24. package/dist/modules/search/i18n/pl.json +2 -0
  25. package/dist/modules/search/lib/reindex-lock.js +29 -6
  26. package/dist/modules/search/lib/reindex-lock.js.map +2 -2
  27. package/dist/modules/search/lib/reindex-progress.js +28 -3
  28. package/dist/modules/search/lib/reindex-progress.js.map +2 -2
  29. package/dist/modules/search/workers/fulltext-index.worker.js +19 -6
  30. package/dist/modules/search/workers/fulltext-index.worker.js.map +2 -2
  31. package/dist/strategies/fulltext.strategy.js +33 -2
  32. package/dist/strategies/fulltext.strategy.js.map +2 -2
  33. package/package.json +4 -4
  34. package/src/__tests__/reindex-strategy-policy.test.ts +2 -0
  35. package/src/__tests__/search-indexer-fulltext-reindex-scope.test.ts +435 -0
  36. package/src/__tests__/workers.test.ts +46 -1
  37. package/src/indexer/search-indexer.ts +31 -7
  38. package/src/modules/search/__integration__/TC-SEARCH-016.spec.ts +247 -0
  39. package/src/modules/search/__integration__/TC-SEARCH-017.spec.ts +141 -0
  40. package/src/modules/search/__integration__/TC-SEARCH-018.spec.ts +114 -0
  41. package/src/modules/search/ai-tools.ts +3 -1
  42. package/src/modules/search/api/__tests__/reindex.routes.test.ts +222 -6
  43. package/src/modules/search/api/openapi.ts +1 -0
  44. package/src/modules/search/api/reindex/route.ts +39 -16
  45. package/src/modules/search/cli.ts +2 -16
  46. package/src/modules/search/frontend/components/sections/FulltextSearchSection.tsx +1 -1
  47. package/src/modules/search/frontend/components/sections/__tests__/FulltextSearchSection.test.tsx +112 -3
  48. package/src/modules/search/i18n/de.json +2 -0
  49. package/src/modules/search/i18n/en.json +2 -0
  50. package/src/modules/search/i18n/es.json +2 -0
  51. package/src/modules/search/i18n/ko.json +2 -0
  52. package/src/modules/search/i18n/pl.json +2 -0
  53. package/src/modules/search/lib/__tests__/reindex-progress.test.ts +157 -0
  54. package/src/modules/search/lib/reindex-lock.ts +66 -8
  55. package/src/modules/search/lib/reindex-progress.ts +55 -2
  56. package/src/modules/search/workers/fulltext-index.worker.ts +21 -6
  57. package/src/strategies/fulltext.strategy.ts +39 -2
@@ -0,0 +1,157 @@
1
+ import type { EntityManager } from '@mikro-orm/postgresql'
2
+ import type { ProgressJob } from '@open-mercato/core/modules/progress/data/entities'
3
+ import type { ProgressService } from '@open-mercato/core/modules/progress/lib/progressService'
4
+ import { startReindexProgressJob } from '../reindex-progress'
5
+
6
+ type JobStatus = 'pending' | 'running' | 'completed' | 'failed' | 'cancelled'
7
+
8
+ type FakeJob = {
9
+ id: string
10
+ status: JobStatus
11
+ errorMessage?: string
12
+ }
13
+
14
+ const TENANT_ID = '11111111-1111-4111-8111-111111111111'
15
+ const ORG_ID = '22222222-2222-4222-8222-222222222222'
16
+ const USER_ID = '33333333-3333-4333-8333-333333333333'
17
+ const SUPERSEDED = 'Superseded by a newer reindex.'
18
+
19
+ function createHarness(initialJobs: FakeJob[], options: { stuckJobIds?: string[] } = {}) {
20
+ const jobs = [...initialJobs]
21
+ const stuck = new Set(options.stuckJobIds ?? [])
22
+ let created = 0
23
+
24
+ const em = {
25
+ findOne: jest.fn(async () => {
26
+ const active = jobs.filter((job) => job.status === 'pending' || job.status === 'running')
27
+ return (active[active.length - 1] ?? null) as unknown as ProgressJob | null
28
+ }),
29
+ }
30
+
31
+ const progressService = {
32
+ failJob: jest.fn(async (jobId: string, input: { errorMessage: string }) => {
33
+ const job = jobs.find((entry) => entry.id === jobId)
34
+ if (!job) throw new Error(`[internal] job ${jobId} not found`)
35
+ if (!stuck.has(jobId)) {
36
+ job.status = 'failed'
37
+ job.errorMessage = input.errorMessage
38
+ }
39
+ return { ...job } as unknown as ProgressJob
40
+ }),
41
+ createJob: jest.fn(async () => {
42
+ created += 1
43
+ const job: FakeJob = { id: `new-job-${created}`, status: 'pending' }
44
+ jobs.push(job)
45
+ return { ...job } as unknown as ProgressJob
46
+ }),
47
+ startJob: jest.fn(async (jobId: string) => {
48
+ const job = jobs.find((entry) => entry.id === jobId)
49
+ if (job) job.status = 'running'
50
+ return { ...job } as unknown as ProgressJob
51
+ }),
52
+ }
53
+
54
+ const start = () =>
55
+ startReindexProgressJob({
56
+ em: em as unknown as EntityManager,
57
+ progressService: progressService as unknown as ProgressService,
58
+ type: 'fulltext',
59
+ tenantId: TENANT_ID,
60
+ organizationId: ORG_ID,
61
+ userId: USER_ID,
62
+ description: 'Reindex all entities (queued)',
63
+ supersededMessage: SUPERSEDED,
64
+ })
65
+
66
+ return { jobs, em, progressService, start }
67
+ }
68
+
69
+ describe('startReindexProgressJob', () => {
70
+ it('fails an active job left by a stale run and creates a new running job', async () => {
71
+ const harness = createHarness([{ id: 'orphan-job', status: 'running' }])
72
+
73
+ const jobId = await harness.start()
74
+
75
+ expect(harness.progressService.failJob).toHaveBeenCalledTimes(1)
76
+ expect(harness.progressService.failJob).toHaveBeenCalledWith(
77
+ 'orphan-job',
78
+ { errorMessage: SUPERSEDED },
79
+ { tenantId: TENANT_ID, organizationId: ORG_ID, userId: USER_ID },
80
+ )
81
+ expect(harness.jobs.find((job) => job.id === 'orphan-job')).toMatchObject({
82
+ status: 'failed',
83
+ errorMessage: SUPERSEDED,
84
+ })
85
+ expect(jobId).toBe('new-job-1')
86
+ expect(harness.progressService.createJob).toHaveBeenCalledTimes(1)
87
+ expect(harness.progressService.createJob).toHaveBeenCalledWith(
88
+ expect.objectContaining({ jobType: 'search.reindex.fulltext', cancellable: true }),
89
+ { tenantId: TENANT_ID, organizationId: ORG_ID, userId: USER_ID },
90
+ )
91
+ expect(harness.progressService.startJob).toHaveBeenCalledWith('new-job-1', {
92
+ tenantId: TENANT_ID,
93
+ organizationId: ORG_ID,
94
+ userId: USER_ID,
95
+ })
96
+ expect(harness.jobs.filter((job) => job.status === 'running').map((job) => job.id)).toEqual(['new-job-1'])
97
+ })
98
+
99
+ it('fails every active job in the scope before creating the new one', async () => {
100
+ const harness = createHarness([
101
+ { id: 'older-orphan', status: 'pending' },
102
+ { id: 'newer-orphan', status: 'running' },
103
+ ])
104
+
105
+ const jobId = await harness.start()
106
+
107
+ expect(harness.progressService.failJob.mock.calls.map(([id]) => id)).toEqual(['newer-orphan', 'older-orphan'])
108
+ expect(jobId).toBe('new-job-1')
109
+ expect(harness.progressService.createJob).toHaveBeenCalledTimes(1)
110
+ })
111
+
112
+ it('does not fail anything when no job is active', async () => {
113
+ const harness = createHarness([{ id: 'finished-job', status: 'completed' }])
114
+
115
+ const jobId = await harness.start()
116
+
117
+ expect(harness.progressService.failJob).not.toHaveBeenCalled()
118
+ expect(jobId).toBe('new-job-1')
119
+ expect(harness.progressService.createJob).toHaveBeenCalledTimes(1)
120
+ expect(harness.progressService.startJob).toHaveBeenCalledTimes(1)
121
+ })
122
+
123
+ it('stops retrying a job that cannot transition and creates exactly one new job', async () => {
124
+ const harness = createHarness([{ id: 'stuck-job', status: 'running' }], { stuckJobIds: ['stuck-job'] })
125
+
126
+ const jobId = await harness.start()
127
+
128
+ expect(harness.progressService.failJob).toHaveBeenCalledTimes(1)
129
+ expect(harness.progressService.failJob).toHaveBeenCalledWith(
130
+ 'stuck-job',
131
+ { errorMessage: SUPERSEDED },
132
+ { tenantId: TENANT_ID, organizationId: ORG_ID, userId: USER_ID },
133
+ )
134
+ expect(harness.progressService.createJob).toHaveBeenCalledTimes(1)
135
+ expect(harness.progressService.startJob).toHaveBeenCalledTimes(1)
136
+ expect(jobId).toBe('new-job-1')
137
+ expect(harness.jobs.filter((job) => job.id.startsWith('new-job-'))).toHaveLength(1)
138
+ })
139
+
140
+ it('returns null without reading progress jobs when the progress module is disabled', async () => {
141
+ const harness = createHarness([{ id: 'orphan-job', status: 'running' }])
142
+
143
+ const jobId = await startReindexProgressJob({
144
+ em: harness.em as unknown as EntityManager,
145
+ progressService: null,
146
+ type: 'fulltext',
147
+ tenantId: TENANT_ID,
148
+ organizationId: ORG_ID,
149
+ userId: USER_ID,
150
+ supersededMessage: SUPERSEDED,
151
+ })
152
+
153
+ expect(jobId).toBeNull()
154
+ expect(harness.em.findOne).not.toHaveBeenCalled()
155
+ expect(harness.jobs.find((job) => job.id === 'orphan-job')?.status).toBe('running')
156
+ })
157
+ })
@@ -29,6 +29,23 @@ export type ReindexLockStatus = {
29
29
  totalCount?: number | null
30
30
  }
31
31
 
32
+ export type ReindexLockCounts = {
33
+ processedCount: number
34
+ totalCount: number | null
35
+ }
36
+
37
+ type LockCountRow = {
38
+ processed_count?: number | string | null
39
+ total_count?: number | string | null
40
+ }
41
+
42
+ function toLockCounts(row: LockCountRow): ReindexLockCounts {
43
+ return {
44
+ processedCount: Number(row.processed_count ?? 0),
45
+ totalCount: row.total_count == null ? null : Number(row.total_count),
46
+ }
47
+ }
48
+
32
49
  function buildScope(
33
50
  type: ReindexLockType,
34
51
  tenantId: string,
@@ -187,14 +204,14 @@ export async function updateReindexProgress(
187
204
  type: ReindexLockType,
188
205
  processedDelta: number,
189
206
  organizationId?: string | null,
190
- ): Promise<void> {
207
+ ): Promise<ReindexLockCounts | null> {
191
208
  try {
192
209
  const scope = buildScope(type, tenantId, organizationId)
193
210
  const entityType = LOCK_ENTITY_TYPES[type]
194
211
  const delta = Math.max(0, processedDelta)
195
212
 
196
213
  // Try to update existing active job first
197
- const result = await db
214
+ const rows = await db
198
215
  .updateTable('entity_index_jobs' as any)
199
216
  .set({
200
217
  processed_count: sql`coalesce(processed_count, 0) + ${delta}`,
@@ -204,16 +221,57 @@ export async function updateReindexProgress(
204
221
  .where(sql<boolean>`tenant_id is not distinct from ${tenantId}`)
205
222
  .where(sql<boolean>`organization_id is not distinct from ${organizationId ?? null}`)
206
223
  .where('finished_at' as any, 'is', null)
207
- .executeTakeFirst()
224
+ .returning(['processed_count' as any, 'total_count' as any])
225
+ .execute() as LockCountRow[]
208
226
 
209
- // Kysely returns numUpdatedRows as bigint; coerce
210
- const updated = Number(result?.numUpdatedRows ?? 0)
227
+ if (rows.length > 0) return toLockCounts(rows[0])
211
228
 
212
229
  // If no active lock exists, recreate it
213
- if (updated === 0) {
214
- await prepareJob(db, scope, 'reindexing')
215
- }
230
+ await prepareJob(db, scope, 'reindexing')
231
+ return null
216
232
  } catch {
217
233
  // Ignore errors when updating progress
234
+ return null
235
+ }
236
+ }
237
+
238
+ /**
239
+ * Record the number of records a queued reindex enqueued on its active lock and return
240
+ * the counts workers have already reported. Workers can finish batches before the
241
+ * enqueueing request knows the total, so the caller uses the returned counts to settle
242
+ * a run whose batches all drained first. Returns null when the lock is no longer active
243
+ * (for example cancelled).
244
+ */
245
+ export async function setReindexLockTotal(
246
+ db: Kysely<any>,
247
+ tenantId: string,
248
+ type: ReindexLockType,
249
+ totalCount: number,
250
+ organizationId?: string | null,
251
+ ): Promise<ReindexLockCounts | null> {
252
+ try {
253
+ const rows = await db
254
+ .updateTable('entity_index_jobs' as any)
255
+ .set({
256
+ total_count: Math.max(0, totalCount),
257
+ heartbeat_at: sql`now()`,
258
+ } as any)
259
+ .where('entity_type' as any, '=', LOCK_ENTITY_TYPES[type])
260
+ .where(sql<boolean>`tenant_id is not distinct from ${tenantId}`)
261
+ .where(sql<boolean>`organization_id is not distinct from ${organizationId ?? null}`)
262
+ .where('finished_at' as any, 'is', null)
263
+ .returning(['processed_count' as any, 'total_count' as any])
264
+ .execute() as LockCountRow[]
265
+ return rows.length > 0 ? toLockCounts(rows[0]) : null
266
+ } catch {
267
+ return null
218
268
  }
219
269
  }
270
+
271
+ /**
272
+ * Whether the counts on a reindex lock show that every enqueued record was processed.
273
+ */
274
+ export function isReindexLockDrained(counts: ReindexLockCounts | null | undefined): boolean {
275
+ if (!counts || counts.totalCount == null) return false
276
+ return counts.processedCount >= counts.totalCount
277
+ }
@@ -56,11 +56,12 @@ export async function ensureReindexProgressJob(params: {
56
56
  totalCount?: number | null
57
57
  description?: string | null
58
58
  }): Promise<string | null> {
59
- if (!params.progressService) return null
59
+ const progressService = params.progressService
60
+ if (!progressService) return null
60
61
  const current = await findActiveJob(params.em, params.type, params.tenantId, params.organizationId)
61
62
  if (current) {
62
63
  if (typeof params.totalCount === 'number') {
63
- await params.progressService.updateProgress(
64
+ await progressService.updateProgress(
64
65
  current.id,
65
66
  { totalCount: params.totalCount },
66
67
  {
@@ -73,6 +74,58 @@ export async function ensureReindexProgressJob(params: {
73
74
  return current.id
74
75
  }
75
76
 
77
+ return createRunningJob({ ...params, progressService })
78
+ }
79
+
80
+ const MAX_SUPERSEDED_JOBS = 20
81
+
82
+ /**
83
+ * Start a fresh progress job for a reindex run that has just acquired its lock.
84
+ *
85
+ * Holding the lock means no other run in this scope is alive, so any job still marked
86
+ * active belongs to a run whose lock went stale; it is failed instead of adopted, so its
87
+ * processed count cannot leak into the new run's totals. Call this before enqueueing
88
+ * batches: workers drop the lock when they find no active job. `supersededMessage` is
89
+ * stored as the failed job's error message, so pass it already translated.
90
+ */
91
+ export async function startReindexProgressJob(params: {
92
+ em: EntityManager
93
+ progressService: ProgressService | null
94
+ type: ReindexProgressType
95
+ tenantId: string
96
+ organizationId?: string | null
97
+ userId?: string | null
98
+ description?: string | null
99
+ supersededMessage: string
100
+ }): Promise<string | null> {
101
+ const progressService = params.progressService
102
+ if (!progressService) return null
103
+ for (let attempt = 0; attempt < MAX_SUPERSEDED_JOBS; attempt += 1) {
104
+ const orphan = await findActiveJob(params.em, params.type, params.tenantId, params.organizationId)
105
+ if (!orphan) break
106
+ const settled = await progressService.failJob(
107
+ orphan.id,
108
+ { errorMessage: params.supersededMessage },
109
+ {
110
+ tenantId: params.tenantId,
111
+ organizationId: params.organizationId ?? null,
112
+ userId: params.userId ?? null,
113
+ },
114
+ )
115
+ if (settled.status === 'pending' || settled.status === 'running') break
116
+ }
117
+ return createRunningJob({ ...params, progressService })
118
+ }
119
+
120
+ async function createRunningJob(params: {
121
+ progressService: ProgressService
122
+ type: ReindexProgressType
123
+ tenantId: string
124
+ organizationId?: string | null
125
+ userId?: string | null
126
+ totalCount?: number | null
127
+ description?: string | null
128
+ }): Promise<string> {
76
129
  const config = REINDEX_JOB_CONFIG[params.type]
77
130
  const created = await params.progressService.createJob(
78
131
  {
@@ -14,8 +14,12 @@ import { recordIndexerLog } from '@open-mercato/shared/lib/indexers/status-log'
14
14
  import { recordIndexerError } from '@open-mercato/shared/lib/indexers/error-log'
15
15
  import type { ProgressService } from '@open-mercato/core/modules/progress/lib/progressService'
16
16
  import { searchDebug, searchDebugWarn, searchError, searchWarn } from '../../../lib/debug'
17
- import { clearReindexLock, updateReindexProgress } from '../lib/reindex-lock'
18
- import { hasActiveReindexProgress, incrementReindexProgress } from '../lib/reindex-progress'
17
+ import { clearReindexLock, isReindexLockDrained, updateReindexProgress } from '../lib/reindex-lock'
18
+ import {
19
+ completeReindexProgress,
20
+ hasActiveReindexProgress,
21
+ incrementReindexProgress,
22
+ } from '../lib/reindex-progress'
19
23
 
20
24
  // Worker metadata for auto-discovery
21
25
  const DEFAULT_CONCURRENCY = 2
@@ -53,11 +57,11 @@ async function advanceFulltextReindexProgress(params: {
53
57
  return
54
58
  }
55
59
 
56
- if (params.db) {
57
- await updateReindexProgress(params.db, params.tenantId, 'fulltext', params.delta, params.organizationId ?? null)
58
- }
60
+ const lockCounts = params.db
61
+ ? await updateReindexProgress(params.db, params.tenantId, 'fulltext', params.delta, params.organizationId ?? null)
62
+ : null
59
63
 
60
- const completed = await incrementReindexProgress({
64
+ let completed = await incrementReindexProgress({
61
65
  em: params.em,
62
66
  progressService: params.progressService,
63
67
  type: 'fulltext',
@@ -65,6 +69,17 @@ async function advanceFulltextReindexProgress(params: {
65
69
  organizationId: params.organizationId ?? null,
66
70
  delta: params.delta,
67
71
  })
72
+ if (!completed && lockCounts && isReindexLockDrained(lockCounts)) {
73
+ await completeReindexProgress({
74
+ em: params.em,
75
+ progressService: params.progressService,
76
+ type: 'fulltext',
77
+ tenantId: params.tenantId,
78
+ organizationId: params.organizationId ?? null,
79
+ resultSummary: { ...lockCounts },
80
+ })
81
+ completed = true
82
+ }
68
83
  if (completed && params.db) {
69
84
  await clearReindexLock(params.db, params.tenantId, 'fulltext', params.organizationId ?? null)
70
85
  }
@@ -78,11 +78,48 @@ export class FullTextSearchStrategy implements SearchStrategy {
78
78
  return this.driver.bulkIndex(docs)
79
79
  }
80
80
 
81
+ /**
82
+ * Whether one entity's documents (optionally one organization's) can be removed without
83
+ * touching the rest of the tenant index: natively through `driver.purge`, or by enumerating
84
+ * them with `driver.listDocumentIds` and deleting them one by one.
85
+ */
86
+ canPurge(): boolean {
87
+ return typeof this.driver.purge === 'function' || typeof this.driver.listDocumentIds === 'function'
88
+ }
89
+
81
90
  async purge(entityId: EntityId, tenantId: string, organizationId?: string | null): Promise<void> {
82
- if (!this.driver.purge) {
91
+ if (this.driver.purge) {
92
+ return this.driver.purge(entityId, tenantId, organizationId)
93
+ }
94
+ if (!this.driver.listDocumentIds) {
83
95
  return
84
96
  }
85
- return this.driver.purge(entityId, tenantId, organizationId)
97
+ const recordIds = await this.listAllDocumentIds(entityId, tenantId, organizationId ?? null)
98
+ for (const recordId of recordIds) {
99
+ await this.driver.delete(recordId, tenantId)
100
+ }
101
+ }
102
+
103
+ private async listAllDocumentIds(
104
+ entityId: EntityId,
105
+ tenantId: string,
106
+ organizationId: string | null,
107
+ ): Promise<string[]> {
108
+ const listDocumentIds = this.driver.listDocumentIds
109
+ if (!listDocumentIds) return []
110
+ const pageSize = 1000
111
+ const recordIds = new Set<string>()
112
+ for (let offset = 0; ; offset += pageSize) {
113
+ const page = await listDocumentIds.call(this.driver, entityId, tenantId, {
114
+ offset,
115
+ limit: pageSize,
116
+ organizationId,
117
+ })
118
+ const knownCount = recordIds.size
119
+ for (const recordId of page) recordIds.add(recordId)
120
+ if (page.length < pageSize || recordIds.size === knownCount) break
121
+ }
122
+ return [...recordIds]
86
123
  }
87
124
 
88
125
  // Additional methods exposed for enrichment and admin purposes