@open-mercato/search 0.10.1-develop.8130.1.8964ae359f → 0.10.1-develop.8143.1.bf633ea932

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,217 @@
1
+ import { registerSearchModuleConfigs } from '@open-mercato/shared/modules/search'
2
+ import type { QueryEngine } from '@open-mercato/shared/lib/query/types'
3
+ import type { Queue, QueuedJob } from '@open-mercato/queue'
4
+ import { SearchIndexer } from '../indexer/search-indexer'
5
+ import { SearchService } from '../service'
6
+ import type { SearchModuleConfig, SearchStrategy } from '../types'
7
+ import type { FulltextIndexJobPayload } from '../queue/fulltext-indexing'
8
+ import type { VectorIndexJobPayload } from '../queue/vector-indexing'
9
+ import { VectorIndexService } from '../vector/services/vector-index.service'
10
+ import type { EmbeddingService } from '../vector/services/embedding'
11
+ import type { VectorDriver } from '../vector/types'
12
+ import { resolveEntityIndexStrategies } from '../lib/entity-strategy-policy'
13
+
14
+ import { handleFulltextIndexJob } from '../modules/search/workers/fulltext-index.worker'
15
+ import { handleVectorIndexJob } from '../modules/search/workers/vector-index.worker'
16
+
17
+ jest.mock('@open-mercato/shared/lib/indexers/status-log', () => ({
18
+ recordIndexerLog: jest.fn().mockResolvedValue(undefined),
19
+ }))
20
+ jest.mock('../vector/lib/vector-logs', () => ({
21
+ logVectorOperation: jest.fn().mockResolvedValue(undefined),
22
+ }))
23
+ jest.mock('../modules/search/lib/reindex-progress', () => ({
24
+ hasActiveReindexProgress: jest.fn().mockResolvedValue(false),
25
+ incrementReindexProgress: jest.fn().mockResolvedValue(false),
26
+ }))
27
+
28
+ const entityId = 'test:entity'
29
+ const tenantId = 'tenant-1'
30
+ const organizationId = 'org-1'
31
+
32
+ function strategy(id: string) {
33
+ return {
34
+ id,
35
+ name: id,
36
+ priority: 1,
37
+ isAvailable: jest.fn().mockResolvedValue(true),
38
+ ensureReady: jest.fn().mockResolvedValue(undefined),
39
+ search: jest.fn().mockResolvedValue([]),
40
+ index: jest.fn().mockResolvedValue(undefined),
41
+ bulkIndex: jest.fn().mockResolvedValue(undefined),
42
+ delete: jest.fn().mockResolvedValue(undefined),
43
+ recreateIndex: jest.fn().mockResolvedValue(undefined),
44
+ } satisfies SearchStrategy & { recreateIndex: () => Promise<void> }
45
+ }
46
+
47
+ function fixture(moduleConfigs: SearchModuleConfig[]) {
48
+ registerSearchModuleConfigs(moduleConfigs)
49
+ const fulltext = strategy('fulltext')
50
+ const vector = strategy('vector')
51
+ const searchService = new SearchService({ strategies: [fulltext, vector] })
52
+ const query = jest.fn().mockResolvedValue({ items: [{ id: 'record-1', name: 'Example' }], total: 1 })
53
+ const queryEngine: QueryEngine = { query }
54
+ const fulltextEnqueue = jest.fn().mockResolvedValue(undefined)
55
+ const vectorEnqueue = jest.fn().mockResolvedValue(undefined)
56
+ const indexer = new SearchIndexer(searchService, moduleConfigs, {
57
+ queryEngine,
58
+ fulltextQueue: { enqueue: fulltextEnqueue } as unknown as Queue<FulltextIndexJobPayload>,
59
+ vectorQueue: { enqueue: vectorEnqueue } as unknown as Queue<VectorIndexJobPayload>,
60
+ })
61
+ return { indexer, fulltext, vector, query, fulltextEnqueue, vectorEnqueue }
62
+ }
63
+
64
+ afterEach(() => registerSearchModuleConfigs([]))
65
+
66
+ const excludedPolicies: Array<[string, SearchModuleConfig[]]> = [
67
+ ['entity restriction', [{ entities: [{ entityId, strategies: ['tokens'] }] }]],
68
+ ['empty entity strategies', [{ entities: [{ entityId, strategies: [] }] }]],
69
+ ['module default', [{ defaultStrategies: ['tokens'], entities: [{ entityId }] }]],
70
+ ['disabled entity', [{ entities: [{ entityId, enabled: false }] }]],
71
+ ]
72
+
73
+ describe.each(excludedPolicies)('%s', (_name, moduleConfigs) => {
74
+ it.each([true, false])('blocks fulltext reindex before reading or queueing data (queue=%s)', async (useQueue) => {
75
+ const state = fixture(moduleConfigs)
76
+ const result = await state.indexer.reindexEntityToFulltext({ entityId, tenantId, organizationId, useQueue })
77
+ expect(result.recordsIndexed).toBe(0)
78
+ expect(state.query).not.toHaveBeenCalled()
79
+ expect(state.fulltextEnqueue).not.toHaveBeenCalled()
80
+ expect(state.fulltext.bulkIndex).not.toHaveBeenCalled()
81
+ expect(state.fulltext.recreateIndex).not.toHaveBeenCalled()
82
+ expect(state.vector.index).not.toHaveBeenCalled()
83
+ })
84
+
85
+ it.each([true, false])('blocks vector reindex before reading or queueing data (queue=%s)', async (useQueue) => {
86
+ const state = fixture(moduleConfigs)
87
+ const result = await state.indexer.reindexEntityToVector({ entityId, tenantId, organizationId, useQueue })
88
+ expect(result.recordsIndexed).toBe(0)
89
+ expect(state.query).not.toHaveBeenCalled()
90
+ expect(state.vectorEnqueue).not.toHaveBeenCalled()
91
+ expect(state.fulltext.index).not.toHaveBeenCalled()
92
+ expect(state.vector.index).not.toHaveBeenCalled()
93
+ })
94
+
95
+ it.each(['index', 'batch-index'] as const)('enforces policy through real queue handlers (%s)', async (jobType) => {
96
+ const state = fixture(moduleConfigs)
97
+ const consumeRecord = jest.spyOn(state.indexer, 'indexRecordById')
98
+ const context = {
99
+ resolve<T>(name: string): T {
100
+ if (name === 'searchIndexer') return state.indexer as T
101
+ if (name === 'searchStrategies') return [state.fulltext, state.vector] as T
102
+ throw new Error('[internal] Service unavailable in worker fixture')
103
+ },
104
+ }
105
+ const payload = jobType === 'index'
106
+ ? { jobType, entityType: entityId, recordId: 'record-1', tenantId, organizationId }
107
+ : { jobType, records: [{ entityId, recordId: 'record-1' }], tenantId, organizationId }
108
+ const job: QueuedJob<FulltextIndexJobPayload & VectorIndexJobPayload> = {
109
+ id: 'job-1', payload, createdAt: new Date().toISOString(),
110
+ }
111
+ const jobContext = { jobId: job.id, attemptNumber: 1, queueName: 'test-queue' }
112
+ await handleFulltextIndexJob(job, jobContext, context)
113
+ await handleVectorIndexJob(job, jobContext, context)
114
+ expect(consumeRecord).toHaveBeenCalled()
115
+ expect(state.fulltext.index).not.toHaveBeenCalled()
116
+ expect(state.fulltext.bulkIndex).not.toHaveBeenCalled()
117
+ expect(state.vector.index).not.toHaveBeenCalled()
118
+ expect(state.vector.bulkIndex).not.toHaveBeenCalled()
119
+ })
120
+
121
+ it('blocks excluded backends during general single and all-entity reindexing', async () => {
122
+ const state = fixture(moduleConfigs)
123
+ await state.indexer.reindexEntity({ entityId, tenantId, organizationId })
124
+ await state.indexer.reindexAll({ tenantId, organizationId })
125
+ expect(state.fulltext.bulkIndex).not.toHaveBeenCalled()
126
+ expect(state.vector.bulkIndex).not.toHaveBeenCalled()
127
+ })
128
+
129
+ it('rechecks policy when queued single and batch references are consumed', async () => {
130
+ const state = fixture(moduleConfigs)
131
+ await state.indexer.indexRecordById({ entityId, recordId: 'record-1', tenantId, organizationId })
132
+ await state.indexer.indexRecordsById({ items: [{ entityId, recordId: 'record-1' }], tenantId, organizationId })
133
+ expect(state.fulltext.index).not.toHaveBeenCalled()
134
+ expect(state.fulltext.bulkIndex).not.toHaveBeenCalled()
135
+ expect(state.vector.index).not.toHaveBeenCalled()
136
+ expect(state.vector.bulkIndex).not.toHaveBeenCalled()
137
+ })
138
+ })
139
+
140
+ it.each([true, false])('keeps allowed fulltext reindex working (queue=%s)', async (useQueue) => {
141
+ const state = fixture([{ defaultStrategies: ['tokens'], entities: [{ entityId, strategies: ['fulltext'] }] }])
142
+ const result = await state.indexer.reindexEntityToFulltext({ entityId, tenantId, organizationId, useQueue })
143
+ expect(result.recordsIndexed).toBe(1)
144
+ if (useQueue) expect(state.fulltextEnqueue).toHaveBeenCalledTimes(1)
145
+ else expect(state.fulltext.bulkIndex).toHaveBeenCalledWith([expect.objectContaining({ entityId, tenantId, organizationId })])
146
+ expect(state.vector.index).not.toHaveBeenCalled()
147
+ })
148
+
149
+ it.each([true, false])('keeps allowed vector reindex working (queue=%s)', async (useQueue) => {
150
+ const state = fixture([{ entities: [{ entityId, strategies: ['vector'] }] }])
151
+ const result = await state.indexer.reindexEntityToVector({ entityId, tenantId, organizationId, useQueue })
152
+ expect(result.recordsIndexed).toBe(1)
153
+ if (useQueue) expect(state.vectorEnqueue).toHaveBeenCalledTimes(1)
154
+ else expect(state.vector.index).toHaveBeenCalledWith(expect.objectContaining({ entityId, tenantId, organizationId }))
155
+ expect(state.fulltext.index).not.toHaveBeenCalled()
156
+ })
157
+
158
+ it('resolves the last declaration and keeps unspecified configurations unrestricted', () => {
159
+ expect(resolveEntityIndexStrategies(entityId, [])).toBeUndefined()
160
+ expect(resolveEntityIndexStrategies(entityId, [{ entities: [{ entityId, strategies: [] }] }, { entities: [{ entityId }] }])).toBeUndefined()
161
+ expect(resolveEntityIndexStrategies(entityId, [{ defaultStrategies: ['fulltext'], entities: [{ entityId, enabled: false }] }])).toEqual([])
162
+ })
163
+
164
+ it.each([true, false])('keeps excluded payloads out of every backend when reindexing all entities (queue=%s)', async (useQueue) => {
165
+ const state = fixture([{ entities: [{ entityId, strategies: ['tokens'] }] }])
166
+ await state.indexer.reindexAllToFulltext({ tenantId, organizationId, useQueue })
167
+ await state.indexer.reindexAllToVector({ tenantId, organizationId, useQueue })
168
+ expect(state.query).not.toHaveBeenCalled()
169
+ expect(state.fulltextEnqueue).not.toHaveBeenCalled()
170
+ expect(state.vectorEnqueue).not.toHaveBeenCalled()
171
+ expect(state.fulltext.bulkIndex).not.toHaveBeenCalled()
172
+ expect(state.vector.index).not.toHaveBeenCalled()
173
+ })
174
+
175
+ it('applies updated policy when previously queued references are consumed', async () => {
176
+ const configs: SearchModuleConfig[] = [{ entities: [{ entityId, strategies: ['fulltext', 'vector'] }] }]
177
+ const state = fixture(configs)
178
+ await state.indexer.reindexEntityToFulltext({ entityId, tenantId, organizationId, useQueue: true })
179
+ const payload = state.fulltextEnqueue.mock.calls[0][0] as FulltextIndexJobPayload
180
+ registerSearchModuleConfigs([{ entities: [{ entityId, strategies: [] }] }])
181
+ await state.indexer.indexRecordsById({ items: payload.records ?? [], tenantId, organizationId })
182
+ expect(state.fulltext.bulkIndex).not.toHaveBeenCalled()
183
+ expect(state.vector.bulkIndex).not.toHaveBeenCalled()
184
+ })
185
+
186
+ it.each(['single', 'reindex'])('blocks legacy vector writes before source callbacks or embedding (%s)', async (mode) => {
187
+ registerSearchModuleConfigs([{ entities: [{ entityId, strategies: ['tokens'] }] }])
188
+ const buildSource = jest.fn().mockResolvedValue({ text: ['Example'] })
189
+ const createEmbedding = jest.fn().mockResolvedValue([1, 2, 3])
190
+ const upsert = jest.fn().mockResolvedValue(undefined)
191
+ const driver = {
192
+ id: 'pgvector',
193
+ ensureReady: jest.fn().mockResolvedValue(undefined),
194
+ getChecksum: jest.fn().mockResolvedValue(null),
195
+ upsert,
196
+ } as unknown as VectorDriver
197
+ let queried = false
198
+ const queryEngine: QueryEngine = { query: jest.fn(async () => {
199
+ const items = queried ? [] : [{ id: 'record-1', name: 'Example' }]
200
+ queried = true
201
+ return { items, total: items.length }
202
+ }) }
203
+ const service = new VectorIndexService({
204
+ drivers: [driver],
205
+ embeddingService: { available: true, createEmbedding } as unknown as EmbeddingService,
206
+ queryEngine,
207
+ moduleConfigs: [{ entities: [{ entityId, buildSource }] }],
208
+ })
209
+ if (mode === 'single') {
210
+ expect(await service.indexRecord({ entityId, recordId: 'record-1', tenantId, organizationId })).toMatchObject({ action: 'skipped' })
211
+ } else {
212
+ await service.reindexEntity({ entityId, tenantId, organizationId })
213
+ }
214
+ expect(buildSource).not.toHaveBeenCalled()
215
+ expect(createEmbedding).not.toHaveBeenCalled()
216
+ expect(upsert).not.toHaveBeenCalled()
217
+ })
package/src/di.ts CHANGED
@@ -1,3 +1,4 @@
1
+ import { resolveEntityIndexStrategies } from './lib/entity-strategy-policy'
1
2
  import { asValue } from 'awilix'
2
3
  import type { Kysely } from 'kysely'
3
4
  import type { FullTextSearchDriver } from './fulltext/types'
@@ -228,7 +229,7 @@ export function registerSearchModule(
228
229
  // Create search service
229
230
  const searchService = new SearchService({
230
231
  strategies,
231
- resolveEntityStrategies: (entityId) => entityConfigMap.get(entityId)?.strategies,
232
+ resolveEntityStrategies: (entityId) => resolveEntityIndexStrategies(entityId, moduleConfigs),
232
233
  defaultStrategies,
233
234
  fallbackStrategy: 'tokens',
234
235
  mergeConfig: options?.mergeConfig ?? {
@@ -1,3 +1,4 @@
1
+ import { isEntityIndexStrategyAllowed } from '../lib/entity-strategy-policy'
1
2
  import type { SearchService } from '../service'
2
3
  import type {
3
4
  SearchModuleConfig,
@@ -677,6 +678,8 @@ export class SearchIndexer {
677
678
  errors: [],
678
679
  }
679
680
 
681
+ if (!isEntityIndexStrategyAllowed(params.entityId, 'fulltext', this.moduleConfigs)) return result
682
+
680
683
  const fulltext = this.getFulltextStrategy()
681
684
  if (!fulltext) {
682
685
  result.success = false
@@ -906,6 +909,8 @@ export class SearchIndexer {
906
909
  errors: [],
907
910
  }
908
911
 
912
+ if (!isEntityIndexStrategyAllowed(params.entityId, 'vector', this.moduleConfigs)) return result
913
+
909
914
  // If useQueue is requested but no queue is available, return error
910
915
  if (params.useQueue !== false && !this.vectorQueue) {
911
916
  result.success = false
@@ -0,0 +1,27 @@
1
+ import { getSearchModuleConfigs } from '@open-mercato/shared/modules/search'
2
+ import type { SearchModuleConfig, SearchStrategyId } from '../types'
3
+
4
+ export function resolveEntityIndexStrategies(
5
+ entityId: string,
6
+ moduleConfigs: SearchModuleConfig[] = getSearchModuleConfigs(),
7
+ ): SearchStrategyId[] | undefined {
8
+ let strategies: SearchStrategyId[] | undefined
9
+ for (const moduleConfig of moduleConfigs) {
10
+ for (const entityConfig of moduleConfig.entities) {
11
+ if (entityConfig.entityId !== entityId) continue
12
+ strategies = entityConfig.enabled === false
13
+ ? []
14
+ : entityConfig.strategies ?? moduleConfig.defaultStrategies
15
+ }
16
+ }
17
+ return strategies
18
+ }
19
+
20
+ export function isEntityIndexStrategyAllowed(
21
+ entityId: string,
22
+ strategyId: SearchStrategyId,
23
+ moduleConfigs?: SearchModuleConfig[],
24
+ ): boolean {
25
+ const strategies = resolveEntityIndexStrategies(entityId, moduleConfigs)
26
+ return strategies === undefined || strategies.includes(strategyId)
27
+ }
package/src/service.ts CHANGED
@@ -8,7 +8,7 @@ import type {
8
8
  IndexableRecord,
9
9
  PresenterEnricherFn,
10
10
  } from './types'
11
- import { getSearchModuleConfigs } from '@open-mercato/shared/modules/search'
11
+ import { resolveEntityIndexStrategies } from './lib/entity-strategy-policy'
12
12
  import { mergeAndRankResults } from './lib/merger'
13
13
  import { searchError } from './lib/debug'
14
14
 
@@ -527,12 +527,7 @@ export class SearchService {
527
527
  const allowedStrategies = this.resolveEntityStrategies(entityId)
528
528
  return allowedStrategies === undefined || allowedStrategies.includes(strategyId)
529
529
  }
530
- let allowedStrategies: SearchStrategyId[] | undefined
531
- for (const moduleConfig of getSearchModuleConfigs()) {
532
- for (const entityConfig of moduleConfig.entities) {
533
- if (entityConfig.entityId === entityId) allowedStrategies = entityConfig.strategies
534
- }
535
- }
530
+ const allowedStrategies = resolveEntityIndexStrategies(entityId)
536
531
  return allowedStrategies === undefined || allowedStrategies.includes(strategyId)
537
532
  }
538
533
 
@@ -1,3 +1,4 @@
1
+ import { isEntityIndexStrategyAllowed } from '../../lib/entity-strategy-policy'
1
2
  import type { QueryEngine } from '@open-mercato/shared/lib/query/types'
2
3
  import type { EntityManager } from '@mikro-orm/postgresql'
3
4
  import type { EntityId } from '@open-mercato/shared/modules/entities'
@@ -399,6 +400,15 @@ export class VectorIndexService {
399
400
  raw: Record<string, any>,
400
401
  opts: { skipDelete?: boolean } = {},
401
402
  ): Promise<VectorIndexOperationResult> {
403
+ if (!isEntityIndexStrategyAllowed(args.entityId, 'vector')) {
404
+ return {
405
+ action: 'skipped',
406
+ existed: false,
407
+ tenantId: args.tenantId,
408
+ organizationId: args.organizationId ?? null,
409
+ reason: 'unsupported',
410
+ }
411
+ }
402
412
  const scopeOrg = args.organizationId ?? null
403
413
  const { record, customFields } = this.extractRecordPayload(args.entityId, raw)
404
414
  const resolvedOrgId = scopeOrg ?? (record.organization_id ?? record.organizationId ?? null)