@open-mercato/search 0.10.1-develop.8225.1.1766b080b2 → 0.10.1-develop.8229.1.61fb2a0f4c

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,425 @@
1
+ import { SearchIndexer } from '../indexer/search-indexer'
2
+ import type { SearchBuildContext, SearchModuleConfig } from '../types'
3
+ import type { QueryEngine, QueryResult } from '@open-mercato/shared/lib/query/types'
4
+ import { registerTelemetryRuntime, type TelemetryRuntime } from '@open-mercato/shared/lib/telemetry/runtime'
5
+ import { searchDebug } from '../lib/debug'
6
+
7
+ jest.mock('../lib/debug', () => ({
8
+ ...jest.requireActual<typeof import('../lib/debug')>('../lib/debug'),
9
+ searchDebug: jest.fn(),
10
+ }))
11
+
12
+ describe('SearchIndexer encrypted custom fields', () => {
13
+ const companyEntityId = 'customers:customer_company_profile'
14
+ const parentEntityId = 'customers:customer_entity'
15
+
16
+ function setup(options: {
17
+ resolver?: (entityId: string, tenantId?: string | null) => Promise<Set<string>>
18
+ parentRecord?: Record<string, unknown>
19
+ loadParent?: boolean
20
+ } = {}) {
21
+ const contexts: SearchBuildContext[] = []
22
+ const hydrated: Array<Record<string, unknown> | undefined> = []
23
+ const moduleConfigs: SearchModuleConfig[] = [
24
+ {
25
+ entities: [
26
+ {
27
+ entityId: companyEntityId,
28
+ enabled: true,
29
+ buildSource: async (ctx) => {
30
+ contexts.push(ctx)
31
+ if (options.loadParent !== false) {
32
+ const engine = ctx.queryEngine as QueryEngine | undefined
33
+ const parent = await engine?.query(parentEntityId, { tenantId: 'tenant-1', filters: { id: 'entity-1' } })
34
+ hydrated.push(parent?.items[0] as Record<string, unknown> | undefined)
35
+ }
36
+ ctx.record.entity_id = 'entity-1'
37
+ return { text: JSON.stringify({ record: ctx.record, customFields: ctx.customFields }) }
38
+ },
39
+ },
40
+ ],
41
+ },
42
+ ]
43
+ const record = {
44
+ id: 'profile-1',
45
+ legal_name: 'Acme Ltd',
46
+ entity_id: null,
47
+ 'cf:tax_number': 'GB123456',
48
+ cf_tax_number: 'GB123456',
49
+ 'cf:industry_note': 'Wholesale',
50
+ }
51
+ const queryEngine: QueryEngine = {
52
+ query: jest.fn(async (entity, opts) => {
53
+ const filters = opts?.filters as { id?: string } | undefined
54
+ if (entity === parentEntityId) {
55
+ const items = options.parentRecord ? [options.parentRecord] : []
56
+ return { items, total: items.length, page: 1, pageSize: 1 } as QueryResult
57
+ }
58
+ const items = filters?.id === record.id ? [{ ...record }] : []
59
+ return { items, total: items.length, page: 1, pageSize: 1 } as QueryResult
60
+ }),
61
+ }
62
+ const searchService = {
63
+ index: jest.fn().mockResolvedValue(undefined),
64
+ bulkIndex: jest.fn().mockResolvedValue(undefined),
65
+ }
66
+ const indexer = new SearchIndexer(searchService as never, moduleConfigs, {
67
+ queryEngine,
68
+ encryptedCustomFieldKeysResolver: options.resolver,
69
+ })
70
+ return { indexer, searchService, contexts, hydrated, record }
71
+ }
72
+
73
+ const encryptedTaxNumber = async (entityId: string) =>
74
+ new Set(entityId === companyEntityId || entityId === parentEntityId ? ['tax_number'] : [])
75
+
76
+ it('keeps encrypted custom fields out of the build context and its text', async () => {
77
+ const { indexer, searchService, contexts, record } = setup({ resolver: encryptedTaxNumber })
78
+
79
+ await indexer.indexRecord({
80
+ entityId: companyEntityId,
81
+ recordId: record.id,
82
+ tenantId: 'tenant-1',
83
+ organizationId: 'org-1',
84
+ record: { ...record },
85
+ customFields: { tax_number: 'GB123456', industry_note: 'Wholesale' },
86
+ })
87
+
88
+ expect(contexts[0].customFields).toEqual({ industry_note: 'Wholesale' })
89
+ expect(contexts[0].record).not.toHaveProperty('cf:tax_number')
90
+ expect(contexts[0].record).not.toHaveProperty('cf_tax_number')
91
+ expect(contexts[0].record['cf:industry_note']).toBe('Wholesale')
92
+ const indexed = searchService.index.mock.calls[0][0]
93
+ expect(indexed.text).not.toContain('GB123456')
94
+ expect(indexed.text).toContain('Wholesale')
95
+ })
96
+
97
+ it('keeps encrypted values in the indexed fields and applies record changes made while building', async () => {
98
+ const { indexer, searchService, record } = setup({ resolver: encryptedTaxNumber })
99
+
100
+ await indexer.indexRecord({
101
+ entityId: companyEntityId,
102
+ recordId: record.id,
103
+ tenantId: 'tenant-1',
104
+ record: { ...record },
105
+ customFields: { tax_number: 'GB123456' },
106
+ })
107
+
108
+ const indexed = searchService.index.mock.calls[0][0]
109
+ expect(indexed.fields['cf:tax_number']).toBe('GB123456')
110
+ expect(indexed.fields.entity_id).toBe('entity-1')
111
+ expect(Object.keys(indexed.fields)).toEqual(Object.keys(record))
112
+ })
113
+
114
+ it('removes encrypted custom fields from records hydrated through the build query engine', async () => {
115
+ const resolver = jest.fn(encryptedTaxNumber)
116
+ const { indexer, hydrated, record } = setup({
117
+ resolver,
118
+ parentRecord: { id: 'entity-1', display_name: 'Acme', 'cf:tax_number': 'GB123456', 'cf:segment': 'B2B' },
119
+ })
120
+
121
+ await indexer.indexRecord({
122
+ entityId: companyEntityId,
123
+ recordId: record.id,
124
+ tenantId: 'tenant-1',
125
+ record: { ...record },
126
+ })
127
+
128
+ expect(hydrated[0]).toEqual({ id: 'entity-1', display_name: 'Acme', 'cf:segment': 'B2B' })
129
+ expect(resolver.mock.calls).toEqual([
130
+ [companyEntityId, 'tenant-1'],
131
+ [parentEntityId, 'tenant-1'],
132
+ ])
133
+ })
134
+
135
+ it('removes encrypted custom fields joined in from customFieldSources of another entity', async () => {
136
+ const contexts: SearchBuildContext[] = []
137
+ const hydrated: Array<Record<string, unknown> | undefined> = []
138
+ const queryEngine: QueryEngine = {
139
+ query: jest.fn(async () => ({
140
+ items: [{ id: 'entity-1', 'cf:tax_number': 'GB123456', 'cf:segment': 'B2B' }],
141
+ total: 1,
142
+ page: 1,
143
+ pageSize: 1,
144
+ }) as QueryResult),
145
+ }
146
+ const indexer = new SearchIndexer({ index: jest.fn().mockResolvedValue(undefined) } as never, [
147
+ {
148
+ entities: [
149
+ {
150
+ entityId: companyEntityId,
151
+ enabled: true,
152
+ buildSource: async (ctx) => {
153
+ contexts.push(ctx)
154
+ const parent = await (ctx.queryEngine as QueryEngine).query(parentEntityId, {
155
+ tenantId: 'tenant-1',
156
+ includeCustomFields: true,
157
+ customFieldSources: [{ entityId: companyEntityId, table: 'customer_companies' }],
158
+ })
159
+ hydrated.push(parent.items[0] as Record<string, unknown>)
160
+ return { text: 'Acme' }
161
+ },
162
+ },
163
+ ],
164
+ },
165
+ ], {
166
+ queryEngine,
167
+ encryptedCustomFieldKeysResolver: async (entityId) => new Set(entityId === companyEntityId ? ['tax_number'] : []),
168
+ })
169
+
170
+ await indexer.indexRecord({ entityId: companyEntityId, recordId: 'profile-1', tenantId: 'tenant-1', record: { id: 'profile-1' } })
171
+
172
+ expect(contexts).toHaveLength(1)
173
+ expect(hydrated[0]).toEqual({ id: 'entity-1', 'cf:segment': 'B2B' })
174
+ })
175
+
176
+ it('applies the same exclusion when records are built in a batch', async () => {
177
+ const { indexer, searchService, contexts, record } = setup({ resolver: encryptedTaxNumber })
178
+
179
+ await indexer.indexRecordsById({
180
+ items: [{ entityId: companyEntityId, recordId: record.id }],
181
+ tenantId: 'tenant-1',
182
+ organizationId: 'org-1',
183
+ })
184
+
185
+ expect(contexts[0].customFields).toEqual({ industry_note: 'Wholesale' })
186
+ expect(contexts[0].record).not.toHaveProperty('cf:tax_number')
187
+ const [indexed] = searchService.bulkIndex.mock.calls[0][0]
188
+ expect(indexed.text).not.toContain('GB123456')
189
+ expect(indexed.fields['cf:tax_number']).toBe('GB123456')
190
+ expect(indexed.fields.entity_id).toBe('entity-1')
191
+ })
192
+
193
+ it('resolves the encrypted definitions once per entity for a whole indexRecordsById batch', async () => {
194
+ const resolver = jest.fn(encryptedTaxNumber)
195
+ const { indexer, searchService, record } = setup({ resolver })
196
+ const records = ['profile-1', 'profile-2', 'profile-3'].map((id) => ({ ...record, id }))
197
+ ;(indexer as unknown as { queryEngine: QueryEngine }).queryEngine = {
198
+ query: jest.fn(async (entity, opts) => {
199
+ if (entity === parentEntityId) return { items: [], total: 0, page: 1, pageSize: 1 } as QueryResult
200
+ const wantedId = (opts?.filters as { id?: string } | undefined)?.id
201
+ const items = records.filter((candidate) => candidate.id === wantedId)
202
+ return { items, total: items.length, page: 1, pageSize: 1 } as QueryResult
203
+ }),
204
+ }
205
+
206
+ await indexer.indexRecordsById({
207
+ items: records.map((candidate) => ({ entityId: companyEntityId, recordId: candidate.id })),
208
+ tenantId: 'tenant-1',
209
+ })
210
+
211
+ expect(searchService.bulkIndex.mock.calls[0][0]).toHaveLength(3)
212
+ expect(resolver.mock.calls.filter(([entityId]) => entityId === companyEntityId)).toHaveLength(1)
213
+ })
214
+
215
+ it('logs only structural metadata for a batch sample', async () => {
216
+ const { indexer, record } = setup({ resolver: encryptedTaxNumber, loadParent: false })
217
+ const mockedSearchDebug = jest.mocked(searchDebug)
218
+ mockedSearchDebug.mockClear()
219
+
220
+ await indexer.indexRecordsById({
221
+ items: [{ entityId: companyEntityId, recordId: record.id }],
222
+ tenantId: 'tenant-1',
223
+ })
224
+
225
+ const sampleCall = mockedSearchDebug.mock.calls.find(([, message]) => message === 'Sample item structure')
226
+ expect(sampleCall?.[2]).toEqual({
227
+ entityId: companyEntityId,
228
+ sampleKeys: Object.keys(record),
229
+ hasId: true,
230
+ })
231
+ expect(JSON.stringify(sampleCall)).not.toContain('Acme Ltd')
232
+ expect(JSON.stringify(sampleCall)).not.toContain('GB123456')
233
+ expect(JSON.stringify(sampleCall)).not.toContain('Wholesale')
234
+ })
235
+
236
+ it('excludes a custom field whose query engine alias sanitizes its key', async () => {
237
+ const { indexer, contexts, record } = setup({ resolver: async () => new Set(['order-ref', 'order_ref']) })
238
+
239
+ await indexer.indexRecord({
240
+ entityId: companyEntityId,
241
+ recordId: record.id,
242
+ tenantId: 'tenant-1',
243
+ record: { ...record, cf_order_ref: 'REF-SECRET' },
244
+ })
245
+
246
+ expect(contexts[0].record).not.toHaveProperty('cf_order_ref')
247
+ })
248
+
249
+ it('applies keys a build callback deletes from a filtered record to the indexed fields', async () => {
250
+ const { indexer, searchService, record } = setup({ resolver: encryptedTaxNumber })
251
+ const config = (indexer as unknown as { entityConfigMap: Map<string, { buildSource?: (ctx: SearchBuildContext) => Promise<unknown> }> }).entityConfigMap.get(companyEntityId)
252
+ const originalBuildSource = config?.buildSource
253
+ if (config) {
254
+ config.buildSource = async (ctx) => {
255
+ delete ctx.record.legal_name
256
+ return originalBuildSource?.(ctx)
257
+ }
258
+ }
259
+
260
+ await indexer.indexRecord({
261
+ entityId: companyEntityId,
262
+ recordId: record.id,
263
+ tenantId: 'tenant-1',
264
+ record: { ...record },
265
+ })
266
+
267
+ const indexed = searchService.index.mock.calls[0][0]
268
+ expect(indexed.fields).not.toHaveProperty('legal_name')
269
+ expect(indexed.fields['cf:tax_number']).toBe('GB123456')
270
+ })
271
+
272
+ it('excludes encrypted keys passed with a cf: or cf_ prefix in customFields', async () => {
273
+ const { indexer, contexts, record } = setup({ resolver: encryptedTaxNumber })
274
+
275
+ await indexer.indexRecord({
276
+ entityId: companyEntityId,
277
+ recordId: record.id,
278
+ tenantId: 'tenant-1',
279
+ record: { ...record },
280
+ customFields: { 'cf:tax_number': 'GB123456', cf_tax_number: 'GB123456', 'cf:industry_note': 'Wholesale' },
281
+ })
282
+
283
+ expect(contexts[0].customFields).toEqual({ 'cf:industry_note': 'Wholesale' })
284
+ })
285
+
286
+ it('memoizes a failed definition lookup for the batch and normalizes non-Error failures', async () => {
287
+ const rejectedValue = { credentials: 'must-not-reach-telemetry' }
288
+ const resolver = jest.fn().mockRejectedValue(rejectedValue)
289
+ const reportError = jest.fn()
290
+ const unregister = registerTelemetryRuntime({ reportError } as unknown as TelemetryRuntime)
291
+ try {
292
+ const { indexer, contexts, record } = setup({ resolver, loadParent: false })
293
+ const records = ['profile-1', 'profile-2'].map((id) => ({ ...record, id }))
294
+ ;(indexer as unknown as { queryEngine: QueryEngine }).queryEngine = {
295
+ query: jest.fn(async (_entity, opts) => {
296
+ const wantedId = (opts?.filters as { id?: string } | undefined)?.id
297
+ const items = records.filter((candidate) => candidate.id === wantedId)
298
+ return { items, total: items.length, page: 1, pageSize: 1 } as QueryResult
299
+ }),
300
+ }
301
+
302
+ await indexer.indexRecordsById({
303
+ items: records.map((candidate) => ({ entityId: companyEntityId, recordId: candidate.id })),
304
+ tenantId: 'tenant-1',
305
+ })
306
+
307
+ const customFieldKeys = contexts.map((ctx) => Object.keys(ctx.record).filter((key) => key.startsWith('cf')))
308
+ expect(customFieldKeys).toEqual([[], []])
309
+ expect(resolver).toHaveBeenCalledTimes(1)
310
+ expect(reportError).toHaveBeenCalledTimes(1)
311
+ expect(reportError).toHaveBeenCalledWith(
312
+ expect.objectContaining({ message: '[internal] Encrypted custom-field resolver rejected with a non-Error value' }),
313
+ {
314
+ module: 'search',
315
+ code: 'search.encrypted_custom_fields_unresolved',
316
+ attributes: { entityId: companyEntityId },
317
+ },
318
+ )
319
+ expect(reportError.mock.calls[0][0]).not.toBe(rejectedValue)
320
+ } finally {
321
+ unregister()
322
+ }
323
+ })
324
+
325
+ it('memoizes a failed definition lookup across a direct vector reindex operation', async () => {
326
+ const resolver = jest.fn().mockRejectedValue(new Error('database unavailable'))
327
+ const reportError = jest.fn()
328
+ const unregister = registerTelemetryRuntime({ reportError } as unknown as TelemetryRuntime)
329
+ try {
330
+ const { indexer, contexts, record } = setup({ resolver, loadParent: false })
331
+ const records = ['profile-1', 'profile-2'].map((id) => ({ ...record, id }))
332
+ ;(indexer as unknown as { queryEngine: QueryEngine }).queryEngine = {
333
+ query: jest.fn(async (_entity, opts) => {
334
+ const wantedId = (opts?.filters as { id?: string } | undefined)?.id
335
+ const items = wantedId
336
+ ? records.filter((candidate) => candidate.id === wantedId)
337
+ : records
338
+ return { items, total: items.length, page: 1, pageSize: 200 } as QueryResult
339
+ }),
340
+ }
341
+
342
+ const result = await indexer.reindexEntityToVector({
343
+ entityId: companyEntityId,
344
+ tenantId: 'tenant-1',
345
+ useQueue: false,
346
+ })
347
+
348
+ expect(result).toEqual(expect.objectContaining({ success: true, recordsIndexed: 2 }))
349
+ expect(contexts).toHaveLength(2)
350
+ expect(contexts.map((ctx) => Object.keys(ctx.record).filter((key) => key.startsWith('cf')))).toEqual([[], []])
351
+ expect(resolver).toHaveBeenCalledTimes(1)
352
+ expect(reportError).toHaveBeenCalledTimes(1)
353
+ } finally {
354
+ unregister()
355
+ }
356
+ })
357
+
358
+ it('excludes every custom field when the encrypted definitions cannot be resolved', async () => {
359
+ const { indexer, searchService, contexts, record } = setup({
360
+ resolver: async () => {
361
+ throw new Error('database unavailable')
362
+ },
363
+ })
364
+
365
+ await indexer.indexRecord({
366
+ entityId: companyEntityId,
367
+ recordId: record.id,
368
+ tenantId: 'tenant-1',
369
+ record: { ...record },
370
+ customFields: { tax_number: 'GB123456', industry_note: 'Wholesale' },
371
+ })
372
+
373
+ expect(contexts[0].customFields).toEqual({})
374
+ expect(Object.keys(contexts[0].record).filter((key) => key.startsWith('cf'))).toEqual([])
375
+ expect(searchService.index.mock.calls[0][0].text).not.toContain('GB123456')
376
+ })
377
+
378
+ it('excludes every custom field when the resolver throws synchronously', async () => {
379
+ const { indexer, contexts, record } = setup({
380
+ resolver: () => {
381
+ throw new Error('resolver misconfigured')
382
+ },
383
+ })
384
+
385
+ await indexer.indexRecord({
386
+ entityId: companyEntityId,
387
+ recordId: record.id,
388
+ tenantId: 'tenant-1',
389
+ record: { ...record },
390
+ customFields: { tax_number: 'GB123456', industry_note: 'Wholesale' },
391
+ })
392
+
393
+ expect(contexts[0].customFields).toEqual({})
394
+ })
395
+
396
+ it('excludes every custom field and reports it when constructed without a resolver', async () => {
397
+ const reportError = jest.fn()
398
+ const unregister = registerTelemetryRuntime({ reportError } as unknown as TelemetryRuntime)
399
+ try {
400
+ const { indexer, searchService, contexts, hydrated, record } = setup({
401
+ parentRecord: { id: 'entity-1', display_name: 'Acme', 'cf:vat_id': 'PL555', cf_vat_id: 'PL555' },
402
+ })
403
+
404
+ await indexer.indexRecord({
405
+ entityId: companyEntityId,
406
+ recordId: record.id,
407
+ tenantId: 'tenant-1',
408
+ record: { ...record },
409
+ customFields: { tax_number: 'GB123456', industry_note: 'Wholesale' },
410
+ })
411
+
412
+ expect(contexts[0].customFields).toEqual({})
413
+ expect(Object.keys(contexts[0].record).filter((key) => key.startsWith('cf'))).toEqual([])
414
+ expect(hydrated[0]).toEqual({ id: 'entity-1', display_name: 'Acme' })
415
+ expect(searchService.index.mock.calls[0][0].text).not.toContain('GB123456')
416
+ expect(searchService.index.mock.calls[0][0].fields['cf:tax_number']).toBe('GB123456')
417
+ expect(reportError).toHaveBeenCalledWith(expect.any(Error), {
418
+ module: 'search',
419
+ code: 'search.encrypted_custom_fields_resolver_missing',
420
+ })
421
+ } finally {
422
+ unregister()
423
+ }
424
+ })
425
+ })