@open-mercato/search 0.10.1-develop.8225.1.1766b080b2 → 0.10.1-develop.8229.1.61fb2a0f4c

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,4 @@
1
- import { VectorIndexService } from '../vector/services/vector-index.service'
1
+ import { VectorIndexService, type VectorIndexServiceOptions } from '../vector/services/vector-index.service'
2
2
  import type { EmbeddingService } from '../vector/services/embedding'
3
3
  import type { VectorDriver, VectorDriverDocument } from '../vector/types'
4
4
  import type { VectorModuleConfig } from '@open-mercato/shared/modules/vector'
@@ -18,6 +18,12 @@ jest.mock('@open-mercato/shared/lib/i18n/server', () => ({
18
18
  resolveTranslations: async () => ({ t: (_key: string, fallback: string) => fallback }),
19
19
  }))
20
20
  jest.mock('../fulltext/drivers', () => ({ createFulltextDriver: jest.fn() }))
21
+ jest.mock('@open-mercato/shared/lib/telemetry/runtime', () => ({
22
+ getTelemetryRuntime: () => ({ reportError: (...args: unknown[]) => mockReportError(...args) }),
23
+ }))
24
+
25
+ const mockReportError = jest.fn((..._args: unknown[]) => {})
26
+ type EncryptedCustomFieldKeysResolver = NonNullable<VectorIndexServiceOptions['encryptedCustomFieldKeysResolver']>
21
27
 
22
28
  type CapturedEmbeddingInput = { value: string | string[] | null }
23
29
 
@@ -80,12 +86,16 @@ function createService(args: {
80
86
  queryEngine: QueryEngine
81
87
  moduleConfigs: VectorModuleConfig[]
82
88
  encryptionService?: TenantDataEncryptionService | null
89
+ encryptedCustomFieldKeysResolver?: EncryptedCustomFieldKeysResolver | null
83
90
  }): VectorIndexService {
84
91
  return new VectorIndexService({
85
92
  drivers: [args.driver],
86
93
  embeddingService: args.embeddingService,
87
94
  queryEngine: args.queryEngine,
88
95
  moduleConfigs: args.moduleConfigs,
96
+ ...(args.encryptedCustomFieldKeysResolver === null
97
+ ? {}
98
+ : { encryptedCustomFieldKeysResolver: args.encryptedCustomFieldKeysResolver ?? (async () => new Set()) }),
89
99
  containerResolver: () => ({
90
100
  resolve: (name: string) => {
91
101
  if (name === 'tenantEncryptionService') {
@@ -173,6 +183,238 @@ describe('VectorIndexService default source field protection (issue #2716)', ()
173
183
  })
174
184
  })
175
185
 
186
+ describe('VectorIndexService encrypted custom-field callback boundaries (PR #8410)', () => {
187
+ it('filters encrypted keys from every callback and hydrated custom-field source with tenant-aware lookups', async () => {
188
+ const rawRecord = {
189
+ id: 'callback-record',
190
+ name: 'Visible Name',
191
+ cf_secret: 'PRIVATE-RECORD',
192
+ cf_note: 'Visible record note',
193
+ 'cf:secret': 'PRIVATE-CUSTOM',
194
+ 'cf:note': 'Visible custom note',
195
+ }
196
+ const originalRecord = { ...rawRecord }
197
+ const callbackContexts: Array<{ record: Record<string, unknown>; customFields: Record<string, unknown> }> = []
198
+ const hydratedRecords: Record<string, unknown>[] = []
199
+ const queryEngine = {
200
+ query: jest.fn(async (entityId: string, options?: Record<string, unknown>) => {
201
+ if (entityId === 'demo:item') return { items: [rawRecord], total: 1 }
202
+ expect(options).toEqual(expect.objectContaining({ tenantId: 'callback-tenant', skipAutoReindex: true }))
203
+ return {
204
+ items: [{
205
+ id: 'related-record',
206
+ 'cf:related_secret': 'PRIVATE-RELATED',
207
+ cf_source_secret: 'PRIVATE-SOURCE',
208
+ 'cf:related_note': 'Visible related note',
209
+ cf_source_note: 'Visible source note',
210
+ }],
211
+ total: 1,
212
+ }
213
+ }),
214
+ } as unknown as QueryEngine
215
+ const resolver = jest.fn<ReturnType<EncryptedCustomFieldKeysResolver>, Parameters<EncryptedCustomFieldKeysResolver>>(async (entityId) => {
216
+ if (entityId === 'demo:item') return new Set(['secret'])
217
+ if (entityId === 'demo:related') return new Set(['related_secret'])
218
+ return new Set(['source_secret'])
219
+ })
220
+ const captureContext = (context: { record: Record<string, unknown>; customFields: Record<string, unknown> }) => {
221
+ callbackContexts.push({ record: { ...context.record }, customFields: { ...context.customFields } })
222
+ }
223
+ const service = createService({
224
+ driver: createDriver(),
225
+ embeddingService: createEmbeddingService({ value: null }),
226
+ queryEngine,
227
+ encryptedCustomFieldKeysResolver: resolver,
228
+ moduleConfigs: [{ entities: [{
229
+ entityId: 'demo:item',
230
+ buildSource: async (context) => {
231
+ captureContext(context)
232
+ const hydrated = await context.queryEngine!.query('demo:related', {
233
+ tenantId: 'callback-tenant',
234
+ customFieldSources: [{
235
+ entityId: 'demo:source',
236
+ join: { fromField: 'id', toField: 'related_id' },
237
+ }],
238
+ })
239
+ hydratedRecords.push(hydrated.items[0])
240
+ return { input: 'Visible source' }
241
+ },
242
+ formatResult: (context) => {
243
+ captureContext(context)
244
+ return { title: 'Visible title' }
245
+ },
246
+ resolveLinks: (context) => {
247
+ captureContext(context)
248
+ return [{ href: '/visible', label: 'Visible' }]
249
+ },
250
+ resolveUrl: (context) => {
251
+ captureContext(context)
252
+ return '/visible'
253
+ },
254
+ }] }],
255
+ })
256
+
257
+ await service.indexRecord({ entityId: 'demo:item', recordId: 'callback-record', tenantId: 'outer-tenant' })
258
+
259
+ expect(callbackContexts).toHaveLength(4)
260
+ for (const context of callbackContexts) {
261
+ expect(context.record).toEqual(expect.objectContaining({ name: 'Visible Name', cf_note: 'Visible record note' }))
262
+ expect(context.record).not.toHaveProperty('cf_secret')
263
+ expect(context.customFields).toEqual({ note: 'Visible custom note' })
264
+ }
265
+ expect(hydratedRecords).toEqual([expect.objectContaining({
266
+ id: 'related-record',
267
+ 'cf:related_note': 'Visible related note',
268
+ cf_source_note: 'Visible source note',
269
+ })])
270
+ expect(JSON.stringify(hydratedRecords)).not.toContain('PRIVATE-')
271
+ expect(resolver).toHaveBeenCalledWith('demo:item', 'outer-tenant')
272
+ expect(resolver).toHaveBeenCalledWith('demo:related', 'callback-tenant')
273
+ expect(resolver).toHaveBeenCalledWith('demo:source', 'callback-tenant')
274
+ expect(rawRecord).toEqual(originalRecord)
275
+ })
276
+
277
+ it('removes every custom field from callbacks and callback queries when the resolver is absent', async () => {
278
+ const contexts: Array<{ record: Record<string, unknown>; customFields: Record<string, unknown> }> = []
279
+ const hydratedRecords: Record<string, unknown>[] = []
280
+ const queryEngine = {
281
+ query: async (entityId: string) => entityId === 'demo:item'
282
+ ? { items: [{ id: 'missing-resolver', name: 'Visible', cf_note: 'PRIVATE-ALIAS', 'cf:secret': 'PRIVATE-CUSTOM' }], total: 1 }
283
+ : { items: [{ id: 'related', 'cf:secret': 'PRIVATE-RELATED', cf_note: 'PRIVATE-RELATED-ALIAS' }], total: 1 },
284
+ } as unknown as QueryEngine
285
+ const capture = (context: { record: Record<string, unknown>; customFields: Record<string, unknown> }) => {
286
+ contexts.push({ record: { ...context.record }, customFields: { ...context.customFields } })
287
+ }
288
+ const service = createService({
289
+ driver: createDriver(),
290
+ embeddingService: createEmbeddingService({ value: null }),
291
+ queryEngine,
292
+ encryptedCustomFieldKeysResolver: null,
293
+ moduleConfigs: [{ entities: [{
294
+ entityId: 'demo:item',
295
+ buildSource: async (context) => {
296
+ capture(context)
297
+ const hydrated = await context.queryEngine!.query('demo:related', { tenantId: 'tenant-closed' })
298
+ hydratedRecords.push(hydrated.items[0])
299
+ return { input: 'Visible source' }
300
+ },
301
+ formatResult: (context) => { capture(context); return { title: 'Visible' } },
302
+ resolveLinks: (context) => { capture(context); return [] },
303
+ resolveUrl: (context) => { capture(context); return null },
304
+ }] }],
305
+ })
306
+
307
+ await service.indexRecord({ entityId: 'demo:item', recordId: 'missing-resolver', tenantId: 'tenant-closed' })
308
+
309
+ expect(contexts).toHaveLength(4)
310
+ for (const context of contexts) {
311
+ expect(context.record).toEqual({ id: 'missing-resolver', name: 'Visible' })
312
+ expect(context.customFields).toEqual({})
313
+ }
314
+ expect(hydratedRecords).toEqual([{ id: 'related' }])
315
+ })
316
+
317
+ it('memoizes normalized resolver failures for one reindex operation and retries on the next operation', async () => {
318
+ mockReportError.mockClear()
319
+ const records = [
320
+ { id: 'first', title: 'First', 'cf:secret': 'PRIVATE-FIRST' },
321
+ { id: 'second', title: 'Second', 'cf:secret': 'PRIVATE-SECOND' },
322
+ ]
323
+ const queryEngine = {
324
+ query: async (entityId: string, options?: { page?: unknown; filters?: unknown }) => {
325
+ if (entityId === 'demo:item') {
326
+ return { items: options?.page ? records : [records[0]], total: options?.page ? 2 : 1 }
327
+ }
328
+ return { items: [{ id: 'related', 'cf:secret': 'PRIVATE-RELATED' }], total: 1 }
329
+ },
330
+ } as unknown as QueryEngine
331
+ const resolver = jest.fn<ReturnType<EncryptedCustomFieldKeysResolver>, Parameters<EncryptedCustomFieldKeysResolver>>(() => {
332
+ throw { secret: 'PRIVATE-REJECTION' }
333
+ })
334
+ const hydrate = async (context: { queryEngine?: QueryEngine }) => {
335
+ const result = await context.queryEngine!.query('demo:related', {
336
+ tenantId: 'tenant-failure',
337
+ customFieldSources: [{
338
+ entityId: 'demo:source',
339
+ join: { fromField: 'id', toField: 'related_id' },
340
+ }],
341
+ })
342
+ expect(result.items).toEqual([{ id: 'related' }])
343
+ }
344
+ const service = createService({
345
+ driver: createDriver(),
346
+ embeddingService: createEmbeddingService({ value: null }),
347
+ queryEngine,
348
+ encryptedCustomFieldKeysResolver: resolver,
349
+ moduleConfigs: [{ entities: [{
350
+ entityId: 'demo:item',
351
+ buildSource: async (context) => { await hydrate(context); return { input: String(context.record.title) } },
352
+ formatResult: async (context) => { await hydrate(context); return { title: String(context.record.title) } },
353
+ resolveLinks: async (context) => { await hydrate(context); return [] },
354
+ resolveUrl: async (context) => { await hydrate(context); return null },
355
+ }] }],
356
+ })
357
+
358
+ await service.reindexEntity({ entityId: 'demo:item', tenantId: 'tenant-failure' })
359
+
360
+ expect(resolver).toHaveBeenCalledTimes(3)
361
+ expect(resolver.mock.calls).toEqual(expect.arrayContaining([
362
+ ['demo:item', 'tenant-failure'],
363
+ ['demo:related', 'tenant-failure'],
364
+ ['demo:source', 'tenant-failure'],
365
+ ]))
366
+ expect(mockReportError).toHaveBeenCalledTimes(3)
367
+ for (const [error, context] of mockReportError.mock.calls) {
368
+ expect(error).toBeInstanceOf(Error)
369
+ expect((error as Error).message).toBe('[internal] Encrypted custom-field resolver rejected with a non-Error value')
370
+ expect((error as Error).message).not.toContain('PRIVATE-REJECTION')
371
+ expect(context).toEqual(expect.objectContaining({ code: 'search.encrypted_custom_fields_unresolved' }))
372
+ }
373
+
374
+ await service.indexRecord({ entityId: 'demo:item', recordId: 'first', tenantId: 'tenant-failure' })
375
+
376
+ expect(resolver).toHaveBeenCalledTimes(6)
377
+ expect(mockReportError).toHaveBeenCalledTimes(6)
378
+ })
379
+
380
+ it('keeps encrypted metadata-like values in the checksum without embedding or storing them', async () => {
381
+ let record = {
382
+ id: 'checksum-record',
383
+ public_label: 'Visible',
384
+ 'cf:title': 'PRIVATE-TITLE-FIRST',
385
+ 'cf:summary': 'PRIVATE-SUMMARY-FIRST',
386
+ }
387
+ const capturedInputs: string[] = []
388
+ const driver = createDriver()
389
+ const service = createService({
390
+ driver,
391
+ embeddingService: {
392
+ available: true,
393
+ createEmbedding: async (input: string | string[]) => {
394
+ capturedInputs.push(Array.isArray(input) ? input.join('\n') : input)
395
+ return [0.1, 0.2, 0.3]
396
+ },
397
+ } as EmbeddingService,
398
+ queryEngine: { query: async () => ({ items: [record], total: 1 }) } as unknown as QueryEngine,
399
+ encryptedCustomFieldKeysResolver: async () => new Set(['title', 'summary']),
400
+ moduleConfigs: [{ entities: [{ entityId: 'demo:item' }] }],
401
+ })
402
+
403
+ await service.indexRecord({ entityId: 'demo:item', recordId: 'checksum-record', tenantId: 'tenant-checksum' })
404
+ record = {
405
+ ...record,
406
+ 'cf:title': 'PRIVATE-TITLE-SECOND',
407
+ 'cf:summary': 'PRIVATE-SUMMARY-SECOND',
408
+ }
409
+ await service.indexRecord({ entityId: 'demo:item', recordId: 'checksum-record', tenantId: 'tenant-checksum' })
410
+
411
+ expect(capturedInputs).toHaveLength(2)
412
+ expect(capturedInputs.join('\n')).not.toContain('PRIVATE-')
413
+ expect(JSON.stringify(driver.upsertCalls)).not.toContain('PRIVATE-')
414
+ expect(driver.upsertCalls[0].checksum).not.toBe(driver.upsertCalls[1].checksum)
415
+ })
416
+ })
417
+
176
418
  describe('Customer vector privacy at provider and storage boundaries (issue #7826)', () => {
177
419
  it.each(['true', 'false'].flatMap((toggle) => ['single', 'by-id', 'batch', 'bulk'].map((mode) => ({ toggle, mode }))))(
178
420
  'protects real-service live declarations with encryption $toggle through $mode indexing', async ({ toggle, mode }) => {
@@ -239,8 +481,12 @@ describe('Customer vector privacy at provider and storage boundaries (issue #782
239
481
  const indexer = createCustomerIndexer({ driver, embeddingService: createEmbeddingService(captured), queryEngine, encryptionService: { getEncryptedFieldNames } })
240
482
  await indexer.indexRecordById({ entityId: 'customers:customer_company_profile', recordId: record.id, tenantId: 'tenant-privacy', organizationId })
241
483
  expect(driver.upsertCalls).toHaveLength(1)
242
- if (scope !== 'other-scope') expect(JSON.stringify(captured.value)).not.toContain('private-')
243
- else expect(JSON.stringify(captured.value)).toContain('private-custom-size')
484
+ if (scope === 'other-scope') {
485
+ expect(JSON.stringify(captured.value)).toContain('private-size')
486
+ expect(JSON.stringify(captured.value)).not.toContain('private-custom-size')
487
+ } else {
488
+ expect(JSON.stringify(captured.value)).not.toContain('private-')
489
+ }
244
490
  for (const entityId of ['customers:customer_company_profile', 'customers:customer_entity']) {
245
491
  expect(getEncryptedFieldNames).toHaveBeenCalledWith(entityId, 'tenant-privacy', organizationId, { ignoreRuntimeHealth: true })
246
492
  }
package/src/di.ts CHANGED
@@ -27,10 +27,18 @@ import type { EncryptionMapEntry } from './lib/field-policy'
27
27
  import type { TenantDataEncryptionService } from '@open-mercato/shared/lib/encryption/tenantDataEncryptionService'
28
28
  import { createPresenterEnricher } from './lib/presenter-enricher'
29
29
  import { createEncryptionMapResolver } from './lib/encryption-map-resolver'
30
- import { createEncryptedCustomFieldKeysResolver, type CustomFieldDefinitionsDatabase, type EncryptedCustomFieldKeysResolver } from './lib/encrypted-custom-field-resolver'
30
+ import {
31
+ createEncryptedCustomFieldKeysResolver,
32
+ type CustomFieldDefinitionsDatabase,
33
+ type EncryptedCustomFieldKeysResolver,
34
+ } from './lib/encrypted-custom-field-resolver'
31
35
  import { reportSearchError } from './lib/debug'
32
36
 
33
37
  export { createEncryptionMapResolver } from './lib/encryption-map-resolver'
38
+ export {
39
+ createEncryptedCustomFieldKeysResolver,
40
+ type EncryptedCustomFieldKeysResolver,
41
+ } from './lib/encrypted-custom-field-resolver'
34
42
 
35
43
  type KyselyEntityManager = {
36
44
  getKysely<Database>(): Kysely<Database>
@@ -56,6 +64,12 @@ function shouldExcludeEncryptedFields(): boolean {
56
64
  return raw === '1' || raw === 'true' || raw === 'yes' || raw === 'on'
57
65
  }
58
66
 
67
+ function createUnavailableEncryptedCustomFieldKeysResolver(): EncryptedCustomFieldKeysResolver {
68
+ return async () => {
69
+ throw new Error('[internal] Custom-field definitions are unavailable; refusing to index potentially encrypted custom fields')
70
+ }
71
+ }
72
+
59
73
  function createUnavailableEncryptionMapResolver(): (entityId: EntityId) => Promise<EncryptionMapEntry[]> {
60
74
  return async () => {
61
75
  throw new Error('[internal] Encryption map lookup is unavailable; refusing to index potentially encrypted fields')
@@ -262,12 +276,12 @@ export function registerSearchModule(
262
276
  let encryptedCustomFieldKeysResolver: EncryptedCustomFieldKeysResolver
263
277
  try {
264
278
  const em = container.resolve<KyselyEntityManager>('em')
265
- encryptedCustomFieldKeysResolver = createEncryptedCustomFieldKeysResolver(em.getKysely<CustomFieldDefinitionsDatabase>())
279
+ encryptedCustomFieldKeysResolver = createEncryptedCustomFieldKeysResolver(
280
+ em.getKysely<CustomFieldDefinitionsDatabase>(),
281
+ )
266
282
  } catch (error) {
267
283
  reportSearchError('SearchModule', error, 'search.custom_field_definitions_unavailable')
268
- encryptedCustomFieldKeysResolver = async () => {
269
- throw new Error('[internal] Custom-field definitions are unavailable')
270
- }
284
+ encryptedCustomFieldKeysResolver = createUnavailableEncryptedCustomFieldKeysResolver()
271
285
  }
272
286
 
273
287
  const searchIndexer = new SearchIndexer(searchService, moduleConfigs, {