@open-mercato/search 0.10.1-develop.7769.1.0e6458bb99 → 0.10.1-develop.7770.1.970b396ebb
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/di.js +8 -2
- package/dist/di.js.map +2 -2
- package/dist/indexer/search-indexer.js +15 -50
- package/dist/indexer/search-indexer.js.map +2 -2
- package/dist/lib/presenter-enricher.js +2 -1
- package/dist/lib/presenter-enricher.js.map +2 -2
- package/dist/strategies/vector.strategy.js +2 -2
- package/dist/strategies/vector.strategy.js.map +2 -2
- package/dist/vector/services/vector-index.service.js +9 -5
- package/dist/vector/services/vector-index.service.js.map +2 -2
- package/package.json +4 -4
- package/src/__tests__/presenter-enricher.test.ts +20 -0
- package/src/__tests__/vector-index-security.test.ts +280 -1
- package/src/di.ts +10 -4
- package/src/indexer/search-indexer.ts +16 -60
- package/src/lib/presenter-enricher.ts +2 -1
- package/src/strategies/vector.strategy.ts +2 -2
- package/src/vector/services/vector-index.service.ts +9 -5
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@open-mercato/search",
|
|
3
|
-
"version": "0.10.1-develop.
|
|
3
|
+
"version": "0.10.1-develop.7770.1.970b396ebb",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -127,9 +127,9 @@
|
|
|
127
127
|
"zod": "^4.4.3"
|
|
128
128
|
},
|
|
129
129
|
"peerDependencies": {
|
|
130
|
-
"@open-mercato/core": "0.10.1-develop.
|
|
131
|
-
"@open-mercato/queue": "0.10.1-develop.
|
|
132
|
-
"@open-mercato/shared": "0.10.1-develop.
|
|
130
|
+
"@open-mercato/core": "0.10.1-develop.7770.1.970b396ebb",
|
|
131
|
+
"@open-mercato/queue": "0.10.1-develop.7770.1.970b396ebb",
|
|
132
|
+
"@open-mercato/shared": "0.10.1-develop.7770.1.970b396ebb"
|
|
133
133
|
},
|
|
134
134
|
"devDependencies": {
|
|
135
135
|
"@types/jest": "^30.0.0",
|
|
@@ -70,6 +70,26 @@ describe('createPresenterEnricher', () => {
|
|
|
70
70
|
mockedDecryptIndexDocForSearch.mockReset()
|
|
71
71
|
})
|
|
72
72
|
|
|
73
|
+
it.each(['throw', 'null'])('keeps protected contacts out of the read-time fallback when callbacks %s', async (failure) => {
|
|
74
|
+
const doc = { id: 'person-1', name: 'Safe display name', email: 'private-contact@example.com', password_hash: 'private-password' }
|
|
75
|
+
mockedDecryptIndexDocForSearch.mockResolvedValue(doc)
|
|
76
|
+
const unavailable = () => {
|
|
77
|
+
if (failure === 'throw') throw new Error('[internal] Injected formatter failure')
|
|
78
|
+
return null
|
|
79
|
+
}
|
|
80
|
+
const config = createConfig({
|
|
81
|
+
fieldPolicy: { searchable: ['name', 'email', 'password_hash'], hashOnly: ['email'], excluded: ['password_hash'] },
|
|
82
|
+
formatResult: unavailable, buildSource: unavailable,
|
|
83
|
+
})
|
|
84
|
+
const enrich = createPresenterEnricher(
|
|
85
|
+
createKyselyMock([{ entity_type: config.entityId, entity_id: doc.id, doc }]),
|
|
86
|
+
new Map([[config.entityId, config]]),
|
|
87
|
+
)
|
|
88
|
+
const [enriched] = await enrich([createResult()], 'tenant-1', 'org-1')
|
|
89
|
+
expect(enriched.presenter?.title).toBe('Safe display name')
|
|
90
|
+
expect(JSON.stringify(enriched.presenter)).not.toContain('private-')
|
|
91
|
+
})
|
|
92
|
+
|
|
73
93
|
it('uses search config presenters and the stored organization scope for doc decryption', async () => {
|
|
74
94
|
const decryptedDoc = {
|
|
75
95
|
id: 'person-1',
|
|
@@ -3,7 +3,21 @@ import type { EmbeddingService } from '../vector/services/embedding'
|
|
|
3
3
|
import type { VectorDriver, VectorDriverDocument } from '../vector/types'
|
|
4
4
|
import type { VectorModuleConfig } from '@open-mercato/shared/modules/vector'
|
|
5
5
|
import type { QueryEngine } from '@open-mercato/shared/lib/query/types'
|
|
6
|
-
import
|
|
6
|
+
import { TenantDataEncryptionService } from '@open-mercato/shared/lib/encryption/tenantDataEncryptionService'
|
|
7
|
+
import type { EntityManager } from '@mikro-orm/postgresql'
|
|
8
|
+
import { searchConfig as customersSearchConfig } from '@open-mercato/core/modules/customers/search'
|
|
9
|
+
import { searchConfig as accountsSearchConfig } from '@open-mercato/core/modules/customer_accounts/search'
|
|
10
|
+
import { SearchIndexer } from '../indexer/search-indexer'
|
|
11
|
+
import { SearchService } from '../service'
|
|
12
|
+
import { VectorSearchStrategy } from '../strategies/vector.strategy'
|
|
13
|
+
import type { SearchStrategy } from '../types'
|
|
14
|
+
import { asValue, createContainer } from 'awilix'
|
|
15
|
+
import { registerSearchModule } from '../di'
|
|
16
|
+
|
|
17
|
+
jest.mock('@open-mercato/shared/lib/i18n/server', () => ({
|
|
18
|
+
resolveTranslations: async () => ({ t: (_key: string, fallback: string) => fallback }),
|
|
19
|
+
}))
|
|
20
|
+
jest.mock('../fulltext/drivers', () => ({ createFulltextDriver: jest.fn() }))
|
|
7
21
|
|
|
8
22
|
type CapturedEmbeddingInput = { value: string | string[] | null }
|
|
9
23
|
|
|
@@ -41,6 +55,25 @@ function createQueryEngine(record: Record<string, unknown>): QueryEngine {
|
|
|
41
55
|
} as unknown as QueryEngine
|
|
42
56
|
}
|
|
43
57
|
|
|
58
|
+
function createCustomerIndexer(args: {
|
|
59
|
+
driver: VectorDriver
|
|
60
|
+
embeddingService: EmbeddingService
|
|
61
|
+
queryEngine: QueryEngine
|
|
62
|
+
encryptionService?: Pick<TenantDataEncryptionService, 'getEncryptedFieldNames'>
|
|
63
|
+
otherStrategy?: SearchStrategy
|
|
64
|
+
}): SearchIndexer {
|
|
65
|
+
const container = createContainer()
|
|
66
|
+
container.register({
|
|
67
|
+
vectorDrivers: asValue([args.driver]),
|
|
68
|
+
vectorEmbeddingService: asValue(args.embeddingService),
|
|
69
|
+
queryEngine: asValue(args.queryEngine),
|
|
70
|
+
tenantEncryptionService: asValue(args.encryptionService ?? { getEncryptedFieldNames: async () => [] }),
|
|
71
|
+
})
|
|
72
|
+
registerSearchModule(container, { skipTokens: true, skipFulltext: true, moduleConfigs: [customersSearchConfig, accountsSearchConfig] })
|
|
73
|
+
if (args.otherStrategy) container.resolve<SearchService>('searchService').registerStrategy(args.otherStrategy)
|
|
74
|
+
return container.resolve<SearchIndexer>('searchIndexer')
|
|
75
|
+
}
|
|
76
|
+
|
|
44
77
|
function createService(args: {
|
|
45
78
|
driver: VectorDriver
|
|
46
79
|
embeddingService: EmbeddingService
|
|
@@ -140,6 +173,252 @@ describe('VectorIndexService default source field protection (issue #2716)', ()
|
|
|
140
173
|
})
|
|
141
174
|
})
|
|
142
175
|
|
|
176
|
+
describe('Customer vector privacy at provider and storage boundaries (issue #7826)', () => {
|
|
177
|
+
it.each(['true', 'false'].flatMap((toggle) => ['single', 'by-id', 'batch', 'bulk'].map((mode) => ({ toggle, mode }))))(
|
|
178
|
+
'protects real-service live declarations with encryption $toggle through $mode indexing', async ({ toggle, mode }) => {
|
|
179
|
+
const previousToggle = process.env.TENANT_DATA_ENCRYPTION
|
|
180
|
+
try {
|
|
181
|
+
process.env.TENANT_DATA_ENCRYPTION = 'true'
|
|
182
|
+
const execute = jest.fn(async (_sql: string, params?: readonly unknown[]) => [{
|
|
183
|
+
entity_id: params?.[0], tenant_id: 'tenant-privacy', organization_id: 'org-privacy',
|
|
184
|
+
fields_json: [{ field: params?.[0] === 'customers:customer_company_profile' ? 'size_bucket' : 'status' }],
|
|
185
|
+
}])
|
|
186
|
+
const encryptionService = new TenantDataEncryptionService({ execute } as unknown as EntityManager, {
|
|
187
|
+
kms: { isHealthy: () => false, getTenantDek: async () => null, createTenantDek: async () => null },
|
|
188
|
+
})
|
|
189
|
+
await expect(encryptionService.getEncryptedFieldNames('customers:customer_company_profile', 'tenant-privacy', 'org-privacy', { ignoreRuntimeHealth: true })).resolves.toEqual(['size_bucket'])
|
|
190
|
+
execute.mockClear()
|
|
191
|
+
process.env.TENANT_DATA_ENCRYPTION = toggle
|
|
192
|
+
const driver = createDriver()
|
|
193
|
+
const captured: CapturedEmbeddingInput = { value: null }
|
|
194
|
+
const otherIndex = jest.fn(async () => {})
|
|
195
|
+
const record = { id: 'real-policy-company', entity_id: 'real-policy-entity', size_bucket: 'private-size' }
|
|
196
|
+
const indexer = createCustomerIndexer({
|
|
197
|
+
driver, embeddingService: createEmbeddingService(captured), encryptionService,
|
|
198
|
+
queryEngine: { query: async (entityId: string) => ({ items: [entityId === 'customers:customer_entity' ? { id: 'real-policy-entity', status: 'private-status' } : record], total: 1 }) } as QueryEngine,
|
|
199
|
+
otherStrategy: { id: 'fulltext', name: 'Fake fulltext', priority: 1, ensureReady: async () => {}, isAvailable: async () => true, search: async () => [], index: otherIndex, delete: async () => {} },
|
|
200
|
+
})
|
|
201
|
+
const params = { entityId: 'customers:customer_company_profile', recordId: record.id, tenantId: 'tenant-privacy', organizationId: 'org-privacy' }
|
|
202
|
+
if (mode === 'single') await indexer.indexRecord({ ...params, record })
|
|
203
|
+
if (mode === 'by-id') await indexer.indexRecordById(params)
|
|
204
|
+
if (mode === 'batch') await indexer.indexRecordsById({ ...params, items: [params] })
|
|
205
|
+
if (mode === 'bulk') await indexer.bulkIndexRecords([{ ...params, record }])
|
|
206
|
+
if (toggle === 'false') {
|
|
207
|
+
expect(captured.value).toBeNull()
|
|
208
|
+
expect(driver.upsertCalls).toHaveLength(0)
|
|
209
|
+
expect(execute).not.toHaveBeenCalled()
|
|
210
|
+
} else {
|
|
211
|
+
expect(captured.value).not.toBeNull()
|
|
212
|
+
expect(driver.upsertCalls).toHaveLength(1)
|
|
213
|
+
expect(JSON.stringify(captured.value)).not.toContain('private-')
|
|
214
|
+
expect(execute).toHaveBeenCalled()
|
|
215
|
+
for (const [, policyParams] of execute.mock.calls) expect(policyParams).toEqual(expect.arrayContaining(['tenant-privacy', 'org-privacy']))
|
|
216
|
+
}
|
|
217
|
+
expect(otherIndex).toHaveBeenCalledTimes(1)
|
|
218
|
+
} finally {
|
|
219
|
+
if (previousToggle === undefined) delete process.env.TENANT_DATA_ENCRYPTION
|
|
220
|
+
else process.env.TENANT_DATA_ENCRYPTION = previousToggle
|
|
221
|
+
}
|
|
222
|
+
},
|
|
223
|
+
)
|
|
224
|
+
|
|
225
|
+
it.each(['protected-scope', 'other-scope', 'tenant-wide', 'related-scope'])('applies live profile and related-entity declarations for the trusted %s', async (scope) => {
|
|
226
|
+
const driver = createDriver()
|
|
227
|
+
const captured: CapturedEmbeddingInput = { value: null }
|
|
228
|
+
const getEncryptedFieldNames = jest.fn(async (entityId: string, tenantId: string | null | undefined, organizationId?: string | null) => {
|
|
229
|
+
if (tenantId !== 'tenant-privacy' || organizationId !== 'protected-scope') return []
|
|
230
|
+
return entityId === 'customers:customer_company_profile' ? ['cf:sizeBucket'] : ['status', 'lifecycleStage']
|
|
231
|
+
})
|
|
232
|
+
const organizationId = scope === 'tenant-wide' || scope === 'related-scope' ? undefined : scope
|
|
233
|
+
const record = { id: 'live-company', entity_id: 'live-entity', tenant_id: 'untrusted-tenant', organization_id: scope === 'tenant-wide' ? 'protected-scope' : scope === 'related-scope' ? undefined : 'untrusted-org', sizeBucket: 'private-size', 'cf:size_bucket': 'private-custom-size' }
|
|
234
|
+
const queryEngine = { query: async (entityId: string) => ({ items: [
|
|
235
|
+
entityId === 'customers:customer_entity'
|
|
236
|
+
? { id: 'live-entity', organization_id: scope === 'related-scope' ? 'protected-scope' : undefined, status: 'private-status', lifecycleStage: 'private-stage' }
|
|
237
|
+
: record,
|
|
238
|
+
], total: 1 }) } as QueryEngine
|
|
239
|
+
const indexer = createCustomerIndexer({ driver, embeddingService: createEmbeddingService(captured), queryEngine, encryptionService: { getEncryptedFieldNames } })
|
|
240
|
+
await indexer.indexRecordById({ entityId: 'customers:customer_company_profile', recordId: record.id, tenantId: 'tenant-privacy', organizationId })
|
|
241
|
+
expect(driver.upsertCalls).toHaveLength(1)
|
|
242
|
+
if (scope !== 'other-scope') expect(JSON.stringify(captured.value)).not.toContain('private-')
|
|
243
|
+
else expect(JSON.stringify(captured.value)).toContain('private-custom-size')
|
|
244
|
+
for (const entityId of ['customers:customer_company_profile', 'customers:customer_entity']) {
|
|
245
|
+
expect(getEncryptedFieldNames).toHaveBeenCalledWith(entityId, 'tenant-privacy', organizationId, { ignoreRuntimeHealth: true })
|
|
246
|
+
}
|
|
247
|
+
})
|
|
248
|
+
|
|
249
|
+
it('does not send decrypted fields when live encryption policy lookup fails', async () => {
|
|
250
|
+
const driver = createDriver()
|
|
251
|
+
const captured: CapturedEmbeddingInput = { value: null }
|
|
252
|
+
const indexer = createCustomerIndexer({
|
|
253
|
+
driver, embeddingService: createEmbeddingService(captured),
|
|
254
|
+
queryEngine: createQueryEngine({ id: 'policy-failure', size_bucket: 'private-size' }),
|
|
255
|
+
encryptionService: { getEncryptedFieldNames: async () => { throw new Error('[internal] Policy unavailable') } },
|
|
256
|
+
})
|
|
257
|
+
await indexer.indexRecordById({ entityId: 'customers:customer_company_profile', recordId: 'policy-failure', tenantId: 'tenant-privacy', organizationId: 'org-privacy' })
|
|
258
|
+
expect(captured.value).toBeNull()
|
|
259
|
+
expect(driver.upsertCalls).toHaveLength(0)
|
|
260
|
+
})
|
|
261
|
+
|
|
262
|
+
it.each(['missing-container', 'missing-service'])('does not embed customer fields with %s', async (failure) => {
|
|
263
|
+
const driver = createDriver()
|
|
264
|
+
const captured: CapturedEmbeddingInput = { value: null }
|
|
265
|
+
const company = customersSearchConfig.entities.find((entity) => entity.entityId === 'customers:customer_company_profile')!
|
|
266
|
+
const indexer = new SearchIndexer(new SearchService({ strategies: [new VectorSearchStrategy(createEmbeddingService(captured), driver)] }), [{ entities: [{
|
|
267
|
+
...company,
|
|
268
|
+
buildSource: (ctx) => company.buildSource!({ ...ctx, container: failure === 'missing-service' ? { resolve: () => undefined } : undefined }),
|
|
269
|
+
}] }])
|
|
270
|
+
await indexer.indexRecord({ entityId: company.entityId, recordId: 'missing-policy', tenantId: 'tenant-privacy', record: { size_bucket: 'private-size' } })
|
|
271
|
+
expect(captured.value).toBeNull()
|
|
272
|
+
expect(driver.upsertCalls).toHaveLength(0)
|
|
273
|
+
})
|
|
274
|
+
|
|
275
|
+
it.each(['throw', 'null', 'empty-string', 'empty-array', 'whitespace'].flatMap((failure) =>
|
|
276
|
+
['single', 'by-id', 'batch', 'bulk'].map((mode) => ({ failure, mode })),
|
|
277
|
+
))('fails closed for $failure sources through $mode indexing without blocking other strategies', async ({ failure, mode }) => {
|
|
278
|
+
const captured: CapturedEmbeddingInput = { value: null }
|
|
279
|
+
const driver = createDriver()
|
|
280
|
+
const otherIndex = jest.fn(async () => {})
|
|
281
|
+
const otherStrategy: SearchStrategy = {
|
|
282
|
+
id: 'fulltext', name: 'Fake fulltext', priority: 1,
|
|
283
|
+
isAvailable: async () => true, ensureReady: async () => {}, search: async () => [],
|
|
284
|
+
index: otherIndex, delete: async () => {},
|
|
285
|
+
}
|
|
286
|
+
const record = { id: 'source-failure', email: 'private-email@example.com', password_hash: 'private-password-hash' }
|
|
287
|
+
const indexer = new SearchIndexer(new SearchService({ strategies: [
|
|
288
|
+
new VectorSearchStrategy(createEmbeddingService(captured), driver), otherStrategy,
|
|
289
|
+
] }), [{ entities: [{
|
|
290
|
+
entityId: 'customer_accounts:customer_user',
|
|
291
|
+
buildSource: () => {
|
|
292
|
+
if (failure === 'throw') throw new Error('[internal] Injected source failure')
|
|
293
|
+
if (failure === 'null') return null
|
|
294
|
+
return { text: failure === 'empty-string' ? '' : failure === 'whitespace' ? ' \n ' : [] }
|
|
295
|
+
},
|
|
296
|
+
}] }], { queryEngine: createQueryEngine(record) })
|
|
297
|
+
const params = { entityId: 'customer_accounts:customer_user', recordId: record.id, tenantId: 'tenant-privacy', organizationId: 'org-privacy' }
|
|
298
|
+
if (mode === 'single') await indexer.indexRecord({ ...params, record })
|
|
299
|
+
if (mode === 'by-id') await indexer.indexRecordById(params)
|
|
300
|
+
if (mode === 'batch') await indexer.indexRecordsById({ ...params, items: [params] })
|
|
301
|
+
if (mode === 'bulk') await indexer.bulkIndexRecords([{ ...params, record }])
|
|
302
|
+
expect(captured.value).toBeNull()
|
|
303
|
+
expect(driver.upsertCalls).toHaveLength(0)
|
|
304
|
+
expect(otherIndex).toHaveBeenCalledTimes(1)
|
|
305
|
+
expect(otherIndex).toHaveBeenCalledWith(expect.objectContaining({ tenantId: 'tenant-privacy', organizationId: 'org-privacy' }))
|
|
306
|
+
})
|
|
307
|
+
|
|
308
|
+
it('keeps protected fields out of a generated presenter when a custom source omits its presenter', async () => {
|
|
309
|
+
const driver = createDriver()
|
|
310
|
+
const service = createService({
|
|
311
|
+
driver,
|
|
312
|
+
embeddingService: createEmbeddingService({ value: null }),
|
|
313
|
+
queryEngine: createQueryEngine({
|
|
314
|
+
id: 'fallback-record',
|
|
315
|
+
title: 'private-title',
|
|
316
|
+
description: 'private-description',
|
|
317
|
+
'cf:summary': 'private-summary',
|
|
318
|
+
}),
|
|
319
|
+
moduleConfigs: [{ entities: [{
|
|
320
|
+
entityId: 'demo:item',
|
|
321
|
+
fieldPolicy: { searchable: ['title', 'description', 'summary'], hashOnly: ['title'], excluded: ['description', 'summary'] },
|
|
322
|
+
buildSource: () => ({ input: 'Safe source', checksumSource: 'safe-checksum' }),
|
|
323
|
+
}] }],
|
|
324
|
+
})
|
|
325
|
+
|
|
326
|
+
await service.indexRecord({ entityId: 'demo:item', recordId: 'fallback-record', tenantId: 'tenant-privacy' })
|
|
327
|
+
expect(driver.upsertCalls).toHaveLength(1)
|
|
328
|
+
expect(JSON.stringify(driver.upsertCalls[0])).not.toContain('private-')
|
|
329
|
+
expect(driver.upsertCalls[0].resultTitle).toBe('fallback-record')
|
|
330
|
+
})
|
|
331
|
+
|
|
332
|
+
const customerIndexingCases = [
|
|
333
|
+
'customers:customer_person_profile',
|
|
334
|
+
'customers:customer_company_profile',
|
|
335
|
+
'customer_accounts:customer_user',
|
|
336
|
+
].flatMap((entityId) => ['by-id', 'single', 'bulk'].map((mode) => ({ entityId, mode })))
|
|
337
|
+
it.each(customerIndexingCases)('keeps contact and excluded custom values out of $entityId $mode embeddings and stored metadata', async ({ entityId, mode }) => {
|
|
338
|
+
const driver = createDriver()
|
|
339
|
+
const captured: CapturedEmbeddingInput = { value: null }
|
|
340
|
+
const record = {
|
|
341
|
+
id: `record-${entityId}`,
|
|
342
|
+
entity_id: `entity-${entityId}`,
|
|
343
|
+
display_name: 'Permitted Presenter Name',
|
|
344
|
+
email: 'private-account@example.com',
|
|
345
|
+
primary_email: 'private-profile@example.com',
|
|
346
|
+
primary_phone: '+48-555-030-7826',
|
|
347
|
+
password_hash: 'private-password-hash',
|
|
348
|
+
'cf:personal_email': 'private-custom@example.com',
|
|
349
|
+
cf_government_id: 'private-government-id',
|
|
350
|
+
'cf:bank_account': 'private-bank-account',
|
|
351
|
+
'cf:summary': 'private-summary',
|
|
352
|
+
cf_unknown: 'private-unknown',
|
|
353
|
+
}
|
|
354
|
+
const related = {
|
|
355
|
+
id: `entity-${entityId}`,
|
|
356
|
+
display_name: 'Permitted Customer Name',
|
|
357
|
+
primary_email: 'private-related@example.com',
|
|
358
|
+
primary_phone: '+48-555-040-7826',
|
|
359
|
+
status: 'active',
|
|
360
|
+
'cf:government_id': 'private-related-government',
|
|
361
|
+
cf_personal_email: 'private-related-custom@example.com',
|
|
362
|
+
}
|
|
363
|
+
const query = jest.fn(async (queriedEntity: string, _options: unknown) => ({
|
|
364
|
+
items: [queriedEntity === 'customers:customer_entity' ? related : record],
|
|
365
|
+
total: 1,
|
|
366
|
+
}))
|
|
367
|
+
const indexer = createCustomerIndexer({ driver, embeddingService: createEmbeddingService(captured), queryEngine: { query } as unknown as QueryEngine })
|
|
368
|
+
|
|
369
|
+
const params = {
|
|
370
|
+
entityId,
|
|
371
|
+
recordId: record.id,
|
|
372
|
+
tenantId: 'tenant-privacy',
|
|
373
|
+
organizationId: 'org-privacy',
|
|
374
|
+
}
|
|
375
|
+
if (mode === 'by-id') {
|
|
376
|
+
await indexer.indexRecordById(params)
|
|
377
|
+
} else {
|
|
378
|
+
const recordParams = { ...params, record, customFields: { personal_email: 'private-separate@example.com' } }
|
|
379
|
+
if (mode === 'bulk') {
|
|
380
|
+
await indexer.bulkIndexRecords([recordParams])
|
|
381
|
+
} else {
|
|
382
|
+
await indexer.indexRecord(recordParams)
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
expect(captured.value).not.toBeNull()
|
|
387
|
+
expect(driver.upsertCalls).toHaveLength(1)
|
|
388
|
+
const serialized = JSON.stringify({ input: captured.value, document: driver.upsertCalls[0] })
|
|
389
|
+
expect(serialized).not.toContain('private-')
|
|
390
|
+
expect(serialized).not.toContain('+48-555-')
|
|
391
|
+
expect(JSON.stringify(captured.value)).not.toContain('Permitted')
|
|
392
|
+
expect(driver.upsertCalls[0].tenantId).toBe('tenant-privacy')
|
|
393
|
+
expect(driver.upsertCalls[0].organizationId).toBe('org-privacy')
|
|
394
|
+
for (const [, options] of query.mock.calls) {
|
|
395
|
+
expect(options).toEqual(expect.objectContaining({ tenantId: 'tenant-privacy', organizationId: 'org-privacy' }))
|
|
396
|
+
}
|
|
397
|
+
})
|
|
398
|
+
|
|
399
|
+
it.each(['excluded', 'hashOnly', 'unlisted'] as const)('does not restore a %s custom summary through vector snapshot fallback', async (protection) => {
|
|
400
|
+
const driver = createDriver()
|
|
401
|
+
const service = createService({
|
|
402
|
+
driver,
|
|
403
|
+
embeddingService: createEmbeddingService({ value: null }),
|
|
404
|
+
queryEngine: createQueryEngine({ id: 'snapshot-record', name: 'Safe name', 'cf:summary': 'private-snapshot' }),
|
|
405
|
+
moduleConfigs: [{ entities: [{
|
|
406
|
+
entityId: 'demo:item',
|
|
407
|
+
fieldPolicy: {
|
|
408
|
+
searchable: protection === 'unlisted' ? ['name'] : ['name', 'summary'],
|
|
409
|
+
hashOnly: protection === 'hashOnly' ? ['summary'] : [],
|
|
410
|
+
excluded: protection === 'excluded' ? ['summary'] : [],
|
|
411
|
+
},
|
|
412
|
+
buildSource: () => ({ input: 'Safe source', presenter: { title: 'Safe name' }, checksumSource: 'safe-checksum' }),
|
|
413
|
+
}] }],
|
|
414
|
+
})
|
|
415
|
+
|
|
416
|
+
await service.indexRecord({ entityId: 'demo:item', recordId: 'snapshot-record', tenantId: 'tenant-privacy' })
|
|
417
|
+
expect(driver.upsertCalls).toHaveLength(1)
|
|
418
|
+
expect(JSON.stringify(driver.upsertCalls[0])).not.toContain('private-snapshot')
|
|
419
|
+
})
|
|
420
|
+
})
|
|
421
|
+
|
|
143
422
|
describe('VectorIndexService encryption fail-closed (issue #2716)', () => {
|
|
144
423
|
const enabledEncryptionThatThrows = (): TenantDataEncryptionService =>
|
|
145
424
|
({
|
package/src/di.ts
CHANGED
|
@@ -95,6 +95,15 @@ export function registerSearchModule(
|
|
|
95
95
|
container: SearchContainer,
|
|
96
96
|
options?: SearchModuleOptions,
|
|
97
97
|
): void {
|
|
98
|
+
const moduleConfigs = (options?.moduleConfigs ?? []).map((moduleConfig) => ({
|
|
99
|
+
...moduleConfig,
|
|
100
|
+
entities: moduleConfig.entities.map((entityConfig) => {
|
|
101
|
+
const buildSource = entityConfig.buildSource
|
|
102
|
+
return buildSource
|
|
103
|
+
? { ...entityConfig, buildSource: (ctx: Parameters<typeof buildSource>[0]) => buildSource.call(entityConfig, { ...ctx, container }) }
|
|
104
|
+
: entityConfig
|
|
105
|
+
}),
|
|
106
|
+
}))
|
|
98
107
|
const strategies: SearchStrategy[] = []
|
|
99
108
|
|
|
100
109
|
// Token strategy (always available unless explicitly skipped)
|
|
@@ -127,7 +136,7 @@ export function registerSearchModule(
|
|
|
127
136
|
|
|
128
137
|
// Build entity config map for field policy resolution
|
|
129
138
|
const entityConfigMap = new Map<EntityId, SearchEntityConfig>()
|
|
130
|
-
for (const moduleConfig of
|
|
139
|
+
for (const moduleConfig of moduleConfigs) {
|
|
131
140
|
for (const entityConfig of moduleConfig.entities) {
|
|
132
141
|
if (entityConfig.enabled !== false) {
|
|
133
142
|
entityConfigMap.set(entityConfig.entityId as EntityId, entityConfig)
|
|
@@ -226,9 +235,6 @@ export function registerSearchModule(
|
|
|
226
235
|
presenterEnricher,
|
|
227
236
|
})
|
|
228
237
|
|
|
229
|
-
// Create search indexer with module configs
|
|
230
|
-
const moduleConfigs = options?.moduleConfigs ?? []
|
|
231
|
-
|
|
232
238
|
// Try to resolve fulltextIndexQueue for queue-based reindexing
|
|
233
239
|
let fulltextQueue: Queue<FulltextIndexJobPayload> | undefined
|
|
234
240
|
try {
|
|
@@ -191,9 +191,16 @@ export class SearchIndexer {
|
|
|
191
191
|
* Index a record in the search service.
|
|
192
192
|
*/
|
|
193
193
|
async indexRecord(params: IndexRecordParams): Promise<void> {
|
|
194
|
+
const indexableRecord = await this.buildIndexableRecord(params)
|
|
195
|
+
if (indexableRecord) {
|
|
196
|
+
await this.searchService.index(indexableRecord)
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
private async buildIndexableRecord(params: IndexRecordParams): Promise<IndexableRecord | null> {
|
|
194
201
|
const config = this.entityConfigMap.get(params.entityId)
|
|
195
202
|
if (!config || config.enabled === false) {
|
|
196
|
-
return
|
|
203
|
+
return null
|
|
197
204
|
}
|
|
198
205
|
|
|
199
206
|
const buildContext: SearchBuildContext = {
|
|
@@ -205,7 +212,7 @@ export class SearchIndexer {
|
|
|
205
212
|
}
|
|
206
213
|
|
|
207
214
|
// Try buildSource first (provides text, presenter, links, checksumSource)
|
|
208
|
-
let text: string | string[] | undefined
|
|
215
|
+
let text: string | string[] | undefined = config.buildSource ? [] : undefined
|
|
209
216
|
let presenter: SearchResultPresenter | undefined
|
|
210
217
|
let url: string | undefined
|
|
211
218
|
let links: SearchResultLink[] | undefined
|
|
@@ -215,7 +222,7 @@ export class SearchIndexer {
|
|
|
215
222
|
try {
|
|
216
223
|
const source = await config.buildSource(buildContext)
|
|
217
224
|
if (source) {
|
|
218
|
-
text = source.text
|
|
225
|
+
text = source.text ?? []
|
|
219
226
|
if (source.presenter) presenter = source.presenter
|
|
220
227
|
if (source.links) links = source.links
|
|
221
228
|
if (source.checksumSource !== undefined) checksumSource = source.checksumSource
|
|
@@ -271,8 +278,7 @@ export class SearchIndexer {
|
|
|
271
278
|
}
|
|
272
279
|
}
|
|
273
280
|
|
|
274
|
-
|
|
275
|
-
const indexableRecord: IndexableRecord = {
|
|
281
|
+
return {
|
|
276
282
|
entityId: params.entityId,
|
|
277
283
|
recordId: params.recordId,
|
|
278
284
|
tenantId: params.tenantId,
|
|
@@ -284,8 +290,6 @@ export class SearchIndexer {
|
|
|
284
290
|
text,
|
|
285
291
|
checksumSource,
|
|
286
292
|
}
|
|
287
|
-
|
|
288
|
-
await this.searchService.index(indexableRecord)
|
|
289
293
|
}
|
|
290
294
|
|
|
291
295
|
/**
|
|
@@ -591,58 +595,10 @@ export class SearchIndexer {
|
|
|
591
595
|
const indexableRecords: IndexableRecord[] = []
|
|
592
596
|
|
|
593
597
|
for (const param of params) {
|
|
594
|
-
const
|
|
595
|
-
if (
|
|
596
|
-
|
|
598
|
+
const indexableRecord = await this.buildIndexableRecord(param)
|
|
599
|
+
if (indexableRecord) {
|
|
600
|
+
indexableRecords.push(indexableRecord)
|
|
597
601
|
}
|
|
598
|
-
|
|
599
|
-
const buildContext: SearchBuildContext = {
|
|
600
|
-
record: param.record,
|
|
601
|
-
customFields: param.customFields ?? {},
|
|
602
|
-
organizationId: param.organizationId,
|
|
603
|
-
tenantId: param.tenantId,
|
|
604
|
-
}
|
|
605
|
-
|
|
606
|
-
let presenter: SearchResultPresenter | undefined
|
|
607
|
-
if (config.formatResult) {
|
|
608
|
-
try {
|
|
609
|
-
const result = await config.formatResult(buildContext)
|
|
610
|
-
if (result) presenter = result
|
|
611
|
-
} catch {
|
|
612
|
-
// Skip presenter on error
|
|
613
|
-
}
|
|
614
|
-
}
|
|
615
|
-
|
|
616
|
-
let url: string | undefined
|
|
617
|
-
if (config.resolveUrl) {
|
|
618
|
-
try {
|
|
619
|
-
const result = await config.resolveUrl(buildContext)
|
|
620
|
-
if (result) url = result
|
|
621
|
-
} catch {
|
|
622
|
-
// Skip URL on error
|
|
623
|
-
}
|
|
624
|
-
}
|
|
625
|
-
|
|
626
|
-
let links: SearchResultLink[] | undefined
|
|
627
|
-
if (config.resolveLinks) {
|
|
628
|
-
try {
|
|
629
|
-
const result = await config.resolveLinks(buildContext)
|
|
630
|
-
if (result) links = result
|
|
631
|
-
} catch {
|
|
632
|
-
// Skip links on error
|
|
633
|
-
}
|
|
634
|
-
}
|
|
635
|
-
|
|
636
|
-
indexableRecords.push({
|
|
637
|
-
entityId: param.entityId,
|
|
638
|
-
recordId: param.recordId,
|
|
639
|
-
tenantId: param.tenantId,
|
|
640
|
-
organizationId: param.organizationId,
|
|
641
|
-
fields: param.record,
|
|
642
|
-
presenter,
|
|
643
|
-
url,
|
|
644
|
-
links,
|
|
645
|
-
})
|
|
646
602
|
}
|
|
647
603
|
|
|
648
604
|
if (indexableRecords.length > 0) {
|
|
@@ -1171,7 +1127,7 @@ export class SearchIndexer {
|
|
|
1171
1127
|
}
|
|
1172
1128
|
|
|
1173
1129
|
// Try buildSource first (provides text, presenter, links, checksumSource)
|
|
1174
|
-
let text: string | string[] | undefined
|
|
1130
|
+
let text: string | string[] | undefined = config.buildSource ? [] : undefined
|
|
1175
1131
|
let presenter: SearchResultPresenter | undefined
|
|
1176
1132
|
let url: string | undefined
|
|
1177
1133
|
let links: SearchResultLink[] | undefined
|
|
@@ -1181,7 +1137,7 @@ export class SearchIndexer {
|
|
|
1181
1137
|
try {
|
|
1182
1138
|
const source = await config.buildSource(buildContext)
|
|
1183
1139
|
if (source) {
|
|
1184
|
-
text = source.text
|
|
1140
|
+
text = source.text ?? []
|
|
1185
1141
|
if (source.presenter) presenter = source.presenter
|
|
1186
1142
|
if (source.links) links = source.links
|
|
1187
1143
|
if (source.checksumSource !== undefined) checksumSource = source.checksumSource
|
|
@@ -18,6 +18,7 @@ import {
|
|
|
18
18
|
} from '@open-mercato/shared/lib/custom-fields/kinds'
|
|
19
19
|
import { createLogger } from '@open-mercato/shared/lib/logger'
|
|
20
20
|
import { extractFallbackPresenter } from './fallback-presenter'
|
|
21
|
+
import { extractSearchableFields } from './field-policy'
|
|
21
22
|
import { needsSearchResultEnrichment } from './search-result-enrichment'
|
|
22
23
|
|
|
23
24
|
/** Maximum number of record IDs per batch query to avoid hitting DB parameter limits */
|
|
@@ -308,7 +309,7 @@ async function computePresenterAndLinks(
|
|
|
308
309
|
|
|
309
310
|
// Fallback presenter: extract from doc fields directly
|
|
310
311
|
if (!presenter) {
|
|
311
|
-
presenter = extractFallbackPresenter(doc, entityId, recordId)
|
|
312
|
+
presenter = extractFallbackPresenter(extractSearchableFields(doc, { fieldPolicy: config?.fieldPolicy }), entityId, recordId)
|
|
312
313
|
}
|
|
313
314
|
|
|
314
315
|
// Resolve URL from config
|
|
@@ -138,10 +138,10 @@ export class VectorSearchStrategy implements SearchStrategy {
|
|
|
138
138
|
async index(record: IndexableRecord): Promise<void> {
|
|
139
139
|
await this.ensureReady()
|
|
140
140
|
// Use text from buildSource if available, otherwise fall back to generic extraction
|
|
141
|
-
const textContent = record.text
|
|
141
|
+
const textContent = record.text !== undefined
|
|
142
142
|
? (Array.isArray(record.text) ? record.text.join('\n') : record.text)
|
|
143
143
|
: this.buildTextContent(record)
|
|
144
|
-
if (!textContent) return
|
|
144
|
+
if (!textContent.trim()) return
|
|
145
145
|
|
|
146
146
|
const embedding = await this.embeddingService.createEmbedding(textContent)
|
|
147
147
|
|
|
@@ -20,6 +20,7 @@ import { computeChecksum } from './checksum'
|
|
|
20
20
|
import { EmbeddingService } from './embedding'
|
|
21
21
|
import { logVectorOperation } from '../lib/vector-logs'
|
|
22
22
|
import { searchDebug, searchDebugWarn } from '../../lib/debug'
|
|
23
|
+
import { extractSearchableFields } from '../../lib/field-policy'
|
|
23
24
|
|
|
24
25
|
type ContainerResolver = () => unknown
|
|
25
26
|
const VECTOR_ENTRY_ENCRYPTION_ENTITY_ID = 'vector:vector_search'
|
|
@@ -476,10 +477,12 @@ export class VectorIndexService {
|
|
|
476
477
|
organizationId: resolvedOrgId,
|
|
477
478
|
})
|
|
478
479
|
|
|
479
|
-
const
|
|
480
|
+
const metadataRecord = extractSearchableFields(record, { fieldPolicy: entry.config.fieldPolicy })
|
|
481
|
+
const metadataCustomFields = extractSearchableFields(customFields, { fieldPolicy: entry.config.fieldPolicy })
|
|
482
|
+
const normalizedPresenter = this.ensurePresenter(presenter, metadataRecord, metadataCustomFields, args.recordId, args.entityId)
|
|
480
483
|
const normalizedLinks = Array.isArray(links) && links.length ? links : null
|
|
481
|
-
const snapshot = this.deriveSnapshot(
|
|
482
|
-
const rawResultTitle = this.resolveResultTitle(normalizedPresenter,
|
|
484
|
+
const snapshot = this.deriveSnapshot(metadataRecord, metadataCustomFields)
|
|
485
|
+
const rawResultTitle = this.resolveResultTitle(normalizedPresenter, metadataRecord, metadataCustomFields, args.recordId)
|
|
483
486
|
const rawResultSubtitle = normalizedPresenter.subtitle ?? snapshot ?? null
|
|
484
487
|
const rawResultIcon = normalizedPresenter.icon ?? this.mapDefaultIcon(args.entityId)
|
|
485
488
|
const resultBadge = normalizedPresenter.badge ?? null
|
|
@@ -761,9 +764,10 @@ export class VectorIndexService {
|
|
|
761
764
|
if (formatted) return formatted
|
|
762
765
|
}
|
|
763
766
|
if (fallback) return fallback
|
|
764
|
-
const
|
|
767
|
+
const fields = extractSearchableFields(ctx.record, { fieldPolicy: config.fieldPolicy })
|
|
768
|
+
const nameLike = fields.displayName || fields.title || fields.name
|
|
765
769
|
if (typeof nameLike === 'string' && nameLike.trim().length > 0) {
|
|
766
|
-
const subtitle =
|
|
770
|
+
const subtitle = fields.description || fields.summary
|
|
767
771
|
return {
|
|
768
772
|
title: nameLike,
|
|
769
773
|
subtitle: typeof subtitle === 'string' ? subtitle : undefined,
|