@open-mercato/search 0.10.1-develop.8225.1.1766b080b2 → 0.10.1-develop.8229.1.61fb2a0f4c
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +8 -7
- package/dist/di.js +16 -5
- package/dist/di.js.map +3 -3
- package/dist/indexer/search-indexer.js +136 -36
- package/dist/indexer/search-indexer.js.map +2 -2
- package/dist/lib/debug.js +7 -3
- package/dist/lib/debug.js.map +2 -2
- package/dist/lib/encrypted-custom-field-resolver.js +14 -6
- package/dist/lib/encrypted-custom-field-resolver.js.map +2 -2
- package/dist/vector/services/vector-index.service.js +141 -27
- package/dist/vector/services/vector-index.service.js.map +2 -2
- package/package.json +4 -4
- package/src/__tests__/debug.test.ts +42 -1
- package/src/__tests__/di-encrypted-custom-fields.test.ts +113 -0
- package/src/__tests__/encrypted-custom-field-resolver.test.ts +76 -28
- package/src/__tests__/plaintext-custom-field-boundaries.test.ts +2 -2
- package/src/__tests__/search-indexer-encrypted-custom-fields.boundary.test.ts +214 -0
- package/src/__tests__/search-indexer-encrypted-custom-fields.customers.test.ts +146 -0
- package/src/__tests__/search-indexer-encrypted-custom-fields.test.ts +425 -0
- package/src/__tests__/vector-index-security.test.ts +249 -3
- package/src/di.ts +19 -5
- package/src/indexer/search-indexer.ts +188 -36
- package/src/lib/debug.ts +16 -3
- package/src/lib/encrypted-custom-field-resolver.ts +35 -16
- package/src/vector/services/vector-index.service.ts +178 -23
|
@@ -1,43 +1,91 @@
|
|
|
1
|
-
import
|
|
2
|
-
|
|
1
|
+
import {
|
|
2
|
+
Kysely,
|
|
3
|
+
PostgresAdapter,
|
|
4
|
+
PostgresIntrospector,
|
|
5
|
+
PostgresQueryCompiler,
|
|
6
|
+
type CompiledQuery,
|
|
7
|
+
type DatabaseConnection,
|
|
8
|
+
type Driver,
|
|
9
|
+
} from 'kysely'
|
|
10
|
+
import {
|
|
11
|
+
createEncryptedCustomFieldKeysResolver,
|
|
12
|
+
type CustomFieldDefinitionsDatabase,
|
|
13
|
+
} from '../lib/encrypted-custom-field-resolver'
|
|
3
14
|
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
15
|
+
function createDatabase(rows: Array<Record<string, unknown>>) {
|
|
16
|
+
const queries: CompiledQuery[] = []
|
|
17
|
+
const connection: DatabaseConnection = {
|
|
18
|
+
executeQuery: async <R>(compiled: CompiledQuery) => {
|
|
19
|
+
queries.push(compiled)
|
|
20
|
+
return { rows: rows as R[] }
|
|
21
|
+
},
|
|
22
|
+
streamQuery: () => {
|
|
23
|
+
throw new Error('[internal] streaming is not supported in this test')
|
|
24
|
+
},
|
|
25
|
+
}
|
|
26
|
+
const driver: Driver = {
|
|
27
|
+
init: async () => undefined,
|
|
28
|
+
acquireConnection: async () => connection,
|
|
29
|
+
beginTransaction: async () => undefined,
|
|
30
|
+
commitTransaction: async () => undefined,
|
|
31
|
+
rollbackTransaction: async () => undefined,
|
|
32
|
+
releaseConnection: async () => undefined,
|
|
33
|
+
destroy: async () => undefined,
|
|
12
34
|
}
|
|
13
|
-
const
|
|
14
|
-
|
|
35
|
+
const db = new Kysely<CustomFieldDefinitionsDatabase>({
|
|
36
|
+
dialect: {
|
|
37
|
+
createAdapter: () => new PostgresAdapter(),
|
|
38
|
+
createDriver: () => driver,
|
|
39
|
+
createIntrospector: (instance) => new PostgresIntrospector(instance),
|
|
40
|
+
createQueryCompiler: () => new PostgresQueryCompiler(),
|
|
41
|
+
},
|
|
42
|
+
})
|
|
43
|
+
return { db, queries }
|
|
15
44
|
}
|
|
16
45
|
|
|
17
|
-
describe('encrypted custom-field
|
|
18
|
-
it('
|
|
19
|
-
const {
|
|
20
|
-
{ key: '
|
|
21
|
-
{ key: '
|
|
22
|
-
{ key: '
|
|
23
|
-
{ key: '', config_json: { encrypted:
|
|
46
|
+
describe('encrypted custom-field keys resolver', () => {
|
|
47
|
+
it('returns the normalized keys of encrypted definitions', async () => {
|
|
48
|
+
const { db } = createDatabase([
|
|
49
|
+
{ key: 'Tax_Number', config_json: { encrypted: true } },
|
|
50
|
+
{ key: 'national_id', config_json: '{"encrypted":true,"label":"National ID"}' },
|
|
51
|
+
{ key: 'tax_number', config_json: { encrypted: true } },
|
|
52
|
+
{ key: 'industry_note', config_json: { encrypted: false } },
|
|
53
|
+
{ key: 'priority', config_json: null },
|
|
54
|
+
{ key: ' ', config_json: { encrypted: true } },
|
|
24
55
|
])
|
|
25
56
|
|
|
26
|
-
const keys = await createEncryptedCustomFieldKeysResolver(
|
|
57
|
+
const keys = await createEncryptedCustomFieldKeysResolver(db)('customers:customer_company_profile', 'tenant-1')
|
|
27
58
|
|
|
28
|
-
expect(keys).toEqual(
|
|
29
|
-
expect(predicates).toEqual([['entity_id', '=', 'example:todo'], ['deleted_at', 'is', null]])
|
|
59
|
+
expect(Array.from(keys).sort()).toEqual(['national_id', 'tax_number'])
|
|
30
60
|
})
|
|
31
61
|
|
|
32
|
-
it('returns
|
|
33
|
-
const {
|
|
62
|
+
it('also returns the query engine alias form of a key with characters outside snake_case', async () => {
|
|
63
|
+
const { db } = createDatabase([{ key: 'Order-Ref', config_json: { encrypted: true } }])
|
|
34
64
|
|
|
35
|
-
await
|
|
65
|
+
const keys = await createEncryptedCustomFieldKeysResolver(db)('customers:customer_company_profile', 'tenant-1')
|
|
66
|
+
|
|
67
|
+
expect(Array.from(keys).sort()).toEqual(['order-ref', 'order_ref'])
|
|
36
68
|
})
|
|
37
69
|
|
|
38
|
-
it('
|
|
39
|
-
const
|
|
70
|
+
it('reads non-deleted encrypted-looking definitions of the entity for the tenant and global scope', async () => {
|
|
71
|
+
const { db, queries } = createDatabase([])
|
|
72
|
+
|
|
73
|
+
await createEncryptedCustomFieldKeysResolver(db)('customers:customer_company_profile', 'tenant-1')
|
|
74
|
+
|
|
75
|
+
expect(queries).toHaveLength(1)
|
|
76
|
+
expect(queries[0].sql).toBe(
|
|
77
|
+
'select "key", "config_json" from "custom_field_defs" where "entity_id" = $1 and "deleted_at" is null'
|
|
78
|
+
+ ' and config_json::text like $2 and ("tenant_id" = $3 or "tenant_id" is null)',
|
|
79
|
+
)
|
|
80
|
+
expect(queries[0].parameters).toEqual(['customers:customer_company_profile', '%encrypted%', 'tenant-1'])
|
|
81
|
+
})
|
|
82
|
+
|
|
83
|
+
it('reads every tenant when no tenant is given', async () => {
|
|
84
|
+
const { db, queries } = createDatabase([])
|
|
85
|
+
|
|
86
|
+
await createEncryptedCustomFieldKeysResolver(db)('customers:customer_company_profile', null)
|
|
40
87
|
|
|
41
|
-
|
|
88
|
+
expect(queries[0].sql).not.toContain('tenant_id')
|
|
89
|
+
expect(queries[0].parameters).toEqual(['customers:customer_company_profile', '%encrypted%'])
|
|
42
90
|
})
|
|
43
91
|
})
|
|
@@ -242,13 +242,13 @@ describe('plaintext custom-field boundaries (#7894)', () => {
|
|
|
242
242
|
expect(resolver).toHaveBeenCalledTimes(2)
|
|
243
243
|
})
|
|
244
244
|
|
|
245
|
-
it('preserves explicit source text
|
|
245
|
+
it('preserves explicit source text while a resolver-less indexer excludes custom fields', async () => {
|
|
246
246
|
const harness = createHarness({ resolver: null, buildSource: async () => ({ text: 'Explicit source text' }) })
|
|
247
247
|
|
|
248
248
|
await harness.indexer.indexRecord({ entityId, recordId: 'record-1', tenantId: 'tenant-1', record: makeFields() })
|
|
249
249
|
|
|
250
250
|
expect(harness.embeddings).toEqual(['Explicit source text'])
|
|
251
|
-
expect(harness.documents[0].fields.cf_secret).
|
|
251
|
+
expect(harness.documents[0].fields.cf_secret).toBeUndefined()
|
|
252
252
|
})
|
|
253
253
|
|
|
254
254
|
it('keeps exclusion metadata effective after JSON serialization for custom direct clients', async () => {
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
import { SearchIndexer } from '../indexer/search-indexer'
|
|
2
|
+
import { SearchService } from '../service'
|
|
3
|
+
import { FullTextSearchStrategy } from '../strategies/fulltext.strategy'
|
|
4
|
+
import { VectorSearchStrategy, type EmbeddingService } from '../strategies/vector.strategy'
|
|
5
|
+
import type { FullTextSearchDocument, FullTextSearchDriver } from '../fulltext/types'
|
|
6
|
+
import type { VectorDriver, VectorDriverDocument } from '../vector/types'
|
|
7
|
+
import type { SearchBuildContext, SearchModuleConfig } from '../types'
|
|
8
|
+
import { registerTelemetryRuntime, type TelemetryRuntime } from '@open-mercato/shared/lib/telemetry/runtime'
|
|
9
|
+
|
|
10
|
+
const companyEntityId = 'customers:customer_company_profile'
|
|
11
|
+
const secret = 'GB-SECRET-123456'
|
|
12
|
+
|
|
13
|
+
function describeRecord(ctx: SearchBuildContext): string {
|
|
14
|
+
return JSON.stringify({ record: ctx.record, customFields: ctx.customFields })
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
const moduleConfigs: SearchModuleConfig[] = [
|
|
18
|
+
{
|
|
19
|
+
entities: [
|
|
20
|
+
{
|
|
21
|
+
entityId: companyEntityId,
|
|
22
|
+
enabled: true,
|
|
23
|
+
buildSource: async (ctx) => ({ text: describeRecord(ctx) }),
|
|
24
|
+
formatResult: async (ctx) => ({ title: 'Acme Ltd', subtitle: describeRecord(ctx) }),
|
|
25
|
+
resolveUrl: async (ctx) => `/backend/companies?context=${encodeURIComponent(describeRecord(ctx))}`,
|
|
26
|
+
resolveLinks: async (ctx) => [{ href: '/backend/companies', label: describeRecord(ctx), kind: 'primary' }],
|
|
27
|
+
},
|
|
28
|
+
],
|
|
29
|
+
},
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
function createRecord(): Record<string, unknown> {
|
|
33
|
+
return {
|
|
34
|
+
id: 'profile-1',
|
|
35
|
+
legal_name: 'Acme Ltd',
|
|
36
|
+
'cf:tax_number': secret,
|
|
37
|
+
'cf:industry_note': 'Wholesale',
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function createFulltextDriver() {
|
|
42
|
+
const documents: FullTextSearchDocument[] = []
|
|
43
|
+
const driver: FullTextSearchDriver = {
|
|
44
|
+
id: 'meilisearch',
|
|
45
|
+
ensureReady: async () => undefined,
|
|
46
|
+
isHealthy: async () => true,
|
|
47
|
+
search: async () => [],
|
|
48
|
+
index: async (doc) => {
|
|
49
|
+
documents.push(doc)
|
|
50
|
+
},
|
|
51
|
+
bulkIndex: async (docs) => {
|
|
52
|
+
documents.push(...docs)
|
|
53
|
+
},
|
|
54
|
+
delete: async () => undefined,
|
|
55
|
+
}
|
|
56
|
+
return { driver, documents }
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function createVectorBackend() {
|
|
60
|
+
const embeddedTexts: string[] = []
|
|
61
|
+
const upserted: VectorDriverDocument[] = []
|
|
62
|
+
const embeddingService: EmbeddingService = {
|
|
63
|
+
available: true,
|
|
64
|
+
createEmbedding: async (text) => {
|
|
65
|
+
embeddedTexts.push(text)
|
|
66
|
+
return [0.1, 0.2]
|
|
67
|
+
},
|
|
68
|
+
}
|
|
69
|
+
const driver: VectorDriver = {
|
|
70
|
+
id: 'pgvector',
|
|
71
|
+
ensureReady: async () => undefined,
|
|
72
|
+
upsert: async (doc) => {
|
|
73
|
+
upserted.push(doc)
|
|
74
|
+
},
|
|
75
|
+
delete: async () => undefined,
|
|
76
|
+
query: async () => [],
|
|
77
|
+
getChecksum: async () => null,
|
|
78
|
+
}
|
|
79
|
+
return { embeddingService, driver, embeddedTexts, upserted }
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const encryptedTaxNumber = async () => new Set(['tax_number'])
|
|
83
|
+
|
|
84
|
+
describe('SearchIndexer encrypted custom fields at the strategy boundary', () => {
|
|
85
|
+
it('keeps encrypted values out of the documents bulkIndexRecords sends to the fulltext driver', async () => {
|
|
86
|
+
const { driver, documents } = createFulltextDriver()
|
|
87
|
+
const searchService = new SearchService({ strategies: [new FullTextSearchStrategy(driver)] })
|
|
88
|
+
const indexer = new SearchIndexer(searchService, moduleConfigs, {
|
|
89
|
+
encryptedCustomFieldKeysResolver: encryptedTaxNumber,
|
|
90
|
+
})
|
|
91
|
+
|
|
92
|
+
await indexer.bulkIndexRecords([
|
|
93
|
+
{
|
|
94
|
+
entityId: companyEntityId,
|
|
95
|
+
recordId: 'profile-1',
|
|
96
|
+
tenantId: 'tenant-1',
|
|
97
|
+
organizationId: 'org-1',
|
|
98
|
+
record: createRecord(),
|
|
99
|
+
customFields: { tax_number: secret, industry_note: 'Wholesale' },
|
|
100
|
+
},
|
|
101
|
+
])
|
|
102
|
+
|
|
103
|
+
expect(documents).toHaveLength(1)
|
|
104
|
+
const [document] = documents
|
|
105
|
+
const callbackOutput = JSON.stringify({ presenter: document.presenter, url: document.url, links: document.links })
|
|
106
|
+
expect(callbackOutput).not.toContain(secret)
|
|
107
|
+
expect(callbackOutput).toContain('Wholesale')
|
|
108
|
+
})
|
|
109
|
+
|
|
110
|
+
it('keeps encrypted values out of the text and presenter indexRecord sends to the embedding provider', async () => {
|
|
111
|
+
const { embeddingService, driver, embeddedTexts, upserted } = createVectorBackend()
|
|
112
|
+
const searchService = new SearchService({ strategies: [new VectorSearchStrategy(embeddingService, driver)] })
|
|
113
|
+
const indexer = new SearchIndexer(searchService, moduleConfigs, {
|
|
114
|
+
encryptedCustomFieldKeysResolver: encryptedTaxNumber,
|
|
115
|
+
})
|
|
116
|
+
|
|
117
|
+
await indexer.indexRecord({
|
|
118
|
+
entityId: companyEntityId,
|
|
119
|
+
recordId: 'profile-1',
|
|
120
|
+
tenantId: 'tenant-1',
|
|
121
|
+
organizationId: 'org-1',
|
|
122
|
+
record: createRecord(),
|
|
123
|
+
customFields: { tax_number: secret, industry_note: 'Wholesale' },
|
|
124
|
+
})
|
|
125
|
+
|
|
126
|
+
expect(embeddedTexts).toHaveLength(1)
|
|
127
|
+
expect(embeddedTexts[0]).not.toContain(secret)
|
|
128
|
+
expect(embeddedTexts[0]).toContain('Wholesale')
|
|
129
|
+
expect(JSON.stringify({ presenter: upserted[0].presenter, url: upserted[0].url, links: upserted[0].links })).not.toContain(secret)
|
|
130
|
+
})
|
|
131
|
+
|
|
132
|
+
it('reports an unresolved definition lookup and leaves every custom field out of bulk documents', async () => {
|
|
133
|
+
const reportError = jest.fn()
|
|
134
|
+
const unregister = registerTelemetryRuntime({ reportError } as unknown as TelemetryRuntime)
|
|
135
|
+
try {
|
|
136
|
+
const { driver, documents } = createFulltextDriver()
|
|
137
|
+
const searchService = new SearchService({ strategies: [new FullTextSearchStrategy(driver)] })
|
|
138
|
+
const failure = new Error('custom_field_defs unavailable')
|
|
139
|
+
const indexer = new SearchIndexer(searchService, moduleConfigs, {
|
|
140
|
+
encryptedCustomFieldKeysResolver: async () => {
|
|
141
|
+
throw failure
|
|
142
|
+
},
|
|
143
|
+
})
|
|
144
|
+
|
|
145
|
+
await indexer.bulkIndexRecords([
|
|
146
|
+
{
|
|
147
|
+
entityId: companyEntityId,
|
|
148
|
+
recordId: 'profile-1',
|
|
149
|
+
tenantId: 'tenant-1',
|
|
150
|
+
record: createRecord(),
|
|
151
|
+
customFields: { tax_number: secret, industry_note: 'Wholesale' },
|
|
152
|
+
},
|
|
153
|
+
])
|
|
154
|
+
|
|
155
|
+
const callbackOutput = JSON.stringify(documents[0].presenter)
|
|
156
|
+
expect(callbackOutput).not.toContain(secret)
|
|
157
|
+
expect(callbackOutput).not.toContain('Wholesale')
|
|
158
|
+
expect(reportError).toHaveBeenCalledWith(failure, {
|
|
159
|
+
module: 'search',
|
|
160
|
+
code: 'search.encrypted_custom_fields_unresolved',
|
|
161
|
+
attributes: { entityId: companyEntityId },
|
|
162
|
+
})
|
|
163
|
+
} finally {
|
|
164
|
+
unregister()
|
|
165
|
+
}
|
|
166
|
+
})
|
|
167
|
+
|
|
168
|
+
it('leaves every custom field out of bulk documents when constructed without a resolver', async () => {
|
|
169
|
+
const { driver, documents } = createFulltextDriver()
|
|
170
|
+
const searchService = new SearchService({ strategies: [new FullTextSearchStrategy(driver)] })
|
|
171
|
+
const indexer = new SearchIndexer(searchService, moduleConfigs)
|
|
172
|
+
|
|
173
|
+
await indexer.bulkIndexRecords([
|
|
174
|
+
{
|
|
175
|
+
entityId: companyEntityId,
|
|
176
|
+
recordId: 'profile-1',
|
|
177
|
+
tenantId: 'tenant-1',
|
|
178
|
+
record: createRecord(),
|
|
179
|
+
customFields: { tax_number: secret, industry_note: 'Wholesale' },
|
|
180
|
+
},
|
|
181
|
+
])
|
|
182
|
+
|
|
183
|
+
expect(documents).toHaveLength(1)
|
|
184
|
+
const callbackOutput = JSON.stringify({ presenter: documents[0].presenter, url: documents[0].url, links: documents[0].links })
|
|
185
|
+
expect(callbackOutput).not.toContain(secret)
|
|
186
|
+
expect(callbackOutput).not.toContain('Wholesale')
|
|
187
|
+
})
|
|
188
|
+
|
|
189
|
+
it('keeps indexing when the telemetry runtime itself throws', async () => {
|
|
190
|
+
const unregister = registerTelemetryRuntime({
|
|
191
|
+
reportError: () => {
|
|
192
|
+
throw new Error('telemetry down')
|
|
193
|
+
},
|
|
194
|
+
} as unknown as TelemetryRuntime)
|
|
195
|
+
try {
|
|
196
|
+
const { driver, documents } = createFulltextDriver()
|
|
197
|
+
const searchService = new SearchService({ strategies: [new FullTextSearchStrategy(driver)] })
|
|
198
|
+
const indexer = new SearchIndexer(searchService, moduleConfigs, {
|
|
199
|
+
encryptedCustomFieldKeysResolver: async () => {
|
|
200
|
+
throw new Error('custom_field_defs unavailable')
|
|
201
|
+
},
|
|
202
|
+
})
|
|
203
|
+
|
|
204
|
+
await indexer.bulkIndexRecords([
|
|
205
|
+
{ entityId: companyEntityId, recordId: 'profile-1', tenantId: 'tenant-1', record: createRecord() },
|
|
206
|
+
])
|
|
207
|
+
|
|
208
|
+
expect(documents).toHaveLength(1)
|
|
209
|
+
expect(JSON.stringify(documents[0].presenter)).not.toContain(secret)
|
|
210
|
+
} finally {
|
|
211
|
+
unregister()
|
|
212
|
+
}
|
|
213
|
+
})
|
|
214
|
+
})
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
jest.mock('@open-mercato/shared/lib/i18n/server', () => ({
|
|
2
|
+
resolveTranslations: async () => ({
|
|
3
|
+
locale: 'en',
|
|
4
|
+
t: (key: string, fallback?: string) => (typeof fallback === 'string' ? fallback : key),
|
|
5
|
+
}),
|
|
6
|
+
}))
|
|
7
|
+
|
|
8
|
+
import { searchConfig } from '@open-mercato/core/modules/customers/search'
|
|
9
|
+
import type { QueryEngine } from '@open-mercato/shared/lib/query/types'
|
|
10
|
+
import type { EntityId } from '@open-mercato/shared/modules/entities'
|
|
11
|
+
import { SearchIndexer } from '../indexer/search-indexer'
|
|
12
|
+
import { SearchService } from '../service'
|
|
13
|
+
import { VectorSearchStrategy, type EmbeddingService } from '../strategies/vector.strategy'
|
|
14
|
+
import type { VectorDriver, VectorDriverDocument } from '../vector/types'
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* The customers Company source writes every custom field of the profile and of its customer
|
|
18
|
+
* entity into its source text, which is the leak #7872 reported. These tests run the real
|
|
19
|
+
* source through the indexer and the vector strategy, so the text the embedding provider
|
|
20
|
+
* receives is the one asserted on.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
const companyEntityId = 'customers:customer_company_profile'
|
|
24
|
+
const customerEntityId = 'customers:customer_entity'
|
|
25
|
+
const profileSecret = 'PROFILE-SECRET-SIZE'
|
|
26
|
+
const entitySecret = 'CREDIT-REVIEW-SECRET'
|
|
27
|
+
|
|
28
|
+
const encryptedKeysByEntity: Record<string, string[]> = {
|
|
29
|
+
[companyEntityId]: ['size_bucket'],
|
|
30
|
+
[customerEntityId]: ['status'],
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const resolveEncryptedKeys = async (entityId: EntityId) => new Set(encryptedKeysByEntity[entityId] ?? [])
|
|
34
|
+
|
|
35
|
+
function createVectorBackend() {
|
|
36
|
+
const embeddedTexts: string[] = []
|
|
37
|
+
const upserted: VectorDriverDocument[] = []
|
|
38
|
+
const embeddingService: EmbeddingService = {
|
|
39
|
+
available: true,
|
|
40
|
+
createEmbedding: async (text) => {
|
|
41
|
+
embeddedTexts.push(text)
|
|
42
|
+
return [0.1, 0.2]
|
|
43
|
+
},
|
|
44
|
+
}
|
|
45
|
+
const driver: VectorDriver = {
|
|
46
|
+
id: 'pgvector',
|
|
47
|
+
ensureReady: async () => undefined,
|
|
48
|
+
upsert: async (doc) => {
|
|
49
|
+
upserted.push(doc)
|
|
50
|
+
},
|
|
51
|
+
delete: async () => undefined,
|
|
52
|
+
query: async () => [],
|
|
53
|
+
getChecksum: async () => null,
|
|
54
|
+
}
|
|
55
|
+
return { embeddingService, driver, embeddedTexts, upserted }
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function createIndexer(rows: Array<Record<string, unknown>>) {
|
|
59
|
+
const backend = createVectorBackend()
|
|
60
|
+
const query = jest.fn(async () => ({ items: rows.map((row) => ({ ...row })), total: rows.length, page: 1, pageSize: 1 }))
|
|
61
|
+
const queryEngine = { query } as unknown as QueryEngine
|
|
62
|
+
const searchService = new SearchService({
|
|
63
|
+
strategies: [new VectorSearchStrategy(backend.embeddingService, backend.driver)],
|
|
64
|
+
})
|
|
65
|
+
const moduleConfig = {
|
|
66
|
+
...searchConfig,
|
|
67
|
+
entities: searchConfig.entities.map((entity) => entity.entityId === companyEntityId && entity.buildSource
|
|
68
|
+
? {
|
|
69
|
+
...entity,
|
|
70
|
+
buildSource: (context: Parameters<NonNullable<typeof entity.buildSource>>[0]) => entity.buildSource!({
|
|
71
|
+
...context,
|
|
72
|
+
container: { resolve: () => ({ getEncryptedFieldNames: async () => [] }) },
|
|
73
|
+
}),
|
|
74
|
+
}
|
|
75
|
+
: entity),
|
|
76
|
+
}
|
|
77
|
+
const indexer = new SearchIndexer(searchService, [moduleConfig], {
|
|
78
|
+
queryEngine,
|
|
79
|
+
encryptedCustomFieldKeysResolver: resolveEncryptedKeys,
|
|
80
|
+
})
|
|
81
|
+
return { indexer, query, ...backend }
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function storedCallbackOutput(document: VectorDriverDocument): string {
|
|
85
|
+
return JSON.stringify({ presenter: document.presenter, url: document.url, links: document.links })
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
describe('SearchIndexer encrypted custom fields through the customers Company source', () => {
|
|
89
|
+
it('keeps encrypted profile and customer-entity fields out of the embedded text and stored presenter', async () => {
|
|
90
|
+
const { indexer, embeddedTexts, upserted } = createIndexer([
|
|
91
|
+
{
|
|
92
|
+
id: 'customer-entity-1',
|
|
93
|
+
kind: 'company',
|
|
94
|
+
'cf:status': entitySecret,
|
|
95
|
+
},
|
|
96
|
+
])
|
|
97
|
+
|
|
98
|
+
await indexer.indexRecord({
|
|
99
|
+
entityId: companyEntityId,
|
|
100
|
+
recordId: 'company-profile-1',
|
|
101
|
+
tenantId: 'tenant-1',
|
|
102
|
+
organizationId: 'org-1',
|
|
103
|
+
record: {
|
|
104
|
+
id: 'company-profile-1',
|
|
105
|
+
entity_id: 'customer-entity-1',
|
|
106
|
+
'cf:size_bucket': profileSecret,
|
|
107
|
+
},
|
|
108
|
+
customFields: { size_bucket: profileSecret },
|
|
109
|
+
})
|
|
110
|
+
|
|
111
|
+
expect(embeddedTexts).toHaveLength(1)
|
|
112
|
+
const [text] = embeddedTexts
|
|
113
|
+
expect(text).toContain('Record ID: company-profile-1')
|
|
114
|
+
expect(text).not.toContain(profileSecret)
|
|
115
|
+
expect(text).not.toContain(entitySecret)
|
|
116
|
+
expect(upserted).toHaveLength(1)
|
|
117
|
+
expect(storedCallbackOutput(upserted[0])).not.toContain(profileSecret)
|
|
118
|
+
expect(storedCallbackOutput(upserted[0])).not.toContain(entitySecret)
|
|
119
|
+
})
|
|
120
|
+
|
|
121
|
+
it('keeps an encrypted profile field joined through customFieldSources out of the embedded text', async () => {
|
|
122
|
+
const { indexer, query, embeddedTexts, upserted } = createIndexer([
|
|
123
|
+
{
|
|
124
|
+
id: 'customer-entity-2',
|
|
125
|
+
kind: 'company',
|
|
126
|
+
'cf:size_bucket': profileSecret,
|
|
127
|
+
},
|
|
128
|
+
])
|
|
129
|
+
|
|
130
|
+
await indexer.indexRecord({
|
|
131
|
+
entityId: companyEntityId,
|
|
132
|
+
recordId: 'company-profile-2',
|
|
133
|
+
tenantId: 'tenant-1',
|
|
134
|
+
organizationId: 'org-1',
|
|
135
|
+
record: { id: 'company-profile-2' },
|
|
136
|
+
})
|
|
137
|
+
|
|
138
|
+
const queryOptions = (query.mock.calls as unknown as Array<[string, { customFieldSources?: Array<{ entityId: string }> }]>)
|
|
139
|
+
.find(([entity]) => entity === customerEntityId)?.[1]
|
|
140
|
+
expect(queryOptions?.customFieldSources?.map((source) => source.entityId)).toContain(companyEntityId)
|
|
141
|
+
expect(embeddedTexts).toHaveLength(1)
|
|
142
|
+
expect(embeddedTexts[0]).toContain('Record ID: company-profile-2')
|
|
143
|
+
expect(embeddedTexts[0]).not.toContain(profileSecret)
|
|
144
|
+
expect(storedCallbackOutput(upserted[0])).not.toContain(profileSecret)
|
|
145
|
+
})
|
|
146
|
+
})
|