@open-mercato/search 0.10.1-develop.8225.1.1766b080b2 → 0.10.1-develop.8229.1.61fb2a0f4c
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +8 -7
- package/dist/di.js +16 -5
- package/dist/di.js.map +3 -3
- package/dist/indexer/search-indexer.js +136 -36
- package/dist/indexer/search-indexer.js.map +2 -2
- package/dist/lib/debug.js +7 -3
- package/dist/lib/debug.js.map +2 -2
- package/dist/lib/encrypted-custom-field-resolver.js +14 -6
- package/dist/lib/encrypted-custom-field-resolver.js.map +2 -2
- package/dist/vector/services/vector-index.service.js +141 -27
- package/dist/vector/services/vector-index.service.js.map +2 -2
- package/package.json +4 -4
- package/src/__tests__/debug.test.ts +42 -1
- package/src/__tests__/di-encrypted-custom-fields.test.ts +113 -0
- package/src/__tests__/encrypted-custom-field-resolver.test.ts +76 -28
- package/src/__tests__/plaintext-custom-field-boundaries.test.ts +2 -2
- package/src/__tests__/search-indexer-encrypted-custom-fields.boundary.test.ts +214 -0
- package/src/__tests__/search-indexer-encrypted-custom-fields.customers.test.ts +146 -0
- package/src/__tests__/search-indexer-encrypted-custom-fields.test.ts +425 -0
- package/src/__tests__/vector-index-security.test.ts +249 -3
- package/src/di.ts +19 -5
- package/src/indexer/search-indexer.ts +188 -36
- package/src/lib/debug.ts +16 -3
- package/src/lib/encrypted-custom-field-resolver.ts +35 -16
- package/src/vector/services/vector-index.service.ts +178 -23
|
@@ -14,8 +14,9 @@ import type { QueryEngine } from '@open-mercato/shared/lib/query/types'
|
|
|
14
14
|
import type { Queue } from '@open-mercato/queue'
|
|
15
15
|
import type { FulltextIndexJobPayload } from '../queue/fulltext-indexing'
|
|
16
16
|
import type { VectorIndexJobPayload, VectorBatchRecord } from '../queue/vector-indexing'
|
|
17
|
-
import { searchDebug, searchDebugWarn, searchError,
|
|
17
|
+
import { reportSearchError, searchDebug, searchDebugWarn, searchError, searchWarn } from '../lib/debug'
|
|
18
18
|
import type { EncryptedCustomFieldKeysResolver } from '../lib/encrypted-custom-field-resolver'
|
|
19
|
+
import { normalizeDefinitionKey } from '@open-mercato/shared/lib/crud/custom-field-definition-index'
|
|
19
20
|
|
|
20
21
|
/**
|
|
21
22
|
* Maximum number of pages to process during reindex to prevent infinite loops.
|
|
@@ -23,7 +24,88 @@ import type { EncryptedCustomFieldKeysResolver } from '../lib/encrypted-custom-f
|
|
|
23
24
|
*/
|
|
24
25
|
const MAX_PAGES = 10000
|
|
25
26
|
|
|
26
|
-
|
|
27
|
+
/**
|
|
28
|
+
* Custom fields kept out of the build context: the normalized keys of encrypted
|
|
29
|
+
* definitions, every custom field when they could not be resolved, or none.
|
|
30
|
+
*/
|
|
31
|
+
type CustomFieldExclusion = Set<string> | 'all' | null
|
|
32
|
+
|
|
33
|
+
type CustomFieldExclusionLookup = (entityId: EntityId, tenantId: string | null) => Promise<CustomFieldExclusion>
|
|
34
|
+
|
|
35
|
+
function toPlaintextExcludedCustomFields(
|
|
36
|
+
exclusion: CustomFieldExclusion,
|
|
37
|
+
): IndexableRecord['plaintextExcludedCustomFields'] {
|
|
38
|
+
if (!exclusion) return undefined
|
|
39
|
+
return exclusion === 'all' ? 'all' : Array.from(exclusion)
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function mergeCustomFieldExclusions(exclusions: CustomFieldExclusion[]): CustomFieldExclusion {
|
|
43
|
+
if (exclusions.includes('all')) return 'all'
|
|
44
|
+
const sets = exclusions.filter((exclusion): exclusion is Set<string> => exclusion instanceof Set)
|
|
45
|
+
if (!sets.length) return null
|
|
46
|
+
if (sets.length === 1) return sets[0]
|
|
47
|
+
return new Set(sets.flatMap((set) => Array.from(set)))
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function isExcludedCustomField(key: string, exclusion: CustomFieldExclusion): boolean {
|
|
51
|
+
if (!exclusion) return false
|
|
52
|
+
if (exclusion === 'all') return true
|
|
53
|
+
return exclusion.has(normalizeDefinitionKey(key))
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function withoutExcludedRecordCustomFields(
|
|
57
|
+
record: Record<string, unknown>,
|
|
58
|
+
exclusion: CustomFieldExclusion,
|
|
59
|
+
): Record<string, unknown> {
|
|
60
|
+
if (!exclusion) return record
|
|
61
|
+
const filtered: Record<string, unknown> = {}
|
|
62
|
+
let removed = false
|
|
63
|
+
for (const [key, value] of Object.entries(record)) {
|
|
64
|
+
const isCustomField = key.startsWith('cf:') || key.startsWith('cf_')
|
|
65
|
+
if (isCustomField && isExcludedCustomField(key.slice(3), exclusion)) {
|
|
66
|
+
removed = true
|
|
67
|
+
continue
|
|
68
|
+
}
|
|
69
|
+
filtered[key] = value
|
|
70
|
+
}
|
|
71
|
+
return removed ? filtered : record
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Build callbacks may enrich the record they receive (for example with a hydrated parent);
|
|
76
|
+
* those changes reach the indexed fields as they did before the record was filtered.
|
|
77
|
+
*/
|
|
78
|
+
function applyBuildRecordChanges(
|
|
79
|
+
buildRecord: Record<string, unknown>,
|
|
80
|
+
record: Record<string, unknown>,
|
|
81
|
+
exclusion: CustomFieldExclusion,
|
|
82
|
+
): void {
|
|
83
|
+
if (buildRecord === record) return
|
|
84
|
+
for (const [key, value] of Object.entries(buildRecord)) {
|
|
85
|
+
if (record[key] !== value) record[key] = value
|
|
86
|
+
}
|
|
87
|
+
for (const key of Object.keys(record)) {
|
|
88
|
+
if (key in buildRecord) continue
|
|
89
|
+
const isCustomField = key.startsWith('cf:') || key.startsWith('cf_')
|
|
90
|
+
if (isCustomField && isExcludedCustomField(key.slice(3), exclusion)) continue
|
|
91
|
+
delete record[key]
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function withoutExcludedCustomFields(
|
|
96
|
+
customFields: Record<string, unknown>,
|
|
97
|
+
exclusion: CustomFieldExclusion,
|
|
98
|
+
): Record<string, unknown> {
|
|
99
|
+
if (!exclusion) return customFields
|
|
100
|
+
const filtered: Record<string, unknown> = {}
|
|
101
|
+
for (const [key, value] of Object.entries(customFields)) {
|
|
102
|
+
const isPrefixed = key.startsWith('cf:') || key.startsWith('cf_')
|
|
103
|
+
if (isExcludedCustomField(key, exclusion)) continue
|
|
104
|
+
if (isPrefixed && isExcludedCustomField(key.slice(3), exclusion)) continue
|
|
105
|
+
filtered[key] = value
|
|
106
|
+
}
|
|
107
|
+
return filtered
|
|
108
|
+
}
|
|
27
109
|
|
|
28
110
|
/**
|
|
29
111
|
* Parameters for indexing a record.
|
|
@@ -37,6 +119,13 @@ export type IndexRecordParams = {
|
|
|
37
119
|
customFields?: Record<string, unknown>
|
|
38
120
|
}
|
|
39
121
|
|
|
122
|
+
type IndexRecordByIdParams = {
|
|
123
|
+
entityId: EntityId
|
|
124
|
+
recordId: string
|
|
125
|
+
tenantId: string
|
|
126
|
+
organizationId?: string | null
|
|
127
|
+
}
|
|
128
|
+
|
|
40
129
|
/**
|
|
41
130
|
* Parameters for indexing a batch of records by id in a single bulk write.
|
|
42
131
|
*/
|
|
@@ -126,6 +215,12 @@ export type SearchIndexerOptions = {
|
|
|
126
215
|
fulltextQueue?: Queue<FulltextIndexJobPayload>
|
|
127
216
|
/** Queue for vector batch indexing */
|
|
128
217
|
vectorQueue?: Queue<VectorIndexJobPayload>
|
|
218
|
+
/**
|
|
219
|
+
* Resolves the custom fields declared `encrypted` for an entity, so their values never reach
|
|
220
|
+
* buildSource, formatResult, resolveUrl or resolveLinks and stay out of the plain text a
|
|
221
|
+
* strategy stores or embeds. Without it, every custom field is left out of those callbacks.
|
|
222
|
+
* `IndexableRecord.fields` keeps them for hash-only tokens.
|
|
223
|
+
*/
|
|
129
224
|
encryptedCustomFieldKeysResolver?: EncryptedCustomFieldKeysResolver
|
|
130
225
|
}
|
|
131
226
|
|
|
@@ -150,6 +245,14 @@ export class SearchIndexer {
|
|
|
150
245
|
this.fulltextQueue = options?.fulltextQueue
|
|
151
246
|
this.vectorQueue = options?.vectorQueue
|
|
152
247
|
this.encryptedCustomFieldKeysResolver = options?.encryptedCustomFieldKeysResolver
|
|
248
|
+
if (!this.encryptedCustomFieldKeysResolver) {
|
|
249
|
+
searchWarn('SearchIndexer', 'No encrypted custom-field resolver; excluding every custom field from build contexts')
|
|
250
|
+
reportSearchError(
|
|
251
|
+
'SearchIndexer',
|
|
252
|
+
new Error('[internal] SearchIndexer has no encryptedCustomFieldKeysResolver; excluding every custom field from build contexts'),
|
|
253
|
+
'search.encrypted_custom_fields_resolver_missing',
|
|
254
|
+
)
|
|
255
|
+
}
|
|
153
256
|
for (const moduleConfig of moduleConfigs) {
|
|
154
257
|
for (const entityConfig of moduleConfig.entities) {
|
|
155
258
|
if (entityConfig.enabled !== false) {
|
|
@@ -159,18 +262,32 @@ export class SearchIndexer {
|
|
|
159
262
|
}
|
|
160
263
|
}
|
|
161
264
|
|
|
162
|
-
|
|
265
|
+
/**
|
|
266
|
+
* Returns a lookup of the custom fields to keep out of build contexts, memoized per entity and
|
|
267
|
+
* tenant for one indexing call or reindex run. A missing resolver or a failed resolution excludes
|
|
268
|
+
* every custom field. Failed resolutions stay cached for the current operation and are retried by
|
|
269
|
+
* the fresh lookup created for the next top-level indexing call.
|
|
270
|
+
*/
|
|
271
|
+
private createCustomFieldExclusionLookup(): CustomFieldExclusionLookup {
|
|
163
272
|
const resolver = this.encryptedCustomFieldKeysResolver
|
|
164
|
-
const cache = new Map<
|
|
165
|
-
return (entityId) => {
|
|
166
|
-
if (!resolver) return Promise.resolve(
|
|
167
|
-
|
|
273
|
+
const cache = new Map<string, Promise<CustomFieldExclusion>>()
|
|
274
|
+
return (entityId, tenantId) => {
|
|
275
|
+
if (!resolver) return Promise.resolve('all')
|
|
276
|
+
const cacheKey = `${entityId}\u0000${tenantId ?? ''}`
|
|
277
|
+
let pending = cache.get(cacheKey)
|
|
168
278
|
if (!pending) {
|
|
169
|
-
pending = Promise.resolve().then(() => resolver(entityId
|
|
170
|
-
|
|
279
|
+
pending = Promise.resolve().then(() => resolver(entityId, tenantId)).catch((error): CustomFieldExclusion => {
|
|
280
|
+
const resolutionError = error instanceof Error
|
|
281
|
+
? error
|
|
282
|
+
: new Error('[internal] Encrypted custom-field resolver rejected with a non-Error value')
|
|
283
|
+
searchWarn('SearchIndexer', 'Encrypted custom fields could not be resolved; excluding every custom field', {
|
|
284
|
+
entityId,
|
|
285
|
+
error: resolutionError.message,
|
|
286
|
+
})
|
|
287
|
+
reportSearchError('SearchIndexer', resolutionError, 'search.encrypted_custom_fields_unresolved', { entityId })
|
|
171
288
|
return 'all'
|
|
172
289
|
})
|
|
173
|
-
cache.set(
|
|
290
|
+
cache.set(cacheKey, pending)
|
|
174
291
|
}
|
|
175
292
|
return pending
|
|
176
293
|
}
|
|
@@ -179,12 +296,27 @@ export class SearchIndexer {
|
|
|
179
296
|
/**
|
|
180
297
|
* Returns a wrapped QueryEngine that forces skipAutoReindex: true on every query.
|
|
181
298
|
* Used to prevent feedback loops when search indexing callbacks (buildSource, formatResult)
|
|
182
|
-
* call queryEngine.query() to hydrate related entities.
|
|
299
|
+
* call queryEngine.query() to hydrate related entities. Encrypted custom fields are removed
|
|
300
|
+
* from the hydrated records, as from the indexed record itself, including those joined in
|
|
301
|
+
* from `customFieldSources`. A query without a tenant is checked against every tenant's definitions.
|
|
183
302
|
*/
|
|
184
|
-
private
|
|
303
|
+
private createBuildQueryEngine(exclusions: CustomFieldExclusionLookup): QueryEngine | undefined {
|
|
185
304
|
if (!this.queryEngine) return undefined
|
|
305
|
+
const queryEngine = this.queryEngine
|
|
186
306
|
const wrapped: QueryEngine = {
|
|
187
|
-
query: (entity, opts) =>
|
|
307
|
+
query: async (entity, opts) => {
|
|
308
|
+
const result = await queryEngine.query(entity, { ...opts, skipAutoReindex: true })
|
|
309
|
+
const sourceEntityIds = [entity, ...(opts?.customFieldSources ?? []).map((source) => source.entityId)]
|
|
310
|
+
const tenantId = opts?.tenantId ?? null
|
|
311
|
+
const exclusion = mergeCustomFieldExclusions(
|
|
312
|
+
await Promise.all(sourceEntityIds.map((sourceEntityId) => exclusions(sourceEntityId as EntityId, tenantId))),
|
|
313
|
+
)
|
|
314
|
+
if (!exclusion || !Array.isArray(result?.items)) return result
|
|
315
|
+
return {
|
|
316
|
+
...result,
|
|
317
|
+
items: result.items.map((item: Record<string, unknown>) => withoutExcludedRecordCustomFields(item, exclusion)),
|
|
318
|
+
}
|
|
319
|
+
},
|
|
188
320
|
}
|
|
189
321
|
return wrapped
|
|
190
322
|
}
|
|
@@ -215,7 +347,14 @@ export class SearchIndexer {
|
|
|
215
347
|
* Index a record in the search service.
|
|
216
348
|
*/
|
|
217
349
|
async indexRecord(params: IndexRecordParams): Promise<void> {
|
|
218
|
-
|
|
350
|
+
await this.indexRecordWithExclusions(params, this.createCustomFieldExclusionLookup())
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
private async indexRecordWithExclusions(
|
|
354
|
+
params: IndexRecordParams,
|
|
355
|
+
exclusions: CustomFieldExclusionLookup,
|
|
356
|
+
): Promise<void> {
|
|
357
|
+
const indexableRecord = await this.buildIndexableRecord(params, exclusions)
|
|
219
358
|
if (indexableRecord) {
|
|
220
359
|
await this.searchService.index(indexableRecord)
|
|
221
360
|
}
|
|
@@ -223,19 +362,20 @@ export class SearchIndexer {
|
|
|
223
362
|
|
|
224
363
|
private async buildIndexableRecord(
|
|
225
364
|
params: IndexRecordParams,
|
|
226
|
-
exclusions = this.
|
|
365
|
+
exclusions: CustomFieldExclusionLookup = this.createCustomFieldExclusionLookup(),
|
|
227
366
|
): Promise<IndexableRecord | null> {
|
|
228
367
|
const config = this.entityConfigMap.get(params.entityId)
|
|
229
368
|
if (!config || config.enabled === false) {
|
|
230
369
|
return null
|
|
231
370
|
}
|
|
232
371
|
|
|
372
|
+
const exclusion = await exclusions(params.entityId, params.tenantId)
|
|
233
373
|
const buildContext: SearchBuildContext = {
|
|
234
|
-
record: params.record,
|
|
235
|
-
customFields: params.customFields ?? {},
|
|
374
|
+
record: withoutExcludedRecordCustomFields(params.record, exclusion),
|
|
375
|
+
customFields: withoutExcludedCustomFields(params.customFields ?? {}, exclusion),
|
|
236
376
|
organizationId: params.organizationId,
|
|
237
377
|
tenantId: params.tenantId,
|
|
238
|
-
queryEngine: this.
|
|
378
|
+
queryEngine: this.createBuildQueryEngine(exclusions),
|
|
239
379
|
}
|
|
240
380
|
|
|
241
381
|
// Try buildSource first (provides text, presenter, links, checksumSource)
|
|
@@ -305,13 +445,15 @@ export class SearchIndexer {
|
|
|
305
445
|
}
|
|
306
446
|
}
|
|
307
447
|
|
|
448
|
+
applyBuildRecordChanges(buildContext.record, params.record, exclusion)
|
|
449
|
+
|
|
308
450
|
return {
|
|
309
451
|
entityId: params.entityId,
|
|
310
452
|
recordId: params.recordId,
|
|
311
453
|
tenantId: params.tenantId,
|
|
312
454
|
organizationId: params.organizationId,
|
|
313
455
|
fields: params.record,
|
|
314
|
-
plaintextExcludedCustomFields:
|
|
456
|
+
plaintextExcludedCustomFields: toPlaintextExcludedCustomFields(exclusion),
|
|
315
457
|
presenter,
|
|
316
458
|
url,
|
|
317
459
|
links,
|
|
@@ -330,6 +472,13 @@ export class SearchIndexer {
|
|
|
330
472
|
tenantId: string
|
|
331
473
|
organizationId?: string | null
|
|
332
474
|
}): Promise<{ action: 'indexed' | 'skipped'; reason?: string }> {
|
|
475
|
+
return this.indexRecordByIdWithExclusions(params, this.createCustomFieldExclusionLookup())
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
private async indexRecordByIdWithExclusions(
|
|
479
|
+
params: IndexRecordByIdParams,
|
|
480
|
+
exclusions: CustomFieldExclusionLookup,
|
|
481
|
+
): Promise<{ action: 'indexed' | 'skipped'; reason?: string }> {
|
|
333
482
|
if (!this.queryEngine) {
|
|
334
483
|
return { action: 'skipped', reason: 'queryEngine not available' }
|
|
335
484
|
}
|
|
@@ -363,14 +512,14 @@ export class SearchIndexer {
|
|
|
363
512
|
}
|
|
364
513
|
}
|
|
365
514
|
|
|
366
|
-
await this.
|
|
515
|
+
await this.indexRecordWithExclusions({
|
|
367
516
|
entityId: params.entityId,
|
|
368
517
|
recordId: params.recordId,
|
|
369
518
|
tenantId: params.tenantId,
|
|
370
519
|
organizationId: params.organizationId,
|
|
371
520
|
record,
|
|
372
521
|
customFields,
|
|
373
|
-
})
|
|
522
|
+
}, exclusions)
|
|
374
523
|
|
|
375
524
|
return { action: 'indexed' }
|
|
376
525
|
} catch (error) {
|
|
@@ -403,8 +552,8 @@ export class SearchIndexer {
|
|
|
403
552
|
list.push(item.recordId)
|
|
404
553
|
recordIdsByEntity.set(item.entityId, list)
|
|
405
554
|
}
|
|
555
|
+
const exclusions = this.createCustomFieldExclusionLookup()
|
|
406
556
|
|
|
407
|
-
const exclusions = this.createPlaintextExclusionLookup()
|
|
408
557
|
const allRecords: IndexableRecord[] = []
|
|
409
558
|
let skipped = 0
|
|
410
559
|
|
|
@@ -523,6 +672,7 @@ export class SearchIndexer {
|
|
|
523
672
|
}
|
|
524
673
|
|
|
525
674
|
// Paginate through all records
|
|
675
|
+
const exclusions = this.createCustomFieldExclusionLookup()
|
|
526
676
|
let page = 1
|
|
527
677
|
const pageSize = 200
|
|
528
678
|
let hasMore = true
|
|
@@ -550,6 +700,7 @@ export class SearchIndexer {
|
|
|
550
700
|
params.organizationId ?? null,
|
|
551
701
|
items,
|
|
552
702
|
config,
|
|
703
|
+
exclusions,
|
|
553
704
|
)
|
|
554
705
|
|
|
555
706
|
// Bulk index records via SearchService (sends to all strategies)
|
|
@@ -623,7 +774,7 @@ export class SearchIndexer {
|
|
|
623
774
|
*/
|
|
624
775
|
async bulkIndexRecords(params: IndexRecordParams[]): Promise<void> {
|
|
625
776
|
const indexableRecords: IndexableRecord[] = []
|
|
626
|
-
const exclusions = this.
|
|
777
|
+
const exclusions = this.createCustomFieldExclusionLookup()
|
|
627
778
|
|
|
628
779
|
for (const param of params) {
|
|
629
780
|
const indexableRecord = await this.buildIndexableRecord(param, exclusions)
|
|
@@ -720,6 +871,7 @@ export class SearchIndexer {
|
|
|
720
871
|
}
|
|
721
872
|
|
|
722
873
|
// Fetch and index records with pagination
|
|
874
|
+
const exclusions = this.createCustomFieldExclusionLookup()
|
|
723
875
|
const pageSize = 200
|
|
724
876
|
let page = 1
|
|
725
877
|
let totalProcessed = 0
|
|
@@ -758,6 +910,7 @@ export class SearchIndexer {
|
|
|
758
910
|
params.organizationId ?? null,
|
|
759
911
|
queryResult.items,
|
|
760
912
|
config,
|
|
913
|
+
exclusions,
|
|
761
914
|
)
|
|
762
915
|
result.recordsDropped = (result.recordsDropped ?? 0) + dropped
|
|
763
916
|
|
|
@@ -955,6 +1108,7 @@ export class SearchIndexer {
|
|
|
955
1108
|
let page = 1
|
|
956
1109
|
let totalProcessed = 0
|
|
957
1110
|
let jobsEnqueued = 0
|
|
1111
|
+
const exclusions = this.createCustomFieldExclusionLookup()
|
|
958
1112
|
|
|
959
1113
|
for (;;) {
|
|
960
1114
|
params.onProgress?.({
|
|
@@ -1014,12 +1168,12 @@ export class SearchIndexer {
|
|
|
1014
1168
|
// Direct indexing (blocking) - index each record via SearchService
|
|
1015
1169
|
for (const { entityId, recordId } of batchRecords) {
|
|
1016
1170
|
try {
|
|
1017
|
-
await this.
|
|
1171
|
+
await this.indexRecordByIdWithExclusions({
|
|
1018
1172
|
entityId: entityId as EntityId,
|
|
1019
1173
|
recordId,
|
|
1020
1174
|
tenantId: params.tenantId,
|
|
1021
1175
|
organizationId: params.organizationId,
|
|
1022
|
-
})
|
|
1176
|
+
}, exclusions)
|
|
1023
1177
|
totalProcessed++
|
|
1024
1178
|
} catch (error) {
|
|
1025
1179
|
searchDebugWarn('SearchIndexer', 'Failed to index record to vector', {
|
|
@@ -1118,23 +1272,19 @@ export class SearchIndexer {
|
|
|
1118
1272
|
organizationId: string | null,
|
|
1119
1273
|
items: Record<string, unknown>[],
|
|
1120
1274
|
config: SearchEntityConfig,
|
|
1121
|
-
exclusions = this.
|
|
1275
|
+
exclusions: CustomFieldExclusionLookup = this.createCustomFieldExclusionLookup(),
|
|
1122
1276
|
): Promise<{ records: IndexableRecord[]; dropped: number }> {
|
|
1123
1277
|
const records: IndexableRecord[] = []
|
|
1124
|
-
const plaintextExcludedCustomFields = await exclusions(entityId)
|
|
1125
1278
|
let dropped = 0
|
|
1279
|
+
const exclusion = await exclusions(entityId, tenantId)
|
|
1280
|
+
const plaintextExcludedCustomFields = toPlaintextExcludedCustomFields(exclusion)
|
|
1281
|
+
const buildQueryEngine = this.createBuildQueryEngine(exclusions)
|
|
1126
1282
|
|
|
1127
|
-
// Debug: log first item to see structure
|
|
1128
1283
|
if (items.length > 0) {
|
|
1129
1284
|
searchDebug('SearchIndexer', 'Sample item structure', {
|
|
1130
1285
|
entityId,
|
|
1131
1286
|
sampleKeys: Object.keys(items[0]),
|
|
1132
|
-
sampleId: items[0].id,
|
|
1133
1287
|
hasId: 'id' in items[0],
|
|
1134
|
-
firstName: items[0].first_name,
|
|
1135
|
-
lastName: items[0].last_name,
|
|
1136
|
-
preferredName: items[0].preferred_name,
|
|
1137
|
-
sampleItem: JSON.stringify(items[0]).slice(0, 500),
|
|
1138
1288
|
})
|
|
1139
1289
|
}
|
|
1140
1290
|
|
|
@@ -1156,11 +1306,11 @@ export class SearchIndexer {
|
|
|
1156
1306
|
}
|
|
1157
1307
|
|
|
1158
1308
|
const buildContext: SearchBuildContext = {
|
|
1159
|
-
record: item,
|
|
1160
|
-
customFields,
|
|
1309
|
+
record: withoutExcludedRecordCustomFields(item, exclusion),
|
|
1310
|
+
customFields: withoutExcludedCustomFields(customFields, exclusion),
|
|
1161
1311
|
organizationId,
|
|
1162
1312
|
tenantId,
|
|
1163
|
-
queryEngine:
|
|
1313
|
+
queryEngine: buildQueryEngine,
|
|
1164
1314
|
}
|
|
1165
1315
|
|
|
1166
1316
|
// Try buildSource first (provides text, presenter, links, checksumSource)
|
|
@@ -1218,6 +1368,8 @@ export class SearchIndexer {
|
|
|
1218
1368
|
}
|
|
1219
1369
|
}
|
|
1220
1370
|
|
|
1371
|
+
applyBuildRecordChanges(buildContext.record, item, exclusion)
|
|
1372
|
+
|
|
1221
1373
|
records.push({
|
|
1222
1374
|
entityId,
|
|
1223
1375
|
recordId,
|
package/src/lib/debug.ts
CHANGED
|
@@ -61,6 +61,11 @@ export function searchError(prefix: string, message: string, data?: Record<strin
|
|
|
61
61
|
componentLogger(prefix).error(message, data)
|
|
62
62
|
}
|
|
63
63
|
|
|
64
|
+
/**
|
|
65
|
+
* Report a caught failure to the active telemetry backend under a stable
|
|
66
|
+
* `search.<reason>` code. A telemetry fault degrades to a warning, so reporting
|
|
67
|
+
* never changes what the caller does next.
|
|
68
|
+
*/
|
|
64
69
|
export function reportSearchError(
|
|
65
70
|
prefix: string,
|
|
66
71
|
error: unknown,
|
|
@@ -68,8 +73,16 @@ export function reportSearchError(
|
|
|
68
73
|
attributes?: Record<string, string | number | boolean | undefined>,
|
|
69
74
|
): void {
|
|
70
75
|
try {
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
76
|
+
const reportedError = error instanceof Error
|
|
77
|
+
? error
|
|
78
|
+
: new Error('[internal] Search failure rejected with a non-Error value')
|
|
79
|
+
getTelemetryRuntime()?.reportError(reportedError, { module: 'search', code, attributes })
|
|
80
|
+
} catch (telemetryError) {
|
|
81
|
+
componentLogger(prefix).warn('Telemetry error report failed', {
|
|
82
|
+
code,
|
|
83
|
+
error: telemetryError instanceof Error
|
|
84
|
+
? telemetryError.message
|
|
85
|
+
: '[internal] Telemetry backend threw a non-Error value',
|
|
86
|
+
})
|
|
74
87
|
}
|
|
75
88
|
}
|
|
@@ -1,32 +1,51 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { sql, type Kysely } from 'kysely'
|
|
2
2
|
import type { EntityId } from '@open-mercato/shared/modules/entities'
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
normalizeDefinitionConfig,
|
|
5
|
+
normalizeDefinitionKey,
|
|
6
|
+
} from '@open-mercato/shared/lib/crud/custom-field-definition-index'
|
|
4
7
|
|
|
5
|
-
export type
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
8
|
+
export type EncryptedCustomFieldKeysResolver = (entityId: EntityId, tenantId?: string | null) => Promise<Set<string>>
|
|
9
|
+
|
|
10
|
+
type CustomFieldDefinitionsTable = {
|
|
11
|
+
entity_id: string
|
|
12
|
+
tenant_id: string | null
|
|
13
|
+
key: string
|
|
14
|
+
config_json: unknown
|
|
15
|
+
deleted_at: Date | null
|
|
12
16
|
}
|
|
13
17
|
|
|
14
|
-
export type
|
|
18
|
+
export type CustomFieldDefinitionsDatabase = {
|
|
19
|
+
custom_field_defs: CustomFieldDefinitionsTable
|
|
20
|
+
}
|
|
15
21
|
|
|
22
|
+
/**
|
|
23
|
+
* Resolve the normalized keys of every custom field declared `encrypted` for an entity.
|
|
24
|
+
* Definitions are read for the record's tenant and global ones, as the query engine joins
|
|
25
|
+
* them; the build context has no organization, so every organization of that tenant counts,
|
|
26
|
+
* and inactive definitions are kept: over-excluding a field is preferable to publishing its
|
|
27
|
+
* plaintext. Without a tenant, every tenant's definitions count. Each key is also added in the
|
|
28
|
+
* form the query engine uses for `cf_` aliases, where characters outside `[a-zA-Z0-9_]` become `_`.
|
|
29
|
+
*/
|
|
16
30
|
export function createEncryptedCustomFieldKeysResolver(
|
|
17
31
|
db: Kysely<CustomFieldDefinitionsDatabase>,
|
|
18
32
|
): EncryptedCustomFieldKeysResolver {
|
|
19
|
-
return async (entityId) => {
|
|
20
|
-
|
|
33
|
+
return async (entityId: EntityId, tenantId?: string | null): Promise<Set<string>> => {
|
|
34
|
+
let query = db
|
|
21
35
|
.selectFrom('custom_field_defs')
|
|
22
36
|
.select(['key', 'config_json'])
|
|
23
37
|
.where('entity_id', '=', entityId)
|
|
24
38
|
.where('deleted_at', 'is', null)
|
|
25
|
-
.
|
|
39
|
+
.where(sql<boolean>`config_json::text like ${'%encrypted%'}`)
|
|
40
|
+
if (tenantId) {
|
|
41
|
+
query = query.where((eb) => eb.or([eb('tenant_id', '=', tenantId), eb('tenant_id', 'is', null)]))
|
|
42
|
+
}
|
|
43
|
+
const rows = await query.execute()
|
|
44
|
+
|
|
26
45
|
const keys = new Set<string>()
|
|
27
|
-
for (const
|
|
28
|
-
if (!normalizeDefinitionConfig(
|
|
29
|
-
const key = normalizeDefinitionKey(
|
|
46
|
+
for (const row of Array.isArray(rows) ? rows : []) {
|
|
47
|
+
if (!normalizeDefinitionConfig(row.config_json).encrypted) continue
|
|
48
|
+
const key = normalizeDefinitionKey(row.key)
|
|
30
49
|
if (!key) continue
|
|
31
50
|
keys.add(key)
|
|
32
51
|
keys.add(key.replace(/[^a-z0-9_]/g, '_'))
|