@open-mercato/search 0.10.1-develop.8225.1.1766b080b2 → 0.10.1-develop.8229.1.61fb2a0f4c
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +8 -7
- package/dist/di.js +16 -5
- package/dist/di.js.map +3 -3
- package/dist/indexer/search-indexer.js +136 -36
- package/dist/indexer/search-indexer.js.map +2 -2
- package/dist/lib/debug.js +7 -3
- package/dist/lib/debug.js.map +2 -2
- package/dist/lib/encrypted-custom-field-resolver.js +14 -6
- package/dist/lib/encrypted-custom-field-resolver.js.map +2 -2
- package/dist/vector/services/vector-index.service.js +141 -27
- package/dist/vector/services/vector-index.service.js.map +2 -2
- package/package.json +4 -4
- package/src/__tests__/debug.test.ts +42 -1
- package/src/__tests__/di-encrypted-custom-fields.test.ts +113 -0
- package/src/__tests__/encrypted-custom-field-resolver.test.ts +76 -28
- package/src/__tests__/plaintext-custom-field-boundaries.test.ts +2 -2
- package/src/__tests__/search-indexer-encrypted-custom-fields.boundary.test.ts +214 -0
- package/src/__tests__/search-indexer-encrypted-custom-fields.customers.test.ts +146 -0
- package/src/__tests__/search-indexer-encrypted-custom-fields.test.ts +425 -0
- package/src/__tests__/vector-index-security.test.ts +249 -3
- package/src/di.ts +19 -5
- package/src/indexer/search-indexer.ts +188 -36
- package/src/lib/debug.ts +16 -3
- package/src/lib/encrypted-custom-field-resolver.ts +35 -16
- package/src/vector/services/vector-index.service.ts +178 -23
|
@@ -17,15 +17,94 @@ import {
|
|
|
17
17
|
} from '../types'
|
|
18
18
|
import type { VectorDriver } from '../types'
|
|
19
19
|
import type { SearchFieldPolicy } from '@open-mercato/shared/modules/search'
|
|
20
|
+
import { normalizeDefinitionKey } from '@open-mercato/shared/lib/crud/custom-field-definition-index'
|
|
20
21
|
import { computeChecksum } from './checksum'
|
|
21
22
|
import { EmbeddingService } from './embedding'
|
|
22
23
|
import { logVectorOperation } from '../lib/vector-logs'
|
|
23
|
-
import { searchDebug, searchDebugWarn } from '../../lib/debug'
|
|
24
|
+
import { reportSearchError, searchDebug, searchDebugWarn, searchWarn } from '../../lib/debug'
|
|
24
25
|
import { extractSearchableFields } from '../../lib/field-policy'
|
|
25
26
|
|
|
26
27
|
type ContainerResolver = () => unknown
|
|
28
|
+
type CustomFieldExclusion = Set<string> | 'all'
|
|
29
|
+
type CustomFieldExclusionLookup = (entityId: EntityId, tenantId: string | null) => Promise<CustomFieldExclusion>
|
|
27
30
|
const VECTOR_ENTRY_ENCRYPTION_ENTITY_ID = 'vector:vector_search'
|
|
28
31
|
|
|
32
|
+
function isExcludedCustomField(key: string, exclusion: CustomFieldExclusion): boolean {
|
|
33
|
+
if (exclusion === 'all') return true
|
|
34
|
+
return exclusion.has(normalizeDefinitionKey(key))
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function withoutExcludedRecordCustomFields(
|
|
38
|
+
record: Record<string, unknown>,
|
|
39
|
+
exclusion: CustomFieldExclusion,
|
|
40
|
+
): Record<string, unknown> {
|
|
41
|
+
if (exclusion !== 'all' && exclusion.size === 0) return record
|
|
42
|
+
const filtered: Record<string, unknown> = {}
|
|
43
|
+
for (const [key, value] of Object.entries(record)) {
|
|
44
|
+
const isCustomField = key.startsWith('cf:') || key.startsWith('cf_')
|
|
45
|
+
if (isCustomField && isExcludedCustomField(key.slice(3), exclusion)) continue
|
|
46
|
+
filtered[key] = value
|
|
47
|
+
}
|
|
48
|
+
return filtered
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function withoutExcludedCustomFields(
|
|
52
|
+
customFields: Record<string, unknown>,
|
|
53
|
+
exclusion: CustomFieldExclusion,
|
|
54
|
+
): Record<string, unknown> {
|
|
55
|
+
if (exclusion !== 'all' && exclusion.size === 0) return customFields
|
|
56
|
+
const filtered: Record<string, unknown> = {}
|
|
57
|
+
for (const [key, value] of Object.entries(customFields)) {
|
|
58
|
+
const isPrefixed = key.startsWith('cf:') || key.startsWith('cf_')
|
|
59
|
+
if (isExcludedCustomField(key, exclusion)) continue
|
|
60
|
+
if (isPrefixed && isExcludedCustomField(key.slice(3), exclusion)) continue
|
|
61
|
+
filtered[key] = value
|
|
62
|
+
}
|
|
63
|
+
return filtered
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function applyBuildRecordChanges(
|
|
67
|
+
buildRecord: Record<string, unknown>,
|
|
68
|
+
record: Record<string, unknown>,
|
|
69
|
+
exclusion: CustomFieldExclusion,
|
|
70
|
+
): void {
|
|
71
|
+
if (buildRecord === record) return
|
|
72
|
+
for (const [key, value] of Object.entries(buildRecord)) {
|
|
73
|
+
if (record[key] !== value) record[key] = value
|
|
74
|
+
}
|
|
75
|
+
for (const key of Object.keys(record)) {
|
|
76
|
+
if (key in buildRecord) continue
|
|
77
|
+
const isCustomField = key.startsWith('cf:') || key.startsWith('cf_')
|
|
78
|
+
if (isCustomField && isExcludedCustomField(key.slice(3), exclusion)) continue
|
|
79
|
+
delete record[key]
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function applyBuildCustomFieldChanges(
|
|
84
|
+
buildCustomFields: Record<string, unknown>,
|
|
85
|
+
customFields: Record<string, unknown>,
|
|
86
|
+
exclusion: CustomFieldExclusion,
|
|
87
|
+
): void {
|
|
88
|
+
if (buildCustomFields === customFields) return
|
|
89
|
+
for (const [key, value] of Object.entries(buildCustomFields)) {
|
|
90
|
+
if (customFields[key] !== value) customFields[key] = value
|
|
91
|
+
}
|
|
92
|
+
for (const key of Object.keys(customFields)) {
|
|
93
|
+
if (key in buildCustomFields || isExcludedCustomField(key, exclusion)) continue
|
|
94
|
+
delete customFields[key]
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function mergeCustomFieldExclusions(exclusions: CustomFieldExclusion[]): CustomFieldExclusion {
|
|
99
|
+
if (exclusions.includes('all')) return 'all'
|
|
100
|
+
return new Set(exclusions.flatMap((exclusion) => Array.from(exclusion)))
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function normalizeResolverError(error: unknown): Error {
|
|
104
|
+
if (error instanceof Error) return error
|
|
105
|
+
return new Error('[internal] Encrypted custom-field resolver rejected with a non-Error value')
|
|
106
|
+
}
|
|
107
|
+
|
|
29
108
|
const DEFAULT_SOURCE_SCOPE_FIELDS = new Set<string>([
|
|
30
109
|
'id',
|
|
31
110
|
'tenant_id',
|
|
@@ -89,6 +168,10 @@ export type VectorIndexServiceOptions = {
|
|
|
89
168
|
moduleConfigs: VectorModuleConfig[]
|
|
90
169
|
defaultDriverId?: VectorDriverId
|
|
91
170
|
containerResolver?: ContainerResolver
|
|
171
|
+
encryptedCustomFieldKeysResolver?: (
|
|
172
|
+
entityId: EntityId,
|
|
173
|
+
tenantId?: string | null,
|
|
174
|
+
) => Promise<Set<string>>
|
|
92
175
|
eventBus?: {
|
|
93
176
|
emitEvent(event: string, payload: any, options?: any): Promise<void>
|
|
94
177
|
}
|
|
@@ -339,6 +422,52 @@ export class VectorIndexService {
|
|
|
339
422
|
return Array.from(unique)
|
|
340
423
|
}
|
|
341
424
|
|
|
425
|
+
private createCustomFieldExclusionLookup(): CustomFieldExclusionLookup {
|
|
426
|
+
const resolver = this.opts.encryptedCustomFieldKeysResolver
|
|
427
|
+
const cache = new Map<string, Promise<CustomFieldExclusion>>()
|
|
428
|
+
return (entityId, tenantId) => {
|
|
429
|
+
const cacheKey = `${entityId}\u0000${tenantId ?? ''}`
|
|
430
|
+
let pending = cache.get(cacheKey)
|
|
431
|
+
if (!pending) {
|
|
432
|
+
pending = resolver
|
|
433
|
+
? Promise.resolve().then(() => resolver(entityId, tenantId)).catch((failure): CustomFieldExclusion => {
|
|
434
|
+
const error = normalizeResolverError(failure)
|
|
435
|
+
searchWarn('VectorIndexService', 'Encrypted custom fields could not be resolved; excluding every custom field', {
|
|
436
|
+
entityId,
|
|
437
|
+
tenantId,
|
|
438
|
+
error: error.message,
|
|
439
|
+
})
|
|
440
|
+
reportSearchError('VectorIndexService', error, 'search.encrypted_custom_fields_unresolved', {
|
|
441
|
+
entityId,
|
|
442
|
+
tenantId: tenantId ?? undefined,
|
|
443
|
+
})
|
|
444
|
+
return 'all'
|
|
445
|
+
})
|
|
446
|
+
: Promise.resolve('all')
|
|
447
|
+
cache.set(cacheKey, pending)
|
|
448
|
+
}
|
|
449
|
+
return pending
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
private createBuildQueryEngine(exclusions: CustomFieldExclusionLookup): QueryEngine {
|
|
454
|
+
const queryEngine = this.opts.queryEngine
|
|
455
|
+
return {
|
|
456
|
+
query: async (entity, opts) => {
|
|
457
|
+
const result = await queryEngine.query(entity, { ...opts, skipAutoReindex: true })
|
|
458
|
+
const sourceEntityIds = [entity, ...(opts?.customFieldSources ?? []).map((source) => source.entityId)]
|
|
459
|
+
const tenantId = opts?.tenantId ?? null
|
|
460
|
+
const exclusion = mergeCustomFieldExclusions(
|
|
461
|
+
await Promise.all(sourceEntityIds.map((sourceEntityId) => exclusions(sourceEntityId as EntityId, tenantId))),
|
|
462
|
+
)
|
|
463
|
+
return {
|
|
464
|
+
...result,
|
|
465
|
+
items: result.items.map((item) => withoutExcludedRecordCustomFields(item as Record<string, unknown>, exclusion) as typeof item),
|
|
466
|
+
}
|
|
467
|
+
},
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
|
|
342
471
|
private async fetchRecord(entityId: EntityId, recordIds: string[], tenantId: string, organizationId?: string | null) {
|
|
343
472
|
const filters: Record<string, any> = { id: { $in: recordIds } }
|
|
344
473
|
const result = await this.opts.queryEngine.query(entityId, {
|
|
@@ -398,6 +527,7 @@ export class VectorIndexService {
|
|
|
398
527
|
driver: VectorDriver,
|
|
399
528
|
args: IndexRecordArgs,
|
|
400
529
|
raw: Record<string, any>,
|
|
530
|
+
exclusions: CustomFieldExclusionLookup,
|
|
401
531
|
opts: { skipDelete?: boolean } = {},
|
|
402
532
|
): Promise<VectorIndexOperationResult> {
|
|
403
533
|
if (!isEntityIndexStrategyAllowed(args.entityId, 'vector')) {
|
|
@@ -412,12 +542,16 @@ export class VectorIndexService {
|
|
|
412
542
|
const scopeOrg = args.organizationId ?? null
|
|
413
543
|
const { record, customFields } = this.extractRecordPayload(args.entityId, raw)
|
|
414
544
|
const resolvedOrgId = scopeOrg ?? (record.organization_id ?? record.organizationId ?? null)
|
|
545
|
+
const exclusion = await exclusions(args.entityId, args.tenantId)
|
|
546
|
+
const buildRecord = withoutExcludedRecordCustomFields(record, exclusion)
|
|
547
|
+
const buildCustomFields = withoutExcludedCustomFields(customFields, exclusion)
|
|
548
|
+
const buildQueryEngine = this.createBuildQueryEngine(exclusions)
|
|
415
549
|
const source = await this.resolveSource(args.entityId, entry.config, {
|
|
416
|
-
record,
|
|
417
|
-
customFields,
|
|
550
|
+
record: buildRecord,
|
|
551
|
+
customFields: buildCustomFields,
|
|
418
552
|
tenantId: args.tenantId,
|
|
419
553
|
organizationId: resolvedOrgId,
|
|
420
|
-
})
|
|
554
|
+
}, buildQueryEngine)
|
|
421
555
|
if (!source) {
|
|
422
556
|
const existing = await driver.getChecksum(args.entityId, args.recordId, args.tenantId)
|
|
423
557
|
if (!opts.skipDelete && existing) {
|
|
@@ -438,7 +572,11 @@ export class VectorIndexService {
|
|
|
438
572
|
}
|
|
439
573
|
}
|
|
440
574
|
|
|
441
|
-
|
|
575
|
+
applyBuildRecordChanges(buildRecord, record, exclusion)
|
|
576
|
+
applyBuildCustomFieldChanges(buildCustomFields, customFields, exclusion)
|
|
577
|
+
const checksumSource = entry.config.buildSource
|
|
578
|
+
? source.checksumSource ?? { record, customFields }
|
|
579
|
+
: { record, customFields }
|
|
442
580
|
const checksum = computeChecksum(checksumSource)
|
|
443
581
|
const current = await driver.getChecksum(args.entityId, args.recordId, args.tenantId)
|
|
444
582
|
if (current && current === checksum) {
|
|
@@ -455,11 +593,11 @@ export class VectorIndexService {
|
|
|
455
593
|
}
|
|
456
594
|
const embedding = await this.opts.embeddingService.createEmbedding(source.input)
|
|
457
595
|
const presenter = await this.resolvePresenter(entry.config, {
|
|
458
|
-
record,
|
|
459
|
-
customFields,
|
|
596
|
+
record: buildRecord,
|
|
597
|
+
customFields: buildCustomFields,
|
|
460
598
|
tenantId: args.tenantId,
|
|
461
599
|
organizationId: resolvedOrgId,
|
|
462
|
-
}, source.presenter ?? null)
|
|
600
|
+
}, buildQueryEngine, source.presenter ?? null)
|
|
463
601
|
if (!presenter?.title) {
|
|
464
602
|
searchDebugWarn('vector.index', 'missing presenter title', {
|
|
465
603
|
entityId: args.entityId,
|
|
@@ -475,20 +613,22 @@ export class VectorIndexService {
|
|
|
475
613
|
})
|
|
476
614
|
}
|
|
477
615
|
const links = await this.resolveLinks(entry.config, {
|
|
478
|
-
record,
|
|
479
|
-
customFields,
|
|
616
|
+
record: buildRecord,
|
|
617
|
+
customFields: buildCustomFields,
|
|
480
618
|
tenantId: args.tenantId,
|
|
481
619
|
organizationId: resolvedOrgId,
|
|
482
|
-
}, source.links ?? null)
|
|
620
|
+
}, buildQueryEngine, source.links ?? null)
|
|
483
621
|
const url = await this.resolveUrl(entry.config, {
|
|
484
|
-
record,
|
|
485
|
-
customFields,
|
|
622
|
+
record: buildRecord,
|
|
623
|
+
customFields: buildCustomFields,
|
|
486
624
|
tenantId: args.tenantId,
|
|
487
625
|
organizationId: resolvedOrgId,
|
|
488
|
-
})
|
|
626
|
+
}, buildQueryEngine)
|
|
489
627
|
|
|
490
|
-
|
|
491
|
-
|
|
628
|
+
applyBuildRecordChanges(buildRecord, record, exclusion)
|
|
629
|
+
applyBuildCustomFieldChanges(buildCustomFields, customFields, exclusion)
|
|
630
|
+
const metadataRecord = extractSearchableFields(buildRecord, { fieldPolicy: entry.config.fieldPolicy })
|
|
631
|
+
const metadataCustomFields = extractSearchableFields(buildCustomFields, { fieldPolicy: entry.config.fieldPolicy })
|
|
492
632
|
const normalizedPresenter = this.ensurePresenter(presenter, metadataRecord, metadataCustomFields, args.recordId, args.entityId)
|
|
493
633
|
const normalizedLinks = Array.isArray(links) && links.length ? links : null
|
|
494
634
|
const snapshot = this.deriveSnapshot(metadataRecord, metadataCustomFields)
|
|
@@ -734,13 +874,13 @@ export class VectorIndexService {
|
|
|
734
874
|
customFields: Record<string, any>
|
|
735
875
|
organizationId?: string | null
|
|
736
876
|
tenantId: string
|
|
737
|
-
}): Promise<VectorIndexSource | null> {
|
|
877
|
+
}, queryEngine: QueryEngine): Promise<VectorIndexSource | null> {
|
|
738
878
|
const baseCtx = {
|
|
739
879
|
record: ctx.record,
|
|
740
880
|
customFields: ctx.customFields,
|
|
741
881
|
organizationId: ctx.organizationId ?? null,
|
|
742
882
|
tenantId: ctx.tenantId,
|
|
743
|
-
queryEngine
|
|
883
|
+
queryEngine,
|
|
744
884
|
container: this.opts.containerResolver ? this.opts.containerResolver() : undefined,
|
|
745
885
|
}
|
|
746
886
|
if (config.buildSource) {
|
|
@@ -759,6 +899,7 @@ export class VectorIndexService {
|
|
|
759
899
|
organizationId?: string | null
|
|
760
900
|
tenantId: string
|
|
761
901
|
},
|
|
902
|
+
queryEngine: QueryEngine,
|
|
762
903
|
fallback?: VectorResultPresenter | null,
|
|
763
904
|
): Promise<VectorResultPresenter | null> {
|
|
764
905
|
const baseCtx = {
|
|
@@ -766,7 +907,7 @@ export class VectorIndexService {
|
|
|
766
907
|
customFields: ctx.customFields,
|
|
767
908
|
organizationId: ctx.organizationId ?? null,
|
|
768
909
|
tenantId: ctx.tenantId,
|
|
769
|
-
queryEngine
|
|
910
|
+
queryEngine,
|
|
770
911
|
container: this.opts.containerResolver ? this.opts.containerResolver() : undefined,
|
|
771
912
|
}
|
|
772
913
|
if (config.formatResult) {
|
|
@@ -794,6 +935,7 @@ export class VectorIndexService {
|
|
|
794
935
|
organizationId?: string | null
|
|
795
936
|
tenantId: string
|
|
796
937
|
},
|
|
938
|
+
queryEngine: QueryEngine,
|
|
797
939
|
fallback?: VectorLinkDescriptor[] | null,
|
|
798
940
|
): Promise<VectorLinkDescriptor[] | null> {
|
|
799
941
|
const baseCtx = {
|
|
@@ -801,7 +943,7 @@ export class VectorIndexService {
|
|
|
801
943
|
customFields: ctx.customFields,
|
|
802
944
|
organizationId: ctx.organizationId ?? null,
|
|
803
945
|
tenantId: ctx.tenantId,
|
|
804
|
-
queryEngine
|
|
946
|
+
queryEngine,
|
|
805
947
|
container: this.opts.containerResolver ? this.opts.containerResolver() : undefined,
|
|
806
948
|
}
|
|
807
949
|
if (config.resolveLinks) {
|
|
@@ -819,6 +961,7 @@ export class VectorIndexService {
|
|
|
819
961
|
organizationId?: string | null
|
|
820
962
|
tenantId: string
|
|
821
963
|
},
|
|
964
|
+
queryEngine: QueryEngine,
|
|
822
965
|
fallback?: string | null,
|
|
823
966
|
): Promise<string | null> {
|
|
824
967
|
if (config.resolveUrl) {
|
|
@@ -827,7 +970,7 @@ export class VectorIndexService {
|
|
|
827
970
|
customFields: ctx.customFields,
|
|
828
971
|
organizationId: ctx.organizationId ?? null,
|
|
829
972
|
tenantId: ctx.tenantId,
|
|
830
|
-
queryEngine
|
|
973
|
+
queryEngine,
|
|
831
974
|
container: this.opts.containerResolver ? this.opts.containerResolver() : undefined,
|
|
832
975
|
})
|
|
833
976
|
if (candidate) return candidate
|
|
@@ -870,7 +1013,7 @@ export class VectorIndexService {
|
|
|
870
1013
|
reason: 'missing_record',
|
|
871
1014
|
}
|
|
872
1015
|
}
|
|
873
|
-
return this.indexExisting(entry, driver, args, raw as Record<string,
|
|
1016
|
+
return this.indexExisting(entry, driver, args, raw as Record<string, unknown>, this.createCustomFieldExclusionLookup())
|
|
874
1017
|
}
|
|
875
1018
|
|
|
876
1019
|
async deleteRecord(args: DeleteRecordArgs): Promise<VectorIndexOperationResult> {
|
|
@@ -907,6 +1050,13 @@ export class VectorIndexService {
|
|
|
907
1050
|
}
|
|
908
1051
|
|
|
909
1052
|
async reindexEntity(args: { entityId: EntityId; tenantId?: string | null; organizationId?: string | null; purgeFirst?: boolean }): Promise<void> {
|
|
1053
|
+
await this.reindexEntityWithExclusions(args, this.createCustomFieldExclusionLookup())
|
|
1054
|
+
}
|
|
1055
|
+
|
|
1056
|
+
private async reindexEntityWithExclusions(
|
|
1057
|
+
args: { entityId: EntityId; tenantId?: string | null; organizationId?: string | null; purgeFirst?: boolean },
|
|
1058
|
+
exclusions: CustomFieldExclusionLookup,
|
|
1059
|
+
): Promise<void> {
|
|
910
1060
|
const entry = this.entityConfig.get(args.entityId)
|
|
911
1061
|
if (!entry) return
|
|
912
1062
|
const driver = this.getDriver(entry.driverId)
|
|
@@ -969,6 +1119,7 @@ export class VectorIndexService {
|
|
|
969
1119
|
organizationId: args.organizationId ?? null,
|
|
970
1120
|
},
|
|
971
1121
|
raw as Record<string, any>,
|
|
1122
|
+
exclusions,
|
|
972
1123
|
{ skipDelete: true },
|
|
973
1124
|
)
|
|
974
1125
|
await logVectorOperation({
|
|
@@ -994,8 +1145,12 @@ export class VectorIndexService {
|
|
|
994
1145
|
}
|
|
995
1146
|
|
|
996
1147
|
async reindexAll(args: { tenantId?: string | null; organizationId?: string | null; purgeFirst?: boolean }): Promise<void> {
|
|
1148
|
+
const exclusions = this.createCustomFieldExclusionLookup()
|
|
997
1149
|
for (const entityId of this.listEnabledEntities()) {
|
|
998
|
-
await this.
|
|
1150
|
+
await this.reindexEntityWithExclusions(
|
|
1151
|
+
{ entityId, tenantId: args.tenantId, organizationId: args.organizationId ?? null, purgeFirst: args.purgeFirst },
|
|
1152
|
+
exclusions,
|
|
1153
|
+
)
|
|
999
1154
|
}
|
|
1000
1155
|
}
|
|
1001
1156
|
|