@open-mercato/search 0.10.1-develop.8225.1.1766b080b2 → 0.10.1-develop.8229.1.61fb2a0f4c

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -17,15 +17,94 @@ import {
17
17
  } from '../types'
18
18
  import type { VectorDriver } from '../types'
19
19
  import type { SearchFieldPolicy } from '@open-mercato/shared/modules/search'
20
+ import { normalizeDefinitionKey } from '@open-mercato/shared/lib/crud/custom-field-definition-index'
20
21
  import { computeChecksum } from './checksum'
21
22
  import { EmbeddingService } from './embedding'
22
23
  import { logVectorOperation } from '../lib/vector-logs'
23
- import { searchDebug, searchDebugWarn } from '../../lib/debug'
24
+ import { reportSearchError, searchDebug, searchDebugWarn, searchWarn } from '../../lib/debug'
24
25
  import { extractSearchableFields } from '../../lib/field-policy'
25
26
 
26
27
  type ContainerResolver = () => unknown
28
+ type CustomFieldExclusion = Set<string> | 'all'
29
+ type CustomFieldExclusionLookup = (entityId: EntityId, tenantId: string | null) => Promise<CustomFieldExclusion>
27
30
  const VECTOR_ENTRY_ENCRYPTION_ENTITY_ID = 'vector:vector_search'
28
31
 
32
+ function isExcludedCustomField(key: string, exclusion: CustomFieldExclusion): boolean {
33
+ if (exclusion === 'all') return true
34
+ return exclusion.has(normalizeDefinitionKey(key))
35
+ }
36
+
37
+ function withoutExcludedRecordCustomFields(
38
+ record: Record<string, unknown>,
39
+ exclusion: CustomFieldExclusion,
40
+ ): Record<string, unknown> {
41
+ if (exclusion !== 'all' && exclusion.size === 0) return record
42
+ const filtered: Record<string, unknown> = {}
43
+ for (const [key, value] of Object.entries(record)) {
44
+ const isCustomField = key.startsWith('cf:') || key.startsWith('cf_')
45
+ if (isCustomField && isExcludedCustomField(key.slice(3), exclusion)) continue
46
+ filtered[key] = value
47
+ }
48
+ return filtered
49
+ }
50
+
51
+ function withoutExcludedCustomFields(
52
+ customFields: Record<string, unknown>,
53
+ exclusion: CustomFieldExclusion,
54
+ ): Record<string, unknown> {
55
+ if (exclusion !== 'all' && exclusion.size === 0) return customFields
56
+ const filtered: Record<string, unknown> = {}
57
+ for (const [key, value] of Object.entries(customFields)) {
58
+ const isPrefixed = key.startsWith('cf:') || key.startsWith('cf_')
59
+ if (isExcludedCustomField(key, exclusion)) continue
60
+ if (isPrefixed && isExcludedCustomField(key.slice(3), exclusion)) continue
61
+ filtered[key] = value
62
+ }
63
+ return filtered
64
+ }
65
+
66
+ function applyBuildRecordChanges(
67
+ buildRecord: Record<string, unknown>,
68
+ record: Record<string, unknown>,
69
+ exclusion: CustomFieldExclusion,
70
+ ): void {
71
+ if (buildRecord === record) return
72
+ for (const [key, value] of Object.entries(buildRecord)) {
73
+ if (record[key] !== value) record[key] = value
74
+ }
75
+ for (const key of Object.keys(record)) {
76
+ if (key in buildRecord) continue
77
+ const isCustomField = key.startsWith('cf:') || key.startsWith('cf_')
78
+ if (isCustomField && isExcludedCustomField(key.slice(3), exclusion)) continue
79
+ delete record[key]
80
+ }
81
+ }
82
+
83
+ function applyBuildCustomFieldChanges(
84
+ buildCustomFields: Record<string, unknown>,
85
+ customFields: Record<string, unknown>,
86
+ exclusion: CustomFieldExclusion,
87
+ ): void {
88
+ if (buildCustomFields === customFields) return
89
+ for (const [key, value] of Object.entries(buildCustomFields)) {
90
+ if (customFields[key] !== value) customFields[key] = value
91
+ }
92
+ for (const key of Object.keys(customFields)) {
93
+ if (key in buildCustomFields || isExcludedCustomField(key, exclusion)) continue
94
+ delete customFields[key]
95
+ }
96
+ }
97
+
98
+ function mergeCustomFieldExclusions(exclusions: CustomFieldExclusion[]): CustomFieldExclusion {
99
+ if (exclusions.includes('all')) return 'all'
100
+ return new Set(exclusions.flatMap((exclusion) => Array.from(exclusion)))
101
+ }
102
+
103
+ function normalizeResolverError(error: unknown): Error {
104
+ if (error instanceof Error) return error
105
+ return new Error('[internal] Encrypted custom-field resolver rejected with a non-Error value')
106
+ }
107
+
29
108
  const DEFAULT_SOURCE_SCOPE_FIELDS = new Set<string>([
30
109
  'id',
31
110
  'tenant_id',
@@ -89,6 +168,10 @@ export type VectorIndexServiceOptions = {
89
168
  moduleConfigs: VectorModuleConfig[]
90
169
  defaultDriverId?: VectorDriverId
91
170
  containerResolver?: ContainerResolver
171
+ encryptedCustomFieldKeysResolver?: (
172
+ entityId: EntityId,
173
+ tenantId?: string | null,
174
+ ) => Promise<Set<string>>
92
175
  eventBus?: {
93
176
  emitEvent(event: string, payload: any, options?: any): Promise<void>
94
177
  }
@@ -339,6 +422,52 @@ export class VectorIndexService {
339
422
  return Array.from(unique)
340
423
  }
341
424
 
425
+ private createCustomFieldExclusionLookup(): CustomFieldExclusionLookup {
426
+ const resolver = this.opts.encryptedCustomFieldKeysResolver
427
+ const cache = new Map<string, Promise<CustomFieldExclusion>>()
428
+ return (entityId, tenantId) => {
429
+ const cacheKey = `${entityId}\u0000${tenantId ?? ''}`
430
+ let pending = cache.get(cacheKey)
431
+ if (!pending) {
432
+ pending = resolver
433
+ ? Promise.resolve().then(() => resolver(entityId, tenantId)).catch((failure): CustomFieldExclusion => {
434
+ const error = normalizeResolverError(failure)
435
+ searchWarn('VectorIndexService', 'Encrypted custom fields could not be resolved; excluding every custom field', {
436
+ entityId,
437
+ tenantId,
438
+ error: error.message,
439
+ })
440
+ reportSearchError('VectorIndexService', error, 'search.encrypted_custom_fields_unresolved', {
441
+ entityId,
442
+ tenantId: tenantId ?? undefined,
443
+ })
444
+ return 'all'
445
+ })
446
+ : Promise.resolve('all')
447
+ cache.set(cacheKey, pending)
448
+ }
449
+ return pending
450
+ }
451
+ }
452
+
453
+ private createBuildQueryEngine(exclusions: CustomFieldExclusionLookup): QueryEngine {
454
+ const queryEngine = this.opts.queryEngine
455
+ return {
456
+ query: async (entity, opts) => {
457
+ const result = await queryEngine.query(entity, { ...opts, skipAutoReindex: true })
458
+ const sourceEntityIds = [entity, ...(opts?.customFieldSources ?? []).map((source) => source.entityId)]
459
+ const tenantId = opts?.tenantId ?? null
460
+ const exclusion = mergeCustomFieldExclusions(
461
+ await Promise.all(sourceEntityIds.map((sourceEntityId) => exclusions(sourceEntityId as EntityId, tenantId))),
462
+ )
463
+ return {
464
+ ...result,
465
+ items: result.items.map((item) => withoutExcludedRecordCustomFields(item as Record<string, unknown>, exclusion) as typeof item),
466
+ }
467
+ },
468
+ }
469
+ }
470
+
342
471
  private async fetchRecord(entityId: EntityId, recordIds: string[], tenantId: string, organizationId?: string | null) {
343
472
  const filters: Record<string, any> = { id: { $in: recordIds } }
344
473
  const result = await this.opts.queryEngine.query(entityId, {
@@ -398,6 +527,7 @@ export class VectorIndexService {
398
527
  driver: VectorDriver,
399
528
  args: IndexRecordArgs,
400
529
  raw: Record<string, any>,
530
+ exclusions: CustomFieldExclusionLookup,
401
531
  opts: { skipDelete?: boolean } = {},
402
532
  ): Promise<VectorIndexOperationResult> {
403
533
  if (!isEntityIndexStrategyAllowed(args.entityId, 'vector')) {
@@ -412,12 +542,16 @@ export class VectorIndexService {
412
542
  const scopeOrg = args.organizationId ?? null
413
543
  const { record, customFields } = this.extractRecordPayload(args.entityId, raw)
414
544
  const resolvedOrgId = scopeOrg ?? (record.organization_id ?? record.organizationId ?? null)
545
+ const exclusion = await exclusions(args.entityId, args.tenantId)
546
+ const buildRecord = withoutExcludedRecordCustomFields(record, exclusion)
547
+ const buildCustomFields = withoutExcludedCustomFields(customFields, exclusion)
548
+ const buildQueryEngine = this.createBuildQueryEngine(exclusions)
415
549
  const source = await this.resolveSource(args.entityId, entry.config, {
416
- record,
417
- customFields,
550
+ record: buildRecord,
551
+ customFields: buildCustomFields,
418
552
  tenantId: args.tenantId,
419
553
  organizationId: resolvedOrgId,
420
- })
554
+ }, buildQueryEngine)
421
555
  if (!source) {
422
556
  const existing = await driver.getChecksum(args.entityId, args.recordId, args.tenantId)
423
557
  if (!opts.skipDelete && existing) {
@@ -438,7 +572,11 @@ export class VectorIndexService {
438
572
  }
439
573
  }
440
574
 
441
- const checksumSource = source.checksumSource ?? { record, customFields }
575
+ applyBuildRecordChanges(buildRecord, record, exclusion)
576
+ applyBuildCustomFieldChanges(buildCustomFields, customFields, exclusion)
577
+ const checksumSource = entry.config.buildSource
578
+ ? source.checksumSource ?? { record, customFields }
579
+ : { record, customFields }
442
580
  const checksum = computeChecksum(checksumSource)
443
581
  const current = await driver.getChecksum(args.entityId, args.recordId, args.tenantId)
444
582
  if (current && current === checksum) {
@@ -455,11 +593,11 @@ export class VectorIndexService {
455
593
  }
456
594
  const embedding = await this.opts.embeddingService.createEmbedding(source.input)
457
595
  const presenter = await this.resolvePresenter(entry.config, {
458
- record,
459
- customFields,
596
+ record: buildRecord,
597
+ customFields: buildCustomFields,
460
598
  tenantId: args.tenantId,
461
599
  organizationId: resolvedOrgId,
462
- }, source.presenter ?? null)
600
+ }, buildQueryEngine, source.presenter ?? null)
463
601
  if (!presenter?.title) {
464
602
  searchDebugWarn('vector.index', 'missing presenter title', {
465
603
  entityId: args.entityId,
@@ -475,20 +613,22 @@ export class VectorIndexService {
475
613
  })
476
614
  }
477
615
  const links = await this.resolveLinks(entry.config, {
478
- record,
479
- customFields,
616
+ record: buildRecord,
617
+ customFields: buildCustomFields,
480
618
  tenantId: args.tenantId,
481
619
  organizationId: resolvedOrgId,
482
- }, source.links ?? null)
620
+ }, buildQueryEngine, source.links ?? null)
483
621
  const url = await this.resolveUrl(entry.config, {
484
- record,
485
- customFields,
622
+ record: buildRecord,
623
+ customFields: buildCustomFields,
486
624
  tenantId: args.tenantId,
487
625
  organizationId: resolvedOrgId,
488
- })
626
+ }, buildQueryEngine)
489
627
 
490
- const metadataRecord = extractSearchableFields(record, { fieldPolicy: entry.config.fieldPolicy })
491
- const metadataCustomFields = extractSearchableFields(customFields, { fieldPolicy: entry.config.fieldPolicy })
628
+ applyBuildRecordChanges(buildRecord, record, exclusion)
629
+ applyBuildCustomFieldChanges(buildCustomFields, customFields, exclusion)
630
+ const metadataRecord = extractSearchableFields(buildRecord, { fieldPolicy: entry.config.fieldPolicy })
631
+ const metadataCustomFields = extractSearchableFields(buildCustomFields, { fieldPolicy: entry.config.fieldPolicy })
492
632
  const normalizedPresenter = this.ensurePresenter(presenter, metadataRecord, metadataCustomFields, args.recordId, args.entityId)
493
633
  const normalizedLinks = Array.isArray(links) && links.length ? links : null
494
634
  const snapshot = this.deriveSnapshot(metadataRecord, metadataCustomFields)
@@ -734,13 +874,13 @@ export class VectorIndexService {
734
874
  customFields: Record<string, any>
735
875
  organizationId?: string | null
736
876
  tenantId: string
737
- }): Promise<VectorIndexSource | null> {
877
+ }, queryEngine: QueryEngine): Promise<VectorIndexSource | null> {
738
878
  const baseCtx = {
739
879
  record: ctx.record,
740
880
  customFields: ctx.customFields,
741
881
  organizationId: ctx.organizationId ?? null,
742
882
  tenantId: ctx.tenantId,
743
- queryEngine: this.opts.queryEngine,
883
+ queryEngine,
744
884
  container: this.opts.containerResolver ? this.opts.containerResolver() : undefined,
745
885
  }
746
886
  if (config.buildSource) {
@@ -759,6 +899,7 @@ export class VectorIndexService {
759
899
  organizationId?: string | null
760
900
  tenantId: string
761
901
  },
902
+ queryEngine: QueryEngine,
762
903
  fallback?: VectorResultPresenter | null,
763
904
  ): Promise<VectorResultPresenter | null> {
764
905
  const baseCtx = {
@@ -766,7 +907,7 @@ export class VectorIndexService {
766
907
  customFields: ctx.customFields,
767
908
  organizationId: ctx.organizationId ?? null,
768
909
  tenantId: ctx.tenantId,
769
- queryEngine: this.opts.queryEngine,
910
+ queryEngine,
770
911
  container: this.opts.containerResolver ? this.opts.containerResolver() : undefined,
771
912
  }
772
913
  if (config.formatResult) {
@@ -794,6 +935,7 @@ export class VectorIndexService {
794
935
  organizationId?: string | null
795
936
  tenantId: string
796
937
  },
938
+ queryEngine: QueryEngine,
797
939
  fallback?: VectorLinkDescriptor[] | null,
798
940
  ): Promise<VectorLinkDescriptor[] | null> {
799
941
  const baseCtx = {
@@ -801,7 +943,7 @@ export class VectorIndexService {
801
943
  customFields: ctx.customFields,
802
944
  organizationId: ctx.organizationId ?? null,
803
945
  tenantId: ctx.tenantId,
804
- queryEngine: this.opts.queryEngine,
946
+ queryEngine,
805
947
  container: this.opts.containerResolver ? this.opts.containerResolver() : undefined,
806
948
  }
807
949
  if (config.resolveLinks) {
@@ -819,6 +961,7 @@ export class VectorIndexService {
819
961
  organizationId?: string | null
820
962
  tenantId: string
821
963
  },
964
+ queryEngine: QueryEngine,
822
965
  fallback?: string | null,
823
966
  ): Promise<string | null> {
824
967
  if (config.resolveUrl) {
@@ -827,7 +970,7 @@ export class VectorIndexService {
827
970
  customFields: ctx.customFields,
828
971
  organizationId: ctx.organizationId ?? null,
829
972
  tenantId: ctx.tenantId,
830
- queryEngine: this.opts.queryEngine,
973
+ queryEngine,
831
974
  container: this.opts.containerResolver ? this.opts.containerResolver() : undefined,
832
975
  })
833
976
  if (candidate) return candidate
@@ -870,7 +1013,7 @@ export class VectorIndexService {
870
1013
  reason: 'missing_record',
871
1014
  }
872
1015
  }
873
- return this.indexExisting(entry, driver, args, raw as Record<string, any>)
1016
+ return this.indexExisting(entry, driver, args, raw as Record<string, unknown>, this.createCustomFieldExclusionLookup())
874
1017
  }
875
1018
 
876
1019
  async deleteRecord(args: DeleteRecordArgs): Promise<VectorIndexOperationResult> {
@@ -907,6 +1050,13 @@ export class VectorIndexService {
907
1050
  }
908
1051
 
909
1052
  async reindexEntity(args: { entityId: EntityId; tenantId?: string | null; organizationId?: string | null; purgeFirst?: boolean }): Promise<void> {
1053
+ await this.reindexEntityWithExclusions(args, this.createCustomFieldExclusionLookup())
1054
+ }
1055
+
1056
+ private async reindexEntityWithExclusions(
1057
+ args: { entityId: EntityId; tenantId?: string | null; organizationId?: string | null; purgeFirst?: boolean },
1058
+ exclusions: CustomFieldExclusionLookup,
1059
+ ): Promise<void> {
910
1060
  const entry = this.entityConfig.get(args.entityId)
911
1061
  if (!entry) return
912
1062
  const driver = this.getDriver(entry.driverId)
@@ -969,6 +1119,7 @@ export class VectorIndexService {
969
1119
  organizationId: args.organizationId ?? null,
970
1120
  },
971
1121
  raw as Record<string, any>,
1122
+ exclusions,
972
1123
  { skipDelete: true },
973
1124
  )
974
1125
  await logVectorOperation({
@@ -994,8 +1145,12 @@ export class VectorIndexService {
994
1145
  }
995
1146
 
996
1147
  async reindexAll(args: { tenantId?: string | null; organizationId?: string | null; purgeFirst?: boolean }): Promise<void> {
1148
+ const exclusions = this.createCustomFieldExclusionLookup()
997
1149
  for (const entityId of this.listEnabledEntities()) {
998
- await this.reindexEntity({ entityId, tenantId: args.tenantId, organizationId: args.organizationId ?? null, purgeFirst: args.purgeFirst })
1150
+ await this.reindexEntityWithExclusions(
1151
+ { entityId, tenantId: args.tenantId, organizationId: args.organizationId ?? null, purgeFirst: args.purgeFirst },
1152
+ exclusions,
1153
+ )
999
1154
  }
1000
1155
  }
1001
1156