@open-mercato/search 0.6.4-develop.4382.1.6b4f656b77 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/dist/lib/merger.js +5 -2
- package/dist/lib/merger.js.map +2 -2
- package/dist/modules/search/__integration__/TC-SEARCH-003.spec.js +64 -0
- package/dist/modules/search/__integration__/TC-SEARCH-003.spec.js.map +7 -0
- package/dist/modules/search/__integration__/TC-SEARCH-004.spec.js +97 -0
- package/dist/modules/search/__integration__/TC-SEARCH-004.spec.js.map +7 -0
- package/dist/modules/search/__integration__/TC-SEARCH-005.spec.js +28 -0
- package/dist/modules/search/__integration__/TC-SEARCH-005.spec.js.map +7 -0
- package/dist/modules/search/__integration__/TC-SEARCH-006.spec.js +48 -0
- package/dist/modules/search/__integration__/TC-SEARCH-006.spec.js.map +7 -0
- package/dist/modules/search/__integration__/TC-SEARCH-007.spec.js +38 -0
- package/dist/modules/search/__integration__/TC-SEARCH-007.spec.js.map +7 -0
- package/dist/modules/search/__integration__/TC-SEARCH-008.spec.js +77 -0
- package/dist/modules/search/__integration__/TC-SEARCH-008.spec.js.map +7 -0
- package/dist/modules/search/__integration__/TC-SEARCH-009.spec.js +54 -0
- package/dist/modules/search/__integration__/TC-SEARCH-009.spec.js.map +7 -0
- package/dist/modules/search/ai-tools.js +61 -2
- package/dist/modules/search/ai-tools.js.map +2 -2
- package/dist/vector/services/vector-index.service.js +45 -25
- package/dist/vector/services/vector-index.service.js.map +2 -2
- package/package.json +9 -10
- package/src/__tests__/merger.test.ts +50 -0
- package/src/__tests__/vector-index-security.test.ts +230 -0
- package/src/lib/merger.ts +16 -2
- package/src/modules/search/README.md +1 -1
- package/src/modules/search/__integration__/TC-SEARCH-003.spec.ts +102 -0
- package/src/modules/search/__integration__/TC-SEARCH-004.spec.ts +135 -0
- package/src/modules/search/__integration__/TC-SEARCH-005.spec.ts +45 -0
- package/src/modules/search/__integration__/TC-SEARCH-006.spec.ts +74 -0
- package/src/modules/search/__integration__/TC-SEARCH-007.spec.ts +72 -0
- package/src/modules/search/__integration__/TC-SEARCH-008.spec.ts +124 -0
- package/src/modules/search/__integration__/TC-SEARCH-009.spec.ts +85 -0
- package/src/modules/search/__tests__/ai-tools.aggregate.test.ts +89 -4
- package/src/modules/search/__tests__/ai-tools.get.test.ts +118 -0
- package/src/modules/search/ai-tools.ts +111 -3
- package/src/vector/services/vector-index.service.ts +57 -25
|
@@ -14,6 +14,7 @@ import {
|
|
|
14
14
|
type VectorIndexEntry,
|
|
15
15
|
} from '../types'
|
|
16
16
|
import type { VectorDriver } from '../types'
|
|
17
|
+
import type { SearchFieldPolicy } from '@open-mercato/shared/modules/search'
|
|
17
18
|
import { computeChecksum } from './checksum'
|
|
18
19
|
import { EmbeddingService } from './embedding'
|
|
19
20
|
import { logVectorOperation } from '../lib/vector-logs'
|
|
@@ -22,6 +23,20 @@ import { searchDebug, searchDebugWarn } from '../../lib/debug'
|
|
|
22
23
|
type ContainerResolver = () => unknown
|
|
23
24
|
const VECTOR_ENTRY_ENCRYPTION_ENTITY_ID = 'vector:vector_search'
|
|
24
25
|
|
|
26
|
+
const DEFAULT_SOURCE_SCOPE_FIELDS = new Set<string>([
|
|
27
|
+
'id',
|
|
28
|
+
'tenant_id',
|
|
29
|
+
'tenantId',
|
|
30
|
+
'organization_id',
|
|
31
|
+
'organizationId',
|
|
32
|
+
'created_at',
|
|
33
|
+
'createdAt',
|
|
34
|
+
'updated_at',
|
|
35
|
+
'updatedAt',
|
|
36
|
+
'deleted_at',
|
|
37
|
+
'deletedAt',
|
|
38
|
+
])
|
|
39
|
+
|
|
25
40
|
const ENRICHMENT_FIELD_HINTS: Record<EntityId, string[]> = {
|
|
26
41
|
'customers:customer_entity': [
|
|
27
42
|
'id',
|
|
@@ -191,17 +206,16 @@ export class VectorIndexService {
|
|
|
191
206
|
links: ((encrypted as any).links ?? args.links) as any,
|
|
192
207
|
payload: ((encrypted as any).payload ?? args.payload) as any,
|
|
193
208
|
}
|
|
194
|
-
} catch {
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
}
|
|
209
|
+
} catch (err) {
|
|
210
|
+
searchDebugWarn('vector.index', 'Vector entry encryption failed; refusing to persist plaintext', {
|
|
211
|
+
entityId: VECTOR_ENTRY_ENCRYPTION_ENTITY_ID,
|
|
212
|
+
tenantId: args.tenantId,
|
|
213
|
+
organizationId: args.organizationId,
|
|
214
|
+
error: err instanceof Error ? err.message : String(err),
|
|
215
|
+
})
|
|
216
|
+
throw err instanceof Error
|
|
217
|
+
? err
|
|
218
|
+
: new Error('[internal] Vector entry encryption failed')
|
|
205
219
|
}
|
|
206
220
|
}
|
|
207
221
|
|
|
@@ -265,17 +279,16 @@ export class VectorIndexService {
|
|
|
265
279
|
links: ((decrypted as any).links ?? args.links) as any,
|
|
266
280
|
payload: ((decrypted as any).payload ?? args.payload) as any,
|
|
267
281
|
}
|
|
268
|
-
} catch {
|
|
269
|
-
return {
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
}
|
|
282
|
+
} catch (err) {
|
|
283
|
+
searchDebugWarn('vector.index', 'Vector entry decryption failed; refusing to return ciphertext as plaintext', {
|
|
284
|
+
entityId: VECTOR_ENTRY_ENCRYPTION_ENTITY_ID,
|
|
285
|
+
tenantId: args.tenantId,
|
|
286
|
+
organizationId: args.organizationId,
|
|
287
|
+
error: err instanceof Error ? err.message : String(err),
|
|
288
|
+
})
|
|
289
|
+
throw err instanceof Error
|
|
290
|
+
? err
|
|
291
|
+
: new Error('[internal] Vector entry decryption failed')
|
|
279
292
|
}
|
|
280
293
|
}
|
|
281
294
|
|
|
@@ -641,10 +654,27 @@ export class VectorIndexService {
|
|
|
641
654
|
return null
|
|
642
655
|
}
|
|
643
656
|
|
|
644
|
-
private buildDefaultSource(
|
|
657
|
+
private buildDefaultSource(
|
|
658
|
+
entityId: EntityId,
|
|
659
|
+
payload: { record: Record<string, any>; customFields: Record<string, any> },
|
|
660
|
+
fieldPolicy?: SearchFieldPolicy,
|
|
661
|
+
): VectorIndexSource {
|
|
645
662
|
const { record, customFields } = payload
|
|
646
663
|
const lines: string[] = []
|
|
647
664
|
|
|
665
|
+
const searchableWhitelist = fieldPolicy?.searchable ? new Set(fieldPolicy.searchable) : null
|
|
666
|
+
const excludedFields = new Set<string>([
|
|
667
|
+
...(fieldPolicy?.excluded ?? []),
|
|
668
|
+
...(fieldPolicy?.hashOnly ?? []),
|
|
669
|
+
])
|
|
670
|
+
|
|
671
|
+
const isEligibleField = (field: string): boolean => {
|
|
672
|
+
if (DEFAULT_SOURCE_SCOPE_FIELDS.has(field)) return false
|
|
673
|
+
if (excludedFields.has(field)) return false
|
|
674
|
+
if (searchableWhitelist && !searchableWhitelist.has(field)) return false
|
|
675
|
+
return true
|
|
676
|
+
}
|
|
677
|
+
|
|
648
678
|
const pushEntry = (label: string, value: unknown) => {
|
|
649
679
|
if (value === null || value === undefined) return
|
|
650
680
|
if (typeof value === 'string' && value.trim().length === 0) return
|
|
@@ -657,16 +687,18 @@ export class VectorIndexService {
|
|
|
657
687
|
|
|
658
688
|
const preferredFields = ['title', 'name', 'displayName', 'summary', 'subject']
|
|
659
689
|
for (const field of preferredFields) {
|
|
690
|
+
if (!isEligibleField(field)) continue
|
|
660
691
|
if (record[field] != null) pushEntry(field, record[field])
|
|
661
692
|
}
|
|
662
693
|
|
|
663
694
|
for (const [key, value] of Object.entries(record)) {
|
|
664
695
|
if (preferredFields.includes(key)) continue
|
|
665
|
-
if (key
|
|
696
|
+
if (!isEligibleField(key)) continue
|
|
666
697
|
pushEntry(key, value)
|
|
667
698
|
}
|
|
668
699
|
|
|
669
700
|
for (const [key, value] of Object.entries(customFields)) {
|
|
701
|
+
if (!isEligibleField(key)) continue
|
|
670
702
|
pushEntry(`custom.${key}`, value)
|
|
671
703
|
}
|
|
672
704
|
|
|
@@ -700,7 +732,7 @@ export class VectorIndexService {
|
|
|
700
732
|
if (built) return built
|
|
701
733
|
return null
|
|
702
734
|
}
|
|
703
|
-
return this.buildDefaultSource(entityId, { record: ctx.record, customFields: ctx.customFields })
|
|
735
|
+
return this.buildDefaultSource(entityId, { record: ctx.record, customFields: ctx.customFields }, config.fieldPolicy)
|
|
704
736
|
}
|
|
705
737
|
|
|
706
738
|
private async resolvePresenter(
|