@open-mercato/search 0.10.1-develop.8225.1.1766b080b2 → 0.10.1-develop.8229.1.61fb2a0f4c
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +8 -7
- package/dist/di.js +16 -5
- package/dist/di.js.map +3 -3
- package/dist/indexer/search-indexer.js +136 -36
- package/dist/indexer/search-indexer.js.map +2 -2
- package/dist/lib/debug.js +7 -3
- package/dist/lib/debug.js.map +2 -2
- package/dist/lib/encrypted-custom-field-resolver.js +14 -6
- package/dist/lib/encrypted-custom-field-resolver.js.map +2 -2
- package/dist/vector/services/vector-index.service.js +141 -27
- package/dist/vector/services/vector-index.service.js.map +2 -2
- package/package.json +4 -4
- package/src/__tests__/debug.test.ts +42 -1
- package/src/__tests__/di-encrypted-custom-fields.test.ts +113 -0
- package/src/__tests__/encrypted-custom-field-resolver.test.ts +76 -28
- package/src/__tests__/plaintext-custom-field-boundaries.test.ts +2 -2
- package/src/__tests__/search-indexer-encrypted-custom-fields.boundary.test.ts +214 -0
- package/src/__tests__/search-indexer-encrypted-custom-fields.customers.test.ts +146 -0
- package/src/__tests__/search-indexer-encrypted-custom-fields.test.ts +425 -0
- package/src/__tests__/vector-index-security.test.ts +249 -3
- package/src/di.ts +19 -5
- package/src/indexer/search-indexer.ts +188 -36
- package/src/lib/debug.ts +16 -3
- package/src/lib/encrypted-custom-field-resolver.ts +35 -16
- package/src/vector/services/vector-index.service.ts +178 -23
|
@@ -1,10 +1,66 @@
|
|
|
1
1
|
import { isEntityIndexStrategyAllowed } from "../../lib/entity-strategy-policy.js";
|
|
2
2
|
import { parseDecryptedFieldValue } from "@open-mercato/shared/lib/encryption/tenantDataEncryptionService";
|
|
3
|
+
import { normalizeDefinitionKey } from "@open-mercato/shared/lib/crud/custom-field-definition-index";
|
|
3
4
|
import { computeChecksum } from "./checksum.js";
|
|
4
5
|
import { logVectorOperation } from "../lib/vector-logs.js";
|
|
5
|
-
import { searchDebug, searchDebugWarn } from "../../lib/debug.js";
|
|
6
|
+
import { reportSearchError, searchDebug, searchDebugWarn, searchWarn } from "../../lib/debug.js";
|
|
6
7
|
import { extractSearchableFields } from "../../lib/field-policy.js";
|
|
7
8
|
const VECTOR_ENTRY_ENCRYPTION_ENTITY_ID = "vector:vector_search";
|
|
9
|
+
function isExcludedCustomField(key, exclusion) {
|
|
10
|
+
if (exclusion === "all") return true;
|
|
11
|
+
return exclusion.has(normalizeDefinitionKey(key));
|
|
12
|
+
}
|
|
13
|
+
function withoutExcludedRecordCustomFields(record, exclusion) {
|
|
14
|
+
if (exclusion !== "all" && exclusion.size === 0) return record;
|
|
15
|
+
const filtered = {};
|
|
16
|
+
for (const [key, value] of Object.entries(record)) {
|
|
17
|
+
const isCustomField = key.startsWith("cf:") || key.startsWith("cf_");
|
|
18
|
+
if (isCustomField && isExcludedCustomField(key.slice(3), exclusion)) continue;
|
|
19
|
+
filtered[key] = value;
|
|
20
|
+
}
|
|
21
|
+
return filtered;
|
|
22
|
+
}
|
|
23
|
+
function withoutExcludedCustomFields(customFields, exclusion) {
|
|
24
|
+
if (exclusion !== "all" && exclusion.size === 0) return customFields;
|
|
25
|
+
const filtered = {};
|
|
26
|
+
for (const [key, value] of Object.entries(customFields)) {
|
|
27
|
+
const isPrefixed = key.startsWith("cf:") || key.startsWith("cf_");
|
|
28
|
+
if (isExcludedCustomField(key, exclusion)) continue;
|
|
29
|
+
if (isPrefixed && isExcludedCustomField(key.slice(3), exclusion)) continue;
|
|
30
|
+
filtered[key] = value;
|
|
31
|
+
}
|
|
32
|
+
return filtered;
|
|
33
|
+
}
|
|
34
|
+
function applyBuildRecordChanges(buildRecord, record, exclusion) {
|
|
35
|
+
if (buildRecord === record) return;
|
|
36
|
+
for (const [key, value] of Object.entries(buildRecord)) {
|
|
37
|
+
if (record[key] !== value) record[key] = value;
|
|
38
|
+
}
|
|
39
|
+
for (const key of Object.keys(record)) {
|
|
40
|
+
if (key in buildRecord) continue;
|
|
41
|
+
const isCustomField = key.startsWith("cf:") || key.startsWith("cf_");
|
|
42
|
+
if (isCustomField && isExcludedCustomField(key.slice(3), exclusion)) continue;
|
|
43
|
+
delete record[key];
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
function applyBuildCustomFieldChanges(buildCustomFields, customFields, exclusion) {
|
|
47
|
+
if (buildCustomFields === customFields) return;
|
|
48
|
+
for (const [key, value] of Object.entries(buildCustomFields)) {
|
|
49
|
+
if (customFields[key] !== value) customFields[key] = value;
|
|
50
|
+
}
|
|
51
|
+
for (const key of Object.keys(customFields)) {
|
|
52
|
+
if (key in buildCustomFields || isExcludedCustomField(key, exclusion)) continue;
|
|
53
|
+
delete customFields[key];
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
function mergeCustomFieldExclusions(exclusions) {
|
|
57
|
+
if (exclusions.includes("all")) return "all";
|
|
58
|
+
return new Set(exclusions.flatMap((exclusion) => Array.from(exclusion)));
|
|
59
|
+
}
|
|
60
|
+
function normalizeResolverError(error) {
|
|
61
|
+
if (error instanceof Error) return error;
|
|
62
|
+
return new Error("[internal] Encrypted custom-field resolver rejected with a non-Error value");
|
|
63
|
+
}
|
|
8
64
|
const DEFAULT_SOURCE_SCOPE_FIELDS = /* @__PURE__ */ new Set([
|
|
9
65
|
"id",
|
|
10
66
|
"tenant_id",
|
|
@@ -228,6 +284,48 @@ class VectorIndexService {
|
|
|
228
284
|
const unique = /* @__PURE__ */ new Set(["id", ...hints]);
|
|
229
285
|
return Array.from(unique);
|
|
230
286
|
}
|
|
287
|
+
createCustomFieldExclusionLookup() {
|
|
288
|
+
const resolver = this.opts.encryptedCustomFieldKeysResolver;
|
|
289
|
+
const cache = /* @__PURE__ */ new Map();
|
|
290
|
+
return (entityId, tenantId) => {
|
|
291
|
+
const cacheKey = `${entityId}\0${tenantId ?? ""}`;
|
|
292
|
+
let pending = cache.get(cacheKey);
|
|
293
|
+
if (!pending) {
|
|
294
|
+
pending = resolver ? Promise.resolve().then(() => resolver(entityId, tenantId)).catch((failure) => {
|
|
295
|
+
const error = normalizeResolverError(failure);
|
|
296
|
+
searchWarn("VectorIndexService", "Encrypted custom fields could not be resolved; excluding every custom field", {
|
|
297
|
+
entityId,
|
|
298
|
+
tenantId,
|
|
299
|
+
error: error.message
|
|
300
|
+
});
|
|
301
|
+
reportSearchError("VectorIndexService", error, "search.encrypted_custom_fields_unresolved", {
|
|
302
|
+
entityId,
|
|
303
|
+
tenantId: tenantId ?? void 0
|
|
304
|
+
});
|
|
305
|
+
return "all";
|
|
306
|
+
}) : Promise.resolve("all");
|
|
307
|
+
cache.set(cacheKey, pending);
|
|
308
|
+
}
|
|
309
|
+
return pending;
|
|
310
|
+
};
|
|
311
|
+
}
|
|
312
|
+
createBuildQueryEngine(exclusions) {
|
|
313
|
+
const queryEngine = this.opts.queryEngine;
|
|
314
|
+
return {
|
|
315
|
+
query: async (entity, opts) => {
|
|
316
|
+
const result = await queryEngine.query(entity, { ...opts, skipAutoReindex: true });
|
|
317
|
+
const sourceEntityIds = [entity, ...(opts?.customFieldSources ?? []).map((source) => source.entityId)];
|
|
318
|
+
const tenantId = opts?.tenantId ?? null;
|
|
319
|
+
const exclusion = mergeCustomFieldExclusions(
|
|
320
|
+
await Promise.all(sourceEntityIds.map((sourceEntityId) => exclusions(sourceEntityId, tenantId)))
|
|
321
|
+
);
|
|
322
|
+
return {
|
|
323
|
+
...result,
|
|
324
|
+
items: result.items.map((item) => withoutExcludedRecordCustomFields(item, exclusion))
|
|
325
|
+
};
|
|
326
|
+
}
|
|
327
|
+
};
|
|
328
|
+
}
|
|
231
329
|
async fetchRecord(entityId, recordIds, tenantId, organizationId) {
|
|
232
330
|
const filters = { id: { $in: recordIds } };
|
|
233
331
|
const result = await this.opts.queryEngine.query(entityId, {
|
|
@@ -276,7 +374,7 @@ class VectorIndexService {
|
|
|
276
374
|
}
|
|
277
375
|
return { record, customFields };
|
|
278
376
|
}
|
|
279
|
-
async indexExisting(entry, driver, args, raw, opts = {}) {
|
|
377
|
+
async indexExisting(entry, driver, args, raw, exclusions, opts = {}) {
|
|
280
378
|
if (!isEntityIndexStrategyAllowed(args.entityId, "vector")) {
|
|
281
379
|
return {
|
|
282
380
|
action: "skipped",
|
|
@@ -289,12 +387,16 @@ class VectorIndexService {
|
|
|
289
387
|
const scopeOrg = args.organizationId ?? null;
|
|
290
388
|
const { record, customFields } = this.extractRecordPayload(args.entityId, raw);
|
|
291
389
|
const resolvedOrgId = scopeOrg ?? (record.organization_id ?? record.organizationId ?? null);
|
|
390
|
+
const exclusion = await exclusions(args.entityId, args.tenantId);
|
|
391
|
+
const buildRecord = withoutExcludedRecordCustomFields(record, exclusion);
|
|
392
|
+
const buildCustomFields = withoutExcludedCustomFields(customFields, exclusion);
|
|
393
|
+
const buildQueryEngine = this.createBuildQueryEngine(exclusions);
|
|
292
394
|
const source = await this.resolveSource(args.entityId, entry.config, {
|
|
293
|
-
record,
|
|
294
|
-
customFields,
|
|
395
|
+
record: buildRecord,
|
|
396
|
+
customFields: buildCustomFields,
|
|
295
397
|
tenantId: args.tenantId,
|
|
296
398
|
organizationId: resolvedOrgId
|
|
297
|
-
});
|
|
399
|
+
}, buildQueryEngine);
|
|
298
400
|
if (!source) {
|
|
299
401
|
const existing = await driver.getChecksum(args.entityId, args.recordId, args.tenantId);
|
|
300
402
|
if (!opts.skipDelete && existing) {
|
|
@@ -314,7 +416,9 @@ class VectorIndexService {
|
|
|
314
416
|
reason: "missing_record"
|
|
315
417
|
};
|
|
316
418
|
}
|
|
317
|
-
|
|
419
|
+
applyBuildRecordChanges(buildRecord, record, exclusion);
|
|
420
|
+
applyBuildCustomFieldChanges(buildCustomFields, customFields, exclusion);
|
|
421
|
+
const checksumSource = entry.config.buildSource ? source.checksumSource ?? { record, customFields } : { record, customFields };
|
|
318
422
|
const checksum = computeChecksum(checksumSource);
|
|
319
423
|
const current = await driver.getChecksum(args.entityId, args.recordId, args.tenantId);
|
|
320
424
|
if (current && current === checksum) {
|
|
@@ -331,11 +435,11 @@ class VectorIndexService {
|
|
|
331
435
|
}
|
|
332
436
|
const embedding = await this.opts.embeddingService.createEmbedding(source.input);
|
|
333
437
|
const presenter = await this.resolvePresenter(entry.config, {
|
|
334
|
-
record,
|
|
335
|
-
customFields,
|
|
438
|
+
record: buildRecord,
|
|
439
|
+
customFields: buildCustomFields,
|
|
336
440
|
tenantId: args.tenantId,
|
|
337
441
|
organizationId: resolvedOrgId
|
|
338
|
-
}, source.presenter ?? null);
|
|
442
|
+
}, buildQueryEngine, source.presenter ?? null);
|
|
339
443
|
if (!presenter?.title) {
|
|
340
444
|
searchDebugWarn("vector.index", "missing presenter title", {
|
|
341
445
|
entityId: args.entityId,
|
|
@@ -351,19 +455,21 @@ class VectorIndexService {
|
|
|
351
455
|
});
|
|
352
456
|
}
|
|
353
457
|
const links = await this.resolveLinks(entry.config, {
|
|
354
|
-
record,
|
|
355
|
-
customFields,
|
|
458
|
+
record: buildRecord,
|
|
459
|
+
customFields: buildCustomFields,
|
|
356
460
|
tenantId: args.tenantId,
|
|
357
461
|
organizationId: resolvedOrgId
|
|
358
|
-
}, source.links ?? null);
|
|
462
|
+
}, buildQueryEngine, source.links ?? null);
|
|
359
463
|
const url = await this.resolveUrl(entry.config, {
|
|
360
|
-
record,
|
|
361
|
-
customFields,
|
|
464
|
+
record: buildRecord,
|
|
465
|
+
customFields: buildCustomFields,
|
|
362
466
|
tenantId: args.tenantId,
|
|
363
467
|
organizationId: resolvedOrgId
|
|
364
|
-
});
|
|
365
|
-
|
|
366
|
-
|
|
468
|
+
}, buildQueryEngine);
|
|
469
|
+
applyBuildRecordChanges(buildRecord, record, exclusion);
|
|
470
|
+
applyBuildCustomFieldChanges(buildCustomFields, customFields, exclusion);
|
|
471
|
+
const metadataRecord = extractSearchableFields(buildRecord, { fieldPolicy: entry.config.fieldPolicy });
|
|
472
|
+
const metadataCustomFields = extractSearchableFields(buildCustomFields, { fieldPolicy: entry.config.fieldPolicy });
|
|
367
473
|
const normalizedPresenter = this.ensurePresenter(presenter, metadataRecord, metadataCustomFields, args.recordId, args.entityId);
|
|
368
474
|
const normalizedLinks = Array.isArray(links) && links.length ? links : null;
|
|
369
475
|
const snapshot = this.deriveSnapshot(metadataRecord, metadataCustomFields);
|
|
@@ -533,13 +639,13 @@ class VectorIndexService {
|
|
|
533
639
|
checksumSource: { record, customFields }
|
|
534
640
|
};
|
|
535
641
|
}
|
|
536
|
-
async resolveSource(entityId, config, ctx) {
|
|
642
|
+
async resolveSource(entityId, config, ctx, queryEngine) {
|
|
537
643
|
const baseCtx = {
|
|
538
644
|
record: ctx.record,
|
|
539
645
|
customFields: ctx.customFields,
|
|
540
646
|
organizationId: ctx.organizationId ?? null,
|
|
541
647
|
tenantId: ctx.tenantId,
|
|
542
|
-
queryEngine
|
|
648
|
+
queryEngine,
|
|
543
649
|
container: this.opts.containerResolver ? this.opts.containerResolver() : void 0
|
|
544
650
|
};
|
|
545
651
|
if (config.buildSource) {
|
|
@@ -549,13 +655,13 @@ class VectorIndexService {
|
|
|
549
655
|
}
|
|
550
656
|
return this.buildDefaultSource(entityId, { record: ctx.record, customFields: ctx.customFields }, config.fieldPolicy);
|
|
551
657
|
}
|
|
552
|
-
async resolvePresenter(config, ctx, fallback) {
|
|
658
|
+
async resolvePresenter(config, ctx, queryEngine, fallback) {
|
|
553
659
|
const baseCtx = {
|
|
554
660
|
record: ctx.record,
|
|
555
661
|
customFields: ctx.customFields,
|
|
556
662
|
organizationId: ctx.organizationId ?? null,
|
|
557
663
|
tenantId: ctx.tenantId,
|
|
558
|
-
queryEngine
|
|
664
|
+
queryEngine,
|
|
559
665
|
container: this.opts.containerResolver ? this.opts.containerResolver() : void 0
|
|
560
666
|
};
|
|
561
667
|
if (config.formatResult) {
|
|
@@ -574,13 +680,13 @@ class VectorIndexService {
|
|
|
574
680
|
}
|
|
575
681
|
return null;
|
|
576
682
|
}
|
|
577
|
-
async resolveLinks(config, ctx, fallback) {
|
|
683
|
+
async resolveLinks(config, ctx, queryEngine, fallback) {
|
|
578
684
|
const baseCtx = {
|
|
579
685
|
record: ctx.record,
|
|
580
686
|
customFields: ctx.customFields,
|
|
581
687
|
organizationId: ctx.organizationId ?? null,
|
|
582
688
|
tenantId: ctx.tenantId,
|
|
583
|
-
queryEngine
|
|
689
|
+
queryEngine,
|
|
584
690
|
container: this.opts.containerResolver ? this.opts.containerResolver() : void 0
|
|
585
691
|
};
|
|
586
692
|
if (config.resolveLinks) {
|
|
@@ -589,14 +695,14 @@ class VectorIndexService {
|
|
|
589
695
|
}
|
|
590
696
|
return fallback ?? null;
|
|
591
697
|
}
|
|
592
|
-
async resolveUrl(config, ctx, fallback) {
|
|
698
|
+
async resolveUrl(config, ctx, queryEngine, fallback) {
|
|
593
699
|
if (config.resolveUrl) {
|
|
594
700
|
const candidate = await config.resolveUrl({
|
|
595
701
|
record: ctx.record,
|
|
596
702
|
customFields: ctx.customFields,
|
|
597
703
|
organizationId: ctx.organizationId ?? null,
|
|
598
704
|
tenantId: ctx.tenantId,
|
|
599
|
-
queryEngine
|
|
705
|
+
queryEngine,
|
|
600
706
|
container: this.opts.containerResolver ? this.opts.containerResolver() : void 0
|
|
601
707
|
});
|
|
602
708
|
if (candidate) return candidate;
|
|
@@ -637,7 +743,7 @@ class VectorIndexService {
|
|
|
637
743
|
reason: "missing_record"
|
|
638
744
|
};
|
|
639
745
|
}
|
|
640
|
-
return this.indexExisting(entry, driver, args, raw);
|
|
746
|
+
return this.indexExisting(entry, driver, args, raw, this.createCustomFieldExclusionLookup());
|
|
641
747
|
}
|
|
642
748
|
async deleteRecord(args) {
|
|
643
749
|
const entry = this.entityConfig.get(args.entityId);
|
|
@@ -672,6 +778,9 @@ class VectorIndexService {
|
|
|
672
778
|
};
|
|
673
779
|
}
|
|
674
780
|
async reindexEntity(args) {
|
|
781
|
+
await this.reindexEntityWithExclusions(args, this.createCustomFieldExclusionLookup());
|
|
782
|
+
}
|
|
783
|
+
async reindexEntityWithExclusions(args, exclusions) {
|
|
675
784
|
const entry = this.entityConfig.get(args.entityId);
|
|
676
785
|
if (!entry) return;
|
|
677
786
|
const driver = this.getDriver(entry.driverId);
|
|
@@ -729,6 +838,7 @@ class VectorIndexService {
|
|
|
729
838
|
organizationId: args.organizationId ?? null
|
|
730
839
|
},
|
|
731
840
|
raw,
|
|
841
|
+
exclusions,
|
|
732
842
|
{ skipDelete: true }
|
|
733
843
|
);
|
|
734
844
|
await logVectorOperation({
|
|
@@ -752,8 +862,12 @@ class VectorIndexService {
|
|
|
752
862
|
}
|
|
753
863
|
}
|
|
754
864
|
async reindexAll(args) {
|
|
865
|
+
const exclusions = this.createCustomFieldExclusionLookup();
|
|
755
866
|
for (const entityId of this.listEnabledEntities()) {
|
|
756
|
-
await this.
|
|
867
|
+
await this.reindexEntityWithExclusions(
|
|
868
|
+
{ entityId, tenantId: args.tenantId, organizationId: args.organizationId ?? null, purgeFirst: args.purgeFirst },
|
|
869
|
+
exclusions
|
|
870
|
+
);
|
|
757
871
|
}
|
|
758
872
|
}
|
|
759
873
|
resolveEntityManager() {
|