@open-mercato/search 0.10.1-develop.8225.1.1766b080b2 → 0.10.1-develop.8229.1.61fb2a0f4c

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,10 +1,66 @@
1
1
  import { isEntityIndexStrategyAllowed } from "../../lib/entity-strategy-policy.js";
2
2
  import { parseDecryptedFieldValue } from "@open-mercato/shared/lib/encryption/tenantDataEncryptionService";
3
+ import { normalizeDefinitionKey } from "@open-mercato/shared/lib/crud/custom-field-definition-index";
3
4
  import { computeChecksum } from "./checksum.js";
4
5
  import { logVectorOperation } from "../lib/vector-logs.js";
5
- import { searchDebug, searchDebugWarn } from "../../lib/debug.js";
6
+ import { reportSearchError, searchDebug, searchDebugWarn, searchWarn } from "../../lib/debug.js";
6
7
  import { extractSearchableFields } from "../../lib/field-policy.js";
7
8
  const VECTOR_ENTRY_ENCRYPTION_ENTITY_ID = "vector:vector_search";
9
+ function isExcludedCustomField(key, exclusion) {
10
+ if (exclusion === "all") return true;
11
+ return exclusion.has(normalizeDefinitionKey(key));
12
+ }
13
+ function withoutExcludedRecordCustomFields(record, exclusion) {
14
+ if (exclusion !== "all" && exclusion.size === 0) return record;
15
+ const filtered = {};
16
+ for (const [key, value] of Object.entries(record)) {
17
+ const isCustomField = key.startsWith("cf:") || key.startsWith("cf_");
18
+ if (isCustomField && isExcludedCustomField(key.slice(3), exclusion)) continue;
19
+ filtered[key] = value;
20
+ }
21
+ return filtered;
22
+ }
23
+ function withoutExcludedCustomFields(customFields, exclusion) {
24
+ if (exclusion !== "all" && exclusion.size === 0) return customFields;
25
+ const filtered = {};
26
+ for (const [key, value] of Object.entries(customFields)) {
27
+ const isPrefixed = key.startsWith("cf:") || key.startsWith("cf_");
28
+ if (isExcludedCustomField(key, exclusion)) continue;
29
+ if (isPrefixed && isExcludedCustomField(key.slice(3), exclusion)) continue;
30
+ filtered[key] = value;
31
+ }
32
+ return filtered;
33
+ }
34
+ function applyBuildRecordChanges(buildRecord, record, exclusion) {
35
+ if (buildRecord === record) return;
36
+ for (const [key, value] of Object.entries(buildRecord)) {
37
+ if (record[key] !== value) record[key] = value;
38
+ }
39
+ for (const key of Object.keys(record)) {
40
+ if (key in buildRecord) continue;
41
+ const isCustomField = key.startsWith("cf:") || key.startsWith("cf_");
42
+ if (isCustomField && isExcludedCustomField(key.slice(3), exclusion)) continue;
43
+ delete record[key];
44
+ }
45
+ }
46
+ function applyBuildCustomFieldChanges(buildCustomFields, customFields, exclusion) {
47
+ if (buildCustomFields === customFields) return;
48
+ for (const [key, value] of Object.entries(buildCustomFields)) {
49
+ if (customFields[key] !== value) customFields[key] = value;
50
+ }
51
+ for (const key of Object.keys(customFields)) {
52
+ if (key in buildCustomFields || isExcludedCustomField(key, exclusion)) continue;
53
+ delete customFields[key];
54
+ }
55
+ }
56
+ function mergeCustomFieldExclusions(exclusions) {
57
+ if (exclusions.includes("all")) return "all";
58
+ return new Set(exclusions.flatMap((exclusion) => Array.from(exclusion)));
59
+ }
60
+ function normalizeResolverError(error) {
61
+ if (error instanceof Error) return error;
62
+ return new Error("[internal] Encrypted custom-field resolver rejected with a non-Error value");
63
+ }
8
64
  const DEFAULT_SOURCE_SCOPE_FIELDS = /* @__PURE__ */ new Set([
9
65
  "id",
10
66
  "tenant_id",
@@ -228,6 +284,48 @@ class VectorIndexService {
228
284
  const unique = /* @__PURE__ */ new Set(["id", ...hints]);
229
285
  return Array.from(unique);
230
286
  }
287
+ createCustomFieldExclusionLookup() {
288
+ const resolver = this.opts.encryptedCustomFieldKeysResolver;
289
+ const cache = /* @__PURE__ */ new Map();
290
+ return (entityId, tenantId) => {
291
+ const cacheKey = `${entityId}\0${tenantId ?? ""}`;
292
+ let pending = cache.get(cacheKey);
293
+ if (!pending) {
294
+ pending = resolver ? Promise.resolve().then(() => resolver(entityId, tenantId)).catch((failure) => {
295
+ const error = normalizeResolverError(failure);
296
+ searchWarn("VectorIndexService", "Encrypted custom fields could not be resolved; excluding every custom field", {
297
+ entityId,
298
+ tenantId,
299
+ error: error.message
300
+ });
301
+ reportSearchError("VectorIndexService", error, "search.encrypted_custom_fields_unresolved", {
302
+ entityId,
303
+ tenantId: tenantId ?? void 0
304
+ });
305
+ return "all";
306
+ }) : Promise.resolve("all");
307
+ cache.set(cacheKey, pending);
308
+ }
309
+ return pending;
310
+ };
311
+ }
312
+ createBuildQueryEngine(exclusions) {
313
+ const queryEngine = this.opts.queryEngine;
314
+ return {
315
+ query: async (entity, opts) => {
316
+ const result = await queryEngine.query(entity, { ...opts, skipAutoReindex: true });
317
+ const sourceEntityIds = [entity, ...(opts?.customFieldSources ?? []).map((source) => source.entityId)];
318
+ const tenantId = opts?.tenantId ?? null;
319
+ const exclusion = mergeCustomFieldExclusions(
320
+ await Promise.all(sourceEntityIds.map((sourceEntityId) => exclusions(sourceEntityId, tenantId)))
321
+ );
322
+ return {
323
+ ...result,
324
+ items: result.items.map((item) => withoutExcludedRecordCustomFields(item, exclusion))
325
+ };
326
+ }
327
+ };
328
+ }
231
329
  async fetchRecord(entityId, recordIds, tenantId, organizationId) {
232
330
  const filters = { id: { $in: recordIds } };
233
331
  const result = await this.opts.queryEngine.query(entityId, {
@@ -276,7 +374,7 @@ class VectorIndexService {
276
374
  }
277
375
  return { record, customFields };
278
376
  }
279
- async indexExisting(entry, driver, args, raw, opts = {}) {
377
+ async indexExisting(entry, driver, args, raw, exclusions, opts = {}) {
280
378
  if (!isEntityIndexStrategyAllowed(args.entityId, "vector")) {
281
379
  return {
282
380
  action: "skipped",
@@ -289,12 +387,16 @@ class VectorIndexService {
289
387
  const scopeOrg = args.organizationId ?? null;
290
388
  const { record, customFields } = this.extractRecordPayload(args.entityId, raw);
291
389
  const resolvedOrgId = scopeOrg ?? (record.organization_id ?? record.organizationId ?? null);
390
+ const exclusion = await exclusions(args.entityId, args.tenantId);
391
+ const buildRecord = withoutExcludedRecordCustomFields(record, exclusion);
392
+ const buildCustomFields = withoutExcludedCustomFields(customFields, exclusion);
393
+ const buildQueryEngine = this.createBuildQueryEngine(exclusions);
292
394
  const source = await this.resolveSource(args.entityId, entry.config, {
293
- record,
294
- customFields,
395
+ record: buildRecord,
396
+ customFields: buildCustomFields,
295
397
  tenantId: args.tenantId,
296
398
  organizationId: resolvedOrgId
297
- });
399
+ }, buildQueryEngine);
298
400
  if (!source) {
299
401
  const existing = await driver.getChecksum(args.entityId, args.recordId, args.tenantId);
300
402
  if (!opts.skipDelete && existing) {
@@ -314,7 +416,9 @@ class VectorIndexService {
314
416
  reason: "missing_record"
315
417
  };
316
418
  }
317
- const checksumSource = source.checksumSource ?? { record, customFields };
419
+ applyBuildRecordChanges(buildRecord, record, exclusion);
420
+ applyBuildCustomFieldChanges(buildCustomFields, customFields, exclusion);
421
+ const checksumSource = entry.config.buildSource ? source.checksumSource ?? { record, customFields } : { record, customFields };
318
422
  const checksum = computeChecksum(checksumSource);
319
423
  const current = await driver.getChecksum(args.entityId, args.recordId, args.tenantId);
320
424
  if (current && current === checksum) {
@@ -331,11 +435,11 @@ class VectorIndexService {
331
435
  }
332
436
  const embedding = await this.opts.embeddingService.createEmbedding(source.input);
333
437
  const presenter = await this.resolvePresenter(entry.config, {
334
- record,
335
- customFields,
438
+ record: buildRecord,
439
+ customFields: buildCustomFields,
336
440
  tenantId: args.tenantId,
337
441
  organizationId: resolvedOrgId
338
- }, source.presenter ?? null);
442
+ }, buildQueryEngine, source.presenter ?? null);
339
443
  if (!presenter?.title) {
340
444
  searchDebugWarn("vector.index", "missing presenter title", {
341
445
  entityId: args.entityId,
@@ -351,19 +455,21 @@ class VectorIndexService {
351
455
  });
352
456
  }
353
457
  const links = await this.resolveLinks(entry.config, {
354
- record,
355
- customFields,
458
+ record: buildRecord,
459
+ customFields: buildCustomFields,
356
460
  tenantId: args.tenantId,
357
461
  organizationId: resolvedOrgId
358
- }, source.links ?? null);
462
+ }, buildQueryEngine, source.links ?? null);
359
463
  const url = await this.resolveUrl(entry.config, {
360
- record,
361
- customFields,
464
+ record: buildRecord,
465
+ customFields: buildCustomFields,
362
466
  tenantId: args.tenantId,
363
467
  organizationId: resolvedOrgId
364
- });
365
- const metadataRecord = extractSearchableFields(record, { fieldPolicy: entry.config.fieldPolicy });
366
- const metadataCustomFields = extractSearchableFields(customFields, { fieldPolicy: entry.config.fieldPolicy });
468
+ }, buildQueryEngine);
469
+ applyBuildRecordChanges(buildRecord, record, exclusion);
470
+ applyBuildCustomFieldChanges(buildCustomFields, customFields, exclusion);
471
+ const metadataRecord = extractSearchableFields(buildRecord, { fieldPolicy: entry.config.fieldPolicy });
472
+ const metadataCustomFields = extractSearchableFields(buildCustomFields, { fieldPolicy: entry.config.fieldPolicy });
367
473
  const normalizedPresenter = this.ensurePresenter(presenter, metadataRecord, metadataCustomFields, args.recordId, args.entityId);
368
474
  const normalizedLinks = Array.isArray(links) && links.length ? links : null;
369
475
  const snapshot = this.deriveSnapshot(metadataRecord, metadataCustomFields);
@@ -533,13 +639,13 @@ class VectorIndexService {
533
639
  checksumSource: { record, customFields }
534
640
  };
535
641
  }
536
- async resolveSource(entityId, config, ctx) {
642
+ async resolveSource(entityId, config, ctx, queryEngine) {
537
643
  const baseCtx = {
538
644
  record: ctx.record,
539
645
  customFields: ctx.customFields,
540
646
  organizationId: ctx.organizationId ?? null,
541
647
  tenantId: ctx.tenantId,
542
- queryEngine: this.opts.queryEngine,
648
+ queryEngine,
543
649
  container: this.opts.containerResolver ? this.opts.containerResolver() : void 0
544
650
  };
545
651
  if (config.buildSource) {
@@ -549,13 +655,13 @@ class VectorIndexService {
549
655
  }
550
656
  return this.buildDefaultSource(entityId, { record: ctx.record, customFields: ctx.customFields }, config.fieldPolicy);
551
657
  }
552
- async resolvePresenter(config, ctx, fallback) {
658
+ async resolvePresenter(config, ctx, queryEngine, fallback) {
553
659
  const baseCtx = {
554
660
  record: ctx.record,
555
661
  customFields: ctx.customFields,
556
662
  organizationId: ctx.organizationId ?? null,
557
663
  tenantId: ctx.tenantId,
558
- queryEngine: this.opts.queryEngine,
664
+ queryEngine,
559
665
  container: this.opts.containerResolver ? this.opts.containerResolver() : void 0
560
666
  };
561
667
  if (config.formatResult) {
@@ -574,13 +680,13 @@ class VectorIndexService {
574
680
  }
575
681
  return null;
576
682
  }
577
- async resolveLinks(config, ctx, fallback) {
683
+ async resolveLinks(config, ctx, queryEngine, fallback) {
578
684
  const baseCtx = {
579
685
  record: ctx.record,
580
686
  customFields: ctx.customFields,
581
687
  organizationId: ctx.organizationId ?? null,
582
688
  tenantId: ctx.tenantId,
583
- queryEngine: this.opts.queryEngine,
689
+ queryEngine,
584
690
  container: this.opts.containerResolver ? this.opts.containerResolver() : void 0
585
691
  };
586
692
  if (config.resolveLinks) {
@@ -589,14 +695,14 @@ class VectorIndexService {
589
695
  }
590
696
  return fallback ?? null;
591
697
  }
592
- async resolveUrl(config, ctx, fallback) {
698
+ async resolveUrl(config, ctx, queryEngine, fallback) {
593
699
  if (config.resolveUrl) {
594
700
  const candidate = await config.resolveUrl({
595
701
  record: ctx.record,
596
702
  customFields: ctx.customFields,
597
703
  organizationId: ctx.organizationId ?? null,
598
704
  tenantId: ctx.tenantId,
599
- queryEngine: this.opts.queryEngine,
705
+ queryEngine,
600
706
  container: this.opts.containerResolver ? this.opts.containerResolver() : void 0
601
707
  });
602
708
  if (candidate) return candidate;
@@ -637,7 +743,7 @@ class VectorIndexService {
637
743
  reason: "missing_record"
638
744
  };
639
745
  }
640
- return this.indexExisting(entry, driver, args, raw);
746
+ return this.indexExisting(entry, driver, args, raw, this.createCustomFieldExclusionLookup());
641
747
  }
642
748
  async deleteRecord(args) {
643
749
  const entry = this.entityConfig.get(args.entityId);
@@ -672,6 +778,9 @@ class VectorIndexService {
672
778
  };
673
779
  }
674
780
  async reindexEntity(args) {
781
+ await this.reindexEntityWithExclusions(args, this.createCustomFieldExclusionLookup());
782
+ }
783
+ async reindexEntityWithExclusions(args, exclusions) {
675
784
  const entry = this.entityConfig.get(args.entityId);
676
785
  if (!entry) return;
677
786
  const driver = this.getDriver(entry.driverId);
@@ -729,6 +838,7 @@ class VectorIndexService {
729
838
  organizationId: args.organizationId ?? null
730
839
  },
731
840
  raw,
841
+ exclusions,
732
842
  { skipDelete: true }
733
843
  );
734
844
  await logVectorOperation({
@@ -752,8 +862,12 @@ class VectorIndexService {
752
862
  }
753
863
  }
754
864
  async reindexAll(args) {
865
+ const exclusions = this.createCustomFieldExclusionLookup();
755
866
  for (const entityId of this.listEnabledEntities()) {
756
- await this.reindexEntity({ entityId, tenantId: args.tenantId, organizationId: args.organizationId ?? null, purgeFirst: args.purgeFirst });
867
+ await this.reindexEntityWithExclusions(
868
+ { entityId, tenantId: args.tenantId, organizationId: args.organizationId ?? null, purgeFirst: args.purgeFirst },
869
+ exclusions
870
+ );
757
871
  }
758
872
  }
759
873
  resolveEntityManager() {