@open-mercato/shared 0.8.1-develop.7206.1.45a3cee87f → 0.8.1-develop.7215.1.60786cf836

package/src/lib/encryption/tenantDataEncryptionService.ts

@@ -34,6 +34,10 @@ const MAP_MISS_TTL_MS = 5 * 60 * 1000
 // up by long-lived processes without a restart (#2746). The service-level cache
 // previously had no TTL and shadowed the KMS's own 15-minute expiry.
 const DEK_CACHE_TTL_MS = 15 * 60 * 1000
+// Matches the `{ ttl: 300 }` passed to the `CacheStrategy` layer for the all-organizations
+// aggregate, so a process that only ever hits its own memory entry (no shared cache configured,
+// or already warm) still picks up a runtime map edit within the same bound (#6066).
+const AGGREGATE_CACHE_TTL_MS = 300 * 1000
 
 function cacheKey(key: MapCacheKey): string {
   return [
@@ -44,6 +48,13 @@ function cacheKey(key: MapCacheKey): string {
   ].join(':')
 }
 
+// Tag for the aggregate of every organization-scoped map of an entity/tenant. The prefix differs
+// from `cacheKey`'s in its first segment rather than by an extra one, so no entity id can spell a
+// tag that collides with an aggregate (`encmap:all-orgs:x:y` would be reachable both ways).
+function allOrganizationsCacheKey(entityId: string, tenantId: string | null): string {
+  return ['encmap-all-orgs', entityId.toLowerCase(), tenantId ?? 'null'].join(':')
+}
+
 function debug(event: string, payload: Record<string, unknown>) {
   if (!isEncryptionDebugEnabled()) return
   try {
@@ -113,11 +124,45 @@ function isEncryptedWithDek(value: unknown, dek: TenantDek): boolean {
   return decryptWithAesGcm(value, dek.key) !== null
 }
 
-function normalizeEncryptedFieldNames(fields: readonly { field?: unknown }[] | null | undefined): string[] {
+function normalizeEncryptedFieldRules(
+  fields: readonly { field?: unknown; hashField?: unknown }[] | null | undefined,
+): EncryptedFieldRule[] {
   if (!Array.isArray(fields)) return []
-  return fields
-    .map((rule) => rule.field)
-    .filter((field): field is string => typeof field === 'string' && field.trim().length > 0)
+  const rules: EncryptedFieldRule[] = []
+  for (const rule of fields) {
+    if (!rule || typeof rule !== 'object') continue
+    const field = rule.field
+    if (typeof field !== 'string' || field.trim().length === 0) continue
+    rules.push({ field, hashField: typeof rule.hashField === 'string' ? rule.hashField : null })
+  }
+  return rules
+}
+
+function normalizeEncryptedFieldNames(fields: readonly { field?: unknown }[] | null | undefined): string[] {
+  return normalizeEncryptedFieldRules(fields).map((rule) => rule.field)
+}
+
+/**
+ * Union of field rules, first declaration wins on the field name. A later duplicate only
+ * contributes its `hashField` when the winning rule declares none, so merging an organization-scoped
+ * map into a tenant-wide one never silently retargets an existing lookup-hash column.
+ */
+function mergeEncryptedFieldRules(groups: readonly EncryptedFieldRule[][]): EncryptedFieldRule[] {
+  const merged: EncryptedFieldRule[] = []
+  const byField = new Map<string, EncryptedFieldRule>()
+  for (const group of groups) {
+    for (const rule of group) {
+      const existing = byField.get(rule.field)
+      if (!existing) {
+        const copy: EncryptedFieldRule = { field: rule.field, hashField: rule.hashField ?? null }
+        byField.set(rule.field, copy)
+        merged.push(copy)
+        continue
+      }
+      if (!existing.hashField && rule.hashField) existing.hashField = rule.hashField
+    }
+  }
+  return merged
 }
 
 function readEncryptedFieldsJson(row: Record<string, unknown>): EncryptedFieldRule[] {
@@ -160,6 +205,7 @@ function getSqlConnection(em: EntityManager): SqlConnection | null {
 
 export class TenantDataEncryptionService {
   private static globalMemoryCache = new Map<string, EncryptionMapRecord>()
+  private static globalAggregateMemoryCache = new Map<string, { at: number; record: EncryptionMapRecord }>()
   private static globalInflightMaps = new Map<string, Promise<EncryptionMapRecord | null>>()
   private static globalDekCache = new Map<string, TenantDek>()
   private static globalInflightDeks = new Map<string, Promise<TenantDek | null>>()
@@ -167,6 +213,7 @@ export class TenantDataEncryptionService {
   private readonly kms: KmsService
   private readonly cache?: CacheStrategy
   private readonly memoryCache = TenantDataEncryptionService.globalMemoryCache
+  private readonly aggregateMemoryCache = TenantDataEncryptionService.globalAggregateMemoryCache
   private readonly dekCache = TenantDataEncryptionService.globalDekCache
   private readonly inflightDeks = TenantDataEncryptionService.globalInflightDeks
   private readonly inflightMaps = TenantDataEncryptionService.globalInflightMaps
@@ -352,7 +399,10 @@ export class TenantDataEncryptionService {
     return this.applySystemDefault(null, key.entityId)
   }
 
-  private async fetchAllOrganizationFieldNames(entityId: string, tenantId: string | null): Promise<string[]> {
+  private async fetchAllOrganizationFieldRules(
+    entityId: string,
+    tenantId: string | null,
+  ): Promise<EncryptedFieldRule[]> {
     const conn = getSqlConnection(this.em)
     if (!conn) return []
     const sql = `
@@ -366,23 +416,97 @@ export class TenantDataEncryptionService {
     `
     const rows = await conn.execute(sql, [entityId, tenantId])
     if (!Array.isArray(rows) || rows.length === 0) return []
-    const names = new Set<string>()
+    const groups: EncryptedFieldRule[][] = []
     for (const row of rows) {
       if (!row || typeof row !== 'object') continue
-      for (const field of normalizeEncryptedFieldNames(readEncryptedFieldsJson(row as Record<string, unknown>))) {
-        names.add(field)
+      groups.push(normalizeEncryptedFieldRules(readEncryptedFieldsJson(row as Record<string, unknown>)))
+    }
+    return mergeEncryptedFieldRules(groups)
+  }
+
+  /**
+   * Cached aggregate of every organization-scoped map for an entity/tenant. `encryptEntityPayload`
+   * and `decryptEntityPayload` consult it on every row at the tenant-wide scope, so the uncached
+   * read this used to be would add a round-trip per flush and per load (#5949).
+   */
+  private async getAllOrganizationFieldRules(
+    entityId: string,
+    tenantId: string | null,
+  ): Promise<EncryptedFieldRule[]> {
+    const tag = allOrganizationsCacheKey(entityId, tenantId)
+    const missExpiresAt = this.missCache.get(tag)
+    if (missExpiresAt) {
+      if (missExpiresAt > Date.now()) return []
+      this.missCache.delete(tag)
+    }
+    const mem = this.aggregateMemoryCache.get(tag)
+    if (mem) {
+      if (mem.at + AGGREGATE_CACHE_TTL_MS > Date.now()) return mem.record.fields
+      this.aggregateMemoryCache.delete(tag)
+    }
+    if (this.cache && typeof this.cache.get === 'function') {
+      const cached = await this.cache.get(tag)
+      if (cached) {
+        const record = cached as EncryptionMapRecord
+        this.aggregateMemoryCache.set(tag, { at: Date.now(), record })
+        return record.fields
       }
     }
-    return Array.from(names)
+    const inflight = this.inflightMaps.get(tag)
+    if (inflight) return (await inflight)?.fields ?? []
+    const pending = (async (): Promise<EncryptionMapRecord | null> => {
+      const fields = await this.fetchAllOrganizationFieldRules(entityId, tenantId)
+      return fields.length ? { entityId, fields } : null
+    })()
+    this.inflightMaps.set(tag, pending)
+    let loaded: EncryptionMapRecord | null
+    try {
+      loaded = await pending
+    } finally {
+      this.inflightMaps.delete(tag)
+    }
+    if (!loaded) {
+      this.missCache.set(tag, Date.now() + MAP_MISS_TTL_MS)
+      return []
+    }
+    this.missCache.delete(tag)
+    this.aggregateMemoryCache.set(tag, { at: Date.now(), record: loaded })
+    if (this.cache && typeof this.cache.set === 'function') {
+      await this.cache.set(tag, loaded, { ttl: 300 })
+    }
+    return loaded.fields
+  }
+
+  /**
+   * The field rules that are actually encrypted at rest for a scope.
+   *
+   * At the tenant-wide scope (`organizationId == null`) `getMap` resolves only the base map, but
+   * rows of the same entity may carry fields declared solely by an organization-scoped map. Reading
+   * or writing such a row with the base map alone stores those fields as plaintext and hands
+   * callers back undecrypted ciphertext, so every scope-aware path unions the organization maps in
+   * (#5949) — the same set `getEncryptedFieldNames` has reported since #2282.
+   */
+  private async resolveFieldRulesForScope(
+    entityId: string,
+    tenantId: string | null,
+    organizationId: string | null,
+    map: EncryptionMapRecord | null,
+  ): Promise<EncryptedFieldRule[]> {
+    const mapRules = normalizeEncryptedFieldRules(map?.fields)
+    if (organizationId != null) return mapRules
+    return mergeEncryptedFieldRules([mapRules, await this.getAllOrganizationFieldRules(entityId, tenantId)])
   }
 
   async invalidateMap(entityId: string, tenantId: string | null, organizationId: string | null): Promise<void> {
-    const tag = cacheKey({ entityId, tenantId, organizationId })
-    this.memoryCache.delete(tag)
-    this.inflightMaps.delete(tag)
-    this.missCache.delete(tag)
-    if (this.cache && typeof (this.cache as any).delete === 'function') {
-      await (this.cache as any).delete(tag)
+    const tags = [cacheKey({ entityId, tenantId, organizationId }), allOrganizationsCacheKey(entityId, tenantId)]
+    for (const tag of tags) {
+      this.memoryCache.delete(tag)
+      this.aggregateMemoryCache.delete(tag)
+      this.inflightMaps.delete(tag)
+      this.missCache.delete(tag)
+      if (this.cache && typeof (this.cache as any).delete === 'function') {
+        await (this.cache as any).delete(tag)
+      }
     }
   }
 
@@ -416,13 +540,13 @@ export class TenantDataEncryptionService {
       return []
     }
     const map = await this.getMap({ entityId, tenantId: tenantId ?? null, organizationId: organizationId ?? null })
-    const fields = new Set(normalizeEncryptedFieldNames(map?.fields))
-    if (organizationId == null) {
-      for (const field of await this.fetchAllOrganizationFieldNames(entityId, tenantId ?? null)) {
-        fields.add(field)
-      }
-    }
-    return Array.from(fields)
+    const fields = await this.resolveFieldRulesForScope(
+      entityId,
+      tenantId ?? null,
+      organizationId ?? null,
+      map,
+    )
+    return fields.map((rule) => rule.field)
   }
 
   private encryptFields(
@@ -506,18 +630,24 @@ export class TenantDataEncryptionService {
       return payload
     }
     const map = await this.getMap({ entityId, tenantId: tenantId ?? null, organizationId: organizationId ?? null })
-    if (!map || !map.fields?.length) {
+    const fields = await this.resolveFieldRulesForScope(
+      entityId,
+      tenantId ?? null,
+      organizationId ?? null,
+      map,
+    )
+    if (!fields.length) {
       debug('⚪️ encrypt.skip.no-map', { entityId, tenantId })
       return payload
     }
-    const keyId = resolveEncryptionKeyId(entityId, map.keyScope, tenantId)
+    const keyId = resolveEncryptionKeyId(entityId, map?.keyScope, tenantId)
     const dek = await this.resolveDekForEncrypt(keyId, { createIfMissing: options?.createMissingDek !== false })
     if (!dek) {
-      debug('⚠️ encrypt.skip.no-dek', { entityId, tenantId, keyScope: map.keyScope ?? 'tenant' })
+      debug('⚠️ encrypt.skip.no-dek', { entityId, tenantId, keyScope: map?.keyScope ?? 'tenant' })
       return payload
     }
-    debug('🔒 encrypt_entity', { entityId, tenantId, organizationId, fields: map.fields.length })
-    return this.encryptFields(payload, map.fields, dek)
+    debug('🔒 encrypt_entity', { entityId, tenantId, organizationId, fields: fields.length })
+    return this.encryptFields(payload, fields, dek)
   }
 
   async decryptEntityPayload(
@@ -531,17 +661,23 @@ export class TenantDataEncryptionService {
       return payload
     }
     const map = await this.getMap({ entityId, tenantId: tenantId ?? null, organizationId: organizationId ?? null })
-    if (!map || !map.fields?.length) {
+    const fields = await this.resolveFieldRulesForScope(
+      entityId,
+      tenantId ?? null,
+      organizationId ?? null,
+      map,
+    )
+    if (!fields.length) {
       debug('⚪️ decrypt.skip.no-map', { entityId, tenantId })
       return payload
     }
-    const keyId = resolveEncryptionKeyId(entityId, map.keyScope, tenantId)
+    const keyId = resolveEncryptionKeyId(entityId, map?.keyScope, tenantId)
     const dek = await this.getDek(keyId)
     if (!dek) {
-      debug('⚠️ decrypt.skip.no-dek', { entityId, tenantId, keyScope: map.keyScope ?? 'tenant' })
+      debug('⚠️ decrypt.skip.no-dek', { entityId, tenantId, keyScope: map?.keyScope ?? 'tenant' })
       return payload
     }
-    debug('🔓 decrypt_entity', { entityId, tenantId, organizationId, fields: map.fields.length })
-    return this.decryptFields(payload, map.fields, dek)
+    debug('🔓 decrypt_entity', { entityId, tenantId, organizationId, fields: fields.length })
+    return this.decryptFields(payload, fields, dek)
   }
 }

package/src/lib/query/engine.ts

@@ -60,9 +60,10 @@ export function isEncryptedLikeField(encrypted: ReadonlySet<string>, field: stri
   return fieldNameCandidates(field).some((candidate) => encrypted.has(candidate))
 }
 
-// The all-orgs union behind `getEncryptedFieldNames(..., organizationId: null)` is an UNCACHED
-// `encryption_maps` read. Encryption maps change on deploys, not per request, so a short TTL
-// removes the per-search round-trip without meaningfully delaying a map rollout.
+// The all-orgs union behind `getEncryptedFieldNames(..., organizationId: null)` reads
+// `encryption_maps`. The service caches that aggregate since #5949, but this set is per-search
+// hot and also spans the name-shape expansion below, so a short TTL keeps the whole resolved
+// set out of the request path without meaningfully delaying a map rollout.
 const ENCRYPTED_LIKE_FIELDS_TTL_MS = 60_000
 const ENCRYPTED_LIKE_FIELDS_CACHE_CAP = 500
 const encryptedLikeFieldsCache = new Map<string, { at: number; fields: Set<string> }>()
@@ -455,10 +456,10 @@ export class BasicQueryEngine implements QueryEngine {
     // `ignoreRuntimeHealth` asks the on-disk question -- a column holds ciphertext even while the
     // KMS is down -- so an outage keeps encrypted columns on the token path (#4622).
     // `organizationId: null` is deliberate, not an omission: the service then unions in every
-    // organization's map (`fetchAllOrganizationFieldNames`), so a field any org encrypts stays on
-    // the token path -- a wider set fails safe. Passing the request's org instead would silently
-    // break encrypted-column search for orgs without their own map. That union is an UNCACHED
-    // `encryption_maps` read, one extra round-trip per searched list request. `null` means
+    // organization's map, so a field any org encrypts stays on the token path -- a wider set
+    // fails safe. Passing the request's org instead would silently break encrypted-column search
+    // for orgs without their own map. The service caches that union (#5949), and the TTL cache
+    // above keeps even the cache lookup off the request path. `null` means
     // the encryption service could not answer at all; keep the pre-existing rewrite-everything
     // behavior then, because guessing "plaintext" would turn encrypted-column search into an
     // ILIKE-on-ciphertext that matches nothing.

package/src/lib/search/config.ts

@@ -21,6 +21,13 @@ export type SearchConfig = {
   entityBlocklistedFields?: Record<string, string[]>
   maxFieldChars?: number
   maxTokensPerField?: number
+  /**
+   * Ceiling on token rows across all fields of one record; `0` disables it. The budget is spent in
+   * the order the document's own keys iterate in, so on an over-budget record *which* fields stay
+   * searchable depends on that key order — see `buildSearchTokenRows` in
+   * `@open-mercato/core/modules/query_index/lib/search-tokens` before recomputing expected tokens
+   * from a document that did not come straight from the indexer.
+   */
   maxTokensPerRecord?: number
 }
 

package/src/modules/widgets/extension-points.ts

@@ -58,7 +58,13 @@ export type ExtensionPointPatternParameter = {
 }
 
 type ExtensionHostDeclarationBase = {
-  source: string
+  /**
+   * File(s) this host is mounted from, relative to the module's source root. A
+   * host mounted from more than one file may declare them all; the first entry
+   * remains the authoritative provenance path, the rest surface on the
+   * generated fact's `additionalSources`.
+   */
+  source: string | readonly string[]
   contextContract?: string
   dataContract?: string
   scopeContract?: string
@@ -398,6 +404,8 @@ export type ModuleExtensionHostFact = {
     | { kind: 'declaration'; path: string; symbol: string }
     | { kind: 'fact-ref'; factSection: string; factKey: string }
     | { kind: 'framework'; path: string; symbol: string }
+  /** Extra mount-point paths beyond the declaration's primary `source` entry. */
+  additionalSources?: string[]
   aliases?: string[]
   patternParameters?: Record<string, ExtensionPointPatternParameter>
   fallbacks?: string[]