@open-mercato/shared 0.6.5-develop.5048.1.fd82f4ae17 → 0.6.5-develop.5116.1.f0af9e5080

package/src/lib/encryption/kms.ts

@@ -22,6 +22,7 @@ export interface KmsService {
   getTenantDek(tenantId: string): Promise<TenantDek | null>
   createTenantDek(tenantId: string): Promise<TenantDek | null>
   isHealthy(): boolean
+  invalidateDek?(tenantId: string): void
 }
 
 class FallbackKmsService implements KmsService {
@@ -76,6 +77,11 @@ class FallbackKmsService implements KmsService {
     }
     return null
   }
+
+  invalidateDek(tenantId: string): void {
+    this.primary.invalidateDek?.(tenantId)
+    this.fallback?.invalidateDek?.(tenantId)
+  }
 }
 
 type VaultClientOpts = {
@@ -90,6 +96,10 @@ type VaultReadResponse = {
   data?: { data?: { key?: string; version?: number }; metadata?: Record<string, unknown> }
 }
 
+// 'conflict' = a check-and-set write lost to a concurrent writer (normal race
+// outcome, Vault still healthy); 'error' = the write genuinely failed.
+type VaultWriteOutcome = 'ok' | 'conflict' | 'error'
+
 function normalizeEnv(value: string | undefined): string {
   if (!value) return ''
   return value.trim().replace(/(?:^['"]|['"]$)/g, '')
@@ -230,11 +240,13 @@ export class HashicorpVaultKmsService implements KmsService {
     }
   }
 
-  private async writeVault(path: string, key: string): Promise<boolean> {
+  private async writeVault(path: string, key: string, opts?: { cas?: number }): Promise<VaultWriteOutcome> {
     if (!this.vaultAddr || !this.vaultToken) {
       this.healthy = false
-      return false
+      return 'error'
     }
+    const body: { data: { key: string }; options?: { cas: number } } = { data: { key } }
+    if (typeof opts?.cas === 'number') body.options = { cas: opts.cas }
     try {
       const res = await fetchWithTimeout(`${this.vaultAddr}/v1/${path}`, {
         method: 'POST',
@@ -242,14 +254,23 @@ export class HashicorpVaultKmsService implements KmsService {
           'X-Vault-Token': this.vaultToken,
           'Content-Type': 'application/json',
         },
-        body: JSON.stringify({ data: { key } }),
+        body: JSON.stringify(body),
         timeoutMs: this.requestTimeoutMs,
       })
-      this.healthy = res.ok
-      if (!res.ok) {
-        console.warn('⚠️ [encryption][kms] Vault write failed', { path, status: res.status })
+      if (res.ok) {
+        this.healthy = true
+        return 'ok'
       }
-      return res.ok
+      // KV v2 returns 400 when a check-and-set write loses to a concurrent
+      // writer (path already at a newer version). That is a normal race outcome,
+      // not an unhealthy Vault — don't flip `healthy`.
+      if (typeof opts?.cas === 'number' && res.status === 400) {
+        console.warn('⚠️ [encryption][kms] Vault write CAS conflict (concurrent DEK create)', { path, status: res.status })
+        return 'conflict'
+      }
+      this.healthy = false
+      console.warn('⚠️ [encryption][kms] Vault write failed', { path, status: res.status })
+      return 'error'
     } catch (err) {
       this.healthy = false
       console.warn('⚠️ [encryption][kms] Vault write error', {
@@ -257,7 +278,7 @@ export class HashicorpVaultKmsService implements KmsService {
         error: (err as Error)?.message || String(err),
         timeoutMs: this.requestTimeoutMs,
       })
-      return false
+      return 'error'
     }
   }
 
@@ -287,16 +308,40 @@ export class HashicorpVaultKmsService implements KmsService {
   }
 
   async createTenantDek(tenantId: string): Promise<TenantDek | null> {
-    const key = generateDek()
     const path = this.buildKeyPath(tenantId)
-    const ok = await this.writeVault(path, key)
-    if (ok) {
+    // Read-before-write: if a DEK already exists for this tenant (another request
+    // or process created it first), adopt it instead of overwriting the active
+    // key — overwriting orphans every row already encrypted under it (#2746).
+    const existing = await this.readVault(path)
+    const existingKey = existing?.data?.data?.key
+    if (existingKey) {
+      return this.remember({ tenantId, key: existingKey, fetchedAt: this.now() })
+    }
+    // A read failure (timeout / 5xx) flips `healthy` off; don't blind-write a new
+    // key over a possibly-existing one we just couldn't read — let the caller fall back.
+    if (!this.healthy) return null
+    const key = generateDek()
+    const outcome = await this.writeVault(path, key, { cas: 0 })
+    if (outcome === 'ok') {
       console.info('🔑 [encryption][kms] Stored tenant DEK in Vault', { tenantId, path })
-    } else {
-      console.warn('⚠️ [encryption][kms] Failed to store tenant DEK in Vault', { tenantId, path })
+      return this.remember({ tenantId, key, fetchedAt: this.now() })
+    }
+    if (outcome === 'conflict') {
+      // A concurrent create won the CAS race — adopt the winner's key so both
+      // callers encrypt under the same DEK.
+      const winner = await this.readVault(path)
+      const winnerKey = winner?.data?.data?.key
+      if (winnerKey) {
+        console.info('🔑 [encryption][kms] Adopted concurrently-created tenant DEK', { tenantId, path })
+        return this.remember({ tenantId, key: winnerKey, fetchedAt: this.now() })
+      }
     }
-    if (!ok) return null
-    return this.remember({ tenantId, key, fetchedAt: this.now() })
+    console.warn('⚠️ [encryption][kms] Failed to store tenant DEK in Vault', { tenantId, path })
+    return null
+  }
+
+  invalidateDek(tenantId: string): void {
+    this.cache.delete(tenantId)
   }
 }
 

package/src/lib/encryption/tenantDataEncryptionService.ts

@@ -22,6 +22,10 @@ type MapCacheKey = {
 }
 
 const MAP_MISS_TTL_MS = 5 * 60 * 1000
+// Mirror the Vault KMS default DEK TTL so a rotated/revoked tenant key is picked
+// up by long-lived processes without a restart (#2746). The service-level cache
+// previously had no TTL and shadowed the KMS's own 15-minute expiry.
+const DEK_CACHE_TTL_MS = 15 * 60 * 1000
 
 function cacheKey(key: MapCacheKey): string {
   return [
@@ -106,11 +110,13 @@ export class TenantDataEncryptionService {
   private static globalMemoryCache = new Map<string, EncryptionMapRecord>()
   private static globalInflightMaps = new Map<string, Promise<EncryptionMapRecord | null>>()
   private static globalDekCache = new Map<string, TenantDek>()
+  private static globalInflightDeks = new Map<string, Promise<TenantDek | null>>()
   private static globalMissCache = new Map<string, number>()
   private readonly kms: KmsService
   private readonly cache?: CacheStrategy
   private readonly memoryCache = TenantDataEncryptionService.globalMemoryCache
   private readonly dekCache = TenantDataEncryptionService.globalDekCache
+  private readonly inflightDeks = TenantDataEncryptionService.globalInflightDeks
   private readonly inflightMaps = TenantDataEncryptionService.globalInflightMaps
   private readonly missCache = TenantDataEncryptionService.globalMissCache
 
@@ -126,10 +132,15 @@ export class TenantDataEncryptionService {
     return isTenantDataEncryptionEnabled() && this.kms.isHealthy()
   }
 
+  private isDekExpired(dek: TenantDek): boolean {
+    return Date.now() - dek.fetchedAt > DEK_CACHE_TTL_MS
+  }
+
   async getDek(tenantId: string | null | undefined): Promise<TenantDek | null> {
     if (!tenantId) return null
     const cached = this.dekCache.get(tenantId)
-    if (cached) return cached
+    if (cached && !this.isDekExpired(cached)) return cached
+    if (cached) this.dekCache.delete(tenantId)
     const dek = await this.kms.getTenantDek(tenantId)
     if (!dek) {
       debug('🔎 dek.miss', { tenantId })
@@ -144,9 +155,22 @@ export class TenantDataEncryptionService {
     const existing = await this.getDek(tenantId)
     if (existing || !tenantId) return existing ?? null
     if (typeof this.kms.createTenantDek !== 'function') return existing ?? null
-    const created = await this.kms.createTenantDek(tenantId)
-    if (created) this.dekCache.set(tenantId, created)
-    return created ?? null
+    // Dedupe concurrent first-time creation within this process so two callers
+    // can't each generate a distinct DEK and overwrite one another (#2746).
+    // Mirrors the encryption-map inflight dedupe (`globalInflightMaps`).
+    const pending = this.inflightDeks.get(tenantId)
+    if (pending) return pending
+    const creation = (async () => {
+      const created = await this.kms.createTenantDek(tenantId)
+      if (created) this.dekCache.set(tenantId, created)
+      return created ?? null
+    })()
+    this.inflightDeks.set(tenantId, creation)
+    try {
+      return await creation
+    } finally {
+      this.inflightDeks.delete(tenantId)
+    }
   }
 
   async createDek(tenantId: string): Promise<TenantDek | null> {
@@ -246,6 +270,15 @@ export class TenantDataEncryptionService {
     }
   }
 
+  // Force a flush of a tenant's cached DEK across the service-level cache and the
+  // underlying KMS cache so an operator can pick up a rotated/revoked key without
+  // a process restart (#2746).
+  invalidateDek(tenantId: string): void {
+    this.dekCache.delete(tenantId)
+    this.inflightDeks.delete(tenantId)
+    this.kms.invalidateDek?.(tenantId)
+  }
+
   async getEncryptedFieldNames(
     entityId: string,
     tenantId: string | null | undefined,

package/src/lib/query/__tests__/resolve-registered-entity-table.test.ts

@@ -0,0 +1,83 @@
+import {
+  resolveRegisteredEntityTableName,
+  resolveEntityTableName,
+  isValidEntityIdShape,
+  ENTITY_ID_PATTERN,
+} from '../engine'
+
+function makeEm(metaByClass: Record<string, string>) {
+  const all = Object.entries(metaByClass).map(([className, tableName]) => ({ className, tableName }))
+  return {
+    getMetadata: () => ({
+      find: (className: string) => {
+        const tableName = metaByClass[className]
+        return tableName ? { tableName } : undefined
+      },
+      getAll: () => all,
+    }),
+  } as any
+}
+
+describe('resolveRegisteredEntityTableName', () => {
+  it('resolves a registered entity via class-name metadata', () => {
+    const em = makeEm({ Todo: 'todos' })
+    expect(resolveRegisteredEntityTableName(em, 'example:todo')).toBe('todos')
+  })
+
+  it('resolves a registered entity via the secondary table-name lookup', () => {
+    const em = makeEm({ SomeOtherClass: 'directory_organizations' })
+    expect(resolveRegisteredEntityTableName(em, 'directory:organization')).toBe('directory_organizations')
+  })
+
+  it('returns null for an entity type that does not map to any registered metadata', () => {
+    const em = makeEm({ Todo: 'todos' })
+    expect(resolveRegisteredEntityTableName(em, 'foo:auth_user')).toBeNull()
+    expect(resolveRegisteredEntityTableName(em, 'foo:user')).toBeNull()
+  })
+
+  it('returns null when no metadata is available', () => {
+    expect(resolveRegisteredEntityTableName(undefined, 'example:todo')).toBeNull()
+    expect(resolveRegisteredEntityTableName({} as any, 'example:todo')).toBeNull()
+  })
+
+  it('never pluralizes attacker-chosen ids into a real table name', () => {
+    const em = makeEm({})
+    expect(resolveRegisteredEntityTableName(em, 'foo:auth_user')).toBeNull()
+    expect(resolveRegisteredEntityTableName(em, 'foo:user')).toBeNull()
+  })
+})
+
+describe('resolveEntityTableName fallback (broad query path)', () => {
+  it('still falls back to a pluralized guess for unregistered ids', () => {
+    const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {})
+    try {
+      const em = makeEm({})
+      expect(resolveEntityTableName(em, 'foo:never_registered_widget')).toBe('never_registered_widgets')
+    } finally {
+      warnSpy.mockRestore()
+    }
+  })
+})
+
+describe('isValidEntityIdShape', () => {
+  it('accepts canonical module:entity ids', () => {
+    expect(isValidEntityIdShape('example:todo')).toBe(true)
+    expect(isValidEntityIdShape('directory:organization')).toBe(true)
+    expect(isValidEntityIdShape('query_index:search_token')).toBe(true)
+  })
+
+  it('rejects ids without exactly two snake_case segments', () => {
+    expect(isValidEntityIdShape('todos')).toBe(false)
+    expect(isValidEntityIdShape('auth_users')).toBe(false)
+    expect(isValidEntityIdShape('foo:bar:baz')).toBe(false)
+    expect(isValidEntityIdShape('Foo:Bar')).toBe(false)
+    expect(isValidEntityIdShape('1bad:entity')).toBe(false)
+    expect(isValidEntityIdShape('foo:')).toBe(false)
+    expect(isValidEntityIdShape(':bar')).toBe(false)
+    expect(isValidEntityIdShape('foo bar:baz')).toBe(false)
+  })
+
+  it('exposes the pattern for schema reuse', () => {
+    expect(ENTITY_ID_PATTERN.test('example:todo')).toBe(true)
+  })
+})

package/src/lib/query/engine.ts

@@ -42,6 +42,15 @@ type ResolvedCustomFieldSource = {
 
 type ResultRow = Record<string, unknown>
 
+/**
+ * Canonical `module:entity` entity-id shape (both segments snake_case,
+ * starting with a lowercase letter). Used to validate caller-supplied entity
+ * ids at security-sensitive boundaries before they reach table resolution.
+ */
+export const ENTITY_ID_PATTERN = /^[a-z][a-z0-9_]*:[a-z][a-z0-9_]*$/
+
+export const isValidEntityIdShape = (value: string): boolean => ENTITY_ID_PATTERN.test(value)
+
 const pluralizeBaseName = (name: string): string => {
   if (!name) return name
   if (name.endsWith('s')) return name
@@ -65,46 +74,66 @@ const candidateClassNames = (rawName: string): string[] => {
   return Array.from(candidates)
 }
 
-export function resolveEntityTableName(em: EntityManager | undefined, entity: EntityId): string {
-  if (entityTableCache.has(entity)) {
-    return entityTableCache.get(entity)!
-  }
+/**
+ * Resolve an entity id to a table name strictly via registered MikroORM metadata.
+ *
+ * Unlike {@link resolveEntityTableName}, this never falls back to a pluralized
+ * guess: it returns `null` when no registered entity matches. Use it for
+ * security-sensitive call sites (e.g. the reindexer) that must refuse to read
+ * arbitrary, attacker-chosen tables that happen to exist in the schema.
+ */
+export function resolveRegisteredEntityTableName(
+  em: EntityManager | undefined,
+  entity: EntityId,
+): string | null {
   const parts = String(entity || '').split(':')
   const rawName = (parts[1] && parts[1].trim().length > 0) ? parts[1] : (parts[0] || '').trim()
   const metadata = (em as any)?.getMetadata?.()
 
-  if (metadata && rawName) {
-    const candidates = candidateClassNames(rawName)
-    for (const candidate of candidates) {
-      try {
-        const meta = metadata.find?.(candidate)
-        if (meta?.tableName) {
-          const tableName = String(meta.tableName)
-          entityTableCache.set(entity, tableName)
-          return tableName
-        }
-      } catch {}
-    }
+  if (!metadata || !rawName) return null
 
-    // Secondary lookup: search ORM metadata by candidate table names
-    const modulePrefix = parts[0] ?? ''
-    const candidateTables = [
-      `${modulePrefix}_${rawName}`,
-      pluralizeBaseName(rawName),
-      `${modulePrefix}_${pluralizeBaseName(rawName)}`,
-    ]
+  const candidates = candidateClassNames(rawName)
+  for (const candidate of candidates) {
     try {
-      const allMeta: any[] = metadata.getAll?.() ?? []
-      for (const meta of allMeta) {
-        if (meta?.tableName && candidateTables.includes(String(meta.tableName))) {
-          const tableName = String(meta.tableName)
-          entityTableCache.set(entity, tableName)
-          return tableName
-        }
+      const meta = metadata.find?.(candidate)
+      if (meta?.tableName) {
+        return String(meta.tableName)
       }
     } catch {}
   }
 
+  // Secondary lookup: search ORM metadata by candidate table names
+  const modulePrefix = parts[0] ?? ''
+  const candidateTables = [
+    `${modulePrefix}_${rawName}`,
+    pluralizeBaseName(rawName),
+    `${modulePrefix}_${pluralizeBaseName(rawName)}`,
+  ]
+  try {
+    const allMeta: any[] = metadata.getAll?.() ?? []
+    for (const meta of allMeta) {
+      if (meta?.tableName && candidateTables.includes(String(meta.tableName))) {
+        return String(meta.tableName)
+      }
+    }
+  } catch {}
+
+  return null
+}
+
+export function resolveEntityTableName(em: EntityManager | undefined, entity: EntityId): string {
+  if (entityTableCache.has(entity)) {
+    return entityTableCache.get(entity)!
+  }
+  const parts = String(entity || '').split(':')
+  const rawName = (parts[1] && parts[1].trim().length > 0) ? parts[1] : (parts[0] || '').trim()
+
+  const registered = resolveRegisteredEntityTableName(em, entity)
+  if (registered) {
+    entityTableCache.set(entity, registered)
+    return registered
+  }
+
   const fallback = pluralizeBaseName(rawName || '')
   console.warn(
     `[QueryEngine] Could not resolve entity "${entity}" via ORM metadata. ` +

package/src/modules/vector.ts

@@ -1,5 +1,6 @@
 import type { EntityId } from './entities'
 import type { QueryEngine } from '../lib/query/types'
+import type { SearchFieldPolicy } from './search'
 
 export type VectorDriverId = 'pgvector' | 'qdrant' | 'chromadb'
 
@@ -71,6 +72,12 @@ export type VectorEntityConfig = {
    * Provide extra deep links rendered next to the search result.
    */
   resolveLinks?: (ctx: VectorBuildContext) => Promise<VectorLinkDescriptor[] | null> | VectorLinkDescriptor[] | null
+  /**
+   * Controls which record/custom fields are eligible for the default embedding source builder.
+   * Applied only when `buildSource` is not provided. `excluded`/`hashOnly` fields are never embedded;
+   * when `searchable` is defined it acts as an allowlist for both record and custom fields.
+   */
+  fieldPolicy?: SearchFieldPolicy
 }
 
 export type VectorModuleConfig = {