@open-mercato/core 0.7.1-develop.7183.1.db9678eeb8 → 0.7.1-develop.7186.1.6e080a5017

package/src/modules/integrations/lib/credentials-service.ts

@@ -1,7 +1,7 @@
 import type { EntityManager } from '@mikro-orm/postgresql'
 import { decryptWithAesGcm, encryptWithAesGcm } from '@open-mercato/shared/lib/encryption/aes'
 import { findOneWithDecryption } from '@open-mercato/shared/lib/encryption/find'
-import { createKmsService } from '@open-mercato/shared/lib/encryption/kms'
+import { createKmsService, resolveEncryptionMode } from '@open-mercato/shared/lib/encryption/kms'
 import { parseDecryptedFieldValue } from '@open-mercato/shared/lib/encryption/tenantDataEncryptionService'
 import {
   getBundle,
@@ -21,15 +21,30 @@ const ENCRYPTED_CREDENTIALS_BLOB_KEY = '__om_encrypted_credentials_blob_v1'
  * configured. The credentials path deliberately fails closed instead of using
  * a hardcoded fallback secret; see security tracker finding #7.
  */
+export type CredentialsEncryptionUnavailableReason = 'no-dek' | 'sealed-while-disabled'
+
+const CREDENTIALS_ENCRYPTION_REMEDY: Record<CredentialsEncryptionUnavailableReason, string> = {
+  'no-dek':
+    'no tenant DEK is available. Configure Vault (VAULT_ADDR/VAULT_TOKEN) or ' +
+    'set TENANT_DATA_ENCRYPTION_FALLBACK_KEY in the environment.',
+  'sealed-while-disabled':
+    'they were sealed while TENANT_DATA_ENCRYPTION was on and it is now off, so no key can open ' +
+    'them. Re-enable TENANT_DATA_ENCRYPTION to read them again, or re-enter the credentials — ' +
+    'saving them while the toggle is off stores them in the clear. Note that ' +
+    '`mercato entities decrypt-database` does not reach this blob: it decrypts the columns an ' +
+    'encryption map covers, and this envelope sits inside the decrypted value.',
+}
+
 export class CredentialsEncryptionUnavailableError extends Error {
   readonly code = 'CREDENTIALS_ENCRYPTION_UNAVAILABLE'
-  constructor(tenantId: string) {
+  readonly reason: CredentialsEncryptionUnavailableReason
+  constructor(tenantId: string, reason: CredentialsEncryptionUnavailableReason = 'no-dek') {
     super(
       `Cannot encrypt or decrypt integration credentials for tenant ${tenantId}: ` +
-        `no tenant DEK is available. Configure Vault (VAULT_ADDR/VAULT_TOKEN) or ` +
-        `set TENANT_DATA_ENCRYPTION_FALLBACK_KEY in the environment.`,
+        CREDENTIALS_ENCRYPTION_REMEDY[reason],
     )
     this.name = 'CredentialsEncryptionUnavailableError'
+    this.reason = reason
   }
 }
 
@@ -37,6 +52,16 @@ export function isCredentialsEncryptionUnavailableError(error: unknown): error i
   return error instanceof CredentialsEncryptionUnavailableError
 }
 
+/**
+ * The one unavailable-reason an operator can act on without restoring a key: the blob predates
+ * `TENANT_DATA_ENCRYPTION=no` and no key exists to open it, so re-entering the credentials is the
+ * only way forward. The admin credentials route degrades to an empty form on this so that the
+ * re-entry is possible at all; `no-dek` (encryption on, KMS unreachable) still fails closed.
+ */
+export function isCredentialsSealedWhileDisabledError(error: unknown): boolean {
+  return isCredentialsEncryptionUnavailableError(error) && error.reason === 'sealed-while-disabled'
+}
+
 function isRecordValue(value: unknown): value is Record<string, unknown> {
   return !!value && typeof value === 'object' && !Array.isArray(value)
 }
@@ -112,8 +137,21 @@ export function createCredentialsService(em: EntityManager) {
     existing.isActive = true
   }
 
-  async function resolveCredentialsDek(scope: IntegrationScope): Promise<string> {
+  /**
+   * Resolve the DEK this tenant's credentials blob is sealed with, or `null` when the operator
+   * has switched tenant data encryption off.
+   *
+   * The `null` is the whole point of going through {@link resolveEncryptionMode} rather than
+   * asking the KMS directly. Under `TENANT_DATA_ENCRYPTION=no` the KMS is a noop and hands back
+   * nothing, which is indistinguishable — to `getTenantDek` alone — from Vault being down. Those
+   * two need opposite answers: an operator who turned encryption off expects plaintext, whereas a
+   * Vault outage must not silently downgrade a secret that is supposed to be sealed. So only
+   * `unavailable` throws.
+   */
+  async function resolveCredentialsDek(scope: IntegrationScope): Promise<string | null> {
     const kms = createKmsService()
+    if (resolveEncryptionMode(kms) === 'disabled') return null
+
     const existing = await kms.getTenantDek(scope.tenantId)
     if (existing?.key) return existing.key
 
@@ -128,6 +166,7 @@ export function createCredentialsService(em: EntityManager) {
     scope: IntegrationScope,
   ): Promise<Record<string, unknown>> {
     const dek = await resolveCredentialsDek(scope)
+    if (!dek) return credentials
     const payload = encryptWithAesGcm(JSON.stringify(credentials), dek)
     return { [ENCRYPTED_CREDENTIALS_BLOB_KEY]: payload.value }
   }
@@ -140,7 +179,14 @@ export function createCredentialsService(em: EntityManager) {
     const encrypted = credentials[ENCRYPTED_CREDENTIALS_BLOB_KEY]
     if (typeof encrypted !== 'string' || !encrypted) return credentials
 
+    // A sealed blob written before encryption was switched off. There is no key to open it with,
+    // and returning the envelope as if it were the credentials would hand an adapter a garbage
+    // secret, so this stays an error even in `disabled` mode rather than a silent empty credential
+    // set. The remedy is re-entering the credentials (see the reason's message); the admin route
+    // catches this specific reason so the form can load empty and accept them.
     const dek = await resolveCredentialsDek(scope)
+    if (!dek) throw new CredentialsEncryptionUnavailableError(scope.tenantId, 'sealed-while-disabled')
+
     const decryptedRaw = decryptWithAesGcm(encrypted, dek)
     if (!decryptedRaw) return {}
 

package/src/modules/integrations/lib/log-service.ts

@@ -1,9 +1,29 @@
 import type { EntityManager, FilterQuery } from '@mikro-orm/postgresql'
 import { findWithDecryption } from '@open-mercato/shared/lib/encryption/find'
+import { createLogger } from '@open-mercato/shared/lib/logger'
+import { getTelemetryRuntime } from '@open-mercato/shared/lib/telemetry/runtime'
+import { groupableCode } from '@open-mercato/shared/lib/telemetry/error-code'
 import type { IntegrationScope } from '@open-mercato/shared/modules/integrations/types'
 import type { ListIntegrationLogsQuery } from '../data/validators'
 import { IntegrationLog } from '../data/entities'
 
+const logger = createLogger('integrations')
+
+/**
+ * The error an `integration_logs` row at `level: 'error'` is reported as.
+ *
+ * A named type rather than a bare `Error` because every integration failure in
+ * the product funnels through one line below: backends that fingerprint on the
+ * error class would otherwise group these together with unrelated framework
+ * errors. The per-row `code` is what separates them from each other.
+ */
+export class IntegrationLogError extends Error {
+  constructor(message: string) {
+    super(message)
+    this.name = 'IntegrationLogError'
+  }
+}
+
 export type IntegrationLogAnalytics = {
   lastActivityAt: string | null
   totalCount: number
@@ -38,6 +58,42 @@ type LogInput = {
   payload?: Record<string, unknown> | null
 }
 
+/**
+ * Report an error row outward, so recording it is also reporting it.
+ *
+ * The row is the durable record; this is the signal. Message, `code` and opaque
+ * ids only — `payload` carries the failed item itself and MUST NOT leave the
+ * database. Wrapped because observability may never alter behaviour: the row is
+ * already flushed by the time this runs, and a telemetry fault degrades to a
+ * warning rather than failing the write its caller depends on.
+ *
+ * The row's own `code` is free-form and written by any module that resolves this
+ * service, third-party ones included, so it is narrowed to a fingerprint before
+ * it is reported: a code the backend cannot group on is worth less than the
+ * catch-all every integration error already shares.
+ */
+function reportErrorLog(input: LogInput, scope: IntegrationScope): void {
+  try {
+    getTelemetryRuntime()?.reportError(new IntegrationLogError(input.message), {
+      module: 'integrations',
+      code: groupableCode(input.code, 'integrations.log_error'),
+      attributes: {
+        integrationId: input.integrationId,
+        runId: input.runId ?? undefined,
+        scopeEntityType: input.scopeEntityType ?? undefined,
+        scopeEntityId: input.scopeEntityId ?? undefined,
+        organizationId: scope.organizationId,
+        tenantId: scope.tenantId,
+      },
+    })
+  } catch (telemetryError) {
+    logger.warn('Failed to report an integration error log to telemetry', {
+      integrationId: input.integrationId,
+      err: telemetryError as Error,
+    })
+  }
+}
+
 export function createIntegrationLogService(em: EntityManager) {
   return {
     async write(input: LogInput, scope: IntegrationScope): Promise<IntegrationLog> {
@@ -54,6 +110,7 @@ export function createIntegrationLogService(em: EntityManager) {
         tenantId: scope.tenantId,
       })
       await em.persist(row).flush()
+      if (input.level === 'error') reportErrorLog(input, scope)
       return row
     },
 

package/src/modules/payment_gateways/workers/status-poller.ts

@@ -44,7 +44,11 @@ export default async function handle(job: QueuedJob<PollerJobPayload>, ctx: Hand
         scopeEntityType: 'payment_transaction',
         scopeEntityId: transaction.id,
         level: 'error',
-        message: 'Payment status polling failed',
+        // The cause belongs in the message, not only in `payload`: the payload is
+        // the durable record and never leaves the database, so an operator paged by
+        // the reported error would otherwise learn only that a gateway failed.
+        message: `Payment status polling failed: ${message}`,
+        code: 'payment_gateways.status_poll_failed',
         payload: {
           transactionId: transaction.id,
           message,

package/src/modules/query_index/lib/search-tokens.ts

@@ -6,6 +6,8 @@ import {
   type SearchConfig,
 } from '@open-mercato/shared/lib/search/config'
 import { tokenizeText } from '@open-mercato/shared/lib/search/tokenize'
+import { looksLikeEncryptedPayload } from '@open-mercato/shared/lib/encryption/aes'
+import { createKmsService, resolveEncryptionMode, type KmsService } from '@open-mercato/shared/lib/encryption/kms'
 import { parseBooleanToken } from '@open-mercato/shared/lib/boolean'
 import { createLogger } from '@open-mercato/shared/lib/logger'
 
@@ -37,6 +39,8 @@ type BuildTokenOptions = {
   tenantId?: string | null
   doc?: Record<string, unknown> | null
   config?: SearchConfig
+  /** Resolved once per write by the exported entry points; see {@link shouldGuardCiphertext}. */
+  guardCiphertext?: boolean
 }
 
 const DEFAULT_SCOPE = { organizationId: null, tenantId: null }
@@ -68,6 +72,87 @@ function collectTextValues(value: unknown): string[] {
   return []
 }
 
+let guardKmsService: KmsService | null = null
+
+/**
+ * Whether the ciphertext guard below is allowed to run for this write.
+ *
+ * The guard recognises an envelope by its SHAPE, which is forgeable: `<16 b64>:<b64>:<24 b64>:v1`
+ * is a string any user can type into a searchable field. `tenantDataEncryptionService` removed the
+ * same structural test for that reason (#2720). So the guard may only run where the shape is the
+ * ONLY test available -- which is exactly where no DEK is reachable:
+ *
+ * - `active`      -- the indexer decrypted the document before handing it here, so a value still
+ *                    shaped like an envelope is plaintext somebody typed. Indexing it is correct,
+ *                    and skipping it would let that person freeze their own record's tokens at a
+ *                    past state. The guard stays off, which also keeps it off the hot path of
+ *                    every normal deployment.
+ * - `disabled`    -- `decryptIndexDocForSearch` is a no-op, so ciphertext arrives undecrypted.
+ * - `unavailable` -- the decrypt was attempted and could not complete, same outcome.
+ *
+ * Resolved once per write rather than per document, over a KMS built once per process:
+ * {@link createKmsService} logs when it falls back, and a reindex calls this once per record. The
+ * toggle itself is still re-read every call by {@link resolveEncryptionMode}; only the KMS is
+ * cached, and it already requires a restart to change, since DEK and map caches are in-process.
+ */
+function shouldGuardCiphertext(): boolean {
+  guardKmsService ??= createKmsService()
+  return resolveEncryptionMode(guardKmsService) !== 'active'
+}
+
+/**
+ * Fields whose value is an AES-GCM envelope rather than the text it is supposed to hold.
+ *
+ * Search tokens are hashes of PLAINTEXT: the indexer decrypts a document before tokenising it
+ * (`indexer.ts` -> `decryptIndexDocForSearch`), which is what lets the token index survive
+ * encryption being switched on or off. That decrypt step is a no-op once
+ * `TENANT_DATA_ENCRYPTION=no`, so an operator who flips the toggle before running
+ * `mercato entities decrypt-database` starts feeding ciphertext into the tokeniser. The tokens
+ * that come out are hashes of base64 noise and match nothing, and because a write REPLACES a
+ * record's tokens, the good plaintext tokens already in the table would be deleted to make room
+ * for them -- turning a recoverable misordering into permanent search loss.
+ *
+ * Detecting the envelope by shape lets the write skip those fields and leave what is already
+ * indexed alone. `guard` gates that detection; see {@link shouldGuardCiphertext} for why it is not
+ * unconditional.
+ */
+function ciphertextFieldsOf(
+  doc: Record<string, unknown> | null | undefined,
+  guard: boolean,
+): Set<string> {
+  const fields = new Set<string>()
+  if (!guard || !doc) return fields
+  for (const [field, value] of Object.entries(doc)) {
+    const values = collectTextValues(value)
+    if (values.length && values.some((text) => looksLikeEncryptedPayload(text))) fields.add(field)
+  }
+  return fields
+}
+
+const warnedCiphertextEntities = new Set<string>()
+
+function warnCiphertextSkipped(entityType: string, tenantId: string | null, fields: Set<string>): void {
+  if (!fields.size) return
+  // Once per entity type per tenant per process: a full reindex would otherwise emit this per
+  // record, while keying on the entity type alone would let the first affected tenant in a shared
+  // process consume the one warning every other tenant's operator needed.
+  const key = `${entityType}|${tenantId ?? ''}`
+  if (warnedCiphertextEntities.has(key)) return
+  warnedCiphertextEntities.add(key)
+  logger.warn(
+    'Search indexing skipped ciphertext fields and preserved their existing tokens. '
+      + 'This means TENANT_DATA_ENCRYPTION was switched off while encrypted data was still at rest. '
+      + 'Run `mercato entities decrypt-database` and reindex; until then these fields are not searchable.',
+    { entityType, tenantId, fields: Array.from(fields).sort((left, right) => left.localeCompare(right)) },
+  )
+}
+
+/** Test seam: both the warning above and the KMS behind the guard are once-per-process. */
+export function resetCiphertextGuardState(): void {
+  warnedCiphertextEntities.clear()
+  guardKmsService = null
+}
+
 function shouldIndexField(
   field: string,
   value: unknown,
@@ -97,9 +182,12 @@ export function buildSearchTokenRows(params: BuildTokenOptions): SearchTokenRow[
   const limits = resolveSearchTokenLimits(config)
   const recordLimit = limits.maxTokensPerRecord > 0 ? limits.maxTokensPerRecord : Number.POSITIVE_INFINITY
   const fieldLimit = limits.maxTokensPerField > 0 ? limits.maxTokensPerField : Number.POSITIVE_INFINITY
+  const ciphertextFields = ciphertextFieldsOf(params.doc, params.guardCiphertext ?? shouldGuardCiphertext())
+  warnCiphertextSkipped(params.entityType, scope.tenantId, ciphertextFields)
 
   for (const [field, rawValue] of Object.entries(params.doc)) {
     if (tokens.length >= recordLimit) break
+    if (ciphertextFields.has(field)) continue
     if (!shouldIndexField(field, rawValue, config, params.entityType)) continue
     const values = collectTextValues(rawValue)
     const seen = new Set<string>()
@@ -149,11 +237,18 @@ export function buildSearchTokenRows(params: BuildTokenOptions): SearchTokenRow[
   return tokens
 }
 
-function buildFieldPairs(recordId: string, doc?: Record<string, unknown> | null): EntityFieldPair[] {
+function buildFieldPairs(
+  recordId: string,
+  doc?: Record<string, unknown> | null,
+  skipFields?: Set<string>,
+): EntityFieldPair[] {
   if (!doc) return []
   const pairs: EntityFieldPair[] = []
   const dedupe = new Set<string>()
   for (const field of Object.keys(doc)) {
+    // The delete below is scoped to these pairs, so omitting a field here is what preserves the
+    // tokens already stored for it rather than merely declining to write new ones.
+    if (skipFields?.has(field)) continue
     const key = `${recordId}|${field}`
     if (dedupe.has(key)) continue
     dedupe.add(key)
@@ -221,12 +316,22 @@ export async function replaceSearchTokensForRecord(
   params: BuildTokenOptions,
   options?: { trx?: SearchTokenExecutor },
 ): Promise<void> {
-  const rows = buildSearchTokenRows(params)
+  const guardCiphertext = params.guardCiphertext ?? shouldGuardCiphertext()
+  const rows = buildSearchTokenRows({ ...params, guardCiphertext })
   const config = params.config ?? resolveSearchConfig()
   if (!config.enabled) return
   const organizationId = params.organizationId ?? null
   const tenantId = params.tenantId ?? null
-  const fieldPairs = buildFieldPairs(String(params.recordId), params.doc)
+  const ciphertextFields = ciphertextFieldsOf(params.doc, guardCiphertext)
+  const fieldPairs = buildFieldPairs(String(params.recordId), params.doc, ciphertextFields)
+
+  // An empty pair list normally means the document is gone, and the delete below then purges the
+  // record wholesale. It can now also mean every field was skipped as ciphertext, where a purge
+  // would destroy precisely the tokens the skip exists to protect. Distinguish the two.
+  if (params.doc && ciphertextFields.size && !fieldPairs.length) {
+    debug('record.preserve-ciphertext', { entityType: params.entityType, recordId: params.recordId })
+    return
+  }
 
   // Same comparison #5402 gave the batch path, over the scope this path actually writes: the
   // delete below is narrowed to the document's own `(entity_id, field)` pairs, so the comparison
@@ -325,13 +430,30 @@ export async function deleteSearchTokensForRecord(
 
 export async function replaceSearchTokensForBatch(
   db: Kysely<any>,
-  payloads: Array<BuildTokenOptions & { doc: Record<string, unknown> }>
+  allPayloads: Array<BuildTokenOptions & { doc: Record<string, unknown> }>
 ): Promise<void> {
-  if (!payloads.length) return
+  if (!allPayloads.length) return
   const config = resolveSearchConfig()
   if (!config.enabled) return
 
-  const rows = payloads.flatMap((payload) => buildSearchTokenRows({ ...payload, config }))
+  // A record carrying ciphertext drops out of the batch entirely, rather than being rewritten
+  // without its encrypted fields. This path deletes by `entity_id` -- it cannot express "replace
+  // these fields and leave those alone" the way the per-record path can -- so partial handling
+  // here would still delete the tokens we are trying to protect. Skipping the record leaves every
+  // one of its tokens, encrypted-field and plaintext-field alike, exactly as it was. The state is
+  // transient by construction: `decrypt-database` followed by a reindex rebuilds all of it.
+  const guardCiphertext = shouldGuardCiphertext()
+  const preservedRecordIds = new Set<string>()
+  const payloads = allPayloads.filter((payload) => {
+    const ciphertextFields = ciphertextFieldsOf(payload.doc, guardCiphertext)
+    if (!ciphertextFields.size) return true
+    warnCiphertextSkipped(payload.entityType, payload.tenantId ?? null, ciphertextFields)
+    preservedRecordIds.add(String(payload.recordId))
+    return false
+  })
+  if (!payloads.length) return
+
+  const rows = payloads.flatMap((payload) => buildSearchTokenRows({ ...payload, config, guardCiphertext }))
   if (!rows.length) {
     const entityType = payloads[0]?.entityType
     if (!entityType) return
@@ -438,6 +560,7 @@ export async function replaceSearchTokensForBatch(
     entityType: payloads[0].entityType,
     recordCount: payloads.length,
     changedCount: changedRecordKeys.size,
+    preservedCiphertextRecordCount: preservedRecordIds.size,
   })
   if (!changedRecordKeys.size) return