@open-mercato/shared 0.6.7 → 0.6.8-develop.6874.1.982d6097d8

package/dist/lib/query/ciphertext-search-warning.js

@@ -0,0 +1,45 @@
+import { createLogger } from "../logger/index.js";
+const logger = createLogger("shared").child({ component: "query" });
+const WARN_CACHE_CAP = 5e3;
+const warned = /* @__PURE__ */ new Set();
+const warnKey = (entity, tenantId, field) => `${entity}|${tenantId ?? "null"}|${field}`;
+const normalizeColumnName = (value) => value.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase();
+function resetCiphertextLikeWarnCache() {
+  warned.clear();
+}
+async function warnOnCiphertextLikeFallback(params) {
+  const { entity, tenantId, reason, service } = params;
+  if (!service || service.isEnabled?.() === false) return;
+  if (typeof service.getEncryptedFieldNames !== "function") return;
+  const candidates = params.fields.filter((field) => typeof field === "string" && !field.startsWith("cf:")).map((field) => normalizeColumnName(field)).filter((field, index, all) => field.length > 0 && all.indexOf(field) === index).filter((field) => !warned.has(warnKey(entity, tenantId, field)));
+  if (!candidates.length) return;
+  try {
+    const encrypted = new Set(
+      (await service.getEncryptedFieldNames(entity, tenantId, null)).map(
+        (field) => normalizeColumnName(String(field))
+      )
+    );
+    for (const field of candidates) {
+      if (!encrypted.has(field)) continue;
+      if (warned.size >= WARN_CACHE_CAP) warned.clear();
+      warned.add(warnKey(entity, tenantId, field));
+      logger.warn("Text search filter cannot match an encrypted column", {
+        entity,
+        field,
+        reason,
+        hint: reason === "search-disabled" ? "OM_SEARCH_ENABLED is off, so the filter runs as ILIKE against ciphertext and matches nothing." : reason === "no-indexable-tokens" ? "The filter value produced no search tokens, so it runs as ILIKE against ciphertext and matches nothing. Use at least OM_SEARCH_MIN_LEN indexable characters, or filter on the deterministic hash column." : "No search_tokens exist for this entity and scope, so the filter runs as ILIKE against ciphertext and matches nothing. Index the entity through query_index and reindex, or filter on the deterministic hash column."
+      });
+    }
+  } catch (err) {
+    logger.debug("Ciphertext search warning check failed", {
+      entity,
+      err: err instanceof Error ? err.message : String(err)
+    });
+  }
+}
+export {
+  normalizeColumnName,
+  resetCiphertextLikeWarnCache,
+  warnOnCiphertextLikeFallback
+};
+//# sourceMappingURL=ciphertext-search-warning.js.map

package/dist/lib/query/ciphertext-search-warning.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../src/lib/query/ciphertext-search-warning.ts"],
+  "sourcesContent": ["import { createLogger } from '../logger'\nimport type { EntityId } from '../../modules/entities'\n\nconst logger = createLogger('shared').child({ component: 'query' })\n\n/**\n * Minimal slice of `TenantDataEncryptionService` needed to decide whether a\n * text filter targets an encrypted column.\n */\nexport type CiphertextWarningEncryptionService = {\n  getEncryptedFieldNames?: (\n    entityId: EntityId,\n    tenantId?: string | null,\n    organizationId?: string | null,\n  ) => Promise<readonly string[]>\n  isEnabled?: () => boolean\n} | null | undefined\n\nexport type CiphertextLikeFallbackReason = 'no-indexable-tokens' | 'no-search-tokens' | 'search-disabled'\n\n// One warning per entity/tenant/field per process \u2014 the fallback repeats on\n// every request, and an operator only needs to learn about it once.\nconst WARN_CACHE_CAP = 5000\nconst warned = new Set<string>()\n\nconst warnKey = (entity: string, tenantId: string | null, field: string): string =>\n  `${entity}|${tenantId ?? 'null'}|${field}`\n\nexport const normalizeColumnName = (value: string): string =>\n  value.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase()\n\n/** Test seam: the warn-once cache is process-global by design. */\nexport function resetCiphertextLikeWarnCache(): void {\n  warned.clear()\n}\n\n/**\n * Warn when a `like`/`ilike` filter is about to run against a column that an\n * encryption map covers while the token index is unavailable.\n *\n * Without this the query degrades silently: the predicate compares a plaintext\n * pattern against ciphertext, matches nothing, and the endpoint returns an\n * empty page that is indistinguishable from a genuine no-result. Issue #2990.\n *\n * Never throws and never blocks the query \u2014 a failure here is logged at debug\n * level and the caller proceeds unchanged.\n */\nexport async function warnOnCiphertextLikeFallback(params: {\n  entity: string\n  fields: readonly string[]\n  tenantId: string | null\n  reason: CiphertextLikeFallbackReason\n  service: CiphertextWarningEncryptionService\n}): Promise<void> {\n  const { entity, tenantId, reason, service } = params\n  if (!service || service.isEnabled?.() === false) return\n  if (typeof service.getEncryptedFieldNames !== 'function') return\n\n  const candidates = params.fields\n    .filter((field) => typeof field === 'string' && !field.startsWith('cf:'))\n    .map((field) => normalizeColumnName(field))\n    .filter((field, index, all) => field.length > 0 && all.indexOf(field) === index)\n    .filter((field) => !warned.has(warnKey(entity, tenantId, field)))\n  if (!candidates.length) return\n\n  try {\n    // `organizationId: null` resolves the global map plus every per-organization\n    // override, so a field encrypted for any organization is reported.\n    const encrypted = new Set(\n      (await service.getEncryptedFieldNames(entity, tenantId, null)).map((field) =>\n        normalizeColumnName(String(field)),\n      ),\n    )\n    for (const field of candidates) {\n      if (!encrypted.has(field)) continue\n      if (warned.size >= WARN_CACHE_CAP) warned.clear()\n      warned.add(warnKey(entity, tenantId, field))\n      logger.warn('Text search filter cannot match an encrypted column', {\n        entity,\n        field,\n        reason,\n        hint: reason === 'search-disabled'\n          ? 'OM_SEARCH_ENABLED is off, so the filter runs as ILIKE against ciphertext and matches nothing.'\n          : reason === 'no-indexable-tokens'\n            ? 'The filter value produced no search tokens, so it runs as ILIKE against ciphertext and matches nothing. Use at least OM_SEARCH_MIN_LEN indexable characters, or filter on the deterministic hash column.'\n            : 'No search_tokens exist for this entity and scope, so the filter runs as ILIKE against ciphertext and matches nothing. Index the entity through query_index and reindex, or filter on the deterministic hash column.',\n      })\n    }\n  } catch (err) {\n    logger.debug('Ciphertext search warning check failed', {\n      entity,\n      err: err instanceof Error ? err.message : String(err),\n    })\n  }\n}\n"],
+  "mappings": "AAAA,SAAS,oBAAoB;AAG7B,MAAM,SAAS,aAAa,QAAQ,EAAE,MAAM,EAAE,WAAW,QAAQ,CAAC;AAmBlE,MAAM,iBAAiB;AACvB,MAAM,SAAS,oBAAI,IAAY;AAE/B,MAAM,UAAU,CAAC,QAAgB,UAAyB,UACxD,GAAG,MAAM,IAAI,YAAY,MAAM,IAAI,KAAK;AAEnC,MAAM,sBAAsB,CAAC,UAClC,MAAM,QAAQ,sBAAsB,OAAO,EAAE,YAAY;AAGpD,SAAS,+BAAqC;AACnD,SAAO,MAAM;AACf;AAaA,eAAsB,6BAA6B,QAMjC;AAChB,QAAM,EAAE,QAAQ,UAAU,QAAQ,QAAQ,IAAI;AAC9C,MAAI,CAAC,WAAW,QAAQ,YAAY,MAAM,MAAO;AACjD,MAAI,OAAO,QAAQ,2BAA2B,WAAY;AAE1D,QAAM,aAAa,OAAO,OACvB,OAAO,CAAC,UAAU,OAAO,UAAU,YAAY,CAAC,MAAM,WAAW,KAAK,CAAC,EACvE,IAAI,CAAC,UAAU,oBAAoB,KAAK,CAAC,EACzC,OAAO,CAAC,OAAO,OAAO,QAAQ,MAAM,SAAS,KAAK,IAAI,QAAQ,KAAK,MAAM,KAAK,EAC9E,OAAO,CAAC,UAAU,CAAC,OAAO,IAAI,QAAQ,QAAQ,UAAU,KAAK,CAAC,CAAC;AAClE,MAAI,CAAC,WAAW,OAAQ;AAExB,MAAI;AAGF,UAAM,YAAY,IAAI;AAAA,OACnB,MAAM,QAAQ,uBAAuB,QAAQ,UAAU,IAAI,GAAG;AAAA,QAAI,CAAC,UAClE,oBAAoB,OAAO,KAAK,CAAC;AAAA,MACnC;AAAA,IACF;AACA,eAAW,SAAS,YAAY;AAC9B,UAAI,CAAC,UAAU,IAAI,KAAK,EAAG;AAC3B,UAAI,OAAO,QAAQ,eAAgB,QAAO,MAAM;AAChD,aAAO,IAAI,QAAQ,QAAQ,UAAU,KAAK,CAAC;AAC3C,aAAO,KAAK,uDAAuD;AAAA,QACjE;AAAA,QACA;AAAA,QACA;AAAA,QACA,MAAM,WAAW,oBACb,kGACA,WAAW,wBACT,6MACA;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF,SAAS,KAAK;AACZ,WAAO,MAAM,0CAA0C;AAAA,MACrD;AAAA,MACA,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,IACtD,CAAC;AAAA,EACH;AACF;",
+  "names": []
+}

package/dist/lib/query/engine.js

@@ -6,12 +6,17 @@ import {
   resolveJoins
 } from "./join-utils.js";
 import { resolveSearchConfig } from "../search/config.js";
+import {
+  createSearchTokenAvailability,
+  isSearchFilterOp
+} from "../search/availability.js";
 import { tokenizeText } from "../search/tokenize.js";
 import { runBeforeQueryPipeline, runAfterQueryPipeline } from "./query-extension-runner.js";
 import {
   buildCustomFieldDefinitionIndexFromRows,
   resolveCfDefIndexOrgCandidates
 } from "../crud/custom-field-definition-index.js";
+import { warnOnCiphertextLikeFallback } from "./ciphertext-search-warning.js";
 import { resolveEncryptedSortFields, resolveEncryptedSortMaxRows, sortRowsInMemory } from "./encrypted-sort.js";
 import { mapWithConcurrency } from "./bounded-decrypt.js";
 import { createLogger } from "../logger/index.js";
@@ -133,8 +138,8 @@ class BasicQueryEngine {
     this.getDbFn = getDbFn;
     this.resolveEncryptionService = resolveEncryptionService;
     this.columnCache = /* @__PURE__ */ new Map();
-    this.tableCache = /* @__PURE__ */ new Map();
     this.searchAliasSeq = 0;
+    this.searchAvailabilityInstance = null;
   }
   getEncryptionService() {
     try {
@@ -149,6 +154,21 @@ class BasicQueryEngine {
     if (typeof emAny?.getKysely === "function") return emAny.getKysely();
     throw new Error("BasicQueryEngine requires an EntityManager exposing getKysely() (MikroORM v7)");
   }
+  searchAvailability() {
+    if (!this.searchAvailabilityInstance) {
+      this.searchAvailabilityInstance = createSearchTokenAvailability({
+        getDb: () => this.getDb(),
+        getConfig: resolveSearchConfig,
+        applyOrganizationScope: (query, column, scope) => this.applyOrganizationScope(
+          query,
+          column,
+          scope
+        ),
+        logDebug: (event, payload) => this.logSearchDebug(event, payload)
+      });
+    }
+    return this.searchAvailabilityInstance;
+  }
   async query(entity, opts = {}) {
     const ext = opts.extensions;
     let effectiveOpts = opts;
@@ -204,11 +224,10 @@ class BasicQueryEngine {
     const { baseFilters, joinFilters } = partitionFilters(table, normalizedFilters, joinMap);
     const cfFilters = normalizedFilters.filter((filter) => String(filter.field).startsWith("cf:"));
     const searchConfig = resolveSearchConfig();
-    const searchEnabled = !skipAutoScope && searchConfig.enabled && await this.tableExists("search_tokens");
-    const hasSearchTokens = searchEnabled ? await this.hasSearchTokens(String(entity), opts.tenantId ?? null, orgScope) : false;
+    const searchFilters = [...baseFilters, ...cfFilters].filter((filter) => isSearchFilterOp(filter.op));
+    const searchEnabled = !skipAutoScope && await this.searchAvailability().staticEnabled();
+    const hasSearchTokens = searchEnabled && searchFilters.length ? await this.searchAvailability().hasTokens(String(entity), opts.tenantId ?? null, orgScope) : false;
     const searchActive = searchEnabled && hasSearchTokens;
-    const joinSearchAvailability = /* @__PURE__ */ new Map();
-    const searchFilters = [...baseFilters, ...cfFilters].filter((filter) => filter.op === "like" || filter.op === "ilike");
     if (searchFilters.length) {
       const fields = searchFilters.map((filter) => String(filter.field));
       this.logSearchDebug("search:init", {
@@ -238,6 +257,35 @@ class BasicQueryEngine {
           organizationScope: orgScope
         });
       }
+      const fallbackFields = searchFilters.filter((filter) => !searchActive || typeof filter.value !== "string" || tokenizeText(filter.value, searchConfig).hashes.length === 0).map((filter) => String(filter.field));
+      if (fallbackFields.length) {
+        await warnOnCiphertextLikeFallback({
+          entity: String(entity),
+          fields: fallbackFields,
+          tenantId: opts.tenantId ?? null,
+          // `searchEnabled` also folds in the missing-table and
+          // omitAutomaticTenantOrgScope cases, which are "no usable tokens"
+          // rather than "the operator switched search off".
+          reason: searchActive ? "no-indexable-tokens" : searchConfig.enabled ? "no-search-tokens" : "search-disabled",
+          service: this.getEncryptionService()
+        });
+      }
+    }
+    for (const [alias, joinedFilters] of joinFilters) {
+      const filters = joinedFilters.filter((entry) => entry.op === "like" || entry.op === "ilike");
+      if (!filters.length) continue;
+      const join = joinMap.get(alias);
+      if (!join?.entityId) continue;
+      const hasJoinedTokens = searchEnabled ? await this.searchAvailability().hasTokens(join.entityId, opts.tenantId ?? null, orgScope) : false;
+      const fallbackFields = filters.filter((filter) => !hasJoinedTokens || typeof filter.value !== "string" || tokenizeText(filter.value, searchConfig).hashes.length === 0).map((filter) => filter.column);
+      if (!fallbackFields.length) continue;
+      await warnOnCiphertextLikeFallback({
+        entity: join.entityId,
+        fields: fallbackFields,
+        tenantId: opts.tenantId ?? null,
+        reason: hasJoinedTokens ? "no-indexable-tokens" : searchConfig.enabled ? "no-search-tokens" : "search-disabled",
+        service: this.getEncryptionService()
+      });
     }
     const recordIdColumn = qualify("id");
     const applyFilterOp = (builder, column, op, value, fieldName) => {
@@ -278,11 +326,7 @@ class BasicQueryEngine {
       if (!searchEnabled || !join.entityId) return { applied: false, builder };
       if (!["like", "ilike"].includes(filter.op)) return { applied: false, builder };
       if (typeof filter.value !== "string" || filter.value.trim().length === 0) return { applied: false, builder };
-      let searchAvailable = joinSearchAvailability.get(join.entityId);
-      if (searchAvailable === void 0) {
-        searchAvailable = await this.hasSearchTokens(join.entityId, opts.tenantId ?? null, orgScope);
-        joinSearchAvailability.set(join.entityId, searchAvailable);
-      }
+      const searchAvailable = await this.searchAvailability().hasTokens(join.entityId, opts.tenantId ?? null, orgScope);
       if (!searchAvailable) return { applied: false, builder };
       const tokens = tokenizeText(String(filter.value), searchConfig);
       if (!tokens.hashes.length) return { applied: false, builder };
@@ -867,36 +911,6 @@ class BasicQueryEngine {
     else this.columnCache.delete(key);
     return present;
   }
-  async tableExists(table) {
-    if (this.tableCache.has(table)) return this.tableCache.get(table) ?? false;
-    const db = this.getDb();
-    const exists = await db.selectFrom("information_schema.tables").select(sql`1`.as("one")).where("table_name", "=", table).limit(1).executeTakeFirst();
-    const present = !!exists;
-    this.tableCache.set(table, present);
-    return present;
-  }
-  async hasSearchTokens(entity, tenantId, orgScope) {
-    try {
-      const db = this.getDb();
-      let query = db.selectFrom("search_tokens").select(sql`1`.as("one")).where("entity_type", "=", entity).limit(1);
-      if (tenantId !== void 0) {
-        query = query.where(sql`tenant_id is not distinct from ${tenantId}`);
-      }
-      if (orgScope) {
-        query = this.applyOrganizationScope(query, "search_tokens.organization_id", orgScope);
-      }
-      const row = await query.executeTakeFirst();
-      return !!row;
-    } catch (err) {
-      this.logSearchDebug("search:has-tokens-error", {
-        entity,
-        tenantId,
-        organizationScope: orgScope,
-        error: err instanceof Error ? err.message : String(err)
-      });
-      return false;
-    }
-  }
   applySearchTokens(q, opts) {
     if (!opts.hashes.length) {
       this.logSearchDebug("search:skip-no-hashes", {