@open-mercato/core 0.7.0 → 0.7.1-develop.7102.1.b41f7e3e51

package/dist/modules/query_index/lib/engine.js

@@ -13,6 +13,8 @@ import {
   resolveJoins
 } from "@open-mercato/shared/lib/query/join-utils";
 import { resolveSearchConfig } from "@open-mercato/shared/lib/search/config";
+import { isEncryptedLikeField, resolveEncryptedLikeFieldSet } from "@open-mercato/shared/lib/query/engine";
+import { isTenantDataEncryptionEnabled } from "@open-mercato/shared/lib/encryption/toggles";
 import {
   createSearchTokenAvailability,
   isSearchFilterOp,
@@ -22,6 +24,7 @@ import { tokenizeText } from "@open-mercato/shared/lib/search/tokenize";
 import { runBeforeQueryPipeline, runAfterQueryPipeline } from "@open-mercato/shared/lib/query/query-extension-runner";
 import { warnOnCiphertextLikeFallback } from "@open-mercato/shared/lib/query/ciphertext-search-warning";
 import { resolveEncryptedSortFields, resolveEncryptedSortMaxRows, sortRowsInMemory } from "@open-mercato/shared/lib/query/encrypted-sort";
+import { resolveListCountCap } from "@open-mercato/shared/lib/query/count-cap";
 import { mapWithConcurrency } from "@open-mercato/shared/lib/query/bounded-decrypt";
 import { createLogger } from "@open-mercato/shared/lib/logger";
 import { parseNumberWithDefault } from "@open-mercato/shared/lib/number";
@@ -387,6 +390,34 @@ class HybridQueryEngine {
       ].filter((filter) => isSearchFilterOp(filter.op));
       const hasSearchTokens = searchEnabled && searchSources.length && sourceSearchFilters.length ? await this.searchAvailability().anySourceHasTokens(searchSources, opts.tenantId ?? null, orgScope) : false;
       const searchRuntime = { ...searchRuntimeBase, searchSources, enabled: searchEnabled && hasSearchTokens };
+      if (searchRuntime.enabled && searchConfig.useIlikeForNonEncryptedFields === true && sourceSearchFilters.some((filter) => !String(filter.field).startsWith("cf:"))) {
+        try {
+          const encryptionService = this.getEncryptionService();
+          const readEncryptedFieldNames = encryptionService?.getEncryptedFieldNames?.bind(encryptionService);
+          if (readEncryptedFieldNames) {
+            searchRuntime.encryptedFields = await resolveEncryptedLikeFieldSet(
+              () => readEncryptedFieldNames(
+                entity,
+                opts.tenantId ?? null,
+                null,
+                { ignoreRuntimeHealth: true }
+              ),
+              String(entity),
+              opts.tenantId ?? null
+            );
+          } else if (isTenantDataEncryptionEnabled()) {
+            searchRuntime.encryptedFields = null;
+          } else {
+            searchRuntime.encryptedFields = /* @__PURE__ */ new Set();
+          }
+        } catch (err) {
+          logger.warn("search: encrypted-field map unavailable; keeping the token rewrite for all columns", {
+            entity: String(entity),
+            error: err instanceof Error ? err.message : String(err)
+          });
+          searchRuntime.encryptedFields = null;
+        }
+      }
       if (searchFilters.length) {
         this.logSearchDebug("search:init", {
           entity,
@@ -769,6 +800,79 @@ class HybridQueryEngine {
       };
       const hasCustomFieldFilters = cfFilters.length > 0;
       const canOptimizeCount = !hasCustomFieldFilters && !hasNonBaseSearchSource;
+      const isRowsetDependentFilter = (filter) => {
+        const baseField = resolveBaseColumn(String(filter.field));
+        if (!baseField) return true;
+        if ((filter.op === "like" || filter.op === "ilike") && searchRuntime.enabled && typeof filter.value === "string" && hasNonBaseSearchSource) return true;
+        return false;
+      };
+      const rowsetRegularFilters = regularBaseFilters.filter((filter) => isRowsetDependentFilter(filter));
+      const outerRegularFilters = regularBaseFilters.filter((filter) => !isRowsetDependentFilter(filter));
+      const orGroupsRowsetDependent = orGroupCfFilters.length > 0 || orGroupFilters.some((filter) => isRowsetDependentFilter(filter));
+      const hasInnerTypedCfSource = preparedCfSources.some((source) => ((opts.customFieldSources ?? []).find((s) => s && (s.alias ?? void 0) === source.alias)?.join?.type ?? "left") === "inner");
+      const needsIndexRowset = hasCustomFieldFilters || rowsetRegularFilters.length > 0 || orGroupsRowsetDependent || hasInnerTypedCfSource;
+      const applyCountShape = async (q) => {
+        let next = applyBaseScope(q);
+        for (const filter of outerRegularFilters) {
+          const baseField = resolveBaseColumn(String(filter.field));
+          if (!baseField) continue;
+          next = this.applyColumnFilter(next, qualify(baseField), filter, {
+            ...searchRuntime,
+            entity,
+            field: String(filter.field),
+            recordIdColumn: qualify("id")
+          });
+        }
+        if (!orGroupsRowsetDependent) next = applyOrGroupedBaseFilters(next);
+        if (needsIndexRowset) {
+          let rowset = db.selectFrom(sql`(select 1)`.as("om_count_seed")).select(sql`1`.as("one"));
+          rowset = applyEntityIndexesJoin(rowset);
+          rowset = applyCustomFieldSourceJoins(rowset);
+          rowset = applyCfFilters(rowset);
+          for (const filter of rowsetRegularFilters) {
+            const baseField = resolveBaseColumn(String(filter.field));
+            if (!baseField) {
+              rowset = this.applyIndexDocFilterFromAlias(
+                rowset,
+                "ei",
+                entity,
+                String(filter.field),
+                filter.op,
+                filter.value,
+                qualify("id"),
+                searchRuntime
+              );
+              continue;
+            }
+            rowset = this.applyColumnFilter(rowset, qualify(baseField), filter, {
+              ...searchRuntime,
+              entity,
+              field: String(filter.field),
+              recordIdColumn: qualify("id")
+            });
+          }
+          if (orGroupsRowsetDependent) rowset = applyOrGroupedBaseFilters(rowset);
+          const capturedRowset = rowset;
+          next = next.where((eb) => eb.exists(capturedRowset));
+        }
+        next = await applyJoinFilters({
+          db,
+          baseTable,
+          builder: next,
+          joinMap,
+          joinFilters,
+          aliasTables,
+          qualifyBase: (column) => qualify(column),
+          applyAliasScope: async (target, alias) => applyAliasScopes(target, alias),
+          applyFilterOp: (target, column, op, value) => applyJoinFilterOpFn(target, column, op, value),
+          applyJoinFilterOp: async (target, filter, qualified, join) => {
+            const applied = await applyJoinSearchFilterOp(target, filter, qualified, join);
+            return { applied, builder: target };
+          },
+          columnExists: (tbl, column) => this.columnExists(tbl, column)
+        });
+        return next;
+      };
       const selectFieldSet = new Set(opts.fields && opts.fields.length ? opts.fields.map(String) : Array.from(columns.keys()));
       if (requiresPlaintextSort) {
         for (const field of encryptedSortFields) selectFieldSet.add(field);
@@ -824,30 +928,11 @@ class HybridQueryEngine {
       const page = opts.page?.page ?? 1;
       const pageSize = opts.page?.pageSize ?? 20;
       const sqlDebugEnabled = this.isSqlDebugEnabled();
-      let total;
-      if (canOptimizeCount) {
-        const optimizedRoot = db.selectFrom(`${baseTable} as b`);
-        let countCore = applyBaseScope(optimizedRoot);
-        countCore = applyRegularBaseFilters(countCore);
-        countCore = applyOrGroupedBaseFilters(countCore);
-        countCore = await applyJoinFilters({
-          db,
-          baseTable,
-          builder: countCore,
-          joinMap,
-          joinFilters,
-          aliasTables,
-          qualifyBase: (column) => qualify(column),
-          applyAliasScope: async (target, alias) => applyAliasScopes(target, alias),
-          applyFilterOp: (target, column, op, value) => applyJoinFilterOpFn(target, column, op, value),
-          applyJoinFilterOp: async (target, filter, qualified, join) => {
-            const applied = await applyJoinSearchFilterOp(target, filter, qualified, join);
-            return { applied, builder: target };
-          },
-          columnExists: (tbl, column) => this.columnExists(tbl, column)
-        });
-        const sub = countCore.select(sql.ref(qualify("id")).as("id")).groupBy(qualify("id")).as("sq");
-        const countQuery = db.selectFrom(sub).select(sql`count(*)`.as("count"));
+      const countCap = resolveListCountCap();
+      const runBoundedCount = async (wasOptimizable) => {
+        const countRoot = db.selectFrom(`${baseTable} as b`);
+        const shape = await applyCountShape(countRoot);
+        const countQuery = countCap !== null ? db.selectFrom(shape.select(sql`1`.as("one")).limit(countCap + 1).as("om_count_probe")).select(sql`count(*)`.as("count")) : shape.select(sql`count(*)`.as("count"));
         if (debugEnabled && sqlDebugEnabled) {
           const compiled = countQuery.compile();
           this.debug("query:sql:count", { entity, sql: compiled.sql, bindings: compiled.parameters });
@@ -856,26 +941,18 @@ class HybridQueryEngine {
           "query:sql:count",
           entity,
           () => countQuery.executeTakeFirst(),
-          { optimized: true },
+          { optimized: wasOptimizable },
           profiler
         );
-        total = this.parseCount(countRow);
-      } else {
-        const countRoot = db.selectFrom(`${baseTable} as b`);
-        const countBuilder = (await applyQueryShape(countRoot)).select(sql`count(distinct ${sql.ref(qualify("id"))})`.as("count"));
-        if (debugEnabled && sqlDebugEnabled) {
-          const compiled = countBuilder.compile();
-          this.debug("query:sql:count", { entity, sql: compiled.sql, bindings: compiled.parameters });
+        const probed = this.parseCount(countRow);
+        if (countCap !== null && probed > countCap) {
+          return { total: countCap, warning: { entity, cap: countCap } };
         }
-        const countRow = await this.captureSqlTiming(
-          "query:sql:count",
-          entity,
-          () => countBuilder.executeTakeFirst(),
-          { optimized: false },
-          profiler
-        );
-        total = this.parseCount(countRow);
-      }
+        return { total: probed };
+      };
+      const counted = await runBoundedCount(canOptimizeCount);
+      const total = counted.total;
+      const listCountCapWarning = counted.warning;
       const dekKeyCache = /* @__PURE__ */ new Map();
       const decryptRow = async (item) => {
         let next = item;
@@ -922,23 +999,25 @@ class HybridQueryEngine {
         const sortFieldNames = Array.from(/* @__PURE__ */ new Set(["id", ...scopeFieldNames, ...resolvedSorts.map((s) => String(s.field))]));
         phase1 = applySelection(phase1, sortFieldNames);
         if (cap !== null) {
-          phase1 = phase1.limit(cap).orderBy(qualify("id"), "asc");
+          phase1 = phase1.limit(cap + 1).orderBy(qualify("id"), "asc");
         }
         if (debugEnabled && sqlDebugEnabled) {
           const compiled = phase1.compile();
           this.debug("query:sql:data:phase1", { entity, sql: compiled.sql, bindings: compiled.parameters });
         }
-        const candidateRows = await this.captureSqlTiming(
+        const candidateRowsRaw = await this.captureSqlTiming(
           "query:sql:data:phase1",
           entity,
           () => phase1.execute(),
           { phase: 1 },
           profiler
         );
+        const sortTruncated = cap !== null && candidateRowsRaw.length > cap;
+        const candidateRows = sortTruncated && cap !== null ? candidateRowsRaw.slice(0, cap) : candidateRowsRaw;
         const decryptedCandidates = await mapWithConcurrency(candidateRows, DECRYPT_CONCURRENCY, decryptRow);
         const orderedCandidates = sortRowsInMemory(decryptedCandidates, resolvedSorts);
         const pageIds = orderedCandidates.slice((page - 1) * pageSize, page * pageSize).map((row) => row.id);
-        if (cap !== null && total > cap) {
+        if (sortTruncated && cap !== null) {
           encryptedSortRowCapWarning = {
             entity,
             sortFields: resolvedSorts.map((s) => String(s.field)),
@@ -990,10 +1069,11 @@ class HybridQueryEngine {
       if (debugEnabled) this.debug("query:complete", { entity, total, items: items.length });
       const typedItems = items;
       let result = { items: typedItems, page, pageSize, total };
-      if (partialIndexWarning || encryptedSortRowCapWarning) {
+      if (partialIndexWarning || encryptedSortRowCapWarning || listCountCapWarning) {
         const meta = {};
         if (partialIndexWarning) meta.partialIndexWarning = partialIndexWarning;
         if (encryptedSortRowCapWarning) meta.encryptedSortRowCapWarning = encryptedSortRowCapWarning;
+        if (listCountCapWarning) meta.listCountCapWarning = listCountCapWarning;
         result.meta = meta;
       }
       result = await applyAfterExtensions(result);
@@ -1419,7 +1499,7 @@ class HybridQueryEngine {
     if (!baseField) {
       return this.buildIndexDocFilterExpression(eb, "ei", entity, fieldName, filter.op, filter.value, "b.id", searchRuntime);
     }
-    if ((filter.op === "like" || filter.op === "ilike") && searchRuntime?.enabled && typeof filter.value === "string") {
+    if ((filter.op === "like" || filter.op === "ilike") && searchRuntime?.enabled && typeof filter.value === "string" && (searchRuntime.encryptedFields == null || isEncryptedLikeField(searchRuntime.encryptedFields, fieldName))) {
       const tokens = tokenizeText(String(filter.value), searchRuntime.config);
       if (tokens.hashes.length) {
         const sources = (searchRuntime.searchSources && searchRuntime.searchSources.length ? searchRuntime.searchSources : [{ entity: String(entity), recordIdColumn: "b.id" }]).filter((src) => src.recordIdColumn && src.entity);
@@ -1439,7 +1519,7 @@ class HybridQueryEngine {
           );
         }
       }
-      return sql`true`;
+      return searchRuntime?.encryptedFields != null && isEncryptedLikeField(searchRuntime.encryptedFields, fieldName) ? sql`false` : sql`true`;
     }
     return this.buildColumnFilterExpression(eb, qualify(baseField), filter.op, filter.value);
   }
@@ -1615,16 +1695,26 @@ class HybridQueryEngine {
     };
     const page = opts.page?.page ?? 1;
     const pageSize = opts.page?.pageSize ?? 20;
+    const countCap = resolveListCountCap();
     const root = db.selectFrom(`custom_entities_storage as ${alias}`);
-    const countQuery = applyScope(root).select(sql`count(distinct ${sql.ref(`${alias}.entity_id`)})`.as("count"));
+    const countShape = applyScope(root);
+    const countQuery = countCap !== null ? db.selectFrom(countShape.select(sql`1`.as("one")).limit(countCap + 1).as("om_count_probe")).select(sql`count(*)`.as("count")) : countShape.select(sql`count(*)`.as("count"));
     const countRow = await countQuery.executeTakeFirst();
-    const total = this.parseCount(countRow);
+    const probed = this.parseCount(countRow);
+    let total = probed;
+    let listCountCapWarning;
+    if (countCap !== null && probed > countCap) {
+      total = countCap;
+      listCountCapWarning = { entity, cap: countCap };
+    }
     let dataQuery = applyScope(db.selectFrom(`custom_entities_storage as ${alias}`));
     dataQuery = applySelection(dataQuery);
     dataQuery = applySort(dataQuery);
     dataQuery = dataQuery.limit(pageSize).offset((page - 1) * pageSize);
     const items = await dataQuery.execute();
-    return { items, page, pageSize, total };
+    const result = { items, page, pageSize, total };
+    if (listCountCapWarning) result.meta = { listCountCapWarning };
+    return result;
   }
   async tableExists(table) {
     const db = this.getDb();
@@ -1979,7 +2069,10 @@ class HybridQueryEngine {
     logger.debug("Search debug event", { event, payload });
   }
   applyColumnFilter(q, column, filter, search) {
-    if ((filter.op === "like" || filter.op === "ilike") && search?.enabled && typeof filter.value === "string") {
+    if ((filter.op === "like" || filter.op === "ilike") && search?.enabled && typeof filter.value === "string" && // Plaintext base columns keep exact SQL ILIKE -- see SearchRuntime.encryptedFields.
+    // Membership runs across name-shape candidates: maps may declare `displayName` while the
+    // filter carries the column name `display_name`.
+    (search.encryptedFields == null || isEncryptedLikeField(search.encryptedFields, search.field))) {
       const tokens = tokenizeText(String(filter.value), search.config);
       const hashes = tokens.hashes;
       if (hashes.length) {
@@ -2009,12 +2102,18 @@ class HybridQueryEngine {
           });
           return q;
         }
+        if (search.encryptedFields != null && isEncryptedLikeField(search.encryptedFields, search.field)) {
+          return q.where(sql`false`);
+        }
       } else {
         this.logSearchDebug("search:skip-empty-hashes", {
           entity: search.entity,
           field: search.field,
           value: filter.value
         });
+        if (search.encryptedFields != null && isEncryptedLikeField(search.encryptedFields, search.field)) {
+          return q.where(sql`false`);
+        }
       }
       return q;
     }