@open-mercato/core 0.6.8-develop.7029.1.a1bb3363af → 0.6.8-develop.7031.1.005201cd70

package/dist/modules/query_index/lib/engine.js

@@ -24,6 +24,7 @@ import { tokenizeText } from "@open-mercato/shared/lib/search/tokenize";
 import { runBeforeQueryPipeline, runAfterQueryPipeline } from "@open-mercato/shared/lib/query/query-extension-runner";
 import { warnOnCiphertextLikeFallback } from "@open-mercato/shared/lib/query/ciphertext-search-warning";
 import { resolveEncryptedSortFields, resolveEncryptedSortMaxRows, sortRowsInMemory } from "@open-mercato/shared/lib/query/encrypted-sort";
+import { resolveListCountCap } from "@open-mercato/shared/lib/query/count-cap";
 import { mapWithConcurrency } from "@open-mercato/shared/lib/query/bounded-decrypt";
 import { createLogger } from "@open-mercato/shared/lib/logger";
 import { parseNumberWithDefault } from "@open-mercato/shared/lib/number";
@@ -799,6 +800,79 @@ class HybridQueryEngine {
       };
       const hasCustomFieldFilters = cfFilters.length > 0;
       const canOptimizeCount = !hasCustomFieldFilters && !hasNonBaseSearchSource;
+      const isRowsetDependentFilter = (filter) => {
+        const baseField = resolveBaseColumn(String(filter.field));
+        if (!baseField) return true;
+        if ((filter.op === "like" || filter.op === "ilike") && searchRuntime.enabled && typeof filter.value === "string" && hasNonBaseSearchSource) return true;
+        return false;
+      };
+      const rowsetRegularFilters = regularBaseFilters.filter((filter) => isRowsetDependentFilter(filter));
+      const outerRegularFilters = regularBaseFilters.filter((filter) => !isRowsetDependentFilter(filter));
+      const orGroupsRowsetDependent = orGroupCfFilters.length > 0 || orGroupFilters.some((filter) => isRowsetDependentFilter(filter));
+      const hasInnerTypedCfSource = preparedCfSources.some((source) => ((opts.customFieldSources ?? []).find((s) => s && (s.alias ?? void 0) === source.alias)?.join?.type ?? "left") === "inner");
+      const needsIndexRowset = hasCustomFieldFilters || rowsetRegularFilters.length > 0 || orGroupsRowsetDependent || hasInnerTypedCfSource;
+      const applyCountShape = async (q) => {
+        let next = applyBaseScope(q);
+        for (const filter of outerRegularFilters) {
+          const baseField = resolveBaseColumn(String(filter.field));
+          if (!baseField) continue;
+          next = this.applyColumnFilter(next, qualify(baseField), filter, {
+            ...searchRuntime,
+            entity,
+            field: String(filter.field),
+            recordIdColumn: qualify("id")
+          });
+        }
+        if (!orGroupsRowsetDependent) next = applyOrGroupedBaseFilters(next);
+        if (needsIndexRowset) {
+          let rowset = db.selectFrom(sql`(select 1)`.as("om_count_seed")).select(sql`1`.as("one"));
+          rowset = applyEntityIndexesJoin(rowset);
+          rowset = applyCustomFieldSourceJoins(rowset);
+          rowset = applyCfFilters(rowset);
+          for (const filter of rowsetRegularFilters) {
+            const baseField = resolveBaseColumn(String(filter.field));
+            if (!baseField) {
+              rowset = this.applyIndexDocFilterFromAlias(
+                rowset,
+                "ei",
+                entity,
+                String(filter.field),
+                filter.op,
+                filter.value,
+                qualify("id"),
+                searchRuntime
+              );
+              continue;
+            }
+            rowset = this.applyColumnFilter(rowset, qualify(baseField), filter, {
+              ...searchRuntime,
+              entity,
+              field: String(filter.field),
+              recordIdColumn: qualify("id")
+            });
+          }
+          if (orGroupsRowsetDependent) rowset = applyOrGroupedBaseFilters(rowset);
+          const capturedRowset = rowset;
+          next = next.where((eb) => eb.exists(capturedRowset));
+        }
+        next = await applyJoinFilters({
+          db,
+          baseTable,
+          builder: next,
+          joinMap,
+          joinFilters,
+          aliasTables,
+          qualifyBase: (column) => qualify(column),
+          applyAliasScope: async (target, alias) => applyAliasScopes(target, alias),
+          applyFilterOp: (target, column, op, value) => applyJoinFilterOpFn(target, column, op, value),
+          applyJoinFilterOp: async (target, filter, qualified, join) => {
+            const applied = await applyJoinSearchFilterOp(target, filter, qualified, join);
+            return { applied, builder: target };
+          },
+          columnExists: (tbl, column) => this.columnExists(tbl, column)
+        });
+        return next;
+      };
       const selectFieldSet = new Set(opts.fields && opts.fields.length ? opts.fields.map(String) : Array.from(columns.keys()));
       if (requiresPlaintextSort) {
         for (const field of encryptedSortFields) selectFieldSet.add(field);
@@ -854,30 +928,11 @@ class HybridQueryEngine {
       const page = opts.page?.page ?? 1;
       const pageSize = opts.page?.pageSize ?? 20;
       const sqlDebugEnabled = this.isSqlDebugEnabled();
-      let total;
-      if (canOptimizeCount) {
-        const optimizedRoot = db.selectFrom(`${baseTable} as b`);
-        let countCore = applyBaseScope(optimizedRoot);
-        countCore = applyRegularBaseFilters(countCore);
-        countCore = applyOrGroupedBaseFilters(countCore);
-        countCore = await applyJoinFilters({
-          db,
-          baseTable,
-          builder: countCore,
-          joinMap,
-          joinFilters,
-          aliasTables,
-          qualifyBase: (column) => qualify(column),
-          applyAliasScope: async (target, alias) => applyAliasScopes(target, alias),
-          applyFilterOp: (target, column, op, value) => applyJoinFilterOpFn(target, column, op, value),
-          applyJoinFilterOp: async (target, filter, qualified, join) => {
-            const applied = await applyJoinSearchFilterOp(target, filter, qualified, join);
-            return { applied, builder: target };
-          },
-          columnExists: (tbl, column) => this.columnExists(tbl, column)
-        });
-        const sub = countCore.select(sql.ref(qualify("id")).as("id")).groupBy(qualify("id")).as("sq");
-        const countQuery = db.selectFrom(sub).select(sql`count(*)`.as("count"));
+      const countCap = resolveListCountCap();
+      const runBoundedCount = async (wasOptimizable) => {
+        const countRoot = db.selectFrom(`${baseTable} as b`);
+        const shape = await applyCountShape(countRoot);
+        const countQuery = countCap !== null ? db.selectFrom(shape.select(sql`1`.as("one")).limit(countCap + 1).as("om_count_probe")).select(sql`count(*)`.as("count")) : shape.select(sql`count(*)`.as("count"));
         if (debugEnabled && sqlDebugEnabled) {
           const compiled = countQuery.compile();
           this.debug("query:sql:count", { entity, sql: compiled.sql, bindings: compiled.parameters });
@@ -886,26 +941,18 @@ class HybridQueryEngine {
           "query:sql:count",
           entity,
           () => countQuery.executeTakeFirst(),
-          { optimized: true },
+          { optimized: wasOptimizable },
           profiler
         );
-        total = this.parseCount(countRow);
-      } else {
-        const countRoot = db.selectFrom(`${baseTable} as b`);
-        const countBuilder = (await applyQueryShape(countRoot)).select(sql`count(distinct ${sql.ref(qualify("id"))})`.as("count"));
-        if (debugEnabled && sqlDebugEnabled) {
-          const compiled = countBuilder.compile();
-          this.debug("query:sql:count", { entity, sql: compiled.sql, bindings: compiled.parameters });
+        const probed = this.parseCount(countRow);
+        if (countCap !== null && probed > countCap) {
+          return { total: countCap, warning: { entity, cap: countCap } };
         }
-        const countRow = await this.captureSqlTiming(
-          "query:sql:count",
-          entity,
-          () => countBuilder.executeTakeFirst(),
-          { optimized: false },
-          profiler
-        );
-        total = this.parseCount(countRow);
-      }
+        return { total: probed };
+      };
+      const counted = await runBoundedCount(canOptimizeCount);
+      const total = counted.total;
+      const listCountCapWarning = counted.warning;
       const dekKeyCache = /* @__PURE__ */ new Map();
       const decryptRow = async (item) => {
         let next = item;
@@ -952,23 +999,25 @@ class HybridQueryEngine {
         const sortFieldNames = Array.from(/* @__PURE__ */ new Set(["id", ...scopeFieldNames, ...resolvedSorts.map((s) => String(s.field))]));
         phase1 = applySelection(phase1, sortFieldNames);
         if (cap !== null) {
-          phase1 = phase1.limit(cap).orderBy(qualify("id"), "asc");
+          phase1 = phase1.limit(cap + 1).orderBy(qualify("id"), "asc");
         }
         if (debugEnabled && sqlDebugEnabled) {
           const compiled = phase1.compile();
           this.debug("query:sql:data:phase1", { entity, sql: compiled.sql, bindings: compiled.parameters });
         }
-        const candidateRows = await this.captureSqlTiming(
+        const candidateRowsRaw = await this.captureSqlTiming(
           "query:sql:data:phase1",
           entity,
           () => phase1.execute(),
           { phase: 1 },
           profiler
         );
+        const sortTruncated = cap !== null && candidateRowsRaw.length > cap;
+        const candidateRows = sortTruncated && cap !== null ? candidateRowsRaw.slice(0, cap) : candidateRowsRaw;
         const decryptedCandidates = await mapWithConcurrency(candidateRows, DECRYPT_CONCURRENCY, decryptRow);
         const orderedCandidates = sortRowsInMemory(decryptedCandidates, resolvedSorts);
         const pageIds = orderedCandidates.slice((page - 1) * pageSize, page * pageSize).map((row) => row.id);
-        if (cap !== null && total > cap) {
+        if (sortTruncated && cap !== null) {
           encryptedSortRowCapWarning = {
             entity,
             sortFields: resolvedSorts.map((s) => String(s.field)),
@@ -1020,10 +1069,11 @@ class HybridQueryEngine {
       if (debugEnabled) this.debug("query:complete", { entity, total, items: items.length });
       const typedItems = items;
       let result = { items: typedItems, page, pageSize, total };
-      if (partialIndexWarning || encryptedSortRowCapWarning) {
+      if (partialIndexWarning || encryptedSortRowCapWarning || listCountCapWarning) {
         const meta = {};
         if (partialIndexWarning) meta.partialIndexWarning = partialIndexWarning;
         if (encryptedSortRowCapWarning) meta.encryptedSortRowCapWarning = encryptedSortRowCapWarning;
+        if (listCountCapWarning) meta.listCountCapWarning = listCountCapWarning;
         result.meta = meta;
       }
       result = await applyAfterExtensions(result);
@@ -1645,16 +1695,26 @@ class HybridQueryEngine {
     };
     const page = opts.page?.page ?? 1;
     const pageSize = opts.page?.pageSize ?? 20;
+    const countCap = resolveListCountCap();
     const root = db.selectFrom(`custom_entities_storage as ${alias}`);
-    const countQuery = applyScope(root).select(sql`count(distinct ${sql.ref(`${alias}.entity_id`)})`.as("count"));
+    const countShape = applyScope(root);
+    const countQuery = countCap !== null ? db.selectFrom(countShape.select(sql`1`.as("one")).limit(countCap + 1).as("om_count_probe")).select(sql`count(*)`.as("count")) : countShape.select(sql`count(*)`.as("count"));
     const countRow = await countQuery.executeTakeFirst();
-    const total = this.parseCount(countRow);
+    const probed = this.parseCount(countRow);
+    let total = probed;
+    let listCountCapWarning;
+    if (countCap !== null && probed > countCap) {
+      total = countCap;
+      listCountCapWarning = { entity, cap: countCap };
+    }
     let dataQuery = applyScope(db.selectFrom(`custom_entities_storage as ${alias}`));
     dataQuery = applySelection(dataQuery);
     dataQuery = applySort(dataQuery);
     dataQuery = dataQuery.limit(pageSize).offset((page - 1) * pageSize);
     const items = await dataQuery.execute();
-    return { items, page, pageSize, total };
+    const result = { items, page, pageSize, total };
+    if (listCountCapWarning) result.meta = { listCountCapWarning };
+    return result;
   }
   async tableExists(table) {
     const db = this.getDb();