@open-mercato/core 0.6.6-develop.5834.1.dc9eb0615e → 0.6.6-develop.5900.1.b3fb52925e

package/dist/modules/query_index/lib/engine.js

@@ -14,7 +14,9 @@ import {
 import { resolveSearchConfig } from "@open-mercato/shared/lib/search/config";
 import { tokenizeText } from "@open-mercato/shared/lib/search/tokenize";
 import { runBeforeQueryPipeline, runAfterQueryPipeline } from "@open-mercato/shared/lib/query/query-extension-runner";
-import { resolveEncryptedSortFields, sortRowsInMemory } from "@open-mercato/shared/lib/query/encrypted-sort";
+import { resolveEncryptedSortFields, resolveEncryptedSortMaxRows, sortRowsInMemory } from "@open-mercato/shared/lib/query/encrypted-sort";
+import { mapWithConcurrency } from "@open-mercato/shared/lib/query/bounded-decrypt";
+const DECRYPT_CONCURRENCY = 8;
 function buildFilterableCustomFieldJoins(sources) {
   if (!sources || sources.length === 0) return [];
   return sources.flatMap((source, index) => {
@@ -662,9 +664,10 @@ class HybridQueryEngine {
         opts.includeCustomFields.map((key) => String(key)).forEach((key) => selectFieldSet.add(`cf:${key}`));
       }
       const selectFields = Array.from(selectFieldSet);
-      const applySelection = (q) => {
+      const applySelection = (q, fieldsOverride) => {
         let next = q;
-        for (const field of selectFields) {
+        const fields = fieldsOverride ?? selectFields;
+        for (const field of fields) {
           const fieldName = String(field);
           if (fieldName.startsWith("cf:")) {
             const alias = this.sanitize(fieldName);
@@ -751,71 +754,122 @@ class HybridQueryEngine {
         );
         total = this.parseCount(countRow);
       }
-      const dataRoot = db.selectFrom(`${baseTable} as b`);
-      let dataBuilder = await applyQueryShape(dataRoot);
-      dataBuilder = applySelection(dataBuilder);
-      dataBuilder = applySort(dataBuilder);
-      if (!requiresPlaintextSort) {
-        dataBuilder = dataBuilder.limit(pageSize).offset((page - 1) * pageSize);
-      }
-      if (debugEnabled && sqlDebugEnabled) {
-        const compiled = dataBuilder.compile();
-        this.debug("query:sql:data", { entity, sql: compiled.sql, bindings: compiled.parameters, page, pageSize });
-      }
-      const itemsRaw = await this.captureSqlTiming(
-        "query:sql:data",
-        entity,
-        () => dataBuilder.execute(),
-        { page, pageSize },
-        profiler
-      );
-      if (debugEnabled) this.debug("query:complete", { entity, total, items: Array.isArray(itemsRaw) ? itemsRaw.length : 0 });
-      let items = itemsRaw;
       const dekKeyCache = /* @__PURE__ */ new Map();
-      if (encSvc?.decryptEntityPayload) {
-        const decrypt = encSvc.decryptEntityPayload.bind(encSvc);
-        items = await Promise.all(
-          items.map(async (item) => {
-            try {
-              const decrypted = await decrypt(
-                entity,
-                item,
-                item?.tenant_id ?? item?.tenantId ?? opts.tenantId ?? null,
-                item?.organization_id ?? item?.organizationId ?? fallbackOrgId ?? null
-              );
-              return { ...item, ...decrypted };
-            } catch (err) {
-              console.error("Error decrypting entity payload", err);
-              return item;
-            }
-          })
+      const decryptRow = async (item) => {
+        let next = item;
+        if (encSvc?.decryptEntityPayload) {
+          const decrypt = encSvc.decryptEntityPayload.bind(encSvc);
+          try {
+            const decrypted = await decrypt(
+              entity,
+              next,
+              next?.tenant_id ?? next?.tenantId ?? opts.tenantId ?? null,
+              next?.organization_id ?? next?.organizationId ?? fallbackOrgId ?? null
+            );
+            next = { ...next, ...decrypted };
+          } catch (err) {
+            console.error("Error decrypting entity payload", err);
+          }
+        }
+        if (encSvc) {
+          try {
+            next = await decryptIndexDocCustomFields(
+              next,
+              {
+                tenantId: next?.tenant_id ?? next?.tenantId ?? opts.tenantId ?? null,
+                organizationId: next?.organization_id ?? next?.organizationId ?? null
+              },
+              encSvc,
+              dekKeyCache
+            );
+          } catch {
+          }
+        }
+        return next;
+      };
+      let items;
+      let encryptedSortRowCapWarning;
+      if (requiresPlaintextSort) {
+        const cap = resolveEncryptedSortMaxRows();
+        const phase1Root = db.selectFrom(`${baseTable} as b`);
+        let phase1 = await applyQueryShape(phase1Root);
+        const sortFieldNames = Array.from(/* @__PURE__ */ new Set(["id", ...resolvedSorts.map((s) => String(s.field))]));
+        phase1 = applySelection(phase1, sortFieldNames);
+        if (cap !== null) {
+          phase1 = phase1.limit(cap).orderBy(qualify("id"), "asc");
+        }
+        if (debugEnabled && sqlDebugEnabled) {
+          const compiled = phase1.compile();
+          this.debug("query:sql:data:phase1", { entity, sql: compiled.sql, bindings: compiled.parameters });
+        }
+        const candidateRows = await this.captureSqlTiming(
+          "query:sql:data:phase1",
+          entity,
+          () => phase1.execute(),
+          { phase: 1 },
+          profiler
         );
-      }
-      if (encSvc) {
-        items = await Promise.all(
-          items.map(async (item) => {
-            try {
-              return await decryptIndexDocCustomFields(
-                item,
-                {
-                  tenantId: item?.tenant_id ?? item?.tenantId ?? opts.tenantId ?? null,
-                  organizationId: item?.organization_id ?? item?.organizationId ?? null
-                },
-                encSvc,
-                dekKeyCache
-              );
-            } catch {
-              return item;
-            }
-          })
+        const decryptedCandidates = await mapWithConcurrency(candidateRows, DECRYPT_CONCURRENCY, decryptRow);
+        const orderedCandidates = sortRowsInMemory(decryptedCandidates, resolvedSorts);
+        const pageIds = orderedCandidates.slice((page - 1) * pageSize, page * pageSize).map((row) => row.id);
+        if (cap !== null && total > cap) {
+          encryptedSortRowCapWarning = {
+            entity,
+            sortFields: resolvedSorts.map((s) => String(s.field)),
+            maxRows: cap,
+            totalMatched: total
+          };
+        }
+        if (pageIds.length === 0) {
+          items = [];
+        } else {
+          const phase2Root = db.selectFrom(`${baseTable} as b`);
+          let phase2 = await applyQueryShape(phase2Root);
+          phase2 = applySelection(phase2);
+          phase2 = phase2.where(qualify("id"), "in", pageIds);
+          if (debugEnabled && sqlDebugEnabled) {
+            const compiled = phase2.compile();
+            this.debug("query:sql:data:phase2", { entity, sql: compiled.sql, bindings: compiled.parameters });
+          }
+          const pageRowsRaw = await this.captureSqlTiming(
+            "query:sql:data:phase2",
+            entity,
+            () => phase2.execute(),
+            { phase: 2 },
+            profiler
+          );
+          const decryptedPageRows = await mapWithConcurrency(pageRowsRaw, DECRYPT_CONCURRENCY, decryptRow);
+          const byId = new Map(decryptedPageRows.map((row) => [String(row.id), row]));
+          items = pageIds.map((id) => byId.get(String(id))).filter((row) => row != null);
+        }
+      } else {
+        const dataRoot = db.selectFrom(`${baseTable} as b`);
+        let dataBuilder = await applyQueryShape(dataRoot);
+        dataBuilder = applySelection(dataBuilder);
+        dataBuilder = applySort(dataBuilder);
+        dataBuilder = dataBuilder.limit(pageSize).offset((page - 1) * pageSize);
+        if (debugEnabled && sqlDebugEnabled) {
+          const compiled = dataBuilder.compile();
+          this.debug("query:sql:data", { entity, sql: compiled.sql, bindings: compiled.parameters, page, pageSize });
+        }
+        const itemsRaw = await this.captureSqlTiming(
+          "query:sql:data",
+          entity,
+          () => dataBuilder.execute(),
+          { page, pageSize },
+          profiler
         );
+        items = await mapWithConcurrency(itemsRaw, DECRYPT_CONCURRENCY, decryptRow);
       }
-      if (requiresPlaintextSort) {
-        items = sortRowsInMemory(items, resolvedSorts).slice((page - 1) * pageSize, page * pageSize);
-      }
+      if (debugEnabled) this.debug("query:complete", { entity, total, items: items.length });
       const typedItems = items;
       let result = { items: typedItems, page, pageSize, total };
-      if (partialIndexWarning) result.meta = { partialIndexWarning };
+      if (partialIndexWarning || encryptedSortRowCapWarning) {
+        const meta = {};
+        if (partialIndexWarning) meta.partialIndexWarning = partialIndexWarning;
+        if (encryptedSortRowCapWarning) meta.encryptedSortRowCapWarning = encryptedSortRowCapWarning;
+        result.meta = meta;
+      }
       result = await applyAfterExtensions(result);
       finishProfile({
         result: "ok",