@open-mercato/shared 0.6.6-develop.5757.1.f5cc26cf92 → 0.6.6-develop.5876.1.6a45c370ac

package/dist/lib/query/engine.js

@@ -12,7 +12,9 @@ import {
   buildCustomFieldDefinitionIndexFromRows,
   resolveCfDefIndexOrgCandidates
 } from "../crud/custom-field-definition-index.js";
-import { resolveEncryptedSortFields, sortRowsInMemory } from "./encrypted-sort.js";
+import { resolveEncryptedSortFields, resolveEncryptedSortMaxRows, sortRowsInMemory } from "./encrypted-sort.js";
+import { mapWithConcurrency } from "./bounded-decrypt.js";
+const DECRYPT_CONCURRENCY = 8;
 const entityTableCache = /* @__PURE__ */ new Map();
 const ENTITY_ID_PATTERN = /^[a-z][a-z0-9_]*:[a-z][a-z0-9_]*$/;
 const isValidEntityIdShape = (value) => ENTITY_ID_PATTERN.test(value);
@@ -176,7 +178,6 @@ class BasicQueryEngine {
     opts = coreOpts;
     const table = resolveEntityTableName(this.em, entity);
     const db = this.getDb();
-    let q = db.selectFrom(table);
     const qualify = (col) => `${table}.${col}`;
     const orgScope = this.resolveOrganizationScope(opts);
     this.searchAliasSeq = 0;
@@ -186,15 +187,6 @@ class BasicQueryEngine {
       );
     }
     const skipAutoScope = opts.omitAutomaticTenantOrgScope === true;
-    if (!skipAutoScope && orgScope && await this.columnExists(table, "organization_id")) {
-      q = this.applyOrganizationScope(q, qualify("organization_id"), orgScope);
-    }
-    if (!skipAutoScope && await this.columnExists(table, "tenant_id")) {
-      q = q.where(qualify("tenant_id"), "=", opts.tenantId);
-    }
-    if (!opts.withDeleted && await this.columnExists(table, "deleted_at")) {
-      q = q.where(qualify("deleted_at"), "is", null);
-    }
     const normalizedFilters = normalizeFilters(opts.filters);
     const resolvedJoins = resolveJoins(
       table,
@@ -307,62 +299,6 @@ class BasicQueryEngine {
     };
     const regularBaseFilters = baseFilters.filter((f) => !f.orGroup);
     const orGroupFilters = baseFilters.filter((f) => f.orGroup);
-    for (const filter of regularBaseFilters) {
-      const fieldName = String(filter.field);
-      let qualified = filter.qualified ?? null;
-      if (!qualified) {
-        const column = await this.resolveBaseColumn(table, fieldName);
-        if (!column) {
-          q = this.applyIndexDocFilter(q, {
-            entity: String(entity),
-            field: fieldName,
-            op: filter.op,
-            value: filter.value,
-            recordIdColumn,
-            tenantId: opts.tenantId ?? null,
-            organizationScope: orgScope,
-            withDeleted: opts.withDeleted === true,
-            searchActive,
-            searchConfig
-          });
-          continue;
-        }
-        qualified = qualify(column);
-      }
-      q = applyFilterOp(q, qualified, filter.op, filter.value, fieldName);
-    }
-    if (orGroupFilters.length > 0) {
-      const groups = /* @__PURE__ */ new Map();
-      for (const f of orGroupFilters) {
-        const group = groups.get(f.orGroup) ?? [];
-        group.push(f);
-        groups.set(f.orGroup, group);
-      }
-      const resolvedGroupFilters = [];
-      for (const [, groupFilters] of groups) {
-        const resolved = [];
-        for (const filter of groupFilters) {
-          const column = await this.resolveBaseColumn(table, String(filter.field));
-          if (column) {
-            resolved.push({
-              qualified: qualify(column),
-              op: filter.op,
-              value: filter.value,
-              fieldName: String(filter.field)
-            });
-          }
-        }
-        if (resolved.length > 0) resolvedGroupFilters.push(resolved);
-      }
-      if (resolvedGroupFilters.length > 0) {
-        q = q.where((eb) => eb.or(
-          resolvedGroupFilters.map((group) => {
-            const parts = group.map((rf) => this.buildColumnOpExpression(eb, rf.qualified, rf.op, rf.value));
-            return parts.length === 1 ? parts[0] : eb.and(parts);
-          })
-        ));
-      }
-    }
     const applyAliasScopes = async (builder, aliasName) => {
       const targetTable = aliasTables.get(aliasName);
       if (!targetTable) return builder;
@@ -375,19 +311,6 @@ class BasicQueryEngine {
       }
       return next;
     };
-    q = await applyJoinFilters({
-      db,
-      baseTable: table,
-      builder: q,
-      joinMap,
-      joinFilters,
-      aliasTables,
-      qualifyBase: (column) => qualify(column),
-      applyAliasScope: (builder, alias) => applyAliasScopes(builder, alias),
-      applyFilterOp: (builder, column, op, value) => applyFilterOp(builder, column, op, value),
-      applyJoinFilterOp,
-      columnExists: (tbl, column) => this.columnExists(tbl, column)
-    });
     const fallbackOrgId = opts.organizationId ?? (Array.isArray(opts.organizationIds) && opts.organizationIds.length === 1 ? opts.organizationIds[0] : null);
     const encryptionService = this.getEncryptionService();
     const resolvedSorts = [];
@@ -407,267 +330,380 @@ class BasicQueryEngine {
       fallbackOrgId
     );
     const requiresPlaintextSort = encryptedSortFields.size > 0;
-    if (opts.fields && opts.fields.length) {
-      const cols = new Set(opts.fields.filter((f) => !f.startsWith("cf:")));
-      if (requiresPlaintextSort) {
-        for (const field of encryptedSortFields) cols.add(field);
-      }
-      for (const c of cols) {
-        q = q.select(sql.ref(qualify(c)).as(c));
-      }
-    } else {
-      q = q.select(sql`${sql.ref(table)}.*`.as("__all"));
-    }
     const tenantId = opts.tenantId;
     const sanitize = (s) => s.replace(/[^a-zA-Z0-9_]/g, "_");
-    const cfSourcesResult = this.configureCustomFieldSources(q, table, entity, db, opts, qualify);
-    q = cfSourcesResult.builder;
-    const cfSources = cfSourcesResult.sources;
-    const entityIdToSource = /* @__PURE__ */ new Map();
-    for (const source of cfSources) {
-      entityIdToSource.set(String(source.entityId), source);
-    }
-    const requestedCustomFieldKeys = Array.isArray(opts.includeCustomFields) ? opts.includeCustomFields.map((key) => String(key)) : [];
-    const cfKeys = /* @__PURE__ */ new Set();
-    const keySource = /* @__PURE__ */ new Map();
-    let resolvedCustomFieldDefinitions;
-    for (const f of opts.fields || []) {
-      if (typeof f === "string" && f.startsWith("cf:")) cfKeys.add(f.slice(3));
-    }
-    for (const f of cfFilters) {
-      if (typeof f.field === "string" && f.field.startsWith("cf:")) cfKeys.add(f.field.slice(3));
-    }
-    if (opts.includeCustomFields === true) {
-      if (entityIdToSource.size > 0) {
-        const entityIdList = Array.from(entityIdToSource.keys());
-        const entityOrder = /* @__PURE__ */ new Map();
-        entityIdList.forEach((id, idx) => entityOrder.set(id, idx));
-        const rows = await db.selectFrom("custom_field_defs").select([
-          "key",
-          "entity_id",
-          "config_json",
-          "kind",
-          "organization_id",
-          "tenant_id",
-          "updated_at",
-          "deleted_at"
-        ]).where("entity_id", "in", entityIdList).where("is_active", "=", true).where((eb) => eb.or([
-          eb("tenant_id", "=", tenantId),
-          eb("tenant_id", "is", null)
-        ])).execute();
-        const orgCandidates = resolveCfDefIndexOrgCandidates(opts.organizationIds, opts.organizationId ?? null);
-        const definitionRows = rows.map((row) => ({
-          key: String(row.key),
-          entityId: String(row.entity_id),
-          kind: row.kind == null ? null : String(row.kind),
-          configJson: row.config_json,
-          organizationId: row.organization_id == null ? null : String(row.organization_id),
-          tenantId: row.tenant_id == null ? null : String(row.tenant_id),
-          deletedAt: row.deleted_at ?? null,
-          updatedAt: row.updated_at ?? null
-        }));
-        resolvedCustomFieldDefinitions = {
-          index: buildCustomFieldDefinitionIndexFromRows(definitionRows, { organizationIds: orgCandidates }),
-          entityIds: entityIdList,
-          tenantId: tenantId ?? null,
-          organizationIds: orgCandidates
-        };
-        const sorted = rows.map((row) => {
-          const raw = row.config_json;
-          let cfg = {};
-          if (raw && typeof raw === "string") {
-            try {
-              cfg = JSON.parse(raw);
-            } catch {
-              cfg = {};
+    const buildQuery = async (projection) => {
+      const isSortKeysProjection = projection === "sortKeys";
+      let q = db.selectFrom(table);
+      if (!skipAutoScope && orgScope && await this.columnExists(table, "organization_id")) {
+        q = this.applyOrganizationScope(q, qualify("organization_id"), orgScope);
+      }
+      if (!skipAutoScope && await this.columnExists(table, "tenant_id")) {
+        q = q.where(qualify("tenant_id"), "=", opts.tenantId);
+      }
+      if (!opts.withDeleted && await this.columnExists(table, "deleted_at")) {
+        q = q.where(qualify("deleted_at"), "is", null);
+      }
+      for (const filter of regularBaseFilters) {
+        const fieldName = String(filter.field);
+        let qualified = filter.qualified ?? null;
+        if (!qualified) {
+          const column = await this.resolveBaseColumn(table, fieldName);
+          if (!column) {
+            q = this.applyIndexDocFilter(q, {
+              entity: String(entity),
+              field: fieldName,
+              op: filter.op,
+              value: filter.value,
+              recordIdColumn,
+              tenantId: opts.tenantId ?? null,
+              organizationScope: orgScope,
+              withDeleted: opts.withDeleted === true,
+              searchActive,
+              searchConfig
+            });
+            continue;
+          }
+          qualified = qualify(column);
+        }
+        q = applyFilterOp(q, qualified, filter.op, filter.value, fieldName);
+      }
+      if (orGroupFilters.length > 0) {
+        const groups = /* @__PURE__ */ new Map();
+        for (const f of orGroupFilters) {
+          const group = groups.get(f.orGroup) ?? [];
+          group.push(f);
+          groups.set(f.orGroup, group);
+        }
+        const resolvedGroupFilters = [];
+        for (const [, groupFilters] of groups) {
+          const resolved = [];
+          for (const filter of groupFilters) {
+            const column = await this.resolveBaseColumn(table, String(filter.field));
+            if (column) {
+              resolved.push({
+                qualified: qualify(column),
+                op: filter.op,
+                value: filter.value,
+                fieldName: String(filter.field)
+              });
             }
-          } else if (raw && typeof raw === "object") {
-            cfg = raw;
           }
-          return {
+          if (resolved.length > 0) resolvedGroupFilters.push(resolved);
+        }
+        if (resolvedGroupFilters.length > 0) {
+          q = q.where((eb) => eb.or(
+            resolvedGroupFilters.map((group) => {
+              const parts = group.map((rf) => this.buildColumnOpExpression(eb, rf.qualified, rf.op, rf.value));
+              return parts.length === 1 ? parts[0] : eb.and(parts);
+            })
+          ));
+        }
+      }
+      q = await applyJoinFilters({
+        db,
+        baseTable: table,
+        builder: q,
+        joinMap,
+        joinFilters,
+        aliasTables,
+        qualifyBase: (column) => qualify(column),
+        applyAliasScope: (builder, alias) => applyAliasScopes(builder, alias),
+        applyFilterOp: (builder, column, op, value) => applyFilterOp(builder, column, op, value),
+        applyJoinFilterOp,
+        columnExists: (tbl, column) => this.columnExists(tbl, column)
+      });
+      if (isSortKeysProjection) {
+        q = q.select(sql.ref(qualify("id")).as("id"));
+        for (const s of resolvedSorts) {
+          if (!s.field.startsWith("cf:")) q = q.select(sql.ref(qualify(s.field)).as(s.field));
+        }
+      } else if (opts.fields && opts.fields.length) {
+        const cols = new Set(opts.fields.filter((f) => !f.startsWith("cf:")));
+        if (requiresPlaintextSort) {
+          for (const field of encryptedSortFields) cols.add(field);
+        }
+        for (const c of cols) {
+          q = q.select(sql.ref(qualify(c)).as(c));
+        }
+      } else {
+        q = q.select(sql`${sql.ref(table)}.*`.as("__all"));
+      }
+      const cfSourcesResult = this.configureCustomFieldSources(q, table, entity, db, opts, qualify);
+      q = cfSourcesResult.builder;
+      const cfSources = cfSourcesResult.sources;
+      const entityIdToSource = /* @__PURE__ */ new Map();
+      for (const source of cfSources) {
+        entityIdToSource.set(String(source.entityId), source);
+      }
+      const requestedCustomFieldKeys = !isSortKeysProjection && Array.isArray(opts.includeCustomFields) ? opts.includeCustomFields.map((key) => String(key)) : [];
+      const cfKeys = /* @__PURE__ */ new Set();
+      const keySource = /* @__PURE__ */ new Map();
+      let resolvedCustomFieldDefinitions2;
+      if (!isSortKeysProjection) {
+        for (const f of opts.fields || []) {
+          if (typeof f === "string" && f.startsWith("cf:")) cfKeys.add(f.slice(3));
+        }
+      }
+      for (const f of cfFilters) {
+        if (typeof f.field === "string" && f.field.startsWith("cf:")) cfKeys.add(f.field.slice(3));
+      }
+      if (!isSortKeysProjection && opts.includeCustomFields === true) {
+        if (entityIdToSource.size > 0) {
+          const entityIdList = Array.from(entityIdToSource.keys());
+          const entityOrder = /* @__PURE__ */ new Map();
+          entityIdList.forEach((id, idx) => entityOrder.set(id, idx));
+          const rows = await db.selectFrom("custom_field_defs").select([
+            "key",
+            "entity_id",
+            "config_json",
+            "kind",
+            "organization_id",
+            "tenant_id",
+            "updated_at",
+            "deleted_at"
+          ]).where("entity_id", "in", entityIdList).where("is_active", "=", true).where((eb) => eb.or([
+            eb("tenant_id", "=", tenantId),
+            eb("tenant_id", "is", null)
+          ])).execute();
+          const orgCandidates = resolveCfDefIndexOrgCandidates(opts.organizationIds, opts.organizationId ?? null);
+          const definitionRows = rows.map((row) => ({
             key: String(row.key),
             entityId: String(row.entity_id),
-            kind: String(row.kind || ""),
-            config: cfg
+            kind: row.kind == null ? null : String(row.kind),
+            configJson: row.config_json,
+            organizationId: row.organization_id == null ? null : String(row.organization_id),
+            tenantId: row.tenant_id == null ? null : String(row.tenant_id),
+            deletedAt: row.deleted_at ?? null,
+            updatedAt: row.updated_at ?? null
+          }));
+          resolvedCustomFieldDefinitions2 = {
+            index: buildCustomFieldDefinitionIndexFromRows(definitionRows, { organizationIds: orgCandidates }),
+            entityIds: entityIdList,
+            tenantId: tenantId ?? null,
+            organizationIds: orgCandidates
           };
-        });
-        sorted.sort((a, b) => {
-          const ai = entityOrder.get(a.entityId) ?? Number.MAX_SAFE_INTEGER;
-          const bi = entityOrder.get(b.entityId) ?? Number.MAX_SAFE_INTEGER;
-          if (ai !== bi) return ai - bi;
-          return a.key.localeCompare(b.key);
-        });
-        const selectedSources = /* @__PURE__ */ new Map();
-        for (const row of sorted) {
-          const source = entityIdToSource.get(row.entityId);
-          if (!source) continue;
-          const cfg = row.config || {};
-          const entityIndex = entityOrder.get(row.entityId) ?? Number.MAX_SAFE_INTEGER;
-          const scores = computeCustomFieldScore(cfg, row.kind, entityIndex);
-          const existing = selectedSources.get(row.key);
-          if (!existing || scores.base > existing.score || scores.base === existing.score && (scores.penalty < existing.penalty || scores.penalty === existing.penalty && scores.entityIndex < existing.entityIndex)) {
-            selectedSources.set(row.key, { source, score: scores.base, penalty: scores.penalty, entityIndex: scores.entityIndex });
+          const sorted = rows.map((row) => {
+            const raw = row.config_json;
+            let cfg = {};
+            if (raw && typeof raw === "string") {
+              try {
+                cfg = JSON.parse(raw);
+              } catch {
+                cfg = {};
+              }
+            } else if (raw && typeof raw === "object") {
+              cfg = raw;
+            }
+            return {
+              key: String(row.key),
+              entityId: String(row.entity_id),
+              kind: String(row.kind || ""),
+              config: cfg
+            };
+          });
+          sorted.sort((a, b) => {
+            const ai = entityOrder.get(a.entityId) ?? Number.MAX_SAFE_INTEGER;
+            const bi = entityOrder.get(b.entityId) ?? Number.MAX_SAFE_INTEGER;
+            if (ai !== bi) return ai - bi;
+            return a.key.localeCompare(b.key);
+          });
+          const selectedSources = /* @__PURE__ */ new Map();
+          for (const row of sorted) {
+            const source = entityIdToSource.get(row.entityId);
+            if (!source) continue;
+            const cfg = row.config || {};
+            const entityIndex = entityOrder.get(row.entityId) ?? Number.MAX_SAFE_INTEGER;
+            const scores = computeCustomFieldScore(cfg, row.kind, entityIndex);
+            const existing = selectedSources.get(row.key);
+            if (!existing || scores.base > existing.score || scores.base === existing.score && (scores.penalty < existing.penalty || scores.penalty === existing.penalty && scores.entityIndex < existing.entityIndex)) {
+              selectedSources.set(row.key, { source, score: scores.base, penalty: scores.penalty, entityIndex: scores.entityIndex });
+            }
+            cfKeys.add(row.key);
+          }
+          for (const [key, entry] of selectedSources.entries()) {
+            keySource.set(key, entry.source);
           }
-          cfKeys.add(row.key);
         }
-        for (const [key, entry] of selectedSources.entries()) {
-          keySource.set(key, entry.source);
+      } else if (!isSortKeysProjection && requestedCustomFieldKeys.length > 0) {
+        for (const key of requestedCustomFieldKeys) cfKeys.add(key);
+      }
+      const unresolvedKeys = Array.from(cfKeys).filter((key) => !keySource.has(key));
+      if (unresolvedKeys.length > 0 && entityIdToSource.size > 0) {
+        const rows = await db.selectFrom("custom_field_defs").select(["key", "entity_id"]).where("entity_id", "in", Array.from(entityIdToSource.keys())).where("key", "in", unresolvedKeys).where("is_active", "=", true).where((eb) => eb.or([
+          eb("tenant_id", "=", tenantId),
+          eb("tenant_id", "is", null)
+        ])).execute();
+        for (const row of rows) {
+          const source = entityIdToSource.get(String(row.entity_id));
+          if (!source) continue;
+          if (!keySource.has(row.key)) keySource.set(row.key, source);
         }
       }
-    } else if (requestedCustomFieldKeys.length > 0) {
-      for (const key of requestedCustomFieldKeys) cfKeys.add(key);
-    }
-    const unresolvedKeys = Array.from(cfKeys).filter((key) => !keySource.has(key));
-    if (unresolvedKeys.length > 0 && entityIdToSource.size > 0) {
-      const rows = await db.selectFrom("custom_field_defs").select(["key", "entity_id"]).where("entity_id", "in", Array.from(entityIdToSource.keys())).where("key", "in", unresolvedKeys).where("is_active", "=", true).where((eb) => eb.or([
-        eb("tenant_id", "=", tenantId),
-        eb("tenant_id", "is", null)
-      ])).execute();
-      for (const row of rows) {
-        const source = entityIdToSource.get(String(row.entity_id));
+      const cfValueExprByKey = {};
+      const cfSelectedAliases = [];
+      const cfJsonAliases2 = /* @__PURE__ */ new Set();
+      const cfMultiAliasByAlias2 = /* @__PURE__ */ new Map();
+      for (const key of cfKeys) {
+        const source = keySource.get(key);
         if (!source) continue;
-        if (!keySource.has(row.key)) keySource.set(row.key, source);
-      }
-    }
-    const cfValueExprByKey = {};
-    const cfSelectedAliases = [];
-    const cfJsonAliases = /* @__PURE__ */ new Set();
-    const cfMultiAliasByAlias = /* @__PURE__ */ new Map();
-    for (const key of cfKeys) {
-      const source = keySource.get(key);
-      if (!source) continue;
-      const entityIdForKey = source.entityId;
-      const recordIdExpr = source.recordIdExpr;
-      const sourceAliasSafe = sanitize(source.alias || "src");
-      const keyAliasSafe = sanitize(key);
-      const defAlias = `cfd_${sourceAliasSafe}_${keyAliasSafe}`;
-      const valAlias = `cfv_${sourceAliasSafe}_${keyAliasSafe}`;
-      q = q.leftJoin(
-        `custom_field_defs as ${defAlias}`,
-        (jb) => jb.on(`${defAlias}.entity_id`, "=", String(entityIdForKey)).on(`${defAlias}.key`, "=", key).on(`${defAlias}.is_active`, "=", true).on((eb) => eb.or([
-          eb(`${defAlias}.tenant_id`, "=", tenantId),
-          eb(`${defAlias}.tenant_id`, "is", null)
-        ]))
-      );
-      q = q.leftJoin(
-        `custom_field_values as ${valAlias}`,
-        (jb) => jb.on(`${valAlias}.entity_id`, "=", String(entityIdForKey)).on(`${valAlias}.field_key`, "=", key).onRef(`${valAlias}.record_id`, "=", recordIdExpr).on((eb) => eb.or([
-          eb(`${valAlias}.tenant_id`, "=", tenantId),
-          eb(`${valAlias}.tenant_id`, "is", null)
-        ]))
-      );
-      const caseExpr = sql`CASE ${sql.ref(`${defAlias}.kind`)}
-           WHEN 'integer' THEN (${sql.ref(`${valAlias}.value_int`)})::text
-           WHEN 'float' THEN (${sql.ref(`${valAlias}.value_float`)})::text
-           WHEN 'boolean' THEN (${sql.ref(`${valAlias}.value_bool`)})::text
-           WHEN 'multiline' THEN (${sql.ref(`${valAlias}.value_multiline`)})::text
-           ELSE (${sql.ref(`${valAlias}.value_text`)})::text
-         END`;
-      cfValueExprByKey[key] = caseExpr;
-      const alias = sanitize(`cf:${key}`);
-      if ((opts.fields || []).includes(`cf:${key}`) || opts.includeCustomFields === true || requestedCustomFieldKeys.length > 0 && requestedCustomFieldKeys.includes(key)) {
-        const multiAlias = `${alias}__is_multi`;
-        const isMultiExpr = sql`bool_or(coalesce((${sql.ref(`${defAlias}.config_json`)}->>'multi')::boolean, false))`;
-        const aggregatedArray = sql`array_remove(array_agg(DISTINCT ${caseExpr}), NULL)`;
-        const projExpr = sql`CASE WHEN ${isMultiExpr}
-                THEN to_jsonb(${aggregatedArray})
-                ELSE to_jsonb(max(${caseExpr}))
+        const entityIdForKey = source.entityId;
+        const recordIdExpr = source.recordIdExpr;
+        const sourceAliasSafe = sanitize(source.alias || "src");
+        const keyAliasSafe = sanitize(key);
+        const defAlias = `cfd_${sourceAliasSafe}_${keyAliasSafe}`;
+        const valAlias = `cfv_${sourceAliasSafe}_${keyAliasSafe}`;
+        q = q.leftJoin(
+          `custom_field_defs as ${defAlias}`,
+          (jb) => jb.on(`${defAlias}.entity_id`, "=", String(entityIdForKey)).on(`${defAlias}.key`, "=", key).on(`${defAlias}.is_active`, "=", true).on((eb) => eb.or([
+            eb(`${defAlias}.tenant_id`, "=", tenantId),
+            eb(`${defAlias}.tenant_id`, "is", null)
+          ]))
+        );
+        q = q.leftJoin(
+          `custom_field_values as ${valAlias}`,
+          (jb) => jb.on(`${valAlias}.entity_id`, "=", String(entityIdForKey)).on(`${valAlias}.field_key`, "=", key).onRef(`${valAlias}.record_id`, "=", recordIdExpr).on((eb) => eb.or([
+            eb(`${valAlias}.tenant_id`, "=", tenantId),
+            eb(`${valAlias}.tenant_id`, "is", null)
+          ]))
+        );
+        const caseExpr = sql`CASE ${sql.ref(`${defAlias}.kind`)}
+             WHEN 'integer' THEN (${sql.ref(`${valAlias}.value_int`)})::text
+             WHEN 'float' THEN (${sql.ref(`${valAlias}.value_float`)})::text
+             WHEN 'boolean' THEN (${sql.ref(`${valAlias}.value_bool`)})::text
+             WHEN 'multiline' THEN (${sql.ref(`${valAlias}.value_multiline`)})::text
+             ELSE (${sql.ref(`${valAlias}.value_text`)})::text
            END`;
-        q = q.select(projExpr.as(alias));
-        q = q.select(isMultiExpr.as(multiAlias));
-        cfSelectedAliases.push(alias);
-        cfJsonAliases.add(alias);
-        cfMultiAliasByAlias.set(alias, multiAlias);
+        cfValueExprByKey[key] = caseExpr;
+        const alias = sanitize(`cf:${key}`);
+        if (!isSortKeysProjection && ((opts.fields || []).includes(`cf:${key}`) || opts.includeCustomFields === true || requestedCustomFieldKeys.length > 0 && requestedCustomFieldKeys.includes(key))) {
+          const multiAlias = `${alias}__is_multi`;
+          const isMultiExpr = sql`bool_or(coalesce((${sql.ref(`${defAlias}.config_json`)}->>'multi')::boolean, false))`;
+          const aggregatedArray = sql`array_remove(array_agg(DISTINCT ${caseExpr}), NULL)`;
+          const projExpr = sql`CASE WHEN ${isMultiExpr}
+                  THEN to_jsonb(${aggregatedArray})
+                  ELSE to_jsonb(max(${caseExpr}))
+             END`;
+          q = q.select(projExpr.as(alias));
+          q = q.select(isMultiExpr.as(multiAlias));
+          cfSelectedAliases.push(alias);
+          cfJsonAliases2.add(alias);
+          cfMultiAliasByAlias2.set(alias, multiAlias);
+        }
       }
-    }
-    for (const f of cfFilters) {
-      if (!f.field.startsWith("cf:")) continue;
-      const key = f.field.slice(3);
-      const expr = cfValueExprByKey[key];
-      if (!expr) continue;
-      if ((f.op === "like" || f.op === "ilike") && searchActive && typeof f.value === "string") {
-        const tokens = tokenizeText(String(f.value), searchConfig);
-        const hashes = tokens.hashes;
-        if (hashes.length) {
-          const result = this.applySearchTokens(q, {
-            entity: String(entity),
-            field: f.field,
-            hashes,
-            recordIdColumn,
-            tenantId: opts.tenantId ?? null,
-            organizationScope: orgScope,
-            tokens: tokens.tokens
-          });
-          this.logSearchDebug("search:cf-filter", {
-            entity: String(entity),
-            field: f.field,
-            tokens: tokens.tokens,
-            hashes,
-            applied: result.applied,
-            tenantId: opts.tenantId ?? null,
-            organizationScope: orgScope
-          });
-          if (result.applied) {
-            q = result.builder;
-            continue;
+      for (const f of cfFilters) {
+        if (!f.field.startsWith("cf:")) continue;
+        const key = f.field.slice(3);
+        const expr = cfValueExprByKey[key];
+        if (!expr) continue;
+        if ((f.op === "like" || f.op === "ilike") && searchActive && typeof f.value === "string") {
+          const tokens = tokenizeText(String(f.value), searchConfig);
+          const hashes = tokens.hashes;
+          if (hashes.length) {
+            const result = this.applySearchTokens(q, {
+              entity: String(entity),
+              field: f.field,
+              hashes,
+              recordIdColumn,
+              tenantId: opts.tenantId ?? null,
+              organizationScope: orgScope,
+              tokens: tokens.tokens
+            });
+            this.logSearchDebug("search:cf-filter", {
+              entity: String(entity),
+              field: f.field,
+              tokens: tokens.tokens,
+              hashes,
+              applied: result.applied,
+              tenantId: opts.tenantId ?? null,
+              organizationScope: orgScope
+            });
+            if (result.applied) {
+              q = result.builder;
+              continue;
+            }
+          } else {
+            this.logSearchDebug("search:cf-skip-empty-hashes", {
+              entity: String(entity),
+              field: f.field,
+              value: f.value
+            });
           }
-        } else {
-          this.logSearchDebug("search:cf-skip-empty-hashes", {
-            entity: String(entity),
-            field: f.field,
-            value: f.value
-          });
         }
+        q = this.applyColumnOp(q, expr, f.op, f.value);
       }
-      q = this.applyColumnOp(q, expr, f.op, f.value);
-    }
-    if (opts.includeExtensions) {
-      const { getModules } = await import("@open-mercato/shared/lib/i18n/server");
-      const allMods = getModules();
-      const allExts = allMods.flatMap((m) => m.entityExtensions || []);
-      const exts = allExts.filter((e) => e.base === entity);
-      const chosen = Array.isArray(opts.includeExtensions) ? exts.filter((e) => opts.includeExtensions.includes(e.extension)) : exts;
-      for (const e of chosen) {
-        const [, extName] = e.extension.split(":");
-        const extTable = extName.endsWith("s") ? extName : `${extName}s`;
-        const alias = `ext_${sanitize(extName)}`;
-        q = q.leftJoin(
-          `${extTable} as ${alias}`,
-          (jb) => jb.onRef(`${alias}.${e.join.extensionKey}`, "=", `${table}.${e.join.baseKey}`)
-        );
+      if (opts.includeExtensions) {
+        const { getModules } = await import("@open-mercato/shared/lib/i18n/server");
+        const allMods = getModules();
+        const allExts = allMods.flatMap((m) => m.entityExtensions || []);
+        const exts = allExts.filter((e) => e.base === entity);
+        const chosen = Array.isArray(opts.includeExtensions) ? exts.filter((e) => opts.includeExtensions.includes(e.extension)) : exts;
+        for (const e of chosen) {
+          const [, extName] = e.extension.split(":");
+          const extTable = extName.endsWith("s") ? extName : `${extName}s`;
+          const alias = `ext_${sanitize(extName)}`;
+          q = q.leftJoin(
+            `${extTable} as ${alias}`,
+            (jb) => jb.onRef(`${alias}.${e.join.extensionKey}`, "=", `${table}.${e.join.baseKey}`)
+          );
+        }
       }
-    }
-    for (const s of resolvedSorts) {
-      if (s.field.startsWith("cf:")) {
-        const key = s.field.slice(3);
-        const alias = sanitize(`cf:${key}`);
-        if (!cfSelectedAliases.includes(alias)) {
-          const expr = cfValueExprByKey[key];
-          if (expr) {
-            q = q.select(sql`max(${expr})`.as(alias));
-            cfSelectedAliases.push(alias);
+      for (const s of resolvedSorts) {
+        if (s.field.startsWith("cf:")) {
+          const key = s.field.slice(3);
+          const alias = sanitize(`cf:${key}`);
+          if (!cfSelectedAliases.includes(alias)) {
+            const expr = cfValueExprByKey[key];
+            if (expr) {
+              q = q.select(sql`max(${expr})`.as(alias));
+              cfSelectedAliases.push(alias);
+            }
           }
+          if (!requiresPlaintextSort) q = q.orderBy(alias, s.dir ?? "asc");
+        } else {
+          if (!requiresPlaintextSort) q = q.orderBy(qualify(s.field), s.dir ?? "asc");
         }
-        if (!requiresPlaintextSort) q = q.orderBy(alias, s.dir ?? "asc");
-      } else {
-        if (!requiresPlaintextSort) q = q.orderBy(qualify(s.field), s.dir ?? "asc");
       }
-    }
+      const hasJoinedAggregates2 = opts.includeExtensions && (Array.isArray(opts.includeExtensions) ? opts.includeExtensions.length > 0 : true) || Object.keys(cfValueExprByKey).length > 0;
+      if (hasJoinedAggregates2) {
+        q = q.groupBy(`${table}.id`);
+      }
+      return { builder: q, hasJoinedAggregates: hasJoinedAggregates2, cfJsonAliases: cfJsonAliases2, cfMultiAliasByAlias: cfMultiAliasByAlias2, resolvedCustomFieldDefinitions: resolvedCustomFieldDefinitions2 };
+    };
     const page = opts.page?.page ?? 1;
     const pageSize = opts.page?.pageSize ?? 20;
-    const hasJoinedAggregates = opts.includeExtensions && (Array.isArray(opts.includeExtensions) ? opts.includeExtensions.length > 0 : true) || Object.keys(cfValueExprByKey).length > 0;
-    if (hasJoinedAggregates) {
-      q = q.groupBy(`${table}.id`);
-    }
+    const {
+      builder: qFull,
+      hasJoinedAggregates,
+      cfJsonAliases,
+      cfMultiAliasByAlias,
+      resolvedCustomFieldDefinitions
+    } = await buildQuery("full");
     const mayMultiplyBaseRows = hasJoinedAggregates || Array.isArray(opts.joins) && opts.joins.length > 0 || Array.isArray(opts.customFieldSources) && opts.customFieldSources.length > 0;
     const countExpr = mayMultiplyBaseRows ? sql`count(distinct ${sql.ref(`${table}.id`)})` : sql`count(*)`;
-    const countBuilder = hasJoinedAggregates ? q.clearSelect().clearOrderBy().clearGroupBy().select(countExpr.as("count")) : q.clearSelect().clearOrderBy().select(countExpr.as("count"));
+    const countBuilder = hasJoinedAggregates ? qFull.clearSelect().clearOrderBy().clearGroupBy().select(countExpr.as("count")) : qFull.clearSelect().clearOrderBy().select(countExpr.as("count"));
     const countRow = await countBuilder.executeTakeFirst();
     const total = Number(countRow?.count ?? 0);
-    const dataQuery = requiresPlaintextSort ? q : q.limit(pageSize).offset((page - 1) * pageSize);
-    const items = await dataQuery.execute();
-    if (cfJsonAliases.size > 0) {
-      for (const row of items) {
+    const svc = encryptionService;
+    const decryptPayload = svc?.decryptEntityPayload?.bind(svc);
+    const decryptRow = async (item) => {
+      if (!decryptPayload) return item;
+      try {
+        const decrypted = await decryptPayload(
+          entity,
+          item,
+          item?.tenant_id ?? item?.tenantId ?? opts.tenantId ?? null,
+          item?.organization_id ?? item?.organizationId ?? fallbackOrgId ?? null
+        );
+        return { ...item, ...decrypted };
+      } catch (err) {
+        console.error("QueryEngine: error decrypting entity payload", err);
+        return item;
+      }
+    };
+    const normalizeCfJsonAliases = (rows) => {
+      if (cfJsonAliases.size === 0) return;
+      for (const row of rows) {
         for (const alias of cfJsonAliases) {
           const multiAlias = cfMultiAliasByAlias.get(alias);
           const isMulti = multiAlias ? Boolean(row[multiAlias]) : false;
@@ -689,30 +725,47 @@ class BasicQueryEngine {
           if (multiAlias) delete row[multiAlias];
         }
       }
+    };
+    let pagedItems;
+    let encryptedSortRowCapWarning;
+    if (requiresPlaintextSort) {
+      const cap = resolveEncryptedSortMaxRows();
+      let qSort = (await buildQuery("sortKeys")).builder;
+      if (cap !== null) {
+        qSort = qSort.limit(cap).orderBy(qualify("id"), "asc");
+      }
+      const candidateRows = await qSort.execute();
+      const decryptedCandidates = decryptPayload ? await mapWithConcurrency(candidateRows, DECRYPT_CONCURRENCY, decryptRow) : candidateRows;
+      const orderedCandidates = sortRowsInMemory(decryptedCandidates, resolvedSorts);
+      const pageIds = orderedCandidates.slice((page - 1) * pageSize, page * pageSize).map((row) => row.id);
+      if (cap !== null && total > cap) {
+        encryptedSortRowCapWarning = {
+          entity,
+          sortFields: resolvedSorts.map((s) => s.field),
+          maxRows: cap,
+          totalMatched: total
+        };
+      }
+      if (pageIds.length === 0) {
+        pagedItems = [];
+      } else {
+        const pageRows = await qFull.where(qualify("id"), "in", pageIds).execute();
+        normalizeCfJsonAliases(pageRows);
+        const decryptedPageRows = decryptPayload ? await mapWithConcurrency(pageRows, DECRYPT_CONCURRENCY, decryptRow) : pageRows;
+        const byId = new Map(decryptedPageRows.map((row) => [String(row.id), row]));
+        pagedItems = pageIds.map((id) => byId.get(String(id))).filter((row) => row != null);
+      }
+    } else {
+      const dataQuery = qFull.limit(pageSize).offset((page - 1) * pageSize);
+      const items = await dataQuery.execute();
+      normalizeCfJsonAliases(items);
+      pagedItems = decryptPayload ? await mapWithConcurrency(items, DECRYPT_CONCURRENCY, decryptRow) : items;
     }
-    const svc = encryptionService;
-    const decryptPayload = svc?.decryptEntityPayload?.bind(svc);
-    let decryptedItems = items;
-    if (decryptPayload) {
-      decryptedItems = await Promise.all(
-        items.map(async (item) => {
-          try {
-            const decrypted = await decryptPayload(
-              entity,
-              item,
-              item?.tenant_id ?? item?.tenantId ?? opts.tenantId ?? null,
-              item?.organization_id ?? item?.organizationId ?? fallbackOrgId ?? null
-            );
-            return { ...item, ...decrypted };
-          } catch (err) {
-            console.error("QueryEngine: error decrypting entity payload", err);
-            return item;
-          }
-        })
-      );
-    }
-    const pagedItems = requiresPlaintextSort ? sortRowsInMemory(decryptedItems, resolvedSorts).slice((page - 1) * pageSize, page * pageSize) : decryptedItems;
     let queryResult = { items: pagedItems, page, pageSize, total };
+    if (encryptedSortRowCapWarning) {
+      const meta = { encryptedSortRowCapWarning };
+      queryResult.meta = meta;
+    }
     if (ext && extensionCtx) {
       const diCtx = ext.resolve ? { resolve: ext.resolve } : noop;
       queryResult = await runAfterQueryPipeline(