@open-mercato/shared 0.6.8-develop.7100.1.fbf66fca35 → 0.7.0

package/src/lib/query/engine.ts

@@ -1,4 +1,4 @@
-import type { QueryEngine, QueryOptions, QueryResult, QueryResultMeta, EncryptedSortRowCapWarning, ListCountCapWarning, QueryCustomFieldSource, QueryExtensionsConfig, Sort } from './types'
+import type { QueryEngine, QueryOptions, QueryResult, QueryResultMeta, EncryptedSortRowCapWarning, QueryCustomFieldSource, QueryExtensionsConfig, Sort } from './types'
 import type { EntityId } from '@open-mercato/shared/modules/entities'
 import type { EntityManager } from '@mikro-orm/postgresql'
 import { type Kysely, sql, type RawBuilder } from 'kysely'
@@ -20,8 +20,6 @@ import {
   type SearchTokenProbeQueryBuilder,
 } from '../search/availability'
 import { tokenizeText } from '../search/tokenize'
-import { fieldNameCandidates } from './encrypted-sort'
-import { isTenantDataEncryptionEnabled } from '../encryption/toggles'
 import { runBeforeQueryPipeline, runAfterQueryPipeline, type QueryExtensionContext } from './query-extension-runner'
 import {
   buildCustomFieldDefinitionIndexFromRows,
@@ -31,9 +29,7 @@ import {
 } from '../crud/custom-field-definition-index'
 import { warnOnCiphertextLikeFallback } from './ciphertext-search-warning'
 import { resolveEncryptedSortFields, resolveEncryptedSortMaxRows, sortRowsInMemory } from './encrypted-sort'
-import { resolveListCountCap } from './count-cap'
 import { mapWithConcurrency } from './bounded-decrypt'
-import { parseNumberWithDefault } from '../number'
 import { createLogger } from '../logger'
 
 const logger = createLogger('shared').child({ component: 'query' })
@@ -47,112 +43,15 @@ const entityTableCache = new Map<string, string>()
 
 type EncryptionResolver = () => {
   decryptEntityPayload?: (entityId: EntityId, payload: Record<string, unknown>, tenantId?: string | null, organizationId?: string | null) => Promise<Record<string, unknown>>
-  getEncryptedFieldNames?: (entityId: EntityId, tenantId?: string | null, organizationId?: string | null, options?: { ignoreRuntimeHealth?: boolean }) => Promise<readonly string[]>
+  getEncryptedFieldNames?: (entityId: EntityId, tenantId?: string | null, organizationId?: string | null) => Promise<readonly string[]>
   isEnabled?: () => boolean
 } | null
 
-/**
- * Membership across name shapes: encryption maps may declare a field as `displayName` or
- * `display_name` (TenantDataEncryptionService resolves both), while query filters carry real
- * column names. Both sides expand through `fieldNameCandidates` at set-build and lookup time.
- */
-export function isEncryptedLikeField(encrypted: ReadonlySet<string>, field: string): boolean {
-  return fieldNameCandidates(field).some((candidate) => encrypted.has(candidate))
-}
-
-// The all-orgs union behind `getEncryptedFieldNames(..., organizationId: null)` is an UNCACHED
-// `encryption_maps` read. Encryption maps change on deploys, not per request, so a short TTL
-// removes the per-search round-trip without meaningfully delaying a map rollout.
-const ENCRYPTED_LIKE_FIELDS_TTL_MS = 60_000
-const ENCRYPTED_LIKE_FIELDS_CACHE_CAP = 500
-const encryptedLikeFieldsCache = new Map<string, { at: number; fields: Set<string> }>()
-
-export async function resolveEncryptedLikeFieldSet(
-  read: () => Promise<readonly string[]>,
-  entity: string,
-  tenantId: string | null,
-): Promise<Set<string>> {
-  const key = `${entity}|${tenantId ?? ''}`
-  const hit = encryptedLikeFieldsCache.get(key)
-  if (hit && Date.now() - hit.at < ENCRYPTED_LIKE_FIELDS_TTL_MS) return hit.fields
-  const names = await read()
-  const fields = new Set<string>()
-  for (const name of names ?? []) {
-    for (const candidate of fieldNameCandidates(String(name))) fields.add(candidate)
-  }
-  if (encryptedLikeFieldsCache.size >= ENCRYPTED_LIKE_FIELDS_CACHE_CAP) encryptedLikeFieldsCache.clear()
-  encryptedLikeFieldsCache.set(key, { at: Date.now(), fields })
-  return fields
-}
-
-/** Test-only: the TTL memo would otherwise leak state across specs. */
-export function clearEncryptedLikeFieldsCache(): void {
-  encryptedLikeFieldsCache.clear()
-}
-
-// Module-scoped on purpose: `createRequestContainer` builds a fresh `BasicQueryEngine`
-// per request, so an instance field alone re-pays the `information_schema` probe on
-// every request (#5605). Schema shape (does a table have this column?) is not
-// per-request state — unlike `tenantEncryptionService` — so sharing the answer across
-// requests is safe. One map per module instance rather than a true process singleton:
-// standalone builds can duplicate this package, which for a memo is harmless (two
-// caches, both correct), so nothing may be built on top of singleton semantics here.
-//
-// Bounded and TTL'd for two reasons. `columnExists` is reached with caller-supplied
-// field names via `resolveBaseColumn` (sort fields and base filter keys arrive raw from
-// the HTTP layer), so an unbounded map would grow monotonically on request input. And a
-// cached `false` is consumed where the tenant/organization/soft-delete predicates are
-// applied, so a schema change that adds one of those columns must not stay invisible
-// until the process restarts — a migration applied against a running `yarn dev` or
-// not-yet-recycled pods converges within the TTL instead. The TTL still removes
-// essentially all of the traffic: a hot column is probed twelve times an hour rather
-// than tens of thousands. Set OM_QUERY_COLUMN_EXISTS_CACHE_MS=0 to disable and probe
-// per request again; OM_QUERY_COLUMN_EXISTS_CACHE_MAX_ENTRIES tunes the bound.
-const COLUMN_EXISTS_CACHE_DEFAULT_TTL_MS = 300_000
-const COLUMN_EXISTS_CACHE_DEFAULT_MAX_ENTRIES = 10_000
-const columnExistsCache = new Map<string, { value: boolean; expiresAt: number }>()
-
-function resolveColumnExistsCacheTtlMs(): number {
-  return parseNumberWithDefault(process.env.OM_QUERY_COLUMN_EXISTS_CACHE_MS, COLUMN_EXISTS_CACHE_DEFAULT_TTL_MS, { integer: true, min: 0 })
-}
-
-function resolveColumnExistsCacheMaxEntries(): number {
-  return parseNumberWithDefault(process.env.OM_QUERY_COLUMN_EXISTS_CACHE_MAX_ENTRIES, COLUMN_EXISTS_CACHE_DEFAULT_MAX_ENTRIES, { integer: true, min: 1 })
-}
-
-function storeColumnExists(key: string, value: boolean, ttlMs: number): void {
-  const maxEntries = resolveColumnExistsCacheMaxEntries()
-  if (columnExistsCache.size >= maxEntries) {
-    const now = Date.now()
-    for (const [entryKey, entry] of columnExistsCache) {
-      if (entry.expiresAt <= now) columnExistsCache.delete(entryKey)
-    }
-    if (columnExistsCache.size >= maxEntries) columnExistsCache.clear()
-  }
-  columnExistsCache.set(key, { value, expiresAt: Date.now() + ttlMs })
-}
-
-/** Test-only: the module-scoped memo would otherwise leak state across specs. */
-export function clearColumnExistsCache(): void {
-  columnExistsCache.clear()
-}
-
-/** Test-only: entry count of the column-existence memo, for the cap regression test. */
-export function columnExistsCacheSize(): number {
-  return columnExistsCache.size
-}
-
 type ResolvedCustomFieldSource = {
   entityId: EntityId
   alias: string
   table: string
   recordIdExpr: RawBuilder<string>
-  /**
-   * The base→source join edge for joined sources (absent on the base source).
-   * The count projection uses it to correlate cf-value EXISTS subqueries
-   * without attaching the source join to the outer query.
-   */
-  hop?: { fromField: string; toField: string; recordIdColumn: string; type: 'left' | 'inner' }
 }
 
 type ResultRow = Record<string, unknown>
@@ -324,6 +223,7 @@ function computeCustomFieldScore(cfg: Record<string, unknown>, kind: string, ent
  * {@link HybridQueryEngine} when the query index is unavailable or incomplete.
  */
 export class BasicQueryEngine implements QueryEngine {
+  private columnCache = new Map<string, boolean>()
   private searchAliasSeq = 0
   private searchAvailabilityInstance: SearchTokenAvailability | null = null
 
@@ -445,62 +345,6 @@ export class BasicQueryEngine implements QueryEngine {
       ? await this.searchAvailability().hasTokens(String(entity), opts.tenantId ?? null, orgScope)
       : false
     const searchActive = searchEnabled && hasSearchTokens
-    // Opt-in via OM_SEARCH_USE_ILIKE_FOR_NON_ENCRYPTED_FIELDS (default false: the pre-existing
-    // rewrite-everything behavior is kept). When enabled, base-column like/ilike is rerouted
-    // through search tokens ONLY for encrypted columns, where
-    // ILIKE against ciphertext cannot match. On a plaintext column SQL ILIKE is exact, and the token
-    // rewrite silently changes the result set: tokenization splits on non-alphanumerics and drops
-    // tokens shorter than minTokenLength, so a document-number search like "ZK 1/2026" degrades to
-    // the tokens {202, 2026} and matches every record from that year instead of the one document.
-    // `ignoreRuntimeHealth` asks the on-disk question -- a column holds ciphertext even while the
-    // KMS is down -- so an outage keeps encrypted columns on the token path (#4622).
-    // `organizationId: null` is deliberate, not an omission: the service then unions in every
-    // organization's map (`fetchAllOrganizationFieldNames`), so a field any org encrypts stays on
-    // the token path -- a wider set fails safe. Passing the request's org instead would silently
-    // break encrypted-column search for orgs without their own map. That union is an UNCACHED
-    // `encryption_maps` read, one extra round-trip per searched list request. `null` means
-    // the encryption service could not answer at all; keep the pre-existing rewrite-everything
-    // behavior then, because guessing "plaintext" would turn encrypted-column search into an
-    // ILIKE-on-ciphertext that matches nothing.
-    let encryptedLikeFields: Set<string> | null = null
-    if (
-      searchActive &&
-      searchConfig.useIlikeForNonEncryptedFields === true &&
-      searchFilters.some((filter) => !String(filter.field).startsWith('cf:'))
-    ) {
-      try {
-        const service = this.getEncryptionService()
-        const readEncryptedFieldNames = service?.getEncryptedFieldNames?.bind(service)
-        if (readEncryptedFieldNames) {
-          encryptedLikeFields = await resolveEncryptedLikeFieldSet(
-            () => readEncryptedFieldNames(
-              String(entity),
-              opts.tenantId ?? null,
-              null,
-              { ignoreRuntimeHealth: true },
-            ),
-            String(entity),
-            opts.tenantId ?? null,
-          )
-        } else if (isTenantDataEncryptionEnabled()) {
-          // Encryption is on but the service is unreachable (a swallowed DI failure looks
-          // exactly like "no service"): treat the map as UNKNOWN and keep the token rewrite,
-          // rather than guessing "plaintext" and running ILIKE against ciphertext.
-          encryptedLikeFields = null
-        } else {
-          // Encryption disabled: nothing is ciphertext at rest, exact ILIKE is always right.
-          encryptedLikeFields = new Set()
-        }
-      } catch (err) {
-        // The fallback is safe (the old rewrite-everything behavior), but taking it silently
-        // would hide that the gate has stopped working.
-        logger.warn('search: encrypted-field map unavailable; keeping the token rewrite for all columns', {
-          entity: String(entity),
-          error: err instanceof Error ? err.message : String(err),
-        })
-        encryptedLikeFields = null
-      }
-    }
     if (searchFilters.length) {
       const fields = searchFilters.map((filter) => String(filter.field))
       this.logSearchDebug('search:init', {
@@ -578,13 +422,7 @@ export class BasicQueryEngine implements QueryEngine {
         searchActive &&
         typeof value === 'string' &&
         fieldName &&
-        typeof column === 'string' &&
-        // Plaintext columns keep exact SQL ILIKE -- see the encryptedLikeFields note above. cf:*
-        // filters never reach this path (they are applied by the custom-field branches), so this
-        // gate only decides base columns. Membership is tested across name-shape candidates --
-        // encryption maps may declare `displayName` while the filter carries the column name
-        // `display_name`, and a raw comparison would misread that ciphertext column as plaintext.
-        (encryptedLikeFields === null || isEncryptedLikeField(encryptedLikeFields, fieldName))
+        typeof column === 'string'
       ) {
         const tokens = tokenizeText(String(value), searchConfig)
         const hashes = tokens.hashes
@@ -708,16 +546,11 @@ export class BasicQueryEngine implements QueryEngine {
     // Builds the fully-scoped query from a fresh root. `projection: 'full'` reproduces
     // today's complete selection (base fields + CF projections + extension joins).
     // `projection: 'sortKeys'` selects only `id` + the sort columns — the slim phase-1
-    // candidate scan used when `requiresPlaintextSort`. `projection: 'count'` carries
-    // scope + filters only: projection joins (CF defs/values, extensions) are omitted
-    // and cf filters are expressed as correlated EXISTS semi-joins, so nothing can
-    // multiply base rows and a LIMIT above the query is an enforceable bound.
-    // Re-running the WHERE/JOIN logic per projection is cheap: every `columnExists`
-    // check is memoized on the module-scoped `columnExistsCache`, so later passes
-    // hit no extra DB calls.
-    const buildQuery = async (projection: 'full' | 'sortKeys' | 'count'): Promise<BuiltQuery> => {
+    // candidate scan used when `requiresPlaintextSort`. Re-running the WHERE/JOIN logic
+    // twice is cheap: every `columnExists` check is memoized on `this.columnCache`,
+    // so the second pass hits no extra DB calls.
+    const buildQuery = async (projection: 'full' | 'sortKeys'): Promise<BuiltQuery> => {
       const isSortKeysProjection = projection === 'sortKeys'
-      const isCountProjection = projection === 'count'
       let q: AnyBuilder = db.selectFrom(table as any)
 
       // Tenant/org/soft-delete scope
@@ -809,10 +642,7 @@ export class BasicQueryEngine implements QueryEngine {
       })
 
       // Selection (base columns only here; cf:* handled later)
-      if (isCountProjection) {
-        // The caller owns the count query's SELECT (a constant inside the
-        // bounded subquery, or the aggregate itself when the cap is off).
-      } else if (isSortKeysProjection) {
+      if (isSortKeysProjection) {
         q = q.select(sql.ref(qualify('id')).as('id'))
         if (await this.columnExists(table, 'tenant_id')) {
           q = q.select(sql.ref(qualify('tenant_id')).as('tenant_id'))
@@ -838,14 +668,14 @@ export class BasicQueryEngine implements QueryEngine {
       }
 
       // Resolve which custom fields to include
-      const cfSourcesResult = this.configureCustomFieldSources(q, table, entity, db, opts, qualify, !isCountProjection)
+      const cfSourcesResult = this.configureCustomFieldSources(q, table, entity, db, opts, qualify)
       q = cfSourcesResult.builder
       const cfSources = cfSourcesResult.sources
       const entityIdToSource = new Map<string, ResolvedCustomFieldSource>()
       for (const source of cfSources) {
         entityIdToSource.set(String(source.entityId), source)
       }
-      const requestedCustomFieldKeys = (projection === 'full' && Array.isArray(opts.includeCustomFields))
+      const requestedCustomFieldKeys = (!isSortKeysProjection && Array.isArray(opts.includeCustomFields))
         ? opts.includeCustomFields.map((key) => String(key))
         : []
       const cfKeys = new Set<string>()
@@ -855,7 +685,7 @@ export class BasicQueryEngine implements QueryEngine {
       // Output-only — never resolved for the slim sortKeys projection.
       let resolvedCustomFieldDefinitions: ResolvedCustomFieldDefinitions | undefined
       // Explicit in fields/filters
-      if (projection === 'full') {
+      if (!isSortKeysProjection) {
         for (const f of (opts.fields || [])) {
           if (typeof f === 'string' && f.startsWith('cf:')) cfKeys.add(f.slice(3))
         }
@@ -863,7 +693,7 @@ export class BasicQueryEngine implements QueryEngine {
       for (const f of cfFilters) {
         if (typeof f.field === 'string' && f.field.startsWith('cf:')) cfKeys.add(f.field.slice(3))
       }
-      if (projection === 'full' && opts.includeCustomFields === true) {
+      if (!isSortKeysProjection && opts.includeCustomFields === true) {
         if (entityIdToSource.size > 0) {
           const entityIdList = Array.from(entityIdToSource.keys())
           const entityOrder = new Map<string, number>()
@@ -990,9 +820,6 @@ export class BasicQueryEngine implements QueryEngine {
       for (const key of cfKeys) {
         const source = keySource.get(key)
         if (!source) continue
-        // The count shape never joins defs/values — cf filters are applied as
-        // correlated EXISTS semi-joins below, so no join can multiply base rows.
-        if (isCountProjection) continue
         const entityIdForKey = source.entityId
         const recordIdExpr = source.recordIdExpr
         const sourceAliasSafe = sanitize(source.alias || 'src')
@@ -1046,16 +873,13 @@ export class BasicQueryEngine implements QueryEngine {
         }
       }
 
-      // Apply cf:* filters (on raw expressions; as EXISTS semi-joins for the count
-      // shape). OR-grouped ones are excluded here and combined with their
-      // disjunct's other leaves right below.
+      // Apply cf:* filters (on raw expressions). OR-grouped ones are excluded here and
+      // combined with their disjunct's other leaves right below.
       for (const f of regularCfFilters) {
         if (!f.field.startsWith('cf:')) continue
         const key = f.field.slice(3)
-        const filterSource = keySource.get(key)
-        if (!filterSource) continue
         const expr = cfValueExprByKey[key]
-        if (!isCountProjection && !expr) continue
+        if (!expr) continue
         if ((f.op === 'like' || f.op === 'ilike') && searchActive && typeof f.value === 'string') {
           const tokens = tokenizeText(String(f.value), searchConfig)
           const hashes = tokens.hashes
@@ -1090,17 +914,6 @@ export class BasicQueryEngine implements QueryEngine {
             })
           }
         }
-        if (isCountProjection) {
-          q = this.applyCfValueExistsFilter(q, {
-            source: filterSource,
-            qualify,
-            tenantId: tenantId ?? null,
-            key,
-            op: f.op,
-            value: f.value,
-          })
-          continue
-        }
         q = this.applyColumnOp(q, expr, f.op, f.value)
       }
 
@@ -1115,35 +928,15 @@ export class BasicQueryEngine implements QueryEngine {
       // `ilike` through the search-token index the way the ungrouped path does. On a
       // field covered by an encryption map such a leaf therefore compares against
       // ciphertext and will not match.
-      //
-      // The count shape never populates cfValueExprByKey (it joins no cf tables), so
-      // its applicability test is key resolution itself — the same condition that
-      // gates the full shape's expression map — and a cf leaf compiles to a
-      // correlated EXISTS instead of a value-expression comparison. Dropping it
-      // instead would narrow the OR and undercount relative to the display query.
-      const cfLeafApplicable = (key: string): boolean =>
-        isCountProjection ? keySource.has(key) : Boolean(cfValueExprByKey[key])
       const applicableGroupFilters = resolvedGroupFilters
-        .map((group) => group.filter((rf) => rf.kind !== 'cf' || cfLeafApplicable(rf.key)))
+        .map((group) => group.filter((rf) => rf.kind !== 'cf' || Boolean(cfValueExprByKey[rf.key])))
         .filter((group) => group.length > 0)
       if (applicableGroupFilters.length > 0) {
         q = q.where((eb: any) => {
           const disjuncts = applicableGroupFilters.map((group) => {
             const parts = group.map((rf) => {
               if (rf.kind === 'column') return this.buildColumnOpExpression(eb, rf.qualified, rf.op, rf.value)
-              if (rf.kind === 'cf') {
-                if (isCountProjection) {
-                  return this.buildCfValueExistsExpression(eb, {
-                    source: keySource.get(rf.key)!,
-                    qualify,
-                    tenantId: tenantId ?? null,
-                    key: rf.key,
-                    op: rf.op,
-                    value: rf.value,
-                  })
-                }
-                return this.buildColumnOpExpression(eb, cfValueExprByKey[rf.key], rf.op, rf.value)
-              }
+              if (rf.kind === 'cf') return this.buildColumnOpExpression(eb, cfValueExprByKey[rf.key], rf.op, rf.value)
               return this.buildIndexDocOpExpression(eb, {
                 entity: String(entity),
                 field: rf.field,
@@ -1161,9 +954,8 @@ export class BasicQueryEngine implements QueryEngine {
         })
       }
 
-      // Entity extensions joins (no selection yet; enables future filters/projections).
-      // Projection-only, so the count shape omits them.
-      if (opts.includeExtensions && !isCountProjection) {
+      // Entity extensions joins (no selection yet; enables future filters/projections)
+      if (opts.includeExtensions) {
         const { getModules } = await import('@open-mercato/shared/lib/i18n/server')
         const allMods = getModules() as any[]
         const allExts = allMods.flatMap((m) => (m as any).entityExtensions || [])
@@ -1190,7 +982,7 @@ export class BasicQueryEngine implements QueryEngine {
       }
 
       // Sorting: base fields and cf:* (use aggregated alias for cf)
-      for (const s of isCountProjection ? [] : resolvedSorts) {
+      for (const s of resolvedSorts) {
         if (s.field.startsWith('cf:')) {
           const key = s.field.slice(3)
           const alias = sanitize(`cf:${key}`)
@@ -1208,12 +1000,8 @@ export class BasicQueryEngine implements QueryEngine {
         }
       }
 
-      // Deduplicate if we joined CFs or extensions by grouping on base id. The count
-      // shape has neither, and must stay barrier-free for its LIMIT to bind.
-      const hasJoinedAggregates = !isCountProjection && (
-        (opts.includeExtensions && (Array.isArray(opts.includeExtensions) ? (opts.includeExtensions.length > 0) : true)) ||
-        Object.keys(cfValueExprByKey).length > 0
-      )
+      // Deduplicate if we joined CFs or extensions by grouping on base id
+      const hasJoinedAggregates = (opts.includeExtensions && (Array.isArray(opts.includeExtensions) ? (opts.includeExtensions.length > 0) : true)) || Object.keys(cfValueExprByKey).length > 0
       if (hasJoinedAggregates) {
         q = q.groupBy(`${table}.id`)
       }
@@ -1233,35 +1021,22 @@ export class BasicQueryEngine implements QueryEngine {
       resolvedCustomFieldDefinitions,
     } = await buildQuery('full')
 
-    // The count is built independently of the display query (the `'count'`
-    // projection): scope + filters only, cf filters as correlated EXISTS
-    // semi-joins, no projection joins. Nothing can multiply base rows, so
-    // `count(*)` needs no DISTINCT (completing #2227) and — when the cap is
-    // active — the LIMIT sits on a row-producing inner query with no
-    // aggregate/sort barrier below it, so it actually bounds the scan.
-    const countCap = resolveListCountCap()
-    const { builder: countShape } = await buildQuery('count')
-    let total: number
-    let listCountCapWarning: ListCountCapWarning | undefined
-    if (countCap !== null) {
-      const probe = countShape.select(sql<number>`1`.as('one')).limit(countCap + 1)
-      const countRow = await db
-        .selectFrom(probe.as('om_count_probe') as any)
-        .select(sql<string>`count(*)`.as('count'))
-        .executeTakeFirst() as { count: unknown } | undefined
-      const probed = Number((countRow as any)?.count ?? 0)
-      if (probed > countCap) {
-        total = countCap
-        listCountCapWarning = { entity, cap: countCap }
-      } else {
-        total = probed
-      }
-    } else {
-      const countRow = await countShape
-        .select(sql<string>`count(*)`.as('count'))
-        .executeTakeFirst() as { count: unknown } | undefined
-      total = Number((countRow as any)?.count ?? 0)
-    }
+    // `count(distinct base.id)` is only required when a join can multiply base rows
+    // (CF/extension aggregates, explicit relation joins, or custom-field sources).
+    // Without such joins base.id is the unique PK, so `count(*)` is equivalent and
+    // lets Postgres skip the redundant DISTINCT sort/hash for an index-only count (#2227).
+    const mayMultiplyBaseRows =
+      hasJoinedAggregates ||
+      (Array.isArray(opts.joins) && opts.joins.length > 0) ||
+      (Array.isArray(opts.customFieldSources) && opts.customFieldSources.length > 0)
+    const countExpr = mayMultiplyBaseRows
+      ? sql<string>`count(distinct ${sql.ref(`${table}.id`)})`
+      : sql<string>`count(*)`
+    const countBuilder = hasJoinedAggregates
+      ? qFull.clearSelect().clearOrderBy().clearGroupBy().select(countExpr.as('count'))
+      : qFull.clearSelect().clearOrderBy().select(countExpr.as('count'))
+    const countRow = await countBuilder.executeTakeFirst() as { count: unknown } | undefined
+    const total = Number((countRow as any)?.count ?? 0)
 
     const svc = encryptionService
     const decryptPayload =
@@ -1322,14 +1097,9 @@ export class BasicQueryEngine implements QueryEngine {
       const cap = resolveEncryptedSortMaxRows()
       let qSort = (await buildQuery('sortKeys')).builder
       if (cap !== null) {
-        // Probe one row past the cap: truncation is detected from the candidate
-        // scan itself, not by comparing against `total` — which may itself be
-        // capped (`OM_LIST_COUNT_CAP`) and would then never exceed the sort cap.
-        qSort = qSort.limit(cap + 1).orderBy(qualify('id'), 'asc' as any)
+        qSort = qSort.limit(cap).orderBy(qualify('id'), 'asc' as any)
       }
-      const candidateRowsRaw = await qSort.execute() as ResultRow[]
-      const sortTruncated = cap !== null && candidateRowsRaw.length > cap
-      const candidateRows = sortTruncated && cap !== null ? candidateRowsRaw.slice(0, cap) : candidateRowsRaw
+      const candidateRows = await qSort.execute() as ResultRow[]
       const decryptedCandidates = decryptPayload
         ? await mapWithConcurrency(candidateRows, DECRYPT_CONCURRENCY, decryptRow)
         : candidateRows
@@ -1338,7 +1108,7 @@ export class BasicQueryEngine implements QueryEngine {
         .slice((page - 1) * pageSize, page * pageSize)
         .map((row) => row.id)
 
-      if (sortTruncated && cap !== null) {
+      if (cap !== null && total > cap) {
         encryptedSortRowCapWarning = {
           entity,
           sortFields: resolvedSorts.map((s) => s.field),
@@ -1374,10 +1144,8 @@ export class BasicQueryEngine implements QueryEngine {
 
     let queryResult: QueryResult<T> = { items: pagedItems as unknown as T[], page, pageSize, total }
 
-    if (encryptedSortRowCapWarning || listCountCapWarning) {
-      const meta: QueryResultMeta = {}
-      if (encryptedSortRowCapWarning) meta.encryptedSortRowCapWarning = encryptedSortRowCapWarning
-      if (listCountCapWarning) meta.listCountCapWarning = listCountCapWarning
+    if (encryptedSortRowCapWarning) {
+      const meta: QueryResultMeta = { encryptedSortRowCapWarning }
       queryResult.meta = meta
     }
 
@@ -1436,143 +1204,6 @@ export class BasicQueryEngine implements QueryEngine {
     }
   }
 
-  /**
-   * Apply a `cf:*` filter as a correlated EXISTS semi-join over
-   * `custom_field_values` (+ `custom_field_defs` for kind-based coercion) —
-   * the count shape's equivalent of the projection path's leftJoin + WHERE.
-   * A semi-join returns each base row at most once, so the count query needs
-   * no DISTINCT or GROUP BY and stays boundable by an outer LIMIT.
-   *
-   * Predicates satisfied by the *absence* of a value row (`eq null`,
-   * `exists: false`) become `NOT EXISTS(value) OR EXISTS(null value)`,
-   * matching the leftJoin form where a missing row yields a NULL expression.
-   */
-  private applyCfValueExistsFilter(
-    q: AnyBuilder,
-    opts: {
-      source: ResolvedCustomFieldSource
-      qualify: (column: string) => string
-      tenantId: string | null
-      key: string
-      op: NormalizedFilter['op']
-      value: unknown
-    },
-  ): AnyBuilder {
-    return q.where((eb: any) => this.buildCfValueExistsExpression(eb, opts))
-  }
-
-  /**
-   * Expression-returning core of `applyCfValueExistsFilter`, so a cf leaf
-   * inside an OR group can compile to an EXISTS predicate on the count shape
-   * instead of being dropped for lacking a `cfValueExprByKey` entry.
-   */
-  private buildCfValueExistsExpression(
-    eb: any,
-    opts: {
-      source: ResolvedCustomFieldSource
-      qualify: (column: string) => string
-      tenantId: string | null
-      key: string
-      op: NormalizedFilter['op']
-      value: unknown
-    },
-  ): any {
-    const { source, qualify, tenantId, key, op, value } = opts
-    const seq = this.searchAliasSeq++
-    const valAlias = `cfev_${seq}`
-    const defAlias = `cfed_${seq}`
-    const srcAlias = `cfes_${seq}`
-    const caseExpr = sql<string | null>`CASE ${sql.ref(`${defAlias}.kind`)}
-         WHEN 'integer' THEN (${sql.ref(`${valAlias}.value_int`)})::text
-         WHEN 'float' THEN (${sql.ref(`${valAlias}.value_float`)})::text
-         WHEN 'boolean' THEN (${sql.ref(`${valAlias}.value_bool`)})::text
-         WHEN 'multiline' THEN (${sql.ref(`${valAlias}.value_multiline`)})::text
-         ELSE (${sql.ref(`${valAlias}.value_text`)})::text
-       END`
-
-    const buildSub = (eb: any): AnyBuilder => {
-      let sub: AnyBuilder = eb
-        .selectFrom(`custom_field_values as ${valAlias}`)
-        .select(sql<number>`1`.as('one'))
-        .leftJoin(`custom_field_defs as ${defAlias}`, (jb: any) =>
-          jb.on(`${defAlias}.entity_id`, '=', String(source.entityId))
-            .on(`${defAlias}.key`, '=', key)
-            .on(`${defAlias}.is_active`, '=', true)
-            .on((jeb: any) => jeb.or([
-              jeb(`${defAlias}.tenant_id`, '=', tenantId),
-              jeb(`${defAlias}.tenant_id`, 'is', null),
-            ])))
-        .where(`${valAlias}.entity_id`, '=', String(source.entityId))
-        .where(`${valAlias}.field_key`, '=', key)
-        .where((web: any) => web.or([
-          web(`${valAlias}.tenant_id`, '=', tenantId),
-          web(`${valAlias}.tenant_id`, 'is', null),
-        ]))
-      if (source.hop) {
-        sub = sub
-          .innerJoin(`${source.table} as ${srcAlias}`, (jb: any) =>
-            jb.on(sql<boolean>`${sql.ref(`${valAlias}.record_id`)} = (${sql.ref(`${srcAlias}.${source.hop!.recordIdColumn}`)})::text`))
-          .whereRef(`${srcAlias}.${source.hop.toField}`, '=', qualify(source.hop.fromField))
-      } else {
-        sub = sub.where(sql<boolean>`${sql.ref(`${valAlias}.record_id`)} = ${source.recordIdExpr}`)
-      }
-      return sub
-    }
-
-    const absenceSatisfiable = (op === 'eq' && value === null) || (op === 'exists' && !value)
-    if (absenceSatisfiable) {
-      return eb.or([
-        eb.not(eb.exists(buildSub(eb))),
-        eb.exists(buildSub(eb).where(sql<boolean>`${caseExpr} is null`)),
-      ])
-    }
-
-    let predicate: RawBuilder<boolean> | null = null
-    switch (op) {
-      case 'eq':
-        predicate = sql<boolean>`${caseExpr} = ${value}`
-        break
-      case 'ne':
-        predicate = value === null
-          ? sql<boolean>`${caseExpr} is not null`
-          : sql<boolean>`${caseExpr} != ${value}`
-        break
-      case 'gt':
-      case 'gte':
-      case 'lt':
-      case 'lte': {
-        const operator = sql.raw(op === 'gt' ? '>' : op === 'gte' ? '>=' : op === 'lt' ? '<' : '<=')
-        predicate = sql<boolean>`${caseExpr} ${operator} ${value}`
-        break
-      }
-      case 'in': {
-        const vals = Array.isArray(value) ? value : [value]
-        predicate = sql<boolean>`${caseExpr} in (${sql.join(vals.map((v) => sql`${v}`), sql`, `)})`
-        break
-      }
-      case 'nin': {
-        const vals = Array.isArray(value) ? value : [value]
-        predicate = sql<boolean>`${caseExpr} not in (${sql.join(vals.map((v) => sql`${v}`), sql`, `)})`
-        break
-      }
-      case 'like':
-        predicate = sql<boolean>`${caseExpr} like ${value}`
-        break
-      case 'ilike':
-        predicate = sql<boolean>`${caseExpr} ilike ${value}`
-        break
-      case 'exists':
-        predicate = sql<boolean>`${caseExpr} is not null`
-        break
-      default:
-        // Mirrors buildColumnOpExpression's unknown-op fallback: a neutral
-        // predicate, so full and count shapes drop the same leaves.
-        return eb.val(true)
-    }
-    const captured = predicate
-    return eb.exists(buildSub(eb).where(captured))
-  }
-
   private buildColumnOpExpression(eb: any, column: string | RawBuilder<unknown>, op: string, value: unknown): any {
     switch (op) {
       case 'eq': return value === null ? eb(column, 'is', null) : eb(column, '=', value)
@@ -1598,9 +1229,11 @@ export class BasicQueryEngine implements QueryEngine {
 
   private async columnExists(table: string, column: string): Promise<boolean> {
     const key = `${table}.${column}`
-    const ttlMs = resolveColumnExistsCacheTtlMs()
-    const cached = columnExistsCache.get(key)
-    if (cached && cached.expiresAt > Date.now()) return cached.value
+    if (this.columnCache.has(key)) {
+      const cached = this.columnCache.get(key)
+      if (cached === true) return true
+      this.columnCache.delete(key)
+    }
     const db = this.getDb()
     const exists = await db
       .selectFrom('information_schema.columns' as any)
@@ -1610,7 +1243,8 @@ export class BasicQueryEngine implements QueryEngine {
       .limit(1)
       .executeTakeFirst()
     const present = !!exists
-    if (ttlMs > 0) storeColumnExists(key, present, ttlMs)
+    if (present) this.columnCache.set(key, true)
+    else this.columnCache.delete(key)
     return present
   }
 
@@ -1817,7 +1451,6 @@ export class BasicQueryEngine implements QueryEngine {
     db: AnyDb,
     opts: QueryOptions,
     qualify: (column: string) => string,
-    attachJoins: boolean = true,
   ): { builder: AnyBuilder; sources: ResolvedCustomFieldSource[] } {
     const sources: ResolvedCustomFieldSource[] = [
       {
@@ -1836,28 +1469,15 @@ export class BasicQueryEngine implements QueryEngine {
       if (!join) {
         throw new Error(`QueryEngine: customFieldSources entry for ${String(srcOpt.entityId)} requires a join configuration`)
       }
-      const joinType: 'left' | 'inner' = (join.type ?? 'left') === 'inner' ? 'inner' : 'left'
-      if (attachJoins) {
-        const joinFn = joinType === 'inner' ? 'innerJoin' : 'leftJoin'
-        next = (next as any)[joinFn](`${joinTable} as ${alias}`, (jb: any) =>
-          jb.onRef(`${alias}.${join.toField}`, '=', qualify(join.fromField)))
-      } else if (joinType === 'inner') {
-        // The count projection carries no projection joins, but an inner-typed
-        // source join restricts the result set — preserve that as a semi-join.
-        next = next.where((eb: any) => eb.exists(
-          eb
-            .selectFrom(`${joinTable} as ${alias}`)
-            .select(sql<number>`1`.as('one'))
-            .whereRef(`${alias}.${join.toField}`, '=', qualify(join.fromField)),
-        ))
-      }
+      const joinFn = (join.type ?? 'left') === 'inner' ? 'innerJoin' : 'leftJoin'
+      next = (next as any)[joinFn](`${joinTable} as ${alias}`, (jb: any) =>
+        jb.onRef(`${alias}.${join.toField}`, '=', qualify(join.fromField)))
       const recordColumn = srcOpt.recordIdColumn ?? 'id'
       sources.push({
         entityId: srcOpt.entityId,
         alias,
         table: joinTable,
         recordIdExpr: sql<string>`${sql.ref(`${alias}.${recordColumn}`)}::text`,
-        hop: { fromField: join.fromField, toField: join.toField, recordIdColumn: recordColumn, type: joinType },
       })
     })
     return { builder: next, sources }